From 14260617053a91b89ebf9abf54aad84cb200d185 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Thu, 8 Aug 2019 20:25:54 -0700 Subject: [PATCH 01/23] OK --- pandas/core/indexes/base.py | 4 +-- pandas/core/ops/__init__.py | 34 ++++++++++++++--------- pandas/core/ops/common.py | 54 +++++++++++++++++++++++++++++++++++++ 3 files changed, 77 insertions(+), 15 deletions(-) create mode 100644 pandas/core/ops/common.py diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py index 356ae20b2240a..38b8dd7457050 100644 --- a/pandas/core/indexes/base.py +++ b/pandas/core/indexes/base.py @@ -115,6 +115,7 @@ def cmp_method(self, other): # technically we could support bool dtyped Index # for now just return the indexing array directly if is_bool_dtype(result): + # TODO: This fails for exactly 1 test, with Int64Index and other="a" and == return result try: return Index(result) @@ -128,7 +129,7 @@ def cmp_method(self, other): def _make_arithmetic_op(op, cls): def index_arithmetic_method(self, other): - if isinstance(other, (ABCSeries, ABCDataFrame, ABCTimedeltaIndex)): + if isinstance(other, (ABCSeries, ABCDataFrame)): return NotImplemented from pandas import Series @@ -139,7 +140,6 @@ def index_arithmetic_method(self, other): return Index(result) name = "__{name}__".format(name=op.__name__) - # TODO: docstring? return set_function_name(index_arithmetic_method, name, cls) diff --git a/pandas/core/ops/__init__.py b/pandas/core/ops/__init__.py index a56521b9c9fec..c801417a02a05 100644 --- a/pandas/core/ops/__init__.py +++ b/pandas/core/ops/__init__.py @@ -74,6 +74,8 @@ rtruediv, rxor, ) +from pandas.core.ops.common import unpack_and_defer + # ----------------------------------------------------------------------------- # Ops Wrapping Utilities @@ -676,12 +678,14 @@ def na_op(x, y): return missing.dispatch_fill_zeros(op, x, y, result) + @unpack_and_defer(op_name) def wrapper(left, right): - if isinstance(right, ABCDataFrame): - return NotImplemented + #if isinstance(right, ABCDataFrame): + # return NotImplemented left, right = _align_method_SERIES(left, right) res_name = get_op_result_name(left, right) + right = lib.item_from_zerodim(right) right = maybe_upcast_for_op(right, left.shape) if is_categorical_dtype(left): @@ -808,6 +812,7 @@ def na_op(x, y): return result + @unpack_and_defer(op_name) def wrapper(self, other, axis=None): # Validate the axis parameter if axis is not None: @@ -828,11 +833,11 @@ def wrapper(self, other, axis=None): # TODO: same for tuples? other = np.asarray(other) - if isinstance(other, ABCDataFrame): # pragma: no cover - # Defer to DataFrame implementation; fail early - return NotImplemented + #if isinstance(other, ABCDataFrame): # pragma: no cover + # # Defer to DataFrame implementation; fail early + # return NotImplemented - elif isinstance(other, ABCSeries) and not self._indexed_same(other): + if isinstance(other, ABCSeries) and not self._indexed_same(other): raise ValueError("Can only compare identically-labeled Series objects") elif ( @@ -938,17 +943,19 @@ def na_op(x, y): fill_int = lambda x: x.fillna(0) fill_bool = lambda x: x.fillna(False).astype(bool) + @unpack_and_defer(op_name) def wrapper(self, other): is_self_int_dtype = is_integer_dtype(self.dtype) self, other = _align_method_SERIES(self, other, align_asobject=True) res_name = get_op_result_name(self, other) + other = lib.item_from_zerodim(other) - if isinstance(other, ABCDataFrame): - # Defer to DataFrame implementation; fail early - return NotImplemented + #if isinstance(other, ABCDataFrame): + # # Defer to DataFrame implementation; fail early + # return NotImplemented - elif isinstance(other, (ABCSeries, ABCIndexClass)): + if isinstance(other, (ABCSeries, ABCIndexClass)): is_other_int_dtype = is_integer_dtype(other.dtype) other = fill_int(other) if is_other_int_dtype else fill_bool(other) @@ -1278,10 +1285,11 @@ def _arith_method_SPARSE_SERIES(cls, op, special): """ op_name = _get_op_name(op, special) + @unpack_and_defer(op_name) def wrapper(self, other): - if isinstance(other, ABCDataFrame): - return NotImplemented - elif isinstance(other, ABCSeries): + #if isinstance(other, ABCDataFrame): + # return NotImplemented + if isinstance(other, ABCSeries): if not isinstance(other, ABCSparseSeries): other = other.to_sparse(fill_value=self.fill_value) return _sparse_series_op(self, other, op, op_name) diff --git a/pandas/core/ops/common.py b/pandas/core/ops/common.py new file mode 100644 index 0000000000000..41a832e228401 --- /dev/null +++ b/pandas/core/ops/common.py @@ -0,0 +1,54 @@ +""" +Boilerplate functions used in defining binary operations. +""" +from functools import wraps + +import numpy as np + +from pandas._libs.lib import item_from_zerodim + +from pandas.core.dtypes.generic import ABCDataFrame, ABCSeries, ABCIndexClass +from pandas.core.dtypes.common import is_list_like + + +def unpack_arraylike(obj, match_len: int): + obj = item_from_zerodim(obj) + + if is_list_like(obj) and len(obj) != match_len: + # NB: we must have already checked for if we need to + # return NotImplemented for DataFrame + raise ValueError("Lengths must match") + + if isinstance(obj, list): + # TODO: what about tuple? + obj = np.asarray(obj) + return obj + + +def unpack_and_defer(name): + def wrapper(method): + return _unpack_and_defer(method, name) + + return wrapper + + +def _unpack_and_defer(method, name): + + is_cmp = name.strip('__') in {'eq', 'ne', 'lt', 'le', 'gt', 'ge'} + + @wraps(method) + def new_method(self, other): + + for cls in [ABCDataFrame, ABCSeries, ABCIndexClass]: + if isinstance(self, cls): + break + if isinstance(other, cls): + if is_cmp and cls is ABCSeries and isinstance(self, ABCIndexClass): + # For comparison ops, Index does *not* defer to Series + break + else: + return NotImplemented + + return method(self, other) + + return new_method From c9e99f338172d58b328424b7774f9ab74715e2bd Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Fri, 9 Aug 2019 07:32:04 -0700 Subject: [PATCH 02/23] OK --- pandas/core/arrays/categorical.py | 9 ++++++--- pandas/core/arrays/datetimelike.py | 15 +++++++++------ pandas/core/arrays/datetimes.py | 6 ++++-- pandas/core/arrays/period.py | 6 ++++-- pandas/core/arrays/timedeltas.py | 6 ++++-- pandas/core/indexes/base.py | 6 ++++-- pandas/core/indexes/range.py | 6 ++++-- 7 files changed, 35 insertions(+), 19 deletions(-) diff --git a/pandas/core/arrays/categorical.py b/pandas/core/arrays/categorical.py index 69fa956b73f28..7b0bbe4967309 100644 --- a/pandas/core/arrays/categorical.py +++ b/pandas/core/arrays/categorical.py @@ -59,6 +59,7 @@ from pandas.core.construction import extract_array, sanitize_array from pandas.core.missing import interpolate_2d from pandas.core.sorting import nargsort +from pandas.core.ops.common import unpack_and_defer from pandas.io.formats import console @@ -77,15 +78,17 @@ ) -def _cat_compare_op(op): +def _cat_compare_op(op): # TODO: op-->opname + + @unpack_and_defer(op) def f(self, other): # On python2, you can usually compare any type to any type, and # Categoricals can be seen as a custom type, but having different # results depending whether categories are the same or not is kind of # insane, so be a bit stricter here and use the python3 idea of # comparing only things of equal type. - if isinstance(other, (ABCDataFrame, ABCSeries, ABCIndexClass)): - return NotImplemented + #if isinstance(other, (ABCDataFrame, ABCSeries, ABCIndexClass)): + # return NotImplemented other = lib.item_from_zerodim(other) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index b3548a1dc20d6..52820dcdacef0 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -48,6 +48,7 @@ from pandas.core.algorithms import checked_add_with_arr, take, unique1d, value_counts import pandas.core.common as com from pandas.core.ops.invalid import make_invalid_op +from pandas.core.ops.common import unpack_and_defer from pandas.tseries import frequencies from pandas.tseries.offsets import DateOffset, Tick @@ -1204,13 +1205,14 @@ def _time_shift(self, periods, freq=None): # to be passed explicitly. return self._generate_range(start=start, end=end, periods=None, freq=self.freq) + @unpack_and_defer("__add__") def __add__(self, other): other = lib.item_from_zerodim(other) - if isinstance(other, (ABCSeries, ABCDataFrame, ABCIndexClass)): - return NotImplemented + #if isinstance(other, (ABCSeries, ABCDataFrame, ABCIndexClass)): + # return NotImplemented # scalar others - elif other is NaT: + if other is NaT: result = self._add_nat() elif isinstance(other, (Tick, timedelta, np.timedelta64)): result = self._add_delta(other) @@ -1270,13 +1272,14 @@ def __radd__(self, other): # alias for __add__ return self.__add__(other) + @unpack_and_defer("__sub__") def __sub__(self, other): other = lib.item_from_zerodim(other) - if isinstance(other, (ABCSeries, ABCDataFrame, ABCIndexClass)): - return NotImplemented + #if isinstance(other, (ABCSeries, ABCDataFrame, ABCIndexClass)): + # return NotImplemented # scalar others - elif other is NaT: + if other is NaT: result = self._sub_nat() elif isinstance(other, (Tick, timedelta, np.timedelta64)): result = self._add_delta(-other) diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index 28537124536e7..672cca80c48e5 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -49,6 +49,7 @@ from pandas.core.dtypes.missing import isna from pandas.core import ops +from pandas.core.ops.common import unpack_and_defer from pandas.core.algorithms import checked_add_with_arr from pandas.core.arrays import datetimelike as dtl from pandas.core.arrays._ranges import generate_regular_range @@ -157,9 +158,10 @@ def _dt_array_cmp(cls, op): opname = "__{name}__".format(name=op.__name__) nat_result = opname == "__ne__" + @unpack_and_defer(opname) def wrapper(self, other): - if isinstance(other, (ABCDataFrame, ABCSeries, ABCIndexClass)): - return NotImplemented + #if isinstance(other, (ABCDataFrame, ABCSeries, ABCIndexClass)): + # return NotImplemented other = lib.item_from_zerodim(other) diff --git a/pandas/core/arrays/period.py b/pandas/core/arrays/period.py index 20ce11c70c344..779623e05bbed 100644 --- a/pandas/core/arrays/period.py +++ b/pandas/core/arrays/period.py @@ -46,6 +46,7 @@ import pandas.core.algorithms as algos from pandas.core.arrays import datetimelike as dtl import pandas.core.common as com +from pandas.core.ops.common import unpack_and_defer from pandas.tseries import frequencies from pandas.tseries.offsets import DateOffset, Tick, _delta_to_tick @@ -69,12 +70,13 @@ def _period_array_cmp(cls, op): opname = "__{name}__".format(name=op.__name__) nat_result = opname == "__ne__" + @unpack_and_defer(opname) def wrapper(self, other): op = getattr(self.asi8, opname) other = lib.item_from_zerodim(other) - if isinstance(other, (ABCDataFrame, ABCSeries, ABCIndexClass)): - return NotImplemented + #if isinstance(other, (ABCDataFrame, ABCSeries, ABCIndexClass)): + # return NotImplemented if is_list_like(other) and len(other) != len(self): raise ValueError("Lengths must match") diff --git a/pandas/core/arrays/timedeltas.py b/pandas/core/arrays/timedeltas.py index 6899e47045c1c..e3219a4a891fd 100644 --- a/pandas/core/arrays/timedeltas.py +++ b/pandas/core/arrays/timedeltas.py @@ -44,6 +44,7 @@ from pandas.core.algorithms import checked_add_with_arr import pandas.core.common as com from pandas.core.ops.invalid import invalid_comparison +from pandas.core.ops.common import unpack_and_defer from pandas.tseries.frequencies import to_offset from pandas.tseries.offsets import Tick @@ -80,10 +81,11 @@ def _td_array_cmp(cls, op): opname = "__{name}__".format(name=op.__name__) nat_result = opname == "__ne__" + @unpack_and_defer(opname) def wrapper(self, other): other = lib.item_from_zerodim(other) - if isinstance(other, (ABCDataFrame, ABCSeries, ABCIndexClass)): - return NotImplemented + #if isinstance(other, (ABCDataFrame, ABCSeries, ABCIndexClass)): + # return NotImplemented if _is_convertible_to_td(other) or other is NaT: try: diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py index 38b8dd7457050..cb68fe6442ed8 100644 --- a/pandas/core/indexes/base.py +++ b/pandas/core/indexes/base.py @@ -72,6 +72,7 @@ import pandas.core.missing as missing from pandas.core.ops import get_op_result_name from pandas.core.ops.invalid import make_invalid_op +from pandas.core.ops.common import unpack_and_defer import pandas.core.sorting as sorting from pandas.core.strings import StringMethods @@ -128,9 +129,10 @@ def cmp_method(self, other): def _make_arithmetic_op(op, cls): + @unpack_and_defer(op.__name__) def index_arithmetic_method(self, other): - if isinstance(other, (ABCSeries, ABCDataFrame)): - return NotImplemented + #if isinstance(other, (ABCSeries, ABCDataFrame)): + # return NotImplemented from pandas import Series diff --git a/pandas/core/indexes/range.py b/pandas/core/indexes/range.py index d2bea5f68b92d..b2d8f77de2e8f 100644 --- a/pandas/core/indexes/range.py +++ b/pandas/core/indexes/range.py @@ -24,6 +24,7 @@ from pandas.core.dtypes.generic import ABCDataFrame, ABCSeries, ABCTimedeltaIndex from pandas.core import ops +from pandas.core.ops.common import unpack_and_defer import pandas.core.common as com import pandas.core.indexes.base as ibase from pandas.core.indexes.base import Index, _index_shared_docs @@ -730,9 +731,10 @@ def __getitem__(self, key): # fall back to Int64Index return super().__getitem__(key) + @unpack_and_defer("__floordiv__") def __floordiv__(self, other): - if isinstance(other, (ABCSeries, ABCDataFrame)): - return NotImplemented + #if isinstance(other, (ABCSeries, ABCDataFrame)): + # return NotImplemented if is_integer(other) and other != 0: if len(self) == 0 or self.start % other == 0 and self.step % other == 0: From 2462d9b27c70ebb3b3d8e7f6b111da87ec6c4a3d Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Fri, 9 Aug 2019 09:12:16 -0700 Subject: [PATCH 03/23] OK --- pandas/core/arrays/sparse.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pandas/core/arrays/sparse.py b/pandas/core/arrays/sparse.py index 8aa83c3fbc37d..81f1b07c784a6 100644 --- a/pandas/core/arrays/sparse.py +++ b/pandas/core/arrays/sparse.py @@ -39,6 +39,7 @@ ) from pandas.core.dtypes.dtypes import register_extension_dtype from pandas.core.dtypes.generic import ( + ABCDataFrame, ABCIndexClass, ABCSeries, ABCSparseArray, @@ -1732,13 +1733,16 @@ def sparse_unary_method(self): @classmethod def _create_arithmetic_method(cls, op): + op_name = op.__name__ + def sparse_arithmetic_method(self, other): - op_name = op.__name__ - if isinstance(other, (ABCSeries, ABCIndexClass)): + if isinstance(other, (ABCDataFrame, ABCSeries, ABCIndexClass)): # Rely on pandas to dispatch to us. return NotImplemented + other = lib.item_from_zerodim(other) + if isinstance(other, SparseArray): return _sparse_array_op(self, other, op, op_name) @@ -1787,10 +1791,12 @@ def _create_comparison_method(cls, op): def cmp_method(self, other): - if isinstance(other, (ABCSeries, ABCIndexClass)): + if isinstance(other, (ABCDataFrame, ABCSeries, ABCIndexClass)): # Rely on pandas to unbox and dispatch to us. return NotImplemented + other = lib.item_from_zerodim(other) + if not is_scalar(other) and not isinstance(other, type(self)): # convert list-like to ndarray other = np.asarray(other) From 2289a85070d2ee8012f9818a72b29a430614fef7 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Fri, 9 Aug 2019 18:05:53 -0700 Subject: [PATCH 04/23] OK --- pandas/core/arrays/categorical.py | 2 +- pandas/core/arrays/datetimes.py | 2 +- pandas/core/arrays/integer.py | 28 ++++++++++++++++------------ pandas/core/arrays/sparse.py | 10 ++++++---- pandas/core/indexes/range.py | 8 ++++---- pandas/core/ops/common.py | 2 ++ pandas/tests/arrays/test_integer.py | 5 +++-- 7 files changed, 33 insertions(+), 24 deletions(-) diff --git a/pandas/core/arrays/categorical.py b/pandas/core/arrays/categorical.py index e1f06abf3b704..bb758813e2536 100644 --- a/pandas/core/arrays/categorical.py +++ b/pandas/core/arrays/categorical.py @@ -90,7 +90,7 @@ def f(self, other): #if isinstance(other, (ABCDataFrame, ABCSeries, ABCIndexClass)): # return NotImplemented - other = lib.item_from_zerodim(other) + #other = lib.item_from_zerodim(other) if not self.ordered: if op in ["__lt__", "__gt__", "__le__", "__ge__"]: diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index 672cca80c48e5..d9f22000af120 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -163,7 +163,7 @@ def wrapper(self, other): #if isinstance(other, (ABCDataFrame, ABCSeries, ABCIndexClass)): # return NotImplemented - other = lib.item_from_zerodim(other) + #other = lib.item_from_zerodim(other) if isinstance(other, (datetime, np.datetime64, str)): if isinstance(other, (datetime, np.datetime64)): diff --git a/pandas/core/arrays/integer.py b/pandas/core/arrays/integer.py index 1f14bd169a228..13e73ee6ecfc3 100644 --- a/pandas/core/arrays/integer.py +++ b/pandas/core/arrays/integer.py @@ -25,6 +25,7 @@ from pandas.core.dtypes.missing import isna, notna from pandas.core import nanops, ops +from pandas.core.ops.common import unpack_and_defer from pandas.core.algorithms import take from pandas.core.arrays import ExtensionArray, ExtensionOpsMixin from pandas.core.tools.numeric import to_numeric @@ -592,14 +593,15 @@ def _values_for_argsort(self) -> np.ndarray: @classmethod def _create_comparison_method(cls, op): - def cmp_method(self, other): + op_name = op.__name__ - op_name = op.__name__ + @unpack_and_defer(op.__name__) + def cmp_method(self, other): mask = None - if isinstance(other, (ABCSeries, ABCIndexClass)): - # Rely on pandas to unbox and dispatch to us. - return NotImplemented + #if isinstance(other, (ABCSeries, ABCIndexClass)): + # # Rely on pandas to unbox and dispatch to us. + # return NotImplemented if isinstance(other, IntegerArray): other, mask = other._data, other._mask @@ -609,7 +611,7 @@ def cmp_method(self, other): if other.ndim > 0 and len(self) != len(other): raise ValueError("Lengths must match to compare") - other = lib.item_from_zerodim(other) + #other = lib.item_from_zerodim(other) # numpy will show a DeprecationWarning on invalid elementwise # comparisons, this will raise in the future @@ -683,14 +685,16 @@ def _maybe_mask_result(self, result, mask, other, op_name): @classmethod def _create_arithmetic_method(cls, op): + op_name = op.__name__ + + @unpack_and_defer(op.__name__) def integer_arithmetic_method(self, other): - op_name = op.__name__ mask = None - if isinstance(other, (ABCSeries, ABCIndexClass)): - # Rely on pandas to unbox and dispatch to us. - return NotImplemented + #if isinstance(other, (ABCSeries, ABCIndexClass)): + # # Rely on pandas to unbox and dispatch to us. + # return NotImplemented if getattr(other, "ndim", 0) > 1: raise NotImplementedError("can only perform ops with 1-d structures") @@ -698,8 +702,8 @@ def integer_arithmetic_method(self, other): if isinstance(other, IntegerArray): other, mask = other._data, other._mask - elif getattr(other, "ndim", None) == 0: - other = other.item() + #elif getattr(other, "ndim", None) == 0: + # other = other.item() elif is_list_like(other): other = np.asarray(other) diff --git a/pandas/core/arrays/sparse.py b/pandas/core/arrays/sparse.py index 81f1b07c784a6..a5d9656a845a4 100644 --- a/pandas/core/arrays/sparse.py +++ b/pandas/core/arrays/sparse.py @@ -56,6 +56,7 @@ from pandas.core.construction import sanitize_array from pandas.core.missing import interpolate_2d import pandas.core.ops as ops +from pandas.core.ops.common import unpack_and_defer import pandas.io.formats.printing as printing @@ -1735,13 +1736,14 @@ def sparse_unary_method(self): def _create_arithmetic_method(cls, op): op_name = op.__name__ + @unpack_and_defer(op_name) def sparse_arithmetic_method(self, other): - if isinstance(other, (ABCDataFrame, ABCSeries, ABCIndexClass)): - # Rely on pandas to dispatch to us. - return NotImplemented + #if isinstance(other, (ABCDataFrame, ABCSeries, ABCIndexClass)): + # # Rely on pandas to dispatch to us. + # return NotImplemented - other = lib.item_from_zerodim(other) + #other = lib.item_from_zerodim(other) if isinstance(other, SparseArray): return _sparse_array_op(self, other, op, op_name) diff --git a/pandas/core/indexes/range.py b/pandas/core/indexes/range.py index b2d8f77de2e8f..4632f912b2d76 100644 --- a/pandas/core/indexes/range.py +++ b/pandas/core/indexes/range.py @@ -769,11 +769,11 @@ def _make_evaluate_binop(op, step=False): op to apply to the step parm if not None if False, use the existing step """ - + @unpack_and_defer(op.__name__) def _evaluate_numeric_binop(self, other): - if isinstance(other, (ABCSeries, ABCDataFrame)): - return NotImplemented - elif isinstance(other, ABCTimedeltaIndex): + #if isinstance(other, (ABCSeries, ABCDataFrame)): + # return NotImplemented + if isinstance(other, ABCTimedeltaIndex): # Defer to TimedeltaIndex implementation return NotImplemented elif isinstance(other, (timedelta, np.timedelta64)): diff --git a/pandas/core/ops/common.py b/pandas/core/ops/common.py index 41a832e228401..ad49b3086f64c 100644 --- a/pandas/core/ops/common.py +++ b/pandas/core/ops/common.py @@ -49,6 +49,8 @@ def new_method(self, other): else: return NotImplemented + other = item_from_zerodim(other) + return method(self, other) return new_method diff --git a/pandas/tests/arrays/test_integer.py b/pandas/tests/arrays/test_integer.py index 50cd1469e5196..e67b0dd5f3248 100644 --- a/pandas/tests/arrays/test_integer.py +++ b/pandas/tests/arrays/test_integer.py @@ -322,8 +322,9 @@ def test_error(self, data, all_arithmetic_operators): ops(pd.Series(pd.date_range("20180101", periods=len(s)))) # 2d - with pytest.raises(NotImplementedError): - opa(pd.DataFrame({"A": s})) + result = opa(pd.DataFrame({"A": s})) + assert result is NotImplemented + with pytest.raises(NotImplementedError): opa(np.arange(len(s)).reshape(-1, len(s))) From 68bab4b8f57f1774a8b93f94554fb998366f0efe Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Fri, 9 Aug 2019 18:40:45 -0700 Subject: [PATCH 05/23] OK --- pandas/core/arrays/categorical.py | 32 +++++++++++++----------------- pandas/core/arrays/datetimelike.py | 7 ------- pandas/core/arrays/datetimes.py | 5 ----- pandas/core/arrays/integer.py | 14 ------------- pandas/core/arrays/period.py | 6 ------ pandas/core/arrays/sparse.py | 14 +------------ pandas/core/arrays/timedeltas.py | 3 --- pandas/core/indexes/range.py | 6 +----- pandas/core/ops/__init__.py | 15 -------------- 9 files changed, 16 insertions(+), 86 deletions(-) diff --git a/pandas/core/arrays/categorical.py b/pandas/core/arrays/categorical.py index bb758813e2536..34afa60c0638a 100644 --- a/pandas/core/arrays/categorical.py +++ b/pandas/core/arrays/categorical.py @@ -7,7 +7,7 @@ from pandas._config import get_option -from pandas._libs import algos as libalgos, hashtable as htable, lib +from pandas._libs import algos as libalgos, hashtable as htable from pandas.compat.numpy import function as nv from pandas.util._decorators import ( Appender, @@ -38,7 +38,7 @@ is_timedelta64_dtype, ) from pandas.core.dtypes.dtypes import CategoricalDtype -from pandas.core.dtypes.generic import ABCDataFrame, ABCIndexClass, ABCSeries +from pandas.core.dtypes.generic import ABCIndexClass, ABCSeries from pandas.core.dtypes.inference import is_hashable from pandas.core.dtypes.missing import isna, notna @@ -78,22 +78,18 @@ ) -def _cat_compare_op(op): # TODO: op-->opname +def _cat_compare_op(opname): # TODO: pass op instead of opname - @unpack_and_defer(op) + @unpack_and_defer(opname) def f(self, other): # On python2, you can usually compare any type to any type, and # Categoricals can be seen as a custom type, but having different # results depending whether categories are the same or not is kind of # insane, so be a bit stricter here and use the python3 idea of # comparing only things of equal type. - #if isinstance(other, (ABCDataFrame, ABCSeries, ABCIndexClass)): - # return NotImplemented - - #other = lib.item_from_zerodim(other) if not self.ordered: - if op in ["__lt__", "__gt__", "__le__", "__ge__"]: + if opname in ["__lt__", "__gt__", "__le__", "__ge__"]: raise TypeError( "Unordered Categoricals can only compare equality or not" ) @@ -120,7 +116,7 @@ def f(self, other): other_codes = other._codes mask = (self._codes == -1) | (other_codes == -1) - f = getattr(self._codes, op) + f = getattr(self._codes, opname) ret = f(other_codes) if mask.any(): # In other series, the leads to False, so do that here too @@ -130,38 +126,38 @@ def f(self, other): if is_scalar(other): if other in self.categories: i = self.categories.get_loc(other) - ret = getattr(self._codes, op)(i) + ret = getattr(self._codes, opname)(i) # check for NaN in self mask = self._codes == -1 ret[mask] = False return ret else: - if op == "__eq__": + if opname == "__eq__": return np.repeat(False, len(self)) - elif op == "__ne__": + elif opname == "__ne__": return np.repeat(True, len(self)) else: msg = ( "Cannot compare a Categorical for op {op} with a " "scalar, which is not a category." ) - raise TypeError(msg.format(op=op)) + raise TypeError(msg.format(op=opname)) else: # allow categorical vs object dtype array comparisons for equality # these are only positional comparisons - if op in ["__eq__", "__ne__"]: - return getattr(np.array(self), op)(np.array(other)) + if opname in ["__eq__", "__ne__"]: + return getattr(np.array(self), opname)(np.array(other)) msg = ( "Cannot compare a Categorical for op {op} with type {typ}." "\nIf you want to compare values, use 'np.asarray(cat) " " other'." ) - raise TypeError(msg.format(op=op, typ=type(other))) + raise TypeError(msg.format(op=opname, typ=type(other))) - f.__name__ = op + f.__name__ = opname return f diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index 84d292da9428c..1b7674f683ee6 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -35,7 +35,6 @@ pandas_dtype, ) from pandas.core.dtypes.generic import ( - ABCDataFrame, ABCIndexClass, ABCPeriodArray, ABCSeries, @@ -1197,9 +1196,6 @@ def _time_shift(self, periods, freq=None): @unpack_and_defer("__add__") def __add__(self, other): - other = lib.item_from_zerodim(other) - #if isinstance(other, (ABCSeries, ABCDataFrame, ABCIndexClass)): - # return NotImplemented # scalar others if other is NaT: @@ -1264,9 +1260,6 @@ def __radd__(self, other): @unpack_and_defer("__sub__") def __sub__(self, other): - other = lib.item_from_zerodim(other) - #if isinstance(other, (ABCSeries, ABCDataFrame, ABCIndexClass)): - # return NotImplemented # scalar others if other is NaT: diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index d9f22000af120..72d9231b7ff86 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -41,7 +41,6 @@ ) from pandas.core.dtypes.dtypes import DatetimeTZDtype from pandas.core.dtypes.generic import ( - ABCDataFrame, ABCIndexClass, ABCPandasArray, ABCSeries, @@ -160,10 +159,6 @@ def _dt_array_cmp(cls, op): @unpack_and_defer(opname) def wrapper(self, other): - #if isinstance(other, (ABCDataFrame, ABCSeries, ABCIndexClass)): - # return NotImplemented - - #other = lib.item_from_zerodim(other) if isinstance(other, (datetime, np.datetime64, str)): if isinstance(other, (datetime, np.datetime64)): diff --git a/pandas/core/arrays/integer.py b/pandas/core/arrays/integer.py index 13e73ee6ecfc3..48c7df03a41ac 100644 --- a/pandas/core/arrays/integer.py +++ b/pandas/core/arrays/integer.py @@ -21,7 +21,6 @@ is_scalar, ) from pandas.core.dtypes.dtypes import register_extension_dtype -from pandas.core.dtypes.generic import ABCIndexClass, ABCSeries from pandas.core.dtypes.missing import isna, notna from pandas.core import nanops, ops @@ -599,10 +598,6 @@ def _create_comparison_method(cls, op): def cmp_method(self, other): mask = None - #if isinstance(other, (ABCSeries, ABCIndexClass)): - # # Rely on pandas to unbox and dispatch to us. - # return NotImplemented - if isinstance(other, IntegerArray): other, mask = other._data, other._mask @@ -611,8 +606,6 @@ def cmp_method(self, other): if other.ndim > 0 and len(self) != len(other): raise ValueError("Lengths must match to compare") - #other = lib.item_from_zerodim(other) - # numpy will show a DeprecationWarning on invalid elementwise # comparisons, this will raise in the future with warnings.catch_warnings(): @@ -692,19 +685,12 @@ def integer_arithmetic_method(self, other): mask = None - #if isinstance(other, (ABCSeries, ABCIndexClass)): - # # Rely on pandas to unbox and dispatch to us. - # return NotImplemented - if getattr(other, "ndim", 0) > 1: raise NotImplementedError("can only perform ops with 1-d structures") if isinstance(other, IntegerArray): other, mask = other._data, other._mask - #elif getattr(other, "ndim", None) == 0: - # other = other.item() - elif is_list_like(other): other = np.asarray(other) if not other.ndim: diff --git a/pandas/core/arrays/period.py b/pandas/core/arrays/period.py index 779623e05bbed..ac5348ec01507 100644 --- a/pandas/core/arrays/period.py +++ b/pandas/core/arrays/period.py @@ -4,7 +4,6 @@ import numpy as np -from pandas._libs import lib from pandas._libs.tslibs import ( NaT, NaTType, @@ -35,7 +34,6 @@ ) from pandas.core.dtypes.dtypes import PeriodDtype from pandas.core.dtypes.generic import ( - ABCDataFrame, ABCIndexClass, ABCPeriodArray, ABCPeriodIndex, @@ -74,10 +72,6 @@ def _period_array_cmp(cls, op): def wrapper(self, other): op = getattr(self.asi8, opname) - other = lib.item_from_zerodim(other) - #if isinstance(other, (ABCDataFrame, ABCSeries, ABCIndexClass)): - # return NotImplemented - if is_list_like(other) and len(other) != len(self): raise ValueError("Lengths must match") diff --git a/pandas/core/arrays/sparse.py b/pandas/core/arrays/sparse.py index a5d9656a845a4..82dada100dc9e 100644 --- a/pandas/core/arrays/sparse.py +++ b/pandas/core/arrays/sparse.py @@ -39,7 +39,6 @@ ) from pandas.core.dtypes.dtypes import register_extension_dtype from pandas.core.dtypes.generic import ( - ABCDataFrame, ABCIndexClass, ABCSeries, ABCSparseArray, @@ -1739,12 +1738,6 @@ def _create_arithmetic_method(cls, op): @unpack_and_defer(op_name) def sparse_arithmetic_method(self, other): - #if isinstance(other, (ABCDataFrame, ABCSeries, ABCIndexClass)): - # # Rely on pandas to dispatch to us. - # return NotImplemented - - #other = lib.item_from_zerodim(other) - if isinstance(other, SparseArray): return _sparse_array_op(self, other, op, op_name) @@ -1791,14 +1784,9 @@ def _create_comparison_method(cls, op): if op_name in {"and_", "or_"}: op_name = op_name[:-1] + @unpack_and_defer(op_name) def cmp_method(self, other): - if isinstance(other, (ABCDataFrame, ABCSeries, ABCIndexClass)): - # Rely on pandas to unbox and dispatch to us. - return NotImplemented - - other = lib.item_from_zerodim(other) - if not is_scalar(other) and not isinstance(other, type(self)): # convert list-like to ndarray other = np.asarray(other) diff --git a/pandas/core/arrays/timedeltas.py b/pandas/core/arrays/timedeltas.py index e3219a4a891fd..2c3396264805c 100644 --- a/pandas/core/arrays/timedeltas.py +++ b/pandas/core/arrays/timedeltas.py @@ -83,9 +83,6 @@ def _td_array_cmp(cls, op): @unpack_and_defer(opname) def wrapper(self, other): - other = lib.item_from_zerodim(other) - #if isinstance(other, (ABCDataFrame, ABCSeries, ABCIndexClass)): - # return NotImplemented if _is_convertible_to_td(other) or other is NaT: try: diff --git a/pandas/core/indexes/range.py b/pandas/core/indexes/range.py index 4632f912b2d76..374ee08f740bb 100644 --- a/pandas/core/indexes/range.py +++ b/pandas/core/indexes/range.py @@ -21,7 +21,7 @@ is_scalar, is_timedelta64_dtype, ) -from pandas.core.dtypes.generic import ABCDataFrame, ABCSeries, ABCTimedeltaIndex +from pandas.core.dtypes.generic import ABCTimedeltaIndex from pandas.core import ops from pandas.core.ops.common import unpack_and_defer @@ -733,8 +733,6 @@ def __getitem__(self, key): @unpack_and_defer("__floordiv__") def __floordiv__(self, other): - #if isinstance(other, (ABCSeries, ABCDataFrame)): - # return NotImplemented if is_integer(other) and other != 0: if len(self) == 0 or self.start % other == 0 and self.step % other == 0: @@ -771,8 +769,6 @@ def _make_evaluate_binop(op, step=False): """ @unpack_and_defer(op.__name__) def _evaluate_numeric_binop(self, other): - #if isinstance(other, (ABCSeries, ABCDataFrame)): - # return NotImplemented if isinstance(other, ABCTimedeltaIndex): # Defer to TimedeltaIndex implementation return NotImplemented diff --git a/pandas/core/ops/__init__.py b/pandas/core/ops/__init__.py index c801417a02a05..00295f828f4bb 100644 --- a/pandas/core/ops/__init__.py +++ b/pandas/core/ops/__init__.py @@ -680,12 +680,9 @@ def na_op(x, y): @unpack_and_defer(op_name) def wrapper(left, right): - #if isinstance(right, ABCDataFrame): - # return NotImplemented left, right = _align_method_SERIES(left, right) res_name = get_op_result_name(left, right) - right = lib.item_from_zerodim(right) right = maybe_upcast_for_op(right, left.shape) if is_categorical_dtype(left): @@ -819,7 +816,6 @@ def wrapper(self, other, axis=None): self._get_axis_number(axis) res_name = get_op_result_name(self, other) - other = lib.item_from_zerodim(other) # TODO: shouldn't we be applying finalize whenever # not isinstance(other, ABCSeries)? @@ -833,10 +829,6 @@ def wrapper(self, other, axis=None): # TODO: same for tuples? other = np.asarray(other) - #if isinstance(other, ABCDataFrame): # pragma: no cover - # # Defer to DataFrame implementation; fail early - # return NotImplemented - if isinstance(other, ABCSeries) and not self._indexed_same(other): raise ValueError("Can only compare identically-labeled Series objects") @@ -949,11 +941,6 @@ def wrapper(self, other): self, other = _align_method_SERIES(self, other, align_asobject=True) res_name = get_op_result_name(self, other) - other = lib.item_from_zerodim(other) - - #if isinstance(other, ABCDataFrame): - # # Defer to DataFrame implementation; fail early - # return NotImplemented if isinstance(other, (ABCSeries, ABCIndexClass)): is_other_int_dtype = is_integer_dtype(other.dtype) @@ -1287,8 +1274,6 @@ def _arith_method_SPARSE_SERIES(cls, op, special): @unpack_and_defer(op_name) def wrapper(self, other): - #if isinstance(other, ABCDataFrame): - # return NotImplemented if isinstance(other, ABCSeries): if not isinstance(other, ABCSparseSeries): other = other.to_sparse(fill_value=self.fill_value) From 7b00b025fb56c4bbc3ba14e59046516ba962849e Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Mon, 12 Aug 2019 08:24:44 -0700 Subject: [PATCH 06/23] OK --- pandas/core/indexes/base.py | 13 +++++++------ pandas/core/indexes/multi.py | 29 ++++++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py index bfada37f4fa00..6d26cce839a87 100644 --- a/pandas/core/indexes/base.py +++ b/pandas/core/indexes/base.py @@ -104,12 +104,14 @@ def cmp_method(self, other): if other.ndim > 0 and len(self) != len(other): raise ValueError("Lengths must match to compare") - if is_object_dtype(self) and not isinstance(self, ABCMultiIndex): + if is_object_dtype(self):# and not isinstance(self, ABCMultiIndex): # don't pass MultiIndex + assert not isinstance(self, ABCMultiIndex) with np.errstate(all="ignore"): result = ops._comp_method_OBJECT_ARRAY(op, self.values, other) else: + # TODO: define this on NumericIndex? with np.errstate(all="ignore"): result = op(self.values, np.asarray(other)) @@ -124,15 +126,14 @@ def cmp_method(self, other): return result name = "__{name}__".format(name=op.__name__) - # TODO: docstring? return set_function_name(cmp_method, name, cls) def _make_arithmetic_op(op, cls): + op_name = "__{name}__".format(name=op.__name__) + @unpack_and_defer(op.__name__) def index_arithmetic_method(self, other): - #if isinstance(other, (ABCSeries, ABCDataFrame)): - # return NotImplemented from pandas import Series @@ -141,8 +142,7 @@ def index_arithmetic_method(self, other): return (Index(result[0]), Index(result[1])) return Index(result) - name = "__{name}__".format(name=op.__name__) - return set_function_name(index_arithmetic_method, name, cls) + return set_function_name(index_arithmetic_method, op_name, cls) class InvalidIndexError(Exception): @@ -5411,6 +5411,7 @@ def _validate_for_numeric_binop(self, other, op): "{opstr} with type: {typ}".format(opstr=opstr, typ=type(other)) ) elif isinstance(other, np.ndarray) and not other.ndim: + assert False, other # this should already have been done. besides other.item() gets timedelta64 objects wrong IIRC other = other.item() if isinstance(other, (Index, ABCSeries, np.ndarray)): diff --git a/pandas/core/indexes/multi.py b/pandas/core/indexes/multi.py index b614952ba1e04..59c2492daecb7 100644 --- a/pandas/core/indexes/multi.py +++ b/pandas/core/indexes/multi.py @@ -1,5 +1,6 @@ from collections import OrderedDict import datetime +import operator from sys import getsizeof import warnings @@ -26,7 +27,7 @@ pandas_dtype, ) from pandas.core.dtypes.dtypes import ExtensionDtype -from pandas.core.dtypes.generic import ABCDataFrame +from pandas.core.dtypes.generic import ABCDataFrame, ABCSeries from pandas.core.dtypes.missing import array_equivalent, isna import pandas.core.algorithms as algos @@ -42,6 +43,7 @@ ) from pandas.core.indexes.frozen import FrozenList, _ensure_frozen import pandas.core.missing as missing +from pandas.core.ops.common import unpack_and_defer from pandas.core.sorting import ( get_group_index, indexer_from_factorized, @@ -61,6 +63,24 @@ ) +def _make_mi_comparison_op(op): + op_name = "__{name}__".format(name=op.__name__) + + @unpack_and_defer(op_name) + def cmp_method(self, other): + + if isinstance(other, (np.ndarray, Index, ABCSeries)): + if len(self) != len(other): + raise ValueError("Lengths must match to compare") + + with np.errstate(all="ignore"): + result = op(self.values, np.asarray(other)) + return result + + cmp_method.__name__ = op_name + return cmp_method + + class MultiIndexUIntEngine(libindex.BaseMultiIndexCodesEngine, libindex.UInt64Engine): """ This class manages a MultiIndex by mapping label combinations to positive @@ -3425,6 +3445,13 @@ def isin(self, values, level=None): else: return np.lib.arraysetops.in1d(level_codes, sought_labels) + __eq__ = _make_mi_comparison_op(operator.eq) + __ne__ = _make_mi_comparison_op(operator.ne) + __lt__ = _make_mi_comparison_op(operator.lt) + __le__ = _make_mi_comparison_op(operator.le) + __gt__ = _make_mi_comparison_op(operator.gt) + __ge__ = _make_mi_comparison_op(operator.ge) + MultiIndex._add_numeric_methods_disabled() MultiIndex._add_numeric_methods_add_sub_disabled() From c2e9fc6d196c442e3baeaa0b496cba322a8408b8 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Mon, 12 Aug 2019 09:20:22 -0700 Subject: [PATCH 07/23] cln --- pandas/core/common.py | 28 ++++++++++++++++++++++++++ pandas/core/computation/common.py | 4 ++-- pandas/core/computation/expressions.py | 20 +++++++++--------- pandas/core/indexes/base.py | 6 ++---- pandas/core/indexes/multi.py | 1 + pandas/core/indexes/range.py | 4 +++- 6 files changed, 45 insertions(+), 18 deletions(-) diff --git a/pandas/core/common.py b/pandas/core/common.py index a507625ccfa01..5cfea75354176 100644 --- a/pandas/core/common.py +++ b/pandas/core/common.py @@ -28,6 +28,34 @@ from pandas.core.dtypes.missing import isna, isnull, notnull # noqa +def pin_method_names(cls): + for attr in dir(cls): + if attr.startswith("__"): + if attr in ["__base__", "__call__", "__class__", "__delattr__", "__dir__"]: + continue + try: + meth = getattr(cls, attr) + except AttributeError: + # e.g. __abstractmethods__ + continue + if callable(meth): + if meth.__name__ != attr: + meth.__name__ = attr + + if getattr(meth, "__module__", None) != cls.__module__: + try: + meth.__module__ = cls.__module__ + except AttributeError: + pass + + if meth.__qualname__ != cls.__name__ + "." + attr: + try: + meth.__qualname__ = cls.__name__ + "." + attr + except AttributeError: + pass + return cls + + class SettingWithCopyError(ValueError): pass diff --git a/pandas/core/computation/common.py b/pandas/core/computation/common.py index b8e212fd2a32e..2a4277e4e0a20 100644 --- a/pandas/core/computation/common.py +++ b/pandas/core/computation/common.py @@ -2,7 +2,7 @@ import numpy as np -import pandas as pd +from pandas._config import get_option # A token value Python's tokenizer probably will never use. _BACKTICK_QUOTED_STRING = 100 @@ -11,7 +11,7 @@ def _ensure_decoded(s): """ if we have bytes, decode them to unicode """ if isinstance(s, (np.bytes_, bytes)): - s = s.decode(pd.get_option("display.encoding")) + s = s.decode(get_option("display.encoding")) return s diff --git a/pandas/core/computation/expressions.py b/pandas/core/computation/expressions.py index d9dc194d484ae..5fb8b1b5c7171 100644 --- a/pandas/core/computation/expressions.py +++ b/pandas/core/computation/expressions.py @@ -99,15 +99,13 @@ def _evaluate_numexpr(op, op_str, a, b, truediv=True, reversed=False, **eval_kwa result = None if _can_use_numexpr(op, op_str, a, b, "evaluate"): - try: - - # we were originally called by a reversed op - # method - if reversed: - a, b = b, a + if reversed: + # we were originally called by a reversed op method + a, b = b, a - a_value = getattr(a, "values", a) - b_value = getattr(b, "values", b) + a_value = getattr(a, "values", a) + b_value = getattr(b, "values", b) + try: result = ne.evaluate( "a_value {op} b_value".format(op=op_str), local_dict={"a_value": a_value, "b_value": b_value}, @@ -138,11 +136,11 @@ def _where_numexpr(cond, a, b): result = None if _can_use_numexpr(None, "where", a, b, "where"): + cond_value = getattr(cond, "values", cond) + a_value = getattr(a, "values", a) + b_value = getattr(b, "values", b) try: - cond_value = getattr(cond, "values", cond) - a_value = getattr(a, "values", a) - b_value = getattr(b, "values", b) result = ne.evaluate( "where(cond_value, a_value, b_value)", local_dict={ diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py index 6d26cce839a87..e19935d8e5a5f 100644 --- a/pandas/core/indexes/base.py +++ b/pandas/core/indexes/base.py @@ -67,6 +67,7 @@ from pandas.core.arrays import ExtensionArray from pandas.core.base import IndexOpsMixin, PandasObject import pandas.core.common as com +from pandas.core.construction import extract_array from pandas.core.indexers import maybe_convert_indices from pandas.core.indexes.frozen import FrozenList import pandas.core.missing as missing @@ -5410,14 +5411,11 @@ def _validate_for_numeric_binop(self, other, op): "cannot evaluate a numeric op " "{opstr} with type: {typ}".format(opstr=opstr, typ=type(other)) ) - elif isinstance(other, np.ndarray) and not other.ndim: - assert False, other # this should already have been done. besides other.item() gets timedelta64 objects wrong IIRC - other = other.item() if isinstance(other, (Index, ABCSeries, np.ndarray)): if len(self) != len(other): raise ValueError("cannot evaluate a numeric op with unequal lengths") - other = com.values_from_object(other) + other = extract_array(other, extract_numpy=True) if other.dtype.kind not in ["f", "i", "u"]: raise TypeError("cannot evaluate a numeric op with a non-numeric dtype") elif isinstance(other, (ABCDateOffset, np.timedelta64, timedelta)): diff --git a/pandas/core/indexes/multi.py b/pandas/core/indexes/multi.py index 59c2492daecb7..a574085a8fe3f 100644 --- a/pandas/core/indexes/multi.py +++ b/pandas/core/indexes/multi.py @@ -162,6 +162,7 @@ def _codes_to_ints(self, codes): return np.bitwise_or.reduce(codes, axis=1) +@com.pin_method_names class MultiIndex(Index): """ A multi-level, or hierarchical, index object for pandas objects. diff --git a/pandas/core/indexes/range.py b/pandas/core/indexes/range.py index 374ee08f740bb..2fa75b3dd7d30 100644 --- a/pandas/core/indexes/range.py +++ b/pandas/core/indexes/range.py @@ -26,6 +26,7 @@ from pandas.core import ops from pandas.core.ops.common import unpack_and_defer import pandas.core.common as com +from pandas.core.construction import extract_array import pandas.core.indexes.base as ibase from pandas.core.indexes.base import Index, _index_shared_docs from pandas.core.indexes.numeric import Int64Index @@ -780,7 +781,8 @@ def _evaluate_numeric_binop(self, other): # Must be an np.ndarray; GH#22390 return op(self._int64index, other) - other = self._validate_for_numeric_binop(other, op) + other = extract_array(other, extract_numpy=True) + #other = self._validate_for_numeric_binop(other, op) attrs = self._get_attributes_dict() attrs = self._maybe_update_attributes(attrs) From afbd04ad9b9c3e3e103d25d8356862ac635d5d48 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Mon, 12 Aug 2019 09:24:07 -0700 Subject: [PATCH 08/23] remove validate_for_numeric_binop --- pandas/core/indexes/base.py | 44 ------------------------------------ pandas/core/indexes/range.py | 1 - 2 files changed, 45 deletions(-) diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py index e19935d8e5a5f..5c35bc946bcc7 100644 --- a/pandas/core/indexes/base.py +++ b/pandas/core/indexes/base.py @@ -67,7 +67,6 @@ from pandas.core.arrays import ExtensionArray from pandas.core.base import IndexOpsMixin, PandasObject import pandas.core.common as com -from pandas.core.construction import extract_array from pandas.core.indexers import maybe_convert_indices from pandas.core.indexes.frozen import FrozenList import pandas.core.missing as missing @@ -5387,49 +5386,6 @@ def _validate_for_numeric_unaryop(self, op, opstr): "{opstr} for type: {typ}".format(opstr=opstr, typ=type(self).__name__) ) - def _validate_for_numeric_binop(self, other, op): - """ - Return valid other; evaluate or raise TypeError if we are not of - the appropriate type. - - Notes - ----- - This is an internal method called by ops. - """ - opstr = "__{opname}__".format(opname=op.__name__) - # if we are an inheritor of numeric, - # but not actually numeric (e.g. DatetimeIndex/PeriodIndex) - if not self._is_numeric_dtype: - raise TypeError( - "cannot evaluate a numeric op {opstr} " - "for type: {typ}".format(opstr=opstr, typ=type(self).__name__) - ) - - if isinstance(other, Index): - if not other._is_numeric_dtype: - raise TypeError( - "cannot evaluate a numeric op " - "{opstr} with type: {typ}".format(opstr=opstr, typ=type(other)) - ) - - if isinstance(other, (Index, ABCSeries, np.ndarray)): - if len(self) != len(other): - raise ValueError("cannot evaluate a numeric op with unequal lengths") - other = extract_array(other, extract_numpy=True) - if other.dtype.kind not in ["f", "i", "u"]: - raise TypeError("cannot evaluate a numeric op with a non-numeric dtype") - elif isinstance(other, (ABCDateOffset, np.timedelta64, timedelta)): - # higher up to handle - pass - elif isinstance(other, (datetime, np.datetime64)): - # higher up to handle - pass - else: - if not (is_float(other) or is_integer(other)): - raise TypeError("can only perform ops with scalar values") - - return other - @classmethod def _add_numeric_methods_binary(cls): """ diff --git a/pandas/core/indexes/range.py b/pandas/core/indexes/range.py index 2fa75b3dd7d30..1697a5fa01075 100644 --- a/pandas/core/indexes/range.py +++ b/pandas/core/indexes/range.py @@ -782,7 +782,6 @@ def _evaluate_numeric_binop(self, other): return op(self._int64index, other) other = extract_array(other, extract_numpy=True) - #other = self._validate_for_numeric_binop(other, op) attrs = self._get_attributes_dict() attrs = self._maybe_update_attributes(attrs) From f3c9c20b919c32315d6729ac5c25d88e1ac633e0 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Mon, 12 Aug 2019 11:05:30 -0700 Subject: [PATCH 09/23] OK --- pandas/core/arrays/datetimelike.py | 70 +++++++++------------- pandas/core/indexes/base.py | 4 +- pandas/tests/arithmetic/test_datetime64.py | 8 ++- 3 files changed, 38 insertions(+), 44 deletions(-) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index 1b7674f683ee6..9ecae7b04e2d8 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -34,11 +34,7 @@ is_unsigned_integer_dtype, pandas_dtype, ) -from pandas.core.dtypes.generic import ( - ABCIndexClass, - ABCPeriodArray, - ABCSeries, -) +from pandas.core.dtypes.generic import ABCIndexClass, ABCPeriodArray, ABCSeries from pandas.core.dtypes.inference import is_array_like from pandas.core.dtypes.missing import is_valid_nat_for_dtype, isna @@ -1228,29 +1224,24 @@ def __add__(self, other): if not is_period_dtype(self): maybe_integer_op_deprecated(self) result = self._addsub_int_array(other, operator.add) - elif is_float_dtype(other): - # Explicitly catch invalid dtypes - raise TypeError( - "cannot add {dtype}-dtype to {cls}".format( - dtype=other.dtype, cls=type(self).__name__ - ) - ) - elif is_period_dtype(other): - # if self is a TimedeltaArray and other is a PeriodArray with - # a timedelta-like (i.e. Tick) freq, this operation is valid. - # Defer to the PeriodArray implementation. - # In remaining cases, this will end up raising TypeError. - return NotImplemented - elif is_extension_array_dtype(other): - # Categorical op will raise; defer explicitly - return NotImplemented - else: # pragma: no cover + # elif is_float_dtype(other): + # # Explicitly catch invalid dtypes + # raise TypeError( + # "cannot add {dtype}-dtype to {cls}".format( + # dtype=other.dtype, cls=type(self).__name__ + # ) + # ) + else: + # Includes Categorical, other ExtensionArrays + # For PeriodDtype, if self is a TimedeltaArray and other is a + # PeriodArray with a timedelta-like (i.e. Tick) freq, this + # operation is valid. Defer to the PeriodArray implementation. + # In remaining cases, this will end up raising TypeError. return NotImplemented if is_timedelta64_dtype(result) and isinstance(result, np.ndarray): from pandas.core.arrays import TimedeltaArray - # TODO: infer freq? return TimedeltaArray(result) return result @@ -1298,29 +1289,26 @@ def __sub__(self, other): if not is_period_dtype(self): maybe_integer_op_deprecated(self) result = self._addsub_int_array(other, operator.sub) - elif isinstance(other, ABCIndexClass): - raise TypeError( - "cannot subtract {cls} and {typ}".format( - cls=type(self).__name__, typ=type(other).__name__ - ) - ) - elif is_float_dtype(other): - # Explicitly catch invalid dtypes - raise TypeError( - "cannot subtract {dtype}-dtype from {cls}".format( - dtype=other.dtype, cls=type(self).__name__ - ) - ) - elif is_extension_array_dtype(other): - # Categorical op will raise; defer explicitly - return NotImplemented - else: # pragma: no cover + # elif isinstance(other, ABCIndexClass): + # raise TypeError( + # "cannot subtract {cls} and {typ}".format( + # cls=type(self).__name__, typ=type(other).__name__ + # ) + # ) + # elif is_float_dtype(other): + # # Explicitly catch invalid dtypes + # raise TypeError( + # "cannot subtract {dtype}-dtype from {cls}".format( + # dtype=other.dtype, cls=type(self).__name__ + # ) + # ) + else: + # Includes ExtensionArrays, float_dtype return NotImplemented if is_timedelta64_dtype(result) and isinstance(result, np.ndarray): from pandas.core.arrays import TimedeltaArray - # TODO: infer freq? return TimedeltaArray(result) return result diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py index 5c35bc946bcc7..dbd48f0471120 100644 --- a/pandas/core/indexes/base.py +++ b/pandas/core/indexes/base.py @@ -104,7 +104,7 @@ def cmp_method(self, other): if other.ndim > 0 and len(self) != len(other): raise ValueError("Lengths must match to compare") - if is_object_dtype(self):# and not isinstance(self, ABCMultiIndex): + if is_object_dtype(self): # and not isinstance(self, ABCMultiIndex): # don't pass MultiIndex assert not isinstance(self, ABCMultiIndex) with np.errstate(all="ignore"): @@ -5418,7 +5418,7 @@ def _add_numeric_methods_unary(cls): def _make_evaluate_unary(op, opstr): def _evaluate_numeric_unary(self): - self._validate_for_numeric_unaryop(op, opstr) + # self._validate_for_numeric_unaryop(op, opstr) attrs = self._get_attributes_dict() attrs = self._maybe_update_attributes(attrs) return Index(op(self.values), **attrs) diff --git a/pandas/tests/arithmetic/test_datetime64.py b/pandas/tests/arithmetic/test_datetime64.py index 3920cfcc002d7..810e3f96feee7 100644 --- a/pandas/tests/arithmetic/test_datetime64.py +++ b/pandas/tests/arithmetic/test_datetime64.py @@ -1097,7 +1097,13 @@ def test_dt64arr_add_timestamp_raises(self, box_with_array): def test_dt64arr_add_sub_float(self, other, box_with_array): dti = DatetimeIndex(["2011-01-01", "2011-01-02"], freq="D") dtarr = tm.box_expected(dti, box_with_array) - msg = "|".join(["unsupported operand type", "cannot (add|subtract)"]) + msg = "|".join( + [ + "unsupported operand type", + "cannot (add|subtract)", + "cannot use operands with types", + ] + ) with pytest.raises(TypeError, match=msg): dtarr + other with pytest.raises(TypeError, match=msg): From 2c72628bd20d9841c2b5be0f3c5027bd4fc4ee6e Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Mon, 12 Aug 2019 11:06:18 -0700 Subject: [PATCH 10/23] delete commented-out --- pandas/core/arrays/datetimelike.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index 9ecae7b04e2d8..5ef7cf0ba7076 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -1224,13 +1224,6 @@ def __add__(self, other): if not is_period_dtype(self): maybe_integer_op_deprecated(self) result = self._addsub_int_array(other, operator.add) - # elif is_float_dtype(other): - # # Explicitly catch invalid dtypes - # raise TypeError( - # "cannot add {dtype}-dtype to {cls}".format( - # dtype=other.dtype, cls=type(self).__name__ - # ) - # ) else: # Includes Categorical, other ExtensionArrays # For PeriodDtype, if self is a TimedeltaArray and other is a @@ -1289,19 +1282,6 @@ def __sub__(self, other): if not is_period_dtype(self): maybe_integer_op_deprecated(self) result = self._addsub_int_array(other, operator.sub) - # elif isinstance(other, ABCIndexClass): - # raise TypeError( - # "cannot subtract {cls} and {typ}".format( - # cls=type(self).__name__, typ=type(other).__name__ - # ) - # ) - # elif is_float_dtype(other): - # # Explicitly catch invalid dtypes - # raise TypeError( - # "cannot subtract {dtype}-dtype from {cls}".format( - # dtype=other.dtype, cls=type(self).__name__ - # ) - # ) else: # Includes ExtensionArrays, float_dtype return NotImplemented From d19282d084b34066e19fc1eb47d906ce4c5b62fe Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Mon, 12 Aug 2019 12:05:31 -0700 Subject: [PATCH 11/23] OK --- pandas/core/indexes/base.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py index dbd48f0471120..f930ef8137bf4 100644 --- a/pandas/core/indexes/base.py +++ b/pandas/core/indexes/base.py @@ -104,8 +104,7 @@ def cmp_method(self, other): if other.ndim > 0 and len(self) != len(other): raise ValueError("Lengths must match to compare") - if is_object_dtype(self): # and not isinstance(self, ABCMultiIndex): - # don't pass MultiIndex + if is_object_dtype(self): assert not isinstance(self, ABCMultiIndex) with np.errstate(all="ignore"): result = ops._comp_method_OBJECT_ARRAY(op, self.values, other) @@ -120,6 +119,7 @@ def cmp_method(self, other): if is_bool_dtype(result): # TODO: This fails for exactly 1 test, with Int64Index and other="a" and == return result + #raise RuntimeError(other, op, self.dtype) try: return Index(result) except TypeError: @@ -5376,16 +5376,6 @@ def _maybe_update_attributes(self, attrs): """ return attrs - def _validate_for_numeric_unaryop(self, op, opstr): - """ - Validate if we can perform a numeric unary operation. - """ - if not self._is_numeric_dtype: - raise TypeError( - "cannot evaluate a numeric op " - "{opstr} for type: {typ}".format(opstr=opstr, typ=type(self).__name__) - ) - @classmethod def _add_numeric_methods_binary(cls): """ @@ -5417,8 +5407,6 @@ def _add_numeric_methods_unary(cls): def _make_evaluate_unary(op, opstr): def _evaluate_numeric_unary(self): - - # self._validate_for_numeric_unaryop(op, opstr) attrs = self._get_attributes_dict() attrs = self._maybe_update_attributes(attrs) return Index(op(self.values), **attrs) From 242d7a0141392bb49d4d07b7878b0751c6a58c50 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Mon, 12 Aug 2019 13:44:56 -0700 Subject: [PATCH 12/23] cleanup --- pandas/core/indexes/base.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py index f930ef8137bf4..2300dabd82154 100644 --- a/pandas/core/indexes/base.py +++ b/pandas/core/indexes/base.py @@ -112,18 +112,12 @@ def cmp_method(self, other): else: # TODO: define this on NumericIndex? with np.errstate(all="ignore"): - result = op(self.values, np.asarray(other)) + result = op(self._values, np.asarray(other)) + - # technically we could support bool dtyped Index - # for now just return the indexing array directly if is_bool_dtype(result): - # TODO: This fails for exactly 1 test, with Int64Index and other="a" and == - return result - #raise RuntimeError(other, op, self.dtype) - try: - return Index(result) - except TypeError: return result + return ops.invalid_comparison(self, other, op) name = "__{name}__".format(name=op.__name__) return set_function_name(cmp_method, name, cls) From 6149859d5e0c1459a5e52e5d3e1e1ed16946f9c3 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Mon, 19 Aug 2019 13:29:11 -0700 Subject: [PATCH 13/23] revert unnecessary --- pandas/core/common.py | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/pandas/core/common.py b/pandas/core/common.py index 5cfea75354176..a507625ccfa01 100644 --- a/pandas/core/common.py +++ b/pandas/core/common.py @@ -28,34 +28,6 @@ from pandas.core.dtypes.missing import isna, isnull, notnull # noqa -def pin_method_names(cls): - for attr in dir(cls): - if attr.startswith("__"): - if attr in ["__base__", "__call__", "__class__", "__delattr__", "__dir__"]: - continue - try: - meth = getattr(cls, attr) - except AttributeError: - # e.g. __abstractmethods__ - continue - if callable(meth): - if meth.__name__ != attr: - meth.__name__ = attr - - if getattr(meth, "__module__", None) != cls.__module__: - try: - meth.__module__ = cls.__module__ - except AttributeError: - pass - - if meth.__qualname__ != cls.__name__ + "." + attr: - try: - meth.__qualname__ = cls.__name__ + "." + attr - except AttributeError: - pass - return cls - - class SettingWithCopyError(ValueError): pass From 05920a1fba174325e92c810e841f0e7de16a66b2 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Mon, 19 Aug 2019 13:30:44 -0700 Subject: [PATCH 14/23] revert --- pandas/core/indexes/base.py | 18 ++++++++---------- pandas/core/indexes/multi.py | 30 +----------------------------- 2 files changed, 9 insertions(+), 39 deletions(-) diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py index 56dc58e9a7242..b983117478c61 100644 --- a/pandas/core/indexes/base.py +++ b/pandas/core/indexes/base.py @@ -72,7 +72,6 @@ import pandas.core.missing as missing from pandas.core.ops import get_op_result_name from pandas.core.ops.invalid import make_invalid_op -from pandas.core.ops.common import unpack_and_defer import pandas.core.sorting as sorting from pandas.core.strings import StringMethods @@ -107,16 +106,14 @@ def cmp_method(self, other): if is_object_dtype(self) and isinstance(other, ABCCategorical): left = type(other)(self._values, dtype=other.dtype) return op(left, other) - elif is_object_dtype(self): - assert not isinstance(self, ABCMultiIndex) + elif is_object_dtype(self) and not isinstance(self, ABCMultiIndex): + # don't pass MultiIndex with np.errstate(all="ignore"): result = ops.comp_method_OBJECT_ARRAY(op, self.values, other) else: - # TODO: define this on NumericIndex? with np.errstate(all="ignore"): - result = op(self._values, np.asarray(other)) - + result = op(self.values, np.asarray(other)) if is_bool_dtype(result): return result @@ -127,10 +124,9 @@ def cmp_method(self, other): def _make_arithmetic_op(op, cls): - op_name = "__{name}__".format(name=op.__name__) - - @unpack_and_defer(op.__name__) def index_arithmetic_method(self, other): + if isinstance(other, (ABCSeries, ABCDataFrame, ABCTimedeltaIndex)): + return NotImplemented from pandas import Series @@ -139,7 +135,9 @@ def index_arithmetic_method(self, other): return (Index(result[0]), Index(result[1])) return Index(result) - return set_function_name(index_arithmetic_method, op_name, cls) + name = "__{name}__".format(name=op.__name__) + # TODO: docstring? + return set_function_name(index_arithmetic_method, name, cls) class InvalidIndexError(Exception): diff --git a/pandas/core/indexes/multi.py b/pandas/core/indexes/multi.py index a574085a8fe3f..b614952ba1e04 100644 --- a/pandas/core/indexes/multi.py +++ b/pandas/core/indexes/multi.py @@ -1,6 +1,5 @@ from collections import OrderedDict import datetime -import operator from sys import getsizeof import warnings @@ -27,7 +26,7 @@ pandas_dtype, ) from pandas.core.dtypes.dtypes import ExtensionDtype -from pandas.core.dtypes.generic import ABCDataFrame, ABCSeries +from pandas.core.dtypes.generic import ABCDataFrame from pandas.core.dtypes.missing import array_equivalent, isna import pandas.core.algorithms as algos @@ -43,7 +42,6 @@ ) from pandas.core.indexes.frozen import FrozenList, _ensure_frozen import pandas.core.missing as missing -from pandas.core.ops.common import unpack_and_defer from pandas.core.sorting import ( get_group_index, indexer_from_factorized, @@ -63,24 +61,6 @@ ) -def _make_mi_comparison_op(op): - op_name = "__{name}__".format(name=op.__name__) - - @unpack_and_defer(op_name) - def cmp_method(self, other): - - if isinstance(other, (np.ndarray, Index, ABCSeries)): - if len(self) != len(other): - raise ValueError("Lengths must match to compare") - - with np.errstate(all="ignore"): - result = op(self.values, np.asarray(other)) - return result - - cmp_method.__name__ = op_name - return cmp_method - - class MultiIndexUIntEngine(libindex.BaseMultiIndexCodesEngine, libindex.UInt64Engine): """ This class manages a MultiIndex by mapping label combinations to positive @@ -162,7 +142,6 @@ def _codes_to_ints(self, codes): return np.bitwise_or.reduce(codes, axis=1) -@com.pin_method_names class MultiIndex(Index): """ A multi-level, or hierarchical, index object for pandas objects. @@ -3446,13 +3425,6 @@ def isin(self, values, level=None): else: return np.lib.arraysetops.in1d(level_codes, sought_labels) - __eq__ = _make_mi_comparison_op(operator.eq) - __ne__ = _make_mi_comparison_op(operator.ne) - __lt__ = _make_mi_comparison_op(operator.lt) - __le__ = _make_mi_comparison_op(operator.le) - __gt__ = _make_mi_comparison_op(operator.gt) - __ge__ = _make_mi_comparison_op(operator.ge) - MultiIndex._add_numeric_methods_disabled() MultiIndex._add_numeric_methods_add_sub_disabled() From d6adfb004ca67fdb785a25b71d18d14fcd9298c4 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Mon, 19 Aug 2019 13:41:30 -0700 Subject: [PATCH 15/23] docstring --- pandas/core/ops/common.py | 75 ++++++++++++++++++++++----------------- 1 file changed, 42 insertions(+), 33 deletions(-) diff --git a/pandas/core/ops/common.py b/pandas/core/ops/common.py index ad49b3086f64c..499ba5b8da20f 100644 --- a/pandas/core/ops/common.py +++ b/pandas/core/ops/common.py @@ -3,54 +3,63 @@ """ from functools import wraps -import numpy as np - from pandas._libs.lib import item_from_zerodim from pandas.core.dtypes.generic import ABCDataFrame, ABCSeries, ABCIndexClass -from pandas.core.dtypes.common import is_list_like -def unpack_arraylike(obj, match_len: int): - obj = item_from_zerodim(obj) +def unpack_and_defer(name: str): + """ + Boilerplate for pandas conventions in arithmetic and comparison methods. + + Parameters + ---------- + name : str - if is_list_like(obj) and len(obj) != match_len: - # NB: we must have already checked for if we need to - # return NotImplemented for DataFrame - raise ValueError("Lengths must match") + Returns + ------- + decorator + """ + def wrapper(method): + return _unpack_and_defer(method, name) - if isinstance(obj, list): - # TODO: what about tuple? - obj = np.asarray(obj) - return obj + return wrapper -def unpack_and_defer(name): - def wrapper(method): - return _unpack_and_defer(method, name) +def _unpack_and_defer(method, name): + """ + Boilerplate for pandas conventions in arithmetic and comparison methods. - return wrapper + Ensure method returns NotImplemented when operating against "senior" + classes. Ensure zero-dimensional ndarrays are always unpacked. + Parameters + ---------- + method : binary method + name : str -def _unpack_and_defer(method, name): + Returns + ------- + method + """ - is_cmp = name.strip('__') in {'eq', 'ne', 'lt', 'le', 'gt', 'ge'} + is_cmp = name.strip('__') in {'eq', 'ne', 'lt', 'le', 'gt', 'ge'} - @wraps(method) - def new_method(self, other): + @wraps(method) + def new_method(self, other): - for cls in [ABCDataFrame, ABCSeries, ABCIndexClass]: - if isinstance(self, cls): - break - if isinstance(other, cls): - if is_cmp and cls is ABCSeries and isinstance(self, ABCIndexClass): - # For comparison ops, Index does *not* defer to Series - break - else: - return NotImplemented + if is_cmp and isinstance(self, ABCIndexClass) and isinstance(other, ABCSeries): + # For comparison ops, Index does *not* defer to Series + pass + else: + for cls in [ABCDataFrame, ABCSeries, ABCIndexClass]: + if isinstance(self, cls): + break + if isinstance(other, cls): + return NotImplemented - other = item_from_zerodim(other) + other = item_from_zerodim(other) - return method(self, other) + return method(self, other) - return new_method + return new_method From 9e30806d051f49f85042a0d7a585f59275b23b6f Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 20 Aug 2019 09:23:20 -0700 Subject: [PATCH 16/23] import fixup --- pandas/core/arrays/categorical.py | 2 +- pandas/core/arrays/datetimelike.py | 2 +- pandas/core/arrays/datetimes.py | 8 ++------ pandas/core/arrays/integer.py | 2 +- pandas/core/arrays/sparse.py | 1 - pandas/core/arrays/timedeltas.py | 2 +- pandas/core/ops/__init__.py | 3 +-- pandas/core/ops/common.py | 2 +- 8 files changed, 8 insertions(+), 14 deletions(-) diff --git a/pandas/core/arrays/categorical.py b/pandas/core/arrays/categorical.py index 24470ab2a9a98..87db8039e0ccb 100644 --- a/pandas/core/arrays/categorical.py +++ b/pandas/core/arrays/categorical.py @@ -59,8 +59,8 @@ import pandas.core.common as com from pandas.core.construction import extract_array, sanitize_array from pandas.core.missing import interpolate_2d -from pandas.core.sorting import nargsort from pandas.core.ops.common import unpack_and_defer +from pandas.core.sorting import nargsort from pandas.io.formats import console diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index 0aca46775bd3e..3153c0c3865fa 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -41,8 +41,8 @@ from pandas.core import missing, nanops from pandas.core.algorithms import checked_add_with_arr, take, unique1d, value_counts import pandas.core.common as com -from pandas.core.ops.invalid import make_invalid_op from pandas.core.ops.common import unpack_and_defer +from pandas.core.ops.invalid import make_invalid_op from pandas.tseries import frequencies from pandas.tseries.offsets import DateOffset, Tick diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index b405dcb136a19..1307e0c90aedd 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -40,19 +40,15 @@ pandas_dtype, ) from pandas.core.dtypes.dtypes import DatetimeTZDtype -from pandas.core.dtypes.generic import ( - ABCIndexClass, - ABCPandasArray, - ABCSeries, -) +from pandas.core.dtypes.generic import ABCIndexClass, ABCPandasArray, ABCSeries from pandas.core.dtypes.missing import isna from pandas.core import ops -from pandas.core.ops.common import unpack_and_defer from pandas.core.algorithms import checked_add_with_arr from pandas.core.arrays import datetimelike as dtl from pandas.core.arrays._ranges import generate_regular_range import pandas.core.common as com +from pandas.core.ops.common import unpack_and_defer from pandas.core.ops.invalid import invalid_comparison from pandas.tseries.frequencies import get_period_alias, to_offset diff --git a/pandas/core/arrays/integer.py b/pandas/core/arrays/integer.py index ac351398ed7c1..76af5c85843ff 100644 --- a/pandas/core/arrays/integer.py +++ b/pandas/core/arrays/integer.py @@ -24,9 +24,9 @@ from pandas.core.dtypes.missing import isna, notna from pandas.core import nanops, ops -from pandas.core.ops.common import unpack_and_defer from pandas.core.algorithms import take from pandas.core.arrays import ExtensionArray, ExtensionOpsMixin +from pandas.core.ops.common import unpack_and_defer from pandas.core.tools.numeric import to_numeric diff --git a/pandas/core/arrays/sparse.py b/pandas/core/arrays/sparse.py index 5d08daeba5742..a067e5774da7d 100644 --- a/pandas/core/arrays/sparse.py +++ b/pandas/core/arrays/sparse.py @@ -39,7 +39,6 @@ ) from pandas.core.dtypes.dtypes import register_extension_dtype from pandas.core.dtypes.generic import ( - ABCDataFrame, ABCIndexClass, ABCSeries, ABCSparseArray, diff --git a/pandas/core/arrays/timedeltas.py b/pandas/core/arrays/timedeltas.py index cf93aebb3f83d..47c58dc3777ff 100644 --- a/pandas/core/arrays/timedeltas.py +++ b/pandas/core/arrays/timedeltas.py @@ -43,8 +43,8 @@ from pandas.core.algorithms import checked_add_with_arr import pandas.core.common as com -from pandas.core.ops.invalid import invalid_comparison from pandas.core.ops.common import unpack_and_defer +from pandas.core.ops.invalid import invalid_comparison from pandas.tseries.frequencies import to_offset from pandas.tseries.offsets import Tick diff --git a/pandas/core/ops/__init__.py b/pandas/core/ops/__init__.py index fe24fff2f9fef..95fa8f0cf5907 100644 --- a/pandas/core/ops/__init__.py +++ b/pandas/core/ops/__init__.py @@ -41,6 +41,7 @@ from pandas._typing import ArrayLike from pandas.core.construction import array, extract_array from pandas.core.ops.array_ops import comp_method_OBJECT_ARRAY, define_na_arithmetic_op +from pandas.core.ops.common import unpack_and_defer from pandas.core.ops.docstrings import ( _arith_doc_FRAME, _flex_comp_doc_FRAME, @@ -66,8 +67,6 @@ rtruediv, rxor, ) -from pandas.core.ops.common import unpack_and_defer - # ----------------------------------------------------------------------------- # Ops Wrapping Utilities diff --git a/pandas/core/ops/common.py b/pandas/core/ops/common.py index 499ba5b8da20f..58a55a99cc2a5 100644 --- a/pandas/core/ops/common.py +++ b/pandas/core/ops/common.py @@ -5,7 +5,7 @@ from pandas._libs.lib import item_from_zerodim -from pandas.core.dtypes.generic import ABCDataFrame, ABCSeries, ABCIndexClass +from pandas.core.dtypes.generic import ABCDataFrame, ABCIndexClass, ABCSeries def unpack_and_defer(name: str): From 63d8f9c3613fcad32b2e52948ae9df092889d665 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 20 Aug 2019 09:58:17 -0700 Subject: [PATCH 17/23] blackify --- pandas/core/indexes/range.py | 1 + pandas/core/ops/common.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pandas/core/indexes/range.py b/pandas/core/indexes/range.py index 496b1069f16c7..162b4014b73b5 100644 --- a/pandas/core/indexes/range.py +++ b/pandas/core/indexes/range.py @@ -768,6 +768,7 @@ def _make_evaluate_binop(op, step=False): op to apply to the step parm if not None if False, use the existing step """ + @unpack_and_defer(op.__name__) def _evaluate_numeric_binop(self, other): if isinstance(other, ABCTimedeltaIndex): diff --git a/pandas/core/ops/common.py b/pandas/core/ops/common.py index 58a55a99cc2a5..8c917faac0c2a 100644 --- a/pandas/core/ops/common.py +++ b/pandas/core/ops/common.py @@ -20,6 +20,7 @@ def unpack_and_defer(name: str): ------- decorator """ + def wrapper(method): return _unpack_and_defer(method, name) @@ -43,7 +44,7 @@ def _unpack_and_defer(method, name): method """ - is_cmp = name.strip('__') in {'eq', 'ne', 'lt', 'le', 'gt', 'ge'} + is_cmp = name.strip("__") in {"eq", "ne", "lt", "le", "gt", "ge"} @wraps(method) def new_method(self, other): From 98f02b953e9e1d39c1d7045509d896aefe7a94ce Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 20 Aug 2019 09:58:36 -0700 Subject: [PATCH 18/23] isort fixup --- pandas/core/indexes/range.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/indexes/range.py b/pandas/core/indexes/range.py index 162b4014b73b5..d864ac92cdd7a 100644 --- a/pandas/core/indexes/range.py +++ b/pandas/core/indexes/range.py @@ -24,12 +24,12 @@ from pandas.core.dtypes.generic import ABCTimedeltaIndex from pandas.core import ops -from pandas.core.ops.common import unpack_and_defer import pandas.core.common as com from pandas.core.construction import extract_array import pandas.core.indexes.base as ibase from pandas.core.indexes.base import Index, _index_shared_docs from pandas.core.indexes.numeric import Int64Index +from pandas.core.ops.common import unpack_and_defer from pandas.io.formats.printing import pprint_thing From f84258f5548307f5c574f6d72b53f0ae22ec60df Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 20 Aug 2019 10:00:49 -0700 Subject: [PATCH 19/23] type ignore --- pandas/core/arrays/datetimelike.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index 3153c0c3865fa..7c6d0896b6e33 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -1323,11 +1323,11 @@ def __rsub__(self, other): return -(self - other) # FIXME: DTA/TDA/PA inplace methods should actually be inplace, GH#24115 - def __iadd__(self, other): + def __iadd__(self, other): # type: ignore # alias for __add__ return self.__add__(other) - def __isub__(self, other): + def __isub__(self, other): # type: ignore # alias for __sub__ return self.__sub__(other) From 66fd7cc51be60b17eb928861d465fb25d92c5a6e Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Mon, 9 Sep 2019 07:43:58 -0700 Subject: [PATCH 20/23] add str type --- pandas/core/ops/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/ops/common.py b/pandas/core/ops/common.py index 8c917faac0c2a..b49948330fed0 100644 --- a/pandas/core/ops/common.py +++ b/pandas/core/ops/common.py @@ -27,7 +27,7 @@ def wrapper(method): return wrapper -def _unpack_and_defer(method, name): +def _unpack_and_defer(method, name: str): """ Boilerplate for pandas conventions in arithmetic and comparison methods. From 85fb1c1ef9e2af832fb25d12c1d5c163cdc1d559 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Mon, 16 Sep 2019 07:25:42 -0700 Subject: [PATCH 21/23] lint fixup --- pandas/core/ops/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pandas/core/ops/__init__.py b/pandas/core/ops/__init__.py index 088be6bfa0e84..85fc83822803f 100644 --- a/pandas/core/ops/__init__.py +++ b/pandas/core/ops/__init__.py @@ -40,12 +40,12 @@ from pandas._typing import ArrayLike from pandas.core.construction import array, extract_array -from pandas.core.ops.common import unpack_and_defer from pandas.core.ops.array_ops import ( comp_method_OBJECT_ARRAY, define_na_arithmetic_op, na_arithmetic_op, ) +from pandas.core.ops.common import unpack_and_defer from pandas.core.ops.docstrings import ( _arith_doc_FRAME, _flex_comp_doc_FRAME, @@ -632,8 +632,6 @@ def _arith_method_SERIES(cls, op, special): _construct_divmod_result if op in [divmod, rdivmod] else _construct_result ) - na_op = define_na_arithmetic_op(op, str_rep, eval_kwargs) - @unpack_and_defer(op_name) def wrapper(left, right): From 6947bc02419d4e4c8b4abca2099ebcd6d317ba18 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 18 Sep 2019 08:54:42 -0700 Subject: [PATCH 22/23] blackify --- pandas/core/arrays/sparse.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pandas/core/arrays/sparse.py b/pandas/core/arrays/sparse.py index e94754647e4f1..c164f54abc243 100644 --- a/pandas/core/arrays/sparse.py +++ b/pandas/core/arrays/sparse.py @@ -38,11 +38,7 @@ pandas_dtype, ) from pandas.core.dtypes.dtypes import register_extension_dtype -from pandas.core.dtypes.generic import ( - ABCIndexClass, - ABCSeries, - ABCSparseArray, -) +from pandas.core.dtypes.generic import ABCIndexClass, ABCSeries, ABCSparseArray from pandas.core.dtypes.missing import isna, na_value_for_dtype, notna from pandas._typing import Dtype From bc2c0d57c3f9f7f3a6f3b9bd1e7bb811ce85d4ff Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 13 Nov 2019 16:48:39 -0800 Subject: [PATCH 23/23] rename unpack_and_defer -> unpack_zerodim_and_defer --- pandas/core/arrays/categorical.py | 4 ++-- pandas/core/arrays/datetimelike.py | 6 +++--- pandas/core/arrays/datetimes.py | 4 ++-- pandas/core/arrays/integer.py | 6 +++--- pandas/core/arrays/period.py | 4 ++-- pandas/core/arrays/sparse/array.py | 6 +++--- pandas/core/arrays/timedeltas.py | 4 ++-- pandas/core/indexes/range.py | 6 +++--- pandas/core/ops/__init__.py | 8 ++++---- pandas/core/ops/common.py | 6 +++--- 10 files changed, 27 insertions(+), 27 deletions(-) diff --git a/pandas/core/arrays/categorical.py b/pandas/core/arrays/categorical.py index 0390c40ae9edc..817972b3356a2 100644 --- a/pandas/core/arrays/categorical.py +++ b/pandas/core/arrays/categorical.py @@ -52,7 +52,7 @@ import pandas.core.common as com from pandas.core.construction import array, extract_array, sanitize_array from pandas.core.missing import interpolate_2d -from pandas.core.ops.common import unpack_and_defer +from pandas.core.ops.common import unpack_zerodim_and_defer from pandas.core.sorting import nargsort from pandas.io.formats import console @@ -75,7 +75,7 @@ def _cat_compare_op(op): opname = "__{op}__".format(op=op.__name__) - @unpack_and_defer(opname) + @unpack_zerodim_and_defer(opname) def f(self, other): # On python2, you can usually compare any type to any type, and # Categoricals can be seen as a custom type, but having different diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index e599b3cbf7f12..e52bc17fcc319 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -41,7 +41,7 @@ from pandas.core import missing, nanops from pandas.core.algorithms import checked_add_with_arr, take, unique1d, value_counts import pandas.core.common as com -from pandas.core.ops.common import unpack_and_defer +from pandas.core.ops.common import unpack_zerodim_and_defer from pandas.core.ops.invalid import make_invalid_op from pandas.tseries import frequencies @@ -1190,7 +1190,7 @@ def _time_shift(self, periods, freq=None): # to be passed explicitly. return self._generate_range(start=start, end=end, periods=None, freq=self.freq) - @unpack_and_defer("__add__") + @unpack_zerodim_and_defer("__add__") def __add__(self, other): # scalar others @@ -1242,7 +1242,7 @@ def __radd__(self, other): # alias for __add__ return self.__add__(other) - @unpack_and_defer("__sub__") + @unpack_zerodim_and_defer("__sub__") def __sub__(self, other): # scalar others diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index 14e9ab2c1f780..8e3c727a14c99 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -48,7 +48,7 @@ from pandas.core.arrays import datetimelike as dtl from pandas.core.arrays._ranges import generate_regular_range import pandas.core.common as com -from pandas.core.ops.common import unpack_and_defer +from pandas.core.ops.common import unpack_zerodim_and_defer from pandas.core.ops.invalid import invalid_comparison from pandas.tseries.frequencies import get_period_alias, to_offset @@ -153,7 +153,7 @@ def _dt_array_cmp(cls, op): opname = "__{name}__".format(name=op.__name__) nat_result = opname == "__ne__" - @unpack_and_defer(opname) + @unpack_zerodim_and_defer(opname) def wrapper(self, other): if isinstance(other, (datetime, np.datetime64, str)): diff --git a/pandas/core/arrays/integer.py b/pandas/core/arrays/integer.py index e59821e0f7d57..e167e556b244a 100644 --- a/pandas/core/arrays/integer.py +++ b/pandas/core/arrays/integer.py @@ -26,7 +26,7 @@ from pandas.core import nanops, ops from pandas.core.algorithms import take from pandas.core.arrays import ExtensionArray, ExtensionOpsMixin -from pandas.core.ops.common import unpack_and_defer +from pandas.core.ops.common import unpack_zerodim_and_defer from pandas.core.tools.numeric import to_numeric @@ -602,7 +602,7 @@ def _values_for_argsort(self) -> np.ndarray: def _create_comparison_method(cls, op): op_name = op.__name__ - @unpack_and_defer(op.__name__) + @unpack_zerodim_and_defer(op.__name__) def cmp_method(self, other): mask = None @@ -692,7 +692,7 @@ def _maybe_mask_result(self, result, mask, other, op_name): def _create_arithmetic_method(cls, op): op_name = op.__name__ - @unpack_and_defer(op.__name__) + @unpack_zerodim_and_defer(op.__name__) def integer_arithmetic_method(self, other): mask = None diff --git a/pandas/core/arrays/period.py b/pandas/core/arrays/period.py index 0c37018f31c75..fdf4059fad569 100644 --- a/pandas/core/arrays/period.py +++ b/pandas/core/arrays/period.py @@ -44,7 +44,7 @@ import pandas.core.algorithms as algos from pandas.core.arrays import datetimelike as dtl import pandas.core.common as com -from pandas.core.ops.common import unpack_and_defer +from pandas.core.ops.common import unpack_zerodim_and_defer from pandas.tseries import frequencies from pandas.tseries.offsets import DateOffset, Tick, _delta_to_tick @@ -68,7 +68,7 @@ def _period_array_cmp(cls, op): opname = "__{name}__".format(name=op.__name__) nat_result = opname == "__ne__" - @unpack_and_defer(opname) + @unpack_zerodim_and_defer(opname) def wrapper(self, other): ordinal_op = getattr(self.asi8, opname) diff --git a/pandas/core/arrays/sparse/array.py b/pandas/core/arrays/sparse/array.py index 828328f372407..943dea4252499 100644 --- a/pandas/core/arrays/sparse/array.py +++ b/pandas/core/arrays/sparse/array.py @@ -44,7 +44,7 @@ from pandas.core.construction import sanitize_array from pandas.core.missing import interpolate_2d import pandas.core.ops as ops -from pandas.core.ops.common import unpack_and_defer +from pandas.core.ops.common import unpack_zerodim_and_defer import pandas.io.formats.printing as printing @@ -1406,7 +1406,7 @@ def sparse_unary_method(self): def _create_arithmetic_method(cls, op): op_name = op.__name__ - @unpack_and_defer(op_name) + @unpack_zerodim_and_defer(op_name) def sparse_arithmetic_method(self, other): if isinstance(other, SparseArray): @@ -1455,7 +1455,7 @@ def _create_comparison_method(cls, op): if op_name in {"and_", "or_"}: op_name = op_name[:-1] - @unpack_and_defer(op_name) + @unpack_zerodim_and_defer(op_name) def cmp_method(self, other): if not is_scalar(other) and not isinstance(other, type(self)): diff --git a/pandas/core/arrays/timedeltas.py b/pandas/core/arrays/timedeltas.py index 701f1a16af277..816beb758dd33 100644 --- a/pandas/core/arrays/timedeltas.py +++ b/pandas/core/arrays/timedeltas.py @@ -45,7 +45,7 @@ from pandas.core import nanops from pandas.core.algorithms import checked_add_with_arr import pandas.core.common as com -from pandas.core.ops.common import unpack_and_defer +from pandas.core.ops.common import unpack_zerodim_and_defer from pandas.core.ops.invalid import invalid_comparison from pandas.tseries.frequencies import to_offset @@ -83,7 +83,7 @@ def _td_array_cmp(cls, op): opname = "__{name}__".format(name=op.__name__) nat_result = opname == "__ne__" - @unpack_and_defer(opname) + @unpack_zerodim_and_defer(opname) def wrapper(self, other): if _is_convertible_to_td(other) or other is NaT: diff --git a/pandas/core/indexes/range.py b/pandas/core/indexes/range.py index 8f1114c1c3f0a..f5cb435b8c1c2 100644 --- a/pandas/core/indexes/range.py +++ b/pandas/core/indexes/range.py @@ -29,7 +29,7 @@ import pandas.core.indexes.base as ibase from pandas.core.indexes.base import Index, _index_shared_docs from pandas.core.indexes.numeric import Int64Index -from pandas.core.ops.common import unpack_and_defer +from pandas.core.ops.common import unpack_zerodim_and_defer from pandas.io.formats.printing import pprint_thing @@ -735,7 +735,7 @@ def __getitem__(self, key): # fall back to Int64Index return super().__getitem__(key) - @unpack_and_defer("__floordiv__") + @unpack_zerodim_and_defer("__floordiv__") def __floordiv__(self, other): if is_integer(other) and other != 0: @@ -772,7 +772,7 @@ def _make_evaluate_binop(op, step=False): if False, use the existing step """ - @unpack_and_defer(op.__name__) + @unpack_zerodim_and_defer(op.__name__) def _evaluate_numeric_binop(self, other): if isinstance(other, ABCTimedeltaIndex): # Defer to TimedeltaIndex implementation diff --git a/pandas/core/ops/__init__.py b/pandas/core/ops/__init__.py index 727c529ea14e0..f7a1258894b89 100644 --- a/pandas/core/ops/__init__.py +++ b/pandas/core/ops/__init__.py @@ -29,7 +29,7 @@ logical_op, ) from pandas.core.ops.array_ops import comp_method_OBJECT_ARRAY # noqa:F401 -from pandas.core.ops.common import unpack_and_defer +from pandas.core.ops.common import unpack_zerodim_and_defer from pandas.core.ops.dispatch import maybe_dispatch_ufunc_to_dunder_op # noqa:F401 from pandas.core.ops.dispatch import should_series_dispatch from pandas.core.ops.docstrings import ( @@ -490,7 +490,7 @@ def _arith_method_SERIES(cls, op, special): op_name = _get_op_name(op, special) eval_kwargs = _gen_eval_kwargs(op_name) - @unpack_and_defer(op_name) + @unpack_zerodim_and_defer(op_name) def wrapper(left, right): left, right = _align_method_SERIES(left, right) @@ -512,7 +512,7 @@ def _comp_method_SERIES(cls, op, special): """ op_name = _get_op_name(op, special) - @unpack_and_defer(op_name) + @unpack_zerodim_and_defer(op_name) def wrapper(self, other): res_name = get_op_result_name(self, other) @@ -538,7 +538,7 @@ def _bool_method_SERIES(cls, op, special): """ op_name = _get_op_name(op, special) - @unpack_and_defer(op_name) + @unpack_zerodim_and_defer(op_name) def wrapper(self, other): self, other = _align_method_SERIES(self, other, align_asobject=True) res_name = get_op_result_name(self, other) diff --git a/pandas/core/ops/common.py b/pandas/core/ops/common.py index b49948330fed0..f4b16cf4a0cf2 100644 --- a/pandas/core/ops/common.py +++ b/pandas/core/ops/common.py @@ -8,7 +8,7 @@ from pandas.core.dtypes.generic import ABCDataFrame, ABCIndexClass, ABCSeries -def unpack_and_defer(name: str): +def unpack_zerodim_and_defer(name: str): """ Boilerplate for pandas conventions in arithmetic and comparison methods. @@ -22,12 +22,12 @@ def unpack_and_defer(name: str): """ def wrapper(method): - return _unpack_and_defer(method, name) + return _unpack_zerodim_and_defer(method, name) return wrapper -def _unpack_and_defer(method, name: str): +def _unpack_zerodim_and_defer(method, name: str): """ Boilerplate for pandas conventions in arithmetic and comparison methods.