From d60f317e21509371bf45d4237ccdab3b9550131c Mon Sep 17 00:00:00 2001 From: makbigc Date: Sun, 20 Jan 2019 14:40:34 +0800 Subject: [PATCH 01/16] Add CompWrapper to clean code --- pandas/core/arrays/__init__.py | 3 +- pandas/core/arrays/base.py | 83 +++++++++++++++++++++++++++++++- pandas/core/arrays/datetimes.py | 14 ++---- pandas/core/arrays/integer.py | 9 +--- pandas/core/arrays/period.py | 12 ++--- pandas/core/arrays/timedeltas.py | 8 +-- 6 files changed, 97 insertions(+), 32 deletions(-) diff --git a/pandas/core/arrays/__init__.py b/pandas/core/arrays/__init__.py index 1033ce784046e..f8e2458e0a44d 100644 --- a/pandas/core/arrays/__init__.py +++ b/pandas/core/arrays/__init__.py @@ -1,7 +1,8 @@ from .array_ import array # noqa from .base import (ExtensionArray, # noqa ExtensionOpsMixin, - ExtensionScalarOpsMixin) + ExtensionScalarOpsMixin, + CompWrapper) from .categorical import Categorical # noqa from .datetimes import DatetimeArray # noqa from .interval import IntervalArray # noqa diff --git a/pandas/core/arrays/base.py b/pandas/core/arrays/base.py index 7aaefef3d03e5..12bcdf83cd555 100644 --- a/pandas/core/arrays/base.py +++ b/pandas/core/arrays/base.py @@ -6,6 +6,7 @@ without warning. """ import operator +from functools import wraps import numpy as np @@ -15,7 +16,7 @@ from pandas.util._decorators import Appender, Substitution from pandas.core.dtypes.common import is_list_like -from pandas.core.dtypes.generic import ABCIndexClass, ABCSeries +from pandas.core.dtypes.generic import ABCDataFrame, ABCIndexClass, ABCSeries from pandas.core.dtypes.missing import isna from pandas.core import ops @@ -1118,3 +1119,83 @@ def _create_arithmetic_method(cls, op): @classmethod def _create_comparison_method(cls, op): return cls._create_method(op, coerce_to_dtype=False) + +''' +def validate_comp_other(comp, list_to_array=False, validate_len=False, + zerodim=False, inst_from_senior_cls=False): + def wrapper(self, other): + if list_to_array is True: + if is_list_like(other): + other = np.asarray(other) + + if validate_len is True: + if is_list_like(other) and len(other) != len(self): + raise ValueError("Lenghts must match") + + if zerodim is True: + import pandas._libs as lib + other = lib.item_from_zerodim(other) + + if inst_from_senior_cls is True: + if isinstance(other, (ABCDataFrame, ABCSeries, ABCIndexClass)): + # Rely on pandas to unbox and dispatch to us. + return NotImplemented + + comp(self, other) + return wrapper +''' + + +class CompWrapper(object): + __key__ = ['list_to_array', 'validate_len', + 'zerodim', 'inst_from_senior_cls'] + + def __init__(self, + list_to_array=None, + validate_len=None, + zerodim=None, + inst_from_senior_cls=None): + self.list_to_array = list_to_array + self.validate_len = validate_len + self.zerodim = zerodim + self.inst_from_senior_cls = inst_from_senior_cls + + def _list_to_array(self, comp): + @wraps(comp) + def wrapper(comp_self, comp_other): + if is_list_like(comp_other): + comp_other = np.asarray(comp_other) + return comp(comp_self, comp_other) + return wrapper + + def _validate_len(self, comp): + @wraps(comp) + def wrapper(comp_self, comp_other): + if is_list_like(comp_other) and len(comp_other) != len(comp_self): + raise ValueError("Lengths must match to compare") + return comp(comp_self, comp_other) + return wrapper + + def _zerodim(self, comp): + @wraps(comp) + def wrapper(comp_self, comp_other): + from pandas._libs import lib + comp_other = lib.item_from_zerodim(comp_other) + return comp(comp_self, comp_other) + return wrapper + + def _inst_from_senior_cls(self, comp): + @wraps(comp) + def wrapper(comp_self, comp_other): + if isinstance(comp_other, (ABCDataFrame, + ABCSeries, ABCIndexClass)): + # Rely on pandas to unbox and dispatch to us. + return NotImplemented + return comp(comp_self, comp_other) + return wrapper + + def __call__(self, comp): + for key in CompWrapper.__key__: + if getattr(self, key) is True: + comp = getattr(self, '_' + key)(comp) + return comp diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index d7a8417a71be2..5b931fd20cb94 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -21,12 +21,12 @@ is_string_dtype, is_timedelta64_dtype, pandas_dtype) from pandas.core.dtypes.dtypes import DatetimeTZDtype from pandas.core.dtypes.generic import ( - ABCDataFrame, ABCIndexClass, ABCPandasArray, ABCSeries) + ABCIndexClass, ABCPandasArray, ABCSeries) from pandas.core.dtypes.missing import isna from pandas.core import ops from pandas.core.algorithms import checked_add_with_arr -from pandas.core.arrays import datetimelike as dtl +from pandas.core.arrays import datetimelike as dtl, CompWrapper from pandas.core.arrays._ranges import generate_regular_range import pandas.core.common as com @@ -130,12 +130,8 @@ def _dt_array_cmp(cls, op): opname = '__{name}__'.format(name=op.__name__) nat_result = True if opname == '__ne__' else False + @CompWrapper(inst_from_senior_cls=True, validate_len=True, zerodim=True) def wrapper(self, other): - if isinstance(other, (ABCDataFrame, ABCSeries, ABCIndexClass)): - return NotImplemented - - other = lib.item_from_zerodim(other) - if isinstance(other, (datetime, np.datetime64, compat.string_types)): if isinstance(other, (datetime, np.datetime64)): # GH#18435 strings get a pass from tzawareness compat @@ -152,8 +148,8 @@ def wrapper(self, other): result.fill(nat_result) elif lib.is_scalar(other) or np.ndim(other) == 0: return ops.invalid_comparison(self, other, op) - elif len(other) != len(self): - raise ValueError("Lengths must match") + #elif len(other) != len(self): + # raise ValueError("Lengths must match") else: if isinstance(other, list): try: diff --git a/pandas/core/arrays/integer.py b/pandas/core/arrays/integer.py index a6a4a49d3a939..21453d26c55b1 100644 --- a/pandas/core/arrays/integer.py +++ b/pandas/core/arrays/integer.py @@ -18,7 +18,7 @@ from pandas.core.dtypes.missing import isna, notna from pandas.core import nanops -from pandas.core.arrays import ExtensionArray, ExtensionOpsMixin +from pandas.core.arrays import ExtensionArray, ExtensionOpsMixin, CompWrapper from pandas.core.tools.numeric import to_numeric @@ -529,22 +529,17 @@ def _values_for_argsort(self): @classmethod def _create_comparison_method(cls, op): + @CompWrapper(validate_len=True, inst_from_senior_cls=True) def cmp_method(self, other): op_name = op.__name__ mask = None - if isinstance(other, (ABCSeries, ABCIndexClass)): - # Rely on pandas to unbox and dispatch to us. - return NotImplemented - if isinstance(other, IntegerArray): other, mask = other._data, other._mask elif is_list_like(other): other = np.asarray(other) - if other.ndim > 0 and len(self) != len(other): - raise ValueError('Lengths must match to compare') other = lib.item_from_zerodim(other) diff --git a/pandas/core/arrays/period.py b/pandas/core/arrays/period.py index e0c71b5609096..2fede8026d0fc 100644 --- a/pandas/core/arrays/period.py +++ b/pandas/core/arrays/period.py @@ -16,14 +16,15 @@ from pandas.core.dtypes.common import ( _TD_DTYPE, ensure_object, is_datetime64_dtype, is_float_dtype, - is_list_like, is_period_dtype, pandas_dtype) + is_period_dtype, pandas_dtype) from pandas.core.dtypes.dtypes import PeriodDtype from pandas.core.dtypes.generic import ( - ABCDataFrame, ABCIndexClass, ABCPeriodIndex, ABCSeries) + ABCIndexClass, ABCPeriodIndex, ABCSeries) from pandas.core.dtypes.missing import isna, notna import pandas.core.algorithms as algos from pandas.core.arrays import datetimelike as dtl +from pandas.core.arrays.base import CompWrapper import pandas.core.common as com from pandas.tseries import frequencies @@ -48,15 +49,10 @@ def _period_array_cmp(cls, op): opname = '__{name}__'.format(name=op.__name__) nat_result = True if opname == '__ne__' else False + @CompWrapper(validate_len=True, inst_from_senior_cls=True) def wrapper(self, other): op = getattr(self.asi8, opname) - if isinstance(other, (ABCDataFrame, ABCSeries, ABCIndexClass)): - return NotImplemented - - if is_list_like(other) and len(other) != len(self): - raise ValueError("Lengths must match") - if isinstance(other, Period): self._check_compatible_with(other) diff --git a/pandas/core/arrays/timedeltas.py b/pandas/core/arrays/timedeltas.py index 4f0c96f7927da..f3cd33ec5af36 100644 --- a/pandas/core/arrays/timedeltas.py +++ b/pandas/core/arrays/timedeltas.py @@ -15,6 +15,7 @@ import pandas.compat as compat from pandas.util._decorators import Appender +from pandas.core.arrays import CompWrapper from pandas.core.dtypes.common import ( _NS_DTYPE, _TD_DTYPE, ensure_int64, is_datetime64_dtype, is_dtype_equal, is_float_dtype, is_integer_dtype, is_list_like, is_object_dtype, is_scalar, @@ -64,10 +65,8 @@ def _td_array_cmp(cls, op): opname = '__{name}__'.format(name=op.__name__) nat_result = True if opname == '__ne__' else False + @CompWrapper(validate_len=True, inst_from_senior_cls=True) def wrapper(self, other): - if isinstance(other, (ABCDataFrame, ABCSeries, ABCIndexClass)): - return NotImplemented - if _is_convertible_to_td(other) or other is NaT: try: other = Timedelta(other) @@ -82,9 +81,6 @@ def wrapper(self, other): elif not is_list_like(other): return ops.invalid_comparison(self, other, op) - elif len(other) != len(self): - raise ValueError("Lengths must match") - else: try: other = type(self)._from_sequence(other)._data From 2c094188d6f43e6bc622a48735547d5057d6f805 Mon Sep 17 00:00:00 2001 From: makbigc Date: Sun, 20 Jan 2019 22:08:27 +0800 Subject: [PATCH 02/16] Remove commented code --- pandas/core/arrays/base.py | 25 ------------------------- pandas/core/arrays/datetimes.py | 2 -- 2 files changed, 27 deletions(-) diff --git a/pandas/core/arrays/base.py b/pandas/core/arrays/base.py index 12bcdf83cd555..5e688c59b4aaf 100644 --- a/pandas/core/arrays/base.py +++ b/pandas/core/arrays/base.py @@ -1120,31 +1120,6 @@ def _create_arithmetic_method(cls, op): def _create_comparison_method(cls, op): return cls._create_method(op, coerce_to_dtype=False) -''' -def validate_comp_other(comp, list_to_array=False, validate_len=False, - zerodim=False, inst_from_senior_cls=False): - def wrapper(self, other): - if list_to_array is True: - if is_list_like(other): - other = np.asarray(other) - - if validate_len is True: - if is_list_like(other) and len(other) != len(self): - raise ValueError("Lenghts must match") - - if zerodim is True: - import pandas._libs as lib - other = lib.item_from_zerodim(other) - - if inst_from_senior_cls is True: - if isinstance(other, (ABCDataFrame, ABCSeries, ABCIndexClass)): - # Rely on pandas to unbox and dispatch to us. - return NotImplemented - - comp(self, other) - return wrapper -''' - class CompWrapper(object): __key__ = ['list_to_array', 'validate_len', diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index 5b931fd20cb94..ca86efaf569be 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -148,8 +148,6 @@ def wrapper(self, other): result.fill(nat_result) elif lib.is_scalar(other) or np.ndim(other) == 0: return ops.invalid_comparison(self, other, op) - #elif len(other) != len(self): - # raise ValueError("Lengths must match") else: if isinstance(other, list): try: From 0c1a296d69b2281a80537b62adf74897fca3af45 Mon Sep 17 00:00:00 2001 From: makbigc Date: Sun, 20 Jan 2019 23:09:29 +0800 Subject: [PATCH 03/16] Isort the files --- pandas/core/arrays/base.py | 2 +- pandas/core/arrays/datetimes.py | 5 ++--- pandas/core/arrays/integer.py | 2 +- pandas/core/arrays/period.py | 3 +-- pandas/core/arrays/timedeltas.py | 2 +- 5 files changed, 6 insertions(+), 8 deletions(-) diff --git a/pandas/core/arrays/base.py b/pandas/core/arrays/base.py index 5e688c59b4aaf..fd994cd89e31a 100644 --- a/pandas/core/arrays/base.py +++ b/pandas/core/arrays/base.py @@ -5,8 +5,8 @@ This is an experimental API and subject to breaking changes without warning. """ -import operator from functools import wraps +import operator import numpy as np diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index ca86efaf569be..263d00522a5c9 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -20,13 +20,12 @@ is_extension_type, is_float_dtype, is_object_dtype, is_period_dtype, is_string_dtype, is_timedelta64_dtype, pandas_dtype) from pandas.core.dtypes.dtypes import DatetimeTZDtype -from pandas.core.dtypes.generic import ( - ABCIndexClass, ABCPandasArray, ABCSeries) +from pandas.core.dtypes.generic import ABCIndexClass, ABCPandasArray, ABCSeries from pandas.core.dtypes.missing import isna from pandas.core import ops from pandas.core.algorithms import checked_add_with_arr -from pandas.core.arrays import datetimelike as dtl, CompWrapper +from pandas.core.arrays import CompWrapper, datetimelike as dtl from pandas.core.arrays._ranges import generate_regular_range import pandas.core.common as com diff --git a/pandas/core/arrays/integer.py b/pandas/core/arrays/integer.py index 21453d26c55b1..d99ace178fdee 100644 --- a/pandas/core/arrays/integer.py +++ b/pandas/core/arrays/integer.py @@ -18,7 +18,7 @@ from pandas.core.dtypes.missing import isna, notna from pandas.core import nanops -from pandas.core.arrays import ExtensionArray, ExtensionOpsMixin, CompWrapper +from pandas.core.arrays import CompWrapper, ExtensionArray, ExtensionOpsMixin from pandas.core.tools.numeric import to_numeric diff --git a/pandas/core/arrays/period.py b/pandas/core/arrays/period.py index 2fede8026d0fc..40da09dd16dcf 100644 --- a/pandas/core/arrays/period.py +++ b/pandas/core/arrays/period.py @@ -18,8 +18,7 @@ _TD_DTYPE, ensure_object, is_datetime64_dtype, is_float_dtype, is_period_dtype, pandas_dtype) from pandas.core.dtypes.dtypes import PeriodDtype -from pandas.core.dtypes.generic import ( - ABCIndexClass, ABCPeriodIndex, ABCSeries) +from pandas.core.dtypes.generic import ABCIndexClass, ABCPeriodIndex, ABCSeries from pandas.core.dtypes.missing import isna, notna import pandas.core.algorithms as algos diff --git a/pandas/core/arrays/timedeltas.py b/pandas/core/arrays/timedeltas.py index f3cd33ec5af36..260642ecce7d3 100644 --- a/pandas/core/arrays/timedeltas.py +++ b/pandas/core/arrays/timedeltas.py @@ -15,7 +15,6 @@ import pandas.compat as compat from pandas.util._decorators import Appender -from pandas.core.arrays import CompWrapper from pandas.core.dtypes.common import ( _NS_DTYPE, _TD_DTYPE, ensure_int64, is_datetime64_dtype, is_dtype_equal, is_float_dtype, is_integer_dtype, is_list_like, is_object_dtype, is_scalar, @@ -28,6 +27,7 @@ from pandas.core import ops from pandas.core.algorithms import checked_add_with_arr +from pandas.core.arrays import CompWrapper import pandas.core.common as com from pandas.tseries.frequencies import to_offset From cb938ce55bb6fba5bd9ac9f3c8e7862d43a051f2 Mon Sep 17 00:00:00 2001 From: makbigc Date: Mon, 21 Jan 2019 21:25:25 +0800 Subject: [PATCH 04/16] Move CompWrapper from arrays/base.py to ops.py --- pandas/core/arrays/__init__.py | 3 +- pandas/core/arrays/base.py | 58 +--------------------------------- pandas/core/ops.py | 56 ++++++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 59 deletions(-) diff --git a/pandas/core/arrays/__init__.py b/pandas/core/arrays/__init__.py index f8e2458e0a44d..1033ce784046e 100644 --- a/pandas/core/arrays/__init__.py +++ b/pandas/core/arrays/__init__.py @@ -1,8 +1,7 @@ from .array_ import array # noqa from .base import (ExtensionArray, # noqa ExtensionOpsMixin, - ExtensionScalarOpsMixin, - CompWrapper) + ExtensionScalarOpsMixin) from .categorical import Categorical # noqa from .datetimes import DatetimeArray # noqa from .interval import IntervalArray # noqa diff --git a/pandas/core/arrays/base.py b/pandas/core/arrays/base.py index fd994cd89e31a..7aaefef3d03e5 100644 --- a/pandas/core/arrays/base.py +++ b/pandas/core/arrays/base.py @@ -5,7 +5,6 @@ This is an experimental API and subject to breaking changes without warning. """ -from functools import wraps import operator import numpy as np @@ -16,7 +15,7 @@ from pandas.util._decorators import Appender, Substitution from pandas.core.dtypes.common import is_list_like -from pandas.core.dtypes.generic import ABCDataFrame, ABCIndexClass, ABCSeries +from pandas.core.dtypes.generic import ABCIndexClass, ABCSeries from pandas.core.dtypes.missing import isna from pandas.core import ops @@ -1119,58 +1118,3 @@ def _create_arithmetic_method(cls, op): @classmethod def _create_comparison_method(cls, op): return cls._create_method(op, coerce_to_dtype=False) - - -class CompWrapper(object): - __key__ = ['list_to_array', 'validate_len', - 'zerodim', 'inst_from_senior_cls'] - - def __init__(self, - list_to_array=None, - validate_len=None, - zerodim=None, - inst_from_senior_cls=None): - self.list_to_array = list_to_array - self.validate_len = validate_len - self.zerodim = zerodim - self.inst_from_senior_cls = inst_from_senior_cls - - def _list_to_array(self, comp): - @wraps(comp) - def wrapper(comp_self, comp_other): - if is_list_like(comp_other): - comp_other = np.asarray(comp_other) - return comp(comp_self, comp_other) - return wrapper - - def _validate_len(self, comp): - @wraps(comp) - def wrapper(comp_self, comp_other): - if is_list_like(comp_other) and len(comp_other) != len(comp_self): - raise ValueError("Lengths must match to compare") - return comp(comp_self, comp_other) - return wrapper - - def _zerodim(self, comp): - @wraps(comp) - def wrapper(comp_self, comp_other): - from pandas._libs import lib - comp_other = lib.item_from_zerodim(comp_other) - return comp(comp_self, comp_other) - return wrapper - - def _inst_from_senior_cls(self, comp): - @wraps(comp) - def wrapper(comp_self, comp_other): - if isinstance(comp_other, (ABCDataFrame, - ABCSeries, ABCIndexClass)): - # Rely on pandas to unbox and dispatch to us. - return NotImplemented - return comp(comp_self, comp_other) - return wrapper - - def __call__(self, comp): - for key in CompWrapper.__key__: - if getattr(self, key) is True: - comp = getattr(self, '_' + key)(comp) - return comp diff --git a/pandas/core/ops.py b/pandas/core/ops.py index 10cebc6f94b92..c7963b11b139b 100644 --- a/pandas/core/ops.py +++ b/pandas/core/ops.py @@ -7,6 +7,7 @@ from __future__ import division import datetime +from functools import wraps import operator import textwrap import warnings @@ -136,6 +137,61 @@ def maybe_upcast_for_op(obj): return obj +class CompWrapper(object): + __key__ = ['list_to_array', 'validate_len', + 'zerodim', 'inst_from_senior_cls'] + + def __init__(self, + list_to_array=None, + validate_len=None, + zerodim=None, + inst_from_senior_cls=None): + self.list_to_array = list_to_array + self.validate_len = validate_len + self.zerodim = zerodim + self.inst_from_senior_cls = inst_from_senior_cls + + def _list_to_array(self, comp): + @wraps(comp) + def wrapper(comp_self, comp_other): + if is_list_like(comp_other): + comp_other = np.asarray(comp_other) + return comp(comp_self, comp_other) + return wrapper + + def _validate_len(self, comp): + @wraps(comp) + def wrapper(comp_self, comp_other): + if is_list_like(comp_other) and len(comp_other) != len(comp_self): + raise ValueError("Lengths must match to compare") + return comp(comp_self, comp_other) + return wrapper + + def _zerodim(self, comp): + @wraps(comp) + def wrapper(comp_self, comp_other): + from pandas._libs import lib + comp_other = lib.item_from_zerodim(comp_other) + return comp(comp_self, comp_other) + return wrapper + + def _inst_from_senior_cls(self, comp): + @wraps(comp) + def wrapper(comp_self, comp_other): + if isinstance(comp_other, (ABCDataFrame, + ABCSeries, ABCIndexClass)): + # Rely on pandas to unbox and dispatch to us. + return NotImplemented + return comp(comp_self, comp_other) + return wrapper + + def __call__(self, comp): + for key in CompWrapper.__key__: + if getattr(self, key) is True: + comp = getattr(self, '_' + key)(comp) + return comp + + # ----------------------------------------------------------------------------- # Reversed Operations not available in the stdlib operator module. # Defining these instead of using lambdas allows us to reference them by name. From 8966432267b80160fae4dc7ed7d4983dd6f6079a Mon Sep 17 00:00:00 2001 From: makbigc Date: Sat, 26 Jan 2019 18:19:10 +0800 Subject: [PATCH 05/16] Import CompWrapper from its new location --- pandas/core/arrays/datetimes.py | 3 ++- pandas/core/arrays/integer.py | 3 ++- pandas/core/arrays/period.py | 2 +- pandas/core/arrays/timedeltas.py | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index 263d00522a5c9..8fd164dc08211 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -25,9 +25,10 @@ from pandas.core import ops from pandas.core.algorithms import checked_add_with_arr -from pandas.core.arrays import CompWrapper, datetimelike as dtl +from pandas.core.arrays import datetimelike as dtl from pandas.core.arrays._ranges import generate_regular_range import pandas.core.common as com +from pandas.core.ops import CompWrapper from pandas.tseries.frequencies import get_period_alias, to_offset from pandas.tseries.offsets import Day, Tick diff --git a/pandas/core/arrays/integer.py b/pandas/core/arrays/integer.py index d99ace178fdee..f8b77f0994568 100644 --- a/pandas/core/arrays/integer.py +++ b/pandas/core/arrays/integer.py @@ -18,7 +18,8 @@ from pandas.core.dtypes.missing import isna, notna from pandas.core import nanops -from pandas.core.arrays import CompWrapper, ExtensionArray, ExtensionOpsMixin +from pandas.core.arrays import ExtensionArray, ExtensionOpsMixin +from pandas.core.ops import CompWrapper from pandas.core.tools.numeric import to_numeric diff --git a/pandas/core/arrays/period.py b/pandas/core/arrays/period.py index 40da09dd16dcf..331d0a190c4e8 100644 --- a/pandas/core/arrays/period.py +++ b/pandas/core/arrays/period.py @@ -23,8 +23,8 @@ import pandas.core.algorithms as algos from pandas.core.arrays import datetimelike as dtl -from pandas.core.arrays.base import CompWrapper import pandas.core.common as com +from pandas.core.ops import CompWrapper from pandas.tseries import frequencies from pandas.tseries.offsets import DateOffset, Tick, _delta_to_tick diff --git a/pandas/core/arrays/timedeltas.py b/pandas/core/arrays/timedeltas.py index 260642ecce7d3..86f7e9a26a9bb 100644 --- a/pandas/core/arrays/timedeltas.py +++ b/pandas/core/arrays/timedeltas.py @@ -27,8 +27,8 @@ from pandas.core import ops from pandas.core.algorithms import checked_add_with_arr -from pandas.core.arrays import CompWrapper import pandas.core.common as com +from pandas.core.ops import CompWrapper from pandas.tseries.frequencies import to_offset from pandas.tseries.offsets import Tick From 99f8afd9609eb3ff854274d56cdc29bbd928b60e Mon Sep 17 00:00:00 2001 From: makbigc Date: Sat, 26 Jan 2019 18:23:10 +0800 Subject: [PATCH 06/16] Add check for EA instance in _inst_from_senior_cls --- pandas/core/ops.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pandas/core/ops.py b/pandas/core/ops.py index c7963b11b139b..4fe35e947e398 100644 --- a/pandas/core/ops.py +++ b/pandas/core/ops.py @@ -30,7 +30,7 @@ is_scalar, is_timedelta64_dtype, needs_i8_conversion) from pandas.core.dtypes.generic import ( ABCDataFrame, ABCIndex, ABCIndexClass, ABCPanel, ABCSeries, ABCSparseArray, - ABCSparseSeries) + ABCSparseSeries, ABCExtensionArray) from pandas.core.dtypes.missing import isna, notna import pandas as pd @@ -178,10 +178,11 @@ def wrapper(comp_self, comp_other): def _inst_from_senior_cls(self, comp): @wraps(comp) def wrapper(comp_self, comp_other): - if isinstance(comp_other, (ABCDataFrame, - ABCSeries, ABCIndexClass)): - # Rely on pandas to unbox and dispatch to us. - return NotImplemented + if isinstance(comp_self, ABCExtensionArray): + if isinstance(comp_other, (ABCDataFrame, ABCSeries, + ABCIndexClass)): + # Rely on pandas to unbox and dispatch to us. + return NotImplemented return comp(comp_self, comp_other) return wrapper From c17322ef9eb99a0d912c32fdfaae2ff8a51e8351 Mon Sep 17 00:00:00 2001 From: makbigc Date: Sat, 26 Jan 2019 18:34:03 +0800 Subject: [PATCH 07/16] Wrap _cat_compare_op with CompWrapper --- pandas/core/arrays/categorical.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pandas/core/arrays/categorical.py b/pandas/core/arrays/categorical.py index 35b662eaae9a5..7529c1857b6b1 100644 --- a/pandas/core/arrays/categorical.py +++ b/pandas/core/arrays/categorical.py @@ -34,6 +34,7 @@ import pandas.core.common as com from pandas.core.config import get_option from pandas.core.missing import interpolate_2d +from pandas.core.ops import CompWrapper from pandas.core.sorting import nargsort from pandas.io.formats import console @@ -52,6 +53,7 @@ Use 'allow_fill=False' to accept the new behavior.""") +@CompWrapper(inst_from_senior_cls=True, zerodim=True) def _cat_compare_op(op): def f(self, other): # On python2, you can usually compare any type to any type, and From 45e1df0f5ae19c559712ef8920e19cc5e71f8613 Mon Sep 17 00:00:00 2001 From: makbigc Date: Sat, 26 Jan 2019 20:33:28 +0800 Subject: [PATCH 08/16] Wrap the func inside _cat_compare_op --- pandas/core/arrays/categorical.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/pandas/core/arrays/categorical.py b/pandas/core/arrays/categorical.py index 7529c1857b6b1..c66382def026a 100644 --- a/pandas/core/arrays/categorical.py +++ b/pandas/core/arrays/categorical.py @@ -53,19 +53,14 @@ Use 'allow_fill=False' to accept the new behavior.""") -@CompWrapper(inst_from_senior_cls=True, zerodim=True) def _cat_compare_op(op): + @CompWrapper(inst_from_senior_cls=True, zerodim=True) def f(self, other): # On python2, you can usually compare any type to any type, and # Categoricals can be seen as a custom type, but having different # results depending whether categories are the same or not is kind of # insane, so be a bit stricter here and use the python3 idea of # comparing only things of equal type. - if isinstance(other, (ABCDataFrame, ABCSeries, ABCIndexClass)): - return NotImplemented - - other = lib.item_from_zerodim(other) - if not self.ordered: if op in ['__lt__', '__gt__', '__le__', '__ge__']: raise TypeError("Unordered Categoricals can only compare " From 72e8b2dfa7c24fd6a6f3f12d0f26907b91ef358b Mon Sep 17 00:00:00 2001 From: makbigc Date: Sat, 26 Jan 2019 21:50:24 +0800 Subject: [PATCH 09/16] Wrap add and sub methods with CompWrapper --- pandas/core/arrays/datetimelike.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index 73e799f9e0a36..6161471276ed9 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -32,6 +32,7 @@ from pandas.core.algorithms import ( checked_add_with_arr, take, unique1d, value_counts) import pandas.core.common as com +from pandas.core.ops import CompWrapper from pandas.tseries import frequencies from pandas.tseries.offsets import DateOffset, Tick @@ -1175,13 +1176,10 @@ def _time_shift(self, periods, freq=None): return self._generate_range(start=start, end=end, periods=None, freq=self.freq) + @CompWrapper(zerodim=True, inst_from_senior_cls=True) def __add__(self, other): - other = lib.item_from_zerodim(other) - if isinstance(other, (ABCSeries, ABCDataFrame)): - return NotImplemented - # scalar others - elif other is NaT: + if other is NaT: result = self._add_nat() elif isinstance(other, (Tick, timedelta, np.timedelta64)): result = self._add_delta(other) @@ -1238,13 +1236,10 @@ def __radd__(self, other): # alias for __add__ return self.__add__(other) + @CompWrapper(zerodim=True, inst_from_senior_cls=True) def __sub__(self, other): - other = lib.item_from_zerodim(other) - if isinstance(other, (ABCSeries, ABCDataFrame)): - return NotImplemented - # scalar others - elif other is NaT: + if other is NaT: result = self._sub_nat() elif isinstance(other, (Tick, timedelta, np.timedelta64)): result = self._add_delta(-other) From b07279f993af9537cdb1261a1fb54a452d40da7b Mon Sep 17 00:00:00 2001 From: makbigc Date: Sat, 26 Jan 2019 21:57:20 +0800 Subject: [PATCH 10/16] Remove CompWrapper --- pandas/core/arrays/integer.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pandas/core/arrays/integer.py b/pandas/core/arrays/integer.py index f8b77f0994568..a6a4a49d3a939 100644 --- a/pandas/core/arrays/integer.py +++ b/pandas/core/arrays/integer.py @@ -19,7 +19,6 @@ from pandas.core import nanops from pandas.core.arrays import ExtensionArray, ExtensionOpsMixin -from pandas.core.ops import CompWrapper from pandas.core.tools.numeric import to_numeric @@ -530,17 +529,22 @@ def _values_for_argsort(self): @classmethod def _create_comparison_method(cls, op): - @CompWrapper(validate_len=True, inst_from_senior_cls=True) def cmp_method(self, other): op_name = op.__name__ mask = None + if isinstance(other, (ABCSeries, ABCIndexClass)): + # Rely on pandas to unbox and dispatch to us. + return NotImplemented + if isinstance(other, IntegerArray): other, mask = other._data, other._mask elif is_list_like(other): other = np.asarray(other) + if other.ndim > 0 and len(self) != len(other): + raise ValueError('Lengths must match to compare') other = lib.item_from_zerodim(other) From 5cf404c4e2dc7b32a755b0eb659848485a1a898a Mon Sep 17 00:00:00 2001 From: makbigc Date: Sat, 26 Jan 2019 22:13:45 +0800 Subject: [PATCH 11/16] CompWrapper handles validate_len in add and sub methods --- pandas/core/arrays/datetimelike.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index 6161471276ed9..c7d80ec715b75 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -988,9 +988,6 @@ def _add_delta_tdi(self, other): Add a delta of a TimedeltaIndex return the i8 result view """ - if len(self) != len(other): - raise ValueError("cannot add indices of unequal length") - if isinstance(other, np.ndarray): # ndarray[timedelta64]; wrap in TimedeltaIndex for op from pandas import TimedeltaIndex @@ -1055,9 +1052,6 @@ def _sub_period_array(self, other): .format(dtype=other.dtype, cls=type(self).__name__)) - if len(self) != len(other): - raise ValueError("cannot subtract arrays/indices of " - "unequal length") if self.freq != other.freq: msg = DIFFERENT_FREQ.format(cls=type(self).__name__, own_freq=self.freqstr, @@ -1176,7 +1170,7 @@ def _time_shift(self, periods, freq=None): return self._generate_range(start=start, end=end, periods=None, freq=self.freq) - @CompWrapper(zerodim=True, inst_from_senior_cls=True) + @CompWrapper(zerodim=True, inst_from_senior_cls=True, validate_len=True) def __add__(self, other): # scalar others if other is NaT: @@ -1236,7 +1230,7 @@ def __radd__(self, other): # alias for __add__ return self.__add__(other) - @CompWrapper(zerodim=True, inst_from_senior_cls=True) + @CompWrapper(zerodim=True, inst_from_senior_cls=True, validate_len=True) def __sub__(self, other): # scalar others if other is NaT: From 96e51488091f065da8dc5cfb987a1d1156b23964 Mon Sep 17 00:00:00 2001 From: makbigc Date: Sun, 27 Jan 2019 13:50:35 +0800 Subject: [PATCH 12/16] Change error msg in test_sub_dti_dti --- pandas/tests/arithmetic/test_datetime64.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/arithmetic/test_datetime64.py b/pandas/tests/arithmetic/test_datetime64.py index 405dc0805a285..99f687dbb485b 100644 --- a/pandas/tests/arithmetic/test_datetime64.py +++ b/pandas/tests/arithmetic/test_datetime64.py @@ -2091,7 +2091,7 @@ def test_sub_dti_dti(self): # different length raises ValueError dti1 = date_range('20130101', periods=3) dti2 = date_range('20130101', periods=4) - msg = 'cannot add indices of unequal length' + msg = 'Lengths must match to compare' with pytest.raises(ValueError, match=msg): dti1 - dti2 From f744ccd8b5361e8524991c84c0bf63f13d82a69f Mon Sep 17 00:00:00 2001 From: makbigc Date: Sun, 27 Jan 2019 17:59:33 +0800 Subject: [PATCH 13/16] Set inst_from_senior_cls as None --- pandas/core/arrays/datetimelike.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index c7d80ec715b75..a894e08be433b 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -1170,8 +1170,11 @@ def _time_shift(self, periods, freq=None): return self._generate_range(start=start, end=end, periods=None, freq=self.freq) - @CompWrapper(zerodim=True, inst_from_senior_cls=True, validate_len=True) + @CompWrapper(zerodim=True, validate_len=True) def __add__(self, other): + if isinstance(other, (ABCSeries, ABCDataFrame)): + return NotImplemented + # scalar others if other is NaT: result = self._add_nat() @@ -1230,8 +1233,11 @@ def __radd__(self, other): # alias for __add__ return self.__add__(other) - @CompWrapper(zerodim=True, inst_from_senior_cls=True, validate_len=True) + @CompWrapper(zerodim=True, validate_len=True) def __sub__(self, other): + if isinstance(other, (ABCSeries, ABCDataFrame)): + return NotImplemented + # scalar others if other is NaT: result = self._sub_nat() From 12343729eaf6ae725115e918577e332aa3c4b29e Mon Sep 17 00:00:00 2001 From: makbigc Date: Sun, 27 Jan 2019 18:19:38 +0800 Subject: [PATCH 14/16] Remove lib --- pandas/core/arrays/categorical.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/core/arrays/categorical.py b/pandas/core/arrays/categorical.py index c66382def026a..e2d0571405d80 100644 --- a/pandas/core/arrays/categorical.py +++ b/pandas/core/arrays/categorical.py @@ -5,7 +5,7 @@ import numpy as np -from pandas._libs import algos as libalgos, lib +from pandas._libs import algos as libalgos import pandas.compat as compat from pandas.compat import lzip, u from pandas.compat.numpy import function as nv @@ -23,7 +23,7 @@ is_timedelta64_dtype) from pandas.core.dtypes.dtypes import CategoricalDtype from pandas.core.dtypes.generic import ( - ABCCategoricalIndex, ABCDataFrame, ABCIndexClass, ABCSeries) + ABCCategoricalIndex, ABCIndexClass, ABCSeries) from pandas.core.dtypes.inference import is_hashable from pandas.core.dtypes.missing import isna, notna From ef40bcd366091b28864370eb6289731fd5702d50 Mon Sep 17 00:00:00 2001 From: makbigc Date: Sun, 27 Jan 2019 23:25:10 +0800 Subject: [PATCH 15/16] Wrap the comp more specifically --- pandas/core/arrays/datetimelike.py | 12 +++++++----- pandas/core/arrays/datetimes.py | 4 +--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index a894e08be433b..991ae5aaeded5 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -983,6 +983,7 @@ def _add_timedeltalike_scalar(self, other): new_values = self._maybe_mask_results(new_values) return new_values.view('i8') + @CompWrapper(validate_len=True) def _add_delta_tdi(self, other): """ Add a delta of a TimedeltaIndex @@ -1032,6 +1033,7 @@ def _sub_nat(self): result.fill(iNaT) return result.view('timedelta64[ns]') + @CompWrapper(validate_len=True) def _sub_period_array(self, other): """ Subtract a Period Array/Index from self. This is only valid if self @@ -1138,7 +1140,7 @@ def _time_shift(self, periods, freq=None): Note this is different from ExtensionArray.shift, which shifts the *position* of each element, padding the end with - missing values. + missing values.x Parameters ---------- @@ -1170,13 +1172,13 @@ def _time_shift(self, periods, freq=None): return self._generate_range(start=start, end=end, periods=None, freq=self.freq) - @CompWrapper(zerodim=True, validate_len=True) + @CompWrapper(zerodim=True) def __add__(self, other): if isinstance(other, (ABCSeries, ABCDataFrame)): return NotImplemented # scalar others - if other is NaT: + elif other is NaT: result = self._add_nat() elif isinstance(other, (Tick, timedelta, np.timedelta64)): result = self._add_delta(other) @@ -1233,13 +1235,13 @@ def __radd__(self, other): # alias for __add__ return self.__add__(other) - @CompWrapper(zerodim=True, validate_len=True) + @CompWrapper(zerodim=True) def __sub__(self, other): if isinstance(other, (ABCSeries, ABCDataFrame)): return NotImplemented # scalar others - if other is NaT: + elif other is NaT: result = self._sub_nat() elif isinstance(other, (Tick, timedelta, np.timedelta64)): result = self._add_delta(-other) diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index 8fd164dc08211..0d3be76b93620 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -697,11 +697,9 @@ def _assert_tzawareness_compat(self, other): # ----------------------------------------------------------------- # Arithmetic Methods + @CompWrapper(validate_len=True) def _sub_datetime_arraylike(self, other): """subtract DatetimeArray/Index or ndarray[datetime64]""" - if len(self) != len(other): - raise ValueError("cannot add indices of unequal length") - if isinstance(other, np.ndarray): assert is_datetime64_dtype(other) other = type(self)(other) From 57463cce3e1f41b32aa0744dc42667e61709a95f Mon Sep 17 00:00:00 2001 From: makbigc Date: Mon, 28 Jan 2019 00:11:01 +0800 Subject: [PATCH 16/16] isort ops.py --- pandas/core/ops.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/core/ops.py b/pandas/core/ops.py index 4fe35e947e398..ac1b5711a5627 100644 --- a/pandas/core/ops.py +++ b/pandas/core/ops.py @@ -29,8 +29,8 @@ is_integer_dtype, is_list_like, is_object_dtype, is_period_dtype, is_scalar, is_timedelta64_dtype, needs_i8_conversion) from pandas.core.dtypes.generic import ( - ABCDataFrame, ABCIndex, ABCIndexClass, ABCPanel, ABCSeries, ABCSparseArray, - ABCSparseSeries, ABCExtensionArray) + ABCDataFrame, ABCExtensionArray, ABCIndex, ABCIndexClass, ABCPanel, + ABCSeries, ABCSparseArray, ABCSparseSeries) from pandas.core.dtypes.missing import isna, notna import pandas as pd