From 50b089526bda81d62b53f7c9b25c5e0c5e92b57d Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 27 Nov 2018 11:18:47 -0800 Subject: [PATCH 1/7] Fix TimedeltaIndex invalid comparisons --- pandas/core/arrays/timedeltas.py | 18 +++++++++++------- pandas/tests/arithmetic/test_timedelta64.py | 18 ++++++++++++------ pandas/util/testing.py | 2 ++ 3 files changed, 25 insertions(+), 13 deletions(-) diff --git a/pandas/core/arrays/timedeltas.py b/pandas/core/arrays/timedeltas.py index da26966cfa94c..3a8cdbdd1848e 100644 --- a/pandas/core/arrays/timedeltas.py +++ b/pandas/core/arrays/timedeltas.py @@ -72,25 +72,29 @@ def _td_array_cmp(cls, op): opname = '__{name}__'.format(name=op.__name__) nat_result = True if opname == '__ne__' else False + meth = getattr(dtl.DatetimeLikeArrayMixin, opname) + def wrapper(self, other): - msg = "cannot compare a {cls} with type {typ}" - meth = getattr(dtl.DatetimeLikeArrayMixin, opname) if _is_convertible_to_td(other) or other is NaT: try: other = _to_m8(other) except ValueError: # failed to parse as timedelta - raise TypeError(msg.format(cls=type(self).__name__, - typ=type(other).__name__)) + return ops.invalid_comparison(self, other, op) + result = meth(self, other) if isna(other): result.fill(nat_result) elif not is_list_like(other): - raise TypeError(msg.format(cls=type(self).__name__, - typ=type(other).__name__)) + return ops.invalid_comparison(self, other, op) + else: - other = type(self)(other)._data + try: + other = type(self)(other)._data + except (ValueError, TypeError): + return ops.invalid_comparison(self, other, op) + result = meth(self, other) result = com.values_from_object(result) diff --git a/pandas/tests/arithmetic/test_timedelta64.py b/pandas/tests/arithmetic/test_timedelta64.py index 58c7216f0eece..be2d29bfdf040 100644 --- a/pandas/tests/arithmetic/test_timedelta64.py +++ b/pandas/tests/arithmetic/test_timedelta64.py @@ -49,14 +49,20 @@ def test_tdi_cmp_str_invalid(self): for left, right in [(tdi, 'a'), ('a', tdi)]: with pytest.raises(TypeError): left > right - with pytest.raises(TypeError): - # FIXME: Shouldn't this return all-False? - left == right - + left >= right with pytest.raises(TypeError): - # FIXME: Shouldn't this return all-True? - left != right + left < right + with pytest.raises(TypeError): + left <= right + + result = left == right + expected = np.array([False, False], dtype=bool) + tm.assert_equal(result, expected) + + result = left != right + expected = np.array([True, True], dtype=bool) + tm.assert_equal(result, expected) @pytest.mark.parametrize('dtype', [None, object]) def test_comp_nat(self, dtype): diff --git a/pandas/util/testing.py b/pandas/util/testing.py index 210620f2092cf..fe3ce4f10bc08 100644 --- a/pandas/util/testing.py +++ b/pandas/util/testing.py @@ -1073,6 +1073,7 @@ def assert_period_array_equal(left, right, obj='PeriodArray'): def assert_datetime_array_equal(left, right, obj='DatetimeArray'): + __tracebackhide__ = True _check_isinstance(left, right, DatetimeArray) assert_numpy_array_equal(left._data, right._data, @@ -1082,6 +1083,7 @@ def assert_datetime_array_equal(left, right, obj='DatetimeArray'): def assert_timedelta_array_equal(left, right, obj='TimedeltaArray'): + __tracebackhide__ = True _check_isinstance(left, right, TimedeltaArray) assert_numpy_array_equal(left._data, right._data, obj='{obj}._data'.format(obj=obj)) From 314d79c2972522f73fc4d88968402aab50cff2d5 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 27 Nov 2018 12:34:53 -0800 Subject: [PATCH 2/7] box test --- pandas/tests/arithmetic/test_timedelta64.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pandas/tests/arithmetic/test_timedelta64.py b/pandas/tests/arithmetic/test_timedelta64.py index be2d29bfdf040..bffde0d5ce6a0 100644 --- a/pandas/tests/arithmetic/test_timedelta64.py +++ b/pandas/tests/arithmetic/test_timedelta64.py @@ -42,11 +42,13 @@ def test_compare_timedelta_series(self): expected = pd.Series([False, True]) tm.assert_series_equal(actual, expected) - def test_tdi_cmp_str_invalid(self): + def test_tdi_cmp_str_invalid(self, box_with_array): # GH#13624 + xbox = box_with_array if box_with_array is not pd.Index else np.ndarray tdi = TimedeltaIndex(['1 day', '2 days']) + tdarr = tm.box_expected(tdi, box_with_array) - for left, right in [(tdi, 'a'), ('a', tdi)]: + for left, right in [(tdarr, 'a'), ('a', tdarr)]: with pytest.raises(TypeError): left > right with pytest.raises(TypeError): @@ -58,10 +60,12 @@ def test_tdi_cmp_str_invalid(self): result = left == right expected = np.array([False, False], dtype=bool) + expected = tm.box_expected(expected, xbox) tm.assert_equal(result, expected) result = left != right expected = np.array([True, True], dtype=bool) + expected = tm.box_expected(expected, xbox) tm.assert_equal(result, expected) @pytest.mark.parametrize('dtype', [None, object]) From f4731eafab5e9149e8d0c8a1fd1bcedcc97aa6b6 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sun, 2 Dec 2018 15:46:50 -0800 Subject: [PATCH 3/7] parametrize more tests --- pandas/core/arrays/datetimes.py | 9 ++ pandas/core/arrays/timedeltas.py | 3 + pandas/core/generic.py | 4 + pandas/core/ops.py | 2 +- pandas/tests/arithmetic/test_datetime64.py | 130 ++++++++++++++------- 5 files changed, 106 insertions(+), 42 deletions(-) diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index 4d3caaacca1c1..553bbd5d6116b 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -173,6 +173,9 @@ class DatetimeArrayMixin(dtl.DatetimeLikeArrayMixin): # by returning NotImplemented timetuple = None + # Needed so that Timestamp.__richcmp__(DateTimeArray) operates pointwise + ndim = 1 + # ensure that operations with numpy arrays defer to our implementation __array_priority__ = 1000 @@ -216,6 +219,12 @@ def __new__(cls, values, freq=None, tz=None, dtype=None): # if dtype has an embedded tz, capture it tz = dtl.validate_tz_from_dtype(dtype, tz) + if not hasattr(values, "dtype"): + if np.ndim(values) == 0: + # i.e. iterator + values = list(values) + values = np.array(values) + if is_object_dtype(values): # kludge; dispatch until the DatetimeArray constructor is complete from pandas import DatetimeIndex diff --git a/pandas/core/arrays/timedeltas.py b/pandas/core/arrays/timedeltas.py index 965a70e8f6def..cb958efe4acbb 100644 --- a/pandas/core/arrays/timedeltas.py +++ b/pandas/core/arrays/timedeltas.py @@ -137,6 +137,9 @@ class TimedeltaArrayMixin(dtl.DatetimeLikeArrayMixin): _typ = "timedeltaarray" __array_priority__ = 1000 + # Needed so that NaT.__richcmp__(DateTimeArray) operates pointwise + ndim = 1 + @property def _box_func(self): return lambda x: Timedelta(x, unit='ns') diff --git a/pandas/core/generic.py b/pandas/core/generic.py index 08c07da39128f..228c8051992ec 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -111,6 +111,10 @@ class NDFrame(PandasObject, SelectionMixin): _metadata = [] _is_copy = None + # dummy attribute so that datetime.__eq__(Series/DataFrame) defers + # by returning NotImplemented + timetuple = None + # ---------------------------------------------------------------------- # Constructors diff --git a/pandas/core/ops.py b/pandas/core/ops.py index 2a21593fab8f5..58d5cb4ff4bb8 100644 --- a/pandas/core/ops.py +++ b/pandas/core/ops.py @@ -967,7 +967,7 @@ def dispatch_to_series(left, right, func, str_rep=None, axis=None): import pandas.core.computation.expressions as expressions right = lib.item_from_zerodim(right) - if lib.is_scalar(right): + if lib.is_scalar(right) or np.ndim(right) == 0: def column_op(a, b): return {i: func(a.iloc[:, i], b) diff --git a/pandas/tests/arithmetic/test_datetime64.py b/pandas/tests/arithmetic/test_datetime64.py index 667c2b4103e00..a636662d1dd21 100644 --- a/pandas/tests/arithmetic/test_datetime64.py +++ b/pandas/tests/arithmetic/test_datetime64.py @@ -25,6 +25,17 @@ DatetimeIndex, TimedeltaIndex) +def assert_all(obj): + """ + Test helper to call call obj.all() the appropriate number of times on + a Series or DataFrame. + """ + if isinstance(obj, pd.DataFrame): + assert obj.all().all() + else: + assert obj.all() + + # ------------------------------------------------------------------ # Comparisons @@ -86,11 +97,11 @@ def test_comparison_invalid(self, box_with_array): [Period('2011-01', freq='M'), NaT, Period('2011-03', freq='M')] ]) @pytest.mark.parametrize('dtype', [None, object]) - def test_nat_comparisons_scalar(self, dtype, data, box): - xbox = box if box is not pd.Index else np.ndarray + def test_nat_comparisons_scalar(self, dtype, data, box_with_array): + xbox = box_with_array if box_with_array is not pd.Index else np.ndarray left = Series(data, dtype=dtype) - left = tm.box_expected(left, box) + left = tm.box_expected(left, box_with_array) expected = [False, False, False] expected = tm.box_expected(expected, xbox) @@ -290,23 +301,24 @@ def test_dti_cmp_datetimelike(self, other, tz_naive_fixture): expected = np.array([True, False]) tm.assert_numpy_array_equal(result, expected) - def dti_cmp_non_datetime(self, tz_naive_fixture): + def dt64arr_cmp_non_datetime(self, tz_naive_fixture, box_with_array): # GH#19301 by convention datetime.date is not considered comparable # to Timestamp or DatetimeIndex. This may change in the future. tz = tz_naive_fixture dti = pd.date_range('2016-01-01', periods=2, tz=tz) + dtarr = tm.box_expected(dti, box_with_array) other = datetime(2016, 1, 1).date() - assert not (dti == other).any() - assert (dti != other).all() + assert not (dtarr == other).any() + assert (dtarr != other).all() with pytest.raises(TypeError): - dti < other + dtarr < other with pytest.raises(TypeError): - dti <= other + dtarr <= other with pytest.raises(TypeError): - dti > other + dtarr > other with pytest.raises(TypeError): - dti >= other + dtarr >= other @pytest.mark.parametrize('other', [None, np.nan, pd.NaT]) def test_dti_eq_null_scalar(self, other, tz_naive_fixture): @@ -323,49 +335,62 @@ def test_dti_ne_null_scalar(self, other, tz_naive_fixture): assert (dti != other).all() @pytest.mark.parametrize('other', [None, np.nan]) - def test_dti_cmp_null_scalar_inequality(self, tz_naive_fixture, other): + def test_dti_cmp_null_scalar_inequality(self, tz_naive_fixture, other, + box_with_array): # GH#19301 tz = tz_naive_fixture dti = pd.date_range('2016-01-01', periods=2, tz=tz) + # FIXME: ValueError with transpose + dtarr = tm.box_expected(dti, box_with_array, transpose=False) with pytest.raises(TypeError): - dti < other + dtarr < other with pytest.raises(TypeError): - dti <= other + dtarr <= other with pytest.raises(TypeError): - dti > other + dtarr > other with pytest.raises(TypeError): - dti >= other + dtarr >= other @pytest.mark.parametrize('dtype', [None, object]) - def test_dti_cmp_nat(self, dtype): + def test_dti_cmp_nat(self, dtype, box_with_array): + xbox = box_with_array if box_with_array is not pd.Index else np.ndarray + left = pd.DatetimeIndex([pd.Timestamp('2011-01-01'), pd.NaT, pd.Timestamp('2011-01-03')]) right = pd.DatetimeIndex([pd.NaT, pd.NaT, pd.Timestamp('2011-01-03')]) + left = tm.box_expected(left, box_with_array) + right = tm.box_expected(right, box_with_array) + lhs, rhs = left, right if dtype is object: lhs, rhs = left.astype(object), right.astype(object) result = rhs == lhs expected = np.array([False, False, True]) - tm.assert_numpy_array_equal(result, expected) + expected = tm.box_expected(expected, xbox) + tm.assert_equal(result, expected) result = lhs != rhs expected = np.array([True, True, False]) - tm.assert_numpy_array_equal(result, expected) + expected = tm.box_expected(expected, xbox) + tm.assert_equal(result, expected) expected = np.array([False, False, False]) - tm.assert_numpy_array_equal(lhs == pd.NaT, expected) - tm.assert_numpy_array_equal(pd.NaT == rhs, expected) + expected = tm.box_expected(expected, xbox) + tm.assert_equal(lhs == pd.NaT, expected) + tm.assert_equal(pd.NaT == rhs, expected) expected = np.array([True, True, True]) - tm.assert_numpy_array_equal(lhs != pd.NaT, expected) - tm.assert_numpy_array_equal(pd.NaT != lhs, expected) + expected = tm.box_expected(expected, xbox) + tm.assert_equal(lhs != pd.NaT, expected) + tm.assert_equal(pd.NaT != lhs, expected) expected = np.array([False, False, False]) - tm.assert_numpy_array_equal(lhs < pd.NaT, expected) - tm.assert_numpy_array_equal(pd.NaT > lhs, expected) + expected = tm.box_expected(expected, xbox) + tm.assert_equal(lhs < pd.NaT, expected) + tm.assert_equal(pd.NaT > lhs, expected) def test_dti_cmp_nat_behaves_like_float_cmp_nan(self): fidx1 = pd.Index([1.0, np.nan, 3.0, np.nan, 5.0, 7.0]) @@ -459,36 +484,47 @@ def test_dti_cmp_nat_behaves_like_float_cmp_nan(self): @pytest.mark.parametrize('op', [operator.eq, operator.ne, operator.gt, operator.ge, operator.lt, operator.le]) - def test_comparison_tzawareness_compat(self, op): + def test_comparison_tzawareness_compat(self, op, box_with_array): # GH#18162 dr = pd.date_range('2016-01-01', periods=6) dz = dr.tz_localize('US/Pacific') + # FIXME: ValueError with transpose + dr = tm.box_expected(dr, box_with_array, transpose=False) + dz = tm.box_expected(dz, box_with_array, transpose=False) + with pytest.raises(TypeError): op(dr, dz) - with pytest.raises(TypeError): - op(dr, list(dz)) + if box_with_array is not pd.DataFrame: + # DataFrame op is invalid until transpose bug is fixed + with pytest.raises(TypeError): + op(dr, list(dz)) with pytest.raises(TypeError): op(dz, dr) - with pytest.raises(TypeError): - op(dz, list(dr)) + if box_with_array is not pd.DataFrame: + # DataFrame op is invalid until transpose bug is fixed + with pytest.raises(TypeError): + op(dz, list(dr)) # Check that there isn't a problem aware-aware and naive-naive do not # raise - assert (dr == dr).all() - assert (dr == list(dr)).all() - assert (dz == dz).all() - assert (dz == list(dz)).all() + assert_all(dr == dr) + assert_all(dz == dz) + if box_with_array is not pd.DataFrame: + # DataFrame doesn't align the lists correctly unless we transpose, + # which we cannot do at the moment + assert (dr == list(dr)).all() + assert (dz == list(dz)).all() # Check comparisons against scalar Timestamps ts = pd.Timestamp('2000-03-14 01:59') ts_tz = pd.Timestamp('2000-03-14 01:59', tz='Europe/Amsterdam') - assert (dr > ts).all() + assert_all(dr > ts) with pytest.raises(TypeError): op(dr, ts_tz) - assert (dz > ts_tz).all() + assert_all(dz > ts_tz) with pytest.raises(TypeError): op(dz, ts) @@ -502,13 +538,18 @@ def test_comparison_tzawareness_compat(self, op): @pytest.mark.parametrize('other', [datetime(2016, 1, 1), Timestamp('2016-01-01'), np.datetime64('2016-01-01')]) - def test_scalar_comparison_tzawareness(self, op, other, tz_aware_fixture): + def test_scalar_comparison_tzawareness(self, op, other, tz_aware_fixture, + box_with_array): tz = tz_aware_fixture dti = pd.date_range('2016-01-01', periods=2, tz=tz) + + # FIXME: ValueError with transpose + dtarr = tm.box_expected(dti, box_with_array, transpose=False) + with pytest.raises(TypeError): - op(dti, other) + op(dtarr, other) with pytest.raises(TypeError): - op(other, dti) + op(other, dtarr) @pytest.mark.parametrize('op', [operator.eq, operator.ne, operator.gt, operator.ge, @@ -558,18 +599,25 @@ def test_dti_cmp_str(self, tz_naive_fixture): @pytest.mark.parametrize('other', ['foo', 99, 4.0, object(), timedelta(days=2)]) - def test_dti_cmp_scalar_invalid(self, other, tz_naive_fixture): + def test_dt64arr_cmp_scalar_invalid(self, other, tz_naive_fixture, + box_with_array): # GH#22074 tz = tz_naive_fixture + xbox = box_with_array if box_with_array is not pd.Index else np.ndarray + rng = date_range('1/1/2000', periods=10, tz=tz) + # FIXME: ValueError with transpose + rng = tm.box_expected(rng, box_with_array, transpose=False) result = rng == other expected = np.array([False] * 10) - tm.assert_numpy_array_equal(result, expected) + expected = tm.box_expected(expected, xbox, transpose=False) + tm.assert_equal(result, expected) result = rng != other expected = np.array([True] * 10) - tm.assert_numpy_array_equal(result, expected) + expected = tm.box_expected(expected, xbox, transpose=False) + tm.assert_equal(result, expected) with pytest.raises(TypeError): rng < other From 4955142682e4a7646fc865fdcbed8be0f2a1785d Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sun, 2 Dec 2018 15:49:19 -0800 Subject: [PATCH 4/7] whatsnew note --- doc/source/whatsnew/v0.24.0.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v0.24.0.rst b/doc/source/whatsnew/v0.24.0.rst index 3913e101d4667..bfac1e87ad450 100644 --- a/doc/source/whatsnew/v0.24.0.rst +++ b/doc/source/whatsnew/v0.24.0.rst @@ -1307,6 +1307,7 @@ Timedelta - Bug in :class:`TimedeltaIndex` where adding ``np.timedelta64('NaT')`` incorrectly returned an all-`NaT` :class:`DatetimeIndex` instead of an all-`NaT` :class:`TimedeltaIndex` (:issue:`23215`) - Bug in :class:`Timedelta` and :func:`to_timedelta()` have inconsistencies in supported unit string (:issue:`21762`) - Bug in :class:`TimedeltaIndex` division where dividing by another :class:`TimedeltaIndex` raised ``TypeError`` instead of returning a :class:`Float64Index` (:issue:`23829`, :issue:`22631`) +- Bug in :class:`TimedeltaIndex` comparison operations where comparing against non-``Timedelta``-like objects would raise ``TypeError`` instead of returning all-``False`` for ``__eq__`` and all-``True`` for ``__ne__`` (:issue:`?????`) Timezones ^^^^^^^^^ From ea38d2d2529b2057655ae0cef3ac47b7021189b4 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sun, 2 Dec 2018 15:52:50 -0800 Subject: [PATCH 5/7] GH ref --- doc/source/whatsnew/v0.24.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.24.0.rst b/doc/source/whatsnew/v0.24.0.rst index bfac1e87ad450..c271547123f78 100644 --- a/doc/source/whatsnew/v0.24.0.rst +++ b/doc/source/whatsnew/v0.24.0.rst @@ -1307,7 +1307,7 @@ Timedelta - Bug in :class:`TimedeltaIndex` where adding ``np.timedelta64('NaT')`` incorrectly returned an all-`NaT` :class:`DatetimeIndex` instead of an all-`NaT` :class:`TimedeltaIndex` (:issue:`23215`) - Bug in :class:`Timedelta` and :func:`to_timedelta()` have inconsistencies in supported unit string (:issue:`21762`) - Bug in :class:`TimedeltaIndex` division where dividing by another :class:`TimedeltaIndex` raised ``TypeError`` instead of returning a :class:`Float64Index` (:issue:`23829`, :issue:`22631`) -- Bug in :class:`TimedeltaIndex` comparison operations where comparing against non-``Timedelta``-like objects would raise ``TypeError`` instead of returning all-``False`` for ``__eq__`` and all-``True`` for ``__ne__`` (:issue:`?????`) +- Bug in :class:`TimedeltaIndex` comparison operations where comparing against non-``Timedelta``-like objects would raise ``TypeError`` instead of returning all-``False`` for ``__eq__`` and all-``True`` for ``__ne__`` (:issue:`24056`) Timezones ^^^^^^^^^ From bd7527d32d7ebc79319e107a72bf7456afb51ca6 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Mon, 3 Dec 2018 14:49:37 -0800 Subject: [PATCH 6/7] missing import --- pandas/core/arrays/timedeltas.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandas/core/arrays/timedeltas.py b/pandas/core/arrays/timedeltas.py index 3f6ef712b3904..89956355c9508 100644 --- a/pandas/core/arrays/timedeltas.py +++ b/pandas/core/arrays/timedeltas.py @@ -22,6 +22,7 @@ ABCDataFrame, ABCIndexClass, ABCSeries, ABCTimedeltaIndex) from pandas.core.dtypes.missing import isna +from pandas.core import ops from pandas.core.algorithms import checked_add_with_arr, unique1d import pandas.core.common as com From 620d52edcfc8aff12de5bf2f179f9516efe3322a Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 4 Dec 2018 10:43:50 -0800 Subject: [PATCH 7/7] skip testing object dtype on older numpy --- pandas/tests/arithmetic/test_datetime64.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pandas/tests/arithmetic/test_datetime64.py b/pandas/tests/arithmetic/test_datetime64.py index ccfa38bb77bc8..02e9c212b56ef 100644 --- a/pandas/tests/arithmetic/test_datetime64.py +++ b/pandas/tests/arithmetic/test_datetime64.py @@ -98,6 +98,11 @@ def test_comparison_invalid(self, box_with_array): ]) @pytest.mark.parametrize('dtype', [None, object]) def test_nat_comparisons_scalar(self, dtype, data, box_with_array): + if box_with_array is tm.to_array and dtype is object: + # dont bother testing ndarray comparison methods as this fails + # on older numpys (since they check object identity) + return + xbox = box_with_array if box_with_array is not pd.Index else np.ndarray left = Series(data, dtype=dtype) @@ -354,6 +359,11 @@ def test_dti_cmp_null_scalar_inequality(self, tz_naive_fixture, other, @pytest.mark.parametrize('dtype', [None, object]) def test_dti_cmp_nat(self, dtype, box_with_array): + if box_with_array is tm.to_array and dtype is object: + # dont bother testing ndarray comparison methods as this fails + # on older numpys (since they check object identity) + return + xbox = box_with_array if box_with_array is not pd.Index else np.ndarray left = pd.DatetimeIndex([pd.Timestamp('2011-01-01'), pd.NaT,