From da6878abceb7cd32d29cb96285d19c108e8ea85a Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 26 Jul 2018 19:25:45 -0700 Subject: [PATCH 1/7] fix tzawareness compat with np.datetime64 --- pandas/core/arrays/datetimes.py | 6 +++++- pandas/tests/indexes/datetimes/test_arithmetic.py | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index 00d53ad82b2dc..010adfa1877fb 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -99,7 +99,7 @@ def wrapper(self, other): meth = getattr(dtl.DatetimeLikeArrayMixin, opname) if isinstance(other, (datetime, np.datetime64, compat.string_types)): - if isinstance(other, datetime): + if isinstance(other, (datetime, np.datetime64)): # GH#18435 strings get a pass from tzawareness compat self._assert_tzawareness_compat(other) @@ -152,6 +152,10 @@ class DatetimeArrayMixin(dtl.DatetimeLikeArrayMixin): 'is_year_end', 'is_leap_year'] _object_ops = ['weekday_name', 'freq', 'tz'] + # dummy attribute so that datetime.__eq__(DatetimeArray) defers + # by returning NotImplemented + timetuple = None + # ----------------------------------------------------------------- # Constructors diff --git a/pandas/tests/indexes/datetimes/test_arithmetic.py b/pandas/tests/indexes/datetimes/test_arithmetic.py index 4ce2b1dd4fd86..f83126e4958ab 100644 --- a/pandas/tests/indexes/datetimes/test_arithmetic.py +++ b/pandas/tests/indexes/datetimes/test_arithmetic.py @@ -275,6 +275,20 @@ def test_comparison_tzawareness_compat(self, op): with pytest.raises(TypeError): op(ts, dz) + @pytest.mark.parametrize('op', [operator.eq, operator.ne, + operator.gt, operator.ge, + operator.lt, operator.le]) + @pytest.mark.parametrize('other', [datetime(2016, 1, 1), + Timestamp('2016-01-01'), + np.datetime64('2016-01-01')]) + def test_scalar_comparison_tzawareness(self, op, other, tz_aware_fixture): + tz = tz_aware_fixture + dti = pd.date_range('2016-01-01', periods=2, tz=tz) + with pytest.raises(TypeError): + op(dti, other) + with pytest.raises(TypeError): + op(other, dti) + @pytest.mark.parametrize('op', [operator.eq, operator.ne, operator.gt, operator.ge, operator.lt, operator.le]) From ba6fdd03b5488c823922924d2c3d4e2a1e96d744 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 26 Jul 2018 19:39:10 -0700 Subject: [PATCH 2/7] Fix DTI comparisons against invalid strings --- doc/source/whatsnew/v0.24.0.txt | 1 + pandas/core/arrays/datetimes.py | 8 ++- pandas/core/ops.py | 30 ++++++++ .../indexes/datetimes/test_arithmetic.py | 69 ++++++++++++++++++- 4 files changed, 105 insertions(+), 3 deletions(-) diff --git a/doc/source/whatsnew/v0.24.0.txt b/doc/source/whatsnew/v0.24.0.txt index 3b04d9937d7f2..6885484d80227 100644 --- a/doc/source/whatsnew/v0.24.0.txt +++ b/doc/source/whatsnew/v0.24.0.txt @@ -461,6 +461,7 @@ Timezones - Bug in :class:`Index` with ``datetime64[ns, tz]`` dtype that did not localize integer data correctly (:issue:`20964`) - Bug in :class:`DatetimeIndex` where constructing with an integer and tz would not localize correctly (:issue:`12619`) - Fixed bug where :meth:`DataFrame.describe` and :meth:`Series.describe` on tz-aware datetimes did not show `first` and `last` result (:issue:`21328`) +- Bug in :class:`DatetimeIndex` comparisons failing to raise ``TypeError`` when comparing timezone-aware ``DatetimeIndex`` against ``np.datetime64`` (:issue:`????`) Offsets ^^^^^^^ diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index 010adfa1877fb..13ece56dae1bc 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -29,6 +29,7 @@ import pandas.core.common as com from pandas.core.algorithms import checked_add_with_arr +from pandas.core import ops from pandas.tseries.frequencies import to_offset from pandas.tseries.offsets import Tick, Day, generate_range @@ -103,7 +104,12 @@ def wrapper(self, other): # GH#18435 strings get a pass from tzawareness compat self._assert_tzawareness_compat(other) - other = _to_m8(other, tz=self.tz) + try: + other = _to_m8(other, tz=self.tz) + except ValueError: + # string that cannot be parsed to Timestamp + return ops.invalid_comparison(self, other, op) + result = meth(self, other) if isna(other): result.fill(nat_result) diff --git a/pandas/core/ops.py b/pandas/core/ops.py index c65d2dcdc478c..30e24fd413ed7 100644 --- a/pandas/core/ops.py +++ b/pandas/core/ops.py @@ -788,6 +788,36 @@ def mask_cmp_op(x, y, op, allowed_types): return result +def invalid_comparison(left, right, op): + """ + If a comparison has mismatched types and is not necessarily meaningful, + follow python3 conventions by: + + - returning all-False for equality + - returning all-True for inequality + - raising TypeError otherwise + + Parameters + ---------- + left : array-like + right : scalar + op : operator.{eq, ne, lt, le, gt} + + Raises + ------ + TypeError : on inequality comparisons + """ + assert lib.is_scalar(right) # other cases handled later + if op is operator.eq: + res_values = np.zeros(left.shape, dtype=bool) + elif op is operator.ne: + res_values = np.ones(left.shape, dtype=bool) + else: + raise TypeError("Invalid comparison between dtype={dtype} and {typ}" + .format(dtype=left.dtype, typ=type(right).__name__)) + return res_values + + # ----------------------------------------------------------------------------- # Functions that add arithmetic methods to objects, given arithmetic factory # methods diff --git a/pandas/tests/indexes/datetimes/test_arithmetic.py b/pandas/tests/indexes/datetimes/test_arithmetic.py index f83126e4958ab..0d396ba0ebb5a 100644 --- a/pandas/tests/indexes/datetimes/test_arithmetic.py +++ b/pandas/tests/indexes/datetimes/test_arithmetic.py @@ -304,12 +304,77 @@ def test_nat_comparison_tzawareness(self, op): result = op(dti.tz_localize('US/Pacific'), pd.NaT) tm.assert_numpy_array_equal(result, expected) - def test_dti_cmp_int_raises(self): + def test_dti_cmp_str(self, tz_naive_fixture): + # regardless of tz, we expect these comparisons are valid + tz = tz_naive_fixture + rng = date_range('1/1/2000', periods=10, tz=tz) + other = '1/1/2000' + + result = rng == other + expected = np.array([True] + [False] * 9) + tm.assert_numpy_array_equal(result, expected) + + result = rng != other + expected = np.array([False] + [True] * 9) + tm.assert_numpy_array_equal(result, expected) + + result = rng < other + expected = np.array([False] * 10) + tm.assert_numpy_array_equal(result, expected) + + result = rng <= other + expected = np.array([True] + [False] * 9) + tm.assert_numpy_array_equal(result, expected) + + result = rng > other + expected = np.array([False] + [True] * 9) + tm.assert_numpy_array_equal(result, expected) + + result = rng >= other + expected = np.array([True] * 10) + tm.assert_numpy_array_equal(result, expected) + + def test_dti_cmp_str_invalid(self, tz_naive_fixture): + tz = tz_naive_fixture + rng = date_range('1/1/2000', periods=10, tz=tz) + other = 'foo' + + result = rng == other + expected = np.array([False] * 10) + tm.assert_numpy_array_equal(result, expected) + + result = rng != other + expected = np.array([True] * 10) + tm.assert_numpy_array_equal(result, expected) + + with pytest.raises(TypeError): + rng < other + with pytest.raises(TypeError): + rng <= other + with pytest.raises(TypeError): + rng > other + with pytest.raises(TypeError): + rng >= other + + def test_dti_cmp_int(self): rng = date_range('1/1/2000', periods=10) - # raise TypeError for now + result = rng == rng[3].value + expected = np.array([False] * 10) + tm.assert_numpy_array_equal(result, expected) + + result = rng != rng[3].value + expected = np.array([True] * 10) + tm.assert_numpy_array_equal(result, expected) + with pytest.raises(TypeError): rng < rng[3].value + with pytest.raises(TypeError): + rng <= rng[3].value + with pytest.raises(TypeError): + rng > rng[3].value + with pytest.raises(TypeError): + rng >= rng[3].value def test_dti_cmp_list(self): rng = date_range('1/1/2000', periods=10) From 6e92dc291a3df7b00e1df40c51f66c35104fc46d Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 26 Jul 2018 19:45:06 -0700 Subject: [PATCH 3/7] Whatsnew, de-duplicate test --- doc/source/whatsnew/v0.24.0.txt | 1 + pandas/core/arrays/datetimes.py | 4 ++- .../indexes/datetimes/test_arithmetic.py | 25 +++---------------- 3 files changed, 7 insertions(+), 23 deletions(-) diff --git a/doc/source/whatsnew/v0.24.0.txt b/doc/source/whatsnew/v0.24.0.txt index 6885484d80227..ebb0c99de6513 100644 --- a/doc/source/whatsnew/v0.24.0.txt +++ b/doc/source/whatsnew/v0.24.0.txt @@ -462,6 +462,7 @@ Timezones - Bug in :class:`DatetimeIndex` where constructing with an integer and tz would not localize correctly (:issue:`12619`) - Fixed bug where :meth:`DataFrame.describe` and :meth:`Series.describe` on tz-aware datetimes did not show `first` and `last` result (:issue:`21328`) - Bug in :class:`DatetimeIndex` comparisons failing to raise ``TypeError`` when comparing timezone-aware ``DatetimeIndex`` against ``np.datetime64`` (:issue:`????`) +- Bug in :class:`DatetimeIndex` comparisons where string comparisons incorrectly raises ``TypeError`` (:issue:`????`) Offsets ^^^^^^^ diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index 13ece56dae1bc..604cedf90b86b 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -5,7 +5,7 @@ import numpy as np from pytz import utc -from pandas._libs import tslib +from pandas._libs import lib, tslib from pandas._libs.tslib import Timestamp, NaT, iNaT from pandas._libs.tslibs import ( normalize_date, @@ -113,6 +113,8 @@ def wrapper(self, other): result = meth(self, other) if isna(other): result.fill(nat_result) + elif lib.is_scalar(other): + return ops.invalid_comparison(self, other, op) else: if isinstance(other, list): other = type(self)(other) diff --git a/pandas/tests/indexes/datetimes/test_arithmetic.py b/pandas/tests/indexes/datetimes/test_arithmetic.py index 0d396ba0ebb5a..99a6a311b4c0d 100644 --- a/pandas/tests/indexes/datetimes/test_arithmetic.py +++ b/pandas/tests/indexes/datetimes/test_arithmetic.py @@ -334,10 +334,11 @@ def test_dti_cmp_str(self, tz_naive_fixture): expected = np.array([True] * 10) tm.assert_numpy_array_equal(result, expected) - def test_dti_cmp_str_invalid(self, tz_naive_fixture): + @pytest.mark.parametrize('other', ['foo', 99, 4.0, + object(), timedelta(days=2)]) + def test_dti_cmp_scalar_invalid(self, other, tz_naive_fixture): tz = tz_naive_fixture rng = date_range('1/1/2000', periods=10, tz=tz) - other = 'foo' result = rng == other expected = np.array([False] * 10) @@ -356,26 +357,6 @@ def test_dti_cmp_str_invalid(self, tz_naive_fixture): with pytest.raises(TypeError): rng >= other - def test_dti_cmp_int(self): - rng = date_range('1/1/2000', periods=10) - - result = rng == rng[3].value - expected = np.array([False] * 10) - tm.assert_numpy_array_equal(result, expected) - - result = rng != rng[3].value - expected = np.array([True] * 10) - tm.assert_numpy_array_equal(result, expected) - - with pytest.raises(TypeError): - rng < rng[3].value - with pytest.raises(TypeError): - rng <= rng[3].value - with pytest.raises(TypeError): - rng > rng[3].value - with pytest.raises(TypeError): - rng >= rng[3].value - def test_dti_cmp_list(self): rng = date_range('1/1/2000', periods=10) From 16202d5a066e61c39c41bbd19da09a64dc939c88 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 26 Jul 2018 20:00:24 -0700 Subject: [PATCH 4/7] Fix DTI comparisons against TDI --- doc/source/whatsnew/v0.24.0.txt | 1 + pandas/core/arrays/datetimes.py | 18 ++++++------- pandas/core/ops.py | 3 +-- .../indexes/datetimes/test_arithmetic.py | 27 +++++++++++++++++++ 4 files changed, 38 insertions(+), 11 deletions(-) diff --git a/doc/source/whatsnew/v0.24.0.txt b/doc/source/whatsnew/v0.24.0.txt index ebb0c99de6513..b69620b4d363e 100644 --- a/doc/source/whatsnew/v0.24.0.txt +++ b/doc/source/whatsnew/v0.24.0.txt @@ -463,6 +463,7 @@ Timezones - Fixed bug where :meth:`DataFrame.describe` and :meth:`Series.describe` on tz-aware datetimes did not show `first` and `last` result (:issue:`21328`) - Bug in :class:`DatetimeIndex` comparisons failing to raise ``TypeError`` when comparing timezone-aware ``DatetimeIndex`` against ``np.datetime64`` (:issue:`????`) - Bug in :class:`DatetimeIndex` comparisons where string comparisons incorrectly raises ``TypeError`` (:issue:`????`) +- Bug in :class:`DatetimeIndex` comparisons when comparing against ``timedelta64[ns]`` dtyped arrays; in some cases ``TypeError`` was incorrectly raised, in others it incorrectly failed to raise (:issue:`????`) Offsets ^^^^^^^ diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index 604cedf90b86b..4ea7e8fa71027 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -18,7 +18,7 @@ from pandas.core.dtypes.common import ( _NS_DTYPE, - is_datetimelike, + is_object_dtype, is_datetime64tz_dtype, is_datetime64_dtype, is_timedelta64_dtype, @@ -121,15 +121,15 @@ def wrapper(self, other): elif not isinstance(other, (np.ndarray, ABCIndexClass, ABCSeries)): # Following Timestamp convention, __eq__ is all-False # and __ne__ is all True, others raise TypeError. - if opname == '__eq__': - return np.zeros(shape=self.shape, dtype=bool) - elif opname == '__ne__': - return np.ones(shape=self.shape, dtype=bool) - raise TypeError('%s type object %s' % - (type(other), str(other))) - - if is_datetimelike(other): + return ops.invalid_comparison(self, other, op) + + if is_datetime64_dtype(other) or is_datetime64tz_dtype(other): self._assert_tzawareness_compat(other) + elif is_object_dtype(other): + raise NotImplementedError + else: + # e.g. is_timedelta64_dtype(other) + return ops.invalid_comparison(self, other, op) result = meth(self, np.asarray(other)) result = com.values_from_object(result) diff --git a/pandas/core/ops.py b/pandas/core/ops.py index 30e24fd413ed7..ed6557fab3e3f 100644 --- a/pandas/core/ops.py +++ b/pandas/core/ops.py @@ -800,14 +800,13 @@ def invalid_comparison(left, right, op): Parameters ---------- left : array-like - right : scalar + right : scalar, array-like op : operator.{eq, ne, lt, le, gt} Raises ------ TypeError : on inequality comparisons """ - assert lib.is_scalar(right) # other cases handled later if op is operator.eq: res_values = np.zeros(left.shape, dtype=bool) elif op is operator.ne: diff --git a/pandas/tests/indexes/datetimes/test_arithmetic.py b/pandas/tests/indexes/datetimes/test_arithmetic.py index 99a6a311b4c0d..090a36b119b85 100644 --- a/pandas/tests/indexes/datetimes/test_arithmetic.py +++ b/pandas/tests/indexes/datetimes/test_arithmetic.py @@ -364,6 +364,33 @@ def test_dti_cmp_list(self): expected = rng == rng tm.assert_numpy_array_equal(result, expected) + @pytest.mark.parametrize('other', [ + pd.timedelta_range('1D', periods=10), + pd.timedelta_range('1D', periods=10).to_series(), + pd.timedelta_range('1D', periods=10).asi8.view('m8[ns]') + ], ids=lambda x: type(x).__name__) + def test_dti_cmp_tdi_tzawareness(self, other): + # reversion test that we _don't_ call _assert_tzawareness_compat + # when comparing against TimedeltaIndex + dti = date_range('2000-01-01', periods=10, tz='Asia/Tokyo') + + result = dti == other + expected = np.array([False] * 10) + tm.assert_numpy_array_equal(result, expected) + + result = dti != other + expected = np.array([True] * 10) + tm.assert_numpy_array_equal(result, expected) + + with pytest.raises(TypeError): + dti < other + with pytest.raises(TypeError): + dti <= other + with pytest.raises(TypeError): + dti > other + with pytest.raises(TypeError): + dti >= other + class TestDatetimeIndexArithmetic(object): From c4ab98c29e31658c2b499e190358d64491fe8687 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 26 Jul 2018 20:14:21 -0700 Subject: [PATCH 5/7] Fix DTI comparisons against object-dtype --- doc/source/whatsnew/v0.24.0.txt | 1 + pandas/core/arrays/datetimes.py | 14 +++++++----- .../indexes/datetimes/test_arithmetic.py | 22 +++++++++++++++++++ 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/doc/source/whatsnew/v0.24.0.txt b/doc/source/whatsnew/v0.24.0.txt index b69620b4d363e..6ecfe2e11d3ce 100644 --- a/doc/source/whatsnew/v0.24.0.txt +++ b/doc/source/whatsnew/v0.24.0.txt @@ -464,6 +464,7 @@ Timezones - Bug in :class:`DatetimeIndex` comparisons failing to raise ``TypeError`` when comparing timezone-aware ``DatetimeIndex`` against ``np.datetime64`` (:issue:`????`) - Bug in :class:`DatetimeIndex` comparisons where string comparisons incorrectly raises ``TypeError`` (:issue:`????`) - Bug in :class:`DatetimeIndex` comparisons when comparing against ``timedelta64[ns]`` dtyped arrays; in some cases ``TypeError`` was incorrectly raised, in others it incorrectly failed to raise (:issue:`????`) +- Bug in :class:`DatetimeIndex` comparisons when comparing against object-dtyped arrays (:issue:`????`) Offsets ^^^^^^^ diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index 4ea7e8fa71027..31ffde0ea12d7 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -117,21 +117,23 @@ def wrapper(self, other): return ops.invalid_comparison(self, other, op) else: if isinstance(other, list): + # FIXME: This can break for object-dtype with mixed types other = type(self)(other) elif not isinstance(other, (np.ndarray, ABCIndexClass, ABCSeries)): # Following Timestamp convention, __eq__ is all-False # and __ne__ is all True, others raise TypeError. return ops.invalid_comparison(self, other, op) - if is_datetime64_dtype(other) or is_datetime64tz_dtype(other): - self._assert_tzawareness_compat(other) - elif is_object_dtype(other): - raise NotImplementedError - else: + if is_object_dtype(other): + result = op(self.astype('O'), np.array(other)) + elif not (is_datetime64_dtype(other) or + is_datetime64tz_dtype(other)): # e.g. is_timedelta64_dtype(other) return ops.invalid_comparison(self, other, op) + else: + self._assert_tzawareness_compat(other) + result = meth(self, np.asarray(other)) - result = meth(self, np.asarray(other)) result = com.values_from_object(result) # Make sure to pass an array to result[...]; indexing with diff --git a/pandas/tests/indexes/datetimes/test_arithmetic.py b/pandas/tests/indexes/datetimes/test_arithmetic.py index 090a36b119b85..f2a0a8735846e 100644 --- a/pandas/tests/indexes/datetimes/test_arithmetic.py +++ b/pandas/tests/indexes/datetimes/test_arithmetic.py @@ -391,6 +391,28 @@ def test_dti_cmp_tdi_tzawareness(self, other): with pytest.raises(TypeError): dti >= other + def test_dti_cmp_object_dtype(self): + dti = date_range('2000-01-01', periods=10, tz='Asia/Tokyo') + + other = dti.astype('O') + + result = dti == other + expected = np.array([True] * 10) + tm.assert_numpy_array_equal(result, expected) + + other = dti.tz_localize(None) + with pytest.raises(TypeError): + # tzawareness failure + dti != other + + other = np.array(list(dti[:5]) + [Timedelta(days=1)] * 5) + result = dti == other + expected = np.array([True] * 5 + [False] * 5) + tm.assert_numpy_array_equal(result, expected) + + with pytest.raises(TypeError): + dti >= other + class TestDatetimeIndexArithmetic(object): From 92b74ec5775cd2c9656bdb0f440c12d5b7229f92 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 26 Jul 2018 20:18:58 -0700 Subject: [PATCH 6/7] Add GH references --- doc/source/whatsnew/v0.24.0.txt | 8 ++++---- pandas/tests/indexes/datetimes/test_arithmetic.py | 4 ++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/doc/source/whatsnew/v0.24.0.txt b/doc/source/whatsnew/v0.24.0.txt index 6ecfe2e11d3ce..83e7d3997fb49 100644 --- a/doc/source/whatsnew/v0.24.0.txt +++ b/doc/source/whatsnew/v0.24.0.txt @@ -461,10 +461,10 @@ Timezones - Bug in :class:`Index` with ``datetime64[ns, tz]`` dtype that did not localize integer data correctly (:issue:`20964`) - Bug in :class:`DatetimeIndex` where constructing with an integer and tz would not localize correctly (:issue:`12619`) - Fixed bug where :meth:`DataFrame.describe` and :meth:`Series.describe` on tz-aware datetimes did not show `first` and `last` result (:issue:`21328`) -- Bug in :class:`DatetimeIndex` comparisons failing to raise ``TypeError`` when comparing timezone-aware ``DatetimeIndex`` against ``np.datetime64`` (:issue:`????`) -- Bug in :class:`DatetimeIndex` comparisons where string comparisons incorrectly raises ``TypeError`` (:issue:`????`) -- Bug in :class:`DatetimeIndex` comparisons when comparing against ``timedelta64[ns]`` dtyped arrays; in some cases ``TypeError`` was incorrectly raised, in others it incorrectly failed to raise (:issue:`????`) -- Bug in :class:`DatetimeIndex` comparisons when comparing against object-dtyped arrays (:issue:`????`) +- Bug in :class:`DatetimeIndex` comparisons failing to raise ``TypeError`` when comparing timezone-aware ``DatetimeIndex`` against ``np.datetime64`` (:issue:`22074`) +- Bug in :class:`DatetimeIndex` comparisons where string comparisons incorrectly raises ``TypeError`` (:issue:`22074`) +- Bug in :class:`DatetimeIndex` comparisons when comparing against ``timedelta64[ns]`` dtyped arrays; in some cases ``TypeError`` was incorrectly raised, in others it incorrectly failed to raise (:issue:`22074`) +- Bug in :class:`DatetimeIndex` comparisons when comparing against object-dtyped arrays (:issue:`22074`) Offsets ^^^^^^^ diff --git a/pandas/tests/indexes/datetimes/test_arithmetic.py b/pandas/tests/indexes/datetimes/test_arithmetic.py index f2a0a8735846e..952532841d4fe 100644 --- a/pandas/tests/indexes/datetimes/test_arithmetic.py +++ b/pandas/tests/indexes/datetimes/test_arithmetic.py @@ -305,6 +305,7 @@ def test_nat_comparison_tzawareness(self, op): tm.assert_numpy_array_equal(result, expected) def test_dti_cmp_str(self, tz_naive_fixture): + # GH#22074 # regardless of tz, we expect these comparisons are valid tz = tz_naive_fixture rng = date_range('1/1/2000', periods=10, tz=tz) @@ -337,6 +338,7 @@ def test_dti_cmp_str(self, tz_naive_fixture): @pytest.mark.parametrize('other', ['foo', 99, 4.0, object(), timedelta(days=2)]) def test_dti_cmp_scalar_invalid(self, other, tz_naive_fixture): + # GH#22074 tz = tz_naive_fixture rng = date_range('1/1/2000', periods=10, tz=tz) @@ -370,6 +372,7 @@ def test_dti_cmp_list(self): pd.timedelta_range('1D', periods=10).asi8.view('m8[ns]') ], ids=lambda x: type(x).__name__) def test_dti_cmp_tdi_tzawareness(self, other): + # GH#22074 # reversion test that we _don't_ call _assert_tzawareness_compat # when comparing against TimedeltaIndex dti = date_range('2000-01-01', periods=10, tz='Asia/Tokyo') @@ -392,6 +395,7 @@ def test_dti_cmp_tdi_tzawareness(self, other): dti >= other def test_dti_cmp_object_dtype(self): + # GH#22074 dti = date_range('2000-01-01', periods=10, tz='Asia/Tokyo') other = dti.astype('O') From 23aa2a0c43e6f0b56b0972374797a1e5bbf85d31 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 26 Jul 2018 21:20:13 -0700 Subject: [PATCH 7/7] fix incorrect tests, update whatsnew --- doc/source/whatsnew/v0.24.0.txt | 8 ++++---- pandas/core/ops.py | 2 +- pandas/tests/frame/test_operators.py | 16 ++++++++++++++-- pandas/tests/frame/test_query_eval.py | 8 ++++++-- pandas/tests/series/test_operators.py | 11 +++++++++-- 5 files changed, 34 insertions(+), 11 deletions(-) diff --git a/doc/source/whatsnew/v0.24.0.txt b/doc/source/whatsnew/v0.24.0.txt index 83e7d3997fb49..3c92401224366 100644 --- a/doc/source/whatsnew/v0.24.0.txt +++ b/doc/source/whatsnew/v0.24.0.txt @@ -439,6 +439,9 @@ Datetimelike - Fixed bug where two :class:`DateOffset` objects with different ``normalize`` attributes could evaluate as equal (:issue:`21404`) - Fixed bug where :meth:`Timestamp.resolution` incorrectly returned 1-microsecond ``timedelta`` instead of 1-nanosecond :class:`Timedelta` (:issue:`21336`,:issue:`21365`) +- Bug in :class:`DatetimeIndex` comparisons where string comparisons incorrectly raises ``TypeError`` (:issue:`22074`) +- Bug in :class:`DatetimeIndex` comparisons when comparing against ``timedelta64[ns]`` dtyped arrays; in some cases ``TypeError`` was incorrectly raised, in others it incorrectly failed to raise (:issue:`22074`) +- Bug in :class:`DatetimeIndex` comparisons when comparing against object-dtyped arrays (:issue:`22074`) Timedelta ^^^^^^^^^ @@ -462,9 +465,6 @@ Timezones - Bug in :class:`DatetimeIndex` where constructing with an integer and tz would not localize correctly (:issue:`12619`) - Fixed bug where :meth:`DataFrame.describe` and :meth:`Series.describe` on tz-aware datetimes did not show `first` and `last` result (:issue:`21328`) - Bug in :class:`DatetimeIndex` comparisons failing to raise ``TypeError`` when comparing timezone-aware ``DatetimeIndex`` against ``np.datetime64`` (:issue:`22074`) -- Bug in :class:`DatetimeIndex` comparisons where string comparisons incorrectly raises ``TypeError`` (:issue:`22074`) -- Bug in :class:`DatetimeIndex` comparisons when comparing against ``timedelta64[ns]`` dtyped arrays; in some cases ``TypeError`` was incorrectly raised, in others it incorrectly failed to raise (:issue:`22074`) -- Bug in :class:`DatetimeIndex` comparisons when comparing against object-dtyped arrays (:issue:`22074`) Offsets ^^^^^^^ @@ -479,7 +479,7 @@ Numeric - Bug in :class:`Series` ``__rmatmul__`` doesn't support matrix vector multiplication (:issue:`21530`) - Bug in :func:`factorize` fails with read-only array (:issue:`12813`) - Fixed bug in :func:`unique` handled signed zeros inconsistently: for some inputs 0.0 and -0.0 were treated as equal and for some inputs as different. Now they are treated as equal for all inputs (:issue:`21866`) -- +- Bug in :class:`Series` comparison against datetime-like scalars and arrays (:issue:`22074`) - Strings diff --git a/pandas/core/ops.py b/pandas/core/ops.py index ed6557fab3e3f..729775d1d8648 100644 --- a/pandas/core/ops.py +++ b/pandas/core/ops.py @@ -1288,7 +1288,7 @@ def na_op(x, y): result = _comp_method_OBJECT_ARRAY(op, x, y) elif is_datetimelike_v_numeric(x, y): - raise TypeError("invalid type comparison") + return invalid_comparison(x, y, op) else: diff --git a/pandas/tests/frame/test_operators.py b/pandas/tests/frame/test_operators.py index fdf50805ad818..b6b783134b424 100644 --- a/pandas/tests/frame/test_operators.py +++ b/pandas/tests/frame/test_operators.py @@ -157,8 +157,20 @@ def test_comparison_invalid(self): def check(df, df2): for (x, y) in [(df, df2), (df2, df)]: - pytest.raises(TypeError, lambda: x == y) - pytest.raises(TypeError, lambda: x != y) + # we expect the result to match Series comparisons for + # == and !=, inequalities should raise + result = x == y + expected = DataFrame({col: x[col] == y[col] + for col in x.columns}, + index=x.index, columns=x.columns) + assert_frame_equal(result, expected) + + result = x != y + expected = DataFrame({col: x[col] != y[col] + for col in x.columns}, + index=x.index, columns=x.columns) + assert_frame_equal(result, expected) + pytest.raises(TypeError, lambda: x >= y) pytest.raises(TypeError, lambda: x > y) pytest.raises(TypeError, lambda: x < y) diff --git a/pandas/tests/frame/test_query_eval.py b/pandas/tests/frame/test_query_eval.py index a226f8de3c8bd..92035850806cb 100644 --- a/pandas/tests/frame/test_query_eval.py +++ b/pandas/tests/frame/test_query_eval.py @@ -463,9 +463,13 @@ def test_date_query_with_non_date(self): df = DataFrame({'dates': date_range('1/1/2012', periods=n), 'nondate': np.arange(n)}) - ops = '==', '!=', '<', '>', '<=', '>=' + result = df.query('dates == nondate', parser=parser, engine=engine) + assert len(result) == 0 - for op in ops: + result = df.query('dates != nondate', parser=parser, engine=engine) + assert_frame_equal(result, df) + + for op in ['<', '>', '<=', '>=']: with pytest.raises(TypeError): df.query('dates %s nondate' % op, parser=parser, engine=engine) diff --git a/pandas/tests/series/test_operators.py b/pandas/tests/series/test_operators.py index ecb74622edf10..fb6e7c412d884 100644 --- a/pandas/tests/series/test_operators.py +++ b/pandas/tests/series/test_operators.py @@ -238,8 +238,15 @@ def test_comparison_invalid(self): s2 = Series(date_range('20010101', periods=5)) for (x, y) in [(s, s2), (s2, s)]: - pytest.raises(TypeError, lambda: x == y) - pytest.raises(TypeError, lambda: x != y) + + result = x == y + expected = Series([False] * 5) + assert_series_equal(result, expected) + + result = x != y + expected = Series([True] * 5) + assert_series_equal(result, expected) + pytest.raises(TypeError, lambda: x >= y) pytest.raises(TypeError, lambda: x > y) pytest.raises(TypeError, lambda: x < y)