From b100f59dd832688ce077552a861a44b694010d63 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 18 Jan 2018 09:18:52 -0800 Subject: [PATCH 01/10] Fix DTI comparison with None, datetime.date --- pandas/core/indexes/datetimes.py | 20 +++++++++++-------- .../indexes/datetimes/test_arithmetic.py | 19 ++++++++++++++++++ 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py index 0349e5c0a448f..a34701c58b6b7 100644 --- a/pandas/core/indexes/datetimes.py +++ b/pandas/core/indexes/datetimes.py @@ -119,8 +119,18 @@ def wrapper(self, other): else: if isinstance(other, list): other = DatetimeIndex(other) - elif not isinstance(other, (np.ndarray, Index, ABCSeries)): - other = _ensure_datetime64(other) + elif not isinstance(other, (np.datetime64, np.ndarray, + Index, ABCSeries)): + # Following Timestamp convention, __eq__ is all-False + # and __ne__ is all True, others raise TypeError. + if opname == '__eq__': + result = np.zeros(shape=self.shape, dtype=bool) + elif opname == '__ne__': + result = np.ones(shape=self.shape, dtype=bool) + else: + raise TypeError('%s type object %s' % + (type(other), str(other))) + return result if is_datetimelike(other): self._assert_tzawareness_compat(other) @@ -147,12 +157,6 @@ def wrapper(self, other): return compat.set_function_name(wrapper, opname, cls) -def _ensure_datetime64(other): - if isinstance(other, np.datetime64): - return other - raise TypeError('%s type object %s' % (type(other), str(other))) - - _midnight = time(0, 0) diff --git a/pandas/tests/indexes/datetimes/test_arithmetic.py b/pandas/tests/indexes/datetimes/test_arithmetic.py index 011b33a4d6f35..c14e07160b3af 100644 --- a/pandas/tests/indexes/datetimes/test_arithmetic.py +++ b/pandas/tests/indexes/datetimes/test_arithmetic.py @@ -41,6 +41,25 @@ def addend(request): return request.param +class TestDatetimeIndexComparisons(object): + @pytest.mark.parametrize('other', [None, + datetime(2016, 1, 1).date()]) + def test_dti_cmp_invalid(self, tz, other): + # GH#19288 + dti = pd.date_range('2016-01-01', periods=2, tz=tz) + + assert not (dti == other).any() + assert (dti != other).all() + with pytest.raises(TypeError): + dti < other + with pytest.raises(TypeError): + dti <= other + with pytest.raises(TypeError): + dti > other + with pytest.raises(TypeError): + dti >= other + + class TestDatetimeIndexArithmetic(object): def test_dti_add_timestamp_raises(self): From 59ac32c1d6065477e46df373ad2ba22cd51e1bf4 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 18 Jan 2018 09:21:41 -0800 Subject: [PATCH 02/10] add GH reference, whatsnew note --- doc/source/whatsnew/v0.23.0.txt | 1 + pandas/tests/indexes/datetimes/test_arithmetic.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.23.0.txt b/doc/source/whatsnew/v0.23.0.txt index 2f3b62febed7d..e6eb286dee254 100644 --- a/doc/source/whatsnew/v0.23.0.txt +++ b/doc/source/whatsnew/v0.23.0.txt @@ -432,6 +432,7 @@ Conversion - Bug in ``.astype()`` to non-ns timedelta units would hold the incorrect dtype (:issue:`19176`, :issue:`19223`, :issue:`12425`) - Bug in subtracting :class:`Series` from ``NaT`` incorrectly returning ``NaT`` (:issue:`19158`) - Bug in comparison of timezone-aware :class:`DatetimeIndex` against ``NaT`` incorrectly raising ``TypeError`` (:issue:`19276`) +- Bug in comparison of :class:`DatetimeIndex` against ``None`` or ``datetime.date`` objects raising ``TypeError`` for ``==`` and ``!=`` comparisons instead of all-``False`` and all-``True``, respectively (:issue:`19301`) Indexing ^^^^^^^^ diff --git a/pandas/tests/indexes/datetimes/test_arithmetic.py b/pandas/tests/indexes/datetimes/test_arithmetic.py index c14e07160b3af..c1c7da84b1f99 100644 --- a/pandas/tests/indexes/datetimes/test_arithmetic.py +++ b/pandas/tests/indexes/datetimes/test_arithmetic.py @@ -45,7 +45,7 @@ class TestDatetimeIndexComparisons(object): @pytest.mark.parametrize('other', [None, datetime(2016, 1, 1).date()]) def test_dti_cmp_invalid(self, tz, other): - # GH#19288 + # GH#19301 dti = pd.date_range('2016-01-01', periods=2, tz=tz) assert not (dti == other).any() From 6720de9d1b064dbc285b1315bab986545ba47960 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sat, 20 Jan 2018 09:50:21 -0800 Subject: [PATCH 03/10] Merge branch 'master' of https://github.com/pandas-dev/pandas into dti_cmp_fix From f2dbb7ef41745a17c31e032f3979e297f3b229db Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sun, 21 Jan 2018 10:50:52 -0800 Subject: [PATCH 04/10] change return order per request --- pandas/core/indexes/datetimes.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py index 015bdf1c341a7..e7164e72ed40a 100644 --- a/pandas/core/indexes/datetimes.py +++ b/pandas/core/indexes/datetimes.py @@ -123,13 +123,11 @@ def wrapper(self, other): # Following Timestamp convention, __eq__ is all-False # and __ne__ is all True, others raise TypeError. if opname == '__eq__': - result = np.zeros(shape=self.shape, dtype=bool) + return np.zeros(shape=self.shape, dtype=bool) elif opname == '__ne__': - result = np.ones(shape=self.shape, dtype=bool) - else: - raise TypeError('%s type object %s' % - (type(other), str(other))) - return result + return np.ones(shape=self.shape, dtype=bool) + raise TypeError('%s type object %s' % + (type(other), str(other))) if is_datetimelike(other): self._assert_tzawareness_compat(other) From 0184b39aaa5c9488cda3bb1cd02eacb98ebe547f Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sun, 21 Jan 2018 10:51:04 -0800 Subject: [PATCH 05/10] de-duplicate tests by giving proper names --- .../indexes/datetimes/test_arithmetic.py | 130 +++++++++--------- 1 file changed, 66 insertions(+), 64 deletions(-) diff --git a/pandas/tests/indexes/datetimes/test_arithmetic.py b/pandas/tests/indexes/datetimes/test_arithmetic.py index fa0f1b8d3375b..8255c586e28b3 100644 --- a/pandas/tests/indexes/datetimes/test_arithmetic.py +++ b/pandas/tests/indexes/datetimes/test_arithmetic.py @@ -61,7 +61,6 @@ def test_dti_cmp_invalid(self, tz, other): with pytest.raises(TypeError): dti >= other - # TODO: De-duplicate with test_comparisons_nat below def test_dti_cmp_nat(self): left = pd.DatetimeIndex([pd.Timestamp('2011-01-01'), pd.NaT, pd.Timestamp('2011-01-03')]) @@ -89,69 +88,7 @@ def test_dti_cmp_nat(self): tm.assert_numpy_array_equal(lhs < pd.NaT, expected) tm.assert_numpy_array_equal(pd.NaT > lhs, expected) - @pytest.mark.parametrize('op', [operator.eq, operator.ne, - operator.gt, operator.ge, - operator.lt, operator.le]) - def test_comparison_tzawareness_compat(self, op): - # GH#18162 - dr = pd.date_range('2016-01-01', periods=6) - dz = dr.tz_localize('US/Pacific') - - with pytest.raises(TypeError): - op(dr, dz) - with pytest.raises(TypeError): - op(dr, list(dz)) - with pytest.raises(TypeError): - op(dz, dr) - with pytest.raises(TypeError): - op(dz, list(dr)) - - # Check that there isn't a problem aware-aware and naive-naive do not - # raise - assert (dr == dr).all() - assert (dr == list(dr)).all() - assert (dz == dz).all() - assert (dz == list(dz)).all() - - # Check comparisons against scalar Timestamps - ts = pd.Timestamp('2000-03-14 01:59') - ts_tz = pd.Timestamp('2000-03-14 01:59', tz='Europe/Amsterdam') - - assert (dr > ts).all() - with pytest.raises(TypeError): - op(dr, ts_tz) - - assert (dz > ts_tz).all() - with pytest.raises(TypeError): - op(dz, ts) - - @pytest.mark.parametrize('op', [operator.eq, operator.ne, - operator.gt, operator.ge, - operator.lt, operator.le]) - def test_nat_comparison_tzawareness(self, op): - # GH#19276 - # tzaware DatetimeIndex should not raise when compared to NaT - dti = pd.DatetimeIndex(['2014-01-01', pd.NaT, '2014-03-01', pd.NaT, - '2014-05-01', '2014-07-01']) - expected = np.array([op == operator.ne] * len(dti)) - result = op(dti, pd.NaT) - tm.assert_numpy_array_equal(result, expected) - - result = op(dti.tz_localize('US/Pacific'), pd.NaT) - tm.assert_numpy_array_equal(result, expected) - - def test_comparisons_coverage(self): - rng = date_range('1/1/2000', periods=10) - - # raise TypeError for now - pytest.raises(TypeError, rng.__lt__, rng[3].value) - - result = rng == list(rng) - exp = rng == rng - tm.assert_numpy_array_equal(result, exp) - - def test_comparisons_nat(self): - + def test_dti_cmp_nat_behaves_like_float_cmp_nan(self): fidx1 = pd.Index([1.0, np.nan, 3.0, np.nan, 5.0, 7.0]) fidx2 = pd.Index([2.0, 3.0, np.nan, np.nan, 6.0, 7.0]) @@ -240,6 +177,71 @@ def test_comparisons_nat(self): expected = np.array([True, True, False, True, True, True]) tm.assert_numpy_array_equal(result, expected) + @pytest.mark.parametrize('op', [operator.eq, operator.ne, + operator.gt, operator.ge, + operator.lt, operator.le]) + def test_comparison_tzawareness_compat(self, op): + # GH#18162 + dr = pd.date_range('2016-01-01', periods=6) + dz = dr.tz_localize('US/Pacific') + + with pytest.raises(TypeError): + op(dr, dz) + with pytest.raises(TypeError): + op(dr, list(dz)) + with pytest.raises(TypeError): + op(dz, dr) + with pytest.raises(TypeError): + op(dz, list(dr)) + + # Check that there isn't a problem aware-aware and naive-naive do not + # raise + assert (dr == dr).all() + assert (dr == list(dr)).all() + assert (dz == dz).all() + assert (dz == list(dz)).all() + + # Check comparisons against scalar Timestamps + ts = pd.Timestamp('2000-03-14 01:59') + ts_tz = pd.Timestamp('2000-03-14 01:59', tz='Europe/Amsterdam') + + assert (dr > ts).all() + with pytest.raises(TypeError): + op(dr, ts_tz) + + assert (dz > ts_tz).all() + with pytest.raises(TypeError): + op(dz, ts) + + @pytest.mark.parametrize('op', [operator.eq, operator.ne, + operator.gt, operator.ge, + operator.lt, operator.le]) + def test_nat_comparison_tzawareness(self, op): + # GH#19276 + # tzaware DatetimeIndex should not raise when compared to NaT + dti = pd.DatetimeIndex(['2014-01-01', pd.NaT, '2014-03-01', pd.NaT, + '2014-05-01', '2014-07-01']) + expected = np.array([op == operator.ne] * len(dti)) + result = op(dti, pd.NaT) + tm.assert_numpy_array_equal(result, expected) + + result = op(dti.tz_localize('US/Pacific'), pd.NaT) + tm.assert_numpy_array_equal(result, expected) + + def test_dti_cmp_int_raises(self): + rng = date_range('1/1/2000', periods=10) + + # raise TypeError for now + with pytest.raises(TypeError): + rng < rng[3].value + + def test_dti_cmp_list(self): + rng = date_range('1/1/2000', periods=10) + + result = rng == list(rng) + expected = rng == rng + tm.assert_numpy_array_equal(result, expected) + class TestDatetimeIndexArithmetic(object): From 78d4eb1e050bd30f5cec439533c9058061d749b6 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sun, 21 Jan 2018 16:26:35 -0800 Subject: [PATCH 06/10] add nan to invalid test --- pandas/tests/indexes/datetimes/test_arithmetic.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/tests/indexes/datetimes/test_arithmetic.py b/pandas/tests/indexes/datetimes/test_arithmetic.py index 8255c586e28b3..5961c33e1a92f 100644 --- a/pandas/tests/indexes/datetimes/test_arithmetic.py +++ b/pandas/tests/indexes/datetimes/test_arithmetic.py @@ -45,7 +45,8 @@ def addend(request): class TestDatetimeIndexComparisons(object): @pytest.mark.parametrize('other', [None, - datetime(2016, 1, 1).date()]) + datetime(2016, 1, 1).date(), + np.nan]) def test_dti_cmp_invalid(self, tz, other): # GH#19301 dti = pd.date_range('2016-01-01', periods=2, tz=tz) From 39ad04d9e49dce9c334afdf0729571845d04db6c Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Mon, 22 Jan 2018 20:05:02 -0800 Subject: [PATCH 07/10] rename test_dti_cmp_invalid --- pandas/tests/indexes/datetimes/test_arithmetic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/indexes/datetimes/test_arithmetic.py b/pandas/tests/indexes/datetimes/test_arithmetic.py index 5961c33e1a92f..b3b1f8450df5c 100644 --- a/pandas/tests/indexes/datetimes/test_arithmetic.py +++ b/pandas/tests/indexes/datetimes/test_arithmetic.py @@ -47,7 +47,7 @@ class TestDatetimeIndexComparisons(object): @pytest.mark.parametrize('other', [None, datetime(2016, 1, 1).date(), np.nan]) - def test_dti_cmp_invalid(self, tz, other): + def test_dti_cmp_non_datetime(self, tz, other): # GH#19301 dti = pd.date_range('2016-01-01', periods=2, tz=tz) From d578aba5a75e0aa20a073745bd8103d8a59daf01 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 23 Jan 2018 19:15:41 -0800 Subject: [PATCH 08/10] add tests for comparisons against datetimelikes --- .../indexes/datetimes/test_arithmetic.py | 53 ++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/pandas/tests/indexes/datetimes/test_arithmetic.py b/pandas/tests/indexes/datetimes/test_arithmetic.py index b3b1f8450df5c..e084319ab0490 100644 --- a/pandas/tests/indexes/datetimes/test_arithmetic.py +++ b/pandas/tests/indexes/datetimes/test_arithmetic.py @@ -14,6 +14,7 @@ from pandas import (Timestamp, Timedelta, Series, DatetimeIndex, TimedeltaIndex, date_range) +from pandas._libs import tslib @pytest.fixture(params=[None, 'UTC', 'Asia/Tokyo', @@ -44,8 +45,58 @@ def addend(request): class TestDatetimeIndexComparisons(object): + @pytest.mark.parametrize('other', [datetime(2016, 1, 1), + Timestamp('2016-01-01'), + np.datetime64('2016-01-01')]) + def test_dti_cmp_datetimelike(self, other, tz): + dti = pd.date_range('2016-01-01', periods=2, tz=tz) + if tz is not None: + if isinstance(other, np.datetime64): + # no tzaware version available + return + elif isinstance(other, Timestamp): + other = other.tz_localize(dti.tzinfo) + else: + other = tslib._localize_pydatetime(other, dti.tzinfo) + + result = dti == other + expected = np.array([True, False]) + tm.assert_numpy_array_equal(result, expected) + + result = dti > other + expected = np.array([False, True]) + tm.assert_numpy_array_equal(result, expected) + + result = dti >= other + expected = np.array([True, True]) + tm.assert_numpy_array_equal(result, expected) + + result = dti < other + expected = np.array([False, False]) + tm.assert_numpy_array_equal(result, expected) + + result = dti <= other + expected = np.array([True, False]) + tm.assert_numpy_array_equal(result, expected) + + def dti_cmp_non_datetime(self, tz): + # GH#19301 by convention datetime.date is not considered comparable + # to Timestamp or DatetimeIndex. This may change in the future. + dti = pd.date_range('2016-01-01', periods=2, tz=tz) + + other = datetime(2016, 1, 1).date() + assert not (dti == other).any() + assert (dti != other).all() + with pytest.raises(TypeError): + dti < other + with pytest.raises(TypeError): + dti <= other + with pytest.raises(TypeError): + dti > other + with pytest.raises(TypeError): + dti >= other + @pytest.mark.parametrize('other', [None, - datetime(2016, 1, 1).date(), np.nan]) def test_dti_cmp_non_datetime(self, tz, other): # GH#19301 From 6ba5d99cc72c1c85f7b882554b88d8e13bfee995 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Wed, 24 Jan 2018 10:05:19 -0800 Subject: [PATCH 09/10] requested name change --- pandas/tests/indexes/datetimes/test_arithmetic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/indexes/datetimes/test_arithmetic.py b/pandas/tests/indexes/datetimes/test_arithmetic.py index e084319ab0490..6213a90438814 100644 --- a/pandas/tests/indexes/datetimes/test_arithmetic.py +++ b/pandas/tests/indexes/datetimes/test_arithmetic.py @@ -98,7 +98,7 @@ def dti_cmp_non_datetime(self, tz): @pytest.mark.parametrize('other', [None, np.nan]) - def test_dti_cmp_non_datetime(self, tz, other): + def test_dti_cmp_null_scalar(self, tz, other): # GH#19301 dti = pd.date_range('2016-01-01', periods=2, tz=tz) From 578a4a3b37a7475b529f47f44fe8115a39de54dc Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 25 Jan 2018 10:11:07 -0800 Subject: [PATCH 10/10] requested edits --- .../tests/indexes/datetimes/test_arithmetic.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/pandas/tests/indexes/datetimes/test_arithmetic.py b/pandas/tests/indexes/datetimes/test_arithmetic.py index 6213a90438814..f0195e240e948 100644 --- a/pandas/tests/indexes/datetimes/test_arithmetic.py +++ b/pandas/tests/indexes/datetimes/test_arithmetic.py @@ -96,14 +96,23 @@ def dti_cmp_non_datetime(self, tz): with pytest.raises(TypeError): dti >= other - @pytest.mark.parametrize('other', [None, - np.nan]) - def test_dti_cmp_null_scalar(self, tz, other): + @pytest.mark.parametrize('other', [None, np.nan, pd.NaT]) + def test_dti_eq_null_scalar(self, other, tz): # GH#19301 dti = pd.date_range('2016-01-01', periods=2, tz=tz) - assert not (dti == other).any() + + @pytest.mark.parametrize('other', [None, np.nan, pd.NaT]) + def test_dti_ne_null_scalar(self, other, tz): + # GH#19301 + dti = pd.date_range('2016-01-01', periods=2, tz=tz) assert (dti != other).all() + + @pytest.mark.parametrize('other', [None, np.nan]) + def test_dti_cmp_null_scalar_inequality(self, tz, other): + # GH#19301 + dti = pd.date_range('2016-01-01', periods=2, tz=tz) + with pytest.raises(TypeError): dti < other with pytest.raises(TypeError):