From 97c32fda5e429fce11a56e2088fb9ea3584d5dc3 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sun, 19 Nov 2017 15:47:06 -0800 Subject: [PATCH 01/19] implement tzawareness_compat for datetimeIndex --- pandas/core/indexes/datetimes.py | 17 +++++++++- pandas/core/internals.py | 18 ++++++++--- .../tests/indexes/datetimes/test_datetime.py | 32 +++++++++++++++++++ 3 files changed, 62 insertions(+), 5 deletions(-) diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py index ba96979435f81..0a957265c556f 100644 --- a/pandas/core/indexes/datetimes.py +++ b/pandas/core/indexes/datetimes.py @@ -11,7 +11,7 @@ from pandas.core.dtypes.common import ( _NS_DTYPE, _INT64_DTYPE, - is_object_dtype, is_datetime64_dtype, + is_object_dtype, is_datetime64_dtype, is_datetime64tz_dtype, is_datetimetz, is_dtype_equal, is_integer, is_float, is_integer_dtype, @@ -104,6 +104,7 @@ def _dt_index_cmp(opname, nat_result=False): """ def wrapper(self, other): + self._assert_tzawareness_compat(other) func = getattr(super(DatetimeIndex, self), opname) if (isinstance(other, datetime) or isinstance(other, compat.string_types)): @@ -649,6 +650,20 @@ def _simple_new(cls, values, name=None, freq=None, tz=None, result._reset_identity() return result + def _assert_tzawareness_compat(self, other): + # adapted from _Timestamp._assert_tzawareness_compat + other_tz = getattr(other, 'tzinfo', None) + if is_datetime64tz_dtype(other): + # Get tzinfo from Series dtype + other_tz = other.dtype.tz + if self.tz is None: + if other_tz is not None: + raise TypeError('Cannot compare tz-naive and tz-aware ' + 'datetime-like objects.') + elif other_tz is None: + raise TypeError('Cannot compare tz-naive and tz-aware ' + 'datetime-like objects') + @property def tzinfo(self): """ diff --git a/pandas/core/internals.py b/pandas/core/internals.py index 665f9ff8eb7a0..65393f9f0338d 100644 --- a/pandas/core/internals.py +++ b/pandas/core/internals.py @@ -3474,10 +3474,20 @@ def replace_list(self, src_list, dest_list, inplace=False, regex=False, # figure out our mask a-priori to avoid repeated replacements values = self.as_matrix() - def comp(s): - if isna(s): - return isna(values) - return _maybe_compare(values, getattr(s, 'asm8', s), operator.eq) + if is_datetime64tz_dtype(self): + # need to be careful not to compare tznaive to tzaware + tz = values.dtype.tz + def comp(s): + if isna(s): + return isna(values) + return _maybe_compare(values, s, operator.eq) + # TODO: Is just not-converting `s` the right thing to do here? + else: + def comp(s): + if isna(s): + return isna(values) + return _maybe_compare(values, getattr(s, 'asm8', s), + operator.eq) masks = [comp(s) for i, s in enumerate(src_list)] diff --git a/pandas/tests/indexes/datetimes/test_datetime.py b/pandas/tests/indexes/datetimes/test_datetime.py index 36f691903d233..95229264c6f36 100644 --- a/pandas/tests/indexes/datetimes/test_datetime.py +++ b/pandas/tests/indexes/datetimes/test_datetime.py @@ -1,3 +1,5 @@ +import operator + import pytest import numpy as np @@ -223,6 +225,36 @@ def test_append_join_nondatetimeindex(self): # it works rng.join(idx, how='outer') + def test_comparison_tzawareness_compat(self): + # GH#18162 + dr = pd.date_range('2016-01-01', periods=6) + dz = dr.tz_localize('US/Pacific') + + ops = [operator.eq, operator.ne, + operator.gt, operator.ge, + operator.lt, operator.le] + + for left, right in [(dr, dz), (dz, dr)]: + for op in ops: + with pytest.raises(TypeError): + op(left, right) + + # Check that there isn't a problem aware-aware and naive-naive do not + # raise + assert (dr == dr).all() + assert (dz == dz).all() + + naive_series = pd.Series(dr) + aware_series = pd.Series(dz) + for op in ops: + with pytest.raises(TypeError): + op(dz, naive_series) + with pytest.raises(TypeError): + op(dr, aware_series) + + # TODO: implement _assert_tzawareness_compat for the reverse + # comparison with the Series on the left-hand side + def test_comparisons_coverage(self): rng = date_range('1/1/2000', periods=10) From f6a0e644353f8e2afd6147342d6aa824ce0b61c5 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sun, 19 Nov 2017 16:20:08 -0800 Subject: [PATCH 02/19] convert before asserting tzawareness_compat --- pandas/core/indexes/datetimes.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py index b4f76f5244379..322962ff1185f 100644 --- a/pandas/core/indexes/datetimes.py +++ b/pandas/core/indexes/datetimes.py @@ -104,10 +104,12 @@ def _dt_index_cmp(opname, nat_result=False): """ def wrapper(self, other): - self._assert_tzawareness_compat(other) func = getattr(super(DatetimeIndex, self), opname) - if (isinstance(other, datetime) or - isinstance(other, compat.string_types)): + if isinstance(other, (datetime, compat.string_types)): + if isinstance(other, compat.string_types): + # We need to convert in order to assert tzawareness + other = Timestamp(other) + self._assert_tzawareness_compat(other) other = _to_m8(other, tz=self.tz) result = func(other) if isna(other): @@ -117,6 +119,7 @@ def wrapper(self, other): other = DatetimeIndex(other) elif not isinstance(other, (np.ndarray, Index, ABCSeries)): other = _ensure_datetime64(other) + self._assert_tzawareness_compat(other) result = func(np.asarray(other)) result = _values_from_object(result) From 28f88965ba19268c740dbf265e2fcecfab690e99 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Fri, 24 Nov 2017 11:14:13 -0800 Subject: [PATCH 03/19] Update test to disallow tzaware vs tznaive comparison --- pandas/tests/series/test_indexing.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pandas/tests/series/test_indexing.py b/pandas/tests/series/test_indexing.py index d141b378fe214..25966abb697de 100644 --- a/pandas/tests/series/test_indexing.py +++ b/pandas/tests/series/test_indexing.py @@ -449,6 +449,13 @@ def test_getitem_setitem_datetimeindex(self): lb = "1990-01-01 04:00:00" rb = "1990-01-01 07:00:00" + with pytest.raises(TypeError): + # tznaive vs tzaware comparison is invalid + # see GH#18376, GH#18162 + ts[(ts.index >= lb) & (ts.index <= rb)] + + lb = "1990-01-01 04:00:00-0500" + rb = "1990-01-01 07:00:00-0500" result = ts[(ts.index >= lb) & (ts.index <= rb)] expected = ts[4:8] assert_series_equal(result, expected) @@ -474,6 +481,13 @@ def test_getitem_setitem_datetimeindex(self): lb = datetime(1990, 1, 1, 4) rb = datetime(1990, 1, 1, 7) + with pytest.raises(TypeError): + # tznaive vs tzaware comparison is invalid + # see GH#18376, GH#18162 + ts[(ts.index >= lb) & (ts.index <= rb)] + + lb = rng.tzinfo.localize(datetime(1990, 1, 1, 4)) + rb = rng.tzinfo.localize(datetime(1990, 1, 1, 7)) result = ts[(ts.index >= lb) & (ts.index <= rb)] expected = ts[4:8] assert_series_equal(result, expected) From 0f2287c4f9c69742b8f3a41052a42bf82cd3226a Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Fri, 24 Nov 2017 13:29:41 -0800 Subject: [PATCH 04/19] flake8 fixup --- pandas/core/internals.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/core/internals.py b/pandas/core/internals.py index f786a9ad9396b..9ea8f72ec9efb 100644 --- a/pandas/core/internals.py +++ b/pandas/core/internals.py @@ -3488,7 +3488,6 @@ def replace_list(self, src_list, dest_list, inplace=False, regex=False, if is_datetime64tz_dtype(self): # need to be careful not to compare tznaive to tzaware - tz = values.dtype.tz def comp(s): if isna(s): return isna(values) From 171238d9140b14c6089703ad8e11fb18f854b8aa Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sat, 25 Nov 2017 09:13:11 -0800 Subject: [PATCH 05/19] Fix typo --- pandas/tests/test_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/test_base.py b/pandas/tests/test_base.py index 31f4ca146040e..9a608cca96319 100644 --- a/pandas/tests/test_base.py +++ b/pandas/tests/test_base.py @@ -114,7 +114,7 @@ def __init__(self, obj): def setup_method(self, method): pass - def test_invalida_delgation(self): + def test_invalid_delegation(self): # these show that in order for the delegation to work # the _delegate_* methods need to be overriden to not raise a TypeError From e95d75ec52949767155088c23116759e759a3107 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sat, 25 Nov 2017 09:13:35 -0800 Subject: [PATCH 06/19] Move part of test per reviewer request --- pandas/tests/indexes/test_base.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/pandas/tests/indexes/test_base.py b/pandas/tests/indexes/test_base.py index c55f53601848c..e93df7231dc4c 100644 --- a/pandas/tests/indexes/test_base.py +++ b/pandas/tests/indexes/test_base.py @@ -2159,6 +2159,28 @@ def test_intersect_str_dates(self): assert len(res) == 0 + def test_comparison_tzawareness_compat(self): + # GH#18162 + dr = pd.date_range('2016-01-01', periods=6) + dz = dr.tz_localize('US/Pacific') + + ops = [operator.eq, operator.ne, + operator.gt, operator.ge, + operator.lt, operator.le] + + # Check that there isn't a problem aware-aware and naive-naive do not + # raise + naive_series = Series(dr) + aware_series = Series(dz) + for op in ops: + with pytest.raises(TypeError): + op(dz, naive_series) + with pytest.raises(TypeError): + op(dr, aware_series) + + # TODO: implement _assert_tzawareness_compat for the reverse + # comparison with the Series on the left-hand side + class TestIndexUtils(object): From ba65aafd95917cbd7a1b11718e486b4fee91a09f Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sat, 25 Nov 2017 09:14:48 -0800 Subject: [PATCH 07/19] update test to include scalar comparisons --- .../tests/indexes/datetimes/test_datetime.py | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/pandas/tests/indexes/datetimes/test_datetime.py b/pandas/tests/indexes/datetimes/test_datetime.py index 5c10f4727596f..80bc7f184adc3 100644 --- a/pandas/tests/indexes/datetimes/test_datetime.py +++ b/pandas/tests/indexes/datetimes/test_datetime.py @@ -269,16 +269,29 @@ def test_comparison_tzawareness_compat(self): assert (dr == dr).all() assert (dz == dz).all() - naive_series = pd.Series(dr) - aware_series = pd.Series(dz) - for op in ops: - with pytest.raises(TypeError): - op(dz, naive_series) - with pytest.raises(TypeError): - op(dr, aware_series) - - # TODO: implement _assert_tzawareness_compat for the reverse - # comparison with the Series on the left-hand side + # Check comparisons against datetime-like strings + str_ts = '2000-03-14 01:59' + str_ts_tz = '2000-03-14 01:59-0500' + + assert (dr > str_ts).all() + with pytest.raises(TypeError): + dr == str_ts_tz + + assert (dz > str_ts_tz).all() + with pytest.raises(TypeError): + dz != str_ts + + # Check comparisons against scalar Timestamps + ts = pd.Timestamp('2000-03-14 01:59') + ts_tz = pd.Timestamp('2000-03-14 01:59', tz='Europe/Amsterdam') + + assert (dr > ts).all() + with pytest.raises(TypeError): + dr == ts_tz + + assert (dz > ts_tz).all() + with pytest.raises(TypeError): + dz != ts def test_comparisons_coverage(self): rng = date_range('1/1/2000', periods=10) From b9e7c6d7e071ce9b43c521e8a5dd8512f3d5aee6 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sat, 25 Nov 2017 14:25:16 -0800 Subject: [PATCH 08/19] revert hack, xfail test --- pandas/core/internals.py | 25 ++++++++++--------------- pandas/tests/indexing/test_coercion.py | 1 + 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/pandas/core/internals.py b/pandas/core/internals.py index a5d179a038420..75f52681c9f3b 100644 --- a/pandas/core/internals.py +++ b/pandas/core/internals.py @@ -3486,22 +3486,17 @@ def replace_list(self, src_list, dest_list, inplace=False, regex=False, # figure out our mask a-priori to avoid repeated replacements values = self.as_matrix() - if is_datetime64tz_dtype(self): - # need to be careful not to compare tznaive to tzaware - def comp(s): - if isna(s): - return isna(values) - return _maybe_compare(values, s, operator.eq) - # TODO: Is just not-converting `s` the right thing to do here? - else: - def comp(s): - if isna(s): - return isna(values) - return _maybe_compare(values, getattr(s, 'asm8', s), - operator.eq) - - masks = [comp(s) for i, s in enumerate(src_list)] + def comp(s): + if isna(s): + return isna(values) + return _maybe_compare(values, getattr(s, 'asm8', s), operator.eq) + try: + masks = [comp(s) for i, s in enumerate(src_list)] + except: + print(src_list) + print(dest_list) + raise result_blocks = [] src_len = len(src_list) - 1 for blk in self.blocks: diff --git a/pandas/tests/indexing/test_coercion.py b/pandas/tests/indexing/test_coercion.py index 752d2deb53304..a126c8c959f60 100644 --- a/pandas/tests/indexing/test_coercion.py +++ b/pandas/tests/indexing/test_coercion.py @@ -1311,6 +1311,7 @@ def test_replace_series_datetime64(self): for to_key in self.rep: self._assert_replace_conversion(from_key, to_key, how='series') + @pytest.mark.xfail def test_replace_series_datetime64tz(self): from_key = 'datetime64[ns, US/Eastern]' for to_key in self.rep: From eefadf75aeb5e1f6119a7a7e49f320349b65cb86 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sun, 26 Nov 2017 08:31:35 -0800 Subject: [PATCH 09/19] parametrize tests, comment for xfail --- .../tests/indexes/datetimes/test_datetime.py | 25 +++++++++---------- pandas/tests/indexes/test_base.py | 18 ++++++------- pandas/tests/indexing/test_coercion.py | 1 + pandas/tests/series/test_indexing.py | 4 +-- 4 files changed, 23 insertions(+), 25 deletions(-) diff --git a/pandas/tests/indexes/datetimes/test_datetime.py b/pandas/tests/indexes/datetimes/test_datetime.py index 80bc7f184adc3..24718bf510a3d 100644 --- a/pandas/tests/indexes/datetimes/test_datetime.py +++ b/pandas/tests/indexes/datetimes/test_datetime.py @@ -250,19 +250,18 @@ def test_append_join_nondatetimeindex(self): # it works rng.join(idx, how='outer') - def test_comparison_tzawareness_compat(self): + @pytest.mark.parametrize('op', [operator.eq, operator.ne, + operator.gt, operator.ge, + operator.lt, operator.le]) + def test_comparison_tzawareness_compat(self, op): # GH#18162 dr = pd.date_range('2016-01-01', periods=6) dz = dr.tz_localize('US/Pacific') - ops = [operator.eq, operator.ne, - operator.gt, operator.ge, - operator.lt, operator.le] - - for left, right in [(dr, dz), (dz, dr)]: - for op in ops: - with pytest.raises(TypeError): - op(left, right) + with pytest.raises(TypeError): + op(dr, dz) + with pytest.raises(TypeError): + op(dz, dr) # Check that there isn't a problem aware-aware and naive-naive do not # raise @@ -275,11 +274,11 @@ def test_comparison_tzawareness_compat(self): assert (dr > str_ts).all() with pytest.raises(TypeError): - dr == str_ts_tz + op(dr, str_ts_tz) assert (dz > str_ts_tz).all() with pytest.raises(TypeError): - dz != str_ts + op(dz, str_ts) # Check comparisons against scalar Timestamps ts = pd.Timestamp('2000-03-14 01:59') @@ -287,11 +286,11 @@ def test_comparison_tzawareness_compat(self): assert (dr > ts).all() with pytest.raises(TypeError): - dr == ts_tz + op(dr, ts_tz) assert (dz > ts_tz).all() with pytest.raises(TypeError): - dz != ts + op(dz, ts) def test_comparisons_coverage(self): rng = date_range('1/1/2000', periods=10) diff --git a/pandas/tests/indexes/test_base.py b/pandas/tests/indexes/test_base.py index b3d3386476c6c..bbd203e335cd8 100644 --- a/pandas/tests/indexes/test_base.py +++ b/pandas/tests/indexes/test_base.py @@ -2240,24 +2240,22 @@ def test_intersect_str_dates(self): assert len(res) == 0 - def test_comparison_tzawareness_compat(self): + @pytest.mark.parametrize('op', [operator.eq, operator.ne, + operator.gt, operator.ge, + operator.lt, operator.le]) + def test_comparison_tzawareness_compat(self, op): # GH#18162 dr = pd.date_range('2016-01-01', periods=6) dz = dr.tz_localize('US/Pacific') - ops = [operator.eq, operator.ne, - operator.gt, operator.ge, - operator.lt, operator.le] - # Check that there isn't a problem aware-aware and naive-naive do not # raise naive_series = Series(dr) aware_series = Series(dz) - for op in ops: - with pytest.raises(TypeError): - op(dz, naive_series) - with pytest.raises(TypeError): - op(dr, aware_series) + with pytest.raises(TypeError): + op(dz, naive_series) + with pytest.raises(TypeError): + op(dr, aware_series) # TODO: implement _assert_tzawareness_compat for the reverse # comparison with the Series on the left-hand side diff --git a/pandas/tests/indexing/test_coercion.py b/pandas/tests/indexing/test_coercion.py index a126c8c959f60..454a05b4e616e 100644 --- a/pandas/tests/indexing/test_coercion.py +++ b/pandas/tests/indexing/test_coercion.py @@ -1311,6 +1311,7 @@ def test_replace_series_datetime64(self): for to_key in self.rep: self._assert_replace_conversion(from_key, to_key, how='series') + # GH #18376, tzawareness-compat bug in BlockManager.replace_list @pytest.mark.xfail def test_replace_series_datetime64tz(self): from_key = 'datetime64[ns, US/Eastern]' diff --git a/pandas/tests/series/test_indexing.py b/pandas/tests/series/test_indexing.py index 18effb26a298f..37c2c1da2e93b 100644 --- a/pandas/tests/series/test_indexing.py +++ b/pandas/tests/series/test_indexing.py @@ -486,8 +486,8 @@ def test_getitem_setitem_datetimeindex(self): # see GH#18376, GH#18162 ts[(ts.index >= lb) & (ts.index <= rb)] - lb = rng.tzinfo.localize(datetime(1990, 1, 1, 4)) - rb = rng.tzinfo.localize(datetime(1990, 1, 1, 7)) + lb = pd.Timestamp(datetime(1990, 1, 1, 4)).tz_localize(rng.tzinfo) + rb = pd.Timestamp(datetime(1990, 1, 1, 7)).tz_localize(rng.tzinfo) result = ts[(ts.index >= lb) & (ts.index <= rb)] expected = ts[4:8] assert_series_equal(result, expected) From abe65f80b47346525f44e96149295fb277bd499e Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Wed, 6 Dec 2017 09:03:52 -0800 Subject: [PATCH 10/19] remove debugging prints --- pandas/core/internals.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/pandas/core/internals.py b/pandas/core/internals.py index e30f2360ac030..9ee95fbee5985 100644 --- a/pandas/core/internals.py +++ b/pandas/core/internals.py @@ -3491,12 +3491,7 @@ def comp(s): return isna(values) return _maybe_compare(values, getattr(s, 'asm8', s), operator.eq) - try: - masks = [comp(s) for i, s in enumerate(src_list)] - except: - print(src_list) - print(dest_list) - raise + masks = [comp(s) for i, s in enumerate(src_list)] result_blocks = [] src_len = len(src_list) - 1 for blk in self.blocks: From 7cba1032b4e687edc6298d0f1a54e31143615e67 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Wed, 6 Dec 2017 09:08:00 -0800 Subject: [PATCH 11/19] rearrange isinstance checks per suggestion --- pandas/core/indexes/datetimes.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py index 6473404d89ba1..b2eb40b523207 100644 --- a/pandas/core/indexes/datetimes.py +++ b/pandas/core/indexes/datetimes.py @@ -105,10 +105,12 @@ def _dt_index_cmp(opname, cls, nat_result=False): def wrapper(self, other): func = getattr(super(DatetimeIndex, self), opname) - if isinstance(other, (datetime, compat.string_types)): - if isinstance(other, compat.string_types): - # We need to convert in order to assert tzawareness - other = Timestamp(other) + + if isinstance(other, compat.string_types): + # We need to convert in order to assert tzawareness + other = Timestamp(other) + + if isinstance(other, datetime): self._assert_tzawareness_compat(other) other = _to_m8(other, tz=self.tz) result = func(other) From 2d68f0154cfbb4eafa65ba87ba70361be21a8a2d Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Wed, 6 Dec 2017 09:11:37 -0800 Subject: [PATCH 12/19] add whatsnew note --- doc/source/whatsnew/v0.22.0.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.22.0.txt b/doc/source/whatsnew/v0.22.0.txt index d34c1f3535509..58470a9aa0cc1 100644 --- a/doc/source/whatsnew/v0.22.0.txt +++ b/doc/source/whatsnew/v0.22.0.txt @@ -352,4 +352,4 @@ Other ^^^^^ - Improved error message when attempting to use a Python keyword as an identifier in a numexpr query (:issue:`18221`) -- +- Fixed bug where comparing :class:`DatetimeIndex` failed to raise ``TypeError`` when attempting to compare timezone-aware and timezone-naive datetimelike objects (:issue:`18162`) From ddf1cea1126f1c5b6267686b6b5ab91dd99214ff Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 7 Dec 2017 07:48:52 -0800 Subject: [PATCH 13/19] fix duplicate line from merge mixup --- doc/source/whatsnew/v0.22.0.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/source/whatsnew/v0.22.0.txt b/doc/source/whatsnew/v0.22.0.txt index 849a739101f1d..56cf5572cc521 100644 --- a/doc/source/whatsnew/v0.22.0.txt +++ b/doc/source/whatsnew/v0.22.0.txt @@ -323,7 +323,6 @@ Categorical Other ^^^^^ -- Improved error message when attempting to use a Python keyword as an identifier in a numexpr query (:issue:`18221`) - Fixed bug where comparing :class:`DatetimeIndex` failed to raise ``TypeError`` when attempting to compare timezone-aware and timezone-naive datetimelike objects (:issue:`18162`) - Improved error message when attempting to use a Python keyword as an identifier in a ``numexpr`` backed query (:issue:`18221`) - From 3b4a01eba79028b4e42ba405d4b821424298a0c8 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 7 Dec 2017 07:50:16 -0800 Subject: [PATCH 14/19] put comment into xfail message --- pandas/tests/indexing/test_coercion.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/tests/indexing/test_coercion.py b/pandas/tests/indexing/test_coercion.py index 454a05b4e616e..f042bdbf004be 100644 --- a/pandas/tests/indexing/test_coercion.py +++ b/pandas/tests/indexing/test_coercion.py @@ -1311,8 +1311,8 @@ def test_replace_series_datetime64(self): for to_key in self.rep: self._assert_replace_conversion(from_key, to_key, how='series') - # GH #18376, tzawareness-compat bug in BlockManager.replace_list - @pytest.mark.xfail + @pytest.mark.xfail('GH #18376, tzawareness-compat bug ' + 'in BlockManager.replace_list') def test_replace_series_datetime64tz(self): from_key = 'datetime64[ns, US/Eastern]' for to_key in self.rep: From aeef3d7ca28e5556d454710d3a77f346bb458c2b Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 7 Dec 2017 09:49:34 -0800 Subject: [PATCH 15/19] Fixup missing keyword --- pandas/tests/indexing/test_coercion.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/tests/indexing/test_coercion.py b/pandas/tests/indexing/test_coercion.py index f042bdbf004be..68d3c86b3eda7 100644 --- a/pandas/tests/indexing/test_coercion.py +++ b/pandas/tests/indexing/test_coercion.py @@ -1311,8 +1311,8 @@ def test_replace_series_datetime64(self): for to_key in self.rep: self._assert_replace_conversion(from_key, to_key, how='series') - @pytest.mark.xfail('GH #18376, tzawareness-compat bug ' - 'in BlockManager.replace_list') + @pytest.mark.xfail(reason='GH #18376, tzawareness-compat bug ' + 'in BlockManager.replace_list') def test_replace_series_datetime64tz(self): from_key = 'datetime64[ns, US/Eastern]' for to_key in self.rep: From f21f0257811cafc15cc7b2887f185bf59072f395 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sat, 16 Dec 2017 10:09:13 -0800 Subject: [PATCH 16/19] give strings a free pass from tzawareness compat --- pandas/core/indexes/datetimes.py | 9 ++++----- pandas/core/internals.py | 1 + pandas/tests/indexes/datetimes/test_datetime.py | 12 ------------ pandas/tests/series/test_indexing.py | 8 ++++---- 4 files changed, 9 insertions(+), 21 deletions(-) diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py index e55e3675b78b0..9085cec4a0853 100644 --- a/pandas/core/indexes/datetimes.py +++ b/pandas/core/indexes/datetimes.py @@ -108,12 +108,11 @@ def _dt_index_cmp(opname, cls, nat_result=False): def wrapper(self, other): func = getattr(super(DatetimeIndex, self), opname) - if isinstance(other, compat.string_types): - # We need to convert in order to assert tzawareness - other = Timestamp(other) + if isinstance(other, (datetime, compat.string_types)): + if isinstance(other, datetime): + # GH#18435 strings get a pass from tzawareness compat + self._assert_tzawareness_compat(other) - if isinstance(other, datetime): - self._assert_tzawareness_compat(other) other = _to_m8(other, tz=self.tz) result = func(other) if isna(other): diff --git a/pandas/core/internals.py b/pandas/core/internals.py index 059abbc1d131d..3a64a0ef84e3d 100644 --- a/pandas/core/internals.py +++ b/pandas/core/internals.py @@ -3482,6 +3482,7 @@ def comp(s): return _maybe_compare(values, getattr(s, 'asm8', s), operator.eq) masks = [comp(s) for i, s in enumerate(src_list)] + result_blocks = [] src_len = len(src_list) - 1 for blk in self.blocks: diff --git a/pandas/tests/indexes/datetimes/test_datetime.py b/pandas/tests/indexes/datetimes/test_datetime.py index 24718bf510a3d..0b56ca6f60294 100644 --- a/pandas/tests/indexes/datetimes/test_datetime.py +++ b/pandas/tests/indexes/datetimes/test_datetime.py @@ -268,18 +268,6 @@ def test_comparison_tzawareness_compat(self, op): assert (dr == dr).all() assert (dz == dz).all() - # Check comparisons against datetime-like strings - str_ts = '2000-03-14 01:59' - str_ts_tz = '2000-03-14 01:59-0500' - - assert (dr > str_ts).all() - with pytest.raises(TypeError): - op(dr, str_ts_tz) - - assert (dz > str_ts_tz).all() - with pytest.raises(TypeError): - op(dz, str_ts) - # Check comparisons against scalar Timestamps ts = pd.Timestamp('2000-03-14 01:59') ts_tz = pd.Timestamp('2000-03-14 01:59', tz='Europe/Amsterdam') diff --git a/pandas/tests/series/test_indexing.py b/pandas/tests/series/test_indexing.py index e90f3634827ed..bcda56df836bb 100644 --- a/pandas/tests/series/test_indexing.py +++ b/pandas/tests/series/test_indexing.py @@ -450,10 +450,10 @@ def test_getitem_setitem_datetimeindex(self): lb = "1990-01-01 04:00:00" rb = "1990-01-01 07:00:00" - with pytest.raises(TypeError): - # tznaive vs tzaware comparison is invalid - # see GH#18376, GH#18162 - ts[(ts.index >= lb) & (ts.index <= rb)] + # GH#18435 strings get a pass from tzawareness compat + result = ts[(ts.index >= lb) & (ts.index <= rb)] + expected = ts[4:8] + assert_series_equal(result, expected) lb = "1990-01-01 04:00:00-0500" rb = "1990-01-01 07:00:00-0500" From a6008e042b45afa1eafb106d0c1b1faa7e89dcc7 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 4 Jan 2018 11:17:24 -0800 Subject: [PATCH 17/19] add list test cases --- pandas/core/indexes/datetimes.py | 7 +++++-- pandas/tests/indexes/datetimes/test_datetime.py | 6 ++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py index 702525a70a561..b7a74b0dbeb32 100644 --- a/pandas/core/indexes/datetimes.py +++ b/pandas/core/indexes/datetimes.py @@ -16,7 +16,7 @@ is_timedelta64_dtype, is_integer, is_float, is_integer_dtype, - is_datetime64_ns_dtype, + is_datetime64_ns_dtype, is_datetimelike, is_period_dtype, is_bool_dtype, is_string_dtype, @@ -122,7 +122,10 @@ def wrapper(self, other): other = DatetimeIndex(other) elif not isinstance(other, (np.ndarray, Index, ABCSeries)): other = _ensure_datetime64(other) - self._assert_tzawareness_compat(other) + + if is_datetimelike(other): + self._assert_tzawareness_compat(other) + result = func(np.asarray(other)) result = _values_from_object(result) diff --git a/pandas/tests/indexes/datetimes/test_datetime.py b/pandas/tests/indexes/datetimes/test_datetime.py index 0b56ca6f60294..41cd654cf22b9 100644 --- a/pandas/tests/indexes/datetimes/test_datetime.py +++ b/pandas/tests/indexes/datetimes/test_datetime.py @@ -260,13 +260,19 @@ def test_comparison_tzawareness_compat(self, op): with pytest.raises(TypeError): op(dr, dz) + with pytest.raises(TypeError): + op(dr, list(dz)) with pytest.raises(TypeError): op(dz, dr) + with pytest.raises(TypeError): + op(dz, list(dr)) # Check that there isn't a problem aware-aware and naive-naive do not # raise assert (dr == dr).all() + assert (dr == list(dr)).all() assert (dz == dz).all() + assert (dz == list(dz)).all() # Check comparisons against scalar Timestamps ts = pd.Timestamp('2000-03-14 01:59') From e2611dc3ee384188d56754e07d971dd593c1e7d3 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 4 Jan 2018 11:48:40 -0800 Subject: [PATCH 18/19] follow precedent to separate out xfailed tests --- pandas/tests/indexing/test_coercion.py | 35 ++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/pandas/tests/indexing/test_coercion.py b/pandas/tests/indexing/test_coercion.py index b518a20390067..de756375db8cb 100644 --- a/pandas/tests/indexing/test_coercion.py +++ b/pandas/tests/indexing/test_coercion.py @@ -822,8 +822,8 @@ def test_replace_series(self, how, to_key, from_key): # tested below return elif from_key in ['datetime64[ns, US/Eastern]', 'datetime64[ns, UTC]']: - pytest.xfail(reason='GH #18376, tzawareness-compat bug ' - 'in BlockManager.replace_list') + # tested below + return if how == 'dict': replacer = dict(zip(self.rep[from_key], self.rep[to_key])) @@ -852,6 +852,37 @@ def test_replace_series(self, how, to_key, from_key): tm.assert_series_equal(result, exp) + # TODO(jbrockmendel) commented out to only have a single xfail printed + @pytest.mark.xfail(reason='GH #18376, tzawareness-compat bug ' + 'in BlockManager.replace_list') + # @pytest.mark.parametrize('how', ['dict', 'series']) + # @pytest.mark.parametrize('to_key', ['timedelta64[ns]', 'bool', 'object', + # 'complex128', 'float64', 'int64']) + # @pytest.mark.parametrize('from_key', ['datetime64[ns, UTC]', + # 'datetime64[ns, US/Eastern]']) + # def test_replace_series_datetime_tz(self, how, to_key, from_key): + def test_replace_series_datetime_tz(self): + how = 'series' + from_key = 'datetime64[ns, US/Eastern]' + to_key = 'timedelta64[ns]' + + index = pd.Index([3, 4], name='xxx') + obj = pd.Series(self.rep[from_key], index=index, name='yyy') + assert obj.dtype == from_key + + if how == 'dict': + replacer = dict(zip(self.rep[from_key], self.rep[to_key])) + elif how == 'series': + replacer = pd.Series(self.rep[to_key], index=self.rep[from_key]) + else: + raise ValueError + + result = obj.replace(replacer) + exp = pd.Series(self.rep[to_key], index=index, name='yyy') + assert exp.dtype == to_key + + tm.assert_series_equal(result, exp) + # TODO(jreback) commented out to only have a single xfail printed @pytest.mark.xfail(reason="different tz, " "currently mask_missing raises SystemError") From 5653c610fd2b8bff5bd4dd4d05d197d871da2df2 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 4 Jan 2018 17:40:27 -0800 Subject: [PATCH 19/19] move whatsnew to conversion --- doc/source/whatsnew/v0.23.0.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.23.0.txt b/doc/source/whatsnew/v0.23.0.txt index 569fdf0bfa8bc..d3194075f76bc 100644 --- a/doc/source/whatsnew/v0.23.0.txt +++ b/doc/source/whatsnew/v0.23.0.txt @@ -310,6 +310,7 @@ Conversion - Bug in :class:`Series` floor-division where operating on a scalar ``timedelta`` raises an exception (:issue:`18846`) - Bug in :class:`FY5253Quarter`, :class:`LastWeekOfMonth` where rollback and rollforward behavior was inconsistent with addition and subtraction behavior (:issue:`18854`) - Bug in :class:`Index` constructor with ``dtype=CategoricalDtype(...)`` where ``categories`` and ``ordered`` are not maintained (issue:`19032`) +- Fixed bug where comparing :class:`DatetimeIndex` failed to raise ``TypeError`` when attempting to compare timezone-aware and timezone-naive datetimelike objects (:issue:`18162`) Indexing @@ -395,5 +396,4 @@ Categorical Other ^^^^^ -- Fixed bug where comparing :class:`DatetimeIndex` failed to raise ``TypeError`` when attempting to compare timezone-aware and timezone-naive datetimelike objects (:issue:`18162`) - Improved error message when attempting to use a Python keyword as an identifier in a ``numexpr`` backed query (:issue:`18221`)