From 83e710543f1ea90206fecea53e6714658f094875 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Wed, 20 Dec 2017 13:58:53 -0800 Subject: [PATCH 1/6] Fix insert NaT into tzaware datetime index closes #16357 --- pandas/core/indexes/datetimes.py | 3 ++- pandas/tests/indexes/datetimes/test_indexing.py | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py index ec5c20d341b50..a6061cf0f1a48 100644 --- a/pandas/core/indexes/datetimes.py +++ b/pandas/core/indexes/datetimes.py @@ -1776,7 +1776,8 @@ def insert(self, loc, item): if isinstance(item, (datetime, np.datetime64)): self._assert_can_do_op(item) - if not self._has_same_tz(item): + if not self._has_same_tz(item) and item is not self._na_value: + # GH#16537 allow pd.NaT through raise ValueError( 'Passed item and index have different timezone') # check freq can be preserved on edge cases diff --git a/pandas/tests/indexes/datetimes/test_indexing.py b/pandas/tests/indexes/datetimes/test_indexing.py index b3ce22962d5d4..485abf85003fc 100644 --- a/pandas/tests/indexes/datetimes/test_indexing.py +++ b/pandas/tests/indexes/datetimes/test_indexing.py @@ -46,6 +46,14 @@ def test_where_tz(self): expected = i2 tm.assert_index_equal(result, expected) + @pytest.mark.parametrize('tz', [None, 'UTC', 'US/Eastern']) + def test_insert_nat(self, tz): + # GH#16537 + idx = pd.DatetimeIndex(['2017-01-01'], tz=tz) + res = idx.insert(0, pd.NaT) + expected = pd.DatetimeIndex(['NaT', '2017-01-01'], tz=tz) + tm.assert_index_equal(res, expected) + def test_insert(self): idx = DatetimeIndex( ['2000-01-04', '2000-01-01', '2000-01-02'], name='idx') From 5a469d2f779c36e9b5bcdac1c19eeb073071040d Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Wed, 20 Dec 2017 14:00:27 -0800 Subject: [PATCH 2/6] Whatsnew note --- doc/source/whatsnew/v0.22.0.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v0.22.0.txt b/doc/source/whatsnew/v0.22.0.txt index 0579a80aad28e..8933c2de0f169 100644 --- a/doc/source/whatsnew/v0.22.0.txt +++ b/doc/source/whatsnew/v0.22.0.txt @@ -294,6 +294,7 @@ Indexing - Bug in tz-aware :class:`DatetimeIndex` where addition/subtraction with a :class:`TimedeltaIndex` or array with ``dtype='timedelta64[ns]'`` was incorrect (:issue:`17558`) - :func:`Index.to_series` now accepts ``index`` and ``name`` kwargs (:issue:`18699`) - :func:`DatetimeIndex.to_series` now accepts ``index`` and ``name`` kwargs (:issue:`18699`) +- Bug in :class:`DatetimeIndex.insert` where inserting ``pd.NaT`` into a timezone-aware index incorrectly raised (:issue:`16357`) I/O ^^^ From 664c800499153e8c782d455fe12338e14d41b6ea Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Wed, 20 Dec 2017 19:38:01 -0800 Subject: [PATCH 3/6] edit whatsnew, test several null types --- doc/source/whatsnew/v0.22.0.txt | 2 +- pandas/tests/indexes/datetimes/test_indexing.py | 14 ++++---------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/doc/source/whatsnew/v0.22.0.txt b/doc/source/whatsnew/v0.22.0.txt index 8933c2de0f169..05097684a988e 100644 --- a/doc/source/whatsnew/v0.22.0.txt +++ b/doc/source/whatsnew/v0.22.0.txt @@ -294,7 +294,7 @@ Indexing - Bug in tz-aware :class:`DatetimeIndex` where addition/subtraction with a :class:`TimedeltaIndex` or array with ``dtype='timedelta64[ns]'`` was incorrect (:issue:`17558`) - :func:`Index.to_series` now accepts ``index`` and ``name`` kwargs (:issue:`18699`) - :func:`DatetimeIndex.to_series` now accepts ``index`` and ``name`` kwargs (:issue:`18699`) -- Bug in :class:`DatetimeIndex.insert` where inserting ``pd.NaT`` into a timezone-aware index incorrectly raised (:issue:`16357`) +- Bug in :func:`DatetimeIndex.insert` where inserting ``NaT`` into a timezone-aware index incorrectly raised (:issue:`16357`) I/O ^^^ diff --git a/pandas/tests/indexes/datetimes/test_indexing.py b/pandas/tests/indexes/datetimes/test_indexing.py index 485abf85003fc..3bd44008c17ca 100644 --- a/pandas/tests/indexes/datetimes/test_indexing.py +++ b/pandas/tests/indexes/datetimes/test_indexing.py @@ -48,11 +48,12 @@ def test_where_tz(self): @pytest.mark.parametrize('tz', [None, 'UTC', 'US/Eastern']) def test_insert_nat(self, tz): - # GH#16537 + # GH#16537, GH#18295 (test missing) idx = pd.DatetimeIndex(['2017-01-01'], tz=tz) - res = idx.insert(0, pd.NaT) expected = pd.DatetimeIndex(['NaT', '2017-01-01'], tz=tz) - tm.assert_index_equal(res, expected) + for null in [None, np.nan, pd.NaT]: + res = idx.insert(0, null) + tm.assert_index_equal(res, expected) def test_insert(self): idx = DatetimeIndex( @@ -153,13 +154,6 @@ def test_insert(self): assert result.tz == expected.tz assert result.freq is None - # GH 18295 (test missing) - expected = DatetimeIndex( - ['20170101', pd.NaT, '20170102', '20170103', '20170104']) - for na in (np.nan, pd.NaT, None): - result = date_range('20170101', periods=4).insert(1, na) - tm.assert_index_equal(result, expected) - def test_delete(self): idx = date_range(start='2000-01-01', periods=5, freq='M', name='idx') From 85ba78c6e5b41345cf199602ccb0e6c5173ec37e Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 21 Dec 2017 09:03:30 -0800 Subject: [PATCH 4/6] edits per reviewer requests --- pandas/core/indexes/datetimes.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py index a6061cf0f1a48..27d3a2d842729 100644 --- a/pandas/core/indexes/datetimes.py +++ b/pandas/core/indexes/datetimes.py @@ -1776,8 +1776,7 @@ def insert(self, loc, item): if isinstance(item, (datetime, np.datetime64)): self._assert_can_do_op(item) - if not self._has_same_tz(item) and item is not self._na_value: - # GH#16537 allow pd.NaT through + if not self._has_same_tz(item) and item is not isna(item): raise ValueError( 'Passed item and index have different timezone') # check freq can be preserved on edge cases From 31e3581c777bbb75bfc5b85c7b8aca0492b9f6d8 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sat, 23 Dec 2017 16:06:12 -0800 Subject: [PATCH 5/6] fix typo --- pandas/core/indexes/datetimes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py index 197c90d20b1d4..3fc3cf9a78a25 100644 --- a/pandas/core/indexes/datetimes.py +++ b/pandas/core/indexes/datetimes.py @@ -1775,7 +1775,7 @@ def insert(self, loc, item): if isinstance(item, (datetime, np.datetime64)): self._assert_can_do_op(item) - if not self._has_same_tz(item) and item is not isna(item): + if not self._has_same_tz(item) and not isna(item): raise ValueError( 'Passed item and index have different timezone') # check freq can be preserved on edge cases From 8934f66a11fb7a899ac2b4fb23e2c623e7abca02 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 28 Dec 2017 17:56:06 -0800 Subject: [PATCH 6/6] parametrize more --- pandas/tests/indexes/datetimes/test_indexing.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pandas/tests/indexes/datetimes/test_indexing.py b/pandas/tests/indexes/datetimes/test_indexing.py index 3bd44008c17ca..48ceefd6368c0 100644 --- a/pandas/tests/indexes/datetimes/test_indexing.py +++ b/pandas/tests/indexes/datetimes/test_indexing.py @@ -46,14 +46,14 @@ def test_where_tz(self): expected = i2 tm.assert_index_equal(result, expected) + @pytest.mark.parametrize('null', [None, np.nan, pd.NaT]) @pytest.mark.parametrize('tz', [None, 'UTC', 'US/Eastern']) - def test_insert_nat(self, tz): + def test_insert_nat(self, tz, null): # GH#16537, GH#18295 (test missing) idx = pd.DatetimeIndex(['2017-01-01'], tz=tz) expected = pd.DatetimeIndex(['NaT', '2017-01-01'], tz=tz) - for null in [None, np.nan, pd.NaT]: - res = idx.insert(0, null) - tm.assert_index_equal(res, expected) + res = idx.insert(0, null) + tm.assert_index_equal(res, expected) def test_insert(self): idx = DatetimeIndex(