From 35912e1c0d3d83ebaf3fa0ccec9b18fefa33e109 Mon Sep 17 00:00:00 2001 From: "Christopher C. Aycock" Date: Thu, 6 Apr 2017 13:27:00 -0400 Subject: [PATCH 1/4] BUG: use entire size of DatetimeTZBlock when coercing result (#15855) --- doc/source/whatsnew/v0.20.0.txt | 1 + pandas/core/internals.py | 2 +- pandas/tests/indexes/datetimes/test_missing.py | 8 ++++++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.20.0.txt b/doc/source/whatsnew/v0.20.0.txt index cb9e2496757ef..f9b6cebb26693 100644 --- a/doc/source/whatsnew/v0.20.0.txt +++ b/doc/source/whatsnew/v0.20.0.txt @@ -999,6 +999,7 @@ Conversion - Bug in ``DataFrame.fillna()`` where the argument ``downcast`` was ignored when fillna value was of type ``dict`` (:issue:`15277`) - Bug in ``.asfreq()``, where frequency was not set for empty ``Series`` (:issue:`14320`) - Bug in ``DataFrame`` construction with nulls and datetimes in a list-like (:issue:`15869`) +- Bug in ``DataFrame.fillna()`` with tz-aware datetimes (:issue:`15855`) Indexing ^^^^^^^^ diff --git a/pandas/core/internals.py b/pandas/core/internals.py index 8db801f8e7212..57361886eab8c 100644 --- a/pandas/core/internals.py +++ b/pandas/core/internals.py @@ -2475,7 +2475,7 @@ def _try_coerce_result(self, result): if isinstance(result, np.ndarray): # allow passing of > 1dim if its trivial if result.ndim > 1: - result = result.reshape(len(result)) + result = result.reshape(np.prod(result.shape)) result = self.values._shallow_copy(result) return result diff --git a/pandas/tests/indexes/datetimes/test_missing.py b/pandas/tests/indexes/datetimes/test_missing.py index 8f3752227b6d0..99e27a4357771 100644 --- a/pandas/tests/indexes/datetimes/test_missing.py +++ b/pandas/tests/indexes/datetimes/test_missing.py @@ -48,3 +48,11 @@ def test_fillna_datetime64(self): pd.Timestamp('2011-01-01 11:00', tz=tz)], dtype=object) self.assert_index_equal(idx.fillna('x'), exp) + + def test_fillna_timezone(self): + # GH 15855 + df = pd.DataFrame({'A': [pd.Timestamp('2012-11-11 00:00:00+01:00'), + pd.NaT]}) + exp = pd.DataFrame({'A': [pd.Timestamp('2012-11-11 00:00:00+01:00'), + pd.Timestamp('2012-11-11 00:00:00+01:00')]}) + self.assert_frame_equal(df.fillna(method='pad'), exp) From 4ad0b12edb9086b23c01a23a55c5502cba91a56e Mon Sep 17 00:00:00 2001 From: "Christopher C. Aycock" Date: Fri, 7 Apr 2017 08:24:35 -0400 Subject: [PATCH 2/4] Moved test --- pandas/tests/frame/test_missing.py | 14 ++++++++++++++ pandas/tests/indexes/datetimes/test_missing.py | 8 -------- pandas/tests/series/test_missing.py | 12 ++++++++++++ 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/pandas/tests/frame/test_missing.py b/pandas/tests/frame/test_missing.py index 93c3ba78a0abf..c209a2bfcbf60 100644 --- a/pandas/tests/frame/test_missing.py +++ b/pandas/tests/frame/test_missing.py @@ -257,6 +257,20 @@ def test_fillna(self): result = df.fillna(value={'Date': df['Date2']}) assert_frame_equal(result, expected) + # with timezone + # GH 15855 + df = pd.DataFrame({'A': [pd.Timestamp('2012-11-11 00:00:00+01:00'), + pd.NaT]}) + exp = pd.DataFrame({'A': [pd.Timestamp('2012-11-11 00:00:00+01:00'), + pd.Timestamp('2012-11-11 00:00:00+01:00')]}) + self.assert_frame_equal(df.fillna(method='pad'), exp) + + df = pd.DataFrame({'A': [pd.NaT, + pd.Timestamp('2012-11-11 00:00:00+01:00')]}) + exp = pd.DataFrame({'A': [pd.Timestamp('2012-11-11 00:00:00+01:00'), + pd.Timestamp('2012-11-11 00:00:00+01:00')]}) + self.assert_frame_equal(df.fillna(method='bfill'), exp) + def test_fillna_downcast(self): # GH 15277 # infer int64 from float64 diff --git a/pandas/tests/indexes/datetimes/test_missing.py b/pandas/tests/indexes/datetimes/test_missing.py index 99e27a4357771..8f3752227b6d0 100644 --- a/pandas/tests/indexes/datetimes/test_missing.py +++ b/pandas/tests/indexes/datetimes/test_missing.py @@ -48,11 +48,3 @@ def test_fillna_datetime64(self): pd.Timestamp('2011-01-01 11:00', tz=tz)], dtype=object) self.assert_index_equal(idx.fillna('x'), exp) - - def test_fillna_timezone(self): - # GH 15855 - df = pd.DataFrame({'A': [pd.Timestamp('2012-11-11 00:00:00+01:00'), - pd.NaT]}) - exp = pd.DataFrame({'A': [pd.Timestamp('2012-11-11 00:00:00+01:00'), - pd.Timestamp('2012-11-11 00:00:00+01:00')]}) - self.assert_frame_equal(df.fillna(method='pad'), exp) diff --git a/pandas/tests/series/test_missing.py b/pandas/tests/series/test_missing.py index 7174283494fe7..6a1ded1b9033c 100644 --- a/pandas/tests/series/test_missing.py +++ b/pandas/tests/series/test_missing.py @@ -258,6 +258,18 @@ def test_datetime64_tz_fillna(self): self.assert_series_equal(expected, result) self.assert_series_equal(pd.isnull(s), null_loc) + # with timezone + # GH 15855 + df = pd.Series([pd.Timestamp('2012-11-11 00:00:00+01:00'), pd.NaT]) + exp = pd.Series([pd.Timestamp('2012-11-11 00:00:00+01:00'), + pd.Timestamp('2012-11-11 00:00:00+01:00')]) + self.assert_series_equal(df.fillna(method='pad'), exp) + + df = pd.Series([pd.NaT, pd.Timestamp('2012-11-11 00:00:00+01:00')]) + exp = pd.Series([pd.Timestamp('2012-11-11 00:00:00+01:00'), + pd.Timestamp('2012-11-11 00:00:00+01:00')]) + self.assert_series_equal(df.fillna(method='bfill'), exp) + def test_datetime64tz_fillna_round_issue(self): # GH 14872 From 1fc53376974de5214e7558b32252b75856cb1a1e Mon Sep 17 00:00:00 2001 From: "Christopher C. Aycock" Date: Fri, 7 Apr 2017 13:48:30 -0400 Subject: [PATCH 3/4] Removed unnecessary 'self' --- pandas/tests/frame/test_missing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/tests/frame/test_missing.py b/pandas/tests/frame/test_missing.py index c209a2bfcbf60..eacf032bbcc85 100644 --- a/pandas/tests/frame/test_missing.py +++ b/pandas/tests/frame/test_missing.py @@ -263,13 +263,13 @@ def test_fillna(self): pd.NaT]}) exp = pd.DataFrame({'A': [pd.Timestamp('2012-11-11 00:00:00+01:00'), pd.Timestamp('2012-11-11 00:00:00+01:00')]}) - self.assert_frame_equal(df.fillna(method='pad'), exp) + assert_frame_equal(df.fillna(method='pad'), exp) df = pd.DataFrame({'A': [pd.NaT, pd.Timestamp('2012-11-11 00:00:00+01:00')]}) exp = pd.DataFrame({'A': [pd.Timestamp('2012-11-11 00:00:00+01:00'), pd.Timestamp('2012-11-11 00:00:00+01:00')]}) - self.assert_frame_equal(df.fillna(method='bfill'), exp) + assert_frame_equal(df.fillna(method='bfill'), exp) def test_fillna_downcast(self): # GH 15277 From 431b6304ec2258278bb9aebb8afcb284714056f6 Mon Sep 17 00:00:00 2001 From: "Christopher C. Aycock" Date: Fri, 7 Apr 2017 14:06:38 -0400 Subject: [PATCH 4/4] Removed unnecessary 'self', again --- pandas/tests/series/test_missing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/tests/series/test_missing.py b/pandas/tests/series/test_missing.py index 6a1ded1b9033c..ea49abeee21c5 100644 --- a/pandas/tests/series/test_missing.py +++ b/pandas/tests/series/test_missing.py @@ -263,12 +263,12 @@ def test_datetime64_tz_fillna(self): df = pd.Series([pd.Timestamp('2012-11-11 00:00:00+01:00'), pd.NaT]) exp = pd.Series([pd.Timestamp('2012-11-11 00:00:00+01:00'), pd.Timestamp('2012-11-11 00:00:00+01:00')]) - self.assert_series_equal(df.fillna(method='pad'), exp) + assert_series_equal(df.fillna(method='pad'), exp) df = pd.Series([pd.NaT, pd.Timestamp('2012-11-11 00:00:00+01:00')]) exp = pd.Series([pd.Timestamp('2012-11-11 00:00:00+01:00'), pd.Timestamp('2012-11-11 00:00:00+01:00')]) - self.assert_series_equal(df.fillna(method='bfill'), exp) + assert_series_equal(df.fillna(method='bfill'), exp) def test_datetime64tz_fillna_round_issue(self): # GH 14872