From 7cd8291d1a50c89af25f30b3ab1145d7fa7bb9ef Mon Sep 17 00:00:00 2001 From: phofl Date: Tue, 22 Sep 2020 22:22:29 +0200 Subject: [PATCH 1/4] Fix bug with pre epoch normalization --- doc/source/whatsnew/v1.1.3.rst | 1 + pandas/_libs/tslibs/conversion.pyx | 2 +- pandas/tests/series/test_datetime_values.py | 11 +++++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.1.3.rst b/doc/source/whatsnew/v1.1.3.rst index 1d386fa372ce1..88d0d54a0c000 100644 --- a/doc/source/whatsnew/v1.1.3.rst +++ b/doc/source/whatsnew/v1.1.3.rst @@ -35,6 +35,7 @@ Fixed regressions - Fixed regression in :meth:`Series.__getitem__` incorrectly raising when the input was a frozenset (:issue:`35747`) - Fixed regression in :meth:`read_excel` with ``engine="odf"`` caused ``UnboundLocalError`` in some cases where cells had nested child nodes (:issue:`36122`,:issue:`35802`) - Fixed regression in :class:`DataFrame` and :class:`Series` comparisons between numeric arrays and strings (:issue:`35700`,:issue:`36377`) +- Fixed regression in :meth:`Series.dt.normalize()` when normalizing pre-epoch dates the result was shifted one day (:issue:`36294`) .. --------------------------------------------------------------------------- diff --git a/pandas/_libs/tslibs/conversion.pyx b/pandas/_libs/tslibs/conversion.pyx index adf1dfbc1ac72..3b52b4d499694 100644 --- a/pandas/_libs/tslibs/conversion.pyx +++ b/pandas/_libs/tslibs/conversion.pyx @@ -830,7 +830,7 @@ cpdef inline datetime localize_pydatetime(datetime dt, object tz): # ---------------------------------------------------------------------- # Normalization -@cython.cdivision +@cython.cdivision(False) cdef inline int64_t normalize_i8_stamp(int64_t local_val) nogil: """ Round the localized nanosecond timestamp down to the previous midnight. diff --git a/pandas/tests/series/test_datetime_values.py b/pandas/tests/series/test_datetime_values.py index 723bd303b1974..8deeb58987267 100644 --- a/pandas/tests/series/test_datetime_values.py +++ b/pandas/tests/series/test_datetime_values.py @@ -702,3 +702,14 @@ def test_week_and_weekofyear_are_deprecated(): series.dt.week with tm.assert_produces_warning(FutureWarning): series.dt.weekofyear + + +def test_normalize_pre_epoch_dates(): + # GH: 36294 + df = pd.DataFrame( + {"year": [1969, 2016], "month": [1, 1], "day": [1, 1], "hour": [9, 9]} + ) + df = pd.to_datetime(df) + result = df.dt.normalize() + expected = pd.to_datetime(pd.Series(["1969-01-01", "2016-01-01"])) + tm.assert_series_equal(result, expected) From 8d65ce39d4abee548d6ebd32e18d64aab1ed444d Mon Sep 17 00:00:00 2001 From: phofl Date: Tue, 22 Sep 2020 23:15:03 +0200 Subject: [PATCH 2/4] Add review comments --- doc/source/whatsnew/v1.1.3.rst | 2 +- pandas/tests/series/test_datetime_values.py | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/doc/source/whatsnew/v1.1.3.rst b/doc/source/whatsnew/v1.1.3.rst index 3e2d3bed06a5b..cd0a53ee8e8ee 100644 --- a/doc/source/whatsnew/v1.1.3.rst +++ b/doc/source/whatsnew/v1.1.3.rst @@ -36,7 +36,7 @@ Fixed regressions - Fixed regression in :meth:`read_excel` with ``engine="odf"`` caused ``UnboundLocalError`` in some cases where cells had nested child nodes (:issue:`36122`,:issue:`35802`) - Fixed regression in :class:`DataFrame` and :class:`Series` comparisons between numeric arrays and strings (:issue:`35700`,:issue:`36377`) - Fixed regression when setting empty :class:`DataFrame` column to a :class:`Series` in preserving name of index in frame (:issue:`36527`) -- Fixed regression in :meth:`Series.dt.normalize()` when normalizing pre-epoch dates the result was shifted one day (:issue:`36294`) +- Fixed regression in :meth:`Series.dt.normalize` when normalizing pre-epoch dates the result was shifted one day (:issue:`36294`) .. --------------------------------------------------------------------------- diff --git a/pandas/tests/series/test_datetime_values.py b/pandas/tests/series/test_datetime_values.py index 8deeb58987267..b0926089bd7b4 100644 --- a/pandas/tests/series/test_datetime_values.py +++ b/pandas/tests/series/test_datetime_values.py @@ -706,10 +706,7 @@ def test_week_and_weekofyear_are_deprecated(): def test_normalize_pre_epoch_dates(): # GH: 36294 - df = pd.DataFrame( - {"year": [1969, 2016], "month": [1, 1], "day": [1, 1], "hour": [9, 9]} - ) - df = pd.to_datetime(df) - result = df.dt.normalize() + s = pd.to_datetime(pd.Series(["1969-01-01 09:00:00", "2016-01-01 09:00:00"])) + result = s.dt.normalize() expected = pd.to_datetime(pd.Series(["1969-01-01", "2016-01-01"])) tm.assert_series_equal(result, expected) From 386d59eca9d9db60995e78344615c4ef9abb2b4c Mon Sep 17 00:00:00 2001 From: phofl Date: Wed, 23 Sep 2020 20:14:33 +0200 Subject: [PATCH 3/4] Add test --- pandas/tests/scalar/timestamp/test_timestamp.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pandas/tests/scalar/timestamp/test_timestamp.py b/pandas/tests/scalar/timestamp/test_timestamp.py index cee7ac450e411..be39f05e7c33c 100644 --- a/pandas/tests/scalar/timestamp/test_timestamp.py +++ b/pandas/tests/scalar/timestamp/test_timestamp.py @@ -564,3 +564,10 @@ def test_dt_subclass_add_timedelta(lh, rh): result = lh + rh expected = SubDatetime(2000, 1, 1, 1) assert result == expected + + +def test_normalize_pre_epoch_dates(): + # GH: 36294 + result = Timestamp("1969-01-01 09:00:00").normalize() + expected = Timestamp("1969-01-01 00:00:00") + assert result == expected From 9e77858739b12814aa67a77dc0ea965e25d6b28b Mon Sep 17 00:00:00 2001 From: phofl Date: Thu, 24 Sep 2020 10:08:11 +0200 Subject: [PATCH 4/4] Move test --- pandas/tests/scalar/timestamp/test_timestamp.py | 7 ------- pandas/tests/scalar/timestamp/test_unary_ops.py | 6 ++++++ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/pandas/tests/scalar/timestamp/test_timestamp.py b/pandas/tests/scalar/timestamp/test_timestamp.py index be39f05e7c33c..cee7ac450e411 100644 --- a/pandas/tests/scalar/timestamp/test_timestamp.py +++ b/pandas/tests/scalar/timestamp/test_timestamp.py @@ -564,10 +564,3 @@ def test_dt_subclass_add_timedelta(lh, rh): result = lh + rh expected = SubDatetime(2000, 1, 1, 1) assert result == expected - - -def test_normalize_pre_epoch_dates(): - # GH: 36294 - result = Timestamp("1969-01-01 09:00:00").normalize() - expected = Timestamp("1969-01-01 00:00:00") - assert result == expected diff --git a/pandas/tests/scalar/timestamp/test_unary_ops.py b/pandas/tests/scalar/timestamp/test_unary_ops.py index 8641bbd0a66f2..e8196cd8328e7 100644 --- a/pandas/tests/scalar/timestamp/test_unary_ops.py +++ b/pandas/tests/scalar/timestamp/test_unary_ops.py @@ -397,6 +397,12 @@ def test_normalize(self, tz_naive_fixture, arg): expected = Timestamp("2013-11-30", tz=tz) assert result == expected + def test_normalize_pre_epoch_dates(self): + # GH: 36294 + result = Timestamp("1969-01-01 09:00:00").normalize() + expected = Timestamp("1969-01-01 00:00:00") + assert result == expected + # -------------------------------------------------------------- @td.skip_if_windows