From e2fa5538a4661007febc3356adf4d1c1476514fc Mon Sep 17 00:00:00 2001 From: Brock Date: Sun, 13 Nov 2022 10:41:56 -0800 Subject: [PATCH 1/4] BUG: rendering dt64tz values with non-pytz --- doc/source/whatsnew/v2.0.0.rst | 2 ++ pandas/_libs/tslibs/vectorized.pyi | 1 - pandas/_libs/tslibs/vectorized.pyx | 4 ++-- pandas/tests/arrays/test_datetimes.py | 29 +++++++++++++++++++++++++++ 4 files changed, 33 insertions(+), 3 deletions(-) diff --git a/doc/source/whatsnew/v2.0.0.rst b/doc/source/whatsnew/v2.0.0.rst index 032bcf09244e5..1f690902fcfdc 100644 --- a/doc/source/whatsnew/v2.0.0.rst +++ b/doc/source/whatsnew/v2.0.0.rst @@ -610,6 +610,8 @@ Datetimelike - Bug in :class:`DatetimeIndex` constructor failing to raise when ``tz=None`` is explicitly specified in conjunction with timezone-aware ``dtype`` or data (:issue:`48659`) - Bug in subtracting a ``datetime`` scalar from :class:`DatetimeIndex` failing to retain the original ``freq`` attribute (:issue:`48818`) - Bug in ``pandas.tseries.holiday.Holiday`` where a half-open date interval causes inconsistent return types from :meth:`USFederalHolidayCalendar.holidays` (:issue:`49075`) +- Bug in rendering :class:`DatetimeIndex` and :class:`Series` and :class:`DataFrame` with timezone-aware dtypes with ``dateutil`` or ``zoneinfo`` timezones near daylight-savings transitions (:issue:`??`) +- Timedelta ^^^^^^^^^ diff --git a/pandas/_libs/tslibs/vectorized.pyi b/pandas/_libs/tslibs/vectorized.pyi index 22f457b9ddc0b..3fd9e2501e611 100644 --- a/pandas/_libs/tslibs/vectorized.pyi +++ b/pandas/_libs/tslibs/vectorized.pyi @@ -33,7 +33,6 @@ def get_resolution( def ints_to_pydatetime( arr: npt.NDArray[np.int64], tz: tzinfo | None = ..., - fold: bool = ..., box: str = ..., reso: int = ..., # NPY_DATETIMEUNIT ) -> npt.NDArray[np.object_]: ... diff --git a/pandas/_libs/tslibs/vectorized.pyx b/pandas/_libs/tslibs/vectorized.pyx index 8661ba4b9b2f1..b95cebd60a847 100644 --- a/pandas/_libs/tslibs/vectorized.pyx +++ b/pandas/_libs/tslibs/vectorized.pyx @@ -94,7 +94,6 @@ def tz_convert_from_utc(ndarray stamps, tzinfo tz, NPY_DATETIMEUNIT reso=NPY_FR_ def ints_to_pydatetime( ndarray stamps, tzinfo tz=None, - bint fold=False, str box="datetime", NPY_DATETIMEUNIT reso=NPY_FR_ns, ) -> np.ndarray: @@ -136,6 +135,7 @@ def ints_to_pydatetime( tzinfo new_tz bint use_date = False, use_ts = False, use_pydt = False object res_val + bint fold = 0 # Note that `result` (and thus `result_flat`) is C-order and # `it` iterates C-order as well, so the iteration matches @@ -168,7 +168,7 @@ def ints_to_pydatetime( else: - local_val = info.utc_val_to_local_val(utc_val, &pos) + local_val = info.utc_val_to_local_val(utc_val, &pos, &fold) if info.use_pytz: # find right representation of dst etc in pytz timezone new_tz = tz._tzinfos[tz._transition_info[pos]] diff --git a/pandas/tests/arrays/test_datetimes.py b/pandas/tests/arrays/test_datetimes.py index 166362a9a8c30..b13791715c33e 100644 --- a/pandas/tests/arrays/test_datetimes.py +++ b/pandas/tests/arrays/test_datetimes.py @@ -3,6 +3,7 @@ """ from datetime import timedelta import operator +from zoneinfo import ZoneInfo import numpy as np import pytest @@ -706,3 +707,31 @@ def test_tz_localize_t2d(self): roundtrip = expected.tz_localize("US/Pacific") tm.assert_datetime_array_equal(roundtrip, dta) + + @pytest.mark.parametrize( + "tz", [ZoneInfo("US/Eastern"), "US/Eastern", "dateutil/US/Eastern"] + ) + def test_iter_zoneinfo_fold(self, tz): + utc_vals = np.array( + [1320552000, 1320555600, 1320559200, 1320562800], dtype=np.int64 + ) + utc_vals *= 1_000_000_000 + + dta = DatetimeArray(utc_vals).tz_localize("UTC").tz_convert(tz) + + left = dta[2] + right = list(dta)[2] + assert str(left) == str(right) + # previously there was a bug where with non-pytz right would be + # Timestamp('2011-11-06 01:00:00-0400', tz='US/Eastern') + # while left would be + # Timestamp('2011-11-06 01:00:00-0500', tz='US/Eastern') + # The .value's would match (so they would compare as equal), + # but the folds would not + assert left.utcoffset() == right.utcoffset() + + # The same bug in ints_to_pydatetime affected .astype, so we test + # that here. + right2 = dta.astype(object)[2] + assert str(left) == str(right2) + assert left.utcoffset() == right2.utcoffset() From 72cf7ca4f31f419617583da128ffdfa906bb4931 Mon Sep 17 00:00:00 2001 From: Brock Date: Sun, 13 Nov 2022 10:46:38 -0800 Subject: [PATCH 2/4] GH ref --- doc/source/whatsnew/v2.0.0.rst | 2 +- pandas/tests/arrays/test_datetimes.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v2.0.0.rst b/doc/source/whatsnew/v2.0.0.rst index 1f690902fcfdc..51ec4361a171e 100644 --- a/doc/source/whatsnew/v2.0.0.rst +++ b/doc/source/whatsnew/v2.0.0.rst @@ -610,7 +610,7 @@ Datetimelike - Bug in :class:`DatetimeIndex` constructor failing to raise when ``tz=None`` is explicitly specified in conjunction with timezone-aware ``dtype`` or data (:issue:`48659`) - Bug in subtracting a ``datetime`` scalar from :class:`DatetimeIndex` failing to retain the original ``freq`` attribute (:issue:`48818`) - Bug in ``pandas.tseries.holiday.Holiday`` where a half-open date interval causes inconsistent return types from :meth:`USFederalHolidayCalendar.holidays` (:issue:`49075`) -- Bug in rendering :class:`DatetimeIndex` and :class:`Series` and :class:`DataFrame` with timezone-aware dtypes with ``dateutil`` or ``zoneinfo`` timezones near daylight-savings transitions (:issue:`??`) +- Bug in rendering :class:`DatetimeIndex` and :class:`Series` and :class:`DataFrame` with timezone-aware dtypes with ``dateutil`` or ``zoneinfo`` timezones near daylight-savings transitions (:issue:`49684`) - Timedelta diff --git a/pandas/tests/arrays/test_datetimes.py b/pandas/tests/arrays/test_datetimes.py index b13791715c33e..fde7e329c5a72 100644 --- a/pandas/tests/arrays/test_datetimes.py +++ b/pandas/tests/arrays/test_datetimes.py @@ -712,6 +712,7 @@ def test_tz_localize_t2d(self): "tz", [ZoneInfo("US/Eastern"), "US/Eastern", "dateutil/US/Eastern"] ) def test_iter_zoneinfo_fold(self, tz): + # GH#49684 utc_vals = np.array( [1320552000, 1320555600, 1320559200, 1320562800], dtype=np.int64 ) From 8b39f6ffa4d47c378f4f32d03530e8e81e4550de Mon Sep 17 00:00:00 2001 From: Brock Date: Sun, 13 Nov 2022 11:25:26 -0800 Subject: [PATCH 3/4] py38 compat --- pandas/tests/arrays/test_datetimes.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/pandas/tests/arrays/test_datetimes.py b/pandas/tests/arrays/test_datetimes.py index fde7e329c5a72..70656639237e9 100644 --- a/pandas/tests/arrays/test_datetimes.py +++ b/pandas/tests/arrays/test_datetimes.py @@ -3,7 +3,11 @@ """ from datetime import timedelta import operator -from zoneinfo import ZoneInfo + +try: + from zoneinfo import ZoneInfo +except ImportError: + ZoneInfo = None import numpy as np import pytest @@ -708,9 +712,11 @@ def test_tz_localize_t2d(self): roundtrip = expected.tz_localize("US/Pacific") tm.assert_datetime_array_equal(roundtrip, dta) - @pytest.mark.parametrize( - "tz", [ZoneInfo("US/Eastern"), "US/Eastern", "dateutil/US/Eastern"] - ) + easts = ["US/Eastern", "dateutil/US/Eastern"] + if ZoneInfo is not None: + easts.append(ZoneInfo("US/Eastern")) + + @pytest.mark.parametrize("tz", easts) def test_iter_zoneinfo_fold(self, tz): # GH#49684 utc_vals = np.array( From f3daa2f09df89fe3cb75b26bcc4fa32da21a0995 Mon Sep 17 00:00:00 2001 From: Brock Date: Sun, 13 Nov 2022 14:17:16 -0800 Subject: [PATCH 4/4] compat for no-tzdata --- pandas/tests/arrays/test_datetimes.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pandas/tests/arrays/test_datetimes.py b/pandas/tests/arrays/test_datetimes.py index 70656639237e9..728a3753da785 100644 --- a/pandas/tests/arrays/test_datetimes.py +++ b/pandas/tests/arrays/test_datetimes.py @@ -714,7 +714,11 @@ def test_tz_localize_t2d(self): easts = ["US/Eastern", "dateutil/US/Eastern"] if ZoneInfo is not None: - easts.append(ZoneInfo("US/Eastern")) + try: + easts.append(ZoneInfo("US/Eastern")) + except KeyError: + # No tzdata + pass @pytest.mark.parametrize("tz", easts) def test_iter_zoneinfo_fold(self, tz):