From 9c9c78d0289efce83cc2523b43f8214ca33e21a2 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Fri, 19 Aug 2022 13:22:09 -0700 Subject: [PATCH 1/3] BUG: DTI.intersection with DST transition --- pandas/core/indexes/datetimes.py | 10 ++++++++++ pandas/tests/indexes/datetimes/test_setops.py | 9 +++++++++ 2 files changed, 19 insertions(+) diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py index 5f5cc5cacac9f..82ae06cac7662 100644 --- a/pandas/core/indexes/datetimes.py +++ b/pandas/core/indexes/datetimes.py @@ -416,6 +416,16 @@ def _formatter_func(self): # -------------------------------------------------------------------- # Set Operation Methods + def _can_range_setop(self, other) -> bool: + # GH 46702: If self or other have non-UTC tzs, DST transitions prevent + # range representation due to no singular step + if self.tz is not None and not timezones.is_utc(self.tz): + return False + other_tz = getattr(other, "tz", None) + if other_tz is not None and not timezones.is_utc(other_tz): + return False + return super()._can_range_setop(other) + def union_many(self, others): """ A bit of a hack to accelerate unioning a collection of indexes. diff --git a/pandas/tests/indexes/datetimes/test_setops.py b/pandas/tests/indexes/datetimes/test_setops.py index e92d726867f32..aa3a5a9ab4535 100644 --- a/pandas/tests/indexes/datetimes/test_setops.py +++ b/pandas/tests/indexes/datetimes/test_setops.py @@ -593,3 +593,12 @@ def test_intersection_bug(self): result = a.intersection(b) tm.assert_index_equal(result, b) assert result.freq == b.freq + + @pytest.mark.parametrize("tz", [None, "UTC", "Europe/Berlin"]) + def test_intersection_dst_transition(self, tz): + # GH 46702: Europe/Berlin has DST transition + idx1 = date_range("2020-03-27", periods=5, freq="D", tz=tz) + idx2 = date_range("2020-03-30", periods=5, freq="D", tz=tz) + result = idx1.intersection(idx2) + expected = date_range("2020-03-30", periods=2, freq="D", tz=tz) + tm.assert_index_equal(result, expected) From 8db8949a3cf729982468e500514c680125d017b6 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Fri, 19 Aug 2022 13:24:55 -0700 Subject: [PATCH 2/3] Add whatsnew note --- doc/source/whatsnew/v1.4.4.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v1.4.4.rst b/doc/source/whatsnew/v1.4.4.rst index 3414082581b80..f92bcfb32a5b2 100644 --- a/doc/source/whatsnew/v1.4.4.rst +++ b/doc/source/whatsnew/v1.4.4.rst @@ -21,6 +21,7 @@ Fixed regressions - Fixed regression in :meth:`DataFrame.loc` not aligning index in some cases when setting a :class:`DataFrame` (:issue:`47578`) - Fixed regression in :meth:`DataFrame.loc` setting a length-1 array like value to a single value in the DataFrame (:issue:`46268`) - Fixed regression in setting ``None`` or non-string value into a ``string``-dtype Series using a mask (:issue:`47628`) +- Fixed regression in :meth:`DatetimeIndex.intersection` when the :class:`DatetimeIndex` has dates crossing daylight savings time (:issue:`46702`) - Fixed regression in :func:`merge` throwing an error when passing a :class:`Series` with a multi-level name (:issue:`47946`) - Fixed regression in :meth:`DataFrame.eval` creating a copy when updating inplace (:issue:`47449`) - From 3c865153709cae0fb48dccb9c62a67aff54b135e Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Fri, 19 Aug 2022 15:28:15 -0700 Subject: [PATCH 3/3] Add fixed offset as well --- pandas/_libs/tslibs/timezones.pxd | 2 +- pandas/_libs/tslibs/timezones.pyi | 1 + pandas/_libs/tslibs/timezones.pyx | 2 +- pandas/core/indexes/datetimes.py | 13 ++++++++++--- pandas/tests/indexes/datetimes/test_setops.py | 5 ++++- 5 files changed, 17 insertions(+), 6 deletions(-) diff --git a/pandas/_libs/tslibs/timezones.pxd b/pandas/_libs/tslibs/timezones.pxd index d1f46b39b2940..c1a4e2bd5e1ac 100644 --- a/pandas/_libs/tslibs/timezones.pxd +++ b/pandas/_libs/tslibs/timezones.pxd @@ -18,6 +18,6 @@ cpdef object get_timezone(tzinfo tz) cpdef tzinfo maybe_get_tz(object tz) cdef timedelta get_utcoffset(tzinfo tz, datetime obj) -cdef bint is_fixed_offset(tzinfo tz) +cpdef bint is_fixed_offset(tzinfo tz) cdef object get_dst_info(tzinfo tz) diff --git a/pandas/_libs/tslibs/timezones.pyi b/pandas/_libs/tslibs/timezones.pyi index d241a35f21cca..4e9f0c6ae6c33 100644 --- a/pandas/_libs/tslibs/timezones.pyi +++ b/pandas/_libs/tslibs/timezones.pyi @@ -18,3 +18,4 @@ def infer_tzinfo( def maybe_get_tz(tz: str | int | np.int64 | tzinfo | None) -> tzinfo | None: ... def get_timezone(tz: tzinfo) -> tzinfo | str: ... def is_utc(tz: tzinfo | None) -> bool: ... +def is_fixed_offset(tz: tzinfo) -> bool: ... diff --git a/pandas/_libs/tslibs/timezones.pyx b/pandas/_libs/tslibs/timezones.pyx index 5992f31e96988..abf8bbc5ca5b9 100644 --- a/pandas/_libs/tslibs/timezones.pyx +++ b/pandas/_libs/tslibs/timezones.pyx @@ -236,7 +236,7 @@ cdef timedelta get_utcoffset(tzinfo tz, datetime obj): return tz.utcoffset(obj) -cdef inline bint is_fixed_offset(tzinfo tz): +cpdef inline bint is_fixed_offset(tzinfo tz): if treat_tz_as_dateutil(tz): if len(tz._trans_idx) == 0 and len(tz._trans_list) == 0: return 1 diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py index 82ae06cac7662..80138c25b0c27 100644 --- a/pandas/core/indexes/datetimes.py +++ b/pandas/core/indexes/datetimes.py @@ -419,10 +419,17 @@ def _formatter_func(self): def _can_range_setop(self, other) -> bool: # GH 46702: If self or other have non-UTC tzs, DST transitions prevent # range representation due to no singular step - if self.tz is not None and not timezones.is_utc(self.tz): + if ( + self.tz is not None + and not timezones.is_utc(self.tz) + and not timezones.is_fixed_offset(self.tz) + ): return False - other_tz = getattr(other, "tz", None) - if other_tz is not None and not timezones.is_utc(other_tz): + if ( + other.tz is not None + and not timezones.is_utc(other.tz) + and not timezones.is_fixed_offset(other.tz) + ): return False return super()._can_range_setop(other) diff --git a/pandas/tests/indexes/datetimes/test_setops.py b/pandas/tests/indexes/datetimes/test_setops.py index aa3a5a9ab4535..be8d70c127e8b 100644 --- a/pandas/tests/indexes/datetimes/test_setops.py +++ b/pandas/tests/indexes/datetimes/test_setops.py @@ -2,6 +2,7 @@ import numpy as np import pytest +import pytz import pandas.util._test_decorators as td @@ -594,7 +595,9 @@ def test_intersection_bug(self): tm.assert_index_equal(result, b) assert result.freq == b.freq - @pytest.mark.parametrize("tz", [None, "UTC", "Europe/Berlin"]) + @pytest.mark.parametrize( + "tz", [None, "UTC", "Europe/Berlin", pytz.FixedOffset(-60)] + ) def test_intersection_dst_transition(self, tz): # GH 46702: Europe/Berlin has DST transition idx1 = date_range("2020-03-27", periods=5, freq="D", tz=tz)