Skip to content

Commit 75d093d

Browse files
authored
DEPR: DatetimeIndex setops with mismatched tzs (#49455)
1 parent 72688c7 commit 75d093d

File tree

5 files changed

+19
-41
lines changed

5 files changed

+19
-41
lines changed

doc/source/whatsnew/v2.0.0.rst

+1
Original file line numberDiff line numberDiff line change
@@ -294,6 +294,7 @@ Removal of prior version deprecations/changes
294294
- Removed the deprecated method ``mad`` from pandas classes (:issue:`11787`)
295295
- Removed the deprecated method ``tshift`` from pandas classes (:issue:`11631`)
296296
- Changed behavior of empty data passed into :class:`Series`; the default dtype will be ``object`` instead of ``float64`` (:issue:`29405`)
297+
- Changed the behavior of :meth:`DatetimeIndex.union`, :meth:`DatetimeIndex.intersection`, and :meth:`DatetimeIndex.symmetric_difference` with mismatched timezones to convert to UTC instead of casting to object dtype (:issue:`39328`)
297298
- Changed the behavior of :func:`to_datetime` with argument "now" with ``utc=False`` to match ``Timestamp("now")`` (:issue:`18705`)
298299
- Changed behavior of :meth:`SparseArray.astype` when given a dtype that is not explicitly ``SparseDtype``, cast to the exact requested dtype rather than silently using a ``SparseDtype`` instead (:issue:`34457`)
299300
- Changed behavior of :class:`DataFrame` constructor given floating-point ``data`` and an integer ``dtype``, when the data cannot be cast losslessly, the floating point dtype is retained, matching :class:`Series` behavior (:issue:`41170`)

pandas/core/indexes/api.py

-8
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@
1313
from pandas.errors import InvalidIndexError
1414

1515
from pandas.core.dtypes.cast import find_common_type
16-
from pandas.core.dtypes.common import is_dtype_equal
1716

1817
from pandas.core.algorithms import safe_sort
1918
from pandas.core.indexes.base import (
@@ -276,7 +275,6 @@ def _find_common_index_dtype(inds):
276275

277276
if kind == "special":
278277
result = indexes[0]
279-
first = result
280278

281279
dtis = [x for x in indexes if isinstance(x, DatetimeIndex)]
282280
dti_tzs = [x for x in dtis if x.tz is not None]
@@ -289,12 +287,6 @@ def _find_common_index_dtype(inds):
289287

290288
if len(dtis) == len(indexes):
291289
sort = True
292-
if not all(is_dtype_equal(x.dtype, first.dtype) for x in indexes):
293-
# i.e. timezones mismatch
294-
# TODO(2.0): once deprecation is enforced, this union will
295-
# cast to UTC automatically.
296-
indexes = [x.tz_convert("UTC") for x in indexes]
297-
298290
result = indexes[0]
299291

300292
elif len(dtis) > 1:

pandas/core/indexes/base.py

+10-15
Original file line numberDiff line numberDiff line change
@@ -3072,10 +3072,9 @@ def _validate_sort_keyword(self, sort):
30723072
)
30733073

30743074
@final
3075-
def _deprecate_dti_setop(self, other: Index, setop: str_t) -> None:
3075+
def _dti_setop_align_tzs(self, other: Index, setop: str_t) -> tuple[Index, Index]:
30763076
"""
3077-
Deprecate setop behavior between timezone-aware DatetimeIndexes with
3078-
mismatched timezones.
3077+
With mismatched timezones, cast both to UTC.
30793078
"""
30803079
# Caller is responsibelf or checking
30813080
# `not is_dtype_equal(self.dtype, other.dtype)`
@@ -3086,14 +3085,10 @@ def _deprecate_dti_setop(self, other: Index, setop: str_t) -> None:
30863085
and other.tz is not None
30873086
):
30883087
# GH#39328, GH#45357
3089-
warnings.warn(
3090-
f"In a future version, the {setop} of DatetimeIndex objects "
3091-
"with mismatched timezones will cast both to UTC instead of "
3092-
"object dtype. To retain the old behavior, "
3093-
f"use `index.astype(object).{setop}(other)`",
3094-
FutureWarning,
3095-
stacklevel=find_stack_level(),
3096-
)
3088+
left = self.tz_convert("UTC")
3089+
right = other.tz_convert("UTC")
3090+
return left, right
3091+
return self, other
30973092

30983093
@final
30993094
def union(self, other, sort=None):
@@ -3193,7 +3188,7 @@ def union(self, other, sort=None):
31933188
"Can only union MultiIndex with MultiIndex or Index of tuples, "
31943189
"try mi.to_flat_index().union(other) instead."
31953190
)
3196-
self._deprecate_dti_setop(other, "union")
3191+
self, other = self._dti_setop_align_tzs(other, "union")
31973192

31983193
dtype = self._find_common_type_compat(other)
31993194
left = self.astype(dtype, copy=False)
@@ -3330,7 +3325,7 @@ def intersection(self, other, sort: bool = False):
33303325
other, result_name = self._convert_can_do_setop(other)
33313326

33323327
if not is_dtype_equal(self.dtype, other.dtype):
3333-
self._deprecate_dti_setop(other, "intersection")
3328+
self, other = self._dti_setop_align_tzs(other, "intersection")
33343329

33353330
if self.equals(other):
33363331
if self.has_duplicates:
@@ -3478,7 +3473,7 @@ def difference(self, other, sort=None):
34783473
self._assert_can_do_setop(other)
34793474
other, result_name = self._convert_can_do_setop(other)
34803475

3481-
# Note: we do NOT call _deprecate_dti_setop here, as there
3476+
# Note: we do NOT call _dti_setop_align_tzs here, as there
34823477
# is no requirement that .difference be commutative, so it does
34833478
# not cast to object.
34843479

@@ -3562,7 +3557,7 @@ def symmetric_difference(self, other, result_name=None, sort=None):
35623557
result_name = result_name_update
35633558

35643559
if not is_dtype_equal(self.dtype, other.dtype):
3565-
self._deprecate_dti_setop(other, "symmetric_difference")
3560+
self, other = self._dti_setop_align_tzs(other, "symmetric_difference")
35663561

35673562
if not self._should_compare(other):
35683563
return self.union(other, sort=sort).rename(result_name)

pandas/core/indexes/datetimes.py

-12
Original file line numberDiff line numberDiff line change
@@ -429,18 +429,6 @@ def _can_range_setop(self, other) -> bool:
429429
return False
430430
return super()._can_range_setop(other)
431431

432-
def _maybe_utc_convert(self, other: Index) -> tuple[DatetimeIndex, Index]:
433-
this = self
434-
435-
if isinstance(other, DatetimeIndex):
436-
if (self.tz is None) ^ (other.tz is None):
437-
raise TypeError("Cannot join tz-naive with tz-aware DatetimeIndex")
438-
439-
if not timezones.tz_compare(self.tz, other.tz):
440-
this = self.tz_convert("UTC")
441-
other = other.tz_convert("UTC")
442-
return this, other
443-
444432
# --------------------------------------------------------------------
445433

446434
def _get_time_micros(self) -> npt.NDArray[np.int64]:

pandas/tests/indexes/datetimes/test_timezones.py

+8-6
Original file line numberDiff line numberDiff line change
@@ -1155,19 +1155,21 @@ def test_dti_convert_tz_aware_datetime_datetime(self, tz):
11551155
@pytest.mark.parametrize("setop", ["union", "intersection", "symmetric_difference"])
11561156
def test_dti_setop_aware(self, setop):
11571157
# non-overlapping
1158+
# GH#39328 as of 2.0 we cast these to UTC instead of object
11581159
rng = date_range("2012-11-15 00:00:00", periods=6, freq="H", tz="US/Central")
11591160

11601161
rng2 = date_range("2012-11-15 12:00:00", periods=6, freq="H", tz="US/Eastern")
11611162

1162-
with tm.assert_produces_warning(FutureWarning):
1163-
# # GH#39328 will cast both to UTC
1164-
result = getattr(rng, setop)(rng2)
1163+
result = getattr(rng, setop)(rng2)
11651164

1166-
expected = getattr(rng.astype("O"), setop)(rng2.astype("O"))
1165+
left = rng.tz_convert("UTC")
1166+
right = rng2.tz_convert("UTC")
1167+
expected = getattr(left, setop)(right)
11671168
tm.assert_index_equal(result, expected)
1169+
assert result.tz == left.tz
11681170
if len(result):
1169-
assert result[0].tz.zone == "US/Central"
1170-
assert result[-1].tz.zone == "US/Eastern"
1171+
assert result[0].tz.zone == "UTC"
1172+
assert result[-1].tz.zone == "UTC"
11711173

11721174
def test_dti_union_mixed(self):
11731175
# GH 21671

0 commit comments

Comments
 (0)