DEPR: DatetimeIndex setops with mismatched tzs (pandas-dev#49455)

Author: jbrockmendel

doc/source/whatsnew/v2.0.0.rst

@@ -294,6 +294,7 @@ Removal of prior version deprecations/changes
 - Removed the deprecated method ``mad`` from pandas classes (:issue:`11787`)
 - Removed the deprecated method ``tshift`` from pandas classes (:issue:`11631`)
 - Changed behavior of empty data passed into :class:`Series`; the default dtype will be ``object`` instead of ``float64`` (:issue:`29405`)
+- Changed the behavior of :meth:`DatetimeIndex.union`, :meth:`DatetimeIndex.intersection`, and :meth:`DatetimeIndex.symmetric_difference` with mismatched timezones to convert to UTC instead of casting to object dtype (:issue:`39328`)
 - Changed the behavior of :func:`to_datetime` with argument "now" with ``utc=False`` to match ``Timestamp("now")`` (:issue:`18705`)
 - Changed behavior of :meth:`SparseArray.astype` when given a dtype that is not explicitly ``SparseDtype``, cast to the exact requested dtype rather than silently using a ``SparseDtype`` instead (:issue:`34457`)
 - Changed behavior of :class:`DataFrame` constructor given floating-point ``data`` and an integer ``dtype``, when the data cannot be cast losslessly, the floating point dtype is retained, matching :class:`Series` behavior (:issue:`41170`)

pandas/core/indexes/api.py

@@ -13,7 +13,6 @@
 from pandas.errors import InvalidIndexError
 
 from pandas.core.dtypes.cast import find_common_type
-from pandas.core.dtypes.common import is_dtype_equal
 
 from pandas.core.algorithms import safe_sort
 from pandas.core.indexes.base import (
@@ -276,7 +275,6 @@ def _find_common_index_dtype(inds):
 
     if kind == "special":
         result = indexes[0]
-        first = result
 
         dtis = [x for x in indexes if isinstance(x, DatetimeIndex)]
         dti_tzs = [x for x in dtis if x.tz is not None]
@@ -289,12 +287,6 @@ def _find_common_index_dtype(inds):
 
         if len(dtis) == len(indexes):
             sort = True
-            if not all(is_dtype_equal(x.dtype, first.dtype) for x in indexes):
-                # i.e. timezones mismatch
-                # TODO(2.0): once deprecation is enforced, this union will
-                #  cast to UTC automatically.
-                indexes = [x.tz_convert("UTC") for x in indexes]
-
             result = indexes[0]
 
         elif len(dtis) > 1:

pandas/core/indexes/base.py

@@ -3072,10 +3072,9 @@ def _validate_sort_keyword(self, sort):
             )
 
     @final
-    def _deprecate_dti_setop(self, other: Index, setop: str_t) -> None:
+    def _dti_setop_align_tzs(self, other: Index, setop: str_t) -> tuple[Index, Index]:
         """
-        Deprecate setop behavior between timezone-aware DatetimeIndexes with
-        mismatched timezones.
+        With mismatched timezones, cast both to UTC.
         """
         # Caller is responsibelf or checking
         #  `not is_dtype_equal(self.dtype, other.dtype)`
@@ -3086,14 +3085,10 @@ def _deprecate_dti_setop(self, other: Index, setop: str_t) -> None:
             and other.tz is not None
         ):
             # GH#39328, GH#45357
-            warnings.warn(
-                f"In a future version, the {setop} of DatetimeIndex objects "
-                "with mismatched timezones will cast both to UTC instead of "
-                "object dtype. To retain the old behavior, "
-                f"use `index.astype(object).{setop}(other)`",
-                FutureWarning,
-                stacklevel=find_stack_level(),
-            )
+            left = self.tz_convert("UTC")
+            right = other.tz_convert("UTC")
+            return left, right
+        return self, other
 
     @final
     def union(self, other, sort=None):
@@ -3193,7 +3188,7 @@ def union(self, other, sort=None):
                     "Can only union MultiIndex with MultiIndex or Index of tuples, "
                     "try mi.to_flat_index().union(other) instead."
                 )
-            self._deprecate_dti_setop(other, "union")
+            self, other = self._dti_setop_align_tzs(other, "union")
 
             dtype = self._find_common_type_compat(other)
             left = self.astype(dtype, copy=False)
@@ -3330,7 +3325,7 @@ def intersection(self, other, sort: bool = False):
         other, result_name = self._convert_can_do_setop(other)
 
         if not is_dtype_equal(self.dtype, other.dtype):
-            self._deprecate_dti_setop(other, "intersection")
+            self, other = self._dti_setop_align_tzs(other, "intersection")
 
         if self.equals(other):
             if self.has_duplicates:
@@ -3478,7 +3473,7 @@ def difference(self, other, sort=None):
         self._assert_can_do_setop(other)
         other, result_name = self._convert_can_do_setop(other)
 
-        # Note: we do NOT call _deprecate_dti_setop here, as there
+        # Note: we do NOT call _dti_setop_align_tzs here, as there
         #  is no requirement that .difference be commutative, so it does
         #  not cast to object.
 
@@ -3562,7 +3557,7 @@ def symmetric_difference(self, other, result_name=None, sort=None):
             result_name = result_name_update
 
         if not is_dtype_equal(self.dtype, other.dtype):
-            self._deprecate_dti_setop(other, "symmetric_difference")
+            self, other = self._dti_setop_align_tzs(other, "symmetric_difference")
 
         if not self._should_compare(other):
             return self.union(other, sort=sort).rename(result_name)

pandas/core/indexes/datetimes.py

@@ -429,18 +429,6 @@ def _can_range_setop(self, other) -> bool:
             return False
         return super()._can_range_setop(other)
 
-    def _maybe_utc_convert(self, other: Index) -> tuple[DatetimeIndex, Index]:
-        this = self
-
-        if isinstance(other, DatetimeIndex):
-            if (self.tz is None) ^ (other.tz is None):
-                raise TypeError("Cannot join tz-naive with tz-aware DatetimeIndex")
-
-            if not timezones.tz_compare(self.tz, other.tz):
-                this = self.tz_convert("UTC")
-                other = other.tz_convert("UTC")
-        return this, other
-
     # --------------------------------------------------------------------
 
     def _get_time_micros(self) -> npt.NDArray[np.int64]:

pandas/tests/indexes/datetimes/test_timezones.py

@@ -1155,19 +1155,21 @@ def test_dti_convert_tz_aware_datetime_datetime(self, tz):
     @pytest.mark.parametrize("setop", ["union", "intersection", "symmetric_difference"])
     def test_dti_setop_aware(self, setop):
         # non-overlapping
+        # GH#39328 as of 2.0 we cast these to UTC instead of object
         rng = date_range("2012-11-15 00:00:00", periods=6, freq="H", tz="US/Central")
 
         rng2 = date_range("2012-11-15 12:00:00", periods=6, freq="H", tz="US/Eastern")
 
-        with tm.assert_produces_warning(FutureWarning):
-            # # GH#39328 will cast both to UTC
-            result = getattr(rng, setop)(rng2)
+        result = getattr(rng, setop)(rng2)
 
-        expected = getattr(rng.astype("O"), setop)(rng2.astype("O"))
+        left = rng.tz_convert("UTC")
+        right = rng2.tz_convert("UTC")
+        expected = getattr(left, setop)(right)
         tm.assert_index_equal(result, expected)
+        assert result.tz == left.tz
         if len(result):
-            assert result[0].tz.zone == "US/Central"
-            assert result[-1].tz.zone == "US/Eastern"
+            assert result[0].tz.zone == "UTC"
+            assert result[-1].tz.zone == "UTC"
 
     def test_dti_union_mixed(self):
         # GH 21671