DEPR/API: Revert handling of i8values to DatetimeIndex (#24708)

Author: TomAugspurger

doc/source/whatsnew/v0.24.0.rst

@@ -1235,7 +1235,6 @@ Datetimelike API Changes
 - :class:`PeriodIndex` subtraction of another ``PeriodIndex`` will now return an object-dtype :class:`Index` of :class:`DateOffset` objects instead of raising a ``TypeError`` (:issue:`20049`)
 - :func:`cut` and :func:`qcut` now returns a :class:`DatetimeIndex` or :class:`TimedeltaIndex` bins when the input is datetime or timedelta dtype respectively and ``retbins=True`` (:issue:`19891`)
 - :meth:`DatetimeIndex.to_period` and :meth:`Timestamp.to_period` will issue a warning when timezone information will be lost (:issue:`21333`)
-- :class:`DatetimeIndex` now accepts :class:`Int64Index` arguments as epoch timestamps (:issue:`20997`)
 - :meth:`PeriodIndex.tz_convert` and :meth:`PeriodIndex.tz_localize` have been removed (:issue:`21781`)
 
 .. _whatsnew_0240.api.other:
@@ -1353,6 +1352,52 @@ the object's ``freq`` attribute (:issue:`21939`, :issue:`23878`).
     dti + pd.Index([1 * dti.freq, 2 * dti.freq])
 
 
+.. _whatsnew_0240.deprecations.integer_tz:
+
+Passing Integer data and a timezone to DatetimeIndex
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The behavior of :class:`DatetimeIndex` when passed integer data and
+a timezone is changing in a future version of pandas. Previously, these
+were interpreted as wall times in the desired timezone. In the future,
+these will be interpreted as wall times in UTC, which are then converted
+to the desired timezone (:issue:`24559`).
+
+The default behavior remains the same, but issues a warning:
+
+.. code-block:: ipython
+
+   In [3]: pd.DatetimeIndex([946684800000000000], tz="US/Central")
+   /bin/ipython:1: FutureWarning:
+       Passing integer-dtype data and a timezone to DatetimeIndex. Integer values
+       will be interpreted differently in a future version of pandas. Previously,
+       these were viewed as datetime64[ns] values representing the wall time
+       *in the specified timezone*. In the future, these will be viewed as
+       datetime64[ns] values representing the wall time *in UTC*. This is similar
+       to a nanosecond-precision UNIX epoch. To accept the future behavior, use
+
+           pd.to_datetime(integer_data, utc=True).tz_convert(tz)
+
+       To keep the previous behavior, use
+
+           pd.to_datetime(integer_data).tz_localize(tz)
+
+    #!/bin/python3
+    Out[3]: DatetimeIndex(['2000-01-01 00:00:00-06:00'], dtype='datetime64[ns, US/Central]', freq=None)
+
+As the warning message explains, opt in to the future behavior by specifying that
+the integer values are UTC, and then converting to the final timezone:
+
+.. ipython:: python
+
+   pd.to_datetime([946684800000000000], utc=True).tz_convert('US/Central')
+
+The old behavior can be retained with by localizing directly to the final timezone:
+
+.. ipython:: python
+
+   pd.to_datetime([946684800000000000]).tz_localize('US/Central')
+
 .. _whatsnew_0240.deprecations.tz_aware_array:
 
 Converting Timezone-Aware Series and Index to NumPy Arrays

pandas/core/arrays/datetimes.py

@@ -33,6 +33,21 @@
 from pandas.tseries.offsets import Day, Tick
 
 _midnight = time(0, 0)
+# TODO(GH-24559): Remove warning, int_as_wall_time parameter.
+_i8_message = """
+    Passing integer-dtype data and a timezone to DatetimeIndex. Integer values
+    will be interpreted differently in a future version of pandas. Previously,
+    these were viewed as datetime64[ns] values representing the wall time
+    *in the specified timezone*. In the future, these will be viewed as
+    datetime64[ns] values representing the wall time *in UTC*. This is similar
+    to a nanosecond-precision UNIX epoch. To accept the future behavior, use
+
+        pd.to_datetime(integer_data, utc=True).tz_convert(tz)
+
+    To keep the previous behavior, use
+
+        pd.to_datetime(integer_data).tz_localize(tz)
+"""
 
 
 def tz_to_dtype(tz):
@@ -342,13 +357,15 @@ def _simple_new(cls, values, freq=None, dtype=_NS_DTYPE):
     @classmethod
     def _from_sequence(cls, data, dtype=None, copy=False,
                        tz=None, freq=None,
-                       dayfirst=False, yearfirst=False, ambiguous='raise'):
+                       dayfirst=False, yearfirst=False, ambiguous='raise',
+                       int_as_wall_time=False):
 
         freq, freq_infer = dtl.maybe_infer_freq(freq)
 
         subarr, tz, inferred_freq = sequence_to_dt64ns(
             data, dtype=dtype, copy=copy, tz=tz,
-            dayfirst=dayfirst, yearfirst=yearfirst, ambiguous=ambiguous)
+            dayfirst=dayfirst, yearfirst=yearfirst,
+            ambiguous=ambiguous, int_as_wall_time=int_as_wall_time)
 
         freq, freq_infer = dtl.validate_inferred_freq(freq, inferred_freq,
                                                       freq_infer)
@@ -1649,7 +1666,8 @@ def to_julian_date(self):
 
 def sequence_to_dt64ns(data, dtype=None, copy=False,
                        tz=None,
-                       dayfirst=False, yearfirst=False, ambiguous='raise'):
+                       dayfirst=False, yearfirst=False, ambiguous='raise',
+                       int_as_wall_time=False):
     """
     Parameters
     ----------
@@ -1661,6 +1679,13 @@ def sequence_to_dt64ns(data, dtype=None, copy=False,
     yearfirst : bool, default False
     ambiguous : str, bool, or arraylike, default 'raise'
         See pandas._libs.tslibs.conversion.tz_localize_to_utc
+    int_as_wall_time : bool, default False
+        Whether to treat ints as wall time in specified timezone, or as
+        nanosecond-precision UNIX epoch (wall time in UTC).
+        This is used in DatetimeIndex.__init__ to deprecate the wall-time
+        behaviour.
+
+        ..versionadded:: 0.24.0
 
     Returns
     -------
@@ -1717,6 +1742,10 @@ def sequence_to_dt64ns(data, dtype=None, copy=False,
             data, inferred_tz = objects_to_datetime64ns(
                 data, dayfirst=dayfirst, yearfirst=yearfirst)
             tz = maybe_infer_tz(tz, inferred_tz)
+            # When a sequence of timestamp objects is passed, we always
+            # want to treat the (now i8-valued) data as UTC timestamps,
+            # not wall times.
+            int_as_wall_time = False
 
     # `data` may have originally been a Categorical[datetime64[ns, tz]],
     # so we need to handle these types.
@@ -1744,8 +1773,16 @@ def sequence_to_dt64ns(data, dtype=None, copy=False,
     else:
         # must be integer dtype otherwise
         # assume this data are epoch timestamps
+        if tz:
+            tz = timezones.maybe_get_tz(tz)
+
         if data.dtype != _INT64_DTYPE:
             data = data.astype(np.int64, copy=False)
+        if int_as_wall_time and tz is not None and not timezones.is_utc(tz):
+            warnings.warn(_i8_message, FutureWarning, stacklevel=4)
+            data = conversion.tz_localize_to_utc(data.view('i8'), tz,
+                                                 ambiguous=ambiguous)
+            data = data.view(_NS_DTYPE)
         result = data.view(_NS_DTYPE)
 
     if copy:

pandas/core/indexes/base.py

@@ -22,7 +22,8 @@
     is_dtype_union_equal, is_extension_array_dtype, is_float, is_float_dtype,
     is_hashable, is_integer, is_integer_dtype, is_interval_dtype, is_iterator,
     is_list_like, is_object_dtype, is_period_dtype, is_scalar,
-    is_signed_integer_dtype, is_timedelta64_dtype, is_unsigned_integer_dtype)
+    is_signed_integer_dtype, is_timedelta64_dtype, is_unsigned_integer_dtype,
+    pandas_dtype)
 import pandas.core.dtypes.concat as _concat
 from pandas.core.dtypes.generic import (
     ABCDataFrame, ABCDateOffset, ABCDatetimeArray, ABCIndexClass,
@@ -732,6 +733,13 @@ def astype(self, dtype, copy=True):
             from .category import CategoricalIndex
             return CategoricalIndex(self.values, name=self.name, dtype=dtype,
                                     copy=copy)
+        elif is_datetime64tz_dtype(dtype):
+            # TODO(GH-24559): Remove this block, use the following elif.
+            # avoid FutureWarning from DatetimeIndex constructor.
+            from pandas import DatetimeIndex
+            tz = pandas_dtype(dtype).tz
+            return (DatetimeIndex(np.asarray(self))
+                    .tz_localize("UTC").tz_convert(tz))
 
         elif is_extension_array_dtype(dtype):
             return Index(np.asarray(self), dtype=dtype, copy=copy)

pandas/core/indexes/datetimes.py

@@ -299,7 +299,8 @@ def __new__(cls, data=None,
 
         dtarr = DatetimeArray._from_sequence(
             data, dtype=dtype, copy=copy, tz=tz, freq=freq,
-            dayfirst=dayfirst, yearfirst=yearfirst, ambiguous=ambiguous)
+            dayfirst=dayfirst, yearfirst=yearfirst, ambiguous=ambiguous,
+            int_as_wall_time=True)
 
         subarr = cls._simple_new(dtarr, name=name,
                                  freq=dtarr.freq, tz=dtarr.tz)

pandas/core/reshape/tile.py

@@ -449,7 +449,10 @@ def _convert_bin_to_datelike_type(bins, dtype):
     bins : Array-like of bins, DatetimeIndex or TimedeltaIndex if dtype is
            datelike
     """
-    if is_datetime64tz_dtype(dtype) or is_datetime_or_timedelta_dtype(dtype):
+    if is_datetime64tz_dtype(dtype):
+        bins = to_datetime(bins.astype(np.int64),
+                           utc=True).tz_convert(dtype.tz)
+    elif is_datetime_or_timedelta_dtype(dtype):
         bins = Index(bins.astype(np.int64), dtype=dtype)
     return bins
 

pandas/tests/dtypes/test_common.py

@@ -209,8 +209,8 @@ def test_is_datetime64tz_dtype():
     assert not com.is_datetime64tz_dtype(object)
     assert not com.is_datetime64tz_dtype([1, 2, 3])
     assert not com.is_datetime64tz_dtype(pd.DatetimeIndex([1, 2, 3]))
-    assert com.is_datetime64tz_dtype(pd.DatetimeIndex(
-        [1, 2, 3], tz="US/Eastern"))
+    assert com.is_datetime64tz_dtype(pd.DatetimeIndex(['2000'],
+                                                      tz="US/Eastern"))
 
 
 def test_is_timedelta64_dtype():
@@ -286,7 +286,7 @@ def test_is_datetimelike():
     assert com.is_datetimelike(pd.PeriodIndex([], freq="A"))
     assert com.is_datetimelike(np.array([], dtype=np.datetime64))
     assert com.is_datetimelike(pd.Series([], dtype="timedelta64[ns]"))
-    assert com.is_datetimelike(pd.DatetimeIndex([1, 2, 3], tz="US/Eastern"))
+    assert com.is_datetimelike(pd.DatetimeIndex(["2000"], tz="US/Eastern"))
 
     dtype = DatetimeTZDtype("ns", tz="US/Eastern")
     s = pd.Series([], dtype=dtype)
@@ -480,7 +480,7 @@ def test_needs_i8_conversion():
     assert com.needs_i8_conversion(np.datetime64)
     assert com.needs_i8_conversion(pd.Series([], dtype="timedelta64[ns]"))
     assert com.needs_i8_conversion(pd.DatetimeIndex(
-        [1, 2, 3], tz="US/Eastern"))
+        ["2000"], tz="US/Eastern"))
 
 
 def test_is_numeric_dtype():
@@ -541,7 +541,7 @@ def test_is_extension_type(check_scipy):
     assert com.is_extension_type(pd.Series(cat))
     assert com.is_extension_type(pd.SparseArray([1, 2, 3]))
     assert com.is_extension_type(pd.SparseSeries([1, 2, 3]))
-    assert com.is_extension_type(pd.DatetimeIndex([1, 2, 3], tz="US/Eastern"))
+    assert com.is_extension_type(pd.DatetimeIndex(['2000'], tz="US/Eastern"))
 
     dtype = DatetimeTZDtype("ns", tz="US/Eastern")
     s = pd.Series([], dtype=dtype)
@@ -635,8 +635,8 @@ def test__get_dtype_fails(input_param):
     (pd.DatetimeIndex([1, 2]), np.datetime64),
     (pd.DatetimeIndex([1, 2]).dtype, np.datetime64),
     ('<M8[ns]', np.datetime64),
-    (pd.DatetimeIndex([1, 2], tz='Europe/London'), pd.Timestamp),
-    (pd.DatetimeIndex([1, 2], tz='Europe/London').dtype,
+    (pd.DatetimeIndex(['2000'], tz='Europe/London'), pd.Timestamp),
+    (pd.DatetimeIndex(['2000'], tz='Europe/London').dtype,
      pd.Timestamp),
     ('datetime64[ns, Europe/London]', pd.Timestamp),
     (pd.SparseSeries([1, 2], dtype='int32'), np.int32),

pandas/tests/indexes/datetimes/test_astype.py

@@ -238,10 +238,10 @@ def _check_rng(rng):
         ['US/Pacific', 'datetime64[ns, US/Pacific]'],
         [None, 'datetime64[ns]']])
     def test_integer_index_astype_datetime(self, tz, dtype):
-        # GH 20997, 20964
+        # GH 20997, 20964, 24559
         val = [pd.Timestamp('2018-01-01', tz=tz).value]
         result = pd.Index(val).astype(dtype)
-        expected = pd.DatetimeIndex(['2018-01-01'], tz=tz)
+        expected = pd.DatetimeIndex(["2018-01-01"], tz=tz)
         tm.assert_index_equal(result, expected)
 
 

pandas/tests/indexes/datetimes/test_construction.py

@@ -118,8 +118,15 @@ def test_construction_with_alt_tz_localize(self, kwargs, tz_aware_fixture):
         tz = tz_aware_fixture
         i = pd.date_range('20130101', periods=5, freq='H', tz=tz)
         kwargs = {key: attrgetter(val)(i) for key, val in kwargs.items()}
-        result = DatetimeIndex(i.tz_localize(None).asi8, **kwargs)
-        expected = i.tz_localize(None).tz_localize('UTC').tz_convert(tz)
+
+        if str(tz) in ('UTC', 'tzutc()'):
+            warn = None
+        else:
+            warn = FutureWarning
+
+        with tm.assert_produces_warning(warn, check_stacklevel=False):
+            result = DatetimeIndex(i.tz_localize(None).asi8, **kwargs)
+        expected = DatetimeIndex(i, **kwargs)
         tm.assert_index_equal(result, expected)
 
         # localize into the provided tz
@@ -377,6 +384,19 @@ def test_range_kwargs_deprecated(self):
         with tm.assert_produces_warning(FutureWarning):
             DatetimeIndex(start='1/1/2000', end='1/10/2000', freq='D')
 
+    def test_integer_values_and_tz_deprecated(self):
+        # GH-24559
+        values = np.array([946684800000000000])
+        with tm.assert_produces_warning(FutureWarning):
+            result = DatetimeIndex(values, tz='US/Central')
+        expected = pd.DatetimeIndex(['2000-01-01T00:00:00'], tz="US/Central")
+        tm.assert_index_equal(result, expected)
+
+        # but UTC is *not* deprecated.
+        with tm.assert_produces_warning(None):
+            result = DatetimeIndex(values, tz='UTC')
+        expected = pd.DatetimeIndex(['2000-01-01T00:00:00'], tz="US/Central")
+
     def test_constructor_coverage(self):
         rng = date_range('1/1/2000', periods=10.5)
         exp = date_range('1/1/2000', periods=10)
@@ -555,21 +575,30 @@ def test_constructor_timestamp_near_dst(self):
                                   ts[1].to_pydatetime()])
         tm.assert_index_equal(result, expected)
 
+    # TODO(GH-24559): Remove the xfail for the tz-aware case.
     @pytest.mark.parametrize('klass', [Index, DatetimeIndex])
     @pytest.mark.parametrize('box', [
         np.array, partial(np.array, dtype=object), list])
     @pytest.mark.parametrize('tz, dtype', [
-        ['US/Pacific', 'datetime64[ns, US/Pacific]'],
-        [None, 'datetime64[ns]']])
+        pytest.param('US/Pacific', 'datetime64[ns, US/Pacific]',
+                     marks=[pytest.mark.xfail(),
+                            pytest.mark.filterwarnings(
+                                "ignore:\\n    Passing:FutureWarning")]),
+        [None, 'datetime64[ns]'],
+    ])
     def test_constructor_with_int_tz(self, klass, box, tz, dtype):
         # GH 20997, 20964
         ts = Timestamp('2018-01-01', tz=tz)
         result = klass(box([ts.value]), dtype=dtype)
         expected = klass([ts])
         assert result == expected
 
+    # This is the desired future behavior
+    @pytest.mark.xfail(reason="Future behavior", strict=False)
+    @pytest.mark.filterwarnings("ignore:\\n    Passing:FutureWarning")
     def test_construction_int_rountrip(self, tz_naive_fixture):
         # GH 12619
+        # TODO(GH-24559): Remove xfail
         tz = tz_naive_fixture
         result = 1293858000000000000
         expected = DatetimeIndex([1293858000000000000], tz=tz).asi8[0]

pandas/tests/indexes/multi/test_integrity.py

@@ -50,7 +50,9 @@ def test_values_multiindex_datetimeindex():
     # Test to ensure we hit the boxing / nobox part of MI.values
     ints = np.arange(10 ** 18, 10 ** 18 + 5)
     naive = pd.DatetimeIndex(ints)
-    aware = pd.DatetimeIndex(ints, tz='US/Central')
+    # TODO(GH-24559): Remove the FutureWarning
+    with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
+        aware = pd.DatetimeIndex(ints, tz='US/Central')
 
     idx = pd.MultiIndex.from_arrays([naive, aware])
     result = idx.values

pandas/tests/indexes/test_base.py

@@ -4,6 +4,7 @@
 from datetime import datetime, timedelta
 from decimal import Decimal
 import math
+import sys
 
 import numpy as np
 import pytest
@@ -401,24 +402,40 @@ def test_constructor_dtypes_datetime(self, tz_naive_fixture, attr, utc,
         # Test constructing with a datetimetz dtype
         # .values produces numpy datetimes, so these are considered naive
         # .asi8 produces integers, so these are considered epoch timestamps
+        # ^the above will be true in a later version. Right now we `.view`
+        # the i8 values as NS_DTYPE, effectively treating them as wall times.
         index = pd.date_range('2011-01-01', periods=5)
         arg = getattr(index, attr)
-        if utc:
-            index = index.tz_localize('UTC').tz_convert(tz_naive_fixture)
-        else:
-            index = index.tz_localize(tz_naive_fixture)
+        index = index.tz_localize(tz_naive_fixture)
         dtype = index.dtype
 
-        result = klass(arg, tz=tz_naive_fixture)
+        # TODO(GH-24559): Remove the sys.modules and warnings
+        # not sure what this is from. It's Py2 only.
+        modules = [sys.modules['pandas.core.indexes.base']]
+
+        if (tz_naive_fixture and attr == "asi8" and
+                str(tz_naive_fixture) not in ('UTC', 'tzutc()')):
+            ex_warn = FutureWarning
+        else:
+            ex_warn = None
+
+        # stacklevel is checked elsewhere. We don't do it here since
+        # Index will have an frame, throwing off the expected.
+        with tm.assert_produces_warning(ex_warn, check_stacklevel=False,
+                                        clear=modules):
+            result = klass(arg, tz=tz_naive_fixture)
         tm.assert_index_equal(result, index)
 
-        result = klass(arg, dtype=dtype)
+        with tm.assert_produces_warning(ex_warn, check_stacklevel=False):
+            result = klass(arg, dtype=dtype)
         tm.assert_index_equal(result, index)
 
-        result = klass(list(arg), tz=tz_naive_fixture)
+        with tm.assert_produces_warning(ex_warn, check_stacklevel=False):
+            result = klass(list(arg), tz=tz_naive_fixture)
         tm.assert_index_equal(result, index)
 
-        result = klass(list(arg), dtype=dtype)
+        with tm.assert_produces_warning(ex_warn, check_stacklevel=False):
+            result = klass(list(arg), dtype=dtype)
         tm.assert_index_equal(result, index)
 
     @pytest.mark.parametrize("attr", ['values', 'asi8'])