DEPR: __array__ for tz-aware Series/Index

This deprecates the current behvior when converting tz-aware Series
or Index to an ndarray. Previously, we converted to M8[ns], throwing
away the timezone information. In the future, we will return an
object-dtype array filled with Timestamps, each of which has the correct
tz.

```python
In [1]: import pandas as pd; import numpy as np

In [2]: ser = pd.Series(pd.date_range('2000', periods=2, tz="CET"))

In [3]: np.asarray(ser)
/bin/ipython:1: FutureWarning: Converting timezone-aware DatetimeArray to timezone-naive ndarray with 'datetime64[ns]' dtype. In the future, this will return an ndarray with 'object' dtype where each element is a 'pandas.Timestamp' with the correct 'tz'.
        To accept the future behavior, pass 'dtype=object'.
        To keep the old behavior, pass 'dtype="datetime64[ns]"'.
  #!/Users/taugspurger/Envs/pandas-dev/bin/python3
Out[3]:
array(['1999-12-31T23:00:00.000000000', '2000-01-01T23:00:00.000000000'],
      dtype='datetime64[ns]')
```

xref pandas-dev#23569

Author: TomAugspurger

pandas/core/arrays/datetimes.py

@@ -522,7 +522,7 @@ def _resolution(self):
     # Array-Like / EA-Interface Methods
 
     def __array__(self, dtype=None):
-        if is_object_dtype(dtype):
+        if is_object_dtype(dtype) or (dtype is None and self.tz):
             return np.array(list(self), dtype=object)
         elif is_int64_dtype(dtype):
             return self.asi8

pandas/core/dtypes/cast.py

@@ -1020,7 +1020,7 @@ def maybe_cast_to_datetime(value, dtype, errors='raise'):
                             # datetime64tz is assumed to be naive which should
                             # be localized to the timezone.
                             is_dt_string = is_string_dtype(value)
-                            value = to_datetime(value, errors=errors)
+                            value = to_datetime(value, errors=errors).array
                             if is_dt_string:
                                 # Strings here are naive, so directly localize
                                 value = value.tz_localize(dtype.tz)

pandas/core/dtypes/dtypes.py

@@ -318,6 +318,7 @@ def _hash_categories(categories, ordered=True):
         from pandas.core.util.hashing import (
             hash_array, _combine_hash_arrays, hash_tuples
         )
+        from pandas.core.dtypes.common import is_datetime64tz_dtype, _NS_DTYPE
 
         if len(categories) and isinstance(categories[0], tuple):
             # assumes if any individual category is a tuple, then all our. ATM
@@ -335,6 +336,11 @@ def _hash_categories(categories, ordered=True):
                     # find a better solution
                     hashed = hash((tuple(categories), ordered))
                     return hashed
+
+            if is_datetime64tz_dtype(categories.dtype):
+                # Avoid future warning.
+                categories = categories.astype(_NS_DTYPE)
+
             cat_array = hash_array(np.asarray(categories), categorize=False)
         if ordered:
             cat_array = np.vstack([

pandas/core/groupby/groupby.py

@@ -26,7 +26,8 @@ class providing the base-class of operations.
 
 from pandas.core.dtypes.cast import maybe_downcast_to_dtype
 from pandas.core.dtypes.common import (
-    ensure_float, is_extension_array_dtype, is_numeric_dtype, is_scalar)
+    _NS_DTYPE, ensure_float, is_datetime64tz_dtype, is_extension_array_dtype,
+    is_numeric_dtype, is_scalar)
 from pandas.core.dtypes.missing import isna, notna
 
 import pandas.core.algorithms as algorithms
@@ -1269,10 +1270,18 @@ def f(self, **kwargs):
             return f
 
         def first_compat(x, axis=0):
+            # This is a bit strange.
+            # We only hit this block when grouping a DatetimeTZBlock *and*
+            # a categorical. Something strange going on with first for
+            # categorical dta.
+            if is_datetime64tz_dtype(x.dtype):
+                dtype = _NS_DTYPE
+            else:
+                dtype = None
 
             def first(x):
 
-                x = np.asarray(x)
+                x = np.asarray(x, dtype=dtype)
                 x = x[notna(x)]
                 if len(x) == 0:
                     return np.nan
@@ -1284,10 +1293,18 @@ def first(x):
                 return first(x)
 
         def last_compat(x, axis=0):
+            # This is a bit strange.
+            # We only hit this block when grouping a DatetimeTZBlock *and*
+            # a categorical. Something strange going on with first for
+            # categorical dta.
+            if is_datetime64tz_dtype(x.dtype):
+                dtype = _NS_DTYPE
+            else:
+                dtype = None
 
             def last(x):
 
-                x = np.asarray(x)
+                x = np.asarray(x, dtype=dtype)
                 x = x[notna(x)]
                 if len(x) == 0:
                     return np.nan

pandas/core/indexes/datetimes.py

@@ -339,6 +339,21 @@ def _simple_new(cls, values, name=None, freq=None, tz=None, dtype=None):
 
     # --------------------------------------------------------------------
 
+    def __array__(self, dtype=None):
+        if dtype is None and isinstance(self._data, DatetimeArray)\
+                and getattr(self.dtype, 'tz', None):
+            msg = (
+                "Converting timezone-aware DatetimeArray to timezone-naive "
+                "ndarray with 'datetime64[ns]' dtype. In the future, this "
+                "will return an ndarray with 'object' dtype where each "
+                "element is a 'pandas.Timestamp' with the correct 'tz'.\n\t"
+                "To accept the future behavior, pass 'dtype=object'.\n\t"
+                "To keep the old behavior, pass 'dtype=\"datetime64[ns]\"'."
+            )
+            warnings.warn(msg, FutureWarning, stacklevel=3)
+            dtype = 'M8[ns]'
+        return np.asarray(self._data, dtype=dtype)
+
     @property
     def dtype(self):
         return self._eadata.dtype
@@ -1114,7 +1129,6 @@ def slice_indexer(self, start=None, end=None, step=None, kind=None):
 
     strftime = ea_passthrough(DatetimeArray.strftime)
     _has_same_tz = ea_passthrough(DatetimeArray._has_same_tz)
-    __array__ = ea_passthrough(DatetimeArray.__array__)
 
     @property
     def offset(self):

pandas/core/indexing.py

@@ -581,7 +581,12 @@ def can_do_equal_len():
                         setter(item, v)
 
                 # we have an equal len ndarray/convertible to our labels
-                elif np.array(value).ndim == 2:
+                # hasattr first, to avoid coercing to ndarray without reason.
+                # But we may be relying on the ndarray coercion to check ndim.
+                # Why not just convert to an ndarray earlier on if needed?
+                elif ((hasattr(value, 'ndim') and value.ndim == 2)
+                      or (not hasattr(value, 'ndim') and
+                          np.array(value).ndim) == 2):
 
                     # note that this coerces the dtype if we are mixed
                     # GH 7551

pandas/core/internals/blocks.py

@@ -2456,6 +2456,12 @@ def get_values(self, dtype=None):
             values = values.reshape(1, -1)
         return values
 
+    def to_dense(self):
+        # we request M8[ns] dtype here, even though it discards tzinfo,
+        # as lots of code (e.g. anything using values_from_object)
+        # expects that behavior.
+        return np.asarray(self.values, dtype=_NS_DTYPE)
+
     def _slice(self, slicer):
         """ return a slice of my values """
         if isinstance(slicer, tuple):
@@ -2600,6 +2606,40 @@ def setitem(self, indexer, value):
                               klass=ObjectBlock,)
             return newb.setitem(indexer, value)
 
+    def quantile(self, qs, interpolation='linear', axis=0, axes=None):
+        # TODO: Add quantile as a reduction method.
+        # We can't just use Block.quantile, as that converts the DTA
+        # to an ndarray[object] via get_values.
+        # This method
+        # 1. Convert DatetimeTZBlock -> DatetimeBlock
+        # 2. Perform the op via Block.quantile
+        # 3. Converts back to tz-aware
+        # Alternatively, we could special case the call to `get_values`
+        # in Block.quantile for DatetimeTZ.
+
+        new_values = np.asarray(self.values, dtype=_NS_DTYPE)
+        if self.ndim == 2:
+            new_values = new_values[None, :]
+
+        new_block = DatetimeBlock(new_values, placement=self.mgr_locs)
+
+        ax, naive = new_block.quantile(qs, interpolation=interpolation,
+                                       axis=axis, axes=axes)
+
+        ndim = getattr(naive, 'ndim', None) or 0
+        if ndim == 0:
+            return ax, self.make_block_scalar(
+                tslibs.Timestamp(naive.values.value, tz=self.values.tz)
+            )
+        else:
+            naive = naive.values.ravel()
+
+        result = DatetimeArray(naive, dtype=self.values.dtype)
+
+        return ax, make_block(result,
+                              placement=np.arange(len(result)),
+                              ndim=ndim)
+
 
 class TimeDeltaBlock(DatetimeLikeBlockMixin, IntBlock):
     __slots__ = ()

pandas/core/internals/construction.py

@@ -34,6 +34,7 @@
 from pandas.core.indexes import base as ibase
 from pandas.core.internals import (
     create_block_manager_from_arrays, create_block_manager_from_blocks)
+from pandas.core.internals.arrays import extract_array
 
 # ---------------------------------------------------------------------
 # BlockManager Interface
@@ -539,7 +540,6 @@ def sanitize_array(data, index, dtype=None, copy=False,
     Sanitize input data to an ndarray, copy if specified, coerce to the
     dtype if specified.
     """
-
     if dtype is not None:
         dtype = pandas_dtype(dtype)
 
@@ -551,8 +551,10 @@ def sanitize_array(data, index, dtype=None, copy=False,
         else:
             data = data.copy()
 
+    data = extract_array(data, extract_numpy=True)
+
     # GH#846
-    if isinstance(data, (np.ndarray, Index, ABCSeries)):
+    if isinstance(data, np.ndarray):
 
         if dtype is not None:
             subarr = np.array(data, copy=False)

pandas/core/reshape/tile.py

@@ -8,7 +8,7 @@
 from pandas._libs.lib import infer_dtype
 
 from pandas.core.dtypes.common import (
-    ensure_int64, is_categorical_dtype, is_datetime64_dtype,
+    _NS_DTYPE, ensure_int64, is_categorical_dtype, is_datetime64_dtype,
     is_datetime64tz_dtype, is_datetime_or_timedelta_dtype, is_integer,
     is_scalar, is_timedelta64_dtype)
 from pandas.core.dtypes.missing import isna
@@ -226,7 +226,10 @@ def cut(x, bins, right=True, labels=None, retbins=False, precision=3,
             raise ValueError('Overlapping IntervalIndex is not accepted.')
 
     else:
-        bins = np.asarray(bins)
+        if is_datetime64tz_dtype(bins):
+            bins = np.asarray(bins, dtype=_NS_DTYPE)
+        else:
+            bins = np.asarray(bins)
         bins = _convert_bin_to_numeric_type(bins, dtype)
         if (np.diff(bins) < 0).any():
             raise ValueError('bins must increase monotonically.')

pandas/core/series.py

@@ -21,7 +21,8 @@
     is_extension_array_dtype, is_extension_type, is_hashable, is_integer,
     is_iterator, is_list_like, is_scalar, is_string_like, is_timedelta64_dtype)
 from pandas.core.dtypes.generic import (
-    ABCDataFrame, ABCDatetimeIndex, ABCSeries, ABCSparseArray, ABCSparseSeries)
+    ABCDataFrame, ABCDatetimeArray, ABCDatetimeIndex, ABCSeries,
+    ABCSparseArray, ABCSparseSeries)
 from pandas.core.dtypes.missing import (
     isna, na_value_for_dtype, notna, remove_na_arraylike)
 
@@ -665,7 +666,20 @@ def __array__(self, result=None):
         """
         The array interface, return my values.
         """
-        return self.get_values()
+        # TODO: change the keyword name from result to dtype?
+        if (result is None and isinstance(self.array, ABCDatetimeArray)
+                and getattr(self.dtype, 'tz', None)):
+            msg = (
+                "Converting timezone-aware DatetimeArray to timezone-naive "
+                "ndarray with 'datetime64[ns]' dtype. In the future, this "
+                "will return an ndarray with 'object' dtype where each "
+                "element is a 'pandas.Timestamp' with the correct 'tz'.\n\t"
+                "To accept the future behavior, pass 'dtype=object'.\n\t"
+                "To keep the old behavior, pass 'dtype=\"datetime64[ns]\"'."
+            )
+            warnings.warn(msg, FutureWarning, stacklevel=3)
+            result = 'M8[ns]'
+        return np.asarray(self.array, result)
 
     def __array_wrap__(self, result, context=None):
         """

pandas/tests/arrays/test_datetimelike.py

@@ -264,11 +264,11 @@ def test_array(self, tz_naive_fixture):
         arr = DatetimeArray(dti)
 
         expected = dti.asi8.view('M8[ns]')
-        result = np.array(arr)
+        result = np.array(arr, dtype='M8[ns]')
         tm.assert_numpy_array_equal(result, expected)
 
         # check that we are not making copies when setting copy=False
-        result = np.array(arr, copy=False)
+        result = np.array(arr, dtype='M8[ns]', copy=False)
         assert result.base is expected.base
         assert result.base is not None
 

pandas/tests/arrays/test_datetimes.py

@@ -178,6 +178,39 @@ def test_fillna_preserves_tz(self, method):
         assert arr[2] is pd.NaT
         assert dti[2] == pd.Timestamp('2000-01-03', tz='US/Central')
 
+    def test_array_interface_tz(self):
+        tz = "US/Central"
+        data = DatetimeArray(pd.date_range('2017', periods=2, tz=tz))
+        result = np.asarray(data)
+
+        expected = np.array([pd.Timestamp('2017-01-01T00:00:00', tz=tz),
+                             pd.Timestamp('2017-01-02T00:00:00', tz=tz)],
+                            dtype=object)
+        tm.assert_numpy_array_equal(result, expected)
+
+        result = np.asarray(data, dtype=object)
+        tm.assert_numpy_array_equal(result, expected)
+
+        result = np.asarray(data, dtype='M8[ns]')
+
+        expected = np.array(['2017-01-01T06:00:00',
+                             '2017-01-02T06:00:00'], dtype="M8[ns]")
+        tm.assert_numpy_array_equal(result, expected)
+
+    def test_array_interface(self):
+        data = DatetimeArray(pd.date_range('2017', periods=2))
+        expected = np.array(['2017-01-01T00:00:00', '2017-01-02T00:00:00'],
+                            dtype='datetime64[ns]')
+
+        result = np.asarray(data)
+        tm.assert_numpy_array_equal(result, expected)
+
+        result = np.asarray(data, dtype=object)
+        expected = np.array([pd.Timestamp('2017-01-01T00:00:00'),
+                             pd.Timestamp('2017-01-02T00:00:00')],
+                            dtype=object)
+        tm.assert_numpy_array_equal(result, expected)
+
 
 class TestSequenceToDT64NS(object):
 

pandas/tests/dtypes/test_missing.py

@@ -278,17 +278,20 @@ def test_array_equivalent():
                             TimedeltaIndex([0, np.nan]))
     assert not array_equivalent(
         TimedeltaIndex([0, np.nan]), TimedeltaIndex([1, np.nan]))
-    assert array_equivalent(DatetimeIndex([0, np.nan], tz='US/Eastern'),
-                            DatetimeIndex([0, np.nan], tz='US/Eastern'))
-    assert not array_equivalent(
-        DatetimeIndex([0, np.nan], tz='US/Eastern'), DatetimeIndex(
-            [1, np.nan], tz='US/Eastern'))
-    assert not array_equivalent(
-        DatetimeIndex([0, np.nan]), DatetimeIndex(
-            [0, np.nan], tz='US/Eastern'))
-    assert not array_equivalent(
-        DatetimeIndex([0, np.nan], tz='CET'), DatetimeIndex(
-            [0, np.nan], tz='US/Eastern'))
+    with catch_warnings():
+        simplefilter("ignore")
+        assert array_equivalent(DatetimeIndex([0, np.nan], tz='US/Eastern'),
+                                DatetimeIndex([0, np.nan], tz='US/Eastern'))
+        assert not array_equivalent(
+            DatetimeIndex([0, np.nan], tz='US/Eastern'), DatetimeIndex(
+                [1, np.nan], tz='US/Eastern'))
+        assert not array_equivalent(
+            DatetimeIndex([0, np.nan]), DatetimeIndex(
+                [0, np.nan], tz='US/Eastern'))
+        assert not array_equivalent(
+            DatetimeIndex([0, np.nan], tz='CET'), DatetimeIndex(
+                [0, np.nan], tz='US/Eastern'))
+
     assert not array_equivalent(
         DatetimeIndex([0, np.nan]), TimedeltaIndex([0, np.nan]))
 

pandas/tests/indexes/datetimes/test_datetime.py

@@ -392,3 +392,45 @@ def test_unique(self, arr, expected):
         # GH 21737
         # Ensure the underlying data is consistent
         assert result[0] == expected[0]
+
+    def test_asarray_tz_naive(self):
+        # This shouldn't produce a warning.
+        idx = pd.date_range('2000', periods=2)
+        # M8[ns] by default
+        with tm.assert_produces_warning(None):
+            result = np.asarray(idx)
+
+        expected = np.array(['2000-01-01', '2000-01-02'], dtype='M8[ns]')
+        tm.assert_numpy_array_equal(result, expected)
+
+        # optionally, object
+        with tm.assert_produces_warning(None):
+            result = np.asarray(idx, dtype=object)
+
+        expected = np.array([pd.Timestamp('2000-01-01'),
+                             pd.Timestamp('2000-01-02')])
+        tm.assert_numpy_array_equal(result, expected)
+
+    def test_asarray_tz_aware(self):
+        tz = 'US/Central'
+        idx = pd.date_range('2000', periods=2, tz=tz)
+        expected = np.array(['2000-01-01T06', '2000-01-02T06'], dtype='M8[ns]')
+        # We warn by default and return an ndarray[M8[ns]]
+        with tm.assert_produces_warning(FutureWarning):
+            result = np.asarray(idx)
+
+        tm.assert_numpy_array_equal(result, expected)
+
+        # Old behavior with no warning
+        with tm.assert_produces_warning(None):
+            result = np.asarray(idx, dtype="M8[ns]")
+
+        tm.assert_numpy_array_equal(result, expected)
+
+        # Future behavior with no warning
+        expected = np.array([pd.Timestamp("2000-01-01", tz=tz),
+                             pd.Timestamp("2000-01-02", tz=tz)])
+        with tm.assert_produces_warning(None):
+            result = np.asarray(idx, dtype=object)
+
+        tm.assert_numpy_array_equal(result, expected)