diff reduction for 24024 (#24543)

Author: jbrockmendel

pandas/core/arrays/datetimes.py

@@ -19,7 +19,7 @@
     is_extension_type, is_float_dtype, is_int64_dtype, is_object_dtype,
     is_period_dtype, is_string_dtype, is_timedelta64_dtype, pandas_dtype)
 from pandas.core.dtypes.dtypes import DatetimeTZDtype
-from pandas.core.dtypes.generic import ABCIndexClass, ABCSeries
+from pandas.core.dtypes.generic import ABCIndexClass, ABCPandasArray, ABCSeries
 from pandas.core.dtypes.missing import isna
 
 from pandas.core import ops
@@ -224,7 +224,7 @@ def _simple_new(cls, values, freq=None, tz=None):
             # for compat with datetime/timedelta/period shared methods,
             #  we can sometimes get here with int64 values.  These represent
             #  nanosecond UTC (or tz-naive) unix timestamps
-            values = values.view('M8[ns]')
+            values = values.view(_NS_DTYPE)
 
         assert values.dtype == 'M8[ns]', values.dtype
 
@@ -417,7 +417,7 @@ def tz(self):
             Returns None when the array is tz-naive.
         """
         # GH 18595
-        return getattr(self._dtype, "tz", None)
+        return getattr(self.dtype, "tz", None)
 
     @tz.setter
     def tz(self, value):
@@ -517,10 +517,6 @@ def astype(self, dtype, copy=True):
     # ----------------------------------------------------------------
     # ExtensionArray Interface
 
-    @property
-    def _ndarray_values(self):
-        return self._data
-
     @Appender(dtl.DatetimeLikeArrayMixin._validate_fill_value.__doc__)
     def _validate_fill_value(self, fill_value):
         if isna(fill_value):
@@ -1568,6 +1564,8 @@ def sequence_to_dt64ns(data, dtype=None, copy=False,
         copy = False
     elif isinstance(data, ABCSeries):
         data = data._values
+    if isinstance(data, ABCPandasArray):
+        data = data.to_numpy()
 
     if hasattr(data, "freq"):
         # i.e. DatetimeArray/Index

pandas/core/arrays/period.py

@@ -269,11 +269,6 @@ def _check_compatible_with(self, other):
     def dtype(self):
         return self._dtype
 
-    @property
-    def _ndarray_values(self):
-        # Ordinals
-        return self._data
-
     @property
     def freq(self):
         """
@@ -475,7 +470,6 @@ def _format_native_types(self, na_rep=u'NaT', date_format=None, **kwargs):
         """
         actually format my specific types
         """
-        # TODO(DatetimeArray): remove
         values = self.astype(object)
 
         if date_format:

pandas/core/arrays/timedeltas.py

@@ -369,8 +369,9 @@ def _addsub_offset_array(self, other, op):
             # TimedeltaIndex can only operate with a subset of DateOffset
             # subclasses.  Incompatible classes will raise AttributeError,
             # which we re-raise as TypeError
-            return dtl.DatetimeLikeArrayMixin._addsub_offset_array(self, other,
-                                                                   op)
+            return super(TimedeltaArrayMixin, self)._addsub_offset_array(
+                other, op
+            )
         except AttributeError:
             raise TypeError("Cannot add/subtract non-tick DateOffset to {cls}"
                             .format(cls=type(self).__name__))

pandas/core/indexes/datetimelike.py

@@ -55,6 +55,7 @@ class DatetimeIndexOpsMixin(ExtensionOpsMixin):
     """
     common ops mixin to support a unified interface datetimelike Index
     """
+    _data = None  # type: DatetimeLikeArrayMixin
 
     # DatetimeLikeArrayMixin assumes subclasses are mutable, so these are
     # properties there.  They can be made into cache_readonly for Index
@@ -72,6 +73,9 @@ class DatetimeIndexOpsMixin(ExtensionOpsMixin):
 
     @property
     def freq(self):
+        """
+        Return the frequency object if it is set, otherwise None.
+        """
         return self._eadata.freq
 
     @freq.setter
@@ -81,6 +85,9 @@ def freq(self, value):
 
     @property
     def freqstr(self):
+        """
+        Return the frequency object as a string if it is set, otherwise None.
+        """
         return self._eadata.freqstr
 
     def unique(self, level=None):
@@ -111,6 +118,20 @@ def wrapper(self, other):
     def _ndarray_values(self):
         return self._eadata._ndarray_values
 
+    # ------------------------------------------------------------------------
+    # Abstract data attributes
+
+    @property
+    def values(self):
+        # type: () -> np.ndarray
+        # Note: PeriodArray overrides this to return an ndarray of objects.
+        return self._eadata._data
+
+    @property
+    @Appender(DatetimeLikeArrayMixin.asi8.__doc__)
+    def asi8(self):
+        return self._eadata.asi8
+
     # ------------------------------------------------------------------------
 
     def equals(self, other):

pandas/core/indexes/datetimes.py

@@ -330,7 +330,6 @@ def _simple_new(cls, values, name=None, freq=None, tz=None, dtype=None):
         result._eadata = dtarr
         result.name = name
         # For groupby perf. See note in indexes/base about _index_data
-        # TODO: make sure this is updated correctly if edited
         result._index_data = result._data
         result._reset_identity()
         return result

pandas/core/indexes/period.py

@@ -18,7 +18,6 @@
 from pandas.core import common as com
 from pandas.core.accessor import delegate_names
 from pandas.core.algorithms import unique1d
-from pandas.core.arrays.datetimelike import DatelikeOps
 from pandas.core.arrays.period import (
     PeriodArray, period_array, validate_dtype_freq)
 from pandas.core.base import _shared_docs
@@ -70,9 +69,9 @@ class PeriodDelegateMixin(DatetimelikeDelegateMixin):
                 typ='property')
 @delegate_names(PeriodArray,
                 PeriodDelegateMixin._delegated_methods,
-                typ="method")
-class PeriodIndex(DatelikeOps, DatetimeIndexOpsMixin, Int64Index,
-                  PeriodDelegateMixin):
+                typ="method",
+                overwrite=True)
+class PeriodIndex(DatetimeIndexOpsMixin, Int64Index, PeriodDelegateMixin):
     """
     Immutable ndarray holding ordinal values indicating regular periods in
     time such as particular years, quarters, months, etc.
@@ -291,20 +290,15 @@ def _eadata(self):
     def values(self):
         return np.asarray(self)
 
-    @property
-    def _values(self):
-        return self._data
-
     @property
     def freq(self):
-        # TODO(DatetimeArray): remove
-        # Can't simply use delegate_names since our base class is defining
-        # freq
         return self._data.freq
 
     @freq.setter
     def freq(self, value):
         value = Period._maybe_convert_freq(value)
+        # TODO: When this deprecation is enforced, PeriodIndex.freq can
+        # be removed entirely, and we'll just inherit.
         msg = ('Setting {cls}.freq has been deprecated and will be '
                'removed in a future version; use {cls}.asfreq instead. '
                'The {cls}.freq setter is not guaranteed to work.')
@@ -897,11 +891,6 @@ def flags(self):
                       FutureWarning, stacklevel=2)
         return self._ndarray_values.flags
 
-    @property
-    def asi8(self):
-        # TODO(DatetimeArray): remove
-        return self.view('i8')
-
     def item(self):
         """
         return the first element of the underlying data as a python

pandas/core/indexes/timedeltas.py

@@ -70,8 +70,8 @@ class TimedeltaDelegateMixin(DatetimelikeDelegateMixin):
 @delegate_names(TimedeltaArray,
                 TimedeltaDelegateMixin._delegated_methods,
                 typ="method", overwrite=False)
-class TimedeltaIndex(DatetimeIndexOpsMixin,
-                     dtl.TimelikeOps, Int64Index, TimedeltaDelegateMixin):
+class TimedeltaIndex(DatetimeIndexOpsMixin, dtl.TimelikeOps, Int64Index,
+                     TimedeltaDelegateMixin):
     """
     Immutable ndarray of timedelta64 data, represented internally as int64, and
     which can be boxed to timedelta objects
@@ -238,7 +238,6 @@ def _simple_new(cls, values, name=None, freq=None, dtype=_TD_DTYPE):
         result._eadata = tdarr
         result.name = name
         # For groupby perf. See note in indexes/base about _index_data
-        # TODO: make sure this is updated correctly if edited
         result._index_data = tdarr._data
 
         result._reset_identity()

pandas/core/internals/blocks.py

@@ -2165,7 +2165,7 @@ def should_store(self, value):
 
 
 class DatetimeLikeBlockMixin(object):
-    """Mixin class for DatetimeBlock and DatetimeTZBlock."""
+    """Mixin class for DatetimeBlock, DatetimeTZBlock, and TimedeltaBlock."""
 
     @property
     def _holder(self):
@@ -2857,15 +2857,17 @@ def to_native_types(self, slicer=None, na_rep=None, date_format=None,
         """ convert to our native types format, slicing if desired """
 
         values = self.values
+        i8values = self.values.view('i8')
+
         if slicer is not None:
-            values = values[..., slicer]
+            i8values = i8values[..., slicer]
 
         from pandas.io.formats.format import _get_format_datetime64_from_values
         format = _get_format_datetime64_from_values(values, date_format)
 
         result = tslib.format_array_from_datetime(
-            values.view('i8').ravel(), tz=getattr(self.values, 'tz', None),
-            format=format, na_rep=na_rep).reshape(values.shape)
+            i8values.ravel(), tz=getattr(self.values, 'tz', None),
+            format=format, na_rep=na_rep).reshape(i8values.shape)
         return np.atleast_2d(result)
 
     def should_store(self, value):
@@ -3115,8 +3117,16 @@ def get_block_type(values, dtype=None):
     dtype = dtype or values.dtype
     vtype = dtype.type
 
-    if is_categorical(values):
+    if is_sparse(dtype):
+        # Need this first(ish) so that Sparse[datetime] is sparse
+        cls = ExtensionBlock
+    elif is_categorical(values):
         cls = CategoricalBlock
+    elif issubclass(vtype, np.datetime64):
+        assert not is_datetime64tz_dtype(values)
+        cls = DatetimeBlock
+    elif is_datetime64tz_dtype(values):
+        cls = DatetimeTZBlock
     elif is_interval_dtype(dtype) or is_period_dtype(dtype):
         cls = ObjectValuesExtensionBlock
     elif is_extension_array_dtype(values):
@@ -3128,11 +3138,6 @@ def get_block_type(values, dtype=None):
         cls = TimeDeltaBlock
     elif issubclass(vtype, np.complexfloating):
         cls = ComplexBlock
-    elif issubclass(vtype, np.datetime64):
-        assert not is_datetime64tz_dtype(values)
-        cls = DatetimeBlock
-    elif is_datetime64tz_dtype(values):
-        cls = DatetimeTZBlock
     elif issubclass(vtype, np.integer):
         cls = IntBlock
     elif dtype == np.bool_:

pandas/core/ops.py

@@ -1539,17 +1539,20 @@ def wrapper(left, right):
             raise TypeError("{typ} cannot perform the operation "
                             "{op}".format(typ=type(left).__name__, op=str_rep))
 
-        elif (is_extension_array_dtype(left) or
-                (is_extension_array_dtype(right) and not is_scalar(right))):
-            # GH#22378 disallow scalar to exclude e.g. "category", "Int64"
-            return dispatch_to_extension_op(op, left, right)
-
         elif is_datetime64_dtype(left) or is_datetime64tz_dtype(left):
+            # Give dispatch_to_index_op a chance for tests like
+            # test_dt64_series_add_intlike, which the index dispatching handles
+            # specifically.
             result = dispatch_to_index_op(op, left, right, pd.DatetimeIndex)
             return construct_result(left, result,
                                     index=left.index, name=res_name,
                                     dtype=result.dtype)
 
+        elif (is_extension_array_dtype(left) or
+                (is_extension_array_dtype(right) and not is_scalar(right))):
+            # GH#22378 disallow scalar to exclude e.g. "category", "Int64"
+            return dispatch_to_extension_op(op, left, right)
+
         elif is_timedelta64_dtype(left):
             result = dispatch_to_index_op(op, left, right, pd.TimedeltaIndex)
             return construct_result(left, result,

pandas/tests/arrays/test_datetimes.py

@@ -16,6 +16,14 @@
 
 
 class TestDatetimeArrayConstructor(object):
+    def test_from_pandas_array(self):
+        arr = pd.array(np.arange(5, dtype=np.int64)) * 3600 * 10**9
+
+        result = DatetimeArray._from_sequence(arr, freq='infer')
+
+        expected = pd.date_range('1970-01-01', periods=5, freq='H')._eadata
+        tm.assert_datetime_array_equal(result, expected)
+
     def test_mismatched_timezone_raises(self):
         arr = DatetimeArray(np.array(['2000-01-01T06:00:00'], dtype='M8[ns]'),
                             dtype=DatetimeTZDtype(tz='US/Central'))

pandas/tests/frame/test_indexing.py

@@ -3245,7 +3245,9 @@ def test_setitem(self):
         b1 = df._data.blocks[1]
         b2 = df._data.blocks[2]
         assert b1.values.equals(b2.values)
-        assert id(b1.values.values.base) != id(b2.values.values.base)
+        if b1.values.values.base is not None:
+            # base being None suffices to assure a copy was made
+            assert id(b1.values.values.base) != id(b2.values.values.base)
 
         # with nan
         df2 = df.copy()