WIP: make _holder changeover

pandas/_libs/src/ujson/python/objToJSON.c

@@ -228,6 +228,11 @@ static PyObject *get_values(PyObject *obj) {
     PRINTMARK();
 
     if (values && !PyArray_CheckExact(values)) {
+
+        if (PyObject_HasAttrString(values, "to_numpy")) {
+            values = PyObject_CallMethod(values, "to_numpy", NULL);
+        }
+
         if (PyObject_HasAttrString(values, "values")) {
             PyObject *subvals = get_values(values);
             PyErr_Clear();
@@ -279,8 +284,8 @@ static PyObject *get_values(PyObject *obj) {
             repr = PyString_FromString("<unknown dtype>");
         }
 
-        PyErr_Format(PyExc_ValueError, "%s or %s are not JSON serializable yet",
-                     PyString_AS_STRING(repr), PyString_AS_STRING(typeRepr));
+        PyErr_Format(PyExc_ValueError, "%R or %R are not JSON serializable yet",
+                     repr, typeRepr);
         Py_DECREF(repr);
         Py_DECREF(typeRepr);
 

pandas/core/arrays/datetimes.py

@@ -19,7 +19,7 @@
     is_extension_type, is_float_dtype, is_int64_dtype, is_object_dtype,
     is_period_dtype, is_string_dtype, is_timedelta64_dtype, pandas_dtype)
 from pandas.core.dtypes.dtypes import DatetimeTZDtype
-from pandas.core.dtypes.generic import ABCIndexClass, ABCSeries
+from pandas.core.dtypes.generic import ABCIndexClass, ABCSeries, ABCPandasArray
 from pandas.core.dtypes.missing import isna
 
 from pandas.core import ops
@@ -240,11 +240,14 @@ def _simple_new(cls, values, freq=None, tz=None):
         result._dtype = dtype
         return result
 
-    def __new__(cls, values, freq=None, tz=None, dtype=None, copy=False,
+    def __init__(self, values, freq=None, tz=None, dtype=None, copy=False,
                 dayfirst=False, yearfirst=False, ambiguous='raise'):
-        return cls._from_sequence(
+        result = type(self)._from_sequence(
             values, freq=freq, tz=tz, dtype=dtype, copy=copy,
             dayfirst=dayfirst, yearfirst=yearfirst, ambiguous=ambiguous)
+        self._data = result._data
+        self._freq = result._freq
+        self._dtype = result._dtype
 
     @classmethod
     def _from_sequence(cls, data, dtype=None, copy=False,
@@ -523,7 +526,9 @@ def _ndarray_values(self):
 
     @Appender(dtl.DatetimeLikeArrayMixin._validate_fill_value.__doc__)
     def _validate_fill_value(self, fill_value):
-        if isna(fill_value):
+        if isna(fill_value) or fill_value == iNaT:
+            # FIXME: shouldn't allow iNaT through here; see discussion
+            #  in GH#24024
             fill_value = iNaT
         elif isinstance(fill_value, (datetime, np.datetime64)):
             self._assert_tzawareness_compat(fill_value)
@@ -1568,6 +1573,8 @@ def sequence_to_dt64ns(data, dtype=None, copy=False,
         copy = False
     elif isinstance(data, ABCSeries):
         data = data._values
+    elif isinstance(data, ABCPandasArray):
+        data = data.to_numpy()
 
     if hasattr(data, "freq"):
         # i.e. DatetimeArray/Index

pandas/core/dtypes/common.py

@@ -10,8 +10,7 @@
 
 from pandas.core.dtypes.dtypes import (
     CategoricalDtype, CategoricalDtypeType, DatetimeTZDtype, ExtensionDtype,
-    IntervalDtype, PandasExtensionDtype, PeriodDtype, _pandas_registry,
-    registry)
+    IntervalDtype, PandasExtensionDtype, PeriodDtype, registry)
 from pandas.core.dtypes.generic import (
     ABCCategorical, ABCCategoricalIndex, ABCDateOffset, ABCDatetimeIndex,
     ABCIndexClass, ABCPeriodArray, ABCPeriodIndex, ABCSeries, ABCSparseArray,
@@ -1984,7 +1983,7 @@ def pandas_dtype(dtype):
         return dtype
 
     # registered extension types
-    result = _pandas_registry.find(dtype) or registry.find(dtype)
+    result = registry.find(dtype)
     if result is not None:
         return result
 

pandas/core/dtypes/concat.py

@@ -12,8 +12,8 @@
     is_extension_array_dtype, is_interval_dtype, is_object_dtype,
     is_period_dtype, is_sparse, is_timedelta64_dtype)
 from pandas.core.dtypes.generic import (
-    ABCDatetimeIndex, ABCPeriodIndex, ABCRangeIndex, ABCSparseDataFrame,
-    ABCTimedeltaIndex)
+    ABCDatetimeArray, ABCDatetimeIndex, ABCIndexClass, ABCPeriodIndex,
+    ABCRangeIndex, ABCSparseDataFrame, ABCTimedeltaIndex)
 
 from pandas import compat
 
@@ -471,7 +471,15 @@ def _concat_datetimetz(to_concat, name=None):
     all inputs must be DatetimeIndex
     it is used in DatetimeIndex.append also
     """
-    return to_concat[0]._concat_same_dtype(to_concat, name=name)
+    # Right now, internals will pass a List[DatetimeArray] here
+    # for reductions like quantile. I would like to disentangle
+    # all this before we get here.
+    sample = to_concat[0]
+
+    if isinstance(sample, ABCIndexClass):
+        return sample._concat_same_dtype(to_concat, name=name)
+    elif isinstance(sample, ABCDatetimeArray):
+        return sample._concat_same_type(to_concat)
 
 
 def _concat_index_same_dtype(indexes, klass=None):

pandas/core/dtypes/dtypes.py

@@ -479,8 +479,8 @@ def _is_boolean(self):
         return is_bool_dtype(self.categories)
 
 
-class DatetimeTZDtype(PandasExtensionDtype):
-
+@register_extension_dtype
+class DatetimeTZDtype(PandasExtensionDtype, ExtensionDtype):
     """
     A np.dtype duck-typed class, suitable for holding a custom datetime with tz
     dtype.
@@ -493,6 +493,7 @@ class DatetimeTZDtype(PandasExtensionDtype):
     str = '|M8[ns]'
     num = 101
     base = np.dtype('M8[ns]')
+    na_value = NaT
     _metadata = ('unit', 'tz')
     _match = re.compile(r"(datetime64|M8)\[(?P<unit>.+), (?P<tz>.+)\]")
     _cache = {}
@@ -570,8 +571,8 @@ def construct_array_type(cls):
         -------
         type
         """
-        from pandas import DatetimeIndex
-        return DatetimeIndex
+        from pandas.core.arrays import DatetimeArrayMixin
+        return DatetimeArrayMixin
 
     @classmethod
     def construct_from_string(cls, string):
@@ -885,10 +886,3 @@ def is_dtype(cls, dtype):
             else:
                 return False
         return super(IntervalDtype, cls).is_dtype(dtype)
-
-
-# TODO(Extension): remove the second registry once all internal extension
-# dtypes are real extension dtypes.
-_pandas_registry = Registry()
-
-_pandas_registry.register(DatetimeTZDtype)

pandas/core/dtypes/generic.py

@@ -67,6 +67,8 @@ def _check(cls, inst):
                                            ("extension",
                                             "categorical",
                                             "periodarray",
+                                            "datetimearray",
+                                            "timedeltaarray",
                                             "npy_extension",
                                             ))
 ABCPandasArray = create_pandas_abc_type("ABCPandasArray",