ENH: DTA to_pydatetime, time, timetz, date, iter support non-nano

Author: jbrockmendel

pandas/_libs/tslibs/vectorized.pyi

@@ -35,6 +35,7 @@ def ints_to_pydatetime(
     freq: BaseOffset | None = ...,
     fold: bool = ...,
     box: str = ...,
+    reso: int = ...,  # NPY_DATETIMEUNIT
 ) -> npt.NDArray[np.object_]: ...
 def tz_convert_from_utc(
     stamps: npt.NDArray[np.int64],

pandas/_libs/tslibs/vectorized.pyx

@@ -33,6 +33,7 @@ from .np_datetime cimport (
     NPY_FR_ns,
     dt64_to_dtstruct,
     npy_datetimestruct,
+    pandas_datetime_to_datetimestruct,
 )
 from .offsets cimport BaseOffset
 from .period cimport get_period_ordinal
@@ -99,7 +100,8 @@ def ints_to_pydatetime(
     tzinfo tz=None,
     BaseOffset freq=None,
     bint fold=False,
-    str box="datetime"
+    str box="datetime",
+    NPY_DATETIMEUNIT reso=NPY_FR_ns,
 ) -> np.ndarray:
     # stamps is int64, arbitrary ndim
     """
@@ -125,12 +127,14 @@ def ints_to_pydatetime(
         * If time, convert to datetime.time
         * If Timestamp, convert to pandas.Timestamp
 
+    reso : NPY_DATETIMEUNIT, default NPY_FR_ns
+
     Returns
     -------
     ndarray[object] of type specified by box
     """
     cdef:
-        Localizer info = Localizer(tz, reso=NPY_FR_ns)
+        Localizer info = Localizer(tz, reso=reso)
         int64_t utc_val, local_val
         Py_ssize_t i, n = stamps.size
         Py_ssize_t pos = -1  # unused, avoid not-initialized warning
@@ -178,10 +182,12 @@ def ints_to_pydatetime(
                 # find right representation of dst etc in pytz timezone
                 new_tz = tz._tzinfos[tz._transition_info[pos]]
 
-            dt64_to_dtstruct(local_val, &dts)
+            pandas_datetime_to_datetimestruct(local_val, reso, &dts)
 
             if use_ts:
-                res_val = create_timestamp_from_ts(utc_val, dts, new_tz, freq, fold)
+                res_val = create_timestamp_from_ts(
+                    utc_val, dts, new_tz, freq, fold, reso=reso
+                )
             elif use_pydt:
                 res_val = datetime(
                     dts.year, dts.month, dts.day, dts.hour, dts.min, dts.sec, dts.us,

pandas/core/arrays/datetimelike.py

@@ -426,15 +426,16 @@ def astype(self, dtype, copy: bool = True):
             if self.dtype.kind == "M":
                 # *much* faster than self._box_values
                 #  for e.g. test_get_loc_tuple_monotonic_above_size_cutoff
-                i8data = self.asi8.ravel()
+                i8data = self.asi8
                 converted = ints_to_pydatetime(
                     i8data,
                     # error: "DatetimeLikeArrayMixin" has no attribute "tz"
                     tz=self.tz,  # type: ignore[attr-defined]
                     freq=self.freq,
                     box="timestamp",
+                    reso=self._reso,
                 )
-                return converted.reshape(self.shape)
+                return converted
 
             elif self.dtype.kind == "m":
                 return ints_to_pytimedelta(self._ndarray, box=True)

pandas/core/arrays/datetimes.py

@@ -650,7 +650,11 @@ def __iter__(self):
                 start_i = i * chunksize
                 end_i = min((i + 1) * chunksize, length)
                 converted = ints_to_pydatetime(
-                    data[start_i:end_i], tz=self.tz, freq=self.freq, box="timestamp"
+                    data[start_i:end_i],
+                    tz=self.tz,
+                    freq=self.freq,
+                    box="timestamp",
+                    reso=self._reso,
                 )
                 yield from converted
 
@@ -1041,7 +1045,7 @@ def to_pydatetime(self) -> npt.NDArray[np.object_]:
         -------
         datetimes : ndarray[object]
         """
-        return ints_to_pydatetime(self.asi8, tz=self.tz)
+        return ints_to_pydatetime(self.asi8, tz=self.tz, reso=self._reso)
 
     def normalize(self) -> DatetimeArray:
         """
@@ -1295,7 +1299,7 @@ def time(self) -> npt.NDArray[np.object_]:
         # keeping their timezone and not using UTC
         timestamps = self._local_timestamps()
 
-        return ints_to_pydatetime(timestamps, box="time")
+        return ints_to_pydatetime(timestamps, box="time", reso=self._reso)
 
     @property
     def timetz(self) -> npt.NDArray[np.object_]:
@@ -1305,7 +1309,7 @@ def timetz(self) -> npt.NDArray[np.object_]:
 
         The time part of the Timestamps.
         """
-        return ints_to_pydatetime(self.asi8, self.tz, box="time")
+        return ints_to_pydatetime(self.asi8, self.tz, box="time", reso=self._reso)
 
     @property
     def date(self) -> npt.NDArray[np.object_]:
@@ -1320,7 +1324,7 @@ def date(self) -> npt.NDArray[np.object_]:
         # keeping their timezone and not using UTC
         timestamps = self._local_timestamps()
 
-        return ints_to_pydatetime(timestamps, box="date")
+        return ints_to_pydatetime(timestamps, box="date", reso=self._reso)
 
     def isocalendar(self) -> DataFrame:
         """

pandas/tests/arrays/test_datetimes.py

@@ -37,6 +37,26 @@ def dtype(self, unit, tz_naive_fixture):
         else:
             return DatetimeTZDtype(unit=unit, tz=tz)
 
+    @pytest.fixture
+    def dta_dti(self, unit, dtype):
+        tz = getattr(dtype, "tz", None)
+
+        dti = pd.date_range("2016-01-01", periods=55, freq="D", tz=tz)
+        if tz is None:
+            arr = np.asarray(dti).astype(f"M8[{unit}]")
+        else:
+            arr = np.asarray(dti.tz_convert("UTC").tz_localize(None)).astype(
+                f"M8[{unit}]"
+            )
+
+        dta = DatetimeArray._simple_new(arr, dtype=dtype)
+        return dta, dti
+
+    @pytest.fixture
+    def dta(self, dta_dti):
+        dta, dti = dta_dti
+        return dta
+
     def test_non_nano(self, unit, reso, dtype):
         arr = np.arange(5, dtype=np.int64).view(f"M8[{unit}]")
         dta = DatetimeArray._simple_new(arr, dtype=dtype)
@@ -52,17 +72,8 @@ def test_non_nano(self, unit, reso, dtype):
     @pytest.mark.parametrize(
         "field", DatetimeArray._field_ops + DatetimeArray._bool_ops
     )
-    def test_fields(self, unit, reso, field, dtype):
-        tz = getattr(dtype, "tz", None)
-        dti = pd.date_range("2016-01-01", periods=55, freq="D", tz=tz)
-        if tz is None:
-            arr = np.asarray(dti).astype(f"M8[{unit}]")
-        else:
-            arr = np.asarray(dti.tz_convert("UTC").tz_localize(None)).astype(
-                f"M8[{unit}]"
-            )
-
-        dta = DatetimeArray._simple_new(arr, dtype=dtype)
+    def test_fields(self, unit, reso, field, dtype, dta_dti):
+        dta, dti = dta_dti
 
         # FIXME: assert (dti == dta).all()
 
@@ -85,6 +96,35 @@ def test_normalize(self, unit):
         res = dta.normalize()
         tm.assert_extension_array_equal(res, expected)
 
+    def test_iter(self, dta):
+        res = next(iter(dta))
+        expected = dta[0]
+
+        assert type(res) is pd.Timestamp
+        assert res.value == expected.value
+        assert res._reso == expected._reso
+        assert res == expected
+
+    def test_astype_object(self, dta):
+        result = dta.astype(object)
+        assert all(x._reso == dta._reso for x in result)
+        assert all(x == y for x, y in zip(result, dta))
+
+    def test_to_pydatetime(self, dta_dti):
+        dta, dti = dta_dti
+
+        result = dta.to_pydatetime()
+        expected = dti.to_pydatetime()
+        tm.assert_numpy_array_equal(result, expected)
+
+    @pytest.mark.parametrize("meth", ["time", "timetz", "date"])
+    def test_time_date(self, dta_dti, meth):
+        dta, dti = dta_dti
+
+        result = getattr(dta, meth)
+        expected = getattr(dti, meth)
+        tm.assert_numpy_array_equal(result, expected)
+
 
 class TestDatetimeArrayComparisons:
     # TODO: merge this into tests/arithmetic/test_datetime64 once it is