Skip to content

ENH: DTA to_pydatetime, time, timetz, date, iter support non-nano #47307

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 10 commits into from
Jun 15, 2022
2 changes: 1 addition & 1 deletion pandas/_libs/tslibs/offsets.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -3115,7 +3115,7 @@ cdef class FY5253Quarter(FY5253Mixin):
for qlen in qtr_lens:
if qlen * 7 <= tdelta.days:
num_qtrs += 1
tdelta -= Timedelta(days=qlen * 7)
tdelta -= (<_Timedelta>Timedelta(days=qlen * 7))._as_reso(norm._reso)
else:
break
else:
Expand Down
1 change: 1 addition & 0 deletions pandas/_libs/tslibs/vectorized.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ def ints_to_pydatetime(
freq: BaseOffset | None = ...,
fold: bool = ...,
box: str = ...,
reso: int = ..., # NPY_DATETIMEUNIT
) -> npt.NDArray[np.object_]: ...
def tz_convert_from_utc(
stamps: npt.NDArray[np.int64],
Expand Down
13 changes: 9 additions & 4 deletions pandas/_libs/tslibs/vectorized.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,8 @@ def ints_to_pydatetime(
tzinfo tz=None,
BaseOffset freq=None,
bint fold=False,
str box="datetime"
str box="datetime",
NPY_DATETIMEUNIT reso=NPY_FR_ns,
) -> np.ndarray:
# stamps is int64, arbitrary ndim
"""
Expand All @@ -126,12 +127,14 @@ def ints_to_pydatetime(
* If time, convert to datetime.time
* If Timestamp, convert to pandas.Timestamp

reso : NPY_DATETIMEUNIT, default NPY_FR_ns

Returns
-------
ndarray[object] of type specified by box
"""
cdef:
Localizer info = Localizer(tz, reso=NPY_FR_ns)
Localizer info = Localizer(tz, reso=reso)
int64_t utc_val, local_val
Py_ssize_t i, n = stamps.size
Py_ssize_t pos = -1 # unused, avoid not-initialized warning
Expand Down Expand Up @@ -179,10 +182,12 @@ def ints_to_pydatetime(
# find right representation of dst etc in pytz timezone
new_tz = tz._tzinfos[tz._transition_info[pos]]

dt64_to_dtstruct(local_val, &dts)
pandas_datetime_to_datetimestruct(local_val, reso, &dts)

if use_ts:
res_val = create_timestamp_from_ts(utc_val, dts, new_tz, freq, fold)
res_val = create_timestamp_from_ts(
utc_val, dts, new_tz, freq, fold, reso=reso
)
elif use_pydt:
res_val = datetime(
dts.year, dts.month, dts.day, dts.hour, dts.min, dts.sec, dts.us,
Expand Down
9 changes: 5 additions & 4 deletions pandas/core/arrays/datetimelike.py
Original file line number Diff line number Diff line change
Expand Up @@ -424,17 +424,18 @@ def astype(self, dtype, copy: bool = True):

if is_object_dtype(dtype):
if self.dtype.kind == "M":
self = cast("DatetimeArray", self)
# *much* faster than self._box_values
# for e.g. test_get_loc_tuple_monotonic_above_size_cutoff
i8data = self.asi8.ravel()
i8data = self.asi8
converted = ints_to_pydatetime(
i8data,
# error: "DatetimeLikeArrayMixin" has no attribute "tz"
tz=self.tz, # type: ignore[attr-defined]
tz=self.tz,
freq=self.freq,
box="timestamp",
reso=self._reso,
)
return converted.reshape(self.shape)
return converted

elif self.dtype.kind == "m":
return ints_to_pytimedelta(self._ndarray, box=True)
Expand Down
14 changes: 9 additions & 5 deletions pandas/core/arrays/datetimes.py
Original file line number Diff line number Diff line change
Expand Up @@ -653,7 +653,11 @@ def __iter__(self):
start_i = i * chunksize
end_i = min((i + 1) * chunksize, length)
converted = ints_to_pydatetime(
data[start_i:end_i], tz=self.tz, freq=self.freq, box="timestamp"
data[start_i:end_i],
tz=self.tz,
freq=self.freq,
box="timestamp",
reso=self._reso,
)
yield from converted

Expand Down Expand Up @@ -1044,7 +1048,7 @@ def to_pydatetime(self) -> npt.NDArray[np.object_]:
-------
datetimes : ndarray[object]
"""
return ints_to_pydatetime(self.asi8, tz=self.tz)
return ints_to_pydatetime(self.asi8, tz=self.tz, reso=self._reso)

def normalize(self) -> DatetimeArray:
"""
Expand Down Expand Up @@ -1301,7 +1305,7 @@ def time(self) -> npt.NDArray[np.object_]:
# keeping their timezone and not using UTC
timestamps = self._local_timestamps()

return ints_to_pydatetime(timestamps, box="time")
return ints_to_pydatetime(timestamps, box="time", reso=self._reso)

@property
def timetz(self) -> npt.NDArray[np.object_]:
Expand All @@ -1311,7 +1315,7 @@ def timetz(self) -> npt.NDArray[np.object_]:

The time part of the Timestamps.
"""
return ints_to_pydatetime(self.asi8, self.tz, box="time")
return ints_to_pydatetime(self.asi8, self.tz, box="time", reso=self._reso)

@property
def date(self) -> npt.NDArray[np.object_]:
Expand All @@ -1326,7 +1330,7 @@ def date(self) -> npt.NDArray[np.object_]:
# keeping their timezone and not using UTC
timestamps = self._local_timestamps()

return ints_to_pydatetime(timestamps, box="date")
return ints_to_pydatetime(timestamps, box="date", reso=self._reso)

def isocalendar(self) -> DataFrame:
"""
Expand Down
29 changes: 29 additions & 0 deletions pandas/tests/arrays/test_datetimes.py
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,35 @@ def test_to_period(self, dta_dti):

tm.assert_extension_array_equal(result, expected)

def test_iter(self, dta):
res = next(iter(dta))
expected = dta[0]

assert type(res) is pd.Timestamp
assert res.value == expected.value
assert res._reso == expected._reso
assert res == expected

def test_astype_object(self, dta):
result = dta.astype(object)
assert all(x._reso == dta._reso for x in result)
assert all(x == y for x, y in zip(result, dta))

def test_to_pydatetime(self, dta_dti):
dta, dti = dta_dti

result = dta.to_pydatetime()
expected = dti.to_pydatetime()
tm.assert_numpy_array_equal(result, expected)

@pytest.mark.parametrize("meth", ["time", "timetz", "date"])
def test_time_date(self, dta_dti, meth):
dta, dti = dta_dti

result = getattr(dta, meth)
expected = getattr(dti, meth)
tm.assert_numpy_array_equal(result, expected)


class TestDatetimeArrayComparisons:
# TODO: merge this into tests/arithmetic/test_datetime64 once it is
Expand Down