Merge branch 'main' into patch-2

Author: jonashaag

pandas/_libs/tslibs/dtypes.pyi

@@ -56,3 +56,19 @@ class Resolution(Enum):
     def get_reso_from_freqstr(cls, freq: str) -> Resolution: ...
     @property
     def attr_abbrev(self) -> str: ...
+
+class NpyDatetimeUnit(Enum):
+    NPY_FR_Y: int
+    NPY_FR_M: int
+    NPY_FR_W: int
+    NPY_FR_D: int
+    NPY_FR_h: int
+    NPY_FR_m: int
+    NPY_FR_s: int
+    NPY_FR_ms: int
+    NPY_FR_us: int
+    NPY_FR_ns: int
+    NPY_FR_ps: int
+    NPY_FR_fs: int
+    NPY_FR_as: int
+    NPY_FR_GENERIC: int

pandas/_libs/tslibs/dtypes.pyx

@@ -257,6 +257,26 @@ class Resolution(Enum):
         return cls.from_attrname(attr_name)
 
 
+class NpyDatetimeUnit(Enum):
+    """
+    Python-space analogue to NPY_DATETIMEUNIT.
+    """
+    NPY_FR_Y = NPY_DATETIMEUNIT.NPY_FR_Y
+    NPY_FR_M = NPY_DATETIMEUNIT.NPY_FR_M
+    NPY_FR_W = NPY_DATETIMEUNIT.NPY_FR_W
+    NPY_FR_D = NPY_DATETIMEUNIT.NPY_FR_D
+    NPY_FR_h = NPY_DATETIMEUNIT.NPY_FR_h
+    NPY_FR_m = NPY_DATETIMEUNIT.NPY_FR_m
+    NPY_FR_s = NPY_DATETIMEUNIT.NPY_FR_s
+    NPY_FR_ms = NPY_DATETIMEUNIT.NPY_FR_ms
+    NPY_FR_us = NPY_DATETIMEUNIT.NPY_FR_us
+    NPY_FR_ns = NPY_DATETIMEUNIT.NPY_FR_ns
+    NPY_FR_ps = NPY_DATETIMEUNIT.NPY_FR_ps
+    NPY_FR_fs = NPY_DATETIMEUNIT.NPY_FR_fs
+    NPY_FR_as = NPY_DATETIMEUNIT.NPY_FR_as
+    NPY_FR_GENERIC = NPY_DATETIMEUNIT.NPY_FR_GENERIC
+
+
 cdef str npy_unit_to_abbrev(NPY_DATETIMEUNIT unit):
     if unit == NPY_DATETIMEUNIT.NPY_FR_ns or unit == NPY_DATETIMEUNIT.NPY_FR_GENERIC:
         # generic -> default to nanoseconds

pandas/_libs/tslibs/offsets.pyi

@@ -108,6 +108,7 @@ def to_offset(freq: None) -> None: ...
 def to_offset(freq: timedelta | BaseOffset | str) -> BaseOffset: ...
 
 class Tick(SingleConstructorOffset):
+    _reso: int
     def __init__(self, n: int = ..., normalize: bool = ...) -> None: ...
     @property
     def delta(self) -> Timedelta: ...

pandas/_libs/tslibs/offsets.pyx

@@ -968,42 +968,49 @@ cdef class Day(Tick):
     _nanos_inc = 24 * 3600 * 1_000_000_000
     _prefix = "D"
     _period_dtype_code = PeriodDtypeCode.D
+    _reso = NPY_DATETIMEUNIT.NPY_FR_D
 
 
 cdef class Hour(Tick):
     _nanos_inc = 3600 * 1_000_000_000
     _prefix = "H"
     _period_dtype_code = PeriodDtypeCode.H
+    _reso = NPY_DATETIMEUNIT.NPY_FR_h
 
 
 cdef class Minute(Tick):
     _nanos_inc = 60 * 1_000_000_000
     _prefix = "T"
     _period_dtype_code = PeriodDtypeCode.T
+    _reso = NPY_DATETIMEUNIT.NPY_FR_m
 
 
 cdef class Second(Tick):
     _nanos_inc = 1_000_000_000
     _prefix = "S"
     _period_dtype_code = PeriodDtypeCode.S
+    _reso = NPY_DATETIMEUNIT.NPY_FR_s
 
 
 cdef class Milli(Tick):
     _nanos_inc = 1_000_000
     _prefix = "L"
     _period_dtype_code = PeriodDtypeCode.L
+    _reso = NPY_DATETIMEUNIT.NPY_FR_ms
 
 
 cdef class Micro(Tick):
     _nanos_inc = 1000
     _prefix = "U"
     _period_dtype_code = PeriodDtypeCode.U
+    _reso = NPY_DATETIMEUNIT.NPY_FR_us
 
 
 cdef class Nano(Tick):
     _nanos_inc = 1
     _prefix = "N"
     _period_dtype_code = PeriodDtypeCode.N
+    _reso = NPY_DATETIMEUNIT.NPY_FR_ns
 
 
 def delta_to_tick(delta: timedelta) -> Tick:

pandas/_libs/tslibs/period.pyx

@@ -1680,14 +1680,17 @@ cdef class _Period(PeriodMixin):
 
     def _add_timedeltalike_scalar(self, other) -> "Period":
         cdef:
-            int64_t nanos, base_nanos
+            int64_t inc
 
         if is_tick_object(self.freq):
-            nanos = delta_to_nanoseconds(other)
-            base_nanos = self.freq.base.nanos
-            if nanos % base_nanos == 0:
-                ordinal = self.ordinal + (nanos // base_nanos)
-                return Period(ordinal=ordinal, freq=self.freq)
+            try:
+                inc = delta_to_nanoseconds(other, reso=self.freq._reso, round_ok=False)
+            except ValueError as err:
+                raise IncompatibleFrequency("Input cannot be converted to "
+                                            f"Period(freq={self.freqstr})") from err
+            # TODO: overflow-check here
+            ordinal = self.ordinal + inc
+            return Period(ordinal=ordinal, freq=self.freq)
         raise IncompatibleFrequency("Input cannot be converted to "
                                     f"Period(freq={self.freqstr})")
 
@@ -2253,7 +2256,7 @@ cdef class _Period(PeriodMixin):
     @property
     def daysinmonth(self) -> int:
         """
-        Get the total number of days of the month that the Period falls in.
+        Get the total number of days of the month that this period falls on.
 
         Returns
         -------

pandas/_libs/tslibs/timedeltas.pxd

@@ -5,7 +5,9 @@ from .np_datetime cimport NPY_DATETIMEUNIT
 
 
 # Exposed for tslib, not intended for outside use.
-cpdef int64_t delta_to_nanoseconds(delta) except? -1
+cpdef int64_t delta_to_nanoseconds(
+    delta, NPY_DATETIMEUNIT reso=*, bint round_ok=*, bint allow_year_month=*
+) except? -1
 cdef convert_to_timedelta64(object ts, str unit)
 cdef bint is_any_td_scalar(object obj)
 

pandas/_libs/tslibs/timedeltas.pyi

@@ -72,7 +72,12 @@ def array_to_timedelta64(
     errors: str = ...,
 ) -> np.ndarray: ...  # np.ndarray[m8ns]
 def parse_timedelta_unit(unit: str | None) -> UnitChoices: ...
-def delta_to_nanoseconds(delta: np.timedelta64 | timedelta | Tick) -> int: ...
+def delta_to_nanoseconds(
+    delta: np.timedelta64 | timedelta | Tick,
+    reso: int = ...,  # NPY_DATETIMEUNIT
+    round_ok: bool = ...,
+    allow_year_month: bool = ...,
+) -> int: ...
 
 class Timedelta(timedelta):
     min: ClassVar[Timedelta]

pandas/_libs/tslibs/timedeltas.pyx

@@ -201,28 +201,76 @@ def ints_to_pytimedelta(ndarray m8values, box=False):
 
 # ----------------------------------------------------------------------
 
-cpdef int64_t delta_to_nanoseconds(delta) except? -1:
-    if is_tick_object(delta):
-        return delta.nanos
-    if isinstance(delta, _Timedelta):
-        if delta._reso == NPY_FR_ns:
-            return delta.value
-        raise NotImplementedError(delta._reso)
 
-    if is_timedelta64_object(delta):
-        return get_timedelta64_value(ensure_td64ns(delta))
+cpdef int64_t delta_to_nanoseconds(
+    delta,
+    NPY_DATETIMEUNIT reso=NPY_FR_ns,
+    bint round_ok=True,
+    bint allow_year_month=False,
+) except? -1:
+    cdef:
+        _Timedelta td
+        NPY_DATETIMEUNIT in_reso
+        int64_t n
+
+    if is_tick_object(delta):
+        n = delta.n
+        in_reso = delta._reso
+        if in_reso == reso:
+            return n
+        else:
+            td = Timedelta._from_value_and_reso(delta.n, reso=in_reso)
+
+    elif isinstance(delta, _Timedelta):
+        td = delta
+        n = delta.value
+        in_reso = delta._reso
+        if in_reso == reso:
+            return n
+
+    elif is_timedelta64_object(delta):
+        in_reso = get_datetime64_unit(delta)
+        n = get_timedelta64_value(delta)
+        if in_reso == reso:
+            return n
+        else:
+            # _from_value_and_reso does not support Year, Month, or unit-less,
+            #  so we have special handling if speciifed
+            try:
+                td = Timedelta._from_value_and_reso(n, reso=in_reso)
+            except NotImplementedError:
+                if allow_year_month:
+                    td64 = ensure_td64ns(delta)
+                    return delta_to_nanoseconds(td64, reso=reso)
+                else:
+                    raise
 
-    if PyDelta_Check(delta):
+    elif PyDelta_Check(delta):
+        in_reso = NPY_DATETIMEUNIT.NPY_FR_us
         try:
-            return (
+            n = (
                 delta.days * 24 * 3600 * 1_000_000
                 + delta.seconds * 1_000_000
                 + delta.microseconds
-            ) * 1000
+                )
         except OverflowError as err:
             raise OutOfBoundsTimedelta(*err.args) from err
 
-    raise TypeError(type(delta))
+        if in_reso == reso:
+            return n
+        else:
+            td = Timedelta._from_value_and_reso(n, reso=in_reso)
+
+    else:
+        raise TypeError(type(delta))
+
+    try:
+        return td._as_reso(reso, round_ok=round_ok).value
+    except OverflowError as err:
+        unit_str = npy_unit_to_abbrev(reso)
+        raise OutOfBoundsTimedelta(
+            f"Cannot cast {str(delta)} to unit={unit_str} without overflow."
+        ) from err
 
 
 @cython.overflowcheck(True)
@@ -1411,6 +1459,7 @@ cdef class _Timedelta(timedelta):
         else:
             mult = get_conversion_factor(self._reso, reso)
             with cython.overflowcheck(True):
+                # Note: caller is responsible for re-raising as OutOfBoundsTimedelta
                 value = self.value * mult
         return type(self)._from_value_and_reso(value, reso=reso)
 

pandas/_libs/tslibs/timestamps.pyx

@@ -353,7 +353,17 @@ cdef class _Timestamp(ABCTimestamp):
             raise NotImplementedError(self._reso)
 
         if is_any_td_scalar(other):
-            nanos = delta_to_nanoseconds(other)
+            if (
+                is_timedelta64_object(other)
+                and get_datetime64_unit(other) == NPY_DATETIMEUNIT.NPY_FR_GENERIC
+            ):
+                # TODO: deprecate allowing this?  We only get here
+                #  with test_timedelta_add_timestamp_interval
+                other = np.timedelta64(other.view("i8"), "ns")
+            # TODO: disallow round_ok, allow_year_month?
+            nanos = delta_to_nanoseconds(
+                other, reso=self._reso, round_ok=True, allow_year_month=True
+            )
             try:
                 result = type(self)(self.value + nanos, tz=self.tzinfo)
             except OverflowError:

pandas/core/arrays/datetimelike.py

@@ -1120,28 +1120,9 @@ def _add_timedeltalike_scalar(self, other):
             new_values.fill(iNaT)
             return type(self)(new_values, dtype=self.dtype)
 
-        # FIXME: this may overflow with non-nano
-        inc = delta_to_nanoseconds(other)
-
-        if not is_period_dtype(self.dtype):
-            # FIXME: don't hardcode 7, 8, 9, 10 here
-            # TODO: maybe patch delta_to_nanoseconds to take reso?
-
-            # error: "DatetimeLikeArrayMixin" has no attribute "_reso"
-            reso = self._reso  # type: ignore[attr-defined]
-            if reso == 10:
-                pass
-            elif reso == 9:
-                # microsecond
-                inc = inc // 1000
-            elif reso == 8:
-                # millisecond
-                inc = inc // 1_000_000
-            elif reso == 7:
-                # second
-                inc = inc // 1_000_000_000
-            else:
-                raise NotImplementedError(reso)
+        # PeriodArray overrides, so we only get here with DTA/TDA
+        # error: "DatetimeLikeArrayMixin" has no attribute "_reso"
+        inc = delta_to_nanoseconds(other, reso=self._reso)  # type: ignore[attr-defined]
 
         new_values = checked_add_with_arr(self.asi8, inc, arr_mask=self._isnan)
         new_values = new_values.view("i8")

pandas/core/arrays/period.py

@@ -782,6 +782,7 @@ def _add_offset(self, other: BaseOffset):
         self._require_matching_freq(other, base=True)
         return self._addsub_int_array_or_scalar(other.n, operator.add)
 
+    # TODO: can we de-duplicate with Period._add_timedeltalike_scalar?
     def _add_timedeltalike_scalar(self, other):
         """
         Parameters
@@ -797,10 +798,15 @@ def _add_timedeltalike_scalar(self, other):
             raise raise_on_incompatible(self, other)
 
         if notna(other):
-            # special handling for np.timedelta64("NaT"), avoid calling
-            #  _check_timedeltalike_freq_compat as that would raise TypeError
-            other = self._check_timedeltalike_freq_compat(other)
-            other = np.timedelta64(other, "ns")
+            # Convert to an integer increment of our own freq, disallowing
+            #  e.g. 30seconds if our freq is minutes.
+            try:
+                inc = delta_to_nanoseconds(other, reso=self.freq._reso, round_ok=False)
+            except ValueError as err:
+                # "Cannot losslessly convert units"
+                raise raise_on_incompatible(self, other) from err
+
+            return self._addsub_int_array_or_scalar(inc, operator.add)
 
         return super()._add_timedeltalike_scalar(other)
 

pandas/core/dtypes/dtypes.py

@@ -670,14 +670,17 @@ class DatetimeTZDtype(PandasExtensionDtype):
 
     type: type[Timestamp] = Timestamp
     kind: str_type = "M"
-    str = "|M8[ns]"
     num = 101
-    base = np.dtype("M8[ns]")
+    base = np.dtype("M8[ns]")  # TODO: depend on reso?
     na_value = NaT
     _metadata = ("unit", "tz")
     _match = re.compile(r"(datetime64|M8)\[(?P<unit>.+), (?P<tz>.+)\]")
     _cache_dtypes: dict[str_type, PandasExtensionDtype] = {}
 
+    @cache_readonly
+    def str(self):
+        return f"|M8[{self._unit}]"
+
     def __init__(self, unit: str_type | DatetimeTZDtype = "ns", tz=None) -> None:
         if isinstance(unit, DatetimeTZDtype):
             # error: "str" has no attribute "tz"
@@ -696,8 +699,8 @@ def __init__(self, unit: str_type | DatetimeTZDtype = "ns", tz=None) -> None:
                     "'DatetimeTZDtype.construct_from_string()' instead."
                 )
                 raise ValueError(msg)
-            else:
-                raise ValueError("DatetimeTZDtype only supports ns units")
+            if unit not in ["s", "ms", "us", "ns"]:
+                raise ValueError("DatetimeTZDtype only supports s, ms, us, ns units")
 
         if tz:
             tz = timezones.maybe_get_tz(tz)
@@ -710,6 +713,19 @@ def __init__(self, unit: str_type | DatetimeTZDtype = "ns", tz=None) -> None:
         self._unit = unit
         self._tz = tz
 
+    @cache_readonly
+    def _reso(self) -> int:
+        """
+        The NPY_DATETIMEUNIT corresponding to this dtype's resolution.
+        """
+        reso = {
+            "s": dtypes.NpyDatetimeUnit.NPY_FR_s,
+            "ms": dtypes.NpyDatetimeUnit.NPY_FR_ms,
+            "us": dtypes.NpyDatetimeUnit.NPY_FR_us,
+            "ns": dtypes.NpyDatetimeUnit.NPY_FR_ns,
+        }[self._unit]
+        return reso.value
+
     @property
     def unit(self) -> str_type:
         """