Skip to content

API: Timedelta constructor pytimedelta, Tick preserve reso #48918

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 11 commits into from
Oct 8, 2022
20 changes: 15 additions & 5 deletions pandas/_libs/tslibs/timedeltas.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -341,8 +341,9 @@ cdef convert_to_timedelta64(object ts, str unit):
elif isinstance(ts, _Timedelta):
# already in the proper format
if ts._reso != NPY_FR_ns:
raise NotImplementedError
ts = np.timedelta64(ts.value, "ns")
ts = ts._as_unit("ns").asm8
else:
ts = np.timedelta64(ts.value, "ns")
elif is_timedelta64_object(ts):
ts = ensure_td64ns(ts)
elif is_integer_object(ts):
Expand Down Expand Up @@ -1706,7 +1707,13 @@ class Timedelta(_Timedelta):
value = parse_timedelta_string(value)
value = np.timedelta64(value)
elif PyDelta_Check(value):
value = convert_to_timedelta64(value, 'ns')
# pytimedelta object -> microsecond resolution
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When the non-nano resolution docs are written, might be good to mention that datetime.timedeltas now default to microsecond resolution?

new_value = delta_to_nanoseconds(
value, reso=NPY_DATETIMEUNIT.NPY_FR_us
)
return cls._from_value_and_reso(
new_value, reso=NPY_DATETIMEUNIT.NPY_FR_us
)
elif is_timedelta64_object(value):
# Retain the resolution if possible, otherwise cast to the nearest
# supported resolution.
Expand All @@ -1720,7 +1727,7 @@ class Timedelta(_Timedelta):
if reso != NPY_DATETIMEUNIT.NPY_FR_GENERIC:
try:
new_value = convert_reso(
get_timedelta64_value(value),
new_value,
reso,
new_reso,
round_ok=True,
Expand All @@ -1730,7 +1737,10 @@ class Timedelta(_Timedelta):
return cls._from_value_and_reso(new_value, reso=new_reso)

elif is_tick_object(value):
value = np.timedelta64(value.nanos, 'ns')
new_reso = get_supported_reso(value._reso)
new_value = delta_to_nanoseconds(value, reso=new_reso)
return cls._from_value_and_reso(new_value, reso=new_reso)

elif is_integer_object(value) or is_float_object(value):
# unit=None is de-facto 'ns'
unit = parse_timedelta_unit(unit)
Expand Down
12 changes: 10 additions & 2 deletions pandas/core/arrays/masked.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@
lib,
missing as libmissing,
)
from pandas._libs.tslibs import (
get_unit_from_dtype,
is_supported_unit,
)
from pandas._typing import (
ArrayLike,
AstypeArg,
Expand Down Expand Up @@ -750,12 +754,16 @@ def _maybe_mask_result(self, result, mask):

return BooleanArray(result, mask, copy=False)

elif result.dtype == "timedelta64[ns]":
elif (
isinstance(result.dtype, np.dtype)
and result.dtype.kind == "m"
and is_supported_unit(get_unit_from_dtype(result.dtype))
):
# e.g. test_numeric_arr_mul_tdscalar_numexpr_path
from pandas.core.arrays import TimedeltaArray

if not isinstance(result, TimedeltaArray):
result = TimedeltaArray._simple_new(result)
result = TimedeltaArray._simple_new(result, dtype=result.dtype)

result[mask] = result.dtype.type("NaT")
return result
Expand Down
4 changes: 2 additions & 2 deletions pandas/core/arrays/timedeltas.py
Original file line number Diff line number Diff line change
Expand Up @@ -257,10 +257,10 @@ def _generate_range(cls, start, end, periods, freq, closed=None):
)

if start is not None:
start = Timedelta(start)
start = Timedelta(start)._as_unit("ns")

if end is not None:
end = Timedelta(end)
end = Timedelta(end)._as_unit("ns")

left_closed, right_closed = validate_endpoints(closed)

Expand Down
19 changes: 19 additions & 0 deletions pandas/tests/arithmetic/test_numeric.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from __future__ import annotations

from collections import abc
from datetime import timedelta
from decimal import Decimal
import operator
from typing import Any
Expand All @@ -27,6 +28,7 @@
Int64Index,
UInt64Index,
)
from pandas.core.arrays import TimedeltaArray
from pandas.core.computation import expressions as expr
from pandas.tests.arithmetic.common import (
assert_invalid_addsub_type,
Expand Down Expand Up @@ -209,6 +211,11 @@ def test_numeric_arr_mul_tdscalar(self, scalar_td, numeric_idx, box_with_array):
tda = expected._data
dtype = scalar_td.dtype
expected = type(tda)._simple_new(tda._ndarray.astype(dtype), dtype=dtype)
elif type(scalar_td) is timedelta and box not in [Index, Series]:
# TODO(2.0): once TDA.astype converts to m8, just do expected.astype
tda = expected._data
dtype = np.dtype("m8[us]")
expected = type(tda)._simple_new(tda._ndarray.astype(dtype), dtype=dtype)

index = tm.box_expected(index, box)
expected = tm.box_expected(expected, box)
Expand Down Expand Up @@ -240,6 +247,13 @@ def test_numeric_arr_mul_tdscalar_numexpr_path(
obj = tm.box_expected(arr, box, transpose=False)

expected = arr_i8.view("timedelta64[D]").astype("timedelta64[ns]")
if type(scalar_td) is timedelta and box is array:
# TODO(2.0): this shouldn't depend on 'box'
expected = expected.astype("timedelta64[us]")
# TODO(2.0): won't be necessary to construct TimedeltaArray
# explicitly.
expected = TimedeltaArray._simple_new(expected, dtype=expected.dtype)

expected = tm.box_expected(expected, box, transpose=False)

result = obj * scalar_td
Expand All @@ -262,6 +276,11 @@ def test_numeric_arr_rdiv_tdscalar(self, three_days, numeric_idx, box_with_array
# i.e. resolution is lower -> use lowest supported resolution
dtype = np.dtype("m8[s]")
expected = type(tda)._simple_new(tda._ndarray.astype(dtype), dtype=dtype)
elif type(three_days) is timedelta and box not in [Index, Series]:
# TODO(2.0): just use expected.astype
tda = expected._data
dtype = np.dtype("m8[us]")
expected = type(tda)._simple_new(tda._ndarray.astype(dtype), dtype=dtype)

index = tm.box_expected(index, box)
expected = tm.box_expected(expected, box)
Expand Down
6 changes: 6 additions & 0 deletions pandas/tests/dtypes/cast/test_promote.py
Original file line number Diff line number Diff line change
Expand Up @@ -480,6 +480,12 @@ def test_maybe_promote_any_with_timedelta64(
"Timedelta scalar"
)
request.node.add_marker(mark)
elif type(fill_value) is datetime.timedelta:
mark = pytest.mark.xfail(
reason="maybe_promote not yet updated to handle non-nano "
"Timedelta scalar"
)
request.node.add_marker(mark)
else:
expected_dtype = np.dtype(object)
exp_val_for_scalar = fill_value
Expand Down
8 changes: 4 additions & 4 deletions pandas/tests/io/json/test_pandas.py
Original file line number Diff line number Diff line change
Expand Up @@ -1039,11 +1039,11 @@ def test_timedelta(self):
tm.assert_frame_equal(frame, result)

def test_mixed_timedelta_datetime(self):
frame = DataFrame({"a": [timedelta(23), Timestamp("20130101")]}, dtype=object)
td = timedelta(23)
ts = Timestamp("20130101")
frame = DataFrame({"a": [td, ts]}, dtype=object)

expected = DataFrame(
{"a": [pd.Timedelta(frame.a[0]).value, Timestamp(frame.a[1]).value]}
)
expected = DataFrame({"a": [pd.Timedelta(td)._as_unit("ns").value, ts.value]})
result = read_json(frame.to_json(date_unit="ns"), dtype={"a": "int64"})
tm.assert_frame_equal(result, expected, check_index_type=False)

Expand Down
39 changes: 36 additions & 3 deletions pandas/tests/scalar/timedelta/test_constructors.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,38 @@ def test_from_td64_retain_resolution():
assert td3._reso == NpyDatetimeUnit.NPY_FR_us.value


def test_from_pytimedelta_us_reso():
# pytimedelta has microsecond resolution, so Timedelta(pytd) inherits that
td = timedelta(days=4, minutes=3)
result = Timedelta(td)
assert result.to_pytimedelta() == td
assert result._reso == NpyDatetimeUnit.NPY_FR_us.value


def test_from_tick_reso():
tick = offsets.Nano()
assert Timedelta(tick)._reso == NpyDatetimeUnit.NPY_FR_ns.value

tick = offsets.Micro()
assert Timedelta(tick)._reso == NpyDatetimeUnit.NPY_FR_us.value

tick = offsets.Milli()
assert Timedelta(tick)._reso == NpyDatetimeUnit.NPY_FR_ms.value

tick = offsets.Second()
assert Timedelta(tick)._reso == NpyDatetimeUnit.NPY_FR_s.value

# everything above Second gets cast to the closest supported reso: second
tick = offsets.Minute()
assert Timedelta(tick)._reso == NpyDatetimeUnit.NPY_FR_s.value

tick = offsets.Hour()
assert Timedelta(tick)._reso == NpyDatetimeUnit.NPY_FR_s.value

tick = offsets.Day()
assert Timedelta(tick)._reso == NpyDatetimeUnit.NPY_FR_s.value


def test_construction():
expected = np.timedelta64(10, "D").astype("m8[ns]").view("i8")
assert Timedelta(10, unit="d").value == expected
Expand Down Expand Up @@ -248,9 +280,10 @@ def test_overflow_on_construction():
with pytest.raises(OutOfBoundsTimedelta, match=msg):
Timedelta(7 * 19999, unit="D")

msg = "Cannot cast 259987 days, 0:00:00 to unit=ns without overflow"
with pytest.raises(OutOfBoundsTimedelta, match=msg):
Timedelta(timedelta(days=13 * 19999))
# used to overflow before non-ns support
td = Timedelta(timedelta(days=13 * 19999))
assert td._reso == NpyDatetimeUnit.NPY_FR_us.value
assert td.days == 13 * 19999


@pytest.mark.parametrize(
Expand Down
19 changes: 12 additions & 7 deletions pandas/tests/scalar/timedelta/test_timedelta.py
Original file line number Diff line number Diff line change
Expand Up @@ -183,13 +183,16 @@ def test_truediv_timedeltalike(self, td):
assert (2.5 * td) / td == 2.5

other = Timedelta(td.value)
msg = "Cannot cast 106752 days 00:00:00 to unit='ns' without overflow"
msg = "Cannot cast 106752 days 00:00:00 to unit='ns' without overflow."
with pytest.raises(OutOfBoundsTimedelta, match=msg):
td / other

with pytest.raises(OutOfBoundsTimedelta, match=msg):
# __rtruediv__
other.to_pytimedelta() / td
# Timedelta(other.to_pytimedelta()) has microsecond resolution,
# so the division doesn't require casting all the way to nanos,
# so succeeds
res = other.to_pytimedelta() / td
expected = other.to_pytimedelta() / td.to_pytimedelta()
assert res == expected

# if there's no overflow, we cast to the higher reso
left = Timedelta._from_value_and_reso(50, NpyDatetimeUnit.NPY_FR_us.value)
Expand Down Expand Up @@ -220,9 +223,11 @@ def test_floordiv_timedeltalike(self, td):
with pytest.raises(OutOfBoundsTimedelta, match=msg):
td // other

with pytest.raises(ValueError, match=msg):
# __rfloordiv__
other.to_pytimedelta() // td
# Timedelta(other.to_pytimedelta()) has microsecond resolution,
# so the floordiv doesn't require casting all the way to nanos,
# so succeeds
res = other.to_pytimedelta() // td
assert res == 0

# if there's no overflow, we cast to the higher reso
left = Timedelta._from_value_and_reso(50050, NpyDatetimeUnit.NPY_FR_us.value)
Expand Down