Skip to content

API: Change Timestamp/Timedelta arithmetic to match numpy #48743

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Sep 27, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 6 additions & 13 deletions pandas/_libs/tslibs/timedeltas.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -787,19 +787,12 @@ def _binary_op_method_timedeltalike(op, name):
# e.g. if original other was timedelta64('NaT')
return NaT

# We allow silent casting to the lower resolution if and only
# if it is lossless.
try:
if self._reso < other._reso:
other = (<_Timedelta>other)._as_reso(self._reso, round_ok=False)
elif self._reso > other._reso:
self = (<_Timedelta>self)._as_reso(other._reso, round_ok=False)
except ValueError as err:
raise ValueError(
"Timedelta addition/subtraction with mismatched resolutions is not "
"allowed when casting to the lower resolution would require "
"lossy rounding."
) from err
# Matching numpy, we cast to the higher resolution. Unlike numpy,
# we raise instead of silently overflowing during this casting.
if self._reso < other._reso:
self = (<_Timedelta>self)._as_reso(other._reso, round_ok=True)
elif self._reso > other._reso:
other = (<_Timedelta>other)._as_reso(self._reso, round_ok=True)

res = op(self.value, other.value)
if res == NPY_NAT:
Expand Down
56 changes: 23 additions & 33 deletions pandas/_libs/tslibs/timestamps.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -433,43 +433,40 @@ cdef class _Timestamp(ABCTimestamp):
# TODO: deprecate allowing this? We only get here
# with test_timedelta_add_timestamp_interval
other = np.timedelta64(other.view("i8"), "ns")
other_reso = NPY_DATETIMEUNIT.NPY_FR_ns
elif (
other_reso == NPY_DATETIMEUNIT.NPY_FR_Y or other_reso == NPY_DATETIMEUNIT.NPY_FR_M
):
# TODO: deprecate allowing these? or handle more like the
# corresponding DateOffsets?
# TODO: no tests get here
other = ensure_td64ns(other)
other_reso = NPY_DATETIMEUNIT.NPY_FR_ns

if other_reso > NPY_DATETIMEUNIT.NPY_FR_ns:
# TODO: no tests
other = ensure_td64ns(other)
if other_reso > self._reso:
# Following numpy, we cast to the higher resolution
# test_sub_timedelta64_mismatched_reso
self = (<_Timestamp>self)._as_reso(other_reso)


if isinstance(other, _Timedelta):
# TODO: share this with __sub__, Timedelta.__add__
# We allow silent casting to the lower resolution if and only
# if it is lossless. See also Timestamp.__sub__
# and Timedelta.__add__
try:
if self._reso < other._reso:
other = (<_Timedelta>other)._as_reso(self._reso, round_ok=False)
elif self._reso > other._reso:
self = (<_Timestamp>self)._as_reso(other._reso, round_ok=False)
except ValueError as err:
raise ValueError(
"Timestamp addition with mismatched resolutions is not "
"allowed when casting to the lower resolution would require "
"lossy rounding."
) from err
# Matching numpy, we cast to the higher resolution. Unlike numpy,
# we raise instead of silently overflowing during this casting.
if self._reso < other._reso:
self = (<_Timestamp>self)._as_reso(other._reso, round_ok=True)
elif self._reso > other._reso:
other = (<_Timedelta>other)._as_reso(self._reso, round_ok=True)

try:
nanos = delta_to_nanoseconds(
other, reso=self._reso, round_ok=False
)
except OutOfBoundsTimedelta:
raise
except ValueError as err:
raise ValueError(
"Addition between Timestamp and Timedelta with mismatched "
"resolutions is not allowed when casting to the lower "
"resolution would require lossy rounding."
) from err

try:
new_value = self.value + nanos
Expand Down Expand Up @@ -556,19 +553,12 @@ cdef class _Timestamp(ABCTimestamp):
"Cannot subtract tz-naive and tz-aware datetime-like objects."
)

# We allow silent casting to the lower resolution if and only
# if it is lossless.
try:
if self._reso < other._reso:
other = (<_Timestamp>other)._as_reso(self._reso, round_ok=False)
elif self._reso > other._reso:
self = (<_Timestamp>self)._as_reso(other._reso, round_ok=False)
except ValueError as err:
raise ValueError(
"Timestamp subtraction with mismatched resolutions is not "
"allowed when casting to the lower resolution would require "
"lossy rounding."
) from err
# Matching numpy, we cast to the higher resolution. Unlike numpy,
# we raise instead of silently overflowing during this casting.
if self._reso < other._reso:
self = (<_Timestamp>self)._as_reso(other._reso, round_ok=False)
elif self._reso > other._reso:
other = (<_Timestamp>other)._as_reso(self._reso, round_ok=False)

# scalar Timestamp/datetime - Timestamp/datetime -> yields a
# Timedelta
Expand Down
31 changes: 15 additions & 16 deletions pandas/tests/scalar/timedelta/test_timedelta.py
Original file line number Diff line number Diff line change
Expand Up @@ -237,38 +237,37 @@ def test_floordiv_numeric(self, td):
assert res._reso == td._reso

def test_addsub_mismatched_reso(self, td):
other = Timedelta(days=1) # can losslessly convert to other resos
# need to cast to since td is out of bounds for ns, so
# so we would raise OverflowError without casting
other = Timedelta(days=1)._as_unit("us")

# td is out of bounds for ns
result = td + other
assert result._reso == td._reso
assert result._reso == other._reso
assert result.days == td.days + 1

result = other + td
assert result._reso == td._reso
assert result._reso == other._reso
assert result.days == td.days + 1

result = td - other
assert result._reso == td._reso
assert result._reso == other._reso
assert result.days == td.days - 1

result = other - td
assert result._reso == td._reso
assert result._reso == other._reso
assert result.days == 1 - td.days

other2 = Timedelta(500) # can't cast losslessly

msg = (
"Timedelta addition/subtraction with mismatched resolutions is "
"not allowed when casting to the lower resolution would require "
"lossy rounding"
)
with pytest.raises(ValueError, match=msg):
other2 = Timedelta(500)
# TODO: should be OutOfBoundsTimedelta
msg = "value too large"
with pytest.raises(OverflowError, match=msg):
td + other2
with pytest.raises(ValueError, match=msg):
with pytest.raises(OverflowError, match=msg):
other2 + td
with pytest.raises(ValueError, match=msg):
with pytest.raises(OverflowError, match=msg):
td - other2
with pytest.raises(ValueError, match=msg):
with pytest.raises(OverflowError, match=msg):
other2 - td

def test_min(self, td):
Expand Down
87 changes: 57 additions & 30 deletions pandas/tests/scalar/timestamp/test_timestamp.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,10 @@
utc,
)

from pandas._libs.tslibs.dtypes import NpyDatetimeUnit
from pandas._libs.tslibs.dtypes import (
NpyDatetimeUnit,
npy_unit_to_abbrev,
)
from pandas._libs.tslibs.timezones import (
dateutil_gettz as gettz,
get_timezone,
Expand Down Expand Up @@ -884,22 +887,29 @@ def test_to_period(self, dt64, ts):
)
def test_addsub_timedeltalike_non_nano(self, dt64, ts, td):

if isinstance(td, Timedelta):
# td._reso is ns
exp_reso = td._reso
else:
# effective td._reso is s
exp_reso = ts._reso

result = ts - td
expected = Timestamp(dt64) - td
assert isinstance(result, Timestamp)
assert result._reso == ts._reso
assert result._reso == exp_reso
assert result == expected

result = ts + td
expected = Timestamp(dt64) + td
assert isinstance(result, Timestamp)
assert result._reso == ts._reso
assert result._reso == exp_reso
assert result == expected

result = td + ts
expected = td + Timestamp(dt64)
assert isinstance(result, Timestamp)
assert result._reso == ts._reso
assert result._reso == exp_reso
assert result == expected

def test_addsub_offset(self, ts_tz):
Expand Down Expand Up @@ -944,27 +954,35 @@ def test_sub_datetimelike_mismatched_reso(self, ts_tz):
result = ts - other
assert isinstance(result, Timedelta)
assert result.value == 0
assert result._reso == min(ts._reso, other._reso)
assert result._reso == max(ts._reso, other._reso)

result = other - ts
assert isinstance(result, Timedelta)
assert result.value == 0
assert result._reso == min(ts._reso, other._reso)
assert result._reso == max(ts._reso, other._reso)

msg = "Timestamp subtraction with mismatched resolutions"
if ts._reso < other._reso:
# Case where rounding is lossy
other2 = other + Timedelta._from_value_and_reso(1, other._reso)
with pytest.raises(ValueError, match=msg):
ts - other2
with pytest.raises(ValueError, match=msg):
other2 - ts
exp = ts._as_unit(npy_unit_to_abbrev(other._reso)) - other2

res = ts - other2
assert res == exp
assert res._reso == max(ts._reso, other._reso)

res = other2 - ts
assert res == -exp
assert res._reso == max(ts._reso, other._reso)
else:
ts2 = ts + Timedelta._from_value_and_reso(1, ts._reso)
with pytest.raises(ValueError, match=msg):
ts2 - other
with pytest.raises(ValueError, match=msg):
other - ts2
exp = ts2 - other._as_unit(npy_unit_to_abbrev(ts2._reso))

res = ts2 - other
assert res == exp
assert res._reso == max(ts._reso, other._reso)
res = other - ts2
assert res == -exp
assert res._reso == max(ts._reso, other._reso)

def test_sub_timedeltalike_mismatched_reso(self, ts_tz):
# case with non-lossy rounding
Expand All @@ -984,32 +1002,41 @@ def test_sub_timedeltalike_mismatched_reso(self, ts_tz):
result = ts + other
assert isinstance(result, Timestamp)
assert result == ts
assert result._reso == min(ts._reso, other._reso)
assert result._reso == max(ts._reso, other._reso)

result = other + ts
assert isinstance(result, Timestamp)
assert result == ts
assert result._reso == min(ts._reso, other._reso)
assert result._reso == max(ts._reso, other._reso)

msg = "Timestamp addition with mismatched resolutions"
if ts._reso < other._reso:
# Case where rounding is lossy
other2 = other + Timedelta._from_value_and_reso(1, other._reso)
with pytest.raises(ValueError, match=msg):
ts + other2
with pytest.raises(ValueError, match=msg):
other2 + ts
exp = ts._as_unit(npy_unit_to_abbrev(other._reso)) + other2
res = ts + other2
assert res == exp
assert res._reso == max(ts._reso, other._reso)
res = other2 + ts
assert res == exp
assert res._reso == max(ts._reso, other._reso)
else:
ts2 = ts + Timedelta._from_value_and_reso(1, ts._reso)
with pytest.raises(ValueError, match=msg):
ts2 + other
with pytest.raises(ValueError, match=msg):
other + ts2
exp = ts2 + other._as_unit(npy_unit_to_abbrev(ts2._reso))

msg = "Addition between Timestamp and Timedelta with mismatched resolutions"
with pytest.raises(ValueError, match=msg):
# With a mismatched td64 as opposed to Timedelta
ts + np.timedelta64(1, "ns")
res = ts2 + other
assert res == exp
assert res._reso == max(ts._reso, other._reso)
res = other + ts2
assert res == exp
assert res._reso == max(ts._reso, other._reso)

def test_sub_timedelta64_mismatched_reso(self, ts_tz):
ts = ts_tz

res = ts + np.timedelta64(1, "ns")
exp = ts._as_unit("ns") + np.timedelta64(1, "ns")
assert exp == res
assert exp._reso == NpyDatetimeUnit.NPY_FR_ns.value

def test_min(self, ts):
assert ts.min <= ts
Expand Down