Skip to content

BUG: ensure_timedelta64ns overflows #34448

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 12 commits into from
Jul 10, 2020
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 28 additions & 4 deletions pandas/_libs/tslibs/conversion.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -217,11 +217,35 @@ def ensure_timedelta64ns(arr: ndarray, copy: bool=True):

Returns
-------
result : ndarray with dtype timedelta64[ns]

ndarray[timedelta64[ns]]
"""
return arr.astype(TD64NS_DTYPE, copy=copy)
# TODO: check for overflows when going from a lower-resolution to nanos
assert arr.dtype.kind == "m", arr.dtype

if arr.dtype == TD64NS_DTYPE:
return arr.copy() if copy else arr

# Re-use the datetime64 machinery to do an overflow-safe `astype`
dtype = arr.dtype.str.replace("m8", "M8")
dummy = arr.view(dtype)
try:
dt64_result = ensure_datetime64ns(dummy, copy)
except OutOfBoundsDatetime as err:
# Re-write the exception in terms of timedelta64 instead of dt64

# Find the value that we are going to report as causing an overflow
tdmin = arr.min()
tdmax = arr.max()
if np.abs(tdmin) >= np.abs(tdmax):
bad_val = tdmin
else:
bad_val = tdmax

unit_str = arr.dtype.str.split("[")[-1][:-1]
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this seems fragile, can you just report it entirely:

In [39]: np.dtype('m8[ms]').name                                                                                                                                   
Out[39]: 'timedelta64[ms]'

raise OutOfBoundsDatetime(
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can we just create OutOfBoundsTimedelta

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i think if we create an OOBTimedelta exception would be ok ith this change

f"Out of bounds for nanosecond timedelta {bad_val}[{unit_str}]"
)

return dt64_result.view(TD64NS_DTYPE)


# ----------------------------------------------------------------------
Expand Down
3 changes: 2 additions & 1 deletion pandas/core/internals/blocks.py
Original file line number Diff line number Diff line change
Expand Up @@ -2321,7 +2321,8 @@ class TimeDeltaBlock(DatetimeLikeBlockMixin, IntBlock):

def __init__(self, values, placement, ndim=None):
if values.dtype != TD64NS_DTYPE:
values = conversion.ensure_timedelta64ns(values)
# e.g. non-nano or int64
values = TimedeltaArray._from_sequence(values)._data
if isinstance(values, TimedeltaArray):
values = values._data
assert isinstance(values, np.ndarray), type(values)
Expand Down
15 changes: 14 additions & 1 deletion pandas/tests/tslibs/test_conversion.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,13 @@
import pytest
from pytz import UTC

from pandas._libs.tslibs import conversion, iNaT, timezones, tzconversion
from pandas._libs.tslibs import (
OutOfBoundsDatetime,
conversion,
iNaT,
timezones,
tzconversion,
)

from pandas import Timestamp, date_range
import pandas._testing as tm
Expand Down Expand Up @@ -80,6 +86,13 @@ def test_ensure_datetime64ns_bigendian():
tm.assert_numpy_array_equal(result, expected)


def test_ensure_timedelta64ns_overflows():
arr = np.arange(10).astype("m8[Y]") * 100
msg = r"Out of bounds for nanosecond timedelta 900\[Y\]"
with pytest.raises(OutOfBoundsDatetime, match=msg):
conversion.ensure_timedelta64ns(arr)


class SubDatetime(datetime):
pass

Expand Down