Skip to content

BUG: ensure_timedelta64ns overflows #34448

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 12 commits into from
Jul 10, 2020
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 11 additions & 4 deletions pandas/_libs/tslibs/conversion.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -217,11 +217,18 @@ def ensure_timedelta64ns(arr: ndarray, copy: bool=True):

Returns
-------
result : ndarray with dtype timedelta64[ns]

ndarray[timedelta64[ns]]
"""
return arr.astype(TD64NS_DTYPE, copy=copy)
# TODO: check for overflows when going from a lower-resolution to nanos
assert arr.dtype.kind == "m", arr.dtype

if arr.dtype == TD64NS_DTYPE:
return arr.copy() if copy else arr

# Re-use the datetime64 machinery to do an overflow-safe `astype`
dtype = arr.dtype.str.replace("m8", "M8")
dummy = arr.view(dtype)
dt64_result = ensure_datetime64ns(dummy, copy)
return dt64_result.view(TD64NS_DTYPE)


# ----------------------------------------------------------------------
Expand Down
3 changes: 2 additions & 1 deletion pandas/core/internals/blocks.py
Original file line number Diff line number Diff line change
Expand Up @@ -2312,7 +2312,8 @@ class TimeDeltaBlock(DatetimeLikeBlockMixin, IntBlock):

def __init__(self, values, placement, ndim=None):
if values.dtype != TD64NS_DTYPE:
values = conversion.ensure_timedelta64ns(values)
# e.g. non-nano or int64
values = TimedeltaArray._from_sequence(values)._data
if isinstance(values, TimedeltaArray):
values = values._data
assert isinstance(values, np.ndarray), type(values)
Expand Down
14 changes: 13 additions & 1 deletion pandas/tests/tslibs/test_conversion.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,13 @@
import pytest
from pytz import UTC

from pandas._libs.tslibs import conversion, iNaT, timezones, tzconversion
from pandas._libs.tslibs import (
OutOfBoundsDatetime,
conversion,
iNaT,
timezones,
tzconversion,
)

from pandas import Timestamp, date_range
import pandas._testing as tm
Expand Down Expand Up @@ -80,6 +86,12 @@ def test_ensure_datetime64ns_bigendian():
tm.assert_numpy_array_equal(result, expected)


def test_ensure_timedelta64ns_overflows():
arr = np.arange(10).astype("m8[Y]") * 100
with pytest.raises(OutOfBoundsDatetime, match="Out of bounds"):
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is the wrong type that is raises for this dtypes

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yah, im not too bothered by the exception type, but the exception message is going to have a datetime string in it that wont be helpful. giving this some thought

conversion.ensure_timedelta64ns(arr)


class SubDatetime(datetime):
pass

Expand Down