Skip to content

Commit 283d672

Browse files
authored
BUG: ensure_timedelta64ns overflows (#34448)
1 parent 0f86110 commit 283d672

File tree

7 files changed

+57
-8
lines changed

7 files changed

+57
-8
lines changed

doc/source/reference/general_utility_functions.rst

+1
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ Exceptions and warnings
4343
errors.NullFrequencyError
4444
errors.NumbaUtilError
4545
errors.OutOfBoundsDatetime
46+
errors.OutOfBoundsTimedelta
4647
errors.ParserError
4748
errors.ParserWarning
4849
errors.PerformanceWarning

pandas/_libs/tslibs/__init__.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
"nat_strings",
88
"is_null_datetimelike",
99
"OutOfBoundsDatetime",
10+
"OutOfBoundsTimedelta",
1011
"IncompatibleFrequency",
1112
"Period",
1213
"Resolution",
@@ -26,7 +27,7 @@
2627
]
2728

2829
from . import dtypes
29-
from .conversion import localize_pydatetime
30+
from .conversion import OutOfBoundsTimedelta, localize_pydatetime
3031
from .dtypes import Resolution
3132
from .nattype import NaT, NaTType, iNaT, is_null_datetimelike, nat_strings
3233
from .np_datetime import OutOfBoundsDatetime

pandas/_libs/tslibs/conversion.pyx

+36-4
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,15 @@ DT64NS_DTYPE = np.dtype('M8[ns]')
5151
TD64NS_DTYPE = np.dtype('m8[ns]')
5252

5353

54+
class OutOfBoundsTimedelta(ValueError):
55+
"""
56+
Raised when encountering a timedelta value that cannot be represented
57+
as a timedelta64[ns].
58+
"""
59+
# Timedelta analogue to OutOfBoundsDatetime
60+
pass
61+
62+
5463
# ----------------------------------------------------------------------
5564
# Unit Conversion Helpers
5665

@@ -228,11 +237,34 @@ def ensure_timedelta64ns(arr: ndarray, copy: bool=True):
228237
229238
Returns
230239
-------
231-
result : ndarray with dtype timedelta64[ns]
232-
240+
ndarray[timedelta64[ns]]
233241
"""
234-
return arr.astype(TD64NS_DTYPE, copy=copy)
235-
# TODO: check for overflows when going from a lower-resolution to nanos
242+
assert arr.dtype.kind == "m", arr.dtype
243+
244+
if arr.dtype == TD64NS_DTYPE:
245+
return arr.copy() if copy else arr
246+
247+
# Re-use the datetime64 machinery to do an overflow-safe `astype`
248+
dtype = arr.dtype.str.replace("m8", "M8")
249+
dummy = arr.view(dtype)
250+
try:
251+
dt64_result = ensure_datetime64ns(dummy, copy)
252+
except OutOfBoundsDatetime as err:
253+
# Re-write the exception in terms of timedelta64 instead of dt64
254+
255+
# Find the value that we are going to report as causing an overflow
256+
tdmin = arr.min()
257+
tdmax = arr.max()
258+
if np.abs(tdmin) >= np.abs(tdmax):
259+
bad_val = tdmin
260+
else:
261+
bad_val = tdmax
262+
263+
raise OutOfBoundsTimedelta(
264+
f"Out of bounds for nanosecond {arr.dtype.name} {bad_val}"
265+
)
266+
267+
return dt64_result.view(TD64NS_DTYPE)
236268

237269

238270
# ----------------------------------------------------------------------

pandas/core/internals/blocks.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -2302,7 +2302,8 @@ class TimeDeltaBlock(DatetimeLikeBlockMixin, IntBlock):
23022302

23032303
def __init__(self, values, placement, ndim=None):
23042304
if values.dtype != TD64NS_DTYPE:
2305-
values = conversion.ensure_timedelta64ns(values)
2305+
# e.g. non-nano or int64
2306+
values = TimedeltaArray._from_sequence(values)._data
23062307
if isinstance(values, TimedeltaArray):
23072308
values = values._data
23082309
assert isinstance(values, np.ndarray), type(values)

pandas/errors/__init__.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
from pandas._config.config import OptionError
88

9-
from pandas._libs.tslibs import OutOfBoundsDatetime
9+
from pandas._libs.tslibs import OutOfBoundsDatetime, OutOfBoundsTimedelta
1010

1111

1212
class NullFrequencyError(ValueError):

pandas/tests/tslibs/test_api.py

+1
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ def test_namespace():
3232
"is_null_datetimelike",
3333
"nat_strings",
3434
"OutOfBoundsDatetime",
35+
"OutOfBoundsTimedelta",
3536
"Period",
3637
"IncompatibleFrequency",
3738
"Resolution",

pandas/tests/tslibs/test_conversion.py

+14-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,13 @@
44
import pytest
55
from pytz import UTC
66

7-
from pandas._libs.tslibs import conversion, iNaT, timezones, tzconversion
7+
from pandas._libs.tslibs import (
8+
OutOfBoundsTimedelta,
9+
conversion,
10+
iNaT,
11+
timezones,
12+
tzconversion,
13+
)
814

915
from pandas import Timestamp, date_range
1016
import pandas._testing as tm
@@ -89,6 +95,13 @@ def test_ensure_datetime64ns_bigendian():
8995
tm.assert_numpy_array_equal(result, expected)
9096

9197

98+
def test_ensure_timedelta64ns_overflows():
99+
arr = np.arange(10).astype("m8[Y]") * 100
100+
msg = r"Out of bounds for nanosecond timedelta64\[Y\] 900"
101+
with pytest.raises(OutOfBoundsTimedelta, match=msg):
102+
conversion.ensure_timedelta64ns(arr)
103+
104+
92105
class SubDatetime(datetime):
93106
pass
94107

0 commit comments

Comments
 (0)