From 776fe0ba590d3c39a0665626aaea49a6067e9e21 Mon Sep 17 00:00:00 2001 From: Brock Date: Fri, 3 Jun 2022 18:44:32 -0700 Subject: [PATCH] REF: remove ensure_datetime64ns, ensure_timedelta64ns --- pandas/_libs/tslibs/conversion.pyi | 10 ----- pandas/_libs/tslibs/conversion.pyx | 39 ------------------- pandas/_libs/tslibs/np_datetime.pyx | 4 ++ pandas/_libs/tslibs/period.pyx | 5 ++- pandas/core/arrays/datetimes.py | 4 +- pandas/core/arrays/timedeltas.py | 10 ++--- pandas/core/dtypes/cast.py | 6 +-- .../arrays/categorical/test_constructors.py | 2 +- .../indexes/datetimes/test_constructors.py | 4 +- pandas/tests/tslibs/test_conversion.py | 7 ++-- 10 files changed, 23 insertions(+), 68 deletions(-) diff --git a/pandas/_libs/tslibs/conversion.pyi b/pandas/_libs/tslibs/conversion.pyi index 16fe853eef815..7e2ebb1b3bad3 100644 --- a/pandas/_libs/tslibs/conversion.pyi +++ b/pandas/_libs/tslibs/conversion.pyi @@ -10,17 +10,7 @@ from pandas._typing import npt DT64NS_DTYPE: np.dtype TD64NS_DTYPE: np.dtype -class OutOfBoundsTimedelta(ValueError): ... - def precision_from_unit( unit: str, ) -> tuple[int, int]: ... # (int64_t, _) -def ensure_datetime64ns( - arr: np.ndarray, # np.ndarray[datetime64[ANY]] - copy: bool = ..., -) -> np.ndarray: ... # np.ndarray[datetime64ns] -def ensure_timedelta64ns( - arr: np.ndarray, # np.ndarray[timedelta64[ANY]] - copy: bool = ..., -) -> np.ndarray: ... # np.ndarray[timedelta64ns] def localize_pydatetime(dt: datetime, tz: tzinfo | None) -> datetime: ... diff --git a/pandas/_libs/tslibs/conversion.pyx b/pandas/_libs/tslibs/conversion.pyx index 2ed1930b01555..808f750c18c9d 100644 --- a/pandas/_libs/tslibs/conversion.pyx +++ b/pandas/_libs/tslibs/conversion.pyx @@ -193,45 +193,6 @@ cdef inline int64_t get_datetime64_nanos(object val) except? -1: return ival -def ensure_datetime64ns(arr: ndarray, copy: bool = True): - """ - Ensure a np.datetime64 array has dtype specifically 'datetime64[ns]' - - Parameters - ---------- - arr : ndarray - copy : bool, default True - - Returns - ------- - ndarray with dtype datetime64[ns] - """ - if (arr).dtype.byteorder == ">": - # GH#29684 we incorrectly get OutOfBoundsDatetime if we dont swap - dtype = arr.dtype - arr = arr.astype(dtype.newbyteorder("<")) - - return astype_overflowsafe(arr, DT64NS_DTYPE, copy=copy) - - -def ensure_timedelta64ns(arr: ndarray, copy: bool = True): - """ - Ensure a np.timedelta64 array has dtype specifically 'timedelta64[ns]' - - Parameters - ---------- - arr : ndarray - copy : bool, default True - - Returns - ------- - ndarray[timedelta64[ns]] - """ - assert arr.dtype.kind == "m", arr.dtype - - return astype_overflowsafe(arr, dtype=TD64NS_DTYPE, copy=copy) - - # ---------------------------------------------------------------------- # _TSObject Conversion diff --git a/pandas/_libs/tslibs/np_datetime.pyx b/pandas/_libs/tslibs/np_datetime.pyx index 337bedafac757..bab4743dce38c 100644 --- a/pandas/_libs/tslibs/np_datetime.pyx +++ b/pandas/_libs/tslibs/np_datetime.pyx @@ -319,6 +319,10 @@ cpdef ndarray astype_overflowsafe( "datetime64/timedelta64 values and dtype must have a unit specified" ) + if (values).dtype.byteorder == ">": + # GH#29684 we incorrectly get OutOfBoundsDatetime if we dont swap + values = values.astype(values.dtype.newbyteorder("<")) + if from_unit == to_unit: # Check this before allocating result for perf, might save some memory if copy: diff --git a/pandas/_libs/tslibs/period.pyx b/pandas/_libs/tslibs/period.pyx index b5444bd5abdd9..557ddd7693744 100644 --- a/pandas/_libs/tslibs/period.pyx +++ b/pandas/_libs/tslibs/period.pyx @@ -47,6 +47,7 @@ from pandas._libs.tslibs.np_datetime cimport ( NPY_DATETIMEUNIT, NPY_FR_D, NPY_FR_us, + astype_overflowsafe, check_dts_bounds, dt64_to_dtstruct, dtstruct_to_dt64, @@ -71,7 +72,7 @@ from pandas._libs.tslibs.timedeltas cimport ( is_any_td_scalar, ) -from pandas._libs.tslibs.conversion import ensure_datetime64ns +from pandas._libs.tslibs.conversion import DT64NS_DTYPE from pandas._libs.tslibs.dtypes cimport ( FR_ANN, @@ -981,7 +982,7 @@ def periodarr_to_dt64arr(const int64_t[:] periodarr, int freq): dta = periodarr.base.view("M8[h]") elif freq == FR_DAY: dta = periodarr.base.view("M8[D]") - return ensure_datetime64ns(dta) + return astype_overflowsafe(dta, dtype=DT64NS_DTYPE) cdef void get_asfreq_info(int from_freq, int to_freq, diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index 7362e029c3d53..39cb249f58f80 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -25,7 +25,7 @@ NaTType, Resolution, Timestamp, - conversion, + astype_overflowsafe, fields, get_resolution, iNaT, @@ -2169,7 +2169,7 @@ def _sequence_to_dt64ns( # tz-naive DatetimeArray or ndarray[datetime64] data = getattr(data, "_ndarray", data) if data.dtype != DT64NS_DTYPE: - data = conversion.ensure_datetime64ns(data) + data = astype_overflowsafe(data, dtype=DT64NS_DTYPE) copy = False if tz is not None: diff --git a/pandas/core/arrays/timedeltas.py b/pandas/core/arrays/timedeltas.py index 0cb1776d6046d..370a0b5f2ae25 100644 --- a/pandas/core/arrays/timedeltas.py +++ b/pandas/core/arrays/timedeltas.py @@ -21,13 +21,11 @@ Tick, Timedelta, Timestamp, + astype_overflowsafe, iNaT, to_offset, ) -from pandas._libs.tslibs.conversion import ( - ensure_timedelta64ns, - precision_from_unit, -) +from pandas._libs.tslibs.conversion import precision_from_unit from pandas._libs.tslibs.fields import get_timedelta_field from pandas._libs.tslibs.timedeltas import ( array_to_timedelta64, @@ -1044,7 +1042,7 @@ def sequence_to_td64ns( elif is_timedelta64_dtype(data.dtype): if data.dtype != TD64NS_DTYPE: # non-nano unit - data = ensure_timedelta64ns(data) + data = astype_overflowsafe(data, dtype=TD64NS_DTYPE) copy = False else: @@ -1086,7 +1084,7 @@ def ints_to_td64ns(data, unit="ns"): dtype_str = f"timedelta64[{unit}]" data = data.view(dtype_str) - data = ensure_timedelta64ns(data) + data = astype_overflowsafe(data, dtype=TD64NS_DTYPE) # the astype conversion makes a copy, so we can avoid re-copying later copy_made = True diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index c6a4effac7a37..ed3f9ee525c9e 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -30,7 +30,7 @@ OutOfBoundsTimedelta, Timedelta, Timestamp, - conversion, + astype_overflowsafe, ) from pandas._libs.tslibs.timedeltas import array_to_timedelta64 from pandas._typing import ( @@ -1419,10 +1419,10 @@ def sanitize_to_nanoseconds(values: np.ndarray, copy: bool = False) -> np.ndarra """ dtype = values.dtype if dtype.kind == "M" and dtype != DT64NS_DTYPE: - values = conversion.ensure_datetime64ns(values) + values = astype_overflowsafe(values, dtype=DT64NS_DTYPE) elif dtype.kind == "m" and dtype != TD64NS_DTYPE: - values = conversion.ensure_timedelta64ns(values) + values = astype_overflowsafe(values, dtype=TD64NS_DTYPE) elif copy: values = values.copy() diff --git a/pandas/tests/arrays/categorical/test_constructors.py b/pandas/tests/arrays/categorical/test_constructors.py index c108ce9fcf100..24f4dd4a1f64a 100644 --- a/pandas/tests/arrays/categorical/test_constructors.py +++ b/pandas/tests/arrays/categorical/test_constructors.py @@ -751,7 +751,7 @@ def test_from_sequence_copy(self): @pytest.mark.xfail( not IS64 or is_platform_windows(), - reason="Incorrectly raising in ensure_datetime64ns", + reason="Incorrectly raising in astype_overflowsafe", ) def test_constructor_datetime64_non_nano(self): categories = np.arange(10).view("M8[D]") diff --git a/pandas/tests/indexes/datetimes/test_constructors.py b/pandas/tests/indexes/datetimes/test_constructors.py index 4ac2c15b7d98e..ea34e636d890f 100644 --- a/pandas/tests/indexes/datetimes/test_constructors.py +++ b/pandas/tests/indexes/datetimes/test_constructors.py @@ -13,7 +13,7 @@ from pandas._libs.tslibs import ( OutOfBoundsDatetime, - conversion, + astype_overflowsafe, ) from pandas.compat import PY39 @@ -975,7 +975,7 @@ def test_index_cast_datetime64_other_units(self): arr = np.arange(0, 100, 10, dtype=np.int64).view("M8[D]") idx = Index(arr) - assert (idx.values == conversion.ensure_datetime64ns(arr)).all() + assert (idx.values == astype_overflowsafe(arr, dtype=np.dtype("M8[ns]"))).all() def test_constructor_int64_nocopy(self): # GH#1624 diff --git a/pandas/tests/tslibs/test_conversion.py b/pandas/tests/tslibs/test_conversion.py index 598be5a0451b6..bf9a45c6d9d7a 100644 --- a/pandas/tests/tslibs/test_conversion.py +++ b/pandas/tests/tslibs/test_conversion.py @@ -6,6 +6,7 @@ from pandas._libs.tslibs import ( OutOfBoundsTimedelta, + astype_overflowsafe, conversion, iNaT, timezones, @@ -106,7 +107,7 @@ def test_tz_convert_readonly(): @pytest.mark.parametrize("dtype", ["M8[ns]", "M8[s]"]) def test_length_zero_copy(dtype, copy): arr = np.array([], dtype=dtype) - result = conversion.ensure_datetime64ns(arr, copy=copy) + result = astype_overflowsafe(arr, copy=copy, dtype=np.dtype("M8[ns]")) if copy: assert not np.shares_memory(result, arr) else: @@ -119,7 +120,7 @@ def test_length_zero_copy(dtype, copy): def test_ensure_datetime64ns_bigendian(): # GH#29684 arr = np.array([np.datetime64(1, "ms")], dtype=">M8[ms]") - result = conversion.ensure_datetime64ns(arr) + result = astype_overflowsafe(arr, dtype=np.dtype("M8[ns]")) expected = np.array([np.datetime64(1, "ms")], dtype="M8[ns]") tm.assert_numpy_array_equal(result, expected) @@ -129,7 +130,7 @@ def test_ensure_timedelta64ns_overflows(): arr = np.arange(10).astype("m8[Y]") * 100 msg = r"Cannot convert 300 years to timedelta64\[ns\] without overflow" with pytest.raises(OutOfBoundsTimedelta, match=msg): - conversion.ensure_timedelta64ns(arr) + astype_overflowsafe(arr, dtype=np.dtype("m8[ns]")) class SubDatetime(datetime):