Skip to content

BUG: to_datetime with out-of-bounds np.datetime64 and errors='ignore' doesn't return input #50992

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Feb 2, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions doc/source/whatsnew/v2.0.0.rst
Original file line number Diff line number Diff line change
Expand Up @@ -1025,6 +1025,7 @@ Datetimelike
- Bug in :func:`to_datetime` was raising ``ValueError`` when parsing mixed-offset :class:`Timestamp` with ``errors='ignore'`` (:issue:`50585`)
- Bug in :func:`to_datetime` was incorrectly handling floating-point inputs within 1 ``unit`` of the overflow boundaries (:issue:`50183`)
- Bug in :func:`to_datetime` with unit of "Y" or "M" giving incorrect results, not matching pointwise :class:`Timestamp` results (:issue:`50870`)
- Bug in :func:`to_datetime` was not returning input with ``errors='ignore'`` when input was out-of-bounds (:issue:`50587`)
-

Timedelta
Expand Down
60 changes: 0 additions & 60 deletions pandas/_libs/tslib.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@ from pandas._libs.tslibs.np_datetime cimport (
NPY_DATETIMEUNIT,
NPY_FR_ns,
check_dts_bounds,
get_datetime64_value,
npy_datetimestruct,
npy_datetimestruct_to_datetime,
pandas_datetime_to_datetimestruct,
Expand Down Expand Up @@ -542,16 +541,6 @@ cpdef array_to_datetime(

cnp.PyArray_MultiIter_NEXT(mi)

except OutOfBoundsDatetime as ex:
ex.args = (f"{ex}, at position {i}",)
if is_coerce:
iresult[i] = NPY_NAT
cnp.PyArray_MultiIter_NEXT(mi)
continue
elif is_raise:
raise
return ignore_errors_out_of_bounds_fallback(values), tz_out

except (TypeError, OverflowError, ValueError) as ex:
ex.args = (f"{ex}, at position {i}",)
if is_coerce:
Expand All @@ -578,55 +567,6 @@ cpdef array_to_datetime(
return result, tz_out


@cython.wraparound(False)
@cython.boundscheck(False)
cdef ndarray ignore_errors_out_of_bounds_fallback(ndarray values):
"""
Fallback for array_to_datetime if an OutOfBoundsDatetime is raised
and errors == "ignore"

Parameters
----------
values : ndarray[object]

Returns
-------
ndarray[object]
"""
cdef:
Py_ssize_t i, n = values.size
object val
cnp.broadcast mi
ndarray[object] oresult
ndarray oresult_nd

oresult_nd = cnp.PyArray_EMPTY(values.ndim, values.shape, cnp.NPY_OBJECT, 0)
mi = cnp.PyArray_MultiIterNew2(oresult_nd, values)
oresult = oresult_nd.ravel()

for i in range(n):
# Analogous to `val = values[i]`
val = <object>(<PyObject**>cnp.PyArray_MultiIter_DATA(mi, 1))[0]

# set as nan except if its a NaT
if checknull_with_nat_and_na(val):
if isinstance(val, float):
oresult[i] = np.nan
else:
oresult[i] = <object>NaT
elif is_datetime64_object(val):
if get_datetime64_value(val) == NPY_NAT:
oresult[i] = <object>NaT
else:
oresult[i] = val.item()
else:
oresult[i] = val

cnp.PyArray_MultiIter_NEXT(mi)

return oresult


@cython.wraparound(False)
@cython.boundscheck(False)
cdef _array_to_datetime_object(
Expand Down
8 changes: 7 additions & 1 deletion pandas/tests/tools/test_to_datetime.py
Original file line number Diff line number Diff line change
Expand Up @@ -1084,9 +1084,15 @@ def test_to_datetime_array_of_dt64s(self, cache, unit):
# numpy is either a python datetime.datetime or datetime.date
tm.assert_index_equal(
to_datetime(dts_with_oob, errors="ignore", cache=cache),
Index([dt.item() for dt in dts_with_oob]),
Index(dts_with_oob),
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can you write a small dedicated test for this. iirc some of the non-nano stuff is liable to change this test and i dont want to accidentally get rid of coverage for this behavior

)

def test_out_of_bounds_errors_ignore(self):
# https://github.com/pandas-dev/pandas/issues/50587
result = to_datetime(np.datetime64("9999-01-01"), errors="ignore")
expected = np.datetime64("9999-01-01")
assert result == expected

def test_to_datetime_tz(self, cache):

# xref 8260
Expand Down