Skip to content

Commit b6f590c

Browse files
author
MarcoGorelli
committed
refactor double-try-except and fix bug
1 parent 3a0db10 commit b6f590c

File tree

3 files changed

+137
-126
lines changed

3 files changed

+137
-126
lines changed

‎doc/source/whatsnew/v2.0.0.rst

+2-1
Original file line numberDiff line numberDiff line change
@@ -823,7 +823,8 @@ Datetimelike
823823
- Bug in :func:`to_datetime` was throwing ``ValueError`` when parsing dates with ISO8601 format where some values were not zero-padded (:issue:`21422`)
824824
- Bug in :func:`to_datetime` was giving incorrect results when using ``format='%Y%m%d'`` and ``errors='ignore'`` (:issue:`26493`)
825825
- Bug in :func:`to_datetime` was failing to parse date strings ``'today'`` and ``'now'`` if ``format`` was not ISO8601 (:issue:`50359`)
826-
- Bug in :func:`Timestamp.utctimetuple` raising a ``TypeError`` (:issue:`32174`)
826+
- Bug in :func:`to_datetime` was raising ``ValueError`` when parsing mixed-offset :class:`Timestamp` with ``errors='ignore'`` (:issue:`50585`)
827+
-
827828

828829
Timedelta
829830
^^^^^^^^^

‎pandas/_libs/tslib.pyx

+117-123
Original file line numberDiff line numberDiff line change
@@ -505,144 +505,138 @@ cpdef array_to_datetime(
505505
result = np.empty(n, dtype="M8[ns]")
506506
iresult = result.view("i8")
507507

508-
try:
509-
for i in range(n):
510-
val = values[i]
508+
for i in range(n):
509+
val = values[i]
511510

512-
try:
513-
if checknull_with_nat_and_na(val):
514-
iresult[i] = NPY_NAT
511+
try:
512+
if checknull_with_nat_and_na(val):
513+
iresult[i] = NPY_NAT
515514

516-
elif PyDateTime_Check(val):
517-
if val.tzinfo is not None:
518-
found_tz = True
519-
else:
520-
found_naive = True
521-
tz_out = convert_timezone(
522-
val.tzinfo,
523-
tz_out,
524-
found_naive,
525-
found_tz,
526-
utc_convert,
527-
)
528-
result[i] = parse_pydatetime(val, &dts, utc_convert)
515+
elif PyDateTime_Check(val):
516+
if val.tzinfo is not None:
517+
found_tz = True
518+
else:
519+
found_naive = True
520+
tz_out = convert_timezone(
521+
val.tzinfo,
522+
tz_out,
523+
found_naive,
524+
found_tz,
525+
utc_convert,
526+
)
527+
result[i] = parse_pydatetime(val, &dts, utc_convert)
528+
529+
elif PyDate_Check(val):
530+
iresult[i] = pydate_to_dt64(val, &dts)
531+
check_dts_bounds(&dts)
529532

530-
elif PyDate_Check(val):
531-
iresult[i] = pydate_to_dt64(val, &dts)
532-
check_dts_bounds(&dts)
533+
elif is_datetime64_object(val):
534+
iresult[i] = get_datetime64_nanos(val, NPY_FR_ns)
533535

534-
elif is_datetime64_object(val):
535-
iresult[i] = get_datetime64_nanos(val, NPY_FR_ns)
536+
elif is_integer_object(val) or is_float_object(val):
537+
# these must be ns unit by-definition
536538

537-
elif is_integer_object(val) or is_float_object(val):
538-
# these must be ns unit by-definition
539+
if val != val or val == NPY_NAT:
540+
iresult[i] = NPY_NAT
541+
elif is_raise or is_ignore:
542+
iresult[i] = val
543+
else:
544+
# coerce
545+
# we now need to parse this as if unit='ns'
546+
# we can ONLY accept integers at this point
547+
# if we have previously (or in future accept
548+
# datetimes/strings, then we must coerce)
549+
iresult[i] = cast_from_unit(val, "ns")
539550

540-
if val != val or val == NPY_NAT:
541-
iresult[i] = NPY_NAT
542-
elif is_raise or is_ignore:
543-
iresult[i] = val
544-
else:
545-
# coerce
546-
# we now need to parse this as if unit='ns'
547-
# we can ONLY accept integers at this point
548-
# if we have previously (or in future accept
549-
# datetimes/strings, then we must coerce)
550-
try:
551-
iresult[i] = cast_from_unit(val, "ns")
552-
except OverflowError:
553-
iresult[i] = NPY_NAT
551+
elif isinstance(val, str):
552+
# string
553+
if type(val) is not str:
554+
# GH#32264 np.str_ object
555+
val = str(val)
554556

555-
elif isinstance(val, str):
556-
# string
557-
if type(val) is not str:
558-
# GH#32264 np.str_ object
559-
val = str(val)
557+
if len(val) == 0 or val in nat_strings:
558+
iresult[i] = NPY_NAT
559+
continue
560560

561-
if len(val) == 0 or val in nat_strings:
562-
iresult[i] = NPY_NAT
561+
string_to_dts_failed = string_to_dts(
562+
val, &dts, &out_bestunit, &out_local,
563+
&out_tzoffset, False, None, False
564+
)
565+
if string_to_dts_failed:
566+
# An error at this point is a _parsing_ error
567+
# specifically _not_ OutOfBoundsDatetime
568+
if parse_today_now(val, &iresult[i], utc):
563569
continue
564570

565-
string_to_dts_failed = string_to_dts(
566-
val, &dts, &out_bestunit, &out_local,
567-
&out_tzoffset, False, None, False
568-
)
569-
if string_to_dts_failed:
570-
# An error at this point is a _parsing_ error
571-
# specifically _not_ OutOfBoundsDatetime
572-
if parse_today_now(val, &iresult[i], utc):
571+
try:
572+
py_dt = parse_datetime_string(val,
573+
dayfirst=dayfirst,
574+
yearfirst=yearfirst)
575+
# If the dateutil parser returned tzinfo, capture it
576+
# to check if all arguments have the same tzinfo
577+
tz = py_dt.utcoffset()
578+
579+
except (ValueError, OverflowError):
580+
if is_coerce:
581+
iresult[i] = NPY_NAT
573582
continue
574-
575-
try:
576-
py_dt = parse_datetime_string(val,
577-
dayfirst=dayfirst,
578-
yearfirst=yearfirst)
579-
# If the dateutil parser returned tzinfo, capture it
580-
# to check if all arguments have the same tzinfo
581-
tz = py_dt.utcoffset()
582-
583-
except (ValueError, OverflowError):
584-
if is_coerce:
585-
iresult[i] = NPY_NAT
586-
continue
587-
raise TypeError(
588-
f"invalid string coercion to datetime "
589-
f"for \"{val}\", at position {i}"
590-
)
591-
592-
if tz is not None:
593-
seen_datetime_offset = True
594-
# dateutil timezone objects cannot be hashed, so
595-
# store the UTC offsets in seconds instead
596-
out_tzoffset_vals.add(tz.total_seconds())
597-
else:
598-
# Add a marker for naive string, to track if we are
599-
# parsing mixed naive and aware strings
600-
out_tzoffset_vals.add("naive")
601-
602-
_ts = convert_datetime_to_tsobject(py_dt, None)
603-
iresult[i] = _ts.value
604-
if not string_to_dts_failed:
605-
# No error reported by string_to_dts, pick back up
606-
# where we left off
607-
value = npy_datetimestruct_to_datetime(NPY_FR_ns, &dts)
608-
if out_local == 1:
609-
seen_datetime_offset = True
610-
# Store the out_tzoffset in seconds
611-
# since we store the total_seconds of
612-
# dateutil.tz.tzoffset objects
613-
out_tzoffset_vals.add(out_tzoffset * 60.)
614-
tz = timezone(timedelta(minutes=out_tzoffset))
615-
value = tz_localize_to_utc_single(value, tz)
616-
out_local = 0
617-
out_tzoffset = 0
618-
else:
619-
# Add a marker for naive string, to track if we are
620-
# parsing mixed naive and aware strings
621-
out_tzoffset_vals.add("naive")
622-
iresult[i] = value
623-
check_dts_bounds(&dts)
624-
625-
else:
626-
if is_coerce:
627-
iresult[i] = NPY_NAT
583+
raise TypeError(
584+
f"invalid string coercion to datetime "
585+
f"for \"{val}\", at position {i}"
586+
)
587+
588+
if tz is not None:
589+
seen_datetime_offset = True
590+
# dateutil timezone objects cannot be hashed, so
591+
# store the UTC offsets in seconds instead
592+
out_tzoffset_vals.add(tz.total_seconds())
628593
else:
629-
raise TypeError(f"{type(val)} is not convertible to datetime")
630-
631-
except OutOfBoundsDatetime as ex:
632-
ex.args = (f"{ex}, at position {i}",)
633-
if is_coerce:
634-
iresult[i] = NPY_NAT
635-
continue
636-
raise
594+
# Add a marker for naive string, to track if we are
595+
# parsing mixed naive and aware strings
596+
out_tzoffset_vals.add("naive")
597+
598+
_ts = convert_datetime_to_tsobject(py_dt, None)
599+
iresult[i] = _ts.value
600+
if not string_to_dts_failed:
601+
# No error reported by string_to_dts, pick back up
602+
# where we left off
603+
value = npy_datetimestruct_to_datetime(NPY_FR_ns, &dts)
604+
if out_local == 1:
605+
seen_datetime_offset = True
606+
# Store the out_tzoffset in seconds
607+
# since we store the total_seconds of
608+
# dateutil.tz.tzoffset objects
609+
out_tzoffset_vals.add(out_tzoffset * 60.)
610+
tz = timezone(timedelta(minutes=out_tzoffset))
611+
value = tz_localize_to_utc_single(value, tz)
612+
out_local = 0
613+
out_tzoffset = 0
614+
else:
615+
# Add a marker for naive string, to track if we are
616+
# parsing mixed naive and aware strings
617+
out_tzoffset_vals.add("naive")
618+
iresult[i] = value
619+
check_dts_bounds(&dts)
637620

638-
except OutOfBoundsDatetime:
639-
if is_raise:
640-
raise
621+
else:
622+
raise TypeError(f"{type(val)} is not convertible to datetime")
641623

642-
return ignore_errors_out_of_bounds_fallback(values), tz_out
624+
except (OutOfBoundsDatetime, ValueError) as ex:
625+
ex.args = (f"{ex}, at position {i}",)
626+
if is_coerce:
627+
iresult[i] = NPY_NAT
628+
continue
629+
elif is_raise:
630+
raise
631+
if isinstance(ex, OutOfBoundsDatetime):
632+
return ignore_errors_out_of_bounds_fallback(values), tz_out
633+
return _array_to_datetime_object(values, errors, dayfirst, yearfirst)
643634

644-
except TypeError:
645-
return _array_to_datetime_object(values, errors, dayfirst, yearfirst)
635+
except TypeError:
636+
if is_coerce:
637+
iresult[i] = NPY_NAT
638+
continue
639+
return _array_to_datetime_object(values, errors, dayfirst, yearfirst)
646640

647641
if seen_datetime_offset and not utc_convert:
648642
# GH#17697

‎pandas/tests/tools/test_to_datetime.py

+18-2
Original file line numberDiff line numberDiff line change
@@ -1094,8 +1094,9 @@ def test_to_datetime_tz(self, cache):
10941094
)
10951095
tm.assert_index_equal(result, expected)
10961096

1097-
def test_to_datetime_tz_mixed_raises(self, cache):
1098-
# mixed tzs will raise
1097+
def test_to_datetime_tz_mixed(self, cache):
1098+
# mixed tzs will raise if errors='raise'
1099+
# https://github.com/pandas-dev/pandas/issues/50585
10991100
arr = [
11001101
Timestamp("2013-01-01 13:00:00", tz="US/Pacific"),
11011102
Timestamp("2013-01-02 14:00:00", tz="US/Eastern"),
@@ -1107,6 +1108,21 @@ def test_to_datetime_tz_mixed_raises(self, cache):
11071108
with pytest.raises(ValueError, match=msg):
11081109
to_datetime(arr, cache=cache)
11091110

1111+
result = to_datetime(arr, cache=cache, errors="ignore")
1112+
expected = Index(
1113+
[
1114+
Timestamp("2013-01-01 13:00:00-08:00"),
1115+
Timestamp("2013-01-02 14:00:00-05:00"),
1116+
],
1117+
dtype="object",
1118+
)
1119+
tm.assert_index_equal(result, expected)
1120+
result = to_datetime(arr, cache=cache, errors="coerce")
1121+
expected = DatetimeIndex(
1122+
["2013-01-01 13:00:00-08:00", "NaT"], dtype="datetime64[ns, US/Pacific]"
1123+
)
1124+
tm.assert_index_equal(result, expected)
1125+
11101126
def test_to_datetime_different_offsets(self, cache):
11111127
# inspired by asv timeseries.ToDatetimeNONISO8601 benchmark
11121128
# see GH-26097 for more

0 commit comments

Comments
 (0)