Skip to content

BUG: Fix pd.to_numeric to have consistent behavior for date-like arguments #43315

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 8 commits into from
Closed

BUG: Fix pd.to_numeric to have consistent behavior for date-like arguments #43315

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
870769f
Handle date arguments as objects
hec10r Aug 30, 2021
1f0b361
Add docstrings
hec10r Aug 30, 2021
08a7bb8
Add support for numpy arrays and remove deprecated conversion
hec10r Aug 31, 2021
05e1f2d
Remove tests that depend on the old behavior
hec10r Aug 31, 2021
5a5fd77
Add test_type_error and generalize test_ignore_error
hec10r Aug 31, 2021
f844dc4
Add test_list_series
hec10r Aug 31, 2021
ffc1a6e
PEP 8
hec10r Aug 31, 2021
bcf1cca
Remove failing tests that depends in the old behavior
hec10r Aug 31, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 19 additions & 9 deletions pandas/core/tools/numeric.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,18 +102,22 @@ def to_numeric(arg, errors="raise", downcast=None):
1 2
2 -3
dtype: int8
>>> s = pd.Series(['apple', '1.0', '2', -3])
>>> s = pd.Series(['apple', '1.0', '2', -3, pd.to_datetime(0), pd.NaT])
>>> pd.to_numeric(s, errors='ignore')
0 apple
1 1.0
2 2
3 -3
0 apple
1 1.0
2 2
3 -3
4 1970-01-01 00:00:00
5 NaT
dtype: object
>>> pd.to_numeric(s, errors='coerce')
0 NaN
1 1.0
2 2.0
3 -3.0
4 NaN
5 NaN
dtype: float64

Downcasting of nullable integer and floating dtypes is supported:
Expand All @@ -137,6 +141,15 @@ def to_numeric(arg, errors="raise", downcast=None):
if errors not in ("ignore", "raise", "coerce"):
raise ValueError("invalid error value specified")

# Handle inputs of "date" type as objects
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

umm what is this?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The idea was to change the behavior of pd.to_datetime for date-like objects. Refer to this discussion: #43280 (comment)

arg_dtype = getattr(arg, "dtype", None)
if is_datetime_or_timedelta_dtype(arg_dtype):
try:
arg = arg._constructor(arg, dtype="O")
except AttributeError:
# when `arg` is a a numpy array
arg = arg.astype("O")

is_series = False
is_index = False
is_scalars = False
Expand All @@ -146,10 +159,7 @@ def to_numeric(arg, errors="raise", downcast=None):
values = arg.values
elif isinstance(arg, ABCIndex):
is_index = True
if needs_i8_conversion(arg.dtype):
values = arg.asi8
else:
values = arg.values
values = arg.values
elif isinstance(arg, (list, tuple)):
values = np.array(arg, dtype="O")
elif is_scalar(arg):
Expand Down
84 changes: 47 additions & 37 deletions pandas/tests/tools/test_to_numeric.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,30 @@ def test_series(last_val):
tm.assert_series_equal(result, expected)


@pytest.mark.parametrize(
"list_data,kwargs",
[
(["-3.14", 7], {}),
(
["-3.14", 7, pd.to_datetime(0), pd.NaT, ["30", -10]],
{"errors": "coerce"}
),
(
["-3.14", 7, pd.to_datetime(0), pd.NaT, ["30", -10]],
{"errors": "ignore"}
),
]
)
def test_list_series(list_data, kwargs):
lis = list_data
ser = Series(list_data)

result = to_numeric(lis, **kwargs)
expected = to_numeric(ser, **kwargs).values

tm.assert_numpy_array_equal(result, expected)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why are you comparing vs a numpy array?

these should use assert_series_equal at a minimum

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Because one input was a list and the other input was a pd.Series. So the first ouput is a np.array and the output for series.values is a np.array when series is instance of pd.Series. It could have been done in other way tough;

...
result = to_numeric(lis, **kwargs)
expected = to_numeric(ser, **kwargs)

tm.assert_series_equal(pd.Series(result), expected)



@pytest.mark.parametrize(
"data",
[
Expand Down Expand Up @@ -111,10 +135,31 @@ def test_error(data, msg):


@pytest.mark.parametrize(
"errors,exp_data", [("ignore", [1, -3.14, "apple"]), ("coerce", [1, -3.14, np.nan])]
"data,msg",
[
([22.06, "-86", pd.NaT], "Invalid object type at position 2"),
(
[pd.to_datetime(0), 22.06, "-86", pd.NaT],
"Invalid object type at position 0"
),
],
)
def test_type_error(data, msg):
ser = Series(data)

with pytest.raises(TypeError, match=msg):
to_numeric(ser, errors="raise")


@pytest.mark.parametrize(
"errors,exp_data",
[
("ignore", [1, -3.14, "apple", pd.to_datetime(0), pd.NaT]),
("coerce", [1, -3.14, np.nan, np.nan, np.nan])
],
)
def test_ignore_error(errors, exp_data):
ser = Series([1, -3.14, "apple"])
ser = Series([1, -3.14, "apple", pd.to_datetime(0), pd.NaT])
result = to_numeric(ser, errors=errors)

expected = Series(exp_data)
Expand Down Expand Up @@ -372,39 +417,6 @@ def test_str(data, exp, transform_assert_equal):
assert_equal(result, expected)


def test_datetime_like(tz_naive_fixture, transform_assert_equal):
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why are you removing these?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Because I was changing the behavior of the function for date-like objects

transform, assert_equal = transform_assert_equal
idx = pd.date_range("20130101", periods=3, tz=tz_naive_fixture)

result = to_numeric(transform(idx))
expected = transform(idx.asi8)
assert_equal(result, expected)


def test_timedelta(transform_assert_equal):
transform, assert_equal = transform_assert_equal
idx = pd.timedelta_range("1 days", periods=3, freq="D")

result = to_numeric(transform(idx))
expected = transform(idx.asi8)
assert_equal(result, expected)


def test_period(transform_assert_equal):
transform, assert_equal = transform_assert_equal

idx = pd.period_range("2011-01", periods=3, freq="M", name="")
inp = transform(idx)

if isinstance(inp, Index):
result = to_numeric(inp)
expected = transform(idx.asi8)
assert_equal(result, expected)
else:
# TODO: PeriodDtype, so support it in to_numeric.
pytest.skip("Missing PeriodDtype support in to_numeric")


@pytest.mark.parametrize(
"errors,expected",
[
Expand Down Expand Up @@ -450,7 +462,6 @@ def test_errors_invalid_value():
[
["1", 2, 3],
[1, 2, 3],
np.array(["1970-01-02", "1970-01-03", "1970-01-04"], dtype="datetime64[D]"),
],
)
@pytest.mark.parametrize(
Expand Down Expand Up @@ -478,7 +489,6 @@ def test_downcast_basic(data, kwargs, exp_dtype):
[
["1", 2, 3],
[1, 2, 3],
np.array(["1970-01-02", "1970-01-03", "1970-01-04"], dtype="datetime64[D]"),
],
)
def test_signed_downcast(data, signed_downcast):
Expand Down