Skip to content

WIP: reduce unwanted type conversions of pd.Series.isin #19508

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 9 commits into from
Closed

WIP: reduce unwanted type conversions of pd.Series.isin #19508

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
a482acd
dirty fix and add integration test
jdoepfert Feb 1, 2018
e5f6245
unit tests pass
jdoepfert Feb 2, 2018
0ea7348
base tests fail
jdoepfert Feb 2, 2018
c94a117
base tests pass
jdoepfert Feb 2, 2018
3b66275
rearrange if clause
jdoepfert Feb 2, 2018
21569bb
add timestamp check
jdoepfert Feb 2, 2018
bbd442a
change test
jdoepfert Feb 2, 2018
cd14c56
extend regression test
jdoepfert Feb 2, 2018
80d9c3d
new approach, some tests fail
jdoepfert Feb 5, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 24 additions & 6 deletions pandas/core/algorithms.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,14 @@
ABCIndexClass, ABCCategorical)
from pandas.core.dtypes.common import (
is_unsigned_integer_dtype, is_signed_integer_dtype,
is_integer_dtype, is_complex_dtype,
is_integer_dtype, is_complex_dtype, is_integer, is_float,
is_object_dtype,
is_categorical_dtype, is_sparse,
is_period_dtype,
is_numeric_dtype, is_float_dtype,
is_bool_dtype, needs_i8_conversion,
is_categorical, is_datetimetz,
is_datetime64_any_dtype, is_datetime64tz_dtype,
is_categorical, is_datetimetz, is_datetime_or_timedelta_dtype,
is_datetime64_any_dtype, is_datetime64tz_dtype, is_datetimelike,
is_timedelta64_dtype, is_interval_dtype,
is_scalar, is_list_like,
_ensure_platform_int, _ensure_object,
Expand All @@ -32,6 +32,7 @@
from pandas.core import common as com
from pandas._libs import algos, lib, hashtable as htable
from pandas._libs.tslib import iNaT
from pandas._libs.tslibs.timestamps import Timestamp


# --------------- #
Expand Down Expand Up @@ -405,7 +406,21 @@ def isin(comps, values):
values = construct_1d_object_array_from_listlike(list(values))

comps, dtype, _ = _ensure_data(comps)
values, _, _ = _ensure_data(values, dtype=dtype)
# Convert `values` to `dtype` if dtype of `values` is like `dtype`
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

you don't need to check all these conditions, rather you can just do

In [14]: l = [1.0, 1]

In [15]: set([type(v) for v in l])
Out[15]: {float, int}

If you have a set of len(1) then you can go ahead and pass the dtype (else you can use object).

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@jreback thx for the comment, not sure if I got you right though. Imagine I have

pd.Series([1, 0]).isin([1.0, 0.5])

Then the len of the set would be one (I only have floats), but the conversion to int would still not be desirable.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

thats a different case which is actually simple. the dtype that ensure_data returns is different from the dtype that the set check returns.

NONE of this checking can be done here, rather it is already done in ensure_data

check_int = (is_integer_dtype(dtype) and
(is_integer_dtype(values) or
all(is_integer(i) for i in values)))
check_float = (is_float_dtype(dtype) and
(is_float_dtype(values) or
all(is_float(i) for i in values)))
check_datetime = (is_datetime_or_timedelta_dtype(dtype) and
(is_datetime_or_timedelta_dtype(values) or
all(is_datetimelike(i) for i in values) or
all(isinstance(i, Timestamp) for i in values)))
if check_int or check_float or check_datetime:
values, _, _ = _ensure_data(values, dtype=dtype)
else:
values, _, _ = _ensure_data(values)

# faster for larger cases to use np.in1d
f = lambda x, y: htable.ismember_object(x, values)
Expand All @@ -414,7 +429,7 @@ def isin(comps, values):
# Ensure np.in1d doesn't get object types or it *may* throw an exception
if len(comps) > 1000000 and not is_object_dtype(comps):
f = lambda x, y: np.in1d(x, y)
elif is_integer_dtype(comps):
elif is_integer_dtype(comps) and is_integer_dtype(values):
try:
values = values.astype('int64', copy=False)
comps = comps.astype('int64', copy=False)
Expand All @@ -423,7 +438,7 @@ def isin(comps, values):
values = values.astype(object)
comps = comps.astype(object)

elif is_float_dtype(comps):
elif is_float_dtype(comps) and is_float_dtype(values):
try:
values = values.astype('float64', copy=False)
comps = comps.astype('float64', copy=False)
Expand All @@ -432,6 +447,9 @@ def isin(comps, values):
except (TypeError, ValueError):
values = values.astype(object)
comps = comps.astype(object)
else:
values = values.astype(object)
comps = comps.astype(object)

return f(comps, values)

Expand Down
13 changes: 13 additions & 0 deletions pandas/tests/test_algos.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -555,6 +555,19 @@ def test_empty(self, empty):
result = algos.isin(vals, empty)
tm.assert_numpy_array_equal(expected, result)

def test_regression_issue_19356(self):
# Regression test for GH19356
l = [-9, -0.5]
expected = np.array([True, False])

series_float = pd.Series([-9.0, 0.0])
result_float = series_float.isin(l)
tm.assert_numpy_array_equal(expected, result_float.values)

series_int = pd.Series([-9, 0])
result_int = series_int.isin(l)
tm.assert_numpy_array_equal(expected, result_int.values)


class TestValueCounts(object):

Expand Down