Skip to content

REF: use should_extension_dispatch for comparison method #27912

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 20 commits into from
Aug 19, 2019
Merged
Changes from 17 commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
e18a291
REF: implement should_extension_dispatch
jbrockmendel Aug 8, 2019
10eacc4
dummy to force CI
jbrockmendel Aug 8, 2019
731ac49
Merge branch 'master' of https://github.com/pandas-dev/pandas into sh…
jbrockmendel Aug 8, 2019
79d6abe
Dummy to force CI
jbrockmendel Aug 8, 2019
919d015
Merge branch 'master' of https://github.com/pandas-dev/pandas into sh…
jbrockmendel Aug 12, 2019
28b287f
Merge branch 'master' of https://github.com/pandas-dev/pandas into sh…
jbrockmendel Aug 13, 2019
9f5bf6e
add types
jbrockmendel Aug 13, 2019
7295a9e
Merge branch 'master' of https://github.com/pandas-dev/pandas into sh…
jbrockmendel Aug 13, 2019
69ea8a2
BUG: fix Sparse reduction (#27890)
jbrockmendel Aug 13, 2019
8b8bca0
use should_extension_dispatch
jbrockmendel Aug 14, 2019
57892a1
simplify comparison method
jbrockmendel Aug 14, 2019
b119a24
Merge branch 'master' of https://github.com/pandas-dev/pandas into sh…
jbrockmendel Aug 14, 2019
4b9ce68
fix numpy warning
jbrockmendel Aug 14, 2019
4f31a3b
Merge branch 'master' of https://github.com/pandas-dev/pandas into sh…
jbrockmendel Aug 14, 2019
1d414df
Merge branch 'master' of https://github.com/pandas-dev/pandas into sh…
jbrockmendel Aug 15, 2019
69f7db2
fix deprecationwaring
jbrockmendel Aug 15, 2019
c2df30a
catch better
jbrockmendel Aug 15, 2019
1ef0be1
Merge branch 'master' of https://github.com/pandas-dev/pandas into sh…
jbrockmendel Aug 15, 2019
97f75dc
Fix check
jbrockmendel Aug 15, 2019
dddd0af
Merge branch 'master' of https://github.com/pandas-dev/pandas into sh…
jbrockmendel Aug 16, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 13 additions & 56 deletions pandas/core/ops/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import datetime
import operator
from typing import Any, Callable, Tuple
import warnings

import numpy as np

Expand All @@ -23,7 +24,6 @@
is_bool_dtype,
is_categorical_dtype,
is_datetime64_dtype,
is_datetime64tz_dtype,
is_datetimelike_v_numeric,
is_extension_array_dtype,
is_integer_dtype,
Expand All @@ -32,7 +32,6 @@
is_period_dtype,
is_scalar,
is_timedelta64_dtype,
needs_i8_conversion,
)
from pandas.core.dtypes.generic import (
ABCDataFrame,
Expand Down Expand Up @@ -605,14 +604,20 @@ def dispatch_to_extension_op(op, left, right):
new_right = extract_array(right, extract_numpy=True)

try:
res_values = op(new_left, new_right)
with warnings.catch_warnings():
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can you add a comment on the condition on what this is filtering. can we pre-empt the DeprecationWarning by a check?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

turns out this was the wrong/suboptimal place to do this check. so this catch_warnings is now removed entirely and this is handled by checking for ABCExtensionArray before we get here

warnings.filterwarnings("ignore", category=DeprecationWarning)
res_values = op(new_left, new_right)
except NullFrequencyError:
# DatetimeIndex and TimedeltaIndex with freq == None raise ValueError
# on add/sub of integers (or int-like). We re-raise as a TypeError.
raise TypeError(
"incompatible type for a datetime/timedelta "
"operation [{name}]".format(name=op.__name__)
)
if isinstance(res_values, bool):
# numpy returned False instead of operating pointwise
assert op.__name__ in ["eq", "ne", "ge", "gt", "le", "lt"], op
return invalid_comparison(new_left, new_right, op)
return res_values


Expand Down Expand Up @@ -758,7 +763,6 @@ def _comp_method_SERIES(cls, op, special):
code duplication.
"""
op_name = _get_op_name(op, special)
masker = _gen_eval_kwargs(op_name).get("masker", False)

def na_op(x, y):
# TODO:
Expand All @@ -777,31 +781,11 @@ def na_op(x, y):

else:

# we want to compare like types
# we only want to convert to integer like if
# we are not NotImplemented, otherwise
# we would allow datetime64 (but viewed as i8) against
# integer comparisons

# we have a datetime/timedelta and may need to convert
assert not needs_i8_conversion(x)
mask = None
if not is_scalar(y) and needs_i8_conversion(y):
mask = isna(x) | isna(y)
y = y.view("i8")
x = x.view("i8")

method = getattr(x, op_name, None)
if method is not None:
with np.errstate(all="ignore"):
result = method(y)
if result is NotImplemented:
return invalid_comparison(x, y, op)
else:
result = op(x, y)

if mask is not None and mask.any():
result[mask] = masker
with np.errstate(all="ignore"):
result = method(y)
if result is NotImplemented:
return invalid_comparison(x, y, op)

return result

Expand Down Expand Up @@ -831,12 +815,6 @@ def wrapper(self, other, axis=None):

if isinstance(other, ABCSeries) and not self._indexed_same(other):
raise ValueError("Can only compare identically-labeled Series objects")
elif (
is_list_like(other)
and len(other) != len(self)
and not isinstance(other, (set, frozenset))
):
raise ValueError("Lengths must match")

elif isinstance(other, (np.ndarray, ABCIndexClass, ABCSeries)):
# TODO: make this treatment consistent across ops and classes.
Expand All @@ -845,28 +823,7 @@ def wrapper(self, other, axis=None):
if len(self) != len(other):
raise ValueError("Lengths must match to compare")

if is_categorical_dtype(self):
# Dispatch to Categorical implementation; CategoricalIndex
# behavior is non-canonical GH#19513
res_values = dispatch_to_extension_op(op, self, other)

elif is_datetime64_dtype(self) or is_datetime64tz_dtype(self):
# Dispatch to DatetimeIndex to ensure identical
# Series/Index behavior
from pandas.core.arrays import DatetimeArray

res_values = dispatch_to_extension_op(op, DatetimeArray(self), other)

elif is_timedelta64_dtype(self):
from pandas.core.arrays import TimedeltaArray

res_values = dispatch_to_extension_op(op, TimedeltaArray(self), other)

elif is_extension_array_dtype(self) or (
is_extension_array_dtype(other) and not is_scalar(other)
):
# Note: the `not is_scalar(other)` condition rules out
# e.g. other == "category"
if should_extension_dispatch(self, other):
res_values = dispatch_to_extension_op(op, self, other)

elif is_scalar(other) and isna(other):
Expand Down