Skip to content

CLN: remove unnecessary check needs_i8_conversion if Index subclass does not support any or all #58006

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 2 additions & 9 deletions pandas/core/indexes/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,6 @@
)
from pandas.core.missing import clean_reindex_fill_method
from pandas.core.ops import get_op_result_name
from pandas.core.ops.invalid import make_invalid_op
from pandas.core.sorting import (
ensure_key_mapped,
get_group_index_sorter,
Expand Down Expand Up @@ -6960,14 +6959,8 @@ def _maybe_disable_logical_methods(self, opname: str_t) -> None:
"""
raise if this Index subclass does not support any or all.
"""
if (
isinstance(self, ABCMultiIndex)
# TODO(3.0): PeriodArray and DatetimeArray any/all will raise,
# so checking needs_i8_conversion will be unnecessary
or (needs_i8_conversion(self.dtype) and self.dtype.kind != "m")
):
# This call will raise
make_invalid_op(opname)(self)
if isinstance(self, ABCMultiIndex) or self.dtype.kind != "m":
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is the != "m" bit still needed?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for the comment. I think we don't need != "m" here. This check worked only together with the check needs_i8_conversion(self.dtype).

I removed != "m" and corrected tests.

I have a question: shouldn't we raise for DatetimeIndex? So far we show FutureWarning:
"'any' with datetime64 dtypes is deprecated and will raise in a future version. Use (obj != pd.Timestamp(0)).any() instead."

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

there is a deprecation in nanany and nanall that i think is intended to be enforced before this needs_i8_conversion check is removed

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

thanks, it was my thought too.
Should I open new PR to enforce the deprecation in nanany and nanall, or can I do it in this PR?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I enforced deprecation any/all with datetime64 in #58029

raise TypeError("cannot perform all with this index type: Index")

@Appender(IndexOpsMixin.argmin.__doc__)
def argmin(self, axis=None, skipna: bool = True, *args, **kwargs) -> int:
Expand Down
21 changes: 14 additions & 7 deletions pandas/tests/indexes/numeric/test_numeric.py
Original file line number Diff line number Diff line change
Expand Up @@ -223,11 +223,14 @@ def test_fillna_float64(self):

def test_logical_compat(self, dtype):
idx = Index(np.arange(5, dtype=dtype))
assert idx.all() == idx.values.all()
assert idx.any() == idx.values.any()
msg = "cannot perform all with this index type: Index"

assert idx.all() == idx.to_series().all()
assert idx.any() == idx.to_series().any()
with pytest.raises(TypeError, match=msg):
assert idx.all() == idx.values.all()
assert idx.any() == idx.values.any()

assert idx.all() == idx.to_series().all()
assert idx.any() == idx.to_series().any()


class TestNumericInt:
Expand Down Expand Up @@ -278,10 +281,14 @@ def test_is_strictly_monotonic(self):
assert not index._is_strictly_monotonic_increasing
assert not index._is_strictly_monotonic_decreasing

def test_logical_compat(self, simple_index):
def test_logical_compat(self, simple_index, request):
idx = simple_index
assert idx.all() == idx.values.all()
assert idx.any() == idx.values.any()
msg = "cannot perform all with this index type: Index"

if request.node.callspec.id not in ["int", "int64"]:
with pytest.raises(TypeError, match=msg):
assert idx.all() == idx.values.all()
assert idx.any() == idx.values.any()

def test_identical(self, simple_index, dtype):
index = simple_index
Expand Down
19 changes: 11 additions & 8 deletions pandas/tests/indexes/test_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -688,12 +688,12 @@ def test_summary(self, index):

def test_logical_compat(self, all_boolean_reductions, simple_index):
index = simple_index
left = getattr(index, all_boolean_reductions)()
assert left == getattr(index.values, all_boolean_reductions)()
right = getattr(index.to_series(), all_boolean_reductions)()
# left might not match right exactly in e.g. string cases where the
# because we use np.any/all instead of .any/all
assert bool(left) == bool(right)
msg = "cannot perform (any|all) with this index type: Index"

with pytest.raises(TypeError, match=msg):
getattr(index, all_boolean_reductions)()
getattr(index.values, all_boolean_reductions)()
getattr(index.to_series(), all_boolean_reductions)()

@pytest.mark.parametrize(
"index", ["string", "int64", "int32", "float64", "float32"], indirect=True
Expand Down Expand Up @@ -1393,8 +1393,11 @@ def test_unique_na(self):

def test_logical_compat(self, simple_index):
index = simple_index
assert index.all() == index.values.all()
assert index.any() == index.values.any()
msg = "cannot perform (any|all) with this index type: Index"

with pytest.raises(TypeError, match=msg):
assert index.all() == index.values.all()
assert index.any() == index.values.any()

@pytest.mark.parametrize("how", ["any", "all"])
@pytest.mark.parametrize("dtype", [None, object, "category"])
Expand Down
22 changes: 12 additions & 10 deletions pandas/tests/indexes/test_old_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -212,22 +212,24 @@ def test_numeric_compat(self, simple_index):
with pytest.raises(TypeError, match=floordiv_err):
1 // idx

def test_logical_compat(self, simple_index):
def test_logical_compat(self, simple_index, request):
if simple_index.dtype in (object, "string"):
pytest.skip("Tested elsewhere.")
idx = simple_index

if idx.dtype.kind in "iufcbm":
assert idx.all() == idx._values.all()
assert idx.all() == idx.to_series().all()
assert idx.any() == idx._values.any()
assert idx.any() == idx.to_series().any()
msg = "cannot perform (any|all) with this index type: Index"

if request.node.callspec.id in [
"".join(["simple_index", t]) for t in ["1", "2", "3", "5", "6", "7"]
]:
with pytest.raises(TypeError, match=msg):
assert idx.all() == idx._values.all()
assert idx.all() == idx.to_series().all()
assert idx.any() == idx._values.any()
assert idx.any() == idx.to_series().any()
else:
msg = "cannot perform (any|all)"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think you can delete L225-L228? i.e. the only thing needed here to pass tests is msg = "does not support reduction '(any|all)'"?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

thanks, I deleted the branch with msg = "datetime64 type does not support operation: '(any|all)'", ci- green.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh OK I see the issue now. So you are branching because DatetimeIndex says does not support operation: '(any|all)' but the other cases say 'does not support reduction '(any|all)'

I don't want to go down a rabbit hole but having a message that differs between operation: and reduction doesn't seem that important. How hard is it to align them so you can just assign msg = "does not support operation: '(any|all)' and have the tests pass?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

L225 looks like it can be safely deleted

if isinstance(idx, IntervalIndex):
msg = (
r"'IntervalArray' with dtype interval\[.*\] does "
"not support reduction '(any|all)'"
)
with pytest.raises(TypeError, match=msg):
idx.all()
with pytest.raises(TypeError, match=msg):
Expand Down
Loading