From dd36d94c3e5f04f21fbde7e12da4ef0ee3eb0472 Mon Sep 17 00:00:00 2001 From: Brock Date: Tue, 15 Aug 2023 14:16:47 -0700 Subject: [PATCH 1/3] ENH: support Index.any/all with float, timedelta64 dtypes --- doc/source/whatsnew/v2.1.0.rst | 2 ++ pandas/core/indexes/base.py | 28 +++++++++++--------- pandas/tests/indexes/numeric/test_numeric.py | 8 ++++++ pandas/tests/indexes/test_base.py | 1 + pandas/tests/indexes/test_old_base.py | 26 +++++++++++------- 5 files changed, 43 insertions(+), 22 deletions(-) diff --git a/doc/source/whatsnew/v2.1.0.rst b/doc/source/whatsnew/v2.1.0.rst index d1a689dc60830..adc73a1d068ec 100644 --- a/doc/source/whatsnew/v2.1.0.rst +++ b/doc/source/whatsnew/v2.1.0.rst @@ -266,6 +266,8 @@ Other enhancements - :meth:`DataFrame.to_parquet` and :func:`read_parquet` will now write and read ``attrs`` respectively (:issue:`54346`) - Added support for the DataFrame Consortium Standard (:issue:`54383`) - Performance improvement in :meth:`.GroupBy.quantile` (:issue:`51722`) +- :meth:`Index.all` and :meth:`Index.any` with floating dtypes and timedelta64 dtypes no longer raise ``TypeError``, matching the :meth:`Series.all` and :meth:`Series.any` behavior (:issue:`??`) +- .. --------------------------------------------------------------------------- .. _whatsnew_210.notable_bug_fixes: diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py index 288fd35892fd0..d816a4c2fe700 100644 --- a/pandas/core/indexes/base.py +++ b/pandas/core/indexes/base.py @@ -7215,11 +7215,12 @@ def any(self, *args, **kwargs): """ nv.validate_any(args, kwargs) self._maybe_disable_logical_methods("any") - # error: Argument 1 to "any" has incompatible type "ArrayLike"; expected - # "Union[Union[int, float, complex, str, bytes, generic], Sequence[Union[int, - # float, complex, str, bytes, generic]], Sequence[Sequence[Any]], - # _SupportsArray]" - return np.any(self.values) # type: ignore[arg-type] + vals = self._values + if not isinstance(vals, np.ndarray): + # i.e. EA, call _reduce instead of "any" to get TypeError instead + # of AttributeError + return vals._reduce("any") + return np.any(vals) def all(self, *args, **kwargs): """ @@ -7262,11 +7263,12 @@ def all(self, *args, **kwargs): """ nv.validate_all(args, kwargs) self._maybe_disable_logical_methods("all") - # error: Argument 1 to "all" has incompatible type "ArrayLike"; expected - # "Union[Union[int, float, complex, str, bytes, generic], Sequence[Union[int, - # float, complex, str, bytes, generic]], Sequence[Sequence[Any]], - # _SupportsArray]" - return np.all(self.values) # type: ignore[arg-type] + vals = self._values + if not isinstance(vals, np.ndarray): + # i.e. EA, call _reduce instead of "all" to get TypeError instead + # of AttributeError + return vals._reduce("all") + return np.all(vals) @final def _maybe_disable_logical_methods(self, opname: str_t) -> None: @@ -7275,9 +7277,9 @@ def _maybe_disable_logical_methods(self, opname: str_t) -> None: """ if ( isinstance(self, ABCMultiIndex) - or needs_i8_conversion(self.dtype) - or isinstance(self.dtype, (IntervalDtype, CategoricalDtype)) - or is_float_dtype(self.dtype) + # TODO(3.0): PeriodArray and DatetimeArray any/all will raise, + # so checking needs_i8_conversion will be unnecessary + or (needs_i8_conversion(self.dtype) and self.dtype.kind != "m") ): # This call will raise make_invalid_op(opname)(self) diff --git a/pandas/tests/indexes/numeric/test_numeric.py b/pandas/tests/indexes/numeric/test_numeric.py index 977c7da7d866f..8cd295802a5d1 100644 --- a/pandas/tests/indexes/numeric/test_numeric.py +++ b/pandas/tests/indexes/numeric/test_numeric.py @@ -227,6 +227,14 @@ def test_fillna_float64(self): exp = Index([1.0, "obj", 3.0], name="x") tm.assert_index_equal(idx.fillna("obj"), exp, exact=True) + def test_logical_compat(self, simple_index): + idx = simple_index + assert idx.all() == idx.values.all() + assert idx.any() == idx.values.any() + + assert idx.all() == idx.to_series().all() + assert idx.any() == idx.to_series().any() + class TestNumericInt: @pytest.fixture(params=[np.int64, np.int32, np.int16, np.int8, np.uint64]) diff --git a/pandas/tests/indexes/test_base.py b/pandas/tests/indexes/test_base.py index b3fb5a26ca63f..8e49f9656a140 100644 --- a/pandas/tests/indexes/test_base.py +++ b/pandas/tests/indexes/test_base.py @@ -693,6 +693,7 @@ def test_format_missing(self, vals, nulls_fixture): def test_logical_compat(self, op, simple_index): index = simple_index assert getattr(index, op)() == getattr(index.values, op)() + assert getattr(index, op)() == getattr(index.to_series(), op)() @pytest.mark.parametrize( "index", ["string", "int64", "int32", "float64", "float32"], indirect=True diff --git a/pandas/tests/indexes/test_old_base.py b/pandas/tests/indexes/test_old_base.py index f8f5a543a9c19..79dc423f12a85 100644 --- a/pandas/tests/indexes/test_old_base.py +++ b/pandas/tests/indexes/test_old_base.py @@ -209,17 +209,25 @@ def test_numeric_compat(self, simple_index): 1 // idx def test_logical_compat(self, simple_index): - if ( - isinstance(simple_index, RangeIndex) - or is_numeric_dtype(simple_index.dtype) - or simple_index.dtype == object - ): + if simple_index.dtype == object: pytest.skip("Tested elsewhere.") idx = simple_index - with pytest.raises(TypeError, match="cannot perform all"): - idx.all() - with pytest.raises(TypeError, match="cannot perform any"): - idx.any() + if idx.dtype.kind in "iufcbm": + assert idx.all() == idx._values.all() + assert idx.all() == idx.to_series().all() + assert idx.any() == idx._values.any() + assert idx.any() == idx.to_series().any() + else: + msg = "cannot perform (any|all)" + if isinstance(idx, IntervalIndex): + msg = ( + r"'IntervalArray' with dtype interval\[.*\] does " + "not support reduction '(any|all)'" + ) + with pytest.raises(TypeError, match=msg): + idx.all() + with pytest.raises(TypeError, match=msg): + idx.any() def test_repr_roundtrip(self, simple_index): if isinstance(simple_index, IntervalIndex): From 632bf7d9dcfc86f447e6f84e3778ed723993d826 Mon Sep 17 00:00:00 2001 From: Brock Date: Tue, 15 Aug 2023 14:18:05 -0700 Subject: [PATCH 2/3] GH ref --- doc/source/whatsnew/v2.1.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v2.1.0.rst b/doc/source/whatsnew/v2.1.0.rst index adc73a1d068ec..8ef2446b37a9f 100644 --- a/doc/source/whatsnew/v2.1.0.rst +++ b/doc/source/whatsnew/v2.1.0.rst @@ -264,9 +264,9 @@ Other enhancements - Many read/to_* functions, such as :meth:`DataFrame.to_pickle` and :func:`read_csv`, support forwarding compression arguments to lzma.LZMAFile (:issue:`52979`) - Reductions :meth:`Series.argmax`, :meth:`Series.argmin`, :meth:`Series.idxmax`, :meth:`Series.idxmin`, :meth:`Index.argmax`, :meth:`Index.argmin`, :meth:`DataFrame.idxmax`, :meth:`DataFrame.idxmin` are now supported for object-dtype objects (:issue:`4279`, :issue:`18021`, :issue:`40685`, :issue:`43697`) - :meth:`DataFrame.to_parquet` and :func:`read_parquet` will now write and read ``attrs`` respectively (:issue:`54346`) +- :meth:`Index.all` and :meth:`Index.any` with floating dtypes and timedelta64 dtypes no longer raise ``TypeError``, matching the :meth:`Series.all` and :meth:`Series.any` behavior (:issue:`54566`) - Added support for the DataFrame Consortium Standard (:issue:`54383`) - Performance improvement in :meth:`.GroupBy.quantile` (:issue:`51722`) -- :meth:`Index.all` and :meth:`Index.any` with floating dtypes and timedelta64 dtypes no longer raise ``TypeError``, matching the :meth:`Series.all` and :meth:`Series.any` behavior (:issue:`??`) - .. --------------------------------------------------------------------------- From d37bcdc6cdbd710ea4a51cb3620213fba3bd0f0d Mon Sep 17 00:00:00 2001 From: Brock Date: Mon, 21 Aug 2023 12:34:29 -0700 Subject: [PATCH 3/3] Update test --- pandas/tests/indexes/test_base.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pandas/tests/indexes/test_base.py b/pandas/tests/indexes/test_base.py index 57b5c69b3f57f..bc04c1c6612f4 100644 --- a/pandas/tests/indexes/test_base.py +++ b/pandas/tests/indexes/test_base.py @@ -692,8 +692,12 @@ def test_format_missing(self, vals, nulls_fixture): @pytest.mark.parametrize("op", ["any", "all"]) def test_logical_compat(self, op, simple_index): index = simple_index - assert getattr(index, op)() == getattr(index.values, op)() - assert getattr(index, op)() == getattr(index.to_series(), op)() + left = getattr(index, op)() + assert left == getattr(index.values, op)() + right = getattr(index.to_series(), op)() + # left might not match right exactly in e.g. string cases where the + # because we use np.any/all instead of .any/all + assert bool(left) == bool(right) @pytest.mark.parametrize( "index", ["string", "int64", "int32", "float64", "float32"], indirect=True