BUG: any/all not returning booleans for object type (#41102)

Author: mzeitlin11

doc/source/whatsnew/v1.3.0.rst

@@ -726,6 +726,7 @@ Numeric
 - Bug in :meth:`DataFrame.transform` would raise ``SpecificationError`` when passed a dictionary and columns were missing; will now raise a ``KeyError`` instead (:issue:`40004`)
 - Bug in :meth:`DataFrameGroupBy.rank` giving incorrect results with ``pct=True`` and equal values between consecutive groups (:issue:`40518`)
 - Bug in :meth:`Series.count` would result in an ``int32`` result on 32-bit platforms when argument ``level=None`` (:issue:`40908`)
+- Bug in :class:`Series` and :class:`DataFrame` reductions with methods ``any`` and ``all`` not returning boolean results for object data (:issue:`12863`, :issue:`35450`, :issue:`27709`)
 - Bug in :meth:`Series.clip` would fail if series contains NA values and has nullable int or float as a data type (:issue:`40851`)
 
 Conversion

pandas/core/nanops.py

@@ -486,6 +486,12 @@ def nanany(
     False
     """
     values, _, _, _, _ = _get_values(values, skipna, fill_value=False, mask=mask)
+
+    # For object type, any won't necessarily return
+    # boolean values (numpy/numpy#4352)
+    if is_object_dtype(values):
+        values = values.astype(bool)
+
     # error: Incompatible return value type (got "Union[bool_, ndarray]", expected
     # "bool")
     return values.any(axis)  # type: ignore[return-value]
@@ -526,6 +532,12 @@ def nanall(
     False
     """
     values, _, _, _, _ = _get_values(values, skipna, fill_value=True, mask=mask)
+
+    # For object type, all won't necessarily return
+    # boolean values (numpy/numpy#4352)
+    if is_object_dtype(values):
+        values = values.astype(bool)
+
     # error: Incompatible return value type (got "Union[bool_, ndarray]", expected
     # "bool")
     return values.all(axis)  # type: ignore[return-value]

pandas/tests/apply/test_series_apply.py

@@ -441,8 +441,8 @@ def test_non_callable_aggregates(how):
                 ("sum", "abc"),
                 ("max", "c"),
                 ("min", "a"),
-                ("all", "c"),  # see GH12863
-                ("any", "a"),
+                ("all", True),
+                ("any", True),
             ],
         ),
     ),

pandas/tests/frame/test_reductions.py

@@ -1068,13 +1068,17 @@ def test_idxmax_dt64_multicolumn_axis1(self):
 
     @pytest.mark.parametrize("opname", ["any", "all"])
     def test_any_all(self, opname, bool_frame_with_na, float_string_frame):
-        assert_bool_op_calc(
-            opname, getattr(np, opname), bool_frame_with_na, has_skipna=True
-        )
         assert_bool_op_api(
             opname, bool_frame_with_na, float_string_frame, has_bool_only=True
         )
 
+    @pytest.mark.parametrize("opname", ["any", "all"])
+    def test_any_all_bool_frame(self, opname, bool_frame_with_na):
+        # GH#12863: numpy gives back non-boolean data for object type
+        # so fill NaNs to compare with pandas behavior
+        df = bool_frame_with_na.fillna(True)
+        assert_bool_op_calc(opname, getattr(np, opname), df, has_skipna=True)
+
     def test_any_all_extra(self):
         df = DataFrame(
             {
@@ -1108,6 +1112,24 @@ def test_any_all_extra(self):
         result = df[["C"]].all(axis=None).item()
         assert result is True
 
+    @pytest.mark.parametrize("axis", [0, 1])
+    @pytest.mark.parametrize("bool_agg_func", ["any", "all"])
+    @pytest.mark.parametrize("skipna", [True, False])
+    def test_any_all_object_dtype(self, axis, bool_agg_func, skipna):
+        # GH#35450
+        df = DataFrame(
+            data=[
+                [1, np.nan, np.nan, True],
+                [np.nan, 2, np.nan, True],
+                [np.nan, np.nan, np.nan, True],
+                [np.nan, np.nan, "5", np.nan],
+            ]
+        )
+
+        result = getattr(df, bool_agg_func)(axis=axis, skipna=skipna)
+        expected = Series([True, True, True, True])
+        tm.assert_series_equal(result, expected)
+
     def test_any_datetime(self):
 
         # GH 23070

pandas/tests/reductions/test_reductions.py

@@ -896,7 +896,7 @@ def test_all_any(self):
 
         # Alternative types, with implicit 'object' dtype.
         s = Series(["abc", True])
-        assert "abc" == s.any()  # 'abc' || True => 'abc'
+        assert s.any()
 
     @pytest.mark.parametrize("klass", [Index, Series])
     def test_numpy_all_any(self, klass):
@@ -913,7 +913,7 @@ def test_all_any_params(self):
         s2 = Series([np.nan, False])
         assert s1.all(skipna=False)  # nan && True => True
         assert s1.all(skipna=True)
-        assert np.isnan(s2.any(skipna=False))  # nan || False => nan
+        assert s2.any(skipna=False)
         assert not s2.any(skipna=True)
 
         # Check level.
@@ -941,6 +941,29 @@ def test_all_any_params(self):
         with pytest.raises(NotImplementedError, match=msg):
             s.all(bool_only=True)
 
+    @pytest.mark.parametrize("bool_agg_func", ["any", "all"])
+    @pytest.mark.parametrize("skipna", [True, False])
+    def test_any_all_object_dtype(self, bool_agg_func, skipna):
+        # GH#12863
+        ser = Series(["a", "b", "c", "d", "e"], dtype=object)
+        result = getattr(ser, bool_agg_func)(skipna=skipna)
+        expected = True
+
+        assert result == expected
+
+    @pytest.mark.parametrize("bool_agg_func", ["any", "all"])
+    @pytest.mark.parametrize(
+        "data", [[False, None], [None, False], [False, np.nan], [np.nan, False]]
+    )
+    def test_any_all_object_dtype_missing(self, data, bool_agg_func):
+        # GH#27709
+        ser = Series(data)
+        result = getattr(ser, bool_agg_func)(skipna=False)
+
+        # None is treated is False, but np.nan is treated as True
+        expected = bool_agg_func == "any" and None not in data
+        assert result == expected
+
     @pytest.mark.parametrize("bool_agg_func", ["any", "all"])
     @pytest.mark.parametrize("skipna", [True, False])
     @pytest.mark.parametrize(

pandas/tests/test_nanops.py

@@ -270,6 +270,7 @@ def _badobj_wrap(self, value, func, allow_complex=True, **kwargs):
                 value = value.astype("f8")
         return func(value, **kwargs)
 
+    @pytest.mark.xfail(reason="GH12863: numpy result won't match for object type")
     @pytest.mark.parametrize(
         "nan_op,np_op", [(nanops.nanany, np.any), (nanops.nanall, np.all)]
     )