From d02aa49c440e553c694e046abb05bb8f698bd38d Mon Sep 17 00:00:00 2001 From: Luke Manley Date: Sat, 25 Feb 2023 21:23:14 +0100 Subject: [PATCH 1/2] BUG: Series.any and Series.all for empty or all-null pyarrow-backed dtypes (#51631) * BUG: ArrowExtensionArray boolean reductions for empty or all null array * whatsnew * clarify whatsnew --- pandas/core/arrays/arrow/array.py | 4 ++++ pandas/tests/extension/test_arrow.py | 16 ++++++++++------ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/pandas/core/arrays/arrow/array.py b/pandas/core/arrays/arrow/array.py index e9122544a18f5..c0e23d98ad411 100644 --- a/pandas/core/arrays/arrow/array.py +++ b/pandas/core/arrays/arrow/array.py @@ -1145,6 +1145,10 @@ def pyarrow_meth(data, skip_nulls, **kwargs): # Let ExtensionArray._reduce raise the TypeError return super()._reduce(name, skipna=skipna, **kwargs) + # GH51624: pyarrow defaults to min_count=1, pandas behavior is min_count=0 + if name in ["any", "all"] and "min_count" not in kwargs: + kwargs["min_count"] = 0 + try: result = pyarrow_meth(data_to_reduce, skip_nulls=skipna, **kwargs) except (AttributeError, NotImplementedError, TypeError) as err: diff --git a/pandas/tests/extension/test_arrow.py b/pandas/tests/extension/test_arrow.py index 8555f26e84c16..c770aeb796706 100644 --- a/pandas/tests/extension/test_arrow.py +++ b/pandas/tests/extension/test_arrow.py @@ -2328,9 +2328,13 @@ def test_dt_tz_localize(unit): tm.assert_series_equal(result, expected) -def test_concat_empty_arrow_backed_series(dtype): - # GH#51734 - ser = pd.Series([], dtype=dtype) - expected = ser.copy() - result = pd.concat([ser[np.array([], dtype=np.bool_)]]) - tm.assert_series_equal(result, expected) +@pytest.mark.parametrize("skipna", [True, False]) +def test_boolean_reduce_series_all_null(all_boolean_reductions, skipna): + # GH51624 + ser = pd.Series([None], dtype="float64[pyarrow]") + result = getattr(ser, all_boolean_reductions)(skipna=skipna) + if skipna: + expected = all_boolean_reductions == "all" + else: + expected = pd.NA + assert result is expected From 6b97adcb77104753b22a9714de040a593032e08d Mon Sep 17 00:00:00 2001 From: Patrick Hoefler Date: Wed, 15 Mar 2023 21:33:18 +0100 Subject: [PATCH 2/2] Add whatsnew --- doc/source/whatsnew/v2.0.0.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v2.0.0.rst b/doc/source/whatsnew/v2.0.0.rst index ba2f504b4c944..a06d37d164bc8 100644 --- a/doc/source/whatsnew/v2.0.0.rst +++ b/doc/source/whatsnew/v2.0.0.rst @@ -1360,6 +1360,7 @@ ExtensionArray - Bug in :class:`Series` constructor unnecessarily overflowing for nullable unsigned integer dtypes (:issue:`38798`, :issue:`25880`) - Bug in setting non-string value into ``StringArray`` raising ``ValueError`` instead of ``TypeError`` (:issue:`49632`) - Bug in :meth:`DataFrame.reindex` not honoring the default ``copy=True`` keyword in case of columns with ExtensionDtype (and as a result also selecting multiple columns with getitem (``[]``) didn't correctly result in a copy) (:issue:`51197`) +- Bug in :meth:`Series.any` and :meth:`Series.all` returning ``NA`` for empty or all null pyarrow-backed data when ``skipna=True`` (:issue:`51624`) - Bug in :class:`~arrays.ArrowExtensionArray` logical operations ``&`` and ``|`` raising ``KeyError`` (:issue:`51688`) Styler