From 2710225944c2b368c2faa51e6ead25359f9c1f84 Mon Sep 17 00:00:00 2001 From: KevsterAmp Date: Wed, 7 Aug 2024 15:58:19 +0800 Subject: [PATCH 1/6] preserver dtpye to bool[pyarrow] when calling Series.isna() --- pandas/core/dtypes/missing.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/pandas/core/dtypes/missing.py b/pandas/core/dtypes/missing.py index b9cd6ae2f13e8..3c8f68d0bc554 100644 --- a/pandas/core/dtypes/missing.py +++ b/pandas/core/dtypes/missing.py @@ -28,6 +28,7 @@ is_string_or_object_np_dtype, ) from pandas.core.dtypes.dtypes import ( + ArrowDtype, CategoricalDtype, DatetimeTZDtype, ExtensionDtype, @@ -206,7 +207,19 @@ def _isna(obj): elif isinstance(obj, ABCSeries): result = _isna_array(obj._values) # box - result = obj._constructor(result, index=obj.index, name=obj.name, copy=False) + type(obj.dtype) + if isinstance(obj.dtype, ArrowDtype): + result = obj._constructor( + result, + index=obj.index, + name=obj.name, + copy=False, + dtype="bool[pyarrow]", + ) + else: + result = obj._constructor( + result, index=obj.index, name=obj.name, copy=False + ) return result elif isinstance(obj, ABCDataFrame): return obj.isna() From 85b3ff3105b9c04b412847bd4136ee9b2aa4921e Mon Sep 17 00:00:00 2001 From: KevsterAmp Date: Wed, 7 Aug 2024 16:08:12 +0800 Subject: [PATCH 2/6] remove unused type(obj.dtype) --- pandas/core/dtypes/missing.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/core/dtypes/missing.py b/pandas/core/dtypes/missing.py index 3c8f68d0bc554..23a5781cbb41b 100644 --- a/pandas/core/dtypes/missing.py +++ b/pandas/core/dtypes/missing.py @@ -207,7 +207,6 @@ def _isna(obj): elif isinstance(obj, ABCSeries): result = _isna_array(obj._values) # box - type(obj.dtype) if isinstance(obj.dtype, ArrowDtype): result = obj._constructor( result, From c099d9bf7191dd7a010fbfafb2d24e7baca9d8d7 Mon Sep 17 00:00:00 2001 From: KevsterAmp Date: Wed, 7 Aug 2024 21:36:13 +0800 Subject: [PATCH 3/6] add bug on whatsnew doc --- doc/source/whatsnew/v3.0.0.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v3.0.0.rst b/doc/source/whatsnew/v3.0.0.rst index 32c98fbf9d655..46fef0b426783 100644 --- a/doc/source/whatsnew/v3.0.0.rst +++ b/doc/source/whatsnew/v3.0.0.rst @@ -539,6 +539,7 @@ Conversion - Bug in :meth:`DataFrame.update` bool dtype being converted to object (:issue:`55509`) - Bug in :meth:`Series.astype` might modify read-only array inplace when casting to a string dtype (:issue:`57212`) - Bug in :meth:`Series.reindex` not maintaining ``float32`` type when a ``reindex`` introduces a missing value (:issue:`45857`) +- Bug in :meth:`Series.isna` returning ``bool`` instead of ``bool[pyarrow]`` for Arrow-based Series (:issue:`59431`) Strings ^^^^^^^ From afa238097595eea86fcaef2902691c5e3820f8fd Mon Sep 17 00:00:00 2001 From: KevsterAmp Date: Wed, 7 Aug 2024 21:45:36 +0800 Subject: [PATCH 4/6] fix issue number on bug addition in whatsnew doc --- doc/source/whatsnew/v3.0.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v3.0.0.rst b/doc/source/whatsnew/v3.0.0.rst index 46fef0b426783..01bf4575e4c55 100644 --- a/doc/source/whatsnew/v3.0.0.rst +++ b/doc/source/whatsnew/v3.0.0.rst @@ -539,7 +539,7 @@ Conversion - Bug in :meth:`DataFrame.update` bool dtype being converted to object (:issue:`55509`) - Bug in :meth:`Series.astype` might modify read-only array inplace when casting to a string dtype (:issue:`57212`) - Bug in :meth:`Series.reindex` not maintaining ``float32`` type when a ``reindex`` introduces a missing value (:issue:`45857`) -- Bug in :meth:`Series.isna` returning ``bool`` instead of ``bool[pyarrow]`` for Arrow-based Series (:issue:`59431`) +- Bug in :meth:`Series.isna` returning ``bool`` instead of ``bool[pyarrow]`` for Arrow-based Series (:issue:`59436`) Strings ^^^^^^^ From 9aac9a8843a473bf86eb58ed2ae676889a7759cb Mon Sep 17 00:00:00 2001 From: KevsterAmp Date: Wed, 21 Aug 2024 20:39:31 +0800 Subject: [PATCH 5/6] improve pyarrow addition --- pandas/core/dtypes/missing.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/pandas/core/dtypes/missing.py b/pandas/core/dtypes/missing.py index 23a5781cbb41b..2e2831598596e 100644 --- a/pandas/core/dtypes/missing.py +++ b/pandas/core/dtypes/missing.py @@ -207,18 +207,10 @@ def _isna(obj): elif isinstance(obj, ABCSeries): result = _isna_array(obj._values) # box - if isinstance(obj.dtype, ArrowDtype): - result = obj._constructor( - result, - index=obj.index, - name=obj.name, - copy=False, - dtype="bool[pyarrow]", - ) - else: - result = obj._constructor( - result, index=obj.index, name=obj.name, copy=False - ) + new_dtype = "bool[pyarrow]" if isinstance(obj.dtype, ArrowDtype) else None + result = obj._constructor( + result, index=obj.index, name=obj.name, copy=False, dtype=new_dtype + ) return result elif isinstance(obj, ABCDataFrame): return obj.isna() From d602fc7eb50d1b22e5cfac1451b80b20fed5dc96 Mon Sep 17 00:00:00 2001 From: KevsterAmp Date: Wed, 21 Aug 2024 20:58:39 +0800 Subject: [PATCH 6/6] add test to bool[pyarrow] implementation --- pandas/tests/series/methods/test_isna.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pandas/tests/series/methods/test_isna.py b/pandas/tests/series/methods/test_isna.py index 92bf2945cc0d1..fca1e003c96b3 100644 --- a/pandas/tests/series/methods/test_isna.py +++ b/pandas/tests/series/methods/test_isna.py @@ -34,3 +34,8 @@ def test_isna(self): expected = Series([False, False, True]) tm.assert_series_equal(ser.isna(), expected) tm.assert_series_equal(ser.notna(), ~expected) + + def test_is_valid_na_for_dtype_pyarrow(self): + s = Series([0, None, 4, 5], dtype="u1[pyarrow]") + assert s.isna().dtype == "bool[pyarrow]" + assert s.isna().dtype != "bool"