Backport PR pandas-dev#33629 on branch 1.0.x (BUG: Fix Categorical use_inf_as_na bug)

Author: dsaxton

doc/source/whatsnew/v1.0.4.rst

@@ -15,6 +15,7 @@ including other versions of pandas.
 
 Fixed regressions
 ~~~~~~~~~~~~~~~~~
+- Bug where :meth:`Series.isna` and :meth:`DataFrame.isna` would raise for categorical dtype when ``pandas.options.mode.use_inf_as_na`` was set to ``True`` (:issue:`33594`)
 - Bug in :meth:`GroupBy.first` and :meth:`GroupBy.last` where None is not preserved in object dtype (:issue:`32800`)
 - 
 

pandas/core/dtypes/missing.py

@@ -269,10 +269,18 @@ def _isna_ndarraylike(obj):
 
 
 def _isna_ndarraylike_old(obj):
+    is_extension = is_extension_array_dtype(obj)
+
     values = getattr(obj, "values", obj)
     dtype = values.dtype
 
-    if is_string_dtype(dtype):
+    if is_extension:
+        if isinstance(obj, (ABCIndexClass, ABCSeries)):
+            values = obj._values
+        else:
+            values = obj
+        result = values.isna() | (values == -np.inf) | (values == np.inf)
+    elif is_string_dtype(dtype):
         # Working around NumPy ticket 1542
         shape = values.shape
 

pandas/tests/arrays/categorical/test_missing.py

@@ -5,7 +5,8 @@
 
 from pandas.core.dtypes.dtypes import CategoricalDtype
 
-from pandas import Categorical, Index, Series, isna
+import pandas as pd
+from pandas import Categorical, DataFrame, Index, Series, isna
 import pandas._testing as tm
 
 
@@ -82,3 +83,53 @@ def test_fillna_iterable_category(self, named):
         expected = Categorical([Point(0, 0), Point(0, 1), Point(0, 0)])
 
         tm.assert_categorical_equal(result, expected)
+
+    @pytest.mark.parametrize(
+        "values, expected",
+        [
+            ([1, 2, 3], np.array([False, False, False])),
+            ([1, 2, np.nan], np.array([False, False, True])),
+            ([1, 2, np.inf], np.array([False, False, True])),
+            ([1, 2, pd.NA], np.array([False, False, True])),
+        ],
+    )
+    def test_use_inf_as_na(self, values, expected):
+        # https://github.com/pandas-dev/pandas/issues/33594
+        with pd.option_context("mode.use_inf_as_na", True):
+            cat = Categorical(values)
+            result = cat.isna()
+            tm.assert_numpy_array_equal(result, expected)
+
+            result = Series(cat).isna()
+            expected = Series(expected)
+            tm.assert_series_equal(result, expected)
+
+            result = DataFrame(cat).isna()
+            expected = DataFrame(expected)
+            tm.assert_frame_equal(result, expected)
+
+    @pytest.mark.parametrize(
+        "values, expected",
+        [
+            ([1, 2, 3], np.array([False, False, False])),
+            ([1, 2, np.nan], np.array([False, False, True])),
+            ([1, 2, np.inf], np.array([False, False, True])),
+            ([1, 2, pd.NA], np.array([False, False, True])),
+        ],
+    )
+    def test_use_inf_as_na_outside_context(self, values, expected):
+        # https://github.com/pandas-dev/pandas/issues/33594
+        # Using isna directly for Categorical will fail in general here
+        cat = Categorical(values)
+
+        with pd.option_context("mode.use_inf_as_na", True):
+            result = pd.isna(cat)
+            tm.assert_numpy_array_equal(result, expected)
+
+            result = pd.isna(Series(cat))
+            expected = Series(expected)
+            tm.assert_series_equal(result, expected)
+
+            result = pd.isna(DataFrame(cat))
+            expected = DataFrame(expected)
+            tm.assert_frame_equal(result, expected)