REGR: isin with nullable types with missing values raising

asv_bench/benchmarks/algos/isin.py

@@ -1,12 +1,11 @@
 import numpy as np
 
-try:
-    from pandas.compat import np_version_under1p20
-except ImportError:
-    from pandas.compat.numpy import _np_version_under1p20 as np_version_under1p20
+from pandas.core.dtypes.cast import is_extension_array_dtype
 
 from pandas import (
     Categorical,
+    Float64Dtype,
+    Int64Dtype,
     NaT,
     Series,
     date_range,
@@ -274,7 +273,15 @@ def time_isin(self, series_type, vals_type):
 
 class IsInLongSeriesLookUpDominates:
     params = [
-        ["int64", "int32", "float64", "float32", "object", "Int64", "Float64"],
+        [
+            "int64",
+            "int32",
+            "float64",
+            "float32",
+            "object",
+            Int64Dtype(),
+            Float64Dtype(),
+        ],
         [5, 1000],
         ["random_hits", "random_misses", "monotone_hits", "monotone_misses"],
     ]
@@ -283,10 +290,6 @@ class IsInLongSeriesLookUpDominates:
     def setup(self, dtype, MaxNumber, series_type):
         N = 10 ** 7
 
-        # https://github.com/pandas-dev/pandas/issues/39844
-        if not np_version_under1p20 and dtype in ("Int64", "Float64"):
-            raise NotImplementedError
-
         if series_type == "random_hits":
             array = np.random.randint(0, MaxNumber, N)
         if series_type == "random_misses":
@@ -297,32 +300,47 @@ def setup(self, dtype, MaxNumber, series_type):
             array = np.arange(N) + MaxNumber
 
         self.series = Series(array).astype(dtype)
-        self.values = np.arange(MaxNumber).astype(dtype)
+
+        if is_extension_array_dtype(dtype):
+            vals_dtype = dtype.type
+        else:
+            vals_dtype = dtype
+        self.values = np.arange(MaxNumber).astype(vals_dtype)
 
     def time_isin(self, dtypes, MaxNumber, series_type):
         self.series.isin(self.values)
 
 
 class IsInLongSeriesValuesDominate:
     params = [
-        ["int64", "int32", "float64", "float32", "object", "Int64", "Float64"],
+        [
+            "int64",
+            "int32",
+            "float64",
+            "float32",
+            "object",
+            Int64Dtype(),
+            Float64Dtype(),
+        ],
         ["random", "monotone"],
     ]
     param_names = ["dtype", "series_type"]
 
     def setup(self, dtype, series_type):
         N = 10 ** 7
 
-        # https://github.com/pandas-dev/pandas/issues/39844
-        if not np_version_under1p20 and dtype in ("Int64", "Float64"):
-            raise NotImplementedError
-
         if series_type == "random":
             vals = np.random.randint(0, 10 * N, N)
         if series_type == "monotone":
             vals = np.arange(N)
 
-        self.values = vals.astype(dtype)
+        if is_extension_array_dtype(dtype):
+            vals_dtype = dtype.type
+        else:
+            vals_dtype = dtype
+
+        self.values = vals.astype(vals_dtype)
+
         M = 10 ** 6 + 1
         self.series = Series(np.arange(M)).astype(dtype)
 

doc/source/whatsnew/v1.3.1.rst

@@ -18,7 +18,7 @@ Fixed regressions
 - :class:`DataFrame` constructed with with an older version of pandas could not be unpickled (:issue:`42345`)
 - Performance regression in constructing a :class:`DataFrame` from a dictionary of dictionaries (:issue:`42338`)
 - Fixed regression in :meth:`DataFrame.agg` dropping values when the DataFrame had an Extension Array dtype, a duplicate index, and ``axis=1`` (:issue:`42380`)
--
+- Fixed regression in :meth:`DataFrame.isin` and :meth:`Series.isin` raising ``TypeError`` with nullable data containing at least one missing value (:issue:`42405`)
 
 .. ---------------------------------------------------------------------------
 

pandas/_libs/lib.pyi

@@ -221,6 +221,7 @@ def array_equivalent_object(
     right: np.ndarray,  # object[:]
 ) -> bool: ...
 def has_infs(arr: np.ndarray) -> bool: ...  # const floating[:]
+def has_NA(arr: np.ndarray) -> bool: ...
 def get_reverse_indexer(
     indexer: np.ndarray,  # const intp_t[:]
     length: int,

pandas/_libs/lib.pyx

@@ -538,6 +538,24 @@ def has_infs(floating[:] arr) -> bool:
     return ret
 
 
+@cython.wraparound(False)
+@cython.boundscheck(False)
+def has_NA(ndarray arr) -> bool:
+    """
+    Return True if NA present in arr, False otherwise
+    """
+    cdef:
+        Py_ssize_t i
+
+    assert arr.ndim == 1, "'arr' must be 1-D."
+
+    for i in range(len(arr)):
+        if arr[i] is C_NA:
+            return True
+
+    return False
+
+
 def maybe_indices_to_slice(ndarray[intp_t] indices, int max_len):
     cdef:
         Py_ssize_t i, n = len(indices)

pandas/core/arrays/masked.py

@@ -403,12 +403,18 @@ def isin(self, values) -> BooleanArray:  # type: ignore[override]
 
         from pandas.core.arrays import BooleanArray
 
-        result = isin(self._data, values)
+        # algorithms.isin will eventually convert values to an ndarray, so no extra
+        # cost to doing it here first
+        values_arr = np.asarray(values)
+        result = isin(self._data, values_arr)
+
         if self._hasna:
-            if libmissing.NA in values:
-                result += self._mask
-            else:
-                result *= np.invert(self._mask)
+            values_have_NA = lib.has_NA(values_arr)
+
+            # For now, NA does not propagate so set result according to presence of NA,
+            # see https://github.com/pandas-dev/pandas/pull/38379 for some discussion
+            result[self._mask] = values_have_NA
+
         mask = np.zeros_like(self, dtype=bool)
         return BooleanArray(result, mask, copy=False)
 

pandas/tests/series/methods/test_isin.py

@@ -156,6 +156,27 @@ def test_isin_float_in_int_series(self, values):
         expected = Series([True, False])
         tm.assert_series_equal(result, expected)
 
+    @pytest.mark.parametrize("dtype", ["boolean", "Int64", "Float64"])
+    @pytest.mark.parametrize(
+        "data,values,expected",
+        [
+            ([0, 1, 0], [1], [False, True, False]),
+            ([0, 1, 0], [1, pd.NA], [False, True, False]),
+            ([0, pd.NA, 0], [1, 0], [True, False, True]),
+            ([0, 1, pd.NA], [1, pd.NA], [False, True, True]),
+            ([0, 1, pd.NA], [1, np.nan], [False, True, False]),
+            ([0, pd.NA, pd.NA], [np.nan, pd.NaT, None], [False, False, False]),
+        ],
+    )
+    def test_isin_masked_types(self, dtype, data, values, expected):
+        # GH#42405
+        ser = Series(data, dtype=dtype)
+
+        result = ser.isin(values)
+        expected = Series(expected, dtype="boolean")
+
+        tm.assert_series_equal(result, expected)
+
 
 @pytest.mark.slow
 def test_isin_large_series_mixed_dtypes_and_nan():