stricter na_value requirements

Author: topper-123

pandas/core/arrays/base.py

@@ -37,6 +37,7 @@
     is_array_like,
     is_dtype_equal,
     is_list_like,
+    is_scalar,
     pandas_dtype,
 )
 from pandas.core.dtypes.dtypes import ExtensionDtype
@@ -360,8 +361,10 @@ def __contains__(self, item) -> bool:
         """
         # comparisons of any item to pd.NA always return pd.NA, so e.g. "a" in [pd.NA]
         # would raise a TypeError. The implementation below works around that.
-        if is_valid_nat_for_dtype(item, self.dtype):
+        if item is self.dtype.na_value:
             return isna(self).any() if self._can_hold_na else False
+        elif is_scalar(item) and isna(item):
+            return False
         else:
             return (item == self).any()
 

pandas/tests/arrays/string_/test_string.py

@@ -527,7 +527,7 @@ def test_to_numpy_na_value(dtype, nulls_fixture):
 
 
 def test_contains():
-    # GH-xxxxx
+    # GH-37867
     arr = pd.array(["a", "b"], dtype="string")
 
     assert "a" in arr
@@ -538,5 +538,5 @@ def test_contains():
     arr = pd.array(["a", pd.NA], dtype="string")
     assert "a" in arr
     assert "x" not in arr
-    assert np.nan in arr
+    assert np.nan not in arr
     assert pd.NA in arr

pandas/tests/extension/base/interface.py

@@ -29,6 +29,23 @@ def test_can_hold_na_valid(self, data):
         # GH-20761
         assert data._can_hold_na is True
 
+    def test_contains(self, data):
+        # GH-37867
+        scalar = data[~data.isna()][0]
+
+        assert scalar in data
+
+        na_value = data.dtype.na_value
+        other_na_value_types = {np.nan, pd.NA, pd.NaT}.difference({na_value})
+        if data.isna().any():
+            assert na_value in data
+            for na_value_type in other_na_value_types:
+                assert na_value_type not in data
+        else:
+            assert na_value not in data
+            for na_value_type in other_na_value_types:
+                assert na_value_type not in data
+
     def test_memory_usage(self, data):
         s = pd.Series(data)
         result = s.memory_usage(index=False)

pandas/tests/extension/json/test_json.py

@@ -143,6 +143,11 @@ def test_custom_asserts(self):
         with pytest.raises(AssertionError, match=msg):
             self.assert_frame_equal(a.to_frame(), b.to_frame())
 
+    @pytest.mark.xfail(reason="comparison method not implemented on JSONArray")
+    def test_contains(self, data):
+        # GH-37867
+        super().test_contains(data)
+
 
 class TestConstructors(BaseJSON, base.BaseConstructorsTests):
     @pytest.mark.skip(reason="not implemented constructor from dtype")