Backport PR pandas-dev#27932: BUG: Ensure that fill_na in Categorical only replaces null values

Author: MarcoGorelli

doc/source/whatsnew/v0.25.1.rst

@@ -25,7 +25,7 @@ Bug fixes
 Categorical
 ^^^^^^^^^^^
 
--
+- Bug in :meth:`Categorical.fillna` would replace all values, not just those that are ``NaN`` (:issue:`26215`)
 -
 -
 

pandas/core/arrays/categorical.py

@@ -1864,8 +1864,8 @@ def fillna(self, value=None, method=None, limit=None):
                     raise ValueError("fill value must be in categories")
 
                 values_codes = _get_codes_for_values(value, self.categories)
-                indexer = np.where(values_codes != -1)
-                codes[indexer] = values_codes[values_codes != -1]
+                indexer = np.where(codes == -1)
+                codes[indexer] = values_codes[indexer]
 
             # If value is not a dict or Series it should be a scalar
             elif is_hashable(value):

pandas/tests/series/test_missing.py

@@ -578,6 +578,28 @@ def test_fillna_categorical(self, fill_value, expected_output):
         exp = Series(Categorical(expected_output, categories=["a", "b"]))
         tm.assert_series_equal(s.fillna(fill_value), exp)
 
+    @pytest.mark.parametrize(
+        "fill_value, expected_output",
+        [
+            (Series(["a", "b", "c", "d", "e"]), ["a", "b", "b", "d", "e"]),
+            (Series(["b", "d", "a", "d", "a"]), ["a", "d", "b", "d", "a"]),
+            (
+                Series(
+                    Categorical(
+                        ["b", "d", "a", "d", "a"], categories=["b", "c", "d", "e", "a"]
+                    )
+                ),
+                ["a", "d", "b", "d", "a"],
+            ),
+        ],
+    )
+    def test_fillna_categorical_with_new_categories(self, fill_value, expected_output):
+        # GH 26215
+        data = ["a", np.nan, "b", np.nan, np.nan]
+        s = Series(Categorical(data, categories=["a", "b", "c", "d", "e"]))
+        exp = Series(Categorical(expected_output, categories=["a", "b", "c", "d", "e"]))
+        tm.assert_series_equal(s.fillna(fill_value), exp)
+
     def test_fillna_categorical_raise(self):
         data = ["a", np.nan, "b", np.nan, np.nan]
         s = Series(Categorical(data, categories=["a", "b"]))