BUG: Dataframe.fillna with np.nan for dtype=category(GH 14021)

closes #14021

Author: John Liekezer <[email protected]>

Closes #14051 from conquistador1492/issue_14021 and squashes the following commits:

a405777 [John Liekezer] BUG: Dataframe.fillna with np.nan for dtype=category(GH 14021)

Author: conquistador1492

doc/source/whatsnew/v0.19.0.txt

@@ -1188,7 +1188,8 @@ Bug Fixes
 - Bug in ``Series`` arithmetic raises ``TypeError`` if it contains datetime-like as ``object`` dtype (:issue:`13043`)
 
 - Bug ``Series.isnull()`` and ``Series.notnull()`` ignore ``Period('NaT')``  (:issue:`13737`)
-- Bug ``Series.fillna()`` and ``Series.dropna()`` don't affect to ``Period('NaT')``  (:issue:`13737`)
+- Bug ``Series.fillna()`` and ``Series.dropna()`` don't affect to ``Period('NaT')``  (:issue:`13737`
+- Bug in ``.fillna(value=np.nan)`` incorrectly raises ``KeyError`` on a ``category`` dtyped ``Series`` (:issue:`14021`)
 
 - Bug in extension dtype creation where the created types were not is/identical (:issue:`13285`)
 - Bug in ``.resample(..)`` where incorrect warnings were triggered by IPython introspection (:issue:`13618`)

pandas/core/categorical.py

@@ -1464,7 +1464,10 @@ def fillna(self, value=None, method=None, limit=None):
             mask = values == -1
             if mask.any():
                 values = values.copy()
-                values[mask] = self.categories.get_loc(value)
+                if isnull(value):
+                    values[mask] = -1
+                else:
+                    values[mask] = self.categories.get_loc(value)
 
         return self._constructor(values, categories=self.categories,
                                  ordered=self.ordered, fastpath=True)

pandas/tests/test_categorical.py

@@ -4118,6 +4118,37 @@ def f():
         res = df.fillna("a")
         tm.assert_frame_equal(res, df_exp)
 
+        # GH 14021
+        # np.nan should always be a is a valid filler
+        cat = Categorical([np.nan, 2, np.nan])
+        val = Categorical([np.nan, np.nan, np.nan])
+        df = DataFrame({"cats": cat, "vals": val})
+        res = df.fillna(df.median())
+        v_exp = [np.nan, np.nan, np.nan]
+        df_exp = pd.DataFrame({"cats": [2, 2, 2], "vals": v_exp},
+                              dtype='category')
+        tm.assert_frame_equal(res, df_exp)
+
+        result = df.cats.fillna(np.nan)
+        tm.assert_series_equal(result, df.cats)
+        result = df.vals.fillna(np.nan)
+        tm.assert_series_equal(result, df.vals)
+
+        idx = pd.DatetimeIndex(['2011-01-01 09:00', '2016-01-01 23:45',
+                                '2011-01-01 09:00', pd.NaT, pd.NaT])
+        df = DataFrame({'a': pd.Categorical(idx)})
+        tm.assert_frame_equal(df.fillna(value=pd.NaT), df)
+
+        idx = pd.PeriodIndex(['2011-01', '2011-01', '2011-01',
+                              pd.NaT, pd.NaT], freq='M')
+        df = DataFrame({'a': pd.Categorical(idx)})
+        tm.assert_frame_equal(df.fillna(value=pd.NaT), df)
+
+        idx = pd.TimedeltaIndex(['1 days', '2 days',
+                                 '1 days', pd.NaT, pd.NaT])
+        df = pd.DataFrame({'a': pd.Categorical(idx)})
+        tm.assert_frame_equal(df.fillna(value=pd.NaT), df)
+
     def test_astype_to_other(self):
 
         s = self.cat['value_group']