Backport PR pandas-dev#42087: REGR: undocumented astype("category").astype(str) type inconsistency between pandas 1.1 & 1.2

Author: simonjayhawkins

doc/source/whatsnew/v1.2.5.rst

@@ -19,6 +19,7 @@ Fixed regressions
 - Regression in :func:`read_csv` when using ``memory_map=True`` with an non-UTF8 encoding (:issue:`40986`)
 - Regression in :meth:`DataFrame.replace` and :meth:`Series.replace` when the values to replace is a NumPy float array (:issue:`40371`)
 - Regression in :func:`ExcelFile` when a corrupt file is opened but not closed (:issue:`41778`)
+- Fixed regression in :meth:`DataFrame.astype` with ``dtype=str`` failing to convert ``NaN`` in categorical columns (:issue:`41797`)
 
 .. ---------------------------------------------------------------------------
 

pandas/core/arrays/categorical.py

@@ -9,7 +9,7 @@
 
 from pandas._config import get_option
 
-from pandas._libs import NaT, algos as libalgos, hashtable as htable
+from pandas._libs import NaT, algos as libalgos, hashtable as htable, lib
 from pandas._libs.lib import no_default
 from pandas._typing import ArrayLike, Dtype, Ordered, Scalar
 from pandas.compat.numpy import function as nv
@@ -429,14 +429,19 @@ def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike:
             try:
                 new_cats = np.asarray(self.categories)
                 new_cats = new_cats.astype(dtype=dtype, copy=copy)
+                fill_value = lib.item_from_zerodim(np.array(np.nan).astype(dtype))
             except (
                 TypeError,  # downstream error msg for CategoricalIndex is misleading
                 ValueError,
             ):
                 msg = f"Cannot cast {self.categories.dtype} dtype to {dtype}"
                 raise ValueError(msg)
 
-            result = take_1d(new_cats, libalgos.ensure_platform_int(self._codes))
+            result = take_1d(
+                new_cats,
+                libalgos.ensure_platform_int(self._codes),
+                fill_value=fill_value,
+            )
 
         return result
 

pandas/tests/frame/methods/test_astype.py

@@ -616,3 +616,11 @@ def test_astype_bytes(self):
         # GH#39474
         result = DataFrame(["foo", "bar", "baz"]).astype(bytes)
         assert result.dtypes[0] == np.dtype("S3")
+
+    def test_astype_categorical_to_string_missing(self):
+        # https://github.com/pandas-dev/pandas/issues/41797
+        df = DataFrame(["a", "b", np.nan])
+        expected = df.astype(str)
+        cat = df.astype("category")
+        result = cat.astype(str)
+        tm.assert_frame_equal(result, expected)