Add fix to raise error when category value 'x' is not predefined but is assigned through df.loc[..]=x

pandas/core/indexing.py

@@ -11,8 +11,10 @@
 from pandas.errors import AbstractMethodError, InvalidIndexError
 from pandas.util._decorators import doc
 
+from pandas.core.dtypes.cast import find_common_type
 from pandas.core.dtypes.common import (
     is_array_like,
+    is_categorical_dtype,
     is_hashable,
     is_integer,
     is_iterator,
@@ -1842,7 +1844,14 @@ def _setitem_with_indexer_missing(self, indexer, value):
         """
         Insert new row(s) or column(s) into the Series or DataFrame.
         """
-        from pandas import Series
+        from pandas import DataFrame, Series
+
+        def check_valid_categorical(new_values, obj_dtype):
+            if is_categorical_dtype(obj_dtype):
+                if (~np.in1d(new_values, obj_dtype.categories.values)).any():
+                    raise ValueError(
+                        "Cannot setitem on a Categorical with a new category"
+                    )
 
         # reindex the axis to the new value
         # and set inplace
@@ -1867,8 +1876,16 @@ def _setitem_with_indexer_missing(self, indexer, value):
                 # GH#22717 handle casting compatibility that np.concatenate
                 #  does incorrectly
                 new_values = concat_compat([self.obj._values, new_values])
+                if is_object_dtype(new_values.dtype):
+                    dtype = None
+                else:
+                    dtype = find_common_type([self.obj.dtype, new_values.dtype])
+            else:
+                dtype = None
+
+            check_valid_categorical(new_values, self.obj.dtype)
             self.obj._mgr = self.obj._constructor(
-                new_values, index=new_index, name=self.obj.name
+                new_values, index=new_index, name=self.obj.name, dtype=dtype
             )._mgr
             self.obj._maybe_update_cacher(clear=True)
 
@@ -1893,7 +1910,16 @@ def _setitem_with_indexer_missing(self, indexer, value):
                     if len(value) != len(self.obj.columns):
                         raise ValueError("cannot set a row with mismatched columns")
 
-                value = Series(value, index=self.obj.columns, name=indexer)
+                if len(set(self.obj.dtypes)) > 1:
+                    value = list(value)
+                    for i in range(len(self.obj.columns)):
+                        value[i] = Series(data=[value[i]], dtype=self.obj.dtypes[i])
+                        check_valid_categorical(value[i], self.obj.dtypes[i])
+                    value = dict(zip(self.obj.columns, value))
+                    value = DataFrame(value)
+                    value.index = [indexer]
+                else:
+                    value = Series(value, index=self.obj.columns, name=indexer)
 
             self.obj._mgr = self.obj.append(value)._mgr
             self.obj._maybe_update_cacher(clear=True)

pandas/tests/series/test_categorical.py

@@ -0,0 +1,54 @@
+import pytest
+
+import pandas as pd
+from pandas import Categorical, Index
+import pandas._testing as tm
+
+
+class TestCategoricalSeries:
+    def test_loc_new_category_series_raises(self):
+        ser = pd.Series(Categorical(["a", "b", "c"]))
+        msg = "Cannot setitem on a Categorical with a new category"
+        with pytest.raises(ValueError, match=msg):
+            ser.loc[3] = "d"
+
+    def test_unused_category_retention(self):
+        # Init case
+        exp_cats = Index(["a", "b", "c", "d"])
+        ser = pd.Series(Categorical(["a", "b", "c"], categories=exp_cats))
+        tm.assert_index_equal(ser.cat.categories, exp_cats)
+
+        # Modify case
+        ser.loc[0] = "b"
+        expected = pd.Series(Categorical(["b", "b", "c"], categories=exp_cats))
+        tm.assert_index_equal(ser.cat.categories, exp_cats)
+        tm.assert_series_equal(ser, expected)
+
+    def test_loc_new_category_row_raises(self):
+        df = pd.DataFrame(
+            {
+                "int": [0, 1, 2],
+                "cat": Categorical(["a", "b", "c"], categories=["a", "b", "c"]),
+            }
+        )
+        msg = "Cannot setitem on a Categorical with a new category"
+        with pytest.raises(ValueError, match=msg):
+            df.loc[3] = [3, "d"]
+
+    def test_loc_new_row_category_dtype_retention(self):
+        df = pd.DataFrame(
+            {
+                "int": [0, 1, 2],
+                "cat": pd.Categorical(["a", "b", "c"], categories=["a", "b", "c"]),
+            }
+        )
+        df.loc[3] = [3, "c"]
+
+        expected = pd.DataFrame(
+            {
+                "int": [0, 1, 2, 3],
+                "cat": pd.Categorical(["a", "b", "c", "c"], categories=["a", "b", "c"]),
+            }
+        )
+
+        tm.assert_frame_equal(df, expected)