Categorical: .codes should be immutable

ERR: codes modification raises ValueError always

Categorical: use Categorical.from_codes() in a few places

Categorical: Fix assigning a Categorical to an existing string column

Author: jankatins

doc/source/categorical.rst

@@ -548,28 +548,25 @@ relevant columns back to `category` and assign the right levels and level orderi
    :suppress:
 
     from pandas.compat import StringIO
-    csv_file = StringIO()
 
 .. ipython:: python
 
     s = pd.Series(pd.Categorical(['a', 'b', 'b', 'a', 'a', 'd']))
     # rename the levels
     s.cat.levels = ["very good", "good", "bad"]
-    # add new levels at the end
-    s.cat.levels = list(s.cat.levels) + ["medium", "very bad"]
-    # reorder the levels
+    # reorder the levels and add missing levels
     s.cat.reorder_levels(["very bad", "bad", "medium", "good", "very good"])
-    df = pd.DataFrame({"s":s, "vals":[1,2,3,4,5,6]})
-    df.to_csv(csv_file)
-    df2 = pd.read_csv(csv_file)
+    df = pd.DataFrame({"cats":s, "vals":[1,2,3,4,5,6]})
+    csv = StringIO()
+    df.to_csv(csv)
+    df2 = pd.read_csv(StringIO(csv.getvalue()))
     df2.dtypes
-    df2["vals"]
+    df2["cats"]
     # Redo the category
-    df2["vals"] = df2["vals"].astype("category")
-    df2["vals"].cat.levels = list(df2["vals"].cat.levels) + ["medium", "very bad"]
-    df2["vals"].cat.reorder_levels(["very bad", "bad", "medium", "good", "very good"])
+    df2["cats"] = df2["cats"].astype("category")
+    df2["cats"].cat.reorder_levels(["very bad", "bad", "medium", "good", "very good"])
     df2.dtypes
-    df2["vals"]
+    df2["cats"]
 
 
 Missing Data

pandas/core/categorical.py

@@ -325,11 +325,24 @@ def from_codes(cls, codes, levels, ordered=True, name=None):
     _codes = None
 
     def _get_codes(self):
-        """ Get the level codes. """
-        # TODO: return a copy so that no manipulation is possible?
-        return self._codes
+        """ Get the level codes.
 
-    codes = property(fget=_get_codes, doc=_codes_doc)
+        Returns
+        -------
+        codes : integer array view
+            A non writable view of the `codes` array.
+        """
+        v = self._codes.view()
+        v.flags.writeable = False
+        return v
+
+    def _set_codes(self, codes):
+        """
+        Not settable by the user directly
+        """
+        raise ValueError("cannot set Categorical codes directly")
+
+    codes = property(fget=_get_codes, fset=_set_codes, doc=_codes_doc)
 
     _levels = None
 

pandas/core/internals.py

@@ -1426,9 +1426,9 @@ def _try_cast(self, element):
         return element
 
     def should_store(self, value):
-        return not issubclass(value.dtype.type,
+        return not (issubclass(value.dtype.type,
                               (np.integer, np.floating, np.complexfloating,
-                               np.datetime64, np.bool_))
+                               np.datetime64, np.bool_)) or com.is_categorical_dtype(value))
 
     def replace(self, to_replace, value, inplace=False, filter=None,
                 regex=False):

pandas/tests/test_categorical.py

@@ -7,7 +7,8 @@
 import numpy as np
 import pandas as pd
 
-from pandas import (Categorical, Index, Series, DataFrame, PeriodIndex, Timestamp)
+from pandas import (Categorical, Index, Series, DataFrame, PeriodIndex,
+                    Timestamp, _np_version_under1p7)
 
 import pandas.core.common as com
 import pandas.compat as compat
@@ -345,12 +346,12 @@ def test_remove_unused_levels(self):
 
     def test_nan_handling(self):
 
-        # Nans are represented as -1 in labels
+        # Nans are represented as -1 in codes
         c = Categorical(["a","b",np.nan,"a"])
         self.assert_numpy_array_equal(c.levels , np.array(["a","b"]))
         self.assert_numpy_array_equal(c._codes , np.array([0,1,-1,0]))
 
-        # If levels have nan included, the label should point to that instead
+        # If levels have nan included, the code should point to that instead
         c = Categorical(["a","b",np.nan,"a"], levels=["a","b",np.nan])
         self.assert_numpy_array_equal(c.levels , np.array(["a","b",np.nan],dtype=np.object_))
         self.assert_numpy_array_equal(c._codes , np.array([0,1,2,0]))
@@ -361,6 +362,36 @@ def test_nan_handling(self):
         self.assert_numpy_array_equal(c.levels , np.array(["a","b",np.nan],dtype=np.object_))
         self.assert_numpy_array_equal(c._codes , np.array([0,1,2,0]))
 
+    def test_codes_immutable(self):
+
+        # Codes should be read only
+        c = Categorical(["a","b","c","a", np.nan])
+        exp = np.array([0,1,2,0, -1])
+        self.assert_numpy_array_equal(c.codes, exp)
+
+        # Assignments to codes should raise
+        def f():
+            c.codes = np.array([0,1,2,0,1])
+        self.assertRaises(ValueError, f)
+
+        # changes in the codes array should raise
+        # np 1.6.1 raises RuntimeError rather than ValueError
+        codes= c.codes
+        def f():
+            codes[4] = 1
+        if _np_version_under1p7:
+            self.assertRaises(RuntimeError, f)
+        else:
+            self.assertRaises(ValueError, f)
+
+        # But even after getting the codes, the original array should still be writeable!
+        c[4] = "a"
+        exp = np.array([0,1,2,0, 0])
+        self.assert_numpy_array_equal(c.codes, exp)
+        c._codes[4] = 2
+        exp = np.array([0,1,2,0, 2])
+        self.assert_numpy_array_equal(c.codes, exp)
+
 
     def test_min_max(self):
 
@@ -549,6 +580,19 @@ def test_creation_astype(self):
         res = s.astype('category')
         tm.assert_series_equal(res, exp)
 
+        df = pd.DataFrame({"cats":[1,2,3,4,5,6], "vals":[1,2,3,4,5,6]})
+        cats = Categorical([1,2,3,4,5,6])
+        exp_df = pd.DataFrame({"cats":cats, "vals":[1,2,3,4,5,6]})
+        df["cats"] =  df["cats"].astype("category")
+        tm.assert_frame_equal(exp_df, df)
+
+
+        df = pd.DataFrame({"cats":['a', 'b', 'b', 'a', 'a', 'd'], "vals":[1,2,3,4,5,6]})
+        cats = Categorical(['a', 'b', 'b', 'a', 'a', 'd'])
+        exp_df = pd.DataFrame({"cats":cats, "vals":[1,2,3,4,5,6]})
+        df["cats"] =  df["cats"].astype("category")
+        tm.assert_frame_equal(exp_df, df)
+
     def test_sideeffects_free(self):
 
         # Passing a categorical to a Series and then changing values in either the series or the

pandas/tests/test_groupby.py

@@ -2901,9 +2901,9 @@ def test_no_dummy_key_names(self):
 
     def test_groupby_categorical(self):
         levels = ['foo', 'bar', 'baz', 'qux']
-        labels = np.random.randint(0, 4, size=100)
+        codes = np.random.randint(0, 4, size=100)
 
-        cats = Categorical(labels, levels, name='myfactor', fastpath=True)
+        cats = Categorical.from_codes(codes, levels, name='myfactor')
 
         data = DataFrame(np.random.randn(100, 4))
 
@@ -3049,18 +3049,18 @@ def test_cython_median(self):
     def test_groupby_categorical_no_compress(self):
         data = Series(np.random.randn(9))
 
-        labels = np.array([0, 0, 0, 1, 1, 1, 2, 2, 2])
-        cats = Categorical(labels, [0, 1, 2], fastpath=True)
+        codes = np.array([0, 0, 0, 1, 1, 1, 2, 2, 2])
+        cats = Categorical.from_codes(codes, [0, 1, 2])
 
         result = data.groupby(cats).mean()
-        exp = data.groupby(labels).mean()
+        exp = data.groupby(codes).mean()
         assert_series_equal(result, exp)
 
-        labels = np.array([0, 0, 0, 1, 1, 1, 3, 3, 3])
-        cats = Categorical(labels, [0, 1, 2, 3], fastpath=True)
+        codes = np.array([0, 0, 0, 1, 1, 1, 3, 3, 3])
+        cats = Categorical.from_codes(codes, [0, 1, 2, 3])
 
         result = data.groupby(cats).mean()
-        exp = data.groupby(labels).mean().reindex(cats.levels)
+        exp = data.groupby(codes).mean().reindex(cats.levels)
         assert_series_equal(result, exp)
 
         cats = Categorical(["a", "a", "a", "b", "b", "b", "c", "c", "c"], levels=["a","b","c","d"])