BUG: Fix .groupby(categorical, sort=False) failing

Author: kernc

doc/source/whatsnew/v0.20.0.txt

@@ -539,6 +539,7 @@ Bug Fixes
 
 - Bug in ``resample``, where a non-string ```loffset`` argument would not be applied when resampling a timeseries (:issue:`13218`)
 
+- Bug in ``Categorical.unique()`` stripping unused categories resulting in ```.groupby(categorical, sort=False)`` not working (:issue:`13179`)
 
 
 

pandas/core/categorical.py

@@ -1851,15 +1851,18 @@ def unique(self):
         """
 
         # unlike np.unique, unique1d does not sort
-        unique_codes = unique1d(self.codes)
+        unique_codes = unique1d(self.codes).astype(self.codes.dtype)
         cat = self.copy()
         # keep nan in codes
         cat._codes = unique_codes
-        # exclude nan from indexer for categories
-        take_codes = unique_codes[unique_codes != -1]
-        if self.ordered:
-            take_codes = sorted(take_codes)
-        return cat.set_categories(cat.categories.take(take_codes))
+        if not self.ordered:
+            take_codes = unique_codes[unique_codes != -1]
+            # Sort categories according to codes order, suffixed by
+            # unused categories in original order (GH-13179)
+            order = (cat.categories[take_codes].tolist() +
+                     cat.categories.tolist()).index
+            cat.set_categories(sorted(cat.categories, key=order), inplace=True)
+        return cat
 
     def equals(self, other):
         """

pandas/tests/groupby/test_categorical.py

@@ -284,6 +284,30 @@ def test_groupby_multi_categorical_as_index(self):
 
             tm.assert_frame_equal(result, expected, check_index_type=True)
 
+    def test_groupby_preserve_categories(self):
+        # GH-13179
+        categories = list('abc')
+
+        # ordered=True
+        df = DataFrame({'A': pd.Categorical(list('ba'),
+                                            categories=categories,
+                                            ordered=True)})
+        index = pd.CategoricalIndex(categories, categories, ordered=True)
+        tm.assert_index_equal(df.groupby('A', sort=True).first().index, index)
+        tm.assert_index_equal(df.groupby('A', sort=False).first().index, index)
+
+        # ordered=False
+        df = DataFrame({'A': pd.Categorical(list('ba'),
+                                            categories=categories,
+                                            ordered=False)})
+        sort_index = pd.CategoricalIndex(categories, categories, ordered=False)
+        nosort_index = pd.CategoricalIndex(list('bac'), list('bac'),
+                                           ordered=False)
+        tm.assert_index_equal(df.groupby('A', sort=True).first().index,
+                              sort_index)
+        tm.assert_index_equal(df.groupby('A', sort=False).first().index,
+                              nosort_index)
+
     def test_groupby_preserve_categorical_dtype(self):
         # GH13743, GH13854
         df = DataFrame({'A': [1, 2, 1, 1, 2],

pandas/tests/test_categorical.py

@@ -1287,8 +1287,9 @@ def test_unique(self):
 
         cat = Categorical(["a", "b", "a", "a"], categories=["a", "b", "c"])
         res = cat.unique()
+        exp = Index(["a", "b", "c"])
         self.assert_index_equal(res.categories, exp)
-        tm.assert_categorical_equal(res, Categorical(exp))
+        tm.assert_categorical_equal(res, Categorical(["a", "b"], exp))
 
         cat = Categorical(["c", "a", "b", "a", "a"],
                           categories=["a", "b", "c"])
@@ -1302,9 +1303,9 @@ def test_unique(self):
         cat = Categorical(["b", np.nan, "b", np.nan, "a"],
                           categories=["a", "b", "c"])
         res = cat.unique()
-        exp = Index(["b", "a"])
+        exp = Index(["b", "a", "c"])
         self.assert_index_equal(res.categories, exp)
-        exp_cat = Categorical(["b", np.nan, "a"], categories=["b", "a"])
+        exp_cat = Categorical(["b", np.nan, "a"], categories=["b", "a", "c"])
         tm.assert_categorical_equal(res, exp_cat)
 
     def test_unique_ordered(self):
@@ -1324,13 +1325,14 @@ def test_unique_ordered(self):
         cat = Categorical(['b', 'a', 'a'], categories=['a', 'b', 'c'],
                           ordered=True)
         res = cat.unique()
-        exp_cat = Categorical(['b', 'a'], categories=['a', 'b'], ordered=True)
+        exp_cat = Categorical(['b', 'a'], categories=['a', 'b', 'c'],
+                              ordered=True)
         tm.assert_categorical_equal(res, exp_cat)
 
         cat = Categorical(['b', 'b', np.nan, 'a'], categories=['a', 'b', 'c'],
                           ordered=True)
         res = cat.unique()
-        exp_cat = Categorical(['b', np.nan, 'a'], categories=['a', 'b'],
+        exp_cat = Categorical(['b', np.nan, 'a'], categories=['a', 'b', 'c'],
                               ordered=True)
         tm.assert_categorical_equal(res, exp_cat)
 
@@ -1345,7 +1347,7 @@ def test_unique_index_series(self):
         tm.assert_categorical_equal(pd.Series(c).unique(), exp)
 
         c = Categorical([1, 1, 2, 2], categories=[3, 2, 1])
-        exp = Categorical([1, 2], categories=[1, 2])
+        exp = Categorical([1, 2], categories=[1, 2, 3])
         tm.assert_categorical_equal(c.unique(), exp)
         tm.assert_index_equal(Index(c).unique(), Index(exp))
         tm.assert_categorical_equal(pd.Series(c).unique(), exp)