fixup! BUG: Fix .groupby(categorical, sort=False) failing

Author: kernc

doc/source/whatsnew/v0.20.0.txt

@@ -120,6 +120,42 @@ Notably, a new numerical index, ``UInt64Index``, has been created (:issue:`14937
 - Bug in ``pd.unique()`` in which unsigned 64-bit integers were causing overflow (:issue:`14915`)
 - Bug in ``pd.value_counts()`` in which unsigned 64-bit integers were being erroneously truncated in the output (:issue:`14934`)
 
+.. _whatsnew_0200.enhancements.groupy_categorical
+
+GroupBy on Categoricals
+^^^^^^^^^^^^^^^^^^^^^^^
+
+In previous version, ``.groupby(..., sort=False)`` would fail with a ``ValueError`` when grouping on a categorical series with some categories not appearing in the data. (:issue:`13179`)
+
+Now, it works.
+
+.. ipython:: python
+
+  chromosomes = np.r_[np.arange(1, 23).astype(str), ['X', 'Y']]
+  df = pd.DataFrame({
+      'A': np.random.randint(100),
+      'B': np.random.randint(100),
+      'C': np.random.randint(100),
+      'chromosomes': pd.Categorical(np.random.choice(chromosomes, 100),
+                                    categories=chromosomes,
+                                    ordered=True)})
+
+Previous Behavior:
+
+.. code-block:: ipython
+
+  In [3]: df[df.chromosomes != '1'].groupby('chromosomes', sort=False).sum()
+  ---------------------------------------------------------------------------
+  ValueError                                Traceback (most recent call last)
+  ...
+  ValueError: items in new_categories are not the same as in old categories
+
+New Behavior:
+
+.. ipython:: python
+
+  df[df.chromosomes != '1'].groupby('chromosomes', sort=False).sum()
+
 .. _whatsnew_0200.enhancements.other:
 
 Other enhancements
@@ -160,7 +196,6 @@ Other enhancements
 
 .. _whatsnew_0200.api_breaking:
 
-
 Backwards incompatible API changes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -539,7 +574,6 @@ Bug Fixes
 
 - Bug in ``resample``, where a non-string ```loffset`` argument would not be applied when resampling a timeseries (:issue:`13218`)
 
-- Bug in ``.groupby`` where ```.groupby(categorical, sort=False)`` would raise ``ValueError`` due to non-matching categories (:issue:`13179`)
 
 
 

pandas/core/categorical.py

@@ -602,6 +602,35 @@ def _get_categories(self):
     categories = property(fget=_get_categories, fset=_set_categories,
                           doc=_categories_doc)
 
+    def _codes_for_groupby(self, sort):
+        """
+        Return a Categorical adjusted for groupby
+
+        Parameters
+        ----------
+        sort : boolean
+            The value of the sort paramter groupby was called with.
+
+        Returns
+        -------
+        Categorical
+            In case of sort=True, self is returned with original categories
+            preserved. In case of sort=False, the new categories are set
+            to the order of appearance in codes (unless ordered=True),
+            followed by any unrepresented categories in original order.
+        """
+        cat = self
+        # sort=False should order groups in as-encountered order (GH-8868)
+        if not sort:
+            cat = self.unique()
+            # But all categories should be present, including those missing
+            # from the data (GH-13179), which .unique() dropped
+            cat.add_categories(self.categories[
+                                   ~self.categories.isin(cat.categories)],
+                               inplace=True)
+            cat = self.reorder_categories(cat.categories)
+        return cat
+
     _ordered = None
 
     def set_ordered(self, value, inplace=False):

pandas/core/groupby.py

@@ -2300,28 +2300,7 @@ def __init__(self, index, grouper=None, obj=None, name=None, level=None,
             # a passed Categorical
             elif is_categorical_dtype(self.grouper):
 
-                # must have an ordered categorical
-                if self.sort:
-                    if not self.grouper.ordered:
-
-                        # technically we cannot group on an unordered
-                        # Categorical
-                        # but this a user convenience to do so; the ordering
-                        # is preserved and if it's a reduction it doesn't make
-                        # any difference
-                        pass
-
-                # fix bug #GH8868 sort=False being ignored in categorical
-                # groupby
-                else:
-                    cat = self.grouper.unique()
-                    all_categories = self.grouper.categories
-                    cat.add_categories(
-                        all_categories[
-                            ~all_categories.isin(cat.categories)],
-                        inplace=True)  # GH-13179
-                    self.grouper = self.grouper.reorder_categories(
-                        cat.categories)
+                self.grouper = self.grouper._codes_for_groupby(self.sort)
 
                 # we make a CategoricalIndex out of the cat grouper
                 # preserving the categories / ordered attributes

pandas/indexes/category.py

@@ -550,6 +550,10 @@ def _append_same_dtype(self, to_concat, name):
         result.name = name
         return result
 
+    def _codes_for_groupby(self, sort):
+        """ Return a Categorical adjusted for groupby """
+        return self.values._codes_for_groupby(sort)
+
     @classmethod
     def _add_comparison_methods(cls):
         """ add in comparison methods """