Closes pandas-dev#49223

Author: gt-on-1234

pandas/core/groupby/categorical.py

@@ -53,7 +53,7 @@ def recode_for_groupby(
         unique_codes = unique1d(c.codes)
 
         take_codes = unique_codes[unique_codes != -1]
-        if c.ordered:
+        if sort:
             take_codes = np.sort(take_codes)
 
         # we recode according to the uniques

pandas/core/groupby/grouper.py

@@ -502,6 +502,15 @@ def __init__(
                 self._group_index,
             ) = index._get_grouper_for_level(mapper, level=ilevel, dropna=dropna)
 
+            if is_categorical_dtype(self.grouping_vector):
+                self._passed_categorical = True
+                self._orig_cats = self.grouping_vector.categories
+
+                # Should the sort arg be just `sort` or `sort or self.grouping_vector.ordered`?
+                self.grouping_vector, self._all_grouper = recode_for_groupby(
+                    self.grouping_vector, sort, observed
+                )
+
         # a passed Grouper like, directly get the grouper in the same way
         # as single grouper groupby, use the group_info to get codes
         elif isinstance(self.grouping_vector, Grouper):
@@ -529,6 +538,8 @@ def __init__(
             self._passed_categorical = True
 
             self._orig_cats = self.grouping_vector.categories
+
+            # Should the sort arg be just `sort` or `sort or self.grouping_vector.ordered`?
             self.grouping_vector, self._all_grouper = recode_for_groupby(
                 self.grouping_vector, sort, observed
             )