BUG: Dataframe.groupby aggregations with categorical columns lead to incorrect results. (#32546)

Author: MarcoGorelli

doc/source/whatsnew/v1.1.0.rst

@@ -976,6 +976,7 @@ Groupby/resample/rolling
 
 - Bug in :meth:`GroupBy.apply` raises ``ValueError`` when the ``by`` axis is not sorted and has duplicates and the applied ``func`` does not mutate passed in objects (:issue:`30667`)
 - Bug in :meth:`DataFrameGroupby.transform` produces incorrect result with transformation functions (:issue:`30918`)
+- Bug in :meth:`Groupby.transform` was returning the wrong result when grouping by multiple keys of which some were categorical and others not (:issue:`32494`)
 - Bug in :meth:`GroupBy.count` causes segmentation fault when grouped-by column contains NaNs (:issue:`32841`)
 - Bug in :meth:`DataFrame.groupby` and :meth:`Series.groupby` produces inconsistent type when aggregating Boolean series (:issue:`32894`)
 - Bug in :meth:`DataFrameGroupBy.sum` and :meth:`SeriesGroupBy.sum` where a large negative number would be returned when the number of non-null values was below ``min_count`` for nullable integer dtypes (:issue:`32861`)

pandas/core/groupby/generic.py

@@ -546,6 +546,7 @@ def _transform_fast(self, result, func_nm: str) -> Series:
         builtin/cythonizable functions
         """
         ids, _, ngroup = self.grouper.group_info
+        result = result.reindex(self.grouper.result_index, copy=False)
         cast = self._transform_should_cast(func_nm)
         out = algorithms.take_1d(result._values, ids)
         if cast:
@@ -1496,6 +1497,7 @@ def _transform_fast(self, result: DataFrame, func_nm: str) -> DataFrame:
         # for each col, reshape to to size of original frame
         # by take operation
         ids, _, ngroup = self.grouper.group_info
+        result = result.reindex(self.grouper.result_index, copy=False)
         output = []
         for i, _ in enumerate(result.columns):
             res = algorithms.take_1d(result.iloc[:, i].values, ids)

pandas/tests/groupby/transform/test_transform.py

@@ -1205,3 +1205,36 @@ def test_transform_lambda_indexing():
         ),
     )
     tm.assert_frame_equal(result, expected)
+
+
+def test_categorical_and_not_categorical_key(observed):
+    # Checks that groupby-transform, when grouping by both a categorical
+    # and a non-categorical key, doesn't try to expand the output to include
+    # non-observed categories but instead matches the input shape.
+    # GH 32494
+    df_with_categorical = pd.DataFrame(
+        {
+            "A": pd.Categorical(["a", "b", "a"], categories=["a", "b", "c"]),
+            "B": [1, 2, 3],
+            "C": ["a", "b", "a"],
+        }
+    )
+    df_without_categorical = pd.DataFrame(
+        {"A": ["a", "b", "a"], "B": [1, 2, 3], "C": ["a", "b", "a"]}
+    )
+
+    # DataFrame case
+    result = df_with_categorical.groupby(["A", "C"], observed=observed).transform("sum")
+    expected = df_without_categorical.groupby(["A", "C"]).transform("sum")
+    tm.assert_frame_equal(result, expected)
+    expected_explicit = pd.DataFrame({"B": [4, 2, 4]})
+    tm.assert_frame_equal(result, expected_explicit)
+
+    # Series case
+    result = df_with_categorical.groupby(["A", "C"], observed=observed)["B"].transform(
+        "sum"
+    )
+    expected = df_without_categorical.groupby(["A", "C"])["B"].transform("sum")
+    tm.assert_series_equal(result, expected)
+    expected_explicit = pd.Series([4, 2, 4], name="B")
+    tm.assert_series_equal(result, expected_explicit)