Groupby transform cleanups

pandas/core/base.py

@@ -565,7 +565,7 @@ def is_any_frame():
         else:
             result = None
 
-        f = self._is_cython_func(arg)
+        f = self._get_cython_func(arg)
         if f and not args and not kwargs:
             return getattr(self, f)(), None
 
@@ -652,7 +652,7 @@ def _shallow_copy(self, obj=None, obj_type=None, **kwargs):
                 kwargs[attr] = getattr(self, attr)
         return obj_type(obj, **kwargs)
 
-    def _is_cython_func(self, arg):
+    def _get_cython_func(self, arg):
         """
         if we define an internal function for this argument, return it
         """

pandas/core/groupby/base.py

@@ -98,6 +98,69 @@ def _gotitem(self, key, ndim, subset=None):
 
 dataframe_apply_whitelist = common_apply_whitelist | frozenset(["dtypes", "corrwith"])
 
-cython_transforms = frozenset(["cumprod", "cumsum", "shift", "cummin", "cummax"])
+# cythonized transformations or canned "agg+broadcast", which do not
+# require postprocessing of the result by transform.
+cythonized_kernels = frozenset(["cumprod", "cumsum", "shift", "cummin", "cummax"])
 
 cython_cast_blacklist = frozenset(["rank", "count", "size", "idxmin", "idxmax"])
+
+# List of aggregation/reduction functions.
+# These map each series/column to a single value
+reduction_functions = frozenset(
+    [
+        "sum",
+        "first",
+        "mean",
+        "all",
+        "any",
+        "bfill",
+        "corr",
+        "count",
+        "cov",
+        "idxmax",
+        "idxmin",
+        "last",
+        "mad",
+        "max",
+        "median",
+        "min",
+        "ngroup",
+        "nth",
+        "nunique",
+        "prod",
+        "quantile",
+        "sem",
+        "size",
+        "skew",
+        "std",
+        "var",
+    ]
+)
+
+# List of transformation functions.
+# These map each object to a like-indexed result object
+transformation_functions = frozenset(
+    [
+        "backfill",
+        "corrwith",
+        "cumcount",
+        "cummax",
+        "cummin",
+        "cumprod",
+        "cumsum",
+        "diff",
+        "ffill",
+        "fillna",
+        "rank",
+        "pad",
+        "pct_change",
+        "shift",
+        "tshift",
+    ]
+)
+
+# Valid values  of `name` for `groupby.transform(name)`
+transform_recognized_functions = reduction_functions | transformation_functions
+transform_recognized_functions -= {
+    "corr"
+}  # returns multindex, exclude from transform(name) for now.

pandas/core/groupby/generic.py

@@ -572,13 +572,19 @@ def _transform_general(self, func, *args, **kwargs):
     def transform(self, func, *args, **kwargs):
 
         # optimized transforms
-        func = self._is_cython_func(func) or func
+        func = self._get_cython_func(func) or func
+
         if isinstance(func, str):
-            if func in base.cython_transforms:
-                # cythonized transform
+            if not (func in base.transform_recognized_functions):
+                msg = "'%s' is not a valid function name for transform(name)"
+                raise ValueError(msg % func)
+            if func in base.cythonized_kernels:
+                # cythonized transformation or canned "reduction+broadcast"
                 return getattr(self, func)(*args, **kwargs)
             else:
-                # cythonized aggregation and merge
+                # If func is a reduction, we need to broadcast the
+                # result to the whole group. Compute func result
+                # and deal with possible broadcasting below.
                 result = getattr(self, func)(*args, **kwargs)
         else:
             return self._transform_general(func, *args, **kwargs)
@@ -589,7 +595,7 @@ def transform(self, func, *args, **kwargs):
 
         obj = self._obj_with_exclusions
 
-        # nuiscance columns
+        # nuisance columns
         if not result.columns.equals(obj.columns):
             return self._transform_general(func, *args, **kwargs)
 
@@ -852,7 +858,7 @@ def aggregate(self, func_or_funcs=None, *args, **kwargs):
             if relabeling:
                 ret.columns = columns
         else:
-            cyfunc = self._is_cython_func(func_or_funcs)
+            cyfunc = self._get_cython_func(func_or_funcs)
             if cyfunc and not args and not kwargs:
                 return getattr(self, cyfunc)()
 
@@ -1004,15 +1010,19 @@ def _aggregate_named(self, func, *args, **kwargs):
     @Substitution(klass="Series", selected="A.")
     @Appender(_transform_template)
     def transform(self, func, *args, **kwargs):
-        func = self._is_cython_func(func) or func
+        func = self._get_cython_func(func) or func
 
-        # if string function
         if isinstance(func, str):
-            if func in base.cython_transforms:
-                # cythonized transform
+            if not (func in base.transform_recognized_functions):
+                msg = "'%s' is not a valid function name for transform(name)"
+                raise ValueError(msg % func)
+            if func in base.cythonized_kernels:
+                # cythonized transform or canned "agg+broadcast"
                 return getattr(self, func)(*args, **kwargs)
             else:
-                # cythonized aggregation and merge
+                # If func is a reduction, we need to broadcast the
+                # result to the whole group. Compute func result
+                # and deal with possible broadcasting below.
                 return self._transform_fast(
                     lambda: getattr(self, func)(*args, **kwargs), func
                 )

pandas/core/resample.py

@@ -1047,7 +1047,7 @@ def _downsample(self, how, **kwargs):
         **kwargs : kw args passed to how function
         """
         self._set_binner()
-        how = self._is_cython_func(how) or how
+        how = self._get_cython_func(how) or how
         ax = self.ax
         obj = self._selected_obj
 
@@ -1197,7 +1197,7 @@ def _downsample(self, how, **kwargs):
         if self.kind == "timestamp":
             return super()._downsample(how, **kwargs)
 
-        how = self._is_cython_func(how) or how
+        how = self._get_cython_func(how) or how
         ax = self.ax
 
         if is_subperiod(ax.freq, self.freq):

pandas/tests/groupby/test_transform.py

@@ -18,6 +18,7 @@
     concat,
     date_range,
 )
+from pandas.core.groupby.base import reduction_functions
 from pandas.core.groupby.groupby import DataError
 from pandas.util import testing as tm
 from pandas.util.testing import assert_frame_equal, assert_series_equal
@@ -1001,3 +1002,47 @@ def test_ffill_not_in_axis(func, key, val):
     expected = df
 
     assert_frame_equal(result, expected)
+
+
+def test_transform_invalid_name_raises():
+    df = DataFrame(dict(a=[0, 1, 1, 2]))
+    g = df.groupby(["a", "b", "b", "c"])
+    with pytest.raises(ValueError, match="not a valid function name"):
+        g.transform("some_arbitrary_name")
+
+    # method exists on the object, but is not a valid transformation/agg
+    assert hasattr(g, "aggregate")  # make sure the method exists
+    with pytest.raises(ValueError, match="not a valid function name"):
+        g.transform("aggregate")
+
+    # Test SeriesGroupBy
+    g = df["a"].groupby(["a", "b", "b", "c"])
+    with pytest.raises(ValueError, match="not a valid function name"):
+        g.transform("some_arbitrary_name")
+
+    # method exists on the object, but is not a valid transformation/agg
+    with pytest.raises(ValueError, match="not a valid function name"):
+        g.transform("aggregate")
+
+
+@pytest.mark.parametrize("func", sorted(reduction_functions))
+def test_transform_agg_by_name(func):
+
+    df = DataFrame(dict(a=[0, 0, 0, 1, 1, 1], b=range(6)))
+    g = df.groupby(np.repeat([0, 1], 3))
+
+    if func == "ngroup":  # GH#27468
+        pytest.xfail("TODO: g.transform('ngroup') doesn't work")
+    if func == "size":  # GH#27469
+        pytest.xfail("TODO: g.transform('size') doesn't work")
+    if func == "corr":
+        pytest.xfail("corr returns multiindex, excluded from transform for now")
+
+    args = {"nth": [0], "quantile": [0.5]}.get(func, [])
+
+    print(func)
+    result = g.transform(func, *args)
+    tm.assert_index_equal(result.index, df.index)
+
+    # check values replicated broadcasted across group
+    assert len(set(result.iloc[-3:, 1])) == 1