REF: dont _try_cast for user-defined functions (pandas-dev#29698)

Author: jbrockmendel

doc/source/whatsnew/v1.0.0.rst

@@ -511,6 +511,7 @@ Groupby/resample/rolling
 - Bug in :meth:`DataFrame.groupby` losing column name information when grouping by a categorical column (:issue:`28787`)
 - Bug in :meth:`DataFrameGroupBy.rolling().quantile()` ignoring ``interpolation`` keyword argument (:issue:`28779`)
 - Bug in :meth:`DataFrame.groupby` where ``any``, ``all``, ``nunique`` and transform functions would incorrectly handle duplicate column labels (:issue:`21668`)
+-
 
 Reshaping
 ^^^^^^^^^

pandas/core/apply.py

@@ -226,6 +226,8 @@ def apply_raw(self):
             if "Function does not reduce" not in str(err):
                 # catch only ValueError raised intentionally in libreduction
                 raise
+            # We expect np.apply_along_axis to give a two-dimensional result, or
+            #  also raise.
             result = np.apply_along_axis(self.f, self.axis, self.values)
 
         # TODO: mixed type case

pandas/core/groupby/generic.py

@@ -1109,12 +1109,12 @@ def _aggregate_frame(self, func, *args, **kwargs) -> DataFrame:
         if axis != obj._info_axis_number:
             for name, data in self:
                 fres = func(data, *args, **kwargs)
-                result[name] = self._try_cast(fres, data)
+                result[name] = fres
         else:
             for name in self.indices:
                 data = self.get_group(name, obj=obj)
                 fres = func(data, *args, **kwargs)
-                result[name] = self._try_cast(fres, data)
+                result[name] = fres
 
         return self._wrap_frame_output(result, obj)
 
@@ -1424,6 +1424,8 @@ def _transform_fast(self, result: DataFrame, func_nm: str) -> DataFrame:
         output = []
         for i, _ in enumerate(result.columns):
             res = algorithms.take_1d(result.iloc[:, i].values, ids)
+            # TODO: we have no test cases that get here with EA dtypes;
+            #  try_cast may not be needed if EAs never get here
             if cast:
                 res = self._try_cast(res, obj.iloc[:, i])
             output.append(res)