diff --git a/pandas/core/groupby/generic.py b/pandas/core/groupby/generic.py
index c766fcaa4f849..2afb77a619a80 100644
--- a/pandas/core/groupby/generic.py
+++ b/pandas/core/groupby/generic.py
@@ -264,7 +264,12 @@ def aggregate(self, func=None, *args, **kwargs):
                 return self._python_agg_general(func, *args, **kwargs)
             except (AssertionError, TypeError):
                 raise
-            except Exception:
+            except (ValueError, KeyError, AttributeError, IndexError):
+                # TODO: IndexError can be removed here following GH#29106
+                # TODO: AttributeError is caused by _index_data hijinx in
+                #  libreduction, can be removed after GH#29160
+                # TODO: KeyError is raised in _python_agg_general,
+                #  see see test_groupby.test_basic
                 result = self._aggregate_named(func, *args, **kwargs)
 
             index = Index(sorted(result), name=self.grouper.names[0])
diff --git a/pandas/core/groupby/ops.py b/pandas/core/groupby/ops.py
index e6f4f2f056058..fbe1598767736 100644
--- a/pandas/core/groupby/ops.py
+++ b/pandas/core/groupby/ops.py
@@ -26,6 +26,7 @@
     is_complex_dtype,
     is_datetime64_any_dtype,
     is_datetime64tz_dtype,
+    is_extension_array_dtype,
     is_integer_dtype,
     is_numeric_dtype,
     is_sparse,
@@ -659,6 +660,12 @@ def _transform(
         return result
 
     def agg_series(self, obj, func):
+        if is_extension_array_dtype(obj.dtype) and obj.dtype.kind != "M":
+            # _aggregate_series_fast would raise TypeError when
+            #  calling libreduction.Slider
+            # TODO: is the datetime64tz case supposed to go through here?
+            return self._aggregate_series_pure_python(obj, func)
+
         try:
             return self._aggregate_series_fast(obj, func)
         except AssertionError:
@@ -683,6 +690,8 @@ def agg_series(self, obj, func):
     def _aggregate_series_fast(self, obj, func):
         func = self._is_builtin_func(func)
 
+        # TODO: pre-empt this, also pre-empt get_result raising TypError if we pass a EA
+        #   for EAs backed by ndarray we may have a performant workaround
         if obj.index._has_complex_internals:
             raise TypeError("Incompatible index for Cython grouper")
 
@@ -717,6 +726,7 @@ def _aggregate_series_pure_python(self, obj, func):
             result[label] = res
 
         result = lib.maybe_convert_objects(result, try_float=0)
+        # TODO: try_cast back to EA?
         return result, counts