diff --git a/pandas/core/groupby.py b/pandas/core/groupby.py
index 168615c060c2b..d15dcc1510577 100644
--- a/pandas/core/groupby.py
+++ b/pandas/core/groupby.py
@@ -432,23 +432,13 @@ def picker(arr):
     def _try_cast(self, result, obj):
         """ try to cast the result to our obj original type,
         we may have roundtripped thru object in the mean-time """
-        try:
-            if obj.ndim > 1:
-                dtype = obj.values.dtype
-            else:
-                dtype = obj.dtype
-
-            if _is_numeric_dtype(dtype):
-
-                # need to respect a non-number here (e.g. Decimal)
-                if len(result) and issubclass(type(result[0]),(np.number,float,int)):
-                    result = _possibly_downcast_to_dtype(result, dtype)
+        if obj.ndim > 1:
+            dtype = obj.values.dtype
+        else:
+            dtype = obj.dtype
 
-            elif issubclass(dtype.type, np.datetime64):
-                if is_datetime64_dtype(obj.dtype):
-                    result = result.astype(obj.dtype)
-        except:
-            pass
+        if not np.isscalar(result):
+            result = _possibly_downcast_to_dtype(result, dtype)
 
         return result