Avoid casting in specific situations

Author: radekbenesinno

pandas/core/groupby/generic.py

@@ -254,8 +254,11 @@ def _aggregate_item_by_item(self, func, *args, **kwargs):
                 data = obj[item]
                 colg = SeriesGroupBy(data, selection=item,
                                      grouper=self.grouper)
-                result[item] = self._try_cast(
-                    colg.aggregate(func, *args, **kwargs), data)
+
+                result[item] = colg.aggregate(func, *args, **kwargs)
+                if func != "idxmin" and func != "idxmax":
+                    result[item] = self._try_cast(result[item], data)
+
             except ValueError:
                 cannot_agg.append(item)
                 continue