ENH: api change, more consistent GroupBy.apply behavior with Series, close #938

Author: wesm

RELEASE.rst

@@ -22,6 +22,20 @@ Where to get it
 * Binary installers on PyPI: http://pypi.python.org/pypi/pandas
 * Documentation: http://pandas.pydata.org
 
+pandas 0.7.3
+============
+
+**Release date:** NOT YET RELEAED
+
+**New features / modules**
+
+**API Changes**
+
+  - Calling apply on grouped Series, e.g. describe(), will no longer yield
+    DataFrame by default. Will have to call unstack() to get prior behavior
+
+**Bug fixes**
+
 pandas 0.7.2
 ============
 

pandas/core/generic.py

@@ -90,7 +90,8 @@ def get(self, key, default=None):
         except KeyError:
             return default
 
-    def groupby(self, by=None, axis=0, level=None, as_index=True, sort=True):
+    def groupby(self, by=None, axis=0, level=None, as_index=True, sort=True,
+                group_keys=True):
         """
         Group series using mapper (dict or key function, apply given function
         to group, return result as series) or by a series of columns
@@ -112,6 +113,8 @@ def groupby(self, by=None, axis=0, level=None, as_index=True, sort=True):
             effectively "SQL-style" grouped output
         sort : boolean, default True
             Sort group keys. Get better performance by turning this off
+        group_keys : boolean, default True
+            When calling apply, add group keys to index to identify pieces
 
         Examples
         --------
@@ -130,7 +133,7 @@ def groupby(self, by=None, axis=0, level=None, as_index=True, sort=True):
         """
         from pandas.core.groupby import groupby
         return groupby(self, by, axis=axis, level=level, as_index=as_index,
-                       sort=sort)
+                       sort=sort, group_keys=group_keys)
 
     def select(self, crit, axis=0):
         """

pandas/core/groupby.py

@@ -87,7 +87,7 @@ class GroupBy(object):
 
     def __init__(self, obj, keys=None, axis=0, level=None,
                  grouper=None, exclusions=None, column=None, as_index=True,
-                 sort=True):
+                 sort=True, group_keys=True):
         self._column = column
 
         if isinstance(obj, NDFrame):
@@ -108,6 +108,7 @@ def __init__(self, obj, keys=None, axis=0, level=None,
         self.as_index = as_index
         self.keys = keys
         self.sort = sort
+        self.group_keys = group_keys
 
         if grouper is None:
             grouper, exclusions = _get_grouper(obj, keys, axis=axis,
@@ -370,23 +371,28 @@ def _python_apply_general(self, func, *args, **kwargs):
     def _wrap_applied_output(self, *args, **kwargs):
         raise NotImplementedError
 
-    def _wrap_frames(self, keys, values, not_indexed_same=False):
+    def _concat_objects(self, keys, values, not_indexed_same=False):
         from pandas.tools.merge import concat
 
-        if not_indexed_same:
+        if not not_indexed_same:
+            result = concat(values, axis=self.axis)
+            ax = self.obj._get_axis(self.axis)
+
+            if isinstance(result, Series):
+                result = result.reindex(ax)
+            else:
+                result = result.reindex_axis(ax, axis=self.axis)
+        elif self.group_keys:
             group_keys = keys
             group_levels = self.grouper.levels
             group_names = self.grouper.names
             result = concat(values, axis=self.axis, keys=group_keys,
                             levels=group_levels, names=group_names)
         else:
             result = concat(values, axis=self.axis)
-            ax = self.obj._get_axis(self.axis)
-            result = result.reindex_axis(ax, axis=self.axis)
 
         return result
 
-
 def _generate_groups(obj, group_index, ngroups, axis=0):
     if isinstance(obj, NDFrame) and not isinstance(obj, DataFrame):
         factory = obj._constructor
@@ -428,10 +434,11 @@ class Grouper(object):
     """
 
     """
-    def __init__(self, axis, groupings, sort=True):
+    def __init__(self, axis, groupings, sort=True, group_keys=True):
         self.axis = axis
         self.groupings = groupings
         self.sort = sort
+        self.group_keys = group_keys
 
     @property
     def shape(self):
@@ -964,21 +971,12 @@ def _get_index():
             return index
 
         if isinstance(values[0], Series):
-            if not_indexed_same:
-                data_dict = dict(zip(keys, values))
-                result = DataFrame(data_dict).T
-                result.index = _get_index()
-                return result
-            else:
-                cat_values = np.concatenate([x.values for x in values])
-                cat_index = values[0].index
-                if len(values) > 1:
-                    cat_index = cat_index.append([x.index for x in values[1:]])
-                return Series(cat_values, index=cat_index)
+            return self._concat_objects(keys, values,
+                                        not_indexed_same=not_indexed_same)
         elif isinstance(values[0], DataFrame):
             # possible that Series -> DataFrame by applied function
-            return self._wrap_frames(keys, values,
-                                     not_indexed_same=not_indexed_same)
+            return self._concat_objects(keys, values,
+                                        not_indexed_same=not_indexed_same)
         else:
             return Series(values, index=_get_index())
 
@@ -1318,8 +1316,8 @@ def _wrap_applied_output(self, keys, values, not_indexed_same=False):
         key_names = self.grouper.names
 
         if isinstance(values[0], DataFrame):
-            return self._wrap_frames(keys, values,
-                                     not_indexed_same=not_indexed_same)
+            return self._concat_objects(keys, values,
+                                        not_indexed_same=not_indexed_same)
         else:
             if len(self.grouper.groupings) > 1:
                 key_index = MultiIndex.from_tuples(keys, names=key_names)

pandas/tests/test_groupby.py

@@ -383,7 +383,7 @@ def test_attr_wrapper(self):
         for name, gp in grouped:
             expected[name] = gp.describe()
         expected = DataFrame(expected).T
-        assert_frame_equal(result, expected)
+        assert_frame_equal(result.unstack(), expected)
 
         # get attribute
         result = grouped.dtype
@@ -395,7 +395,7 @@ def test_attr_wrapper(self):
     def test_series_describe_multikey(self):
         ts = tm.makeTimeSeries()
         grouped = ts.groupby([lambda x: x.year, lambda x: x.month])
-        result = grouped.describe()
+        result = grouped.describe().unstack()
         assert_series_equal(result['mean'], grouped.mean())
         assert_series_equal(result['std'], grouped.std())
         assert_series_equal(result['min'], grouped.min())
@@ -405,7 +405,7 @@ def test_series_describe_single(self):
         grouped = ts.groupby(lambda x: x.month)
         result = grouped.apply(lambda x: x.describe())
         expected = grouped.describe()
-        assert_frame_equal(result, expected)
+        assert_series_equal(result, expected)
 
     def test_series_agg_multikey(self):
         ts = tm.makeTimeSeries()
@@ -449,7 +449,7 @@ def test_frame_describe_multikey(self):
 
         for col in self.tsframe:
             expected = grouped[col].describe()
-            assert_frame_equal(result[col].unstack(), expected)
+            assert_series_equal(result[col], expected)
 
         groupedT = self.tsframe.groupby({'A' : 0, 'B' : 0,
                                          'C' : 1, 'D' : 1}, axis=1)
@@ -1581,6 +1581,31 @@ def test_dont_clobber_name_column(self):
         result = df.groupby('key').apply(lambda x: x)
         assert_frame_equal(result, df)
 
+    def test_skip_group_keys(self):
+        from pandas import concat
+
+        tsf = tm.makeTimeDataFrame()
+
+        grouped = tsf.groupby(lambda x: x.month, group_keys=False)
+        result = grouped.apply(lambda x: x.sort_index(by='A')[:3])
+
+        pieces = []
+        for key, group in grouped:
+            pieces.append(group.sort_index(by='A')[:3])
+
+        expected = concat(pieces)
+        assert_frame_equal(result, expected)
+
+        grouped = tsf['A'].groupby(lambda x: x.month, group_keys=False)
+        result = grouped.apply(lambda x: x.order()[:3])
+
+        pieces = []
+        for key, group in grouped:
+            pieces.append(group.order()[:3])
+
+        expected = concat(pieces)
+        assert_series_equal(result, expected)
+
 def _check_groupby(df, result, keys, field, f=lambda x: x.sum()):
     tups = map(tuple, df[keys].values)
     tups = com._asarray_tuplesafe(tups)