TST: groupby.describe levels don't appear as column (#4792)

Restructure describe def

Fix another test

Refactoring tests

linting & patch groupby tests

add whatsnew

fix docstring

Author: mroeschke

doc/source/whatsnew/v0.20.0.txt

@@ -366,6 +366,7 @@ Other API Changes
 - ``inplace`` arguments now require a boolean value, else a ``ValueError`` is thrown (:issue:`14189`)
 - ``pandas.api.types.is_datetime64_ns_dtype`` will now report ``True`` on a tz-aware dtype, similar to ``pandas.api.types.is_datetime64_any_dtype``
  - ``DataFrame.asof()`` will return a null filled ``Series`` instead the scalar ``NaN`` if a match is not found (:issue:`15118`)
+ - ``groupby.describe()`` now labels the `describe()` metrics in the column instead of the index (:issue:`4792`)
 .. _whatsnew_0200.deprecations:
 
 Deprecations

pandas/core/groupby.py

@@ -80,7 +80,6 @@
     'mean', 'sum', 'min', 'max',
     'cumcount',
     'resample',
-    'describe',
     'rank', 'quantile',
     'fillna',
     'mad',
@@ -1138,6 +1137,17 @@ def ohlc(self):
         return self._apply_to_column_groupbys(
             lambda x: x._cython_agg_general('ohlc'))
 
+    @Substitution(name='groupby')
+    @Appender(_doc_template)
+    def describe(self, **kwargs):
+        """
+        Provide summary statistics for each group, excluding NaN values
+        """
+        result = self.apply(lambda x: x.describe(**kwargs))
+        if self.axis == 1:
+            return result.T
+        return result.unstack()
+
     @Substitution(name='groupby')
     @Appender(_doc_template)
     def resample(self, rule, *args, **kwargs):
@@ -3039,6 +3049,13 @@ def nlargest(self, n=5, keep='first'):
     def nsmallest(self, n=5, keep='first'):
         return self.apply(lambda x: x.nsmallest(n=n, keep=keep))
 
+    @Appender(Series.describe.__doc__)
+    def describe(self, **kwargs):
+        result = self.apply(lambda x: x.describe(**kwargs))
+        if self.axis == 1:
+            return result.T
+        return result.unstack()
+
     def value_counts(self, normalize=False, sort=True, ascending=False,
                      bins=None, dropna=True):
 

pandas/tests/groupby/test_groupby.py

@@ -1447,7 +1447,7 @@ def test_attr_wrapper(self):
         for name, gp in grouped:
             expected[name] = gp.describe()
         expected = DataFrame(expected).T
-        assert_frame_equal(result.unstack(), expected)
+        assert_frame_equal(result, expected)
 
         # get attribute
         result = grouped.dtype
@@ -1459,7 +1459,7 @@ def test_attr_wrapper(self):
     def test_series_describe_multikey(self):
         ts = tm.makeTimeSeries()
         grouped = ts.groupby([lambda x: x.year, lambda x: x.month])
-        result = grouped.describe().unstack()
+        result = grouped.describe()
         assert_series_equal(result['mean'], grouped.mean(), check_names=False)
         assert_series_equal(result['std'], grouped.std(), check_names=False)
         assert_series_equal(result['min'], grouped.min(), check_names=False)
@@ -1468,7 +1468,7 @@ def test_series_describe_single(self):
         ts = tm.makeTimeSeries()
         grouped = ts.groupby(lambda x: x.month)
         result = grouped.apply(lambda x: x.describe())
-        expected = grouped.describe()
+        expected = grouped.describe().stack()
         assert_series_equal(result, expected)
 
     def test_series_index_name(self):
@@ -1479,17 +1479,27 @@ def test_series_index_name(self):
     def test_frame_describe_multikey(self):
         grouped = self.tsframe.groupby([lambda x: x.year, lambda x: x.month])
         result = grouped.describe()
-
+        desc_groups = []
         for col in self.tsframe:
-            expected = grouped[col].describe()
-            assert_series_equal(result[col], expected, check_names=False)
+            group = grouped[col].describe()
+            group_col = pd.MultiIndex([[col] * len(group.columns),
+                                       group.columns],
+                                      [[0] * len(group.columns),
+                                       range(len(group.columns))])
+            group = pd.DataFrame(group.values,
+                                 columns=group_col,
+                                 index=group.index)
+            desc_groups.append(group)
+        expected = pd.concat(desc_groups, axis=1)
+        tm.assert_frame_equal(result, expected)
 
         groupedT = self.tsframe.groupby({'A': 0, 'B': 0,
                                          'C': 1, 'D': 1}, axis=1)
         result = groupedT.describe()
-
-        for name, group in groupedT:
-            assert_frame_equal(result[name], group.describe())
+        expected = self.tsframe.describe().T
+        expected.index = pd.MultiIndex([[0, 0, 1, 1], expected.index],
+                                       [range(4), range(len(expected.index))])
+        tm.assert_frame_equal(result, expected)
 
     def test_frame_describe_tupleindex(self):
 
@@ -1499,10 +1509,27 @@ def test_frame_describe_tupleindex(self):
                          'z': [100, 200, 300, 400, 500] * 3})
         df1['k'] = [(0, 0, 1), (0, 1, 0), (1, 0, 0)] * 5
         df2 = df1.rename(columns={'k': 'key'})
-        result = df1.groupby('k').describe()
-        expected = df2.groupby('key').describe()
-        expected.index.set_names(result.index.names, inplace=True)
-        assert_frame_equal(result, expected)
+        tm.assertRaises(ValueError, lambda: df1.groupby('k').describe())
+        tm.assertRaises(ValueError, lambda: df2.groupby('key').describe())
+
+    def test_frame_describe_multiindex_level_not_as_column(self):
+        # GH 4792
+        prices = {pd.Timestamp('2011-01-06 10:59:05', tz=None): 24990,
+                  pd.Timestamp('2011-01-06 12:43:33', tz=None): 25499,
+                  pd.Timestamp('2011-01-06 12:54:09', tz=None): 25499}
+        volumes = {pd.Timestamp('2011-01-06 10:59:05', tz=None): 1500000000,
+                   pd.Timestamp('2011-01-06 12:43:33', tz=None): 5000000000,
+                   pd.Timestamp('2011-01-06 12:54:09', tz=None): 100000000}
+        df = pd.DataFrame({'PRICE': prices,
+                           'VOLUME': volumes})
+        result = df.groupby('PRICE').VOLUME.describe()
+        data = [df[df.PRICE == 24990].VOLUME.describe().values.tolist(),
+                df[df.PRICE == 25499].VOLUME.describe().values.tolist()]
+        expected = pd.DataFrame(data,
+                                index=pd.Index([24990, 25499], name='PRICE'),
+                                columns=['count', 'mean', 'std', 'min',
+                                         '25%', '50%', '75%', 'max'])
+        tm.assert_frame_equal(result, expected)
 
     def test_frame_groupby(self):
         grouped = self.tsframe.groupby(lambda x: x.weekday())
@@ -2994,16 +3021,25 @@ def test_non_cython_api(self):
         assert_frame_equal(result, expected)
 
         # describe
-        expected = DataFrame(dict(B=concat(
-            [df.loc[[0, 1], 'B'].describe(), df.loc[[2], 'B'].describe()],
-            keys=[1, 3])))
-        expected.index.names = ['A', None]
+        expected = pd.concat([(df[df.A == 1].B
+                                            .describe()
+                                            .to_frame()
+                                            .unstack()
+                                            .to_frame()
+                                            .T),
+                              (df[df.A == 3].B
+                                            .describe()
+                                            .to_frame()
+                                            .unstack()
+                                            .to_frame()
+                                            .T)])
+        expected.index = pd.Index([1, 3], name='A')
         result = g.describe()
         assert_frame_equal(result, expected)
 
-        expected = concat(
-            [df.loc[[0, 1], ['A', 'B']].describe(),
-             df.loc[[2], ['A', 'B']].describe()], keys=[0, 1])
+        expected = pd.concat([df[df.A == 1].describe().unstack().to_frame().T,
+                              df[df.A == 3].describe().unstack().to_frame().T])
+        expected.index = pd.Index([0, 1])
         result = gni.describe()
         assert_frame_equal(result, expected)
 
@@ -5149,7 +5185,6 @@ def test_groupby_whitelist(self):
             'tail',
             'cumcount',
             'resample',
-            'describe',
             'rank',
             'quantile',
             'fillna',
@@ -5186,7 +5221,6 @@ def test_groupby_whitelist(self):
             'tail',
             'cumcount',
             'resample',
-            'describe',
             'rank',
             'quantile',
             'fillna',