TST: add unit tests for PR #965, agg by level, close #934

Author: wesm

pandas/core/frame.py

@@ -3586,23 +3586,23 @@ def mad(self, axis=0, skipna=True, level=None):
             demeaned = frame.sub(frame.mean(axis=1), axis=0)
         return np.abs(demeaned).mean(axis=axis, skipna=skipna)
 
-    @Substitution(name='unbiased variance', shortname='var',
+    @Substitution(name='variance', shortname='var',
                   na_action=_doc_exclude_na, extras='')
     @Appender(_stat_doc)
     def var(self, axis=0, skipna=True, level=None, ddof=1):
         if level is not None:
             return self._agg_by_level('var', axis=axis, level=level,
-                                      skipna=skipna)
+                                      skipna=skipna, ddof=ddof)
         return self._reduce(nanops.nanvar, axis=axis, skipna=skipna,
                             numeric_only=None, ddof=ddof)
 
-    @Substitution(name='unbiased standard deviation', shortname='std',
+    @Substitution(name='standard deviation', shortname='std',
                   na_action=_doc_exclude_na, extras='')
     @Appender(_stat_doc)
     def std(self, axis=0, skipna=True, level=None, ddof=1):
         if level is not None:
             return self._agg_by_level('std', axis=axis, level=level,
-                                      skipna=skipna)
+                                      skipna=skipna, ddof=ddof)
         return np.sqrt(self.var(axis=axis, skipna=skipna, ddof=ddof))
 
     @Substitution(name='unbiased skewness', shortname='skew',
@@ -3615,12 +3615,12 @@ def skew(self, axis=0, skipna=True, level=None):
         return self._reduce(nanops.nanskew, axis=axis, skipna=skipna,
                             numeric_only=None)
 
-    def _agg_by_level(self, name, axis=0, level=0, skipna=True):
+    def _agg_by_level(self, name, axis=0, level=0, skipna=True, **kwds):
         grouped = self.groupby(level=level, axis=axis)
         if hasattr(grouped, name) and skipna:
-            return getattr(grouped, name)()
+            return getattr(grouped, name)(**kwds)
         method = getattr(type(self), name)
-        applyf = lambda x: method(x, axis=axis, skipna=skipna)
+        applyf = lambda x: method(x, axis=axis, skipna=skipna, **kwds)
         return grouped.aggregate(applyf)
 
     def _reduce(self, op, axis=0, skipna=True, numeric_only=None, **kwds):

pandas/core/groupby.py

@@ -263,21 +263,30 @@ def mean(self):
             f = lambda x: x.mean(axis=self.axis)
             return self._python_agg_general(f)
 
-    def std(self):
+    def std(self, ddof=1):
         """
         Compute standard deviation of groups, excluding missing values
 
         For multiple groupings, the result index will be a MultiIndex
         """
-        return self._cython_agg_general('std')
+        # todo, implement at cython level?
+        if ddof == 1:
+            return self._cython_agg_general('std')
+        else:
+            f = lambda x: x.std(ddof=ddof)
+            return self._python_agg_general(f)
 
-    def var(self):
+    def var(self, ddof=1):
         """
         Compute variance of groups, excluding missing values
 
         For multiple groupings, the result index will be a MultiIndex
         """
-        return self._cython_agg_general('var')
+        if ddof == 1:
+            return self._cython_agg_general('var')
+        else:
+            f = lambda x: x.var(ddof=ddof)
+            return self._python_agg_general(f)
 
     def size(self):
         """

pandas/core/series.py

@@ -1005,22 +1005,24 @@ def max(self, axis=None, out=None, skipna=True, level=None):
             return self._agg_by_level('max', level=level, skipna=skipna)
         return nanops.nanmax(self.values, skipna=skipna)
 
-    @Substitution(name='unbiased standard deviation', shortname='stdev',
+    @Substitution(name='standard deviation', shortname='stdev',
                   na_action=_doc_exclude_na, extras='')
     @Appender(_stat_doc)
     def std(self, axis=None, dtype=None, out=None, ddof=1, skipna=True,
             level=None):
         if level is not None:
-            return self._agg_by_level('std', level=level, skipna=skipna)
+            return self._agg_by_level('std', level=level, skipna=skipna,
+                                      ddof=ddof)
         return np.sqrt(nanops.nanvar(self.values, skipna=skipna, ddof=ddof))
 
-    @Substitution(name='unbiased variance', shortname='var',
+    @Substitution(name='variance', shortname='var',
                   na_action=_doc_exclude_na, extras='')
     @Appender(_stat_doc)
     def var(self, axis=None, dtype=None, out=None, ddof=1, skipna=True,
             level=None):
         if level is not None:
-            return self._agg_by_level('var', level=level, skipna=skipna)
+            return self._agg_by_level('var', level=level, skipna=skipna,
+                                      ddof=ddof)
         return nanops.nanvar(self.values, skipna=skipna, ddof=ddof)
 
     @Substitution(name='unbiased skewness', shortname='skew',
@@ -1032,12 +1034,12 @@ def skew(self, skipna=True, level=None):
 
         return nanops.nanskew(self.values, skipna=skipna)
 
-    def _agg_by_level(self, name, level=0, skipna=True):
+    def _agg_by_level(self, name, level=0, skipna=True, **kwds):
         grouped = self.groupby(level=level)
         if hasattr(grouped, name) and skipna:
-            return getattr(grouped, name)()
+            return getattr(grouped, name)(**kwds)
         method = getattr(type(self), name)
-        applyf = lambda x: method(x, skipna=skipna)
+        applyf = lambda x: method(x, skipna=skipna, **kwds)
         return grouped.aggregate(applyf)
 
     def idxmin(self, axis=None, out=None, skipna=True):

pandas/tests/test_frame.py

@@ -4293,14 +4293,21 @@ def test_mad(self):
         f = lambda x: np.abs(x - x.mean()).mean()
         self._check_stat_op('mad', f)
 
-    def test_var(self):
+    def test_var_std(self):
         alt = lambda x: np.var(x, ddof=1)
         self._check_stat_op('var', alt)
 
-    def test_std(self):
         alt = lambda x: np.std(x, ddof=1)
         self._check_stat_op('std', alt)
 
+        result = self.tsframe.std(ddof=4)
+        expected = self.tsframe.apply(lambda x: x.std(ddof=4))
+        assert_almost_equal(result, expected)
+
+        result = self.tsframe.var(ddof=4)
+        expected = self.tsframe.apply(lambda x: x.var(ddof=4))
+        assert_almost_equal(result, expected)
+
     def test_skew(self):
         from scipy.stats import skew
 

pandas/tests/test_multilevel.py

@@ -930,6 +930,24 @@ def test_frame_group_ops(self):
 
             assert_frame_equal(leftside, rightside)
 
+    def test_std_var_pass_ddof(self):
+        index = MultiIndex.from_arrays([np.arange(5).repeat(10),
+                                        np.tile(np.arange(10), 5)])
+        df = DataFrame(np.random.randn(len(index), 5), index=index)
+
+        for meth in ['var', 'std']:
+            ddof = 4
+            alt = lambda x: getattr(x, meth)(ddof=ddof)
+
+            result = getattr(df[0], meth)(level=0, ddof=ddof)
+            expected = df[0].groupby(level=0).agg(alt)
+            assert_series_equal(result, expected)
+
+            result = getattr(df, meth)(level=0, ddof=ddof)
+            expected = df.groupby(level=0).agg(alt)
+            assert_frame_equal(result, expected)
+
+
     def test_frame_series_agg_multiple_levels(self):
         result = self.ymd.sum(level=['year', 'month'])
         expected = self.ymd.groupby(level=['year', 'month']).sum()

pandas/tests/test_series.py

@@ -880,14 +880,21 @@ def test_min(self):
     def test_max(self):
         self._check_stat_op('max', np.max, check_objects=True)
 
-    def test_std(self):
+    def test_var_std(self):
         alt = lambda x: np.std(x, ddof=1)
         self._check_stat_op('std', alt)
 
-    def test_var(self):
         alt = lambda x: np.var(x, ddof=1)
         self._check_stat_op('var', alt)
 
+        result = self.ts.std(ddof=4)
+        expected = np.std(self.ts.values, ddof=4)
+        assert_almost_equal(result, expected)
+
+        result = self.ts.var(ddof=4)
+        expected = np.var(self.ts.values, ddof=4)
+        assert_almost_equal(result, expected)
+
     def test_skew(self):
         from scipy.stats import skew
         alt =lambda x: skew(x, bias=False)