BUG: stat method refactoring to start addressing pandas-dev#382

Author: wesm

pandas/core/frame.py

@@ -31,6 +31,7 @@
 from pandas.core.internals import BlockManager, make_block, form_blocks
 from pandas.core.series import Series, _is_bool_indexer
 from pandas.util import py3compat
+import pandas.core.nanops as nanops
 import pandas.core.common as common
 import pandas.core.datetools as datetools
 import pandas._tseries as lib
@@ -2710,57 +2711,36 @@ def _count_level(self, level, axis=0, numeric_only=False):
         else:
             return result
 
-    def sum(self, axis=0, numeric_only=True, skipna=True, level=None):
+    def sum(self, axis=0, numeric_only=None, skipna=True, level=None):
         if level is not None:
             return self._agg_by_level('sum', axis=axis, level=level,
                                       skipna=skipna)
-
-        y, axis_labels = self._get_agg_data(axis, numeric_only=numeric_only)
-
-        if len(axis_labels) == 0:
-            return Series([], index=[])
-
-        if y.dtype == np.object_:
-            the_sum = y.sum(axis)
-        else:
-            mask = np.isfinite(y)
-
-            if skipna and not issubclass(y.dtype.type, np.integer):
-                np.putmask(y, -mask, 0)
-
-            the_sum = y.sum(axis)
-            the_count = mask.sum(axis)
-
-            ct_mask = the_count == 0
-            if ct_mask.any():
-                the_sum[ct_mask] = nan
-
-        return Series(the_sum, index=axis_labels)
+        return self._reduce(nanops.nansum, axis=axis, skipna=skipna,
+                            numeric_only=numeric_only)
     _add_stat_doc(sum, 'sum', 'sum', extras=_numeric_only_doc)
 
+    def mean(self, axis=0, skipna=True, level=None):
+        if level is not None:
+            return self._agg_by_level('mean', axis=axis, level=level,
+                                      skipna=skipna)
+        return self._reduce(nanops.nanmean, axis=axis, skipna=skipna,
+                            numeric_only=None)
+    _add_stat_doc(mean, 'mean', 'mean')
+
     def min(self, axis=0, skipna=True, level=None):
         if level is not None:
             return self._agg_by_level('min', axis=axis, level=level,
                                       skipna=skipna)
-
-        values, axis_labels = self._get_agg_data(axis, numeric_only=True)
-
-        if skipna and not issubclass(values.dtype.type, np.integer):
-            np.putmask(values, -np.isfinite(values), np.inf)
-
-        return Series(values.min(axis), index=axis_labels)
+        return self._reduce(nanops.nanmin, axis=axis, skipna=skipna,
+                            numeric_only=None)
     _add_stat_doc(min, 'minimum', 'min')
 
     def max(self, axis=0, skipna=True, level=None):
         if level is not None:
             return self._agg_by_level('max', axis=axis, level=level,
                                       skipna=skipna)
-
-        values, axis_labels = self._get_agg_data(axis, numeric_only=True)
-        if skipna and not issubclass(values.dtype.type, np.integer):
-            np.putmask(values, -np.isfinite(values), -np.inf)
-
-        return Series(values.max(axis), index=axis_labels)
+        return self._reduce(nanops.nanmax, axis=axis, skipna=skipna,
+                            numeric_only=None)
     _add_stat_doc(max, 'maximum', 'max')
 
     def prod(self, axis=0, skipna=True, level=None):
@@ -2781,16 +2761,6 @@ def prod(self, axis=0, skipna=True, level=None):
                   na_action='NA/null values are treated as 1')
     product = prod
 
-    def mean(self, axis=0, skipna=True, level=None):
-        if level is not None:
-            return self._agg_by_level('mean', axis=axis, level=level,
-                                      skipna=skipna)
-
-        summed = self.sum(axis, numeric_only=True, skipna=skipna)
-        count = self.count(axis, numeric_only=True).astype(float)
-        return summed / count
-    _add_stat_doc(mean, 'mean', 'mean')
-
     def median(self, axis=0, skipna=True, level=None):
         if level is not None:
             return self._agg_by_level('median', axis=axis, level=level,
@@ -2839,59 +2809,51 @@ def var(self, axis=0, skipna=True, level=None):
         if level is not None:
             return self._agg_by_level('var', axis=axis, level=level,
                                       skipna=skipna)
-
-        y, axis_labels = self._get_agg_data(axis, numeric_only=True)
-
-        mask = np.isnan(y)
-        count = (y.shape[axis] - mask.sum(axis)).astype(float)
-
-        if skipna:
-            np.putmask(y, mask, 0)
-
-        X = y.sum(axis)
-        XX = (y ** 2).sum(axis)
-
-        theVar = (XX - X ** 2 / count) / (count - 1)
-
-        return Series(theVar, index=axis_labels)
+        return self._reduce(nanops.nanvar, axis=axis, skipna=skipna,
+                            numeric_only=None)
     _add_stat_doc(var, 'unbiased variance', 'var')
 
     def std(self, axis=0, skipna=True, level=None):
         if level is not None:
             return self._agg_by_level('std', axis=axis, level=level,
                                       skipna=skipna)
-
         return np.sqrt(self.var(axis=axis, skipna=skipna))
     _add_stat_doc(std, 'unbiased standard deviation', 'std')
 
     def skew(self, axis=0, skipna=True, level=None):
         if level is not None:
             return self._agg_by_level('skew', axis=axis, level=level,
                                       skipna=skipna)
+        return self._reduce(nanops.nanskew, axis=axis, skipna=skipna,
+                            numeric_only=None)
+    _add_stat_doc(skew, 'unbiased skewness', 'skew')
 
-        y, axis_labels = self._get_agg_data(axis, numeric_only=True)
-
-        mask = np.isnan(y)
-        count = (y.shape[axis] - mask.sum(axis)).astype(float)
-
-        if skipna:
-            np.putmask(y, mask, 0)
-
-        A = y.sum(axis) / count
-        B = (y ** 2).sum(axis) / count - A ** 2
-        C = (y ** 3).sum(axis) / count - A ** 3 - 3 * A * B
-
-        # floating point error
-        B = np.where(np.abs(B) < 1e-14, 0, B)
-        C = np.where(np.abs(C) < 1e-14, 0, C)
-
-        result = ((np.sqrt((count ** 2 - count)) * C) /
-                  ((count - 2) * np.sqrt(B) ** 3))
+    def _reduce(self, op, axis=0, skipna=True, numeric_only=None):
 
-        result = np.where(B == 0, 0, result)
+        f = lambda x: op(x, axis=axis, skipna=skipna, copy=True)
+        labels = self._get_agg_axis(axis)
+        if numeric_only is None:
+            try:
+                values = self.values
+                if not self._is_mixed_type:
+                    values = values.copy()
+                result = f(values)
+            except Exception:
+                data = self._get_numeric_data()
+                result = f(data.values)
+                labels = data._get_agg_axis(axis)
+        else:
+            if numeric_only:
+                data = self._get_numeric_data()
+                values = data.values
+                labels = data._get_agg_axis(axis)
+            else:
+                values = self.values
+            result = f(values)
 
-        return Series(result, index=axis_labels)
-    _add_stat_doc(skew, 'unbiased skewness', 'skew')
+        if result.dtype == np.object_:
+            result = result.astype('f8')
+        return Series(result, index=labels)
 
     def idxmin(self, axis=0, skipna=True):
         """

pandas/core/nanops.py

@@ -0,0 +1,100 @@
+import numpy as np
+
+from pandas.core.common import isnull, notnull
+
+def nansum(values, axis=0, skipna=True, copy=True):
+    if values.dtype == np.object_:
+        the_sum = values.sum(axis)
+    else:
+        mask = notnull(values)
+
+        if skipna and not issubclass(values.dtype.type, np.integer):
+            if copy:
+                values = values.copy()
+            np.putmask(values, -mask, 0)
+
+        the_sum = values.sum(axis)
+        the_count = mask.sum(axis)
+
+        ct_mask = the_count == 0
+        if ct_mask.any():
+            the_sum[ct_mask] = np.nan
+
+    return the_sum
+
+def nanmean(values, axis=0, skipna=True, copy=True):
+    if values.dtype == np.object_:
+        the_mean = values.sum(axis) / float(values.shape[axis])
+    else:
+        mask = notnull(values)
+
+        if skipna and not issubclass(values.dtype.type, np.integer):
+            if copy:
+                values = values.copy()
+            np.putmask(values, -mask, 0)
+
+        the_sum = values.sum(axis)
+        the_count = mask.sum(axis)
+        the_mean = the_sum / the_count.astype('f8')
+
+        ct_mask = the_count == 0
+        if ct_mask.any():
+            the_mean[ct_mask] = np.nan
+
+    return the_mean
+
+def nanvar(values, axis=0, skipna=True, copy=True):
+    mask = isnull(values)
+    count = (values.shape[axis] - mask.sum(axis)).astype(float)
+
+    if skipna:
+        if copy:
+            values = values.copy()
+        np.putmask(values, mask, 0)
+
+    X = values.sum(axis)
+    XX = (values ** 2).sum(axis)
+    return (XX - X ** 2 / count) / (count - 1)
+
+def nanskew(values, axis=0, skipna=True, copy=True):
+    if not isinstance(values.dtype.type, np.floating):
+        values = values.astype('f8')
+
+    mask = isnull(values)
+    count = (values.shape[axis] - mask.sum(axis)).astype(float)
+
+    if skipna:
+        if copy:
+            values = values.copy()
+        np.putmask(values, mask, 0)
+
+    A = values.sum(axis) / count
+    B = (values ** 2).sum(axis) / count - A ** 2
+    C = (values ** 3).sum(axis) / count - A ** 3 - 3 * A * B
+
+    # floating point error
+    B = np.where(np.abs(B) < 1e-14, 0, B)
+    C = np.where(np.abs(C) < 1e-14, 0, C)
+
+    result = ((np.sqrt((count ** 2 - count)) * C) /
+              ((count - 2) * np.sqrt(B) ** 3))
+
+    result = np.where(B == 0, 0, result)
+
+    return result
+
+def nanmin(values, axis=0, skipna=True, copy=True):
+    if skipna and not issubclass(values.dtype.type, np.integer):
+        if copy:
+            values = values.copy()
+        np.putmask(values, isnull(values), np.inf)
+    return values.min(axis)
+
+def nanmax(values, axis=0, skipna=True, copy=True):
+    if skipna and not issubclass(values.dtype.type, np.integer):
+        if copy:
+            values = values.copy()
+        np.putmask(values, isnull(values), -np.inf)
+
+    return values.max(axis)
+

pandas/tests/test_frame.py

@@ -3002,12 +3002,6 @@ def test_get_numeric_data(self):
         expected = df.ix[:, []]
         assert_frame_equal(result, expected)
 
-    def test_statistics(self):
-        # unnecessary?
-        sumFrame = self.frame.apply(np.sum)
-        for col, series in self.frame.iteritems():
-            self.assertEqual(sumFrame[col], series.sum())
-
     def test_count(self):
         f = lambda s: notnull(s).sum()
         self._check_stat_op('count', f, has_skipna=False)
@@ -3024,6 +3018,24 @@ def test_count(self):
     def test_sum(self):
         self._check_stat_op('sum', np.sum)
 
+    def test_stat_ops_attempt_obj_array(self):
+        data = {
+            'a': [-0.00049987540199591344, -0.0016467257772919831,
+                   0.00067695870775883013],
+            'b': [-0, -0, 0.0],
+            'c': [0.00031111847529610595, 0.0014902627951905339,
+                  -0.00094099200035979691]
+        }
+        df = DataFrame(data, index=['foo', 'bar', 'baz'],
+                       dtype='O')
+        methods = ['sum', 'mean', 'var', 'std', 'skew', 'min', 'max']
+
+        for meth in methods:
+            self.assert_(df.values.dtype == np.object_)
+            result = getattr(df, meth)(1)
+            expected = getattr(df.astype('f8'), meth)(1)
+            assert_series_equal(result, expected)
+
     def test_mean(self):
         self._check_stat_op('mean', np.mean)