ENH: refactor Series stat ops to use common nanops, test pandas-dev#384 for Series

Author: wesm

RELEASE.rst

@@ -101,6 +101,8 @@ pandas 0.6.0
   - Can pass multiple values columns to `pivot_table` (GH #381)
   - Can call `DataFrame.delevel` with standard Index with name set (GH #393)
   - Use Series name in GroupBy for result index (GH #363)
+  - Refactor Series/DataFrame stat methods to use common set of NaN-friendly
+    function
 
 **Bug fixes**
 
@@ -142,6 +144,9 @@ pandas 0.6.0
   - Don't lose index names in `MultiIndex.droplevel` (GH #394)
   - Infer more proper return type in `DataFrame.apply` when no columns or rows
     depending on whether the passed function is a reduction (GH #389)
+  - Always return NA/NaN from Series.min/max and DataFrame.min/max when all of a
+    row/column/values are NA (GH #384)
+
 
 Thanks
 ------
@@ -157,6 +162,7 @@ Thanks
 - Wouter Overmeire
 - Nathan Pinger
 - Christian Prinoth
+- Skipper Seabold
 - Chang She
 - Ted Square
 - Aman Thakral

pandas/core/nanops.py

@@ -3,48 +3,53 @@
 from pandas.core.common import isnull, notnull
 import pandas._tseries as lib
 
-def nansum(values, axis=0, skipna=True, copy=True):
+try:
+    import bottleneck as bn
+    _USE_BOTTLENECK = True
+except ImportError:
+    _USE_BOTTLENECK = False
+
+def nansum(values, axis=None, skipna=True, copy=True):
     if values.dtype == np.object_:
         the_sum = values.sum(axis)
     else:
-        mask = notnull(values)
+        mask = isnull(values)
 
         if skipna and not issubclass(values.dtype.type, np.integer):
             if copy:
                 values = values.copy()
-            np.putmask(values, -mask, 0)
+            np.putmask(values, mask, 0)
 
         the_sum = values.sum(axis)
-        the_count = mask.sum(axis)
-
-        ct_mask = the_count == 0
-        if ct_mask.any():
-            the_sum[ct_mask] = np.nan
+        the_sum = _maybe_null_out(the_sum, axis, mask)
 
     return the_sum
 
-def nanmean(values, axis=0, skipna=True, copy=True):
+def nanmean(values, axis=None, skipna=True, copy=True):
     if values.dtype == np.object_:
         the_mean = values.sum(axis) / float(values.shape[axis])
     else:
-        mask = notnull(values)
+        mask = isnull(values)
 
         if skipna and not issubclass(values.dtype.type, np.integer):
             if copy:
                 values = values.copy()
-            np.putmask(values, -mask, 0)
+            np.putmask(values, mask, 0)
 
         the_sum = values.sum(axis)
-        the_count = mask.sum(axis)
-        the_mean = the_sum / the_count.astype('f8')
+        count = _get_counts(mask, axis)
 
-        ct_mask = the_count == 0
-        if ct_mask.any():
-            the_mean[ct_mask] = np.nan
+        if axis is not None:
+            the_mean = the_sum / count
+            ct_mask = count == 0
+            if ct_mask.any():
+                the_mean[ct_mask] = np.nan
+        else:
+            the_mean = the_sum / count if count > 0 else np.nan
 
     return the_mean
 
-def nanmedian(values, axis=0, skipna=True, copy=True):
+def nanmedian(values, axis=None, skipna=True, copy=True):
     def get_median(x):
         mask = notnull(x)
         if not skipna and not mask.all():
@@ -57,11 +62,18 @@ def get_median(x):
     if axis == 0:
         values = values.T
 
-    return np.asarray([get_median(arr) for arr in values])
+    if values.ndim > 1:
+        return np.asarray([get_median(arr) for arr in values])
+    else:
+        return get_median(values)
 
-def nanvar(values, axis=0, skipna=True, copy=True):
+def nanvar(values, axis=None, skipna=True, copy=True, ddof=1):
     mask = isnull(values)
-    count = (values.shape[axis] - mask.sum(axis)).astype(float)
+
+    if axis is not None:
+        count = (values.shape[axis] - mask.sum(axis)).astype(float)
+    else:
+        count = float(values.size - mask.sum())
 
     if skipna:
         if copy:
@@ -70,14 +82,14 @@ def nanvar(values, axis=0, skipna=True, copy=True):
 
     X = values.sum(axis)
     XX = (values ** 2).sum(axis)
-    return (XX - X ** 2 / count) / (count - 1)
+    return (XX - X ** 2 / count) / (count - ddof)
 
-def nanskew(values, axis=0, skipna=True, copy=True):
+def nanskew(values, axis=None, skipna=True, copy=True):
     if not isinstance(values.dtype.type, np.floating):
         values = values.astype('f8')
 
     mask = isnull(values)
-    count = (values.shape[axis] - mask.sum(axis)).astype(float)
+    count = _get_counts(mask, axis)
 
     if skipna:
         if copy:
@@ -89,49 +101,67 @@ def nanskew(values, axis=0, skipna=True, copy=True):
     C = (values ** 3).sum(axis) / count - A ** 3 - 3 * A * B
 
     # floating point error
-    B = np.where(np.abs(B) < 1e-14, 0, B)
-    C = np.where(np.abs(C) < 1e-14, 0, C)
+    B = _zero_out_fperr(B)
+    C = _zero_out_fperr(C)
 
     result = ((np.sqrt((count ** 2 - count)) * C) /
               ((count - 2) * np.sqrt(B) ** 3))
 
-    result = np.where(B == 0, 0, result)
-
-    return result
+    if isinstance(result, np.ndarray):
+        return np.where(B == 0, 0, result)
+    else:
+        return 0 if B == 0 else result
 
-def nanmin(values, axis=0, skipna=True, copy=True):
+def nanmin(values, axis=None, skipna=True, copy=True):
     mask = isnull(values)
     if skipna and not issubclass(values.dtype.type, np.integer):
         if copy:
             values = values.copy()
         np.putmask(values, mask, np.inf)
     result = values.min(axis)
+    return _maybe_null_out(result, axis, mask)
 
-    null_mask = (mask.shape[axis] - mask.sum(axis)) == 0
-    if null_mask.any():
-        result = result.astype('f8')
-        result[null_mask] = np.nan
-    return result
-
-def nanmax(values, axis=0, skipna=True, copy=True):
+def nanmax(values, axis=None, skipna=True, copy=True):
     mask = isnull(values)
     if skipna and not issubclass(values.dtype.type, np.integer):
         if copy:
             values = values.copy()
         np.putmask(values, mask, -np.inf)
     result = values.max(axis)
+    return _maybe_null_out(result, axis, mask)
 
-    null_mask = (mask.shape[axis] - mask.sum(axis)) == 0
-    if null_mask.any():
-        result = result.astype('f8')
-        result[null_mask] = np.nan
-    return result
-
-def nanprod(values, axis=0, skipna=True, copy=True):
+def nanprod(values, axis=None, skipna=True, copy=True):
     mask = isnull(values)
     if skipna and not issubclass(values.dtype.type, np.integer):
+        if copy:
+            values = values.copy()
         values[mask] = 1
     result = values.prod(axis)
-    count = mask.shape[axis] - mask.sum(axis)
-    result[count == 0] = np.nan
+    return _maybe_null_out(result, axis, mask)
+
+def _get_counts(mask, axis):
+    if axis is not None:
+        count = (mask.shape[axis] - mask.sum(axis)).astype(float)
+    else:
+        count = float(mask.size - mask.sum())
+
+    return count
+
+def _maybe_null_out(result, axis, mask):
+    if axis is not None:
+        null_mask = (mask.shape[axis] - mask.sum(axis)) == 0
+        if null_mask.any():
+            result = result.astype('f8')
+            result[null_mask] = np.nan
+    else:
+        null_mask = mask.size - mask.sum()
+        if null_mask == 0:
+            result = np.nan
+
     return result
+
+def _zero_out_fperr(arg):
+    if isinstance(arg, np.ndarray):
+        return np.where(np.abs(arg) < 1e-14, 0, arg)
+    else:
+        return 0 if np.abs(arg) < 1e-14 else arg

pandas/core/series.py

@@ -22,6 +22,7 @@
 from pandas.util import py3compat
 import pandas.core.common as common
 import pandas.core.datetools as datetools
+import pandas.core.nanops as nanops
 import pandas._tseries as lib
 
 __all__ = ['Series', 'TimeSeries']
@@ -640,23 +641,13 @@ def nunique(self):
     def sum(self, axis=0, dtype=None, out=None, skipna=True, level=None):
         if level is not None:
             return self._agg_by_level('sum', level=level, skipna=skipna)
-
-        values = self.values.copy()
-
-        if skipna:
-            mask = isnull(values)
-            if mask.all():
-                return np.nan
-            np.putmask(values, mask, 0)
-
-        return values.sum()
+        return nanops.nansum(self.values, skipna=skipna, copy=True)
     _add_stat_doc(sum, 'sum', 'sum', extras=_doc_ndarray_interface)
 
     def mean(self, axis=0, dtype=None, out=None, skipna=True, level=None):
         if level is not None:
             return self._agg_by_level('mean', level=level, skipna=skipna)
-
-        return self._ndarray_statistic('mean', dtype=dtype, skipna=skipna)
+        return nanops.nanmean(self.values, skipna=skipna)
     _add_stat_doc(mean, 'mean', 'mean', extras=_doc_ndarray_interface)
 
     def mad(self, skipna=True, level=None):
@@ -670,99 +661,47 @@ def mad(self, skipna=True, level=None):
     def median(self, skipna=True, level=None):
         if level is not None:
             return self._agg_by_level('median', level=level, skipna=skipna)
-
-        arr = self.values
-        if arr.dtype != np.float_:
-            arr = arr.astype(float)
-        mask = notnull(arr)
-
-        if skipna:
-            arr = arr[mask]
-        else:
-            if not mask.all():
-                return np.nan
-
-        return lib.median(arr)
+        return nanops.nanmedian(self.values, skipna=skipna)
     _add_stat_doc(median, 'median', 'median')
 
-    def prod(self, axis=0, dtype=None, out=None, skipna=True, level=None):
+    def prod(self, axis=None, dtype=None, out=None, skipna=True, level=None):
         if level is not None:
             return self._agg_by_level('prod', level=level, skipna=skipna)
-
-        return self._ndarray_statistic('prod', dtype=dtype, skipna=skipna)
+        return nanops.nanprod(self.values, skipna=skipna)
     _add_stat_doc(prod, 'product', 'product')
 
     def min(self, axis=None, out=None, skipna=True, level=None):
         if level is not None:
             return self._agg_by_level('min', level=level, skipna=skipna)
-
-        arr = self.values.copy()
-
-        if skipna:
-            if not issubclass(arr.dtype.type, np.integer):
-                np.putmask(arr, isnull(arr), np.inf)
-
-        return arr.min()
+        return nanops.nanmin(self.values, skipna=skipna, copy=True)
     _add_stat_doc(min, 'minimum', 'min')
 
     def max(self, axis=None, out=None, skipna=True, level=None):
         if level is not None:
             return self._agg_by_level('max', level=level, skipna=skipna)
-
-        arr = self.values.copy()
-
-        if skipna:
-            if not issubclass(arr.dtype.type, np.integer):
-                np.putmask(arr, isnull(arr), -np.inf)
-
-        return arr.max()
+        return nanops.nanmax(self.values, skipna=skipna, copy=True)
     _add_stat_doc(max, 'maximum', 'max')
 
     def std(self, axis=None, dtype=None, out=None, ddof=1, skipna=True,
             level=None):
         if level is not None:
             return self._agg_by_level('std', level=level, skipna=skipna)
-
-        if skipna:
-            nona = remove_na(self.values)
-            if len(nona) < 2:
-                return nan
-            return ndarray.std(nona, axis, dtype, out, ddof)
-
-        return self.values.std(axis, dtype, out, ddof)
+        return np.sqrt(nanops.nanvar(self.values, skipna=skipna, copy=True,
+                                     ddof=ddof))
     _add_stat_doc(std, 'unbiased standard deviation', 'stdev')
 
     def var(self, axis=None, dtype=None, out=None, ddof=1, skipna=True,
             level=None):
         if level is not None:
             return self._agg_by_level('var', level=level, skipna=skipna)
-
-        if skipna:
-            nona = remove_na(self.values)
-            if len(nona) < 2:
-                return nan
-            return ndarray.var(nona, axis, dtype, out, ddof)
-
-        return self.values.var(axis, dtype, out, ddof)
+        return nanops.nanvar(self.values, skipna=skipna, copy=True, ddof=ddof)
     _add_stat_doc(var, 'unbiased variance', 'var')
 
     def skew(self, skipna=True, level=None):
         if level is not None:
             return self._agg_by_level('skew', level=level, skipna=skipna)
 
-        y = np.array(self.values)
-        mask = notnull(y)
-        count = mask.sum()
-
-        if count < len(self) and not skipna:
-            return np.nan
-
-        np.putmask(y, -mask, 0)
-        A = y.sum() / count
-        B = (y**2).sum() / count  - A**2
-        C = (y**3).sum() / count - A**3 - 3*A*B
-
-        return (np.sqrt((count**2-count))*C) / ((count-2)*np.sqrt(B)**3)
+        return nanops.nanskew(self.values, skipna=skipna, copy=True)
     _add_stat_doc(skew, 'unbiased skewness', 'skew')
 
     def idxmin(self, axis=None, out=None, skipna=True):
@@ -803,18 +742,6 @@ def idxmax(self, axis=None, out=None, skipna=True):
                 np.putmask(arr, isnull(arr), -np.inf)
         return self.index[arr.argmax()]
 
-    def _ndarray_statistic(self, funcname, dtype=None, skipna=True):
-        arr = self.values
-        retVal = getattr(arr, funcname)(dtype=dtype)
-
-        if skipna and isnull(retVal):
-            arr = remove_na(arr)
-            if len(arr) == 0:
-                return np.nan
-            retVal = getattr(arr, funcname)(dtype=dtype)
-
-        return retVal
-
     def _agg_by_level(self, name, level=0, skipna=True):
         method = getattr(type(self), name)
         applyf = lambda x: method(x, skipna=skipna)

pandas/tests/test_series.py

@@ -587,7 +587,7 @@ def _check_stat_op(self, name, alternate):
         assert_almost_equal(f(nona), alternate(nona))
 
         allna = self.series * nan
-        self.assert_(isnull(f(allna)))
+        self.assert_(np.isnan(f(allna)))
 
     def _check_accum_op(self, name):
         func = getattr(np, name)