ENH: make any/all conform to sum/mean interface. Fixed bug in copy keyword in BlockManager.get_numeric_data #1416

Author: Chang She

pandas/core/frame.py

@@ -528,18 +528,6 @@ def shape(self):
     def empty(self):
         return not (len(self.columns) > 0 and len(self.index) > 0)
 
-    def any(self):
-        if not self._is_mixed_type:
-            if self.dtypes[0] == np.bool_:
-                return self.values.any()
-        raise ValueError('Cannot call any() on mixed or non-boolean DataFrame')
-
-    def all(self):
-        if not self._is_mixed_type:
-            if self.dtypes[0] == np.bool_:
-                return self.values.all()
-        raise ValueError('Cannot call all() on mixed or non-boolean DataFrame')
-
     def __nonzero__(self):
         raise ValueError("Cannot call bool() on DataFrame.")
 
@@ -4057,6 +4045,62 @@ def _count_level(self, level, axis=0, numeric_only=False):
         else:
             return result
 
+    def any(self, axis=0, bool_only=None, skipna=True, level=None):
+        """
+        Return whether any element is True over requested axis.
+        %(na_action)s
+
+        Parameters
+        ----------
+        axis : {0, 1}
+            0 for row-wise, 1 for column-wise
+        skipna : boolean, default True
+            Exclude NA/null values. If an entire row/column is NA, the result
+            will be NA
+        level : int, default None
+            If the axis is a MultiIndex (hierarchical), count along a
+            particular level, collapsing into a DataFrame
+        bool_only : boolean, default None
+            Only include boolean data.
+
+        Returns
+        -------
+        any : Series (or DataFrame if level specified)
+        """
+        if level is not None:
+            return self._agg_by_level('any', axis=axis, level=level,
+                                      skipna=skipna)
+        return self._reduce(nanops.nanany, axis=axis, skipna=skipna,
+                            numeric_only=bool_only, filter_type='bool')
+
+    def all(self, axis=0, bool_only=None, skipna=True, level=None):
+        """
+        Return whether any element is True over requested axis.
+        %(na_action)s
+
+        Parameters
+        ----------
+        axis : {0, 1}
+            0 for row-wise, 1 for column-wise
+        skipna : boolean, default True
+            Exclude NA/null values. If an entire row/column is NA, the result
+            will be NA
+        level : int, default None
+            If the axis is a MultiIndex (hierarchical), count along a
+            particular level, collapsing into a DataFrame
+        bool_only : boolean, default None
+            Only include boolean data.
+
+        Returns
+        -------
+        any : Series (or DataFrame if level specified)
+        """
+        if level is not None:
+            return self._agg_by_level('all', axis=axis, level=level,
+                                      skipna=skipna)
+        return self._reduce(nanops.nanall, axis=axis, skipna=skipna,
+                            numeric_only=bool_only, filter_type='bool')
+
     @Substitution(name='sum', shortname='sum', na_action=_doc_exclude_na,
                   extras=_numeric_only_doc)
     @Appender(_stat_doc)
@@ -4183,20 +4227,33 @@ def _agg_by_level(self, name, axis=0, level=0, skipna=True, **kwds):
         applyf = lambda x: method(x, axis=axis, skipna=skipna, **kwds)
         return grouped.aggregate(applyf)
 
-    def _reduce(self, op, axis=0, skipna=True, numeric_only=None, **kwds):
+    def _reduce(self, op, axis=0, skipna=True, numeric_only=None,
+                filter_type=None, **kwds):
         f = lambda x: op(x, axis=axis, skipna=skipna, **kwds)
         labels = self._get_agg_axis(axis)
         if numeric_only is None:
             try:
                 values = self.values
                 result = f(values)
             except Exception:
-                data = self._get_numeric_data()
+                if filter_type is None or filter_type == 'numeric':
+                    data = self._get_numeric_data()
+                elif filter_type == 'bool':
+                    data = self._get_bool_data()
+                else:
+                    raise ValueError('Invalid filter_type %s ' %
+                                     str(filter_type))
                 result = f(data.values)
                 labels = data._get_agg_axis(axis)
         else:
             if numeric_only:
-                data = self._get_numeric_data()
+                if filter_type is None or filter_type == 'numeric':
+                    data = self._get_numeric_data()
+                elif filter_type == 'bool':
+                    data = self._get_bool_data()
+                else:
+                    raise ValueError('Invalid filter_type %s ' %
+                                     str(filter_type))
                 values = data.values
                 labels = data._get_agg_axis(axis)
             else:
@@ -4205,7 +4262,13 @@ def _reduce(self, op, axis=0, skipna=True, numeric_only=None, **kwds):
 
         if result.dtype == np.object_:
             try:
-                result = result.astype('f8')
+                if filter_type is None or filter_type == 'numeric':
+                    result = result.astype('f8')
+                elif filter_type == 'bool':
+                    result = result.astype('b')
+                else:
+                    raise ValueError('Invalid dtype %s ' % str(filter_type))
+
             except (ValueError, TypeError):
                 pass
 
@@ -4274,6 +4337,16 @@ def _get_numeric_data(self):
             else:
                 return self.ix[:, []]
 
+    def _get_bool_data(self):
+        if self._is_mixed_type:
+            bool_data = self._data.get_bool_data()
+            return DataFrame(bool_data, copy=False)
+        else:
+            if self.values.dtype == np.bool_:
+                return self
+            else:
+                return self.ix[:, []]
+
     def quantile(self, q=0.5, axis=0):
         """
         Return values at the given quantile over requested axis, a la

pandas/core/internals.py

@@ -492,6 +492,10 @@ def __init__(self, blocks, axes, do_integrity_check=True):
         if do_integrity_check:
             self._verify_integrity()
 
+    @classmethod
+    def make_empty(self):
+        return BlockManager([], [[], []])
+
     def __nonzero__(self):
         return True
 
@@ -589,10 +593,28 @@ def is_consolidated(self):
         dtypes = [blk.dtype.type for blk in self.blocks]
         return len(dtypes) == len(set(dtypes))
 
-    def get_numeric_data(self, copy=False):
-        num_blocks = [b for b in self.blocks
-                      if (isinstance(b, (IntBlock, FloatBlock, ComplexBlock))
-                          and not isinstance(b, DatetimeBlock))]
+    def get_numeric_data(self, copy=False, type_list=None):
+        """
+        Parameters
+        ----------
+        copy : boolean, default False
+            Whether to copy the blocks
+        type_list : tuple of type, default None
+            Numeric types by default (Float/Complex/Int but not Datetime)
+        """
+        if type_list is None:
+            def filter_blocks(block):
+                return (isinstance(block, (IntBlock, FloatBlock, ComplexBlock))
+                        and not isinstance(block, DatetimeBlock))
+        else:
+            type_list = self._get_clean_block_types(type_list)
+            filter_blocks = lambda block: isinstance(block, type_list)
+
+        maybe_copy = lambda b: b.copy() if copy else b
+        num_blocks = [maybe_copy(b) for b in self.blocks if filter_blocks(b)]
+
+        if len(num_blocks) == 0:
+            return BlockManager.make_empty()
 
         indexer = np.sort(np.concatenate([b.ref_locs for b in num_blocks]))
         new_items = self.items.take(indexer)
@@ -606,6 +628,26 @@ def get_numeric_data(self, copy=False):
         new_axes[0] = new_items
         return BlockManager(new_blocks, new_axes, do_integrity_check=False)
 
+    def _get_clean_block_types(self, type_list):
+        if not isinstance(type_list, tuple):
+            try:
+                type_list = tuple(type_list)
+            except TypeError:
+                type_list = (type_list,)
+
+        type_map = {int : IntBlock, float : FloatBlock,
+                    complex : ComplexBlock,
+                    np.datetime64 : DatetimeBlock,
+                    datetime : DatetimeBlock,
+                    bool : BoolBlock,
+                    object : ObjectBlock}
+
+        type_list = tuple([type_map.get(t, t) for t in type_list])
+        return type_list
+
+    def get_bool_data(self, copy=False):
+        return self.get_numeric_data(copy=copy, type_list=(BoolBlock,))
+
     def get_slice(self, slobj, axis=0):
         new_axes = list(self.axes)
         new_axes[axis] = new_axes[axis][slobj]

pandas/core/nanops.py

@@ -57,6 +57,22 @@ def _has_infs(result):
     else:
         return np.isinf(result) or np.isneginf(result)
 
+def nanany(values, axis=None, skipna=True):
+    mask = isnull(values)
+
+    if skipna:
+        values = values.copy()
+        np.putmask(values, mask, False)
+    return values.any(axis)
+
+def nanall(values, axis=None, skipna=True):
+    mask = isnull(values)
+
+    if skipna:
+        values = values.copy()
+        np.putmask(values, mask, True)
+    return values.all(axis)
+
 def _nansum(values, axis=None, skipna=True):
     mask = isnull(values)
 

pandas/tests/test_frame.py

@@ -1,4 +1,3 @@
-
 # pylint: disable-msg=W0612,E1101
 from copy import deepcopy
 from datetime import datetime, timedelta
@@ -2769,7 +2768,6 @@ def _test_seq(df, idx_ser, col_ser):
         rs = df.le(df)
         self.assert_(not rs.ix[0, 0])
 
-
         # scalar
         assert_frame_equal(df.eq(0), df == 0)
         assert_frame_equal(df.ne(0), df != 0)
@@ -6246,25 +6244,73 @@ def test_bool_raises_value_error_1069(self):
         self.failUnlessRaises(ValueError, lambda: bool(df))
 
     def test_any_all(self):
-        df = DataFrame([[True, True, False]])
-        self.assert_(df.any())
-        self.assert_(not df.all())
+        self._check_bool_op('any', np.any, has_skipna=True, has_bool_only=True)
+        self._check_bool_op('all', np.all, has_skipna=True, has_bool_only=True)
 
-        df = DataFrame([[True, True, True]])
-        self.assert_(df.any())
-        self.assert_(df.all())
+    def _check_bool_op(self, name, alternative, frame=None, has_skipna=True,
+                       has_bool_only=False):
+        if frame is None:
+            frame = self.frame > 0
+            # set some NAs
+            frame = DataFrame(frame.values.astype(object), frame.index,
+                              frame.columns)
+            frame.ix[5:10] = np.nan
+            frame.ix[15:20, -2:] = np.nan
 
-        df = -df
-        self.assert_(not df.any())
-        self.assert_(not df.all())
+        f = getattr(frame, name)
+
+        if has_skipna:
+            def skipna_wrapper(x):
+                nona = x.dropna().values
+                return alternative(nona)
 
-        df = DataFrame([[1, 2, 3]])
-        self.assertRaises(ValueError, df.any)
-        self.assertRaises(ValueError, df.all)
+            def wrapper(x):
+                return alternative(x.values)
 
-        df = DataFrame([[1, 2, 3], [True, True, False]])
-        self.assertRaises(ValueError, df.any)
-        self.assertRaises(ValueError, df.all)
+            result0 = f(axis=0, skipna=False)
+            result1 = f(axis=1, skipna=False)
+            assert_series_equal(result0, frame.apply(wrapper))
+            assert_series_equal(result1, frame.apply(wrapper, axis=1),
+                                check_dtype=False) # HACK: win32
+        else:
+            skipna_wrapper = alternative
+            wrapper = alternative
+
+        result0 = f(axis=0)
+        result1 = f(axis=1)
+        assert_series_equal(result0, frame.apply(skipna_wrapper))
+        assert_series_equal(result1, frame.apply(skipna_wrapper, axis=1),
+                            check_dtype=False)
+
+        # result = f(axis=1)
+        # comp = frame.apply(alternative, axis=1).reindex(result.index)
+        # assert_series_equal(result, comp)
+
+        self.assertRaises(Exception, f, axis=2)
+
+        # make sure works on mixed-type frame
+        mixed = self.mixed_frame
+        mixed['_bool_'] = np.random.randn(len(mixed)) > 0
+        getattr(mixed, name)(axis=0)
+        getattr(mixed, name)(axis=1)
+
+        if has_bool_only:
+            getattr(mixed, name)(axis=0, bool_only=True)
+            getattr(mixed, name)(axis=1, bool_only=True)
+            getattr(frame, name)(axis=0, bool_only=False)
+            getattr(frame, name)(axis=1, bool_only=False)
+
+        # all NA case
+        if has_skipna:
+            all_na = frame * np.NaN
+            r0 = getattr(all_na, name)(axis=0)
+            r1 = getattr(all_na, name)(axis=1)
+            if name == 'any':
+                self.assert_(not r0.any())
+                self.assert_(not r1.any())
+            else:
+                self.assert_(r0.all())
+                self.assert_(r1.all())
 
 if __name__ == '__main__':
     # unittest.main()

pandas/tests/test_internals.py

@@ -4,9 +4,10 @@
 
 import numpy as np
 
-from pandas import Index, MultiIndex, DataFrame
+from pandas import Index, MultiIndex, DataFrame, Series
 from pandas.core.internals import *
 import pandas.core.internals as internals
+import pandas.util.testing as tm
 
 from pandas.util.testing import (assert_almost_equal, assert_frame_equal, randn)
 
@@ -385,6 +386,38 @@ def test_xs(self):
 
         assert_frame_equal(DataFrame(result), DataFrame(expected))
 
+    def test_get_numeric_data(self):
+        int_ser = Series(np.array([0, 1, 2]))
+        float_ser = Series(np.array([0., 1., 2.]))
+        complex_ser = Series(np.array([0j, 1j, 2j]))
+        str_ser = Series(np.array(['a', 'b', 'c']))
+        bool_ser = Series(np.array([True, False, True]))
+        obj_ser = Series(np.array([1, 'a', 5]))
+        dt_ser = Series(tm.makeDateIndex(3))
+        #check types
+        df = DataFrame({'int' : int_ser, 'float' : float_ser,
+                        'complex' : complex_ser, 'str' : str_ser,
+                        'bool' : bool_ser, 'obj' : obj_ser,
+                        'dt' : dt_ser})
+        xp = DataFrame({'int' : int_ser, 'float' : float_ser,
+                        'complex' : complex_ser})
+        rs = DataFrame(df._data.get_numeric_data())
+        assert_frame_equal(xp, rs)
+
+        xp = DataFrame({'bool' : bool_ser})
+        rs = DataFrame(df._data.get_numeric_data(type_list=bool))
+        assert_frame_equal(xp, rs)
+
+        rs = DataFrame(df._data.get_numeric_data(type_list=bool))
+        df.ix[0, 'bool'] = not df.ix[0, 'bool']
+
+        self.assertEqual(rs.ix[0, 'bool'], df.ix[0, 'bool'])
+
+        rs = DataFrame(df._data.get_numeric_data(type_list=bool, copy=True))
+        df.ix[0, 'bool'] = not df.ix[0, 'bool']
+
+        self.assertEqual(rs.ix[0, 'bool'], not df.ix[0, 'bool'])
+
 if __name__ == '__main__':
     # unittest.main()
     import nose