ENH: add np.nan funcs to _cython_table

doc/source/whatsnew/v0.23.1.txt

@@ -94,3 +94,9 @@ Categorical
 ^^^^^^^^^^^
 
 -
+
+Numeric
+^^^^^^^
+
+- :meth:`~DataFrame.agg` now correctly handles numpy NaN-aware methods like :meth:`numpy.nansum` (:issue:`19629`)
+- :meth:`~DataFrame.agg` now correctly handles built-in methods like ``sum`` when axis=1 (:issue:`21134`)

pandas/conftest.py

@@ -149,3 +149,20 @@ def tz_aware_fixture(request):
     Fixture for trying explicit timezones: {0}
     """
     return request.param
+
+
+@pytest.fixture(
+    # params: Python 3.5 randomizes dict access and xdist doesn't like that
+    # in fixtures. In order to get predetermined values we need to sort
+    # the list deterministically
+    # GH 21123
+    params=list(sorted(pd.core.base.SelectionMixin._cython_table.items(),
+                       key=lambda x: x[0].__name__)),
+    ids=lambda x: "({}-{!r})".format(x[0].__name__, x[1]),
+)
+def cython_table_items(request):
+    """
+    Fixture for returning the items in
+    pandas.core.base.SelectionMixin._cython_table
+    """
+    return request.param

pandas/core/base.py

@@ -22,7 +22,8 @@
 from pandas.core import common as com, algorithms
 import pandas.core.nanops as nanops
 import pandas._libs.lib as lib
-from pandas.compat.numpy import function as nv
+from pandas.compat.numpy import (function as nv, _np_version_under1p10,
+                                 _np_version_under1p12)
 from pandas.compat import PYPY
 from pandas.util._decorators import (Appender, cache_readonly,
                                      deprecate_kwarg, Substitution)
@@ -191,17 +192,31 @@ class SelectionMixin(object):
         np.all: 'all',
         np.any: 'any',
         np.sum: 'sum',
+        np.nansum: 'sum',
         np.mean: 'mean',
+        np.nanmean: 'mean',
         np.prod: 'prod',
         np.std: 'std',
+        np.nanstd: 'std',
         np.var: 'var',
+        np.nanvar: 'var',
         np.median: 'median',
+        np.nanmedian: 'median',
         np.max: 'max',
+        np.nanmax: 'max',
         np.min: 'min',
+        np.nanmin: 'min',
         np.cumprod: 'cumprod',
-        np.cumsum: 'cumsum'
+        np.cumsum: 'cumsum',
     }
 
+    if not _np_version_under1p10:
+        _cython_table[np.nanprod] = 'prod'
+
+    if not _np_version_under1p12:
+        _cython_table[np.nancumprod] = 'cumprod'
+        _cython_table[np.nancumsum] = 'cumsum'
+
     @property
     def _selection_name(self):
         """
@@ -316,13 +331,14 @@ def _try_aggregate_string_function(self, arg, *args, **kwargs):
 
         raise ValueError("{arg} is an unknown string function".format(arg=arg))
 
-    def _aggregate(self, arg, *args, **kwargs):
+    def _aggregate(self, arg, axis=0, *args, **kwargs):
         """
         provide an implementation for the aggregators
 
         Parameters
         ----------
         arg : string, dict, function
+        axis : int
         *args : args to pass on to the function
         **kwargs : kwargs to pass on to the function
 
@@ -335,25 +351,26 @@ def _aggregate(self, arg, *args, **kwargs):
         how can be a string describe the required post-processing, or
         None if not required
         """
+        obj = self if axis == 0 else self.T
         is_aggregator = lambda x: isinstance(x, (list, tuple, dict))
         is_nested_renamer = False
 
         _axis = kwargs.pop('_axis', None)
         if _axis is None:
-            _axis = getattr(self, 'axis', 0)
+            _axis = getattr(obj, 'axis', 0)
         _level = kwargs.pop('_level', None)
 
         if isinstance(arg, compat.string_types):
-            return self._try_aggregate_string_function(arg, *args,
-                                                       **kwargs), None
+            return obj._try_aggregate_string_function(arg, *args,
+                                                      **kwargs), None
 
         if isinstance(arg, dict):
 
             # aggregate based on the passed dict
             if _axis != 0:  # pragma: no cover
                 raise ValueError('Can only pass dict with axis=0')
 
-            obj = self._selected_obj
+            selected_obj = obj._selected_obj
 
             def nested_renaming_depr(level=4):
                 # deprecation of nested renaming
@@ -388,16 +405,16 @@ def nested_renaming_depr(level=4):
                     if isinstance(v, dict):
                         is_nested_renamer = True
 
-                        if k not in obj.columns:
+                        if k not in selected_obj.columns:
                             msg = ('cannot perform renaming for {key} with a '
                                    'nested dictionary').format(key=k)
                             raise SpecificationError(msg)
                         nested_renaming_depr(4 + (_level or 0))
 
-                    elif isinstance(obj, ABCSeries):
+                    elif isinstance(selected_obj, ABCSeries):
                         nested_renaming_depr()
-                    elif isinstance(obj, ABCDataFrame) and \
-                            k not in obj.columns:
+                    elif isinstance(selected_obj, ABCDataFrame) and \
+                            k not in selected_obj.columns:
                         raise KeyError(
                             "Column '{col}' does not exist!".format(col=k))
 
@@ -407,8 +424,8 @@ def nested_renaming_depr(level=4):
                 # deprecation of renaming keys
                 # GH 15931
                 keys = list(compat.iterkeys(arg))
-                if (isinstance(obj, ABCDataFrame) and
-                        len(obj.columns.intersection(keys)) != len(keys)):
+                if (isinstance(selected_obj, ABCDataFrame) and len(
+                        selected_obj.columns.intersection(keys)) != len(keys)):
                     nested_renaming_depr()
 
             from pandas.core.reshape.concat import concat
@@ -417,7 +434,7 @@ def _agg_1dim(name, how, subset=None):
                 """
                 aggregate a 1-dim with how
                 """
-                colg = self._gotitem(name, ndim=1, subset=subset)
+                colg = obj._gotitem(name, ndim=1, subset=subset)
                 if colg.ndim != 1:
                     raise SpecificationError("nested dictionary is ambiguous "
                                              "in aggregation")
@@ -427,8 +444,8 @@ def _agg_2dim(name, how):
                 """
                 aggregate a 2-dim with how
                 """
-                colg = self._gotitem(self._selection, ndim=2,
-                                     subset=obj)
+                colg = obj._gotitem(obj._selection, ndim=2,
+                                    subset=selected_obj)
                 return colg.aggregate(how, _level=None)
 
             def _agg(arg, func):
@@ -458,20 +475,22 @@ def _agg(arg, func):
 
                 else:
 
-                    if self._selection is not None:
+                    if obj._selection is not None:
                         keys = None
 
             # some selection on the object
-            elif self._selection is not None:
+            elif obj._selection is not None:
 
-                sl = set(self._selection_list)
+                sl = set(obj._selection_list)
 
                 # we are a Series like object,
                 # but may have multiple aggregations
                 if len(sl) == 1:
 
-                    result = _agg(arg, lambda fname,
-                                  agg_how: _agg_1dim(self._selection, agg_how))
+                    result = _agg(
+                        arg,
+                        lambda fname, agg_how: _agg_1dim(
+                            obj._selection, agg_how))
 
                 # we are selecting the same set as we are aggregating
                 elif not len(sl - set(keys)):
@@ -516,7 +535,7 @@ def is_any_frame():
                 return concat([result[k] for k in keys],
                               keys=keys, axis=1), True
 
-            elif isinstance(self, ABCSeries) and is_any_series():
+            elif isinstance(obj, ABCSeries) and is_any_series():
 
                 # we have a dict of Series
                 # return a MI Series
@@ -541,20 +560,20 @@ def is_any_frame():
 
                 # we have a dict of scalars
                 result = Series(result,
-                                name=getattr(self, 'name', None))
+                                name=getattr(obj, 'name', None))
 
             return result, True
         elif is_list_like(arg) and arg not in compat.string_types:
             # we require a list, but not an 'str'
-            return self._aggregate_multiple_funcs(arg,
-                                                  _level=_level,
-                                                  _axis=_axis), None
+            return obj._aggregate_multiple_funcs(arg,
+                                                 _level=_level,
+                                                 _axis=_axis), None
         else:
             result = None
 
-        f = self._is_cython_func(arg)
-        if f and not args and not kwargs:
-            return getattr(self, f)(), None
+        f = obj._is_cython_func(arg)
+        if f is not None:
+            return getattr(obj, f)(*args, **kwargs), None
 
         # caller can react
         return result, True

pandas/core/frame.py

@@ -5818,13 +5818,11 @@ def _gotitem(self,
     def aggregate(self, func, axis=0, *args, **kwargs):
         axis = self._get_axis_number(axis)
 
-        # TODO: flipped axis
         result = None
-        if axis == 0:
-            try:
-                result, how = self._aggregate(func, axis=0, *args, **kwargs)
-            except TypeError:
-                pass
+        try:
+            result, how = self._aggregate(func, axis=axis, *args, **kwargs)
+        except TypeError:
+            pass
         if result is None:
             return self.apply(func, axis=axis, args=args, **kwargs)
         return result

pandas/core/groupby/groupby.py

@@ -4086,7 +4086,10 @@ def _post_process_cython_aggregate(self, obj):
     def aggregate(self, arg, *args, **kwargs):
 
         _level = kwargs.pop('_level', None)
-        result, how = self._aggregate(arg, _level=_level, *args, **kwargs)
+        _agg_kwargs = kwargs.copy()
+        axis = _agg_kwargs.pop('axis', 0)
+        result, how = self._aggregate(arg, axis, _level=_level,
+                                      *args, **_agg_kwargs)
         if how is None:
             return result
 

pandas/tests/frame/test_apply.py

@@ -1056,3 +1056,26 @@ def test_non_callable_aggregates(self):
         expected = df.size
 
         assert result == expected
+
+    @pytest.mark.parametrize("df", [
+        pd.DataFrame([[1, 2], [3, 4]]),
+        pd.DataFrame([[np.nan, 2], [3, 4]]),
+        pd.DataFrame(),
+    ])
+    def test_agg_function_input(self, df, cython_table_items):
+        # test whether the functions (keys) in
+        # pd.core.base.SelectionMixin._cython_table give the same result
+        # as the related strings (values) when used in df.agg. Examples:
+        # - ``df.agg(np.nansum)`` should give the same result as
+        #   ``df.agg('sum')``
+        # - ``df.agg(sum)`` should give the same result as ``df.agg('sum')``
+        # etc.
+        # GH21123
+        np_func, str_func = cython_table_items
+
+        tm.assert_almost_equal(df.agg(np_func),
+                               df.agg(str_func),
+                               )
+        tm.assert_almost_equal(df.agg(np_func, axis=1),
+                               df.agg(str_func, axis=1),
+                               )

pandas/tests/series/test_apply.py

@@ -587,3 +587,23 @@ def test_map_missing_mixed(self, vals, mapping, exp):
         result = s.map(mapping)
 
         tm.assert_series_equal(result, pd.Series(exp))
+
+    @pytest.mark.parametrize("series", [
+        pd.Series([1, 2, 3, 4]),
+        pd.Series([np.nan, 2, 3, 4]),
+        pd.Series(),
+    ])
+    def test_agg_function_input(self, series, cython_table_items):
+        # test whether the functions (keys) in
+        # pd.core.base.SelectionMixin._cython_table give the same result
+        # as the related strings (values), when used in ser.agg. Examples:
+        # - ``ser.agg(np.nansum)`` should give the same result as
+        #   ``ser.agg('sum')``
+        # - ``ser.agg(sum)`` should give the same result as ``ser.agg('sum')``
+        # etc.
+        # GH21123
+        np_func, str_func = cython_table_items
+
+        tm.assert_almost_equal(series.agg(np_func),
+                               series.agg(str_func),
+                               )