Merge remote-tracking branch 'upstream/master' into typing

Author: simonjayhawkins

ci/code_checks.sh

@@ -122,7 +122,7 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
     MSG='Check for non-standard imports' ; echo $MSG
     invgrep -R --include="*.py*" -E "from pandas.core.common import " pandas
     invgrep -R --include="*.py*" -E "from collections.abc import " pandas
-    # invgrep -R --include="*.py*" -E "from numpy import nan " pandas  # GH#24822 not yet implemented since the offending imports have not all been removed
+    invgrep -R --include="*.py*" -E "from numpy import nan " pandas
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     MSG='Check for use of exec' ; echo $MSG

pandas/core/dtypes/cast.py

@@ -360,22 +360,32 @@ def maybe_promote(dtype, fill_value=np.nan):
         if isinstance(fill_value, datetime) and fill_value.tzinfo is not None:
             # Trying to insert tzaware into tznaive, have to cast to object
             dtype = np.dtype(np.object_)
+        elif is_integer(fill_value) or (is_float(fill_value) and not isna(fill_value)):
+            dtype = np.dtype(np.object_)
         else:
             try:
                 fill_value = tslibs.Timestamp(fill_value).to_datetime64()
             except (TypeError, ValueError):
                 dtype = np.dtype(np.object_)
     elif issubclass(dtype.type, np.timedelta64):
-        try:
-            fv = tslibs.Timedelta(fill_value)
-        except ValueError:
+        if (
+            is_integer(fill_value)
+            or (is_float(fill_value) and not np.isnan(fill_value))
+            or isinstance(fill_value, str)
+        ):
+            # TODO: What about str that can be a timedelta?
             dtype = np.dtype(np.object_)
         else:
-            if fv is NaT:
-                # NaT has no `to_timedelta64` method
-                fill_value = np.timedelta64("NaT", "ns")
+            try:
+                fv = tslibs.Timedelta(fill_value)
+            except ValueError:
+                dtype = np.dtype(np.object_)
             else:
-                fill_value = fv.to_timedelta64()
+                if fv is NaT:
+                    # NaT has no `to_timedelta64` method
+                    fill_value = np.timedelta64("NaT", "ns")
+                else:
+                    fill_value = fv.to_timedelta64()
     elif is_datetime64tz_dtype(dtype):
         if isna(fill_value):
             fill_value = NaT

pandas/core/frame.py

@@ -6737,14 +6737,6 @@ def apply(
         DataFrame.aggregate: Only perform aggregating type operations.
         DataFrame.transform: Only perform transforming type operations.
 
-        Notes
-        -----
-        In the current implementation apply calls `func` twice on the
-        first column/row to decide whether it can take a fast or slow
-        code path. This can lead to unexpected behavior if `func` has
-        side-effects, as they will take effect twice for the first
-        column/row.
-
         Examples
         --------
 

pandas/core/groupby/generic.py

@@ -142,6 +142,9 @@ def pinner(cls):
 class SeriesGroupBy(GroupBy):
     _apply_whitelist = base.series_apply_whitelist
 
+    def _iterate_slices(self):
+        yield self._selection_name, self._selected_obj
+
     @property
     def _selection_name(self):
         """
@@ -333,7 +336,7 @@ def _aggregate_multiple_funcs(self, arg, _level):
 
         return DataFrame(results, columns=columns)
 
-    def _wrap_output(self, output, index, names=None):
+    def _wrap_series_output(self, output, index, names=None):
         """ common agg/transform wrapping logic """
         output = output[self._selection_name]
 
@@ -346,13 +349,15 @@ def _wrap_output(self, output, index, names=None):
             return Series(output, index=index, name=name)
 
     def _wrap_aggregated_output(self, output, names=None):
-        result = self._wrap_output(
+        result = self._wrap_series_output(
             output=output, index=self.grouper.result_index, names=names
         )
         return self._reindex_output(result)._convert(datetime=True)
 
     def _wrap_transformed_output(self, output, names=None):
-        return self._wrap_output(output=output, index=self.obj.index, names=names)
+        return self._wrap_series_output(
+            output=output, index=self.obj.index, names=names
+        )
 
     def _wrap_applied_output(self, keys, values, not_indexed_same=False):
         if len(keys) == 0:
@@ -876,7 +881,7 @@ def aggregate(self, func=None, *args, **kwargs):
             if self.grouper.nkeys > 1:
                 return self._python_agg_general(func, *args, **kwargs)
             elif args or kwargs:
-                result = self._aggregate_generic(func, *args, **kwargs)
+                result = self._aggregate_frame(func, *args, **kwargs)
             else:
 
                 # try to treat as if we are passing a list
@@ -885,7 +890,7 @@ def aggregate(self, func=None, *args, **kwargs):
                         [func], _level=_level, _axis=self.axis
                     )
                 except Exception:
-                    result = self._aggregate_generic(func)
+                    result = self._aggregate_frame(func)
                 else:
                     result.columns = Index(
                         result.columns.levels[0], name=self._selected_obj.columns.name
@@ -1009,7 +1014,7 @@ def _cython_agg_blocks(self, how, alt=None, numeric_only=True, min_count=-1):
 
         return new_items, new_blocks
 
-    def _aggregate_generic(self, func, *args, **kwargs):
+    def _aggregate_frame(self, func, *args, **kwargs):
         if self.grouper.nkeys != 1:
             raise AssertionError("Number of keys must be 1")
 
@@ -1032,7 +1037,7 @@ def _aggregate_generic(self, func, *args, **kwargs):
                     wrapper = lambda x: func(x, *args, **kwargs)
                     result[name] = data.apply(wrapper, axis=axis)
 
-        return self._wrap_generic_output(result, obj)
+        return self._wrap_frame_output(result, obj)
 
     def _aggregate_item_by_item(self, func, *args, **kwargs):
         # only for axis==0
@@ -1516,7 +1521,7 @@ def _gotitem(self, key, ndim, subset=None):
 
         raise AssertionError("invalid ndim for _gotitem")
 
-    def _wrap_generic_output(self, result, obj):
+    def _wrap_frame_output(self, result, obj):
         result_index = self.grouper.levels[0]
 
         if self.axis == 0:

pandas/core/groupby/groupby.py

@@ -11,6 +11,8 @@ class providing the base-class of operations.
 from contextlib import contextmanager
 import datetime
 from functools import partial, wraps
+import inspect
+import re
 import types
 from typing import Dict, FrozenSet, List, Optional, Tuple, Type, Union
 
@@ -613,48 +615,53 @@ def _make_wrapper(self, name):
             return self.apply(lambda self: getattr(self, name))
 
         f = getattr(type(self._selected_obj), name)
+        sig = inspect.signature(f)
 
         def wrapper(*args, **kwargs):
             # a little trickery for aggregation functions that need an axis
             # argument
-            kwargs_with_axis = kwargs.copy()
-            if "axis" not in kwargs_with_axis or kwargs_with_axis["axis"] is None:
-                kwargs_with_axis["axis"] = self.axis
-
-            def curried_with_axis(x):
-                return f(x, *args, **kwargs_with_axis)
+            if "axis" in sig.parameters:
+                if kwargs.get("axis", None) is None:
+                    kwargs["axis"] = self.axis
 
             def curried(x):
                 return f(x, *args, **kwargs)
 
             # preserve the name so we can detect it when calling plot methods,
             # to avoid duplicates
-            curried.__name__ = curried_with_axis.__name__ = name
+            curried.__name__ = name
 
             # special case otherwise extra plots are created when catching the
             # exception below
             if name in base.plotting_methods:
                 return self.apply(curried)
 
             try:
-                return self.apply(curried_with_axis)
-            except Exception:
-                try:
-                    return self.apply(curried)
-                except Exception:
-
-                    # related to : GH3688
-                    # try item-by-item
-                    # this can be called recursively, so need to raise
-                    # ValueError
-                    # if we don't have this method to indicated to aggregate to
-                    # mark this column as an error
-                    try:
-                        return self._aggregate_item_by_item(name, *args, **kwargs)
-                    except AttributeError:
-                        # e.g. SparseArray has no flags attr
-                        raise ValueError
-
+                return self.apply(curried)
+            except TypeError as err:
+                if not re.search(
+                    "reduction operation '.*' not allowed for this dtype", str(err)
+                ):
+                    # We don't have a cython implementation
+                    # TODO: is the above comment accurate?
+                    raise
+
+            # related to : GH3688
+            # try item-by-item
+            # this can be called recursively, so need to raise
+            # ValueError
+            # if we don't have this method to indicated to aggregate to
+            # mark this column as an error
+            try:
+                return self._aggregate_item_by_item(name, *args, **kwargs)
+            except AttributeError:
+                # e.g. SparseArray has no flags attr
+                # FIXME: 'SeriesGroupBy' has no attribute '_aggregate_item_by_item'
+                #  occurs in idxmax() case
+                #  in tests.groupby.test_function.test_non_cython_api
+                raise ValueError
+
+        wrapper.__name__ = name
         return wrapper
 
     def get_group(self, name, obj=None):
@@ -747,7 +754,7 @@ def _python_apply_general(self, f):
         )
 
     def _iterate_slices(self):
-        yield self._selection_name, self._selected_obj
+        raise AbstractMethodError(self)
 
     def transform(self, func, *args, **kwargs):
         raise AbstractMethodError(self)
@@ -872,6 +879,12 @@ def _cython_transform(self, how, numeric_only=True, **kwargs):
     def _wrap_aggregated_output(self, output, names=None):
         raise AbstractMethodError(self)
 
+    def _wrap_transformed_output(self, output, names=None):
+        raise AbstractMethodError(self)
+
+    def _wrap_applied_output(self, keys, values, not_indexed_same=False):
+        raise AbstractMethodError(self)
+
     def _cython_agg_general(self, how, alt=None, numeric_only=True, min_count=-1):
         output = {}
         for name, obj in self._iterate_slices():
@@ -922,9 +935,6 @@ def _python_agg_general(self, func, *args, **kwargs):
 
         return self._wrap_aggregated_output(output)
 
-    def _wrap_applied_output(self, *args, **kwargs):
-        raise AbstractMethodError(self)
-
     def _concat_objects(self, keys, values, not_indexed_same=False):
         from pandas.core.reshape.concat import concat
 

pandas/core/internals/concat.py

@@ -288,7 +288,7 @@ def get_empty_dtype_and_na(join_units):
             return np.float64, np.nan
 
     if is_uniform_reindex(join_units):
-        # XXX: integrate property
+        # FIXME: integrate property
         empty_dtype = join_units[0].block.dtype
         upcasted_na = join_units[0].block.fill_value
         return empty_dtype, upcasted_na
@@ -339,6 +339,7 @@ def get_empty_dtype_and_na(join_units):
     if not upcast_classes:
         upcast_classes = null_upcast_classes
 
+    # TODO: de-duplicate with maybe_promote?
     # create the result
     if "object" in upcast_classes:
         return np.dtype(np.object_), np.nan
@@ -357,7 +358,7 @@ def get_empty_dtype_and_na(join_units):
     elif "datetime" in upcast_classes:
         return np.dtype("M8[ns]"), tslibs.iNaT
     elif "timedelta" in upcast_classes:
-        return np.dtype("m8[ns]"), tslibs.iNaT
+        return np.dtype("m8[ns]"), np.timedelta64("NaT", "ns")
     else:  # pragma
         try:
             g = np.find_common_type(upcast_classes, [])

pandas/core/internals/managers.py

@@ -2035,7 +2035,7 @@ def concatenate_block_managers(mgrs_indexers, axes, concat_axis, copy):
             values = b.values
             if copy:
                 values = values.copy()
-            elif not copy:
+            else:
                 values = values.view()
             b = b.make_block_same_class(values, placement=placement)
         elif is_uniform_join_units(join_units):

pandas/core/ops/__init__.py

@@ -766,9 +766,9 @@ def f(self, other, axis=default_axis, level=None):
     return f
 
 
-def _comp_method_FRAME(cls, func, special):
-    str_rep = _get_opstr(func)
-    op_name = _get_op_name(func, special)
+def _comp_method_FRAME(cls, op, special):
+    str_rep = _get_opstr(op)
+    op_name = _get_op_name(op, special)
 
     @Appender("Wrapper for comparison method {name}".format(name=op_name))
     def f(self, other):
@@ -781,18 +781,18 @@ def f(self, other):
                 raise ValueError(
                     "Can only compare identically-labeled DataFrame objects"
                 )
-            new_data = dispatch_to_series(self, other, func, str_rep)
+            new_data = dispatch_to_series(self, other, op, str_rep)
             return self._construct_result(new_data)
 
         elif isinstance(other, ABCSeries):
             return _combine_series_frame(
-                self, other, func, fill_value=None, axis=None, level=None
+                self, other, op, fill_value=None, axis=None, level=None
             )
         else:
 
             # straight boolean comparisons we want to allow all columns
             # (regardless of dtype to pass thru) See #4537 for discussion.
-            new_data = dispatch_to_series(self, other, func)
+            new_data = dispatch_to_series(self, other, op)
             return self._construct_result(new_data)
 
     f.__name__ = op_name

pandas/core/ops/array_ops.py

@@ -118,14 +118,14 @@ def masked_arith_op(x, y, op):
     return result
 
 
-def define_na_arithmetic_op(op, str_rep, eval_kwargs):
+def define_na_arithmetic_op(op, str_rep: str, eval_kwargs):
     def na_op(x, y):
         return na_arithmetic_op(x, y, op, str_rep, eval_kwargs)
 
     return na_op
 
 
-def na_arithmetic_op(left, right, op, str_rep, eval_kwargs):
+def na_arithmetic_op(left, right, op, str_rep: str, eval_kwargs):
     """
     Return the result of evaluating op on the passed in values.
 
@@ -173,6 +173,7 @@ def arithmetic_op(
         Cannot be a DataFrame or Index.  Series is *not* excluded.
     op : {operator.add, operator.sub, ...}
         Or one of the reversed variants from roperator.
+    str_rep : str
 
     Returns
     -------
@@ -279,8 +280,16 @@ def comparison_op(
     return res_values
 
 
-def na_logical_op(x, y, op):
+def na_logical_op(x: np.ndarray, y, op):
     try:
+        # For exposition, write:
+        #  yarr = isinstance(y, np.ndarray)
+        #  yint = is_integer(y) or (yarr and y.dtype.kind == "i")
+        #  ybool = is_bool(y) or (yarr and y.dtype.kind == "b")
+        #  xint = x.dtype.kind == "i"
+        #  xbool = x.dtype.kind == "b"
+        # Then Cases where this goes through without raising include:
+        #  (xint or xbool) and (yint or bool)
         result = op(x, y)
     except TypeError:
         if isinstance(y, np.ndarray):
@@ -304,9 +313,9 @@ def na_logical_op(x, y, op):
                 NotImplementedError,
             ):
                 raise TypeError(
-                    "cannot compare a dtyped [{dtype}] array "
-                    "with a scalar of type [{typ}]".format(
-                        dtype=x.dtype, typ=type(y).__name__
+                    "Cannot perform '{op}' with a dtyped [{dtype}] array "
+                    "and scalar of type [{typ}]".format(
+                        op=op.__name__, dtype=x.dtype, typ=type(y).__name__
                     )
                 )