Merge branch 'master' of https://github.com/pandas-dev/pandas into ref-numeric-validate_fill_value

Author: jbrockmendel

ci/deps/azure-39.yaml

@@ -15,3 +15,8 @@ dependencies:
   - numpy
   - python-dateutil
   - pytz
+
+  # optional dependencies
+  - pytables
+  - scipy
+  - pyarrow=1.0

doc/source/user_guide/groupby.rst

@@ -524,6 +524,15 @@ index are the group names and whose values are the sizes of each group.
 
    grouped.describe()
 
+Another aggregation example is to compute the number of unique values of each group. This is similar to the ``value_counts`` function, except that it only counts unique values.
+
+.. ipython:: python
+
+   ll = [['foo', 1], ['foo', 2], ['foo', 2], ['bar', 1], ['bar', 1]]
+   df4 = pd.DataFrame(ll, columns=["A", "B"])
+   df4
+   df4.groupby("A")["B"].nunique()
+
 .. note::
 
    Aggregation functions **will not** return the groups that you are aggregating over

doc/source/whatsnew/v1.1.5.rst

@@ -28,6 +28,7 @@ Bug fixes
 - Bug in metadata propagation for ``groupby`` iterator (:issue:`37343`)
 - Bug in indexing on a :class:`Series` with ``CategoricalDtype`` after unpickling (:issue:`37631`)
 - Bug in :class:`RollingGroupby` with the resulting :class:`MultiIndex` when grouping by a label that is in the index (:issue:`37641`)
+- Bug in pytables methods in python 3.9 (:issue:`38041`)
 -
 
 .. ---------------------------------------------------------------------------

doc/source/whatsnew/v1.2.0.rst

@@ -49,7 +49,6 @@
 )
 from pandas.core.dtypes.generic import (
     ABCExtensionArray,
-    ABCIndex,
     ABCIndexClass,
     ABCMultiIndex,
     ABCSeries,
@@ -69,7 +68,7 @@
 # dtype access    #
 # --------------- #
 def _ensure_data(
-    values, dtype: Optional[DtypeObj] = None
+    values: ArrayLike, dtype: Optional[DtypeObj] = None
 ) -> Tuple[np.ndarray, DtypeObj]:
     """
     routine to ensure that our data is of the correct
@@ -95,6 +94,12 @@ def _ensure_data(
     pandas_dtype : np.dtype or ExtensionDtype
     """
 
+    if dtype is not None:
+        # We only have non-None dtype when called from `isin`, and
+        #  both Datetimelike and Categorical dispatch before getting here.
+        assert not needs_i8_conversion(dtype)
+        assert not is_categorical_dtype(dtype)
+
     if not isinstance(values, ABCMultiIndex):
         # extract_array would raise
         values = extract_array(values, extract_numpy=True)
@@ -131,21 +136,20 @@ def _ensure_data(
         return ensure_object(values), np.dtype("object")
 
     # datetimelike
-    vals_dtype = getattr(values, "dtype", None)
-    if needs_i8_conversion(vals_dtype) or needs_i8_conversion(dtype):
-        if is_period_dtype(vals_dtype) or is_period_dtype(dtype):
+    if needs_i8_conversion(values.dtype) or needs_i8_conversion(dtype):
+        if is_period_dtype(values.dtype) or is_period_dtype(dtype):
             from pandas import PeriodIndex
 
-            values = PeriodIndex(values)
+            values = PeriodIndex(values)._data
             dtype = values.dtype
-        elif is_timedelta64_dtype(vals_dtype) or is_timedelta64_dtype(dtype):
+        elif is_timedelta64_dtype(values.dtype) or is_timedelta64_dtype(dtype):
             from pandas import TimedeltaIndex
 
-            values = TimedeltaIndex(values)
+            values = TimedeltaIndex(values)._data
             dtype = values.dtype
         else:
             # Datetime
-            if values.ndim > 1 and is_datetime64_ns_dtype(vals_dtype):
+            if values.ndim > 1 and is_datetime64_ns_dtype(values.dtype):
                 # Avoid calling the DatetimeIndex constructor as it is 1D only
                 # Note: this is reached by DataFrame.rank calls GH#27027
                 # TODO(EA2D): special case not needed with 2D EAs
@@ -155,12 +159,12 @@ def _ensure_data(
 
             from pandas import DatetimeIndex
 
-            values = DatetimeIndex(values)
+            values = DatetimeIndex(values)._data
             dtype = values.dtype
 
         return values.asi8, dtype
 
-    elif is_categorical_dtype(vals_dtype) and (
+    elif is_categorical_dtype(values.dtype) and (
         is_categorical_dtype(dtype) or dtype is None
     ):
         values = values.codes
@@ -237,11 +241,11 @@ def _ensure_arraylike(values):
 }
 
 
-def _get_hashtable_algo(values):
+def _get_hashtable_algo(values: np.ndarray):
     """
     Parameters
     ----------
-    values : arraylike
+    values : np.ndarray
 
     Returns
     -------
@@ -255,15 +259,15 @@ def _get_hashtable_algo(values):
     return htable, values
 
 
-def _get_values_for_rank(values):
+def _get_values_for_rank(values: ArrayLike):
     if is_categorical_dtype(values):
-        values = values._values_for_rank()
+        values = cast("Categorical", values)._values_for_rank()
 
     values, _ = _ensure_data(values)
     return values
 
 
-def get_data_algo(values):
+def get_data_algo(values: ArrayLike):
     values = _get_values_for_rank(values)
 
     ndtype = _check_object_for_strings(values)
@@ -421,20 +425,28 @@ def isin(comps: AnyArrayLike, values: AnyArrayLike) -> np.ndarray:
             f"to isin(), you passed a [{type(values).__name__}]"
         )
 
-    if not isinstance(values, (ABCIndex, ABCSeries, ABCExtensionArray, np.ndarray)):
+    if not isinstance(
+        values, (ABCIndexClass, ABCSeries, ABCExtensionArray, np.ndarray)
+    ):
         values = construct_1d_object_array_from_listlike(list(values))
         # TODO: could use ensure_arraylike here
+    elif isinstance(values, ABCMultiIndex):
+        # Avoid raising in extract_array
+        values = np.array(values)
 
     comps = _ensure_arraylike(comps)
     comps = extract_array(comps, extract_numpy=True)
-    if is_categorical_dtype(comps):
+    if is_categorical_dtype(comps.dtype):
         # TODO(extension)
         # handle categoricals
         return cast("Categorical", comps).isin(values)
 
-    if needs_i8_conversion(comps):
+    if needs_i8_conversion(comps.dtype):
         # Dispatch to DatetimeLikeArrayMixin.isin
         return array(comps).isin(values)
+    elif needs_i8_conversion(values.dtype) and not is_object_dtype(comps.dtype):
+        # e.g. comps are integers and values are datetime64s
+        return np.zeros(comps.shape, dtype=bool)
 
     comps, dtype = _ensure_data(comps)
     values, _ = _ensure_data(values, dtype=dtype)
@@ -474,7 +486,7 @@ def isin(comps: AnyArrayLike, values: AnyArrayLike) -> np.ndarray:
 
 
 def factorize_array(
-    values, na_sentinel: int = -1, size_hint=None, na_value=None, mask=None
+    values: np.ndarray, na_sentinel: int = -1, size_hint=None, na_value=None, mask=None
 ) -> Tuple[np.ndarray, np.ndarray]:
     """
     Factorize an array-like to codes and uniques.
@@ -838,7 +850,7 @@ def value_counts_arraylike(values, dropna: bool):
     return keys, counts
 
 
-def duplicated(values, keep="first") -> np.ndarray:
+def duplicated(values: ArrayLike, keep: str = "first") -> np.ndarray:
     """
     Return boolean ndarray denoting duplicate values.
 

pandas/core/algorithms.py

@@ -5,8 +5,15 @@
     ExtensionArray
 """
 import operator
+from typing import Any, Callable
+import warnings
 
-from pandas.core.ops import roperator
+import numpy as np
+
+from pandas._libs import lib
+
+from pandas.core.construction import extract_array
+from pandas.core.ops import maybe_dispatch_ufunc_to_dunder_op, roperator
 from pandas.core.ops.common import unpack_zerodim_and_defer
 
 
@@ -140,3 +147,138 @@ def __pow__(self, other):
     @unpack_zerodim_and_defer("__rpow__")
     def __rpow__(self, other):
         return self._arith_method(other, roperator.rpow)
+
+
+def array_ufunc(self, ufunc: Callable, method: str, *inputs: Any, **kwargs: Any):
+    """
+    Compatibility with numpy ufuncs.
+
+    See also
+    --------
+    numpy.org/doc/stable/reference/arrays.classes.html#numpy.class.__array_ufunc__
+    """
+    from pandas.core.generic import NDFrame
+    from pandas.core.internals import BlockManager
+
+    cls = type(self)
+
+    # for binary ops, use our custom dunder methods
+    result = maybe_dispatch_ufunc_to_dunder_op(self, ufunc, method, *inputs, **kwargs)
+    if result is not NotImplemented:
+        return result
+
+    # Determine if we should defer.
+    no_defer = (np.ndarray.__array_ufunc__, cls.__array_ufunc__)
+
+    for item in inputs:
+        higher_priority = (
+            hasattr(item, "__array_priority__")
+            and item.__array_priority__ > self.__array_priority__
+        )
+        has_array_ufunc = (
+            hasattr(item, "__array_ufunc__")
+            and type(item).__array_ufunc__ not in no_defer
+            and not isinstance(item, self._HANDLED_TYPES)
+        )
+        if higher_priority or has_array_ufunc:
+            return NotImplemented
+
+    # align all the inputs.
+    types = tuple(type(x) for x in inputs)
+    alignable = [x for x, t in zip(inputs, types) if issubclass(t, NDFrame)]
+
+    if len(alignable) > 1:
+        # This triggers alignment.
+        # At the moment, there aren't any ufuncs with more than two inputs
+        # so this ends up just being x1.index | x2.index, but we write
+        # it to handle *args.
+
+        if len(set(types)) > 1:
+            # We currently don't handle ufunc(DataFrame, Series)
+            # well. Previously this raised an internal ValueError. We might
+            # support it someday, so raise a NotImplementedError.
+            raise NotImplementedError(
+                "Cannot apply ufunc {} to mixed DataFrame and Series "
+                "inputs.".format(ufunc)
+            )
+        axes = self.axes
+        for obj in alignable[1:]:
+            # this relies on the fact that we aren't handling mixed
+            # series / frame ufuncs.
+            for i, (ax1, ax2) in enumerate(zip(axes, obj.axes)):
+                axes[i] = ax1.union(ax2)
+
+        reconstruct_axes = dict(zip(self._AXIS_ORDERS, axes))
+        inputs = tuple(
+            x.reindex(**reconstruct_axes) if issubclass(t, NDFrame) else x
+            for x, t in zip(inputs, types)
+        )
+    else:
+        reconstruct_axes = dict(zip(self._AXIS_ORDERS, self.axes))
+
+    if self.ndim == 1:
+        names = [getattr(x, "name") for x in inputs if hasattr(x, "name")]
+        name = names[0] if len(set(names)) == 1 else None
+        reconstruct_kwargs = {"name": name}
+    else:
+        reconstruct_kwargs = {}
+
+    def reconstruct(result):
+        if lib.is_scalar(result):
+            return result
+        if result.ndim != self.ndim:
+            if method == "outer":
+                if self.ndim == 2:
+                    # we already deprecated for Series
+                    msg = (
+                        "outer method for ufunc {} is not implemented on "
+                        "pandas objects. Returning an ndarray, but in the "
+                        "future this will raise a 'NotImplementedError'. "
+                        "Consider explicitly converting the DataFrame "
+                        "to an array with '.to_numpy()' first."
+                    )
+                    warnings.warn(msg.format(ufunc), FutureWarning, stacklevel=4)
+                    return result
+                raise NotImplementedError
+            return result
+        if isinstance(result, BlockManager):
+            # we went through BlockManager.apply
+            result = self._constructor(result, **reconstruct_kwargs, copy=False)
+        else:
+            # we converted an array, lost our axes
+            result = self._constructor(
+                result, **reconstruct_axes, **reconstruct_kwargs, copy=False
+            )
+        # TODO: When we support multiple values in __finalize__, this
+        # should pass alignable to `__fianlize__` instead of self.
+        # Then `np.add(a, b)` would consider attrs from both a and b
+        # when a and b are NDFrames.
+        if len(alignable) == 1:
+            result = result.__finalize__(self)
+        return result
+
+    if self.ndim > 1 and (
+        len(inputs) > 1 or ufunc.nout > 1  # type: ignore[attr-defined]
+    ):
+        # Just give up on preserving types in the complex case.
+        # In theory we could preserve them for them.
+        # * nout>1 is doable if BlockManager.apply took nout and
+        #   returned a Tuple[BlockManager].
+        # * len(inputs) > 1 is doable when we know that we have
+        #   aligned blocks / dtypes.
+        inputs = tuple(np.asarray(x) for x in inputs)
+        result = getattr(ufunc, method)(*inputs)
+    elif self.ndim == 1:
+        # ufunc(series, ...)
+        inputs = tuple(extract_array(x, extract_numpy=True) for x in inputs)
+        result = getattr(ufunc, method)(*inputs, **kwargs)
+    else:
+        # ufunc(dataframe)
+        mgr = inputs[0]._mgr
+        result = mgr.apply(getattr(ufunc, method))
+
+    if ufunc.nout > 1:  # type: ignore[attr-defined]
+        result = tuple(reconstruct(x) for x in result)
+    else:
+        result = reconstruct(result)
+    return result

pandas/core/arraylike.py

@@ -430,6 +430,10 @@ def visit_Subscript(self, node, **kwargs):
         except AttributeError:
             pass
 
+        if isinstance(slobj, Term):
+            # In py39 np.ndarray lookups with Term containing int raise
+            slobj = slobj.value
+
         try:
             return self.const_type(value[slobj], self.env)
         except TypeError as err: