Merge branch 'master' of github.com:pandas-dev/pandas

Author: taytzehao

asv_bench/benchmarks/groupby.py

@@ -480,7 +480,19 @@ class GroupByCythonAgg:
     param_names = ["dtype", "method"]
     params = [
         ["float64"],
-        ["sum", "prod", "min", "max", "mean", "median", "var", "first", "last"],
+        [
+            "sum",
+            "prod",
+            "min",
+            "max",
+            "mean",
+            "median",
+            "var",
+            "first",
+            "last",
+            "any",
+            "all",
+        ],
     ]
 
     def setup(self, dtype, method):

ci/azure/windows.yml

@@ -8,7 +8,7 @@ jobs:
     vmImage: ${{ parameters.vmImage }}
   strategy:
     matrix:
-      py37_np16:
+      py37_np17:
         ENV_FILE: ci/deps/azure-windows-37.yaml
         CONDA_PY: "37"
         PATTERN: "not slow and not network"

ci/deps/actions-37-minimum_versions.yaml

@@ -18,7 +18,7 @@ dependencies:
   - jinja2=2.10
   - numba=0.46.0
   - numexpr=2.6.8
-  - numpy=1.16.5
+  - numpy=1.17.3
   - openpyxl=3.0.0
   - pytables=3.5.1
   - python-dateutil=2.7.3

ci/deps/azure-macos-37.yaml

@@ -19,7 +19,7 @@ dependencies:
   - matplotlib=2.2.3
   - nomkl
   - numexpr
-  - numpy=1.16.5
+  - numpy=1.17.3
   - openpyxl
   - pyarrow=0.15.1
   - pytables

ci/deps/azure-windows-37.yaml

@@ -24,7 +24,7 @@ dependencies:
   - moto>=1.3.14
   - flask
   - numexpr
-  - numpy=1.16.*
+  - numpy=1.17.*
   - openpyxl
   - pyarrow=0.15
   - pytables

doc/source/getting_started/install.rst

@@ -222,7 +222,7 @@ Dependencies
 Package                                                          Minimum supported version
 ================================================================ ==========================
 `setuptools <https://setuptools.readthedocs.io/en/latest/>`__    38.6.0
-`NumPy <https://numpy.org>`__                                    1.16.5
+`NumPy <https://numpy.org>`__                                    1.17.3
 `python-dateutil <https://dateutil.readthedocs.io/en/stable/>`__ 2.7.3
 `pytz <https://pypi.org/project/pytz/>`__                        2017.3
 ================================================================ ==========================

doc/source/whatsnew/v1.3.0.rst

@@ -217,6 +217,9 @@ Other enhancements
 - :class:`RangeIndex` can now be constructed by passing a ``range`` object directly e.g. ``pd.RangeIndex(range(3))`` (:issue:`12067`)
 - :meth:`round` being enabled for the nullable integer and floating dtypes (:issue:`38844`)
 - :meth:`pandas.read_csv` and :meth:`pandas.read_json` expose the argument ``encoding_errors`` to control how encoding errors are handled (:issue:`39450`)
+- :meth:`.GroupBy.any` and :meth:`.GroupBy.all` use Kleene logic with nullable data types (:issue:`37506`)
+- :meth:`.GroupBy.any` and :meth:`.GroupBy.all` return a ``BooleanDtype`` for columns with nullable data types (:issue:`33449`)
+-
 
 .. ---------------------------------------------------------------------------
 
@@ -465,7 +468,7 @@ If installed, we now require:
 +-----------------+-----------------+----------+---------+
 | Package         | Minimum Version | Required | Changed |
 +=================+=================+==========+=========+
-| numpy           | 1.16.5          |    X     |         |
+| numpy           | 1.17.3          |    X     |    X    |
 +-----------------+-----------------+----------+---------+
 | pytz            | 2017.3          |    X     |         |
 +-----------------+-----------------+----------+---------+
@@ -711,7 +714,7 @@ Missing
 
 - Bug in :class:`Grouper` now correctly propagates ``dropna`` argument and :meth:`DataFrameGroupBy.transform` now correctly handles missing values for ``dropna=True`` (:issue:`35612`)
 - Bug in :func:`isna`, and :meth:`Series.isna`, :meth:`Index.isna`, :meth:`DataFrame.isna` (and the corresponding ``notna`` functions) not recognizing ``Decimal("NaN")`` objects (:issue:`39409`)
--
+- Bug in :meth:`DataFrame.fillna` not accepting dictionary for ``downcast`` keyword (:issue:`40809`)
 
 MultiIndex
 ^^^^^^^^^^
@@ -787,6 +790,8 @@ Groupby/resample/rolling
 - Bug in :meth:`Series.asfreq` and :meth:`DataFrame.asfreq` dropping rows when the index is not sorted (:issue:`39805`)
 - Bug in aggregation functions for :class:`DataFrame` not respecting ``numeric_only`` argument when ``level`` keyword was given (:issue:`40660`)
 - Bug in :class:`core.window.RollingGroupby` where ``as_index=False`` argument in ``groupby`` was ignored (:issue:`39433`)
+- Bug in :meth:`.GroupBy.any` and :meth:`.GroupBy.all` raising ``ValueError`` when using with nullable type columns holding ``NA`` even with ``skipna=True`` (:issue:`40585`)
+
 
 Reshaping
 ^^^^^^^^^

environment.yml

@@ -3,7 +3,7 @@ channels:
   - conda-forge
 dependencies:
   # required
-  - numpy>=1.16.5
+  - numpy>=1.17.3
   - python=3
   - python-dateutil>=2.7.3
   - pytz

pandas/__init__.py

@@ -20,7 +20,6 @@
 
 # numpy compat
 from pandas.compat import (
-    np_version_under1p17 as _np_version_under1p17,
     np_version_under1p18 as _np_version_under1p18,
     is_numpy_dev as _is_numpy_dev,
 )

pandas/_libs/groupby.pyx

@@ -388,40 +388,47 @@ def group_fillna_indexer(ndarray[int64_t] out, ndarray[intp_t] labels,
 
 @cython.boundscheck(False)
 @cython.wraparound(False)
-def group_any_all(uint8_t[::1] out,
-                  const uint8_t[::1] values,
+def group_any_all(int8_t[::1] out,
+                  const int8_t[::1] values,
                   const intp_t[:] labels,
                   const uint8_t[::1] mask,
                   str val_test,
-                  bint skipna) -> None:
+                  bint skipna,
+                  bint nullable) -> None:
     """
-    Aggregated boolean values to show truthfulness of group elements.
+    Aggregated boolean values to show truthfulness of group elements. If the
+    input is a nullable type (nullable=True), the result will be computed
+    using Kleene logic.
 
     Parameters
     ----------
-    out : np.ndarray[np.uint8]
+    out : np.ndarray[np.int8]
         Values into which this method will write its results.
     labels : np.ndarray[np.intp]
         Array containing unique label for each group, with its
         ordering matching up to the corresponding record in `values`
-    values : np.ndarray[np.uint8]
+    values : np.ndarray[np.int8]
         Containing the truth value of each element.
     mask : np.ndarray[np.uint8]
         Indicating whether a value is na or not.
     val_test : {'any', 'all'}
         String object dictating whether to use any or all truth testing
     skipna : bool
         Flag to ignore nan values during truth testing
+    nullable : bool
+        Whether or not the input is a nullable type. If True, the
+        result will be computed using Kleene logic
 
     Notes
     -----
     This method modifies the `out` parameter rather than returning an object.
-    The returned values will either be 0 or 1 (False or True, respectively).
+    The returned values will either be 0, 1 (False or True, respectively), or
+    -1 to signify a masked position in the case of a nullable input.
     """
     cdef:
         Py_ssize_t i, N = len(labels)
         intp_t lab
-        uint8_t flag_val
+        int8_t flag_val
 
     if val_test == 'all':
         # Because the 'all' value of an empty iterable in Python is True we can
@@ -444,6 +451,16 @@ def group_any_all(uint8_t[::1] out,
             if lab < 0 or (skipna and mask[i]):
                 continue
 
+            if nullable and mask[i]:
+                # Set the position as masked if `out[lab] != flag_val`, which
+                # would indicate True/False has not yet been seen for any/all,
+                # so by Kleene logic the result is currently unknown
+                if out[lab] != flag_val:
+                    out[lab] = -1
+                continue
+
+            # If True and 'any' or False and 'all', the result is
+            # already determined
             if values[i] == flag_val:
                 out[lab] = flag_val
 

pandas/compat/__init__.py

@@ -16,7 +16,6 @@
     is_numpy_dev,
     np_array_datetime64_compat,
     np_datetime64_compat,
-    np_version_under1p17,
     np_version_under1p18,
     np_version_under1p19,
     np_version_under1p20,
@@ -133,7 +132,6 @@ def get_lzma_file(lzma):
     "is_numpy_dev",
     "np_array_datetime64_compat",
     "np_datetime64_compat",
-    "np_version_under1p17",
     "np_version_under1p18",
     "np_version_under1p19",
     "np_version_under1p20",

pandas/compat/numpy/__init__.py

@@ -8,12 +8,11 @@
 # numpy versioning
 _np_version = np.__version__
 _nlv = LooseVersion(_np_version)
-np_version_under1p17 = _nlv < LooseVersion("1.17")
 np_version_under1p18 = _nlv < LooseVersion("1.18")
 np_version_under1p19 = _nlv < LooseVersion("1.19")
 np_version_under1p20 = _nlv < LooseVersion("1.20")
 is_numpy_dev = ".dev" in str(_nlv)
-_min_numpy_ver = "1.16.5"
+_min_numpy_ver = "1.17.3"
 
 
 if _nlv < _min_numpy_ver:
@@ -65,6 +64,5 @@ def np_array_datetime64_compat(arr, dtype="M8[ns]"):
 __all__ = [
     "np",
     "_np_version",
-    "np_version_under1p17",
     "is_numpy_dev",
 ]

pandas/core/array_algos/masked_reductions.py

@@ -8,7 +8,6 @@
 import numpy as np
 
 from pandas._libs import missing as libmissing
-from pandas.compat import np_version_under1p17
 
 from pandas.core.nanops import check_below_min_count
 
@@ -46,11 +45,7 @@ def _sumprod(
     else:
         if check_below_min_count(values.shape, mask, min_count):
             return libmissing.NA
-
-        if np_version_under1p17:
-            return func(values[~mask])
-        else:
-            return func(values, where=~mask)
+        return func(values, where=~mask)
 
 
 def sum(

pandas/core/arrays/datetimes.py

@@ -190,7 +190,7 @@ class DatetimeArray(dtl.TimelikeOps, dtl.DatelikeOps):
     _infer_matches = ("datetime", "datetime64", "date")
 
     # define my properties & methods for delegation
-    _bool_ops = [
+    _bool_ops: list[str] = [
         "is_month_start",
         "is_month_end",
         "is_quarter_start",
@@ -199,8 +199,8 @@ class DatetimeArray(dtl.TimelikeOps, dtl.DatelikeOps):
         "is_year_end",
         "is_leap_year",
     ]
-    _object_ops = ["freq", "tz"]
-    _field_ops = [
+    _object_ops: list[str] = ["freq", "tz"]
+    _field_ops: list[str] = [
         "year",
         "month",
         "day",
@@ -220,9 +220,9 @@ class DatetimeArray(dtl.TimelikeOps, dtl.DatelikeOps):
         "microsecond",
         "nanosecond",
     ]
-    _other_ops = ["date", "time", "timetz"]
-    _datetimelike_ops = _field_ops + _object_ops + _bool_ops + _other_ops
-    _datetimelike_methods = [
+    _other_ops: list[str] = ["date", "time", "timetz"]
+    _datetimelike_ops: list[str] = _field_ops + _object_ops + _bool_ops + _other_ops
+    _datetimelike_methods: list[str] = [
         "to_period",
         "tz_localize",
         "tz_convert",

pandas/core/arrays/interval.py

@@ -87,7 +87,7 @@
 
 IntervalArrayT = TypeVar("IntervalArrayT", bound="IntervalArray")
 
-_interval_shared_docs = {}
+_interval_shared_docs: dict[str, str] = {}
 
 _shared_docs_kwargs = {
     "klass": "IntervalArray",

pandas/core/arrays/period.py

@@ -157,9 +157,9 @@ class PeriodArray(PeriodMixin, dtl.DatelikeOps):
 
     # Names others delegate to us
     _other_ops: list[str] = []
-    _bool_ops = ["is_leap_year"]
-    _object_ops = ["start_time", "end_time", "freq"]
-    _field_ops = [
+    _bool_ops: list[str] = ["is_leap_year"]
+    _object_ops: list[str] = ["start_time", "end_time", "freq"]
+    _field_ops: list[str] = [
         "year",
         "month",
         "day",

pandas/core/generic.py

@@ -1797,9 +1797,9 @@ def _drop_labels_or_levels(self, keys, axis: int = 0):
 
         Parameters
         ----------
-        keys: str or list of str
+        keys : str or list of str
             labels or levels to drop
-        axis: int, default 0
+        axis : int, default 0
             Axis that levels are associated with (0 for index, 1 for columns)
 
         Returns
@@ -6446,11 +6446,13 @@ def fillna(
                     )
 
                 result = self if inplace else self.copy()
+                is_dict = isinstance(downcast, dict)
                 for k, v in value.items():
                     if k not in result:
                         continue
                     obj = result[k]
-                    obj.fillna(v, limit=limit, inplace=True, downcast=downcast)
+                    downcast_k = downcast if not is_dict else downcast.get(k)
+                    obj.fillna(v, limit=limit, inplace=True, downcast=downcast_k)
                 return result if not inplace else None
 
             elif not is_list_like(value):

pandas/core/groupby/base.py

@@ -3,8 +3,9 @@
 hold the allowlist of methods that are exposed on the
 SeriesGroupBy and the DataFrameGroupBy objects.
 """
+from __future__ import annotations
+
 import collections
-from typing import List
 
 from pandas._typing import final
 
@@ -19,7 +20,7 @@
 
 
 class ShallowMixin(PandasObject):
-    _attributes: List[str] = []
+    _attributes: list[str] = []
 
     @final
     def _shallow_copy(self, obj, **kwargs):
@@ -39,7 +40,7 @@ class GotItemMixin(PandasObject):
     Provide the groupby facilities to the mixed object.
     """
 
-    _attributes: List[str]
+    _attributes: list[str]
 
     @final
     def _gotitem(self, key, ndim, subset=None):
@@ -106,12 +107,16 @@ def _gotitem(self, key, ndim, subset=None):
     | plotting_methods
 )
 
-series_apply_allowlist = (
+series_apply_allowlist: frozenset[str] = (
     common_apply_allowlist
-    | {"nlargest", "nsmallest", "is_monotonic_increasing", "is_monotonic_decreasing"}
+    | frozenset(
+        {"nlargest", "nsmallest", "is_monotonic_increasing", "is_monotonic_decreasing"}
+    )
 ) | frozenset(["dtype", "unique"])
 
-dataframe_apply_allowlist = common_apply_allowlist | frozenset(["dtypes", "corrwith"])
+dataframe_apply_allowlist: frozenset[str] = common_apply_allowlist | frozenset(
+    ["dtypes", "corrwith"]
+)
 
 # cythonized transformations or canned "agg+broadcast", which do not
 # require postprocessing of the result by transform.