Merge pull request #175 from pandas-dev/master

Sync Fork from Upstream Repo

Author: sthagen

ci/azure/windows.yml

@@ -8,7 +8,7 @@ jobs:
     vmImage: ${{ parameters.vmImage }}
   strategy:
     matrix:
-      py37_np16:
+      py37_np17:
         ENV_FILE: ci/deps/azure-windows-37.yaml
         CONDA_PY: "37"
         PATTERN: "not slow and not network"

ci/deps/actions-37-minimum_versions.yaml

@@ -18,7 +18,7 @@ dependencies:
   - jinja2=2.10
   - numba=0.46.0
   - numexpr=2.6.8
-  - numpy=1.16.5
+  - numpy=1.17.3
   - openpyxl=3.0.0
   - pytables=3.5.1
   - python-dateutil=2.7.3

ci/deps/azure-macos-37.yaml

@@ -19,7 +19,7 @@ dependencies:
   - matplotlib=2.2.3
   - nomkl
   - numexpr
-  - numpy=1.16.5
+  - numpy=1.17.3
   - openpyxl
   - pyarrow=0.15.1
   - pytables

ci/deps/azure-windows-37.yaml

@@ -24,7 +24,7 @@ dependencies:
   - moto>=1.3.14
   - flask
   - numexpr
-  - numpy=1.16.*
+  - numpy=1.17.*
   - openpyxl
   - pyarrow=0.15
   - pytables

doc/source/getting_started/install.rst

@@ -222,7 +222,7 @@ Dependencies
 Package                                                          Minimum supported version
 ================================================================ ==========================
 `setuptools <https://setuptools.readthedocs.io/en/latest/>`__    38.6.0
-`NumPy <https://numpy.org>`__                                    1.16.5
+`NumPy <https://numpy.org>`__                                    1.17.3
 `python-dateutil <https://dateutil.readthedocs.io/en/stable/>`__ 2.7.3
 `pytz <https://pypi.org/project/pytz/>`__                        2017.3
 ================================================================ ==========================

doc/source/whatsnew/v1.3.0.rst

@@ -468,7 +468,7 @@ If installed, we now require:
 +-----------------+-----------------+----------+---------+
 | Package         | Minimum Version | Required | Changed |
 +=================+=================+==========+=========+
-| numpy           | 1.16.5          |    X     |         |
+| numpy           | 1.17.3          |    X     |    X    |
 +-----------------+-----------------+----------+---------+
 | pytz            | 2017.3          |    X     |         |
 +-----------------+-----------------+----------+---------+
@@ -714,7 +714,7 @@ Missing
 
 - Bug in :class:`Grouper` now correctly propagates ``dropna`` argument and :meth:`DataFrameGroupBy.transform` now correctly handles missing values for ``dropna=True`` (:issue:`35612`)
 - Bug in :func:`isna`, and :meth:`Series.isna`, :meth:`Index.isna`, :meth:`DataFrame.isna` (and the corresponding ``notna`` functions) not recognizing ``Decimal("NaN")`` objects (:issue:`39409`)
--
+- Bug in :meth:`DataFrame.fillna` not accepting dictionary for ``downcast`` keyword (:issue:`40809`)
 
 MultiIndex
 ^^^^^^^^^^

environment.yml

@@ -3,7 +3,7 @@ channels:
   - conda-forge
 dependencies:
   # required
-  - numpy>=1.16.5
+  - numpy>=1.17.3
   - python=3
   - python-dateutil>=2.7.3
   - pytz

pandas/__init__.py

@@ -20,7 +20,6 @@
 
 # numpy compat
 from pandas.compat import (
-    np_version_under1p17 as _np_version_under1p17,
     np_version_under1p18 as _np_version_under1p18,
     is_numpy_dev as _is_numpy_dev,
 )

pandas/compat/__init__.py

@@ -16,7 +16,6 @@
     is_numpy_dev,
     np_array_datetime64_compat,
     np_datetime64_compat,
-    np_version_under1p17,
     np_version_under1p18,
     np_version_under1p19,
     np_version_under1p20,
@@ -133,7 +132,6 @@ def get_lzma_file(lzma):
     "is_numpy_dev",
     "np_array_datetime64_compat",
     "np_datetime64_compat",
-    "np_version_under1p17",
     "np_version_under1p18",
     "np_version_under1p19",
     "np_version_under1p20",

pandas/compat/numpy/__init__.py

@@ -8,12 +8,11 @@
 # numpy versioning
 _np_version = np.__version__
 _nlv = LooseVersion(_np_version)
-np_version_under1p17 = _nlv < LooseVersion("1.17")
 np_version_under1p18 = _nlv < LooseVersion("1.18")
 np_version_under1p19 = _nlv < LooseVersion("1.19")
 np_version_under1p20 = _nlv < LooseVersion("1.20")
 is_numpy_dev = ".dev" in str(_nlv)
-_min_numpy_ver = "1.16.5"
+_min_numpy_ver = "1.17.3"
 
 
 if _nlv < _min_numpy_ver:
@@ -65,6 +64,5 @@ def np_array_datetime64_compat(arr, dtype="M8[ns]"):
 __all__ = [
     "np",
     "_np_version",
-    "np_version_under1p17",
     "is_numpy_dev",
 ]

pandas/core/array_algos/masked_reductions.py

@@ -8,7 +8,6 @@
 import numpy as np
 
 from pandas._libs import missing as libmissing
-from pandas.compat import np_version_under1p17
 
 from pandas.core.nanops import check_below_min_count
 
@@ -46,11 +45,7 @@ def _sumprod(
     else:
         if check_below_min_count(values.shape, mask, min_count):
             return libmissing.NA
-
-        if np_version_under1p17:
-            return func(values[~mask])
-        else:
-            return func(values, where=~mask)
+        return func(values, where=~mask)
 
 
 def sum(

pandas/core/generic.py

@@ -6446,11 +6446,13 @@ def fillna(
                     )
 
                 result = self if inplace else self.copy()
+                is_dict = isinstance(downcast, dict)
                 for k, v in value.items():
                     if k not in result:
                         continue
                     obj = result[k]
-                    obj.fillna(v, limit=limit, inplace=True, downcast=downcast)
+                    downcast_k = downcast if not is_dict else downcast.get(k)
+                    obj.fillna(v, limit=limit, inplace=True, downcast=downcast_k)
                 return result if not inplace else None
 
             elif not is_list_like(value):

pandas/core/groupby/generic.py

@@ -177,9 +177,6 @@ def pinner(cls):
 class SeriesGroupBy(GroupBy[Series]):
     _apply_allowlist = base.series_apply_allowlist
 
-    # Defined as a cache_readonly in SelectionMixin
-    _obj_with_exclusions: Series
-
     def _iterate_slices(self) -> Iterable[Series]:
         yield self._selected_obj
 
@@ -930,9 +927,6 @@ def pct_change(self, periods=1, fill_method="pad", limit=None, freq=None):
 @pin_allowlisted_properties(DataFrame, base.dataframe_apply_allowlist)
 class DataFrameGroupBy(GroupBy[DataFrame]):
 
-    # Defined as a cache_readonly in SelectionMixin
-    _obj_with_exclusions: DataFrame
-
     _apply_allowlist = base.dataframe_apply_allowlist
 
     _agg_examples_doc = dedent(

pandas/core/indexing.py

@@ -1873,7 +1873,11 @@ def _setitem_single_column(self, loc: int, value, plane_indexer):
         if com.is_null_slice(pi) or com.is_full_slice(pi, len(self.obj)):
             ser = value
         elif is_array_like(value) and is_exact_shape_match(ser, value):
-            ser = value
+            if is_list_like(pi):
+                ser = value[np.argsort(pi)]
+            else:
+                # in case of slice
+                ser = value[pi]
         else:
             # set the item, possibly having a dtype change
             ser = ser.copy()

pandas/io/formats/style.py

@@ -1558,55 +1558,10 @@ def highlight_between(
 
         .. figure:: ../../_static/style/hbetw_props.png
         """
-
-        def f(
-            data: FrameOrSeries,
-            props: str,
-            left: Scalar | Sequence | np.ndarray | FrameOrSeries | None = None,
-            right: Scalar | Sequence | np.ndarray | FrameOrSeries | None = None,
-            inclusive: bool | str = True,
-        ) -> np.ndarray:
-            if np.iterable(left) and not isinstance(left, str):
-                left = _validate_apply_axis_arg(
-                    left, "left", None, data  # type: ignore[arg-type]
-                )
-
-            if np.iterable(right) and not isinstance(right, str):
-                right = _validate_apply_axis_arg(
-                    right, "right", None, data  # type: ignore[arg-type]
-                )
-
-            # get ops with correct boundary attribution
-            if inclusive == "both":
-                ops = (operator.ge, operator.le)
-            elif inclusive == "neither":
-                ops = (operator.gt, operator.lt)
-            elif inclusive == "left":
-                ops = (operator.ge, operator.lt)
-            elif inclusive == "right":
-                ops = (operator.gt, operator.le)
-            else:
-                raise ValueError(
-                    f"'inclusive' values can be 'both', 'left', 'right', or 'neither' "
-                    f"got {inclusive}"
-                )
-
-            g_left = (
-                ops[0](data, left)
-                if left is not None
-                else np.full(data.shape, True, dtype=bool)
-            )
-            l_right = (
-                ops[1](data, right)
-                if right is not None
-                else np.full(data.shape, True, dtype=bool)
-            )
-            return np.where(g_left & l_right, props, "")
-
         if props is None:
             props = f"background-color: {color};"
         return self.apply(
-            f,  # type: ignore[arg-type]
+            _highlight_between,  # type: ignore[arg-type]
             axis=axis,
             subset=subset,
             props=props,
@@ -1831,3 +1786,51 @@ def css(rgba) -> str:
                 index=data.index,
                 columns=data.columns,
             )
+
+
+def _highlight_between(
+    data: FrameOrSeries,
+    props: str,
+    left: Scalar | Sequence | np.ndarray | FrameOrSeries | None = None,
+    right: Scalar | Sequence | np.ndarray | FrameOrSeries | None = None,
+    inclusive: bool | str = True,
+) -> np.ndarray:
+    """
+    Return an array of css props based on condition of data values within given range.
+    """
+    if np.iterable(left) and not isinstance(left, str):
+        left = _validate_apply_axis_arg(
+            left, "left", None, data  # type: ignore[arg-type]
+        )
+
+    if np.iterable(right) and not isinstance(right, str):
+        right = _validate_apply_axis_arg(
+            right, "right", None, data  # type: ignore[arg-type]
+        )
+
+    # get ops with correct boundary attribution
+    if inclusive == "both":
+        ops = (operator.ge, operator.le)
+    elif inclusive == "neither":
+        ops = (operator.gt, operator.lt)
+    elif inclusive == "left":
+        ops = (operator.ge, operator.lt)
+    elif inclusive == "right":
+        ops = (operator.gt, operator.le)
+    else:
+        raise ValueError(
+            f"'inclusive' values can be 'both', 'left', 'right', or 'neither' "
+            f"got {inclusive}"
+        )
+
+    g_left = (
+        ops[0](data, left)
+        if left is not None
+        else np.full(data.shape, True, dtype=bool)
+    )
+    l_right = (
+        ops[1](data, right)
+        if right is not None
+        else np.full(data.shape, True, dtype=bool)
+    )
+    return np.where(g_left & l_right, props, "")

pandas/tests/api/test_api.py

@@ -192,7 +192,6 @@ class TestPDApi(Base):
         "_hashtable",
         "_lib",
         "_libs",
-        "_np_version_under1p17",
         "_np_version_under1p18",
         "_is_numpy_dev",
         "_testing",

pandas/tests/computation/test_eval.py

@@ -12,10 +12,6 @@
 import numpy as np
 import pytest
 
-from pandas.compat import (
-    is_platform_windows,
-    np_version_under1p17,
-)
 from pandas.errors import PerformanceWarning
 import pandas.util._test_decorators as td
 
@@ -220,22 +216,6 @@ def test_simple_cmp_ops(self, cmp_op):
 
     @pytest.mark.parametrize("op", _good_arith_ops)
     def test_binary_arith_ops(self, op, lhs, rhs, request):
-
-        if (
-            op == "/"
-            and isinstance(lhs, DataFrame)
-            and isinstance(rhs, DataFrame)
-            and not lhs.isna().any().any()
-            and rhs.shape == (10, 5)
-            and np_version_under1p17
-            and is_platform_windows()
-            and compat.PY38
-        ):
-            mark = pytest.mark.xfail(
-                reason="GH#37328 floating point precision on Windows builds"
-            )
-            request.node.add_marker(mark)
-
         self.check_binary_arith_op(lhs, op, rhs)
 
     def test_modulus(self, lhs, rhs):

pandas/tests/extension/base/dim2.py

@@ -4,26 +4,10 @@
 import numpy as np
 import pytest
 
-from pandas.compat import np_version_under1p17
-
 import pandas as pd
-from pandas.core.arrays import (
-    FloatingArray,
-    IntegerArray,
-)
 from pandas.tests.extension.base.base import BaseExtensionTests
 
 
-def maybe_xfail_masked_reductions(arr, request):
-    if (
-        isinstance(arr, (FloatingArray, IntegerArray))
-        and np_version_under1p17
-        and arr.ndim == 2
-    ):
-        mark = pytest.mark.xfail(reason="masked_reductions does not implement")
-        request.node.add_marker(mark)
-
-
 class Dim2CompatTests(BaseExtensionTests):
     def test_swapaxes(self, data):
         arr2d = data.repeat(2).reshape(-1, 2)
@@ -148,7 +132,6 @@ def test_reductions_2d_axis_none(self, data, method, request):
             pytest.skip("test is not applicable for this type/dtype")
 
         arr2d = data.reshape(1, -1)
-        maybe_xfail_masked_reductions(arr2d, request)
 
         err_expected = None
         err_result = None
@@ -177,7 +160,6 @@ def test_reductions_2d_axis0(self, data, method, request):
             pytest.skip("test is not applicable for this type/dtype")
 
         arr2d = data.reshape(1, -1)
-        maybe_xfail_masked_reductions(arr2d, request)
 
         kwargs = {}
         if method == "std":
@@ -225,7 +207,6 @@ def test_reductions_2d_axis1(self, data, method, request):
             pytest.skip("test is not applicable for this type/dtype")
 
         arr2d = data.reshape(1, -1)
-        maybe_xfail_masked_reductions(arr2d, request)
 
         try:
             result = getattr(arr2d, method)(axis=1)