Merge branch 'master' into ref-gbop

Author: jbrockmendel

.pre-commit-config.yaml

@@ -1,5 +1,7 @@
 minimum_pre_commit_version: 2.9.2
 exclude: ^LICENSES/|\.(html|csv|svg)$
+ci:
+    autofix_prs: false
 repos:
 -   repo: https://github.com/MarcoGorelli/absolufy-imports
     rev: v0.3.0

.travis.yml

@@ -35,16 +35,9 @@ matrix:
   fast_finish: true
 
   include:
-    - arch: arm64
+    - arch: arm64-graviton2
       env:
-        - JOB="3.7, arm64" PYTEST_WORKERS=1 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)"
-
-  allow_failures:
-    # Moved to allowed_failures 2020-09-29 due to timeouts https://github.com/pandas-dev/pandas/issues/36719
-    - arch: arm64
-      env:
-        - JOB="3.7, arm64" PYTEST_WORKERS=1 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)"
-
+        - JOB="3.7, arm64" PYTEST_WORKERS="auto" ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)"
 
 before_install:
   - echo "before_install"

doc/_templates/sidebar-nav-bs.html

@@ -0,0 +1,9 @@
+<nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
+  <div class="bd-toc-item active">
+    {% if pagename.startswith("reference") %}
+    {{ generate_nav_html("sidebar", maxdepth=4, collapse=True, includehidden=True, titles_only=True) }}
+    {% else %}
+    {{ generate_nav_html("sidebar", maxdepth=4, collapse=False, includehidden=True, titles_only=True) }}
+    {% endif %}
+  </div>
+</nav>

doc/source/_static/style/hbetw_axNone.png

@@ -23,7 +23,7 @@ Styler properties
    :toctree: api/
 
    Styler.env
-   Styler.template
+   Styler.template_html
    Styler.loader
 
 Style application
@@ -53,6 +53,7 @@ Builtin styles
    Styler.highlight_null
    Styler.highlight_max
    Styler.highlight_min
+   Styler.highlight_between
    Styler.background_gradient
    Styler.bar
 

doc/source/_static/style/hbetw_basic.png

@@ -498,7 +498,7 @@ values across a level. For instance:
    )
    df = pd.DataFrame(np.random.randn(4, 2), index=midx)
    df
-   df2 = df.mean(level=0)
+   df2 = df.groupby(level=0).mean()
    df2
    df2.reindex(df.index, level=0)
 

doc/source/_static/style/hbetw_props.png

@@ -633,7 +633,7 @@ even if some categories are not present in the data:
         data=[[1, 2, 3], [4, 5, 6]],
         columns=pd.MultiIndex.from_arrays([["A", "B", "B"], columns]),
     )
-    df.sum(axis=1, level=1)
+    df.groupby(axis=1, level=1).sum()
 
 Groupby will also show "unused" categories:
 

doc/source/_static/style/hbetw_seq.png

@@ -320,14 +320,6 @@ number:
 
    s.groupby(level="second").sum()
 
-The aggregation functions such as ``sum`` will take the level parameter
-directly. Additionally, the resulting index will be named according to the
-chosen level:
-
-.. ipython:: python
-
-   s.sum(level="second")
-
 Grouping with multiple levels is supported.
 
 .. ipython:: python

doc/source/reference/style.rst

@@ -1710,7 +1710,7 @@
     "            Styler.loader,  # the default\n",
     "        ])\n",
     "    )\n",
-    "    template = env.get_template(\"myhtml.tpl\")"
+    "    template_html = env.get_template(\"myhtml.tpl\")"
    ]
   },
   {

doc/source/user_guide/advanced.rst

@@ -154,6 +154,7 @@ Other enhancements:
 - ``Series.all`` and ``Series.any`` now support the ``level`` and ``skipna`` parameters (:issue:`8302`):
 
   .. ipython:: python
+     :okwarning:
 
      s = pd.Series([False, True, False], index=[0, 0, 1])
      s.any(level=0)

doc/source/user_guide/categorical.rst

@@ -117,6 +117,9 @@ We provided some focused development on :class:`.Styler`, including altering met
 to accept more universal CSS language for arguments, such as ``'color:red;'`` instead of
 ``[('color', 'red')]`` (:issue:`39564`). This is also added to the built-in methods
 to allow custom CSS highlighting instead of default background coloring (:issue:`40242`).
+Enhancements to other built-in methods include extending the :meth:`.Styler.background_gradient`
+method to shade elements based on a given gradient map and not be restricted only to
+values in the DataFrame (:issue:`39930` :issue:`22727` :issue:`28901`).
 
 The :meth:`.Styler.apply` now consistently allows functions with ``ndarray`` output to
 allow more flexible development of UDFs when ``axis`` is ``None`` ``0`` or ``1`` (:issue:`39393`).
@@ -206,6 +209,7 @@ Other enhancements
 - :meth:`.Styler.background_gradient` now allows the ability to supply a specific gradient map (:issue:`22727`)
 - :meth:`.Styler.clear` now clears :attr:`Styler.hidden_index` and :attr:`Styler.hidden_columns` as well (:issue:`40484`)
 - Builtin highlighting methods in :class:`Styler` have a more consistent signature and css customisability (:issue:`40242`)
+- :meth:`.Styler.highlight_between` added to list of builtin styling methods (:issue:`39821`)
 - :meth:`Series.loc.__getitem__` and :meth:`Series.loc.__setitem__` with :class:`MultiIndex` now raising helpful error message when indexer has too many dimensions (:issue:`35349`)
 - :meth:`pandas.read_stata` and :class:`StataReader` support reading data from compressed files.
 - Add support for parsing ``ISO 8601``-like timestamps with negative signs to :meth:`pandas.Timedelta` (:issue:`37172`)
@@ -563,6 +567,7 @@ Deprecations
 - Deprecated allowing partial failure in :meth:`Series.transform` and :meth:`DataFrame.transform` when ``func`` is list-like or dict-like and raises anything but ``TypeError``; ``func`` raising anything but a ``TypeError`` will raise in a future version (:issue:`40211`)
 - Deprecated support for ``np.ma.mrecords.MaskedRecords`` in the :class:`DataFrame` constructor, pass ``{name: data[name] for name in data.dtype.names}`` instead (:issue:`40363`)
 - Deprecated the use of ``**kwargs`` in :class:`.ExcelWriter`; use the keyword argument ``engine_kwargs`` instead (:issue:`40430`)
+- Deprecated the ``level`` keyword for :class:`DataFrame` and :class:`Series` aggregations; use groupby instead (:issue:`39983`)
 
 .. ---------------------------------------------------------------------------
 
@@ -818,24 +823,29 @@ ExtensionArray
 - Fixed a bug where some properties of subclasses of :class:`PandasExtensionDtype` where improperly cached (:issue:`40329`)
 -
 
+Styler
+^^^^^^
+
+- Bug in :class:`Styler` where ``subset`` arg in methods raised an error for some valid multiindex slices (:issue:`33562`)
+- :class:`Styler` rendered HTML output minor alterations to support w3 good code standard (:issue:`39626`)
+- Bug in :class:`Styler` where rendered HTML was missing a column class identifier for certain header cells (:issue:`39716`)
+- Bug in :meth:`Styler.background_gradient` where text-color was not determined correctly (:issue:`39888`)
+- Bug in :class:`Styler` where multiple elements in CSS-selectors were not correctly added to ``table_styles`` (:issue:`39942`)
+- Bug in :class:`.Styler` where copying from Jupyter dropped top left cell and misaligned headers (:issue:`12147`)
+- Bug in :class:`.Styler.where` where ``kwargs`` were not passed to the applicable callable (:issue:`40845`)
+- Bug in :class:`Styler` which caused CSS to duplicate on multiple renders. (:issue:`39395`, :issue:`40334`)
+
+
 Other
 ^^^^^
 - Bug in :class:`Index` constructor sometimes silently ignoring a specified ``dtype`` (:issue:`38879`)
 - Bug in :func:`pandas.api.types.infer_dtype` not recognizing Series, Index or array with a period dtype (:issue:`23553`)
 - Bug in :func:`pandas.api.types.infer_dtype` raising an error for general :class:`.ExtensionArray` objects. It will now return ``"unknown-array"`` instead of raising (:issue:`37367`)
 - Bug in constructing a :class:`Series` from a list and a :class:`PandasDtype` (:issue:`39357`)
-- Bug in :class:`Styler` which caused CSS to duplicate on multiple renders. (:issue:`39395`, :issue:`40334`)
 - ``inspect.getmembers(Series)`` no longer raises an ``AbstractMethodError`` (:issue:`38782`)
 - Bug in :meth:`Series.where` with numeric dtype and ``other = None`` not casting to ``nan`` (:issue:`39761`)
 - :meth:`Index.where` behavior now mirrors :meth:`Index.putmask` behavior, i.e. ``index.where(mask, other)`` matches ``index.putmask(~mask, other)`` (:issue:`39412`)
 - Bug in :func:`pandas.testing.assert_series_equal`, :func:`pandas.testing.assert_frame_equal`, :func:`pandas.testing.assert_index_equal` and :func:`pandas.testing.assert_extension_array_equal` incorrectly raising when an attribute has an unrecognized NA type (:issue:`39461`)
-- Bug in :class:`Styler` where ``subset`` arg in methods raised an error for some valid multiindex slices (:issue:`33562`)
-- :class:`Styler` rendered HTML output minor alterations to support w3 good code standard (:issue:`39626`)
-- Bug in :class:`Styler` where rendered HTML was missing a column class identifier for certain header cells (:issue:`39716`)
-- Bug in :meth:`Styler.background_gradient` where text-color was not determined correctly (:issue:`39888`)
-- Bug in :class:`Styler` where multiple elements in CSS-selectors were not correctly added to ``table_styles`` (:issue:`39942`)
-- Bug in :class:`.Styler` where copying from Jupyter dropped top left cell and misaligned headers (:issue:`12147`)
-- Bug in :class:`.Styler.where` where ``kwargs`` were not passed to the applicable callable (:issue:`40845`)
 - Bug in :meth:`DataFrame.equals`, :meth:`Series.equals`, :meth:`Index.equals` with object-dtype containing ``np.datetime64("NaT")`` or ``np.timedelta64("NaT")`` (:issue:`39650`)
 - Bug in :func:`pandas.util.show_versions` where console JSON output was not proper JSON (:issue:`39701`)
 - Bug in :meth:`DataFrame.convert_dtypes` incorrectly raised ValueError when called on an empty DataFrame (:issue:`40393`)

doc/source/user_guide/groupby.rst

@@ -48,6 +48,8 @@
 
 """
 
+from __future__ import annotations
+
 from collections import namedtuple
 from contextlib import (
     ContextDecorator,
@@ -57,12 +59,7 @@
 from typing import (
     Any,
     Callable,
-    Dict,
     Iterable,
-    List,
-    Optional,
-    Tuple,
-    Type,
     cast,
 )
 import warnings
@@ -73,16 +70,16 @@
 RegisteredOption = namedtuple("RegisteredOption", "key defval doc validator cb")
 
 # holds deprecated option metadata
-_deprecated_options: Dict[str, DeprecatedOption] = {}
+_deprecated_options: dict[str, DeprecatedOption] = {}
 
 # holds registered option metadata
-_registered_options: Dict[str, RegisteredOption] = {}
+_registered_options: dict[str, RegisteredOption] = {}
 
 # holds the current values for registered options
-_global_config: Dict[str, Any] = {}
+_global_config: dict[str, Any] = {}
 
 # keys which have a special meaning
-_reserved_keys: List[str] = ["all"]
+_reserved_keys: list[str] = ["all"]
 
 
 class OptionError(AttributeError, KeyError):
@@ -194,7 +191,7 @@ def get_default_val(pat: str):
 class DictWrapper:
     """ provide attribute-style access to a nested dict"""
 
-    def __init__(self, d: Dict[str, Any], prefix: str = ""):
+    def __init__(self, d: dict[str, Any], prefix: str = ""):
         object.__setattr__(self, "d", d)
         object.__setattr__(self, "prefix", prefix)
 
@@ -428,8 +425,8 @@ def register_option(
     key: str,
     defval: object,
     doc: str = "",
-    validator: Optional[Callable[[Any], Any]] = None,
-    cb: Optional[Callable[[str], Any]] = None,
+    validator: Callable[[Any], Any] | None = None,
+    cb: Callable[[str], Any] | None = None,
 ) -> None:
     """
     Register an option in the package-wide pandas config object
@@ -500,7 +497,7 @@ def register_option(
 
 
 def deprecate_option(
-    key: str, msg: Optional[str] = None, rkey: Optional[str] = None, removal_ver=None
+    key: str, msg: str | None = None, rkey: str | None = None, removal_ver=None
 ) -> None:
     """
     Mark option `key` as deprecated, if code attempts to access this option,
@@ -547,7 +544,7 @@ def deprecate_option(
 # functions internal to the module
 
 
-def _select_options(pat: str) -> List[str]:
+def _select_options(pat: str) -> list[str]:
     """
     returns a list of keys matching `pat`
 
@@ -565,7 +562,7 @@ def _select_options(pat: str) -> List[str]:
     return [k for k in keys if re.search(pat, k, re.I)]
 
 
-def _get_root(key: str) -> Tuple[Dict[str, Any], str]:
+def _get_root(key: str) -> tuple[dict[str, Any], str]:
     path = key.split(".")
     cursor = _global_config
     for p in path[:-1]:
@@ -674,7 +671,7 @@ def pp_options_list(keys: Iterable[str], width=80, _print: bool = False):
     from itertools import groupby
     from textwrap import wrap
 
-    def pp(name: str, ks: Iterable[str]) -> List[str]:
+    def pp(name: str, ks: Iterable[str]) -> list[str]:
         pfx = "- " + name + ".[" if name else ""
         ls = wrap(
             ", ".join(ks),
@@ -687,7 +684,7 @@ def pp(name: str, ks: Iterable[str]) -> List[str]:
             ls[-1] = ls[-1] + "]"
         return ls
 
-    ls: List[str] = []
+    ls: list[str] = []
     singles = [x for x in sorted(keys) if x.find(".") < 0]
     if singles:
         ls += pp("", singles)
@@ -760,7 +757,7 @@ def inner(key: str, *args, **kwds):
 # arg in register_option
 
 
-def is_type_factory(_type: Type[Any]) -> Callable[[Any], None]:
+def is_type_factory(_type: type[Any]) -> Callable[[Any], None]:
     """
 
     Parameters
@@ -826,7 +823,7 @@ def inner(x) -> None:
     return inner
 
 
-def is_nonnegative_int(value: Optional[int]) -> None:
+def is_nonnegative_int(value: int | None) -> None:
     """
     Verify that value is None or a positive int.
 

doc/source/user_guide/style.ipynb

@@ -2,15 +2,16 @@
 Unopinionated display configuration.
 """
 
+from __future__ import annotations
+
 import locale
 import sys
-from typing import Optional
 
 from pandas._config import config as cf
 
 # -----------------------------------------------------------------------------
 # Global formatting options
-_initial_defencoding: Optional[str] = None
+_initial_defencoding: str | None = None
 
 
 def detect_console_encoding() -> str:

doc/source/whatsnew/v0.15.2.rst

@@ -74,6 +74,7 @@ cdef class IndexEngine:
         return val in self.mapping
 
     cpdef get_loc(self, object val):
+        # -> Py_ssize_t | slice | ndarray[bool]
         cdef:
             Py_ssize_t loc
 
@@ -109,6 +110,7 @@ cdef class IndexEngine:
             raise KeyError(val)
 
     cdef inline _get_loc_duplicates(self, object val):
+        # -> Py_ssize_t | slice | ndarray[bool]
         cdef:
             Py_ssize_t diff
 
@@ -142,6 +144,7 @@ cdef class IndexEngine:
     cdef _unpack_bool_indexer(self,
                               ndarray[uint8_t, ndim=1, cast=True] indexer,
                               object val):
+        # Returns ndarray[bool] or int
         cdef:
             ndarray[intp_t, ndim=1] found
             int count

doc/source/whatsnew/v1.3.0.rst

@@ -44,8 +44,8 @@ cdef class {{name}}Engine(IndexEngine):
             raise KeyError(val)
     {{endif}}
 
-    cdef void _call_map_locations(self, ndarray values):
-        self.mapping.map_locations(algos.ensure_{{name.lower()}}(values))
+    cdef void _call_map_locations(self, ndarray[{{dtype}}_t] values):
+        self.mapping.map_locations(values)
 
     cdef _maybe_get_bool_indexer(self, object val):
         # Returns ndarray[bool] or int