Merge remote-tracking branch 'upstream/master' into styler_tooltips_feature

# Conflicts:
#	doc/source/whatsnew/v1.3.0.rst

Author: attack68

.github/workflows/ci.yml

@@ -2,74 +2,81 @@ name: CI
 
 on:
   push:
-    branches: master
+    branches: [master]
   pull_request:
     branches:
       - master
       - 1.2.x
 
 env:
   ENV_FILE: environment.yml
+  PANDAS_CI: 1
 
 jobs:
   checks:
     name: Checks
     runs-on: ubuntu-latest
-    steps:
-
-    - name: Setting conda path
-      run: echo "${HOME}/miniconda3/bin" >> $GITHUB_PATH
+    defaults:
+      run:
+        shell: bash -l {0}
 
+    steps:
     - name: Checkout
       uses: actions/checkout@v1
 
     - name: Looking for unwanted patterns
       run: ci/code_checks.sh patterns
       if: always()
 
-    - name: Setup environment and build pandas
-      run: ci/setup_env.sh
-      if: always()
+    - name: Cache conda
+      uses: actions/cache@v2
+      with:
+        path: ~/conda_pkgs_dir
+        key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }}
 
-    - name: Linting
+    - uses: conda-incubator/setup-miniconda@v2
+      with:
+        activate-environment: pandas-dev
+        channel-priority: strict
+        environment-file: ${{ env.ENV_FILE }}
+        use-only-tar-bz2: true
+
+    - name: Environment Detail
       run: |
-        source activate pandas-dev
-        ci/code_checks.sh lint
+        conda info
+        conda list
+
+    - name: Build Pandas
+      run: |
+        python setup.py build_ext -j 2
+        python -m pip install -e . --no-build-isolation --no-use-pep517
+
+    - name: Linting
+      run: ci/code_checks.sh lint
       if: always()
 
     - name: Checks on imported code
-      run: |
-        source activate pandas-dev
-        ci/code_checks.sh code
+      run: ci/code_checks.sh code
       if: always()
 
     - name: Running doctests
-      run: |
-        source activate pandas-dev
-        ci/code_checks.sh doctests
+      run: ci/code_checks.sh doctests
       if: always()
 
     - name: Docstring validation
-      run: |
-        source activate pandas-dev
-        ci/code_checks.sh docstrings
+      run: ci/code_checks.sh docstrings
       if: always()
 
     - name: Typing validation
-      run: |
-        source activate pandas-dev
-        ci/code_checks.sh typing
+      run: ci/code_checks.sh typing
       if: always()
 
     - name: Testing docstring validation script
-      run: |
-        source activate pandas-dev
-        pytest --capture=no --strict-markers scripts
+      run: pytest --capture=no --strict-markers scripts
       if: always()
 
     - name: Running benchmarks
       run: |
-        source activate pandas-dev
         cd asv_bench
         asv check -E existing
         git remote add upstream https://github.com/pandas-dev/pandas.git
@@ -106,7 +113,6 @@ jobs:
       run: |
         source activate pandas-dev
         python web/pandas_web.py web/pandas --target-path=web/build
-
     - name: Build documentation
       run: |
         source activate pandas-dev
@@ -132,3 +138,22 @@ jobs:
     - name: Upload dev docs
       run: rsync -az --delete doc/build/html/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas/pandas-docs/dev
       if: github.event_name == 'push'
+
+  data_manager:
+    name: Test experimental data manager
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Setting conda path
+      run: echo "${HOME}/miniconda3/bin" >> $GITHUB_PATH
+
+    - name: Checkout
+      uses: actions/checkout@v1
+
+    - name: Setup environment and build pandas
+      run: ci/setup_env.sh
+
+    - name: Run tests
+      run: |
+        source activate pandas-dev
+        pytest pandas/tests/frame/methods --array-manager

.pre-commit-config.yaml

@@ -20,11 +20,9 @@ repos:
         types: [text]
         args: [--append-config=flake8/cython-template.cfg]
 -   repo: https://github.com/PyCQA/isort
-    rev: 5.6.4
+    rev: 5.7.0
     hooks:
     -   id: isort
-        types: [text] # overwrite upstream `types: [python]`
-        types_or: [python, cython]
 -   repo: https://github.com/asottile/pyupgrade
     rev: v2.7.4
     hooks:
@@ -128,7 +126,7 @@ repos:
         entry: python scripts/validate_unwanted_patterns.py --validation-type="bare_pytest_raises"
         types: [python]
         files: ^pandas/tests/
-        exclude: ^pandas/tests/(computation|extension|io)/
+        exclude: ^pandas/tests/extension/
     -   id: inconsistent-namespace-usage
         name: 'Check for inconsistent use of pandas namespace in tests'
         entry: python scripts/check_for_inconsistent_pandas_namespace.py
@@ -154,6 +152,11 @@ repos:
             \#\ type:\s?ignore(?!\[)
         language: pygrep
         types: [python]
+    -   id: np-bool
+        name: Check for use of np.bool instead of np.bool_
+        entry: np\.bool[^_8]
+        language: pygrep
+        types_or: [python, cython, rst]
     -   id: no-os-remove
         name: Check code for instances of os.remove
         entry: os\.remove

asv_bench/benchmarks/groupby.py

@@ -627,7 +627,7 @@ class TransformBools:
     def setup(self):
         N = 120000
         transition_points = np.sort(np.random.choice(np.arange(N), 1400))
-        transitions = np.zeros(N, dtype=np.bool)
+        transitions = np.zeros(N, dtype=np.bool_)
         transitions[transition_points] = True
         self.g = transitions.cumsum()
         self.df = DataFrame({"signal": np.random.rand(N)})

asv_bench/benchmarks/io/excel.py

@@ -43,6 +43,7 @@ class ReadExcel:
     params = ["xlrd", "openpyxl", "odf"]
     param_names = ["engine"]
     fname_excel = "spreadsheet.xlsx"
+    fname_excel_xls = "spreadsheet.xls"
     fname_odf = "spreadsheet.ods"
 
     def _create_odf(self):
@@ -63,10 +64,16 @@ def setup_cache(self):
         self.df = _generate_dataframe()
 
         self.df.to_excel(self.fname_excel, sheet_name="Sheet1")
+        self.df.to_excel(self.fname_excel_xls, sheet_name="Sheet1")
         self._create_odf()
 
     def time_read_excel(self, engine):
-        fname = self.fname_odf if engine == "odf" else self.fname_excel
+        if engine == "xlrd":
+            fname = self.fname_excel_xls
+        elif engine == "odf":
+            fname = self.fname_odf
+        else:
+            fname = self.fname_excel
         read_excel(fname, engine=engine)
 
 

ci/deps/actions-37-locale.yaml

@@ -30,7 +30,7 @@ dependencies:
   - openpyxl
   - pandas-gbq
   - google-cloud-bigquery>=1.27.2 # GH 36436
-  - pyarrow>=0.17
+  - pyarrow=0.17 # GH 38803
   - pytables>=3.5.1
   - scipy
   - xarray=0.12.3

ci/deps/azure-37-slow.yaml

@@ -36,3 +36,4 @@ dependencies:
   - xlwt
   - moto
   - flask
+  - numba

ci/deps/azure-38-slow.yaml

@@ -34,3 +34,4 @@ dependencies:
   - xlwt
   - moto
   - flask
+  - numba

doc/make.py

@@ -46,6 +46,7 @@ def __init__(
         warnings_are_errors=False,
     ):
         self.num_jobs = num_jobs
+        self.include_api = include_api
         self.verbosity = verbosity
         self.warnings_are_errors = warnings_are_errors
 
@@ -188,7 +189,14 @@ def _add_redirects(self):
                 if not row or row[0].strip().startswith("#"):
                     continue
 
-                path = os.path.join(BUILD_PATH, "html", *row[0].split("/")) + ".html"
+                html_path = os.path.join(BUILD_PATH, "html")
+                path = os.path.join(html_path, *row[0].split("/")) + ".html"
+
+                if not self.include_api and (
+                    os.path.join(html_path, "reference") in path
+                    or os.path.join(html_path, "generated") in path
+                ):
+                    continue
 
                 try:
                     title = self._get_page_title(row[1])
@@ -198,11 +206,6 @@ def _add_redirects(self):
                     # sphinx specific stuff
                     title = "this page"
 
-                if os.path.exists(path):
-                    raise RuntimeError(
-                        f"Redirection would overwrite an existing file: {path}"
-                    )
-
                 with open(path, "w") as moved_page_fd:
                     html = f"""\
 <html>

doc/source/_static/spreadsheets/conditional.png

@@ -77,29 +77,32 @@
 try:
     import nbconvert
 except ImportError:
-    logger.warn("nbconvert not installed. Skipping notebooks.")
+    logger.warning("nbconvert not installed. Skipping notebooks.")
     exclude_patterns.append("**/*.ipynb")
 else:
     try:
         nbconvert.utils.pandoc.get_pandoc_version()
     except nbconvert.utils.pandoc.PandocMissing:
-        logger.warn("Pandoc not installed. Skipping notebooks.")
+        logger.warning("Pandoc not installed. Skipping notebooks.")
         exclude_patterns.append("**/*.ipynb")
 
 # sphinx_pattern can be '-api' to exclude the API pages,
 # the path to a file, or a Python object
 # (e.g. '10min.rst' or 'pandas.DataFrame.head')
 source_path = os.path.dirname(os.path.abspath(__file__))
 pattern = os.environ.get("SPHINX_PATTERN")
+single_doc = pattern is not None and pattern != "-api"
+include_api = pattern != "-api"
 if pattern:
     for dirname, dirs, fnames in os.walk(source_path):
+        reldir = os.path.relpath(dirname, source_path)
         for fname in fnames:
             if os.path.splitext(fname)[-1] in (".rst", ".ipynb"):
                 fname = os.path.relpath(os.path.join(dirname, fname), source_path)
 
                 if fname == "index.rst" and os.path.abspath(dirname) == source_path:
                     continue
-                elif pattern == "-api" and dirname == "reference":
+                elif pattern == "-api" and reldir.startswith("reference"):
                     exclude_patterns.append(fname)
                 elif pattern != "-api" and fname != pattern:
                     exclude_patterns.append(fname)
@@ -109,11 +112,11 @@
 with open(os.path.join(source_path, "index.rst"), "w") as f:
     f.write(
         t.render(
-            include_api=pattern is None,
-            single_doc=(pattern if pattern is not None and pattern != "-api" else None),
+            include_api=include_api,
+            single_doc=(pattern if single_doc else None),
         )
     )
-autosummary_generate = True if pattern is None else ["index"]
+autosummary_generate = True if include_api else ["index"]
 autodoc_typehints = "none"
 
 # numpydoc
@@ -315,7 +318,7 @@
         # ... and each of its public methods
         moved_api_pages.append((f"{old}.{method}", f"{new}.{method}"))
 
-if pattern is None:
+if include_api:
     html_additional_pages = {
         "generated/" + page[0]: "api_redirect.html" for page in moved_api_pages
     }
@@ -411,7 +414,7 @@
 # latex_use_modindex = True
 
 
-if pattern is None:
+if include_api:
     intersphinx_mapping = {
         "dateutil": ("https://dateutil.readthedocs.io/en/latest/", None),
         "matplotlib": ("https://matplotlib.org/", None),
@@ -432,7 +435,7 @@
 
 
 ipython_warning_is_error = False
-ipython_exec_lines = [
+ipython_execlines = [
     "import numpy as np",
     "import pandas as pd",
     # This ensures correct rendering on system with console encoding != utf8

doc/source/_static/spreadsheets/filter.png

@@ -698,6 +698,12 @@ to run its checks with::
 
 without needing to have done ``pre-commit install`` beforehand.
 
+If you want to run checks on all recently commited files on upstream/master you can use::
+
+    pre-commit run --from-ref=upstream/master --to-ref=HEAD --all-files
+
+without needing to have done ``pre-commit install`` beforehand.
+
 .. note::
 
     If you have conflicting installations of ``virtualenv``, then you may get an

doc/source/_static/spreadsheets/find.png

@@ -85,6 +85,14 @@ Featuretools is a Python library for automated feature engineering built on top
 
 Compose is a machine learning tool for labeling data and prediction engineering. It allows you to structure the labeling process by parameterizing prediction problems and transforming time-driven relational data into target values with cutoff times that can be used for supervised learning.
 
+`STUMPY <https://github.com/TDAmeritrade/stumpy>`__
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+STUMPY is a powerful and scalable Python library for modern time series analysis.
+At its core, STUMPY efficiently computes something called a
+`matrix profile <https://stumpy.readthedocs.io/en/latest/Tutorial_The_Matrix_Profile.html>`__,
+which can be used for a wide variety of time series data mining tasks.
+
 .. _ecosystem.visualization:
 
 Visualization

doc/source/_static/logo_excel.svg renamed to doc/source/_static/spreadsheets/logo_excel.svg

@@ -62,6 +62,12 @@ see the :ref:`indexing documentation<indexing>` for much more on how to use an
 ``Index`` effectively.
 
 
+Copies vs. in place operations
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. include:: includes/copies.rst
+
+
 Data input / output
 -------------------