Merge remote-tracking branch 'upstream/master' into DOC-Fix-algo-index-nan

Author: ShaharNaveh

.pre-commit-config.yaml

@@ -1,5 +1,7 @@
 minimum_pre_commit_version: 2.9.2
 exclude: ^LICENSES/|\.(html|csv|svg)$
+ci:
+    autofix_prs: false
 repos:
 -   repo: https://github.com/MarcoGorelli/absolufy-imports
     rev: v0.3.0
@@ -36,7 +38,10 @@ repos:
     rev: 3.9.0
     hooks:
     -   id: flake8
-        additional_dependencies: [flake8-comprehensions>=3.1.0, flake8-bugbear>=21.3.2]
+        additional_dependencies:
+            - flake8-comprehensions==3.1.0
+            - flake8-bugbear==21.3.2
+            - pandas-dev-flaker==0.2.0
     -   id: flake8
         name: flake8 (cython)
         types: [cython]
@@ -69,7 +74,11 @@ repos:
     rev: v1.2.2
     hooks:
     -   id: yesqa
-        additional_dependencies: [flake8==3.9.0]
+        additional_dependencies:
+            - flake8==3.9.0
+            - flake8-comprehensions==3.1.0
+            - flake8-bugbear==21.3.2
+            - pandas-dev-flaker==0.2.0
 -   repo: local
     hooks:
     -   id: flake8-rst
@@ -80,28 +89,6 @@ repos:
         types: [rst]
         args: [--filename=*.rst]
         additional_dependencies: [flake8-rst==0.7.0, flake8==3.7.9]
-    -   id: frame-or-series-union
-        name: Check for use of Union[Series, DataFrame] instead of FrameOrSeriesUnion alias
-        entry: Union\[.*(Series,.*DataFrame|DataFrame,.*Series).*\]
-        language: pygrep
-        types: [python]
-        exclude: ^pandas/_typing\.py$
-    -   id: inconsistent-namespace-usage
-        name: 'Check for inconsistent use of pandas namespace'
-        entry: python scripts/check_for_inconsistent_pandas_namespace.py
-        language: python
-        types: [python]
-    -   id: no-os-remove
-        name: Check code for instances of os.remove
-        entry: os\.remove
-        language: pygrep
-        types: [python]
-        files: ^pandas/tests/
-        exclude: |
-            (?x)^
-            pandas/tests/io/excel/test_writers\.py
-            |pandas/tests/io/pytables/common\.py
-            |pandas/tests/io/pytables/test_store\.py$
     -   id: unwanted-patterns
         name: Unwanted patterns
         language: pygrep
@@ -111,52 +98,10 @@ repos:
             \#\ type:\ (?!ignore)
             |\#\ type:\s?ignore(?!\[)
 
-            # foo._class__ instead of type(foo)
-            |\.__class__
-
-            # np.bool/np.object instead of np.bool_/np.object_
-            |np\.bool[^_8]
-            |np\.object[^_8]
-
-            # imports from pandas.core.common instead of `import pandas.core.common as com`
-            |from\ pandas\.core\.common\ import
-            |from\ pandas\.core\ import\ common
-
-            # imports from collections.abc instead of `from collections import abc`
-            |from\ collections\.abc\ import
-
-            # Numpy
-            |from\ numpy\ import\ random
-            |from\ numpy\.random\ import
-
             # Incorrect code-block / IPython directives
             |\.\.\ code-block\ ::
             |\.\.\ ipython\ ::
         types_or: [python, cython, rst]
-        exclude: ^doc/source/development/code_style\.rst  # contains examples of patterns to avoid
-    -   id: unwanted-patterns-in-tests
-        name: Unwanted patterns in tests
-        language: pygrep
-        entry: |
-            (?x)
-            # pytest.xfail instead of pytest.mark.xfail
-            pytest\.xfail
-
-            # imports from pandas._testing instead of `import pandas._testing as tm`
-            |from\ pandas\._testing\ import
-            |from\ pandas\ import\ _testing\ as\ tm
-
-            # No direct imports from conftest
-            |conftest\ import
-            |import\ conftest
-
-            # pandas.testing instead of tm
-            |pd\.testing\.
-
-            # pd.api.types instead of from pandas.api.types import ...
-            |(pd|pandas)\.api\.types\.
-        files: ^pandas/tests/
-        types_or: [python, cython, rst]
     -   id: pip-to-conda
         name: Generate pip dependency from conda
         description: This hook checks if the conda environment.yml and requirements-dev.txt are equal
@@ -178,35 +123,6 @@ repos:
         language: python
         types: [rst]
         files: ^doc/source/(development|reference)/
-    -   id: unwanted-patterns-bare-pytest-raises
-        name: Check for use of bare pytest raises
-        language: python
-        entry: python scripts/validate_unwanted_patterns.py --validation-type="bare_pytest_raises"
-        types: [python]
-        files: ^pandas/tests/
-        exclude: ^pandas/tests/extension/
-    -   id: unwanted-patterns-private-function-across-module
-        name: Check for use of private functions across modules
-        language: python
-        entry: python scripts/validate_unwanted_patterns.py --validation-type="private_function_across_module"
-        types: [python]
-        exclude: ^(asv_bench|pandas/tests|doc)/
-    -   id: unwanted-patterns-private-import-across-module
-        name: Check for import of private attributes across modules
-        language: python
-        entry: python scripts/validate_unwanted_patterns.py --validation-type="private_import_across_module"
-        types: [python]
-        exclude: ^(asv_bench|pandas/tests|doc)/
-    -   id: unwanted-patterns-strings-to-concatenate
-        name: Check for use of not concatenated strings
-        language: python
-        entry: python scripts/validate_unwanted_patterns.py --validation-type="strings_to_concatenate"
-        types_or: [python, cython]
-    -   id: unwanted-patterns-strings-with-wrong-placed-whitespace
-        name: Check for strings with wrong placed spaces
-        language: python
-        entry: python scripts/validate_unwanted_patterns.py --validation-type="strings_with_wrong_placed_whitespace"
-        types_or: [python, cython]
     -   id: use-pd_array-in-core
         name: Import pandas.array as pd_array in core
         language: python

.travis.yml

@@ -35,16 +35,11 @@ matrix:
   fast_finish: true
 
   include:
-    - arch: arm64
+    - arch: arm64-graviton2
+      virt: lxd
+      group: edge
       env:
-        - JOB="3.7, arm64" PYTEST_WORKERS=1 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)"
-
-  allow_failures:
-    # Moved to allowed_failures 2020-09-29 due to timeouts https://github.com/pandas-dev/pandas/issues/36719
-    - arch: arm64
-      env:
-        - JOB="3.7, arm64" PYTEST_WORKERS=1 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)"
-
+        - JOB="3.7, arm64" PYTEST_WORKERS="auto" ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)"
 
 before_install:
   - echo "before_install"

asv_bench/benchmarks/gil.py

@@ -31,7 +31,7 @@
 except ImportError:
     from pandas import algos
 try:
-    from pandas._testing import test_parallel
+    from pandas._testing import test_parallel  # noqa: PDF014
 
     have_real_test_parallel = True
 except ImportError:

asv_bench/benchmarks/groupby.py

@@ -480,7 +480,19 @@ class GroupByCythonAgg:
     param_names = ["dtype", "method"]
     params = [
         ["float64"],
-        ["sum", "prod", "min", "max", "mean", "median", "var", "first", "last"],
+        [
+            "sum",
+            "prod",
+            "min",
+            "max",
+            "mean",
+            "median",
+            "var",
+            "first",
+            "last",
+            "any",
+            "all",
+        ],
     ]
 
     def setup(self, dtype, method):

asv_bench/benchmarks/pandas_vb_common.py

@@ -70,7 +70,7 @@ class BaseIO:
     def remove(self, f):
         """Remove created files"""
         try:
-            os.remove(f)
+            os.remove(f)  # noqa: PDF008
         except OSError:
             # On Windows, attempting to remove a file that is in use
             # causes an exception to be raised

ci/azure/windows.yml

@@ -8,7 +8,7 @@ jobs:
     vmImage: ${{ parameters.vmImage }}
   strategy:
     matrix:
-      py37_np16:
+      py37_np17:
         ENV_FILE: ci/deps/azure-windows-37.yaml
         CONDA_PY: "37"
         PATTERN: "not slow and not network"

ci/code_checks.sh

@@ -64,27 +64,6 @@ fi
 ### PATTERNS ###
 if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
 
-    MSG='Check for use of exec' ; echo $MSG
-    invgrep -R --include="*.py*" -E "[^a-zA-Z0-9_]exec\(" pandas
-    RET=$(($RET + $?)) ; echo $MSG "DONE"
-
-    MSG='Check for pytest warns' ; echo $MSG
-    invgrep -r -E --include '*.py' 'pytest\.warns' pandas/tests/
-    RET=$(($RET + $?)) ; echo $MSG "DONE"
-
-    MSG='Check for pytest raises without context' ; echo $MSG
-    invgrep -r -E --include '*.py' "[[:space:]] pytest.raises" pandas/tests/
-    RET=$(($RET + $?)) ; echo $MSG "DONE"
-
-    MSG='Check for use of builtin filter function' ; echo $MSG
-    invgrep -R --include="*.py" -P '(?<!def)[\(\s]filter\(' pandas
-    RET=$(($RET + $?)) ; echo $MSG "DONE"
-
-    # Check for the following code in testing: `np.testing` and `np.array_equal`
-    MSG='Check for invalid testing' ; echo $MSG
-    invgrep -r -E --include '*.py' --exclude testing.py '(numpy|np)(\.testing|\.array_equal)' pandas/tests/
-    RET=$(($RET + $?)) ; echo $MSG "DONE"
-
     # Check for the following code in the extension array base tests: `tm.assert_frame_equal` and `tm.assert_series_equal`
     MSG='Check for invalid EA testing' ; echo $MSG
     invgrep -r -E --include '*.py' --exclude base.py 'tm.assert_(series|frame)_equal' pandas/tests/extension/base
@@ -98,15 +77,6 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
     invgrep -R --include="*.rst" -E "[a-zA-Z0-9]\`\`?[a-zA-Z0-9]" doc/source/
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
-    # Check for the following code in testing: `unittest.mock`, `mock.Mock()` or `mock.patch`
-    MSG='Check that unittest.mock is not used (pytest builtin monkeypatch fixture should be used instead)' ; echo $MSG
-    invgrep -r -E --include '*.py' '(unittest(\.| import )mock|mock\.Mock\(\)|mock\.patch)' pandas/tests/
-    RET=$(($RET + $?)) ; echo $MSG "DONE"
-
-    MSG='Check for use of {foo!r} instead of {repr(foo)}' ; echo $MSG
-    invgrep -R --include=*.{py,pyx} '!r}' pandas
-    RET=$(($RET + $?)) ; echo $MSG "DONE"
-    echo $MSG "DONE"
 fi
 
 ### CODE ###

ci/deps/actions-37-minimum_versions.yaml

@@ -18,7 +18,7 @@ dependencies:
   - jinja2=2.10
   - numba=0.46.0
   - numexpr=2.6.8
-  - numpy=1.16.5
+  - numpy=1.17.3
   - openpyxl=3.0.0
   - pytables=3.5.1
   - python-dateutil=2.7.3

ci/deps/azure-macos-37.yaml

@@ -19,7 +19,7 @@ dependencies:
   - matplotlib=2.2.3
   - nomkl
   - numexpr
-  - numpy=1.16.5
+  - numpy=1.17.3
   - openpyxl
   - pyarrow=0.15.1
   - pytables

ci/deps/azure-windows-37.yaml

@@ -24,7 +24,7 @@ dependencies:
   - moto>=1.3.14
   - flask
   - numexpr
-  - numpy=1.16.*
+  - numpy=1.17.*
   - openpyxl
   - pyarrow=0.15
   - pytables

doc/_templates/sidebar-nav-bs.html

@@ -0,0 +1,9 @@
+<nav class="bd-links" id="bd-docs-nav" aria-label="Main navigation">
+  <div class="bd-toc-item active">
+    {% if pagename.startswith("reference") %}
+    {{ generate_nav_html("sidebar", maxdepth=4, collapse=True, includehidden=True, titles_only=True) }}
+    {% else %}
+    {{ generate_nav_html("sidebar", maxdepth=4, collapse=False, includehidden=True, titles_only=True) }}
+    {% endif %}
+  </div>
+</nav>