Merge remote-tracking branch 'upstream/master' into bisect

Author: simonjayhawkins

.github/workflows/ci.yml

@@ -125,7 +125,7 @@ jobs:
     # This can be removed when the ipython directive fails when there are errors,
     # including the `tee sphinx.log` in te previous step (https://github.com/ipython/ipython/issues/11547)
     - name: Check ipython directive errors
-      run: "! grep -B1 \"^<<<-------------------------------------------------------------------------$\" sphinx.log"
+      run: "! grep -B10 \"^<<<-------------------------------------------------------------------------$\" sphinx.log"
 
     - name: Install ssh key
       run: |

.github/workflows/stale-pr.yml

@@ -2,7 +2,7 @@ name: "Stale PRs"
 on:
   schedule:
   # * is a special character in YAML so you have to quote this string
-  - cron: "0 */6 * * *"
+  - cron: "0 0 * * *"
 
 jobs:
   stale:
@@ -11,8 +11,8 @@ jobs:
     - uses: actions/stale@v3
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
-        stale-pr-message: "This pull request is stale because it has been open for thirty days with no activity."
-        skip-stale-pr-message: true
+        stale-pr-message: "This pull request is stale because it has been open for thirty days with no activity. Please update or respond to this comment if you're still interested in working on this."
+        skip-stale-pr-message: false
         stale-pr-label: "Stale"
         exempt-pr-labels: "Needs Review,Blocked,Needs Discussion"
         days-before-stale: 30

.pre-commit-config.yaml

@@ -4,7 +4,7 @@ repos:
     hooks:
     -   id: black
 -   repo: https://gitlab.com/pycqa/flake8
-    rev: 3.8.3
+    rev: 3.8.4
     hooks:
     -   id: flake8
         additional_dependencies: [flake8-comprehensions>=3.1.0]
@@ -34,20 +34,6 @@ repos:
     rev: v1.6.0
     hooks:
       - id: rst-backticks
-        # these exclusions should be removed and the files fixed
-        exclude: (?x)(
-          categorical\.rst|
-          contributing\.rst|
-          contributing_docstring\.rst|
-          extending\.rst|
-          ecosystem\.rst|
-          comparison_with_sql\.rst|
-          install\.rst|
-          calculate_statistics\.rst|
-          combine_dataframes\.rst|
-          v0\.|
-          v1\.0\.|
-          v1\.1\.[012])
 -   repo: local
     hooks:
     -   id: pip_to_conda
@@ -57,3 +43,7 @@ repos:
         entry: python -m scripts.generate_pip_deps_from_conda
         files: ^(environment.yml|requirements-dev.txt)$
         pass_filenames: false
+-   repo: https://github.com/asottile/yesqa
+    rev: v1.2.2
+    hooks:
+    -   id: yesqa

.travis.yml

@@ -46,16 +46,16 @@ matrix:
     - env:
         - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" PATTERN="(not slow and not network and not clipboard)"
 
-    - arch: arm64
-      env:
-        - JOB="3.7, arm64" PYTEST_WORKERS=1 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)"
-
     - env:
         - JOB="3.7, locale" ENV_FILE="ci/deps/travis-37-locale.yaml" PATTERN="((not slow and not network and not clipboard) or (single and db))" LOCALE_OVERRIDE="zh_CN.UTF-8" SQL="1"
       services:
         - mysql
         - postgresql
 
+    - arch: arm64
+      env:
+        - JOB="3.7, arm64" PYTEST_WORKERS=1 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)"
+
     - env:
         # Enabling Deprecations when running tests
         # PANDAS_TESTING_MODE="deprecate" causes DeprecationWarning messages to be displayed in the logs
@@ -65,6 +65,12 @@ matrix:
         - mysql
         - postgresql
 
+  allow_failures:
+    # Moved to allowed_failures 2020-09-29 due to timeouts https://github.com/pandas-dev/pandas/issues/36719
+    - arch: arm64
+      env:
+        - JOB="3.7, arm64" PYTEST_WORKERS=1 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)"
+
 
 before_install:
   - echo "before_install"

asv_bench/benchmarks/dtypes.py

@@ -1,5 +1,9 @@
+import string
+
 import numpy as np
 
+from pandas import DataFrame
+import pandas._testing as tm
 from pandas.api.types import pandas_dtype
 
 from .pandas_vb_common import (
@@ -62,4 +66,57 @@ def time_infer(self, dtype):
         lib.infer_dtype(self.data_dict[dtype], skipna=False)
 
 
+class SelectDtypes:
+
+    params = [
+        tm.ALL_INT_DTYPES
+        + tm.ALL_EA_INT_DTYPES
+        + tm.FLOAT_DTYPES
+        + tm.COMPLEX_DTYPES
+        + tm.DATETIME64_DTYPES
+        + tm.TIMEDELTA64_DTYPES
+        + tm.BOOL_DTYPES
+    ]
+    param_names = ["dtype"]
+
+    def setup(self, dtype):
+        N, K = 5000, 50
+        self.index = tm.makeStringIndex(N)
+        self.columns = tm.makeStringIndex(K)
+
+        def create_df(data):
+            return DataFrame(data, index=self.index, columns=self.columns)
+
+        self.df_int = create_df(np.random.randint(low=100, size=(N, K)))
+        self.df_float = create_df(np.random.randn(N, K))
+        self.df_bool = create_df(np.random.choice([True, False], size=(N, K)))
+        self.df_string = create_df(
+            np.random.choice(list(string.ascii_letters), size=(N, K))
+        )
+
+    def time_select_dtype_int_include(self, dtype):
+        self.df_int.select_dtypes(include=dtype)
+
+    def time_select_dtype_int_exclude(self, dtype):
+        self.df_int.select_dtypes(exclude=dtype)
+
+    def time_select_dtype_float_include(self, dtype):
+        self.df_float.select_dtypes(include=dtype)
+
+    def time_select_dtype_float_exclude(self, dtype):
+        self.df_float.select_dtypes(exclude=dtype)
+
+    def time_select_dtype_bool_include(self, dtype):
+        self.df_bool.select_dtypes(include=dtype)
+
+    def time_select_dtype_bool_exclude(self, dtype):
+        self.df_bool.select_dtypes(exclude=dtype)
+
+    def time_select_dtype_string_include(self, dtype):
+        self.df_string.select_dtypes(include=dtype)
+
+    def time_select_dtype_string_exclude(self, dtype):
+        self.df_string.select_dtypes(exclude=dtype)
+
+
 from .pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/indexing.py

@@ -191,7 +191,7 @@ def setup(self, index):
         }
         index = indexes[index]
         self.s = Series(np.random.rand(N), index=index)
-        self.indexer = [True, False, True, True, False] * 20000
+        self.indexer = np.random.randint(0, N, size=N)
 
     def time_take(self, index):
         self.s.take(self.indexer)

asv_bench/benchmarks/pandas_vb_common.py

@@ -15,7 +15,7 @@
 
 # Compatibility import for the testing module
 try:
-    import pandas._testing as tm  # noqa
+    import pandas._testing as tm
 except ImportError:
     import pandas.util.testing as tm  # noqa
 

asv_bench/benchmarks/tslibs/offsets.py

@@ -9,7 +9,7 @@
 from pandas import offsets
 
 try:
-    import pandas.tseries.holiday  # noqa
+    import pandas.tseries.holiday
 except ImportError:
     pass
 

ci/code_checks.sh

@@ -335,7 +335,7 @@ if [[ -z "$CHECK" || "$CHECK" == "doctests" ]]; then
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     MSG='Doctests strings.py' ; echo $MSG
-    pytest -q --doctest-modules pandas/core/strings.py
+    pytest -q --doctest-modules pandas/core/strings/
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     # Directories

ci/deps/azure-37-minimum_versions.yaml

@@ -20,7 +20,7 @@ dependencies:
   - numexpr=2.6.8
   - numpy=1.16.5
   - openpyxl=2.6.0
-  - pytables=3.4.4
+  - pytables=3.5.1
   - python-dateutil=2.7.3
   - pytz=2017.3
   - pyarrow=0.15

ci/deps/travis-37-locale.yaml

@@ -13,7 +13,7 @@ dependencies:
 
   # pandas dependencies
   - beautifulsoup4
-  - blosc=1.14.3
+  - blosc=1.15.0
   - python-blosc
   - fastparquet=0.3.2
   - html5lib
@@ -30,7 +30,7 @@ dependencies:
   - pyarrow>=0.17
   - psycopg2=2.7
   - pymysql=0.7.11
-  - pytables
+  - pytables>=3.5.1
   - python-dateutil
   - pytz
   - scipy

codecov.yml

@@ -1,7 +1,7 @@
 codecov:
   branch: master
 
-comment: off
+comment: false
 
 coverage:
   status:
@@ -11,3 +11,6 @@ coverage:
     patch:
       default:
         target: '50'
+
+github_checks:
+    annotations: false

doc/source/conf.py

@@ -146,7 +146,7 @@
 # built documents.
 #
 # The short X.Y version.
-import pandas  # noqa: E402 isort:skip
+import pandas  # isort:skip
 
 # version = '%s r%s' % (pandas.__version__, svn_version())
 version = str(pandas.__version__)
@@ -441,14 +441,14 @@
 # Add custom Documenter to handle attributes/methods of an AccessorProperty
 # eg pandas.Series.str and pandas.Series.dt (see GH9322)
 
-import sphinx  # noqa: E402 isort:skip
-from sphinx.util import rpartition  # noqa: E402 isort:skip
-from sphinx.ext.autodoc import (  # noqa: E402 isort:skip
+import sphinx  # isort:skip
+from sphinx.util import rpartition  # isort:skip
+from sphinx.ext.autodoc import (  # isort:skip
     AttributeDocumenter,
     Documenter,
     MethodDocumenter,
 )
-from sphinx.ext.autosummary import Autosummary  # noqa: E402 isort:skip
+from sphinx.ext.autosummary import Autosummary  # isort:skip
 
 
 class AccessorDocumenter(MethodDocumenter):

doc/source/development/code_style.rst

@@ -9,7 +9,7 @@ pandas code style guide
 .. contents:: Table of contents:
    :local:
 
-*pandas* follows the `PEP8 <https://www.python.org/dev/peps/pep-0008/>`_
+pandas follows the `PEP8 <https://www.python.org/dev/peps/pep-0008/>`_
 standard and uses `Black <https://black.readthedocs.io/en/stable/>`_
 and `Flake8 <https://flake8.pycqa.org/en/latest/>`_ to ensure a
 consistent code format throughout the project. For details see the
@@ -172,5 +172,6 @@ Reading from a url
 .. code-block:: python
 
     from pandas.io.common import urlopen
-    with urlopen('http://www.google.com') as url:
+
+    with urlopen("http://www.google.com") as url:
         raw_text = url.read()

doc/source/development/contributing.rst

@@ -31,13 +31,13 @@ comment letting others know they are working on an issue. While this is ok, you
 check each issue individually, and it's not possible to find the unassigned ones.
 
 For this reason, we implemented a workaround consisting of adding a comment with the exact
-text `take`. When you do it, a GitHub action will automatically assign you the issue
+text ``take``. When you do it, a GitHub action will automatically assign you the issue
 (this will take seconds, and may require refreshing the page to see it).
 By doing this, it's possible to filter the list of issues and find only the unassigned ones.
 
 So, a good way to find an issue to start contributing to pandas is to check the list of
 `unassigned good first issues <https://github.com/pandas-dev/pandas/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22+no%3Aassignee>`_
-and assign yourself one you like by writing a comment with the exact text `take`.
+and assign yourself one you like by writing a comment with the exact text ``take``.
 
 If for whatever reason you are not able to continue working with the issue, please try to
 unassign it, so other people know it's available again. You can check the list of
@@ -133,7 +133,7 @@ want to clone your fork to your machine::
     cd pandas-yourname
     git remote add upstream https://github.com/pandas-dev/pandas.git
 
-This creates the directory `pandas-yourname` and connects your repository to
+This creates the directory ``pandas-yourname`` and connects your repository to
 the upstream (main project) *pandas* repository.
 
 Note that performing a shallow clone (with ``--depth==N``, for some ``N`` greater
@@ -155,12 +155,12 @@ Using a Docker container
 
 Instead of manually setting up a development environment, you can use `Docker
 <https://docs.docker.com/get-docker/>`_ to automatically create the environment with just several
-commands. Pandas provides a `DockerFile` in the root directory to build a Docker image
+commands. pandas provides a ``DockerFile`` in the root directory to build a Docker image
 with a full pandas development environment.
 
 **Docker Commands**
 
-Pass your GitHub username in the `DockerFile` to use your own fork::
+Pass your GitHub username in the ``DockerFile`` to use your own fork::
 
     # Build the image pandas-yourname-env
     docker build --tag pandas-yourname-env .
@@ -172,7 +172,7 @@ Even easier, you can integrate Docker with the following IDEs:
 **Visual Studio Code**
 
 You can use the DockerFile to launch a remote session with Visual Studio Code,
-a popular free IDE, using the `.devcontainer.json` file.
+a popular free IDE, using the ``.devcontainer.json`` file.
 See https://code.visualstudio.com/docs/remote/containers for details.
 
 **PyCharm (Professional)**
@@ -190,7 +190,7 @@ Note that you might need to rebuild the C extensions if/when you merge with upst
 Installing a C compiler
 ~~~~~~~~~~~~~~~~~~~~~~~
 
-Pandas uses C extensions (mostly written using Cython) to speed up certain
+pandas uses C extensions (mostly written using Cython) to speed up certain
 operations. To install pandas from source, you need to compile these C
 extensions, which means you need a C compiler. This process depends on which
 platform you're using.
@@ -782,7 +782,7 @@ As part of :ref:`Continuous Integration <contributing.ci>` checks we run::
 
     isort --check-only pandas
 
-to check that imports are correctly formatted as per the `setup.cfg`.
+to check that imports are correctly formatted as per the ``setup.cfg``.
 
 If you see output like the below in :ref:`Continuous Integration <contributing.ci>` checks:
 
@@ -979,7 +979,7 @@ For example, quite a few functions in pandas accept a ``dtype`` argument. This c
    def as_type(dtype: Dtype) -> ...:
        ...
 
-This module will ultimately house types for repeatedly used concepts like "path-like", "array-like", "numeric", etc... and can also hold aliases for commonly appearing parameters like `axis`. Development of this module is active so be sure to refer to the source for the most up to date list of available types.
+This module will ultimately house types for repeatedly used concepts like "path-like", "array-like", "numeric", etc... and can also hold aliases for commonly appearing parameters like ``axis``. Development of this module is active so be sure to refer to the source for the most up to date list of available types.
 
 Validating type hints
 ~~~~~~~~~~~~~~~~~~~~~
@@ -1219,7 +1219,7 @@ This test shows off several useful features of Hypothesis, as well as
 demonstrating a good use-case: checking properties that should hold over
 a large or complicated domain of inputs.
 
-To keep the Pandas test suite running quickly, parametrized tests are
+To keep the pandas test suite running quickly, parametrized tests are
 preferred if the inputs or logic are simple, with Hypothesis tests reserved
 for cases with complex logic or where there are too many combinations of
 options or subtle interactions to test (or think of!) all of them.
@@ -1302,7 +1302,7 @@ Or with one of the following constructs::
 
 Using `pytest-xdist <https://pypi.org/project/pytest-xdist>`_, one can
 speed up local testing on multicore machines. To use this feature, you will
-need to install `pytest-xdist` via::
+need to install ``pytest-xdist`` via::
 
     pip install pytest-xdist
 
@@ -1465,7 +1465,7 @@ The following defines how a commit message should be structured.  Please referen
 relevant GitHub issues in your commit message using GH1234 or #1234.  Either style
 is fine, but the former is generally preferred:
 
-* a subject line with `< 80` chars.
+* a subject line with ``< 80`` chars.
 * One blank line.
 * Optionally, a commit message body.