Merge remote-tracking branch 'upstream/master' into bisect

Author: simonjayhawkins

.github/actions/build_pandas/action.yml

@@ -13,5 +13,5 @@ runs:
     - name: Build Pandas
       run: |
         python setup.py build_ext -j 2
-        python -m pip install -e . --no-build-isolation --no-use-pep517
+        python -m pip install -e . --no-build-isolation --no-use-pep517 --no-index
       shell: bash -l {0}

.github/workflows/ci.yml

@@ -45,6 +45,15 @@ jobs:
         environment-file: ${{ env.ENV_FILE }}
         use-only-tar-bz2: true
 
+    - name: Install node.js (for pyright)
+      uses: actions/setup-node@v2
+      with:
+        node-version: "16"
+
+    - name: Install pyright
+      # note: keep version in sync with .pre-commit-config.yaml
+      run: npm install -g [email protected]
+
     - name: Build Pandas
       uses: ./.github/actions/build_pandas
 
@@ -127,8 +136,11 @@ jobs:
         echo "${{ secrets.server_ip }} ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBE1Kkopomm7FHG5enATf7SgnpICZ4W2bw+Ho+afqin+w7sMcrsa0je7sbztFAV8YchDkiBKnWTG4cRT+KZgZCaY=" > ~/.ssh/known_hosts
       if: ${{github.event_name == 'push' && github.ref == 'refs/heads/master'}}
 
+    - name: Copy cheatsheets into site directory
+      run: cp doc/cheatsheet/Pandas_Cheat_Sheet* web/build/
+
     - name: Upload web
-      run: rsync -az --delete --exclude='pandas-docs' --exclude='docs' --exclude='Pandas_Cheat_Sheet*' web/build/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas
+      run: rsync -az --delete --exclude='pandas-docs' --exclude='docs' web/build/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas
       if: ${{github.event_name == 'push' && github.ref == 'refs/heads/master'}}
 
     - name: Upload dev docs
@@ -137,6 +149,7 @@ jobs:
 
     - name: Move docs into site directory
       run: mv doc/build/html web/build/docs
+
     - name: Save website as an artifact
       uses: actions/upload-artifact@v2
       with:

.github/workflows/database.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - master
+      - 1.3.x
   pull_request:
     branches:
       - master
@@ -79,7 +80,7 @@ jobs:
     - uses: conda-incubator/setup-miniconda@v2
       with:
         activate-environment: pandas-dev
-        channel-priority: flexible
+        channel-priority: strict
         environment-file: ${{ matrix.ENV_FILE }}
         use-only-tar-bz2: true
 

.github/workflows/pre-commit.yml

@@ -16,4 +16,6 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - uses: actions/setup-python@v2
+      with:
+        python-version: '3.9.7'
     - uses: pre-commit/[email protected]

.github/workflows/python-dev.yml

@@ -17,16 +17,25 @@ env:
   PANDAS_CI: 1
   PATTERN: "not slow and not network and not clipboard"
   COVERAGE: true
-  PYTEST_TARGET:  pandas
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macOS-latest]
+        pytest_target: ["pandas/tests/[a-h]*", "pandas/tests/[i-z]*"]
+        include:
+          # No need to split tests on windows
+          - os: windows-latest
+            pytest_target: pandas
+
     name: actions-310-dev
-    timeout-minutes: 60
+    timeout-minutes: 80
 
     concurrency:
-      group: ${{ github.ref }}-dev
+      group: ${{ github.ref }}-${{ matrix.os }}-${{ matrix.pytest_target }}-dev
       cancel-in-progress: ${{github.event_name == 'pull_request'}}
 
     steps:
@@ -40,12 +49,12 @@ jobs:
         python-version: '3.10-dev'
 
     - name: Install dependencies
+      shell: bash
       run: |
         python -m pip install --upgrade pip setuptools wheel
         pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy
-        pip install git+https://github.com/pytest-dev/pytest.git
         pip install git+https://github.com/nedbat/coveragepy.git
-        pip install cython python-dateutil pytz hypothesis pytest-xdist pytest-cov
+        pip install cython python-dateutil pytz hypothesis pytest>=6.2.5 pytest-xdist pytest-cov
         pip list
 
     - name: Build Pandas
@@ -58,10 +67,11 @@ jobs:
         python -c "import pandas; pandas.show_versions();"
 
     - name: Test with pytest
+      env:
+          PYTEST_TARGET: ${{ matrix.pytest_target }}
+      shell: bash
       run: |
         ci/run_tests.sh
-      # GH 41935
-      continue-on-error: true
 
     - name: Publish test results
       uses: actions/upload-artifact@master

.github/workflows/sdist.yml

@@ -23,7 +23,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9"]
+        python-version: ["3.8", "3.9", "3.10"]
     concurrency:
       group: ${{github.ref}}-${{matrix.python-version}}-sdist
       cancel-in-progress: ${{github.event_name == 'pull_request'}}
@@ -53,13 +53,24 @@ jobs:
     - uses: conda-incubator/setup-miniconda@v2
       with:
         activate-environment: pandas-sdist
-        python-version: ${{ matrix.python-version }}
+        python-version: '${{ matrix.python-version }}'
 
     - name: Install pandas from sdist
       run: |
-        conda list
+        pip list
         python -m pip install dist/*.gz
 
+    - name: Force oldest supported NumPy
+      run: |
+        case "${{matrix.python-version}}" in
+        3.8)
+          pip install numpy==1.18.5 ;;
+        3.9)
+          pip install numpy==1.19.3 ;;
+        3.10)
+          pip install numpy==1.21.2 ;;
+        esac
+
     - name: Import pandas
       run: |
         cd ..

.pre-commit-config.yaml

@@ -9,7 +9,7 @@ repos:
     -   id: absolufy-imports
         files: ^pandas/
 -   repo: https://github.com/python/black
-    rev: 21.7b0
+    rev: 21.9b0
     hooks:
     -   id: black
 -   repo: https://github.com/codespell-project/codespell
@@ -39,10 +39,11 @@ repos:
     rev: 3.9.2
     hooks:
     -   id: flake8
-        additional_dependencies:
-            - flake8-comprehensions==3.1.0
-            - flake8-bugbear==21.3.2
-            - pandas-dev-flaker==0.2.0
+        additional_dependencies: &flake8_dependencies
+        - flake8==3.9.2
+        - flake8-comprehensions==3.1.0
+        - flake8-bugbear==21.3.2
+        - pandas-dev-flaker==0.2.0
     -   id: flake8
         alias: flake8-cython
         name: flake8 (cython)
@@ -58,7 +59,7 @@ repos:
     hooks:
     -   id: isort
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v2.23.3
+    rev: v2.29.0
     hooks:
     -   id: pyupgrade
         args: [--py38-plus]
@@ -76,11 +77,18 @@ repos:
     rev: v1.2.3
     hooks:
     -   id: yesqa
-        additional_dependencies:
-            - flake8==3.9.2
-            - flake8-comprehensions==3.1.0
-            - flake8-bugbear==21.3.2
-            - pandas-dev-flaker==0.2.0
+        additional_dependencies: *flake8_dependencies
+-   repo: local
+    hooks:
+    -   id: pyright
+        name: pyright
+        entry: pyright
+        language: node
+        pass_filenames: false
+        types: [python]
+        stages: [manual]
+        # note: keep version in sync with .github/workflows/ci.yml
+        additional_dependencies: ['[email protected]']
 -   repo: local
     hooks:
     -   id: flake8-rst
@@ -124,6 +132,12 @@ repos:
         entry: 'np\.random\.seed'
         files: ^asv_bench/benchmarks
         exclude: ^asv_bench/benchmarks/pandas_vb_common\.py
+    -   id: np-testing-array-equal
+        name: Check for usage of numpy testing or array_equal
+        language: pygrep
+        entry: '(numpy|np)(\.testing|\.array_equal)'
+        files: ^pandas/tests/
+        types: [python]
     -   id: invalid-ea-testing
         name: Check for invalid EA testing
         language: pygrep

asv_bench/benchmarks/groupby.py

@@ -603,6 +603,38 @@ def time_sum(self):
         self.df.groupby(["a"])["b"].sum()
 
 
+class String:
+    # GH#41596
+    param_names = ["dtype", "method"]
+    params = [
+        ["str", "string[python]"],
+        [
+            "sum",
+            "prod",
+            "min",
+            "max",
+            "mean",
+            "median",
+            "var",
+            "first",
+            "last",
+            "any",
+            "all",
+        ],
+    ]
+
+    def setup(self, dtype, method):
+        cols = list("abcdefghjkl")
+        self.df = DataFrame(
+            np.random.randint(0, 100, size=(1_000_000, len(cols))),
+            columns=cols,
+            dtype=dtype,
+        )
+
+    def time_str_func(self, dtype, method):
+        self.df.groupby("a")[self.df.columns[1:]].agg(method)
+
+
 class Categories:
     def setup(self):
         N = 10 ** 5

asv_bench/benchmarks/index_object.py

@@ -86,6 +86,12 @@ def time_iter_dec(self):
         for _ in self.idx_dec:
             pass
 
+    def time_sort_values_asc(self):
+        self.idx_inc.sort_values()
+
+    def time_sort_values_des(self):
+        self.idx_inc.sort_values(ascending=False)
+
 
 class IndexEquals:
     def setup(self):

asv_bench/benchmarks/indexing_engines.py

@@ -1,5 +1,5 @@
 """
-Benchmarks in this fiel depend exclusively on code in _libs/
+Benchmarks in this file depend exclusively on code in _libs/
 
 If a PR does not edit anything in _libs, it is very unlikely that benchmarks
 in this file will be affected.
@@ -35,25 +35,49 @@ class NumericEngineIndexing:
     params = [
         _get_numeric_engines(),
         ["monotonic_incr", "monotonic_decr", "non_monotonic"],
+        [True, False],
+        [10 ** 5, 2 * 10 ** 6],  # 2e6 is above SIZE_CUTOFF
     ]
-    param_names = ["engine_and_dtype", "index_type"]
+    param_names = ["engine_and_dtype", "index_type", "unique", "N"]
 
-    def setup(self, engine_and_dtype, index_type):
+    def setup(self, engine_and_dtype, index_type, unique, N):
         engine, dtype = engine_and_dtype
-        N = 10 ** 5
-        values = list([1] * N + [2] * N + [3] * N)
-        arr = {
-            "monotonic_incr": np.array(values, dtype=dtype),
-            "monotonic_decr": np.array(list(reversed(values)), dtype=dtype),
-            "non_monotonic": np.array([1, 2, 3] * N, dtype=dtype),
-        }[index_type]
+
+        if index_type == "monotonic_incr":
+            if unique:
+                arr = np.arange(N * 3, dtype=dtype)
+            else:
+                values = list([1] * N + [2] * N + [3] * N)
+                arr = np.array(values, dtype=dtype)
+        elif index_type == "monotonic_decr":
+            if unique:
+                arr = np.arange(N * 3, dtype=dtype)[::-1]
+            else:
+                values = list([1] * N + [2] * N + [3] * N)
+                arr = np.array(values, dtype=dtype)[::-1]
+        else:
+            assert index_type == "non_monotonic"
+            if unique:
+                arr = np.empty(N * 3, dtype=dtype)
+                arr[:N] = np.arange(N * 2, N * 3, dtype=dtype)
+                arr[N:] = np.arange(N * 2, dtype=dtype)
+            else:
+                arr = np.array([1, 2, 3] * N, dtype=dtype)
 
         self.data = engine(arr)
         # code belows avoids populating the mapping etc. while timing.
         self.data.get_loc(2)
 
-    def time_get_loc(self, engine_and_dtype, index_type):
-        self.data.get_loc(2)
+        self.key_middle = arr[len(arr) // 2]
+        self.key_early = arr[2]
+
+    def time_get_loc(self, engine_and_dtype, index_type, unique, N):
+        self.data.get_loc(self.key_early)
+
+    def time_get_loc_near_middle(self, engine_and_dtype, index_type, unique, N):
+        # searchsorted performance may be different near the middle of a range
+        #  vs near an endpoint
+        self.data.get_loc(self.key_middle)
 
 
 class ObjectEngineIndexing:

asv_bench/benchmarks/inference.py

@@ -277,6 +277,16 @@ def time_dup_string_tzoffset_dates(self, cache):
         to_datetime(self.dup_string_with_tz, cache=cache)
 
 
+# GH 43901
+class ToDatetimeInferDatetimeFormat:
+    def setup(self):
+        rng = date_range(start="1/1/2000", periods=100000, freq="H")
+        self.strings = rng.strftime("%Y-%m-%d %H:%M:%S").tolist()
+
+    def time_infer_datetime_format(self):
+        to_datetime(self.strings, infer_datetime_format=True)
+
+
 class ToTimedelta:
     def setup(self):
         self.ints = np.random.randint(0, 60, size=10000)