Merge branch 'main' into deprecate_stacklevel

Author: jreback

.circleci/config.yml

@@ -13,7 +13,7 @@ jobs:
       PANDAS_CI: "1"
     steps:
       - checkout
-      - run: ci/setup_env.sh
+      - run: .circleci/setup_env.sh
       - run: PATH=$HOME/miniconda3/envs/pandas-dev/bin:$HOME/miniconda3/condabin:$PATH ci/run_tests.sh
 
 workflows:

ci/setup_env.sh renamed to .circleci/setup_env.sh

@@ -51,8 +51,7 @@ echo
 echo "update conda"
 conda config --set ssl_verify false
 conda config --set quiet true --set always_yes true --set changeps1 false
-# TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941
-conda install -y -c conda-forge -n base 'mamba>=0.21.2' pip
+conda install -y -c conda-forge -n base 'mamba>=0.21.2' pip setuptools
 
 echo "conda info -a"
 conda info -a
@@ -67,8 +66,6 @@ echo "mamba env update --file=${ENV_FILE}"
 # See https://github.com/mamba-org/mamba/issues/633
 mamba create -q -n pandas-dev
 time mamba env update -n pandas-dev --file="${ENV_FILE}"
-# TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941
-mamba install -n pandas-dev 'setuptools<60'
 
 echo "conda list -n pandas-dev"
 conda list -n pandas-dev
@@ -104,6 +101,6 @@ echo "Build extensions"
 python setup.py build_ext -q -j3
 
 echo "Install pandas"
-python -m pip install --no-build-isolation -e .
+python -m pip install --no-build-isolation --no-use-pep517 -e .
 
 echo "done"

.github/actions/setup-conda/action.yml

@@ -0,0 +1,27 @@
+name: Set up Conda environment
+inputs:
+  environment-file:
+    description: Conda environment file to use.
+    default: environment.yml
+  pyarrow-version:
+    description: If set, overrides the PyArrow version in the Conda environment to the given string.
+    required: false
+runs:
+  using: composite
+  steps:
+    - name: Set Arrow version in ${{ inputs.environment-file }} to ${{ inputs.pyarrow-version }}
+      run: |
+        grep -q '  - pyarrow' ${{ inputs.environment-file }}
+        sed -i"" -e "s/  - pyarrow/  - pyarrow=${{ inputs.pyarrow-version }}/" ${{ inputs.environment-file }}
+        cat ${{ inputs.environment-file }}
+      shell: bash
+      if: ${{ inputs.pyarrow-version }}
+
+    - name: Install ${{ inputs.environment-file }}
+      uses: conda-incubator/setup-miniconda@v2
+      with:
+        environment-file: ${{ inputs.environment-file }}
+        channel-priority: ${{ runner.os == 'macOS' && 'flexible' || 'strict' }}
+        channels: conda-forge
+        mamba-version: "0.23"
+        use-mamba: true

.github/actions/setup/action.yml

@@ -0,0 +1,49 @@
+name: 32 Bit Linux
+
+on:
+  push:
+    branches:
+      - main
+      - 1.4.x
+  pull_request:
+    branches:
+      - main
+      - 1.4.x
+    paths-ignore:
+      - "doc/**"
+
+jobs:
+  pytest:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Run 32-bit manylinux2014 Docker Build / Tests
+        run: |
+          # Without this (line 34), versioneer will not be able to determine the pandas version.
+          # This is because of a security update to git that blocks it from reading the config folder if
+          # it is not owned by the current user. We hit this since the "mounted" folder is not hit by the
+          # Docker container.
+          # xref https://github.com/pypa/manylinux/issues/1309
+          docker pull quay.io/pypa/manylinux2014_i686
+          docker run --platform linux/386 -v $(pwd):/pandas quay.io/pypa/manylinux2014_i686 \
+          /bin/bash -xc "cd pandas && \
+          git config --global --add safe.directory /pandas && \
+          /opt/python/cp38-cp38/bin/python -m venv ~/virtualenvs/pandas-dev && \
+          . ~/virtualenvs/pandas-dev/bin/activate && \
+          python -m pip install --no-deps -U pip wheel 'setuptools<60.0.0' && \
+          pip install cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio>=0.17 hypothesis && \
+          python setup.py build_ext -q -j2 && \
+          python -m pip install --no-build-isolation --no-use-pep517 -e . && \
+          export PANDAS_CI=1 && \
+          pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml"
+
+      - name: Publish test results for Python 3.8-32 bit full Linux
+        uses: actions/upload-artifact@v3
+        with:
+          name: Test results
+          path: test-data.xml
+        if: failure()

.github/workflows/32-bit-linux.yml

@@ -67,15 +67,6 @@ jobs:
         environment-file: ${{ env.ENV_FILE }}
         use-only-tar-bz2: true
 
-    - name: Install node.js (for pyright)
-      uses: actions/setup-node@v3
-      with:
-        node-version: "16"
-
-    - name: Install pyright
-      # note: keep version in sync with .pre-commit-config.yaml
-      run: npm install -g [email protected]
-
     - name: Build Pandas
       id: build
       uses: ./.github/actions/build_pandas
@@ -96,8 +87,16 @@ jobs:
       run: ci/code_checks.sh docstrings
       if: ${{ steps.build.outcome == 'success' }}
 
-    - name: Run typing validation
-      run: ci/code_checks.sh typing
+    - name: Use existing environment for type checking
+      run: |
+        echo $PATH >> $GITHUB_PATH
+        echo "PYTHONHOME=$PYTHONHOME" >> $GITHUB_ENV
+        echo "PYTHONPATH=$PYTHONPATH" >> $GITHUB_ENV
+
+    - name: Typing
+      uses: pre-commit/[email protected]
+      with:
+        extra_args: --hook-stage manual --all-files
       if: ${{ steps.build.outcome == 'success' }}
 
     - name: Run docstring validation script tests

.github/workflows/code-checks.yml

@@ -24,43 +24,46 @@ jobs:
       group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-web-docs
       cancel-in-progress: true
 
+    defaults:
+      run:
+        shell: bash -el {0}
+
     steps:
     - name: Checkout
       uses: actions/checkout@v3
       with:
         fetch-depth: 0
 
-    - name: Set up pandas
-      uses: ./.github/actions/setup
+    - name: Set up Conda
+      uses: ./.github/actions/setup-conda
+
+    - name: Build Pandas
+      uses: ./.github/actions/build_pandas
 
     - name: Build website
-      run: |
-        source activate pandas-dev
-        python web/pandas_web.py web/pandas --target-path=web/build
+      run: python web/pandas_web.py web/pandas --target-path=web/build
 
     - name: Build documentation
-      run: |
-        source activate pandas-dev
-        doc/make.py --warnings-are-errors
+      run: doc/make.py --warnings-are-errors
 
     - name: Install ssh key
       run: |
         mkdir -m 700 -p ~/.ssh
         echo "${{ secrets.server_ssh_key }}" > ~/.ssh/id_rsa
         chmod 600 ~/.ssh/id_rsa
         echo "${{ secrets.server_ip }} ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBE1Kkopomm7FHG5enATf7SgnpICZ4W2bw+Ho+afqin+w7sMcrsa0je7sbztFAV8YchDkiBKnWTG4cRT+KZgZCaY=" > ~/.ssh/known_hosts
-      if: ${{github.event_name == 'push' && github.ref == 'refs/heads/main'}}
+      if: github.event_name == 'push' && github.ref == 'refs/heads/main'
 
     - name: Copy cheatsheets into site directory
       run: cp doc/cheatsheet/Pandas_Cheat_Sheet* web/build/
 
     - name: Upload web
       run: rsync -az --delete --exclude='pandas-docs' --exclude='docs' web/build/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas
-      if: ${{github.event_name == 'push' && github.ref == 'refs/heads/main'}}
+      if: github.event_name == 'push' && github.ref == 'refs/heads/main'
 
     - name: Upload dev docs
       run: rsync -az --delete doc/build/html/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas/pandas-docs/dev
-      if: ${{github.event_name == 'push' && github.ref == 'refs/heads/main'}}
+      if: github.event_name == 'push' && github.ref == 'refs/heads/main'
 
     - name: Move docs into site directory
       run: mv doc/build/html web/build/docs

.github/workflows/docbuild-and-upload.yml

@@ -43,32 +43,18 @@ jobs:
       with:
         fetch-depth: 0
 
-    - name: Install Dependencies
-      uses: conda-incubator/setup-[email protected]
+    - name: Set up Conda
+      uses: ./.github/actions/setup-conda
       with:
-        mamba-version: "*"
-        channels: conda-forge
-        activate-environment: pandas-dev
-        channel-priority: ${{ matrix.os == 'macos-latest' && 'flexible' || 'strict' }}
         environment-file: ci/deps/${{ matrix.env_file }}
-        use-only-tar-bz2: true
-
-    # ImportError: 2): Library not loaded: @rpath/libssl.1.1.dylib
-    # Referenced from: /Users/runner/miniconda3/envs/pandas-dev/lib/libthrift.0.13.0.dylib
-    # Reason: image not found
-    - name: Upgrade pyarrow on MacOS
-      run: conda install -n pandas-dev -c conda-forge --no-update-deps pyarrow=6
-      if: ${{ matrix.os == 'macos-latest' }}
+        pyarrow-version: ${{ matrix.os == 'macos-latest' && '6' || '' }}
 
     - name: Build Pandas
       uses: ./.github/actions/build_pandas
 
     - name: Test
       run: ci/run_tests.sh
 
-    - name: Build Version
-      run: conda list
-
     - name: Publish test results
       uses: actions/upload-artifact@v3
       with:

.github/workflows/macos-windows.yml

@@ -155,24 +155,11 @@ jobs:
         channel-priority: flexible
         environment-file: ${{ env.ENV_FILE }}
         use-only-tar-bz2: true
-      if: ${{ env.IS_PYPY == 'false' }} # No pypy3.8 support
 
     - name: Upgrade Arrow version
       run: conda install -n pandas-dev -c conda-forge --no-update-deps pyarrow=${{ matrix.pyarrow_version }}
       if: ${{ matrix.pyarrow_version }}
 
-    - name: Setup PyPy
-      uses: actions/setup-python@v3
-      with:
-        python-version: "pypy-3.8"
-      if: ${{ env.IS_PYPY == 'true' }}
-
-    - name: Setup PyPy dependencies
-      run: |
-        # TODO: re-enable cov, its slowing the tests down though
-        pip install Cython numpy python-dateutil pytz pytest>=6.0 pytest-xdist>=1.31.0 pytest-asyncio>=0.17 hypothesis>=5.5.3
-      if: ${{ env.IS_PYPY == 'true' }}
-
     - name: Build Pandas
       uses: ./.github/actions/build_pandas
 

.github/workflows/posix.yml

@@ -2,7 +2,7 @@
 # Unfreeze(by commentingthe if: false() condition) once the
 # next Python Dev version has released beta 1 and both Cython and numpy support it
 # After that Python has released, migrate the workflows to the
-# posix GHA workflows/Azure pipelines and "freeze" this file by
+# posix GHA workflows and "freeze" this file by
 # uncommenting the if: false() condition
 # Feel free to modify this comment as necessary.
 
@@ -54,11 +54,10 @@ jobs:
       with:
         python-version: '3.11-dev'
 
-    # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941
     - name: Install dependencies
       shell: bash -el {0}
       run: |
-        python -m pip install --upgrade pip "setuptools<60.0.0" wheel
+        python -m pip install --upgrade pip setuptools wheel
         pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy
         pip install git+https://github.com/nedbat/coveragepy.git
         pip install cython python-dateutil pytz hypothesis pytest>=6.2.5 pytest-xdist pytest-cov

.github/workflows/python-dev.yml

@@ -41,10 +41,9 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
 
-    # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip "setuptools<60.0.0" wheel
+        python -m pip install --upgrade pip setuptools wheel
 
         # GH 39416
         pip install numpy
@@ -66,10 +65,8 @@ jobs:
         channels: conda-forge
         python-version: '${{ matrix.python-version }}'
 
-    # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941
     - name: Install pandas from sdist
       run: |
-        python -m pip install --upgrade "setuptools<60.0.0"
         pip list
         python -m pip install dist/*.gz
 

.github/workflows/sdist.yml

@@ -1,5 +1,7 @@
-minimum_pre_commit_version: 2.9.2
+minimum_pre_commit_version: 2.15.0
 exclude: ^LICENSES/|\.(html|csv|svg)$
+# reserve "manual" for mypy and pyright
+default_stages: [commit, merge-commit, push, prepare-commit-msg, commit-msg, post-checkout, post-commit, post-merge, post-rewrite]
 ci:
     autofix_prs: false
 repos:
@@ -9,7 +11,7 @@ repos:
     -   id: absolufy-imports
         files: ^pandas/
 -   repo: https://github.com/jendrikseipp/vulture
-    rev: 'v2.3'
+    rev: 'v2.4'
     hooks:
       - id: vulture
         entry: python scripts/run_vulture.py
@@ -31,7 +33,9 @@ repos:
     -   id: debug-statements
     -   id: end-of-file-fixer
         exclude: \.txt$
+        stages: [commit, merge-commit, push, prepare-commit-msg, commit-msg, post-checkout, post-commit, post-merge, post-rewrite]
     -   id: trailing-whitespace
+        stages: [commit, merge-commit, push, prepare-commit-msg, commit-msg, post-checkout, post-commit, post-merge, post-rewrite]
 -   repo: https://github.com/cpplint/cpplint
     rev: 1.6.0
     hooks:
@@ -56,7 +60,7 @@ repos:
     hooks:
     -   id: isort
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v2.32.0
+    rev: v2.32.1
     hooks:
     -   id: pyupgrade
         args: [--py38-plus]
@@ -71,7 +75,7 @@ repos:
         types: [text]  # overwrite types: [rst]
         types_or: [python, rst]
 -   repo: https://github.com/sphinx-contrib/sphinx-lint
-    rev: v0.4.1
+    rev: v0.6
     hooks:
     - id: sphinx-lint
 -   repo: https://github.com/asottile/yesqa
@@ -84,12 +88,22 @@ repos:
     -   id: pyright
         name: pyright
         entry: pyright
+        # note: assumes python env is setup and activated
         language: node
         pass_filenames: false
         types: [python]
         stages: [manual]
-        # note: keep version in sync with .github/workflows/code-checks.yml
-        additional_dependencies: ['[email protected]']
+        additional_dependencies: ['[email protected]']
+-   repo: local
+    hooks:
+    -   id: mypy
+        name: mypy
+        entry: mypy
+        # note: assumes python env is setup and activated
+        language: system
+        pass_filenames: false
+        types: [python]
+        stages: [manual]
 -   repo: local
     hooks:
     -   id: flake8-rst