Refactor CI code

.circleci/config.yml

@@ -13,7 +13,7 @@ jobs:
       PANDAS_CI: "1"
     steps:
       - checkout
-      - run: ci/setup_env.sh
+      - run: .circleci/setup_env.sh
       - run: PATH=$HOME/miniconda3/envs/pandas-dev/bin:$HOME/miniconda3/condabin:$PATH ci/run_tests.sh
 
 workflows:

.github/actions/build-pandas/action.yml

@@ -0,0 +1,39 @@
+name: Build pandas
+description: Rebuilds the C extensions and installs pandas
+runs:
+  using: composite
+  steps:
+    - name: Environment Detail
+      run: |
+        if which micromamba; then
+          micromamba info
+          micromamba list
+          micromamba info | grep -Ei 'environment.+:' | grep -qEiv 'environment.+:.+none'
+        fi
+        if which pip; then
+          pip list
+        fi
+        python --version
+      shell: bash -el {0}
+
+    - name: Get Python version
+      id: get-python-version
+      run: python3 -c "import platform as p; print(f'::set-output name=version::{p.python_implementation()}-{p.python_version()}')"
+      shell: bash -el {0}
+
+    - name: Set up sccache
+      uses: ./.github/actions/setup-sccache
+      with:
+        extra-cache-key: ${{ steps.get-python-version.outputs.version }}
+
+    - name: Build Pandas
+      run: |
+        which python
+        which pip
+        time DISTUTILS_C_COMPILER_LAUNCHER=sccache python setup.py build_ext -v -j 2
+        pip install -v -e . --no-build-isolation --no-use-pep517 --no-index
+      shell: bash -el {0}
+
+    - name: Build Version
+      run: pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd
+      shell: bash -el {0}

.github/actions/run-tests/action.yml

@@ -0,0 +1,34 @@
+name: Run tests and report results
+inputs:
+  check-pyarrow-version:
+    required: false
+runs:
+  using: composite
+  steps:
+    - name: Check PyArrow version
+      run: |
+        # Double check that we have the expected PyArrow
+        pushd /tmp
+        python -c "import pandas; pandas.show_versions()" | egrep -i "pyarrow.+: ${{ matrix.check-pyarrow-version }}"
+        popd
+      shell: bash -el {0}
+      if: ${{ inputs.check-pyarrow-version }}
+
+    - name: Test
+      run: ci/run_tests.sh
+      shell: bash -el {0}
+
+    - name: Publish test results
+      uses: actions/upload-artifact@v2
+      with:
+        name: Test results
+        path: test-data.xml
+      if: failure()
+
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v2
+      with:
+        flags: unittests
+        name: codecov-pandas
+        fail_ci_if_error: false
+      if: failure()

.github/actions/setup-python/action.yml

@@ -0,0 +1,65 @@
+name: Setup Python and install requirements
+inputs:
+  python-version:
+    required: true
+  architecture:
+    default: x64
+runs:
+  using: composite
+  steps:
+    # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941
+    - name: Create temporary requirements.txt
+      run: |
+        # Drop cache at least once per month
+        month_today="$(date '+%Y-%m')"
+        cat > requirements.txt <<EOF
+          # $month_today
+
+          # Python deps
+          pip
+          setuptools<60.0.0
+          wheel
+
+          # Pandas deps
+          # GH 39416
+          numpy
+          cython
+          python-dateutil
+          pytz
+
+          # Test deps
+          git+https://github.com/nedbat/coveragepy.git
+          hypothesis
+          pytest>=6.2.5
+          pytest-xdist
+          pytest-cov
+          pytest-asyncio>=0.17
+        EOF
+      shell: bash -el {0}
+
+    - name: Set up Python
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ inputs.python-version }}
+        architecture: ${{ inputs.architecture }}
+        cache: pip
+
+    - name: Fix $PATH on macOS
+      run: |
+        # On macOS, the Python version we installed above is too late in $PATH
+        # to be effective if using "bash -l" (which we need for code that works
+        # with Conda envs).
+        cat >> ~/.bash_profile <<EOF
+          export PATH="\$(echo "\$PATH" | grep -Eio '[^:]+hostedtoolcache/python[^:]+bin'):\$PATH" \
+        EOF
+      shell: bash -el {0}
+      if: ${{ runner.os == 'macOS' }}
+
+    - name: Install dependencies
+      run: |
+        cat requirements.txt
+        # TODO https://github.com/numpy/numpy/issues/21196
+        bits32=$(python -c 'import sys; print(int(sys.maxsize > 2**32))')
+        NPY_DISABLE_SVML=$bits32 pip install -r requirements.txt
+        pip list
+      shell: bash -el {0}

.github/actions/setup-sccache/action.yml

@@ -0,0 +1,28 @@
+name: Setup sccache
+inputs:
+  extra-cache-key:
+    required: false
+    default: ''
+runs:
+  using: composite
+  steps:
+    - name: Get Date
+      id: get-date
+      run: echo "::set-output name=today::$(/bin/date -u '+%Y%m%d')"
+      shell: bash
+
+    - name: Fix Windows temporary directory
+      # On Windows, for some reason the default temporary directory provided to sccache
+      # may become read-only at some point. Work around by having a private tempdir.
+      id: mktemp
+      run: echo "::set-output name=tmpdir::$(cygpath -w $(mktemp -d))"
+      shell: bash
+      if: ${{ runner.os == 'Windows' }}
+
+    - name: Setup sccache
+      uses: hendrikmuhs/[email protected]
+      with:
+        variant: sccache
+        key: ${{ runner.os }}--${{ runner.arch }}--${{ github.workflow }}--${{ steps.get-date.outputs.today }}--${{ inputs.extra-cache-key }}
+      env:
+        TMP: "${{ steps.mktemp.outputs.tmpdir }}"

.github/actions/setup/action.yml

@@ -1,12 +1,50 @@
 name: Set up pandas
 description: Runs all the setup steps required to have a built pandas ready to use
+inputs:
+  environment-file:
+    default: environment.yml
+  pyarrow-version:
+      required: false
+  is-pypy:
+    default: false
+  environment-name:
+    default: pandas-dev
+  python-version:
+    required: false
 runs:
   using: composite
   steps:
-    - name: Setting conda path
-      run: echo "${HOME}/miniconda3/bin" >> $GITHUB_PATH
-      shell: bash -l {0}
+    - name: Set Arrow version in ${{ inputs.environment-file }} to ${{ inputs.pyarrow-version }}
+      run: |
+        grep -q '  - pyarrow' ${{ inputs.environment-file }}
+        sed -i"" -e "s/  - pyarrow/  - pyarrow=${{ inputs.pyarrow-version }}/" ${{ inputs.environment-file }}
+        cat ${{ inputs.environment-file }}
+      shell: bash
+      if: ${{ inputs.pyarrow-version }}
 
-    - name: Setup environment and build pandas
-      run: ci/setup_env.sh
-      shell: bash -l {0}
+    - name: Pin setuptools (GH#44980)
+      run: |
+        echo '  - setuptools <60' >> ${{ inputs.environment-file }}
+      shell: bash
+
+    - name: Install ${{ inputs.environment-file }} (Python ${{ inputs.python-version }})
+      uses: mamba-org/provision-with-micromamba@main
+      with:
+        environment-name: ${{ inputs.environment-name }}
+        environment-file: ${{ inputs.environment-file }}
+        extra-specs: |
+          ${{ inputs.python-version && format('python={0}', inputs.python-version) }}
+        cache-env: true
+      if: ${{ inputs.is-pypy == 'false' }} # No pypy3.8 support
+
+    - name: Setup PyPy
+      uses: actions/setup-python@v3
+      with:
+        python-version: "pypy-3.8"
+      if: ${{ inputs.is-pypy == 'true' }}
+
+    - name: Setup PyPy dependencies
+      # TODO: re-enable cov, its slowing the tests down though
+      run: pip install Cython numpy python-dateutil pytz pytest>=6.0 pytest-xdist>=1.31.0 hypothesis>=5.5.3 pytest-asyncio>=0.17
+      shell: bash
+      if: ${{ inputs.is-pypy == 'true' }}

.github/workflows/asv-bot.yml

@@ -6,8 +6,7 @@ on:
       - created
 
 env:
-  ENV_FILE: environment.yml
-  COMMENT: ${{github.event.comment.body}}
+  COMMENT: ${{ github.event.comment.body }}
 
 jobs:
   autotune:
@@ -17,7 +16,7 @@ jobs:
     runs-on: ubuntu-latest
     defaults:
       run:
-        shell: bash -l {0}
+        shell: bash -el {0}
 
     concurrency:
       # Set concurrency to prevent abuse(full runs are ~5.5 hours !!!)
@@ -29,24 +28,14 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
-      - name: Cache conda
-        uses: actions/cache@v2
-        with:
-          path: ~/conda_pkgs_dir
-          key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }}
-
-        # Although asv sets up its own env, deps are still needed
-        # during discovery process
-      - uses: conda-incubator/setup-miniconda@v2
-        with:
-          activate-environment: pandas-dev
-          channel-priority: strict
-          environment-file: ${{ env.ENV_FILE }}
-          use-only-tar-bz2: true
+      # Although asv sets up its own env, deps are still needed
+      # during discovery process
+      - name: Set up Conda
+        uses: ./.github/actions/setup
 
       - name: Run benchmarks
         id: bench

.github/workflows/autoupdate-pre-commit-config.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v3
       - name: Cache multiple paths
         uses: actions/cache@v2
         with: