WIP Add ccache to Azure builds

.circleci/config.yml

@@ -1,22 +1 @@
 version: 2.1
-
-jobs:
-  test-arm:
-    machine:
-      image: ubuntu-2004:202101-01
-    resource_class: arm.medium
-    environment:
-      ENV_FILE: ci/deps/circle-38-arm64.yaml
-      PYTEST_WORKERS: auto
-      PATTERN: "not slow and not network and not clipboard and not arm_slow"
-      PYTEST_TARGET: "pandas"
-      PANDAS_CI: "1"
-    steps:
-      - checkout
-      - run: ci/setup_env.sh
-      - run: PATH=$HOME/miniconda3/envs/pandas-dev/bin:$HOME/miniconda3/condabin:$PATH ci/run_tests.sh
-
-workflows:
-  test:
-    jobs:
-      - test-arm

.github/actions/build-pandas/action.yml

@@ -0,0 +1,52 @@
+name: Build pandas
+description: Rebuilds the C extensions and installs pandas
+inputs:
+  use-login-shell:
+    description: "Use 'bash -l' as shell (required for Conda envs)"
+    default: true
+runs:
+  using: composite
+  steps:
+    # Create a shell wrapper to be able to call "bash" or "bash -l" depending
+    # on the "use-login-shell" arguments.
+    # We need this because GHA does not allow ${{ inputs. }} in "shell: " arguments.
+    - name: Set shell
+      shell: bash
+      run: |
+        if [ ${{ inputs.use-login-shell }} = true ]; then
+          args="-l"
+        fi
+        echo "exec bash $args \"\$@\"" > /tmp/_build_pandas_shell
+        cat /tmp/_build_pandas_shell
+
+    - name: Environment Detail
+      shell: bash /tmp/_build_pandas_shell {0}
+      run: |
+        if which conda; then
+          conda info
+          conda list
+        fi
+        if which pip; then
+          pip list
+        fi
+        python --version
+
+    - name: Get Python version
+      id: get-python-version
+      shell: bash /tmp/_build_pandas_shell {0}
+      run: python3 -c "import platform as p; print(f'::set-output name=version::{p.python_version()}-{p.python_branch()}')"
+
+    - name: Set up sccache
+      uses: ./.github/actions/setup-sccache
+      with:
+        extra-cache-key: ${{ steps.get-python-version.outputs.version }}
+
+    - name: Build Pandas
+      shell: bash /tmp/_build_pandas_shell {0}
+      run: |
+        time DISTUTILS_C_COMPILER_LAUNCHER=sccache python setup.py build_ext -vv -j 2
+        python -m pip install -vv -e . --no-build-isolation --no-use-pep517 --no-index
+
+    - name: Build Version
+      shell: bash /tmp/_build_pandas_shell {0}
+      run: pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd

.github/actions/setup-sccache/action.yml

@@ -0,0 +1,18 @@
+name: Setup sccache
+inputs:
+  extra-cache-key:
+    required: false
+    default: ''
+runs:
+  using: composite
+  steps:
+    - name: Get Date
+      id: get-date
+      run: echo "::set-output name=today::$(/bin/date -u '+%Y%m%d')"
+      shell: bash
+
+    - name: Setup sccache
+      uses: jonashaag/ccache-action@sccache-2
+      with:
+        sccache: true
+        key: ${{ runner.os }}--${{ runner.arch }}--${{ github.workflow }}--${{ steps.get-date.outputs.today }}--${{ inputs.extra-cache-key }}

.github/actions/setup/action.yml

@@ -1,12 +1,64 @@
 name: Set up pandas
 description: Runs all the setup steps required to have a built pandas ready to use
+inputs:
+  environment-file:
+    default: environment.yml
+  pyarrow-version:
+      required: false
+  is-pypy:
+    default: false
+  activate-environment:
+    default: pandas-dev
+  python-version:
+    required: false
 runs:
   using: composite
   steps:
-    - name: Setting conda path
-      run: echo "${HOME}/miniconda3/bin" >> $GITHUB_PATH
-      shell: bash -l {0}
+    - name: Get Date
+      id: get-date
+      run: echo "::set-output name=today::$(/bin/date -u '+%Y%m%d')"
+      shell: bash
 
-    - name: Setup environment and build pandas
-      run: ci/setup_env.sh
-      shell: bash -l {0}
+    - name: Cache Conda packages
+      uses: actions/cache@v2
+      with:
+        path: ~/conda_pkgs_dir
+        key: conda-${{ runner.os }}-${{ runner.arch }}-${{ inputs.environment-file }}-${{ steps.get-date.outputs.today }}
+
+    - name: Set Arrow version in ${{ inputs.environment-file }} to ${{ inputs.pyarrow-version }}
+      run: |
+        grep -q '\- pyarrow' ${{ inputs.environment-file }}
+        sed -i "s/- pyarrow/- pyarrow=${{ inputs.pyarrow-version }}/" ${{ inputs.environment-file }}
+        cat ${{ inputs.environment-file }}
+      shell: bash
+      if: ${{ inputs.pyarrow-version }}
+
+    - name: Setup Mambaforge and install ${{ inputs.environment-file }} (Python ${{ inputs.python-version }})
+      uses: conda-incubator/setup-miniconda@v2
+      with:
+        mamba-version: "0.21.2"
+        use-mamba: true
+        channels: conda-forge
+        activate-environment: ${{ inputs.activate-environment }}
+        channel-priority: strict
+        environment-file: ${{ inputs.environment-file }}
+        python-version: ${{ inputs.python-version }}
+        use-only-tar-bz2: true
+      if: ${{ inputs.is-pypy == 'false' }} # No pypy3.8 support
+
+    - name: Pin setuptools (GH#44980)
+      run: mamba install -n ${{ inputs.activate-environment }} 'setuptools<60.0.0'
+      shell: bash
+      if: ${{ inputs.is-pypy == 'false' }} # No pypy3.8 support
+
+    - name: Setup PyPy
+      uses: actions/setup-python@v2
+      with:
+        python-version: "pypy-3.8"
+      if: ${{ inputs.is-pypy == 'true' }}
+
+    - name: Setup PyPy dependencies
+      # TODO: re-enable cov, its slowing the tests down though
+      run: pip install Cython numpy python-dateutil pytz pytest>=6.0 pytest-xdist>=1.31.0 hypothesis>=5.5.3
+      shell: bash
+      if: ${{ inputs.is-pypy == 'true' }}

.github/workflows/assign.yml

@@ -7,6 +7,7 @@ jobs:
   issue_assign:
     runs-on: ubuntu-latest
     steps:
+    - run: false
     - if: github.event.comment.body == 'take'
       run: |
         echo "Assigning issue ${{ github.event.issue.number }} to ${{ github.event.comment.user.login }}"

.github/workflows/asv-bot.yml

@@ -6,8 +6,7 @@ on:
       - created
 
 env:
-  ENV_FILE: environment.yml
-  COMMENT: ${{github.event.comment.body}}
+  COMMENT: ${{ github.event.comment.body }}
 
 jobs:
   autotune:
@@ -28,25 +27,16 @@ jobs:
       cancel-in-progress: false
 
     steps:
+      - run: false
       - name: Checkout
         uses: actions/checkout@v2
         with:
           fetch-depth: 0
 
-      - name: Cache conda
-        uses: actions/cache@v2
-        with:
-          path: ~/conda_pkgs_dir
-          key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }}
-
-        # Although asv sets up its own env, deps are still needed
-        # during discovery process
-      - uses: conda-incubator/setup-miniconda@v2
-        with:
-          activate-environment: pandas-dev
-          channel-priority: strict
-          environment-file: ${{ env.ENV_FILE }}
-          use-only-tar-bz2: true
+      # Although asv sets up its own env, deps are still needed
+      # during discovery process
+      - name: Set up Conda
+        uses: ./.github/actions/setup
 
       - name: Run benchmarks
         id: bench

.github/workflows/autoupdate-pre-commit-config.yml

@@ -11,6 +11,7 @@ jobs:
     name: Autoupdate pre-commit config
     runs-on: ubuntu-latest
     steps:
+      - run: false
       - name: Set up Python
         uses: actions/setup-python@v2
       - name: Cache multiple paths

.github/workflows/code-checks.yml

@@ -11,7 +11,6 @@ on:
       - 1.4.x
 
 env:
-  ENV_FILE: environment.yml
   PANDAS_CI: 1
 
 jobs:
@@ -23,6 +22,7 @@ jobs:
       group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-pre-commit
       cancel-in-progress: true
     steps:
+    - run: false
     - name: Checkout
       uses: actions/checkout@v2
 
@@ -47,25 +47,19 @@ jobs:
       cancel-in-progress: true
 
     steps:
+    - run: false
     - name: Checkout
       uses: actions/checkout@v2
       with:
         fetch-depth: 0
 
-    - name: Cache conda
-      uses: actions/cache@v2
-      with:
-        path: ~/conda_pkgs_dir
-        key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }}
+    - name: Set up Conda
+      uses: ./.github/actions/setup
 
-    - uses: conda-incubator/setup-miniconda@v2
-      with:
-        mamba-version: "*"
-        channels: conda-forge
-        activate-environment: pandas-dev
-        channel-priority: strict
-        environment-file: ${{ env.ENV_FILE }}
-        use-only-tar-bz2: true
+    - name: Build Pandas
+      uses: ./.github/actions/build-pandas
+      id: build
+      continue-on-error: true
 
     - name: Install node.js (for pyright)
       uses: actions/setup-node@v2
@@ -76,10 +70,6 @@ jobs:
       # note: keep version in sync with .pre-commit-config.yaml
       run: npm install -g [email protected]
 
-    - name: Build Pandas
-      id: build
-      uses: ./.github/actions/build_pandas
-
     - name: Run checks on imported code
       run: ci/code_checks.sh code
       if: ${{ steps.build.outcome == 'success' }}
@@ -113,29 +103,17 @@ jobs:
       cancel-in-progress: true
 
     steps:
+    - run: false
     - name: Checkout
       uses: actions/checkout@v2
       with:
         fetch-depth: 0
 
-    - name: Cache conda
-      uses: actions/cache@v2
-      with:
-        path: ~/conda_pkgs_dir
-        key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }}
-
-    - uses: conda-incubator/setup-miniconda@v2
-      with:
-        mamba-version: "*"
-        channels: conda-forge
-        activate-environment: pandas-dev
-        channel-priority: strict
-        environment-file: ${{ env.ENV_FILE }}
-        use-only-tar-bz2: true
+    - name: Set up Conda
+      uses: ./.github/actions/setup
 
     - name: Build Pandas
-      id: build
-      uses: ./.github/actions/build_pandas
+      uses: ./.github/actions/build-pandas
 
     - name: Run ASV benchmarks
       run: |
@@ -148,7 +126,6 @@ jobs:
         if grep "failed" benchmarks.log > /dev/null ; then
             exit 1
         fi
-      if: ${{ steps.build.outcome == 'success' }}
 
     - name: Publish benchmarks artifact
       uses: actions/upload-artifact@v2
@@ -170,6 +147,7 @@ jobs:
       cancel-in-progress: true
 
     steps:
+      - run: false
       - name: Clean up dangling images
         run: docker image prune -f
 

.github/workflows/comment_bot.yml

@@ -12,6 +12,7 @@ jobs:
     if: startsWith(github.event.comment.body, '@github-actions pre-commit')
     runs-on: ubuntu-latest
     steps:
+      - run: false
       - uses: actions/checkout@v2
       - uses: r-lib/actions/pr-fetch@v2
         with: