Add ccache to GHA builds

- Refactor duplicated code from GHA workflows to the 'build-pandas' and
  'setup' actions.
- Use Mamba over Conda everywhere.
- Add sccache using 'hendrikmuhs/ccache-action'.
- General cleanup of GHA workflows.

Author: jonashaag

.github/actions/build-pandas/action.yml

@@ -0,0 +1,52 @@
+name: Build pandas
+description: Rebuilds the C extensions and installs pandas
+inputs:
+  use-login-shell:
+    description: "Use 'bash -l' as shell (required for Conda envs)"
+    default: true
+runs:
+  using: composite
+  steps:
+    # Create a shell wrapper to be able to call "bash" or "bash -l" depending
+    # on the "use-login-shell" arguments.
+    # We need this because GHA does not allow ${{ inputs. }} in "shell: " arguments.
+    - name: Set shell
+      shell: bash
+      run: |
+        if [ ${{ inputs.use-login-shell }} = true ]; then
+          args="-l"
+        fi
+        echo "exec bash $args \"\$@\"" > /tmp/_build_pandas_shell
+        cat /tmp/_build_pandas_shell
+
+    - name: Environment Detail
+      shell: bash /tmp/_build_pandas_shell {0}
+      run: |
+        if which conda; then
+          conda info
+          conda list
+        fi
+        if which pip; then
+          pip list
+        fi
+        python --version
+
+    - name: Get Python version
+      id: get-python-version
+      shell: bash /tmp/_build_pandas_shell {0}
+      run: python3 -c "import platform as p; print(f'::set-output name=version::{p.python_version()}-{p.python_branch()}')"
+
+    - name: Set up sccache
+      uses: ./.github/actions/setup-sccache
+      with:
+        extra-cache-key: ${{ steps.get-python-version.outputs.version }}
+
+    - name: Build Pandas
+      shell: bash /tmp/_build_pandas_shell {0}
+      run: |
+        time DISTUTILS_C_COMPILER_LAUNCHER=sccache python setup.py build_ext -vv -j 2
+        python -m pip install -vv -e . --no-build-isolation --no-use-pep517 --no-index
+
+    - name: Build Version
+      shell: bash /tmp/_build_pandas_shell {0}
+      run: pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd

.github/actions/build_pandas/action.yml

@@ -0,0 +1,18 @@
+name: Setup sccache
+inputs:
+  extra-cache-key:
+    required: false
+    default: ''
+runs:
+  using: composite
+  steps:
+    - name: Get Date
+      id: get-date
+      run: echo "::set-output name=today::$(/bin/date -u '+%Y%m%d')"
+      shell: bash
+
+    - name: Setup sccache
+      uses: jonashaag/ccache-action@sccache-2
+      with:
+        sccache: true
+        key: ${{ runner.os }}--${{ runner.arch }}--${{ github.workflow }}--${{ steps.get-date.outputs.today }}--${{ inputs.extra-cache-key }}

.github/actions/setup-sccache/action.yml

@@ -1,12 +1,64 @@
 name: Set up pandas
 description: Runs all the setup steps required to have a built pandas ready to use
+inputs:
+  environment-file:
+    default: environment.yml
+  pyarrow-version:
+      required: false
+  is-pypy:
+    default: false
+  activate-environment:
+    default: pandas-dev
+  python-version:
+    required: false
 runs:
   using: composite
   steps:
-    - name: Setting conda path
-      run: echo "${HOME}/miniconda3/bin" >> $GITHUB_PATH
-      shell: bash -l {0}
+    - name: Get Date
+      id: get-date
+      run: echo "::set-output name=today::$(/bin/date -u '+%Y%m%d')"
+      shell: bash
 
-    - name: Setup environment and build pandas
-      run: ci/setup_env.sh
-      shell: bash -l {0}
+    - name: Cache Conda packages
+      uses: actions/cache@v2
+      with:
+        path: ~/conda_pkgs_dir
+        key: conda-${{ runner.os }}-${{ runner.arch }}-${{ inputs.environment-file }}-${{ steps.get-date.outputs.today }}
+
+    - name: Set Arrow version in ${{ inputs.environment-file }} to ${{ inputs.pyarrow-version }}
+      run: |
+        grep -q '\- pyarrow' ${{ inputs.environment-file }}
+        sed -i "s/- pyarrow/- pyarrow=${{ inputs.pyarrow-version }}/" ${{ inputs.environment-file }}
+        cat ${{ inputs.environment-file }}
+      shell: bash
+      if: ${{ inputs.pyarrow-version }}
+
+    - name: Setup Mambaforge and install ${{ inputs.environment-file }} (Python ${{ inputs.python-version }})
+      uses: conda-incubator/setup-miniconda@v2
+      with:
+        mamba-version: "0.21.2"
+        use-mamba: true
+        channels: conda-forge
+        activate-environment: ${{ inputs.activate-environment }}
+        channel-priority: strict
+        environment-file: ${{ inputs.environment-file }}
+        python-version: ${{ inputs.python-version }}
+        use-only-tar-bz2: true
+      if: ${{ inputs.is-pypy == 'false' }} # No pypy3.8 support
+
+    - name: Pin setuptools (GH#44980)
+      run: mamba install -n ${{ inputs.activate-environment }} 'setuptools<60.0.0'
+      shell: bash
+      if: ${{ inputs.is-pypy == 'false' }} # No pypy3.8 support
+
+    - name: Setup PyPy
+      uses: actions/setup-python@v2
+      with:
+        python-version: "pypy-3.8"
+      if: ${{ inputs.is-pypy == 'true' }}
+
+    - name: Setup PyPy dependencies
+      # TODO: re-enable cov, its slowing the tests down though
+      run: pip install Cython numpy python-dateutil pytz pytest>=6.0 pytest-xdist>=1.31.0 hypothesis>=5.5.3
+      shell: bash
+      if: ${{ inputs.is-pypy == 'true' }}

.github/actions/setup/action.yml

@@ -6,8 +6,7 @@ on:
       - created
 
 env:
-  ENV_FILE: environment.yml
-  COMMENT: ${{github.event.comment.body}}
+  COMMENT: ${{ github.event.comment.body }}
 
 jobs:
   autotune:
@@ -33,20 +32,10 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Cache conda
-        uses: actions/cache@v2
-        with:
-          path: ~/conda_pkgs_dir
-          key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }}
-
-        # Although asv sets up its own env, deps are still needed
-        # during discovery process
-      - uses: conda-incubator/setup-miniconda@v2
-        with:
-          activate-environment: pandas-dev
-          channel-priority: strict
-          environment-file: ${{ env.ENV_FILE }}
-          use-only-tar-bz2: true
+      # Although asv sets up its own env, deps are still needed
+      # during discovery process
+      - name: Set up Conda
+        uses: ./.github/actions/setup
 
       - name: Run benchmarks
         id: bench

.github/workflows/asv-bot.yml

@@ -11,7 +11,6 @@ on:
       - 1.4.x
 
 env:
-  ENV_FILE: environment.yml
   PANDAS_CI: 1
 
 jobs:
@@ -52,20 +51,13 @@ jobs:
       with:
         fetch-depth: 0
 
-    - name: Cache conda
-      uses: actions/cache@v2
-      with:
-        path: ~/conda_pkgs_dir
-        key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }}
+    - name: Set up Conda
+      uses: ./.github/actions/setup
 
-    - uses: conda-incubator/setup-miniconda@v2
-      with:
-        mamba-version: "*"
-        channels: conda-forge
-        activate-environment: pandas-dev
-        channel-priority: strict
-        environment-file: ${{ env.ENV_FILE }}
-        use-only-tar-bz2: true
+    - name: Build Pandas
+      uses: ./.github/actions/build-pandas
+      id: build
+      continue-on-error: true
 
     - name: Install node.js (for pyright)
       uses: actions/setup-node@v2
@@ -76,10 +68,6 @@ jobs:
       # note: keep version in sync with .pre-commit-config.yaml
       run: npm install -g [email protected]
 
-    - name: Build Pandas
-      id: build
-      uses: ./.github/actions/build_pandas
-
     - name: Run checks on imported code
       run: ci/code_checks.sh code
       if: ${{ steps.build.outcome == 'success' }}
@@ -94,11 +82,9 @@ jobs:
 
     - name: Run typing validation
       run: ci/code_checks.sh typing
-      if: ${{ steps.build.outcome == 'success' }}
 
     - name: Run docstring validation script tests
       run: pytest scripts
-      if: ${{ steps.build.outcome == 'success' }}
 
   asv-benchmarks:
     name: ASV Benchmarks
@@ -118,24 +104,11 @@ jobs:
       with:
         fetch-depth: 0
 
-    - name: Cache conda
-      uses: actions/cache@v2
-      with:
-        path: ~/conda_pkgs_dir
-        key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }}
-
-    - uses: conda-incubator/setup-miniconda@v2
-      with:
-        mamba-version: "*"
-        channels: conda-forge
-        activate-environment: pandas-dev
-        channel-priority: strict
-        environment-file: ${{ env.ENV_FILE }}
-        use-only-tar-bz2: true
+    - name: Set up Conda
+      uses: ./.github/actions/setup
 
     - name: Build Pandas
-      id: build
-      uses: ./.github/actions/build_pandas
+      uses: ./.github/actions/build-pandas
 
     - name: Run ASV benchmarks
       run: |
@@ -148,7 +121,6 @@ jobs:
         if grep "failed" benchmarks.log > /dev/null ; then
             exit 1
         fi
-      if: ${{ steps.build.outcome == 'success' }}
 
     - name: Publish benchmarks artifact
       uses: actions/upload-artifact@v2

.github/workflows/code-checks.yml

@@ -13,14 +13,21 @@ on:
       - "doc/**"
 
 env:
-  ENV_FILE: environment.yml
   PANDAS_CI: 1
+  PYTEST_WORKERS: "auto"
+  PYTEST_TARGET: pandas
+  PATTERN: "not network and not clipboard and not single_cpu"
+  PANDAS_DATA_MANAGER: array
 
 jobs:
   data_manager:
     name: Test experimental data manager
     runs-on: ubuntu-latest
     timeout-minutes: 120
+    defaults:
+      run:
+        shell: bash -l {0}
+
     services:
       moto:
         image: motoserver/moto
@@ -29,6 +36,7 @@ jobs:
           AWS_SECRET_ACCESS_KEY: foobar_secret
         ports:
           - 5000:5000
+
     concurrency:
       # https://github.community/t/concurrecy-not-work-for-push/183068/7
       group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-data_manager
@@ -40,15 +48,11 @@ jobs:
       with:
         fetch-depth: 0
 
-    - name: Set up pandas
+    - name: Set up Conda
       uses: ./.github/actions/setup
 
+    - name: Build pandas
+      uses: ./.github/actions/build-pandas
+
     - name: Run tests
-      env:
-        PANDAS_DATA_MANAGER: array
-        PATTERN: "not network and not clipboard and not single_cpu"
-        PYTEST_WORKERS: "auto"
-        PYTEST_TARGET: pandas
-      run: |
-        source activate pandas-dev
-        ci/run_tests.sh
+      run: ci/run_tests.sh

.github/workflows/datamanger.yml renamed to .github/workflows/datamanager.yml

@@ -11,13 +11,15 @@ on:
       - 1.4.x
 
 env:
-  ENV_FILE: environment.yml
   PANDAS_CI: 1
 
 jobs:
   web_and_docs:
     name: Doc Build and Upload
     runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -l {0}
 
     concurrency:
       # https://github.community/t/concurrecy-not-work-for-push/183068/7
@@ -30,17 +32,17 @@ jobs:
       with:
         fetch-depth: 0
 
-    - name: Set up pandas
+    - name: Set up Conda
       uses: ./.github/actions/setup
 
+    - name: Build pandas
+      uses: ./.github/actions/build-pandas
+
     - name: Build website
-      run: |
-        source activate pandas-dev
-        python web/pandas_web.py web/pandas --target-path=web/build
+      run: python web/pandas_web.py web/pandas --target-path=web/build
+
     - name: Build documentation
-      run: |
-        source activate pandas-dev
-        doc/make.py --warnings-are-errors
+      run: doc/make.py --warnings-are-errors
 
     - name: Install ssh key
       run: |