CI: Add ccache

.circleci/setup_env.sh

@@ -54,11 +54,7 @@ if pip list | grep -q ^pandas; then
     pip uninstall -y pandas || true
 fi
 
-echo "Build extensions"
-# GH 47305: Parallel build can causes flaky ImportError from pandas/_libs/tslibs
-python setup.py build_ext -q -j1
-
 echo "Install pandas"
-python -m pip install --no-build-isolation --no-use-pep517 -e .
+python -m pip install --no-build-isolation -ve .
 
 echo "done"

.github/actions/build-pandas/action.yml

@@ -0,0 +1,22 @@
+# TODO: merge setup-ccache, setup-conda, build-pandas into a single action?
+name: Build pandas
+description: Rebuilds the C extensions and installs pandas
+runs:
+  using: composite
+  steps:
+    - name: Set up Ccache
+      uses: ./.github/actions/setup-ccache
+
+    - name: Build Pandas
+      if : ${{ runner.os != 'Windows' }}
+      run: |
+        python -m pip install -ve . --no-build-isolation
+      shell: bash -el {0}
+
+    - name: Build Pandas (Windows)
+      if: ${{ runner.os == 'Windows' }}
+      run: |
+        call micromamba activate test
+        call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat"
+        python -m pip install -ve . --no-build-isolation
+      shell: cmd /C call {0}

.github/actions/setup-ccache/action.yml

@@ -0,0 +1,33 @@
+name: Setup sccache
+runs:
+  using: composite
+  steps:
+    - name: Make cache key
+      id: cache-key
+      run: |
+        key="${{ runner.os }}--${{ runner.arch }}--${{ github.workflow }}"
+        # Date: Daily invalidation of all ccaches as an extra safety measure.
+        key="$key--$(/bin/date -u '+%Y%m%d')"
+        # Python version: Separate caches for each Python version. This reduces the number of cache misses.
+        key="$key--$(python -V)"
+        # Cache version: Bump this number to manually invalidate the cache.
+        key="$key--0"
+
+        echo "cache-key=$key" >> $GITHUB_OUTPUT
+      shell: bash
+
+    # On Windows, for some reason the default temporary directory provided to sccache
+    # may become read-only at some point. Work around by having a private tempdir.
+    - name: Fix Windows temporary directory
+      id: mktemp
+      run: echo "tmpdir=$(cygpath -w $(mktemp -d))" >> $GITHUB_OUTPUT
+      shell: bash
+      if: ${{ runner.os == 'Windows' }}
+
+    - name: Setup sccache
+      uses: hendrikmuhs/[email protected]
+      with:
+        variant: sccache
+        key: ${{ steps.cache-key.outputs.cache-key }}
+      env:
+        TMP: "${{ steps.mktemp.outputs.tmpdir }}"

.github/actions/setup-conda/action.yml

@@ -3,12 +3,6 @@ inputs:
   environment-file:
     description: Conda environment file to use.
     default: environment.yml
-  environment-name:
-    description: Name to use for the Conda environment
-    default: test
-  extra-specs:
-    description: Extra packages to install
-    required: false
   pyarrow-version:
     description: If set, overrides the PyArrow version in the Conda environment to the given string.
     required: false
@@ -19,16 +13,17 @@ runs:
       run: |
         grep -q '  - pyarrow' ${{ inputs.environment-file }}
         sed -i"" -e "s/  - pyarrow/  - pyarrow=${{ inputs.pyarrow-version }}/" ${{ inputs.environment-file }}
+        echo ::group::Patched environment.yml contents
         cat ${{ inputs.environment-file }}
+        echo ::endgroup::
       shell: bash
       if: ${{ inputs.pyarrow-version }}
 
     - name: Install ${{ inputs.environment-file }}
       uses: mamba-org/provision-with-micromamba@v12
       with:
         environment-file: ${{ inputs.environment-file }}
-        environment-name: ${{ inputs.environment-name }}
-        extra-specs: ${{ inputs.extra-specs }}
+        environment-name: test
         channels: conda-forge
         channel-priority: ${{ runner.os == 'macOS' && 'flexible' || 'strict' }}
         condarc-file: ci/condarc.yml

.github/workflows/32-bit-linux.yml

@@ -38,13 +38,14 @@ jobs:
           /opt/python/cp38-cp38/bin/python -m venv ~/virtualenvs/pandas-dev && \
           . ~/virtualenvs/pandas-dev/bin/activate && \
           python -m pip install --no-deps -U pip wheel 'setuptools<60.0.0' && \
+          pip install cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio>=0.17 hypothesis && \
+          pip install "meson[ninja] @ git+https://github.com/mesonbuild/meson.git@master" && \
+          pip install "git+https://github.com/mesonbuild/meson-python.git@main" && \
           python -m pip install versioneer[toml] && \
-          python -m pip install cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio>=0.17 hypothesis && \
-          python setup.py build_ext -q -j1 && \
-          python -m pip install --no-build-isolation --no-use-pep517 -e . && \
-          python -m pip list && \
           export PANDAS_CI=1 && \
-          pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml"
+          python -m pip install --no-build-isolation -ve . && \
+          python -m pip list && \
+          pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml --import-mode=importlib"
 
       - name: Publish test results for Python 3.8-32 bit full Linux
         uses: actions/upload-artifact@v3

.github/workflows/code-checks.yml

@@ -62,7 +62,7 @@ jobs:
 
     - name: Build Pandas
       id: build
-      uses: ./.github/actions/build_pandas
+      uses: ./.github/actions/build-pandas
 
     # The following checks are independent of each other and should still be run if one fails
     - name: Check for no warnings when building single-page docs
@@ -125,7 +125,7 @@ jobs:
 
     - name: Build Pandas
       id: build
-      uses: ./.github/actions/build_pandas
+      uses: ./.github/actions/build-pandas
 
     - name: Run ASV benchmarks
       run: |

.github/workflows/docbuild-and-upload.yml

@@ -44,7 +44,7 @@ jobs:
       uses: ./.github/actions/setup-conda
 
     - name: Build Pandas
-      uses: ./.github/actions/build_pandas
+      uses: ./.github/actions/build-pandas
 
     - name: Set up maintainers cache
       uses: actions/cache@v3

.github/workflows/macos-windows.yml

@@ -56,7 +56,7 @@ jobs:
         pyarrow-version: ${{ matrix.os == 'macos-latest' && '9' || '' }}
 
     - name: Build Pandas
-      uses: ./.github/actions/build_pandas
+      uses: ./.github/actions/build-pandas
 
     - name: Test
       uses: ./.github/actions/run-tests

.github/workflows/package-checks.yml

@@ -42,11 +42,15 @@ jobs:
 
       - name: Install required dependencies
         run: |
-          python -m pip install --upgrade pip setuptools wheel python-dateutil pytz numpy cython
+          # TODO: Remove when we fully migrate to meson
+          # since the PEP 517 build will pull build dependencies automatically
+          python -m pip install --upgrade pip wheel python-dateutil pytz numpy cython
+          python -m pip install "meson[ninja] @ git+https://github.com/mesonbuild/meson.git@master"
+          python -m pip install git+https://github.com/mesonbuild/meson-python.git@main
           python -m pip install versioneer[toml]
         shell: bash -el {0}
 
       - name: Pip install with extra
         run: |
-          python -m pip install -e .[${{ matrix.extra }}] --no-build-isolation
+          python -m pip install .[${{ matrix.extra }}] -v --no-build-isolation
         shell: bash -el {0}

.github/workflows/python-dev.yml

@@ -75,15 +75,30 @@ jobs:
         python -m pip install --upgrade pip setuptools wheel
         python -m pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy
         python -m pip install git+https://github.com/nedbat/coveragepy.git
+        python -m pip install python-dateutil pytz cython
+        # TODO: update when upstream releases fixes
+        python -m pip install "meson[ninja] @ git+https://github.com/mesonbuild/meson.git@master"
+        python -m pip install "git+https://github.com/mesonbuild/meson-python.git@main"
+        python -m pip install hypothesis==6.52.1 pytest>=6.2.5 pytest-xdist pytest-cov pytest-asyncio>=0.17
         python -m pip install versioneer[toml]
-        python -m pip install python-dateutil pytz cython hypothesis==6.52.1 pytest>=6.2.5 pytest-xdist pytest-cov pytest-asyncio>=0.17
         python -m pip list
 
-    # GH 47305: Parallel build can cause flaky ImportError from pandas/_libs/tslibs
+      # Sigh, someone (numpy?) is depending on mingw, which pandas doesn't compile with.
+      # Also, meson doesn't detect visual c++ unless cl.exe is in path.
+      # TODO: File a bug with meson-python about this.
     - name: Build Pandas
+      if : ${{ runner.os != 'Windows' }}
       run: |
-        python setup.py build_ext -q -j1
-        python -m pip install -e . --no-build-isolation --no-use-pep517 --no-index
+        python3 -m pip install -ve . --no-build-isolation
+      shell: bash -el {0}
+
+    - name: Build Pandas (Windows)
+      if: ${{ runner.os == 'Windows' }}
+      run: |
+        call micromamba activate test
+        call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat"
+        python -m pip install -ve . --no-build-isolation
+      shell: cmd /C call {0}
 
     - name: Build Version
       run: |

.github/workflows/sdist.yml

@@ -46,30 +46,29 @@ jobs:
 
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip setuptools wheel
+        python -m pip install --upgrade pip setuptools wheel build
         python -m pip install versioneer[toml]
 
-        # GH 39416
-        pip install numpy
-
     - name: Build pandas sdist
       run: |
         pip list
-        python setup.py sdist --formats=gztar
+        python -m build --sdist
 
     - name: Upload sdist artifact
       uses: actions/upload-artifact@v3
       with:
         name: ${{matrix.python-version}}-sdist.gz
         path: dist/*.gz
 
-    - name: Set up Conda
-      uses: ./.github/actions/setup-conda
+    - name: Set up empty Conda environment
+      uses: mamba-org/provision-with-micromamba@v12
       with:
         environment-file: false
-        environment-name: pandas-sdist
+        environment-name: sdist
         extra-specs: |
           python =${{ matrix.python-version }}
+        channels: conda-forge
+        condarc-file: ci/condarc.yml
 
     - name: Install pandas from sdist
       run: |

.github/workflows/ubuntu.yml

@@ -27,58 +27,60 @@ jobs:
     timeout-minutes: 180
     strategy:
       matrix:
-        env_file: [actions-38.yaml, actions-39.yaml, actions-310.yaml]
+        #env_file: [actions-38.yaml, actions-39.yaml, actions-310.yaml]
+        env_file: [actions-38.yaml]
         pattern: ["not single_cpu", "single_cpu"]
-        pyarrow_version: ["7", "8", "9", "10"]
-        include:
-          - name: "Downstream Compat"
-            env_file: actions-38-downstream_compat.yaml
-            pattern: "not slow and not network and not single_cpu"
-            pytest_target: "pandas/tests/test_downstream.py"
-          - name: "Minimum Versions"
-            env_file: actions-38-minimum_versions.yaml
-            pattern: "not slow and not network and not single_cpu"
-            error_on_warnings: "0"
-          - name: "Locale: it_IT"
-            env_file: actions-38.yaml
-            pattern: "not slow and not network and not single_cpu"
-            extra_apt: "language-pack-it"
-            # Use the utf8 version as the default, it has no bad side-effect.
-            lang: "it_IT.utf8"
-            lc_all: "it_IT.utf8"
-            # Also install it_IT (its encoding is ISO8859-1) but do not activate it.
-            # It will be temporarily activated during tests with locale.setlocale
-            extra_loc: "it_IT"
-          - name: "Locale: zh_CN"
-            env_file: actions-38.yaml
-            pattern: "not slow and not network and not single_cpu"
-            extra_apt: "language-pack-zh-hans"
-            # Use the utf8 version as the default, it has no bad side-effect.
-            lang: "zh_CN.utf8"
-            lc_all: "zh_CN.utf8"
-            # Also install zh_CN (its encoding is gb2312) but do not activate it.
-            # It will be temporarily activated during tests with locale.setlocale
-            extra_loc: "zh_CN"
-          - name: "Copy-on-Write"
-            env_file: actions-310.yaml
-            pattern: "not slow and not network and not single_cpu"
-            pandas_copy_on_write: "1"
-            error_on_warnings: "0"
-          - name: "Data Manager"
-            env_file: actions-38.yaml
-            pattern: "not slow and not network and not single_cpu"
-            pandas_data_manager: "array"
-            error_on_warnings: "0"
-          - name: "Pypy"
-            env_file: actions-pypy-38.yaml
-            pattern: "not slow and not network and not single_cpu"
-            test_args: "--max-worker-restart 0"
-            error_on_warnings: "0"
-          - name: "Numpy Dev"
-            env_file: actions-310-numpydev.yaml
-            pattern: "not slow and not network and not single_cpu"
-            test_args: "-W error::DeprecationWarning:numpy -W error::FutureWarning:numpy"
-            error_on_warnings: "0"
+        #pyarrow_version: ["7", "8", "9", "10"]
+        pyarrow_version: ["10"]
+        #include:
+        #  - name: "Downstream Compat"
+        #    env_file: actions-38-downstream_compat.yaml
+        #    pattern: "not slow and not network and not single_cpu"
+        #    pytest_target: "pandas/tests/test_downstream.py"
+        #  - name: "Minimum Versions"
+        #    env_file: actions-38-minimum_versions.yaml
+        #    pattern: "not slow and not network and not single_cpu"
+        #    error_on_warnings: "0"
+        #  - name: "Locale: it_IT"
+        #    env_file: actions-38.yaml
+        #    pattern: "not slow and not network and not single_cpu"
+        #    extra_apt: "language-pack-it"
+        #    # Use the utf8 version as the default, it has no bad side-effect.
+        #    lang: "it_IT.utf8"
+        #    lc_all: "it_IT.utf8"
+        #    # Also install it_IT (its encoding is ISO8859-1) but do not activate it.
+        #    # It will be temporarily activated during tests with locale.setlocale
+        #    extra_loc: "it_IT"
+        #  - name: "Locale: zh_CN"
+        #    env_file: actions-38.yaml
+        #    pattern: "not slow and not network and not single_cpu"
+        #    extra_apt: "language-pack-zh-hans"
+        #    # Use the utf8 version as the default, it has no bad side-effect.
+        #    lang: "zh_CN.utf8"
+        #    lc_all: "zh_CN.utf8"
+        #    # Also install zh_CN (its encoding is gb2312) but do not activate it.
+        #    # It will be temporarily activated during tests with locale.setlocale
+        #    extra_loc: "zh_CN"
+        #  - name: "Copy-on-Write"
+        #    env_file: actions-310.yaml
+        #    pattern: "not slow and not network and not single_cpu"
+        #    pandas_copy_on_write: "1"
+        #    error_on_warnings: "0"
+        #  - name: "Data Manager"
+        #    env_file: actions-38.yaml
+        #    pattern: "not slow and not network and not single_cpu"
+        #    pandas_data_manager: "array"
+        #    error_on_warnings: "0"
+        #  - name: "Pypy"
+        #    env_file: actions-pypy-38.yaml
+        #    pattern: "not slow and not network and not single_cpu"
+        #    test_args: "--max-worker-restart 0"
+        #    error_on_warnings: "0"
+        #  - name: "Numpy Dev"
+        #    env_file: actions-310-numpydev.yaml
+        #    pattern: "not slow and not network and not single_cpu"
+        #    test_args: "-W error::DeprecationWarning:numpy -W error::FutureWarning:numpy"
+        #    error_on_warnings: "0"
         exclude:
           - env_file: actions-38.yaml
             pyarrow_version: "7"
@@ -173,7 +175,7 @@ jobs:
         pyarrow-version: ${{ matrix.pyarrow_version }}
 
     - name: Build Pandas
-      uses: ./.github/actions/build_pandas
+      uses: ./.github/actions/build-pandas
 
     - name: Test
       uses: ./.github/actions/run-tests