From 5d557a76a0cf642087d6ce9e4d7d9794d95a9a3a Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Wed, 16 Oct 2024 15:38:22 +0200 Subject: [PATCH] CI: Add Windows wheels for the free-threaded build --- .gitattributes | 3 ++- .github/workflows/unit-tests.yml | 4 ++-- .github/workflows/wheels.yml | 18 ------------------ MANIFEST.in | 2 ++ pyproject.toml | 15 +++++++-------- scripts/cibw_before_build.sh | 6 +++--- scripts/cibw_before_build_windows.sh | 13 +++++++++++++ scripts/cibw_before_test_windows.sh | 5 +++++ 8 files changed, 34 insertions(+), 32 deletions(-) create mode 100644 scripts/cibw_before_build_windows.sh create mode 100644 scripts/cibw_before_test_windows.sh diff --git a/.gitattributes b/.gitattributes index b3d70ca8b24fb..f77da2339b20f 100644 --- a/.gitattributes +++ b/.gitattributes @@ -85,4 +85,5 @@ pandas/tests/io/parser/data export-ignore # Include cibw script in sdist since it's needed for building wheels scripts/cibw_before_build.sh -export-ignore -scripts/cibw_before_test.sh -export-ignore +scripts/cibw_before_build_windows.sh -export-ignore +scripts/cibw_before_test_windows.sh -export-ignore diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 60b234d613a38..479de16eac2af 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -388,8 +388,8 @@ jobs: - name: Build Environment run: | python --version - python -m pip install --upgrade pip setuptools wheel meson[ninja]==1.2.1 meson-python==0.13.1 - python -m pip install --pre --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy cython + python -m pip install --upgrade pip setuptools wheel numpy meson[ninja]==1.2.1 meson-python==0.13.1 + python -m pip install --pre --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple cython python -m pip install versioneer[toml] python -m pip install python-dateutil pytz tzdata hypothesis>=6.84.0 pytest>=7.3.2 pytest-xdist>=3.4.0 pytest-cov python -m pip install -ve . --no-build-isolation --no-index --no-deps -Csetup-args="--werror" diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index ce48cfa463974..9c9a8f5121b06 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -111,10 +111,6 @@ jobs: - buildplat: [ubuntu-22.04, pyodide_wasm32] python: ["cp312", "3.12"] cibw_build_frontend: 'build' - # TODO: Build free-threaded wheels for Windows - exclude: - - buildplat: [windows-2022, win_amd64] - python: ["cp313t", "3.13"] env: IS_PUSH: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }} @@ -181,20 +177,6 @@ jobs: shell: bash -el {0} run: for whl in $(ls wheelhouse); do wheel unpack wheelhouse/$whl -d /tmp; done - # Testing on windowsservercore instead of GHA runner to fail on missing DLLs - - name: Test Windows Wheels - if: ${{ matrix.buildplat[1] == 'win_amd64' }} - shell: pwsh - run: | - $TST_CMD = @" - python -m pip install hypothesis>=6.84.0 pytest>=7.3.2 pytest-xdist>=3.4.0; - python -m pip install `$(Get-Item pandas\wheelhouse\*.whl); - python -c `'import pandas as pd; pd.test(extra_args=[`\"--no-strict-data-files`\", `\"-m not clipboard and not single_cpu and not slow and not network and not db`\"])`'; - "@ - # add rc to the end of the image name if the Python version is unreleased - docker pull python:${{ matrix.python[1] == '3.13' && '3.13-rc' || format('{0}-windowsservercore', matrix.python[1]) }} - docker run --env PANDAS_CI='1' -v ${PWD}:C:\pandas python:${{ matrix.python[1] == '3.13' && '3.13-rc' || format('{0}-windowsservercore', matrix.python[1]) }} powershell -Command $TST_CMD - - uses: actions/upload-artifact@v4 with: name: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }} diff --git a/MANIFEST.in b/MANIFEST.in index a7d7d7eb4e062..c59151f340545 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -65,3 +65,5 @@ graft pandas/_libs/include # Include cibw script in sdist since it's needed for building wheels include scripts/cibw_before_build.sh +include scripts/cibw_before_build_windows.sh +include scripts/cibw_before_test_windows.sh diff --git a/pyproject.toml b/pyproject.toml index 6dfee8f4910db..0c76ecd0b15b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -160,7 +160,13 @@ free-threaded-support = true before-build = "PACKAGE_DIR={package} bash {package}/scripts/cibw_before_build.sh" [tool.cibuildwheel.windows] -before-build = "pip install delvewheel && bash {package}/scripts/cibw_before_build.sh" +before-build = "pip install delvewheel && bash {package}/scripts/cibw_before_build_windows.sh" +before-test = "bash {package}/scripts/cibw_before_test_windows.sh" +test-command = """ + set PANDAS_CI='1' && \ + python -c "import pandas as pd; \ + pd.test(extra_args=['--no-strict-data-files', '-m not clipboard and not single_cpu and not slow and not network and not db']);" \ + """ repair-wheel-command = "delvewheel repair -w {dest_dir} {wheel}" [[tool.cibuildwheel.overrides]] @@ -175,13 +181,6 @@ test-command = """ select = "*-musllinux*" before-test = "apk update && apk add musl-locales" -[[tool.cibuildwheel.overrides]] -select = "*-win*" -# We test separately for Windows, since we use -# the windowsservercore docker image to check if any dlls are -# missing from the wheel -test-command = "" - [[tool.cibuildwheel.overrides]] # Don't strip wheels on macOS. # macOS doesn't support stripping wheels with linker diff --git a/scripts/cibw_before_build.sh b/scripts/cibw_before_build.sh index 679b91e3280ec..4cdbf8db0ba89 100644 --- a/scripts/cibw_before_build.sh +++ b/scripts/cibw_before_build.sh @@ -5,8 +5,8 @@ done # TODO: Delete when there's a PyPI Cython release that supports free-threaded Python 3.13. FREE_THREADED_BUILD="$(python -c"import sysconfig; print(bool(sysconfig.get_config_var('Py_GIL_DISABLED')))")" -if [[ $FREE_THREADED_BUILD == "True" ]]; then +if [[ $FREE_THREADED_BUILD == "True" ]]; then python -m pip install -U pip - python -m pip install -i https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy cython - python -m pip install ninja meson-python versioneer[toml] + python -m pip install -i https://pypi.anaconda.org/scientific-python-nightly-wheels/simple cython + python -m pip install numpy ninja meson-python versioneer[toml] fi diff --git a/scripts/cibw_before_build_windows.sh b/scripts/cibw_before_build_windows.sh new file mode 100644 index 0000000000000..5153ebd691f3b --- /dev/null +++ b/scripts/cibw_before_build_windows.sh @@ -0,0 +1,13 @@ +# Add 3rd party licenses, like numpy does +for file in $PACKAGE_DIR/LICENSES/*; do + cat $file >> $PACKAGE_DIR/LICENSE +done + +# TODO: Delete when there's a PyPI Cython release that supports free-threaded Python 3.13 +# and a NumPy Windows wheel for the free-threaded build on PyPI. +FREE_THREADED_BUILD="$(python -c"import sysconfig; print(bool(sysconfig.get_config_var('Py_GIL_DISABLED')))")" +if [[ $FREE_THREADED_BUILD == "True" ]]; then + python -m pip install -U pip + python -m pip install -i https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy cython + python -m pip install ninja meson-python versioneer[toml] +fi diff --git a/scripts/cibw_before_test_windows.sh b/scripts/cibw_before_test_windows.sh new file mode 100644 index 0000000000000..dd02bc23dd5a1 --- /dev/null +++ b/scripts/cibw_before_test_windows.sh @@ -0,0 +1,5 @@ +# TODO: Delete when there's a NumPy Windows wheel for the free-threaded build on PyPI. +FREE_THREADED_BUILD="$(python -c"import sysconfig; print(bool(sysconfig.get_config_var('Py_GIL_DISABLED')))")" +if [[ $FREE_THREADED_BUILD == "True" ]]; then + python -m pip install -i https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy +fi