From 4d43b4bddc8fa471364806ab340101239a2b9ceb Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Tue, 4 Oct 2022 16:24:56 -0400 Subject: [PATCH 01/13] REF: Refactor cibuildwheel logic into pyproject.toml --- .github/workflows/wheels.yml | 16 +++------------- pyproject.toml | 24 ++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 17bec7dcd3ad2..4d44bba43555b 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -54,9 +54,8 @@ jobs: # TODO: support PyPy? python: [["cp38", "3.8"], ["cp39", "3.9"], ["cp310", "3.10"], ["cp311", "3.11-dev"]]# "pp38", "pp39"] env: - IS_32_BIT: ${{ matrix.buildplat[1] == 'win32' }} IS_PUSH: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }} - IS_SCHEDULE_DISPATCH: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} + IS_SCHEDULE_DISPATCH: true #${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} steps: - name: Checkout pandas uses: actions/checkout@v3 @@ -72,15 +71,6 @@ jobs: uses: pypa/cibuildwheel@v2.9.0 env: CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }} - CIBW_ENVIRONMENT: IS_32_BIT='${{ env.IS_32_BIT }}' - # We can't test directly with cibuildwheel, since we need to have to wheel location - # to mount into the docker image - CIBW_TEST_COMMAND_LINUX: "python {project}/ci/test_wheels.py" - CIBW_TEST_COMMAND_MACOS: "python {project}/ci/test_wheels.py" - CIBW_TEST_REQUIRES: hypothesis==6.52.1 pytest>=6.2.5 pytest-xdist pytest-asyncio>=0.17 - CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "python ci/fix_wheels.py {wheel} {dest_dir}" - CIBW_ARCHS_MACOS: x86_64 universal2 - CIBW_BUILD_VERBOSITY: 3 # Used to test the built wheels - uses: actions/setup-python@v3 @@ -118,7 +108,7 @@ jobs: - name: Upload wheels if: success() - shell: bash + shell: bash -el {0} env: PANDAS_STAGING_UPLOAD_TOKEN: ${{ secrets.PANDAS_STAGING_UPLOAD_TOKEN }} PANDAS_NIGHTLY_UPLOAD_TOKEN: ${{ secrets.PANDAS_NIGHTLY_UPLOAD_TOKEN }} @@ -195,7 +185,7 @@ jobs: - name: Upload sdist if: success() - shell: bash + shell: bash -el {0} env: PANDAS_STAGING_UPLOAD_TOKEN: ${{ secrets.PANDAS_STAGING_UPLOAD_TOKEN }} PANDAS_NIGHTLY_UPLOAD_TOKEN: ${{ secrets.PANDAS_NIGHTLY_UPLOAD_TOKEN }} diff --git a/pyproject.toml b/pyproject.toml index e5b50a6705a99..b4f605e0af91c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,30 @@ requires = [ # https://github.com/python-versioneer/python-versioneer/issues/193 # build-backend = "setuptools.build_meta" +[tool.cibuildwheel] +skip = "cp36-* cp37-* pp37-* *-manylinux_i686 *_ppc64le *_s390x *-musllinux*" +build-verbosity = "3" +test-requires = "hypothesis==6.52.1 pytest>=6.2.5 pytest-xdist pytest-asyncio>=0.17" +test-command = "python {project}/ci/test_wheels.py" + +[tool.cibuildwheel.macos] +archs = "x86_64 universal2" +test-skip = "*_arm64 *_universal2:arm64" + +[tool.cibuildwheel.windows] +repair-wheel-command = "python ci/fix_wheels.py {wheel} {dest_dir}" + +[[tool.cibuildwheel.overrides]] +select = "*-win*" +# We test separately for Windows, since we use +# the base windows docker image to check if any dlls are +# missing from the wheel +test-command = "" + +[[tool.cibuildwheel.overrides]] +select = "*-win32" +environment = { IS_32_BIT="true" } + [tool.black] target-version = ['py38', 'py39'] exclude = ''' From f414bd90b1989446e44e95d54703a79a34f95d17 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Tue, 4 Oct 2022 18:19:22 -0400 Subject: [PATCH 02/13] BLD: Build arm64 wheels on CircleCI --- .circleci/config.yml | 65 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6133037bf3b7d..0ade38704ae1a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,8 +15,71 @@ jobs: - checkout - run: .circleci/setup_env.sh - run: PATH=$HOME/miniconda3/envs/pandas-dev/bin:$HOME/miniconda3/condabin:$PATH ci/run_tests.sh - + build-aarch64-38: + machine: + image: ubuntu-2004:202101-01 + resource_class: arm.large + steps: + - checkout + - run: + name: Build Aarch64 wheels (3.8) + command: | + pip3 install cibuildwheel==2.9.0 + cibuildwheel --output-dir wheelhouse + environment: + CIBW_BUILD: cp38-manylinux_aarch64 + - store_artifacts: + path: wheelhouse/ + build-aarch64-39: + machine: + image: ubuntu-2004:202101-01 + resource_class: arm.large + steps: + - checkout + - run: + name: Build Aarch64 wheels (3.9) + command: | + pip3 install cibuildwheel==2.9.0 + cibuildwheel --output-dir wheelhouse + environment: + CIBW_BUILD: cp39-manylinux_aarch64 + - store_artifacts: + path: wheelhouse/ + build-aarch64-310: + machine: + image: ubuntu-2004:202101-01 + resource_class: arm.large + steps: + - checkout + - run: + name: Build Aarch64 wheels (3.10) + command: | + pip3 install cibuildwheel==2.9.0 + cibuildwheel --output-dir wheelhouse + environment: + CIBW_BUILD: cp310-manylinux_aarch64 + - store_artifacts: + path: wheelhouse/ + build-aarch64-311: + machine: + image: ubuntu-2004:202101-01 + resource_class: arm.large + steps: + - checkout + - run: + name: Build Aarch64 wheels (3.11) + command: | + pip3 install cibuildwheel==2.9.0 + cibuildwheel --output-dir wheelhouse + environment: + CIBW_BUILD: cp311-manylinux_aarch64 + - store_artifacts: + path: wheelhouse/ workflows: test: jobs: - test-arm + - build-aarch64-38 + - build-aarch64-39 + - build-aarch64-310 + - build-aarch64-311 From f4e9678983ee56d929cc1a6b2fde50042fdb44ce Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sat, 26 Nov 2022 18:59:04 -0500 Subject: [PATCH 03/13] Update wheels.yml --- .github/workflows/wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 98d59936fb3ad..b0ff6a3110f6a 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -55,7 +55,7 @@ jobs: python: [["cp38", "3.8"], ["cp39", "3.9"], ["cp310", "3.10"], ["cp311", "3.11"]]# "pp38", "pp39"] env: IS_PUSH: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }} - IS_SCHEDULE_DISPATCH: true #${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} + IS_SCHEDULE_DISPATCH: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} steps: - name: Checkout pandas uses: actions/checkout@v3 From d75c7b7d9b968f47fe846f7a2daa627c4b287db5 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sun, 22 Jan 2023 08:17:22 -0800 Subject: [PATCH 04/13] Update config.yml --- .circleci/config.yml | 74 +++++++++++++++----------------------------- 1 file changed, 25 insertions(+), 49 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0ade38704ae1a..c66e21469c80e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,10 +15,13 @@ jobs: - checkout - run: .circleci/setup_env.sh - run: PATH=$HOME/miniconda3/envs/pandas-dev/bin:$HOME/miniconda3/condabin:$PATH ci/run_tests.sh - build-aarch64-38: + build-aarch: machine: image: ubuntu-2004:202101-01 resource_class: arm.large + environment: + TAG = << pipeline.git.tag >> + TRIGGER_SOURCE = << pipeline.trigger_source >> steps: - checkout - run: @@ -27,59 +30,32 @@ jobs: pip3 install cibuildwheel==2.9.0 cibuildwheel --output-dir wheelhouse environment: - CIBW_BUILD: cp38-manylinux_aarch64 - - store_artifacts: - path: wheelhouse/ - build-aarch64-39: - machine: - image: ubuntu-2004:202101-01 - resource_class: arm.large - steps: - - checkout - - run: - name: Build Aarch64 wheels (3.9) - command: | - pip3 install cibuildwheel==2.9.0 - cibuildwheel --output-dir wheelhouse - environment: - CIBW_BUILD: cp39-manylinux_aarch64 - - store_artifacts: - path: wheelhouse/ - build-aarch64-310: - machine: - image: ubuntu-2004:202101-01 - resource_class: arm.large - steps: - - checkout - - run: - name: Build Aarch64 wheels (3.10) - command: | - pip3 install cibuildwheel==2.9.0 - cibuildwheel --output-dir wheelhouse - environment: - CIBW_BUILD: cp310-manylinux_aarch64 - - store_artifacts: - path: wheelhouse/ - build-aarch64-311: - machine: - image: ubuntu-2004:202101-01 - resource_class: arm.large - steps: - - checkout + CIBW_BUILD: << parameters.CIBW_BUILD >> - run: - name: Build Aarch64 wheels (3.11) + name: Upload wheels command: | - pip3 install cibuildwheel==2.9.0 - cibuildwheel --output-dir wheelhouse - environment: - CIBW_BUILD: cp311-manylinux_aarch64 + if ! [[ -z ${TAG} ]]; then + export IS_PUSH="true" + fi + if [[ $TRIGGER_SOURCE == "scheduled_pipeline"]]; then + export IS_SCHEDULE_DISPATCH="true" + fi + source ci/upload_wheels.sh + set_upload_vars + upload_wheels - store_artifacts: path: wheelhouse/ workflows: test: + # Don't run trigger this one when scheduled pipeline runs + when: + not: + equal: [ scheduled_pipeline, << pipeline.trigger_source >> ] jobs: - test-arm - - build-aarch64-38 - - build-aarch64-39 - - build-aarch64-310 - - build-aarch64-311 + build-wheels: + jobs: + - build-aarch64 + matrix: + parameters: + CIBW_BUILD: ["cp38-manylinux_aarch64", "cp39-manylinux_aarch64", "cp310-manylinux_aarch64", "cp311-manylinux_aarch64"] From eb0d89f0b9cd2a288afe65b9d5b7d50882e434a3 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sun, 22 Jan 2023 08:20:17 -0800 Subject: [PATCH 05/13] fix syntax --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c66e21469c80e..6568c516cb56f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -55,7 +55,7 @@ workflows: - test-arm build-wheels: jobs: - - build-aarch64 + - build-aarch64: matrix: parameters: CIBW_BUILD: ["cp38-manylinux_aarch64", "cp39-manylinux_aarch64", "cp310-manylinux_aarch64", "cp311-manylinux_aarch64"] From 45d792724fdf4ddb14c210327a53f532f982ff89 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sun, 22 Jan 2023 08:20:41 -0800 Subject: [PATCH 06/13] fix typo --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6568c516cb56f..7bf700e732c01 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,7 +15,7 @@ jobs: - checkout - run: .circleci/setup_env.sh - run: PATH=$HOME/miniconda3/envs/pandas-dev/bin:$HOME/miniconda3/condabin:$PATH ci/run_tests.sh - build-aarch: + build-aarch64: machine: image: ubuntu-2004:202101-01 resource_class: arm.large From abf36b6023dac590622abd1cf5a4b03012327cc2 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sun, 22 Jan 2023 08:25:46 -0800 Subject: [PATCH 07/13] Update config.yml --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7bf700e732c01..9a0e51c3bb3d7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -30,7 +30,7 @@ jobs: pip3 install cibuildwheel==2.9.0 cibuildwheel --output-dir wheelhouse environment: - CIBW_BUILD: << parameters.CIBW_BUILD >> + CIBW_BUILD: << parameters.cibw-build >> - run: name: Upload wheels command: | @@ -58,4 +58,4 @@ workflows: - build-aarch64: matrix: parameters: - CIBW_BUILD: ["cp38-manylinux_aarch64", "cp39-manylinux_aarch64", "cp310-manylinux_aarch64", "cp311-manylinux_aarch64"] + cibw-build: ["cp38-manylinux_aarch64", "cp39-manylinux_aarch64", "cp310-manylinux_aarch64", "cp311-manylinux_aarch64"] From 64a2c7be0c63d1f66721fb7150b5bcd39a17b3fb Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sun, 22 Jan 2023 08:28:59 -0800 Subject: [PATCH 08/13] define parameters --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9a0e51c3bb3d7..83ad0187143df 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,6 +16,9 @@ jobs: - run: .circleci/setup_env.sh - run: PATH=$HOME/miniconda3/envs/pandas-dev/bin:$HOME/miniconda3/condabin:$PATH ci/run_tests.sh build-aarch64: + parameters: + cibw-build: + type: string machine: image: ubuntu-2004:202101-01 resource_class: arm.large From 3b53f3c5a221d6409b92942efda3a6477fc66461 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sun, 22 Jan 2023 09:58:19 -0800 Subject: [PATCH 09/13] fix shell script syntax --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 83ad0187143df..7fb0d82e0e85a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -37,10 +37,10 @@ jobs: - run: name: Upload wheels command: | - if ! [[ -z ${TAG} ]]; then + if [[ -n ${TAG} ]]; then export IS_PUSH="true" fi - if [[ $TRIGGER_SOURCE == "scheduled_pipeline"]]; then + if [[ $TRIGGER_SOURCE == "scheduled_pipeline" ]]; then export IS_SCHEDULE_DISPATCH="true" fi source ci/upload_wheels.sh From 1611f83f8eb4552cc0c429a34d4079c4a4eb2510 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 8 Feb 2023 18:00:55 -0500 Subject: [PATCH 10/13] try to run less --- .circleci/config.yml | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f79deff8c2d3e..27c0e6196bb18 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -31,7 +31,19 @@ jobs: steps: - checkout - run: - name: Build Aarch64 wheels (3.8) + name: Check if build is necessary + command: | + # Check if tag is defined or TRIGGER_SOURCE is scheduled + if [[ -n ${TAG} ]]; then + export IS_PUSH="true" + elif [[ $TRIGGER_SOURCE == "scheduled_pipeline" ]]; then + export IS_SCHEDULE_DISPATCH="true" + # Look for the build label + elif ! (curl https://api.github.com/repos/pandas-dev/pandas/issues/$CIRCLE_PR_NUMBER | grep -q 'Build'); then + circleci-agent step halt + fi + - run: + name: Build aarch64 wheels command: | pip3 install cibuildwheel==2.9.0 cibuildwheel --output-dir wheelhouse @@ -63,6 +75,9 @@ workflows: build-wheels: jobs: - build-aarch64: + filters: + tags: + only: /^v.*/ matrix: parameters: cibw-build: ["cp38-manylinux_aarch64", "cp39-manylinux_aarch64", "cp310-manylinux_aarch64", "cp311-manylinux_aarch64"] From e624cdbb9764787c197842591ad0a7f21616d098 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Thu, 9 Feb 2023 07:24:07 -0500 Subject: [PATCH 11/13] Update config.yml --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 27c0e6196bb18..1bd2cd712f490 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -39,7 +39,7 @@ jobs: elif [[ $TRIGGER_SOURCE == "scheduled_pipeline" ]]; then export IS_SCHEDULE_DISPATCH="true" # Look for the build label - elif ! (curl https://api.github.com/repos/pandas-dev/pandas/issues/$CIRCLE_PR_NUMBER | grep -q 'Build'); then + elif ! (curl https://api.github.com/repos/pandas-dev/pandas/issues/$CIRCLE_PR_NUMBER | jq '.labels' | grep -q 'Build'); then circleci-agent step halt fi - run: From 28052d92a9df626770a3f835ad0c33661c4b1249 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Fri, 10 Feb 2023 07:18:51 -0500 Subject: [PATCH 12/13] Update config.yml --- .circleci/config.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1bd2cd712f490..f1720533a1b93 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -38,7 +38,9 @@ jobs: export IS_PUSH="true" elif [[ $TRIGGER_SOURCE == "scheduled_pipeline" ]]; then export IS_SCHEDULE_DISPATCH="true" - # Look for the build label + # Look for the build label/[wheel build] in commit + elif (git log --format=oneline -n 1 $CIRCLE_SHA1) | grep -q '[wheel build]'; ; then + : # Do nothing elif ! (curl https://api.github.com/repos/pandas-dev/pandas/issues/$CIRCLE_PR_NUMBER | jq '.labels' | grep -q 'Build'); then circleci-agent step halt fi From 7609b46d0a3b7dcb3e7c8ac1972179f37c475277 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Fri, 10 Feb 2023 12:10:23 -0500 Subject: [PATCH 13/13] Update config.yml --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f1720533a1b93..772a1c8821dcf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -39,7 +39,7 @@ jobs: elif [[ $TRIGGER_SOURCE == "scheduled_pipeline" ]]; then export IS_SCHEDULE_DISPATCH="true" # Look for the build label/[wheel build] in commit - elif (git log --format=oneline -n 1 $CIRCLE_SHA1) | grep -q '[wheel build]'; ; then + elif (git log --format=oneline -n 1 $CIRCLE_SHA1) | grep -q '[wheel build]'; then : # Do nothing elif ! (curl https://api.github.com/repos/pandas-dev/pandas/issues/$CIRCLE_PR_NUMBER | jq '.labels' | grep -q 'Build'); then circleci-agent step halt