Skip to content

Commit 00ef7c6

Browse files
lithomas1mroeschke
authored and
noatamir
committed
BLD: Build wheels using cibuildwheel (pandas-dev#48283)
* BLD: Build wheels using cibuildwheel * update from code review Co-Authored-By: Matthew Roeschke <[email protected]> * fix 3.11 version * changes from code review * Update test_wheels.py * sync run time with pandas-wheels Co-authored-by: Matthew Roeschke <[email protected]>
1 parent 21ce807 commit 00ef7c6

File tree

6 files changed

+345
-0
lines changed

6 files changed

+345
-0
lines changed

‎.github/workflows/wheels.yml

+180
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,180 @@
1+
# Workflow to build wheels for upload to PyPI.
2+
# Inspired by numpy's cibuildwheel config https://github.com/numpy/numpy/blob/main/.github/workflows/wheels.yml
3+
#
4+
# In an attempt to save CI resources, wheel builds do
5+
# not run on each push but only weekly and for releases.
6+
# Wheel builds can be triggered from the Actions page
7+
# (if you have the perms) on a commit to master.
8+
#
9+
# Alternatively, you can add labels to the pull request in order to trigger wheel
10+
# builds.
11+
# The label(s) that trigger builds are:
12+
# - Build
13+
name: Wheel builder
14+
15+
on:
16+
schedule:
17+
# ┌───────────── minute (0 - 59)
18+
# │ ┌───────────── hour (0 - 23)
19+
# │ │ ┌───────────── day of the month (1 - 31)
20+
# │ │ │ ┌───────────── month (1 - 12 or JAN-DEC)
21+
# │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT)
22+
# │ │ │ │ │
23+
- cron: "27 3 */1 * *"
24+
push:
25+
pull_request:
26+
types: [labeled, opened, synchronize, reopened]
27+
workflow_dispatch:
28+
29+
concurrency:
30+
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
31+
cancel-in-progress: true
32+
33+
jobs:
34+
build_wheels:
35+
name: Build wheel for ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}
36+
if: >-
37+
github.event_name == 'schedule' ||
38+
github.event_name == 'workflow_dispatch' ||
39+
(github.event_name == 'pull_request' &&
40+
contains(github.event.pull_request.labels.*.name, 'Build')) ||
41+
(github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && ( ! endsWith(github.ref, 'dev0')))
42+
runs-on: ${{ matrix.buildplat[0] }}
43+
strategy:
44+
# Ensure that a wheel builder finishes even if another fails
45+
fail-fast: false
46+
matrix:
47+
# Github Actions doesn't support pairing matrix values together, let's improvise
48+
# https://github.com/github/feedback/discussions/7835#discussioncomment-1769026
49+
buildplat:
50+
- [ubuntu-20.04, manylinux_x86_64]
51+
- [macos-11, macosx_*]
52+
- [windows-2019, win_amd64]
53+
- [windows-2019, win32]
54+
# TODO: support PyPy?
55+
python: [["cp38", "3.8"], ["cp39", "3.9"], ["cp310", "3.10"], ["cp311", "3.11-dev"]]# "pp38", "pp39"]
56+
env:
57+
IS_32_BIT: ${{ matrix.buildplat[1] == 'win32' }}
58+
IS_PUSH: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }}
59+
IS_SCHEDULE_DISPATCH: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
60+
steps:
61+
- name: Checkout pandas
62+
uses: actions/checkout@v3
63+
with:
64+
submodules: true
65+
# versioneer.py requires the latest tag to be reachable. Here we
66+
# fetch the complete history to get access to the tags.
67+
# A shallow clone can work when the following issue is resolved:
68+
# https://github.com/actions/checkout/issues/338
69+
fetch-depth: 0
70+
71+
- name: Build wheels
72+
uses: pypa/[email protected]
73+
env:
74+
CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}
75+
CIBW_ENVIRONMENT: IS_32_BIT='${{ env.IS_32_BIT }}'
76+
# We can't test directly with cibuildwheel, since we need to have to wheel location
77+
# to mount into the docker image
78+
CIBW_TEST_COMMAND_LINUX: "python {project}/ci/test_wheels.py"
79+
CIBW_TEST_COMMAND_MACOS: "python {project}/ci/test_wheels.py"
80+
CIBW_TEST_REQUIRES: hypothesis==6.52.1 pytest>=6.2.5 pytest-xdist pytest-asyncio>=0.17
81+
CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "python ci/fix_wheels.py {wheel} {dest_dir}"
82+
CIBW_ARCHS_MACOS: x86_64 universal2
83+
CIBW_BUILD_VERBOSITY: 3
84+
85+
# Used to push the built wheels
86+
- uses: actions/setup-python@v3
87+
with:
88+
python-version: ${{ matrix.python[1] }}
89+
90+
- name: Test wheels (Windows 64-bit only)
91+
if: ${{ matrix.buildplat[1] == 'win_amd64' }}
92+
shell: cmd
93+
run: |
94+
python ci/test_wheels.py wheelhouse
95+
96+
- uses: actions/upload-artifact@v3
97+
with:
98+
name: ${{ matrix.python[0] }}-${{ startsWith(matrix.buildplat[1], 'macosx') && 'macosx' || matrix.buildplat[1] }}
99+
path: ./wheelhouse/*.whl
100+
101+
- name: Upload wheels
102+
if: success()
103+
shell: bash
104+
env:
105+
PANDAS_STAGING_UPLOAD_TOKEN: ${{ secrets.PANDAS_STAGING_UPLOAD_TOKEN }}
106+
PANDAS_NIGHTLY_UPLOAD_TOKEN: ${{ secrets.PANDAS_NIGHTLY_UPLOAD_TOKEN }}
107+
run: |
108+
source ci/upload_wheels.sh
109+
set_upload_vars
110+
# trigger an upload to
111+
# https://anaconda.org/scipy-wheels-nightly/pandas
112+
# for cron jobs or "Run workflow" (restricted to main branch).
113+
# Tags will upload to
114+
# https://anaconda.org/multibuild-wheels-staging/pandas
115+
# The tokens were originally generated at anaconda.org
116+
upload_wheels
117+
build_sdist:
118+
name: Build sdist
119+
if: >-
120+
github.event_name == 'schedule' ||
121+
github.event_name == 'workflow_dispatch' ||
122+
(github.event_name == 'pull_request' &&
123+
contains(github.event.pull_request.labels.*.name, 'Build')) ||
124+
(github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && ( ! endsWith(github.ref, 'dev0')))
125+
runs-on: ubuntu-latest
126+
env:
127+
IS_PUSH: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }}
128+
IS_SCHEDULE_DISPATCH: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
129+
steps:
130+
- name: Checkout pandas
131+
uses: actions/checkout@v3
132+
with:
133+
submodules: true
134+
# versioneer.py requires the latest tag to be reachable. Here we
135+
# fetch the complete history to get access to the tags.
136+
# A shallow clone can work when the following issue is resolved:
137+
# https://github.com/actions/checkout/issues/338
138+
fetch-depth: 0
139+
# Used to push the built wheels
140+
- uses: actions/setup-python@v3
141+
with:
142+
# Build sdist on lowest supported Python
143+
python-version: '3.8'
144+
- name: Build sdist
145+
run: |
146+
pip install build
147+
python -m build --sdist
148+
- name: Test the sdist
149+
run: |
150+
# TODO: Don't run test suite, and instead build wheels from sdist
151+
# by splitting the wheel builders into a two stage job
152+
# (1. Generate sdist 2. Build wheels from sdist)
153+
# This tests the sdists, and saves some build time
154+
python -m pip install dist/*.gz
155+
pip install hypothesis==6.52.1 pytest>=6.2.5 pytest-xdist pytest-asyncio>=0.17
156+
cd .. # Not a good idea to test within the src tree
157+
python -c "import pandas; print(pandas.__version__);
158+
pandas.test(extra_args=['-m not clipboard and not single_cpu', '--skip-slow', '--skip-network', '--skip-db', '-n=2']);
159+
pandas.test(extra_args=['-m not clipboard and single_cpu', '--skip-slow', '--skip-network', '--skip-db'])"
160+
- uses: actions/upload-artifact@v3
161+
with:
162+
name: sdist
163+
path: ./dist/*
164+
165+
- name: Upload sdist
166+
if: success()
167+
shell: bash
168+
env:
169+
PANDAS_STAGING_UPLOAD_TOKEN: ${{ secrets.PANDAS_STAGING_UPLOAD_TOKEN }}
170+
PANDAS_NIGHTLY_UPLOAD_TOKEN: ${{ secrets.PANDAS_NIGHTLY_UPLOAD_TOKEN }}
171+
run: |
172+
source ci/upload_wheels.sh
173+
set_upload_vars
174+
# trigger an upload to
175+
# https://anaconda.org/scipy-wheels-nightly/pandas
176+
# for cron jobs or "Run workflow" (restricted to main branch).
177+
# Tags will upload to
178+
# https://anaconda.org/multibuild-wheels-staging/pandas
179+
# The tokens were originally generated at anaconda.org
180+
upload_wheels

‎.pre-commit-config.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,8 @@ repos:
5353
rev: 5.0.4
5454
hooks:
5555
- id: flake8
56+
# Need to patch os.remove rule in pandas-dev-flaker
57+
exclude: ^ci/fix_wheels.py
5658
additional_dependencies: &flake8_dependencies
5759
- flake8==5.0.4
5860
- flake8-bugbear==22.7.1

‎ci/fix_wheels.py

+54
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
import os
2+
import shutil
3+
import sys
4+
import zipfile
5+
6+
try:
7+
_, wheel_path, dest_dir = sys.argv
8+
# Figure out whether we are building on 32 or 64 bit python
9+
is_32 = sys.maxsize <= 2**32
10+
PYTHON_ARCH = "x86" if is_32 else "x64"
11+
except ValueError:
12+
# Too many/little values to unpack
13+
raise ValueError(
14+
"User must pass the path to the wheel and the destination directory."
15+
)
16+
# Wheels are zip files
17+
if not os.path.isdir(dest_dir):
18+
print(f"Created directory {dest_dir}")
19+
os.mkdir(dest_dir)
20+
shutil.copy(wheel_path, dest_dir) # Remember to delete if process fails
21+
wheel_name = os.path.basename(wheel_path)
22+
success = True
23+
exception = None
24+
repaired_wheel_path = os.path.join(dest_dir, wheel_name)
25+
with zipfile.ZipFile(repaired_wheel_path, "a") as zipf:
26+
try:
27+
# TODO: figure out how licensing works for the redistributables
28+
base_redist_dir = (
29+
f"C:/Program Files (x86)/Microsoft Visual Studio/2019/"
30+
f"Enterprise/VC/Redist/MSVC/14.29.30133/{PYTHON_ARCH}/"
31+
f"Microsoft.VC142.CRT/"
32+
)
33+
zipf.write(
34+
os.path.join(base_redist_dir, "msvcp140.dll"),
35+
"pandas/_libs/window/msvcp140.dll",
36+
)
37+
zipf.write(
38+
os.path.join(base_redist_dir, "concrt140.dll"),
39+
"pandas/_libs/window/concrt140.dll",
40+
)
41+
if not is_32:
42+
zipf.write(
43+
os.path.join(base_redist_dir, "vcruntime140_1.dll"),
44+
"pandas/_libs/window/vcruntime140_1.dll",
45+
)
46+
except Exception as e:
47+
success = False
48+
exception = e
49+
50+
if not success:
51+
os.remove(repaired_wheel_path)
52+
raise exception
53+
else:
54+
print(f"Successfully repaired wheel was written to {repaired_wheel_path}")

‎ci/test_wheels.py

+58
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
import glob
2+
import os
3+
import platform
4+
import shutil
5+
import subprocess
6+
import sys
7+
8+
if os.name == "nt":
9+
py_ver = platform.python_version()
10+
is_32_bit = os.getenv("IS_32_BIT") == "true"
11+
try:
12+
wheel_dir = sys.argv[1]
13+
wheel_path = glob.glob(f"{wheel_dir}/*.whl")[0]
14+
except IndexError:
15+
# Not passed
16+
wheel_path = None
17+
print(f"IS_32_BIT is {is_32_bit}")
18+
print(f"Path to built wheel is {wheel_path}")
19+
if is_32_bit:
20+
sys.exit(0) # No way to test Windows 32-bit(no docker image)
21+
if wheel_path is None:
22+
raise ValueError("Wheel path must be passed in if on 64-bit Windows")
23+
print(f"Pulling docker image to test Windows 64-bit Python {py_ver}")
24+
subprocess.run(f"docker pull python:{py_ver}-windowsservercore", check=True)
25+
pandas_base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
26+
print(f"pandas project dir is {pandas_base_dir}")
27+
dist_dir = os.path.join(pandas_base_dir, "dist")
28+
print(f"Copying wheel into pandas_base_dir/dist ({dist_dir})")
29+
os.mkdir(dist_dir)
30+
shutil.copy(wheel_path, dist_dir)
31+
print(os.listdir(dist_dir))
32+
subprocess.run(
33+
rf"docker run -v %cd%:c:\pandas "
34+
f"python:{py_ver}-windowsservercore /pandas/ci/test_wheels_windows.bat",
35+
check=True,
36+
shell=True,
37+
cwd=pandas_base_dir,
38+
)
39+
else:
40+
import pandas as pd
41+
42+
pd.test(
43+
extra_args=[
44+
"-m not clipboard and not single_cpu",
45+
"--skip-slow",
46+
"--skip-network",
47+
"--skip-db",
48+
"-n=2",
49+
]
50+
)
51+
pd.test(
52+
extra_args=[
53+
"-m not clipboard and single_cpu",
54+
"--skip-slow",
55+
"--skip-network",
56+
"--skip-db",
57+
]
58+
)

‎ci/test_wheels_windows.bat

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
set test_command=import pandas as pd; print(pd.__version__); ^
2+
pd.test(extra_args=['-m not clipboard and not single_cpu', '--skip-slow', '--skip-network', '--skip-db', '-n=2']); ^
3+
pd.test(extra_args=['-m not clipboard and single_cpu', '--skip-slow', '--skip-network', '--skip-db'])
4+
5+
python --version
6+
pip install pytz six numpy python-dateutil
7+
pip install hypothesis==6.52.1 pytest>=6.2.5 pytest-xdist pytest-asyncio>=0.17
8+
pip install --find-links=pandas/dist --no-index pandas
9+
python -c "%test_command%"

‎ci/upload_wheels.sh

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
# Modified from numpy's https://github.com/numpy/numpy/blob/main/tools/wheels/upload_wheels.sh
2+
3+
set_upload_vars() {
4+
echo "IS_PUSH is $IS_PUSH"
5+
echo "IS_SCHEDULE_DISPATCH is $IS_SCHEDULE_DISPATCH"
6+
if [[ "$IS_PUSH" == "true" ]]; then
7+
echo push and tag event
8+
export ANACONDA_ORG="multibuild-wheels-staging"
9+
export TOKEN="$PANDAS_STAGING_UPLOAD_TOKEN"
10+
export ANACONDA_UPLOAD="true"
11+
elif [[ "$IS_SCHEDULE_DISPATCH" == "true" ]]; then
12+
echo scheduled or dispatched event
13+
export ANACONDA_ORG="scipy-wheels-nightly"
14+
export TOKEN="$PANDAS_NIGHTLY_UPLOAD_TOKEN"
15+
export ANACONDA_UPLOAD="true"
16+
else
17+
echo non-dispatch event
18+
export ANACONDA_UPLOAD="false"
19+
fi
20+
}
21+
upload_wheels() {
22+
echo ${PWD}
23+
if [[ ${ANACONDA_UPLOAD} == true ]]; then
24+
if [ -z ${TOKEN} ]; then
25+
echo no token set, not uploading
26+
else
27+
conda install -q -y anaconda-client
28+
# sdists are located under dist folder when built through setup.py
29+
if compgen -G "./dist/*.gz"; then
30+
echo "Found sdist"
31+
anaconda -q -t ${TOKEN} upload --skip -u ${ANACONDA_ORG} ./dist/*.gz
32+
elif compgen -G "./wheelhouse/*.whl"; then
33+
echo "Found wheel"
34+
anaconda -q -t ${TOKEN} upload --skip -u ${ANACONDA_ORG} ./wheelhouse/*.whl
35+
else
36+
echo "Files do not exist"
37+
return 1
38+
fi
39+
echo "PyPI-style index: https://pypi.anaconda.org/$ANACONDA_ORG/simple"
40+
fi
41+
fi
42+
}

0 commit comments

Comments
 (0)