diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 35a93618d..782ea7fa7 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -4,85 +4,49 @@ on: push: branches: - master + - cibuildwheel tags: - 'v*' -env: - LIBGIT2_VERSION: 1.3.0 - jobs: - linux: - name: Linux - runs-on: ubuntu-20.04 + build_wheels: + name: Build wheels on ${{ matrix.os }} + runs-on: ${{ matrix.os }} strategy: matrix: - arch: [x86_64, aarch64] - steps: - - name: Checkout pygit2 - uses: actions/checkout@v2 - - - name: Set up QEMU - if: ${{ matrix.arch == 'aarch64' }} - uses: docker/setup-qemu-action@v1 + os: [ubuntu-20.04, macOS-11] - - run: | - /bin/sh misc/build-all-manylinux-wheels.sh ${{ matrix.arch }} - - - name: Save wheels - uses: actions/upload-artifact@v2 - with: - name: wheels - path: dist/*.whl - - macos: - name: macos-${{ matrix.py.ver }} - runs-on: macos-latest - env: - MACOSX_DEPLOYMENT_TARGET: "10.9" - strategy: - fail-fast: false - matrix: - py: - - ver: '3.7' - release: '3.7.9' # last Python.org binary release - url: https://www.python.org/ftp/python/3.7.9/python-3.7.9-macosx10.9.pkg - - ver: '3.8' - release: '3.8.10' # last Python.org binary release - url: https://www.python.org/ftp/python/3.8.10/python-3.8.10-macosx10.9.pkg - - ver: '3.9' - release: '3.9.7' - url: https://www.python.org/ftp/python/3.9.7/python-3.9.7-macosx10.9.pkg - - ver: '3.10' - release: '3.10.0' - url: https://www.python.org/ftp/python/3.10.0/python-3.10.0post2-macos11.pkg steps: - - name: Checkout pygit2 - uses: actions/checkout@v2 + - uses: actions/checkout@v2 - - name: Setup python - run: | - PKG=$(basename "${{ matrix.py.url }}") - wget --no-verbose -N "${{ matrix.py.url }}" - sudo installer -pkg $PKG -target / - export PATH=/Library/Frameworks/Python.framework/Versions/${{ matrix.py.ver }}/bin:$PATH - echo "/Library/Frameworks/Python.framework/Versions/${{ matrix.py.ver }}/bin" >> $GITHUB_PATH - which python3 - python3 -VV - test "$(python3 -V)" = "Python ${{ matrix.py.release }}" + - uses: actions/setup-python@v2 - - name: Build - run: | - /bin/sh build.sh wheel bundle test + - name: Set up QEMU + if: runner.os == 'Linux' + uses: docker/setup-qemu-action@v1 + with: + platforms: all - - name: Save wheel - uses: actions/upload-artifact@v2 - with: - name: wheels - path: wheelhouse/*.whl + - name: Build wheels + uses: pypa/cibuildwheel@v2.3.0 + env: + CIBW_BEFORE_ALL: sh build.sh + CIBW_ENVIRONMENT: LIBGIT2_VERSION=1.3.0 LIBSSH2_VERSION=1.10.0 LIBGIT2=/project/ci + CIBW_ENVIRONMENT_MACOS: LIBGIT2_VERSION=1.3.0 LIBSSH2_VERSION=1.10.0 DYLD_LIBRARY_PATH=/usr/local/lib OPENSSL_VERSION=3.0.0 + CIBW_SKIP: "*-musllinux*" + CIBW_REPAIR_WHEEL_COMMAND_LINUX: "DYLD_LIBRARY_PATH=/project/ci/lib64 auditwheel repair -w {dest_dir} {wheel}" + CIBW_REPAIR_WHEEL_COMMAND_MACOS: "DYLD_LIBRARY_PATH=/usr/local/lib delocate-listdeps {wheel} && DYLD_LIBRARY_PATH=/usr/local/lib delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}" + CIBW_ARCHS_MACOS: x86_64 universal2 + CIBW_ARCHS_LINUX: x86_64 aarch64 + + - uses: actions/upload-artifact@v2 + with: + name: wheels + path: ./wheelhouse/*.whl pypi: if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') - needs: [linux, macos] + needs: [build_wheels] runs-on: ubuntu-20.04 steps: diff --git a/build.sh b/build.sh index 36fc12d70..38fa8938d 100644 --- a/build.sh +++ b/build.sh @@ -12,7 +12,9 @@ # LIBSSH2_OPENSSL - Where to find openssl # LIBSSH2_PREFIX - Where to find libssh2 # LIBSSH2_VERSION= - Build the given version of libssh2 -# LIBGIT2_VERSION= - Build the given version of libgit2 +# LIBGIT2_VERSION= - Build the given version of libgit2 +# OPENSSL_VERSION= - Build the given version of OpenSSL +# (only needed for Mac universal on CI) # # Either use LIBSSH2_PREFIX, or LIBSSH2_VERSION, or none (if libssh2 is already # in the path, or if you don't want to use it). @@ -55,13 +57,24 @@ KERNEL=`uname -s` BUILD_TYPE=${BUILD_TYPE:-Debug} PYTHON=${PYTHON:-python3} -PYTHON_TAG=$($PYTHON build_tag.py) +if [ "$CIBUILDWHEEL" != "1" ]; then + PYTHON_TAG=$($PYTHON build_tag.py) +fi + PREFIX="${PREFIX:-$(pwd)/ci/$PYTHON_TAG}" export LDFLAGS="-Wl,-rpath,$PREFIX/lib" -# Create a virtual environment -$PYTHON -m venv $PREFIX -cd ci +if [ "$CIBUILDWHEEL" = "1" ]; then + rm -rf ci + mkdir ci || true + cd ci + if [ "$KERNEL" = "Linux" ]; then + yum install wget openssl-devel libssh2-devel zlib-devel -y + fi +else + # Create a virtual environment + $PYTHON -m venv $PREFIX +fi # Install zlib # XXX Build libgit2 with USE_BUNDLED_ZLIB instead? @@ -76,17 +89,67 @@ if [ -n "$ZLIB_VERSION" ]; then cd .. fi +# Install openssl +if [ -n "$OPENSSL_VERSION" ]; then + if [ "$CIBUILDWHEEL" != "1" ] || [ "$KERNEL" != "Darwin" ]; then + echo "OPENSSL_VERSION should only be set when building" + echo "macOS universal2 wheels on GitHub!" + echo "Please unset and try again" + exit 1 + fi + FILENAME=openssl-$OPENSSL_VERSION + wget https://www.openssl.org/source/$FILENAME.tar.gz -N --no-check-certificate + + tar xf $FILENAME.tar.gz + mv $FILENAME openssl-x86 + + tar xf $FILENAME.tar.gz + mv $FILENAME openssl-arm + + cd openssl-x86 + ./Configure darwin64-x86_64-cc shared + make + cd ../openssl-arm + ./Configure enable-rc5 zlib darwin64-arm64-cc no-asm + make + cd .. + + mkdir openssl-universal + + LIBSSL=$(basename openssl-x86/libssl.*.dylib) + lipo -create openssl-x86/libssl.*.dylib openssl-arm/libssl.*.dylib -output openssl-universal/$LIBSSL + LIBCRYPTO=$(basename openssl-x86/libcrypto.*.dylib) + lipo -create openssl-x86/libcrypto.*.dylib openssl-arm/libcrypto.*.dylib -output openssl-universal/$LIBCRYPTO + cd openssl-universal + install_name_tool -id "@rpath/$LIBSSL" $LIBSSL + install_name_tool -id "@rpath/$LIBCRYPTO" $LIBCRYPTO + OPENSSL_PREFIX=$(pwd) + cd .. +fi + # Install libssh2 if [ -n "$LIBSSH2_VERSION" ]; then FILENAME=libssh2-$LIBSSH2_VERSION wget https://www.libssh2.org/download/$FILENAME.tar.gz -N --no-check-certificate tar xf $FILENAME.tar.gz cd $FILENAME - cmake . \ - -DCMAKE_INSTALL_PREFIX=$PREFIX \ - -DBUILD_SHARED_LIBS=ON \ - -DBUILD_EXAMPLES=OFF \ - -DBUILD_TESTING=OFF + if [ "$KERNEL" = "Darwin" ] && [ "$CIBUILDWHEEL" = "1" ]; then + cmake . \ + -DCMAKE_INSTALL_PREFIX=$PREFIX \ + -DBUILD_SHARED_LIBS=ON \ + -DBUILD_EXAMPLES=OFF \ + -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \ + -DOPENSSL_CRYPTO_LIBRARY="../openssl-universal/$LIBCRYPTO" \ + -DOPENSSL_SSL_LIBRARY="../openssl-universal/$LIBSSL" \ + -DOPENSSL_INCLUDE_DIR="../openssl-x86/include" \ + -DBUILD_TESTING=OFF + else + cmake . \ + -DCMAKE_INSTALL_PREFIX=$PREFIX \ + -DBUILD_SHARED_LIBS=ON \ + -DBUILD_EXAMPLES=OFF \ + -DBUILD_TESTING=OFF + fi cmake --build . --target install cd .. LIBSSH2_PREFIX=$PREFIX @@ -98,18 +161,43 @@ if [ -n "$LIBGIT2_VERSION" ]; then wget https://github.com/libgit2/libgit2/archive/refs/tags/v$LIBGIT2_VERSION.tar.gz -N -O $FILENAME.tar.gz tar xf $FILENAME.tar.gz cd $FILENAME - CMAKE_PREFIX_PATH=$OPENSSL_PREFIX:$LIBSSH2_PREFIX cmake . \ - -DCMAKE_INSTALL_PREFIX=$PREFIX \ - -DBUILD_SHARED_LIBS=ON \ - -DBUILD_CLAR=OFF \ - -DCMAKE_BUILD_TYPE=$BUILD_TYPE + if [ "$KERNEL" = "Darwin" ] && [ "$CIBUILDWHEEL" = "1" ]; then + CMAKE_PREFIX_PATH=$OPENSSL_PREFIX:$LIBSSH2_PREFIX cmake . \ + -DBUILD_SHARED_LIBS=ON \ + -DBUILD_CLAR=OFF \ + -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \ + -DOPENSSL_CRYPTO_LIBRARY="../openssl-universal/$LIBCRYPTO" \ + -DOPENSSL_SSL_LIBRARY="../openssl-universal/$LIBSSL" \ + -DOPENSSL_INCLUDE_DIR="../openssl-x86/include" \ + -DCMAKE_BUILD_TYPE=$BUILD_TYPE + else + CMAKE_PREFIX_PATH=$OPENSSL_PREFIX:$LIBSSH2_PREFIX cmake . \ + -DCMAKE_INSTALL_PREFIX=$PREFIX \ + -DBUILD_SHARED_LIBS=ON \ + -DBUILD_CLAR=OFF \ + -DCMAKE_BUILD_TYPE=$BUILD_TYPE + fi cmake --build . --target install cd .. export LIBGIT2=$PREFIX fi +if [ "$CIBUILDWHEEL" = "1" ]; then + # This is gross. auditwheel/delocate-wheel are not so good + # at finding libraries in random places, so we have to + # put them in the loader path. + if [ "$KERNEL" = "Darwin" ]; then + cp -r $OPENSSL_PREFIX/*.dylib /usr/local/lib + cp -r $LIBSSH2_PREFIX/lib/*.dylib /usr/local/lib + cp -r $FILENAME/*.dylib /usr/local/lib + else + cp -r $PREFIX/lib64/*.so* /usr/local/lib + fi + # we're done building dependencies, cibuildwheel action will take over + exit 0 +fi + # Build pygit2 -cd .. $PREFIX/bin/pip install -U pip wheel if [ "$1" = "wheel" ]; then shift diff --git a/misc/build-all-manylinux-wheels.sh b/misc/build-all-manylinux-wheels.sh deleted file mode 100755 index 9ec95dc59..000000000 --- a/misc/build-all-manylinux-wheels.sh +++ /dev/null @@ -1,19 +0,0 @@ -#! /usr/bin/env bash - -set -x -set -e - -ARCH=$1 - -# Wait for docker pull to complete downloading container -if [ $ARCH = 'aarch64' ]; then - manylinux_image="ghcr.io/pyca/cryptography-manylinux2014_${ARCH}" -else - manylinux_image="ghcr.io/pyca/cryptography-manylinux2014:${ARCH}" -fi -docker pull "${manylinux_image}" & -wait - -# Build wheels -docker run --rm -v `pwd`:/io/src "${manylinux_image}" /io/src/misc/build-manylinux-wheels.sh & -wait diff --git a/misc/build-manylinux-wheels.sh b/misc/build-manylinux-wheels.sh deleted file mode 100755 index c7eeed45b..000000000 --- a/misc/build-manylinux-wheels.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash - -set -xe - -ARCH=`uname -m` -PYTHONS="cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310" -export PYCA_OPENSSL_PATH=/opt/pyca/cryptography/openssl -export OPENSSL_PATH=/opt/openssl -export CFLAGS="-fPIC" -export PKG_CONFIG_PATH="${OPENSSL_PATH}/lib/pkgconfig:${PYCA_OPENSSL_PATH}/lib/pkgconfig" - -# Install requirements -yum -y install git libffi-devel -yum -y install openssl wget -if [ ${ARCH} == 'aarch64' ]; then - yum -y install cmake -else - yum -y install cmake3 -fi - -# Copy source directory so we don't mangle with it -SRC_DIR=/io/src -TMP_DIR=/io/tmp -rm $TMP_DIR -rf -cp -a $SRC_DIR $TMP_DIR - -# Build -pushd $TMP_DIR -export OPENSSL_PREFIX=/opt/pyca/cryptography/openssl -export ZLIB_VERSION=1.2.11 -export LIBSSH2_VERSION=1.10.0 -export LIBGIT2_VERSION=1.3.0 -export BUILD_TYPE=Release -for PY in $PYTHONS; do - PYTHON=/opt/python/${PY}/bin/python sh build.sh wheel bundle test -done -popd - -# Copy wheels to source dist folder -ls -l $TMP_DIR/wheelhouse -mkdir -p $SRC_DIR/dist -cp $TMP_DIR/wheelhouse/pygit2-*_$ARCH.whl $SRC_DIR/dist -chown -R --reference="${SRC_DIR}/setup.py" ${SRC_DIR}/dist