From f8d5c0841c3c17ca923ba8a1f8809ea0cd352990 Mon Sep 17 00:00:00 2001 From: Simon Cozens Date: Thu, 25 Nov 2021 08:57:44 +0000 Subject: [PATCH 1/6] Use cibuildwheel --- .github/workflows/wheels.yml | 85 +++++++------------------- install-deps-mac.sh | 106 +++++++++++++++++++++++++++++++++ install-deps.sh | 112 +++++++++++++++++++++++++++++++++++ 3 files changed, 238 insertions(+), 65 deletions(-) create mode 100644 install-deps-mac.sh create mode 100644 install-deps.sh diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 35a93618d..31675cd45 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -4,85 +4,40 @@ on: push: branches: - master + - cibuildwheel tags: - 'v*' -env: - LIBGIT2_VERSION: 1.3.0 - jobs: - linux: - name: Linux - runs-on: ubuntu-20.04 + build_wheels: + name: Build wheels on ${{ matrix.os }} + runs-on: ${{ matrix.os }} strategy: matrix: - arch: [x86_64, aarch64] - steps: - - name: Checkout pygit2 - uses: actions/checkout@v2 - - - name: Set up QEMU - if: ${{ matrix.arch == 'aarch64' }} - uses: docker/setup-qemu-action@v1 - - - run: | - /bin/sh misc/build-all-manylinux-wheels.sh ${{ matrix.arch }} - - - name: Save wheels - uses: actions/upload-artifact@v2 - with: - name: wheels - path: dist/*.whl + os: [ubuntu-20.04, macOS-11] - macos: - name: macos-${{ matrix.py.ver }} - runs-on: macos-latest - env: - MACOSX_DEPLOYMENT_TARGET: "10.9" - strategy: - fail-fast: false - matrix: - py: - - ver: '3.7' - release: '3.7.9' # last Python.org binary release - url: https://www.python.org/ftp/python/3.7.9/python-3.7.9-macosx10.9.pkg - - ver: '3.8' - release: '3.8.10' # last Python.org binary release - url: https://www.python.org/ftp/python/3.8.10/python-3.8.10-macosx10.9.pkg - - ver: '3.9' - release: '3.9.7' - url: https://www.python.org/ftp/python/3.9.7/python-3.9.7-macosx10.9.pkg - - ver: '3.10' - release: '3.10.0' - url: https://www.python.org/ftp/python/3.10.0/python-3.10.0post2-macos11.pkg steps: - - name: Checkout pygit2 - uses: actions/checkout@v2 + - uses: actions/checkout@v2 - - name: Setup python - run: | - PKG=$(basename "${{ matrix.py.url }}") - wget --no-verbose -N "${{ matrix.py.url }}" - sudo installer -pkg $PKG -target / - export PATH=/Library/Frameworks/Python.framework/Versions/${{ matrix.py.ver }}/bin:$PATH - echo "/Library/Frameworks/Python.framework/Versions/${{ matrix.py.ver }}/bin" >> $GITHUB_PATH - which python3 - python3 -VV - test "$(python3 -V)" = "Python ${{ matrix.py.release }}" + - uses: actions/setup-python@v2 - - name: Build - run: | - /bin/sh build.sh wheel bundle test + - name: Build wheels + uses: pypa/cibuildwheel@v2.2.2 + env: + CIBW_BEFORE_ALL_LINUX: sh install-deps.sh + CIBW_BEFORE_ALL_MACOS: sh install-deps-mac.sh + CIBW_ENVIRONMENT: LIBGIT2_VERSION=1.3.0 LIBSSH2_VERSION=1.10.0 DYLD_LIBRARY_PATH=/usr/local/lib + CIBW_SKIP: "*-musllinux*" + CIBW_REPAIR_WHEEL_COMMAND_MACOS: "DYLD_LIBRARY_PATH=/usr/local/lib delocate-listdeps {wheel} && DYLD_LIBRARY_PATH=/usr/local/lib delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}" + CIBW_ARCHS_MACOS: x86_64 universal2 - - name: Save wheel - uses: actions/upload-artifact@v2 - with: - name: wheels - path: wheelhouse/*.whl + - uses: actions/upload-artifact@v2 + with: + path: ./wheelhouse/*.whl pypi: if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') - needs: [linux, macos] + needs: [build_wheels] runs-on: ubuntu-20.04 steps: diff --git a/install-deps-mac.sh b/install-deps-mac.sh new file mode 100644 index 000000000..79b27f57b --- /dev/null +++ b/install-deps-mac.sh @@ -0,0 +1,106 @@ +#!/bin/sh +set -x # Print every command and variable +set -e # Fail fast + +# Variables +ARCH=`uname -m` +KERNEL=`uname -s` +BUILD_TYPE=${BUILD_TYPE:-Debug} +PYTHON=${PYTHON:-python3} + +PREFIX="${PREFIX:-$(pwd)/ci/}" +export LDFLAGS="-Wl,-rpath,$PREFIX/lib" + +OPENSSL_VERSION=3.0.0 + +rm -rf ci +mkdir ci || true +cd ci + + +# Install zlib +# XXX Build libgit2 with USE_BUNDLED_ZLIB instead? +if [ -n "$ZLIB_VERSION" ]; then + FILENAME=zlib-$ZLIB_VERSION + wget https://www.zlib.net/$FILENAME.tar.gz -N + tar xf $FILENAME.tar.gz + cd $FILENAME + ./configure --prefix=$PREFIX + make + make install + cd .. +fi + +# Install openssl +FILENAME=openssl-$OPENSSL_VERSION +wget https://www.openssl.org/source/$FILENAME.tar.gz -N --no-check-certificate + +tar xf $FILENAME.tar.gz +mv $FILENAME openssl-x86 + +tar xf $FILENAME.tar.gz +mv $FILENAME openssl-arm + +cd openssl-x86 +./Configure darwin64-x86_64-cc shared +make +cd ../openssl-arm +./Configure enable-rc5 zlib darwin64-arm64-cc no-asm +make +cd .. + +mkdir openssl-universal + +LIBSSL=$(basename openssl-x86/libssl.*.dylib) +lipo -create openssl-x86/libssl.*.dylib openssl-arm/libssl.*.dylib -output openssl-universal/$LIBSSL +LIBCRYPTO=$(basename openssl-x86/libcrypto.*.dylib) +lipo -create openssl-x86/libcrypto.*.dylib openssl-arm/libcrypto.*.dylib -output openssl-universal/$LIBCRYPTO +cd openssl-universal +install_name_tool -id "@rpath/$LIBSSL" $LIBSSL +install_name_tool -id "@rpath/$LIBCRYPTO" $LIBCRYPTO +OPENSSL_PREFIX=$(pwd) +cd .. + + +# Install libssh2 +if [ -n "$LIBSSH2_VERSION" ]; then + FILENAME=libssh2-$LIBSSH2_VERSION + wget https://www.libssh2.org/download/$FILENAME.tar.gz -N --no-check-certificate + tar xf $FILENAME.tar.gz + cd $FILENAME + cmake . \ + -DCMAKE_INSTALL_PREFIX=$PREFIX \ + -DBUILD_SHARED_LIBS=ON \ + -DBUILD_EXAMPLES=OFF \ + -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \ + -DOPENSSL_CRYPTO_LIBRARY="../openssl-universal/$LIBCRYPTO" \ + -DOPENSSL_SSL_LIBRARY="../openssl-universal/$LIBSSL" \ + -DOPENSSL_INCLUDE_DIR="../openssl-x86/include" \ + -DBUILD_TESTING=OFF + cmake --build . --target install + cd .. + LIBSSH2_PREFIX=$PREFIX +fi + +# Install libgit2 +if [ -n "$LIBGIT2_VERSION" ]; then + FILENAME=libgit2-$LIBGIT2_VERSION + wget https://github.com/libgit2/libgit2/archive/refs/tags/v$LIBGIT2_VERSION.tar.gz -N -O $FILENAME.tar.gz + tar xf $FILENAME.tar.gz + cd $FILENAME + CMAKE_PREFIX_PATH=$OPENSSL_PREFIX:$LIBSSH2_PREFIX cmake . \ + -DBUILD_SHARED_LIBS=ON \ + -DBUILD_CLAR=OFF \ + -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \ + -DCMAKE_BUILD_TYPE=$BUILD_TYPE \ + -DOPENSSL_CRYPTO_LIBRARY="../openssl-universal/$LIBCRYPTO" \ + -DOPENSSL_SSL_LIBRARY="../openssl-universal/$LIBSSL" \ + -DOPENSSL_INCLUDE_DIR="../openssl-x86/include" + cmake --build . --target install + cd .. +fi + +# This is gross +cp -r $OPENSSL_PREFIX/*.dylib /usr/local/lib +cp -r $LIBSSH2_PREFIX/lib/*.dylib /usr/local/lib +cp -r $FILENAME/*.dylib /usr/local/lib diff --git a/install-deps.sh b/install-deps.sh new file mode 100644 index 000000000..5e1ec44bb --- /dev/null +++ b/install-deps.sh @@ -0,0 +1,112 @@ +#!/bin/sh + +# +# Synopsis: +# +# sh build.sh - Build inplace +# sh build.sh test - Build inplace, and run the tests +# sh build.sh wheel - Build a wheel, install, and run the tests +# +# Environment variables: +# +# LIBSSH2_OPENSSL - Where to find openssl +# LIBSSH2_PREFIX - Where to find libssh2 +# LIBSSH2_VERSION= - Build the given version of libssh2 +# LIBGIT2_VERSION= - Build the given version of libgit2 +# +# Either use LIBSSH2_PREFIX, or LIBSSH2_VERSION, or none (if libssh2 is already +# in the path, or if you don't want to use it). +# +# Examples. +# +# Build inplace, libgit2 must be available in the path: +# +# sh build.sh +# +# Build libgit2 1.3.0 (will use libssh2 if available), then build pygit2 +# inplace: +# +# LIBGIT2_VERSION=1.3.0 sh build.sh +# +# Build libssh2 1.10.0 and libgit2 1.3.0, then build pygit2 inplace: +# +# LIBSSH2_VERSION=1.10.0 LIBGIT2_VERSION=1.3.0 sh build.sh +# +# Tell where libssh2 is installed, build libgit2 1.3.0, then build pygit2 +# inplace: +# +# LIBSSH2_PREFIX=/usr/local LIBGIT2_VERSION=1.3.0 sh build.sh +# +# Build inplace and run the tests: +# +# sh build.sh test +# +# Build a wheel: +# +# sh build.sh wheel +# + +set -x # Print every command and variable +set -e # Fail fast + +# Variables +ARCH=`uname -m` +KERNEL=`uname -s` +BUILD_TYPE=${BUILD_TYPE:-Debug} +PYTHON=${PYTHON:-python3} + +PREFIX="${PREFIX:-$(pwd)/ci/}" +export LDFLAGS="-Wl,-rpath,$PREFIX/lib" + +rm -rf ci +mkdir ci || true +cd ci + +if [ "$KERNEL" = "Linux" ]; then + yum install wget openssl-devel libssh2-devel zlib-devel -y +fi + + +# Install zlib +# XXX Build libgit2 with USE_BUNDLED_ZLIB instead? +if [ -n "$ZLIB_VERSION" ]; then + FILENAME=zlib-$ZLIB_VERSION + wget https://www.zlib.net/$FILENAME.tar.gz -N + tar xf $FILENAME.tar.gz + cd $FILENAME + ./configure --prefix=$PREFIX + make + make install + cd .. +fi + +# Install libssh2 +if [ -n "$LIBSSH2_VERSION" ]; then + FILENAME=libssh2-$LIBSSH2_VERSION + wget https://www.libssh2.org/download/$FILENAME.tar.gz -N --no-check-certificate + tar xf $FILENAME.tar.gz + cd $FILENAME + cmake . \ + -DCMAKE_INSTALL_PREFIX=$PREFIX \ + -DBUILD_SHARED_LIBS=ON \ + -DBUILD_EXAMPLES=OFF \ + -DBUILD_TESTING=OFF + cmake --build . --target install + cd .. + LIBSSH2_PREFIX=$PREFIX +fi + +# Install libgit2 +if [ -n "$LIBGIT2_VERSION" ]; then + FILENAME=libgit2-$LIBGIT2_VERSION + wget https://github.com/libgit2/libgit2/archive/refs/tags/v$LIBGIT2_VERSION.tar.gz -N -O $FILENAME.tar.gz + tar xf $FILENAME.tar.gz + cd $FILENAME + CMAKE_PREFIX_PATH=$OPENSSL_PREFIX:$LIBSSH2_PREFIX cmake . \ + -DBUILD_SHARED_LIBS=ON \ + -DBUILD_CLAR=OFF \ + -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \ + -DCMAKE_BUILD_TYPE=$BUILD_TYPE + cmake --build . --target install + cd .. +fi From b531894247478f94d7d49575709bc071bf6d4872 Mon Sep 17 00:00:00 2001 From: Simon Cozens Date: Fri, 26 Nov 2021 09:36:34 +0000 Subject: [PATCH 2/6] Restore aarch64 builds --- .github/workflows/wheels.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 31675cd45..842957cd1 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -21,6 +21,12 @@ jobs: - uses: actions/setup-python@v2 + - name: Set up QEMU + if: runner.os == 'Linux' + uses: docker/setup-qemu-action@v1 + with: + platforms: all + - name: Build wheels uses: pypa/cibuildwheel@v2.2.2 env: @@ -30,6 +36,7 @@ jobs: CIBW_SKIP: "*-musllinux*" CIBW_REPAIR_WHEEL_COMMAND_MACOS: "DYLD_LIBRARY_PATH=/usr/local/lib delocate-listdeps {wheel} && DYLD_LIBRARY_PATH=/usr/local/lib delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}" CIBW_ARCHS_MACOS: x86_64 universal2 + CIBW_ARCHS_LINUX: x86_64 aarch64 - uses: actions/upload-artifact@v2 with: From 10ebe70fcdb77c794fbc6ea179e34d0314cd0919 Mon Sep 17 00:00:00 2001 From: Simon Cozens Date: Fri, 26 Nov 2021 11:09:05 +0000 Subject: [PATCH 3/6] No longer needed --- .github/workflows/wheels.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 842957cd1..d9783b188 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -4,7 +4,6 @@ on: push: branches: - master - - cibuildwheel tags: - 'v*' From 1a9c401374b1531f7bd39125eb8b385774c51555 Mon Sep 17 00:00:00 2001 From: Simon Cozens Date: Mon, 29 Nov 2021 11:01:50 +0000 Subject: [PATCH 4/6] Unify the two dependency scripts, for starters --- .github/workflows/wheels.yml | 6 +- install-deps-mac.sh | 106 ----------------------------------- install-deps.sh | 85 ++++++++++++++++++++++++---- 3 files changed, 78 insertions(+), 119 deletions(-) delete mode 100644 install-deps-mac.sh diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index d9783b188..bea402565 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -29,9 +29,9 @@ jobs: - name: Build wheels uses: pypa/cibuildwheel@v2.2.2 env: - CIBW_BEFORE_ALL_LINUX: sh install-deps.sh - CIBW_BEFORE_ALL_MACOS: sh install-deps-mac.sh - CIBW_ENVIRONMENT: LIBGIT2_VERSION=1.3.0 LIBSSH2_VERSION=1.10.0 DYLD_LIBRARY_PATH=/usr/local/lib + CIBW_BEFORE_ALL: sh install-deps.sh + CIBW_ENVIRONMENT: LIBGIT2_VERSION=1.3.0 LIBSSH2_VERSION=1.10.0 + CIBW_ENVIRONMENT_MACOS: LIBGIT2_VERSION=1.3.0 LIBSSH2_VERSION=1.10.0 DYLD_LIBRARY_PATH=/usr/local/lib OPENSSL_VERSION=3.0.0 CIBW_SKIP: "*-musllinux*" CIBW_REPAIR_WHEEL_COMMAND_MACOS: "DYLD_LIBRARY_PATH=/usr/local/lib delocate-listdeps {wheel} && DYLD_LIBRARY_PATH=/usr/local/lib delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}" CIBW_ARCHS_MACOS: x86_64 universal2 diff --git a/install-deps-mac.sh b/install-deps-mac.sh deleted file mode 100644 index 79b27f57b..000000000 --- a/install-deps-mac.sh +++ /dev/null @@ -1,106 +0,0 @@ -#!/bin/sh -set -x # Print every command and variable -set -e # Fail fast - -# Variables -ARCH=`uname -m` -KERNEL=`uname -s` -BUILD_TYPE=${BUILD_TYPE:-Debug} -PYTHON=${PYTHON:-python3} - -PREFIX="${PREFIX:-$(pwd)/ci/}" -export LDFLAGS="-Wl,-rpath,$PREFIX/lib" - -OPENSSL_VERSION=3.0.0 - -rm -rf ci -mkdir ci || true -cd ci - - -# Install zlib -# XXX Build libgit2 with USE_BUNDLED_ZLIB instead? -if [ -n "$ZLIB_VERSION" ]; then - FILENAME=zlib-$ZLIB_VERSION - wget https://www.zlib.net/$FILENAME.tar.gz -N - tar xf $FILENAME.tar.gz - cd $FILENAME - ./configure --prefix=$PREFIX - make - make install - cd .. -fi - -# Install openssl -FILENAME=openssl-$OPENSSL_VERSION -wget https://www.openssl.org/source/$FILENAME.tar.gz -N --no-check-certificate - -tar xf $FILENAME.tar.gz -mv $FILENAME openssl-x86 - -tar xf $FILENAME.tar.gz -mv $FILENAME openssl-arm - -cd openssl-x86 -./Configure darwin64-x86_64-cc shared -make -cd ../openssl-arm -./Configure enable-rc5 zlib darwin64-arm64-cc no-asm -make -cd .. - -mkdir openssl-universal - -LIBSSL=$(basename openssl-x86/libssl.*.dylib) -lipo -create openssl-x86/libssl.*.dylib openssl-arm/libssl.*.dylib -output openssl-universal/$LIBSSL -LIBCRYPTO=$(basename openssl-x86/libcrypto.*.dylib) -lipo -create openssl-x86/libcrypto.*.dylib openssl-arm/libcrypto.*.dylib -output openssl-universal/$LIBCRYPTO -cd openssl-universal -install_name_tool -id "@rpath/$LIBSSL" $LIBSSL -install_name_tool -id "@rpath/$LIBCRYPTO" $LIBCRYPTO -OPENSSL_PREFIX=$(pwd) -cd .. - - -# Install libssh2 -if [ -n "$LIBSSH2_VERSION" ]; then - FILENAME=libssh2-$LIBSSH2_VERSION - wget https://www.libssh2.org/download/$FILENAME.tar.gz -N --no-check-certificate - tar xf $FILENAME.tar.gz - cd $FILENAME - cmake . \ - -DCMAKE_INSTALL_PREFIX=$PREFIX \ - -DBUILD_SHARED_LIBS=ON \ - -DBUILD_EXAMPLES=OFF \ - -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \ - -DOPENSSL_CRYPTO_LIBRARY="../openssl-universal/$LIBCRYPTO" \ - -DOPENSSL_SSL_LIBRARY="../openssl-universal/$LIBSSL" \ - -DOPENSSL_INCLUDE_DIR="../openssl-x86/include" \ - -DBUILD_TESTING=OFF - cmake --build . --target install - cd .. - LIBSSH2_PREFIX=$PREFIX -fi - -# Install libgit2 -if [ -n "$LIBGIT2_VERSION" ]; then - FILENAME=libgit2-$LIBGIT2_VERSION - wget https://github.com/libgit2/libgit2/archive/refs/tags/v$LIBGIT2_VERSION.tar.gz -N -O $FILENAME.tar.gz - tar xf $FILENAME.tar.gz - cd $FILENAME - CMAKE_PREFIX_PATH=$OPENSSL_PREFIX:$LIBSSH2_PREFIX cmake . \ - -DBUILD_SHARED_LIBS=ON \ - -DBUILD_CLAR=OFF \ - -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \ - -DCMAKE_BUILD_TYPE=$BUILD_TYPE \ - -DOPENSSL_CRYPTO_LIBRARY="../openssl-universal/$LIBCRYPTO" \ - -DOPENSSL_SSL_LIBRARY="../openssl-universal/$LIBSSL" \ - -DOPENSSL_INCLUDE_DIR="../openssl-x86/include" - cmake --build . --target install - cd .. -fi - -# This is gross -cp -r $OPENSSL_PREFIX/*.dylib /usr/local/lib -cp -r $LIBSSH2_PREFIX/lib/*.dylib /usr/local/lib -cp -r $FILENAME/*.dylib /usr/local/lib diff --git a/install-deps.sh b/install-deps.sh index 5e1ec44bb..e47da51a7 100644 --- a/install-deps.sh +++ b/install-deps.sh @@ -80,17 +80,61 @@ if [ -n "$ZLIB_VERSION" ]; then cd .. fi +# Install openssl +if [ -n "$OPENSSL_VERSION" ]; then + FILENAME=openssl-$OPENSSL_VERSION + wget https://www.openssl.org/source/$FILENAME.tar.gz -N --no-check-certificate + + tar xf $FILENAME.tar.gz + mv $FILENAME openssl-x86 + + tar xf $FILENAME.tar.gz + mv $FILENAME openssl-arm + + cd openssl-x86 + ./Configure darwin64-x86_64-cc shared + make + cd ../openssl-arm + ./Configure enable-rc5 zlib darwin64-arm64-cc no-asm + make + cd .. + + mkdir openssl-universal + + LIBSSL=$(basename openssl-x86/libssl.*.dylib) + lipo -create openssl-x86/libssl.*.dylib openssl-arm/libssl.*.dylib -output openssl-universal/$LIBSSL + LIBCRYPTO=$(basename openssl-x86/libcrypto.*.dylib) + lipo -create openssl-x86/libcrypto.*.dylib openssl-arm/libcrypto.*.dylib -output openssl-universal/$LIBCRYPTO + cd openssl-universal + install_name_tool -id "@rpath/$LIBSSL" $LIBSSL + install_name_tool -id "@rpath/$LIBCRYPTO" $LIBCRYPTO + OPENSSL_PREFIX=$(pwd) + cd .. +fi + # Install libssh2 if [ -n "$LIBSSH2_VERSION" ]; then FILENAME=libssh2-$LIBSSH2_VERSION wget https://www.libssh2.org/download/$FILENAME.tar.gz -N --no-check-certificate tar xf $FILENAME.tar.gz cd $FILENAME - cmake . \ - -DCMAKE_INSTALL_PREFIX=$PREFIX \ - -DBUILD_SHARED_LIBS=ON \ - -DBUILD_EXAMPLES=OFF \ - -DBUILD_TESTING=OFF + if [ "$KERNEL" = "Darwin" ]; then + cmake . \ + -DCMAKE_INSTALL_PREFIX=$PREFIX \ + -DBUILD_SHARED_LIBS=ON \ + -DBUILD_EXAMPLES=OFF \ + -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \ + -DOPENSSL_CRYPTO_LIBRARY="../openssl-universal/$LIBCRYPTO" \ + -DOPENSSL_SSL_LIBRARY="../openssl-universal/$LIBSSL" \ + -DOPENSSL_INCLUDE_DIR="../openssl-x86/include" \ + -DBUILD_TESTING=OFF + else + cmake . \ + -DCMAKE_INSTALL_PREFIX=$PREFIX \ + -DBUILD_SHARED_LIBS=ON \ + -DBUILD_EXAMPLES=OFF \ + -DBUILD_TESTING=OFF + fi cmake --build . --target install cd .. LIBSSH2_PREFIX=$PREFIX @@ -102,11 +146,32 @@ if [ -n "$LIBGIT2_VERSION" ]; then wget https://github.com/libgit2/libgit2/archive/refs/tags/v$LIBGIT2_VERSION.tar.gz -N -O $FILENAME.tar.gz tar xf $FILENAME.tar.gz cd $FILENAME - CMAKE_PREFIX_PATH=$OPENSSL_PREFIX:$LIBSSH2_PREFIX cmake . \ - -DBUILD_SHARED_LIBS=ON \ - -DBUILD_CLAR=OFF \ - -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \ - -DCMAKE_BUILD_TYPE=$BUILD_TYPE + if [ "$KERNEL" = "Darwin" ]; then + CMAKE_PREFIX_PATH=$OPENSSL_PREFIX:$LIBSSH2_PREFIX cmake . \ + -DBUILD_SHARED_LIBS=ON \ + -DBUILD_CLAR=OFF \ + -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \ + -DOPENSSL_CRYPTO_LIBRARY="../openssl-universal/$LIBCRYPTO" \ + -DOPENSSL_SSL_LIBRARY="../openssl-universal/$LIBSSL" \ + -DOPENSSL_INCLUDE_DIR="../openssl-x86/include" \ + -DCMAKE_BUILD_TYPE=$BUILD_TYPE + else + CMAKE_PREFIX_PATH=$OPENSSL_PREFIX:$LIBSSH2_PREFIX cmake . \ + -DBUILD_SHARED_LIBS=ON \ + -DBUILD_CLAR=OFF \ + -DCMAKE_BUILD_TYPE=$BUILD_TYPE + fi cmake --build . --target install cd .. fi + +if [ "$KERNEL" = "Darwin" ]; then + if [ "$GITHUB_ACTIONS" != "true" ]; then + echo "Refusing to install Universal openssl locally!" + exit 1 + fi + # This is gross + cp -r $OPENSSL_PREFIX/*.dylib /usr/local/lib + cp -r $LIBSSH2_PREFIX/lib/*.dylib /usr/local/lib + cp -r $FILENAME/*.dylib /usr/local/lib +fi From f1f4ac955f6c1e61e805b275bb0d67650d3ae883 Mon Sep 17 00:00:00 2001 From: Simon Cozens Date: Mon, 29 Nov 2021 11:09:33 +0000 Subject: [PATCH 5/6] Now unify in-place build/CI build scripts --- .github/workflows/wheels.yml | 6 +- build.sh | 120 ++++++++++++++++++++---- install-deps.sh | 177 ----------------------------------- 3 files changed, 108 insertions(+), 195 deletions(-) delete mode 100644 install-deps.sh diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index bea402565..b5d974d2a 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -4,6 +4,7 @@ on: push: branches: - master + - cibuildwheel tags: - 'v*' @@ -29,10 +30,11 @@ jobs: - name: Build wheels uses: pypa/cibuildwheel@v2.2.2 env: - CIBW_BEFORE_ALL: sh install-deps.sh - CIBW_ENVIRONMENT: LIBGIT2_VERSION=1.3.0 LIBSSH2_VERSION=1.10.0 + CIBW_BEFORE_ALL: sh build.sh + CIBW_ENVIRONMENT: LIBGIT2_VERSION=1.3.0 LIBSSH2_VERSION=1.10.0 LIBGIT2=/project/ci CIBW_ENVIRONMENT_MACOS: LIBGIT2_VERSION=1.3.0 LIBSSH2_VERSION=1.10.0 DYLD_LIBRARY_PATH=/usr/local/lib OPENSSL_VERSION=3.0.0 CIBW_SKIP: "*-musllinux*" + CIBW_REPAIR_WHEEL_COMMAND_LINUX: "DYLD_LIBRARY_PATH=/project/ci/lib64 auditwheel repair -w {dest_dir} {wheel}" CIBW_REPAIR_WHEEL_COMMAND_MACOS: "DYLD_LIBRARY_PATH=/usr/local/lib delocate-listdeps {wheel} && DYLD_LIBRARY_PATH=/usr/local/lib delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}" CIBW_ARCHS_MACOS: x86_64 universal2 CIBW_ARCHS_LINUX: x86_64 aarch64 diff --git a/build.sh b/build.sh index 36fc12d70..38fa8938d 100644 --- a/build.sh +++ b/build.sh @@ -12,7 +12,9 @@ # LIBSSH2_OPENSSL - Where to find openssl # LIBSSH2_PREFIX - Where to find libssh2 # LIBSSH2_VERSION= - Build the given version of libssh2 -# LIBGIT2_VERSION= - Build the given version of libgit2 +# LIBGIT2_VERSION= - Build the given version of libgit2 +# OPENSSL_VERSION= - Build the given version of OpenSSL +# (only needed for Mac universal on CI) # # Either use LIBSSH2_PREFIX, or LIBSSH2_VERSION, or none (if libssh2 is already # in the path, or if you don't want to use it). @@ -55,13 +57,24 @@ KERNEL=`uname -s` BUILD_TYPE=${BUILD_TYPE:-Debug} PYTHON=${PYTHON:-python3} -PYTHON_TAG=$($PYTHON build_tag.py) +if [ "$CIBUILDWHEEL" != "1" ]; then + PYTHON_TAG=$($PYTHON build_tag.py) +fi + PREFIX="${PREFIX:-$(pwd)/ci/$PYTHON_TAG}" export LDFLAGS="-Wl,-rpath,$PREFIX/lib" -# Create a virtual environment -$PYTHON -m venv $PREFIX -cd ci +if [ "$CIBUILDWHEEL" = "1" ]; then + rm -rf ci + mkdir ci || true + cd ci + if [ "$KERNEL" = "Linux" ]; then + yum install wget openssl-devel libssh2-devel zlib-devel -y + fi +else + # Create a virtual environment + $PYTHON -m venv $PREFIX +fi # Install zlib # XXX Build libgit2 with USE_BUNDLED_ZLIB instead? @@ -76,17 +89,67 @@ if [ -n "$ZLIB_VERSION" ]; then cd .. fi +# Install openssl +if [ -n "$OPENSSL_VERSION" ]; then + if [ "$CIBUILDWHEEL" != "1" ] || [ "$KERNEL" != "Darwin" ]; then + echo "OPENSSL_VERSION should only be set when building" + echo "macOS universal2 wheels on GitHub!" + echo "Please unset and try again" + exit 1 + fi + FILENAME=openssl-$OPENSSL_VERSION + wget https://www.openssl.org/source/$FILENAME.tar.gz -N --no-check-certificate + + tar xf $FILENAME.tar.gz + mv $FILENAME openssl-x86 + + tar xf $FILENAME.tar.gz + mv $FILENAME openssl-arm + + cd openssl-x86 + ./Configure darwin64-x86_64-cc shared + make + cd ../openssl-arm + ./Configure enable-rc5 zlib darwin64-arm64-cc no-asm + make + cd .. + + mkdir openssl-universal + + LIBSSL=$(basename openssl-x86/libssl.*.dylib) + lipo -create openssl-x86/libssl.*.dylib openssl-arm/libssl.*.dylib -output openssl-universal/$LIBSSL + LIBCRYPTO=$(basename openssl-x86/libcrypto.*.dylib) + lipo -create openssl-x86/libcrypto.*.dylib openssl-arm/libcrypto.*.dylib -output openssl-universal/$LIBCRYPTO + cd openssl-universal + install_name_tool -id "@rpath/$LIBSSL" $LIBSSL + install_name_tool -id "@rpath/$LIBCRYPTO" $LIBCRYPTO + OPENSSL_PREFIX=$(pwd) + cd .. +fi + # Install libssh2 if [ -n "$LIBSSH2_VERSION" ]; then FILENAME=libssh2-$LIBSSH2_VERSION wget https://www.libssh2.org/download/$FILENAME.tar.gz -N --no-check-certificate tar xf $FILENAME.tar.gz cd $FILENAME - cmake . \ - -DCMAKE_INSTALL_PREFIX=$PREFIX \ - -DBUILD_SHARED_LIBS=ON \ - -DBUILD_EXAMPLES=OFF \ - -DBUILD_TESTING=OFF + if [ "$KERNEL" = "Darwin" ] && [ "$CIBUILDWHEEL" = "1" ]; then + cmake . \ + -DCMAKE_INSTALL_PREFIX=$PREFIX \ + -DBUILD_SHARED_LIBS=ON \ + -DBUILD_EXAMPLES=OFF \ + -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \ + -DOPENSSL_CRYPTO_LIBRARY="../openssl-universal/$LIBCRYPTO" \ + -DOPENSSL_SSL_LIBRARY="../openssl-universal/$LIBSSL" \ + -DOPENSSL_INCLUDE_DIR="../openssl-x86/include" \ + -DBUILD_TESTING=OFF + else + cmake . \ + -DCMAKE_INSTALL_PREFIX=$PREFIX \ + -DBUILD_SHARED_LIBS=ON \ + -DBUILD_EXAMPLES=OFF \ + -DBUILD_TESTING=OFF + fi cmake --build . --target install cd .. LIBSSH2_PREFIX=$PREFIX @@ -98,18 +161,43 @@ if [ -n "$LIBGIT2_VERSION" ]; then wget https://github.com/libgit2/libgit2/archive/refs/tags/v$LIBGIT2_VERSION.tar.gz -N -O $FILENAME.tar.gz tar xf $FILENAME.tar.gz cd $FILENAME - CMAKE_PREFIX_PATH=$OPENSSL_PREFIX:$LIBSSH2_PREFIX cmake . \ - -DCMAKE_INSTALL_PREFIX=$PREFIX \ - -DBUILD_SHARED_LIBS=ON \ - -DBUILD_CLAR=OFF \ - -DCMAKE_BUILD_TYPE=$BUILD_TYPE + if [ "$KERNEL" = "Darwin" ] && [ "$CIBUILDWHEEL" = "1" ]; then + CMAKE_PREFIX_PATH=$OPENSSL_PREFIX:$LIBSSH2_PREFIX cmake . \ + -DBUILD_SHARED_LIBS=ON \ + -DBUILD_CLAR=OFF \ + -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \ + -DOPENSSL_CRYPTO_LIBRARY="../openssl-universal/$LIBCRYPTO" \ + -DOPENSSL_SSL_LIBRARY="../openssl-universal/$LIBSSL" \ + -DOPENSSL_INCLUDE_DIR="../openssl-x86/include" \ + -DCMAKE_BUILD_TYPE=$BUILD_TYPE + else + CMAKE_PREFIX_PATH=$OPENSSL_PREFIX:$LIBSSH2_PREFIX cmake . \ + -DCMAKE_INSTALL_PREFIX=$PREFIX \ + -DBUILD_SHARED_LIBS=ON \ + -DBUILD_CLAR=OFF \ + -DCMAKE_BUILD_TYPE=$BUILD_TYPE + fi cmake --build . --target install cd .. export LIBGIT2=$PREFIX fi +if [ "$CIBUILDWHEEL" = "1" ]; then + # This is gross. auditwheel/delocate-wheel are not so good + # at finding libraries in random places, so we have to + # put them in the loader path. + if [ "$KERNEL" = "Darwin" ]; then + cp -r $OPENSSL_PREFIX/*.dylib /usr/local/lib + cp -r $LIBSSH2_PREFIX/lib/*.dylib /usr/local/lib + cp -r $FILENAME/*.dylib /usr/local/lib + else + cp -r $PREFIX/lib64/*.so* /usr/local/lib + fi + # we're done building dependencies, cibuildwheel action will take over + exit 0 +fi + # Build pygit2 -cd .. $PREFIX/bin/pip install -U pip wheel if [ "$1" = "wheel" ]; then shift diff --git a/install-deps.sh b/install-deps.sh deleted file mode 100644 index e47da51a7..000000000 --- a/install-deps.sh +++ /dev/null @@ -1,177 +0,0 @@ -#!/bin/sh - -# -# Synopsis: -# -# sh build.sh - Build inplace -# sh build.sh test - Build inplace, and run the tests -# sh build.sh wheel - Build a wheel, install, and run the tests -# -# Environment variables: -# -# LIBSSH2_OPENSSL - Where to find openssl -# LIBSSH2_PREFIX - Where to find libssh2 -# LIBSSH2_VERSION= - Build the given version of libssh2 -# LIBGIT2_VERSION= - Build the given version of libgit2 -# -# Either use LIBSSH2_PREFIX, or LIBSSH2_VERSION, or none (if libssh2 is already -# in the path, or if you don't want to use it). -# -# Examples. -# -# Build inplace, libgit2 must be available in the path: -# -# sh build.sh -# -# Build libgit2 1.3.0 (will use libssh2 if available), then build pygit2 -# inplace: -# -# LIBGIT2_VERSION=1.3.0 sh build.sh -# -# Build libssh2 1.10.0 and libgit2 1.3.0, then build pygit2 inplace: -# -# LIBSSH2_VERSION=1.10.0 LIBGIT2_VERSION=1.3.0 sh build.sh -# -# Tell where libssh2 is installed, build libgit2 1.3.0, then build pygit2 -# inplace: -# -# LIBSSH2_PREFIX=/usr/local LIBGIT2_VERSION=1.3.0 sh build.sh -# -# Build inplace and run the tests: -# -# sh build.sh test -# -# Build a wheel: -# -# sh build.sh wheel -# - -set -x # Print every command and variable -set -e # Fail fast - -# Variables -ARCH=`uname -m` -KERNEL=`uname -s` -BUILD_TYPE=${BUILD_TYPE:-Debug} -PYTHON=${PYTHON:-python3} - -PREFIX="${PREFIX:-$(pwd)/ci/}" -export LDFLAGS="-Wl,-rpath,$PREFIX/lib" - -rm -rf ci -mkdir ci || true -cd ci - -if [ "$KERNEL" = "Linux" ]; then - yum install wget openssl-devel libssh2-devel zlib-devel -y -fi - - -# Install zlib -# XXX Build libgit2 with USE_BUNDLED_ZLIB instead? -if [ -n "$ZLIB_VERSION" ]; then - FILENAME=zlib-$ZLIB_VERSION - wget https://www.zlib.net/$FILENAME.tar.gz -N - tar xf $FILENAME.tar.gz - cd $FILENAME - ./configure --prefix=$PREFIX - make - make install - cd .. -fi - -# Install openssl -if [ -n "$OPENSSL_VERSION" ]; then - FILENAME=openssl-$OPENSSL_VERSION - wget https://www.openssl.org/source/$FILENAME.tar.gz -N --no-check-certificate - - tar xf $FILENAME.tar.gz - mv $FILENAME openssl-x86 - - tar xf $FILENAME.tar.gz - mv $FILENAME openssl-arm - - cd openssl-x86 - ./Configure darwin64-x86_64-cc shared - make - cd ../openssl-arm - ./Configure enable-rc5 zlib darwin64-arm64-cc no-asm - make - cd .. - - mkdir openssl-universal - - LIBSSL=$(basename openssl-x86/libssl.*.dylib) - lipo -create openssl-x86/libssl.*.dylib openssl-arm/libssl.*.dylib -output openssl-universal/$LIBSSL - LIBCRYPTO=$(basename openssl-x86/libcrypto.*.dylib) - lipo -create openssl-x86/libcrypto.*.dylib openssl-arm/libcrypto.*.dylib -output openssl-universal/$LIBCRYPTO - cd openssl-universal - install_name_tool -id "@rpath/$LIBSSL" $LIBSSL - install_name_tool -id "@rpath/$LIBCRYPTO" $LIBCRYPTO - OPENSSL_PREFIX=$(pwd) - cd .. -fi - -# Install libssh2 -if [ -n "$LIBSSH2_VERSION" ]; then - FILENAME=libssh2-$LIBSSH2_VERSION - wget https://www.libssh2.org/download/$FILENAME.tar.gz -N --no-check-certificate - tar xf $FILENAME.tar.gz - cd $FILENAME - if [ "$KERNEL" = "Darwin" ]; then - cmake . \ - -DCMAKE_INSTALL_PREFIX=$PREFIX \ - -DBUILD_SHARED_LIBS=ON \ - -DBUILD_EXAMPLES=OFF \ - -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \ - -DOPENSSL_CRYPTO_LIBRARY="../openssl-universal/$LIBCRYPTO" \ - -DOPENSSL_SSL_LIBRARY="../openssl-universal/$LIBSSL" \ - -DOPENSSL_INCLUDE_DIR="../openssl-x86/include" \ - -DBUILD_TESTING=OFF - else - cmake . \ - -DCMAKE_INSTALL_PREFIX=$PREFIX \ - -DBUILD_SHARED_LIBS=ON \ - -DBUILD_EXAMPLES=OFF \ - -DBUILD_TESTING=OFF - fi - cmake --build . --target install - cd .. - LIBSSH2_PREFIX=$PREFIX -fi - -# Install libgit2 -if [ -n "$LIBGIT2_VERSION" ]; then - FILENAME=libgit2-$LIBGIT2_VERSION - wget https://github.com/libgit2/libgit2/archive/refs/tags/v$LIBGIT2_VERSION.tar.gz -N -O $FILENAME.tar.gz - tar xf $FILENAME.tar.gz - cd $FILENAME - if [ "$KERNEL" = "Darwin" ]; then - CMAKE_PREFIX_PATH=$OPENSSL_PREFIX:$LIBSSH2_PREFIX cmake . \ - -DBUILD_SHARED_LIBS=ON \ - -DBUILD_CLAR=OFF \ - -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \ - -DOPENSSL_CRYPTO_LIBRARY="../openssl-universal/$LIBCRYPTO" \ - -DOPENSSL_SSL_LIBRARY="../openssl-universal/$LIBSSL" \ - -DOPENSSL_INCLUDE_DIR="../openssl-x86/include" \ - -DCMAKE_BUILD_TYPE=$BUILD_TYPE - else - CMAKE_PREFIX_PATH=$OPENSSL_PREFIX:$LIBSSH2_PREFIX cmake . \ - -DBUILD_SHARED_LIBS=ON \ - -DBUILD_CLAR=OFF \ - -DCMAKE_BUILD_TYPE=$BUILD_TYPE - fi - cmake --build . --target install - cd .. -fi - -if [ "$KERNEL" = "Darwin" ]; then - if [ "$GITHUB_ACTIONS" != "true" ]; then - echo "Refusing to install Universal openssl locally!" - exit 1 - fi - # This is gross - cp -r $OPENSSL_PREFIX/*.dylib /usr/local/lib - cp -r $LIBSSH2_PREFIX/lib/*.dylib /usr/local/lib - cp -r $FILENAME/*.dylib /usr/local/lib -fi From 16456c57580c725b60ac780a1e36828ba31c3415 Mon Sep 17 00:00:00 2001 From: Simon Cozens Date: Thu, 2 Dec 2021 08:51:36 +0000 Subject: [PATCH 6/6] Address feedback --- .github/workflows/wheels.yml | 3 ++- misc/build-all-manylinux-wheels.sh | 19 ------------- misc/build-manylinux-wheels.sh | 43 ------------------------------ 3 files changed, 2 insertions(+), 63 deletions(-) delete mode 100755 misc/build-all-manylinux-wheels.sh delete mode 100755 misc/build-manylinux-wheels.sh diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index b5d974d2a..782ea7fa7 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -28,7 +28,7 @@ jobs: platforms: all - name: Build wheels - uses: pypa/cibuildwheel@v2.2.2 + uses: pypa/cibuildwheel@v2.3.0 env: CIBW_BEFORE_ALL: sh build.sh CIBW_ENVIRONMENT: LIBGIT2_VERSION=1.3.0 LIBSSH2_VERSION=1.10.0 LIBGIT2=/project/ci @@ -41,6 +41,7 @@ jobs: - uses: actions/upload-artifact@v2 with: + name: wheels path: ./wheelhouse/*.whl pypi: diff --git a/misc/build-all-manylinux-wheels.sh b/misc/build-all-manylinux-wheels.sh deleted file mode 100755 index 9ec95dc59..000000000 --- a/misc/build-all-manylinux-wheels.sh +++ /dev/null @@ -1,19 +0,0 @@ -#! /usr/bin/env bash - -set -x -set -e - -ARCH=$1 - -# Wait for docker pull to complete downloading container -if [ $ARCH = 'aarch64' ]; then - manylinux_image="ghcr.io/pyca/cryptography-manylinux2014_${ARCH}" -else - manylinux_image="ghcr.io/pyca/cryptography-manylinux2014:${ARCH}" -fi -docker pull "${manylinux_image}" & -wait - -# Build wheels -docker run --rm -v `pwd`:/io/src "${manylinux_image}" /io/src/misc/build-manylinux-wheels.sh & -wait diff --git a/misc/build-manylinux-wheels.sh b/misc/build-manylinux-wheels.sh deleted file mode 100755 index c7eeed45b..000000000 --- a/misc/build-manylinux-wheels.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash - -set -xe - -ARCH=`uname -m` -PYTHONS="cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310" -export PYCA_OPENSSL_PATH=/opt/pyca/cryptography/openssl -export OPENSSL_PATH=/opt/openssl -export CFLAGS="-fPIC" -export PKG_CONFIG_PATH="${OPENSSL_PATH}/lib/pkgconfig:${PYCA_OPENSSL_PATH}/lib/pkgconfig" - -# Install requirements -yum -y install git libffi-devel -yum -y install openssl wget -if [ ${ARCH} == 'aarch64' ]; then - yum -y install cmake -else - yum -y install cmake3 -fi - -# Copy source directory so we don't mangle with it -SRC_DIR=/io/src -TMP_DIR=/io/tmp -rm $TMP_DIR -rf -cp -a $SRC_DIR $TMP_DIR - -# Build -pushd $TMP_DIR -export OPENSSL_PREFIX=/opt/pyca/cryptography/openssl -export ZLIB_VERSION=1.2.11 -export LIBSSH2_VERSION=1.10.0 -export LIBGIT2_VERSION=1.3.0 -export BUILD_TYPE=Release -for PY in $PYTHONS; do - PYTHON=/opt/python/${PY}/bin/python sh build.sh wheel bundle test -done -popd - -# Copy wheels to source dist folder -ls -l $TMP_DIR/wheelhouse -mkdir -p $SRC_DIR/dist -cp $TMP_DIR/wheelhouse/pygit2-*_$ARCH.whl $SRC_DIR/dist -chown -R --reference="${SRC_DIR}/setup.py" ${SRC_DIR}/dist