Use cibuildwheel; build MacOS universal

.github/workflows/wheels.yml

@@ -4,85 +4,49 @@ on:
   push:
     branches:
     - master
+    - cibuildwheel
     tags:
     - 'v*'
 
-env:
-  LIBGIT2_VERSION: 1.3.0
-
 jobs:
-  linux:
-    name: Linux
-    runs-on: ubuntu-20.04
+  build_wheels:
+    name: Build wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        arch: [x86_64, aarch64]
-    steps:
-    - name: Checkout pygit2
-      uses: actions/checkout@v2
-
-    - name: Set up QEMU
-      if: ${{ matrix.arch == 'aarch64' }}
-      uses: docker/setup-qemu-action@v1
+        os: [ubuntu-20.04, macOS-11]
 
-    - run: |
-        /bin/sh misc/build-all-manylinux-wheels.sh ${{ matrix.arch }}
-
-    - name: Save wheels
-      uses: actions/upload-artifact@v2
-      with:
-        name: wheels
-        path: dist/*.whl
-
-  macos:
-    name: macos-${{ matrix.py.ver }}
-    runs-on: macos-latest
-    env:
-      MACOSX_DEPLOYMENT_TARGET: "10.9"
-    strategy:
-      fail-fast: false
-      matrix:
-        py:
-          - ver: '3.7'
-            release: '3.7.9'  # last Python.org binary release
-            url: https://www.python.org/ftp/python/3.7.9/python-3.7.9-macosx10.9.pkg
-          - ver: '3.8'
-            release: '3.8.10' # last Python.org binary release
-            url: https://www.python.org/ftp/python/3.8.10/python-3.8.10-macosx10.9.pkg
-          - ver: '3.9'
-            release: '3.9.7'
-            url: https://www.python.org/ftp/python/3.9.7/python-3.9.7-macosx10.9.pkg
-          - ver: '3.10'
-            release: '3.10.0'
-            url: https://www.python.org/ftp/python/3.10.0/python-3.10.0post2-macos11.pkg
     steps:
-    - name: Checkout pygit2
-      uses: actions/checkout@v2
+      - uses: actions/checkout@v2
 
-    - name: Setup python
-      run: |
-        PKG=$(basename "${{ matrix.py.url }}")
-        wget --no-verbose -N "${{ matrix.py.url }}"
-        sudo installer -pkg $PKG -target /
-        export PATH=/Library/Frameworks/Python.framework/Versions/${{ matrix.py.ver }}/bin:$PATH
-        echo "/Library/Frameworks/Python.framework/Versions/${{ matrix.py.ver }}/bin" >> $GITHUB_PATH
-        which python3
-        python3 -VV
-        test "$(python3 -V)" = "Python ${{ matrix.py.release }}"
+      - uses: actions/setup-python@v2
 
-    - name: Build
-      run: |
-        /bin/sh build.sh wheel bundle test
+      - name: Set up QEMU
+        if: runner.os == 'Linux'
+        uses: docker/setup-qemu-action@v1
+        with:
+          platforms: all
 
-    - name: Save wheel
-      uses: actions/upload-artifact@v2
-      with:
-        name: wheels
-        path: wheelhouse/*.whl
+      - name: Build wheels
+        uses: pypa/[email protected]
+        env:
+          CIBW_BEFORE_ALL: sh build.sh
+          CIBW_ENVIRONMENT: LIBGIT2_VERSION=1.3.0 LIBSSH2_VERSION=1.10.0 LIBGIT2=/project/ci
+          CIBW_ENVIRONMENT_MACOS: LIBGIT2_VERSION=1.3.0 LIBSSH2_VERSION=1.10.0 DYLD_LIBRARY_PATH=/usr/local/lib OPENSSL_VERSION=3.0.0
+          CIBW_SKIP: "*-musllinux*"
+          CIBW_REPAIR_WHEEL_COMMAND_LINUX: "DYLD_LIBRARY_PATH=/project/ci/lib64 auditwheel repair -w {dest_dir} {wheel}"
+          CIBW_REPAIR_WHEEL_COMMAND_MACOS: "DYLD_LIBRARY_PATH=/usr/local/lib delocate-listdeps {wheel} && DYLD_LIBRARY_PATH=/usr/local/lib delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}"
+          CIBW_ARCHS_MACOS: x86_64 universal2
+          CIBW_ARCHS_LINUX: x86_64 aarch64
+
+      - uses: actions/upload-artifact@v2
+        with:
+          name: wheels
+          path: ./wheelhouse/*.whl
 
   pypi:
     if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
-    needs: [linux, macos]
+    needs: [build_wheels]
     runs-on: ubuntu-20.04
 
     steps:

build.sh

@@ -12,7 +12,9 @@
 #   LIBSSH2_OPENSSL           - Where to find openssl
 #   LIBSSH2_PREFIX            - Where to find libssh2
 #   LIBSSH2_VERSION=<Version> - Build the given version of libssh2
-#   LIBGIT2_VERSION=<Versoin> - Build the given version of libgit2
+#   LIBGIT2_VERSION=<Version> - Build the given version of libgit2
+#   OPENSSL_VERSION=<Version> - Build the given version of OpenSSL
+#                               (only needed for Mac universal on CI)
 #
 # Either use LIBSSH2_PREFIX, or LIBSSH2_VERSION, or none (if libssh2 is already
 # in the path, or if you don't want to use it).
@@ -55,13 +57,24 @@ KERNEL=`uname -s`
 BUILD_TYPE=${BUILD_TYPE:-Debug}
 PYTHON=${PYTHON:-python3}
 
-PYTHON_TAG=$($PYTHON build_tag.py)
+if [ "$CIBUILDWHEEL" != "1" ]; then
+    PYTHON_TAG=$($PYTHON build_tag.py)
+fi
+
 PREFIX="${PREFIX:-$(pwd)/ci/$PYTHON_TAG}"
 export LDFLAGS="-Wl,-rpath,$PREFIX/lib"
 
-# Create a virtual environment
-$PYTHON -m venv $PREFIX
-cd ci
+if [ "$CIBUILDWHEEL" = "1" ]; then
+    rm -rf ci
+    mkdir ci || true
+    cd ci
+    if [ "$KERNEL" = "Linux" ]; then
+        yum install wget openssl-devel libssh2-devel zlib-devel -y
+    fi
+else
+    # Create a virtual environment
+    $PYTHON -m venv $PREFIX
+fi
 
 # Install zlib
 # XXX Build libgit2 with USE_BUNDLED_ZLIB instead?
@@ -76,17 +89,67 @@ if [ -n "$ZLIB_VERSION" ]; then
     cd ..
 fi
 
+# Install openssl
+if [ -n "$OPENSSL_VERSION" ]; then
+    if [ "$CIBUILDWHEEL" != "1" ] || [ "$KERNEL" != "Darwin" ]; then
+        echo "OPENSSL_VERSION should only be set when building"
+        echo "macOS universal2 wheels on GitHub!"
+        echo "Please unset and try again"
+        exit 1
+    fi
+    FILENAME=openssl-$OPENSSL_VERSION
+    wget https://www.openssl.org/source/$FILENAME.tar.gz -N --no-check-certificate
+
+    tar xf $FILENAME.tar.gz
+    mv $FILENAME openssl-x86
+
+    tar xf $FILENAME.tar.gz
+    mv $FILENAME openssl-arm
+
+    cd openssl-x86
+    ./Configure darwin64-x86_64-cc shared
+    make
+    cd ../openssl-arm
+    ./Configure enable-rc5 zlib darwin64-arm64-cc no-asm
+    make
+    cd ..
+
+    mkdir openssl-universal
+
+    LIBSSL=$(basename openssl-x86/libssl.*.dylib)
+    lipo -create openssl-x86/libssl.*.dylib openssl-arm/libssl.*.dylib -output openssl-universal/$LIBSSL
+    LIBCRYPTO=$(basename openssl-x86/libcrypto.*.dylib)
+    lipo -create openssl-x86/libcrypto.*.dylib openssl-arm/libcrypto.*.dylib -output openssl-universal/$LIBCRYPTO
+    cd openssl-universal
+    install_name_tool -id "@rpath/$LIBSSL" $LIBSSL
+    install_name_tool -id "@rpath/$LIBCRYPTO" $LIBCRYPTO
+    OPENSSL_PREFIX=$(pwd)
+    cd ..
+fi
+
 # Install libssh2
 if [ -n "$LIBSSH2_VERSION" ]; then
     FILENAME=libssh2-$LIBSSH2_VERSION
     wget https://www.libssh2.org/download/$FILENAME.tar.gz -N --no-check-certificate
     tar xf $FILENAME.tar.gz
     cd $FILENAME
-    cmake . \
-            -DCMAKE_INSTALL_PREFIX=$PREFIX \
-            -DBUILD_SHARED_LIBS=ON \
-            -DBUILD_EXAMPLES=OFF \
-            -DBUILD_TESTING=OFF
+    if [ "$KERNEL" = "Darwin" ] && [ "$CIBUILDWHEEL" = "1" ]; then
+        cmake . \
+                -DCMAKE_INSTALL_PREFIX=$PREFIX \
+                -DBUILD_SHARED_LIBS=ON \
+                -DBUILD_EXAMPLES=OFF \
+                -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \
+                -DOPENSSL_CRYPTO_LIBRARY="../openssl-universal/$LIBCRYPTO" \
+                -DOPENSSL_SSL_LIBRARY="../openssl-universal/$LIBSSL" \
+                -DOPENSSL_INCLUDE_DIR="../openssl-x86/include" \
+                -DBUILD_TESTING=OFF
+    else
+        cmake . \
+                -DCMAKE_INSTALL_PREFIX=$PREFIX \
+                -DBUILD_SHARED_LIBS=ON \
+                -DBUILD_EXAMPLES=OFF \
+                -DBUILD_TESTING=OFF
+    fi
     cmake --build . --target install
     cd ..
     LIBSSH2_PREFIX=$PREFIX
@@ -98,18 +161,43 @@ if [ -n "$LIBGIT2_VERSION" ]; then
     wget https://github.com/libgit2/libgit2/archive/refs/tags/v$LIBGIT2_VERSION.tar.gz -N -O $FILENAME.tar.gz
     tar xf $FILENAME.tar.gz
     cd $FILENAME
-    CMAKE_PREFIX_PATH=$OPENSSL_PREFIX:$LIBSSH2_PREFIX cmake . \
-            -DCMAKE_INSTALL_PREFIX=$PREFIX \
-            -DBUILD_SHARED_LIBS=ON \
-            -DBUILD_CLAR=OFF \
-            -DCMAKE_BUILD_TYPE=$BUILD_TYPE
+    if [ "$KERNEL" = "Darwin" ] && [ "$CIBUILDWHEEL" = "1" ]; then
+        CMAKE_PREFIX_PATH=$OPENSSL_PREFIX:$LIBSSH2_PREFIX cmake . \
+                -DBUILD_SHARED_LIBS=ON \
+                -DBUILD_CLAR=OFF \
+                -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" \
+                -DOPENSSL_CRYPTO_LIBRARY="../openssl-universal/$LIBCRYPTO" \
+                -DOPENSSL_SSL_LIBRARY="../openssl-universal/$LIBSSL" \
+                -DOPENSSL_INCLUDE_DIR="../openssl-x86/include" \
+                -DCMAKE_BUILD_TYPE=$BUILD_TYPE
+    else
+        CMAKE_PREFIX_PATH=$OPENSSL_PREFIX:$LIBSSH2_PREFIX cmake . \
+                -DCMAKE_INSTALL_PREFIX=$PREFIX \
+                -DBUILD_SHARED_LIBS=ON \
+                -DBUILD_CLAR=OFF \
+                -DCMAKE_BUILD_TYPE=$BUILD_TYPE
+    fi
     cmake --build . --target install
     cd ..
     export LIBGIT2=$PREFIX
 fi
 
+if [ "$CIBUILDWHEEL" = "1" ]; then
+    # This is gross. auditwheel/delocate-wheel are not so good
+    # at finding libraries in random places, so we have to
+    # put them in the loader path.
+    if [ "$KERNEL" = "Darwin" ]; then
+        cp -r $OPENSSL_PREFIX/*.dylib /usr/local/lib
+        cp -r $LIBSSH2_PREFIX/lib/*.dylib /usr/local/lib
+        cp -r $FILENAME/*.dylib /usr/local/lib
+    else
+        cp -r $PREFIX/lib64/*.so* /usr/local/lib
+    fi
+    # we're done building dependencies, cibuildwheel action will take over
+    exit 0
+fi
+
 # Build pygit2
-cd ..
 $PREFIX/bin/pip install -U pip wheel
 if [ "$1" = "wheel" ]; then
     shift