From a575618b8867cb14bb7987bcbcc5420155eb4f99 Mon Sep 17 00:00:00 2001 From: mayeut Date: Mon, 5 Apr 2021 17:31:53 +0200 Subject: [PATCH 1/3] Add ppc64le & s390x manylinux2014 builds --- .travis.yml | 21 +++++++--- scikit-ci.yml | 14 +------ ...ylinux2014-aarch64-build-and-test-wheel.sh | 32 --------------- ...anylinux2014-build-and-install-openssl.sh} | 20 ++++++--- scripts/manylinux2014-build-and-test-wheel.sh | 41 +++++++++++++++++++ 5 files changed, 72 insertions(+), 56 deletions(-) delete mode 100755 scripts/manylinux2014-aarch64-build-and-test-wheel.sh rename scripts/{manylinux2014-aarch64-build-and-install-openssl.sh => manylinux2014-build-and-install-openssl.sh} (77%) create mode 100755 scripts/manylinux2014-build-and-test-wheel.sh diff --git a/.travis.yml b/.travis.yml index 566c1f44b..785661d8f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,10 @@ branches: - master - /^[0-9]+(\.[0-9]+)*(\.post[0-9]+)?$/ +os: linux +dist: focal language: python +python: "3.8" matrix: include: @@ -18,12 +21,14 @@ matrix: env: - PYTHON_VERSION=2.7.18 - - os: linux - arch: arm64-graviton2 + - arch: arm64-graviton2 virt: vm group: edge - dist: focal - language: generic + + - arch: ppc64le + + - arch: s390x + cache: @@ -63,8 +68,8 @@ before_install: -v $(pwd):/io \ --env TRAVIS=True \ --env TRAVIS_OS_NAME="${TRAVIS_OS_NAME}" \ - quay.io/pypa/manylinux2014_aarch64 \ - /io/scripts/manylinux2014-aarch64-build-and-test-wheel.sh + quay.io/pypa/manylinux2014_$(uname -m) \ + /io/scripts/manylinux2014-build-and-test-wheel.sh fi install: @@ -72,6 +77,10 @@ install: if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then ci install elif [[ "$TRAVIS_OS_NAME" == "linux" ]]; then + MACHINE=$(uname -m) + if [[ "${MACHINE}" == "s390x" ]] || [[ "${MACHINE}" == "ppc64le" ]]; then + pip install $(pwd)/cryptography*.whl + fi pip install twine fi diff --git a/scikit-ci.yml b/scikit-ci.yml index 62b2ec7f3..3facc1c4e 100644 --- a/scikit-ci.yml +++ b/scikit-ci.yml @@ -13,10 +13,6 @@ before_install: PATH: /opt/python/$/bin:$ SETUP_CMAKE_ARGS: -DCMAKE_JOB_POOL_COMPILE:STRING=compile -DCMAKE_JOB_POOL_LINK:STRING=link '-DCMAKE_JOB_POOLS:STRING=compile=4;link=3' commands: - - | - if [[ ${DEFAULT_DOCKCROSS_IMAGE} == *"manylinux2014-aarch64"* ]]; then - ./scripts/manylinux2014-aarch64-build-and-install-openssl.sh - fi - python: | import ci, os image_name=os.environ["DEFAULT_DOCKCROSS_IMAGE"].split(":")[0].split("/")[1] @@ -37,14 +33,6 @@ before_install: # SETUP_CMAKE_ARGS if arch in ["x86", "x64"]: os.environ["SETUP_CMAKE_ARGS"] = "-DOPENSSL_ROOT_DIR:PATH=/usr/local/ssl " + os.environ["SETUP_CMAKE_ARGS"] - # SETUP_BDIST_WHEEL_ARGS - if arch == "aarch64": - os.environ["SETUP_BDIST_WHEEL_ARGS"] = "--plat-name %s" % os.environ["AUDITWHEEL_PLAT"] - setup_cmake_args = [] - # Remove this after addressing https://github.com/dockcross/dockcross/issues/431 - setup_cmake_args.append("-DOPENSSL_ROOT_DIR:PATH=/tmp/openssl-install") - setup_cmake_args.append("-DSTRIP_EXECUTABLE:FILEPATH=" + os.environ["CROSS_ROOT"] + "/" + os.environ["CROSS_TRIPLE"] + "/bin/strip") - os.environ["SETUP_CMAKE_ARGS"] = " ".join(setup_cmake_args) + " " + os.environ["SETUP_CMAKE_ARGS"] ci.driver.Driver.save_env(os.environ) travis: @@ -64,7 +52,7 @@ before_install: os.environ["SETUP_BDIST_WHEEL_ARGS"] = "--plat-name %s" % os.environ["AUDITWHEEL_PLAT"] setup_cmake_args = [] setup_cmake_args.append("-DSTRIP_EXECUTABLE:FILEPATH=/opt/rh/devtoolset-9/root/usr/" + "/bin/strip") - if platform.machine() == "aarch64": + if platform.machine() in {"aarch64", "ppc64le", "s390x"}: # Remove this after addressing https://github.com/dockcross/dockcross/issues/431 setup_cmake_args.append("-DOPENSSL_ROOT_DIR:PATH=/tmp/openssl-install") os.environ["SETUP_CMAKE_ARGS"] = " ".join(setup_cmake_args) diff --git a/scripts/manylinux2014-aarch64-build-and-test-wheel.sh b/scripts/manylinux2014-aarch64-build-and-test-wheel.sh deleted file mode 100755 index a5eeb5bb0..000000000 --- a/scripts/manylinux2014-aarch64-build-and-test-wheel.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -set -e -set -x - -MANYLINUX_PYTHON=cp38-cp38 -export PATH="/opt/python/${MANYLINUX_PYTHON}/bin:$PATH" - -yum install -y wget - -cd /io -./scripts/manylinux2014-aarch64-build-and-install-openssl.sh - -ci_before_install() { - /opt/python/${MANYLINUX_PYTHON}/bin/python scripts/ssl-check.py - /opt/python/${MANYLINUX_PYTHON}/bin/pip install scikit-ci scikit-ci-addons scikit-build -} - -ci_install() { - /opt/python/${MANYLINUX_PYTHON}/bin/ci install -} -ci_test() { - /opt/python/${MANYLINUX_PYTHON}/bin/ci test -} -ci_after_success() { - /opt/python/${MANYLINUX_PYTHON}/bin/ci after_test -} - -ci_before_install -ci_install -ci_test -ci_after_success diff --git a/scripts/manylinux2014-aarch64-build-and-install-openssl.sh b/scripts/manylinux2014-build-and-install-openssl.sh similarity index 77% rename from scripts/manylinux2014-aarch64-build-and-install-openssl.sh rename to scripts/manylinux2014-build-and-install-openssl.sh index de6fa6c3a..2aa17b48f 100755 --- a/scripts/manylinux2014-aarch64-build-and-install-openssl.sh +++ b/scripts/manylinux2014-build-and-install-openssl.sh @@ -21,27 +21,37 @@ check_var CROSS_TRIPLE # OPENSSL_INSTALL_DIR=${CROSS_ROOT}/${CROSS_TRIPLE} # Support using older manylinux2014-aarch64 images where 'sudo' is broken -OPENSSL_INSTALL_DIR=/tmp/openssl-install + +OPENSSL_INSTALL_DIR=$1 +shift cd /tmp # Download -wget http://www.openssl.org/source/${OPENSSL_ROOT}.tar.gz +curl -fsSLO http://www.openssl.org/source/${OPENSSL_ROOT}.tar.gz check_sha256sum ${OPENSSL_ROOT}.tar.gz ${OPENSSL_HASH} tar -xzf ${OPENSSL_ROOT}.tar.gz rm -rf ${OPENSSL_ROOT}.tar.gz +if [ "$(uname -m)" == "s390x" ]; then + TARGET=linux64-s390x +else + TARGET=linux-$(uname -m) +fi + # Configure cd ${OPENSSL_ROOT} ./Configure \ - linux-aarch64 \ + ${TARGET} \ --cross-compile-prefix= \ --prefix=${OPENSSL_INSTALL_DIR} \ - shared + $* # Build make -j$(nproc) # Install -make install +make install > /dev/null +cd /tmp +rm -rf ${OPENSSL_ROOT} diff --git a/scripts/manylinux2014-build-and-test-wheel.sh b/scripts/manylinux2014-build-and-test-wheel.sh new file mode 100755 index 000000000..3dcf257e7 --- /dev/null +++ b/scripts/manylinux2014-build-and-test-wheel.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +set -e +set -x + +MANYLINUX_PYTHON_BIN=/opt/python/cp38-cp38/bin +export PATH="${MANYLINUX_PYTHON_BIN}:$PATH" + +cd /io +./scripts/manylinux2014-build-and-install-openssl.sh /tmp/openssl-install shared + +MACHINE=$(uname -m) +if [ "${MACHINE}" == "s390x" ] || [ "${MACHINE}" == "ppc64le" ]; then + # build cryptography from sources + yum install -y libffi-devel + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path + export PATH=${HOME}/.cargo/bin:${PATH} + ./scripts/manylinux2014-build-and-install-openssl.sh /tmp/openssl-cryptography no-shared -fPIC + CFLAGS="-I/tmp/openssl-cryptography/include" LDFLAGS="-L/tmp/openssl-cryptography/lib" ${MANYLINUX_PYTHON_BIN}/pip wheel --no-binary :all: cryptography + ${MANYLINUX_PYTHON_BIN}/pip install ./cryptography*.whl +fi + +ci_before_install() { + ${MANYLINUX_PYTHON_BIN}/python scripts/ssl-check.py + ${MANYLINUX_PYTHON_BIN}/pip install scikit-ci scikit-ci-addons scikit-build +} + +ci_install() { + ${MANYLINUX_PYTHON_BIN}/ci install +} +ci_test() { + ${MANYLINUX_PYTHON_BIN}/ci test +} +ci_after_success() { + ${MANYLINUX_PYTHON_BIN}/ci after_test +} + +ci_before_install +ci_install +ci_test +ci_after_success From 5772f7e0dc35bd68a6fa7e7a3dca8a75c609be30 Mon Sep 17 00:00:00 2001 From: mayeut Date: Sat, 10 Apr 2021 01:19:46 +0200 Subject: [PATCH 2/3] Use same arguments as cryptography for rust compiler installation --- scripts/manylinux2014-build-and-test-wheel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/manylinux2014-build-and-test-wheel.sh b/scripts/manylinux2014-build-and-test-wheel.sh index 3dcf257e7..1454c9305 100755 --- a/scripts/manylinux2014-build-and-test-wheel.sh +++ b/scripts/manylinux2014-build-and-test-wheel.sh @@ -13,7 +13,7 @@ MACHINE=$(uname -m) if [ "${MACHINE}" == "s390x" ] || [ "${MACHINE}" == "ppc64le" ]; then # build cryptography from sources yum install -y libffi-devel - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path + curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain stable --profile minimal --no-modify-path export PATH=${HOME}/.cargo/bin:${PATH} ./scripts/manylinux2014-build-and-install-openssl.sh /tmp/openssl-cryptography no-shared -fPIC CFLAGS="-I/tmp/openssl-cryptography/include" LDFLAGS="-L/tmp/openssl-cryptography/lib" ${MANYLINUX_PYTHON_BIN}/pip wheel --no-binary :all: cryptography From 68e3b75a081764d12c50e4e00b78c09c0281d95d Mon Sep 17 00:00:00 2001 From: mayeut Date: Sat, 10 Apr 2021 01:50:51 +0200 Subject: [PATCH 3/3] Limit to parallel builds to 4 threads on ppc64le --- scikit-ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scikit-ci.yml b/scikit-ci.yml index 3facc1c4e..db993f4f5 100644 --- a/scikit-ci.yml +++ b/scikit-ci.yml @@ -56,7 +56,12 @@ before_install: # Remove this after addressing https://github.com/dockcross/dockcross/issues/431 setup_cmake_args.append("-DOPENSSL_ROOT_DIR:PATH=/tmp/openssl-install") os.environ["SETUP_CMAKE_ARGS"] = " ".join(setup_cmake_args) - os.environ["SETUP_MAKE_ARGS"] = "-j$(nproc)" + if platform.machine() in {"ppc64le"}: + # the container sees all 16 threads but building with all of them + # triggers the OoM killer due to RAM limitations + os.environ["SETUP_MAKE_ARGS"] = "-j4" + else: + os.environ["SETUP_MAKE_ARGS"] = "-j$(nproc)" ci.driver.Driver.save_env(os.environ) install: