From 8d0ac38fbf154104ccb74ff01dabf37bc487e629 Mon Sep 17 00:00:00 2001 From: mattip Date: Mon, 20 Jan 2020 14:44:29 +1100 Subject: [PATCH 1/3] clean up cruft from old openblas handling --- config.sh | 15 ++++++++----- env_vars.sh | 4 +++- env_vars_32.sh | 5 +---- gfortran-install | 2 +- on_openblas.rst | 55 --------------------------------------------- openblas_support.py | 42 ---------------------------------- 6 files changed, 15 insertions(+), 108 deletions(-) delete mode 100644 on_openblas.rst delete mode 100644 openblas_support.py diff --git a/config.sh b/config.sh index bc65dfe..052ba3f 100644 --- a/config.sh +++ b/config.sh @@ -8,10 +8,9 @@ function build_wheel { local lib_plat=$PLAT if [ -n "$IS_OSX" ]; then install_gfortran - else - # For manylinux2010 builds with manylinux1 openblas builds - $use_sudo yum install -y libgfortran-4.4.7 fi + echo gcc --version + echo `gcc --version` build_libs $lib_plat # Fix version error for development wheels by using bdist_wheel build_bdist_wheel $@ @@ -44,7 +43,13 @@ function run_tests { $PYTHON_EXE -c "$(get_test_cmd)" # Check bundled license file $PYTHON_EXE ../check_license.py - # Show BLAS / LAPACK used. Since this uses a wheel we cannot use - # tools/openblas_config.py; tools is not part of what is shipped + # Show BLAS / LAPACK used. Assumes this is one-directory-in + # so we can find tools/openblas_config.py; + if [ -e ../numpy/tools/openblas_support.py ]; then + $PYTHON_EXE ../numpy/tools/openblas_support.py --check_version + else + echo could not find ../numpy, ls -d .. is + ls -d .. + fi $PYTHON_EXE -c 'import numpy; numpy.show_config()' } diff --git a/env_vars.sh b/env_vars.sh index 31e0dc3..b1056ce 100644 --- a/env_vars.sh +++ b/env_vars.sh @@ -1,6 +1,5 @@ #!/usr/bin/env bash # Environment variables for build -OPENBLAS_VERSION="v0.3.7" MACOSX_DEPLOYMENT_TARGET=10.9 CFLAGS="-std=c99 -fno-strict-aliasing" # Macos's linker doesn't support stripping symbols @@ -13,3 +12,6 @@ if [ "$(uname)" != "Darwin" ]; then # the current version of manybuild pass "-strip-all" to CPPFLAGS and FFLAGS STRIP_FLAGS="" fi +# For verbosity: report where each command came from +export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }' +set -x diff --git a/env_vars_32.sh b/env_vars_32.sh index 72f3009..3f02e87 100644 --- a/env_vars_32.sh +++ b/env_vars_32.sh @@ -3,11 +3,8 @@ # The important difference from the 64-bit build is `-msse2` to # compile sse loops for ufuncs. set -x -OPENBLAS_VERSION="v0.3.7" MACOSX_DEPLOYMENT_TARGET=10.9 -# Causes failure for pre-1.19 in np.reciprocal -# CFLAGS="-msse2 -std=c99 -fno-strict-aliasing" -CFLAGS="-std=c99 -fno-strict-aliasing" +CFLAGS="-msse2 -std=c99 -fno-strict-aliasing" # Macos's linker doesn't support stripping symbols if [ "$(uname)" != "Darwin" ]; then LDFLAGS="-Wl,--strip-debug" diff --git a/gfortran-install b/gfortran-install index e9764d0..d430fe6 160000 --- a/gfortran-install +++ b/gfortran-install @@ -1 +1 @@ -Subproject commit e9764d0a8f4e3adcd385c319e7742177193484c7 +Subproject commit d430fe6e38b6c5149c53f775a4437964e2f7b883 diff --git a/on_openblas.rst b/on_openblas.rst deleted file mode 100644 index a4634d8..0000000 --- a/on_openblas.rst +++ /dev/null @@ -1,55 +0,0 @@ -######################## -Updating OpenBLAS builds -######################## - -A white-water ride... - -******* -Windows -******* - -Get access to or clone : https://github.com/matthew-brett/build-openblas - -Edit the OpenBLAS commit in the `appveyor.yml` file, and commit. - -This results in zipped OpenBLAS archives at https://github.com/matthew-brett/build-openblas. The archive names are from the OpenBLAS commit, the Mingw-w64 gcc version, and the bitness (32, 64). - -Download these build zip files to your machine, and calculate the SHA 256 hexdigests with something like:: - - curl -LO https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com/openblas-v0.3.0_gcc7_1_0_win32.zip - curl -LO https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com/openblas-v0.3.0_gcc7_1_0_win32.zip - shasum -a 256 openblas-v0.3.0*.zip - -Modify the `.appveyor.yml` or `appveyor.yml` files in: - -* https://github.com/MacPython/numpy-wheels -* https://github.com/MacPython/scipy-wheels - -with the URLs of the new zip files (above), and the SHA 256 hex digests, e.g.:: - - OPENBLAS_32: https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com/openblas-5f998ef_gcc7_1_0_win32.zip - OPENBLAS_64: https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com/openblas-5f998ef_gcc7_1_0_win64.zip - OPENBLAS_32_SHA256: 0a12804b08d475179a0411936f509b44d7512f084b4a81c2fa3abe8c9ac4ee09 - OPENBLAS_64_SHA256: 8f11d8a5a79795a3a1ccb2326c979a0ca426e623eee93f8e35239e3c21e62cd6 - -These builds take a long time. The 64 bit builds can take over 2 hours. I -(`matthew-brett`) got special permission from the Appeveyor team to run builds -for up to 2 hours 30 minutes, on the basis that they were important, and -infrequent. - -********** -Linux, OSX -********** - -See: https://github.com/MacPython/openblas-libs - -These builds, a bit confusingly, have names built from (from left to right, -separated by hyphens): - -#. ``openblas``. -#. ``git describe`` run on the OpenBLAS commit in the submodule. This commit - in turn comes from the `BUILD_COMMIT` specification in `.travis.yml`. -#. result of `uname`; -#. the platform (``i686`` or ``x86_64``), maybe followed by a specified suffix. -#. The suffix above is, by default, empty on Linux, and comes from the SHA - hexdigest of the GFortran archive for macOS. diff --git a/openblas_support.py b/openblas_support.py deleted file mode 100644 index 52d283a..0000000 --- a/openblas_support.py +++ /dev/null @@ -1,42 +0,0 @@ -import os -import textwrap - -def make_init(dirname): - ''' - Create a _distributor_init.py file for OpenBlas - ''' - with open(os.path.join(dirname, '_distributor_init.py'), 'wt') as fid: - fid.write(textwrap.dedent(""" - ''' - Helper to preload windows dlls to prevent dll not found errors. - Once a DLL is preloaded, its namespace is made available to any - subsequent DLL. This file originated in the numpy-wheels repo, - and is created as part of the scripts that build the wheel. - ''' - import os - from ctypes import WinDLL - import glob - if os.name == 'nt': - # convention for storing / loading the DLL from - # numpy/.libs/, if present - try: - basedir = os.path.dirname(__file__) - except: - pass - else: - libs_dir = os.path.abspath(os.path.join(basedir, '.libs')) - DLL_filenames = [] - if os.path.isdir(libs_dir): - for filename in glob.glob(os.path.join(libs_dir, - '*openblas*dll')): - # NOTE: would it change behavior to load ALL - # DLLs at this path vs. the name restriction? - WinDLL(os.path.abspath(filename)) - DLL_filenames.append(filename) - if len(DLL_filenames) > 1: - import warnings - warnings.warn("loaded more than 1 DLL from .libs:\\n%s" % - "\\n".join(DLL_filenames), - stacklevel=1) - """)) - From 6c9a1df2eec008f9fbece35e4ad192890cab5a7f Mon Sep 17 00:00:00 2001 From: mattip Date: Fri, 8 May 2020 07:23:29 +0300 Subject: [PATCH 2/3] WIP: revamp travis.yml for pypy, aarch64. Comment out failing builds --- .travis.yml | 118 +++++++++++++++++++++++++++++++++++++++++++++++ .travis.yml.back | 113 --------------------------------------------- config.sh | 14 ++---- env_vars_32.sh | 5 +- multibuild | 2 +- 5 files changed, 128 insertions(+), 124 deletions(-) create mode 100644 .travis.yml delete mode 100644 .travis.yml.back diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..55a05d0 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,118 @@ +env: + global: + - REPO_DIR=numpy + # Also see CRON_COMMIT below + - BUILD_COMMIT=master + - BUILD_DEPENDS=cython==0.29.16 + - TEST_DEPENDS="pytest hypothesis cffi pytz" + # Commit when running from cron job + - CRON_COMMIT=master + - EXTRA_ARGV="'--disable-pytest-warnings'" + +language: python +dist: bionic +services: docker +os: linux + +jobs: + include: + - os: linux + arch: x86_64 + env: + - MB_PYTHON_VERSION=pypy3.6-7.3.1 + - MB_ML_VER=2010 + - DOCKER_TEST_IMAGE=multibuild/xenial_{PLAT} + + # The manylinux2014 arm64 image has libbz2.so.1 and libbz.so.1.0.6 but not + # libbz2.so.1.0, leading to this error with pypy: + # pypy3.6-v7.3.1-aarch64/bin/pypy: error while loading shared libraries: + # libbz2.so.1.0: cannot open shared object file: No such file or directory + #- os: linux + # arch: arm64 + # env: + # - PLAT=aarch64 + # - MB_ML_VER=2014 + # - MB_PYTHON_VERSION=pypy3.6-7.3.1 + # - DOCKER_TEST_IMAGE=multibuild/xenial_{PLAT} + + # The pypy3 osx package has bin/pypy3 not bin/pypy, so + # multibuild/common_utils.sh has a "ln" command (not "ln -s"). Apparently + # it does not work, we get the error message: + # multibuild/common_utils.sh: line 476: + # /Users/travis/build/MacPython/numpy-wheels/pypy3.6-v7.3.1-linux64/bin/pypy: + # cannot execute binary file + # - os: osx + # language: generic + # env: + # - MB_PYTHON_VERSION=pypy3.6-7.3.1 + # - MB_PYTHON_OSX_VER=10.9 + + - os: linux + arch: arm64 + env: + - PLAT=aarch64 + - MB_ML_VER=2014 + - MB_PYTHON_VERSION=3.6 + - DEBUG_PRINT=1 + - DOCKER_TEST_IMAGE=multibuild/xenial_{PLAT} + - os: linux + arch: arm64 + env: + - PLAT=aarch64 + - MB_ML_VER=2014 + - MB_PYTHON_VERSION=3.8 + - DEBUG_PRINT=1 + - DOCKER_TEST_IMAGE=multibuild/xenial_{PLAT} + - os: linux + arch: arm64 + env: + - PLAT=aarch64 + - MB_ML_VER=2014 + - MB_PYTHON_VERSION=3.7 + - DEBUG_PRINT=1 + - DOCKER_TEST_IMAGE=multibuild/xenial_{PLAT} + +before_install: + - if [ "$TRAVIS_EVENT_TYPE" == "cron" ]; then + CONTAINER="pre-release"; + BUILD_COMMIT=${CRON_COMMIT}; + NPY_RELAXED_STRIDES_DEBUG=1; + else + CONTAINER=wheels; + fi + # Set DEBUG_PRINT environment variable in settings + - if [ -n "${DEBUG_PRINT}" ]; then set -x; fi + - source multibuild/common_utils.sh + - source multibuild/travis_steps.sh + - source extra_functions.sh + - before_install + +install: + # Maybe get and clean and patch source + - clean_code $REPO_DIR $BUILD_COMMIT + - ./patch_code.sh $REPO_DIR + - build_wheel $REPO_DIR $PLAT + +script: + - install_run $PLAT + +after_success: + # trigger an upload to the shared ecosystem + # infrastructure at: https://anaconda.org/scipy-wheels-nightly + # for cron jobs only (restricted to master branch once + # per week) + # The tokens are set from + # https://travis-ci.org/github/MacPython/numpy-wheels/settings + # originally generated at + # anaconda.org/scipy-wheels-nightly/settings/access + - if [ "$TRAVIS_BRANCH" == "master" ]; then + ANACONDA_ORG="scipy-wheels-nightly"; + TOKEN=${NUMPY_NIGHTLY_UPLOAD_TOKEN}; + else + ANACONDA_ORG="multibuild-wheels-staging"; + TOKEN=${NUMPY_STAGING_UPLOAD_TOKEN}; + fi + - pip install git+https://github.com/Anaconda-Server/anaconda-client; + - if [ -n "${TOKEN}" ] ;then + anaconda -t ${TOKEN} upload u ${ANACONDA_ORG} ${TRAVIS_BUILD_DIR}/wheelhouse/*.whl; + fi diff --git a/.travis.yml.back b/.travis.yml.back deleted file mode 100644 index fcf26e3..0000000 --- a/.travis.yml.back +++ /dev/null @@ -1,113 +0,0 @@ -env: - global: - - REPO_DIR=numpy - # Also see DAILY_COMMIT below - - BUILD_COMMIT=e94cec800304a6a467cf90ce4e7d3e207770b4b4 - - BUILD_DEPENDS=cython - - TEST_DEPENDS="pytest hypothesis" - - WHEELHOUSE_UPLOADER_USERNAME=travis-worker - # Following generated with - # travis encrypt -r MacPython/numpy-wheels WHEELHOUSE_UPLOADER_SECRET= - - secure: - "FEFwGNflXOjuNjv4f2oOq0m8I3b6OvG5AVV/tH9Fsr6jtPNwQOx7PMOVzISBrWfAueeFubz/IapXBeDP8fRio9RVOcEHBhVJQYeJ46VubOx4X/4pRUD7ehGLxjYK30BGVdNh7z/tL6Xr5+iCuxxmOBgirG0pQVeBZXDchPJaUK0=" - # Commit when running from daily branch - - DAILY_COMMIT=master - - EXTRA_ARGV="'--disable-pytest-warnings'" - -language: python -dist: bionic -services: docker -os: linux - -jobs: - include: - - os: linux - env: - - MB_PYTHON_VERSION=3.6 - - MB_ML_VER=2010 - - PLAT=x86_64 - - os: linux - env: - - MB_PYTHON_VERSION=3.6 - - MB_ML_VER=2010 - - PLAT=i686 - - ENV_VARS_PATH=env_vars_32.sh - - DOCKER_TEST_IMAGE=multibuild/xenial_{PLAT} - - os: linux - env: - - MB_PYTHON_VERSION=3.7 - - MB_ML_VER=2010 - - PLAT=x86_64 - - os: linux - env: - - MB_PYTHON_VERSION=3.7 - - MB_ML_VER=2010 - - PLAT=i686 - - ENV_VARS_PATH=env_vars_32.sh - - DOCKER_TEST_IMAGE=multibuild/xenial_{PLAT} - - os: linux - env: - - MB_PYTHON_VERSION=3.8 - - MB_ML_VER=2010 - - PLAT=x86_64 - - os: linux - env: - - MB_PYTHON_VERSION=3.8 - - MB_ML_VER=2010 - - PLAT=i686 - - ENV_VARS_PATH=env_vars_32.sh - - DOCKER_TEST_IMAGE=multibuild/xenial_{PLAT} - - os: osx - language: generic - osx_image: xcode10.1 - env: - - MB_PYTHON_VERSION=3.6 - - MB_PYTHON_OSX_VER=10.9 - - os: osx - language: generic - osx_image: xcode10.1 - env: - - MB_PYTHON_VERSION=3.7 - - MB_PYTHON_OSX_VER=10.9 - - os: osx - language: generic - osx_image: xcode10.1 - env: - - MB_PYTHON_VERSION=3.8 - - MB_PYTHON_OSX_VER=10.9 - -before_install: - - if [ "$TRAVIS_BRANCH" == "master" ]; then - CONTAINER="pre-release"; - BUILD_COMMIT=${DAILY_COMMIT:-$BUILD_COMMIT}; - NPY_RELAXED_STRIDES_DEBUG=1; - else - CONTAINER=wheels; - UPLOAD_ARGS="--no-update-index"; - fi - # Set DEBUG_PRINT environment variable in settings - - if [ -n "${DEBUG_PRINT}" ]; then set -x; fi - - source multibuild/common_utils.sh - - source multibuild/travis_steps.sh - - before_install - -install: - # Maybe get and clean and patch source - - clean_code $REPO_DIR $BUILD_COMMIT - - ./patch_code.sh $REPO_DIR - - if [ "$TRAVIS_OS_NAME" == "linux" ]; then - export CFLAGS=${CFLAGS}" -Wno-sign-compare -Wno-unused-result\ - -Wno-strict-aliasing"; - fi - - build_wheel $REPO_DIR $PLAT - -script: - - install_run $PLAT - -after_success: - # Upload wheels to Rackspace container - - pip install wheelhouse-uploader - - python -m wheelhouse_uploader upload --local-folder - ${TRAVIS_BUILD_DIR}/wheelhouse/ - $UPLOAD_ARGS - $CONTAINER diff --git a/config.sh b/config.sh index 052ba3f..b19f891 100644 --- a/config.sh +++ b/config.sh @@ -21,9 +21,9 @@ function build_libs { # download and un-tar the openblas libraries. The python call returns # the un-tar root directory, then the files are copied into /usr/local. # Could utilize a site.cfg instead to prevent the copy. - python -mpip install urllib3 - python -c"import platform; print('platform.uname().machine', platform.uname().machine)" - basedir=$(python numpy/tools/openblas_support.py) + $PYTHON_EXE -mpip install urllib3 + $PYTHON_EXE -c"import platform; print('platform.uname().machine', platform.uname().machine)" + basedir=$($PYTHON_EXE numpy/tools/openblas_support.py) $use_sudo cp -r $basedir/lib/* /usr/local/lib $use_sudo cp $basedir/include/* /usr/local/include } @@ -43,13 +43,9 @@ function run_tests { $PYTHON_EXE -c "$(get_test_cmd)" # Check bundled license file $PYTHON_EXE ../check_license.py - # Show BLAS / LAPACK used. Assumes this is one-directory-in - # so we can find tools/openblas_config.py; - if [ -e ../numpy/tools/openblas_support.py ]; then + # Test BLAS / LAPACK used + if [ -n "$IS_LINUX" -o -n "$IS_OSX" ]; then $PYTHON_EXE ../numpy/tools/openblas_support.py --check_version - else - echo could not find ../numpy, ls -d .. is - ls -d .. fi $PYTHON_EXE -c 'import numpy; numpy.show_config()' } diff --git a/env_vars_32.sh b/env_vars_32.sh index 3f02e87..7136cea 100644 --- a/env_vars_32.sh +++ b/env_vars_32.sh @@ -4,7 +4,10 @@ # compile sse loops for ufuncs. set -x MACOSX_DEPLOYMENT_TARGET=10.9 -CFLAGS="-msse2 -std=c99 -fno-strict-aliasing" + +# Fails test_umath.TestAVXUfuncs with reciprocal on Azure +# CFLAGS="-msse2 -std=c99 -fno-strict-aliasing" +CFLAGS="-std=c99 -fno-strict-aliasing" # Macos's linker doesn't support stripping symbols if [ "$(uname)" != "Darwin" ]; then LDFLAGS="-Wl,--strip-debug" diff --git a/multibuild b/multibuild index 2a3f785..35fbf1f 160000 --- a/multibuild +++ b/multibuild @@ -1 +1 @@ -Subproject commit 2a3f785096d791a5cd07c4320a041e37a055be5c +Subproject commit 35fbf1f828595209876b75c2363c104d954e173e From 156222335315f19f1839a47e9e2b6a75be872be2 Mon Sep 17 00:00:00 2001 From: mattip Date: Wed, 13 May 2020 07:49:08 +0300 Subject: [PATCH 3/3] refactor azure to specify BUILD_COMMIT once, not twice --- README.rst | 4 ++-- azure-pipelines.yml | 3 +++ azure/posix.yml | 1 - azure/windows.yml | 1 - 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 5f628f3..0ee3f35 100644 --- a/README.rst +++ b/README.rst @@ -58,7 +58,7 @@ where user is the ``ANACONDA_ORG`` value in the ``yml`` files. Triggering a build ================== -You will likely want to edit the ``azure/*`` files to +You will likely want to edit the ``azure-pipelines.yml`` files to specify the ``BUILD_COMMIT`` before triggering a build - see below. You will need write permission to the github repository to trigger new builds. @@ -75,7 +75,7 @@ Which numpy commit does the repository build? PRs merged to this repo from a fork, and commits directly pushed to this repo will build the commit specified in the ``BUILD_COMMIT`` at the top of the -``azure/windows.yml`` and ``azure/posix.yml`` files, the wheels will be +``azure-pipelines.yml`` file, the wheels will be uploaded to https://anaconda.org/multibuild-wheels-staging/numpy. The ``NIGHTLY_BUILD_COMMIT`` is built once a week (sorry for the misnomer), and uploaded to https://anaconda.org/scipy-wheels-nightly/. diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e78f9cb..0e00001 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -13,6 +13,9 @@ trigger: pr: - master +variables: + BUILD_COMMIT: "master" + jobs: - template: azure/windows.yml parameters: diff --git a/azure/posix.yml b/azure/posix.yml index 9c90bdc..a106a2f 100644 --- a/azure/posix.yml +++ b/azure/posix.yml @@ -9,7 +9,6 @@ jobs: vmImage: ${{ parameters.vmImage }} variables: REPO_DIR: "numpy" - BUILD_COMMIT: "master" PLAT: "x86_64" CYTHON_BUILD_DEP: "cython==0.29.16" NIGHTLY_BUILD_COMMIT: "master" diff --git a/azure/windows.yml b/azure/windows.yml index 14eb6a0..1b7ef37 100644 --- a/azure/windows.yml +++ b/azure/windows.yml @@ -8,7 +8,6 @@ jobs: pool: vmImage: ${{ parameters.vmImage }} variables: - BUILD_COMMIT: "master" NIGHTLY_BUILD_COMMIT: "master" JUNITXML: "test-data.xml" TEST_DIR: '$(Agent.WorkFolder)/tmp_for_test'