reuse numpy azure configuration for windows, cleanups

Author: mattip

.appveyor.yml

@@ -0,0 +1,10 @@
+image: Ubuntu
+
+install:
+- echo done
+
+test_script:
+- echo done
+
+build: off
+

azure-pipelines.yml

@@ -1,7 +1,7 @@
 schedules:
 - cron: "27 3 */1 * *"
   # 3:27am UTC everyday
-  displayName: Nighthly build
+  displayName: Nightly build
   branches:
     include:
     - master
@@ -22,27 +22,32 @@ jobs:
         py_3.6_32:
           PYTHON_VERSION: "3.6.x"
           PYTHON_ARCH: "x86"
+          BITS: 32
         py_3.6_64:
           PYTHON_VERSION: "3.6.x"
+          PYTHON_ARCH: 'x64'
+          BITS: 64
         py_3.7_32:
           PYTHON_VERSION: "3.7.x"
           PYTHON_ARCH: "x86"
-          NIGHTLY_BUILD: "true"
+          BITS: 32
         py_3.7_64:
           PYTHON_VERSION: "3.7.x"
-          NIGHTLY_BUILD: "true"
+          PYTHON_ARCH: 'x64'
+          BITS: 64
         py_3.8_32:
           PYTHON_VERSION: "3.8.x"
           PYTHON_ARCH: "x86"
-          NIGHTLY_BUILD: "true"
+          BITS: 32
         py_3.8_64:
           PYTHON_VERSION: "3.8.x"
-          NIGHTLY_BUILD: "true"
+          PYTHON_ARCH: 'x64'
+          BITS: 64
 
   - template: azure/posix.yml
     parameters:
       name: linux
-      vmImage: ubuntu-16.04
+      vmImage: ubuntu-18.04
       matrix:
         py_3.6_32:
           MB_PYTHON_VERSION: "3.6"
@@ -56,24 +61,20 @@ jobs:
         py_3.7_32:
           MB_PYTHON_VERSION: "3.7"
           PLAT: "i686"
-          NIGHTLY_BUILD: "true"
           MB_ML_VER: "2010"
           ENV_VARS_PATH: "env_vars_32.sh"
           DOCKER_TEST_IMAGE: "multibuild/xenial_{PLAT}"
         py_3.7_64:
           MB_PYTHON_VERSION: "3.7"
-          NIGHTLY_BUILD: "true"
           MB_ML_VER: "2010"
         py_3.8_32:
           MB_PYTHON_VERSION: "3.8"
           PLAT: "i686"
-          NIGHTLY_BUILD: "true"
           MB_ML_VER: "2010"
           ENV_VARS_PATH: "env_vars_32.sh"
           DOCKER_TEST_IMAGE: "multibuild/xenial_{PLAT}"
         py_3.8_64:
           MB_PYTHON_VERSION: "3.8"
-          NIGHTLY_BUILD: "true"
           MB_ML_VER: "2010"
 
   - template: azure/posix.yml
@@ -83,12 +84,7 @@ jobs:
       matrix:
         py_3.6_64:
           MB_PYTHON_VERSION: "3.6"
-          MB_PYTHON_OSX_VER: 10.9
         py_3.7_64:
           MB_PYTHON_VERSION: "3.7"
-          NIGHTLY_BUILD: "true"
-          MB_PYTHON_OSX_VER: 10.9
         py_3.8_64:
           MB_PYTHON_VERSION: "3.8"
-          NIGHTLY_BUILD: "true"
-          MB_PYTHON_OSX_VER: 10.9

azure/posix.yml

@@ -9,11 +9,10 @@ jobs:
       vmImage: ${{ parameters.vmImage }}
     variables:
       REPO_DIR: "numpy"
-      BUILD_COMMIT: "master"
+      BUILD_COMMIT: 'v1.18.3'
       PLAT: "x86_64"
       CYTHON_BUILD_DEP: "cython==0.29.16"
       NIGHTLY_BUILD_COMMIT: "master"
-      NIGHTLY_BUILD: "false"
       TEST_DEPENDS: "pytest hypothesis cffi pytz"
       JUNITXML: "test-data.xml"
       TEST_DIR: "tmp_for_test"
@@ -33,16 +32,11 @@ jobs:
       - bash: |
           set -e
 
-          SKIP_BUILD="false"
           if [ "$BUILD_REASON" == "Schedule" ]; then
             BUILD_COMMIT=$NIGHTLY_BUILD_COMMIT
-            if [ "$NIGHTLY_BUILD" != "true" ]; then
-              SKIP_BUILD="true"
-            fi
           fi
           echo "Building numpy@$BUILD_COMMIT"
           echo "##vso[task.setvariable variable=BUILD_COMMIT]$BUILD_COMMIT"
-          echo "##vso[task.setvariable variable=SKIP_BUILD]$SKIP_BUILD"
 
           # Platform variables used in multibuild scripts
           if [ `uname` == 'Darwin' ]; then
@@ -60,6 +54,7 @@ jobs:
 
       - bash: |
           set -e
+          echo $BUILD_COMMIT
           pip install virtualenv wheel
           BUILD_DEPENDS="$CYTHON_BUILD_DEP"
 
@@ -74,55 +69,50 @@ jobs:
           ./patch_code.sh $REPO_DIR
           build_wheel $REPO_DIR $PLAT
         displayName: Build wheel
-        condition: eq(variables['SKIP_BUILD'], 'false')
 
       - bash: |
-          set -xe
+          set -e
           source multibuild/common_utils.sh
           source multibuild/travis_steps.sh
           source extra_functions.sh
           setup_test_venv
           install_run $PLAT
           teardown_test_venv
         displayName: Install wheel and test
-        condition: eq(variables['SKIP_BUILD'], 'false')
 
       - bash: |
           echo "##vso[task.prependpath]$CONDA/bin"
           sudo chown -R $USER $CONDA
         displayName: Add conda to PATH
-        condition: and(succeeded(), eq(variables['SKIP_BUILD'], 'false'), ne(variables['Build.Reason'], 'PullRequest'))
 
       - bash: conda install -q -y anaconda-client
         displayName: Install anaconda-client
-        condition: and(succeeded(), eq(variables['SKIP_BUILD'], 'false'), ne(variables['Build.Reason'], 'PullRequest'))
 
       - bash: |
           set -e
           if [ "$BUILD_REASON" == "Schedule" ]; then
             ANACONDA_ORG="scipy-wheels-nightly"
-            TOKEN="$NUMPY_NIGHTLY_UPLOAD_TOKEN"
+            TOKEN="$MAPPED_NUMPY_NIGHTLY_UPLOAD_TOKEN"
           else
             ANACONDA_ORG="multibuild-wheels-staging"
-            TOKEN="$NUMPY_STAGING_UPLOAD_TOKEN"
+            TOKEN="$MAPPED_NUMPY_STAGING_UPLOAD_TOKEN"
           fi
           if [ "$TOKEN" == "" ]; then
             echo "##[warning] Could not find anaconda.org upload token in secret variables"
           fi
           echo "##vso[task.setvariable variable=TOKEN]$TOKEN"
           echo "##vso[task.setvariable variable=ANACONDA_ORG]$ANACONDA_ORG"
         displayName: Retrieve secret upload token
-        condition: and(succeeded(), eq(variables['SKIP_BUILD'], 'false'), ne(variables['Build.Reason'], 'PullRequest'))
         env:
           # Secret variables need to mapped to env variables explicitly:
-          NUMPY_NIGHTLY_UPLOAD_TOKEN: $(NUMPY_NIGHTLY_UPLOAD_TOKEN)
-          NUMPY_STAGING_UPLOAD_TOKEN: $(NUMPY_STAGING_UPLOAD_TOKEN)
+          MAPPED_NUMPY_NIGHTLY_UPLOAD_TOKEN: $(NUMPY_NIGHTLY_UPLOAD_TOKEN)
+          MAPPED_NUMPY_STAGING_UPLOAD_TOKEN: $(NUMPY_STAGING_UPLOAD_TOKEN)
 
       - bash: |
           set -e
           # The --force option forces a replacement if the remote file already
           # exists.
-          ls wheelhouse/*.whl
+          echo uploading wheelhouse/*.whl
           anaconda -t $TOKEN upload --force -u $ANACONDA_ORG wheelhouse/*.whl
           echo "PyPI-style index: https://pypi.anaconda.org/$ANACONDA_ORG/simple"
         displayName: Upload to anaconda.org (only if secret token is retrieved)

azure/windows.yml

@@ -8,10 +8,8 @@ jobs:
     pool:
       vmImage: ${{ parameters.vmImage }}
     variables:
-      BUILD_COMMIT: "v1.18.2"
+      BUILD_COMMIT: "v1.18.3"
       NIGHTLY_BUILD_COMMIT: "master"
-      NIGHTLY_BUILD: "false"
-      PYTHON_ARCH: "x64"
       JUNITXML: "test-data.xml"
       TEST_DIR: '$(Agent.WorkFolder)/tmp_for_test'
     strategy:
@@ -24,6 +22,7 @@ jobs:
       - task: UsePythonVersion@0
         inputs:
           versionSpec: $(PYTHON_VERSION)
+          addToPath: true
           architecture: $(PYTHON_ARCH)
         displayName: Set python version
 
@@ -37,16 +36,11 @@ jobs:
         displayName: Check that we have the expected version and architecture for Python
       - bash: |
           set -e
-          SKIP_BUILD="false"
           if [ "$BUILD_REASON" == "Schedule" ]; then
             BUILD_COMMIT=$NIGHTLY_BUILD_COMMIT
-            if [ "$NIGHTLY_BUILD" != "true" ]; then
-              SKIP_BUILD="true"
-            fi
           fi
           echo "Building numpy@$BUILD_COMMIT"
           echo "##vso[task.setvariable variable=BUILD_COMMIT]$BUILD_COMMIT"
-          echo "##vso[task.setvariable variable=SKIP_BUILD]$SKIP_BUILD"
           # Store original Python path to be able to create test_venv pointing
           # to same Python version.
           PYTHON_EXE=`which python`
@@ -55,13 +49,18 @@ jobs:
       - bash: |
           set -e
           cd numpy
+          git fetch origin HEAD
           git checkout $BUILD_COMMIT
           git clean -fxd
           git reset --hard
         displayName: Checkout numpy commit
-        condition: eq(variables['SKIP_BUILD'], 'false')
+
+      - powershell: |
+          choco install -y mingw --forcex86 --force --version=5.3.0
+        displayName: 'Install 32-bit mingw for 32-bit builds'
+        condition: eq(variables['BITS'], 32)
       - bash: |
-          set -e
+          set -xe
           pushd numpy
           pip install twine wheel urllib3
           # a bit overkill, all we really need is cython
@@ -70,65 +69,78 @@ jobs:
           # handle license
           cp ../LICENSE_win32.txt LICENSE.txt
 
-          # Download and install static "openblas.a" to PYTHON/lib
-          PYTHON_DIR=$(python -c"import sys, os; print(os.path.dirname(sys.executable))")
-          LIB=$(python tools/openblas_support.py)
-          echo Copying $LIB to $PYTHON_DIR/lib
-          cp $LIB $PYTHON_DIR/lib/openblas.a
+          # handle _distributor_init.py
+          PYTHONPATH=tools python -c "import openblas_support; openblas_support.make_init('numpy')"
+
+          # Download and get the path to "openblas.a". We cannot copy it
+          # to $PYTHON_EXE's directory since that is on a different drive which
+          # mingw does not like. Instead copy it to a directory and set OPENBLAS
+          target=$(python tools/openblas_support.py)
+          mkdir -p openblas
+          echo Copying $target to openblas
+          cp $target openblas
+          echo "##vso[task.setvariable variable=OPENBLAS]openblas"
+        displayName: Prepare the build
 
+      - powershell: |
+          If ($(BITS) -eq 32) {
+              $env:CFLAGS = "-m32"
+              $env:LDFLAGS = "-m32"
+              $env:PATH = "C:\\tools\\mingw32\\bin;" + $env:PATH
+              refreshenv
+          }
           # Build the wheel
+          pushd numpy
           python setup.py build
           python setup.py bdist_wheel
-          ls dist
           twine check dist/*
           popd
         displayName: Build wheel
-        condition: eq(variables['SKIP_BUILD'], 'false')
       - bash: |
           set -ex
           source extra_functions.sh
           source config.sh
           setup_test_venv
-          pip install -r numpy/test_requirements.txt
-          pip install numpy/dist/numpy-*.whl
+          python -m pip install -r numpy/test_requirements.txt
+          python -m pip install numpy/dist/numpy-*.whl
+          mkdir -p for_test
+          pushd for_test
           run_tests
+          popd
           teardown_test_venv
         displayName: Install wheel and test
-        condition: eq(variables['SKIP_BUILD'], 'false')
 
       - bash: echo "##vso[task.prependpath]$CONDA/Scripts"
         displayName: Add conda to PATH
-        condition: and(succeeded(), eq(variables['SKIP_BUILD'], 'false'), ne(variables['Build.Reason'], 'PullRequest'))
 
       - bash: conda install -q -y anaconda-client
         displayName: Install anaconda-client
-        condition: and(succeeded(), eq(variables['SKIP_BUILD'], 'false'), ne(variables['Build.Reason'], 'PullRequest'))
 
       - bash: |
           set -e
           if [ "$BUILD_REASON" == "Schedule" ]; then
             ANACONDA_ORG="scipy-wheels-nightly"
-            TOKEN="$NUMPY_NIGHTLY_UPLOAD_TOKEN"
+            TOKEN="$MAPPED_NUMPY_NIGHTLY_UPLOAD_TOKEN"
           else
             ANACONDA_ORG="multibuild-wheels-staging"
-            TOKEN="$NUMPY_STAGING_UPLOAD_TOKEN"
+            TOKEN="$MAPPED_NUMPY_STAGING_UPLOAD_TOKEN"
           fi
           if [ "$TOKEN" == "" ]; then
             echo "##[warning] Could not find anaconda.org upload token in secret variables"
           fi
           echo "##vso[task.setvariable variable=TOKEN]$TOKEN"
           echo "##vso[task.setvariable variable=ANACONDA_ORG]$ANACONDA_ORG"
         displayName: Retrieve secret upload token
-        condition: and(succeeded(), eq(variables['SKIP_BUILD'], 'false'), ne(variables['Build.Reason'], 'PullRequest'))
         env:
           # Secret variables need to mapped to env variables explicitly:
-          NUMPY_NIGHTLY_UPLOAD_TOKEN: $(NUMPY_NIGHTLY_UPLOAD_TOKEN)
-          NUMPY_STAGING_UPLOAD_TOKEN: $(NUMPY_STAGING_UPLOAD_TOKEN)
+          MAPPED_NUMPY_NIGHTLY_UPLOAD_TOKEN: $(NUMPY_NIGHTLY_UPLOAD_TOKEN)
+          MAPPED_NUMPY_STAGING_UPLOAD_TOKEN: $(NUMPY_STAGING_UPLOAD_TOKEN)
+
       - bash: |
           set -e
           # The --force option forces a replacement if the remote file already
           # exists.
-          ls numpy/dist/numpy-*.whl
+          echo uploading numpy/dist/numpy-*.whl
           anaconda -t $TOKEN upload --force -u $ANACONDA_ORG numpy/dist/numpy-*.whl
           echo "PyPI-style index: https://pypi.anaconda.org/$ANACONDA_ORG/simple"
         displayName: Upload to anaconda.org (only if secret token is retrieved)

config.sh

@@ -41,10 +41,10 @@ function run_tests {
     if [ -n "$IS_LINUX" ]; then
         apt-get -y update && apt-get install -y gfortran
     fi
-    python -c "$(get_test_cmd)"
+    $PYTHON_EXE -c "$(get_test_cmd)"
     # Check bundled license file
-    python ../check_license.py
+    $PYTHON_EXE ../check_license.py
     # Show BLAS / LAPACK used. Since this uses a wheel we cannot use
     # tools/openblas_config.py; tools is not part of what is shipped
-    python -c 'import numpy; numpy.show_config()'
+    $PYTHON_EXE -c 'import numpy; numpy.show_config()'
 }

extra_functions.sh

@@ -7,11 +7,14 @@ function setup_test_venv {
         if [ $(uname) == "Darwin" ]; then
             source test_venv/bin/activate
         else
+            mkdir -p test_venv/libs
             source test_venv/Scripts/activate
         fi
         # Note: the idiom "python -m pip install ..." is necessary to upgrade
         # pip itself on Windows. Otherwise one would get a permission error on
         # pip.exe.
+        PYTHON_EXE=python
+        PIP_CMD="$PYTHON_EXE -m pip"
         python -m pip install --upgrade pip wheel
         if [ "$TEST_DEPENDS" != "" ]; then
             pip install $TEST_DEPENDS
@@ -28,4 +31,4 @@ function teardown_test_venv {
     fi
 } 
 # Work around bug in multibuild
-if [ ! -o PIP_CMD ]; then PIP_CMD="$PYTHON_EXE -mpip"; fi
+if [ ! -o PIP_CMD ]; then PIP_CMD="$PYTHON_EXE -m pip"; fi