remove NIGHTLY_BUILD, clean up scripts

Author: mattip

azure-pipelines.yml

@@ -1,7 +1,7 @@
 schedules:
 - cron: "27 3 */1 * *"
   # 3:27am UTC everyday
-  displayName: Nighthly build
+  displayName: Nightly build
   branches:
     include:
     - master
@@ -30,21 +30,17 @@ jobs:
         py_3.7_32:
           PYTHON_VERSION: "3.7.x"
           PYTHON_ARCH: "x86"
-          NIGHTLY_BUILD: "true"
           BITS: 32
         py_3.7_64:
           PYTHON_VERSION: "3.7.x"
-          NIGHTLY_BUILD: "true"
           PYTHON_ARCH: 'x64'
           BITS: 64
         py_3.8_32:
           PYTHON_VERSION: "3.8.x"
           PYTHON_ARCH: "x86"
-          NIGHTLY_BUILD: "true"
           BITS: 32
         py_3.8_64:
           PYTHON_VERSION: "3.8.x"
-          NIGHTLY_BUILD: "true"
           PYTHON_ARCH: 'x64'
           BITS: 64
 
@@ -65,24 +61,20 @@ jobs:
         py_3.7_32:
           MB_PYTHON_VERSION: "3.7"
           PLAT: "i686"
-          NIGHTLY_BUILD: "true"
           MB_ML_VER: "2010"
           ENV_VARS_PATH: "env_vars_32.sh"
           DOCKER_TEST_IMAGE: "multibuild/xenial_{PLAT}"
         py_3.7_64:
           MB_PYTHON_VERSION: "3.7"
-          NIGHTLY_BUILD: "true"
           MB_ML_VER: "2010"
         py_3.8_32:
           MB_PYTHON_VERSION: "3.8"
           PLAT: "i686"
-          NIGHTLY_BUILD: "true"
           MB_ML_VER: "2010"
           ENV_VARS_PATH: "env_vars_32.sh"
           DOCKER_TEST_IMAGE: "multibuild/xenial_{PLAT}"
         py_3.8_64:
           MB_PYTHON_VERSION: "3.8"
-          NIGHTLY_BUILD: "true"
           MB_ML_VER: "2010"
 
   - template: azure/posix.yml
@@ -92,12 +84,7 @@ jobs:
       matrix:
         py_3.6_64:
           MB_PYTHON_VERSION: "3.6"
-          MB_PYTHON_OSX_VER: 10.9
         py_3.7_64:
           MB_PYTHON_VERSION: "3.7"
-          NIGHTLY_BUILD: "true"
-          MB_PYTHON_OSX_VER: 10.9
         py_3.8_64:
           MB_PYTHON_VERSION: "3.8"
-          NIGHTLY_BUILD: "true"
-          MB_PYTHON_OSX_VER: 10.9

azure/posix.yml

@@ -9,11 +9,10 @@ jobs:
       vmImage: ${{ parameters.vmImage }}
     variables:
       REPO_DIR: "numpy"
-      BUILD_COMMIT: "master"
+      BUILD_COMMIT: 'master'
       PLAT: "x86_64"
       CYTHON_BUILD_DEP: "cython==0.29.16"
       NIGHTLY_BUILD_COMMIT: "master"
-      NIGHTLY_BUILD: "false"
       TEST_DEPENDS: "pytest hypothesis cffi pytz"
       JUNITXML: "test-data.xml"
       TEST_DIR: "tmp_for_test"
@@ -33,16 +32,11 @@ jobs:
       - bash: |
           set -e
 
-          SKIP_BUILD="false"
           if [ "$BUILD_REASON" == "Schedule" ]; then
             BUILD_COMMIT=$NIGHTLY_BUILD_COMMIT
-            if [ "$NIGHTLY_BUILD" != "true" ]; then
-              SKIP_BUILD="true"
-            fi
           fi
           echo "Building numpy@$BUILD_COMMIT"
           echo "##vso[task.setvariable variable=BUILD_COMMIT]$BUILD_COMMIT"
-          echo "##vso[task.setvariable variable=SKIP_BUILD]$SKIP_BUILD"
 
           # Platform variables used in multibuild scripts
           if [ `uname` == 'Darwin' ]; then
@@ -60,6 +54,7 @@ jobs:
 
       - bash: |
           set -e
+          echo $BUILD_COMMIT
           pip install virtualenv wheel
           BUILD_DEPENDS="$CYTHON_BUILD_DEP"
 
@@ -74,28 +69,24 @@ jobs:
           ./patch_code.sh $REPO_DIR
           build_wheel $REPO_DIR $PLAT
         displayName: Build wheel
-        condition: eq(variables['SKIP_BUILD'], 'false')
 
       - bash: |
-          set -xe
+          set -e
           source multibuild/common_utils.sh
           source multibuild/travis_steps.sh
           source extra_functions.sh
           setup_test_venv
           install_run $PLAT
           teardown_test_venv
         displayName: Install wheel and test
-        condition: eq(variables['SKIP_BUILD'], 'false')
 
       - bash: |
+          set -e
           echo "##vso[task.prependpath]$CONDA/bin"
           sudo chown -R $USER $CONDA
-        displayName: Add conda to PATH
-        condition: and(succeeded(), eq(variables['SKIP_BUILD'], 'false'), ne(variables['Build.Reason'], 'PullRequest'))
-
-      - bash: conda install -q -y anaconda-client
+          conda install -q -y anaconda-client
+          echo $PATH
         displayName: Install anaconda-client
-        condition: and(succeeded(), eq(variables['SKIP_BUILD'], 'false'), ne(variables['Build.Reason'], 'PullRequest'))
 
       - bash: |
           set -e
@@ -112,7 +103,6 @@ jobs:
           echo "##vso[task.setvariable variable=TOKEN]$TOKEN"
           echo "##vso[task.setvariable variable=ANACONDA_ORG]$ANACONDA_ORG"
         displayName: Retrieve secret upload token
-        condition: and(succeeded(), eq(variables['SKIP_BUILD'], 'false'), ne(variables['Build.Reason'], 'PullRequest'))
         env:
           # Secret variables need to mapped to env variables explicitly:
           NUMPY_NIGHTLY_UPLOAD_TOKEN: $(NUMPY_NIGHTLY_UPLOAD_TOKEN)
@@ -123,6 +113,7 @@ jobs:
           # The --force option forces a replacement if the remote file already
           # exists.
           ls wheelhouse/*.whl
+          echo $PATH
           anaconda -t $TOKEN upload --force -u $ANACONDA_ORG wheelhouse/*.whl
           echo "PyPI-style index: https://pypi.anaconda.org/$ANACONDA_ORG/simple"
         displayName: Upload to anaconda.org (only if secret token is retrieved)

azure/windows.yml

@@ -8,7 +8,7 @@ jobs:
     pool:
       vmImage: ${{ parameters.vmImage }}
     variables:
-      BUILD_COMMIT: "v1.18.2"
+      BUILD_COMMIT: "master"
       NIGHTLY_BUILD_COMMIT: "master"
       JUNITXML: "test-data.xml"
       TEST_DIR: '$(Agent.WorkFolder)/tmp_for_test'
@@ -36,16 +36,11 @@ jobs:
         displayName: Check that we have the expected version and architecture for Python
       - bash: |
           set -e
-          SKIP_BUILD="false"
           if [ "$BUILD_REASON" == "Schedule" ]; then
             BUILD_COMMIT=$NIGHTLY_BUILD_COMMIT
-            if [ "$NIGHTLY_BUILD" != "true" ]; then
-              SKIP_BUILD="true"
-            fi
           fi
           echo "Building numpy@$BUILD_COMMIT"
           echo "##vso[task.setvariable variable=BUILD_COMMIT]$BUILD_COMMIT"
-          echo "##vso[task.setvariable variable=SKIP_BUILD]$SKIP_BUILD"
           # Store original Python path to be able to create test_venv pointing
           # to same Python version.
           PYTHON_EXE=`which python`
@@ -54,68 +49,72 @@ jobs:
       - bash: |
           set -e
           cd numpy
+          git fetch origin HEAD
           git checkout $BUILD_COMMIT
           git clean -fxd
           git reset --hard
         displayName: Checkout numpy commit
-        condition: eq(variables['SKIP_BUILD'], 'false')
 
       - powershell: |
           choco install -y mingw --forcex86 --force --version=5.3.0
-          $env:CFLAGS = "-m32"
-          $env:LDFLAGS = "-m32"
-          $env:PATH = "C:\\tools\\mingw32\\bin;" + $env:PATH
-          refreshenv
         displayName: 'Install 32-bit mingw for 32-bit builds'
         condition: eq(variables['BITS'], 32)
-      - powershell: |
+      - bash: |
+          set -xe
+          pushd numpy
           pip install twine wheel urllib3
           # a bit overkill, all we really need is cython
-          pip install --timeout=60 -r numpy/test_requirements.txt
+          pip install --timeout=60 -r test_requirements.txt
 
           # handle license
-          cp LICENSE_win32.txt numpy/LICENSE.txt
+          cp ../LICENSE_win32.txt LICENSE.txt
 
-          # Download and copy static "openblas.a", put it into LIB
-          $libdir = Join-Path $PSScriptRoot "OPENBLAS_LIB"
-          mkdir $libdir
-          $openblas_lib=$( python numpy/tools/openblas_support.py )
-          echo Copying $openblas_lib to $libdir
-          cp $openblas_lib $libdir/openblas.a
-          $env:LIB = $libdir + ";" + $env:LIB
+          # handle _distributor_init.py
+          PYTHONPATH=tools python -c "import openblas_support; openblas_support.make_init('numpy')"
+
+          # Download and get the path to "openblas.a". We cannot copy it
+          # to $PYTHON_EXE's directory since that is on a different drive which
+          # mingw does not like. Instead copy it to a directory and set OPENBLAS
+          target=$(python tools/openblas_support.py)
+          mkdir -p openblas
+          echo Copying $target to openblas
+          cp $target openblas
+          echo "##vso[task.setvariable variable=OPENBLAS]openblas"
         displayName: Prepare the build
-        condition: eq(variables['SKIP_BUILD'], 'false')
 
-      - bash: |
+      - powershell: |
+          If ($(BITS) -eq 32) {
+              $env:CFLAGS = "-m32"
+              $env:LDFLAGS = "-m32"
+              $env:PATH = "C:\\tools\\mingw32\\bin;" + $env:PATH
+              refreshenv
+          }
           # Build the wheel
-          set -e
           pushd numpy
           python setup.py build
           python setup.py bdist_wheel
-          ls dist
           twine check dist/*
           popd
         displayName: Build wheel
-        condition: eq(variables['SKIP_BUILD'], 'false')
       - bash: |
           set -ex
           source extra_functions.sh
           source config.sh
           setup_test_venv
-          pip install -r numpy/test_requirements.txt
-          pip install numpy/dist/numpy-*.whl
+          python -m pip install -r numpy/test_requirements.txt
+          python -m pip install numpy/dist/numpy-*.whl
+          mkdir -p for_test
+          pushd for_test
           run_tests
+          popd
           teardown_test_venv
         displayName: Install wheel and test
-        condition: eq(variables['SKIP_BUILD'], 'false')
-
-      - bash: echo "##vso[task.prependpath]$CONDA/Scripts"
-        displayName: Add conda to PATH
-        condition: and(succeeded(), eq(variables['SKIP_BUILD'], 'false'), ne(variables['Build.Reason'], 'PullRequest'))
 
-      - bash: conda install -q -y anaconda-client
+      - bash: |
+          set -ex
+          echo "##vso[task.prependpath]$CONDA/Scripts"
+          conda install -q -y anaconda-client
         displayName: Install anaconda-client
-        condition: and(succeeded(), eq(variables['SKIP_BUILD'], 'false'), ne(variables['Build.Reason'], 'PullRequest'))
 
       - bash: |
           set -e
@@ -132,7 +131,6 @@ jobs:
           echo "##vso[task.setvariable variable=TOKEN]$TOKEN"
           echo "##vso[task.setvariable variable=ANACONDA_ORG]$ANACONDA_ORG"
         displayName: Retrieve secret upload token
-        condition: and(succeeded(), eq(variables['SKIP_BUILD'], 'false'), ne(variables['Build.Reason'], 'PullRequest'))
         env:
           # Secret variables need to mapped to env variables explicitly:
           NUMPY_NIGHTLY_UPLOAD_TOKEN: $(NUMPY_NIGHTLY_UPLOAD_TOKEN)

extra_functions.sh

@@ -7,11 +7,14 @@ function setup_test_venv {
         if [ $(uname) == "Darwin" ]; then
             source test_venv/bin/activate
         else
+            mkdir -p test_venv/libs
             source test_venv/Scripts/activate
         fi
         # Note: the idiom "python -m pip install ..." is necessary to upgrade
         # pip itself on Windows. Otherwise one would get a permission error on
         # pip.exe.
+        PYTHON_EXE=python
+        PIP_CMD="$PYTHON_EXE -m pip"
         python -m pip install --upgrade pip wheel
         if [ "$TEST_DEPENDS" != "" ]; then
             pip install $TEST_DEPENDS
@@ -28,4 +31,4 @@ function teardown_test_venv {
     fi
 } 
 # Work around bug in multibuild
-if [ ! -o PIP_CMD ]; then PIP_CMD="$PYTHON_EXE -mpip"; fi
+if [ ! -o PIP_CMD ]; then PIP_CMD="$PYTHON_EXE -m pip"; fi