Gs/fix submodules checkout

.github/workflows/build_wheels.yml → .github/workflows/build_wheels_linux.yml

@@ -1,83 +1,17 @@
-name: Build PYPI wheels for opencv-python
+name: Build PYPI wheels for opencv-python on Linux x86_64
 
 on:
+  push:
+    branches:
+      - gs/fix-submodules-checkout
   pull_request:
     branches:
-      - master
+      - 3.4
   release:
     types: [published, edited]
 
 
 jobs:
-  build-windows-x86_64:
-    runs-on: ${{ matrix.os }}
-    defaults:
-      run:
-        shell: powershell
-
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [windows-latest]
-        python-version: [3.6, 3.7, 3.8, 3.9]
-        platform: [x86, x64]
-        with_contrib: [0, 1]
-        without_gui: [0, 1]
-        build_sdist: [0]
-
-    env:
-      ACTIONS_ALLOW_UNSECURE_COMMANDS: true
-      SDIST: ${{ matrix.build_sdist || 0}}
-      ENABLE_HEADLESS: ${{ matrix.without_gui }}
-      ENABLE_CONTRIB: ${{ matrix.with_contrib }}
-
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v2
-      with:
-        submodules: true
-        fetch-depth: 0
-
-    - name: Update submodules
-      run: |
-        git submodule update --remote
-
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-        architecture: ${{ matrix.platform }}
-
-    - name: Setup MSBuild.exe
-      uses: warrenbuckley/Setup-MSBuild@v1
-
-    - name: build script
-      run: |
-        python --version
-        python -m pip install --upgrade pip
-        python -m pip install --upgrade setuptools
-        set "CI_BUILD=1" && python -m pip wheel --wheel-dir=%cd%\wheelhouse . --verbose
-      shell: cmd
-
-    - name: before test
-      run: |
-        cd ${{ github.workspace }}/tests
-        &python -m pip install --user --no-warn-script-location (ls "../wheelhouse/opencv*.whl")
-        if ($LastExitCode -ne 0) {throw $LastExitCode}
-      shell: powershell
-
-    - name: run test
-      run: |
-        cd ${{ github.workspace }}/tests
-        python -m unittest test
-      shell: cmd
-
-    - name: saving artifacts
-      uses: actions/upload-artifact@v2
-      with:
-        name: wheels
-        path: wheelhouse/opencv*.whl
-
   build:
     runs-on: ${{ matrix.os }}
     defaults:
@@ -87,7 +21,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest]
+        os: [ubuntu-latest]
         python-version: [3.6, 3.7, 3.8, 3.9]
         platform: [x64]
         with_contrib: [0, 1]
@@ -116,12 +50,15 @@ jobs:
     - name: Checkout
       uses: actions/checkout@v2
       with:
-        submodules: true
+        submodules: false
         fetch-depth: 0
 
+    - name: Update multibuild submodule
+      run: git submodule update --init --recursive multibuild
+
     - name: Update submodules
-      run: |
-        git submodule update --remote
+      if: github.event_name == 'pull_request'
+      run: git submodule update --remote
 
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v2
@@ -140,7 +77,7 @@ jobs:
         echo "BUILD_DEPENDS=$(echo $NP_BUILD_DEP)" >> $GITHUB_ENV;
         echo "TEST_DEPENDS=$(echo $NP_TEST_DEP)" >> $GITHUB_ENV;
 
-    - name: before install
+    - name: build
       run: |
         set -e
         # Check out and prepare the source
@@ -157,8 +94,32 @@ jobs:
         echo $ENABLE_HEADLESS > headless.enabled
         set -x
         build_wheel $REPO_DIR $PLAT
+
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        submodules: true
+        fetch-depth: 0
+
+    - name: install and test
+      run: |
+        set -e
+        # Check out and prepare the source
+        # Multibuild doesn't have releases, so --depth would break eventually (see
+        # https://superuser.com/questions/1240216/server-does-not-allow-request-for-unadvertised)
+        git submodule update --init multibuild
+        source multibuild/common_utils.sh
+        # https://github.com/matthew-brett/multibuild/issues/116
+        if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export ARCH_FLAGS=" "; fi
+        source multibuild/travis_steps.sh
+        # This sets -x
+        # source travis_multibuild_customize.sh
+        echo $ENABLE_CONTRIB > contrib.enabled
+        echo $ENABLE_HEADLESS > headless.enabled
+        set -x
         install_run $PLAT
         set +x
+
     - name: saving artifacts
       uses: actions/upload-artifact@v2
       with:
@@ -205,10 +166,14 @@ jobs:
     - name: Checkout
       uses: actions/checkout@v2
       with:
-        submodules: true
+        submodules: false
         fetch-depth: 0
 
+    - name: Update multibuild submodule
+      run: git submodule update --init --recursive multibuild
+
     - name: Update submodules
+      if: github.event_name == 'pull_request'
       run: |
         git submodule update --remote
 
@@ -238,6 +203,7 @@ jobs:
         # Install and run tests
         set -x
         echo "skipping tests because of sdist"
+
     - name: saving artifacts
       uses: actions/upload-artifact@v2
       with:
@@ -246,7 +212,7 @@ jobs:
 
   test_release_opencv_python:
     if: github.event_name == 'release' && github.event.release.prerelease
-    needs: [build, build-windows-x86_64, build_sdist]
+    needs: [build, build_sdist]
     runs-on: ubuntu-latest
     environment: test-opencv-python-release
     defaults:
@@ -265,7 +231,7 @@ jobs:
 
   release_opencv_python:
     if: github.event_name == 'release' && !github.event.release.prerelease
-    needs: [build, build-windows-x86_64, build_sdist]
+    needs: [build, build_sdist]
     runs-on: ubuntu-latest
     environment: opencv-python-release
     defaults:

.github/workflows/build_wheels_linux_arm.yml

@@ -0,0 +1,177 @@
+name: Build PYPI wheels for opencv-python on Linux ARM
+
+on:
+  push:
+    branches:
+      - asen/check_3.4_wo_submodules
+  pull_request:
+    branches:
+      - 3.4
+  release:
+    types: [published, edited]
+
+
+jobs:
+  build_arm:
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        shell: bash
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ARM64]
+        python-version: [3.6, 3.7, 3.8, 3.9]
+        platform: [x64]
+        with_contrib: [0, 1]
+        without_gui: [0, 1]
+        build_sdist: [0]
+
+    env:
+      ACTIONS_ALLOW_UNSECURE_COMMANDS: true
+      REPO_DIR: .
+      BUILD_COMMIT: master
+      PROJECT_SPEC: opencv-python
+      MB_PYTHON_VERSION: ${{ matrix.python-version }}
+      TRAVIS_PYTHON_VERSION: ${{ matrix.python-version }}
+      PLAT: aarch64
+      MB_ML_VER: 2014
+      NP_TEST_DEP: numpy
+      TRAVIS_BUILD_DIR: ${{ github.workspace }}
+      CONFIG_PATH: travis_config.sh
+      DOCKER_IMAGE: quay.io/skvark/manylinux2014_${PLAT}
+      DOCKER_TEST_IMAGE: multibuild/xenial_arm64v8
+      USE_CCACHE: 0
+      UNICODE_WIDTH: 32
+      SDIST: ${{ matrix.build_sdist || 0}}
+      ENABLE_HEADLESS: ${{ matrix.without_gui }}
+      ENABLE_CONTRIB: ${{ matrix.with_contrib }}
+
+    steps:
+    - name: Cleanup
+      if: always()
+      uses: AutoModality/action-clean@v1
+
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        submodules: false
+        fetch-depth: 0
+
+    - name: Update multibuild submodule
+      run: git submodule update --init --recursive multibuild
+
+    - name: Update submodules
+      if: github.event_name == 'pull_request'
+      run: git submodule update --remote
+
+    - name: Setup Environment variables
+      run: |
+        if [ "ARM64" == "${{ matrix.os }}" ]; then echo "PLAT=aarch64" >> $GITHUB_ENV; fi
+        echo "BUILD_DEPENDS=$(echo $NP_BUILD_DEP)" >> $GITHUB_ENV;
+        echo "TEST_DEPENDS=$(echo $NP_TEST_DEP)" >> $GITHUB_ENV;
+
+    - name: before install
+      run: |
+        set -e
+        if [[ $SDIST == 0 ]]; then
+          # Check out and prepare the source
+          # Multibuild doesn't have releases, so --depth would break eventually (see
+          # https://superuser.com/questions/1240216/server-does-not-allow-request-for-unadvertised)
+          git submodule update --init multibuild
+          source multibuild/common_utils.sh
+          # https://github.com/matthew-brett/multibuild/issues/116
+          if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export ARCH_FLAGS=" "; fi
+          source multibuild/travis_steps.sh
+          # This sets -x
+          # source travis_multibuild_customize.sh
+          echo $ENABLE_CONTRIB > contrib.enabled
+          echo $ENABLE_HEADLESS > headless.enabled
+          echo "end"
+          # Not interested in travis internal scripts' output
+        fi
+        set +x
+        # Build and package
+        set -x
+        ls
+        if [[ $SDIST == 1 ]]; then
+          python -m pip install --upgrade pip
+          python -m pip install scikit-build
+          python setup.py sdist
+        else
+          build_wheel $REPO_DIR $PLAT
+        fi
+        set +x
+        # Install and run tests
+        set -x
+        if [[ $SDIST == 1 ]]; then
+          echo "skipping tests because of sdist"
+          rc=0
+        else
+          install_run $PLAT && rc=$? || rc=$?
+        fi
+        set +x
+        #otherwise, Travis logic terminates prematurely
+        #https://travis-ci.community/t/shell-session-update-command-not-found-in-build-log-causes-build-to-fail-if-trap-err-is-set/817
+        trap ERR
+        test "$rc" -eq 0
+
+    - name: saving artifacts
+      uses: actions/upload-artifact@v2
+      with:
+        name: wheels
+        path: wheelhouse/opencv*.whl
+
+  test_release_opencv_python:
+    if: github.event_name == 'release' && github.event.release.prerelease
+    needs: [build_arm]
+    runs-on: ubuntu-latest
+    environment: test-opencv-python-release
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/download-artifact@v2
+        with:
+          name: wheels
+          path: wheelhouse/
+
+      - name: Upload all wheels
+        run: |
+          python -m pip install twine
+          python -m twine upload --repository testpypi -u ${{ secrets.PYPI_USERNAME }} -p ${{ secrets.PYPI_PASSWORD }} --skip-existing wheelhouse/opencv_*
+
+  release_opencv_python:
+    if: github.event_name == 'release' && !github.event.release.prerelease
+    needs: [build_arm]
+    runs-on: ubuntu-latest
+    environment: opencv-python-release
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/download-artifact@v2
+        with:
+          name: wheels
+          path: wheelhouse/
+
+      - name: Upload wheels for opencv_python
+        run: |
+          python -m pip install twine
+          python -m twine upload -u ${{ secrets.OPENCV_PYTHON_USERNAME }} -p ${{ secrets.OPENCV_PYTHON_PASSWORD }} --skip-existing wheelhouse/opencv_python-*
+
+      - name: Upload wheels for opencv_contrib_python
+        run: |
+          python -m pip install twine
+          python -m twine upload -u ${{ secrets.OPENCV_CONTRIB_PYTHON_USERNAME }} -p ${{ secrets.OPENCV_CONTRIB_PYTHON_PASSWORD }} --skip-existing wheelhouse/opencv_contrib_python-*
+
+      - name: Upload wheels for opencv_python_headless
+        run: |
+          python -m pip install twine
+          python -m twine upload -u ${{ secrets.OPENCV_PYTHON_HEADLESS_USERNAME }} -p ${{ secrets.OPENCV_PYTHON_HEADLESS_PASSWORD }} --skip-existing wheelhouse/opencv_python_headless-*
+
+      - name: Upload wheels for opencv_contrib_python_headless
+        run: |
+          python -m pip install twine
+          python -m twine upload -u ${{ secrets.OPENCV_CONTRIB_PYTHON_HEADLESS_USERNAME }} -p ${{ secrets.OPENCV_CONTRIB_PYTHON_HEADLESS_PASSWORD }} --skip-existing wheelhouse/opencv_contrib_python_headless-*