From 093137bb009dbca4936c4b163ac77874aebc9929 Mon Sep 17 00:00:00 2001 From: Grigory Serebryakov Date: Mon, 5 Apr 2021 10:59:56 +0300 Subject: [PATCH 01/41] Add initial config for github actions --- .github/workflows/build_wheels.yml | 146 +++++++++++++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 .github/workflows/build_wheels.yml diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml new file mode 100644 index 00000000..b47f1976 --- /dev/null +++ b/.github/workflows/build_wheels.yml @@ -0,0 +1,146 @@ +name: Build PYPI wheels for opencv-python + +on: + push: + branches: + - main + pull_request: + branches: + - main + release: + # Only use the types keyword to narrow down the activity types that will trigger your workflow. + types: [published, created, edited] + + +jobs: + build: + container: + image: quay.io/skvark/manylinux2014_x86_64 + runs-on: ${{ matrix.os }} + defaults: + run: + shell: bash + + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] #, macos-latest] + python-version: [3.6] #, 3.7, 3.8, 3.9] + platform: [x64] + + env: + REPO_DIR: opencv + BUILD_COMMIT: master + PROJECT_SPEC: opencv + UNICODE_WIDTH: 32 + PLAT: x86_64 + MB_PYTHON_VERSION: ${{ matrix.python-version }} + TRAVIS_PYTHON_VERSION: ${{ matrix.python-version }} + MB_ML_VER: 2014 + NP_BUILD_DEP: numpy==1.11.1 + NP_TEST_DEP: numpy + TRAVIS_BUILD_DIR: ${{ github.workspace }} + CONFIG_PATH: travis_config.sh + USE_CCACHE: 1 + UNICODE_WIDTH: 32 + SDIST: 0 + + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + submodules: true + fetch-depth: 0 + + - name: Update submodules + run: | + git submodule update --remote + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Setup Environment variables + run: | + if [ "macos-latest" == "${{ matrix.os }}" ]; then echo "TRAVIS_OS_NAME=osx" >> $GITHUB_ENV; else echo "TRAVIS_OS_NAME=${{ matrix.os }}" >> $GITHUB_ENV; fi + if [ "schedule" == "${{ github.event_name }}" ]; then echo "TRAVIS_EVENT_TYPE=cron" >> $GITHUB_ENV; else echo "TRAVIS_EVENT_TYPE=${{ github.event_name }}" >> $GITHUB_ENV; fi + if [ "schedule" == "${{ github.event_name }}" ]; then echo "BUILD_COMMIT=master" >> $GITHUB_ENV; else echo "BUILD_COMMIT=$BUILD_COMMIT" >> $GITHUB_ENV; fi + echo "BUILD_DEPENDS=$(echo $NP_BUILD_DEP)" >> $GITHUB_ENV; + echo "TEST_DEPENDS=$(echo $NP_TEST_DEP)" >> $GITHUB_ENV; + + - name: before install + run: | + set -e + + if [[ $SDIST == 0 ]]; then + # Check out and prepare the source + # Multibuild doesn't have releases, so --depth would break eventually (see + # https://superuser.com/questions/1240216/server-does-not-allow-request-for-unadvertised) + git submodule update --init multibuild + + source multibuild/common_utils.sh + + # https://github.com/matthew-brett/multibuild/issues/116 + if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export ARCH_FLAGS=" "; fi + + source multibuild/travis_steps.sh + # This sets -x + + source travis_multibuild_customize.sh + echo $ENABLE_CONTRIB > contrib.enabled + echo $ENABLE_HEADLESS > headless.enabled + + echo "end" + # Not interested in travis internal scripts' output + fi + + set +x + build_index_wheel $PROJECT_SPEC # download source from pypi + # build_wheel $REPO_DIR $PLAT # versioneer does not work with submodules + install_run $PLAT + - name: build + run: | + # Build and package + set -x + + if [[ $SDIST == 1 ]]; then + python -m pip install --upgrade pip + python -m pip install scikit-build + python setup.py sdist + else + build_wheel $REPO_DIR $PLAT + fi + + set +x + + - name: install and test + run: | + # Install and run tests + set -x + if [[ $SDIST == 1 ]]; then + echo "skipping tests because of sdist" + else + install_run $PLAT && rc=$? || rc=$? + fi + + set +x + + #otherwise, Travis logic terminates prematurely + #https://travis-ci.community/t/shell-session-update-command-not-found-in-build-log-causes-build-to-fail-if-trap-err-is-set/817 + trap ERR + test "$rc" -eq 0 + + # - name: Upload wheels + # env: + # # PYPI repository + # TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + # TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + # # PYPITEST repository + # # TWINE_USERNAME: ${{ secrets.PYPITEST_USERNAME }} + # # TWINE_PASSWORD: ${{ secrets.PYPITEST_PASSWORD }} + # # TWINE_REPOSITORY_URL: 'https://test.pypi.org/legacy/' + # run: | + # twine upload --skip-existing ${TRAVIS_BUILD_DIR}/wheelhouse/* + # # Upload wheels to PYPITEST + # #twine upload --skip-existing ${TRAVIS_BUILD_DIR}/wheelhouse/* From eb06701f915622b84a0cf068f4a0eb6b397befee Mon Sep 17 00:00:00 2001 From: Grigory Serebryakov Date: Mon, 5 Apr 2021 11:16:58 +0300 Subject: [PATCH 02/41] Allow actions for all branches and PRs to master by default --- .github/workflows/build_wheels.yml | 4 ++-- opencv | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index b47f1976..bf6f2498 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -3,10 +3,10 @@ name: Build PYPI wheels for opencv-python on: push: branches: - - main + - all pull_request: branches: - - main + - master release: # Only use the types keyword to narrow down the activity types that will trigger your workflow. types: [published, created, edited] diff --git a/opencv b/opencv index 1363496c..b19f8603 160000 --- a/opencv +++ b/opencv @@ -1 +1 @@ -Subproject commit 1363496c1106606684d40447f5d1149b2c66a9f8 +Subproject commit b19f8603843ac63274379c9e36331db6d4917001 From 69297d84ce6caf8ed0e37e0d2a5fdfa11d277c76 Mon Sep 17 00:00:00 2001 From: Grigory Serebryakov Date: Mon, 5 Apr 2021 11:19:00 +0300 Subject: [PATCH 03/41] Fix typo --- .github/workflows/build_wheels.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index bf6f2498..d6d97512 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -32,7 +32,6 @@ jobs: REPO_DIR: opencv BUILD_COMMIT: master PROJECT_SPEC: opencv - UNICODE_WIDTH: 32 PLAT: x86_64 MB_PYTHON_VERSION: ${{ matrix.python-version }} TRAVIS_PYTHON_VERSION: ${{ matrix.python-version }} From 46a4ca9441005ff9f7360eea3cea821cdb7ad17d Mon Sep 17 00:00:00 2001 From: Grigory Serebryakov Date: Mon, 5 Apr 2021 11:26:07 +0300 Subject: [PATCH 04/41] Do not set docker image info for multibuild, as we do it in github actions --- .github/workflows/build_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index d6d97512..213033ae 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -86,7 +86,7 @@ jobs: source multibuild/travis_steps.sh # This sets -x - source travis_multibuild_customize.sh + # source travis_multibuild_customize.sh echo $ENABLE_CONTRIB > contrib.enabled echo $ENABLE_HEADLESS > headless.enabled From db300f2e8a41d766ce16ea8b69ebf07bed074d76 Mon Sep 17 00:00:00 2001 From: Grigory Serebryakov Date: Mon, 5 Apr 2021 11:44:00 +0300 Subject: [PATCH 05/41] Do not run github action in docker as multiduild will do it for us --- .github/workflows/build_wheels.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 213033ae..247496c0 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -14,8 +14,6 @@ on: jobs: build: - container: - image: quay.io/skvark/manylinux2014_x86_64 runs-on: ${{ matrix.os }} defaults: run: From 86b0eceae4f803b8ac1df6b2b80714154a71cd00 Mon Sep 17 00:00:00 2001 From: Grigory Serebryakov Date: Mon, 5 Apr 2021 12:00:54 +0300 Subject: [PATCH 06/41] Remove error steps - somehow I've missed them when cleaned for the build --- .github/workflows/build_wheels.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 247496c0..04e5c9c5 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -93,9 +93,6 @@ jobs: fi set +x - build_index_wheel $PROJECT_SPEC # download source from pypi - # build_wheel $REPO_DIR $PLAT # versioneer does not work with submodules - install_run $PLAT - name: build run: | # Build and package From 89082e53dfde1fbc993c23ff9b953b13ee46c30b Mon Sep 17 00:00:00 2001 From: Grigory Serebryakov Date: Mon, 5 Apr 2021 12:04:54 +0300 Subject: [PATCH 07/41] Merge all build logic into one step as somehow build wheel is not known when sourced and used in different steps --- .github/workflows/build_wheels.yml | 53 ++++++++++++++---------------- 1 file changed, 25 insertions(+), 28 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 04e5c9c5..92bf5f4b 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -93,37 +93,34 @@ jobs: fi set +x - - name: build - run: | - # Build and package - set -x - if [[ $SDIST == 1 ]]; then - python -m pip install --upgrade pip - python -m pip install scikit-build - python setup.py sdist - else - build_wheel $REPO_DIR $PLAT - fi + # Build and package + set -x - set +x + if [[ $SDIST == 1 ]]; then + python -m pip install --upgrade pip + python -m pip install scikit-build + python setup.py sdist + else + build_wheel $REPO_DIR $PLAT + fi - - name: install and test - run: | - # Install and run tests - set -x - if [[ $SDIST == 1 ]]; then - echo "skipping tests because of sdist" - else - install_run $PLAT && rc=$? || rc=$? - fi - - set +x - - #otherwise, Travis logic terminates prematurely - #https://travis-ci.community/t/shell-session-update-command-not-found-in-build-log-causes-build-to-fail-if-trap-err-is-set/817 - trap ERR - test "$rc" -eq 0 + set +x + + # Install and run tests + set -x + if [[ $SDIST == 1 ]]; then + echo "skipping tests because of sdist" + else + install_run $PLAT && rc=$? || rc=$? + fi + + set +x + + #otherwise, Travis logic terminates prematurely + #https://travis-ci.community/t/shell-session-update-command-not-found-in-build-log-causes-build-to-fail-if-trap-err-is-set/817 + trap ERR + test "$rc" -eq 0 # - name: Upload wheels # env: From 4b1a401d7aa459a2deb023946cc91451a2021c3c Mon Sep 17 00:00:00 2001 From: Grigory Serebryakov Date: Mon, 5 Apr 2021 12:20:08 +0300 Subject: [PATCH 08/41] Debug: try to change directory to get inside the repo sources --- .github/workflows/build_wheels.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 92bf5f4b..ceae54ad 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -97,6 +97,8 @@ jobs: # Build and package set -x + ls + cd opencv-python if [[ $SDIST == 1 ]]; then python -m pip install --upgrade pip python -m pip install scikit-build From e55080279d6d291b86cb87a1518ce5bb193872a4 Mon Sep 17 00:00:00 2001 From: Grigory Serebryakov Date: Tue, 6 Apr 2021 16:49:24 +0300 Subject: [PATCH 09/41] Adjust env variables to enable build --- .github/workflows/build_wheels.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index ceae54ad..2d40361f 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -27,9 +27,9 @@ jobs: platform: [x64] env: - REPO_DIR: opencv + REPO_DIR: . BUILD_COMMIT: master - PROJECT_SPEC: opencv + PROJECT_SPEC: opencv-python PLAT: x86_64 MB_PYTHON_VERSION: ${{ matrix.python-version }} TRAVIS_PYTHON_VERSION: ${{ matrix.python-version }} @@ -98,7 +98,6 @@ jobs: set -x ls - cd opencv-python if [[ $SDIST == 1 ]]; then python -m pip install --upgrade pip python -m pip install scikit-build From 4a1452e9708cd72393ffb3b19533804a7ca9f83e Mon Sep 17 00:00:00 2001 From: Grigory Serebryakov Date: Tue, 6 Apr 2021 23:59:15 +0300 Subject: [PATCH 10/41] Specify docker image to use in github builds --- .github/workflows/build_wheels.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 2d40361f..8b3175ad 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -38,6 +38,7 @@ jobs: NP_TEST_DEP: numpy TRAVIS_BUILD_DIR: ${{ github.workspace }} CONFIG_PATH: travis_config.sh + DOCKER_IMAGE: quay.io/skvark/manylinux2014_${PLAT} USE_CCACHE: 1 UNICODE_WIDTH: 32 SDIST: 0 From a6c746d4ede7eed93bb53d7ba235eb5a247eeccb Mon Sep 17 00:00:00 2001 From: Grigory Serebryakov Date: Wed, 7 Apr 2021 00:08:31 +0300 Subject: [PATCH 11/41] Enable back all python versions in build matrix --- .github/workflows/build_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 8b3175ad..752643f6 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -23,7 +23,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest] #, macos-latest] - python-version: [3.6] #, 3.7, 3.8, 3.9] + python-version: [3.6, 3.7, 3.8, 3.9] platform: [x64] env: From 940e5b11927e4c4d06cc577669a28db439e7b819 Mon Sep 17 00:00:00 2001 From: Grigory Serebryakov Date: Wed, 7 Apr 2021 00:24:28 +0300 Subject: [PATCH 12/41] Remove numpy as build dependency as prebuilt docker should handle it for us --- .github/workflows/build_wheels.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 752643f6..a8dcf930 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -34,7 +34,6 @@ jobs: MB_PYTHON_VERSION: ${{ matrix.python-version }} TRAVIS_PYTHON_VERSION: ${{ matrix.python-version }} MB_ML_VER: 2014 - NP_BUILD_DEP: numpy==1.11.1 NP_TEST_DEP: numpy TRAVIS_BUILD_DIR: ${{ github.workspace }} CONFIG_PATH: travis_config.sh From 60e6a36cc03f3e01b5228884a4540b8c9251d669 Mon Sep 17 00:00:00 2001 From: Grigory Serebryakov Date: Wed, 7 Apr 2021 17:37:36 +0300 Subject: [PATCH 13/41] Make linux build matrix wider - enable contrib, headless/gui, source/binary distribution --- .github/workflows/build_wheels.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index a8dcf930..3d51d583 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -25,6 +25,9 @@ jobs: os: [ubuntu-latest] #, macos-latest] python-version: [3.6, 3.7, 3.8, 3.9] platform: [x64] + build_sdist: [0, 1] + with_contrib: [0, 1] + without_gui: [0, 1] env: REPO_DIR: . @@ -40,7 +43,9 @@ jobs: DOCKER_IMAGE: quay.io/skvark/manylinux2014_${PLAT} USE_CCACHE: 1 UNICODE_WIDTH: 32 - SDIST: 0 + SDIST: ${{ matrix.build_sdist }} + ENABLE_HEADLESS: ${{ matrix.without_gui }} + ENABLE_CONTRIB: ${{ matrix.with_contrib }} steps: - name: Checkout From 5c33d11a45fddb3219fc07fca4dfed626a7f2721 Mon Sep 17 00:00:00 2001 From: Grigory Serebryakov Date: Wed, 7 Apr 2021 17:45:39 +0300 Subject: [PATCH 14/41] Fix return code setting for skipped tests in case of sdist --- .github/workflows/build_wheels.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 3d51d583..16a93524 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -117,6 +117,7 @@ jobs: set -x if [[ $SDIST == 1 ]]; then echo "skipping tests because of sdist" + rc=0 else install_run $PLAT && rc=$? || rc=$? fi From 219c4a1ed0f4ce068df2c1b75ef5de257d4ae638 Mon Sep 17 00:00:00 2001 From: Grigory Serebryakov Date: Wed, 7 Apr 2021 18:27:06 +0300 Subject: [PATCH 15/41] Build source distribution with only one version of python - as we do not care abnout python version when doing the source distribution --- .github/workflows/build_wheels.yml | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 16a93524..1386e06d 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -25,9 +25,30 @@ jobs: os: [ubuntu-latest] #, macos-latest] python-version: [3.6, 3.7, 3.8, 3.9] platform: [x64] - build_sdist: [0, 1] with_contrib: [0, 1] without_gui: [0, 1] + build_sdist: [0] + include: + - os: ubuntu-latest + python-version: 3.8 + build_sdist: 1 + with_contrib: 0 + without_gui: 0 + - os: ubuntu-latest + python-version: 3.8 + build_sdist: 1 + with_contrib: 1 + without_gui: 0 + - os: ubuntu-latest + python-version: 3.8 + build_sdist: 1 + with_contrib: 0 + without_gui: 1 + - os: ubuntu-latest + python-version: 3.8 + build_sdist: 1 + with_contrib: 1 + without_gui: 1 env: REPO_DIR: . From 665b09b33537a43f64fa1d6b301db21f3247665a Mon Sep 17 00:00:00 2001 From: Grigory Serebryakov Date: Wed, 7 Apr 2021 18:29:47 +0300 Subject: [PATCH 16/41] Try to fix config --- .github/workflows/build_wheels.yml | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 1386e06d..ba0c2c64 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -32,23 +32,6 @@ jobs: - os: ubuntu-latest python-version: 3.8 build_sdist: 1 - with_contrib: 0 - without_gui: 0 - - os: ubuntu-latest - python-version: 3.8 - build_sdist: 1 - with_contrib: 1 - without_gui: 0 - - os: ubuntu-latest - python-version: 3.8 - build_sdist: 1 - with_contrib: 0 - without_gui: 1 - - os: ubuntu-latest - python-version: 3.8 - build_sdist: 1 - with_contrib: 1 - without_gui: 1 env: REPO_DIR: . From 4b6885d164553b5d5094d1838f003e3e347c745a Mon Sep 17 00:00:00 2001 From: Grigory Serebryakov Date: Wed, 7 Apr 2021 21:03:13 +0300 Subject: [PATCH 17/41] Try to fix config --- .github/workflows/build_wheels.yml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index ba0c2c64..0d3a8737 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -32,6 +32,23 @@ jobs: - os: ubuntu-latest python-version: 3.8 build_sdist: 1 + with_contrib: 0 + without_gui: 0 + - os: ubuntu-latest + python-version: 3.8 + build_sdist: 1 + with_contrib: 0 + without_gui: 1 + - os: ubuntu-latest + python-version: 3.8 + build_sdist: 1 + with_contrib: 1 + without_gui: 0 + - os: ubuntu-latest + python-version: 3.8 + build_sdist: 1 + with_contrib: 1 + without_gui: 1 env: REPO_DIR: . @@ -47,7 +64,7 @@ jobs: DOCKER_IMAGE: quay.io/skvark/manylinux2014_${PLAT} USE_CCACHE: 1 UNICODE_WIDTH: 32 - SDIST: ${{ matrix.build_sdist }} + SDIST: ${{ matrix.build_sdist || 0}} ENABLE_HEADLESS: ${{ matrix.without_gui }} ENABLE_CONTRIB: ${{ matrix.with_contrib }} From c1c51f8b088c8a1e1ce6395499bdc9bdfc2603a6 Mon Sep 17 00:00:00 2001 From: Grigory Serebryakov Date: Wed, 7 Apr 2021 21:20:54 +0300 Subject: [PATCH 18/41] Enable OSX builds --- .github/workflows/build_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 0d3a8737..c8209cd6 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -22,7 +22,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest] #, macos-latest] + os: [ubuntu-latest, macos-latest] python-version: [3.6, 3.7, 3.8, 3.9] platform: [x64] with_contrib: [0, 1] From af32e404567f8ecd2ee7901ef10a66e3fa424e91 Mon Sep 17 00:00:00 2001 From: Grigory Serebryakov Date: Wed, 7 Apr 2021 21:28:16 +0300 Subject: [PATCH 19/41] Quickfix to enable OSX build --- travis_osx_brew_cache.sh | 95 +++++++++++++++++++++------------------- 1 file changed, 49 insertions(+), 46 deletions(-) diff --git a/travis_osx_brew_cache.sh b/travis_osx_brew_cache.sh index e3a69cc3..401265f2 100644 --- a/travis_osx_brew_cache.sh +++ b/travis_osx_brew_cache.sh @@ -4,6 +4,9 @@ #Should be in Travis' cache BREW_LOCAL_BOTTLE_METADATA="$HOME/local_bottle_metadata" +#FIXME: temporary fix to enable the build, should be replaced with the proper path to the cache dir +mkdir -p $BREW_LOCAL_BOTTLE_METADATA + # Starting reference point for elapsed build time; seconds since the epoch. #TRAVIS_TIMER_START_TIME is set at the start of a log fold, in nanoseconds since the epoch BREW_TIME_START=$(($TRAVIS_TIMER_START_TIME/10**9)) @@ -48,33 +51,33 @@ function brew_add_local_bottles { # so that `brew` commands can find them. # If the package was updated, removes the corresponding files # and the bottle's entry in the formula, if any. - - # Bottle entry in formula: + + # Bottle entry in formula: # bottle do # <...> # sha256 "" => : # <...> - # end - + # end + echo "Cached bottles:" ls "$(brew --cache)/downloads" || true #may not exist initially since it's "$(brew --cache)" that is in Travis cache echo "Saved .json's and links:" ls "$BREW_LOCAL_BOTTLE_METADATA" - + for JSON in "$BREW_LOCAL_BOTTLE_METADATA"/*.json; do [ -e "$JSON" ] || break # OSX 10.11 bash has no nullglob local PACKAGE JSON_VERSION JSON_REBUILD OS_CODENAME BOTTLE_HASH - + _brew_parse_bottle_json "$JSON" PACKAGE JSON_VERSION JSON_REBUILD OS_CODENAME BOTTLE_HASH echo "Adding local bottle: $PACKAGE ${JSON_VERSION}_${JSON_REBUILD}" - + local FORMULA_VERSION FORMULA_REBUILD FORMULA_BOTTLE_HASH - + _brew_parse_package_info "$PACKAGE" "$OS_CODENAME" FORMULA_VERSION FORMULA_REBUILD FORMULA_BOTTLE_HASH local FORMULA_HAS_BOTTLE; [ -n "$FORMULA_BOTTLE_HASH" ] && FORMULA_HAS_BOTTLE=1 || true - + local BOTTLE_LINK BOTTLE=""; BOTTLE_LINK="${JSON}.bottle.lnk"; local BOTTLE_EXISTS= BOTTLE_MISMATCH= VERSION_MISMATCH= @@ -88,10 +91,10 @@ function brew_add_local_bottles { if [ -f "$BOTTLE_LINK" ]; then BOTTLE=$(cat "$BOTTLE_LINK"); BOTTLE=$(cd "$(dirname "$BOTTLE")"; pwd)/$(basename "$BOTTLE") - + if [ -e "$BOTTLE" ]; then BOTTLE_EXISTS=1; - + # The hash in `brew --cache $PACKAGE` entry is generated from download URL, # which itself is generated from base URL and version # (see Homebrew/Library/Homebrew/download_strategy.rb:cached_location). @@ -110,7 +113,7 @@ function brew_add_local_bottles { else echo "Link file is missing or of invalid type!" >&2 fi - + # Delete cached bottle and all metadata if invalid if [[ -z "$BOTTLE_EXISTS" || -n "$VERSION_MISMATCH" || -n "$BOTTLE_MISMATCH" ]]; then echo "Deleting the cached bottle and all metadata" @@ -133,11 +136,11 @@ function brew_add_local_bottles { git commit -m "Removed obsolete local bottle ${JSON_VERSION}_${JSON_REBUILD} :${OS_CODENAME}" "$FORMULA" ) fi - + if [ -n "$BOTTLE" -a -n "$BOTTLE_EXISTS" ]; then rm "$BOTTLE"; fi rm -f "$BOTTLE_LINK" rm "$JSON" - + #(Re)add metadata to the formula otherwise else if [ "$FORMULA_BOTTLE_HASH" == "$BOTTLE_HASH" ]; then @@ -156,7 +159,7 @@ function brew_cache_cleanup { #Lefovers from some failure probably rm -f "$BREW_LOCAL_BOTTLE_METADATA"/*.tar.gz - + #`brew cleanup` may delete locally-built bottles that weren't needed this time # so we're saving and restoring them local BOTTLE_LINK BOTTLE @@ -187,16 +190,16 @@ function brew_go_bootstrap_mode { local EXIT_CODE=${1:-1} echo "Going into cache bootstrap mode" - + BREW_BOOTSTRAP_MODE=1 - + #Can't just `exit` because that would terminate the build without saving the cache #Have to replace further actions with no-ops - + local MESSAGE=""; if [ "$EXIT_CODE" -ne 0 ]; then MESSAGE='Building dependencies took too long. Restart the build in Travis UI to continue from cache.'; fi - + eval ' function '"$cmd"' { return 0; } function repair_wheelhouse { return 0; } @@ -205,11 +208,11 @@ function brew_go_bootstrap_mode { echo \ ' echo -e "\n'"$MESSAGE"'\n"' fi)"\ - ' + ' # Travis runs user scripts via `eval` i.e. in the same shell process. # So have to unset errexit in order to get to cache save stage set +e; return '"$EXIT_CODE"' - }' + }' } @@ -228,15 +231,15 @@ function _brew_install_and_cache_within_time_limit { if grep -qxFf <(cat <<<"$_BREW_ALREADY_INSTALLED") <<<"$PACKAGE"; then MARKED_INSTALLED=1 fi - + if [ -n "$MARKED_INSTALLED" ] || (brew list --versions "$PACKAGE" >/dev/null && ! (brew outdated | grep -qxF "$PACKAGE")); then echo "Already installed and the latest version: $PACKAGE" if [ -z "$MARKED_INSTALLED" ]; then _brew_mark_installed "$PACKAGE"; fi return 0 fi - + local BUILD_FROM_SOURCE INCLUDE_BUILD KEG_ONLY - + _brew_is_bottle_available "$PACKAGE" KEG_ONLY || BUILD_FROM_SOURCE=1 [ -n "$BUILD_FROM_SOURCE" ] && INCLUDE_BUILD="--include-build" || true @@ -254,7 +257,7 @@ function _brew_install_and_cache_within_time_limit { _brew_install_and_cache "$PACKAGE" "$([[ -z "$BUILD_FROM_SOURCE" ]] && echo 1 || echo 0)" "$KEG_ONLY" || return 2 _brew_check_elapsed_build_time "$TIME_START" "$TIME_LIMIT" || return $? } - + function _brew_parse_bottle_json { # Parse JSON file resulting from `brew bottle --json` @@ -271,9 +274,9 @@ function _brew_parse_bottle_json { print tag_name print tag_dict["sha256"] ' "$JSON") - + unset JSON - + { local i v; for i in {1..5}; do read -r v eval "${1:?}=\"$v\"" @@ -284,7 +287,7 @@ function _brew_parse_bottle_json { function _brew_parse_package_info { # Get and parse `brew info --json` about a package # and save data into specified variables - + local PACKAGE OS_CODENAME PACKAGE="${1:?}"; shift OS_CODENAME="${1:?}"; shift @@ -300,9 +303,9 @@ function _brew_parse_package_info { print bottle_data["files"].get(sys.argv[2],{"sha256":"!?"})["sha256"] #prevent losing trailing blank line to command substitution ' \ "$PACKAGE" "$OS_CODENAME"); JSON_DATA="${JSON_DATA%\!\?}" #!? can't occur in a hash - + unset PACKAGE OS_CODENAME - + { local i v; for i in {1..3}; do read -r v eval "${1:?}=\"$v\"" @@ -346,13 +349,13 @@ function _brew_install_and_cache { # Install bottle or make and cache bottle. # assumes that deps were already installed # and not already the latest version - + local PACKAGE USE_BOTTLE KEG_ONLY PACKAGE="${1:?}" USE_BOTTLE="${2:?}" KEG_ONLY="${3:?}" local VERB - + if brew list --versions "$PACKAGE"; then # Install alongside the old version to avoid to have to update "runtime dependents" # https://discourse.brew.sh/t/can-i-install-a-new-version-without-having-to-upgrade-runtime-dependents/4443 @@ -363,7 +366,7 @@ function _brew_install_and_cache { else VERB=install fi - + if [[ "$USE_BOTTLE" -gt 0 ]]; then echo "Installing bottle for: $PACKAGE" brew $VERB "$PACKAGE" @@ -380,12 +383,12 @@ function _brew_install_and_cache { #proper procedure as per https://discourse.brew.sh/t/how-are-bottle-and-postinstall-related-is-it-safe-to-run-bottle-after-postinstall/3410/4 brew uninstall --ignore-dependencies "$PACKAGE" brew $VERB "$BOTTLE" - + local JSON; JSON=$(sed -E 's/bottle(.[[:digit:]]+)?\.tar\.gz$/bottle.json/' <<<"$BOTTLE") - + #`brew bottle --merge` doesn't return nonzero on nonexisting json file test -f "$JSON" -a -f "$BOTTLE" - + brew bottle --merge --write "$JSON" local CACHED_BOTTLE; CACHED_BOTTLE="$(brew --cache "$PACKAGE")" mv "$BOTTLE" "$CACHED_BOTTLE"; @@ -394,9 +397,9 @@ function _brew_install_and_cache { #Symlinks aren't cached by Travis. Will just save paths in files then. local BOTTLE_LINK; BOTTLE_LINK="${CACHED_JSON}.bottle.lnk" echo "$CACHED_BOTTLE" >"$BOTTLE_LINK" - + fi - + _brew_mark_installed "$PACKAGE" } @@ -411,11 +414,11 @@ function _brew_check_elapsed_build_time { local TIME_START TIME_LIMIT ELAPSED_TIME TIME_START="${1:?}" TIME_LIMIT="${2:?}" - + ELAPSED_TIME=$(($(date +%s) - $TIME_START)) echo "Elapsed time: "$(($ELAPSED_TIME/60))"m (${ELAPSED_TIME}s)" - - if [[ "$ELAPSED_TIME" -gt $TIME_LIMIT ]]; then + + if [[ "$ELAPSED_TIME" -gt $TIME_LIMIT ]]; then brew_go_bootstrap_mode return 1 fi @@ -426,19 +429,19 @@ function _brew_check_slow_building_ahead { #If the package's projected build completion is higher than hard limit, # skip it and arrange for further build to be skipped and return 1 - + local PACKAGE TIME_START TIME_HARD_LIMIT PACKAGE="${1:?}" TIME_START="${2:?}" TIME_HARD_LIMIT="${3:?}" - - local PROJECTED_BUILD_TIME + + local PROJECTED_BUILD_TIME PROJECTED_BUILD_TIME=$(echo "$BREW_SLOW_BUILIDING_PACKAGES" | awk '$1=="'"$PACKAGE"'"{print $2}') [ -z "$PROJECTED_BUILD_TIME" ] && return 0 || true - + local PROJECTED_BUILD_END_ELAPSED_TIME PROJECTED_BUILD_END_ELAPSED_TIME=$(( $(date +%s) - TIME_START + PROJECTED_BUILD_TIME * 60)) - + if [[ "$PROJECTED_BUILD_END_ELAPSED_TIME" -ge "$TIME_HARD_LIMIT" ]]; then echo -e "\nProjected build end elapsed time for $PACKAGE: $((PROJECTED_BUILD_END_ELAPSED_TIME/60))m ($PROJECTED_BUILD_END_ELAPSED_TIMEs)" brew_go_bootstrap_mode From fbebaa61bf92c604349b4f7248d443d8b213352d Mon Sep 17 00:00:00 2001 From: Grigory Serebryakov Date: Wed, 7 Apr 2021 21:29:52 +0300 Subject: [PATCH 20/41] Temporary disable linux and all pythons execpt one to reduce the build matrix size --- .github/workflows/build_wheels.yml | 46 +++++++++++++++--------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index c8209cd6..d66d90f0 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -22,33 +22,33 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest] - python-version: [3.6, 3.7, 3.8, 3.9] + os: [macos-latest] # ubuntu-latest, + python-version: [3.8] # [3.6, 3.7, 3.8, 3.9] platform: [x64] with_contrib: [0, 1] without_gui: [0, 1] build_sdist: [0] - include: - - os: ubuntu-latest - python-version: 3.8 - build_sdist: 1 - with_contrib: 0 - without_gui: 0 - - os: ubuntu-latest - python-version: 3.8 - build_sdist: 1 - with_contrib: 0 - without_gui: 1 - - os: ubuntu-latest - python-version: 3.8 - build_sdist: 1 - with_contrib: 1 - without_gui: 0 - - os: ubuntu-latest - python-version: 3.8 - build_sdist: 1 - with_contrib: 1 - without_gui: 1 + # include: + # - os: ubuntu-latest + # python-version: 3.8 + # build_sdist: 1 + # with_contrib: 0 + # without_gui: 0 + # - os: ubuntu-latest + # python-version: 3.8 + # build_sdist: 1 + # with_contrib: 0 + # without_gui: 1 + # - os: ubuntu-latest + # python-version: 3.8 + # build_sdist: 1 + # with_contrib: 1 + # without_gui: 0 + # - os: ubuntu-latest + # python-version: 3.8 + # build_sdist: 1 + # with_contrib: 1 + # without_gui: 1 env: REPO_DIR: . From a271af1b851fbcb49cb3a2566dca0ab8df4e305c Mon Sep 17 00:00:00 2001 From: Grigory Serebryakov Date: Wed, 7 Apr 2021 22:01:10 +0300 Subject: [PATCH 21/41] Disable cache stage that was implemented for travis --- travis_config.sh | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/travis_config.sh b/travis_config.sh index 9ba0bb11..c9480b9b 100644 --- a/travis_config.sh +++ b/travis_config.sh @@ -99,24 +99,26 @@ function pre_build { export HOMEBREW_NO_AUTO_UPDATE=1 #after the cache stage, all bottles and Homebrew metadata should be already cached locally - if [ -n "$CACHE_STAGE" ]; then - brew update - generate_ffmpeg_formula - brew_add_local_bottles - fi + # if [ -n "$CACHE_STAGE" ]; then + # brew update + # generate_ffmpeg_formula + # brew_add_local_bottles + # fi echo 'Installing FFmpeg' - if [ -n "$CACHE_STAGE" ]; then - brew_install_and_cache_within_time_limit ffmpeg_opencv || { [ $? -gt 1 ] && return 2 || return 0; } - else - brew unlink python@2 + # if [ -n "$CACHE_STAGE" ]; then + # brew_install_and_cache_within_time_limit ffmpeg_opencv || { [ $? -gt 1 ] && return 2 || return 0; } + # else + brew update generate_ffmpeg_formula + brew_add_local_bottles + brew unlink python@2 brew install ffmpeg_opencv - fi + # fi # echo 'Installing qt5' - + # if [ -n "$CACHE_STAGE" ]; then # echo "Qt5 has bottle, no caching needed" # else From 169df1fd1740fe674fbf2c2ddb2087e5d8b63d84 Mon Sep 17 00:00:00 2001 From: Grigory Serebryakov Date: Wed, 7 Apr 2021 22:03:42 +0300 Subject: [PATCH 22/41] Do not unlink python2 for github osx builds --- travis_config.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travis_config.sh b/travis_config.sh index c9480b9b..1bf9805b 100644 --- a/travis_config.sh +++ b/travis_config.sh @@ -113,7 +113,7 @@ function pre_build { brew update generate_ffmpeg_formula brew_add_local_bottles - brew unlink python@2 + # brew unlink python@2 brew install ffmpeg_opencv # fi From d2e35409991e0673dac88db07bd0169ef9a7a0fc Mon Sep 17 00:00:00 2001 From: Grigory Serebryakov Date: Wed, 7 Apr 2021 22:16:31 +0300 Subject: [PATCH 23/41] Fix ffmpeg bottle build command --- travis_config.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travis_config.sh b/travis_config.sh index 1bf9805b..9e8b454f 100644 --- a/travis_config.sh +++ b/travis_config.sh @@ -114,7 +114,7 @@ function pre_build { generate_ffmpeg_formula brew_add_local_bottles # brew unlink python@2 - brew install ffmpeg_opencv + brew install --build-bottle ffmpeg_opencv # fi # echo 'Installing qt5' From 093968edd068f54ae955c022434b484866406da1 Mon Sep 17 00:00:00 2001 From: Grigory Serebryakov Date: Wed, 7 Apr 2021 22:51:27 +0300 Subject: [PATCH 24/41] Disable travis cache stage --- travis_config.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/travis_config.sh b/travis_config.sh index 9e8b454f..aac66317 100644 --- a/travis_config.sh +++ b/travis_config.sh @@ -95,7 +95,8 @@ function pre_build { if [ -n "$IS_OSX" ]; then echo "Running for OSX" - local CACHE_STAGE; (echo "$TRAVIS_BUILD_STAGE_NAME" | grep -qiF "final") || CACHE_STAGE=1 + local CACHE_STAGE;# (echo "$TRAVIS_BUILD_STAGE_NAME" | grep -qiF "final") || CACHE_STAGE=1 + CACHE_STAGE= export HOMEBREW_NO_AUTO_UPDATE=1 #after the cache stage, all bottles and Homebrew metadata should be already cached locally From db31b23ee8ec1ae5a0294b03e6c501349f271d03 Mon Sep 17 00:00:00 2001 From: Andrey Senyaev Date: Wed, 14 Apr 2021 12:25:53 +0300 Subject: [PATCH 25/41] Changed build_wheels to Windows platform --- .github/workflows/build_wheels.yml | 125 +++++++---------------------- 1 file changed, 28 insertions(+), 97 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index d66d90f0..2536458d 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -22,7 +22,7 @@ jobs: strategy: fail-fast: false matrix: - os: [macos-latest] # ubuntu-latest, + os: [windows-latest] # ubuntu-latest, python-version: [3.8] # [3.6, 3.7, 3.8, 3.9] platform: [x64] with_contrib: [0, 1] @@ -50,24 +50,6 @@ jobs: # with_contrib: 1 # without_gui: 1 - env: - REPO_DIR: . - BUILD_COMMIT: master - PROJECT_SPEC: opencv-python - PLAT: x86_64 - MB_PYTHON_VERSION: ${{ matrix.python-version }} - TRAVIS_PYTHON_VERSION: ${{ matrix.python-version }} - MB_ML_VER: 2014 - NP_TEST_DEP: numpy - TRAVIS_BUILD_DIR: ${{ github.workspace }} - CONFIG_PATH: travis_config.sh - DOCKER_IMAGE: quay.io/skvark/manylinux2014_${PLAT} - USE_CCACHE: 1 - UNICODE_WIDTH: 32 - SDIST: ${{ matrix.build_sdist || 0}} - ENABLE_HEADLESS: ${{ matrix.without_gui }} - ENABLE_CONTRIB: ${{ matrix.with_contrib }} - steps: - name: Checkout uses: actions/checkout@v2 @@ -80,86 +62,35 @@ jobs: git submodule update --remote - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Setup Environment variables run: | - if [ "macos-latest" == "${{ matrix.os }}" ]; then echo "TRAVIS_OS_NAME=osx" >> $GITHUB_ENV; else echo "TRAVIS_OS_NAME=${{ matrix.os }}" >> $GITHUB_ENV; fi - if [ "schedule" == "${{ github.event_name }}" ]; then echo "TRAVIS_EVENT_TYPE=cron" >> $GITHUB_ENV; else echo "TRAVIS_EVENT_TYPE=${{ github.event_name }}" >> $GITHUB_ENV; fi - if [ "schedule" == "${{ github.event_name }}" ]; then echo "BUILD_COMMIT=master" >> $GITHUB_ENV; else echo "BUILD_COMMIT=$BUILD_COMMIT" >> $GITHUB_ENV; fi - echo "BUILD_DEPENDS=$(echo $NP_BUILD_DEP)" >> $GITHUB_ENV; - echo "TEST_DEPENDS=$(echo $NP_TEST_DEP)" >> $GITHUB_ENV; - - - name: before install + if [[ ! $env:PYTHON ]] { + curl -o install_python.ps1 https://raw.githubusercontent.com/matthew-brett/multibuild/devel/install_python.ps1 + .\install_python.ps1 + } + if [[ ! $env:PYTHON ]]; then echo "No PYTHON"; fi + set PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH% + python --version + + - name: build script run: | - set -e - - if [[ $SDIST == 0 ]]; then - # Check out and prepare the source - # Multibuild doesn't have releases, so --depth would break eventually (see - # https://superuser.com/questions/1240216/server-does-not-allow-request-for-unadvertised) - git submodule update --init multibuild - - source multibuild/common_utils.sh - - # https://github.com/matthew-brett/multibuild/issues/116 - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export ARCH_FLAGS=" "; fi - - source multibuild/travis_steps.sh - # This sets -x - - # source travis_multibuild_customize.sh - echo $ENABLE_CONTRIB > contrib.enabled - echo $ENABLE_HEADLESS > headless.enabled + "%PYTHON%\\python.exe" -m pip install --upgrade pip + "%PYTHON%\\python.exe" -m pip install --upgrade setuptools + set "CI_BUILD=1" && "%PYTHON%\\python.exe" -m pip wheel --wheel-dir=%cd%\dist . --verbose - echo "end" - # Not interested in travis internal scripts' output - fi - - set +x - - # Build and package - set -x - - ls - if [[ $SDIST == 1 ]]; then - python -m pip install --upgrade pip - python -m pip install scikit-build - python setup.py sdist - else - build_wheel $REPO_DIR $PLAT - fi - - set +x - - # Install and run tests - set -x - if [[ $SDIST == 1 ]]; then - echo "skipping tests because of sdist" - rc=0 - else - install_run $PLAT && rc=$? || rc=$? - fi - - set +x + - name: before test + run: | + cd ${{ github.workspace }}/tests + export PYTHONWARNINGS = "ignore:::pip._internal.cli.base_command" + &"${{ matrix.python-version }}\\python.exe" -m pip install --user --no-warn-script-location (ls "../dist/opencv_*.whl") + if [[ $? -ne 0 ]]; then exit $?; fi - #otherwise, Travis logic terminates prematurely - #https://travis-ci.community/t/shell-session-update-command-not-found-in-build-log-causes-build-to-fail-if-trap-err-is-set/817 - trap ERR - test "$rc" -eq 0 + - name: run test + run: | + cd ${{ github.workspace }}/tests + "%PYTHON%\\python.exe" -m unittest test - # - name: Upload wheels - # env: - # # PYPI repository - # TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - # TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - # # PYPITEST repository - # # TWINE_USERNAME: ${{ secrets.PYPITEST_USERNAME }} - # # TWINE_PASSWORD: ${{ secrets.PYPITEST_PASSWORD }} - # # TWINE_REPOSITORY_URL: 'https://test.pypi.org/legacy/' - # run: | - # twine upload --skip-existing ${TRAVIS_BUILD_DIR}/wheelhouse/* - # # Upload wheels to PYPITEST - # #twine upload --skip-existing ${TRAVIS_BUILD_DIR}/wheelhouse/* + - name: saving artifacts + uses: actions/upload-artifact@v2 + with: + name: wheels + path: dist\opencv*.whl From 591c15a38b2d64e34322a1a943d847d667be6f80 Mon Sep 17 00:00:00 2001 From: Andrey Senyaev <76472231+asenyaev@users.noreply.github.com> Date: Fri, 16 Apr 2021 11:22:40 +0300 Subject: [PATCH 26/41] Try new pipeline --- .github/workflows/build_wheels.yml | 65 +++++++++++------------------- 1 file changed, 23 insertions(+), 42 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 2536458d..99510911 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -17,38 +17,20 @@ jobs: runs-on: ${{ matrix.os }} defaults: run: - shell: bash + shell: powershell strategy: fail-fast: false matrix: os: [windows-latest] # ubuntu-latest, - python-version: [3.8] # [3.6, 3.7, 3.8, 3.9] - platform: [x64] + python-version: [3.6] # [3.6, 3.7, 3.8, 3.9] + platform: [x64] # [x32, x64] with_contrib: [0, 1] without_gui: [0, 1] build_sdist: [0] - # include: - # - os: ubuntu-latest - # python-version: 3.8 - # build_sdist: 1 - # with_contrib: 0 - # without_gui: 0 - # - os: ubuntu-latest - # python-version: 3.8 - # build_sdist: 1 - # with_contrib: 0 - # without_gui: 1 - # - os: ubuntu-latest - # python-version: 3.8 - # build_sdist: 1 - # with_contrib: 1 - # without_gui: 0 - # - os: ubuntu-latest - # python-version: 3.8 - # build_sdist: 1 - # with_contrib: 1 - # without_gui: 1 + + env: + ACTIONS_ALLOW_UNSECURE_COMMANDS: true steps: - name: Checkout @@ -60,37 +42,36 @@ jobs: - name: Update submodules run: | git submodule update --remote - - name: Set up Python ${{ matrix.python-version }} - run: | - if [[ ! $env:PYTHON ]] { - curl -o install_python.ps1 https://raw.githubusercontent.com/matthew-brett/multibuild/devel/install_python.ps1 - .\install_python.ps1 - } - if [[ ! $env:PYTHON ]]; then echo "No PYTHON"; fi - set PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH% - python --version + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Setup MSBuild.exe + uses: warrenbuckley/Setup-MSBuild@v1 - name: build script run: | - "%PYTHON%\\python.exe" -m pip install --upgrade pip - "%PYTHON%\\python.exe" -m pip install --upgrade setuptools - set "CI_BUILD=1" && "%PYTHON%\\python.exe" -m pip wheel --wheel-dir=%cd%\dist . --verbose + python --version + python -m pip install --upgrade pip + python -m pip install --upgrade setuptools + set "CI_BUILD=1" && python -m pip wheel --wheel-dir=%cd%\dist . --verbose + shell: cmd - name: before test run: | cd ${{ github.workspace }}/tests - export PYTHONWARNINGS = "ignore:::pip._internal.cli.base_command" - &"${{ matrix.python-version }}\\python.exe" -m pip install --user --no-warn-script-location (ls "../dist/opencv_*.whl") - if [[ $? -ne 0 ]]; then exit $?; fi - + $env:PYTHONWARNINGS = "ignore:::pip._internal.cli.base_command" + &python -m pip install --user --no-warn-script-location (ls "../dist/opencv_*.whl") + if ($LastExitCode -ne 0) {throw $LastExitCode} - name: run test run: | cd ${{ github.workspace }}/tests - "%PYTHON%\\python.exe" -m unittest test + python -m unittest test + shell: cmd - name: saving artifacts uses: actions/upload-artifact@v2 with: - name: wheels + name: wheels-${{ matrix.python-version }}-${{ matrix.platform }}-${{ matrix.with_contrib }}-${{ matrix.without_gui }}-${{ matrix.build_sdist }} path: dist\opencv*.whl From f56fb7894a3d37b123868ba77e81dc0d982fb241 Mon Sep 17 00:00:00 2001 From: Andrey Senyaev <76472231+asenyaev@users.noreply.github.com> Date: Fri, 16 Apr 2021 11:55:26 +0300 Subject: [PATCH 27/41] Try windows with a new configuration --- .github/workflows/build_wheels.yml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 99510911..20283b51 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -22,12 +22,17 @@ jobs: strategy: fail-fast: false matrix: - os: [windows-latest] # ubuntu-latest, + os: [windows-latest, ubuntu-latest, macos-latest] # ubuntu-latest, python-version: [3.6] # [3.6, 3.7, 3.8, 3.9] - platform: [x64] # [x32, x64] + platform: [x32, x64] with_contrib: [0, 1] without_gui: [0, 1] build_sdist: [0] + exclude: + - os: macos-latest + platform: x32 + - os: ubuntu-latest + platform: x32 env: ACTIONS_ALLOW_UNSECURE_COMMANDS: true @@ -42,15 +47,18 @@ jobs: - name: Update submodules run: | git submodule update --remote + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - name: Setup MSBuild.exe + if: ${{ matrix.os == 'windows-latest' }} uses: warrenbuckley/Setup-MSBuild@v1 - name: build script + if: ${{ matrix.os == 'windows-latest' }} run: | python --version python -m pip install --upgrade pip @@ -59,19 +67,23 @@ jobs: shell: cmd - name: before test + if: ${{ matrix.os == 'windows-latest' }} run: | cd ${{ github.workspace }}/tests $env:PYTHONWARNINGS = "ignore:::pip._internal.cli.base_command" &python -m pip install --user --no-warn-script-location (ls "../dist/opencv_*.whl") if ($LastExitCode -ne 0) {throw $LastExitCode} + - name: run test + if: ${{ matrix.os == 'windows-latest' }} run: | cd ${{ github.workspace }}/tests python -m unittest test shell: cmd - name: saving artifacts + if: ${{ matrix.os == 'windows-latest' }} uses: actions/upload-artifact@v2 with: - name: wheels-${{ matrix.python-version }}-${{ matrix.platform }}-${{ matrix.with_contrib }}-${{ matrix.without_gui }}-${{ matrix.build_sdist }} + name: ${{ matrix.os }}-wheels-${{ matrix.python-version }}-${{ matrix.platform }}-${{ matrix.with_contrib }}-${{ matrix.without_gui }}-${{ matrix.build_sdist }} path: dist\opencv*.whl From a1be42022d1efe995136e85051aa040326a051c3 Mon Sep 17 00:00:00 2001 From: Andrey Senyaev <76472231+asenyaev@users.noreply.github.com> Date: Fri, 16 Apr 2021 11:58:41 +0300 Subject: [PATCH 28/41] Try bash on Windows --- .github/workflows/build_wheels.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 20283b51..f2ac0f76 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -17,7 +17,7 @@ jobs: runs-on: ${{ matrix.os }} defaults: run: - shell: powershell + shell: bash strategy: fail-fast: false @@ -73,6 +73,7 @@ jobs: $env:PYTHONWARNINGS = "ignore:::pip._internal.cli.base_command" &python -m pip install --user --no-warn-script-location (ls "../dist/opencv_*.whl") if ($LastExitCode -ne 0) {throw $LastExitCode} + shell: powershell - name: run test if: ${{ matrix.os == 'windows-latest' }} From 834e9b3dcb5ce4fe457a5254ebbdb345d95a7e06 Mon Sep 17 00:00:00 2001 From: Andrey Senyaev <76472231+asenyaev@users.noreply.github.com> Date: Fri, 16 Apr 2021 12:05:32 +0300 Subject: [PATCH 29/41] Added ubuntu and macos into pipeline --- .github/workflows/build_wheels.yml | 72 +++++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index f2ac0f76..d4e25ab0 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -36,6 +36,22 @@ jobs: env: ACTIONS_ALLOW_UNSECURE_COMMANDS: true + REPO_DIR: . + BUILD_COMMIT: master + PROJECT_SPEC: opencv-python + PLAT: x86_64 + MB_PYTHON_VERSION: ${{ matrix.python-version }} + TRAVIS_PYTHON_VERSION: ${{ matrix.python-version }} + MB_ML_VER: 2014 + NP_TEST_DEP: numpy + TRAVIS_BUILD_DIR: ${{ github.workspace }} + CONFIG_PATH: travis_config.sh + DOCKER_IMAGE: quay.io/skvark/manylinux2014_${PLAT} + USE_CCACHE: 1 + UNICODE_WIDTH: 32 + SDIST: ${{ matrix.build_sdist || 0}} + ENABLE_HEADLESS: ${{ matrix.without_gui }} + ENABLE_CONTRIB: ${{ matrix.with_contrib }} steps: - name: Checkout @@ -52,6 +68,61 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} + + - name: Setup Environment variables + if: ${{ matrix.os == 'ubuntu-latest' }} || ${{ matrix.os == 'macos-latest' }} + run: | + if [ "macos-latest" == "${{ matrix.os }}" ]; then echo "TRAVIS_OS_NAME=osx" >> $GITHUB_ENV; else echo "TRAVIS_OS_NAME=${{ matrix.os }}" >> $GITHUB_ENV; fi + if [ "schedule" == "${{ github.event_name }}" ]; then echo "TRAVIS_EVENT_TYPE=cron" >> $GITHUB_ENV; else echo "TRAVIS_EVENT_TYPE=${{ github.event_name }}" >> $GITHUB_ENV; fi + if [ "schedule" == "${{ github.event_name }}" ]; then echo "BUILD_COMMIT=master" >> $GITHUB_ENV; else echo "BUILD_COMMIT=$BUILD_COMMIT" >> $GITHUB_ENV; fi + echo "BUILD_DEPENDS=$(echo $NP_BUILD_DEP)" >> $GITHUB_ENV; + echo "TEST_DEPENDS=$(echo $NP_TEST_DEP)" >> $GITHUB_ENV; + + - name: before install + if: ${{ matrix.os == 'ubuntu-latest' }} || ${{ matrix.os == 'macos-latest' }} + run: | + set -e + if [[ $SDIST == 0 ]]; then + # Check out and prepare the source + # Multibuild doesn't have releases, so --depth would break eventually (see + # https://superuser.com/questions/1240216/server-does-not-allow-request-for-unadvertised) + git submodule update --init multibuild + source multibuild/common_utils.sh + # https://github.com/matthew-brett/multibuild/issues/116 + if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export ARCH_FLAGS=" "; fi + source multibuild/travis_steps.sh + # This sets -x + # source travis_multibuild_customize.sh + echo $ENABLE_CONTRIB > contrib.enabled + echo $ENABLE_HEADLESS > headless.enabled + echo "end" + # Not interested in travis internal scripts' output + fi + set +x + # Build and package + set -x + ls + if [[ $SDIST == 1 ]]; then + python -m pip install --upgrade pip + python -m pip install scikit-build + python setup.py sdist + else + build_wheel $REPO_DIR $PLAT + fi + set +x + # Install and run tests + set -x + if [[ $SDIST == 1 ]]; then + echo "skipping tests because of sdist" + rc=0 + else + install_run $PLAT && rc=$? || rc=$? + fi + set +x + #otherwise, Travis logic terminates prematurely + #https://travis-ci.community/t/shell-session-update-command-not-found-in-build-log-causes-build-to-fail-if-trap-err-is-set/817 + trap ERR + test "$rc" -eq 0 - name: Setup MSBuild.exe if: ${{ matrix.os == 'windows-latest' }} @@ -83,7 +154,6 @@ jobs: shell: cmd - name: saving artifacts - if: ${{ matrix.os == 'windows-latest' }} uses: actions/upload-artifact@v2 with: name: ${{ matrix.os }}-wheels-${{ matrix.python-version }}-${{ matrix.platform }}-${{ matrix.with_contrib }}-${{ matrix.without_gui }}-${{ matrix.build_sdist }} From 8a4a96783fe71419a449f4e4b23199fca2fb0b0e Mon Sep 17 00:00:00 2001 From: Andrey Senyaev <76472231+asenyaev@users.noreply.github.com> Date: Fri, 16 Apr 2021 12:10:42 +0300 Subject: [PATCH 30/41] Fix if condition for windows --- .github/workflows/build_wheels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index d4e25ab0..97aa8fe7 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -70,7 +70,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Setup Environment variables - if: ${{ matrix.os == 'ubuntu-latest' }} || ${{ matrix.os == 'macos-latest' }} + if: ${{ matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' }} run: | if [ "macos-latest" == "${{ matrix.os }}" ]; then echo "TRAVIS_OS_NAME=osx" >> $GITHUB_ENV; else echo "TRAVIS_OS_NAME=${{ matrix.os }}" >> $GITHUB_ENV; fi if [ "schedule" == "${{ github.event_name }}" ]; then echo "TRAVIS_EVENT_TYPE=cron" >> $GITHUB_ENV; else echo "TRAVIS_EVENT_TYPE=${{ github.event_name }}" >> $GITHUB_ENV; fi @@ -79,7 +79,7 @@ jobs: echo "TEST_DEPENDS=$(echo $NP_TEST_DEP)" >> $GITHUB_ENV; - name: before install - if: ${{ matrix.os == 'ubuntu-latest' }} || ${{ matrix.os == 'macos-latest' }} + if: ${{ matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' }} run: | set -e if [[ $SDIST == 0 ]]; then From c7db7042b85a59deda63c8cc1f90d53aae044fee Mon Sep 17 00:00:00 2001 From: Andrey Senyaev <76472231+asenyaev@users.noreply.github.com> Date: Fri, 16 Apr 2021 12:38:28 +0300 Subject: [PATCH 31/41] Turned on all python versions, uploading artifacts --- .github/workflows/build_wheels.yml | 55 ++++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 97aa8fe7..56d1505a 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -22,8 +22,8 @@ jobs: strategy: fail-fast: false matrix: - os: [windows-latest, ubuntu-latest, macos-latest] # ubuntu-latest, - python-version: [3.6] # [3.6, 3.7, 3.8, 3.9] + os: [windows-latest, ubuntu-latest, macos-latest] + python-version: [3.6, 3.7, 3.8, 3.9] platform: [x32, x64] with_contrib: [0, 1] without_gui: [0, 1] @@ -154,7 +154,58 @@ jobs: shell: cmd - name: saving artifacts + if: ${{ matrix.os == 'windows-latest' }} uses: actions/upload-artifact@v2 with: name: ${{ matrix.os }}-wheels-${{ matrix.python-version }}-${{ matrix.platform }}-${{ matrix.with_contrib }}-${{ matrix.without_gui }}-${{ matrix.build_sdist }} path: dist\opencv*.whl + + - name: saving artifacts + if: ${{ matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' }} + uses: actions/upload-artifact@v2 + with: + name: ${{ matrix.os }}-wheels-${{ matrix.python-version }}-${{ matrix.platform }}-${{ matrix.with_contrib }}-${{ matrix.without_gui }}-${{ matrix.build_sdist }} + path: ./wheelhouse/opencv*.whl + + + # - name: Upload wheels ${{ matrix.os }} + # if: ${{ matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' }} + # env: + # # PYPI repository + # TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + # TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + # # PYPITEST repository + # # TWINE_USERNAME: ${{ secrets.PYPITEST_USERNAME }} + # # TWINE_PASSWORD: ${{ secrets.PYPITEST_PASSWORD }} + # # TWINE_REPOSITORY_URL: 'https://test.pypi.org/legacy/' + # run: | + # twine upload --skip-existing ${TRAVIS_BUILD_DIR}/wheelhouse/* + # # Upload wheels to PYPITEST + # #twine upload --skip-existing ${TRAVIS_BUILD_DIR}/wheelhouse/* + + # - name: Upload wheels ${{ matrix.os }} + # if: ${{ matrix.os == 'windows-latest' && startsWith(github.ref, 'refs/tags/')}} + # env: + # USER: fXgF9uyy6sT0JoVOR7BoqA== + # PASS: 0bXSOVjf9x8L7nErTivu92TF1FwNosTjFJQPmxp8Dys= + # run: | + # cd ${{ github.workspace }} + # if (${Env:ENABLE_CONTRIB} -eq 0) { + # if (${Env:ENABLE_HEADLESS} -eq 0) { + # echo "This is a default build. Deployment will be done to PyPI entry opencv-python." + # } + # else { + # echo "This is a headless build. Deployment will be done to PyPI entry opencv-python-headless." + # } + # } + # else { + # if (${Env:ENABLE_HEADLESS} -eq 0) { + # echo "This is a contrib build. Deployment will be done to PyPI entry opencv-contrib-python." + # } + # else { + # echo "This is a headless contrib build. Deployment will be done to PyPI entry opencv-contrib-python-headless." + # } + # } + # &python -m pip install twine + # &python -m twine upload -u ${Env:USER} -p ${Env:PASS} --skip-existing dist/opencv* + # shell: powershell From b3b8d09a6f3145da205a66c176c4fa67e64e65a7 Mon Sep 17 00:00:00 2001 From: Andrey Senyaev <76472231+asenyaev@users.noreply.github.com> Date: Wed, 21 Apr 2021 19:24:15 +0300 Subject: [PATCH 32/41] Fixed Windows and created Release job --- .github/workflows/build_wheels.yml | 191 +++++++++++++++-------------- 1 file changed, 98 insertions(+), 93 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 56d1505a..8d3b132f 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -8,11 +8,80 @@ on: branches: - master release: - # Only use the types keyword to narrow down the activity types that will trigger your workflow. - types: [published, created, edited] + types: [published, edited] jobs: + build-windows-x86_64: + runs-on: ${{ matrix.os }} + defaults: + run: + shell: powershell + + strategy: + fail-fast: false + matrix: + os: [windows-latest] + python-version: [3.6, 3.7, 3.8, 3.9] + platform: [x86, x64] + with_contrib: [0, 1] + without_gui: [0, 1] + build_sdist: [0] + + env: + ACTIONS_ALLOW_UNSECURE_COMMANDS: true + SDIST: ${{ matrix.build_sdist || 0}} + ENABLE_HEADLESS: ${{ matrix.without_gui }} + ENABLE_CONTRIB: ${{ matrix.with_contrib }} + + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + submodules: true + fetch-depth: 0 + + - name: Update submodules + run: | + git submodule update --remote + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + architecture: ${{ matrix.platform }} + + - name: Setup MSBuild.exe + uses: warrenbuckley/Setup-MSBuild@v1 + + - name: build script + run: | + python --version + python -m pip install --upgrade pip + python -m pip install --upgrade setuptools + set "CI_BUILD=1" && python -m pip wheel --wheel-dir=%cd%\wheelhouse . --verbose + shell: cmd + + - name: before test + run: | + cd ${{ github.workspace }}/tests + $env:PYTHONWARNINGS = "ignore:::pip._internal.cli.base_command" + &python -m pip install --user --no-warn-script-location (ls "../wheelhouse/opencv*.whl") + if ($LastExitCode -ne 0) {throw $LastExitCode} + shell: powershell + + - name: run test + run: | + cd ${{ github.workspace }}/tests + python -m unittest test + shell: cmd + + - name: saving artifacts + uses: actions/upload-artifact@v2 + with: + name: wheels + path: wheelhouse/opencv*.whl + build: runs-on: ${{ matrix.os }} defaults: @@ -22,17 +91,12 @@ jobs: strategy: fail-fast: false matrix: - os: [windows-latest, ubuntu-latest, macos-latest] + os: [ubuntu-latest, macos-latest] python-version: [3.6, 3.7, 3.8, 3.9] - platform: [x32, x64] + platform: [x64] with_contrib: [0, 1] without_gui: [0, 1] build_sdist: [0] - exclude: - - os: macos-latest - platform: x32 - - os: ubuntu-latest - platform: x32 env: ACTIONS_ALLOW_UNSECURE_COMMANDS: true @@ -68,9 +132,9 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} + architecture: ${{ matrix.platform }} - name: Setup Environment variables - if: ${{ matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' }} run: | if [ "macos-latest" == "${{ matrix.os }}" ]; then echo "TRAVIS_OS_NAME=osx" >> $GITHUB_ENV; else echo "TRAVIS_OS_NAME=${{ matrix.os }}" >> $GITHUB_ENV; fi if [ "schedule" == "${{ github.event_name }}" ]; then echo "TRAVIS_EVENT_TYPE=cron" >> $GITHUB_ENV; else echo "TRAVIS_EVENT_TYPE=${{ github.event_name }}" >> $GITHUB_ENV; fi @@ -79,7 +143,6 @@ jobs: echo "TEST_DEPENDS=$(echo $NP_TEST_DEP)" >> $GITHUB_ENV; - name: before install - if: ${{ matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' }} run: | set -e if [[ $SDIST == 0 ]]; then @@ -123,89 +186,31 @@ jobs: #https://travis-ci.community/t/shell-session-update-command-not-found-in-build-log-causes-build-to-fail-if-trap-err-is-set/817 trap ERR test "$rc" -eq 0 - - - name: Setup MSBuild.exe - if: ${{ matrix.os == 'windows-latest' }} - uses: warrenbuckley/Setup-MSBuild@v1 - - - name: build script - if: ${{ matrix.os == 'windows-latest' }} - run: | - python --version - python -m pip install --upgrade pip - python -m pip install --upgrade setuptools - set "CI_BUILD=1" && python -m pip wheel --wheel-dir=%cd%\dist . --verbose - shell: cmd - - - name: before test - if: ${{ matrix.os == 'windows-latest' }} - run: | - cd ${{ github.workspace }}/tests - $env:PYTHONWARNINGS = "ignore:::pip._internal.cli.base_command" - &python -m pip install --user --no-warn-script-location (ls "../dist/opencv_*.whl") - if ($LastExitCode -ne 0) {throw $LastExitCode} - shell: powershell - - - name: run test - if: ${{ matrix.os == 'windows-latest' }} - run: | - cd ${{ github.workspace }}/tests - python -m unittest test - shell: cmd - - name: saving artifacts - if: ${{ matrix.os == 'windows-latest' }} uses: actions/upload-artifact@v2 with: - name: ${{ matrix.os }}-wheels-${{ matrix.python-version }}-${{ matrix.platform }}-${{ matrix.with_contrib }}-${{ matrix.without_gui }}-${{ matrix.build_sdist }} - path: dist\opencv*.whl + name: wheels + path: wheelhouse/opencv*.whl - - name: saving artifacts - if: ${{ matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' }} - uses: actions/upload-artifact@v2 - with: - name: ${{ matrix.os }}-wheels-${{ matrix.python-version }}-${{ matrix.platform }}-${{ matrix.with_contrib }}-${{ matrix.without_gui }}-${{ matrix.build_sdist }} - path: ./wheelhouse/opencv*.whl - - - # - name: Upload wheels ${{ matrix.os }} - # if: ${{ matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' }} - # env: - # # PYPI repository - # TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - # TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - # # PYPITEST repository - # # TWINE_USERNAME: ${{ secrets.PYPITEST_USERNAME }} - # # TWINE_PASSWORD: ${{ secrets.PYPITEST_PASSWORD }} - # # TWINE_REPOSITORY_URL: 'https://test.pypi.org/legacy/' - # run: | - # twine upload --skip-existing ${TRAVIS_BUILD_DIR}/wheelhouse/* - # # Upload wheels to PYPITEST - # #twine upload --skip-existing ${TRAVIS_BUILD_DIR}/wheelhouse/* - - # - name: Upload wheels ${{ matrix.os }} - # if: ${{ matrix.os == 'windows-latest' && startsWith(github.ref, 'refs/tags/')}} - # env: - # USER: fXgF9uyy6sT0JoVOR7BoqA== - # PASS: 0bXSOVjf9x8L7nErTivu92TF1FwNosTjFJQPmxp8Dys= - # run: | - # cd ${{ github.workspace }} - # if (${Env:ENABLE_CONTRIB} -eq 0) { - # if (${Env:ENABLE_HEADLESS} -eq 0) { - # echo "This is a default build. Deployment will be done to PyPI entry opencv-python." - # } - # else { - # echo "This is a headless build. Deployment will be done to PyPI entry opencv-python-headless." - # } - # } - # else { - # if (${Env:ENABLE_HEADLESS} -eq 0) { - # echo "This is a contrib build. Deployment will be done to PyPI entry opencv-contrib-python." - # } - # else { - # echo "This is a headless contrib build. Deployment will be done to PyPI entry opencv-contrib-python-headless." - # } - # } - # &python -m pip install twine - # &python -m twine upload -u ${Env:USER} -p ${Env:PASS} --skip-existing dist/opencv* - # shell: powershell + release: + if: startsWith(github.ref, 'refs/tags/v') + needs: [build, build-windows-x86_64] + runs-on: ubuntu-latest + defaults: + run: + shell: bash + steps: + - uses: actions/download-artifact@v2 + with: + name: wheels + path: wheelhouse/ + + - name: Upload wheels ${{ matrix.os }} + if: ${{ matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' }} && startsWith(github.ref, 'refs/tags/') + env: + # PYPI repository credentials + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + python -m pip install twine + python -m twine upload -u ${Env:TWINE_USERNAME} -p ${Env:TWINE_PASSWORD} --skip-existing wheelhouse/opencv* From b8b409fbc1c4c7029d2e2f2e791f93ea014ebfd8 Mon Sep 17 00:00:00 2001 From: Grigory Serebryakov Date: Thu, 22 Apr 2021 23:22:06 +0300 Subject: [PATCH 33/41] Add missed source distribution build --- .github/workflows/build_wheels.yml | 89 ++++++++++++++++++++++++++++-- 1 file changed, 84 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 8d3b132f..7e074209 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -44,7 +44,7 @@ jobs: - name: Update submodules run: | git submodule update --remote - + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: @@ -127,7 +127,7 @@ jobs: - name: Update submodules run: | git submodule update --remote - + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: @@ -141,7 +141,7 @@ jobs: if [ "schedule" == "${{ github.event_name }}" ]; then echo "BUILD_COMMIT=master" >> $GITHUB_ENV; else echo "BUILD_COMMIT=$BUILD_COMMIT" >> $GITHUB_ENV; fi echo "BUILD_DEPENDS=$(echo $NP_BUILD_DEP)" >> $GITHUB_ENV; echo "TEST_DEPENDS=$(echo $NP_TEST_DEP)" >> $GITHUB_ENV; - + - name: before install run: | set -e @@ -192,9 +192,89 @@ jobs: name: wheels path: wheelhouse/opencv*.whl + + build_sdist: + runs-on: ${{ matrix.os }} + defaults: + run: + shell: bash + + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + python-version: [3.8] + platform: [x64] + with_contrib: [0, 1] + without_gui: [0] + build_sdist: [1] + + env: + ACTIONS_ALLOW_UNSECURE_COMMANDS: true + REPO_DIR: . + BUILD_COMMIT: master + PROJECT_SPEC: opencv-python + PLAT: x86_64 + MB_PYTHON_VERSION: ${{ matrix.python-version }} + TRAVIS_PYTHON_VERSION: ${{ matrix.python-version }} + MB_ML_VER: 2014 + NP_TEST_DEP: numpy + TRAVIS_BUILD_DIR: ${{ github.workspace }} + CONFIG_PATH: travis_config.sh + DOCKER_IMAGE: quay.io/skvark/manylinux2014_${PLAT} + USE_CCACHE: 1 + UNICODE_WIDTH: 32 + SDIST: ${{ matrix.build_sdist || 0}} + ENABLE_HEADLESS: ${{ matrix.without_gui || 0 }} + ENABLE_CONTRIB: ${{ matrix.with_contrib || 0}} + + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + submodules: true + fetch-depth: 0 + + - name: Update submodules + run: | + git submodule update --remote + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + architecture: ${{ matrix.platform }} + + - name: Setup Environment variables + run: | + if [ "macos-latest" == "${{ matrix.os }}" ]; then echo "TRAVIS_OS_NAME=osx" >> $GITHUB_ENV; else echo "TRAVIS_OS_NAME=${{ matrix.os }}" >> $GITHUB_ENV; fi + if [ "schedule" == "${{ github.event_name }}" ]; then echo "TRAVIS_EVENT_TYPE=cron" >> $GITHUB_ENV; else echo "TRAVIS_EVENT_TYPE=${{ github.event_name }}" >> $GITHUB_ENV; fi + if [ "schedule" == "${{ github.event_name }}" ]; then echo "BUILD_COMMIT=master" >> $GITHUB_ENV; else echo "BUILD_COMMIT=$BUILD_COMMIT" >> $GITHUB_ENV; fi + echo "BUILD_DEPENDS=$(echo $NP_BUILD_DEP)" >> $GITHUB_ENV; + echo "TEST_DEPENDS=$(echo $NP_TEST_DEP)" >> $GITHUB_ENV; + + - name: before install + run: | + set -e + # Build and package + set -x + python -m pip install --upgrade pip + python -m pip install scikit-build + python setup.py sdist + set +x + # Install and run tests + set -x + echo "skipping tests because of sdist" + - name: saving artifacts + uses: actions/upload-artifact@v2 + with: + name: wheels + path: dist/opencv*.tar.gz + + release: if: startsWith(github.ref, 'refs/tags/v') - needs: [build, build-windows-x86_64] + needs: [build, build-windows-x86_64, build_sdist] runs-on: ubuntu-latest defaults: run: @@ -206,7 +286,6 @@ jobs: path: wheelhouse/ - name: Upload wheels ${{ matrix.os }} - if: ${{ matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' }} && startsWith(github.ref, 'refs/tags/') env: # PYPI repository credentials TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} From d512b45113885945c5f35c3b832cf3cc865cc0cb Mon Sep 17 00:00:00 2001 From: Grigory Serebryakov Date: Fri, 23 Apr 2021 00:09:52 +0300 Subject: [PATCH 34/41] Move x86 python builds to github actions. Travis does only aarch64 now --- .github/workflows/build_wheels.yml | 5 +- .travis.yml | 609 +---------------------------- 2 files changed, 17 insertions(+), 597 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 7e074209..59db62cc 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -93,7 +93,7 @@ jobs: matrix: os: [ubuntu-latest, macos-latest] python-version: [3.6, 3.7, 3.8, 3.9] - platform: [x64] + platform: [x86, x64] with_contrib: [0, 1] without_gui: [0, 1] build_sdist: [0] @@ -103,7 +103,6 @@ jobs: REPO_DIR: . BUILD_COMMIT: master PROJECT_SPEC: opencv-python - PLAT: x86_64 MB_PYTHON_VERSION: ${{ matrix.python-version }} TRAVIS_PYTHON_VERSION: ${{ matrix.python-version }} MB_ML_VER: 2014 @@ -139,6 +138,8 @@ jobs: if [ "macos-latest" == "${{ matrix.os }}" ]; then echo "TRAVIS_OS_NAME=osx" >> $GITHUB_ENV; else echo "TRAVIS_OS_NAME=${{ matrix.os }}" >> $GITHUB_ENV; fi if [ "schedule" == "${{ github.event_name }}" ]; then echo "TRAVIS_EVENT_TYPE=cron" >> $GITHUB_ENV; else echo "TRAVIS_EVENT_TYPE=${{ github.event_name }}" >> $GITHUB_ENV; fi if [ "schedule" == "${{ github.event_name }}" ]; then echo "BUILD_COMMIT=master" >> $GITHUB_ENV; else echo "BUILD_COMMIT=$BUILD_COMMIT" >> $GITHUB_ENV; fi + if [ "x64" == "${{ matrix.platform }}" ]; then echo "PLAT=x86_64" >> $GITHUB_ENV; fi + if [ "x86" == "${{ matrix.platform }}" ]; then echo "PLAT=i686" >> $GITHUB_ENV; fi echo "BUILD_DEPENDS=$(echo $NP_BUILD_DEP)" >> $GITHUB_ENV; echo "TEST_DEPENDS=$(echo $NP_TEST_DEP)" >> $GITHUB_ENV; diff --git a/.travis.yml b/.travis.yml index 3c2efde8..6ba06d4c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,6 @@ cache: # Add more cache stages (s2 etc) and corresponding OSX jobs like s1 # if brew builds start to take longer than one Travis time limit stages: - - s1 - final jobs: @@ -32,227 +31,7 @@ jobs: exclude: - language: ruby include: - # source distributions - - os: linux - stage: s1 - script: skip - env: - - SDIST=1 - - ENABLE_CONTRIB=0 - - ENABLE_HEADLESS=0 - python: "3.8" - language: python - dist: xenial - - os: linux - stage: s1 - script: skip - env: - - SDIST=1 - - ENABLE_CONTRIB=0 - - ENABLE_HEADLESS=1 - python: "3.8" - language: python - dist: xenial - - os: linux - stage: s1 - script: skip - env: - - SDIST=1 - - ENABLE_CONTRIB=1 - - ENABLE_HEADLESS=0 - python: "3.8" - language: python - dist: xenial - - os: linux - stage: s1 - script: skip - env: - - SDIST=1 - - ENABLE_CONTRIB=1 - - ENABLE_HEADLESS=1 - python: "3.8" - language: python - dist: xenial - - # default builds for MacOS - #further jobs in the list will use the same stage until the next assignment - - stage: final - - os: osx - language: generic - osx_image: xcode9.4 - env: - - MB_PYTHON_VERSION=3.6 - - ENABLE_CONTRIB=0 - - ENABLE_HEADLESS=0 - - TEST_DEPENDS=numpy==1.13.3 - - - os: osx - language: generic - osx_image: xcode9.4 - env: - - MB_PYTHON_VERSION=3.7 - - ENABLE_CONTRIB=0 - - ENABLE_HEADLESS=0 - - TEST_DEPENDS=numpy==1.14.5 - - - os: osx - language: generic - osx_image: xcode9.4 - env: - - MB_PYTHON_VERSION=3.8 - - ENABLE_CONTRIB=0 - - ENABLE_HEADLESS=0 - - TEST_DEPENDS=numpy==1.17.3 - - - os: osx - language: generic - osx_image: xcode9.4 - env: - - MB_PYTHON_VERSION=3.9 - - ENABLE_CONTRIB=0 - - ENABLE_HEADLESS=0 - - TEST_DEPENDS=numpy==1.19.3 - - - # headless builds for MacOS - - os: osx - language: generic - osx_image: xcode9.4 - env: - - MB_PYTHON_VERSION=3.6 - - ENABLE_CONTRIB=0 - - ENABLE_HEADLESS=1 - - TEST_DEPENDS=numpy==1.13.3 - - - os: osx - language: generic - osx_image: xcode9.4 - env: - - MB_PYTHON_VERSION=3.7 - - ENABLE_CONTRIB=0 - - ENABLE_HEADLESS=1 - - TEST_DEPENDS=numpy==1.14.5 - - - os: osx - language: generic - osx_image: xcode9.4 - env: - - MB_PYTHON_VERSION=3.8 - - ENABLE_CONTRIB=0 - - ENABLE_HEADLESS=1 - - TEST_DEPENDS=numpy==1.17.3 - - - os: osx - language: generic - osx_image: xcode9.4 - env: - - MB_PYTHON_VERSION=3.9 - - ENABLE_CONTRIB=0 - - ENABLE_HEADLESS=1 - - TEST_DEPENDS=numpy==1.19.3 - - - # Contrib builds for MacOS - - os: osx - language: generic - osx_image: xcode9.4 - env: - - MB_PYTHON_VERSION=3.6 - - ENABLE_CONTRIB=1 - - ENABLE_HEADLESS=0 - - TEST_DEPENDS=numpy==1.13.3 - - - os: osx - language: generic - osx_image: xcode9.4 - env: - - MB_PYTHON_VERSION=3.7 - - ENABLE_CONTRIB=1 - - ENABLE_HEADLESS=0 - - TEST_DEPENDS=numpy==1.14.5 - - - os: osx - language: generic - osx_image: xcode9.4 - env: - - MB_PYTHON_VERSION=3.8 - - ENABLE_CONTRIB=1 - - ENABLE_HEADLESS=0 - - TEST_DEPENDS=numpy==1.17.3 - - - os: osx - language: generic - osx_image: xcode9.4 - env: - - MB_PYTHON_VERSION=3.9 - - ENABLE_CONTRIB=1 - - ENABLE_HEADLESS=0 - - TEST_DEPENDS=numpy==1.19.3 - - - # headless contrib builds for MacOS - - os: osx - language: generic - osx_image: xcode9.4 - env: - - MB_PYTHON_VERSION=3.6 - - ENABLE_CONTRIB=1 - - ENABLE_HEADLESS=1 - - TEST_DEPENDS=numpy==1.13.3 - - - os: osx - language: generic - osx_image: xcode9.4 - env: - - MB_PYTHON_VERSION=3.7 - - ENABLE_CONTRIB=1 - - ENABLE_HEADLESS=1 - - TEST_DEPENDS=numpy==1.14.5 - - - os: osx - language: generic - osx_image: xcode9.4 - env: - - MB_PYTHON_VERSION=3.8 - - ENABLE_CONTRIB=1 - - ENABLE_HEADLESS=1 - - TEST_DEPENDS=numpy==1.17.3 - - - os: osx - language: generic - osx_image: xcode9.4 - env: - - MB_PYTHON_VERSION=3.9 - - ENABLE_CONTRIB=1 - - ENABLE_HEADLESS=1 - - TEST_DEPENDS=numpy==1.19.3 - - # default builds for Linux - - os: linux - language: generic - dist: xenial - services: docker - env: - - MB_PYTHON_VERSION=3.6 - - ENABLE_CONTRIB=0 - - ENABLE_HEADLESS=0 - - TEST_DEPENDS=numpy==1.13.3 - cache: - directories: $HOME/.ccache - - os: linux - language: generic - dist: xenial - services: docker - env: - - MB_PYTHON_VERSION=3.6 - - PLAT=i686 - - ENABLE_CONTRIB=0 - - ENABLE_HEADLESS=0 - - TEST_DEPENDS=numpy==1.13.3 - - USE_CCACHE=0 - cache: - directories: $HOME/.ccache - os: linux arch: arm64 language: generic @@ -268,30 +47,6 @@ jobs: - USE_CCACHE=0 cache: directories: $HOME/.ccache - - os: linux - language: generic - dist: xenial - services: docker - env: - - MB_PYTHON_VERSION=3.7 - - ENABLE_CONTRIB=0 - - ENABLE_HEADLESS=0 - - TEST_DEPENDS=numpy==1.14.5 - cache: - directories: $HOME/.ccache - - os: linux - language: generic - dist: xenial - services: docker - env: - - MB_PYTHON_VERSION=3.7 - - PLAT=i686 - - ENABLE_CONTRIB=0 - - ENABLE_HEADLESS=0 - - TEST_DEPENDS=numpy==1.14.5 - - USE_CCACHE=0 - cache: - directories: $HOME/.ccache - os: linux arch: arm64 language: generic @@ -307,30 +62,6 @@ jobs: - USE_CCACHE=0 cache: directories: $HOME/.ccache - - os: linux - language: generic - dist: xenial - services: docker - env: - - MB_PYTHON_VERSION=3.8 - - ENABLE_CONTRIB=0 - - ENABLE_HEADLESS=0 - - TEST_DEPENDS=numpy==1.17.3 - cache: - directories: $HOME/.ccache - - os: linux - language: generic - dist: xenial - services: docker - env: - - MB_PYTHON_VERSION=3.8 - - PLAT=i686 - - ENABLE_CONTRIB=0 - - ENABLE_HEADLESS=0 - - TEST_DEPENDS=numpy==1.17.3 - - USE_CCACHE=0 - cache: - directories: $HOME/.ccache - os: linux arch: arm64 language: generic @@ -346,30 +77,6 @@ jobs: - USE_CCACHE=0 cache: directories: $HOME/.ccache - - os: linux - language: generic - dist: xenial - services: docker - env: - - MB_PYTHON_VERSION=3.9 - - ENABLE_CONTRIB=0 - - ENABLE_HEADLESS=0 - - TEST_DEPENDS=numpy==1.19.3 - cache: - directories: $HOME/.ccache - - os: linux - language: generic - dist: xenial - services: docker - env: - - MB_PYTHON_VERSION=3.9 - - PLAT=i686 - - ENABLE_CONTRIB=0 - - ENABLE_HEADLESS=0 - - TEST_DEPENDS=numpy==1.19.3 - - USE_CCACHE=0 - cache: - directories: $HOME/.ccache - os: linux arch: arm64 language: generic @@ -387,30 +94,6 @@ jobs: directories: $HOME/.ccache # headless builds for Linux - - os: linux - language: generic - dist: xenial - services: docker - env: - - MB_PYTHON_VERSION=3.6 - - ENABLE_CONTRIB=0 - - ENABLE_HEADLESS=1 - - TEST_DEPENDS=numpy==1.13.3 - cache: - directories: $HOME/.ccache - - os: linux - language: generic - dist: xenial - services: docker - env: - - MB_PYTHON_VERSION=3.6 - - PLAT=i686 - - ENABLE_CONTRIB=0 - - ENABLE_HEADLESS=1 - - TEST_DEPENDS=numpy==1.13.3 - - USE_CCACHE=0 - cache: - directories: $HOME/.ccache - os: linux arch: arm64 language: generic @@ -426,30 +109,6 @@ jobs: - USE_CCACHE=0 cache: directories: $HOME/.ccache - - os: linux - language: generic - dist: xenial - services: docker - env: - - MB_PYTHON_VERSION=3.7 - - ENABLE_CONTRIB=0 - - ENABLE_HEADLESS=1 - - TEST_DEPENDS=numpy==1.14.5 - cache: - directories: $HOME/.ccache - - os: linux - language: generic - dist: xenial - services: docker - env: - - MB_PYTHON_VERSION=3.7 - - PLAT=i686 - - ENABLE_CONTRIB=0 - - ENABLE_HEADLESS=1 - - TEST_DEPENDS=numpy==1.14.5 - - USE_CCACHE=0 - cache: - directories: $HOME/.ccache - os: linux arch: arm64 language: generic @@ -465,30 +124,6 @@ jobs: - USE_CCACHE=0 cache: directories: $HOME/.ccache - - os: linux - language: generic - dist: xenial - services: docker - env: - - MB_PYTHON_VERSION=3.8 - - ENABLE_CONTRIB=0 - - ENABLE_HEADLESS=1 - - TEST_DEPENDS=numpy==1.17.3 - cache: - directories: $HOME/.ccache - - os: linux - language: generic - dist: xenial - services: docker - env: - - MB_PYTHON_VERSION=3.8 - - PLAT=i686 - - ENABLE_CONTRIB=0 - - ENABLE_HEADLESS=1 - - TEST_DEPENDS=numpy==1.17.3 - - USE_CCACHE=0 - cache: - directories: $HOME/.ccache - os: linux arch: arm64 language: generic @@ -504,30 +139,6 @@ jobs: - USE_CCACHE=0 cache: directories: $HOME/.ccache - - os: linux - language: generic - dist: xenial - services: docker - env: - - MB_PYTHON_VERSION=3.9 - - ENABLE_CONTRIB=0 - - ENABLE_HEADLESS=1 - - TEST_DEPENDS=numpy==1.19.3 - cache: - directories: $HOME/.ccache - - os: linux - language: generic - dist: xenial - services: docker - env: - - MB_PYTHON_VERSION=3.9 - - PLAT=i686 - - ENABLE_CONTRIB=0 - - ENABLE_HEADLESS=1 - - TEST_DEPENDS=numpy==1.19.3 - - USE_CCACHE=0 - cache: - directories: $HOME/.ccache - os: linux arch: arm64 language: generic @@ -545,30 +156,6 @@ jobs: directories: $HOME/.ccache # contrib builds for Linux - - os: linux - language: generic - dist: xenial - services: docker - env: - - MB_PYTHON_VERSION=3.6 - - TEST_DEPENDS=numpy==1.13.3 - - ENABLE_CONTRIB=1 - - ENABLE_HEADLESS=0 - cache: - directories: $HOME/.ccache - - os: linux - language: generic - dist: xenial - services: docker - env: - - MB_PYTHON_VERSION=3.6 - - PLAT=i686 - - TEST_DEPENDS=numpy==1.13.3 - - ENABLE_CONTRIB=1 - - ENABLE_HEADLESS=0 - - USE_CCACHE=0 - cache: - directories: $HOME/.ccache - os: linux arch: arm64 language: generic @@ -584,30 +171,6 @@ jobs: - USE_CCACHE=0 cache: directories: $HOME/.ccache - - os: linux - language: generic - dist: xenial - services: docker - env: - - MB_PYTHON_VERSION=3.7 - - TEST_DEPENDS=numpy==1.14.5 - - ENABLE_CONTRIB=1 - - ENABLE_HEADLESS=0 - cache: - directories: $HOME/.ccache - - os: linux - language: generic - dist: xenial - services: docker - env: - - MB_PYTHON_VERSION=3.7 - - PLAT=i686 - - TEST_DEPENDS=numpy==1.14.5 - - ENABLE_CONTRIB=1 - - ENABLE_HEADLESS=0 - - USE_CCACHE=0 - cache: - directories: $HOME/.ccache - os: linux arch: arm64 language: generic @@ -623,30 +186,6 @@ jobs: - USE_CCACHE=0 cache: directories: $HOME/.ccache - - os: linux - language: generic - dist: xenial - services: docker - env: - - MB_PYTHON_VERSION=3.8 - - TEST_DEPENDS=numpy==1.17.3 - - ENABLE_CONTRIB=1 - - ENABLE_HEADLESS=0 - cache: - directories: $HOME/.ccache - - os: linux - language: generic - dist: xenial - services: docker - env: - - MB_PYTHON_VERSION=3.8 - - PLAT=i686 - - TEST_DEPENDS=numpy==1.17.3 - - ENABLE_CONTRIB=1 - - ENABLE_HEADLESS=0 - - USE_CCACHE=0 - cache: - directories: $HOME/.ccache - os: linux arch: arm64 language: generic @@ -662,30 +201,6 @@ jobs: - USE_CCACHE=0 cache: directories: $HOME/.ccache - - os: linux - language: generic - dist: xenial - services: docker - env: - - MB_PYTHON_VERSION=3.9 - - TEST_DEPENDS=numpy==1.19.3 - - ENABLE_CONTRIB=1 - - ENABLE_HEADLESS=0 - cache: - directories: $HOME/.ccache - - os: linux - language: generic - dist: xenial - services: docker - env: - - MB_PYTHON_VERSION=3.9 - - PLAT=i686 - - TEST_DEPENDS=numpy==1.19.3 - - ENABLE_CONTRIB=1 - - ENABLE_HEADLESS=0 - - USE_CCACHE=0 - cache: - directories: $HOME/.ccache - os: linux arch: arm64 language: generic @@ -704,30 +219,6 @@ jobs: # headless contrib builds for Linux - - os: linux - language: generic - dist: xenial - services: docker - env: - - MB_PYTHON_VERSION=3.6 - - TEST_DEPENDS=numpy==1.13.3 - - ENABLE_CONTRIB=1 - - ENABLE_HEADLESS=1 - cache: - directories: $HOME/.ccache - - os: linux - language: generic - dist: xenial - services: docker - env: - - MB_PYTHON_VERSION=3.6 - - PLAT=i686 - - TEST_DEPENDS=numpy==1.13.3 - - ENABLE_CONTRIB=1 - - ENABLE_HEADLESS=1 - - USE_CCACHE=0 - cache: - directories: $HOME/.ccache - os: linux arch: arm64 language: generic @@ -743,30 +234,6 @@ jobs: - USE_CCACHE=0 cache: directories: $HOME/.ccache - - os: linux - language: generic - dist: xenial - services: docker - env: - - MB_PYTHON_VERSION=3.7 - - TEST_DEPENDS=numpy==1.14.5 - - ENABLE_CONTRIB=1 - - ENABLE_HEADLESS=1 - cache: - directories: $HOME/.ccache - - os: linux - language: generic - dist: xenial - services: docker - env: - - MB_PYTHON_VERSION=3.7 - - PLAT=i686 - - TEST_DEPENDS=numpy==1.14.5 - - ENABLE_CONTRIB=1 - - ENABLE_HEADLESS=1 - - USE_CCACHE=0 - cache: - directories: $HOME/.ccache - os: linux arch: arm64 language: generic @@ -782,30 +249,6 @@ jobs: - USE_CCACHE=0 cache: directories: $HOME/.ccache - - os: linux - language: generic - dist: xenial - services: docker - env: - - MB_PYTHON_VERSION=3.8 - - TEST_DEPENDS=numpy==1.17.3 - - ENABLE_CONTRIB=1 - - ENABLE_HEADLESS=1 - cache: - directories: $HOME/.ccache - - os: linux - language: generic - dist: xenial - services: docker - env: - - MB_PYTHON_VERSION=3.8 - - PLAT=i686 - - TEST_DEPENDS=numpy==1.17.3 - - ENABLE_CONTRIB=1 - - ENABLE_HEADLESS=1 - - USE_CCACHE=0 - cache: - directories: $HOME/.ccache - os: linux arch: arm64 language: generic @@ -821,30 +264,6 @@ jobs: - USE_CCACHE=0 cache: directories: $HOME/.ccache - - os: linux - language: generic - dist: xenial - services: docker - env: - - MB_PYTHON_VERSION=3.9 - - TEST_DEPENDS=numpy==1.19.3 - - ENABLE_CONTRIB=1 - - ENABLE_HEADLESS=1 - cache: - directories: $HOME/.ccache - - os: linux - language: generic - dist: xenial - services: docker - env: - - MB_PYTHON_VERSION=3.9 - - PLAT=i686 - - TEST_DEPENDS=numpy==1.19.3 - - ENABLE_CONTRIB=1 - - ENABLE_HEADLESS=1 - - USE_CCACHE=0 - cache: - directories: $HOME/.ccache - os: linux arch: arm64 language: generic @@ -969,23 +388,23 @@ after_success: | fi - # Save to Azure storage always + # # Save to Azure storage always - if [[ $TRAVIS_CPU_ARCH != "arm64" ]]; then - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then - curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash - else - brew install azure-cli - fi + # if [[ $TRAVIS_CPU_ARCH != "arm64" ]]; then + # if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then + # curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + # else + # brew install azure-cli + # fi - az storage container create -n ${TRAVIS_COMMIT} --public-access blob + # az storage container create -n ${TRAVIS_COMMIT} --public-access blob - if [[ $SDIST == 1 ]]; then - az storage blob upload-batch -d ${TRAVIS_COMMIT} -s ${TRAVIS_BUILD_DIR}/dist --pattern *.gz - else - az storage blob upload-batch -d ${TRAVIS_COMMIT} -s ${TRAVIS_BUILD_DIR}/wheelhouse --pattern opencv*.whl - fi - fi + # if [[ $SDIST == 1 ]]; then + # az storage blob upload-batch -d ${TRAVIS_COMMIT} -s ${TRAVIS_BUILD_DIR}/dist --pattern *.gz + # else + # az storage blob upload-batch -d ${TRAVIS_COMMIT} -s ${TRAVIS_BUILD_DIR}/wheelhouse --pattern opencv*.whl + # fi + # fi set +x From 176a99b5d5c3f1febd9db937783f7109e3d4e5b9 Mon Sep 17 00:00:00 2001 From: Grigory Serebryakov Date: Fri, 23 Apr 2021 01:03:53 +0300 Subject: [PATCH 35/41] Split linux and OSX builds to exclude x86 OSX builds from build matrix --- .github/workflows/build_wheels.yml | 118 ++++++++++++++++++++++++++++- 1 file changed, 115 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 59db62cc..1842abf4 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -82,7 +82,8 @@ jobs: name: wheels path: wheelhouse/opencv*.whl - build: + + build_linux: runs-on: ${{ matrix.os }} defaults: run: @@ -91,7 +92,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest] python-version: [3.6, 3.7, 3.8, 3.9] platform: [x86, x64] with_contrib: [0, 1] @@ -193,6 +194,117 @@ jobs: name: wheels path: wheelhouse/opencv*.whl + build_macos: + runs-on: ${{ matrix.os }} + defaults: + run: + shell: bash + + strategy: + fail-fast: false + matrix: + os: [macos-latest] + python-version: [3.6, 3.7, 3.8, 3.9] + platform: [x64] + with_contrib: [0, 1] + without_gui: [0, 1] + build_sdist: [0] + + env: + ACTIONS_ALLOW_UNSECURE_COMMANDS: true + REPO_DIR: . + BUILD_COMMIT: master + PROJECT_SPEC: opencv-python + MB_PYTHON_VERSION: ${{ matrix.python-version }} + TRAVIS_PYTHON_VERSION: ${{ matrix.python-version }} + MB_ML_VER: 2014 + NP_TEST_DEP: numpy + TRAVIS_BUILD_DIR: ${{ github.workspace }} + CONFIG_PATH: travis_config.sh + DOCKER_IMAGE: quay.io/skvark/manylinux2014_${PLAT} + USE_CCACHE: 1 + UNICODE_WIDTH: 32 + SDIST: ${{ matrix.build_sdist || 0}} + ENABLE_HEADLESS: ${{ matrix.without_gui }} + ENABLE_CONTRIB: ${{ matrix.with_contrib }} + + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + submodules: true + fetch-depth: 0 + + - name: Update submodules + run: | + git submodule update --remote + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + architecture: ${{ matrix.platform }} + + - name: Setup Environment variables + run: | + if [ "macos-latest" == "${{ matrix.os }}" ]; then echo "TRAVIS_OS_NAME=osx" >> $GITHUB_ENV; else echo "TRAVIS_OS_NAME=${{ matrix.os }}" >> $GITHUB_ENV; fi + if [ "schedule" == "${{ github.event_name }}" ]; then echo "TRAVIS_EVENT_TYPE=cron" >> $GITHUB_ENV; else echo "TRAVIS_EVENT_TYPE=${{ github.event_name }}" >> $GITHUB_ENV; fi + if [ "schedule" == "${{ github.event_name }}" ]; then echo "BUILD_COMMIT=master" >> $GITHUB_ENV; else echo "BUILD_COMMIT=$BUILD_COMMIT" >> $GITHUB_ENV; fi + if [ "x64" == "${{ matrix.platform }}" ]; then echo "PLAT=x86_64" >> $GITHUB_ENV; fi + if [ "x86" == "${{ matrix.platform }}" ]; then echo "PLAT=i686" >> $GITHUB_ENV; fi + echo "BUILD_DEPENDS=$(echo $NP_BUILD_DEP)" >> $GITHUB_ENV; + echo "TEST_DEPENDS=$(echo $NP_TEST_DEP)" >> $GITHUB_ENV; + + - name: before install + run: | + set -e + if [[ $SDIST == 0 ]]; then + # Check out and prepare the source + # Multibuild doesn't have releases, so --depth would break eventually (see + # https://superuser.com/questions/1240216/server-does-not-allow-request-for-unadvertised) + git submodule update --init multibuild + source multibuild/common_utils.sh + # https://github.com/matthew-brett/multibuild/issues/116 + if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export ARCH_FLAGS=" "; fi + source multibuild/travis_steps.sh + # This sets -x + # source travis_multibuild_customize.sh + echo $ENABLE_CONTRIB > contrib.enabled + echo $ENABLE_HEADLESS > headless.enabled + echo "end" + # Not interested in travis internal scripts' output + fi + set +x + # Build and package + set -x + ls + if [[ $SDIST == 1 ]]; then + python -m pip install --upgrade pip + python -m pip install scikit-build + python setup.py sdist + else + build_wheel $REPO_DIR $PLAT + fi + set +x + # Install and run tests + set -x + if [[ $SDIST == 1 ]]; then + echo "skipping tests because of sdist" + rc=0 + else + install_run $PLAT && rc=$? || rc=$? + fi + set +x + #otherwise, Travis logic terminates prematurely + #https://travis-ci.community/t/shell-session-update-command-not-found-in-build-log-causes-build-to-fail-if-trap-err-is-set/817 + trap ERR + test "$rc" -eq 0 + - name: saving artifacts + uses: actions/upload-artifact@v2 + with: + name: wheels + path: wheelhouse/opencv*.whl + build_sdist: runs-on: ${{ matrix.os }} @@ -275,7 +387,7 @@ jobs: release: if: startsWith(github.ref, 'refs/tags/v') - needs: [build, build-windows-x86_64, build_sdist] + needs: [build_linux, build_macos, build-windows-x86_64, build_sdist] runs-on: ubuntu-latest defaults: run: From aceb2a3f732a4fcdf881ce2a92354c0862567803 Mon Sep 17 00:00:00 2001 From: Grigory Serebryakov Date: Fri, 23 Apr 2021 01:18:37 +0300 Subject: [PATCH 36/41] Reduce architectures to x64 for both OSX and Ubuntu as x86 is supported for windows only --- .github/workflows/build_wheels.yml | 118 +---------------------------- 1 file changed, 3 insertions(+), 115 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 1842abf4..c5eb58a4 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -82,119 +82,7 @@ jobs: name: wheels path: wheelhouse/opencv*.whl - - build_linux: - runs-on: ${{ matrix.os }} - defaults: - run: - shell: bash - - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest] - python-version: [3.6, 3.7, 3.8, 3.9] - platform: [x86, x64] - with_contrib: [0, 1] - without_gui: [0, 1] - build_sdist: [0] - - env: - ACTIONS_ALLOW_UNSECURE_COMMANDS: true - REPO_DIR: . - BUILD_COMMIT: master - PROJECT_SPEC: opencv-python - MB_PYTHON_VERSION: ${{ matrix.python-version }} - TRAVIS_PYTHON_VERSION: ${{ matrix.python-version }} - MB_ML_VER: 2014 - NP_TEST_DEP: numpy - TRAVIS_BUILD_DIR: ${{ github.workspace }} - CONFIG_PATH: travis_config.sh - DOCKER_IMAGE: quay.io/skvark/manylinux2014_${PLAT} - USE_CCACHE: 1 - UNICODE_WIDTH: 32 - SDIST: ${{ matrix.build_sdist || 0}} - ENABLE_HEADLESS: ${{ matrix.without_gui }} - ENABLE_CONTRIB: ${{ matrix.with_contrib }} - - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - submodules: true - fetch-depth: 0 - - - name: Update submodules - run: | - git submodule update --remote - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - architecture: ${{ matrix.platform }} - - - name: Setup Environment variables - run: | - if [ "macos-latest" == "${{ matrix.os }}" ]; then echo "TRAVIS_OS_NAME=osx" >> $GITHUB_ENV; else echo "TRAVIS_OS_NAME=${{ matrix.os }}" >> $GITHUB_ENV; fi - if [ "schedule" == "${{ github.event_name }}" ]; then echo "TRAVIS_EVENT_TYPE=cron" >> $GITHUB_ENV; else echo "TRAVIS_EVENT_TYPE=${{ github.event_name }}" >> $GITHUB_ENV; fi - if [ "schedule" == "${{ github.event_name }}" ]; then echo "BUILD_COMMIT=master" >> $GITHUB_ENV; else echo "BUILD_COMMIT=$BUILD_COMMIT" >> $GITHUB_ENV; fi - if [ "x64" == "${{ matrix.platform }}" ]; then echo "PLAT=x86_64" >> $GITHUB_ENV; fi - if [ "x86" == "${{ matrix.platform }}" ]; then echo "PLAT=i686" >> $GITHUB_ENV; fi - echo "BUILD_DEPENDS=$(echo $NP_BUILD_DEP)" >> $GITHUB_ENV; - echo "TEST_DEPENDS=$(echo $NP_TEST_DEP)" >> $GITHUB_ENV; - - - name: before install - run: | - set -e - if [[ $SDIST == 0 ]]; then - # Check out and prepare the source - # Multibuild doesn't have releases, so --depth would break eventually (see - # https://superuser.com/questions/1240216/server-does-not-allow-request-for-unadvertised) - git submodule update --init multibuild - source multibuild/common_utils.sh - # https://github.com/matthew-brett/multibuild/issues/116 - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export ARCH_FLAGS=" "; fi - source multibuild/travis_steps.sh - # This sets -x - # source travis_multibuild_customize.sh - echo $ENABLE_CONTRIB > contrib.enabled - echo $ENABLE_HEADLESS > headless.enabled - echo "end" - # Not interested in travis internal scripts' output - fi - set +x - # Build and package - set -x - ls - if [[ $SDIST == 1 ]]; then - python -m pip install --upgrade pip - python -m pip install scikit-build - python setup.py sdist - else - build_wheel $REPO_DIR $PLAT - fi - set +x - # Install and run tests - set -x - if [[ $SDIST == 1 ]]; then - echo "skipping tests because of sdist" - rc=0 - else - install_run $PLAT && rc=$? || rc=$? - fi - set +x - #otherwise, Travis logic terminates prematurely - #https://travis-ci.community/t/shell-session-update-command-not-found-in-build-log-causes-build-to-fail-if-trap-err-is-set/817 - trap ERR - test "$rc" -eq 0 - - name: saving artifacts - uses: actions/upload-artifact@v2 - with: - name: wheels - path: wheelhouse/opencv*.whl - - build_macos: + build: runs-on: ${{ matrix.os }} defaults: run: @@ -203,7 +91,7 @@ jobs: strategy: fail-fast: false matrix: - os: [macos-latest] + os: [ubuntu-latest, macos-latest] python-version: [3.6, 3.7, 3.8, 3.9] platform: [x64] with_contrib: [0, 1] @@ -387,7 +275,7 @@ jobs: release: if: startsWith(github.ref, 'refs/tags/v') - needs: [build_linux, build_macos, build-windows-x86_64, build_sdist] + needs: [build, build-windows-x86_64, build_sdist] runs-on: ubuntu-latest defaults: run: From d376197b3d0ba5a5f2691f28c461996efcb7b7a5 Mon Sep 17 00:00:00 2001 From: Grigory Serebryakov Date: Fri, 23 Apr 2021 23:49:14 +0300 Subject: [PATCH 37/41] Add PyPI secrets usage --- .github/workflows/build_wheels.yml | 80 ++++++++++++++++++++++++++++-- 1 file changed, 75 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index c5eb58a4..9306bff9 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -207,7 +207,7 @@ jobs: python-version: [3.8] platform: [x64] with_contrib: [0, 1] - without_gui: [0] + without_gui: [0, 1] build_sdist: [1] env: @@ -273,10 +273,80 @@ jobs: path: dist/opencv*.tar.gz - release: + release_opencv_python: + if: startsWith(github.ref, 'refs/tags/v') + needs: [build, build-windows-x86_64, build_sdist] + runs-on: ubuntu-latest + environment: opencv-python-release + defaults: + run: + shell: bash + steps: + - uses: actions/download-artifact@v2 + with: + name: wheels + path: wheelhouse/ + + - name: Upload wheels ${{ matrix.os }} + env: + # PYPI repository credentials + TWINE_USERNAME: ${{ secrets.OPENCV_PYTHON_USERNAME }} + TWINE_PASSWORD: ${{ secrets.OPENCV_PYTHON_PASSWORD }} + run: | + python -m pip install twine + python -m twine upload -u ${Env:TWINE_USERNAME} -p ${Env:TWINE_PASSWORD} --skip-existing wheelhouse/opencv_python-* + + release_opencv_contrib_python: + if: startsWith(github.ref, 'refs/tags/v') + needs: [build, build-windows-x86_64, build_sdist] + runs-on: ubuntu-latest + environment: opencv-python-release + defaults: + run: + shell: bash + steps: + - uses: actions/download-artifact@v2 + with: + name: wheels + path: wheelhouse/ + + - name: Upload wheels ${{ matrix.os }} + env: + # PYPI repository credentials + TWINE_USERNAME: ${{ secrets.OPENCV_CONTRIB_PYTHON_USERNAME }} + TWINE_PASSWORD: ${{ secrets.OPENCV_CONTRIB_PYTHON_PASSWORD }} + run: | + python -m pip install twine + python -m twine upload -u ${Env:TWINE_USERNAME} -p ${Env:TWINE_PASSWORD} --skip-existing wheelhouse/opencv_contrib_python-* + + release_opencv_python_headless: + if: startsWith(github.ref, 'refs/tags/v') + needs: [build, build-windows-x86_64, build_sdist] + runs-on: ubuntu-latest + environment: opencv-python-release + defaults: + run: + shell: bash + steps: + - uses: actions/download-artifact@v2 + with: + name: wheels + path: wheelhouse/ + + - name: Upload wheels ${{ matrix.os }} + env: + # PYPI repository credentials + TWINE_USERNAME: ${{ secrets.OPENCV_PYTHON_HEADLESS_USERNAME }} + TWINE_PASSWORD: ${{ secrets.OPENCV_PYTHON_HEADLESS_PASSWORD }} + run: | + python -m pip install twine + python -m twine upload -u ${Env:TWINE_USERNAME} -p ${Env:TWINE_PASSWORD} --skip-existing wheelhouse/opencv_python_headless-* + + release_opencv_contrib_python_headless: if: startsWith(github.ref, 'refs/tags/v') needs: [build, build-windows-x86_64, build_sdist] runs-on: ubuntu-latest + environment: opencv-python-release defaults: run: shell: bash @@ -289,8 +359,8 @@ jobs: - name: Upload wheels ${{ matrix.os }} env: # PYPI repository credentials - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + TWINE_USERNAME: ${{ secrets.OPENCV_CONTRIB_PYTHON_HEADLESS_USERNAME }} + TWINE_PASSWORD: ${{ secrets.OPENCV_CONTRIB_PYTHON_HEADLESS_PASSWORD }} run: | python -m pip install twine - python -m twine upload -u ${Env:TWINE_USERNAME} -p ${Env:TWINE_PASSWORD} --skip-existing wheelhouse/opencv* + python -m twine upload -u ${Env:TWINE_USERNAME} -p ${Env:TWINE_PASSWORD} --skip-existing wheelhouse/opencv_contrib_python_headless-* \ No newline at end of file From f316fb8947c8441c56f15c53701ede40235d8443 Mon Sep 17 00:00:00 2001 From: Grigory Serebryakov Date: Wed, 28 Apr 2021 22:04:47 +0300 Subject: [PATCH 38/41] Publish packages built in PRs to testPyPI --- .github/workflows/build_wheels.yml | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 9306bff9..959e02b0 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -3,7 +3,7 @@ name: Build PYPI wheels for opencv-python on: push: branches: - - all + - master pull_request: branches: - master @@ -272,6 +272,29 @@ jobs: name: wheels path: dist/opencv*.tar.gz + test_release_opencv_python_all: + if: ${{github.event.pull_request.types}} + needs: [build, build-windows-x86_64, build_sdist] + runs-on: ubuntu-latest + environment: test-opencv-python-release + defaults: + run: + shell: bash + steps: + - uses: actions/download-artifact@v2 + with: + name: wheels + path: wheelhouse/ + + - name: Upload wheels ${{ matrix.os }} + env: + # PYPI repository credentials + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + python -m pip install twine + python -m twine upload --repository testpypi -u ${Env:TWINE_USERNAME} -p ${Env:TWINE_PASSWORD} --skip-existing wheelhouse/opencv_* + release_opencv_python: if: startsWith(github.ref, 'refs/tags/v') From 7483e1bfc4c050e7acb470267707dec1b24b7b5d Mon Sep 17 00:00:00 2001 From: Grigory Serebryakov Date: Thu, 29 Apr 2021 11:13:19 +0300 Subject: [PATCH 39/41] Push to test PyPI on each package build. Remove unneded build on push to master --- .github/workflows/build_wheels.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 959e02b0..dcefff53 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -1,9 +1,6 @@ name: Build PYPI wheels for opencv-python on: - push: - branches: - - master pull_request: branches: - master @@ -273,7 +270,6 @@ jobs: path: dist/opencv*.tar.gz test_release_opencv_python_all: - if: ${{github.event.pull_request.types}} needs: [build, build-windows-x86_64, build_sdist] runs-on: ubuntu-latest environment: test-opencv-python-release From 377e3cf5595409a7e11c73c69fffed4ae9c70379 Mon Sep 17 00:00:00 2001 From: Grigory Serebryakov Date: Thu, 29 Apr 2021 21:17:54 +0300 Subject: [PATCH 40/41] Apply review comments --- .github/workflows/build_wheels.yml | 59 ++++++++++-------------------- 1 file changed, 19 insertions(+), 40 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index dcefff53..ea6732dc 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -62,7 +62,6 @@ jobs: - name: before test run: | cd ${{ github.workspace }}/tests - $env:PYTHONWARNINGS = "ignore:::pip._internal.cli.base_command" &python -m pip install --user --no-warn-script-location (ls "../wheelhouse/opencv*.whl") if ($LastExitCode -ne 0) {throw $LastExitCode} shell: powershell @@ -126,6 +125,7 @@ jobs: - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 + if: ${{ 'macos-latest' == matrix.os }} with: python-version: ${{ matrix.python-version }} architecture: ${{ matrix.platform }} @@ -143,47 +143,22 @@ jobs: - name: before install run: | set -e - if [[ $SDIST == 0 ]]; then - # Check out and prepare the source - # Multibuild doesn't have releases, so --depth would break eventually (see - # https://superuser.com/questions/1240216/server-does-not-allow-request-for-unadvertised) - git submodule update --init multibuild - source multibuild/common_utils.sh - # https://github.com/matthew-brett/multibuild/issues/116 - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export ARCH_FLAGS=" "; fi - source multibuild/travis_steps.sh - # This sets -x - # source travis_multibuild_customize.sh - echo $ENABLE_CONTRIB > contrib.enabled - echo $ENABLE_HEADLESS > headless.enabled - echo "end" - # Not interested in travis internal scripts' output - fi - set +x - # Build and package - set -x - ls - if [[ $SDIST == 1 ]]; then - python -m pip install --upgrade pip - python -m pip install scikit-build - python setup.py sdist - else - build_wheel $REPO_DIR $PLAT - fi - set +x - # Install and run tests + # Check out and prepare the source + # Multibuild doesn't have releases, so --depth would break eventually (see + # https://superuser.com/questions/1240216/server-does-not-allow-request-for-unadvertised) + git submodule update --init multibuild + source multibuild/common_utils.sh + # https://github.com/matthew-brett/multibuild/issues/116 + if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export ARCH_FLAGS=" "; fi + source multibuild/travis_steps.sh + # This sets -x + # source travis_multibuild_customize.sh + echo $ENABLE_CONTRIB > contrib.enabled + echo $ENABLE_HEADLESS > headless.enabled set -x - if [[ $SDIST == 1 ]]; then - echo "skipping tests because of sdist" - rc=0 - else - install_run $PLAT && rc=$? || rc=$? - fi + build_wheel $REPO_DIR $PLAT + install_run $PLAT set +x - #otherwise, Travis logic terminates prematurely - #https://travis-ci.community/t/shell-session-update-command-not-found-in-build-log-causes-build-to-fail-if-trap-err-is-set/817 - trap ERR - test "$rc" -eq 0 - name: saving artifacts uses: actions/upload-artifact@v2 with: @@ -287,8 +262,12 @@ jobs: # PYPI repository credentials TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + TEST_ENV_VAR: test run: | python -m pip install twine + echo ${{ secrets.OPENCV_PYTHON_USERNAME }} + echo ${Env:TWINE_USERNAME} + echo ${Env:TEST_ENV_VAR} python -m twine upload --repository testpypi -u ${Env:TWINE_USERNAME} -p ${Env:TWINE_PASSWORD} --skip-existing wheelhouse/opencv_* From 6945db0b530aa9cbc729a50207cf9f928d1ea5fa Mon Sep 17 00:00:00 2001 From: Grigory Serebryakov Date: Fri, 30 Apr 2021 20:09:32 +0300 Subject: [PATCH 41/41] Combine all release steps into one job --- .github/workflows/build_wheels.yml | 101 +++-------------------------- 1 file changed, 8 insertions(+), 93 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index ea6732dc..cbed8b6c 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -244,33 +244,6 @@ jobs: name: wheels path: dist/opencv*.tar.gz - test_release_opencv_python_all: - needs: [build, build-windows-x86_64, build_sdist] - runs-on: ubuntu-latest - environment: test-opencv-python-release - defaults: - run: - shell: bash - steps: - - uses: actions/download-artifact@v2 - with: - name: wheels - path: wheelhouse/ - - - name: Upload wheels ${{ matrix.os }} - env: - # PYPI repository credentials - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - TEST_ENV_VAR: test - run: | - python -m pip install twine - echo ${{ secrets.OPENCV_PYTHON_USERNAME }} - echo ${Env:TWINE_USERNAME} - echo ${Env:TEST_ENV_VAR} - python -m twine upload --repository testpypi -u ${Env:TWINE_USERNAME} -p ${Env:TWINE_PASSWORD} --skip-existing wheelhouse/opencv_* - - release_opencv_python: if: startsWith(github.ref, 'refs/tags/v') needs: [build, build-windows-x86_64, build_sdist] @@ -284,81 +257,23 @@ jobs: with: name: wheels path: wheelhouse/ - - - name: Upload wheels ${{ matrix.os }} - env: - # PYPI repository credentials - TWINE_USERNAME: ${{ secrets.OPENCV_PYTHON_USERNAME }} - TWINE_PASSWORD: ${{ secrets.OPENCV_PYTHON_PASSWORD }} + - name: Upload wheels for opencv_python run: | python -m pip install twine - python -m twine upload -u ${Env:TWINE_USERNAME} -p ${Env:TWINE_PASSWORD} --skip-existing wheelhouse/opencv_python-* - - release_opencv_contrib_python: - if: startsWith(github.ref, 'refs/tags/v') - needs: [build, build-windows-x86_64, build_sdist] - runs-on: ubuntu-latest - environment: opencv-python-release - defaults: - run: - shell: bash - steps: - - uses: actions/download-artifact@v2 - with: - name: wheels - path: wheelhouse/ + python -m twine upload -u ${{ secrets.OPENCV_PYTHON_USERNAME }} -p ${{ secrets.OPENCV_PYTHON_PASSWORD }} --skip-existing wheelhouse/opencv_python-* - - name: Upload wheels ${{ matrix.os }} - env: - # PYPI repository credentials - TWINE_USERNAME: ${{ secrets.OPENCV_CONTRIB_PYTHON_USERNAME }} - TWINE_PASSWORD: ${{ secrets.OPENCV_CONTRIB_PYTHON_PASSWORD }} + - name: Upload wheels for opencv_contrib_python run: | python -m pip install twine - python -m twine upload -u ${Env:TWINE_USERNAME} -p ${Env:TWINE_PASSWORD} --skip-existing wheelhouse/opencv_contrib_python-* + python -m twine upload -u ${{ secrets.OPENCV_CONTRIB_PYTHON_USERNAME }} -p ${{ secrets.OPENCV_CONTRIB_PYTHON_PASSWORD }} --skip-existing wheelhouse/opencv_contrib_python-* - release_opencv_python_headless: - if: startsWith(github.ref, 'refs/tags/v') - needs: [build, build-windows-x86_64, build_sdist] - runs-on: ubuntu-latest - environment: opencv-python-release - defaults: - run: - shell: bash - steps: - - uses: actions/download-artifact@v2 - with: - name: wheels - path: wheelhouse/ - - - name: Upload wheels ${{ matrix.os }} - env: - # PYPI repository credentials - TWINE_USERNAME: ${{ secrets.OPENCV_PYTHON_HEADLESS_USERNAME }} - TWINE_PASSWORD: ${{ secrets.OPENCV_PYTHON_HEADLESS_PASSWORD }} + - name: Upload wheels for opencv_python_headless run: | python -m pip install twine - python -m twine upload -u ${Env:TWINE_USERNAME} -p ${Env:TWINE_PASSWORD} --skip-existing wheelhouse/opencv_python_headless-* + python -m twine upload -u ${{ secrets.OPENCV_PYTHON_HEADLESS_USERNAME }} -p ${{ secrets.OPENCV_PYTHON_HEADLESS_PASSWORD }} --skip-existing wheelhouse/opencv_python_headless-* - release_opencv_contrib_python_headless: - if: startsWith(github.ref, 'refs/tags/v') - needs: [build, build-windows-x86_64, build_sdist] - runs-on: ubuntu-latest - environment: opencv-python-release - defaults: - run: - shell: bash - steps: - - uses: actions/download-artifact@v2 - with: - name: wheels - path: wheelhouse/ + - name: Upload wheels for opencv_contrib_python_headless - - name: Upload wheels ${{ matrix.os }} - env: - # PYPI repository credentials - TWINE_USERNAME: ${{ secrets.OPENCV_CONTRIB_PYTHON_HEADLESS_USERNAME }} - TWINE_PASSWORD: ${{ secrets.OPENCV_CONTRIB_PYTHON_HEADLESS_PASSWORD }} run: | python -m pip install twine - python -m twine upload -u ${Env:TWINE_USERNAME} -p ${Env:TWINE_PASSWORD} --skip-existing wheelhouse/opencv_contrib_python_headless-* \ No newline at end of file + python -m twine upload -u ${{ secrets.OPENCV_CONTRIB_PYTHON_HEADLESS_USERNAME }} -p ${{ secrets.OPENCV_CONTRIB_PYTHON_HEADLESS_PASSWORD }} --skip-existing wheelhouse/opencv_contrib_python_headless-*