From 488d6f1d9d4855006180f4377c8b4897073fbbcf Mon Sep 17 00:00:00 2001 From: Matteo Pologruto Date: Mon, 19 Sep 2022 16:01:53 +0200 Subject: [PATCH 1/9] Use a matrix create parallel builds for each os Using a matrix to run each build task greatly improves performances, since they can all start concurrently. The finishing time of the job will be equal to the one of the longer task to build, instead of being the sum of each individual task's finishing time. --- .github/workflows/publish-go-nightly-task.yml | 15 ++++++++++++++- .github/workflows/publish-go-tester-task.yml | 15 ++++++++++++++- .github/workflows/release-go-task.yml | 15 ++++++++++++++- DistTasks.yml | 13 ------------- 4 files changed, 42 insertions(+), 16 deletions(-) diff --git a/.github/workflows/publish-go-nightly-task.yml b/.github/workflows/publish-go-nightly-task.yml index 897e4e9fea9..d69b0446960 100644 --- a/.github/workflows/publish-go-nightly-task.yml +++ b/.github/workflows/publish-go-nightly-task.yml @@ -22,6 +22,19 @@ jobs: create-nightly-artifacts: runs-on: ubuntu-latest + strategy: + matrix: + os: + - Windows_32bit + - Windows_64bit + - Linux_32bit + - Linux_64bit + - Linux_ARMv6 + - Linux_ARMv7 + - Linux_ARM64 + - macOS_64bit + - macOS_ARM64 + steps: - name: Checkout repository uses: actions/checkout@v3 @@ -35,7 +48,7 @@ jobs: - name: Build env: NIGHTLY: true - run: task dist:all + run: task dist:${{ matrix.os }} - name: Upload artifacts uses: actions/upload-artifact@v3 diff --git a/.github/workflows/publish-go-tester-task.yml b/.github/workflows/publish-go-tester-task.yml index 12f0a768686..7376e1d31a7 100644 --- a/.github/workflows/publish-go-tester-task.yml +++ b/.github/workflows/publish-go-tester-task.yml @@ -57,6 +57,19 @@ jobs: if: needs.run-determination.outputs.result == 'true' runs-on: ubuntu-latest + strategy: + matrix: + os: + - Windows_32bit + - Windows_64bit + - Linux_32bit + - Linux_64bit + - Linux_ARMv6 + - Linux_ARMv7 + - Linux_ARM64 + - macOS_64bit + - macOS_ARM64 + steps: - name: Checkout repository uses: actions/checkout@v3 @@ -77,7 +90,7 @@ jobs: fi PACKAGE_NAME_PREFIX="$PACKAGE_NAME_PREFIX-${{ github.sha }}-" export PACKAGE_NAME_PREFIX - task dist:all + task dist:${{ matrix.os }} # Transfer builds to artifacts job - name: Upload combined builds artifact diff --git a/.github/workflows/release-go-task.yml b/.github/workflows/release-go-task.yml index ffda4895ef5..364a364b246 100644 --- a/.github/workflows/release-go-task.yml +++ b/.github/workflows/release-go-task.yml @@ -19,6 +19,19 @@ jobs: create-release-artifacts: runs-on: ubuntu-latest + strategy: + matrix: + os: + - Windows_32bit + - Windows_64bit + - Linux_32bit + - Linux_64bit + - Linux_ARMv6 + - Linux_ARMv7 + - Linux_ARM64 + - macOS_64bit + - macOS_ARM64 + steps: - name: Checkout repository uses: actions/checkout@v3 @@ -40,7 +53,7 @@ jobs: version: 3.x - name: Build - run: task dist:all + run: task dist:${{ matrix.os }} - name: Upload artifacts uses: actions/upload-artifact@v3 diff --git a/DistTasks.yml b/DistTasks.yml index 86476672ef0..5a354cd991d 100644 --- a/DistTasks.yml +++ b/DistTasks.yml @@ -23,19 +23,6 @@ vars: CHECKSUM_FILE: "{{.VERSION}}-checksums.txt" tasks: - all: - desc: Build for distribution for all platforms - cmds: - - task: Windows_32bit - - task: Windows_64bit - - task: Linux_32bit - - task: Linux_64bit - - task: Linux_ARMv6 - - task: Linux_ARMv7 - - task: Linux_ARM64 - - task: macOS_64bit - - task: macOS_ARM64 - Windows_32bit: desc: Builds Windows 32 bit binaries dir: "{{.DIST_DIR}}" From dfbf112a2819fe2dfaf3a629ecd8c5670d69dee8 Mon Sep 17 00:00:00 2001 From: Matteo Pologruto Date: Wed, 21 Sep 2022 13:08:23 +0200 Subject: [PATCH 2/9] Calculate checksums during release creation Checksums of the output files where previously calculated during the initial creation of the artifacts, during the notarization process and, finally, at the release creation's step. The whole process has been simplified and checksums are now computed only during the creation of the release. --- .github/workflows/publish-go-nightly-task.yml | 24 ++++++------------- .github/workflows/publish-go-tester-task.yml | 15 ++++++++++++ .github/workflows/release-go-task.yml | 24 ++++++------------- DistTasks.yml | 10 -------- 4 files changed, 29 insertions(+), 44 deletions(-) diff --git a/.github/workflows/publish-go-nightly-task.yml b/.github/workflows/publish-go-nightly-task.yml index d69b0446960..b15eb023476 100644 --- a/.github/workflows/publish-go-nightly-task.yml +++ b/.github/workflows/publish-go-nightly-task.yml @@ -140,14 +140,10 @@ jobs: run: | gon "${{ env.GON_CONFIG_PATH }}" - - name: Re-package binary and output checksum + - name: Re-package binary id: re-package working-directory: ${{ env.DIST_DIR }} - # This step performs the following: - # 1. Repackage the signed binary replaced in place by Gon (ignoring the output zip file) - # 2. Recalculate package checksum - # 3. Output the new checksum to include in the nnnnnn-checksums.txt file - # (it cannot be done there because of workflow job parallelization) + # Repackage the signed binary replaced in place by Gon (ignoring the output zip file) run: | # GitHub's upload/download-artifact@v2 actions don't preserve file permissions, # so we need to add execution permission back until the action is made to do this. @@ -157,11 +153,9 @@ jobs: tar -czvf "$PACKAGE_FILENAME" \ -C "${{ env.PROJECT_NAME }}_osx_${{ matrix.artifact.name }}/" "${{ env.PROJECT_NAME }}" \ -C ../../ LICENSE.txt - CHECKSUM_LINE="$(shasum -a 256 $PACKAGE_FILENAME)" echo "PACKAGE_FILENAME=$PACKAGE_FILENAME" >> $GITHUB_ENV - echo "::set-output name=checksum-${{ matrix.artifact.name }}::$CHECKSUM_LINE" - - name: Upload artifacts + - name: Upload artifact uses: actions/upload-artifact@v3 with: if-no-files-found: error @@ -179,15 +173,11 @@ jobs: name: ${{ env.ARTIFACT_NAME }} path: ${{ env.DIST_DIR }} - - name: Update checksum + - name: Output checksum + working-directory: ${{ env.DIST_DIR}} run: | - declare -a checksum_lines=("${{ needs.notarize-macos.outputs.checksum-darwin_amd64 }}" "${{ needs.notarize-macos.outputs.checksum-darwin_arm64 }}") - for checksum_line in "${checksum_lines[@]}" - do - CHECKSUM=$(echo ${checksum_line} | cut -d " " -f 1) - PACKAGE_FILENAME=$(echo ${checksum_line} | cut -d " " -f 2) - perl -pi -w -e "s/.*${PACKAGE_FILENAME}/${CHECKSUM} ${PACKAGE_FILENAME}/g;" ${{ env.DIST_DIR }}/*-checksums.txt - done + TAG="nightly-$(date -u +"%Y%m%d")" + sha256sum ${{ env.PROJECT_NAME }}_${TAG}* >> ${TAG}-checksums.txt - name: Upload release files on Arduino downloads servers uses: docker://plugins/s3 diff --git a/.github/workflows/publish-go-tester-task.yml b/.github/workflows/publish-go-tester-task.yml index 7376e1d31a7..e29702bbbd3 100644 --- a/.github/workflows/publish-go-tester-task.yml +++ b/.github/workflows/publish-go-tester-task.yml @@ -24,6 +24,8 @@ on: repository_dispatch: env: + # As defined by the Taskfile's PROJECT_NAME variable + PROJECT_NAME: arduino-cli # As defined by the Taskfile's DIST_DIR variable DIST_DIR: dist BUILDS_ARTIFACT: build-artifacts @@ -135,6 +137,19 @@ jobs: name: ${{ env.BUILDS_ARTIFACT }} path: ${{ env.BUILDS_ARTIFACT }} + # Calculate checksums once + - name: Output checksum + if: matrix.artifact.name == 'checksums' + working-directory: ${{ env.BUILDS_ARTIFACT}} + run: | + PACKAGE_NAME_PREFIX="test" + if [ "${{ github.event_name }}" = "pull_request" ]; then + PACKAGE_NAME_PREFIX="$PACKAGE_NAME_PREFIX-${{ github.event.number }}" + fi + PACKAGE_NAME_PREFIX="$PACKAGE_NAME_PREFIX-${{ github.sha }}-" + TAG="${PACKAGE_NAME_PREFIX}git-snapshot" + sha256sum ${{ env.PROJECT_NAME }}_${TAG}* >> ${TAG}-checksums.txt + - name: Upload individual build artifact uses: actions/upload-artifact@v3 with: diff --git a/.github/workflows/release-go-task.yml b/.github/workflows/release-go-task.yml index 364a364b246..b2fc04bb7ab 100644 --- a/.github/workflows/release-go-task.yml +++ b/.github/workflows/release-go-task.yml @@ -145,14 +145,10 @@ jobs: run: | gon "${{ env.GON_CONFIG_PATH }}" - - name: Re-package binary and output checksum + - name: Re-package binary id: re-package working-directory: ${{ env.DIST_DIR }} - # This step performs the following: - # 1. Repackage the signed binary replaced in place by Gon (ignoring the output zip file) - # 2. Recalculate package checksum - # 3. Output the new checksum to include in the nnnnnn-checksums.txt file - # (it cannot be done there because of workflow job parallelization) + # Repackage the signed binary replaced in place by Gon (ignoring the output zip file) run: | # GitHub's upload/download-artifact@v2 actions don't preserve file permissions, # so we need to add execution permission back until the action is made to do this. @@ -162,11 +158,9 @@ jobs: tar -czvf "$PACKAGE_FILENAME" \ -C "${{ env.PROJECT_NAME }}_osx_${{ matrix.artifact.name }}/" "${{ env.PROJECT_NAME }}" \ -C ../../ LICENSE.txt - CHECKSUM_LINE="$(shasum -a 256 $PACKAGE_FILENAME)" echo "PACKAGE_FILENAME=$PACKAGE_FILENAME" >> $GITHUB_ENV - echo "::set-output name=checksum-${{ matrix.artifact.name }}::$CHECKSUM_LINE" - - name: Upload artifacts + - name: Upload artifact uses: actions/upload-artifact@v3 with: if-no-files-found: error @@ -184,15 +178,11 @@ jobs: name: ${{ env.ARTIFACT_NAME }} path: ${{ env.DIST_DIR }} - - name: Update checksum + - name: Output checksum + working-directory: ${{ env.DIST_DIR}} run: | - declare -a checksum_lines=("${{ needs.notarize-macos.outputs.checksum-darwin_amd64 }}" "${{ needs.notarize-macos.outputs.checksum-darwin_arm64 }}") - for checksum_line in "${checksum_lines[@]}" - do - CHECKSUM=$(echo ${checksum_line} | cut -d " " -f 1) - PACKAGE_FILENAME=$(echo ${checksum_line} | cut -d " " -f 2) - perl -pi -w -e "s/.*${PACKAGE_FILENAME}/${CHECKSUM} ${PACKAGE_FILENAME}/g;" ${{ env.DIST_DIR }}/*-checksums.txt - done + TAG="${GITHUB_REF/refs\/tags\//}" + sha256sum ${{ env.PROJECT_NAME }}_${TAG}* >> ${TAG}-checksums.txt - name: Identify Prerelease # This is a workaround while waiting for create-release action diff --git a/DistTasks.yml b/DistTasks.yml index 5a354cd991d..9b459e32aef 100644 --- a/DistTasks.yml +++ b/DistTasks.yml @@ -20,7 +20,6 @@ version: "3" vars: CONTAINER: "docker.elastic.co/beats-dev/golang-crossbuild" GO_VERSION: "1.17.8" - CHECKSUM_FILE: "{{.VERSION}}-checksums.txt" tasks: Windows_32bit: @@ -35,7 +34,6 @@ tasks: -p "{{.BUILD_PLATFORM}}" zip {{.PACKAGE_NAME}} {{.PLATFORM_DIR}}/{{.PROJECT_NAME}}.exe ../LICENSE.txt -j - sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}} vars: PLATFORM_DIR: "{{.PROJECT_NAME}}_windows_386" @@ -57,7 +55,6 @@ tasks: -p "{{.BUILD_PLATFORM}}" zip {{.PACKAGE_NAME}} {{.PLATFORM_DIR}}/{{.PROJECT_NAME}}.exe ../LICENSE.txt -j - sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}} vars: PLATFORM_DIR: "{{.PROJECT_NAME}}_windows_amd64" @@ -79,7 +76,6 @@ tasks: -p "{{.BUILD_PLATFORM}}" tar cz -C {{.PLATFORM_DIR}} {{.PROJECT_NAME}} -C ../.. LICENSE.txt -f {{.PACKAGE_NAME}} - sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}} vars: PLATFORM_DIR: "{{.PROJECT_NAME}}_linux_amd32" @@ -101,7 +97,6 @@ tasks: -p "{{.BUILD_PLATFORM}}" tar cz -C {{.PLATFORM_DIR}} {{.PROJECT_NAME}} -C ../.. LICENSE.txt -f {{.PACKAGE_NAME}} - sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}} vars: PLATFORM_DIR: "{{.PROJECT_NAME}}_linux_amd64" @@ -123,7 +118,6 @@ tasks: -p "{{.BUILD_PLATFORM}}" tar cz -C {{.PLATFORM_DIR}} {{.PROJECT_NAME}} -C ../.. LICENSE.txt -f {{.PACKAGE_NAME}} - sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}} vars: PLATFORM_DIR: "{{.PROJECT_NAME}}_linux_arm_7" @@ -145,7 +139,6 @@ tasks: -p "{{.BUILD_PLATFORM}}" tar cz -C {{.PLATFORM_DIR}} {{.PROJECT_NAME}} -C ../.. LICENSE.txt -f {{.PACKAGE_NAME}} - sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}} vars: PLATFORM_DIR: "{{.PROJECT_NAME}}_linux_arm_6" @@ -195,7 +188,6 @@ tasks: -p "{{.BUILD_PLATFORM}}" tar cz -C {{.PLATFORM_DIR}} {{.PROJECT_NAME}} -C ../.. LICENSE.txt -f {{.PACKAGE_NAME}} - sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}} vars: PLATFORM_DIR: "{{.PROJECT_NAME}}_linux_arm_6" @@ -217,7 +209,6 @@ tasks: -p "{{.BUILD_PLATFORM}}" tar cz -C {{.PLATFORM_DIR}} {{.PROJECT_NAME}} -C ../.. LICENSE.txt -f {{.PACKAGE_NAME}} - sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}} vars: PLATFORM_DIR: "{{.PROJECT_NAME}}_osx_darwin_amd64" @@ -252,7 +243,6 @@ tasks: -p "{{.BUILD_PLATFORM}}" tar cz -C {{.PLATFORM_DIR}} {{.PROJECT_NAME}} -C ../.. LICENSE.txt -f {{.PACKAGE_NAME}} - sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}} vars: PLATFORM_DIR: "{{.PROJECT_NAME}}_osx_darwin_arm64" From 55f6d3c565f5fc1aad39571ae9588879eec236ce Mon Sep 17 00:00:00 2001 From: Matteo Pologruto Date: Wed, 21 Sep 2022 17:11:07 +0200 Subject: [PATCH 3/9] Set condition to create changelog once The changelog is the same for each OS. It does not make sense to generate it more than once. --- .github/workflows/release-go-task.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/release-go-task.yml b/.github/workflows/release-go-task.yml index b2fc04bb7ab..56479508b05 100644 --- a/.github/workflows/release-go-task.yml +++ b/.github/workflows/release-go-task.yml @@ -39,6 +39,8 @@ jobs: fetch-depth: 0 - name: Create changelog + # Avoid creating the same changelog for each os + if: matrix.os == 'Windows_32bit' uses: arduino/create-changelog@v1 with: tag-regex: '^[0-9]+\.[0-9]+\.[0-9]+.*$' From 9fed4a9f3cfdcea7e43cd698cf4c7855801ce998 Mon Sep 17 00:00:00 2001 From: Matteo Pologruto Date: Mon, 19 Sep 2022 16:15:04 +0200 Subject: [PATCH 4/9] Disable s3 push for testing --- .github/workflows/publish-go-nightly-task.yml | 56 +++++++++---------- .github/workflows/release-go-task.yml | 32 +++++------ 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/.github/workflows/publish-go-nightly-task.yml b/.github/workflows/publish-go-nightly-task.yml index b15eb023476..d3a190a7640 100644 --- a/.github/workflows/publish-go-nightly-task.yml +++ b/.github/workflows/publish-go-nightly-task.yml @@ -179,31 +179,31 @@ jobs: TAG="nightly-$(date -u +"%Y%m%d")" sha256sum ${{ env.PROJECT_NAME }}_${TAG}* >> ${TAG}-checksums.txt - - name: Upload release files on Arduino downloads servers - uses: docker://plugins/s3 - env: - PLUGIN_SOURCE: "${{ env.DIST_DIR }}/*" - PLUGIN_TARGET: "${{ env.AWS_PLUGIN_TARGET }}nightly" - PLUGIN_STRIP_PREFIX: "${{ env.DIST_DIR }}/" - PLUGIN_BUCKET: ${{ secrets.DOWNLOADS_BUCKET }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - report: - runs-on: ubuntu-latest - needs: publish-nightly - if: failure() # Run if publish-nightly or any of its job dependencies failed - - steps: - - name: Report failure - uses: masci/datadog@v1 - with: - api-key: ${{ secrets.DD_API_KEY }} - events: | - - title: "${{ env.PROJECT_NAME }} nightly build failed" - text: "Nightly build workflow has failed" - alert_type: "error" - host: ${{ github.repository }} - tags: - - "project:${{ env.PROJECT_NAME }}" - - "workflow:${{ github.workflow }}" + # - name: Upload release files on Arduino downloads servers + # uses: docker://plugins/s3 + # env: + # PLUGIN_SOURCE: "${{ env.DIST_DIR }}/*" + # PLUGIN_TARGET: "${{ env.AWS_PLUGIN_TARGET }}nightly" + # PLUGIN_STRIP_PREFIX: "${{ env.DIST_DIR }}/" + # PLUGIN_BUCKET: ${{ secrets.DOWNLOADS_BUCKET }} + # AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + + # report: + # runs-on: ubuntu-latest + # needs: publish-nightly + # if: failure() # Run if publish-nightly or any of its job dependencies failed + + # steps: + # - name: Report failure + # uses: masci/datadog@v1 + # with: + # api-key: ${{ secrets.DD_API_KEY }} + # events: | + # - title: "${{ env.PROJECT_NAME }} nightly build failed" + # text: "Nightly build workflow has failed" + # alert_type: "error" + # host: ${{ github.repository }} + # tags: + # - "project:${{ env.PROJECT_NAME }}" + # - "workflow:${{ github.workflow }}" diff --git a/.github/workflows/release-go-task.yml b/.github/workflows/release-go-task.yml index 56479508b05..e8d01e03c5c 100644 --- a/.github/workflows/release-go-task.yml +++ b/.github/workflows/release-go-task.yml @@ -206,19 +206,19 @@ jobs: # (all the files we need are in the DIST_DIR root) artifacts: ${{ env.DIST_DIR }}/* - - name: Upload release files on Arduino downloads servers - uses: docker://plugins/s3 - env: - PLUGIN_SOURCE: "${{ env.DIST_DIR }}/*" - PLUGIN_TARGET: ${{ env.AWS_PLUGIN_TARGET }} - PLUGIN_STRIP_PREFIX: "${{ env.DIST_DIR }}/" - PLUGIN_BUCKET: ${{ secrets.DOWNLOADS_BUCKET }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - - name: Update Homebrew formula - if: steps.prerelease.outputs.IS_PRE != 'true' - uses: dawidd6/action-homebrew-bump-formula@v3 - with: - token: ${{ secrets.ARDUINOBOT_GITHUB_TOKEN }} - formula: arduino-cli + # - name: Upload release files on Arduino downloads servers + # uses: docker://plugins/s3 + # env: + # PLUGIN_SOURCE: "${{ env.DIST_DIR }}/*" + # PLUGIN_TARGET: ${{ env.AWS_PLUGIN_TARGET }} + # PLUGIN_STRIP_PREFIX: "${{ env.DIST_DIR }}/" + # PLUGIN_BUCKET: ${{ secrets.DOWNLOADS_BUCKET }} + # AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + + # - name: Update Homebrew formula + # if: steps.prerelease.outputs.IS_PRE != 'true' + # uses: dawidd6/action-homebrew-bump-formula@v3 + # with: + # token: ${{ secrets.ARDUINOBOT_GITHUB_TOKEN }} + # formula: arduino-cli From b3664cd5cd0a42c32dc5b65fb0a503f027b4867d Mon Sep 17 00:00:00 2001 From: Matteo Pologruto Date: Thu, 22 Sep 2022 12:19:47 +0200 Subject: [PATCH 5/9] Upload nightly artifacts for testing --- .github/workflows/publish-go-nightly-task.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/publish-go-nightly-task.yml b/.github/workflows/publish-go-nightly-task.yml index d3a190a7640..805dc995765 100644 --- a/.github/workflows/publish-go-nightly-task.yml +++ b/.github/workflows/publish-go-nightly-task.yml @@ -179,6 +179,13 @@ jobs: TAG="nightly-$(date -u +"%Y%m%d")" sha256sum ${{ env.PROJECT_NAME }}_${TAG}* >> ${TAG}-checksums.txt + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + if-no-files-found: error + name: ${{ env.ARTIFACT_NAME }} + path: ${{ env.DIST_DIR }} + # - name: Upload release files on Arduino downloads servers # uses: docker://plugins/s3 # env: From 630506e6ec352024b069b45b72d19fc004266679 Mon Sep 17 00:00:00 2001 From: Matteo Pologruto Date: Thu, 22 Sep 2022 16:15:47 +0200 Subject: [PATCH 6/9] Fix linux_arm_6 typo --- DistTasks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DistTasks.yml b/DistTasks.yml index 9b459e32aef..1b23c823914 100644 --- a/DistTasks.yml +++ b/DistTasks.yml @@ -190,7 +190,7 @@ tasks: tar cz -C {{.PLATFORM_DIR}} {{.PROJECT_NAME}} -C ../.. LICENSE.txt -f {{.PACKAGE_NAME}} vars: - PLATFORM_DIR: "{{.PROJECT_NAME}}_linux_arm_6" + PLATFORM_DIR: "{{.PROJECT_NAME}}_linux_arm_64" BUILD_COMMAND: "go build -o {{.DIST_DIR}}/{{.PLATFORM_DIR}}/{{.PROJECT_NAME}} {{.LDFLAGS}}" BUILD_PLATFORM: "linux/arm64" CONTAINER_TAG: "{{.GO_VERSION}}-arm" From 8114e21fe6304521a5e7db3bb57cb87efc5f0a55 Mon Sep 17 00:00:00 2001 From: Matteo Pologruto Date: Mon, 26 Sep 2022 16:01:20 +0200 Subject: [PATCH 7/9] Stop uploading nigthly artifacts --- .github/workflows/publish-go-nightly-task.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/publish-go-nightly-task.yml b/.github/workflows/publish-go-nightly-task.yml index 805dc995765..d3a190a7640 100644 --- a/.github/workflows/publish-go-nightly-task.yml +++ b/.github/workflows/publish-go-nightly-task.yml @@ -179,13 +179,6 @@ jobs: TAG="nightly-$(date -u +"%Y%m%d")" sha256sum ${{ env.PROJECT_NAME }}_${TAG}* >> ${TAG}-checksums.txt - - name: Upload artifacts - uses: actions/upload-artifact@v3 - with: - if-no-files-found: error - name: ${{ env.ARTIFACT_NAME }} - path: ${{ env.DIST_DIR }} - # - name: Upload release files on Arduino downloads servers # uses: docker://plugins/s3 # env: From 370016b7be2817f909bf989b66d85af215cc8ead Mon Sep 17 00:00:00 2001 From: Matteo Pologruto Date: Mon, 26 Sep 2022 16:07:45 +0200 Subject: [PATCH 8/9] Enable s3 pushing --- .github/workflows/publish-go-nightly-task.yml | 56 +++++++++---------- .github/workflows/release-go-task.yml | 32 +++++------ 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/.github/workflows/publish-go-nightly-task.yml b/.github/workflows/publish-go-nightly-task.yml index d3a190a7640..b15eb023476 100644 --- a/.github/workflows/publish-go-nightly-task.yml +++ b/.github/workflows/publish-go-nightly-task.yml @@ -179,31 +179,31 @@ jobs: TAG="nightly-$(date -u +"%Y%m%d")" sha256sum ${{ env.PROJECT_NAME }}_${TAG}* >> ${TAG}-checksums.txt - # - name: Upload release files on Arduino downloads servers - # uses: docker://plugins/s3 - # env: - # PLUGIN_SOURCE: "${{ env.DIST_DIR }}/*" - # PLUGIN_TARGET: "${{ env.AWS_PLUGIN_TARGET }}nightly" - # PLUGIN_STRIP_PREFIX: "${{ env.DIST_DIR }}/" - # PLUGIN_BUCKET: ${{ secrets.DOWNLOADS_BUCKET }} - # AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - # report: - # runs-on: ubuntu-latest - # needs: publish-nightly - # if: failure() # Run if publish-nightly or any of its job dependencies failed - - # steps: - # - name: Report failure - # uses: masci/datadog@v1 - # with: - # api-key: ${{ secrets.DD_API_KEY }} - # events: | - # - title: "${{ env.PROJECT_NAME }} nightly build failed" - # text: "Nightly build workflow has failed" - # alert_type: "error" - # host: ${{ github.repository }} - # tags: - # - "project:${{ env.PROJECT_NAME }}" - # - "workflow:${{ github.workflow }}" + - name: Upload release files on Arduino downloads servers + uses: docker://plugins/s3 + env: + PLUGIN_SOURCE: "${{ env.DIST_DIR }}/*" + PLUGIN_TARGET: "${{ env.AWS_PLUGIN_TARGET }}nightly" + PLUGIN_STRIP_PREFIX: "${{ env.DIST_DIR }}/" + PLUGIN_BUCKET: ${{ secrets.DOWNLOADS_BUCKET }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + + report: + runs-on: ubuntu-latest + needs: publish-nightly + if: failure() # Run if publish-nightly or any of its job dependencies failed + + steps: + - name: Report failure + uses: masci/datadog@v1 + with: + api-key: ${{ secrets.DD_API_KEY }} + events: | + - title: "${{ env.PROJECT_NAME }} nightly build failed" + text: "Nightly build workflow has failed" + alert_type: "error" + host: ${{ github.repository }} + tags: + - "project:${{ env.PROJECT_NAME }}" + - "workflow:${{ github.workflow }}" diff --git a/.github/workflows/release-go-task.yml b/.github/workflows/release-go-task.yml index e8d01e03c5c..56479508b05 100644 --- a/.github/workflows/release-go-task.yml +++ b/.github/workflows/release-go-task.yml @@ -206,19 +206,19 @@ jobs: # (all the files we need are in the DIST_DIR root) artifacts: ${{ env.DIST_DIR }}/* - # - name: Upload release files on Arduino downloads servers - # uses: docker://plugins/s3 - # env: - # PLUGIN_SOURCE: "${{ env.DIST_DIR }}/*" - # PLUGIN_TARGET: ${{ env.AWS_PLUGIN_TARGET }} - # PLUGIN_STRIP_PREFIX: "${{ env.DIST_DIR }}/" - # PLUGIN_BUCKET: ${{ secrets.DOWNLOADS_BUCKET }} - # AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - # - name: Update Homebrew formula - # if: steps.prerelease.outputs.IS_PRE != 'true' - # uses: dawidd6/action-homebrew-bump-formula@v3 - # with: - # token: ${{ secrets.ARDUINOBOT_GITHUB_TOKEN }} - # formula: arduino-cli + - name: Upload release files on Arduino downloads servers + uses: docker://plugins/s3 + env: + PLUGIN_SOURCE: "${{ env.DIST_DIR }}/*" + PLUGIN_TARGET: ${{ env.AWS_PLUGIN_TARGET }} + PLUGIN_STRIP_PREFIX: "${{ env.DIST_DIR }}/" + PLUGIN_BUCKET: ${{ secrets.DOWNLOADS_BUCKET }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + + - name: Update Homebrew formula + if: steps.prerelease.outputs.IS_PRE != 'true' + uses: dawidd6/action-homebrew-bump-formula@v3 + with: + token: ${{ secrets.ARDUINOBOT_GITHUB_TOKEN }} + formula: arduino-cli From 86b1fb57c299a79fba9d98574c1c2c2efbfca7b5 Mon Sep 17 00:00:00 2001 From: Matteo Pologruto Date: Tue, 27 Sep 2022 17:37:13 +0200 Subject: [PATCH 9/9] Upload build artifacts separately Previously, the different builds were firstly uploaded using a single artifact, which was then downloaded to create different ones and eventually deleted. Now, since builds are created concurrently, the same matrix can be used to directly upload an artifact for each build. It's necessary to use a second job to calculate the checksum related to each build and save them all in a single .txt file. --- .github/workflows/publish-go-tester-task.yml | 105 ++++++++----------- 1 file changed, 46 insertions(+), 59 deletions(-) diff --git a/.github/workflows/publish-go-tester-task.yml b/.github/workflows/publish-go-tester-task.yml index e29702bbbd3..f2bc0fc84cd 100644 --- a/.github/workflows/publish-go-tester-task.yml +++ b/.github/workflows/publish-go-tester-task.yml @@ -55,6 +55,7 @@ jobs: echo "::set-output name=result::$RESULT" build: + name: Build ${{ matrix.os.name }} needs: run-determination if: needs.run-determination.outputs.result == 'true' runs-on: ubuntu-latest @@ -62,15 +63,33 @@ jobs: strategy: matrix: os: - - Windows_32bit - - Windows_64bit - - Linux_32bit - - Linux_64bit - - Linux_ARMv6 - - Linux_ARMv7 - - Linux_ARM64 - - macOS_64bit - - macOS_ARM64 + - dist: Windows_32bit + path: "*Windows_32bit.zip" + name: Windows_X86-32 + - dist: Windows_64bit + path: "*Windows_64bit.zip" + name: Windows_X86-64 + - dist: Linux_32bit + path: "*Linux_32bit.tar.gz" + name: Linux_X86-32 + - dist: Linux_64bit + path: "*Linux_64bit.tar.gz" + name: Linux_X86-64 + - dist: Linux_ARMv6 + path: "*Linux_ARMv6.tar.gz" + name: Linux_ARMv6 + - dist: Linux_ARMv7 + path: "*Linux_ARMv7.tar.gz" + name: Linux_ARMv7 + - dist: Linux_ARM64 + path: "*Linux_ARM64.tar.gz" + name: Linux_ARM64 + - dist: macOS_64bit + path: "*macOS_64bit.tar.gz" + name: macOS_64 + - dist: macOS_ARM64 + path: "*macOS_ARM64.tar.gz" + name: macOS_ARM64 steps: - name: Checkout repository @@ -92,54 +111,25 @@ jobs: fi PACKAGE_NAME_PREFIX="$PACKAGE_NAME_PREFIX-${{ github.sha }}-" export PACKAGE_NAME_PREFIX - task dist:${{ matrix.os }} + task dist:${{ matrix.os.dist }} - # Transfer builds to artifacts job - - name: Upload combined builds artifact + - name: Upload build artifact uses: actions/upload-artifact@v3 with: - path: ${{ env.DIST_DIR }} - name: ${{ env.BUILDS_ARTIFACT }} + path: ${{ env.DIST_DIR }}/${{ matrix.os.path }} + name: ${{ matrix.os.name }} - artifacts: - name: ${{ matrix.artifact.name }} artifact + checksums: needs: build runs-on: ubuntu-latest - strategy: - matrix: - artifact: - - path: "*checksums.txt" - name: checksums - - path: "*Linux_32bit.tar.gz" - name: Linux_X86-32 - - path: "*Linux_64bit.tar.gz" - name: Linux_X86-64 - - path: "*Linux_ARM64.tar.gz" - name: Linux_ARM64 - - path: "*Linux_ARMv6.tar.gz" - name: Linux_ARMv6 - - path: "*Linux_ARMv7.tar.gz" - name: Linux_ARMv7 - - path: "*macOS_64bit.tar.gz" - name: macOS_64 - - path: "*macOS_ARM64.tar.gz" - name: macOS_ARM64 - - path: "*Windows_32bit.zip" - name: Windows_X86-32 - - path: "*Windows_64bit.zip" - name: Windows_X86-64 - steps: - - name: Download combined builds artifact + - name: Download build artifacts uses: actions/download-artifact@v3 with: - name: ${{ env.BUILDS_ARTIFACT }} path: ${{ env.BUILDS_ARTIFACT }} - # Calculate checksums once - name: Output checksum - if: matrix.artifact.name == 'checksums' working-directory: ${{ env.BUILDS_ARTIFACT}} run: | PACKAGE_NAME_PREFIX="test" @@ -148,20 +138,17 @@ jobs: fi PACKAGE_NAME_PREFIX="$PACKAGE_NAME_PREFIX-${{ github.sha }}-" TAG="${PACKAGE_NAME_PREFIX}git-snapshot" - sha256sum ${{ env.PROJECT_NAME }}_${TAG}* >> ${TAG}-checksums.txt - - - name: Upload individual build artifact + declare -a artifacts=($(ls -d */)) + for artifact in ${artifacts[@]} + do + cd $artifact + checksum=$(sha256sum ${{ env.PROJECT_NAME }}_${TAG}*) + cd .. + echo $checksum >> ${TAG}-checksums.txt + done + + - name: Upload checksum artifact uses: actions/upload-artifact@v3 with: - path: ${{ env.BUILDS_ARTIFACT }}/${{ matrix.artifact.path }} - name: ${{ matrix.artifact.name }} - - clean: - needs: artifacts - runs-on: ubuntu-latest - - steps: - - name: Remove unneeded combined builds artifact - uses: geekyeggo/delete-artifact@v1 - with: - name: ${{ env.BUILDS_ARTIFACT }} + path: ${{ env.BUILDS_ARTIFACT }}/*checksums.txt + name: checksums