Skip to content

[skip-changelog] Use a matrix to create parallel builds for each OS #1883

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
Sep 29, 2022
39 changes: 21 additions & 18 deletions .github/workflows/publish-go-nightly-task.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,19 @@ jobs:
create-nightly-artifacts:
runs-on: ubuntu-latest

strategy:
matrix:
os:
- Windows_32bit
- Windows_64bit
- Linux_32bit
- Linux_64bit
- Linux_ARMv6
- Linux_ARMv7
- Linux_ARM64
- macOS_64bit
- macOS_ARM64

steps:
- name: Checkout repository
uses: actions/checkout@v3
Expand All @@ -35,7 +48,7 @@ jobs:
- name: Build
env:
NIGHTLY: true
run: task dist:all
run: task dist:${{ matrix.os }}

- name: Upload artifacts
uses: actions/upload-artifact@v3
Expand Down Expand Up @@ -127,14 +140,10 @@ jobs:
run: |
gon "${{ env.GON_CONFIG_PATH }}"

- name: Re-package binary and output checksum
- name: Re-package binary
id: re-package
working-directory: ${{ env.DIST_DIR }}
# This step performs the following:
# 1. Repackage the signed binary replaced in place by Gon (ignoring the output zip file)
# 2. Recalculate package checksum
# 3. Output the new checksum to include in the nnnnnn-checksums.txt file
# (it cannot be done there because of workflow job parallelization)
# Repackage the signed binary replaced in place by Gon (ignoring the output zip file)
run: |
# GitHub's upload/download-artifact@v2 actions don't preserve file permissions,
# so we need to add execution permission back until the action is made to do this.
Expand All @@ -144,11 +153,9 @@ jobs:
tar -czvf "$PACKAGE_FILENAME" \
-C "${{ env.PROJECT_NAME }}_osx_${{ matrix.artifact.name }}/" "${{ env.PROJECT_NAME }}" \
-C ../../ LICENSE.txt
CHECKSUM_LINE="$(shasum -a 256 $PACKAGE_FILENAME)"
echo "PACKAGE_FILENAME=$PACKAGE_FILENAME" >> $GITHUB_ENV
echo "::set-output name=checksum-${{ matrix.artifact.name }}::$CHECKSUM_LINE"

- name: Upload artifacts
- name: Upload artifact
uses: actions/upload-artifact@v3
with:
if-no-files-found: error
Expand All @@ -166,15 +173,11 @@ jobs:
name: ${{ env.ARTIFACT_NAME }}
path: ${{ env.DIST_DIR }}

- name: Update checksum
- name: Output checksum
working-directory: ${{ env.DIST_DIR}}
run: |
declare -a checksum_lines=("${{ needs.notarize-macos.outputs.checksum-darwin_amd64 }}" "${{ needs.notarize-macos.outputs.checksum-darwin_arm64 }}")
for checksum_line in "${checksum_lines[@]}"
do
CHECKSUM=$(echo ${checksum_line} | cut -d " " -f 1)
PACKAGE_FILENAME=$(echo ${checksum_line} | cut -d " " -f 2)
perl -pi -w -e "s/.*${PACKAGE_FILENAME}/${CHECKSUM} ${PACKAGE_FILENAME}/g;" ${{ env.DIST_DIR }}/*-checksums.txt
done
TAG="nightly-$(date -u +"%Y%m%d")"
sha256sum ${{ env.PROJECT_NAME }}_${TAG}* >> ${TAG}-checksums.txt

- name: Upload release files on Arduino downloads servers
uses: docker://plugins/s3
Expand Down
107 changes: 61 additions & 46 deletions .github/workflows/publish-go-tester-task.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ on:
repository_dispatch:

env:
# As defined by the Taskfile's PROJECT_NAME variable
PROJECT_NAME: arduino-cli
# As defined by the Taskfile's DIST_DIR variable
DIST_DIR: dist
BUILDS_ARTIFACT: build-artifacts
Expand Down Expand Up @@ -53,10 +55,42 @@ jobs:
echo "::set-output name=result::$RESULT"

build:
name: Build ${{ matrix.os.name }}
needs: run-determination
if: needs.run-determination.outputs.result == 'true'
runs-on: ubuntu-latest

strategy:
matrix:
os:
- dist: Windows_32bit
path: "*Windows_32bit.zip"
name: Windows_X86-32
- dist: Windows_64bit
path: "*Windows_64bit.zip"
name: Windows_X86-64
- dist: Linux_32bit
path: "*Linux_32bit.tar.gz"
name: Linux_X86-32
- dist: Linux_64bit
path: "*Linux_64bit.tar.gz"
name: Linux_X86-64
- dist: Linux_ARMv6
path: "*Linux_ARMv6.tar.gz"
name: Linux_ARMv6
- dist: Linux_ARMv7
path: "*Linux_ARMv7.tar.gz"
name: Linux_ARMv7
- dist: Linux_ARM64
path: "*Linux_ARM64.tar.gz"
name: Linux_ARM64
- dist: macOS_64bit
path: "*macOS_64bit.tar.gz"
name: macOS_64
- dist: macOS_ARM64
path: "*macOS_ARM64.tar.gz"
name: macOS_ARM64

steps:
- name: Checkout repository
uses: actions/checkout@v3
Expand All @@ -77,63 +111,44 @@ jobs:
fi
PACKAGE_NAME_PREFIX="$PACKAGE_NAME_PREFIX-${{ github.sha }}-"
export PACKAGE_NAME_PREFIX
task dist:all
task dist:${{ matrix.os.dist }}

# Transfer builds to artifacts job
- name: Upload combined builds artifact
- name: Upload build artifact
uses: actions/upload-artifact@v3
with:
path: ${{ env.DIST_DIR }}
name: ${{ env.BUILDS_ARTIFACT }}
path: ${{ env.DIST_DIR }}/${{ matrix.os.path }}
name: ${{ matrix.os.name }}

artifacts:
name: ${{ matrix.artifact.name }} artifact
checksums:
needs: build
runs-on: ubuntu-latest

strategy:
matrix:
artifact:
- path: "*checksums.txt"
name: checksums
- path: "*Linux_32bit.tar.gz"
name: Linux_X86-32
- path: "*Linux_64bit.tar.gz"
name: Linux_X86-64
- path: "*Linux_ARM64.tar.gz"
name: Linux_ARM64
- path: "*Linux_ARMv6.tar.gz"
name: Linux_ARMv6
- path: "*Linux_ARMv7.tar.gz"
name: Linux_ARMv7
- path: "*macOS_64bit.tar.gz"
name: macOS_64
- path: "*macOS_ARM64.tar.gz"
name: macOS_ARM64
- path: "*Windows_32bit.zip"
name: Windows_X86-32
- path: "*Windows_64bit.zip"
name: Windows_X86-64

steps:
- name: Download combined builds artifact
- name: Download build artifacts
uses: actions/download-artifact@v3
with:
name: ${{ env.BUILDS_ARTIFACT }}
path: ${{ env.BUILDS_ARTIFACT }}

- name: Upload individual build artifact
uses: actions/upload-artifact@v3
with:
path: ${{ env.BUILDS_ARTIFACT }}/${{ matrix.artifact.path }}
name: ${{ matrix.artifact.name }}

clean:
needs: artifacts
runs-on: ubuntu-latest
- name: Output checksum
working-directory: ${{ env.BUILDS_ARTIFACT}}
run: |
PACKAGE_NAME_PREFIX="test"
if [ "${{ github.event_name }}" = "pull_request" ]; then
PACKAGE_NAME_PREFIX="$PACKAGE_NAME_PREFIX-${{ github.event.number }}"
fi
PACKAGE_NAME_PREFIX="$PACKAGE_NAME_PREFIX-${{ github.sha }}-"
TAG="${PACKAGE_NAME_PREFIX}git-snapshot"
declare -a artifacts=($(ls -d */))
for artifact in ${artifacts[@]}
do
cd $artifact
checksum=$(sha256sum ${{ env.PROJECT_NAME }}_${TAG}*)
cd ..
echo $checksum >> ${TAG}-checksums.txt
done

steps:
- name: Remove unneeded combined builds artifact
uses: geekyeggo/delete-artifact@v1
- name: Upload checksum artifact
uses: actions/upload-artifact@v3
with:
name: ${{ env.BUILDS_ARTIFACT }}
path: ${{ env.BUILDS_ARTIFACT }}/*checksums.txt
name: checksums
41 changes: 23 additions & 18 deletions .github/workflows/release-go-task.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,28 @@ jobs:
create-release-artifacts:
runs-on: ubuntu-latest

strategy:
matrix:
os:
- Windows_32bit
- Windows_64bit
- Linux_32bit
- Linux_64bit
- Linux_ARMv6
- Linux_ARMv7
- Linux_ARM64
- macOS_64bit
- macOS_ARM64

steps:
- name: Checkout repository
uses: actions/checkout@v3
with:
fetch-depth: 0

- name: Create changelog
# Avoid creating the same changelog for each os
if: matrix.os == 'Windows_32bit'
uses: arduino/create-changelog@v1
with:
tag-regex: '^[0-9]+\.[0-9]+\.[0-9]+.*$'
Expand All @@ -40,7 +55,7 @@ jobs:
version: 3.x

- name: Build
run: task dist:all
run: task dist:${{ matrix.os }}

- name: Upload artifacts
uses: actions/upload-artifact@v3
Expand Down Expand Up @@ -132,14 +147,10 @@ jobs:
run: |
gon "${{ env.GON_CONFIG_PATH }}"

- name: Re-package binary and output checksum
- name: Re-package binary
id: re-package
working-directory: ${{ env.DIST_DIR }}
# This step performs the following:
# 1. Repackage the signed binary replaced in place by Gon (ignoring the output zip file)
# 2. Recalculate package checksum
# 3. Output the new checksum to include in the nnnnnn-checksums.txt file
# (it cannot be done there because of workflow job parallelization)
# Repackage the signed binary replaced in place by Gon (ignoring the output zip file)
run: |
# GitHub's upload/download-artifact@v2 actions don't preserve file permissions,
# so we need to add execution permission back until the action is made to do this.
Expand All @@ -149,11 +160,9 @@ jobs:
tar -czvf "$PACKAGE_FILENAME" \
-C "${{ env.PROJECT_NAME }}_osx_${{ matrix.artifact.name }}/" "${{ env.PROJECT_NAME }}" \
-C ../../ LICENSE.txt
CHECKSUM_LINE="$(shasum -a 256 $PACKAGE_FILENAME)"
echo "PACKAGE_FILENAME=$PACKAGE_FILENAME" >> $GITHUB_ENV
echo "::set-output name=checksum-${{ matrix.artifact.name }}::$CHECKSUM_LINE"

- name: Upload artifacts
- name: Upload artifact
uses: actions/upload-artifact@v3
with:
if-no-files-found: error
Expand All @@ -171,15 +180,11 @@ jobs:
name: ${{ env.ARTIFACT_NAME }}
path: ${{ env.DIST_DIR }}

- name: Update checksum
- name: Output checksum
working-directory: ${{ env.DIST_DIR}}
run: |
declare -a checksum_lines=("${{ needs.notarize-macos.outputs.checksum-darwin_amd64 }}" "${{ needs.notarize-macos.outputs.checksum-darwin_arm64 }}")
for checksum_line in "${checksum_lines[@]}"
do
CHECKSUM=$(echo ${checksum_line} | cut -d " " -f 1)
PACKAGE_FILENAME=$(echo ${checksum_line} | cut -d " " -f 2)
perl -pi -w -e "s/.*${PACKAGE_FILENAME}/${CHECKSUM} ${PACKAGE_FILENAME}/g;" ${{ env.DIST_DIR }}/*-checksums.txt
done
TAG="${GITHUB_REF/refs\/tags\//}"
sha256sum ${{ env.PROJECT_NAME }}_${TAG}* >> ${TAG}-checksums.txt

- name: Identify Prerelease
# This is a workaround while waiting for create-release action
Expand Down
Loading