Skip to content

Update release workflows using parallelization #277

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Oct 14, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 0 additions & 23 deletions workflow-templates/assets/release-go-task/DistTasks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,22 +20,8 @@ version: "3"
vars:
CONTAINER: "docker.elastic.co/beats-dev/golang-crossbuild"
GO_VERSION: "1.17.8"
CHECKSUM_FILE: "{{.VERSION}}-checksums.txt"

tasks:
all:
desc: Build for distribution for all platforms
cmds:
- task: Windows_32bit
- task: Windows_64bit
- task: Linux_32bit
- task: Linux_64bit
- task: Linux_ARMv6
- task: Linux_ARMv7
- task: Linux_ARM64
- task: macOS_64bit
- task: macOS_ARM64

Windows_32bit:
desc: Builds Windows 32 bit binaries
dir: "{{.DIST_DIR}}"
Expand All @@ -48,7 +34,6 @@ tasks:
-p "{{.BUILD_PLATFORM}}"

zip {{.PACKAGE_NAME}} {{.PLATFORM_DIR}}/{{.PROJECT_NAME}}.exe ../LICENSE.txt -j
sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}}

vars:
PLATFORM_DIR: "{{.PROJECT_NAME}}_windows_386"
Expand All @@ -70,7 +55,6 @@ tasks:
-p "{{.BUILD_PLATFORM}}"

zip {{.PACKAGE_NAME}} {{.PLATFORM_DIR}}/{{.PROJECT_NAME}}.exe ../LICENSE.txt -j
sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}}

vars:
PLATFORM_DIR: "{{.PROJECT_NAME}}_windows_amd64"
Expand All @@ -92,7 +76,6 @@ tasks:
-p "{{.BUILD_PLATFORM}}"

tar cz -C {{.PLATFORM_DIR}} {{.PROJECT_NAME}} -C ../.. LICENSE.txt -f {{.PACKAGE_NAME}}
sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}}

vars:
PLATFORM_DIR: "{{.PROJECT_NAME}}_linux_amd32"
Expand All @@ -114,7 +97,6 @@ tasks:
-p "{{.BUILD_PLATFORM}}"

tar cz -C {{.PLATFORM_DIR}} {{.PROJECT_NAME}} -C ../.. LICENSE.txt -f {{.PACKAGE_NAME}}
sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}}

vars:
PLATFORM_DIR: "{{.PROJECT_NAME}}_linux_amd64"
Expand All @@ -136,7 +118,6 @@ tasks:
-p "{{.BUILD_PLATFORM}}"

tar cz -C {{.PLATFORM_DIR}} {{.PROJECT_NAME}} -C ../.. LICENSE.txt -f {{.PACKAGE_NAME}}
sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}}

vars:
PLATFORM_DIR: "{{.PROJECT_NAME}}_linux_arm_7"
Expand All @@ -158,7 +139,6 @@ tasks:
-p "{{.BUILD_PLATFORM}}"

tar cz -C {{.PLATFORM_DIR}} {{.PROJECT_NAME}} -C ../.. LICENSE.txt -f {{.PACKAGE_NAME}}
sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}}

vars:
PLATFORM_DIR: "{{.PROJECT_NAME}}_linux_arm_6"
Expand Down Expand Up @@ -208,7 +188,6 @@ tasks:
-p "{{.BUILD_PLATFORM}}"

tar cz -C {{.PLATFORM_DIR}} {{.PROJECT_NAME}} -C ../.. LICENSE.txt -f {{.PACKAGE_NAME}}
sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}}

vars:
PLATFORM_DIR: "{{.PROJECT_NAME}}_linux_arm_64"
Expand All @@ -230,7 +209,6 @@ tasks:
-p "{{.BUILD_PLATFORM}}"

tar cz -C {{.PLATFORM_DIR}} {{.PROJECT_NAME}} -C ../.. LICENSE.txt -f {{.PACKAGE_NAME}}
sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}}

vars:
PLATFORM_DIR: "{{.PROJECT_NAME}}_osx_darwin_amd64"
Expand Down Expand Up @@ -265,7 +243,6 @@ tasks:
-p "{{.BUILD_PLATFORM}}"

tar cz -C {{.PLATFORM_DIR}} {{.PROJECT_NAME}} -C ../.. LICENSE.txt -f {{.PACKAGE_NAME}}
sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}}

vars:
PLATFORM_DIR: "{{.PROJECT_NAME}}_osx_darwin_arm64"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,19 @@ jobs:
create-nightly-artifacts:
runs-on: ubuntu-latest

strategy:
matrix:
os:
- Windows_32bit
- Windows_64bit
- Linux_32bit
- Linux_64bit
- Linux_ARMv6
- Linux_ARMv7
- Linux_ARM64
- macOS_64bit
- macOS_ARM64

steps:
- name: Checkout repository
uses: actions/checkout@v3
Expand All @@ -35,7 +48,7 @@ jobs:
- name: Build
env:
NIGHTLY: true
run: task dist:all
run: task dist:${{ matrix.os }}

- name: Upload artifacts
uses: actions/upload-artifact@v3
Expand Down Expand Up @@ -128,14 +141,10 @@ jobs:
run: |
gon "${{ env.GON_CONFIG_PATH }}"

- name: Re-package binary and output checksum
- name: Re-package binary
id: re-package
working-directory: ${{ env.DIST_DIR }}
# This step performs the following:
# 1. Repackage the signed binary replaced in place by Gon (ignoring the output zip file)
# 2. Recalculate package checksum
# 3. Output the new checksum to include in the nnnnnn-checksums.txt file
# (it cannot be done there because of workflow job parallelization)
# Repackage the signed binary replaced in place by Gon (ignoring the output zip file)
run: |
# GitHub's upload/download-artifact actions don't preserve file permissions,
# so we need to add execution permission back until the action is made to do this.
Expand All @@ -145,11 +154,9 @@ jobs:
tar -czvf "$PACKAGE_FILENAME" \
-C "${{ env.PROJECT_NAME }}_osx_${{ matrix.artifact.name }}/" "${{ env.PROJECT_NAME }}" \
-C ../../ LICENSE.txt
CHECKSUM_LINE="$(shasum -a 256 $PACKAGE_FILENAME)"
echo "PACKAGE_FILENAME=$PACKAGE_FILENAME" >> $GITHUB_ENV
echo "::set-output name=checksum-${{ matrix.artifact.name }}::$CHECKSUM_LINE"

- name: Upload artifacts
- name: Upload artifact
uses: actions/upload-artifact@v3
with:
if-no-files-found: error
Expand All @@ -167,15 +174,11 @@ jobs:
name: ${{ env.ARTIFACT_NAME }}
path: ${{ env.DIST_DIR }}

- name: Update checksum
- name: Create checksum file
working-directory: ${{ env.DIST_DIR}}
run: |
declare -a checksum_lines=("${{ needs.notarize-macos.outputs.checksum-darwin_amd64 }}" "${{ needs.notarize-macos.outputs.checksum-darwin_arm64 }}")
for checksum_line in "${checksum_lines[@]}"
do
CHECKSUM=$(echo ${checksum_line} | cut -d " " -f 1)
PACKAGE_FILENAME=$(echo ${checksum_line} | cut -d " " -f 2)
perl -pi -w -e "s/.*${PACKAGE_FILENAME}/${CHECKSUM} ${PACKAGE_FILENAME}/g;" ${{ env.DIST_DIR }}/*-checksums.txt
done
TAG="nightly-$(date -u +"%Y%m%d")"
sha256sum ${{ env.PROJECT_NAME }}_${TAG}* > ${TAG}-checksums.txt

- name: Upload release files on Arduino downloads servers
uses: docker://plugins/s3
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,10 @@ on:
repository_dispatch:

env:
# As defined by the Taskfile's PROJECT_NAME variable
PROJECT_NAME: TODO_PROJECT_NAME
# As defined by the Taskfile's DIST_DIR variable
DIST_DIR: dist
BUILDS_ARTIFACT: build-artifacts

jobs:
run-determination:
Expand All @@ -52,86 +53,107 @@ jobs:

echo "::set-output name=result::$RESULT"

build:
package-name-prefix:
needs: run-determination
if: needs.run-determination.outputs.result == 'true'
runs-on: ubuntu-latest

outputs:
prefix: ${{ steps.calculation.outputs.prefix }}
steps:
- name: Checkout repository
uses: actions/checkout@v3

- name: Install Task
uses: arduino/setup-task@v1
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
version: 3.x

- name: Build
- name: package name prefix calculation
id: calculation
run: |
PACKAGE_NAME_PREFIX="test"
if [ "${{ github.event_name }}" = "pull_request" ]; then
PACKAGE_NAME_PREFIX="$PACKAGE_NAME_PREFIX-${{ github.event.number }}"
fi
PACKAGE_NAME_PREFIX="$PACKAGE_NAME_PREFIX-${{ github.sha }}-"
export PACKAGE_NAME_PREFIX
task dist:all

# Transfer builds to artifacts job
- name: Upload combined builds artifact
uses: actions/upload-artifact@v3
with:
path: ${{ env.DIST_DIR }}
name: ${{ env.BUILDS_ARTIFACT }}
echo "::set-output name=prefix::$PACKAGE_NAME_PREFIX"

artifacts:
name: ${{ matrix.artifact.name }} artifact
needs: build
build:
needs: package-name-prefix
name: Build ${{ matrix.os.name }}
runs-on: ubuntu-latest

strategy:
matrix:
artifact:
- path: "*checksums.txt"
name: checksums
- path: "*Linux_32bit.tar.gz"
os:
- task: Windows_32bit
path: "*Windows_32bit.zip"
name: Windows_X86-32
- task: Windows_64bit
path: "*Windows_64bit.zip"
name: Windows_X86-64
- task: Linux_32bit
path: "*Linux_32bit.tar.gz"
name: Linux_X86-32
- path: "*Linux_64bit.tar.gz"
- task: Linux_64bit
path: "*Linux_64bit.tar.gz"
name: Linux_X86-64
- path: "*Linux_ARM64.tar.gz"
name: Linux_ARM64
- path: "*Linux_ARMv6.tar.gz"
- task: Linux_ARMv6
path: "*Linux_ARMv6.tar.gz"
name: Linux_ARMv6
- path: "*Linux_ARMv7.tar.gz"
- task: Linux_ARMv7
path: "*Linux_ARMv7.tar.gz"
name: Linux_ARMv7
- path: "*macOS_64bit.tar.gz"
- task: Linux_ARM64
path: "*Linux_ARM64.tar.gz"
name: Linux_ARM64
- task: macOS_64bit
path: "*macOS_64bit.tar.gz"
name: macOS_64
- path: "*macOS_ARM64.tar.gz"
- task: macOS_ARM64
path: "*macOS_ARM64.tar.gz"
name: macOS_ARM64
- path: "*Windows_32bit.zip"
name: Windows_X86-32
- path: "*Windows_64bit.zip"
name: Windows_X86-64

steps:
- name: Download combined builds artifact
uses: actions/download-artifact@v3
- name: Checkout repository
uses: actions/checkout@v3

- name: Install Task
uses: arduino/setup-task@v1
with:
name: ${{ env.BUILDS_ARTIFACT }}
path: ${{ env.BUILDS_ARTIFACT }}
repo-token: ${{ secrets.GITHUB_TOKEN }}
version: 3.x

- name: Upload individual build artifact
- name: Build
run: |
PACKAGE_NAME_PREFIX=${{ needs.package-name-prefix.outputs.prefix }}
export PACKAGE_NAME_PREFIX
task dist:${{ matrix.os.task }}

# Transfer builds to artifacts job
- name: Upload build artifact
uses: actions/upload-artifact@v3
with:
path: ${{ env.BUILDS_ARTIFACT }}/${{ matrix.artifact.path }}
name: ${{ matrix.artifact.name }}
path: ${{ env.DIST_DIR }}/${{ matrix.os.path }}
name: ${{ matrix.os.name }}

clean:
needs: artifacts
checksums:
needs:
- build
- package-name-prefix
runs-on: ubuntu-latest

steps:
- name: Remove unneeded combined builds artifact
uses: geekyeggo/delete-artifact@v2
- name: Download build artifacts
uses: actions/download-artifact@v3

- name: Create checksum file
run: |
TAG="${{ needs.package-name-prefix.outputs.prefix }}git-snapshot"
declare -a artifacts=($(ls -d */))
for artifact in ${artifacts[@]}
do
cd $artifact
checksum=$(sha256sum ${{ env.PROJECT_NAME }}_${TAG}*)
cd ..
echo $checksum >> ${TAG}-checksums.txt
done

- name: Upload checksum artifact
uses: actions/upload-artifact@v3
with:
name: ${{ env.BUILDS_ARTIFACT }}
path: ./*checksums.txt
name: checksums
Loading