Skip to content

Commit 8ea1f4c

Browse files
MatteoPologrutoMatteoPologruto
authored
Merge pull request #277 from MatteoPologruto/parallel-dist
Update release workflows using parallelization
2 parents 77b0c91 + 5c9debd commit 8ea1f4c

File tree

8 files changed

+238
-195
lines changed

8 files changed

+238
-195
lines changed

‎workflow-templates/assets/release-go-task/DistTasks.yml

-23
Original file line numberDiff line numberDiff line change
@@ -20,22 +20,8 @@ version: "3"
2020
vars:
2121
CONTAINER: "docker.elastic.co/beats-dev/golang-crossbuild"
2222
GO_VERSION: "1.17.8"
23-
CHECKSUM_FILE: "{{.VERSION}}-checksums.txt"
2423

2524
tasks:
26-
all:
27-
desc: Build for distribution for all platforms
28-
cmds:
29-
- task: Windows_32bit
30-
- task: Windows_64bit
31-
- task: Linux_32bit
32-
- task: Linux_64bit
33-
- task: Linux_ARMv6
34-
- task: Linux_ARMv7
35-
- task: Linux_ARM64
36-
- task: macOS_64bit
37-
- task: macOS_ARM64
38-
3925
Windows_32bit:
4026
desc: Builds Windows 32 bit binaries
4127
dir: "{{.DIST_DIR}}"
@@ -48,7 +34,6 @@ tasks:
4834
-p "{{.BUILD_PLATFORM}}"
4935
5036
zip {{.PACKAGE_NAME}} {{.PLATFORM_DIR}}/{{.PROJECT_NAME}}.exe ../LICENSE.txt -j
51-
sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}}
5237
5338
vars:
5439
PLATFORM_DIR: "{{.PROJECT_NAME}}_windows_386"
@@ -70,7 +55,6 @@ tasks:
7055
-p "{{.BUILD_PLATFORM}}"
7156
7257
zip {{.PACKAGE_NAME}} {{.PLATFORM_DIR}}/{{.PROJECT_NAME}}.exe ../LICENSE.txt -j
73-
sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}}
7458
7559
vars:
7660
PLATFORM_DIR: "{{.PROJECT_NAME}}_windows_amd64"
@@ -92,7 +76,6 @@ tasks:
9276
-p "{{.BUILD_PLATFORM}}"
9377
9478
tar cz -C {{.PLATFORM_DIR}} {{.PROJECT_NAME}} -C ../.. LICENSE.txt -f {{.PACKAGE_NAME}}
95-
sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}}
9679
9780
vars:
9881
PLATFORM_DIR: "{{.PROJECT_NAME}}_linux_amd32"
@@ -114,7 +97,6 @@ tasks:
11497
-p "{{.BUILD_PLATFORM}}"
11598
11699
tar cz -C {{.PLATFORM_DIR}} {{.PROJECT_NAME}} -C ../.. LICENSE.txt -f {{.PACKAGE_NAME}}
117-
sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}}
118100
119101
vars:
120102
PLATFORM_DIR: "{{.PROJECT_NAME}}_linux_amd64"
@@ -136,7 +118,6 @@ tasks:
136118
-p "{{.BUILD_PLATFORM}}"
137119
138120
tar cz -C {{.PLATFORM_DIR}} {{.PROJECT_NAME}} -C ../.. LICENSE.txt -f {{.PACKAGE_NAME}}
139-
sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}}
140121
141122
vars:
142123
PLATFORM_DIR: "{{.PROJECT_NAME}}_linux_arm_7"
@@ -158,7 +139,6 @@ tasks:
158139
-p "{{.BUILD_PLATFORM}}"
159140
160141
tar cz -C {{.PLATFORM_DIR}} {{.PROJECT_NAME}} -C ../.. LICENSE.txt -f {{.PACKAGE_NAME}}
161-
sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}}
162142
163143
vars:
164144
PLATFORM_DIR: "{{.PROJECT_NAME}}_linux_arm_6"
@@ -208,7 +188,6 @@ tasks:
208188
-p "{{.BUILD_PLATFORM}}"
209189
210190
tar cz -C {{.PLATFORM_DIR}} {{.PROJECT_NAME}} -C ../.. LICENSE.txt -f {{.PACKAGE_NAME}}
211-
sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}}
212191
213192
vars:
214193
PLATFORM_DIR: "{{.PROJECT_NAME}}_linux_arm_64"
@@ -230,7 +209,6 @@ tasks:
230209
-p "{{.BUILD_PLATFORM}}"
231210
232211
tar cz -C {{.PLATFORM_DIR}} {{.PROJECT_NAME}} -C ../.. LICENSE.txt -f {{.PACKAGE_NAME}}
233-
sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}}
234212
235213
vars:
236214
PLATFORM_DIR: "{{.PROJECT_NAME}}_osx_darwin_amd64"
@@ -265,7 +243,6 @@ tasks:
265243
-p "{{.BUILD_PLATFORM}}"
266244
267245
tar cz -C {{.PLATFORM_DIR}} {{.PROJECT_NAME}} -C ../.. LICENSE.txt -f {{.PACKAGE_NAME}}
268-
sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}}
269246
270247
vars:
271248
PLATFORM_DIR: "{{.PROJECT_NAME}}_osx_darwin_arm64"

‎workflow-templates/dependabot/workflow-template-copies/.github/workflows/publish-go-nightly-task.yml

+21-18
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,19 @@ jobs:
2222
create-nightly-artifacts:
2323
runs-on: ubuntu-latest
2424

25+
strategy:
26+
matrix:
27+
os:
28+
- Windows_32bit
29+
- Windows_64bit
30+
- Linux_32bit
31+
- Linux_64bit
32+
- Linux_ARMv6
33+
- Linux_ARMv7
34+
- Linux_ARM64
35+
- macOS_64bit
36+
- macOS_ARM64
37+
2538
steps:
2639
- name: Checkout repository
2740
uses: actions/checkout@v3
@@ -35,7 +48,7 @@ jobs:
3548
- name: Build
3649
env:
3750
NIGHTLY: true
38-
run: task dist:all
51+
run: task dist:${{ matrix.os }}
3952

4053
- name: Upload artifacts
4154
uses: actions/upload-artifact@v3
@@ -128,14 +141,10 @@ jobs:
128141
run: |
129142
gon "${{ env.GON_CONFIG_PATH }}"
130143
131-
- name: Re-package binary and output checksum
144+
- name: Re-package binary
132145
id: re-package
133146
working-directory: ${{ env.DIST_DIR }}
134-
# This step performs the following:
135-
# 1. Repackage the signed binary replaced in place by Gon (ignoring the output zip file)
136-
# 2. Recalculate package checksum
137-
# 3. Output the new checksum to include in the nnnnnn-checksums.txt file
138-
# (it cannot be done there because of workflow job parallelization)
147+
# Repackage the signed binary replaced in place by Gon (ignoring the output zip file)
139148
run: |
140149
# GitHub's upload/download-artifact actions don't preserve file permissions,
141150
# so we need to add execution permission back until the action is made to do this.
@@ -145,11 +154,9 @@ jobs:
145154
tar -czvf "$PACKAGE_FILENAME" \
146155
-C "${{ env.PROJECT_NAME }}_osx_${{ matrix.artifact.name }}/" "${{ env.PROJECT_NAME }}" \
147156
-C ../../ LICENSE.txt
148-
CHECKSUM_LINE="$(shasum -a 256 $PACKAGE_FILENAME)"
149157
echo "PACKAGE_FILENAME=$PACKAGE_FILENAME" >> $GITHUB_ENV
150-
echo "::set-output name=checksum-${{ matrix.artifact.name }}::$CHECKSUM_LINE"
151158
152-
- name: Upload artifacts
159+
- name: Upload artifact
153160
uses: actions/upload-artifact@v3
154161
with:
155162
if-no-files-found: error
@@ -167,15 +174,11 @@ jobs:
167174
name: ${{ env.ARTIFACT_NAME }}
168175
path: ${{ env.DIST_DIR }}
169176

170-
- name: Update checksum
177+
- name: Create checksum file
178+
working-directory: ${{ env.DIST_DIR}}
171179
run: |
172-
declare -a checksum_lines=("${{ needs.notarize-macos.outputs.checksum-darwin_amd64 }}" "${{ needs.notarize-macos.outputs.checksum-darwin_arm64 }}")
173-
for checksum_line in "${checksum_lines[@]}"
174-
do
175-
CHECKSUM=$(echo ${checksum_line} | cut -d " " -f 1)
176-
PACKAGE_FILENAME=$(echo ${checksum_line} | cut -d " " -f 2)
177-
perl -pi -w -e "s/.*${PACKAGE_FILENAME}/${CHECKSUM} ${PACKAGE_FILENAME}/g;" ${{ env.DIST_DIR }}/*-checksums.txt
178-
done
180+
TAG="nightly-$(date -u +"%Y%m%d")"
181+
sha256sum ${{ env.PROJECT_NAME }}_${TAG}* > ${TAG}-checksums.txt
179182
180183
- name: Upload release files on Arduino downloads servers
181184
uses: docker://plugins/s3

‎workflow-templates/dependabot/workflow-template-copies/.github/workflows/publish-go-tester-task.yml

+73-51
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,10 @@ on:
2424
repository_dispatch:
2525

2626
env:
27+
# As defined by the Taskfile's PROJECT_NAME variable
28+
PROJECT_NAME: TODO_PROJECT_NAME
2729
# As defined by the Taskfile's DIST_DIR variable
2830
DIST_DIR: dist
29-
BUILDS_ARTIFACT: build-artifacts
3031

3132
jobs:
3233
run-determination:
@@ -52,86 +53,107 @@ jobs:
5253
5354
echo "::set-output name=result::$RESULT"
5455
55-
build:
56+
package-name-prefix:
5657
needs: run-determination
5758
if: needs.run-determination.outputs.result == 'true'
5859
runs-on: ubuntu-latest
59-
60+
outputs:
61+
prefix: ${{ steps.calculation.outputs.prefix }}
6062
steps:
61-
- name: Checkout repository
62-
uses: actions/checkout@v3
63-
64-
- name: Install Task
65-
uses: arduino/setup-task@v1
66-
with:
67-
repo-token: ${{ secrets.GITHUB_TOKEN }}
68-
version: 3.x
69-
70-
- name: Build
63+
- name: package name prefix calculation
64+
id: calculation
7165
run: |
7266
PACKAGE_NAME_PREFIX="test"
7367
if [ "${{ github.event_name }}" = "pull_request" ]; then
7468
PACKAGE_NAME_PREFIX="$PACKAGE_NAME_PREFIX-${{ github.event.number }}"
7569
fi
7670
PACKAGE_NAME_PREFIX="$PACKAGE_NAME_PREFIX-${{ github.sha }}-"
77-
export PACKAGE_NAME_PREFIX
78-
task dist:all
7971
80-
# Transfer builds to artifacts job
81-
- name: Upload combined builds artifact
82-
uses: actions/upload-artifact@v3
83-
with:
84-
path: ${{ env.DIST_DIR }}
85-
name: ${{ env.BUILDS_ARTIFACT }}
72+
echo "::set-output name=prefix::$PACKAGE_NAME_PREFIX"
8673
87-
artifacts:
88-
name: ${{ matrix.artifact.name }} artifact
89-
needs: build
74+
build:
75+
needs: package-name-prefix
76+
name: Build ${{ matrix.os.name }}
9077
runs-on: ubuntu-latest
9178

9279
strategy:
9380
matrix:
94-
artifact:
95-
- path: "*checksums.txt"
96-
name: checksums
97-
- path: "*Linux_32bit.tar.gz"
81+
os:
82+
- task: Windows_32bit
83+
path: "*Windows_32bit.zip"
84+
name: Windows_X86-32
85+
- task: Windows_64bit
86+
path: "*Windows_64bit.zip"
87+
name: Windows_X86-64
88+
- task: Linux_32bit
89+
path: "*Linux_32bit.tar.gz"
9890
name: Linux_X86-32
99-
- path: "*Linux_64bit.tar.gz"
91+
- task: Linux_64bit
92+
path: "*Linux_64bit.tar.gz"
10093
name: Linux_X86-64
101-
- path: "*Linux_ARM64.tar.gz"
102-
name: Linux_ARM64
103-
- path: "*Linux_ARMv6.tar.gz"
94+
- task: Linux_ARMv6
95+
path: "*Linux_ARMv6.tar.gz"
10496
name: Linux_ARMv6
105-
- path: "*Linux_ARMv7.tar.gz"
97+
- task: Linux_ARMv7
98+
path: "*Linux_ARMv7.tar.gz"
10699
name: Linux_ARMv7
107-
- path: "*macOS_64bit.tar.gz"
100+
- task: Linux_ARM64
101+
path: "*Linux_ARM64.tar.gz"
102+
name: Linux_ARM64
103+
- task: macOS_64bit
104+
path: "*macOS_64bit.tar.gz"
108105
name: macOS_64
109-
- path: "*macOS_ARM64.tar.gz"
106+
- task: macOS_ARM64
107+
path: "*macOS_ARM64.tar.gz"
110108
name: macOS_ARM64
111-
- path: "*Windows_32bit.zip"
112-
name: Windows_X86-32
113-
- path: "*Windows_64bit.zip"
114-
name: Windows_X86-64
115109

116110
steps:
117-
- name: Download combined builds artifact
118-
uses: actions/download-artifact@v3
111+
- name: Checkout repository
112+
uses: actions/checkout@v3
113+
114+
- name: Install Task
115+
uses: arduino/setup-task@v1
119116
with:
120-
name: ${{ env.BUILDS_ARTIFACT }}
121-
path: ${{ env.BUILDS_ARTIFACT }}
117+
repo-token: ${{ secrets.GITHUB_TOKEN }}
118+
version: 3.x
122119

123-
- name: Upload individual build artifact
120+
- name: Build
121+
run: |
122+
PACKAGE_NAME_PREFIX=${{ needs.package-name-prefix.outputs.prefix }}
123+
export PACKAGE_NAME_PREFIX
124+
task dist:${{ matrix.os.task }}
125+
126+
# Transfer builds to artifacts job
127+
- name: Upload build artifact
124128
uses: actions/upload-artifact@v3
125129
with:
126-
path: ${{ env.BUILDS_ARTIFACT }}/${{ matrix.artifact.path }}
127-
name: ${{ matrix.artifact.name }}
130+
path: ${{ env.DIST_DIR }}/${{ matrix.os.path }}
131+
name: ${{ matrix.os.name }}
128132

129-
clean:
130-
needs: artifacts
133+
checksums:
134+
needs:
135+
- build
136+
- package-name-prefix
131137
runs-on: ubuntu-latest
132138

133139
steps:
134-
- name: Remove unneeded combined builds artifact
135-
uses: geekyeggo/delete-artifact@v2
140+
- name: Download build artifacts
141+
uses: actions/download-artifact@v3
142+
143+
- name: Create checksum file
144+
run: |
145+
TAG="${{ needs.package-name-prefix.outputs.prefix }}git-snapshot"
146+
declare -a artifacts=($(ls -d */))
147+
for artifact in ${artifacts[@]}
148+
do
149+
cd $artifact
150+
checksum=$(sha256sum ${{ env.PROJECT_NAME }}_${TAG}*)
151+
cd ..
152+
echo $checksum >> ${TAG}-checksums.txt
153+
done
154+
155+
- name: Upload checksum artifact
156+
uses: actions/upload-artifact@v3
136157
with:
137-
name: ${{ env.BUILDS_ARTIFACT }}
158+
path: ./*checksums.txt
159+
name: checksums

0 commit comments

Comments
 (0)