Skip to content

Commit 543323f

Browse files
lucasssvazlucasssvaz
committed
Merge branch 'ci/cron_optimization'
2 parents 6683a0d + 06f2fa0 commit 543323f

File tree

5 files changed

+199
-113
lines changed

5 files changed

+199
-113
lines changed

‎.github/workflows/cron.yml

+165-99
Original file line numberDiff line numberDiff line change
@@ -1,144 +1,210 @@
11
name: Cron Build
22

3-
on:
3+
on:
44
schedule:
55
# ┌───────────── minute (0 - 59)
66
# │ ┌───────────── hour (0 - 23)
77
# │ │ ┌───────────── day of the month (1 - 31)
88
# │ │ │ ┌───────────── month (1 - 12 or JAN-DEC)
99
# │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT)
10-
# │ │ │ │ │
10+
# │ │ │ │ │
1111
# │ │ │ │ │
1212
# │ │ │ │ │
1313
# * * * * *
1414
- cron: '0 */6 * * *'
15+
workflow_dispatch:
16+
inputs:
17+
idf_branch:
18+
description: 'IDF branch to build (JSON list of strings e.g. ["release/v5.1"] or "all" to build all branches)'
19+
required: true
20+
default: 'all'
21+
target:
22+
description: 'Target to build (JSON list of strings e.g. ["esp32", "esp32s2"] or "all" to build all targets)'
23+
required: true
24+
default: 'all'
1525

1626
defaults:
1727
run:
1828
shell: bash
1929

2030
jobs:
21-
run:
22-
name: Build with IDF ${{ matrix.idf_branch }}
31+
gen-matrix:
32+
name: Generate matrix
2333
runs-on: ubuntu-latest
24-
34+
outputs:
35+
matrix: ${{ steps.gen-matrix.outputs.matrix }}
36+
branches: ${{ steps.gen-matrix.outputs.branches }}
37+
steps:
38+
- name: Generate matrix
39+
id: gen-matrix
40+
run: |
41+
apt update && apt install jq
42+
43+
# Change this based on the IDF branches we want to build
44+
all_branches="[\"release/v5.1\"]"
45+
46+
# Change this based on the COMMON targets for all branches we want to build.
47+
common_targets="[\"esp32\", \"esp32s2\", \"esp32s3\", \"esp32c2\", \"esp32c3\", \"esp32c6\", \"esp32h2\"]"
48+
49+
# For additional targets per branch, add them here
50+
additional_targets="[{\"idf_branch\": \"release/v5.3\", \"target\": \"esp32p4\"}]"
51+
52+
branches="${{ github.event.inputs.idf_branch }}"
53+
targets="${{ github.event.inputs.target }}"
54+
55+
if [ -z "$branches" ] || [ "$branches" == "all" ]; then
56+
branches=$all_branches
57+
fi
58+
59+
if [ -z "$targets" ]; then
60+
targets="all"
61+
fi
62+
63+
echo "Inputs:"
64+
echo "idf_branch: ${{ github.event.inputs.idf_branch }}"
65+
echo "target: ${{ github.event.inputs.target }}"
66+
67+
matrix="{"
68+
matrix+="\"idf_branch\": $branches,"
69+
70+
if [ "$targets" == "all" ]; then
71+
matrix+="\"target\": $common_targets,"
72+
matrix+="\"include\": "
73+
# Add all additional targets that are in the selected branches
74+
matrix+=$(echo $additional_targets | jq --argjson branches "$branches" '[.[] | select(.idf_branch as $branch | $branches | index($branch))]')
75+
else
76+
matrix+="\"target\": $targets"
77+
fi
78+
79+
matrix+="}"
80+
81+
echo "Branches: $branches"
82+
echo "Targets: $targets"
83+
echo "Matrix:"
84+
echo "$matrix" | jq .
85+
echo "matrix=$matrix" >> $GITHUB_OUTPUT
86+
echo "branches=$branches" >> $GITHUB_OUTPUT
87+
88+
build-libs:
89+
name: Build with IDF ${{ matrix.idf_branch }} for ${{ matrix.target }}
90+
runs-on: ubuntu-latest
91+
needs: gen-matrix
2592
strategy:
2693
fail-fast: false
27-
matrix:
28-
idf_branch: [release/v5.1, release/v4.4] #, release/v3.3]
94+
matrix: ${{ fromJson(needs.gen-matrix.outputs.matrix) }}
2995
steps:
3096
- uses: actions/checkout@v4
3197
with:
3298
fetch-depth: 0
99+
100+
- name: Check if result should be deployed
101+
id: check
102+
env:
103+
GITHUB_TOKEN: ${{ secrets.PUSH_TOKEN }}
104+
GIT_AUTHOR_EMAIL: ${{ secrets.PUSH_EMAIL }}
105+
GIT_COMMITTER_EMAIL: ${{ secrets.PUSH_EMAIL }}
106+
IDF_BRANCH: ${{ matrix.idf_branch }}
107+
run: bash ./tools/check-deploy-needed.sh
108+
33109
- name: Install dependencies
110+
if: env.libs_has_commit == '0' || env.ar_has_commit == '0'
34111
run: bash ./tools/prepare-ci.sh
112+
35113
- name: Build
36114
env:
37115
GITHUB_TOKEN: ${{ secrets.PUSH_TOKEN }}
38116
GIT_AUTHOR_EMAIL: ${{ secrets.PUSH_EMAIL }}
39117
GIT_COMMITTER_EMAIL: ${{ secrets.PUSH_EMAIL }}
40118
IDF_BRANCH: ${{ matrix.idf_branch }}
119+
TARGETS: ${{ matrix.target }}
120+
if: env.libs_has_commit == '0' || env.ar_has_commit == '0'
41121
run: |
42-
git checkout ${{ matrix.idf_branch }} || echo "Using master branch"
122+
git checkout ${{ env.IDF_BRANCH }} || echo "Using master branch"
43123
bash ./tools/cron.sh
124+
44125
- name: Upload build
45-
if: failure()
126+
if: failure() && (env.libs_has_commit == '0' || env.ar_has_commit == '0')
46127
uses: actions/upload-artifact@v4
47128
with:
48-
name: build
129+
name: build-${{ matrix.idf_branch }}-${{ matrix.target }}
49130
path: build
50-
- name: Upload archive
131+
132+
- name: Upload library files
133+
if: env.libs_has_commit == '0' || env.ar_has_commit == '0'
51134
uses: actions/upload-artifact@v4
52135
with:
53-
name: artifacts
136+
name: libs-${{ matrix.idf_branch }}-${{ matrix.target }}
54137
path: dist
55138

139+
combine-artifacts:
140+
name: Combine artifacts for IDF ${{ matrix.idf_branch }}
141+
runs-on: ubuntu-latest
142+
needs: [gen-matrix, build-libs]
143+
strategy:
144+
fail-fast: false
145+
matrix:
146+
idf_branch: ${{ fromJson(needs.gen-matrix.outputs.branches) }}
147+
steps:
148+
- uses: actions/checkout@v4
149+
with:
150+
fetch-depth: 0
151+
152+
- name: Check if result should be deployed
153+
id: check
154+
env:
155+
GITHUB_TOKEN: ${{ secrets.PUSH_TOKEN }}
156+
GIT_AUTHOR_EMAIL: ${{ secrets.PUSH_EMAIL }}
157+
GIT_COMMITTER_EMAIL: ${{ secrets.PUSH_EMAIL }}
158+
IDF_BRANCH: ${{ matrix.idf_branch }}
159+
run: bash ./tools/check-deploy-needed.sh
160+
161+
- name: Download artifacts
162+
if: env.libs_has_commit == '0' || env.ar_has_commit == '0'
163+
uses: actions/download-artifact@v4
164+
with:
165+
path: dist
166+
pattern: libs-${{ matrix.idf_branch }}-*
167+
merge-multiple: true
168+
169+
- name: Combine artifacts
170+
if: env.libs_has_commit == '0' || env.ar_has_commit == '0'
171+
shell: bash
172+
run: |
173+
set -e
174+
mkdir -p out
175+
find dist -name 'arduino-esp32-libs-esp*.tar.gz'
176+
for file in $files; do
177+
tar zxvf $file -C out
178+
cat out/tools/esp32-arduino-libs/versions.txt >> out/tools/esp32-arduino-libs/versions_full.txt
179+
done
180+
181+
# Merge versions.txt files
182+
awk -i inplace '!seen[$0]++' out/tools/esp32-arduino-libs/versions_full.txt
183+
mv -f out/tools/esp32-arduino-libs/versions_full.txt out/tools/esp32-arduino-libs/versions.txt
184+
185+
cd out/tools/esp32-arduino-libs && tar zcf ../../../dist/esp32-arduino-libs.tar.gz * && cd ../../..
186+
cp out/package_esp32_index.template.json dist/package_esp32_index.template.json
187+
188+
- name: Upload full esp32-arduino-libs archive
189+
if: env.libs_has_commit == '0' || env.ar_has_commit == '0'
190+
uses: actions/upload-artifact@v4
191+
with:
192+
name: esp32-arduino-libs
193+
path: dist/esp32-arduino-libs.tar.gz
56194

57-
# check:
58-
# name: Check if result should be deployed
59-
# runs-on: ubuntu-latest
60-
# strategy:
61-
# matrix:
62-
# branch: [release/v5.1, release/v4.4] #, release/v3.3]
63-
# outputs:
64-
# idf_branch: ${{ steps.check.outputs.idf_branch }}
65-
# idf_commit: ${{ steps.check.outputs.idf_commit }}
66-
# ar_branch: ${{ steps.check.outputs.ar_branch }}
67-
# ar_new_commit_message: ${{ steps.check.outputs.ar_new_commit_message }}
68-
# ar_new_branch_name: ${{ steps.check.outputs.ar_new_branch_name }}
69-
# ar_new_pr_title: ${{ steps.check.outputs.ar_new_pr_title }}
70-
# ar_has_commit: ${{ steps.check.outputs.ar_has_commit }}
71-
# ar_has_branch: ${{ steps.check.outputs.ar_has_branch }}
72-
# ar_has_pr: ${{ steps.check.outputs.ar_has_pr }}
73-
# libs_version: ${{ steps.check.outputs.libs_version }}
74-
# libs_has_commit: ${{ steps.check.outputs.libs_has_commit }}
75-
# libs_has_branch: ${{ steps.check.outputs.libs_has_branch }}
76-
# steps:
77-
# - uses: actions/checkout@v3
78-
# - id: check
79-
# env:
80-
# GITHUB_TOKEN: ${{ secrets.PUSH_TOKEN }}
81-
# GIT_AUTHOR_EMAIL: ${{ secrets.PUSH_EMAIL }}
82-
# GIT_COMMITTER_EMAIL: ${{ secrets.PUSH_EMAIL }}
83-
# IDF_BRANCH: ${{ matrix.idf_branch }}
84-
# run: bash ./tools/check-deploy-needed.sh
85-
86-
# build:
87-
# name: Build Libs for ${{ matrix.target }}
88-
# runs-on: ubuntu-latest
89-
# needs: check
90-
# if: needs.check.outputs.libs_has_commit == '0' || needs.check.outputs.ar_has_commit == '0'
91-
# strategy:
92-
# matrix:
93-
# target: [esp32, esp32s2, esp32s3, esp32c3, esp32c6, esp32h2]
94-
# fail-fast: false
95-
# steps:
96-
# - uses: actions/checkout@v3
97-
# # - name: Install dependencies
98-
# # run: bash ./tools/prepare-ci.sh
99-
# - shell: bash
100-
# name: Build Libs for ${{ matrix.target }}
101-
# run: echo ${{ matrix.target }}
102-
# # run: bash ./build.sh -t ${{ matrix.target }}
103-
# # - name: Upload archive
104-
# # uses: actions/upload-artifact@v3
105-
# # with:
106-
# # name: artifacts
107-
# # path: dist
108-
109-
# deploy:
110-
# name: Deploy build
111-
# runs-on: ubuntu-latest
112-
# needs: [check, build]
113-
# steps:
114-
# - uses: actions/checkout@v3
115-
# - shell: bash
116-
# env:
117-
# GITHUB_TOKEN: ${{ secrets.PUSH_TOKEN }}
118-
# GIT_AUTHOR_EMAIL: ${{ secrets.PUSH_EMAIL }}
119-
# GIT_COMMITTER_EMAIL: ${{ secrets.PUSH_EMAIL }}
120-
# IDF_BRANCH: ${{ needs.check.outputs.idf_branch }}
121-
# IDF_COMMIT: ${{ needs.check.outputs.idf_commit }}
122-
# AR_BRANCH: ${{ needs.check.outputs.ar_branch }}
123-
# AR_NEW_COMMIT_MESSAGE: ${{ needs.check.outputs.ar_new_commit_message }}
124-
# AR_NEW_BRANCH_NAME: ${{ needs.check.outputs.ar_new_branch_name }}
125-
# AR_NEW_PR_TITLE: ${{ needs.check.outputs.ar_new_pr_title }}
126-
# AR_HAS_COMMIT: ${{ needs.check.outputs.ar_has_commit }}
127-
# AR_HAS_BRANCH: ${{ needs.check.outputs.ar_has_branch }}
128-
# AR_HAS_PR: ${{ needs.check.outputs.ar_has_pr }}
129-
# LIBS_VERSION: ${{ needs.check.outputs.libs_version }}
130-
# LIBS_HAS_COMMIT: ${{ needs.check.outputs.libs_has_commit }}
131-
# LIBS_HAS_BRANCH: ${{ needs.check.outputs.libs_has_branch }}
132-
# run: |
133-
# echo "IDF_COMMIT: $IDF_COMMIT"
134-
# echo "AR_BRANCH: $AR_BRANCH"
135-
# echo "AR_NEW_COMMIT_MESSAGE: $AR_NEW_COMMIT_MESSAGE"
136-
# echo "AR_NEW_BRANCH_NAME: $AR_NEW_BRANCH_NAME"
137-
# echo "AR_NEW_PR_TITLE: $AR_NEW_PR_TITLE"
138-
# echo "AR_HAS_COMMIT: $AR_HAS_COMMIT"
139-
# echo "AR_HAS_BRANCH: $AR_HAS_BRANCH"
140-
# echo "AR_HAS_PR: $AR_HAS_PR"
141-
# echo "LIBS_VERSION: $LIBS_VERSION"
142-
# echo "LIBS_HAS_COMMIT: $LIBS_HAS_COMMIT"
143-
# echo "LIBS_HAS_BRANCH: $LIBS_HAS_BRANCH"
195+
- name: Upload package_esp32_index.template.json
196+
if: env.libs_has_commit == '0' || env.ar_has_commit == '0'
197+
uses: actions/upload-artifact@v4
198+
with:
199+
name: package-esp32-index-json
200+
path: dist/package_esp32_index.template.json
144201

202+
- name: Push changes
203+
if: github.repository == 'espressif/esp32-arduino-lib-builder' && (env.libs_has_commit == '0' || env.ar_has_commit == '0')
204+
env:
205+
GITHUB_TOKEN: ${{ secrets.PUSH_TOKEN }}
206+
GIT_AUTHOR_EMAIL: ${{ secrets.PUSH_EMAIL }}
207+
GIT_COMMITTER_EMAIL: ${{ secrets.PUSH_EMAIL }}
208+
IDF_BRANCH: ${{ matrix.idf_branch }}
209+
run: |
210+
bash ./tools/push-to-arduino.sh

‎build.sh

+5
Original file line numberDiff line numberDiff line change
@@ -299,13 +299,15 @@ done
299299

300300
# update package_esp32_index.template.json
301301
if [ "$BUILD_TYPE" = "all" ]; then
302+
echo "* Generating package_esp32_index.template.json..."
302303
python3 ./tools/gen_tools_json.py -i "$IDF_PATH" -j "$AR_COMPS/arduino/package/package_esp32_index.template.json" -o "$AR_OUT/"
303304
python3 ./tools/gen_tools_json.py -i "$IDF_PATH" -o "$TOOLS_JSON_OUT/"
304305
if [ $? -ne 0 ]; then exit 1; fi
305306
fi
306307

307308
# Generate PlatformIO manifest file
308309
if [ "$BUILD_TYPE" = "all" ]; then
310+
echo "* Generating PlatformIO manifest file..."
309311
pushd $IDF_PATH
310312
ibr=$(git describe --all 2>/dev/null)
311313
ic=$(git -C "$IDF_PATH" rev-parse --short HEAD)
@@ -316,18 +318,21 @@ fi
316318

317319
# copy everything to arduino-esp32 installation
318320
if [ $COPY_OUT -eq 1 ] && [ -d "$ESP32_ARDUINO" ]; then
321+
echo "* Copying to Arduino..."
319322
./tools/copy-to-arduino.sh
320323
if [ $? -ne 0 ]; then exit 1; fi
321324
fi
322325

323326
# push changes to esp32-arduino-libs and create pull request into arduino-esp32
324327
if [ $DEPLOY_OUT -eq 1 ]; then
328+
echo "* Pushing to Arduino..."
325329
./tools/push-to-arduino.sh
326330
if [ $? -ne 0 ]; then exit 1; fi
327331
fi
328332

329333
# archive the build
330334
if [ $ARCHIVE_OUT -eq 1 ]; then
335+
echo "* Archiving build..."
331336
./tools/archive-build.sh "$TARGET"
332337
if [ $? -ne 0 ]; then exit 1; fi
333338
fi

‎tools/check-deploy-needed.sh

+23-12
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
#/bin/bash
22

3+
if [ "$GITHUB_EVENT_NAME" != "schedule" ] && [ "$GITHUB_EVENT_NAME" != "workflow_dispatch" ] && [ "$GITHUB_EVENT_NAME" != "repository_dispatch" -o "$GITHUB_EVENT_ACTION" != "deploy" ]; then
4+
echo "Wrong event '$GITHUB_EVENT_NAME'!"
5+
exit 1
6+
fi
7+
38
source ./tools/config.sh
49

510
IDF_COMMIT=`github_last_commit "$IDF_REPO" "$IDF_BRANCH"`
@@ -70,16 +75,22 @@ echo "LIBS_VERSION: $LIBS_VERSION"
7075
echo "LIBS_HAS_COMMIT: $LIBS_HAS_COMMIT"
7176
echo "LIBS_HAS_BRANCH: $LIBS_HAS_BRANCH"
7277

73-
if [ ! -x $GITHUB_OUTPUT ]; then
74-
echo "idf_commit=$IDF_COMMIT" >> "$GITHUB_OUTPUT"
75-
echo "ar_branch=$AR_BRANCH" >> "$GITHUB_OUTPUT"
76-
echo "ar_new_commit_message=$AR_NEW_COMMIT_MESSAGE" >> "$GITHUB_OUTPUT"
77-
echo "ar_new_branch_name=$AR_NEW_BRANCH_NAME" >> "$GITHUB_OUTPUT"
78-
echo "ar_new_pr_title=$AR_NEW_PR_TITLE" >> "$GITHUB_OUTPUT"
79-
echo "ar_has_commit=$AR_HAS_COMMIT" >> "$GITHUB_OUTPUT"
80-
echo "ar_has_branch=$AR_HAS_BRANCH" >> "$GITHUB_OUTPUT"
81-
echo "ar_has_pr=$AR_HAS_PR" >> "$GITHUB_OUTPUT"
82-
echo "libs_version=$LIBS_VERSION" >> "$GITHUB_OUTPUT"
83-
echo "libs_has_commit=$LIBS_HAS_COMMIT" >> "$GITHUB_OUTPUT"
84-
echo "libs_has_branch=$LIBS_HAS_BRANCH" >> "$GITHUB_OUTPUT"
78+
if [ ! -x $GITHUB_ENV ]; then
79+
echo "idf_commit=$IDF_COMMIT" >> "$GITHUB_ENV"
80+
echo "ar_branch=$AR_BRANCH" >> "$GITHUB_ENV"
81+
echo "ar_new_commit_message=$AR_NEW_COMMIT_MESSAGE" >> "$GITHUB_ENV"
82+
echo "ar_new_branch_name=$AR_NEW_BRANCH_NAME" >> "$GITHUB_ENV"
83+
echo "ar_new_pr_title=$AR_NEW_PR_TITLE" >> "$GITHUB_ENV"
84+
echo "ar_has_commit=$AR_HAS_COMMIT" >> "$GITHUB_ENV"
85+
echo "ar_has_branch=$AR_HAS_BRANCH" >> "$GITHUB_ENV"
86+
echo "ar_has_pr=$AR_HAS_PR" >> "$GITHUB_ENV"
87+
echo "libs_version=$LIBS_VERSION" >> "$GITHUB_ENV"
88+
echo "libs_has_commit=$LIBS_HAS_COMMIT" >> "$GITHUB_ENV"
89+
echo "libs_has_branch=$LIBS_HAS_BRANCH" >> "$GITHUB_ENV"
90+
fi
91+
92+
if [ "$LIBS_HAS_COMMIT" == "0" ] || [ "$AR_HAS_COMMIT" == "0" ]; then
93+
echo "Deploy needed"
94+
else
95+
echo "Deploy not needed. Skipping..."
8596
fi

0 commit comments

Comments
 (0)