Skip to content

Commit 9f3010f

Browse files
lucasssvazP-R-O-C-H-Y
lucasssvaz
and
P-R-O-C-H-Y
authored
ci(json): Improve requirement checking (#10554)
* ci(json): Change requirements for sketches that use WiFi * ci(checks): Optimize requirement checking * fix(tests): Additional checks before running tests --------- Co-authored-by: Jan Procházka <[email protected]>
1 parent d1df696 commit 9f3010f

File tree

93 files changed

+357
-352
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

93 files changed

+357
-352
lines changed

Diff for: ‎.github/scripts/install-platformio-esp32.sh

+7-86
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@ TOOLCHAIN_VERSION="12.2.0+20230208"
77
ESPTOOLPY_VERSION="~1.40501.0"
88
ESPRESSIF_ORGANIZATION_NAME="espressif"
99
SDKCONFIG_DIR="$PLATFORMIO_ESP32_PATH/tools/esp32-arduino-libs"
10+
SCRIPTS_DIR="./.github/scripts"
11+
COUNT_SKETCHES="${SCRIPTS_DIR}/sketch_utils.sh count"
12+
CHECK_REQUIREMENTS="${SCRIPTS_DIR}/sketch_utils.sh check_requirements"
1013

1114
echo "Installing Python Wheel ..."
1215
pip install wheel > /dev/null 2>&1
@@ -74,64 +77,6 @@ function build_pio_sketch(){ # build_pio_sketch <board> <options> <path-to-ino>
7477
python -m platformio ci --board "$board" "$sketch_dir" --project-option="$options"
7578
}
7679

77-
function count_sketches(){ # count_sketches <examples-path>
78-
local examples="$1"
79-
rm -rf sketches.txt
80-
if [ ! -d "$examples" ]; then
81-
touch sketches.txt
82-
return 0
83-
fi
84-
local sketches=$(find $examples -name *.ino)
85-
local sketchnum=0
86-
for sketch in $sketches; do
87-
local sketchdir=$(dirname $sketch)
88-
local sketchdirname=$(basename $sketchdir)
89-
local sketchname=$(basename $sketch)
90-
if [[ "${sketchdirname}.ino" != "$sketchname" ]]; then
91-
continue
92-
elif [ -f $sketchdir/ci.json ]; then
93-
# If the target is listed as false, skip the sketch. Otherwise, include it.
94-
is_target=$(jq -r '.targets[esp32]' $sketchdir/ci.json)
95-
if [[ "$is_target" == "false" ]]; then
96-
continue
97-
fi
98-
99-
# Check if the sketch requires any configuration options (AND)
100-
requirements=$(jq -r '.requires[]? // empty' $sketchdir/ci.json)
101-
if [[ "$requirements" != "null" && "$requirements" != "" ]]; then
102-
for requirement in $requirements; do
103-
requirement=$(echo $requirement | xargs)
104-
found_line=$(grep -E "^$requirement" "$SDKCONFIG_DIR/esp32/sdkconfig")
105-
if [[ "$found_line" == "" ]]; then
106-
continue 2
107-
fi
108-
done
109-
fi
110-
111-
# Check if the sketch requires any configuration options (OR)
112-
requirements_or=$(jq -r '.requires_any[]? // empty' $sketchdir/ci.json)
113-
if [[ "$requirements_or" != "null" && "$requirements_or" != "" ]]; then
114-
found=false
115-
for requirement in $requirements_or; do
116-
requirement=$(echo $requirement | xargs)
117-
found_line=$(grep -E "^$requirement" "$SDKCONFIG_DIR/esp32/sdkconfig")
118-
if [[ "$found_line" != "" ]]; then
119-
found=true
120-
break
121-
fi
122-
done
123-
if [[ "$found" == "false" ]]; then
124-
continue
125-
fi
126-
fi
127-
fi
128-
129-
echo $sketch >> sketches.txt
130-
sketchnum=$(($sketchnum + 1))
131-
done
132-
return $sketchnum
133-
}
134-
13580
function build_pio_sketches(){ # build_pio_sketches <board> <options> <examples-path> <chunk> <total-chunks>
13681
if [ "$#" -lt 3 ]; then
13782
echo "ERROR: Illegal number of parameters"
@@ -160,7 +105,7 @@ function build_pio_sketches(){ # build_pio_sketches <board> <options> <examples-
160105
fi
161106

162107
set +e
163-
count_sketches "$examples"
108+
${COUNT_SKETCHES} "$examples" "esp32"
164109
local sketchcount=$?
165110
set -e
166111
local sketches=$(cat sketches.txt)
@@ -204,33 +149,9 @@ function build_pio_sketches(){ # build_pio_sketches <board> <options> <examples-
204149
continue
205150
fi
206151

207-
# Check if the sketch requires any configuration options (AND)
208-
requirements=$(jq -r '.requires[]? // empty' $sketchdir/ci.json)
209-
if [[ "$requirements" != "null" && "$requirements" != "" ]]; then
210-
for requirement in $requirements; do
211-
requirement=$(echo $requirement | xargs)
212-
found_line=$(grep -E "^$requirement" "$SDKCONFIG_DIR/esp32/sdkconfig")
213-
if [[ "$found_line" == "" ]]; then
214-
continue 2
215-
fi
216-
done
217-
fi
218-
219-
# Check if the sketch requires any configuration options (OR)
220-
requirements_or=$(jq -r '.requires_any[]? // empty' $sketchdir/ci.json)
221-
if [[ "$requirements_or" != "null" && "$requirements_or" != "" ]]; then
222-
found=false
223-
for requirement in $requirements_or; do
224-
requirement=$(echo $requirement | xargs)
225-
found_line=$(grep -E "^$requirement" "$SDKCONFIG_DIR/esp32/sdkconfig")
226-
if [[ "$found_line" != "" ]]; then
227-
found=true
228-
break
229-
fi
230-
done
231-
if [[ "$found" == "false" ]]; then
232-
continue
233-
fi
152+
local has_requirements=$(${CHECK_REQUIREMENTS} $sketchdir "$SDKCONFIG_DIR/esp32/sdkconfig")
153+
if [ "$has_requirements" == "0" ]; then
154+
continue
234155
fi
235156
fi
236157

Diff for: ‎.github/scripts/sketch_utils.sh

+53-56
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,49 @@ else
88
SDKCONFIG_DIR="tools/esp32-arduino-libs"
99
fi
1010

11+
function check_requirements(){ # check_requirements <sketchdir> <sdkconfig_path>
12+
local sketchdir=$1
13+
local sdkconfig_path=$2
14+
local has_requirements=1
15+
16+
if [ ! -f "$sdkconfig_path" ] || [ ! -f "$sketchdir/ci.json" ]; then
17+
echo "ERROR: sdkconfig or ci.json not found" 1>&2
18+
# Return 1 on error to force the sketch to be built and fail. This way the
19+
# CI will fail and the user will know that the sketch has a problem.
20+
else
21+
# Check if the sketch requires any configuration options (AND)
22+
local requirements=$(jq -r '.requires[]? // empty' "$sketchdir/ci.json")
23+
if [[ "$requirements" != "null" && "$requirements" != "" ]]; then
24+
for requirement in $requirements; do
25+
requirement=$(echo $requirement | xargs)
26+
found_line=$(grep -E "^$requirement" "$sdkconfig_path")
27+
if [[ "$found_line" == "" ]]; then
28+
has_requirements=0
29+
fi
30+
done
31+
fi
32+
33+
# Check if the sketch requires any configuration options (OR)
34+
local requirements_or=$(jq -r '.requires_any[]? // empty' "$sketchdir/ci.json")
35+
if [[ "$requirements_or" != "null" && "$requirements_or" != "" ]]; then
36+
local found=false
37+
for requirement in $requirements_or; do
38+
requirement=$(echo $requirement | xargs)
39+
found_line=$(grep -E "^$requirement" "$sdkconfig_path")
40+
if [[ "$found_line" != "" ]]; then
41+
found=true
42+
break
43+
fi
44+
done
45+
if [[ "$found" == "false" ]]; then
46+
has_requirements=0
47+
fi
48+
fi
49+
fi
50+
51+
echo $has_requirements
52+
}
53+
1154
function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [extra-options]
1255
while [ ! -z "$1" ]; do
1356
case "$1" in
@@ -171,35 +214,10 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
171214
exit 0
172215
fi
173216

174-
# Check if the sketch requires any configuration options (AND)
175-
requirements=$(jq -r '.requires[]? // empty' $sketchdir/ci.json)
176-
if [[ "$requirements" != "null" && "$requirements" != "" ]]; then
177-
for requirement in $requirements; do
178-
requirement=$(echo $requirement | xargs)
179-
found_line=$(grep -E "^$requirement" "$SDKCONFIG_DIR/$target/sdkconfig")
180-
if [[ "$found_line" == "" ]]; then
181-
echo "Target $target does not meet the requirement $requirement for $sketchname. Skipping."
182-
exit 0
183-
fi
184-
done
185-
fi
186-
187-
# Check if the sketch excludes any configuration options (OR)
188-
requirements_or=$(jq -r '.requires_any[]? // empty' $sketchdir/ci.json)
189-
if [[ "$requirements_or" != "null" && "$requirements_or" != "" ]]; then
190-
found=false
191-
for requirement in $requirements_or; do
192-
requirement=$(echo $requirement | xargs)
193-
found_line=$(grep -E "^$requirement" "$SDKCONFIG_DIR/$target/sdkconfig")
194-
if [[ "$found_line" != "" ]]; then
195-
found=true
196-
break
197-
fi
198-
done
199-
if [[ "$found" == "false" ]]; then
200-
echo "Target $target meets none of the requirements in requires_any for $sketchname. Skipping."
201-
exit 0
202-
fi
217+
local has_requirements=$(check_requirements "$sketchdir" "$SDKCONFIG_DIR/$target/sdkconfig")
218+
if [ "$has_requirements" == "0" ]; then
219+
echo "Target $target does not meet the requirements for $sketchname. Skipping."
220+
exit 0
203221
fi
204222
fi
205223

@@ -348,33 +366,9 @@ function count_sketches(){ # count_sketches <path> [target] [file] [ignore-requi
348366
fi
349367

350368
if [ "$ignore_requirements" != "1" ]; then
351-
# Check if the sketch requires any configuration options (AND)
352-
requirements=$(jq -r '.requires[]? // empty' $sketchdir/ci.json)
353-
if [[ "$requirements" != "null" && "$requirements" != "" ]]; then
354-
for requirement in $requirements; do
355-
requirement=$(echo $requirement | xargs)
356-
found_line=$(grep -E "^$requirement" $SDKCONFIG_DIR/$target/sdkconfig)
357-
if [[ "$found_line" == "" ]]; then
358-
continue 2
359-
fi
360-
done
361-
fi
362-
363-
# Check if the sketch excludes any configuration options (OR)
364-
requirements_or=$(jq -r '.requires_any[]? // empty' $sketchdir/ci.json)
365-
if [[ "$requirements_or" != "null" && "$requirements_or" != "" ]]; then
366-
found=false
367-
for requirement in $requirements_or; do
368-
requirement=$(echo $requirement | xargs)
369-
found_line=$(grep -E "^$requirement" $SDKCONFIG_DIR/$target/sdkconfig)
370-
if [[ "$found_line" != "" ]]; then
371-
found=true
372-
break
373-
fi
374-
done
375-
if [[ "$found" == "false" ]]; then
376-
continue 2
377-
fi
369+
local has_requirements=$(check_requirements "$sketchdir" "$SDKCONFIG_DIR/$target/sdkconfig")
370+
if [ "$has_requirements" == "0" ]; then
371+
continue
378372
fi
379373
fi
380374
fi
@@ -552,6 +546,7 @@ Available commands:
552546
count: Count sketches.
553547
build: Build a sketch.
554548
chunk_build: Build a chunk of sketches.
549+
check_requirements: Check if target meets sketch requirements.
555550
"
556551

557552
cmd=$1
@@ -569,6 +564,8 @@ case "$cmd" in
569564
;;
570565
"chunk_build") build_sketches $*
571566
;;
567+
"check_requirements") check_requirements $*
568+
;;
572569
*)
573570
echo "ERROR: Unrecognized command"
574571
echo "$USAGE"

Diff for: ‎.github/scripts/tests_run.sh

+14-33
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ function run_test() {
99
local sketchname=$(basename $sketchdir)
1010
local result=0
1111
local error=0
12+
local sdkconfig_path
1213

1314
if [ $options -eq 0 ] && [ -f $sketchdir/ci.json ]; then
1415
len=`jq -r --arg target $target '.fqbn[$target] | length' $sketchdir/ci.json`
@@ -20,9 +21,9 @@ function run_test() {
2021
fi
2122

2223
if [ $len -eq 1 ]; then
23-
SDKCONFIG_PATH="$HOME/.arduino/tests/$sketchname/build.tmp/sdkconfig"
24+
sdkconfig_path="$HOME/.arduino/tests/$sketchname/build.tmp/sdkconfig"
2425
else
25-
SDKCONFIG_PATH="$HOME/.arduino/tests/$sketchname/build0.tmp/sdkconfig"
26+
sdkconfig_path="$HOME/.arduino/tests/$sketchname/build0.tmp/sdkconfig"
2627
fi
2728

2829
if [ -f $sketchdir/ci.json ]; then
@@ -35,39 +36,19 @@ function run_test() {
3536
printf "\n\n\n"
3637
return 0
3738
fi
39+
fi
3840

39-
# Check if the sketch requires any configuration options (AND)
40-
requirements=$(jq -r '.requires[]? // empty' $sketchdir/ci.json)
41-
if [[ "$requirements" != "null" && "$requirements" != "" ]]; then
42-
for requirement in $requirements; do
43-
requirement=$(echo $requirement | xargs)
44-
found_line=$(grep -E "^$requirement" "$SDKCONFIG_PATH")
45-
if [[ "$found_line" == "" ]]; then
46-
printf "\033[93mTarget $target does not meet the requirement $requirement for $sketchname. Skipping.\033[0m\n"
47-
printf "\n\n\n"
48-
return 0
49-
fi
50-
done
51-
fi
41+
if [ ! -f $sdkconfig_path ]; then
42+
printf "\033[93mSketch $sketchname not built\nMight be due to missing target requirements or build failure\033[0m\n"
43+
printf "\n\n\n"
44+
return 0
45+
fi
5246

53-
# Check if the sketch requires any configuration options (OR)
54-
requirements_or=$(jq -r '.requires_any[]? // empty' $sketchdir/ci.json)
55-
if [[ "$requirements_or" != "null" && "$requirements_or" != "" ]]; then
56-
found=false
57-
for requirement in $requirements_or; do
58-
requirement=$(echo $requirement | xargs)
59-
found_line=$(grep -E "^$requirement" "$SDKCONFIG_PATH")
60-
if [[ "$found_line" != "" ]]; then
61-
found=true
62-
break
63-
fi
64-
done
65-
if [[ "$found" == "false" ]]; then
66-
printf "\033[93mTarget $target meets none of the requirements in requires_any for $sketchname. Skipping.\033[0m\n"
67-
printf "\n\n\n"
68-
return 0
69-
fi
70-
fi
47+
local right_target=$(grep -E "^CONFIG_IDF_TARGET=\"$target\"$" "$sdkconfig_path")
48+
if [ -z "$right_target" ]; then
49+
printf "\033[91mError: Sketch $sketchname compiled for different target\n\033[0m\n"
50+
printf "\n\n\n"
51+
return 1
7152
fi
7253

7354
if [ $len -eq 1 ]; then

Diff for: ‎libraries/ArduinoOTA/examples/BasicOTA/ci.json

+3-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
{
2-
"requires": [
3-
"CONFIG_SOC_WIFI_SUPPORTED=y"
2+
"requires_any": [
3+
"CONFIG_SOC_WIFI_SUPPORTED=y",
4+
"CONFIG_ESP_WIFI_REMOTE_ENABLED=y"
45
]
56
}

Diff for: ‎libraries/AsyncUDP/examples/AsyncUDPClient/ci.json

+3-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
{
2-
"requires": [
3-
"CONFIG_SOC_WIFI_SUPPORTED=y"
2+
"requires_any": [
3+
"CONFIG_SOC_WIFI_SUPPORTED=y",
4+
"CONFIG_ESP_WIFI_REMOTE_ENABLED=y"
45
]
56
}

Diff for: ‎libraries/AsyncUDP/examples/AsyncUDPMulticastServer/ci.json

+3-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
{
2-
"requires": [
3-
"CONFIG_SOC_WIFI_SUPPORTED=y"
2+
"requires_any": [
3+
"CONFIG_SOC_WIFI_SUPPORTED=y",
4+
"CONFIG_ESP_WIFI_REMOTE_ENABLED=y"
45
]
56
}

Diff for: ‎libraries/AsyncUDP/examples/AsyncUDPServer/ci.json

+3-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
{
2-
"requires": [
3-
"CONFIG_SOC_WIFI_SUPPORTED=y"
2+
"requires_any": [
3+
"CONFIG_SOC_WIFI_SUPPORTED=y",
4+
"CONFIG_ESP_WIFI_REMOTE_ENABLED=y"
45
]
56
}

Diff for: ‎libraries/DNSServer/examples/CaptivePortal/ci.json

+3-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
{
2-
"requires": [
3-
"CONFIG_SOC_WIFI_SUPPORTED=y"
2+
"requires_any": [
3+
"CONFIG_SOC_WIFI_SUPPORTED=y",
4+
"CONFIG_ESP_WIFI_REMOTE_ENABLED=y"
45
]
56
}

Diff for: ‎libraries/ESP32/examples/Time/SimpleTime/ci.json

+3-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
{
2-
"requires": [
3-
"CONFIG_SOC_WIFI_SUPPORTED=y"
2+
"requires_any": [
3+
"CONFIG_SOC_WIFI_SUPPORTED=y",
4+
"CONFIG_ESP_WIFI_REMOTE_ENABLED=y"
45
]
56
}

Diff for: ‎libraries/ESP_NOW/examples/ESP_NOW_Broadcast_Master/ci.json

+3-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
{
2-
"requires": [
3-
"CONFIG_SOC_WIFI_SUPPORTED=y"
2+
"requires_any": [
3+
"CONFIG_SOC_WIFI_SUPPORTED=y",
4+
"CONFIG_ESP_WIFI_REMOTE_ENABLED=y"
45
]
56
}

0 commit comments

Comments
 (0)