Skip to content

Commit 7a6dae0

Browse files
Ouss4Ouss4
authored
Refactor the CI scripts (#6191)
The reason behind this refactoring is that all of the sketch related functions can (and will) be used for other purposes. Build in the sketch directory: This will make it easy to handle artifacts after the build. Separate sketch related functions from IDE installation script. This is the main commit. Create a separate job for the Cmake check. This check was part of one of the Linux build. I believe that it's not the best place for such a check. Checking for the skip landmarks and validity of the the sketch directory were already done by count_sketches.
1 parent 9f08cf4 commit 7a6dae0

File tree

4 files changed

+245
-202
lines changed

4 files changed

+245
-202
lines changed

Diff for: ‎.github/scripts/install-arduino-ide.sh

-156
Original file line numberDiff line numberDiff line change
@@ -34,9 +34,6 @@ else
3434
fi
3535
export OS_NAME
3636

37-
ARDUINO_BUILD_DIR="$HOME/.arduino/build.tmp"
38-
ARDUINO_CACHE_DIR="$HOME/.arduino/cache.tmp"
39-
4037
if [ "$OS_IS_MACOS" == "1" ]; then
4138
export ARDUINO_IDE_PATH="/Applications/Arduino.app/Contents/Java"
4239
export ARDUINO_USR_PATH="$HOME/Documents/Arduino"
@@ -81,156 +78,3 @@ if [ ! -d "$ARDUINO_IDE_PATH" ]; then
8178
echo ""
8279
fi
8380

84-
function build_sketch(){ # build_sketch <fqbn> <path-to-ino> [extra-options]
85-
if [ "$#" -lt 2 ]; then
86-
echo "ERROR: Illegal number of parameters"
87-
echo "USAGE: build_sketch <fqbn> <path-to-ino> [extra-options]"
88-
return 1
89-
fi
90-
91-
local fqbn="$1"
92-
local sketch="$2"
93-
local xtra_opts="$3"
94-
local win_opts=""
95-
if [ "$OS_IS_WINDOWS" == "1" ]; then
96-
local ctags_version=`ls "$ARDUINO_IDE_PATH/tools-builder/ctags/"`
97-
local preprocessor_version=`ls "$ARDUINO_IDE_PATH/tools-builder/arduino-preprocessor/"`
98-
win_opts="-prefs=runtime.tools.ctags.path=$ARDUINO_IDE_PATH/tools-builder/ctags/$ctags_version -prefs=runtime.tools.arduino-preprocessor.path=$ARDUINO_IDE_PATH/tools-builder/arduino-preprocessor/$preprocessor_version"
99-
fi
100-
101-
#echo ""
102-
#echo "Compiling '"$(basename "$sketch")"' ..."
103-
mkdir -p "$ARDUINO_BUILD_DIR"
104-
mkdir -p "$ARDUINO_CACHE_DIR"
105-
$ARDUINO_IDE_PATH/arduino-builder -compile -logger=human -core-api-version=10810 \
106-
-fqbn=$fqbn \
107-
-warnings="all" \
108-
-tools "$ARDUINO_IDE_PATH/tools-builder" \
109-
-tools "$ARDUINO_IDE_PATH/tools" \
110-
-built-in-libraries "$ARDUINO_IDE_PATH/libraries" \
111-
-hardware "$ARDUINO_IDE_PATH/hardware" \
112-
-hardware "$ARDUINO_USR_PATH/hardware" \
113-
-libraries "$ARDUINO_USR_PATH/libraries" \
114-
-build-cache "$ARDUINO_CACHE_DIR" \
115-
-build-path "$ARDUINO_BUILD_DIR" \
116-
$win_opts $xtra_opts "$sketch"
117-
}
118-
119-
function count_sketches(){ # count_sketches <examples-path> <target-mcu>
120-
local examples="$1"
121-
local target="$2"
122-
rm -rf sketches.txt
123-
if [ ! -d "$examples" ]; then
124-
touch sketches.txt
125-
return 0
126-
fi
127-
local sketches=$(find $examples -name *.ino)
128-
local sketchnum=0
129-
for sketch in $sketches; do
130-
local sketchdir=$(dirname $sketch)
131-
local sketchdirname=$(basename $sketchdir)
132-
local sketchname=$(basename $sketch)
133-
if [[ "$sketchdirname.ino" != "$sketchname" ]]; then
134-
continue
135-
elif [[ -f "$sketchdir/.skip.$target" ]]; then
136-
continue
137-
else
138-
echo $sketch >> sketches.txt
139-
sketchnum=$(($sketchnum + 1))
140-
fi
141-
done
142-
return $sketchnum
143-
}
144-
145-
function build_sketches(){ # build_sketches <fqbn> <target-mcu> <examples-path> <chunk> <total-chunks> [extra-options]
146-
local fqbn=$1
147-
local target="$2"
148-
local examples=$3
149-
local chunk_idex=$4
150-
local chunks_num=$5
151-
local xtra_opts=$6
152-
153-
if [ "$#" -lt 3 ]; then
154-
echo "ERROR: Illegal number of parameters"
155-
echo "USAGE: build_sketches <fqbn> <target-mcu <examples-path> [<chunk> <total-chunks>] [extra-options]"
156-
return 1
157-
fi
158-
159-
if [ "$#" -lt 5 ]; then
160-
chunk_idex="0"
161-
chunks_num="1"
162-
xtra_opts=$4
163-
fi
164-
165-
if [ "$chunks_num" -le 0 ]; then
166-
echo "ERROR: Chunks count must be positive number"
167-
return 1
168-
fi
169-
if [ "$chunk_idex" -ge "$chunks_num" ] && [ "$chunks_num" -ge 2 ]; then
170-
echo "ERROR: Chunk index must be less than chunks count"
171-
return 1
172-
fi
173-
174-
set +e
175-
count_sketches "$examples" "$target"
176-
local sketchcount=$?
177-
set -e
178-
local sketches=$(cat sketches.txt)
179-
rm -rf sketches.txt
180-
181-
local chunk_size=$(( $sketchcount / $chunks_num ))
182-
local all_chunks=$(( $chunks_num * $chunk_size ))
183-
if [ "$all_chunks" -lt "$sketchcount" ]; then
184-
chunk_size=$(( $chunk_size + 1 ))
185-
fi
186-
187-
local start_index=0
188-
local end_index=0
189-
if [ "$chunk_idex" -ge "$chunks_num" ]; then
190-
start_index=$chunk_idex
191-
end_index=$sketchcount
192-
else
193-
start_index=$(( $chunk_idex * $chunk_size ))
194-
if [ "$sketchcount" -le "$start_index" ]; then
195-
echo "Skipping job"
196-
return 0
197-
fi
198-
199-
end_index=$(( $(( $chunk_idex + 1 )) * $chunk_size ))
200-
if [ "$end_index" -gt "$sketchcount" ]; then
201-
end_index=$sketchcount
202-
fi
203-
fi
204-
205-
local start_num=$(( $start_index + 1 ))
206-
echo "Found $sketchcount Sketches for target '$target'";
207-
echo "Chunk Index : $chunk_idex"
208-
echo "Chunk Count : $chunks_num"
209-
echo "Chunk Size : $chunk_size"
210-
echo "Start Sketch: $start_num"
211-
echo "End Sketch : $end_index"
212-
213-
local sketchnum=0
214-
for sketch in $sketches; do
215-
local sketchdir=$(dirname $sketch)
216-
local sketchdirname=$(basename $sketchdir)
217-
local sketchname=$(basename $sketch)
218-
if [ "${sketchdirname}.ino" != "$sketchname" ] \
219-
|| [ -f "$sketchdir/.skip.$target" ]; then
220-
continue
221-
fi
222-
sketchnum=$(($sketchnum + 1))
223-
if [ "$sketchnum" -le "$start_index" ] \
224-
|| [ "$sketchnum" -gt "$end_index" ]; then
225-
continue
226-
fi
227-
echo ""
228-
echo "Building Sketch Index $(($sketchnum - 1)) - $sketchdirname"
229-
build_sketch "$fqbn" "$sketch" "$xtra_opts"
230-
local result=$?
231-
if [ $result -ne 0 ]; then
232-
return $result
233-
fi
234-
done
235-
return 0
236-
}

Diff for: ‎.github/scripts/on-push.sh

+54-46
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,40 @@
22

33
set -e
44

5+
function build(){
6+
local target=$1
7+
local fqbn=$2
8+
local chunk_index=$3
9+
local chunks_cnt=$4
10+
local sketches=$5
11+
12+
local BUILD_SKETCH="${SCRIPTS_DIR}/sketch_utils.sh build"
13+
local BUILD_SKETCHES="${SCRIPTS_DIR}/sketch_utils.sh chunk_build"
14+
15+
local args="$ARDUINO_IDE_PATH $ARDUINO_USR_PATH"
16+
17+
args+=" \"$fqbn\""
18+
19+
if [ "$OS_IS_LINUX" == "1" ]; then
20+
args+=" $target"
21+
args+=" $ARDUINO_ESP32_PATH/libraries"
22+
args+=" $chunk_index $chunks_cnt"
23+
${BUILD_SKETCHES} ${args}
24+
else
25+
if [ "$OS_IS_WINDOWS" == "1" ]; then
26+
local ctags_version=`ls "$ARDUINO_IDE_PATH/tools-builder/ctags/"`
27+
local preprocessor_version=`ls "$ARDUINO_IDE_PATH/tools-builder/arduino-preprocessor/"`
28+
win_opts="-prefs=runtime.tools.ctags.path=$ARDUINO_IDE_PATH/tools-builder/ctags/$ctags_version
29+
-prefs=runtime.tools.arduino-preprocessor.path=$ARDUINO_IDE_PATH/tools-builder/arduino-preprocessor/$preprocessor_version"
30+
args+=" ${win_opts}"
31+
fi
32+
33+
for sketch in ${sketches}; do
34+
${BUILD_SKETCH} ${args} ${sketch}
35+
done
36+
fi
37+
}
38+
539
if [ -z "$GITHUB_WORKSPACE" ]; then
640
export GITHUB_WORKSPACE="$PWD"
741
export GITHUB_REPOSITORY="espressif/arduino-esp32"
@@ -22,57 +56,31 @@ fi
2256
#echo "Updating submodules ..."
2357
#git -C "$GITHUB_WORKSPACE" submodule update --init --recursive > /dev/null 2>&1
2458

59+
SCRIPTS_DIR="./.github/scripts"
2560
if [ "$BUILD_PIO" -eq 0 ]; then
26-
# ArduinoIDE ESP32 Test
27-
TARGET="esp32"
28-
FQBN="espressif:esp32:esp32:PSRAM=enabled,PartitionScheme=huge_app"
2961
source ./.github/scripts/install-arduino-ide.sh
30-
source ./.github/scripts/install-arduino-core-esp32.sh
31-
if [ "$OS_IS_WINDOWS" == "1" ]; then
32-
build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/WiFiClientSecure/examples/WiFiClientSecure/WiFiClientSecure.ino" && \
33-
build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/BLE/examples/BLE_server/BLE_server.ino" && \
34-
build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/ESP32/examples/Camera/CameraWebServer/CameraWebServer.ino"
35-
elif [ "$OS_IS_MACOS" == "1" ]; then
36-
build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/WiFi/examples/WiFiClient/WiFiClient.ino" && \
37-
build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/WiFiClientSecure/examples/WiFiClientSecure/WiFiClientSecure.ino" && \
38-
build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/BluetoothSerial/examples/SerialToSerialBT/SerialToSerialBT.ino" && \
39-
build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/BLE/examples/BLE_server/BLE_server.ino" && \
40-
build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/ESP32/examples/Camera/CameraWebServer/CameraWebServer.ino"
41-
else
42-
# CMake Test
43-
if [ "$CHUNK_INDEX" -eq 0 ]; then
44-
bash "$ARDUINO_ESP32_PATH/.github/scripts/check-cmakelists.sh"
45-
fi
46-
build_sketches "$FQBN" "$TARGET" "$ARDUINO_ESP32_PATH/libraries" "$CHUNK_INDEX" "$CHUNKS_CNT"
47-
fi
62+
source ${SCRIPTS_DIR}/install-arduino-core-esp32.sh
4863

49-
# ArduinoIDE ESP32S2 Test
50-
TARGET="esp32s2"
51-
FQBN="espressif:esp32:esp32s2:PSRAM=enabled,PartitionScheme=huge_app"
52-
if [ "$OS_IS_WINDOWS" == "1" ]; then
53-
build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/WiFi/examples/WiFiClient/WiFiClient.ino" && \
54-
build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/WiFiClientSecure/examples/WiFiClientSecure/WiFiClientSecure.ino"
55-
elif [ "$OS_IS_MACOS" == "1" ]; then
56-
build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/WiFi/examples/WiFiClient/WiFiClient.ino" && \
57-
build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/WiFiClientSecure/examples/WiFiClientSecure/WiFiClientSecure.ino"
58-
else
59-
build_sketches "$FQBN" "$TARGET" "$ARDUINO_ESP32_PATH/libraries" "$CHUNK_INDEX" "$CHUNKS_CNT"
60-
fi
64+
FQBN_ESP32="espressif:esp32:esp32:PSRAM=enabled,PartitionScheme=huge_app"
65+
FQBN_ESP32S2="espressif:esp32:esp32s2:PSRAM=enabled,PartitionScheme=huge_app"
66+
FQBN_ESP32C3="espressif:esp32:esp32c3:PartitionScheme=huge_app"
6167

62-
# ArduinoIDE ESP32C3 Test
63-
TARGET="esp32c3"
64-
FQBN="espressif:esp32:esp32c3:PartitionScheme=huge_app"
65-
if [ "$OS_IS_WINDOWS" == "1" ]; then
66-
build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/WiFi/examples/WiFiClient/WiFiClient.ino" && \
67-
build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/WiFiClientSecure/examples/WiFiClientSecure/WiFiClientSecure.ino"
68-
elif [ "$OS_IS_MACOS" == "1" ]; then
69-
build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/WiFi/examples/WiFiClient/WiFiClient.ino" && \
70-
build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/WiFiClientSecure/examples/WiFiClientSecure/WiFiClientSecure.ino"
71-
else
72-
build_sketches "$FQBN" "$TARGET" "$ARDUINO_ESP32_PATH/libraries" "$CHUNK_INDEX" "$CHUNKS_CNT"
73-
fi
68+
SKETCHES_ESP32="\
69+
$ARDUINO_ESP32_PATH/libraries/WiFiClientSecure/examples/WiFiClientSecure/WiFiClientSecure.ino\
70+
$ARDUINO_ESP32_PATH/libraries/BLE/examples/BLE_server/BLE_server.ino\
71+
$ARDUINO_ESP32_PATH/libraries/ESP32/examples/Camera/CameraWebServer/CameraWebServer.ino\
72+
"
73+
74+
SKETCHES_ESP32XX="\
75+
$ARDUINO_ESP32_PATH/libraries/WiFiClientSecure/examples/WiFiClientSecure/WiFiClientSecure.ino\
76+
$ARDUINO_ESP32_PATH/libraries/WiFi/examples/WiFiClient/WiFiClient.ino\
77+
"
78+
79+
build "esp32" $FQBN_ESP32 $CHUNK_INDEX $CHUNKS_CNT $SKETCHES_ESP32
80+
build "esp32s2" $FQBN_ESP32S2 $CHUNK_INDEX $CHUNKS_CNT $SKETCHES_ESP32XX
81+
build "esp32c3" $FQBN_ESP32C3 $CHUNK_INDEX $CHUNKS_CNT $SKETCHES_ESP32XX
7482
else
75-
source ./.github/scripts/install-platformio-esp32.sh
83+
source ./${SCRIPTS_DIR}/install-platformio-esp32.sh
7684
# PlatformIO ESP32 Test
7785
BOARD="esp32dev"
7886
OPTIONS="board_build.partitions = huge_app.csv"

0 commit comments

Comments
 (0)