diff --git a/.github/scripts/on-push.sh b/.github/scripts/on-push.sh index bf311b93166..30fd1653deb 100755 --- a/.github/scripts/on-push.sh +++ b/.github/scripts/on-push.sh @@ -9,7 +9,8 @@ function build(){ local fqbn=$2 local chunk_index=$3 local chunks_cnt=$4 - shift; shift; shift; shift; + local build_log=$5 + shift; shift; shift; shift; shift; local sketches=$* local BUILD_SKETCH="${SCRIPTS_DIR}/sketch_utils.sh build" @@ -22,6 +23,9 @@ function build(){ if [ "$OS_IS_LINUX" == "1" ]; then args+=" -p $ARDUINO_ESP32_PATH/libraries" args+=" -i $chunk_index -m $chunks_cnt" + if [ $build_log -eq 1 ]; then + args+=" -l $build_log" + fi ${BUILD_SKETCHES} ${args} else for sketch in ${sketches}; do @@ -45,6 +49,7 @@ fi CHUNK_INDEX=$1 CHUNKS_CNT=$2 +BUILD_LOG=$3 BUILD_PIO=0 if [ "$#" -lt 2 ] || [ "$CHUNKS_CNT" -le 0 ]; then CHUNK_INDEX=0 @@ -55,6 +60,10 @@ elif [ "$CHUNK_INDEX" -eq "$CHUNKS_CNT" ]; then BUILD_PIO=1 fi +if [ "$BUILD_LOG" -le 0 ]; then + BUILD_LOG=0 +fi + #echo "Updating submodules ..." #git -C "$GITHUB_WORKSPACE" submodule update --init --recursive > /dev/null 2>&1 @@ -77,13 +86,28 @@ if [ "$BUILD_PIO" -eq 0 ]; then $ARDUINO_ESP32_PATH/libraries/ESP32/examples/Camera/CameraWebServer/CameraWebServer.ino\ $ARDUINO_ESP32_PATH/libraries/Insights/examples/MinimalDiagnostics/MinimalDiagnostics.ino\ " + #create sizes_file + sizes_file="$GITHUB_WORKSPACE/cli_compile_$CHUNK_INDEX.json" - build "esp32s3" $FQBN_ESP32S3 $CHUNK_INDEX $CHUNKS_CNT $SKETCHES_ESP32 - build "esp32s2" $FQBN_ESP32S2 $CHUNK_INDEX $CHUNKS_CNT $SKETCHES_ESP32 - build "esp32c3" $FQBN_ESP32C3 $CHUNK_INDEX $CHUNKS_CNT $SKETCHES_ESP32 - build "esp32c6" $FQBN_ESP32C6 $CHUNK_INDEX $CHUNKS_CNT $SKETCHES_ESP32 - build "esp32h2" $FQBN_ESP32H2 $CHUNK_INDEX $CHUNKS_CNT $SKETCHES_ESP32 - build "esp32" $FQBN_ESP32 $CHUNK_INDEX $CHUNKS_CNT $SKETCHES_ESP32 + if [ "$BUILD_LOG" -eq 1 ]; then + #create sizes_file and echo start of JSON array with "boards" key + echo "{\"boards\": [" > $sizes_file + fi + + #build sketches for different targets + build "esp32s3" $FQBN_ESP32S3 $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32 + build "esp32s2" $FQBN_ESP32S2 $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32 + build "esp32c3" $FQBN_ESP32C3 $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32 + build "esp32c6" $FQBN_ESP32C6 $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32 + build "esp32h2" $FQBN_ESP32H2 $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32 + build "esp32" $FQBN_ESP32 $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32 + + if [ "$BUILD_LOG" -eq 1 ]; then + #remove last comma from the last JSON object + sed -i '$ s/.$//' "$sizes_file" + #echo end of JSON array + echo "]}" >> $sizes_file + fi else source ${SCRIPTS_DIR}/install-platformio-esp32.sh # PlatformIO ESP32 Test diff --git a/.github/scripts/sketch_utils.sh b/.github/scripts/sketch_utils.sh index 70a21a15452..e8da865a067 100755 --- a/.github/scripts/sketch_utils.sh +++ b/.github/scripts/sketch_utils.sh @@ -27,6 +27,14 @@ function build_sketch(){ # build_sketch [ex shift sketchdir=$1 ;; + -i ) + shift + chunk_index=$1 + ;; + -l ) + shift + log_compilation=$1 + ;; * ) break ;; @@ -140,6 +148,9 @@ function build_sketch(){ # build_sketch [ex build_dir="$HOME/.arduino/tests/$sketchname/build.tmp" fi + output_file="$HOME/.arduino/cli_compile_output.txt" + sizes_file="$GITHUB_WORKSPACE/cli_compile_$chunk_index.json" + mkdir -p "$ARDUINO_CACHE_DIR" for i in `seq 0 $(($len - 1))` do @@ -164,13 +175,39 @@ function build_sketch(){ # build_sketch [ex --build-property "compiler.warning_flags.all=-Wall -Werror=all -Wextra" \ --build-cache-path "$ARDUINO_CACHE_DIR" \ --build-path "$build_dir" \ - $xtra_opts "${sketchdir}" + $xtra_opts "${sketchdir}" \ + > $output_file exit_status=$? if [ $exit_status -ne 0 ]; then echo ""ERROR: Compilation failed with error code $exit_status"" exit $exit_status fi + + if [ $log_compilation ]; then + #Extract the program storage space and dynamic memory usage in bytes and percentage in separate variables from the output, just the value without the string + flash_bytes=$(grep -oE 'Sketch uses ([0-9]+) bytes' $output_file | awk '{print $3}') + flash_percentage=$(grep -oE 'Sketch uses ([0-9]+) bytes \(([0-9]+)%\)' $output_file | awk '{print $5}' | tr -d '(%)') + ram_bytes=$(grep -oE 'Global variables use ([0-9]+) bytes' $output_file | awk '{print $4}') + ram_percentage=$(grep -oE 'Global variables use ([0-9]+) bytes \(([0-9]+)%\)' $output_file | awk '{print $6}' | tr -d '(%)') + + # Extract the directory path excluding the filename + directory_path=$(dirname "$sketch") + # Define the constant part + constant_part="/home/runner/Arduino/hardware/espressif/esp32/libraries/" + # Extract the desired substring using sed + lib_sketch_name=$(echo "$directory_path" | sed "s|$constant_part||") + #append json file where key is fqbn, sketch name, sizes -> extracted values + echo "{\"name\": \"$lib_sketch_name\", + \"sizes\": [{ + \"flash_bytes\": $flash_bytes, + \"flash_percentage\": $flash_percentage, + \"ram_bytes\": $ram_bytes, + \"ram_percentage\": $ram_percentage + }] + }," >> "$sizes_file" + fi + elif [ -f "$ide_path/arduino-builder" ]; then echo "Building $sketchname with arduino-builder and FQBN=$currfqbn" echo "Build path = $build_dir" @@ -204,6 +241,7 @@ function build_sketch(){ # build_sketch [ex # $xtra_opts "${sketchdir}/${sketchname}.ino" fi done + unset fqbn unset xtra_opts unset options @@ -277,6 +315,10 @@ function build_sketches(){ # build_sketches > "$sizes_file" + fi + local sketchnum=0 - args+=" -ai $ide_path -au $user_path" + args+=" -ai $ide_path -au $user_path -i $chunk_index" + if [ $log_compilation ]; then + args+=" -l $log_compilation" + fi for sketch in $sketches; do local sketchdir=$(dirname $sketch) local sketchdirname=$(basename $sketchdir) @@ -358,6 +411,18 @@ function build_sketches(){ # build_sketches > "$sizes_file" + #echo end of board sizes_file json + echo "}," >> "$sizes_file" + fi + return 0 } @@ -389,4 +454,3 @@ case "$cmd" in echo "$USAGE" exit 2 esac - diff --git a/.github/workflows/lib.yml b/.github/workflows/lib.yml index da2e5ed6a04..0e98618ad5c 100644 --- a/.github/workflows/lib.yml +++ b/.github/workflows/lib.yml @@ -121,16 +121,21 @@ jobs: git add ${{ env.RESULT_LIBRARY_TEST_FILE }} git commit -m "Generated External Libraries Test Results" git push origin HEAD:gh-pages - - event_file: - name: "Event File" - if: | - contains(github.event.pull_request.labels.*.name, 'lib_test') - needs: compile-sketch + + #Upload PR number as artifact + upload-pr-number: + name: Upload PR number + if: github.event_name == 'pull_request' runs-on: ubuntu-latest steps: - - name: Upload - uses: actions/upload-artifact@v4 - with: - name: Event File - path: ${{github.event_path}} + - name: Save the PR number in an artifact + shell: bash + env: + PR_NUM: ${{ github.event.number }} + run: echo $PR_NUM > pr_num.txt + - name: Upload PR number + uses: actions/upload-artifact@v4 + with: + name: pr_number + path: ./pr_num.txt + overwrite: true diff --git a/.github/workflows/publishlib.yml b/.github/workflows/publishlib.yml index abfebbf4ab7..7e7a4579ca9 100644 --- a/.github/workflows/publishlib.yml +++ b/.github/workflows/publishlib.yml @@ -11,7 +11,6 @@ env: # It's convenient to set variables for values used multiple times in the workflow SKETCHES_REPORTS_PATH: artifacts/libraries-report GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} - PR_NUMBER: ${{ github.event.workflow_run.pull_requests[0].number }} jobs: lib-test-results: @@ -26,21 +25,32 @@ jobs: run: | mkdir -p artifacts && cd artifacts mkdir -p libraries-report + mkdir -p workflows artifacts_url=${{ github.event.workflow_run.artifacts_url }} gh api "$artifacts_url" -q '.artifacts[] | [.name, .archive_download_url] | @tsv' | while read artifact do IFS=$'\t' read name url <<< "$artifact" gh api $url > "$name.zip" unzip -j "$name.zip" -d "temp_$name" - mv "temp_$name"/* libraries-report + if [[ "$name" == "pr_num" ]]; then + mv "temp_$name"/* workflows + else + mv "temp_$name"/* libraries-report + fi rm -r "temp_$name" done echo "Contents of parent directory:" ls -R .. + + - name: Read the pr_num file + id: pr_num_reader + uses: juliangruber/read-file-action@v1 + with: + path: ./artifacts/workflows/pr_num.txt - name: Report results uses: P-R-O-C-H-Y/report-size-deltas@libs with: sketches-reports-source: ${{ env.SKETCHES_REPORTS_PATH }} github-token: ${{ env.GITHUB_TOKEN }} - pr-number: ${{ env.PR_NUMBER }} \ No newline at end of file + pr-number: "${{ steps.pr_num_reader.outputs.content }}" diff --git a/.github/workflows/publishsizes.yml b/.github/workflows/publishsizes.yml index 92ac4cbb42c..8c4a957b22b 100644 --- a/.github/workflows/publishsizes.yml +++ b/.github/workflows/publishsizes.yml @@ -11,7 +11,6 @@ env: # It's convenient to set variables for values used multiple times in the workflow SKETCHES_REPORTS_PATH: artifacts/sizes-report GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} - PR_NUMBER: ${{ github.event.workflow_run.pull_requests[0].number }} jobs: sizes-test-results: @@ -22,21 +21,34 @@ jobs: github.event.workflow_run.conclusion == 'success' steps: - - name: Download and Extract Artifacts + - name: Checkout code + uses: actions/checkout@v4 # This step checks out the repository's code at gh-pages branch + with: + ref: gh-pages + + - name: Create folder structure run: | mkdir -p artifacts && cd artifacts mkdir -p sizes-report mkdir -p sizes-report/master mkdir -p sizes-report/pr + + - name: Download JSON file + run: | + mv master_cli_compile/*.json artifacts/sizes-report/master/ + + - name: Download and Extract Artifacts + run: | + cd artifacts artifacts_url=${{ github.event.workflow_run.artifacts_url }} gh api "$artifacts_url" -q '.artifacts[] | [.name, .archive_download_url] | @tsv' | while read artifact do IFS=$'\t' read name url <<< "$artifact" gh api $url > "$name.zip" unzip -j "$name.zip" -d "temp_$name" - if [[ "$name" == *"master"* ]]; then - mv "temp_$name"/* sizes-report/master - elif [[ "$name" == *"pr"* ]]; then + if [[ "$name" == "pr_num" ]]; then + mv "temp_$name"/* sizes-report + elif [[ "$name" == "pr_cli"* ]]; then mv "temp_$name"/* sizes-report/pr else mv "temp_$name"/* sizes-report @@ -45,10 +57,16 @@ jobs: done echo "Contents of parent directory:" ls -R .. + + - name: Read the pr_num file + id: pr_num_reader + uses: juliangruber/read-file-action@v1 + with: + path: ./artifacts/sizes-report/pr_num.txt - name: Report results uses: P-R-O-C-H-Y/report-size-deltas@sizes_v2 with: sketches-reports-source: ${{ env.SKETCHES_REPORTS_PATH }} github-token: ${{ env.GITHUB_TOKEN }} - pr-number: ${{ env.PR_NUMBER }} \ No newline at end of file + pr-number: "${{ steps.pr_num_reader.outputs.content }}" diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index c7c48225ca4..9d912db35fb 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -46,7 +46,15 @@ jobs: 'tools/get.py', '.github/scripts/install-arduino-ide.sh') }} - name: Build Sketches - run: bash ./.github/scripts/on-push.sh ${{ matrix.chunk }} 15 + run: bash ./.github/scripts/on-push.sh ${{ matrix.chunk }} 15 1 + + #Upload cli compile json as artifact + - name: Upload cli compile json + uses: actions/upload-artifact@v4 + with: + name: pr_cli_compile_${{ matrix.chunk }} + path: cli_compile_${{ matrix.chunk }}.json + overwrite: true # Windows and MacOS build-arduino-win-mac: @@ -110,3 +118,58 @@ jobs: idf.py create-project test echo CONFIG_FREERTOS_HZ=1000 > test/sdkconfig.defaults idf.py -C test -DEXTRA_COMPONENT_DIRS=$PWD/components build + + # Save artifacts to gh-pages + save-master-artifacts: + name: Save master artifacts + needs: build-arduino-linux + if: github.event_name == 'push' && github.ref == 'refs/heads/master' + runs-on: ubuntu-latest + steps: + # Check out repository + - name: Checkout repository + uses: actions/checkout@v4 + with: + token: ${{secrets.GITHUB_TOKEN}} + fetch-depth: '0' + + - name: Switch branch + run: + git checkout remotes/origin/gh-pages + + - name: Download sketches reports artifact + uses: actions/download-artifact@v4 + with: + pattern: pr_cli_compile_* + merge-multiple: true + path: master_cli_compile + + - name: List files in the directory + run: ls -R + + - name: Commit json files to gh-pages if on master + if: github.event_name == 'push' && github.ref == 'refs/heads/master' + run: | + git config user.name github-actions + git config user.email github-actions@github.com + git add --all + git commit -m "Updated cli compile json files" + git push origin HEAD:gh-pages + + #Upload PR number as artifact + upload-pr-number: + name: Upload PR number + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Save the PR number in an artifact + shell: bash + env: + PR_NUM: ${{ github.event.number }} + run: echo $PR_NUM > pr_num.txt + - name: Upload PR number + uses: actions/upload-artifact@v4 + with: + name: pr_number + path: ./pr_num.txt + overwrite: true