Merge branch 'ci/cron_optimization'

lucasssvaz · lucasssvaz · commit 439df1dc7b85 · 2024-07-03T19:59:12.000-03:00
diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml
@@ -1,13 +1,13 @@
 name: Cron Build
 
-on: 
+on:
   schedule:
 #             ┌───────────── minute (0 - 59)
 #             │ ┌───────────── hour (0 - 23)
 #             │ │ ┌───────────── day of the month (1 - 31)
 #             │ │ │ ┌───────────── month (1 - 12 or JAN-DEC)
 #             │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT)
-#             │ │ │ │ │                                   
+#             │ │ │ │ │
 #             │ │ │ │ │
 #             │ │ │ │ │
 #             * * * * *
@@ -17,128 +17,185 @@ defaults:
   run:
     shell: bash
 
+env:
+  LATEST_RELEASE_BRANCH: "release/v5.1" # Change this to the latest release branch so the checkouts are done correctly
+
 jobs:
-  run:
-    name: Build with IDF ${{ matrix.idf_branch }}
+  gen-matrix:
+    name: Generate matrix
     runs-on: ubuntu-latest
-    
+    outputs:
+      matrix: ${{ steps.gen-matrix.outputs.matrix }}
+      branches: ${{ steps.gen-matrix.outputs.branches }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Generate matrix
+        id: gen-matrix
+        run: |
+          set -e
+
+          # Change this based on the IDF branches we want to build. Don't forget to update env.LATEST_RELEASE_BRANCH
+          all_branches=("release/v5.1")
+
+          # Change this based on the COMMON targets for all branches we want to build.
+          common_targets="[\"esp32\", \"esp32s2\", \"esp32s3\", \"esp32c2\", \"esp32c3\", \"esp32c6\", \"esp32h2\"]"
+
+          # For additional targets per branch, add them here
+          additional_targets="[{\"idf_branch\": \"release/v5.3\", \"target\": \"esp32p4\"}]"
+
+          branches="["
+          matrix="{"
+
+          for branch in ${all_branches[@]}; do
+              if [ "$branch" == "$LATEST_RELEASE_BRANCH" ]; then
+                  git checkout master
+              else
+                  git checkout $branch
+              fi
+              export IDF_BRANCH=$branch
+              ./tools/check-deploy-needed.sh
+              if [ "$DEPLOY_NEEDED" == "1" ]; then
+                  branches+="\"$branch\","
+              fi
+          done
+
+          branches="${branches%,}]"
+          matrix+="\"idf_branch\": $branches,"
+          matrix+="\"target\": $common_targets,"
+
+          matrix+="\"include\": "
+          # Add all additional targets that are in the selected branches
+          matrix+=$(echo $additional_targets | jq --argjson branches "$branches" '[.[] | select(.idf_branch as $branch | $branches | index($branch))]')
+
+          matrix+="}"
+
+          echo "Branches: $branches"
+
+          echo "Matrix:"
+          echo "$matrix" | jq .
+
+          if [ ! -x $GITHUB_OUTPUT ]; then
+              echo "matrix=$matrix" >> $GITHUB_OUTPUT
+              echo "branches=$branches" >> $GITHUB_OUTPUT
+          fi
+
+  build-libs:
+    name: Build with IDF ${{ matrix.idf_branch }} for ${{ matrix.target }}
+    runs-on: ubuntu-latest
+    if: needs.gen-matrix.outputs.branches != '[]'
+    needs: gen-matrix
     strategy:
       fail-fast: false
-      matrix:
-        idf_branch: [release/v5.1, release/v4.4] #, release/v3.3]
+      matrix: ${{ fromJson(needs.gen-matrix.outputs.matrix) }}
     steps:
     - uses: actions/checkout@v4
       with:
-        fetch-depth: 0
+        # Useful workaround for the checkout action to work with the matrix
+        # https://github.com/actions/runner/issues/409#issuecomment-1013325196
+        ref: ${{ matrix.idf_branch == env.LATEST_RELEASE_BRANCH && 'master' || matrix.idf_branch }}
+
     - name: Install dependencies
       run: bash ./tools/prepare-ci.sh
+
     - name: Build
       env:
-        GITHUB_TOKEN: ${{ secrets.PUSH_TOKEN }}
+        GITHUB_TOKEN: ${{ secrets.PUSH_TOKEN || secrets.GITHUB_TOKEN }}
         GIT_AUTHOR_EMAIL: ${{ secrets.PUSH_EMAIL }}
         GIT_COMMITTER_EMAIL: ${{ secrets.PUSH_EMAIL }}
         IDF_BRANCH: ${{ matrix.idf_branch }}
+        TARGETS: ${{ matrix.target }}
       run: |
-        git checkout ${{ matrix.idf_branch }} || echo "Using master branch"
         bash ./tools/cron.sh
+
+    - name: Replace invalid characters in the artifact name
+      run: |
+        branch=${{ matrix.idf_branch }}
+        echo "libs_branch=${branch//\//_}" >> $GITHUB_ENV
+
     - name: Upload build
       if: failure()
       uses: actions/upload-artifact@v4
       with:
-        name: build
+        name: build-${{ env.libs_branch }}-${{ matrix.target }}
         path: build
-    - name: Upload archive
+
+    - name: Upload library files
       uses: actions/upload-artifact@v4
       with:
-        name: artifacts
+        name: libs-${{ env.libs_branch }}-${{ matrix.target }}
         path: dist
 
+  combine-artifacts:
+    name: Combine artifacts for IDF ${{ matrix.idf_branch }}
+    runs-on: ubuntu-latest
+    needs: [gen-matrix, build-libs]
+    # Condition is evaluated before the job is run so it won't cause a failure
+    if: needs.gen-matrix.outputs.branches != '[]'
+    strategy:
+      fail-fast: false
+      matrix:
+        idf_branch: ${{ fromJson(needs.gen-matrix.outputs.branches) }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # Useful workaround for the checkout action to work with the matrix
+          # https://github.com/actions/runner/issues/409#issuecomment-1013325196
+          ref: ${{ matrix.idf_branch == env.LATEST_RELEASE_BRANCH && 'master' || matrix.idf_branch }}
+
+      - name: Replace invalid characters in the artifact name
+        run: |
+          branch=${{ matrix.idf_branch }}
+          echo "libs_branch=${branch//\//_}" >> $GITHUB_ENV
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: dist
+          pattern: libs-${{ env.libs_branch }}-*
+          merge-multiple: true
+
+      - name: Combine artifacts
+        run: |
+          set -e
+          mkdir -p out
+
+          libs_folder="out/tools/esp32-arduino-libs"
+
+          files=$(find dist -name 'arduino-esp32-libs-esp*.tar.gz')
+          for file in $files; do
+              echo "Extracting $file"
+              tar zxvf $file -C out
+              cat $libs_folder/versions.txt >> $libs_folder/versions_full.txt
+          done
+
+          # Merge versions.txt files
+          awk -i inplace '!seen[$0]++' $libs_folder/versions_full.txt
+          mv -f $libs_folder/versions_full.txt $libs_folder/versions.txt
+
+          cd $libs_folder && tar zcf ../../../dist/esp32-arduino-libs.tar.gz * && cd ../../..
+          cp out/package_esp32_index.template.json dist/package_esp32_index.template.json
+
+      - name: Upload full esp32-arduino-libs archive
+        uses: actions/upload-artifact@v4
+        with:
+          name: esp32-arduino-libs
+          path: dist/esp32-arduino-libs.tar.gz
 
-  # check:
-  #   name: Check if result should be deployed
-  #   runs-on: ubuntu-latest
-  #   strategy:
-  #     matrix:
-  #       branch: [release/v5.1, release/v4.4] #, release/v3.3]
-  #   outputs:
-  #     idf_branch: ${{ steps.check.outputs.idf_branch }}
-  #     idf_commit: ${{ steps.check.outputs.idf_commit }}
-  #     ar_branch: ${{ steps.check.outputs.ar_branch }}
-  #     ar_new_commit_message: ${{ steps.check.outputs.ar_new_commit_message }}
-  #     ar_new_branch_name: ${{ steps.check.outputs.ar_new_branch_name }}
-  #     ar_new_pr_title: ${{ steps.check.outputs.ar_new_pr_title }}
-  #     ar_has_commit: ${{ steps.check.outputs.ar_has_commit }}
-  #     ar_has_branch: ${{ steps.check.outputs.ar_has_branch }}
-  #     ar_has_pr: ${{ steps.check.outputs.ar_has_pr }}
-  #     libs_version: ${{ steps.check.outputs.libs_version }}
-  #     libs_has_commit: ${{ steps.check.outputs.libs_has_commit }}
-  #     libs_has_branch: ${{ steps.check.outputs.libs_has_branch }}
-  #   steps:
-  #   - uses: actions/checkout@v3
-  #   - id: check
-  #     env:
-  #       GITHUB_TOKEN: ${{ secrets.PUSH_TOKEN }}
-  #       GIT_AUTHOR_EMAIL: ${{ secrets.PUSH_EMAIL }}
-  #       GIT_COMMITTER_EMAIL: ${{ secrets.PUSH_EMAIL }}
-  #       IDF_BRANCH: ${{ matrix.idf_branch }}
-  #     run: bash ./tools/check-deploy-needed.sh
-
-  # build:
-  #   name: Build Libs for ${{ matrix.target }}
-  #   runs-on: ubuntu-latest
-  #   needs: check
-  #   if: needs.check.outputs.libs_has_commit == '0' || needs.check.outputs.ar_has_commit == '0'
-  #   strategy:
-  #     matrix:
-  #       target: [esp32, esp32s2, esp32s3, esp32c3, esp32c6, esp32h2]
-  #     fail-fast: false
-  #   steps:
-  #   - uses: actions/checkout@v3
-  #   # - name: Install dependencies
-  #   #   run: bash ./tools/prepare-ci.sh
-  #   - shell: bash
-  #     name: Build Libs for ${{ matrix.target }}
-  #     run: echo ${{ matrix.target }}
-  #   #   run: bash ./build.sh -t ${{ matrix.target }}
-  #   # - name: Upload archive
-  #   #   uses: actions/upload-artifact@v3
-  #   #   with:
-  #   #     name: artifacts
-  #   #     path: dist
-
-  # deploy:
-  #   name: Deploy build
-  #   runs-on: ubuntu-latest
-  #   needs: [check, build]
-  #   steps:
-  #   - uses: actions/checkout@v3
-  #   - shell: bash
-  #     env:
-  #       GITHUB_TOKEN: ${{ secrets.PUSH_TOKEN }}
-  #       GIT_AUTHOR_EMAIL: ${{ secrets.PUSH_EMAIL }}
-  #       GIT_COMMITTER_EMAIL: ${{ secrets.PUSH_EMAIL }}
-  #       IDF_BRANCH: ${{ needs.check.outputs.idf_branch }}
-  #       IDF_COMMIT: ${{ needs.check.outputs.idf_commit }}
-  #       AR_BRANCH: ${{ needs.check.outputs.ar_branch }}
-  #       AR_NEW_COMMIT_MESSAGE: ${{ needs.check.outputs.ar_new_commit_message }}
-  #       AR_NEW_BRANCH_NAME: ${{ needs.check.outputs.ar_new_branch_name }}
-  #       AR_NEW_PR_TITLE: ${{ needs.check.outputs.ar_new_pr_title }}
-  #       AR_HAS_COMMIT: ${{ needs.check.outputs.ar_has_commit }}
-  #       AR_HAS_BRANCH: ${{ needs.check.outputs.ar_has_branch }}
-  #       AR_HAS_PR: ${{ needs.check.outputs.ar_has_pr }}
-  #       LIBS_VERSION: ${{ needs.check.outputs.libs_version }}
-  #       LIBS_HAS_COMMIT: ${{ needs.check.outputs.libs_has_commit }}
-  #       LIBS_HAS_BRANCH: ${{ needs.check.outputs.libs_has_branch }}
-  #     run: |
-  #       echo "IDF_COMMIT: $IDF_COMMIT"
-  #       echo "AR_BRANCH: $AR_BRANCH"
-  #       echo "AR_NEW_COMMIT_MESSAGE: $AR_NEW_COMMIT_MESSAGE"
-  #       echo "AR_NEW_BRANCH_NAME: $AR_NEW_BRANCH_NAME"
-  #       echo "AR_NEW_PR_TITLE: $AR_NEW_PR_TITLE"
-  #       echo "AR_HAS_COMMIT: $AR_HAS_COMMIT"
-  #       echo "AR_HAS_BRANCH: $AR_HAS_BRANCH"
-  #       echo "AR_HAS_PR: $AR_HAS_PR"
-  #       echo "LIBS_VERSION: $LIBS_VERSION"
-  #       echo "LIBS_HAS_COMMIT: $LIBS_HAS_COMMIT"
-  #       echo "LIBS_HAS_BRANCH: $LIBS_HAS_BRANCH"
+      - name: Upload package_esp32_index.template.json
+        uses: actions/upload-artifact@v4
+        with:
+          name: package-esp32-index-json
+          path: dist/package_esp32_index.template.json
 
+      - name: Push changes
+        if: github.repository == 'espressif/esp32-arduino-lib-builder'
+        env:
+          GITHUB_TOKEN: ${{ secrets.PUSH_TOKEN }}
+          GIT_AUTHOR_EMAIL: ${{ secrets.PUSH_EMAIL }}
+          GIT_COMMITTER_EMAIL: ${{ secrets.PUSH_EMAIL }}
+          IDF_BRANCH: ${{ matrix.idf_branch }}
+        run: |
+          bash ./tools/push-to-arduino.sh
diff --git a/build.sh b/build.sh
@@ -299,13 +299,15 @@ done
 
 # update package_esp32_index.template.json
 if [ "$BUILD_TYPE" = "all" ]; then
+    echo "* Generating package_esp32_index.template.json..."
     python3 ./tools/gen_tools_json.py -i "$IDF_PATH" -j "$AR_COMPS/arduino/package/package_esp32_index.template.json" -o "$AR_OUT/"
     python3 ./tools/gen_tools_json.py -i "$IDF_PATH" -o "$TOOLS_JSON_OUT/"
     if [ $? -ne 0 ]; then exit 1; fi
 fi
 
 # Generate PlatformIO manifest file
 if [ "$BUILD_TYPE" = "all" ]; then
+    echo "* Generating PlatformIO manifest file..."
     pushd $IDF_PATH
     ibr=$(git describe --all 2>/dev/null)
     ic=$(git -C "$IDF_PATH" rev-parse --short HEAD)
@@ -316,18 +318,21 @@ fi
 
 # copy everything to arduino-esp32 installation
 if [ $COPY_OUT -eq 1 ] && [ -d "$ESP32_ARDUINO" ]; then
+    echo "* Copying to Arduino..."
     ./tools/copy-to-arduino.sh
     if [ $? -ne 0 ]; then exit 1; fi
 fi
 
 # push changes to esp32-arduino-libs and create pull request into arduino-esp32
 if [ $DEPLOY_OUT -eq 1 ]; then
+    echo "* Pushing to Arduino..."
     ./tools/push-to-arduino.sh
     if [ $? -ne 0 ]; then exit 1; fi
 fi
 
 # archive the build
 if [ $ARCHIVE_OUT -eq 1 ]; then
+    echo "* Archiving build..."
     ./tools/archive-build.sh "$TARGET"
     if [ $? -ne 0 ]; then exit 1; fi
 fi
diff --git a/tools/check-deploy-needed.sh b/tools/check-deploy-needed.sh
@@ -1,5 +1,10 @@
 #/bin/bash
 
+if [ "$GITHUB_EVENT_NAME" != "schedule" ] && [ "$GITHUB_EVENT_NAME" != "workflow_dispatch" ] && [ "$GITHUB_EVENT_NAME" != "repository_dispatch" -o "$GITHUB_EVENT_ACTION" != "deploy" ]; then
+	echo "Wrong event '$GITHUB_EVENT_NAME'!"
+	exit 1
+fi
+
 source ./tools/config.sh
 
 IDF_COMMIT=`github_last_commit "$IDF_REPO" "$IDF_BRANCH"`
@@ -70,16 +75,24 @@ echo "LIBS_VERSION: $LIBS_VERSION"
 echo "LIBS_HAS_COMMIT: $LIBS_HAS_COMMIT"
 echo "LIBS_HAS_BRANCH: $LIBS_HAS_BRANCH"
 
-if [ ! -x $GITHUB_OUTPUT ]; then
-	echo "idf_commit=$IDF_COMMIT" >> "$GITHUB_OUTPUT"
-	echo "ar_branch=$AR_BRANCH" >> "$GITHUB_OUTPUT"
-	echo "ar_new_commit_message=$AR_NEW_COMMIT_MESSAGE" >> "$GITHUB_OUTPUT"
-	echo "ar_new_branch_name=$AR_NEW_BRANCH_NAME" >> "$GITHUB_OUTPUT"
-	echo "ar_new_pr_title=$AR_NEW_PR_TITLE" >> "$GITHUB_OUTPUT"
-	echo "ar_has_commit=$AR_HAS_COMMIT" >> "$GITHUB_OUTPUT"
-	echo "ar_has_branch=$AR_HAS_BRANCH" >> "$GITHUB_OUTPUT"
-	echo "ar_has_pr=$AR_HAS_PR" >> "$GITHUB_OUTPUT"
-	echo "libs_version=$LIBS_VERSION" >> "$GITHUB_OUTPUT"
-	echo "libs_has_commit=$LIBS_HAS_COMMIT" >> "$GITHUB_OUTPUT"
-	echo "libs_has_branch=$LIBS_HAS_BRANCH" >> "$GITHUB_OUTPUT"
+if [ ! -x $GITHUB_ENV ]; then
+	echo "idf_commit=$IDF_COMMIT" >> "$GITHUB_ENV"
+	echo "ar_branch=$AR_BRANCH" >> "$GITHUB_ENV"
+	echo "ar_new_commit_message=$AR_NEW_COMMIT_MESSAGE" >> "$GITHUB_ENV"
+	echo "ar_new_branch_name=$AR_NEW_BRANCH_NAME" >> "$GITHUB_ENV"
+	echo "ar_new_pr_title=$AR_NEW_PR_TITLE" >> "$GITHUB_ENV"
+	echo "ar_has_commit=$AR_HAS_COMMIT" >> "$GITHUB_ENV"
+	echo "ar_has_branch=$AR_HAS_BRANCH" >> "$GITHUB_ENV"
+	echo "ar_has_pr=$AR_HAS_PR" >> "$GITHUB_ENV"
+	echo "libs_version=$LIBS_VERSION" >> "$GITHUB_ENV"
+	echo "libs_has_commit=$LIBS_HAS_COMMIT" >> "$GITHUB_ENV"
+	echo "libs_has_branch=$LIBS_HAS_BRANCH" >> "$GITHUB_ENV"
+fi
+
+if [ "$LIBS_HAS_COMMIT" == "0" ] || [ "$AR_HAS_COMMIT" == "0" ]; then
+	echo "Deploy needed"
+	export DEPLOY_NEEDED="1"
+else
+	echo "Deploy not needed. Skipping..."
+	export DEPLOY_NEEDED="0"
 fi
diff --git a/tools/cron.sh b/tools/cron.sh
@@ -1,8 +1,9 @@
 #!/bin/bash
 
-if [ ! "$GITHUB_EVENT_NAME" == "schedule" ]; then
-    echo "Wrong event '$GITHUB_EVENT_NAME'!"
-    exit 1
+if [ -z "$TARGETS" ]; then
+    TARGETS="all"
 fi
 
-bash ./build.sh -d
+export IDF_CCACHE_ENABLE=1
+
+bash ./build.sh -e -t $TARGETS
diff --git a/tools/install-esp-idf.sh b/tools/install-esp-idf.sh
