Split cron jobs using matrix

Author: lucasssvaz

.github/workflows/cron.yml

@@ -1,31 +1,49 @@
 name: Cron Build
 
-on: 
+on:
   schedule:
 #             ┌───────────── minute (0 - 59)
 #             │ ┌───────────── hour (0 - 23)
 #             │ │ ┌───────────── day of the month (1 - 31)
 #             │ │ │ ┌───────────── month (1 - 12 or JAN-DEC)
 #             │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT)
-#             │ │ │ │ │                                   
+#             │ │ │ │ │
 #             │ │ │ │ │
 #             │ │ │ │ │
 #             * * * * *
     - cron:  '0 */6 * * *'
+  workflow_dispatch:
+    inputs:
+      idf_branch:
+        description: 'IDF branch to build'
+        required: true
+        default: 'release/v5.1'
+      target:
+        description: 'Target to build'
+        required: true
+        default: 'all'
 
 defaults:
   run:
     shell: bash
 
 jobs:
-  run:
-    name: Build with IDF ${{ matrix.idf_branch }}
+  build-libs:
+    name: Build with IDF ${{ matrix.idf_branch }} for ${{ matrix.target }}
     runs-on: ubuntu-latest
-    
+    if: github.event_name == 'schedule'
     strategy:
       fail-fast: false
       matrix:
         idf_branch: [release/v5.1, release/v4.4] #, release/v3.3]
+        target: [esp32, esp32s2, esp32s3, esp32c3]
+        include:
+          - idf_branch: release/v5.1
+            target: esp32c2
+          - idf_branch: release/v5.1
+            target: esp32c6
+          - idf_branch: release/v5.1
+            target: esp32h2
     steps:
     - uses: actions/checkout@v4
       with:
@@ -39,8 +57,94 @@ jobs:
         GIT_COMMITTER_EMAIL: ${{ secrets.PUSH_EMAIL }}
         IDF_BRANCH: ${{ matrix.idf_branch }}
       run: |
-        git checkout ${{ matrix.idf_branch }} || echo "Using master branch"
-        bash ./tools/cron.sh
+        git checkout ${{ env.IDF_BRANCH }} || echo "Using master branch"
+        if [ ! "$GITHUB_EVENT_NAME" == "schedule" ]; then
+          echo "Wrong event '$GITHUB_EVENT_NAME'!"
+          exit 1
+        fi
+
+        bash ./build.sh -e -t ${{ matrix.target }}
+
+    - name: Upload build
+      if: failure()
+      uses: actions/upload-artifact@v4
+      with:
+        name: build-${{ matrix.idf_branch }}-${{ matrix.target }}
+        path: build
+
+    - name: Upload library files
+      uses: actions/upload-artifact@v4
+      with:
+        name: libs-${{ matrix.idf_branch }}-${{ matrix.target }}
+        path: dist
+
+  combine-artifacts:
+    name: Combine artifacts for IDF ${{ matrix.idf_branch }}
+    runs-on: ubuntu-latest
+    if: github.event_name == 'schedule'
+    needs: build-libs
+    strategy:
+      fail-fast: false
+      matrix:
+        idf_branch: [release/v5.1, release/v4.4]
+    steps:
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: dist
+          pattern: libs-${{ matrix.idf_branch }}-*
+          merge-multiple: true
+
+      - name: Combine artifacts
+        shell: bash
+        run: |
+          set -e
+          mkdir -p out
+          find dist -name 'arduino-esp32-libs-esp*.tar.gz' -exec tar zxvf {} -C out \;
+          for file in $files; do
+            tar zxvf $file -C out
+            cat out/tools/esp32-arduino-libs/versions.txt >> out/tools/esp32-arduino-libs/versions_full.txt
+          done
+          awk -i inplace '!seen[$0]++' out/tools/esp32-arduino-libs/versions_full.txt
+          mv -f out/tools/esp32-arduino-libs/versions_full.txt out/tools/esp32-arduino-libs/versions.txt
+          cd out/tools/esp32-arduino-libs && tar zcf ../../../dist/esp32-arduino-libs.tar.gz * && cd ../../..
+          cp out/package_esp32_index.template.json dist/package_esp32_index.template.json
+
+      - name: Push changes
+        if: ${{ github.repository == 'espressif/esp32-arduino-lib-builder' }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.PUSH_TOKEN }}
+          GIT_AUTHOR_EMAIL: ${{ secrets.PUSH_EMAIL }}
+          GIT_COMMITTER_EMAIL: ${{ secrets.PUSH_EMAIL }}
+          IDF_BRANCH: ${{ matrix.idf_branch }}
+        run: |
+          ./tools/push-to-arduino.sh
+          if [ $? -ne 0 ]; then exit 1; fi
+
+  build-libs-dispatch:
+    name: Dispatch build with IDF ${{ inputs.idf_branch }}
+    runs-on: ubuntu-latest
+    if: github.event_name == 'workflow_dispatch'
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - name: Install dependencies
+      run: bash ./tools/prepare-ci.sh
+    - name: Build
+      env:
+        GITHUB_TOKEN: ${{ secrets.PUSH_TOKEN }}
+        GIT_AUTHOR_EMAIL: ${{ secrets.PUSH_EMAIL }}
+        GIT_COMMITTER_EMAIL: ${{ secrets.PUSH_EMAIL }}
+        IDF_BRANCH: ${{ inputs.idf_branch }}
+      run: |
+        git checkout ${{ env.IDF_BRANCH }} || echo "Using master branch"
+        if [ ! "$GITHUB_EVENT_NAME" == "schedule" ]; then
+          echo "Wrong event '$GITHUB_EVENT_NAME'!"
+          exit 1
+        fi
+
+        bash ./build.sh -d -t ${{ inputs.target }}
     - name: Upload build
       if: failure()
       uses: actions/upload-artifact@v4

build.sh

@@ -142,7 +142,7 @@ if [ "$BUILD_TYPE" != "all" ]; then
             # Skip building for targets that are not in the $TARGET array
             continue
         fi
-                
+
         configs="configs/defconfig.common;configs/defconfig.$target;configs/defconfig.debug_$BUILD_DEBUG"
         for defconf in `echo "$target_json" | jq -c '.features[]' | tr -d '"'`; do
             configs="$configs;configs/defconfig.$defconf"
@@ -187,7 +187,7 @@ for target_json in `jq -c '.targets[]' configs/builds.json`; do
             continue
         fi
     fi
-    
+
     # Skip chips that should not be a part of the final libs
     # WARNING!!! this logic needs to be updated when cron builds are split into jobs
     if [ "$TARGET" = "all" ] && [ $target_skip -eq 1 ]; then
@@ -299,13 +299,15 @@ done
 
 # update package_esp32_index.template.json
 if [ "$BUILD_TYPE" = "all" ]; then
+    echo "* Generating package_esp32_index.template.json..."
     python3 ./tools/gen_tools_json.py -i "$IDF_PATH" -j "$AR_COMPS/arduino/package/package_esp32_index.template.json" -o "$AR_OUT/"
     python3 ./tools/gen_tools_json.py -i "$IDF_PATH" -o "$TOOLS_JSON_OUT/"
     if [ $? -ne 0 ]; then exit 1; fi
 fi
 
 # Generate PlatformIO manifest file
 if [ "$BUILD_TYPE" = "all" ]; then
+    echo "* Generating PlatformIO manifest file..."
     pushd $IDF_PATH
     ibr=$(git describe --all 2>/dev/null)
     ic=$(git -C "$IDF_PATH" rev-parse --short HEAD)
@@ -316,18 +318,21 @@ fi
 
 # copy everything to arduino-esp32 installation
 if [ $COPY_OUT -eq 1 ] && [ -d "$ESP32_ARDUINO" ]; then
+    echo "* Copying to Arduino..."
     ./tools/copy-to-arduino.sh
     if [ $? -ne 0 ]; then exit 1; fi
 fi
 
 # push changes to esp32-arduino-libs and create pull request into arduino-esp32
 if [ $DEPLOY_OUT -eq 1 ]; then
+    echo "* Pushing to Arduino..."
     ./tools/push-to-arduino.sh
     if [ $? -ne 0 ]; then exit 1; fi
 fi
 
 # archive the build
 if [ $ARCHIVE_OUT -eq 1 ]; then
+    echo "* Archiving build..."
     ./tools/archive-build.sh "$TARGET"
     if [ $? -ne 0 ]; then exit 1; fi
 fi

tools/cron.sh

@@ -51,7 +51,7 @@ source $IDF_PATH/export.sh
 # SETUP ARDUINO DEPLOY
 #
 
-if [ "$GITHUB_EVENT_NAME" == "schedule" ] || [ "$GITHUB_EVENT_NAME" == "repository_dispatch" -a "$GITHUB_EVENT_ACTION" == "deploy" ]; then
+if [ "$GITHUB_EVENT_NAME" == "schedule" ] || [ "$GITHUB_EVENT_NAME" == "workflow_dispatch" ] || [ "$GITHUB_EVENT_NAME" == "repository_dispatch" -a "$GITHUB_EVENT_ACTION" == "deploy" ]; then
 	# format new branch name and pr title
 	if [ -x $commit_predefined ]; then #commit was not specified at build time
 		AR_NEW_BRANCH_NAME="idf-$IDF_BRANCH"

tools/install-esp-idf.sh

@@ -43,7 +43,7 @@ if [ $LIBS_HAS_COMMIT == "0" ] || [ $AR_HAS_COMMIT == "0" ]; then
 	# make changes to the files
 	echo "Patching files in esp32-arduino-libs branch '$AR_NEW_BRANCH_NAME'..."
 	rm -rf $IDF_LIBS_DIR/* && cp -Rf $AR_TOOLS/esp32-arduino-libs/* $IDF_LIBS_DIR/
-	
+
 	cd $IDF_LIBS_DIR
 	if [ -f "README.md" ]; then
 		rm -rf "README.md"
@@ -52,9 +52,9 @@ if [ $LIBS_HAS_COMMIT == "0" ] || [ $AR_HAS_COMMIT == "0" ]; then
 	# did any of the files change?
 	if [ -n "$(git status --porcelain)" ]; then
 		echo "Pushing changes to esp32-arduino-libs branch '$AR_NEW_BRANCH_NAME'..."
-	    git add . && git commit --message "$AR_NEW_COMMIT_MESSAGE" && git push -u origin $AR_NEW_BRANCH_NAME
+	  git add . && git commit --message "$AR_NEW_COMMIT_MESSAGE" && git push -u origin $AR_NEW_BRANCH_NAME
 		if [ $? -ne 0 ]; then
-		    echo "ERROR: Pushing to branch '$AR_NEW_BRANCH_NAME' failed"
+		  echo "ERROR: Pushing to branch '$AR_NEW_BRANCH_NAME' failed"
 			exit 1
 		fi
 		IDF_LIBS_COMMIT=`git rev-parse --verify HEAD`
@@ -122,13 +122,13 @@ if [ $AR_HAS_COMMIT == "0" ]; then
 	# make changes to the files
 	echo "Patching files in branch '$AR_NEW_BRANCH_NAME'..."
 	rm -rf "$AR_COMPS/arduino/package/package_esp32_index.template.json" && cp -f "$AR_OUT/package_esp32_index.template.json" "$AR_COMPS/arduino/package/package_esp32_index.template.json"
-	
+
 	cd $AR_COMPS/arduino
 
 	# did any of the files change?
 	if [ -n "$(git status --porcelain)" ]; then
 		echo "Pushing changes to branch '$AR_NEW_BRANCH_NAME'..."
-	    git add . && git commit --message "$AR_NEW_COMMIT_MESSAGE" && git push -u origin $AR_NEW_BRANCH_NAME
+	  git add . && git commit --message "$AR_NEW_COMMIT_MESSAGE" && git push -u origin $AR_NEW_BRANCH_NAME
 		if [ $? -ne 0 ]; then
 		    echo "ERROR: Pushing to branch '$AR_NEW_BRANCH_NAME' failed"
 			exit 1