diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml index 00405c4ba..a9ec31ce3 100644 --- a/.github/workflows/cron.yml +++ b/.github/workflows/cron.yml @@ -1,144 +1,36 @@ -name: Cron Build +name: Cron Deploy -on: +on: schedule: # ┌───────────── minute (0 - 59) # │ ┌───────────── hour (0 - 23) # │ │ ┌───────────── day of the month (1 - 31) # │ │ │ ┌───────────── month (1 - 12 or JAN-DEC) # │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT) -# │ │ │ │ │ +# │ │ │ │ │ # │ │ │ │ │ # │ │ │ │ │ # * * * * * - cron: '0 */6 * * *' + workflow_dispatch: # For testing defaults: run: shell: bash jobs: - run: + build-libs: name: Build with IDF ${{ matrix.idf_branch }} - runs-on: ubuntu-latest - + uses: ./.github/workflows/cron_build.yml + with: + idf_branch: ${{ matrix.idf_branch }} + lib_builder_branch: ${{ matrix.lib_builder_branch }} + targets: ${{ matrix.targets }} + secrets: inherit strategy: fail-fast: false matrix: - idf_branch: [release/v5.1, release/v4.4] #, release/v3.3] - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Install dependencies - run: bash ./tools/prepare-ci.sh - - name: Build - env: - GITHUB_TOKEN: ${{ secrets.PUSH_TOKEN }} - GIT_AUTHOR_EMAIL: ${{ secrets.PUSH_EMAIL }} - GIT_COMMITTER_EMAIL: ${{ secrets.PUSH_EMAIL }} - IDF_BRANCH: ${{ matrix.idf_branch }} - run: | - git checkout ${{ matrix.idf_branch }} || echo "Using master branch" - bash ./tools/cron.sh - - name: Upload build - if: failure() - uses: actions/upload-artifact@v4 - with: - name: build - path: build - - name: Upload archive - uses: actions/upload-artifact@v4 - with: - name: artifacts - path: dist - - - # check: - # name: Check if result should be deployed - # runs-on: ubuntu-latest - # strategy: - # matrix: - # branch: [release/v5.1, release/v4.4] #, release/v3.3] - # outputs: - # idf_branch: ${{ steps.check.outputs.idf_branch }} - # idf_commit: ${{ steps.check.outputs.idf_commit }} - # ar_branch: ${{ steps.check.outputs.ar_branch }} - # ar_new_commit_message: ${{ steps.check.outputs.ar_new_commit_message }} - # ar_new_branch_name: ${{ steps.check.outputs.ar_new_branch_name }} - # ar_new_pr_title: ${{ steps.check.outputs.ar_new_pr_title }} - # ar_has_commit: ${{ steps.check.outputs.ar_has_commit }} - # ar_has_branch: ${{ steps.check.outputs.ar_has_branch }} - # ar_has_pr: ${{ steps.check.outputs.ar_has_pr }} - # libs_version: ${{ steps.check.outputs.libs_version }} - # libs_has_commit: ${{ steps.check.outputs.libs_has_commit }} - # libs_has_branch: ${{ steps.check.outputs.libs_has_branch }} - # steps: - # - uses: actions/checkout@v3 - # - id: check - # env: - # GITHUB_TOKEN: ${{ secrets.PUSH_TOKEN }} - # GIT_AUTHOR_EMAIL: ${{ secrets.PUSH_EMAIL }} - # GIT_COMMITTER_EMAIL: ${{ secrets.PUSH_EMAIL }} - # IDF_BRANCH: ${{ matrix.idf_branch }} - # run: bash ./tools/check-deploy-needed.sh - - # build: - # name: Build Libs for ${{ matrix.target }} - # runs-on: ubuntu-latest - # needs: check - # if: needs.check.outputs.libs_has_commit == '0' || needs.check.outputs.ar_has_commit == '0' - # strategy: - # matrix: - # target: [esp32, esp32s2, esp32s3, esp32c3, esp32c6, esp32h2] - # fail-fast: false - # steps: - # - uses: actions/checkout@v3 - # # - name: Install dependencies - # # run: bash ./tools/prepare-ci.sh - # - shell: bash - # name: Build Libs for ${{ matrix.target }} - # run: echo ${{ matrix.target }} - # # run: bash ./build.sh -t ${{ matrix.target }} - # # - name: Upload archive - # # uses: actions/upload-artifact@v3 - # # with: - # # name: artifacts - # # path: dist - - # deploy: - # name: Deploy build - # runs-on: ubuntu-latest - # needs: [check, build] - # steps: - # - uses: actions/checkout@v3 - # - shell: bash - # env: - # GITHUB_TOKEN: ${{ secrets.PUSH_TOKEN }} - # GIT_AUTHOR_EMAIL: ${{ secrets.PUSH_EMAIL }} - # GIT_COMMITTER_EMAIL: ${{ secrets.PUSH_EMAIL }} - # IDF_BRANCH: ${{ needs.check.outputs.idf_branch }} - # IDF_COMMIT: ${{ needs.check.outputs.idf_commit }} - # AR_BRANCH: ${{ needs.check.outputs.ar_branch }} - # AR_NEW_COMMIT_MESSAGE: ${{ needs.check.outputs.ar_new_commit_message }} - # AR_NEW_BRANCH_NAME: ${{ needs.check.outputs.ar_new_branch_name }} - # AR_NEW_PR_TITLE: ${{ needs.check.outputs.ar_new_pr_title }} - # AR_HAS_COMMIT: ${{ needs.check.outputs.ar_has_commit }} - # AR_HAS_BRANCH: ${{ needs.check.outputs.ar_has_branch }} - # AR_HAS_PR: ${{ needs.check.outputs.ar_has_pr }} - # LIBS_VERSION: ${{ needs.check.outputs.libs_version }} - # LIBS_HAS_COMMIT: ${{ needs.check.outputs.libs_has_commit }} - # LIBS_HAS_BRANCH: ${{ needs.check.outputs.libs_has_branch }} - # run: | - # echo "IDF_COMMIT: $IDF_COMMIT" - # echo "AR_BRANCH: $AR_BRANCH" - # echo "AR_NEW_COMMIT_MESSAGE: $AR_NEW_COMMIT_MESSAGE" - # echo "AR_NEW_BRANCH_NAME: $AR_NEW_BRANCH_NAME" - # echo "AR_NEW_PR_TITLE: $AR_NEW_PR_TITLE" - # echo "AR_HAS_COMMIT: $AR_HAS_COMMIT" - # echo "AR_HAS_BRANCH: $AR_HAS_BRANCH" - # echo "AR_HAS_PR: $AR_HAS_PR" - # echo "LIBS_VERSION: $LIBS_VERSION" - # echo "LIBS_HAS_COMMIT: $LIBS_HAS_COMMIT" - # echo "LIBS_HAS_BRANCH: $LIBS_HAS_BRANCH" - + include: + - idf_branch: "release/v5.1" + lib_builder_branch: "master" + targets: "esp32,esp32s2,esp32s3,esp32c3,esp32c6,esp32h2" diff --git a/.github/workflows/cron_build.yml b/.github/workflows/cron_build.yml new file mode 100644 index 000000000..f8767617c --- /dev/null +++ b/.github/workflows/cron_build.yml @@ -0,0 +1,161 @@ +name: Cron Build Matrix + +on: + workflow_call: + inputs: + idf_branch: + type: string + required: true + description: 'IDF branch to build' + lib_builder_branch: + type: string + required: true + description: 'Branch of the lib-builder to use' + targets: + type: string + required: true + description: 'Targets to build' + +env: + IDF_BRANCH: ${{ inputs.idf_branch }} + +jobs: + check-if-needed: + name: Check if deploy is needed for ${{ inputs.idf_branch }} + runs-on: ubuntu-latest + outputs: + idf_commit: ${{ steps.check.outputs.idf_commit }} + ar_branch: ${{ steps.check.outputs.ar_branch }} + ar_new_commit_message: ${{ steps.check.outputs.ar_new_commit_message }} + ar_new_branch_name: ${{ steps.check.outputs.ar_new_branch_name }} + ar_new_pr_title: ${{ steps.check.outputs.ar_new_pr_title }} + ar_has_commit: ${{ steps.check.outputs.ar_has_commit }} + ar_has_branch: ${{ steps.check.outputs.ar_has_branch }} + ar_has_pr: ${{ steps.check.outputs.ar_has_pr }} + libs_release_tag: ${{ steps.check.outputs.libs_release_tag }} + libs_version: ${{ steps.check.outputs.libs_version }} + libs_release_id: ${{ steps.check.outputs.libs_release_id }} + libs_has_release: ${{ steps.check.outputs.libs_has_release }} + libs_asset_id: ${{ steps.check.outputs.libs_asset_id }} + libs_has_asset: ${{ steps.check.outputs.libs_has_asset }} + deploy_needed: ${{ steps.check.outputs.deploy_needed }} + targets_list: ${{ steps.check.outputs.targets_list }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.lib_builder_branch }} + + - name: Check deploy and generate variables + id: check + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + source ./tools/check-deploy-needed.sh + targets_list=$(echo "${{ inputs.targets }}" | sed 's/ *, */,/g' | sed 's/^/["/' | sed 's/$/"]/' | sed 's/,/","/g') + echo "Targets list: $targets_list" + echo "targets_list=$targets_list" >> $GITHUB_OUTPUT + + build-libs: + name: Build for ${{ matrix.target }} (${{ inputs.idf_branch }}) + runs-on: ubuntu-latest + if: needs.check-if-needed.outputs.deploy_needed == '1' + needs: check-if-needed + strategy: + fail-fast: false + matrix: + target: ${{ fromJson(needs.check-if-needed.outputs.targets_list) }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.lib_builder_branch }} + + - name: Install dependencies + run: bash ./tools/prepare-ci.sh + + - name: Build + env: + GITHUB_TOKEN: ${{ secrets.PUSH_TOKEN || secrets.GITHUB_TOKEN }} + GIT_AUTHOR_EMAIL: ${{ secrets.PUSH_EMAIL }} + GIT_COMMITTER_EMAIL: ${{ secrets.PUSH_EMAIL }} + TARGET: ${{ matrix.target }} + run: | + bash ./tools/cron.sh + + - name: Replace invalid characters in the artifact name + run: | + branch=${{ inputs.idf_branch }} + echo "libs_branch=${branch//\//_}" >> $GITHUB_ENV + + - name: Upload build + if: failure() + uses: actions/upload-artifact@v4 + with: + name: build-${{ env.libs_branch }}-${{ matrix.target }} + path: build + + - name: Upload library files + uses: actions/upload-artifact@v4 + with: + name: libs-${{ env.libs_branch }}-${{ matrix.target }} + path: dist + + combine-artifacts: + name: Combine artifacts and push changes for IDF ${{ inputs.idf_branch }} + runs-on: ubuntu-latest + needs: [check-if-needed, build-libs] + if: needs.check-if-needed.outputs.deploy_needed == '1' + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.lib_builder_branch }} + + - name: Replace invalid characters in the artifact name + run: | + branch=${{ inputs.idf_branch }} + echo "libs_branch=${branch//\//_}" >> $GITHUB_ENV + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + path: dist + pattern: libs-${{ env.libs_branch }}-* + merge-multiple: true + + - name: Combine artifacts + run: bash ./tools/combine-artifacts.sh + + - name: Upload full esp32-arduino-libs archive + uses: actions/upload-artifact@v4 + with: + name: esp32-arduino-libs-${{ env.libs_branch }} + path: dist/esp32-arduino-libs.zip + compression-level: 0 + + - name: Push changes + if: github.repository == 'espressif/esp32-arduino-lib-builder' + env: + GITHUB_TOKEN: ${{ secrets.PUSH_TOKEN }} + GIT_AUTHOR_EMAIL: ${{ secrets.PUSH_EMAIL }} + GIT_COMMITTER_EMAIL: ${{ secrets.PUSH_EMAIL }} + IDF_COMMIT: ${{ needs.check-if-needed.outputs.idf_commit }} + AR_BRANCH: ${{ needs.check-if-needed.outputs.ar_branch }} + AR_NEW_COMMIT_MESSAGE: ${{ needs.check-if-needed.outputs.ar_new_commit_message }} + AR_NEW_BRANCH_NAME: ${{ needs.check-if-needed.outputs.ar_new_branch_name }} + AR_NEW_PR_TITLE: ${{ needs.check-if-needed.outputs.ar_new_pr_title }} + AR_HAS_COMMIT: ${{ needs.check-if-needed.outputs.ar_has_commit }} + AR_HAS_BRANCH: ${{ needs.check-if-needed.outputs.ar_has_branch }} + AR_HAS_PR: ${{ needs.check-if-needed.outputs.ar_has_pr }} + LIBS_RELEASE_TAG: ${{ needs.check-if-needed.outputs.libs_release_tag }} + LIBS_VERSION: ${{ needs.check-if-needed.outputs.libs_version }} + LIBS_RELEASE_ID: ${{ needs.check-if-needed.outputs.libs_release_id }} + LIBS_HAS_RELEASE: ${{ needs.check-if-needed.outputs.libs_has_release }} + LIBS_ASSET_ID: ${{ needs.check-if-needed.outputs.libs_asset_id }} + LIBS_HAS_ASSET: ${{ needs.check-if-needed.outputs.libs_has_asset }} + run: | + bash ./tools/push-to-arduino.sh + + - name: Upload package_esp32_index.template.json + uses: actions/upload-artifact@v4 + with: + name: package-esp32-index-json-${{ env.libs_branch }} + path: out/package_esp32_index.template.json diff --git a/build.sh b/build.sh index ca412b7ff..9bceda087 100755 --- a/build.sh +++ b/build.sh @@ -18,6 +18,8 @@ fi # Fixes building some components. See https://github.com/espressif/arduino-esp32/issues/10167 export IDF_COMPONENT_OVERWRITE_MANAGED_COMPONENTS=1 +CCACHE_ENABLE=1 + TARGET="all" BUILD_TYPE="all" BUILD_DEBUG="default" @@ -29,8 +31,9 @@ if [ -z $DEPLOY_OUT ]; then fi function print_help() { - echo "Usage: build.sh [-s] [-A ] [-I ] [-D ] [-i ] [-c ] [-t ] [-b ] [config ...]" + echo "Usage: build.sh [-s] [-n] [-A ] [-I ] [-D ] [-i ] [-c ] [-t ] [-b ] [config ...]" echo " -s Skip installing/updating of ESP-IDF and all components" + echo " -n Disable ccache" echo " -A Set which branch of arduino-esp32 to be used for compilation" echo " -I Set which branch of ESP-IDF to be used for compilation" echo " -i Set which commit of ESP-IDF to be used for compilation" @@ -49,6 +52,9 @@ while getopts ":A:I:i:c:t:b:D:sde" opt; do s ) SKIP_ENV=1 ;; + n ) + CCACHE_ENABLE=0 + ;; d ) DEPLOY_OUT=1 ;; @@ -99,6 +105,8 @@ done shift $((OPTIND -1)) CONFIGS=$@ +export IDF_CCACHE_ENABLE=$CCACHE_ENABLE + # Output the TARGET array echo "TARGET(s): ${TARGET[@]}" @@ -146,7 +154,7 @@ if [ "$BUILD_TYPE" != "all" ]; then # Skip building for targets that are not in the $TARGET array continue fi - + configs="configs/defconfig.common;configs/defconfig.$target;configs/defconfig.debug_$BUILD_DEBUG" for defconf in `echo "$target_json" | jq -c '.features[]' | tr -d '"'`; do configs="$configs;configs/defconfig.$defconf" @@ -191,7 +199,7 @@ for target_json in `jq -c '.targets[]' configs/builds.json`; do continue fi fi - + # Skip chips that should not be a part of the final libs # WARNING!!! this logic needs to be updated when cron builds are split into jobs if [ "$TARGET" = "all" ] && [ $target_skip -eq 1 ]; then @@ -291,6 +299,7 @@ done # update package_esp32_index.template.json if [ "$BUILD_TYPE" = "all" ]; then + echo "* Generating package_esp32_index.template.json..." python3 ./tools/gen_tools_json.py -i "$IDF_PATH" -j "$AR_COMPS/arduino/package/package_esp32_index.template.json" -o "$AR_OUT/" python3 ./tools/gen_tools_json.py -i "$IDF_PATH" -o "$TOOLS_JSON_OUT/" if [ $? -ne 0 ]; then exit 1; fi @@ -298,6 +307,7 @@ fi # Generate PlatformIO manifest file if [ "$BUILD_TYPE" = "all" ]; then + echo "* Generating PlatformIO manifest file..." pushd $IDF_PATH ibr=$(git describe --all 2>/dev/null) ic=$(git -C "$IDF_PATH" rev-parse --short HEAD) @@ -308,18 +318,21 @@ fi # copy everything to arduino-esp32 installation if [ $COPY_OUT -eq 1 ] && [ -d "$ESP32_ARDUINO" ]; then + echo "* Copying to Arduino..." ./tools/copy-to-arduino.sh if [ $? -ne 0 ]; then exit 1; fi fi # push changes to esp32-arduino-libs and create pull request into arduino-esp32 if [ $DEPLOY_OUT -eq 1 ]; then + echo "* Pushing to Arduino..." ./tools/push-to-arduino.sh if [ $? -ne 0 ]; then exit 1; fi fi # archive the build if [ $ARCHIVE_OUT -eq 1 ]; then + echo "* Archiving build..." ./tools/archive-build.sh "$TARGET" if [ $? -ne 0 ]; then exit 1; fi fi diff --git a/tools/check-deploy-needed.sh b/tools/check-deploy-needed.sh index 52401942a..230d0bf71 100755 --- a/tools/check-deploy-needed.sh +++ b/tools/check-deploy-needed.sh @@ -4,6 +4,11 @@ source ./tools/config.sh IDF_COMMIT=`github_last_commit "$IDF_REPO" "$IDF_BRANCH"` +if [ -z $IDF_COMMIT ]; then + echo "Failed to get IDF commit for branch $IDF_BRANCH" + exit 1 +fi + if [ -z $GITHUB_HEAD_REF ]; then current_branch=`git branch --show-current` else @@ -31,7 +36,8 @@ AR_NEW_BRANCH_NAME="idf-$IDF_BRANCH" AR_NEW_COMMIT_MESSAGE="IDF $IDF_BRANCH $IDF_COMMIT" AR_NEW_PR_TITLE="IDF $IDF_BRANCH" -LIBS_VERSION="idf-"${IDF_BRANCH//\//_}"-$IDF_COMMIT" +LIBS_RELEASE_TAG="idf-"${IDF_BRANCH//\//_}"" +LIBS_VERSION="$LIBS_RELEASE_TAG-$IDF_COMMIT" AR_HAS_BRANCH=`github_branch_exists "$AR_REPO" "$AR_NEW_BRANCH_NAME"` if [ "$AR_HAS_BRANCH" == "1" ]; then @@ -41,8 +47,10 @@ else fi AR_HAS_PR=`github_pr_exists "$AR_REPO" "$AR_NEW_BRANCH_NAME"` -LIBS_HAS_BRANCH=`github_branch_exists "$AR_LIBS_REPO" "$AR_NEW_BRANCH_NAME"` -LIBS_HAS_COMMIT=`github_commit_exists "$AR_LIBS_REPO" "$AR_NEW_BRANCH_NAME" "$IDF_COMMIT"` +LIBS_RELEASE_ID=`github_release_id "$AR_LIBS_REPO" "$LIBS_RELEASE_TAG"` +LIBS_HAS_RELEASE=`if [ -n "$LIBS_RELEASE_ID" ]; then echo "1"; else echo "0"; fi` +LIBS_ASSET_ID=`github_release_asset_id "$AR_LIBS_REPO" "$LIBS_RELEASE_ID" "esp32-arduino-libs-$LIBS_VERSION.zip"` +LIBS_HAS_ASSET=`if [ -n "$LIBS_ASSET_ID" ]; then echo "1"; else echo "0"; fi` export IDF_COMMIT @@ -54,9 +62,25 @@ export AR_HAS_COMMIT export AR_HAS_BRANCH export AR_HAS_PR +export LIBS_RELEASE_TAG export LIBS_VERSION -export LIBS_HAS_COMMIT -export LIBS_HAS_BRANCH +export LIBS_RELEASE_ID +export LIBS_HAS_RELEASE +export LIBS_ASSET_ID +export LIBS_HAS_ASSET + +if [ "$LIBS_HAS_RELEASE" == "1" ]; then + if [ "$LIBS_HAS_ASSET" == "0" ] || [ "$AR_HAS_COMMIT" == "0" ]; then + echo "Deploy needed" + export DEPLOY_NEEDED="1" + else + echo "Deploy not needed. Skipping..." + export DEPLOY_NEEDED="0" + fi +else + echo "Release for tag \"$LIBS_RELEASE_TAG\" not found. Please create the release first." + exit 1 +fi echo "IDF_COMMIT: $IDF_COMMIT" echo "AR_BRANCH: $AR_BRANCH" @@ -66,9 +90,13 @@ echo "AR_NEW_PR_TITLE: $AR_NEW_PR_TITLE" echo "AR_HAS_COMMIT: $AR_HAS_COMMIT" echo "AR_HAS_BRANCH: $AR_HAS_BRANCH" echo "AR_HAS_PR: $AR_HAS_PR" +echo "LIBS_RELEASE_TAG: $LIBS_RELEASE_TAG" echo "LIBS_VERSION: $LIBS_VERSION" -echo "LIBS_HAS_COMMIT: $LIBS_HAS_COMMIT" -echo "LIBS_HAS_BRANCH: $LIBS_HAS_BRANCH" +echo "LIBS_RELEASE_ID: $LIBS_RELEASE_ID" +echo "LIBS_HAS_RELEASE: $LIBS_HAS_RELEASE" +echo "LIBS_ASSET_ID: $LIBS_ASSET_ID" +echo "LIBS_HAS_ASSET: $LIBS_HAS_ASSET" +echo "DEPLOY_NEEDED: $DEPLOY_NEEDED" if [ ! -x $GITHUB_OUTPUT ]; then echo "idf_commit=$IDF_COMMIT" >> "$GITHUB_OUTPUT" @@ -79,7 +107,12 @@ if [ ! -x $GITHUB_OUTPUT ]; then echo "ar_has_commit=$AR_HAS_COMMIT" >> "$GITHUB_OUTPUT" echo "ar_has_branch=$AR_HAS_BRANCH" >> "$GITHUB_OUTPUT" echo "ar_has_pr=$AR_HAS_PR" >> "$GITHUB_OUTPUT" + echo "libs_release_tag=$LIBS_RELEASE_TAG" >> "$GITHUB_OUTPUT" echo "libs_version=$LIBS_VERSION" >> "$GITHUB_OUTPUT" - echo "libs_has_commit=$LIBS_HAS_COMMIT" >> "$GITHUB_OUTPUT" - echo "libs_has_branch=$LIBS_HAS_BRANCH" >> "$GITHUB_OUTPUT" + echo "libs_release_id=$LIBS_RELEASE_ID" >> "$GITHUB_OUTPUT" + echo "libs_has_release=$LIBS_HAS_RELEASE" >> "$GITHUB_OUTPUT" + echo "libs_asset_id=$LIBS_ASSET_ID" >> "$GITHUB_OUTPUT" + echo "libs_has_asset=$LIBS_HAS_ASSET" >> "$GITHUB_OUTPUT" + echo "deploy_needed=$DEPLOY_NEEDED" >> "$GITHUB_OUTPUT" fi + diff --git a/tools/combine-artifacts.sh b/tools/combine-artifacts.sh new file mode 100755 index 000000000..8cb1c25e2 --- /dev/null +++ b/tools/combine-artifacts.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -e +mkdir -p out + +libs_folder="out/tools/esp32-arduino-libs" + +files=$(find dist -name 'arduino-esp32-libs-esp*.tar.gz') +for file in $files; do + echo "Extracting $file" + tar zxf $file -C out + cat $libs_folder/versions.txt >> $libs_folder/versions_full.txt +done + +# Merge versions.txt files +awk -i inplace '!seen[$0]++' $libs_folder/versions_full.txt +mv -f $libs_folder/versions_full.txt $libs_folder/versions.txt + +echo "Creating zip file" +cd out/tools && zip -q -r ../../dist/esp32-arduino-libs.zip * && cd ../.. diff --git a/tools/config.sh b/tools/config.sh index ef917e846..b05a879b2 100755 --- a/tools/config.sh +++ b/tools/config.sh @@ -51,7 +51,6 @@ AR_GEN_PART_PY="$AR_TOOLS/gen_esp32part.py" AR_SDK="$AR_TOOLS/esp32-arduino-libs/$IDF_TARGET" PIO_SDK="FRAMEWORK_SDK_DIR, \"$IDF_TARGET\"" TOOLS_JSON_OUT="$AR_TOOLS/esp32-arduino-libs" -IDF_LIBS_DIR="$AR_ROOT/../esp32-arduino-libs" if [ -d "$IDF_PATH" ]; then export IDF_COMMIT=$(git -C "$IDF_PATH" rev-parse --short HEAD) @@ -133,6 +132,37 @@ function github_pr_exists(){ # github_pr_exists if [ ! "$pr_num" == "" ] && [ ! "$pr_num" == "null" ]; then echo 1; else echo 0; fi } +function github_release_id(){ # github_release_id + local repo_path="$1" + local release_tag="$2" + local release=`curl -s -k -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3.raw+json" "https://api.github.com/repos/$repo_path/releases" | jq --arg release_tag "$release_tag" -r '.[] | select(.tag_name == $release_tag) | .id'` + if [ ! "$release" == "" ] && [ ! "$release" == "null" ]; then echo "$release"; else echo ""; fi +} + +function github_release_asset_id(){ # github_release_asset_id + local repo_path="$1" + local release_id="$2" + local release_file="$3" + local release_asset=`curl -s -k -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3.raw+json" "https://api.github.com/repos/$repo_path/releases/$release_id/assets" | jq --arg release_file "$release_file" -r '.[] | select(.name == $release_file) | .id'` + if [ ! "$release_asset" == "" ] && [ ! "$release_asset" == "null" ]; then echo "$release_asset"; else echo ""; fi +} + +function github_release_asset_upload(){ # github_release_asset_upload + local repo_path="$1" + local release_id="$2" + local release_file_name="$3" + local release_file_path="$4" + local file_extension="${release_file_name##*.}" + local release_asset=`curl -s -k -X POST -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3.raw+json" -H "Content-Type: application/$file_extension" --data-binary "@$release_file_path" "https://uploads.github.com/repos/$repo_path/releases/$release_id/assets?name=$release_file_name" | jq -r '.id'` + if [ ! "$release_asset" == "" ] && [ ! "$release_asset" == "null" ]; then echo "$release_asset"; else echo ""; fi +} + +function github_release_asset_delete(){ # github_release_asset_delete + local repo_path="$1" + local release_asset_id="$2" + local res=$(curl -s -k -o /dev/null -w "%{http_code}" -X DELETE -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3.raw+json" "https://api.github.com/repos/$repo_path/releases/assets/$release_asset_id") + if [ "$res" -eq 204 ]; then echo 1; else echo 0; fi +} function git_branch_exists(){ # git_branch_exists diff --git a/tools/cron.sh b/tools/cron.sh index 000fdc4dc..1ca166717 100755 --- a/tools/cron.sh +++ b/tools/cron.sh @@ -1,8 +1,7 @@ #!/bin/bash -if [ ! "$GITHUB_EVENT_NAME" == "schedule" ]; then - echo "Wrong event '$GITHUB_EVENT_NAME'!" - exit 1 +if [ -z "$TARGET" ]; then + TARGET="all" fi -bash ./build.sh -d +bash ./build.sh -e -t $TARGET diff --git a/tools/install-arduino.sh b/tools/install-arduino.sh index 51e1162a4..abd1526fe 100755 --- a/tools/install-arduino.sh +++ b/tools/install-arduino.sh @@ -46,17 +46,3 @@ if [ "$AR_BRANCH" ]; then git -C "$AR_COMPS/arduino" pull --ff-only fi if [ $? -ne 0 ]; then exit 1; fi - -# -# CLONE/UPDATE ESP32-ARDUINO-LIBS -# -if [ ! -d "$IDF_LIBS_DIR" ]; then - echo "Cloning esp32-arduino-libs..." - git clone "$AR_LIBS_REPO_URL" "$IDF_LIBS_DIR" -else - echo "Updating esp32-arduino-libs..." - git -C "$IDF_LIBS_DIR" fetch && \ - git -C "$IDF_LIBS_DIR" pull --ff-only -fi -if [ $? -ne 0 ]; then exit 1; fi - diff --git a/tools/install-esp-idf.sh b/tools/install-esp-idf.sh index 320ce0cb0..c6f65dfbc 100755 --- a/tools/install-esp-idf.sh +++ b/tools/install-esp-idf.sh @@ -49,52 +49,3 @@ fi # source $IDF_PATH/export.sh - -# -# SETUP ARDUINO DEPLOY -# - -if [ "$GITHUB_EVENT_NAME" == "schedule" ] || [ "$GITHUB_EVENT_NAME" == "repository_dispatch" -a "$GITHUB_EVENT_ACTION" == "deploy" ]; then - # format new branch name and pr title - if [ -x $commit_predefined ]; then #commit was not specified at build time - AR_NEW_BRANCH_NAME="idf-$IDF_BRANCH" - AR_NEW_COMMIT_MESSAGE="IDF $IDF_BRANCH $IDF_COMMIT" - AR_NEW_PR_TITLE="IDF $IDF_BRANCH" - else - AR_NEW_BRANCH_NAME="idf-$IDF_COMMIT" - AR_NEW_COMMIT_MESSAGE="IDF $IDF_COMMIT" - AR_NEW_PR_TITLE="$AR_NEW_COMMIT_MESSAGE" - fi - LIBS_VERSION="idf-"${IDF_BRANCH//\//_}"-$IDF_COMMIT" - - AR_HAS_COMMIT=`git_commit_exists "$AR_COMPS/arduino" "$AR_NEW_COMMIT_MESSAGE"` - AR_HAS_BRANCH=`git_branch_exists "$AR_COMPS/arduino" "$AR_NEW_BRANCH_NAME"` - AR_HAS_PR=`github_pr_exists "$AR_REPO" "$AR_NEW_BRANCH_NAME"` - - LIBS_HAS_COMMIT=`git_commit_exists "$IDF_LIBS_DIR" "$AR_NEW_COMMIT_MESSAGE"` - LIBS_HAS_BRANCH=`git_branch_exists "$IDF_LIBS_DIR" "$AR_NEW_BRANCH_NAME"` - - if [ "$LIBS_HAS_COMMIT" == "1" ]; then - echo "Commit '$AR_NEW_COMMIT_MESSAGE' Already Exists in esp32-arduino-libs" - fi - - if [ "$AR_HAS_COMMIT" == "1" ]; then - echo "Commit '$AR_NEW_COMMIT_MESSAGE' Already Exists in arduino-esp32" - fi - - if [ "$LIBS_HAS_COMMIT" == "1" ] && [ "$AR_HAS_COMMIT" == "1" ]; then - exit 0 - fi - - export AR_NEW_BRANCH_NAME - export AR_NEW_COMMIT_MESSAGE - export AR_NEW_PR_TITLE - - export AR_HAS_COMMIT - export AR_HAS_BRANCH - export AR_HAS_PR - - export LIBS_VERSION - export LIBS_HAS_COMMIT - export LIBS_HAS_BRANCH -fi diff --git a/tools/push-to-arduino.sh b/tools/push-to-arduino.sh index f4d794d0c..c4d1959d3 100755 --- a/tools/push-to-arduino.sh +++ b/tools/push-to-arduino.sh @@ -1,16 +1,12 @@ #!/bin/bash -source ./tools/config.sh + +source ./tools/install-arduino.sh if [ -x $GITHUB_TOKEN ]; then echo "ERROR: GITHUB_TOKEN was not defined" exit 1 fi -if ! [ -d "$AR_COMPS/arduino" ]; then - echo "ERROR: Target arduino folder does not exist!" - exit 1 -fi - # setup git for pushing git config --global github.user "$GITHUB_ACTOR" git config --global user.name "$GITHUB_ACTOR" @@ -24,79 +20,96 @@ git config --global user.email "$GITHUB_ACTOR@github.com" # esp32-arduino-libs # -if [ $LIBS_HAS_COMMIT == "0" ] || [ $AR_HAS_COMMIT == "0" ]; then +LIBS_ZIP_FILENAME="esp32-arduino-libs-$LIBS_VERSION.zip" +LIBS_JSON_FILENAME="package-$LIBS_VERSION.json" +IDF_LIBS_ZIP_URL="https://github.com/$AR_LIBS_REPO/releases/download/$LIBS_RELEASE_TAG/$LIBS_ZIP_FILENAME" +IDF_LIBS_JSON_URL="https://github.com/$AR_LIBS_REPO/releases/download/$LIBS_RELEASE_TAG/$LIBS_JSON_FILENAME" + +if [ $AR_HAS_COMMIT == "0" ] || [ $LIBS_HAS_ASSET == "0" ]; then cd "$AR_ROOT" - # create branch if necessary - if [ "$LIBS_HAS_BRANCH" == "1" ]; then - echo "Branch '$AR_NEW_BRANCH_NAME' Already Exists" - echo "Switching to esp32-arduino-libs branch '$AR_NEW_BRANCH_NAME'..." - git -C "$IDF_LIBS_DIR" checkout $AR_NEW_BRANCH_NAME - else - echo "Creating esp32-arduino-libs branch '$AR_NEW_BRANCH_NAME'..." - git -C "$IDF_LIBS_DIR" checkout -b $AR_NEW_BRANCH_NAME + mkdir -p dist + + # check if the release exists + if [ $LIBS_HAS_RELEASE == "0" ]; then + echo "Release for tag \"$LIBS_RELEASE_TAG\" not found. Please create the release first." + exit 1 fi - if [ $? -ne 0 ]; then - echo "ERROR: Checkout of branch '$AR_NEW_BRANCH_NAME' failed" + + # Delete old assets for the version + if [ $LIBS_HAS_ASSET == "1" ]; then + echo "Deleting existing assets for version '$LIBS_VERSION'..." + if [ `github_release_asset_delete "$AR_LIBS_REPO" "$LIBS_ASSET_ID"` == "0" ]; then + echo "ERROR: Failed to delete asset '$LIBS_ZIP_FILENAME'" + fi + JSON_ASSET_ID=`github_release_asset_id "$AR_LIBS_REPO" "$LIBS_RELEASE_ID" "$LIBS_JSON_FILENAME"` + if [ "$JSON_ASSET_ID" != "" ] && [ `github_release_asset_delete "$AR_LIBS_REPO" "$JSON_ASSET_ID"` == "0" ]; then + echo "ERROR: Failed to delete asset '$LIBS_JSON_FILENAME'" + fi + fi + + echo "Creating asset '$LIBS_ZIP_FILENAME'..." + + mv -f "dist/esp32-arduino-libs.zip" "dist/$LIBS_ZIP_FILENAME" + LIBS_ASSET_ID=`github_release_asset_upload "$AR_LIBS_REPO" "$LIBS_RELEASE_ID" "$LIBS_ZIP_FILENAME" "dist/$LIBS_ZIP_FILENAME"` + if [ -z "$LIBS_ASSET_ID" ]; then + echo "ERROR: Failed to upload asset '$LIBS_ZIP_FILENAME'" exit 1 fi - # make changes to the files - echo "Patching files in esp32-arduino-libs branch '$AR_NEW_BRANCH_NAME'..." - rm -rf $IDF_LIBS_DIR/* && cp -Rf $AR_TOOLS/esp32-arduino-libs/* $IDF_LIBS_DIR/ - - cd $IDF_LIBS_DIR - if [ -f "README.md" ]; then - rm -rf "README.md" + echo "Finished uploading asset '$LIBS_ZIP_FILENAME'. Asset ID: $LIBS_ASSET_ID" + + # Calculate the local file checksum and size + local_checksum=$(sha256sum "dist/$LIBS_ZIP_FILENAME" | awk '{print $1}') + local_size=$(stat -c%s "dist/$LIBS_ZIP_FILENAME") + + echo "Downloading asset '$LIBS_ZIP_FILENAME' and checking integrity..." + + # Download the file + remote_file="remote-$LIBS_ZIP_FILENAME" + curl -s -L -o "$remote_file" "$IDF_LIBS_ZIP_URL" + + # Check if the download was successful + if [ $? -ne 0 ]; then + echo "Error downloading file from $IDF_LIBS_ZIP_URL" + exit 1 fi - # did any of the files change? - if [ -n "$(git status --porcelain)" ]; then - echo "Pushing changes to esp32-arduino-libs branch '$AR_NEW_BRANCH_NAME'..." - git add . && git commit --message "$AR_NEW_COMMIT_MESSAGE" && git push -u origin $AR_NEW_BRANCH_NAME - if [ $? -ne 0 ]; then - echo "ERROR: Pushing to branch '$AR_NEW_BRANCH_NAME' failed" - exit 1 - fi - IDF_LIBS_COMMIT=`git rev-parse --verify HEAD` - IDF_LIBS_DL_URL="https://codeload.github.com/espressif/esp32-arduino-libs/zip/$IDF_LIBS_COMMIT" - # ToDo: this URL needs to get into Arduino's package.json + # Calculate the remote file checksum and size + remote_checksum=$(sha256sum "$remote_file" | awk '{print $1}') + remote_size=$(stat -c%s "$remote_file") - # Download the file - filename="esp32-arduino-libs-$IDF_LIBS_COMMIT.zip" - curl -s -o "$filename" "$IDF_LIBS_DL_URL" + echo "Local: $local_size bytes, $local_checksum" + echo "Remote: $remote_size bytes, $remote_checksum" - # Check if the download was successful - if [ $? -ne 0 ]; then - echo "Error downloading file from $IDF_LIBS_DL_URL" - exit 1 + # Check if the checksums match + if [ "$local_checksum" != "$remote_checksum" ]; then + echo "Checksum mismatch for downloaded file" + echo "Deleting asset and exiting..." + if [ `github_release_asset_delete "$AR_LIBS_REPO" "$LIBS_ASSET_ID"` == "0" ]; then + echo "ERROR: Failed to delete asset '$LIBS_ZIP_FILENAME'" fi + exit 1 + fi - # Calculate the size in bytes and SHA-256 sum - size=$(stat -c%s "$filename") - sha256sum=$(sha256sum "$filename" | awk '{print $1}') - - # Clean up the downloaded file - rm "$filename" - - # Print the results - echo "Tool: esp32-arduino-libs" - echo "Version: $LIBS_VERSION" - echo "URL: $IDF_LIBS_DL_URL" - echo "File: $filename" - echo "Size: $size bytes" - echo "SHA-256: $sha256sum" - echo "JSON: $AR_OUT/package_esp32_index.template.json" - cd "$AR_ROOT" - python3 tools/add_sdk_json.py -j "$AR_OUT/package_esp32_index.template.json" -n "esp32-arduino-libs" -v "$LIBS_VERSION" -u "$IDF_LIBS_DL_URL" -f "$filename" -s "$size" -c "$sha256sum" - if [ $? -ne 0 ]; then exit 1; fi + # Clean up the downloaded file + rm "$remote_file" + + # Print the results + echo "Tool: esp32-arduino-libs" + echo "Version: $LIBS_VERSION" + echo "URL: $IDF_LIBS_ZIP_URL" + echo "File: $LIBS_ZIP_FILENAME" + echo "Size: $local_size bytes" + echo "SHA-256: $local_checksum" + echo "JSON: $AR_OUT/package_esp32_index.template.json" + cd "$AR_ROOT" + python3 tools/add_sdk_json.py -j "$AR_OUT/package_esp32_index.template.json" -n "esp32-arduino-libs" -v "$LIBS_VERSION" -u "$IDF_LIBS_ZIP_URL" -f "$LIBS_ZIP_FILENAME" -s "$local_size" -c "$local_checksum" + if [ $? -ne 0 ]; then exit 1; fi - else - echo "No changes in esp32-arduino-libs branch '$AR_NEW_BRANCH_NAME'" - if [ $LIBS_HAS_BRANCH == "0" ]; then - echo "Delete created branch '$AR_NEW_BRANCH_NAME'" - git branch -d $AR_NEW_BRANCH_NAME - fi - exit 0 + JSON_ASSET_ID=`github_release_asset_upload "$AR_LIBS_REPO" "$LIBS_RELEASE_ID" "$LIBS_JSON_FILENAME" "$AR_OUT/package_esp32_index.template.json"` + if [ -z "$JSON_ASSET_ID" ]; then + echo "ERROR: Failed to upload asset '$LIBS_JSON_FILENAME'" + exit 1 fi fi @@ -104,7 +117,7 @@ fi # esp32-arduino # -if [ $AR_HAS_COMMIT == "0" ]; then +if [ $AR_HAS_COMMIT == "0" ] || [ $LIBS_HAS_ASSET == "0" ]; then cd "$AR_ROOT" # create or checkout the branch if [ ! $AR_HAS_BRANCH == "0" ]; then @@ -115,31 +128,31 @@ if [ $AR_HAS_COMMIT == "0" ]; then git -C "$AR_COMPS/arduino" checkout -b $AR_NEW_BRANCH_NAME fi if [ $? -ne 0 ]; then - echo "ERROR: Checkout of branch '$AR_NEW_BRANCH_NAME' failed" + echo "ERROR: Checkout of branch '$AR_NEW_BRANCH_NAME' failed" exit 1 fi # make changes to the files echo "Patching files in branch '$AR_NEW_BRANCH_NAME'..." rm -rf "$AR_COMPS/arduino/package/package_esp32_index.template.json" && cp -f "$AR_OUT/package_esp32_index.template.json" "$AR_COMPS/arduino/package/package_esp32_index.template.json" - + cd $AR_COMPS/arduino # did any of the files change? if [ -n "$(git status --porcelain)" ]; then echo "Pushing changes to branch '$AR_NEW_BRANCH_NAME'..." - git add . && git commit --message "$AR_NEW_COMMIT_MESSAGE" && git push -u origin $AR_NEW_BRANCH_NAME + git add . && git commit --message "$AR_NEW_COMMIT_MESSAGE" && git push -u origin $AR_NEW_BRANCH_NAME if [ $? -ne 0 ]; then - echo "ERROR: Pushing to branch '$AR_NEW_BRANCH_NAME' failed" + echo "ERROR: Pushing to branch '$AR_NEW_BRANCH_NAME' failed" exit 1 fi else - echo "No changes in branch '$AR_NEW_BRANCH_NAME'" - if [ $AR_HAS_BRANCH == "0" ]; then - echo "Delete created branch '$AR_NEW_BRANCH_NAME'" - git branch -d $AR_NEW_BRANCH_NAME - fi - exit 0 + echo "No changes in branch '$AR_NEW_BRANCH_NAME'" + if [ $AR_HAS_BRANCH == "0" ]; then + echo "Delete created branch '$AR_NEW_BRANCH_NAME'" + git branch -d $AR_NEW_BRANCH_NAME + fi + exit 0 fi # CREATE PULL REQUEST