builder updates (#84)

* Update GH actions

* Update python

* CONFIG_SPI_FLASH_YIELD_DURING_ERASE=y

* Use MacOS ARM

* Update push.yml

* Update push.yml

* add lib Builder branch

* add env

* add comments

* CONFIG_ETH_TRANSMIT_MUTEX

* Fix build of build Platformio manifest (fail in second run)

* fix manifest script build error

* rm outdated code

* don't know why it needs to be restored?!?

* Build Pio framework manifest (#78)

* Update build.sh

* Update gen_pio_frmwk_manifest.py

* Delete core_version.h

* generate core version.h

* Update build.sh

Author: Jason2866

.github/workflows/push.yml

@@ -5,19 +5,22 @@ on:
 jobs:
   build-libs:
     name: Build Arduino Libs
-    runs-on: ubuntu-22.04
+    runs-on: macos-14
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
-        python-version: '3.10'
+        python-version: '3.11'
     - name: Install dependencies
       run: bash ./tools/prepare-ci.sh
+    - name: Get current branch
+      run: |
+        echo "GIT_BRANCH=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_ENV
     - name: Build Arduino Libs
       run: bash ./build.sh
     - name: Release
-      uses: jason2866/action-gh-release@v1.2
+      uses: jason2866/action-gh-release@v1.3
       with:
         tag_name: ${{ github.run_number }}
         body_path: release-info.txt

build.sh

@@ -144,9 +144,10 @@ mkdir -p "$AR_TOOLS/esp32-arduino-libs"
 rm -rf release-info.txt
 IDF_Commit_short=$(git -C "$IDF_PATH" rev-parse --short HEAD || echo "")
 AR_Commit_short=$(git -C "$AR_COMPS/arduino" rev-parse --short HEAD || echo "")
-echo "Framework built from 
-- $IDF_REPO branch [$IDF_BRANCH](https://github.com/$IDF_REPO/tree/$IDF_BRANCH) commit [$IDF_Commit_short](https://github.com/$IDF_REPO/commits/$IDF_BRANCH/#:~:text=$IDF_Commit_short) 
-- $AR_REPO branch [$AR_BRANCH](https://github.com/$AR_REPO/tree/$AR_BRANCH) commit [$AR_Commit_short](https://github.com/$AR_REPO/commits/$AR_BRANCH/#:~:text=$AR_Commit_short)" >> release-info.txt
+echo "Framework built from
+- $IDF_REPO branch [$IDF_BRANCH](https://github.com/$IDF_REPO/tree/$IDF_BRANCH) commit [$IDF_Commit_short](https://github.com/$IDF_REPO/commits/$IDF_BRANCH/#:~:text=$IDF_Commit_short)
+- $AR_REPO branch [$AR_BRANCH](https://github.com/$AR_REPO/tree/$AR_BRANCH) commit [$AR_Commit_short](https://github.com/$AR_REPO/commits/$AR_BRANCH/#:~:text=$AR_Commit_short)
+- Arduino lib builder branch: $GIT_BRANCH" >> release-info.txt
 
 #targets_count=`jq -c '.targets[] | length' configs/builds.json`
 for target_json in `jq -c '.targets[]' configs/builds.json`; do
@@ -234,19 +235,38 @@ for component in `ls "$AR_MANAGED_COMPS"`; do
     fi
 done
 
+export IDF_COMMIT=$(git -C "$IDF_PATH" rev-parse --short HEAD)
+
 # update package_esp32_index.template.json
 if [ "$BUILD_TYPE" = "all" ]; then
     python3 ./tools/gen_tools_json.py -i "$IDF_PATH" -j "$AR_COMPS/arduino/package/package_esp32_index.template.json" -o "$AR_OUT/"
     python3 ./tools/gen_tools_json.py -i "$IDF_PATH" -o "$TOOLS_JSON_OUT/"
     if [ $? -ne 0 ]; then exit 1; fi
 fi
 
-# Generate PlatformIO manifest file
+# Generate PlatformIO library manifest file
 if [ "$BUILD_TYPE" = "all" ]; then
-    python3 ./tools/gen_platformio_manifest.py -o "$TOOLS_JSON_OUT/" -s $(git -C "$IDF_PATH" symbolic-ref --short HEAD || git -C "$IDF_PATH" tag --points-at HEAD) -c $(git -C "$IDF_PATH" rev-parse --short HEAD)
+    python3 ./tools/gen_pio_lib_manifest.py -o "$TOOLS_JSON_OUT/" -s "$IDF_BRANCH" -c "$IDF_COMMIT"
     if [ $? -ne 0 ]; then exit 1; fi
 fi
 
+AR_VERSION=$(jq -c '.version' "$AR_COMPS/arduino/package.json" | tr -d '"')
+AR_VERSION_UNDERSCORE=`echo "$AR_VERSION" | tr . _`
+
+# Generate PlatformIO framework manifest file
+rm -rf "$AR_ROOT/package.json"
+if [ "$BUILD_TYPE" = "all" ]; then
+    python3 ./tools/gen_pio_frmwk_manifest.py -o "$AR_ROOT/" -s "v$AR_VERSION" -c "$IDF_COMMIT"
+    if [ $? -ne 0 ]; then exit 1; fi
+fi
+
+# Generate core_version.h
+rm -rf "$AR_ROOT/core_version.h"
+echo "#define ARDUINO_ESP32_GIT_VER 0x$AR_Commit_short
+#define ARDUINO_ESP32_GIT_DESC $AR_VERSION
+#define ARDUINO_ESP32_RELEASE_$AR_VERSION_UNDERSCORE
+#define ARDUINO_ESP32_RELEASE \"$AR_VERSION_UNDERSCORE\"" >> "$AR_ROOT/core_version.h"
+
 # copy everything to arduino-esp32 installation
 if [ $COPY_OUT -eq 1 ] && [ -d "$ESP32_ARDUINO" ]; then
     ./tools/copy-to-arduino.sh

configs/defconfig.common

@@ -42,6 +42,7 @@ CONFIG_ESP_WIFI_CSI_ENABLED=n
 CONFIG_ESP_WIFI_ENABLE_WPA3_SAE=y
 # CONFIG_ESP_WIFI_IRAM_OPT is not set
 # CONFIG_ESP_WIFI_RX_IRAM_OPT is not set
+CONFIG_ETH_TRANSMIT_MUTEX=y
 CONFIG_ETH_SPI_ETHERNET_DM9051=y
 CONFIG_ETH_SPI_ETHERNET_W5500=y
 CONFIG_ETH_SPI_ETHERNET_KSZ8851SNL=y
@@ -188,6 +189,7 @@ CONFIG_MBEDTLS_ECP_DP_SECP256R1_ENABLED=y
 CONFIG_OPENSSL_ASSERT_DO_NOTHING=y
 CONFIG_PTHREAD_TASK_STACK_SIZE_DEFAULT=2048
 CONFIG_SPI_FLASH_DANGEROUS_WRITE_ALLOWED=y
+CONFIG_SPI_FLASH_YIELD_DURING_ERASE=y
 CONFIG_SPI_FLASH_ERASE_YIELD_DURATION_MS=10
 CONFIG_SPI_FLASH_ERASE_YIELD_TICKS=2
 CONFIG_SPI_FLASH_WRITE_CHUNK_SIZE=4096

core_version.h

@@ -1,7 +1,5 @@
 #!/bin/bash
 
-IDF_COMMIT=$(git -C "$IDF_PATH" rev-parse --short HEAD || echo "")
-IDF_BRANCH=$(git -C "$IDF_PATH" symbolic-ref --short HEAD || git -C "$IDF_PATH" tag --points-at HEAD || echo "")
 idf_version_string=${IDF_BRANCH//\//_}"-$IDF_COMMIT"
 
 archive_path="dist/arduino-esp32-libs-$idf_version_string.tar.gz"

package.json

@@ -6,7 +6,7 @@ if [ -z $IDF_PATH ]; then
 fi
 
 if [ -z $IDF_BRANCH ]; then
-    IDF_BRANCH="release/v5.1"
+    export IDF_BRANCH="release/v5.1"
 fi
 
 # Arduino branch to use
@@ -60,17 +60,20 @@ TOOLS_JSON_OUT="$AR_TOOLS/esp32-arduino-libs"
 IDF_LIBS_DIR="$AR_ROOT/../esp32-arduino-libs"
 
 if [ "$IDF_COMMIT" ]; then
-    	echo "Using IDF commit $IDF_COMMIT"
 	export IDF_COMMIT
-elif [ -d "$IDF_PATH" ]; then
-	export IDF_COMMIT=$(git -C "$IDF_PATH" rev-parse --short HEAD)
-	export IDF_BRANCH=$(git -C "$IDF_PATH" symbolic-ref --short HEAD || git -C "$IDF_PATH" tag --points-at HEAD)
 fi
 
+if [ -d "$IDF_PATH" ]; then
+    export IDF_COMMIT=$(git -C "$IDF_PATH" rev-parse --short HEAD)
+fi
+
+echo "Using IDF branch $IDF_BRANCH"
+echo "Using IDF commit $IDF_COMMIT"
+
 if [ "$AR_COMMIT" ]; then
-    echo "Using commit $AR_COMMIT for Arduino"
+    echo "Using Arduino commit $AR_COMMIT"
 else
-    AR_COMMIT=$(git -C "$AR_COMPS/arduino" rev-parse --short HEAD || echo "")
+    export AR_COMMIT=$(git -C "$AR_COMPS/arduino" rev-parse --short HEAD || echo "")
 fi
 
 #rm -rf release-info.txt
@@ -103,6 +106,7 @@ function get_os(){
 AR_OS=`get_os`
 
 export SED="sed"
+export AWK="awk"
 export SSTAT="stat -c %s"
 
 if [[ "$AR_OS" == "macos" ]]; then
@@ -115,6 +119,7 @@ if [[ "$AR_OS" == "macos" ]]; then
         exit 1
     fi
     export SED="gsed"
+    export AWK="gawk"
     export SSTAT="stat -f %z"
 fi
 

tools/archive-build.sh

@@ -0,0 +1,86 @@
+import argparse
+import json
+import os
+import re
+import sys
+
+MANIFEST_DATA = {
+    "name": "framework-arduinoespressif32",
+    "description": "Platformio Tasmota Arduino framework for the Espressif ESP32 series of SoCs",
+    "keywords": ["framework", "tasmota", "arduino", "espressif", "esp32"],
+    "license": "LGPL-2.1-or-later",
+    "repository": {
+        "type": "git",
+        "url": "https://github.com/tasmota/arduino-esp32",
+    },
+}
+
+
+def convert_version(version_string):
+    """A helper function that converts a custom IDF version string
+    extracted from a Git repository to a suitable SemVer alternative. For example:
+    'release/v5.1' becomes '5.1.0',
+    'v7.7.7' becomes '7.7.7'
+    """
+
+    regex_pattern = (
+        r"v(?P<MAJOR>0|[1-9]\d*)\.(?P<MINOR>0|[1-9]\d*)\.*(?P<PATCH>0|[1-9]\d*)*"
+    )
+    match = re.search(regex_pattern, version_string)
+    if not match:
+        sys.stderr.write(
+            f"Failed to find a regex match for '{regex_pattern}' in '{version_string}'\n"
+        )
+        return ""
+
+    major, minor, patch = match.groups()
+    if not patch:
+        patch = "0"
+
+    return ".".join((major, minor, patch))
+
+
+def main(dst_dir, version_string, commit_hash):
+
+    converted_version = convert_version(version_string)
+    if not converted_version:
+        sys.stderr.write(f"Failed to convert version '{version_string}'\n")
+        return -1
+
+    manifest_file_path = os.path.join(dst_dir, "package.json")
+    with open(manifest_file_path, "w", encoding="utf8") as fp:
+        MANIFEST_DATA["version"] = f"{converted_version}+sha.{commit_hash}"
+        json.dump(MANIFEST_DATA, fp, indent=2)
+
+    print(
+        f"Generated PlatformIO framework manifest file '{manifest_file_path}' with '{converted_version}' version"
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-o",
+        "--dst-dir",
+        dest="dst_dir",
+        required=True,
+        help="Destination folder where the 'package.json' manifest will be located",
+    )
+    parser.add_argument(
+        "-s",
+        "--version-string",
+        dest="version_string",
+        required=True,
+        help="ESP-IDF version string used for compiling libraries",
+    )
+    parser.add_argument(
+        "-c",
+        "--commit-hash",
+        dest="commit_hash",
+        required=True,
+        help="ESP-IDF revision in form of a commit hash",
+    )
+    args = parser.parse_args()
+
+    sys.exit(main(args.dst_dir, args.version_string, args.commit_hash))

tools/config.sh

@@ -7,7 +7,7 @@
 MANIFEST_DATA = {
     "name": "framework-arduinoespressif32-libs",
     "description": "Precompiled libraries for Arduino Wiring-based Framework for the Espressif ESP32 series of SoCs",
-    "keywords": ["framework", "arduino", "espressif", "esp32"],
+    "keywords": ["framework", "tasmota", "arduino", "espressif", "esp32"],
     "license": "LGPL-2.1-or-later",
     "repository": {
         "type": "git",
@@ -53,7 +53,7 @@ def main(dst_dir, version_string, commit_hash):
         json.dump(MANIFEST_DATA, fp, indent=2)
 
     print(
-        f"Generated PlatformIO manifest file '{manifest_file_path}' with '{converted_version}' version"
+        f"Generated PlatformIO libraries manifest file '{manifest_file_path}' with '{converted_version}' version"
     )
     return 0
 

tools/gen_pio_frmwk_manifest.py

@@ -59,7 +59,7 @@
         if tool_name.endswith('-elf'):
             tool_name += '-gcc'
         print('Found {0}, version: {1}'.format(tool_name, tool_version))
-        
+
         if simple_output == False:
             dep_found = False
             dep_skip = False

tools/gen_platformio_manifest.py renamed to tools/gen_pio_lib_manifest.py

@@ -63,8 +63,6 @@ fi
 if [ ! -x $idf_was_installed ] || [ ! -x $commit_predefined ]; then
         git submodule update --recursive
 	$IDF_PATH/install.sh
-	export IDF_COMMIT=$(git -C "$IDF_PATH" rev-parse --short HEAD)
-	export IDF_BRANCH=$(git -C "$IDF_PATH" symbolic-ref --short HEAD || git -C "$IDF_PATH" tag --points-at HEAD)
 
 	# Temporarily patch the ESP32-S2 I2C LL driver to keep the clock source
 	cd $IDF_PATH

tools/gen_tools_json.py

@@ -1,4 +1,16 @@
 #!/bin/bash
 
-sudo apt-get install -y gperf cmake ninja-build
-pip3 install wheel future pyelftools
+# Ubuntu setup
+# Change in archive-build.sh gawk to awk
+#sudo apt-get install -y gperf cmake ninja-build
+#pip3 install wheel future pyelftools
+
+# MacOS (ARM) setup
+# Change in archive-build.sh awk to gawk
+brew install gsed
+brew install gawk
+brew install gperf
+brew install ninja
+brew install ccache
+python -m pip install --upgrade pip
+pip install wheel future pyelftools