From 89859f7f4c07ce3e5d487a4b689dbda78f5d5c3a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 31 May 2018 20:19:10 +0300 Subject: [PATCH 1/6] Calculate an absolute path for a custom partitions table (#1452) --- tools/platformio-build.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/platformio-build.py b/tools/platformio-build.py index b2129409ad5..622e7712eab 100644 --- a/tools/platformio-build.py +++ b/tools/platformio-build.py @@ -24,7 +24,7 @@ # Extends: https://github.com/platformio/platform-espressif32/blob/develop/builder/main.py -from os.path import isdir, isfile, join +from os.path import abspath, isdir, isfile, join from SCons.Script import DefaultEnvironment @@ -161,8 +161,9 @@ fwpartitions_dir = join(FRAMEWORK_DIR, "tools", "partitions") partitions_csv = env.BoardConfig().get("build.partitions", "default.csv") env.Replace( - PARTITIONS_TABLE_CSV=join(fwpartitions_dir, partitions_csv) if isfile( - join(fwpartitions_dir, partitions_csv)) else partitions_csv) + PARTITIONS_TABLE_CSV=abspath( + join(fwpartitions_dir, partitions_csv) if isfile( + join(fwpartitions_dir, partitions_csv)) else partitions_csv)) partition_table = env.Command( join("$BUILD_DIR", "partitions.bin"), From 90c99fb80904c94b40f59647540c0f1a70b8fad4 Mon Sep 17 00:00:00 2001 From: Martin Vychodil Date: Tue, 12 Jun 2018 21:27:26 +0200 Subject: [PATCH 2/6] * Arduino-ESP32 release management scripted (ready-to-merge) --- .travis.yml | 61 ++-- build-release.sh | 348 ++++++++++++++++++++++ build-tests.sh | 72 +++++ deploy.sh | 147 +++++++++ package/merge_packages.py | 78 +++++ package/package_esp32_index.template.json | 52 ++-- 6 files changed, 694 insertions(+), 64 deletions(-) create mode 100644 build-release.sh create mode 100644 build-tests.sh create mode 100644 deploy.sh create mode 100644 package/merge_packages.py diff --git a/.travis.yml b/.travis.yml index 3929a491e89..95d93a07414 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,44 +7,28 @@ python: os: - linux +env: + global: + #2DO: replace by Arduino-ESP32 specific output of traviscli: travis encrypt ESP32_GITHUB_TOKEN="" + #- secure: "rZ5dID535OdWTzXReLawQQXzPeBeDoTy293FV3VdSxG4bQGfk52MH6pd9XzaVGNyvfeXlWpeOwgJauOEiCBQSe4j71SsWZUFWOSz2IJguF6HiHYi3ftG1oHPHi1J6kEZ6XdrHi6rPTmv2x8Bj5Wk6SV+eOfT0OlM89BovVnhOvXoCH65g4rqvc01XLngE/Lj5mxZIpWMDyBfdhQQcqmp6fZLxAziOauD4Xjd1U1IxGiSlYjWin4948P2UezuT9PbFRTfCKjedA5nXAtz6gwz0+SGM8gjTZRXMQ6aYtbQNSZmdX3HsHZiyVDFmdaYiZVYcr7iGvJgmgjvb9AUmpEVTybr7ph0SpA0il4gutojAiJik5sG4/fvNlsYnNiAxgCf7SmyE6z/pAnODrNN7CVc/jdHihmc5hNQgkbcLrPI902UXyFUuP4GIBK5Ph1qA88HKtg1gp0xe1yPh64KMi0SXtX8NIimOeOCNFV1Yp9/9sSDy2sbZRBHmE36LZH6duMGbpzPU9bp7xYy5SlcTnXCqd6II8JkICWJdkfQNZ39JuDIqJZFPKO+9RfglFq5TnPq64SupySe6wrhPjpHHrs7tikNsl4j9/OU0i1tqZc/H6hiZ0EF7nL0hO3Vm9lKbMlcwOy5DSv2OSe7uSnDX4bbbvvBWKJiX1okuwb9olQhmUc=" + - REMOTE_URL=https://github.com/$TRAVIS_REPO_SLUG/releases/download/$TRAVIS_TAG + script: - #- set -e - - echo -e "travis_fold:start:sketch_test_env_prepare" - - pip install pyserial - - wget -O arduino.tar.xz https://www.arduino.cc/download.php?f=/arduino-nightly-linux64.tar.xz - - tar xf arduino.tar.xz - - mv arduino-nightly $HOME/arduino_ide - - mkdir -p $HOME/Arduino/libraries - - cd $HOME/arduino_ide/hardware - - mkdir espressif - - cd espressif - - ln -s $TRAVIS_BUILD_DIR esp32 - - cd esp32 - - git submodule update --init --recursive - - cd tools - - python get.py - - export PATH="$HOME/arduino_ide:$TRAVIS_BUILD_DIR/tools/xtensa-esp32-elf/bin:$PATH" - - which arduino - - cd $TRAVIS_BUILD_DIR - - source tools/common.sh - - echo -e "travis_fold:end:sketch_test_env_prepare" - - echo -e "travis_fold:start:sketch_test" - - build_sketches $HOME/arduino_ide $TRAVIS_BUILD_DIR/libraries "-l $HOME/Arduino/libraries" - - echo -e "travis_fold:end:sketch_test" - - echo -e "travis_fold:start:size_report" - - cat size.log - - echo -e "travis_fold:end:size_report" + #2DO: uncomment after successful TCI/GH integration + # build the package and run platformIO tests + # - bash $TRAVIS_BUILD_DIR/tools/build-tests.sh + + # zip the package if tagged build, otherwise finish here + - bash $TRAVIS_BUILD_DIR/tools/build-release.sh -a$ESP32_GITHUB_TOKEN + +deploy: + provider: script + skip_cleanup: true + script: bash $TRAVIS_BUILD_DIR/tools/deploy.sh -t$TRAVIS_TAG -a$ESP32_GITHUB_TOKEN -s$TRAVIS_REPO_SLUG -drelease + + on: + tags: true - # test library examples with PlatformIO - - echo -e "travis_fold:start:platformio_test_env_prepare" - - pip install -U https://github.com/platformio/platformio/archive/develop.zip - - platformio platform install https://github.com/platformio/platform-espressif32.git#feature/stage - - sed -i 's/https:\/\/github\.com\/espressif\/arduino-esp32\.git/*/' ~/.platformio/platforms/espressif32/platform.json - - ln -s $TRAVIS_BUILD_DIR ~/.platformio/packages/framework-arduinoespressif32 - - echo -e "travis_fold:end:platformio_test_env_prepare" - - echo -e "travis_fold:start:platformio_test" - - "python -c \"import glob,os,subprocess,sys; map(lambda p: (sys.stdout.write('Library example: %s\\n' % p), subprocess.call(['pio', 'ci', p, '--board', 'esp32dev'])), set([os.path.dirname(p) for p in glob.glob('libraries/*/examples/*/*.ino') + glob.glob('libraries/*/examples/*/*/*.ino')]))\"" - - echo -e "travis_fold:end:platformio_test" notifications: email: @@ -52,7 +36,8 @@ notifications: on_failure: change webhooks: urls: - - https://webhooks.gitter.im/e/cb057279c430d91a47a8 + #2DO: replace with Arduino-ESP32 specific link + #- https://webhooks.gitter.im/e/6dfa09a8a8ffd43f9297 on_success: change # options: [always|never|change] default: always on_failure: always # options: [always|never|change] default: always - on_start: false # default: false + on_start: never # options: [always|never|change] default: always \ No newline at end of file diff --git a/build-release.sh b/build-release.sh new file mode 100644 index 00000000000..76c57a73c95 --- /dev/null +++ b/build-release.sh @@ -0,0 +1,348 @@ +#!/bin/bash + +############################################################ +# $1 - download link +# $2 - JSON output file +function downloadAndMergePackageJSON() +{ + echo " --- Package JSON definition merge BEGIN ---" + + jsonLink=$1 + jsonOut=$2 + curlAuthToken=$3 + outDirectory=$4 + + echo "Remote package JSON: $jsonLink (source)" + echo "Current package JSON: $jsonOut (target)" + + old_json=$outDirectory/oldJson.json + merged_json=$outDirectory/mergedJson.json + + #DEBUG + #echo " Local tmp for remote JSON: $old_json" + #echo " Merge output JSON: $merged_json" + + echo "Downloading JSON package definition: $jsonLink ..." + + # Authentication through HTTP headers might fail on redirection due to bug in cURL (https://curl.haxx.se/docs/adv_2018-b3bf.html - headers are resent to the target location including the original authentication) + # Notes: + # - eg AmazonAWS fails with Bad Request due to having maximum 1 authentication mechanism per a request, might be general issue + # - it's a first-class credential leakage + # - the fix is available in cURL 7.58.0+, however, TravisCI is not yet updated (May 29, 2018) - see https://docs.travis-ci.com/user/build-environment-updates + # - TravisCI workaround: updating build environment through .travis.yml (ie install required version of cURL using apt-get, see https://docs.travis-ci.com/user/installing-dependencies/) + # - previous point not used on purpose (build time increase, possible failure corrupts whole build, etc) but it's good to know there's a way out of hell + # - local workaround: authentication through 'access_token' as GET parameter works smoothly, however, HTTP headers are preferred + + #curl --verbose -sH "Authorization: token $curlAuthToken" -L -o "$old_json" "$jsonLink" + curl -L -o "$old_json" "$jsonLink?access_token=$curlAuthToken" + + #curl -L -o "$old_json" "$jsonLink" + + echo "Merging $old_json into $jsonOut ..." + + echo + set +e + stdbuf -oL python package/merge_packages.py "$jsonOut" "$old_json" > "$merged_json" + set -e #supposed to be ON by default + echo + + set -v + if [ ! -s $merged_json ]; then + rm -f "$merged_json" + echo "Nothing to merge ($merged_json empty), $jsonOut unchanged" + else + rm -f "$jsonOut" + mv "$merged_json" "$jsonOut" + echo "Data successfully merged to $jsonOut" + fi + + rm -f "$old_json" + set +v + echo " --- Package JSON definition merge END ---" +} +############################################################ + +#Cmdline options +# -a: GitHub API access token +# -d: output directory to store the (pre)release filedir set + +set -e + +echo +echo "===================================================================" +echo "RELEASE PACKAGE PUBLISHING ARRANGEMENTS (GitHub/Arduino compliance)" +echo "===================================================================" +echo + +# cURL authentication token +while getopts ":a:,:d:" opt; do + case $opt in + a) + curlAuth=$OPTARG + echo "ACCESS TOKEN: $curlAuth" >&2 + ;; + d) + releaseDir=$OPTARG + echo "RELEASE OUTPUT DIRECTORY: $releaseDir" >&2 + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument." >&2 + exit 1 + ;; + esac +done + +# where we at? +if [ -z "$TRAVIS_BUILD_DIR" ]; then + echo "Non-TravisCI environment" + cd "$( dirname ${BASH_SOURCE[0]} )"/.. + bTravisRun=0 +else + echo "TravisCI run" + cd $TRAVIS_BUILD_DIR + bTravisRun=1 +fi + +# no tag, no love +if [ -z "$TRAVIS_TAG" ] && [ $bTravisRun -eq 1 ]; then + echo "Non-tagged builds not supported in Travis CI environment, exiting" + exit 0 +fi + +currentDir=`pwd` +echo "Current working directory: $currentDir" + +srcdir=$currentDir + +if [ -z "$releaseDir" ]; then + releaseDir=release +fi +echo "Release output directory: $releaseDir" + + +# get current branch name and commit hash +branch_name="" +verx="" +extent="" + +if [ -z "$TRAVIS_TAG" ]; then + branch_name=`git rev-parse --abbrev-ref HEAD 2>/dev/null` + ver=`sed -n -E 's/version=([0-9.]+)/\1/p' platform.txt` + verx=`git rev-parse --short=8 HEAD 2>/dev/null` +else + ver=$TRAVIS_TAG +fi + +# Package name (case-insensitive) +shopt -s nocasematch + +if [ ! -z "$branch_name" ] && [ "$branch_name" != "master" ]; then + extent="-$branch_name-$verx" +fi + +package_name=esp32-$ver$extent + +shopt -u nocasematch + +echo "Version: $ver" +echo "Branch name: $branch_name" +echo "Git revision (8B): $verx" +echo "Extension: $extent" +echo "Travis CI tag: $TRAVIS_TAG" +echo "Package name: $package_name" + +# Set REMOTE_URL environment variable to the address where the package will be +# available for download. This gets written into package json file. + +if [ -z "$REMOTE_URL" ]; then + REMOTE_URL="http://localhost:8000" + echo "REMOTE_URL not defined, using default" +fi + +echo "Remote: $REMOTE_URL" + +# Create directory for the package +outdir=$releaseDir/$package_name +echo "Temporary output directory: $outdir" + +rm -rf $releaseDir +mkdir -p $outdir + +# Copy package required stuff: + +# +cp -f $srcdir/boards.txt $outdir/ +cp -f $srcdir/platform.txt $outdir/ +cp -f $srcdir/programmers.txt $outdir/ + +# +# cores/ +# libraries/ +# variants/ +cp -Rf $srcdir/cores $outdir/ +cp -Rf $srcdir/libraries $outdir/ +cp -Rf $srcdir/variants $outdir/ + +# +# tools/partitions/ +mkdir -p $outdir/tools/partitions +cp -f $srcdir/tools/partitions/boot_app0.bin $outdir/tools/partitions +cp -f $srcdir/tools/partitions/default.csv $outdir/tools/partitions +cp -f $srcdir/tools/partitions/minimal.csv $outdir/tools/partitions +cp -f $srcdir/tools/partitions/min_spiffs.csv $outdir/tools/partitions +cp -f $srcdir/tools/partitions/no_ota.csv $outdir/tools/partitions + +# tools/sdk/ +cp -Rf $srcdir/tools/sdk $outdir/tools/ + +# tools/ +cp -f $srcdir/tools/espota.exe $outdir/tools/ +cp -f $srcdir/tools/espota.py $outdir/tools/ +cp -f $srcdir/tools/esptool.py $outdir/tools/ +cp -f $srcdir/tools/gen_esp32part.py $outdir/tools/ +cp -f $srcdir/tools/gen_esp32part.exe $outdir/tools/ + +find $outdir -name '*.DS_Store' -exec rm -f {} \; + +# Do some replacements in platform.txt file, which are required because IDE +# handles tool paths differently when package is installed in hardware folder +cat $srcdir/platform.txt | \ +sed 's/runtime.tools.xtensa-esp32-elf-gcc.path={runtime.platform.path}\/tools\/xtensa-esp32-elf//g' | \ +sed 's/tools.esptool.path={runtime.platform.path}\/tools\/esptool/tools.esptool.path=\{runtime.tools.esptool.path\}/g' \ + > $outdir/platform.txt + +# Put core version and short hash of git version into core_version.h +ver_define=`echo $plain_ver | tr "[:lower:].\055" "[:upper:]_"` +echo Ver define: $ver_define +echo \#define ARDUINO_ESP32_GIT_VER 0x`git rev-parse --short=8 HEAD 2>/dev/null` >$outdir/cores/esp32/core_version.h +echo \#define ARDUINO_ESP32_GIT_DESC `git describe --tags 2>/dev/null` >>$outdir/cores/esp32/core_version.h +echo \#define ARDUINO_ESP32_RELEASE_$ver_define >>$outdir/cores/esp32/core_version.h +echo \#define ARDUINO_ESP32_RELEASE \"$ver_define\" >>$outdir/cores/esp32/core_version.h + +# Store submodules' current versions +git submodule status > $releaseDir/submodules.txt + +# remove all .git* files +find $outdir -name '*.git*' -type f -delete + +# Zip the package +package_name_zip=$package_name.zip + +echo "----------------------------------------------------------" +echo "Making $package_name ZIP archive..." +echo + +pushd $releaseDir >/dev/null + +zip -qr $package_name_zip $package_name + + +echo "----------------------------------------------------------" +echo "Making $package_name JSON definition file(s)..." +echo + +# Calculate SHA sum and size +sha=`shasum -a 256 $package_name_zip | cut -f 1 -d ' '` +size=`/bin/ls -l $package_name_zip | awk '{print $5}'` +echo Size: $size +echo SHA-256: $sha + +popd >/dev/null + +PACKAGE_JSON_DEV="package_esp32_dev_index.json" +PACKAGE_JSON_REL="package_esp32_index.json" + +# figure out the package type (release / pre-release) +shopt -s nocasematch +if [[ $TRAVIS_TAG == *-RC* ]]; then + bIsPrerelease=1 + package_name_json=$PACKAGE_JSON_DEV + echo "Package type: PRE-RELEASE, JSON def.file: $PACKAGE_JSON_DEV" +else + bIsPrerelease=0 + package_name_json=$PACKAGE_JSON_REL + echo "Package type: RELEASE, JSON def.files: $PACKAGE_JSON_REL, $PACKAGE_JSON_DEV" +fi +shopt -u nocasematch + +# Generate JSON package definition +echo +echo "----------------------------------------------------------" +echo "Preparing current package definition ($package_name_json)..." + +# JSON contents +jq_arg=".packages[0].platforms[0].version = \"$ver\" | \ + .packages[0].platforms[0].url = \"$REMOTE_URL/$package_name_zip\" |\ + .packages[0].platforms[0].archiveFileName = \"$package_name_zip\"" + +jq_arg="$jq_arg |\ + .packages[0].platforms[0].size = \"$size\" |\ + .packages[0].platforms[0].checksum = \"SHA-256:$sha\"" + + +# Cleanup temporary work dir +rm -rf $outdir + + +# Get previous release name +echo +echo "----------------------------------------------------------" +echo "Getting previous releases versions..." + +releasesJson=$releaseDir/releases.json + +curl -sH "Authorization: token $curlAuth" https://api.github.com/repos/$TRAVIS_REPO_SLUG/releases > $releasesJson + +# Previous final release (prerelase == false) +prev_release=$(jq -r '. | map(select(.draft == false and .prerelease == false)) | sort_by(.created_at | - fromdateiso8601) | .[0].tag_name' ${releasesJson}) +# Previous release (possibly a pre-release) +prev_any_release=$(jq -r '. | map(select(.draft == false)) | sort_by(.created_at | - fromdateiso8601) | .[0].tag_name' ${releasesJson}) +# Previous pre-release +prev_pre_release=$(jq -r '. | map(select(.draft == false and .prerelease == true)) | sort_by(.created_at | - fromdateiso8601) | .[0].tag_name' ${releasesJson}) + +rm -f "$releasesJson" + +echo "Previous release: $prev_release" +echo "Previous (pre-?)release: $prev_any_release" +echo "Previous pre-release: $prev_pre_release" + +# always get DEV version of JSON (included in both RC/REL) +echo +echo "----------------------------------------------------------" +echo "Generating $PACKAGE_JSON_DEV..." +echo + +cat $srcdir/package/package_esp32_index.template.json | jq "$jq_arg" > $releaseDir/$PACKAGE_JSON_DEV +if [ ! -z "$prev_any_release" ] && [ "$prev_any_release" != "null" ]; then + downloadAndMergePackageJSON "https://github.com/$TRAVIS_REPO_SLUG/releases/download/${prev_any_release}/${PACKAGE_JSON_DEV}" "$releaseDir/${PACKAGE_JSON_DEV}" "${curlAuth}" "$releaseDir" + + # Release notes: GIT log comments (prev_any_release, current_release> + git log --oneline $prev_any_release.. > $releaseDir/commits.txt +fi + +# for RELEASE run update REL JSON as well +if [ $bIsPrerelease -eq 0 ]; then + + echo + echo "----------------------------------------------------------" + echo "Generating $PACKAGE_JSON_REL..." + echo + + cat $srcdir/package/package_esp32_index.template.json | jq "$jq_arg" > $releaseDir/$PACKAGE_JSON_REL + if [ ! -z "$prev_release" ] && [ "$prev_release" != "null" ]; then + downloadAndMergePackageJSON "https://github.com/$TRAVIS_REPO_SLUG/releases/download/${prev_release}/${PACKAGE_JSON_REL}" "$releaseDir/${PACKAGE_JSON_REL}" "${curlAuth}" "$releaseDir" + + # Release notes: GIT log comments (prev_release, current_release> + git log --oneline $prev_release.. > $releaseDir/commits.txt + fi +fi + +echo +echo "==============================================================" +echo "Package '$package_name' ready for publishing, script finished." +echo "==============================================================" +echo diff --git a/build-tests.sh b/build-tests.sh new file mode 100644 index 00000000000..4651cc7e73a --- /dev/null +++ b/build-tests.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +#- set -e +echo -e "travis_fold:start:sketch_test_env_prepare" + +pip install pyserial + +wget -O arduino.tar.xz https://www.arduino.cc/download.php?f=/arduino-nightly-linux64.tar.xz + +tar xf arduino.tar.xz + +mv arduino-nightly $HOME/arduino_ide + +mkdir -p $HOME/Arduino/libraries + +cd $HOME/arduino_ide/hardware + +mkdir espressif + +cd espressif + +ln -s $TRAVIS_BUILD_DIR esp32 + +cd esp32 + +git submodule update --init --recursive + +cd tools + +python get.py + +export PATH="$HOME/arduino_ide:$TRAVIS_BUILD_DIR/tools/xtensa-esp32-elf/bin:$PATH" + +which arduino + +cd $TRAVIS_BUILD_DIR + +source tools/common.sh + +echo -e "travis_fold:end:sketch_test_env_prepare" + +echo -e "travis_fold:start:sketch_test" + +build_sketches $HOME/arduino_ide $TRAVIS_BUILD_DIR/libraries "-l $HOME/Arduino/libraries" + +echo -e "travis_fold:end:sketch_test" + +echo -e "travis_fold:start:size_report" + +cat size.log + +echo -e "travis_fold:end:size_report" + +test library examples with PlatformIO + +echo -e "travis_fold:start:platformio_test_env_prepare" + +pip install -U https://github.com/platformio/platformio/archive/develop.zip + +platformio platform install https://github.com/platformio/platform-espressif32.git#feature/stage + +sed -i 's/https:\/\/github\.com\/espressif\/arduino-esp32\.git/*/' ~/.platformio/platforms/espressif32/platform.json + +ln -s $TRAVIS_BUILD_DIR ~/.platformio/packages/framework-arduinoespressif32 + +echo -e "travis_fold:end:platformio_test_env_prepare" + +echo -e "travis_fold:start:platformio_test" + +"python -c \"import glob,os,subprocess,sys; map(lambda p: (sys.stdout.write('Library example: %s\\n' % p), subprocess.call(['pio', 'ci', p, '--board', 'esp32dev'])), set([os.path.dirname(p) for p in glob.glob('libraries/*/examples/*/*.ino') + glob.glob('libraries/*/examples/*/*/*.ino')]))\"" + +echo -e "travis_fold:end:platformio_test" \ No newline at end of file diff --git a/deploy.sh b/deploy.sh new file mode 100644 index 00000000000..5ce5a690136 --- /dev/null +++ b/deploy.sh @@ -0,0 +1,147 @@ +#!/bin/bash + +set -e + +#Cmdline options +# -t: tag (*_RC* determines prerelease version, can be overriden be -p) +# -a: GitHub API access token +# -s: GitHub repository slug (user/repo) +# -p: prerelease true/false +# -f: files to upload (ie assets. delim = ';', must come quoted) +# -d: directory to upload (by adding dir contents to assets) +while getopts ":t:,:a:,:s:,:p:,:f:,:d:" opt; do + case $opt in + t) + varTagName=$OPTARG + echo "TAG: $varTagName" >&2 + ;; + a) + varAccessToken=$OPTARG + echo "ACCESS TOKEN: $varAccessToken" >&2 + ;; + s) + varRepoSlug=$OPTARG + echo "REPO SLUG: $varRepoSlug" >&2 + ;; + p) + varPrerelease=$OPTARG + echo "PRERELEASE: $varPrerelease" >&2 + ;; + f) + varAssets=$OPTARG + echo "ASSETS: $varAssets" >&2 + ;; + d) + varAssetsDir=$OPTARG + echo "ASSETS DIR: $varAssetsDir" >&2 + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument." >&2 + exit 1 + ;; + esac +done + +#Check tag name for release/prerelease (prerelease tag contains '_RC' as for release-candidate. case-insensitive) +shopt -s nocasematch +if [ -z $varPrerelease ]; then + if [[ $varTagName == *-RC* ]]; then + varPrerelease=true + else + varPrerelease=false + fi +fi +shopt -u nocasematch + +# +# Prepare Markdown release notes: +################################# +# +# - tag's description: +# ignore first 3 lines - commiter, tagname, blank +# first line of message: heading +# other lines: converted to bullets +# empty lines ignored +# if '* ' found as a first char pair, it's converted to '- ' to keep bulleting unified +relNotesRaw=`git show -s --format=%b $varTagName` +readarray -t msgArray <<<"$relNotesRaw" + +arrLen=${#msgArray[@]} +if [ $arrLen > 3 ]; then + ind=3 + while [ $ind -lt $arrLen ]; do + if [ $ind -eq 3 ]; then + releaseNotes="#### ${msgArray[ind]}\\n" + else + oneLine="$(echo -e "${msgArray[ind]}" | sed -e 's/^[[:space:]]*//')" + + if [ ${#oneLine} -gt 0 ]; then + if [ "${oneLine:0:2}" == "* " ]; then oneLine=$(echo ${oneLine/\*/-}); fi + if [ "${oneLine:0:2}" != "- " ]; then releaseNotes+="- "; fi + releaseNotes+="$oneLine\\n" + fi + fi + let ind=$ind+1 + done +else + releaseNotes="#### Release of $varTagName\\n" +fi + +# - list of commits (commits.txt must exit in the output dir) +commitFile=$varAssetsDir/commits.txt +if [ -e "$commitFile" ]; then + + releaseNotes+="\\n##### Commits\\n" + + IFS=$'\n' + for next in `cat $commitFile` + do + IFS=' ' read -r commitId commitMsg <<< "$next" + releaseNotes+="- [$commitId](https://github.com/$varRepoSlug/commit/$commitId) $commitMsg\\n" + done + rm -f $commitFile +fi + +releaseNotes=$(perl -pe 's/\r?\n/\\n/' <<< ${releaseNotes}) + +#JSON parameters to create a new release +curlData="{\"tag_name\": \"$varTagName\",\"target_commitish\": \"master\",\"name\": \"v$varTagName\",\"body\": \"$releaseNotes\",\"draft\": false,\"prerelease\": $varPrerelease}" + +#Create the release (initial source file assets created by GitHub) +releaseId=$(curl --data "$curlData" https://api.github.com/repos/$varRepoSlug/releases?access_token=$varAccessToken | jq -r '.id') +echo Release ID: $releaseId + +# Assets defined by dir contents +if [ ! -z $varAssetsDir ]; then + varAssetsTemp=$(ls -p $varAssetsDir | grep -v / | tr '\n' ';') + for item in $(echo $varAssetsTemp | tr ";" "\n") + do + varAssets+=$varAssetsDir/$item; + varAssets+=';' + done +fi + +echo +echo varAssets: $varAssets + +#Upload additional assets +if [ ! -z $varAssets ]; then + curlAuth="Authorization: token $varAccessToken" + for filename in $(echo $varAssets | tr ";" "\n") + do + echo + echo + echo Uploading $filename... + + curl -X POST -sH "$curlAuth" -H "Content-Type: application/octet-stream" --data-binary @"$filename" https://uploads.github.com/repos/$varRepoSlug/releases/$releaseId/assets?name=$(basename $filename) + done +fi + +echo +echo + + diff --git a/package/merge_packages.py b/package/merge_packages.py new file mode 100644 index 00000000000..c05787c8d8f --- /dev/null +++ b/package/merge_packages.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python +# This script merges two Arduino Board Manager package json files. +# Usage: +# python merge_packages.py package_esp8266com_index.json version/new/package_esp8266com_index.json +# Written by Ivan Grokhotkov, 2015 +# +from __future__ import print_function +from distutils.version import LooseVersion +import re +import json +import sys + +def load_package(filename): + pkg = json.load(open(filename))['packages'][0] + print("Loaded package {0} from {1}".format(pkg['name'], filename), file=sys.stderr) + print("{0} platform(s), {1} tools".format(len(pkg['platforms']), len(pkg['tools'])), file=sys.stderr) + return pkg + +def merge_objects(versions, obj): + for o in obj: + name = o['name'].encode('ascii') + ver = o['version'].encode('ascii') + if not name in versions: + print("found new object, {0}".format(name), file=sys.stderr) + versions[name] = {} + if not ver in versions[name]: + print("found new version {0} for object {1}".format(ver, name), file=sys.stderr) + versions[name][ver] = o + return versions + +# Normalize ESP release version string (x.x.x) by adding '-rc' (x.x.x-rc9223372036854775807) to ensure having REL above any RC +# Dummy approach, functional anyway for current ESP package versioning (unlike NormalizedVersion/LooseVersion/StrictVersion & similar crap) +def pkgVersionNormalized(versionString): + + verStr = str(versionString) + verParts = re.split('\.|-rc', verStr, flags=re.IGNORECASE) + + if len(verParts) == 3: + verStr = str(versionString) + '-rc' + str(sys.maxint) + elif len(verParts) != 4: + print("pkgVersionNormalized WARNING: unexpected version format: {0})".format(verStr), file=sys.stderr) + + return verStr + + +def main(args): + if len(args) < 3: + print("Usage: {0} ".format(args[0]), file=sys.stderr) + return 1 + + tools = {} + platforms = {} + pkg1 = load_package(args[1]) + tools = merge_objects(tools, pkg1['tools']); + platforms = merge_objects(platforms, pkg1['platforms']); + pkg2 = load_package(args[2]) + tools = merge_objects(tools, pkg2['tools']); + platforms = merge_objects(platforms, pkg2['platforms']); + + pkg1['tools'] = [] + pkg1['platforms'] = [] + + for name in tools: + for version in tools[name]: + print("Adding tool {0}-{1}".format(name, version), file=sys.stderr) + pkg1['tools'].append(tools[name][version]) + + for name in platforms: + for version in platforms[name]: + print("Adding platform {0}-{1}".format(name, version), file=sys.stderr) + pkg1['platforms'].append(platforms[name][version]) + + pkg1['platforms'] = sorted(pkg1['platforms'], key=lambda k: LooseVersion(pkgVersionNormalized(k['version'])), reverse=True) + + json.dump({'packages':[pkg1]}, sys.stdout, indent=2) + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/package/package_esp32_index.template.json b/package/package_esp32_index.template.json index d68918f1257..0ce6d86f5be 100644 --- a/package/package_esp32_index.template.json +++ b/package/package_esp32_index.template.json @@ -43,7 +43,7 @@ { "packager": "esp32", "name": "mkspiffs", - "version": "0.2.2" + "version": "0.2.3" } ] } @@ -105,49 +105,49 @@ }, { "name": "mkspiffs", - "version": "0.2.2", + "version": "0.2.3", "systems": [ { "host": "i686-mingw32", - "url": "https://github.com/igrr/mkspiffs/releases/download/0.2.2/mkspiffs-0.2.2-arduino-esp32-win32.zip", - "archiveFileName": "mkspiffs-0.2.2-arduino-esp32-win32.zip", - "checksum": "SHA-256:988baa2827005a20a7c7028f0c2d45d19df2e0a7d42319f4a7a5776a3f0dff2e", - "size": "347207" + "url": "https://github.com/igrr/mkspiffs/releases/download/0.2.3/mkspiffs-0.2.3-arduino-esp32-win32.zip", + "archiveFileName": "mkspiffs-0.2.3-arduino-esp32-win32.zip", + "checksum": "SHA-256:b647f2c2efe6949819c85ea9404271b55c7c9c25bcb98d3b98a1d0ba771adf56", + "size": "249809" }, { "host": "x86_64-apple-darwin", - "url": "https://github.com/igrr/mkspiffs/releases/download/0.2.2/mkspiffs-0.2.2-arduino-esp32-osx.tar.gz", - "archiveFileName": "mkspiffs-0.2.2-arduino-esp32-osx.tar.gz", - "checksum": "SHA-256:7aee138be9a73fe7fd1f75cf3f3695f0afae812d04fcbf74b17da330f66ae4cd", - "size": "130211" + "url": "https://github.com/igrr/mkspiffs/releases/download/0.2.3/mkspiffs-0.2.3-arduino-esp32-osx.tar.gz", + "archiveFileName": "mkspiffs-0.2.3-arduino-esp32-osx.tar.gz", + "checksum": "SHA-256:9f43fc74a858cf564966b5035322c3e5e61c31a647c5a1d71b388ed6efc48423", + "size": "130270" }, { "host": "i386-apple-darwin", - "url": "https://github.com/igrr/mkspiffs/releases/download/0.2.2/mkspiffs-0.2.2-arduino-esp32-osx.tar.gz", - "archiveFileName": "mkspiffs-0.2.2-arduino-esp32-osx.tar.gz", - "checksum": "SHA-256:7aee138be9a73fe7fd1f75cf3f3695f0afae812d04fcbf74b17da330f66ae4cd", - "size": "130211" + "url": "https://github.com/igrr/mkspiffs/releases/download/0.2.3/mkspiffs-0.2.3-arduino-esp32-osx.tar.gz", + "archiveFileName": "mkspiffs-0.2.3-arduino-esp32-osx.tar.gz", + "checksum": "SHA-256:9f43fc74a858cf564966b5035322c3e5e61c31a647c5a1d71b388ed6efc48423", + "size": "130270" }, { "host": "x86_64-pc-linux-gnu", - "url": "https://github.com/igrr/mkspiffs/releases/download/0.2.2/mkspiffs-0.2.2-arduino-esp32-linux64.tar.gz", - "archiveFileName": "mkspiffs-0.2.2-arduino-esp32-linux64.tar.gz", - "checksum": "SHA-256:17f89d9b38d4f68f2f03f7561b951d1d3b6d6f5b74d35b6d3eb8da3440be3400", - "size": "50611" + "url": "https://github.com/igrr/mkspiffs/releases/download/0.2.3/mkspiffs-0.2.3-arduino-esp32-linux64.tar.gz", + "archiveFileName": "mkspiffs-0.2.3-arduino-esp32-linux64.tar.gz", + "checksum": "SHA-256:5e1a4ff41385e842f389f6b5254102a547e566a06b49babeffa93ef37115cb5d", + "size": "50646" }, { "host": "i686-pc-linux-gnu", - "url": "https://github.com/igrr/mkspiffs/releases/download/0.2.2/mkspiffs-0.2.2-arduino-esp32-linux32.tar.gz", - "archiveFileName": "mkspiffs-0.2.2-arduino-esp32-linux32.tar.gz", - "checksum": "SHA-256:181fca76210de04a23eb7af028d9886de5a73e638c63d351a691a24cfb9f03d3", - "size": "48730" + "url": "https://github.com/igrr/mkspiffs/releases/download/0.2.3/mkspiffs-0.2.3-arduino-esp32-linux32.tar.gz", + "archiveFileName": "mkspiffs-0.2.3-arduino-esp32-linux32.tar.gz", + "checksum": "SHA-256:464463a93e8833209cdc29ba65e1a12fec31718dc10075c195a2445b2c3f6cb0", + "size": "48751" }, { "host": "arm-linux-gnueabihf", - "url": "https://github.com/igrr/mkspiffs/releases/download/0.2.2/mkspiffs-0.2.2-arduino-esp32-linux-armhf.tar.gz", - "archiveFileName": "mkspiffs-0.2.2-arduino-esp32-linux-armhf.tar.gz", - "checksum": "SHA-256:2e99cbdf5ee60b27d6ade096d4caf03a90edfd5f4edf4da2a8674d770aa4ca1b", - "size": "40658" + "url": "https://github.com/igrr/mkspiffs/releases/download/0.2.3/mkspiffs-0.2.3-arduino-esp32-linux-armhf.tar.gz", + "archiveFileName": "mkspiffs-0.2.3-arduino-esp32-linux-armhf.tar.gz", + "checksum": "SHA-256:ade3dc00117912ac08a1bdbfbfe76b12d21a34bc5fa1de0cfc45fe7a8d0a0185", + "size": "40665" } ] } From 8051ab5f71bf58035fdb64a964e475656a49c3a9 Mon Sep 17 00:00:00 2001 From: Martin Vychodil Date: Thu, 21 Jun 2018 11:37:44 +0200 Subject: [PATCH 3/6] * secure env for espressif/arduino-esp32 --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 95d93a07414..719a853835e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,8 +9,7 @@ os: env: global: - #2DO: replace by Arduino-ESP32 specific output of traviscli: travis encrypt ESP32_GITHUB_TOKEN="" - #- secure: "rZ5dID535OdWTzXReLawQQXzPeBeDoTy293FV3VdSxG4bQGfk52MH6pd9XzaVGNyvfeXlWpeOwgJauOEiCBQSe4j71SsWZUFWOSz2IJguF6HiHYi3ftG1oHPHi1J6kEZ6XdrHi6rPTmv2x8Bj5Wk6SV+eOfT0OlM89BovVnhOvXoCH65g4rqvc01XLngE/Lj5mxZIpWMDyBfdhQQcqmp6fZLxAziOauD4Xjd1U1IxGiSlYjWin4948P2UezuT9PbFRTfCKjedA5nXAtz6gwz0+SGM8gjTZRXMQ6aYtbQNSZmdX3HsHZiyVDFmdaYiZVYcr7iGvJgmgjvb9AUmpEVTybr7ph0SpA0il4gutojAiJik5sG4/fvNlsYnNiAxgCf7SmyE6z/pAnODrNN7CVc/jdHihmc5hNQgkbcLrPI902UXyFUuP4GIBK5Ph1qA88HKtg1gp0xe1yPh64KMi0SXtX8NIimOeOCNFV1Yp9/9sSDy2sbZRBHmE36LZH6duMGbpzPU9bp7xYy5SlcTnXCqd6II8JkICWJdkfQNZ39JuDIqJZFPKO+9RfglFq5TnPq64SupySe6wrhPjpHHrs7tikNsl4j9/OU0i1tqZc/H6hiZ0EF7nL0hO3Vm9lKbMlcwOy5DSv2OSe7uSnDX4bbbvvBWKJiX1okuwb9olQhmUc=" + - secure: "l/4Dt+KQ/mACtGAHDUsPr66fUte840PZoQ4xpPikqWZI0uARu4l+Ym7+sHinnT6fBqrj8AJeBYGz4nFa8NK4LutZn9mSD40w+sxl0wSV4oHV8rzKe3Cd8+sMG3+o33yWoikMNjSvqa73Q0rm+SgrlInNdZbuAyixL+a2alaWSnGPm4F2xwUGj+S33TOy5P/Xp77CYtCV5S8vzyk/eEdNhoF0GYePJVdfuzCOUjXMyT5OWxORkzzQ7Hnn/Ka/RDfV8Si4HgujLQBrK5q6iPnNBFqBSqilYBepSMn4opnOBpIm0SCgePz7XQEFC83buA7GUcnCnfg38bf+dCwHaODf1d1PmqVRYt2QmfinexXtM4afAtL0iBUDtvrfnXHzwW9w82VeZhpbJSVh9DUQvB0IlsZeCz9J9PUBAi3N+SMX+9l+BomYwRUlPuKY+Ef2JKk9q6mxtUkky5R0daAlVxEhpVdQks1rT+T+NMoDMemxQ3SKEiqAHh6EgHecruszffmZ71uLX9MpERpew0qN+UFiafws+jkTjx+3yF9yut0Hf9sMbeAYzzkGzRqJTUEBJ6B29Cql8M0yRXCNN/8wuuTHhG8esstozga4ZQoIVrq7mEAgup376PTcNfr1+imbbWVQ7lJdYIuDe6OS5V3OX6np11vgK/DbhfyzvQv9Z1zAGnM=" - REMOTE_URL=https://github.com/$TRAVIS_REPO_SLUG/releases/download/$TRAVIS_TAG script: From 1ca67777557ac2f016a404c0f7e0ca07200cb447 Mon Sep 17 00:00:00 2001 From: Martin Vychodil Date: Thu, 21 Jun 2018 11:56:43 +0200 Subject: [PATCH 4/6] * build tests enabled * gitter webhook enabled --- .travis.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 719a853835e..2b94d570049 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,9 +13,8 @@ env: - REMOTE_URL=https://github.com/$TRAVIS_REPO_SLUG/releases/download/$TRAVIS_TAG script: - #2DO: uncomment after successful TCI/GH integration # build the package and run platformIO tests - # - bash $TRAVIS_BUILD_DIR/tools/build-tests.sh + - bash $TRAVIS_BUILD_DIR/tools/build-tests.sh # zip the package if tagged build, otherwise finish here - bash $TRAVIS_BUILD_DIR/tools/build-release.sh -a$ESP32_GITHUB_TOKEN @@ -35,8 +34,7 @@ notifications: on_failure: change webhooks: urls: - #2DO: replace with Arduino-ESP32 specific link - #- https://webhooks.gitter.im/e/6dfa09a8a8ffd43f9297 + - https://webhooks.gitter.im/e/6dfa09a8a8ffd43f9297 on_success: change # options: [always|never|change] default: always on_failure: always # options: [always|never|change] default: always on_start: never # options: [always|never|change] default: always \ No newline at end of file From f451d600aa512c8b84df93be3185589e02bef21a Mon Sep 17 00:00:00 2001 From: Martin Vychodil Date: Thu, 21 Jun 2018 12:00:51 +0200 Subject: [PATCH 5/6] * gitter room link fixed * better comment --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2b94d570049..fd347ec3630 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ env: - REMOTE_URL=https://github.com/$TRAVIS_REPO_SLUG/releases/download/$TRAVIS_TAG script: - # build the package and run platformIO tests + # run sketch tests - bash $TRAVIS_BUILD_DIR/tools/build-tests.sh # zip the package if tagged build, otherwise finish here @@ -34,7 +34,7 @@ notifications: on_failure: change webhooks: urls: - - https://webhooks.gitter.im/e/6dfa09a8a8ffd43f9297 + - https://webhooks.gitter.im/e/cb057279c430d91a47a8 on_success: change # options: [always|never|change] default: always on_failure: always # options: [always|never|change] default: always on_start: never # options: [always|never|change] default: always \ No newline at end of file From 2a37ab621ec98aa9afb992921fe0e67c28422091 Mon Sep 17 00:00:00 2001 From: Martin Vychodil Date: Thu, 21 Jun 2018 12:12:58 +0200 Subject: [PATCH 6/6] * filepaths fixed --- build-release.sh => tools/build-release.sh | 0 build-tests.sh => tools/build-tests.sh | 0 deploy.sh => tools/deploy.sh | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename build-release.sh => tools/build-release.sh (100%) rename build-tests.sh => tools/build-tests.sh (100%) rename deploy.sh => tools/deploy.sh (100%) diff --git a/build-release.sh b/tools/build-release.sh similarity index 100% rename from build-release.sh rename to tools/build-release.sh diff --git a/build-tests.sh b/tools/build-tests.sh similarity index 100% rename from build-tests.sh rename to tools/build-tests.sh diff --git a/deploy.sh b/tools/deploy.sh similarity index 100% rename from deploy.sh rename to tools/deploy.sh