From 4b0cef5adce8ad9f61525e41f67ca0cfb4e8476b Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Mon, 6 Feb 2017 18:18:16 +0800 Subject: [PATCH 1/5] package: add support for nightly packages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Arduino IDE has support for “nightly” versions of boards manager packages (https://github.com/arduino/Arduino/pull/3449). These versions are distinguished by the lack of “size” and “checksum” fields in json file. IDE always uninstalls previously installed nightly version before installing the new one. This change adds support for generation of nightly package builds. --- package/build_boards_manager_package.sh | 39 +++++++++++++++---- .../package_esp8266com_index.template.json | 2 - 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/package/build_boards_manager_package.sh b/package/build_boards_manager_package.sh index 8859603d80..ff1a5b4d17 100755 --- a/package/build_boards_manager_package.sh +++ b/package/build_boards_manager_package.sh @@ -1,8 +1,19 @@ #!/bin/bash # +next="2.4.0" # Figure out how will the package be called -ver=`git describe --tags --always` +ver=`git describe --exact-match` +if [ $? -ne 0 ]; then + # not tagged version; generate nightly package + date_str=`date +"%Y%m%d"` + is_nightly=1 + plain_ver="${next}-nightly" + ver="${plain_ver}+${date_str}" +else + plain_ver=$ver +fi + package_name=esp8266-$ver echo "Version: $ver" echo "Package name: $package_name" @@ -71,7 +82,7 @@ $SED 's/recipe.hooks.core.prebuild.1.pattern.*//g' \ > $outdir/platform.txt # Put core version and short hash of git version into core_version.h -ver_define=`echo $ver | tr "[:lower:].-" "[:upper:]_"` +ver_define=`echo $plain_ver | tr "[:lower:].-+" "[:upper:]_"` echo Ver define: $ver_define echo \#define ARDUINO_ESP8266_GIT_VER 0x`git rev-parse --short=8 HEAD 2>/dev/null` >$outdir/cores/esp8266/core_version.h echo \#define ARDUINO_ESP8266_RELEASE_$ver_define >>$outdir/cores/esp8266/core_version.h @@ -90,14 +101,26 @@ echo Size: $size echo SHA-256: $sha echo "Making package_esp8266com_index.json" -cat $srcdir/package/package_esp8266com_index.template.json | \ -jq ".packages[0].platforms[0].version = \"$ver\" | \ + +jq_arg=".packages[0].platforms[0].version = \"$ver\" | \ .packages[0].platforms[0].url = \"$PKG_URL\" |\ .packages[0].platforms[0].archiveFileName = \"$package_name.zip\" |\ - .packages[0].platforms[0].checksum = \"SHA-256:$sha\" |\ - .packages[0].platforms[0].size = \"$size\" |\ - .packages[0].platforms[0].help.online = \"$DOC_URL\"" \ - > package_esp8266com_index.json + .packages[0].platforms[0].help.online = \"$DOC_URL\"" + +if [ -z "$is_nightly" ]; then + jq_arg="$jq_arg |\ + .packages[0].platforms[0].size = \"$size\" |\ + .packages[0].platforms[0].checksum = \"SHA-256:$sha\" |" +fi + +cat $srcdir/package/package_esp8266com_index.template.json | \ + jq "$jq_arg" > package_esp8266com_index.json + +old_json=package_esp8266com_index_stable.json +wget -O $old_json http://arduino.esp8266.com/$branch/package_esp8266com_index.json +new_json=package_esp8266com_index.json + +python ../../merge_packages.py $new_json $old_json >tmp && mv tmp $new_json && rm $old_json popd popd diff --git a/package/package_esp8266com_index.template.json b/package/package_esp8266com_index.template.json index 5493d51278..703d8ab3ca 100644 --- a/package/package_esp8266com_index.template.json +++ b/package/package_esp8266com_index.template.json @@ -16,8 +16,6 @@ "category": "ESP8266", "url": "", "archiveFileName": "", - "checksum": "", - "size": "", "help": { "online": "" }, From dacd718d17d9eacce2e35c52c65bbac65b2d6a0f Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Sat, 18 Feb 2017 18:26:55 +0800 Subject: [PATCH 2/5] package: add script to build and deploy nightly packages --- .travis.yml | 37 ++------------ package/build_boards_manager_package.sh | 30 ++++++++---- package/deploy_nightly_package.sh | 46 ++++++++++++++++++ platform.txt | 2 +- tests/common.sh | 64 +++++++++++++++++++++++++ 5 files changed, 134 insertions(+), 45 deletions(-) create mode 100755 package/deploy_nightly_package.sh diff --git a/.travis.yml b/.travis.yml index 05fb744fe3..a8a7bbc936 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,8 @@ sudo: false language: bash os: - linux +env: + - BUILD_TYPE=build addons: apt: @@ -11,40 +13,7 @@ addons: - g++-4.8 script: - - set -e - - export CXX="g++-4.8" CC="gcc-4.8" GCOV="gcov-4.8" - - echo -e "travis_fold:start:host_tests" - - pushd $TRAVIS_BUILD_DIR/tests/host - - make - - make clean-objects - - echo -e "travis_fold:end:host_tests" - - echo -e "travis_fold:start:sketch_test_env_prepare" - - popd - - wget -O arduino.tar.xz https://www.arduino.cc/download.php?f=/arduino-nightly-linux64.tar.xz - - tar xf arduino.tar.xz - - mv arduino-nightly $HOME/arduino_ide - - cd $HOME/arduino_ide/hardware - - mkdir esp8266com - - cd esp8266com - - ln -s $TRAVIS_BUILD_DIR esp8266 - - cd esp8266/tools - - python get.py - - export PATH="$HOME/arduino_ide:$TRAVIS_BUILD_DIR/tools/xtensa-lx106-elf/bin:$PATH" - - which arduino - - cd $TRAVIS_BUILD_DIR - - source tests/common.sh - - install_libraries - - echo -e "travis_fold:end:sketch_test_env_prepare" - - echo -e "travis_fold:start:sketch_test" - - build_sketches $HOME/arduino_ide $TRAVIS_BUILD_DIR/libraries "-l $HOME/Arduino/libraries" - - echo -e "travis_fold:end:sketch_test" - - echo -e "travis_fold:start:size_report" - - cat size.log - - echo -e "travis_fold:end:size_report" - -after_success: - - pushd $TRAVIS_BUILD_DIR/tests/host - - bash <(curl -s https://codecov.io/bash) -X gcov + - $TRAVIS_BUILD_DIR/tests/common.sh notifications: email: diff --git a/package/build_boards_manager_package.sh b/package/build_boards_manager_package.sh index ff1a5b4d17..ea3cdb2b8d 100755 --- a/package/build_boards_manager_package.sh +++ b/package/build_boards_manager_package.sh @@ -1,6 +1,8 @@ #!/bin/bash # -next="2.4.0" + +# Extract next version from platform.txt +next=`sed -n -E 's/version=([0-9.]+)/\1/p' ../platform.txt` # Figure out how will the package be called ver=`git describe --exact-match` @@ -27,14 +29,14 @@ fi echo "Remote: $REMOTE_URL" if [ -z "$PKG_URL" ]; then - PKG_URL="$REMOTE_URL/versions/$ver/$package_name.zip" + if [ -z "$PKG_URL_PREFIX" ]; then + PKG_URL_PREFIX="$REMOTE_URL/versions/$ver" + fi + PKG_URL="$PKG_URL_PREFIX/$package_name.zip" fi echo "Package: $PKG_URL" - -if [ -z "$DOC_URL" ]; then - DOC_URL="$REMOTE_URL/versions/$ver/doc/reference.html" -fi echo "Docs: $DOC_URL" + pushd .. # Create directory for the package outdir=package/versions/$ver/$package_name @@ -82,7 +84,7 @@ $SED 's/recipe.hooks.core.prebuild.1.pattern.*//g' \ > $outdir/platform.txt # Put core version and short hash of git version into core_version.h -ver_define=`echo $plain_ver | tr "[:lower:].-+" "[:upper:]_"` +ver_define=`echo $plain_ver | tr "[:lower:].\055" "[:upper:]_"` echo Ver define: $ver_define echo \#define ARDUINO_ESP8266_GIT_VER 0x`git rev-parse --short=8 HEAD 2>/dev/null` >$outdir/cores/esp8266/core_version.h echo \#define ARDUINO_ESP8266_RELEASE_$ver_define >>$outdir/cores/esp8266/core_version.h @@ -104,8 +106,7 @@ echo "Making package_esp8266com_index.json" jq_arg=".packages[0].platforms[0].version = \"$ver\" | \ .packages[0].platforms[0].url = \"$PKG_URL\" |\ - .packages[0].platforms[0].archiveFileName = \"$package_name.zip\" |\ - .packages[0].platforms[0].help.online = \"$DOC_URL\"" + .packages[0].platforms[0].archiveFileName = \"$package_name.zip\"" if [ -z "$is_nightly" ]; then jq_arg="$jq_arg |\ @@ -113,13 +114,22 @@ if [ -z "$is_nightly" ]; then .packages[0].platforms[0].checksum = \"SHA-256:$sha\" |" fi +if [ ! -z "$DOC_URL" ]; then + jq_arg="$jq_arg |\ + .packages[0].platforms[0].help.online = \"$DOC_URL\"" +fi + cat $srcdir/package/package_esp8266com_index.template.json | \ jq "$jq_arg" > package_esp8266com_index.json old_json=package_esp8266com_index_stable.json -wget -O $old_json http://arduino.esp8266.com/$branch/package_esp8266com_index.json +for i in $(seq 1 5); do + echo "Downloading old package, try $i" + curl -L -o $old_json http://arduino.esp8266.com/stable/package_esp8266com_index.json && break +done new_json=package_esp8266com_index.json +set +e python ../../merge_packages.py $new_json $old_json >tmp && mv tmp $new_json && rm $old_json popd diff --git a/package/deploy_nightly_package.sh b/package/deploy_nightly_package.sh new file mode 100755 index 0000000000..70edb10fc3 --- /dev/null +++ b/package/deploy_nightly_package.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +set -e +export PKG_URL_PREFIX=https://dl.bintray.com/igrr/arduino-esp8266/: +commit=`git rev-parse --short HEAD` + +./build_boards_manager_package.sh + +ver=`ls -1 versions` +bintray_slug=igrr/arduino-esp8266/arduino-esp8266-core + +# Upload to bintray +# URL to the file will look like this: https://dl.bintray.com/igrr/arduino-esp8266/:esp8266-2.4.0-nightly+20170218.zip +curl --progress-bar \ + -T versions/$ver/esp8266-$ver.zip \ + -uigrr:$BINTRAY_API_KEY \ + -o curl.out \ + https://api.bintray.com/content/$bintray_slug/$ver/esp8266-$ver.zip + + +# Publish the uploaded file +curl -uigrr:$BINTRAY_API_KEY \ + -X POST \ + https://api.bintray.com/content/$bintray_slug/$ver/publish + +# Load deploy key +echo -n $ESP8266_ARDUINO_DEPLOY_KEY_B64 > ~/.ssh/esp8266_arduino_deploy_b64 +base64 --decode --ignore-garbage ~/.ssh/esp8266_arduino_deploy_b64 > ~/.ssh/esp8266_arduino_deploy +chmod 600 ~/.ssh/esp8266_arduino_deploy +echo -e "Host $DEPLOY_HOST_NAME\n\tUser $DEPLOY_USER_NAME\n\tStrictHostKeyChecking no\n\tIdentityFile ~/.ssh/esp8266_arduino_deploy" >> ~/.ssh/config + +# Generate the badge (used in Readme.md) +release_date=$(date "+%b_%d,_%Y") +curl -o versions/$ver/badge.svg https://img.shields.io/badge/updated-$release_date-blue.svg + +# Check old version +oldver=$(ssh $DEPLOY_HOST_NAME "cat $DEPLOY_PATH_NIGHTLY/version") + +if [ "$oldver" = "$ver" ]; then + echo "Nightly version hasn't changed, not updating" +else + # Upload new json, version file, and the badge + echo $ver > versions/$ver/version + scp versions/$ver/{package_esp8266com_index.json,version,badge.svg} $DEPLOY_HOST_NAME:$DEPLOY_PATH_NIGHTLY/ + curl --data-urlencode "message=Updating **nightly** package from $oldver to $ver" $GITTER_WEBHOOK +fi diff --git a/platform.txt b/platform.txt index f8f79de6c9..5bbb59c79d 100644 --- a/platform.txt +++ b/platform.txt @@ -6,7 +6,7 @@ # https://github.com/arduino/Arduino/wiki/Arduino-IDE-1.5---3rd-party-Hardware-specification name=ESP8266 Modules -version=2.3.0 +version=2.4.0 runtime.tools.xtensa-lx106-elf-gcc.path={runtime.platform.path}/tools/xtensa-lx106-elf runtime.tools.esptool.path={runtime.platform.path}/tools/esptool diff --git a/tests/common.sh b/tests/common.sh index 61757dbe77..39ba5b813d 100755 --- a/tests/common.sh +++ b/tests/common.sh @@ -84,3 +84,67 @@ function install_libraries() popd } + +function install_ide() +{ + local ide_path=$1 + local core_path=$2 + wget -O arduino.tar.xz https://www.arduino.cc/download.php?f=/arduino-nightly-linux64.tar.xz + tar xf arduino.tar.xz + mv arduino-nightly $ide_path + cd $ide_path/hardware + mkdir esp8266com + cd esp8266com + ln -s $core_path esp8266 + cd esp8266/tools + python get.py + export PATH="$ide_path:$core_path/tools/xtensa-lx106-elf/bin:$PATH" +} + +function run_host_tests() +{ + pushd host + make + make clean-objects + popd +} + + +function run_travis_ci_build() +{ + export CXX="g++-4.8" CC="gcc-4.8" GCOV="gcov-4.8" + echo -e "travis_fold:start:host_tests" + cd $TRAVIS_BUILD_DIR/tests + run_host_tests + echo -e "travis_fold:end:host_tests" + echo -e "travis_fold:start:sketch_test_env_prepare" + cd $TRAVIS_BUILD_DIR + install_ide $HOME/arduino_ide $TRAVIS_BUILD_DIR + which arduino + cd $TRAVIS_BUILD_DIR + install_libraries + echo -e "travis_fold:end:sketch_test_env_prepare" + echo -e "travis_fold:start:sketch_test" + build_sketches $HOME/arduino_ide $TRAVIS_BUILD_DIR/libraries "-l $HOME/Arduino/libraries" + echo -e "travis_fold:end:sketch_test" + echo -e "travis_fold:start:size_report" + cat size.log + echo -e "travis_fold:end:size_report" + pushd $TRAVIS_BUILD_DIR/tests/host + bash <(curl -s https://codecov.io/bash) -X gcov +} + +function deploy_nightly_package() +{ + cd $TRAVIS_BUILD_DIR/package + ./deploy_nightly_package.sh +} + +set -e + +if [ "$BUILD_TYPE" = "deploy_nightly_package" ]; then + deploy_nightly_package +elif [ "$BUILD_TYPE" = "build" ]; then + run_travis_ci_build +fi + From 5f9966218faee32b5e25280a6f987058d9b70f6b Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Sun, 19 Feb 2017 13:47:42 +0800 Subject: [PATCH 3/5] readme: remove staging package, add nightly package --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 014add389b..5958d12336 100644 --- a/README.md +++ b/README.md @@ -38,10 +38,10 @@ Boards manager link: `http://arduino.esp8266.com/stable/package_esp8266com_index Documentation: [http://esp8266.github.io/Arduino/versions/2.3.0/](http://esp8266.github.io/Arduino/versions/2.3.0/) -##### Staging version ![](http://arduino.esp8266.com/staging/badge.svg) -Boards manager link: `http://arduino.esp8266.com/staging/package_esp8266com_index.json` +##### Nightly version ![](http://arduino.esp8266.com/nightly/badge.svg) +Boards manager link: `http://arduino.esp8266.com/nightly/package_esp8266com_index.json` -Documentation: [http://esp8266.github.io/Arduino/versions/2.3.0-rc2/](http://esp8266.github.io/Arduino/versions/2.3.0-rc2/) +No online documentation available for nightly version yet — work in progress. ### Using git version [![Linux build status](https://travis-ci.org/esp8266/Arduino.svg)](https://travis-ci.org/esp8266/Arduino) [![codecov.io](https://codecov.io/github/esp8266/Arduino/coverage.svg?branch=master)](https://codecov.io/github/esp8266/Arduino?branch=master) From 160c4239b13da7bf647d5cff84805f083484a1b5 Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Mon, 20 Feb 2017 22:43:09 +0800 Subject: [PATCH 4/5] fix nightly package URL prefix --- package/deploy_nightly_package.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/deploy_nightly_package.sh b/package/deploy_nightly_package.sh index 70edb10fc3..50e9cc5fcd 100755 --- a/package/deploy_nightly_package.sh +++ b/package/deploy_nightly_package.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e -export PKG_URL_PREFIX=https://dl.bintray.com/igrr/arduino-esp8266/: +export PKG_URL_PREFIX=https://dl.bintray.com/igrr/arduino-esp8266 commit=`git rev-parse --short HEAD` ./build_boards_manager_package.sh From aba7f27b4470150233055644c67d90e7ad21c22c Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Tue, 21 Feb 2017 00:13:33 +0800 Subject: [PATCH 5/5] exclude documentation from boards manager package --- package/build_boards_manager_package.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/package/build_boards_manager_package.sh b/package/build_boards_manager_package.sh index ea3cdb2b8d..42a8f6c928 100755 --- a/package/build_boards_manager_package.sh +++ b/package/build_boards_manager_package.sh @@ -50,6 +50,7 @@ cat << EOF > exclude.txt .gitignore .travis.yml package +doc EOF # Also include all files which are ignored by git git ls-files --other --directory >> exclude.txt