Add Win32 build to CI system (esp8266#6493)

Build a single INO under a Windows VM on Travis to ensure
that the Win32 toolchain works properly.

In build.py, instead of making a string with spaces and then
splitting on " ", just make the list itself with individual parameters.
This will allow spaces in paths to be supported properly.

Author: earlephilhower

.travis.yml

@@ -107,12 +107,18 @@ jobs:
       script: $TRAVIS_BUILD_DIR/tests/buildm.sh
       env: CC=gcc-7 CXX=g++-7
 
-    - name: "MacOS can build sketches"
+    - name: "Mac OSX can build sketches"
       os: osx
       stage: build
       script: $TRAVIS_BUILD_DIR/tests/build.sh
       env: MACOSX=1 BUILD_PARITY=custom mod=500 rem=1
 
+    - name: "Windows can build sketches"
+      os: windows
+      stage: build
+      script: $TRAVIS_BUILD_DIR/tests/build.sh
+      env: WINDOWS=1 BUILD_PARITY=custom mod=500 rem=1
+
     - name: "Boards"
       stage: build
       script: $TRAVIS_BUILD_DIR/tests/ci/build_boards.sh

tests/common.sh

@@ -44,8 +44,7 @@ function print_size_info()
             segments[$seg]=$size
         fi
 
-
-    done < <(xtensa-lx106-elf-size --format=sysv $elf_file)
+    done < <(xtensa-lx106-elf-size --format=sysv $elf_file | sed 's/\r//g' )
 
     total_ram=$((${segments[data]} + ${segments[rodata]} + ${segments[bss]}))
     total_flash=$((${segments[data]} + ${segments[rodata]} + ${segments[text]} + ${segments[irom0text]}))
@@ -66,6 +65,10 @@ function build_sketches()
     local lwip=$6
     mkdir -p $build_dir
     local build_cmd="python3 tools/build.py -b generic -v -w all -s 4M1M -v -k --build_cache $cache_dir -p $PWD/$build_dir -n $lwip $build_arg "
+    if [ "$WINDOWS" = "1" ]; then
+        # Paths to the arduino builder need to be / referenced, not our native ones
+        build_cmd=$(echo $build_cmd --ide_path $arduino | sed 's/ \/c\// \//g' ) # replace '/c/' with '/'
+    fi
     local sketches=$(find $srcpath -name *.ino | sort)
     print_size_info >size.log
     export ARDUINO_IDE_PATH=$arduino
@@ -107,6 +110,14 @@ function build_sketches()
         fi
         echo -e "\n ------------ Building $sketch ------------ \n";
         # $arduino --verify $sketch;
+	if [ "$WINDOWS" == "1" ]; then
+            sketch=$(echo $sketch | sed 's/^\/c//')
+            # MINGW will try to be helpful and silently convert args that look like paths to point to a spot inside the MinGW dir.  This breaks everything.
+            # http://www.mingw.org/wiki/Posix_path_conversion
+            # https://stackoverflow.com/questions/7250130/how-to-stop-mingw-and-msys-from-mangling-path-names-given-at-the-command-line#34386471
+            export MSYS2_ARG_CONV_EXC="*"
+            export MSYS_NO_PATHCONV=1
+        fi
         echo "$build_cmd $sketch"
         time ($build_cmd $sketch >build.log)
         local result=$?
@@ -135,7 +146,7 @@ function install_libraries()
     pushd $HOME/Arduino/libraries
 
     # install ArduinoJson library
-    { test -r ArduinoJson-v6.11.0.zip || wget https://github.com/bblanchon/ArduinoJson/releases/download/v6.11.0/ArduinoJson-v6.11.0.zip; } && unzip -q ArduinoJson-v6.11.0.zip
+    { test -r ArduinoJson-v6.11.0.zip || wget -nv https://github.com/bblanchon/ArduinoJson/releases/download/v6.11.0/ArduinoJson-v6.11.0.zip; } && unzip -q ArduinoJson-v6.11.0.zip
 
     popd
 }
@@ -145,17 +156,27 @@ function install_ide()
     local ide_path=$1
     local core_path=$2
     local debug=$3
-    if [ "$MACOSX" = "1" ]; then
+    if [ "$WINDOWS" = "1" ]; then
+        # Acquire needed packages from Windows package manager
+        choco install --no-progress python3
+        export PATH="/c/Python37:$PATH"  # Ensure it's live from now on...
+        cp /c/Python37/python.exe /c/Python37/python3.exe
+        choco install --no-progress unzip
+        choco install --no-progress sed
+        #choco install --no-progress golang
+        test -r arduino-nightly-windows.zip || wget -nv -O arduino-nightly-windows.zip https://www.arduino.cc/download.php?f=/arduino-nightly-windows.zip
+        unzip -q arduino-nightly-windows.zip
+    elif [ "$MACOSX" = "1" ]; then
         # MACOS only has next-to-obsolete Python2 installed.  Install Python 3 from python.org
         wget https://www.python.org/ftp/python/3.7.4/python-3.7.4-macosx10.9.pkg
         sudo installer -pkg python-3.7.4-macosx10.9.pkg -target /
         # Install the Python3 certificates, because SSL connections fail w/o them and of course they aren't installed by default.
-	( cd "/Applications/Python 3.7/" && sudo "./Install Certificates.command" )
+        ( cd "/Applications/Python 3.7/" && sudo "./Install Certificates.command" )
         # Hack to place arduino-builder in the same spot as sane OSes
         test -r arduino.zip || wget -O arduino.zip https://downloads.arduino.cc/arduino-nightly-macosx.zip
         unzip -q arduino.zip
-	mv Arduino.app arduino-nightly
-	mv arduino-nightly/Contents/Java/* arduino-nightly/.
+        mv Arduino.app arduino-nightly
+        mv arduino-nightly/Contents/Java/* arduino-nightly/.
     else
         test -r arduino.tar.xz || wget -O arduino.tar.xz https://www.arduino.cc/download.php?f=/arduino-nightly-linux64.tar.xz
         tar xf arduino.tar.xz
@@ -164,7 +185,11 @@ function install_ide()
     cd $ide_path/hardware
     mkdir esp8266com
     cd esp8266com
-    ln -s $core_path esp8266
+    if [ "$WINDOWS" = "1" ]; then
+        cp -a $core_path esp8266
+    else
+        ln -s $core_path esp8266
+    fi
     local debug_flags=""
     if [ "$debug" = "debug" ]; then
         debug_flags="-DDEBUG_ESP_PORT=Serial -DDEBUG_ESP_SSL -DDEBUG_ESP_TLS_MEM -DDEBUG_ESP_HTTP_CLIENT -DDEBUG_ESP_HTTP_SERVER -DDEBUG_ESP_CORE -DDEBUG_ESP_WIFI -DDEBUG_ESP_HTTP_UPDATE -DDEBUG_ESP_UPDATER -DDEBUG_ESP_OTA -DDEBUG_ESP_OOM"
@@ -176,8 +201,14 @@ function install_ide()
     cat esp8266/platform.local.txt
     echo -e "\n----\n"
     cd esp8266/tools
-    python3 get.py
-    export PATH="$ide_path:$core_path/tools/xtensa-lx106-elf/bin:$PATH"
+    python3 get.py -q
+    if [ "$WINDOWS" = "1" ]; then
+        # Because the symlinks don't work well under Win32, we need to add the path to this copy, not the original...
+        relbin=$(realpath $PWD/xtensa-lx106-elf/bin)
+        export PATH="$ide_path:$relbin:$PATH"
+    else
+        export PATH="$ide_path:$core_path/tools/xtensa-lx106-elf/bin:$PATH"
+    fi
 }
 
 function install_arduino()

tools/build.py

@@ -24,28 +24,42 @@
 import sys
 import os
 import argparse
+import platform
 import subprocess
 import tempfile
 import shutil
 
+
+# Arduino-builder needs forward-slash paths for passed in params or it cannot
+# launch the needed toolset.
+def windowsize_paths(l):
+    """Convert forward-slash paths to backslash paths referenced from C:"""
+    out = []
+    for i in l:
+        if i.startswith('/'):
+            i = 'C:' + i
+        out += [i.replace('/', '\\')]
+    return out
+
 def compile(tmp_dir, sketch, cache, tools_dir, hardware_dir, ide_path, f, args):
-    cmd = ide_path + '/arduino-builder '
-    cmd += '-compile -logger=human '
-    cmd += '-build-path "' + tmp_dir + '" '
-    cmd += '-tools "' +  ide_path + '/tools-builder" '
+    cmd = []
+    cmd += [ide_path + '/arduino-builder']
+    cmd += ['-compile', '-logger=human']
+    cmd += ['-build-path', tmp_dir]
+    cmd += ['-tools', ide_path + '/tools-builder']
     if cache != "":
-        cmd += '-build-cache "' + cache + '" '
+        cmd += ['-build-cache', cache ]
     if args.library_path:
         for lib_dir in args.library_path:
-            cmd += '-libraries "' + lib_dir + '" '
-    cmd += '-hardware "' + ide_path + '/hardware" '
+            cmd += ['-libraries', lib_dir]
+    cmd += ['-hardware', ide_path + '/hardware']
     if args.hardware_dir:
         for hw_dir in args.hardware_dir:
-            cmd += '-hardware "' + hw_dir + '" '
+            cmd += ['-hardware', hw_dir]
     else:
-        cmd += '-hardware "' + hardware_dir + '" '
+        cmd += ['-hardware', hardware_dir]
     # Debug=Serial,DebugLevel=Core____
-    cmd += '-fqbn=esp8266com:esp8266:{board_name}:' \
+    fqbn = '-fqbn=esp8266com:esp8266:{board_name}:' \
             'xtal={cpu_freq},' \
             'FlashFreq={flash_freq},' \
             'FlashMode={flash_mode},' \
@@ -54,20 +68,22 @@ def compile(tmp_dir, sketch, cache, tools_dir, hardware_dir, ide_path, f, args):
             'ip={lwIP},' \
             'ResetMethod=nodemcu'.format(**vars(args))
     if args.debug_port and args.debug_level:
-        cmd += 'dbg={debug_port},lvl={debug_level}'.format(**vars(args))
-    cmd += ' '
-    cmd += '-built-in-libraries "' + ide_path + '/libraries" '
-    cmd += '-ide-version=10607 '
-    cmd += '-warnings={warnings} '.format(**vars(args))
+        fqbn += 'dbg={debug_port},lvl={debug_level}'.format(**vars(args))
+    cmd += [fqbn]
+    cmd += ['-built-in-libraries', ide_path + '/libraries']
+    cmd += ['-ide-version=10607']
+    cmd += ['-warnings={warnings}'.format(**vars(args))]
     if args.verbose:
-        cmd += '-verbose '
-    cmd += sketch
+        cmd += ['-verbose']
+    cmd += [sketch]
+
+    if platform.system() == "Windows":
+        cmd = windowsize_paths(cmd)
 
     if args.verbose:
-        print('Building: ' + cmd, file=f)
+        print('Building: ' + " ".join(cmd), file=f)
 
-    cmds = cmd.split(' ')
-    p = subprocess.Popen(cmds, stdout=f, stderr=subprocess.STDOUT)
+    p = subprocess.Popen(cmd, stdout=f, stderr=subprocess.STDOUT)
     p.wait()
     return p.returncode
 
@@ -127,6 +143,7 @@ def main():
     hardware_dir = os.path.dirname(os.path.realpath(__file__)) + '/../cores'
 
     output_name = tmp_dir + '/' + os.path.basename(sketch_path) + '.bin'
+
     if args.verbose:
         print("Sketch: ", sketch_path)
         print("Build dir: ", tmp_dir)

tools/get.py

@@ -15,6 +15,9 @@
 import tarfile
 import zipfile
 import re
+
+verbose = True
+
 if sys.version_info[0] == 3:
     from urllib.request import urlretrieve
 else:
@@ -38,10 +41,12 @@ def mkdir_p(path):
             raise
 
 def report_progress(count, blockSize, totalSize):
-    percent = int(count*blockSize*100/totalSize)
-    percent = min(100, percent)
-    sys.stdout.write("\r%d%%" % percent)
-    sys.stdout.flush()
+    global verbose
+    if verbose:
+        percent = int(count*blockSize*100/totalSize)
+        percent = min(100, percent)
+        sys.stdout.write("\r%d%%" % percent)
+        sys.stdout.flush()
 
 def unpack(filename, destination):
     dirname = ''
@@ -111,6 +116,11 @@ def identify_platform():
     return arduino_platform_names[sys_name][bits]
 
 def main():
+    global verbose
+    # Support optional "-q" quiet mode simply
+    if len(sys.argv) == 2:
+        if sys.argv[1] == "-q":
+            verbose = False
     print('Platform: {0}'.format(identify_platform()))
     tools_to_download = load_tools_list('../package/package_esp8266com_index.template.json', identify_platform())
     mkdir_p(dist_dir)