Allow multiple instances of the same 'required_tool' in cores

arduino/cores/cores.go

@@ -19,11 +19,11 @@ package cores
 
 import (
 	"encoding/json"
+	"sort"
 	"strings"
 
-	paths "github.com/arduino/go-paths-helper"
-
 	"github.com/arduino/arduino-cli/arduino/resources"
+	paths "github.com/arduino/go-paths-helper"
 	properties "github.com/arduino/go-properties-orderedmap"
 	semver "go.bug.st/relaxed-semver"
 )
@@ -79,6 +79,20 @@ func (bm *BoardManifest) HasUsbID(vid, pid string) bool {
 // ToolDependencies is a set of tool dependency
 type ToolDependencies []*ToolDependency
 
+// Sort sorts the ToolDependencies by name and (if multiple instance of the same
+// tool is required) by version.
+func (deps ToolDependencies) Sort() {
+	sort.Slice(deps, func(i, j int) bool {
+		if deps[i].ToolPackager != deps[j].ToolPackager {
+			return deps[i].ToolPackager < deps[j].ToolPackager
+		}
+		if deps[i].ToolName != deps[j].ToolName {
+			return deps[i].ToolName < deps[j].ToolName
+		}
+		return deps[i].ToolVersion.LessThan(deps[j].ToolVersion)
+	})
+}
+
 // ToolDependency is a tuple that uniquely identifies a specific version of a Tool
 type ToolDependency struct {
 	ToolName     string

arduino/cores/packagemanager/package_manager.go

@@ -403,23 +403,26 @@ func (pm *PackageManager) FindToolsRequiredForBoard(board *cores.Board) ([]*core
 	}
 
 	// replace the default tools above with the specific required by the current platform
+	requiredTools := []*cores.ToolRelease{}
+	platform.Dependencies.Sort()
 	for _, toolDep := range platform.Dependencies {
 		pm.Log.WithField("tool", toolDep).Infof("Required tool")
 		tool := pm.FindToolDependency(toolDep)
 		if tool == nil {
 			return nil, fmt.Errorf("tool release not found: %s", toolDep)
 		}
-		foundTools[tool.Tool.Name] = tool
+		requiredTools = append(requiredTools, tool)
+		delete(foundTools, tool.Tool.Name)
 	}
 
-	requiredTools := []*cores.ToolRelease{}
 	for _, toolRel := range foundTools {
 		requiredTools = append(requiredTools, toolRel)
 	}
 	return requiredTools, nil
 }
 
-// FindToolDependency FIXMEDOC
+// FindToolDependency returns the ToolRelease referenced by the ToolDependency or nil if
+// the referenced tool doesn't exists.
 func (pm *PackageManager) FindToolDependency(dep *cores.ToolDependency) *cores.ToolRelease {
 	toolRelease, err := pm.Package(dep.ToolPackager).Tool(dep.ToolName).Release(dep.ToolVersion).Get()
 	if err != nil {

arduino/cores/packagemanager/package_manager_test.go

@@ -21,14 +21,13 @@ import (
 	"net/url"
 	"testing"
 
-	"go.bug.st/relaxed-semver"
-
 	"github.com/arduino/arduino-cli/arduino/cores"
 	"github.com/arduino/arduino-cli/arduino/cores/packagemanager"
 	"github.com/arduino/arduino-cli/configs"
 	"github.com/arduino/go-paths-helper"
 	"github.com/arduino/go-properties-orderedmap"
 	"github.com/stretchr/testify/require"
+	semver "go.bug.st/relaxed-semver"
 )
 
 var customHardware = paths.New("testdata", "custom_hardware")
@@ -102,6 +101,7 @@ func TestFindToolsRequiredForBoard(t *testing.T) {
 	}
 	loadIndex("https://dl.espressif.com/dl/package_esp32_index.json")
 	loadIndex("http://arduino.esp8266.com/stable/package_esp8266com_index.json")
+	loadIndex("https://adafruit.github.io/arduino-board-index/package_adafruit_index.json")
 	require.NoError(t, pm.LoadHardware(conf))
 	esp32, err := pm.FindBoardWithFQBN("esp32:esp32:esp32")
 	require.NoError(t, err)
@@ -138,4 +138,35 @@ func TestFindToolsRequiredForBoard(t *testing.T) {
 	testConflictingToolsInDifferentPackages()
 	testConflictingToolsInDifferentPackages()
 	testConflictingToolsInDifferentPackages()
+
+	feather, err := pm.FindBoardWithFQBN("adafruit:samd:adafruit_feather_m0_express")
+	require.NoError(t, err)
+	require.NotNil(t, feather)
+	featherTools, err := pm.FindToolsRequiredForBoard(feather)
+	require.NoError(t, err)
+	require.NotNil(t, featherTools)
+
+	// Test when a package index requires two different version of the same tool
+	// See: https://github.com/arduino/arduino-cli/issues/166#issuecomment-528295989
+	bossac17 := pm.FindToolDependency(&cores.ToolDependency{
+		ToolPackager: "arduino",
+		ToolName:     "bossac",
+		ToolVersion:  semver.ParseRelaxed("1.7.0"),
+	})
+	require.NotNil(t, bossac17)
+	bossac18 := pm.FindToolDependency(&cores.ToolDependency{
+		ToolPackager: "arduino",
+		ToolName:     "bossac",
+		ToolVersion:  semver.ParseRelaxed("1.8.0-48-gb176eee"),
+	})
+	require.NotNil(t, bossac18)
+	require.Contains(t, featherTools, bossac17)
+	require.Contains(t, featherTools, bossac18)
+
+	// Check if the runtime variable is set correctly to the latest version
+	uploadProperties := properties.NewMap()
+	for _, requiredTool := range featherTools {
+		uploadProperties.Merge(requiredTool.RuntimeProperties())
+	}
+	require.Equal(t, bossac18.InstallDir.String(), uploadProperties.Get("runtime.tools.bossac.path"))
 }

arduino/cores/packagemanager/testdata/data_dir_1/packages/adafruit/hardware/samd/1.5.3/keywords.txt

@@ -0,0 +1,3 @@
+SerialGSM	KEYWORD1
+SerialSARA	KEYWORD1
+INPUT_PULLDOWN	LITERAL1	Constants	RESERVED_WORD_2

arduino/cores/packagemanager/testdata/data_dir_1/packages/adafruit/hardware/samd/1.5.3/platform.txt

@@ -0,0 +1,193 @@
+# Copyright (c) 2014-2015 Arduino LLC.  All right reserved.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+# Arduino SAMD Core and platform.
+#
+# For more info:
+# https://github.com/arduino/Arduino/wiki/Arduino-IDE-1.5---3rd-party-Hardware-specification
+
+name=Adafruit SAMD (32-bits ARM Cortex-M0+ and Cortex-M4) Boards
+version=1.5.3
+
+# Compile variables
+# -----------------
+
+compiler.warning_flags=-w
+compiler.warning_flags.none=-w
+compiler.warning_flags.default=
+compiler.warning_flags.more=-Wall
+compiler.warning_flags.all=-Wall -Wextra
+
+compiler.path={runtime.tools.arm-none-eabi-gcc.path}/bin/
+compiler.c.cmd=arm-none-eabi-gcc
+compiler.c.flags=-mcpu={build.mcu} -mthumb -c -Os {compiler.warning_flags} -std=gnu11 -ffunction-sections -fdata-sections -nostdlib --param max-inline-insns-single=500 -MMD -D__SKETCH_NAME__="""{build.project_name}"""
+compiler.c.elf.cmd=arm-none-eabi-gcc
+compiler.c.elf.flags=-Os -Wl,--gc-sections -save-temps
+compiler.S.cmd=arm-none-eabi-gcc
+compiler.S.flags=-c -g -x assembler-with-cpp -MMD
+compiler.cpp.cmd=arm-none-eabi-g++
+compiler.cpp.flags=-mcpu={build.mcu} -mthumb -c -Os {compiler.warning_flags} -std=gnu++11 -ffunction-sections -fdata-sections -fno-threadsafe-statics -nostdlib --param max-inline-insns-single=500 -fno-rtti -fno-exceptions -MMD -D__SKETCH_NAME__="""{build.project_name}"""
+compiler.ar.cmd=arm-none-eabi-ar
+compiler.ar.flags=rcs
+compiler.objcopy.cmd=arm-none-eabi-objcopy
+compiler.objcopy.eep.flags=-O ihex -j .eeprom --set-section-flags=.eeprom=alloc,load --no-change-warnings --change-section-lma .eeprom=0
+compiler.elf2hex.flags=-O binary
+compiler.elf2hex.cmd=arm-none-eabi-objcopy
+compiler.ldflags=-mcpu={build.mcu} -mthumb -Wl,--cref -Wl,--check-sections -Wl,--gc-sections -Wl,--unresolved-symbols=report-all -Wl,--warn-common -Wl,--warn-section-align
+compiler.size.cmd=arm-none-eabi-size
+compiler.define=-DARDUINO=
+
+# this can be overriden in boards.txt
+build.extra_flags=
+build.cache_flags=
+build.flags.optimize=
+build.flags.maxspi=
+build.flags.maxqspi=
+build.flags.usbstack=
+build.flags.debug=
+
+# These can be overridden in platform.local.txt
+compiler.c.extra_flags=
+compiler.c.elf.extra_flags=
+#compiler.c.elf.extra_flags=-v
+compiler.cpp.extra_flags=
+compiler.S.extra_flags=
+compiler.ar.extra_flags=
+compiler.elf2hex.extra_flags=
+
+compiler.arm.cmsis.c.flags="-I{runtime.tools.CMSIS-4.5.0.path}/CMSIS/Include/" "-I{runtime.tools.CMSIS-Atmel-1.2.0.path}/CMSIS/Device/ATMEL/"
+compiler.arm.cmsis.ldflags="-L{runtime.tools.CMSIS-4.5.0.path}/CMSIS/Lib/GCC/"  -larm_cortexM0l_math
+
+# USB Flags
+# ---------
+build.usb_flags=-DUSB_VID={build.vid} -DUSB_PID={build.pid} -DUSBCON '-DUSB_MANUFACTURER={build.usb_manufacturer}' '-DUSB_PRODUCT={build.usb_product}' {build.flags.usbstack} {build.flags.debug} "-I{build.core.path}/Adafruit_TinyUSB_Core" "-I{build.core.path}/Adafruit_TinyUSB_Core/tinyusb/src"
+
+# Default usb manufacturer will be replaced at compile time using
+# numeric vendor ID if available or by board's specific value.
+build.usb_manufacturer="Unknown"
+
+
+# Compile patterns
+# ----------------
+
+## Compile c files
+recipe.c.o.pattern="{compiler.path}{compiler.c.cmd}" {compiler.c.flags} -DF_CPU={build.f_cpu} -DARDUINO={runtime.ide.version} -DARDUINO_{build.board} -DARDUINO_ARCH_{build.arch} {compiler.c.extra_flags} {build.extra_flags} {build.cache_flags}  {build.flags.debug} {build.flags.optimize} {build.flags.maxspi} {build.flags.maxqspi} {compiler.arm.cmsis.c.flags} {includes} "{source_file}" -o "{object_file}"
+
+## Compile c++ files
+recipe.cpp.o.pattern="{compiler.path}{compiler.cpp.cmd}" {compiler.cpp.flags} -DF_CPU={build.f_cpu} -DARDUINO={runtime.ide.version} -DARDUINO_{build.board} -DARDUINO_ARCH_{build.arch} {compiler.cpp.extra_flags} {build.extra_flags} {build.cache_flags} {build.flags.debug} {build.flags.optimize} {build.flags.maxspi} {build.flags.maxqspi} {compiler.arm.cmsis.c.flags} {includes} "{source_file}" -o "{object_file}"
+
+## Compile S files
+recipe.S.o.pattern="{compiler.path}{compiler.S.cmd}" {compiler.S.flags} -DF_CPU={build.f_cpu} -DARDUINO={runtime.ide.version} -DARDUINO_{build.board} -DARDUINO_ARCH_{build.arch} {compiler.S.extra_flags} {build.extra_flags} {build.cache_flags} {compiler.arm.cmsis.c.flags} {includes} "{source_file}" -o "{object_file}"
+
+## Create archives
+recipe.ar.pattern="{compiler.path}{compiler.ar.cmd}" {compiler.ar.flags} {compiler.ar.extra_flags} "{archive_file_path}" "{object_file}"
+
+## Combine gc-sections, archives, and objects
+recipe.c.combine.pattern="{compiler.path}{compiler.c.elf.cmd}"  "-L{build.path}" {compiler.c.elf.flags} {compiler.c.elf.extra_flags} "-T{build.variant.path}/{build.ldscript}" "-Wl,-Map,{build.path}/{build.project_name}.map" --specs=nano.specs --specs=nosys.specs {compiler.ldflags} -o "{build.path}/{build.project_name}.elf" {object_files} -Wl,--start-group {compiler.arm.cmsis.ldflags} "-L{build.variant.path}" -lm "{build.path}/{archive_file}" -Wl,--end-group
+
+## Create output (bin file)
+recipe.objcopy.bin.pattern="{compiler.path}{compiler.elf2hex.cmd}" {compiler.elf2hex.flags} {compiler.elf2hex.extra_flags} "{build.path}/{build.project_name}.elf" "{build.path}/{build.project_name}.bin"
+
+## Save hex
+recipe.output.tmp_file={build.project_name}.bin
+recipe.output.save_file={build.project_name}.{build.variant}.bin
+
+## Compute size
+recipe.size.pattern="{compiler.path}{compiler.size.cmd}" -A "{build.path}/{build.project_name}.elf"
+recipe.size.regex=\.text\s+([0-9]+).*
+
+
+# Uploader tools
+# --------------
+
+#
+# BOSSA
+#
+
+tools.bossac.path={runtime.tools.bossac-1.7.0.path}
+tools.bossac.cmd=bossac
+
+tools.bossac.upload.params.verbose=-i -d
+tools.bossac.upload.params.quiet=
+tools.bossac.upload.pattern="{path}/{cmd}" {upload.verbose} --port={serial.port.file} -U {upload.native_usb} -i -e -w -v "{build.path}/{build.project_name}.bin" -R
+
+tools.bossac_remote.upload.pattern=/usr/bin/run-bossac {upload.verbose} --port=ttyATH0 -U {upload.native_usb} -e -w -v /tmp/sketch.bin -R
+
+tools.bossac.network_cmd={runtime.tools.arduinoOTA.path}/bin/arduinoOTA
+tools.bossac.upload.network_pattern="{network_cmd}" -address {serial.port} -port 65280 -username arduino -password "{network.password}" -sketch "{build.path}/{build.project_name}.bin" -upload /sketch -b
+
+# v1.8.0
+
+tools.bossac18.path={runtime.tools.bossac-1.8.0-48-gb176eee.path}
+tools.bossac18.cmd=bossac
+
+tools.bossac18.upload.params.verbose=-i -d
+tools.bossac18.upload.params.quiet=
+tools.bossac18.upload.pattern="{path}/{cmd}" {upload.verbose} --port={serial.port.file} -U -i --offset={upload.offset} -w -v "{build.path}/{build.project_name}.bin" -R
+
+tools.bossac18.network_cmd={runtime.tools.arduinoOTA.path}/bin/arduinoOTA
+tools.bossac18.upload.network_pattern="{network_cmd}" -address {serial.port} -port 65280 -username arduino -password "{network.password}" -sketch "{build.path}/{build.project_name}.bin" -upload /sketch -b
+
+
+#
+# OpenOCD sketch upload
+#
+
+tools.openocd.path={runtime.tools.openocd.path}
+tools.openocd.cmd=bin/openocd
+tools.openocd.cmd.windows=bin/openocd.exe
+
+tools.openocd.upload.params.verbose=-d2
+tools.openocd.upload.params.quiet=-d0
+tools.openocd.upload.pattern="{path}/{cmd}" {upload.verbose} -s "{path}/share/openocd/scripts/" -f "{runtime.platform.path}/variants/{build.variant}/{build.openocdscript}" -c "telnet_port disabled; program {{{build.path}/{build.project_name}.bin}} verify reset 0x00002000; shutdown"
+
+tools.openocd.program.params.verbose=-d2
+tools.openocd.program.params.quiet=-d0
+tools.openocd.program.pattern="{path}/{cmd}" {program.verbose} -s "{path}/share/openocd/scripts/" -f "{runtime.platform.path}/variants/{build.variant}/{build.openocdscript}" -c "telnet_port disabled; program {{{build.path}/{build.project_name}.elf}} verify reset; shutdown"
+
+tools.openocd.erase.params.verbose=-d3
+tools.openocd.erase.params.quiet=-d0
+tools.openocd.erase.pattern=
+
+tools.openocd.bootloader.params.verbose=-d3
+tools.openocd.bootloader.params.quiet=-d0
+tools.openocd.bootloader.pattern="{path}/{cmd}" {bootloader.verbose} -s "{path}/share/openocd/scripts/" -f interface/{program.protocol}.cfg -c "{program.setup_command}" -f "{runtime.platform.path}/variants/{build.variant}/{build.openocdscript}" -c "telnet_port disabled; init; halt; at91samd bootloader 0; program {{{runtime.platform.path}/bootloaders/{bootloader.file}}} verify reset; shutdown"
+
+#
+# OpenOCD sketch upload - version with configurable bootloader size
+# FIXME: this programmer is a workaround for default options being overwritten by uploadUsingPreferences
+#
+
+tools.openocd-withbootsize.path={runtime.tools.openocd-0.9.0-arduino.path}
+tools.openocd-withbootsize.cmd=bin/openocd
+tools.openocd-withbootsize.cmd.windows=bin/openocd.exe
+
+tools.openocd-withbootsize.upload.params.verbose=-d2
+tools.openocd-withbootsize.upload.params.quiet=-d0
+tools.openocd-withbootsize.upload.pattern="{path}/{cmd}" {upload.verbose} -s "{path}/share/openocd/scripts/" -f "{runtime.platform.path}/variants/{build.variant}/{build.openocdscript}" -c "telnet_port disabled; program {{{build.path}/{build.project_name}.bin}} verify reset {bootloader.size}; shutdown"
+
+# Program flashes the binary at 0x0000, so use the linker script without_bootloader
+tools.openocd-withbootsize.program.params.verbose=-d2
+tools.openocd-withbootsize.program.params.quiet=-d0
+tools.openocd-withbootsize.program.pattern="{path}/{cmd}" {program.verbose} -s "{path}/share/openocd/scripts/" -f "{runtime.platform.path}/variants/{build.variant}/{build.openocdscript}" -c "telnet_port disabled; program {{{build.path}/{build.project_name}.elf}} verify reset; shutdown"
+
+tools.openocd-withbootsize.erase.params.verbose=-d3
+tools.openocd-withbootsize.erase.params.quiet=-d0
+tools.openocd-withbootsize.erase.pattern=
+
+tools.openocd-withbootsize.bootloader.params.verbose=-d2
+tools.openocd-withbootsize.bootloader.params.quiet=-d0
+tools.openocd-withbootsize.bootloader.pattern="{path}/{cmd}" {bootloader.verbose} -s "{path}/share/openocd/scripts/" -f "{runtime.platform.path}/variants/{build.variant}/{build.openocdscript}" -c "telnet_port disabled; init; halt; at91samd bootloader 0; program {{{runtime.platform.path}/bootloaders/{bootloader.file}}} verify reset; shutdown"

arduino/cores/packagemanager/testdata/data_dir_1/packages/adafruit/hardware/samd/1.5.3/programmers.txt

@@ -0,0 +1,30 @@
+# Copyright (c) 2014-2015 Arduino LLC.  All right reserved.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+adafruit_jlink.name=J-Link over OpenOCD
+adafruit_jlink.communication=USB
+adafruit_jlink.protocol=jlink
+adafruit_jlink.program.protocol=jlink
+adafruit_jlink.program.tool=openocd
+adafruit_jlink.program.setup_command=interface jlink; transport select swd; reset_config none separate; set WORKAREASIZE 0;
+
+
+adafruit_atmel_ice.name=Atmel-ICE over OpenOCD
+adafruit_atmel_ice.communication=USB
+adafruit_atmel_ice.protocol=cmsis-dap
+adafruit_atmel_ice.program.protocol=cmsis-dap
+adafruit_atmel_ice.program.tool=openocd
+adafruit_atmel_ice.program.setup_command=cmsis_dap_vid_pid 0x03eb 0x2141; transport select swd;

commands/upload/upload.go

@@ -116,6 +116,7 @@ func Upload(ctx context.Context, req *rpc.UploadReq, outStream io.Writer, errStr
 
 	if requiredTools, err := pm.FindToolsRequiredForBoard(board); err == nil {
 		for _, requiredTool := range requiredTools {
+			logrus.WithField("tool", requiredTool).Info("Tool required for upload")
 			uploadProperties.Merge(requiredTool.RuntimeProperties())
 		}
 	}