Merge branch 'master' into feat/purge-build-cache

Author: Luca Bianconi

Diff for: arduino/builder/sketch_test.go

@@ -17,7 +17,6 @@ package builder_test
 
 import (
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -31,7 +30,7 @@ import (
 )
 
 func tmpDirOrDie() *paths.Path {
-	dir, err := ioutil.TempDir(os.TempDir(), "builder_test")
+	dir, err := os.MkdirTemp(os.TempDir(), "builder_test")
 	if err != nil {
 		panic(fmt.Sprintf("error creating tmp dir: %v", err))
 	}
@@ -44,7 +43,7 @@ func TestSaveSketch(t *testing.T) {
 	sketchFile := filepath.Join("testdata", sketchName)
 	tmp := tmpDirOrDie()
 	defer tmp.RemoveAll()
-	source, err := ioutil.ReadFile(sketchFile)
+	source, err := os.ReadFile(sketchFile)
 	if err != nil {
 		t.Fatalf("unable to read golden file %s: %v", sketchFile, err)
 	}

Diff for: arduino/cores/packagemanager/package_manager.go

@@ -614,29 +614,58 @@ func (pme *Explorer) GetTool(toolID string) *cores.Tool {
 	}
 }
 
-// FindToolsRequiredForBoard FIXMEDOC
-func (pme *Explorer) FindToolsRequiredForBoard(board *cores.Board) ([]*cores.ToolRelease, error) {
-	pme.log.Infof("Searching tools required for board %s", board)
-
-	// core := board.Properties["build.core"]
-	platform := board.PlatformRelease
-
-	// maps "PACKAGER:TOOL" => ToolRelease
-	foundTools := map[string]*cores.ToolRelease{}
-
-	// a Platform may not specify required tools (because it's a platform that comes from a
-	// user/hardware dir without a package_index.json) then add all available tools
-	for _, targetPackage := range pme.packages {
-		for _, tool := range targetPackage.Tools {
-			rel := tool.GetLatestInstalled()
-			if rel != nil {
-				foundTools[rel.Tool.Name] = rel
+// FindToolsRequiredForBuild returns the list of ToolReleases needed to build for the specified
+// plaftorm. The buildPlatform may be different depending on the selected board.
+func (pme *Explorer) FindToolsRequiredForBuild(platform, buildPlatform *cores.PlatformRelease) ([]*cores.ToolRelease, error) {
+
+	// maps tool name => all available ToolRelease
+	allToolsAlternatives := map[string][]*cores.ToolRelease{}
+	for _, tool := range pme.GetAllInstalledToolsReleases() {
+		alternatives := allToolsAlternatives[tool.Tool.Name]
+		alternatives = append(alternatives, tool)
+		allToolsAlternatives[tool.Tool.Name] = alternatives
+	}
+
+	// selectBest select the tool with best matching, applying the following rules
+	// in order of priority:
+	// - the tool comes from the requested packager
+	// - the tool comes from the build platform packager
+	// - the tool has the greatest version
+	// - the tool packager comes first in alphabetic order
+	packagerPriority := map[string]int{}
+	packagerPriority[platform.Platform.Package.Name] = 2
+	if buildPlatform != nil {
+		packagerPriority[buildPlatform.Platform.Package.Name] = 1
+	}
+	selectBest := func(tools []*cores.ToolRelease) *cores.ToolRelease {
+		selected := tools[0]
+		for _, tool := range tools[1:] {
+			if packagerPriority[tool.Tool.Package.Name] != packagerPriority[selected.Tool.Package.Name] {
+				if packagerPriority[tool.Tool.Package.Name] > packagerPriority[selected.Tool.Package.Name] {
+					selected = tool
+				}
+				continue
+			}
+			if !tool.Version.Equal(selected.Version) {
+				if tool.Version.GreaterThan(selected.Version) {
+					selected = tool
+				}
+				continue
+			}
+			if tool.Tool.Package.Name < selected.Tool.Package.Name {
+				selected = tool
 			}
 		}
+		return selected
 	}
 
-	// replace the default tools above with the specific required by the current platform
+	// First select the specific tools required by the current platform
 	requiredTools := []*cores.ToolRelease{}
+	// The Sorting of the tool dependencies is required because some platforms may depends
+	// on more than one version of the same tool. For example adafruit:samd has both
+	// [email protected] and [email protected]. To allow the runtime property
+	// {runtime.tools.bossac.path} to be correctly set to the 1.8.0 version we must ensure
+	// that the returned array is sorted by version.
 	platform.ToolDependencies.Sort()
 	for _, toolDep := range platform.ToolDependencies {
 		pme.log.WithField("tool", toolDep).Infof("Required tool")
@@ -645,11 +674,15 @@ func (pme *Explorer) FindToolsRequiredForBoard(board *cores.Board) ([]*cores.Too
 			return nil, fmt.Errorf(tr("tool release not found: %s"), toolDep)
 		}
 		requiredTools = append(requiredTools, tool)
-		delete(foundTools, tool.Tool.Name)
+		delete(allToolsAlternatives, tool.Tool.Name)
 	}
 
-	for _, toolRel := range foundTools {
-		requiredTools = append(requiredTools, toolRel)
+	// Since a Platform may not specify the required tools (because it's a platform that comes
+	// from a user/hardware dir without a package_index.json) then add all available tools giving
+	// priority to tools coming from the same packager or referenced packager
+	for _, tools := range allToolsAlternatives {
+		tool := selectBest(tools)
+		requiredTools = append(requiredTools, tool)
 	}
 	return requiredTools, nil
 }

Diff for: arduino/cores/packagemanager/package_manager_test.go

@@ -287,6 +287,8 @@ func TestFindToolsRequiredForBoard(t *testing.T) {
 	loadIndex("https://dl.espressif.com/dl/package_esp32_index.json")
 	loadIndex("http://arduino.esp8266.com/stable/package_esp8266com_index.json")
 	loadIndex("https://adafruit.github.io/arduino-board-index/package_adafruit_index.json")
+	loadIndex("https://test.com/package_test_index.json") // this is not downloaded, it just picks the "local cached" file package_test_index.json
+
 	// We ignore the errors returned since they might not be necessarily blocking
 	// but just warnings for the user, like in the case a board is not loaded
 	// because of malformed menus
@@ -310,8 +312,13 @@ func TestFindToolsRequiredForBoard(t *testing.T) {
 	})
 	require.NotNil(t, esptool0413)
 
+	testPlatform := pme.FindPlatformRelease(&packagemanager.PlatformReference{
+		Package:              "test",
+		PlatformArchitecture: "avr",
+		PlatformVersion:      semver.MustParse("1.1.0")})
+
 	testConflictingToolsInDifferentPackages := func() {
-		tools, err := pme.FindToolsRequiredForBoard(esp32)
+		tools, err := pme.FindToolsRequiredForBuild(esp32.PlatformRelease, nil)
 		require.NoError(t, err)
 		require.Contains(t, tools, esptool231)
 		require.NotContains(t, tools, esptool0413)
@@ -331,10 +338,44 @@ func TestFindToolsRequiredForBoard(t *testing.T) {
 	testConflictingToolsInDifferentPackages()
 	testConflictingToolsInDifferentPackages()
 
+	{
+		// Test buildPlatform dependencies
+		arduinoBossac180 := pme.FindToolDependency(&cores.ToolDependency{
+			ToolPackager: "arduino",
+			ToolName:     "bossac",
+			ToolVersion:  semver.ParseRelaxed("1.8.0-48-gb176eee"),
+		})
+		require.NotNil(t, arduinoBossac180)
+		testBossac175 := pme.FindToolDependency(&cores.ToolDependency{
+			ToolPackager: "test",
+			ToolName:     "bossac",
+			ToolVersion:  semver.ParseRelaxed("1.7.5"),
+		})
+		require.NotNil(t, testBossac175)
+
+		tools, err := pme.FindToolsRequiredForBuild(esp32.PlatformRelease, nil)
+		require.NoError(t, err)
+		require.Contains(t, tools, esptool231)
+		require.NotContains(t, tools, esptool0413)
+		// When building without testPlatform dependency, arduino:bossac should be selected
+		// since it has the higher version
+		require.NotContains(t, tools, testBossac175)
+		require.Contains(t, tools, arduinoBossac180)
+
+		tools, err = pme.FindToolsRequiredForBuild(esp32.PlatformRelease, testPlatform)
+		require.NoError(t, err)
+		require.Contains(t, tools, esptool231)
+		require.NotContains(t, tools, esptool0413)
+		// When building with testPlatform dependency, test:bossac should be selected
+		// because it has dependency priority
+		require.Contains(t, tools, testBossac175)
+		require.NotContains(t, tools, arduinoBossac180)
+	}
+
 	feather, err := pme.FindBoardWithFQBN("adafruit:samd:adafruit_feather_m0_express")
 	require.NoError(t, err)
 	require.NotNil(t, feather)
-	featherTools, err := pme.FindToolsRequiredForBoard(feather)
+	featherTools, err := pme.FindToolsRequiredForBuild(feather.PlatformRelease, nil)
 	require.NoError(t, err)
 	require.NotNil(t, featherTools)
 

Diff for: arduino/cores/packagemanager/testdata/data_dir_1/package_test_index.json

@@ -0,0 +1,45 @@
+{
+  "packages": [
+    {
+      "name": "test",
+      "websiteURL": "https://test.com",
+      "email": "[email protected]",
+      "help": {
+        "online": "https://test.com"
+      },
+      "maintainer": "Test",
+      "platforms": [
+        {
+          "architecture": "avr",
+          "boards": [],
+          "name": "Test AVR Boards",
+          "category": "Test",
+          "version": "1.1.0",
+          "archiveFileName": "test-1.1.0.tar.bz2",
+          "checksum": "SHA-256:4e72d4267d9a8d86874edcd021dc661854a5136c0eed947a6fe10366bc51e373",
+          "help": {
+            "online": "https://test.com"
+          },
+          "url": "https://test.com/test-1.1.0.tar.bz2",
+          "toolsDependencies": [],
+          "size": "12345"
+        }
+      ],
+      "tools": [
+        {
+          "name": "bossac",
+          "version": "1.7.5",
+          "systems": [
+            {
+              "host": "i386-apple-darwin11",
+              "checksum": "MD5:603bcce8e59683ac27054b3197a53254",
+              "size": "96372129",
+              "archiveFileName": "bossac.tar.bz2",
+              "url": "https://test.com/bossac.tar.bz2"
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}

Diff for: arduino/cores/packagemanager/testdata/data_dir_1/packages/test/hardware/avr/1.1.0/boards.txt

@@ -17,7 +17,7 @@ package httpclient
 
 import (
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
@@ -42,7 +42,7 @@ func TestUserAgentHeader(t *testing.T) {
 	response, err := client.Do(request)
 	require.NoError(t, err)
 
-	b, err := ioutil.ReadAll(response.Body)
+	b, err := io.ReadAll(response.Body)
 	require.NoError(t, err)
 
 	require.Equal(t, "test-user-agent", string(b))

Diff for: arduino/cores/packagemanager/testdata/data_dir_1/packages/test/tools/bossac/1.7.5/.keep

@@ -24,7 +24,6 @@ import (
 	"fmt"
 	"hash"
 	"io"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -154,7 +153,7 @@ func computeDirChecksum(root string) (string, error) {
 
 // CheckDirChecksum reads checksum from the package.json and compares it with a recomputed value.
 func CheckDirChecksum(root string) (bool, error) {
-	packageJSON, err := ioutil.ReadFile(filepath.Join(root, packageFileName))
+	packageJSON, err := os.ReadFile(filepath.Join(root, packageFileName))
 	if err != nil {
 		return false, err
 	}

Diff for: arduino/httpclient/httpclient_test.go

@@ -16,9 +16,9 @@
 package resources
 
 import (
-	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
+	"os"
 	"strings"
 	"testing"
 
@@ -67,7 +67,7 @@ func TestDownloadApplyUserAgentHeaderUsingConfig(t *testing.T) {
 	//    User-Agent: arduino-cli/0.0.0-test.preview  (amd64; linux; go1.12.4) Commit:deadbeef/Build:2019-06-12 11:11:11.111
 	//    Accept-Encoding: gzip
 
-	b, err := ioutil.ReadFile(tmp.String() + "/cache/echo.txt") // just pass the file name
+	b, err := os.ReadFile(tmp.String() + "/cache/echo.txt") // just pass the file name
 	require.NoError(t, err)
 
 	requestLines := strings.Split(string(b), "\r\n")

Diff for: arduino/resources/checksums.go

@@ -20,7 +20,6 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"os"
 	"path"
@@ -53,7 +52,7 @@ func main() {
 	// To avoid polluting an existing arduino-cli installation, the example
 	// client uses a temp folder to keep cores, libraries and the like.
 	// You can point `dataDir` to a location that better fits your needs.
-	dataDir, err = ioutil.TempDir("", "arduino-rpc-client")
+	dataDir, err = os.MkdirTemp("", "arduino-rpc-client")
 	if err != nil {
 		log.Fatal(err)
 	}

Diff for: arduino/resources/helpers_test.go

@@ -159,10 +159,7 @@ func (s *ArduinoCoreServerImpl) UpdateLibrariesIndex(req *rpc.UpdateLibrariesInd
 	err := commands.UpdateLibrariesIndex(stream.Context(), req,
 		func(p *rpc.DownloadProgress) { stream.Send(&rpc.UpdateLibrariesIndexResponse{DownloadProgress: p}) },
 	)
-	if err != nil {
-		return convertErrorToRPCStatus(err)
-	}
-	return stream.Send(&rpc.UpdateLibrariesIndexResponse{})
+	return convertErrorToRPCStatus(err)
 }
 
 // Create FIXMEDOC
@@ -357,10 +354,7 @@ func (s *ArduinoCoreServerImpl) LibraryInstall(req *rpc.LibraryInstallRequest, s
 		func(p *rpc.DownloadProgress) { stream.Send(&rpc.LibraryInstallResponse{Progress: p}) },
 		func(p *rpc.TaskProgress) { stream.Send(&rpc.LibraryInstallResponse{TaskProgress: p}) },
 	)
-	if err != nil {
-		return convertErrorToRPCStatus(err)
-	}
-	return stream.Send(&rpc.LibraryInstallResponse{})
+	return convertErrorToRPCStatus(err)
 }
 
 // LibraryUpgrade FIXMEDOC
@@ -370,21 +364,15 @@ func (s *ArduinoCoreServerImpl) LibraryUpgrade(req *rpc.LibraryUpgradeRequest, s
 		func(p *rpc.DownloadProgress) { stream.Send(&rpc.LibraryUpgradeResponse{Progress: p}) },
 		func(p *rpc.TaskProgress) { stream.Send(&rpc.LibraryUpgradeResponse{TaskProgress: p}) },
 	)
-	if err != nil {
-		return convertErrorToRPCStatus(err)
-	}
-	return stream.Send(&rpc.LibraryUpgradeResponse{})
+	return convertErrorToRPCStatus(err)
 }
 
 // LibraryUninstall FIXMEDOC
 func (s *ArduinoCoreServerImpl) LibraryUninstall(req *rpc.LibraryUninstallRequest, stream rpc.ArduinoCoreService_LibraryUninstallServer) error {
 	err := lib.LibraryUninstall(stream.Context(), req,
 		func(p *rpc.TaskProgress) { stream.Send(&rpc.LibraryUninstallResponse{TaskProgress: p}) },
 	)
-	if err != nil {
-		return convertErrorToRPCStatus(err)
-	}
-	return stream.Send(&rpc.LibraryUninstallResponse{})
+	return convertErrorToRPCStatus(err)
 }
 
 // LibraryUpgradeAll FIXMEDOC
@@ -393,10 +381,7 @@ func (s *ArduinoCoreServerImpl) LibraryUpgradeAll(req *rpc.LibraryUpgradeAllRequ
 		func(p *rpc.DownloadProgress) { stream.Send(&rpc.LibraryUpgradeAllResponse{Progress: p}) },
 		func(p *rpc.TaskProgress) { stream.Send(&rpc.LibraryUpgradeAllResponse{TaskProgress: p}) },
 	)
-	if err != nil {
-		return convertErrorToRPCStatus(err)
-	}
-	return stream.Send(&rpc.LibraryUpgradeAllResponse{})
+	return convertErrorToRPCStatus(err)
 }
 
 // LibraryResolveDependencies FIXMEDOC
@@ -429,10 +414,7 @@ func (s *ArduinoCoreServerImpl) ZipLibraryInstall(req *rpc.ZipLibraryInstallRequ
 		stream.Context(), req,
 		func(p *rpc.TaskProgress) { stream.Send(&rpc.ZipLibraryInstallResponse{TaskProgress: p}) },
 	)
-	if err != nil {
-		return convertErrorToRPCStatus(err)
-	}
-	return stream.Send(&rpc.ZipLibraryInstallResponse{})
+	return convertErrorToRPCStatus(err)
 }
 
 // GitLibraryInstall FIXMEDOC
@@ -441,10 +423,7 @@ func (s *ArduinoCoreServerImpl) GitLibraryInstall(req *rpc.GitLibraryInstallRequ
 		stream.Context(), req,
 		func(p *rpc.TaskProgress) { stream.Send(&rpc.GitLibraryInstallResponse{TaskProgress: p}) },
 	)
-	if err != nil {
-		return convertErrorToRPCStatus(err)
-	}
-	return stream.Send(&rpc.GitLibraryInstallResponse{})
+	return convertErrorToRPCStatus(err)
 }
 
 // EnumerateMonitorPortSettings FIXMEDOC