[breaking] Implementation of sketch profiles

arduino/cores/cores.go

@@ -16,12 +16,16 @@
 package cores
 
 import (
+	"crypto/sha256"
+	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	"net/url"
 	"sort"
 	"strings"
 
 	"github.com/arduino/arduino-cli/arduino/resources"
+	"github.com/arduino/arduino-cli/arduino/utils"
 	"github.com/arduino/arduino-cli/i18n"
 	rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
 	paths "github.com/arduino/go-paths-helper"
@@ -122,6 +126,17 @@ func (dep *ToolDependency) String() string {
 	return dep.ToolPackager + ":" + dep.ToolName + "@" + dep.ToolVersion.String()
 }
 
+// InternalUniqueIdentifier returns the unique identifier for this object
+func (dep *ToolDependency) InternalUniqueIdentifier(platformIndexURL *url.URL) string {
+	h := sha256.New()
+	h.Write([]byte(dep.String()))
+	if platformIndexURL != nil {
+		h.Write([]byte(platformIndexURL.String()))
+	}
+	res := dep.String() + "_" + hex.EncodeToString(h.Sum([]byte{}))[:16]
+	return utils.SanitizeName(res)
+}
+
 // DiscoveryDependencies is a list of DiscoveryDependency
 type DiscoveryDependencies []*DiscoveryDependency
 

arduino/cores/packagemanager/download.go

@@ -18,6 +18,7 @@ package packagemanager
 import (
 	"fmt"
 
+	"github.com/arduino/arduino-cli/arduino"
 	"github.com/arduino/arduino-cli/arduino/cores"
 	rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
 	"go.bug.st/downloader/v2"
@@ -128,6 +129,9 @@ func (pm *PackageManager) DownloadToolRelease(tool *cores.ToolRelease, config *d
 
 // DownloadPlatformRelease downloads a PlatformRelease. If the platform is already downloaded a
 // nil Downloader is returned.
-func (pm *PackageManager) DownloadPlatformRelease(platform *cores.PlatformRelease, config *downloader.Config, label string, progressCB rpc.DownloadProgressCB) error {
-	return platform.Resource.Download(pm.DownloadDir, config, label, progressCB)
+func (pm *PackageManager) DownloadPlatformRelease(platform *cores.PlatformRelease, config *downloader.Config, progressCB rpc.DownloadProgressCB) error {
+	if platform.Resource == nil {
+		return &arduino.PlatformNotFoundError{Platform: platform.String()}
+	}
+	return platform.Resource.Download(pm.DownloadDir, config, platform.String(), progressCB)
 }

arduino/cores/packagemanager/loader.go

@@ -128,7 +128,7 @@ func (pm *PackageManager) LoadHardwareFromDirectory(path *paths.Path) []error {
 		toolsSubdirPath := packagerPath.Join("tools")
 		if toolsSubdirPath.IsDir() {
 			pm.Log.Infof("Checking existence of 'tools' path: %s", toolsSubdirPath)
-			merr = append(merr, pm.loadToolsFromPackage(targetPackage, toolsSubdirPath)...)
+			merr = append(merr, pm.LoadToolsFromPackageDir(targetPackage, toolsSubdirPath)...)
 		}
 		// If the Package does not contain Platforms or Tools we remove it since does not contain anything valuable
 		if len(targetPackage.Platforms) == 0 && len(targetPackage.Tools) == 0 {
@@ -589,7 +589,9 @@ func convertUploadToolsToPluggableDiscovery(props *properties.Map) {
 	props.Merge(propsToAdd)
 }
 
-func (pm *PackageManager) loadToolsFromPackage(targetPackage *cores.Package, toolsPath *paths.Path) []error {
+// LoadToolsFromPackageDir loads a set of tools from the given toolsPath. The tools will be loaded
+// in the given *Package.
+func (pm *PackageManager) LoadToolsFromPackageDir(targetPackage *cores.Package, toolsPath *paths.Path) []error {
 	pm.Log.Infof("Loading tools from dir: %s", toolsPath)
 
 	var merr []error
@@ -712,7 +714,7 @@ func (pm *PackageManager) LoadToolsFromBundleDirectory(toolsPath *paths.Path) er
 	} else {
 		// otherwise load the tools inside the unnamed package
 		unnamedPackage := pm.Packages.GetOrCreatePackage("")
-		pm.loadToolsFromPackage(unnamedPackage, toolsPath)
+		pm.LoadToolsFromPackageDir(unnamedPackage, toolsPath)
 	}
 	return nil
 }

arduino/cores/packagemanager/profiles.go

@@ -0,0 +1,187 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2020-2022 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to [email protected].
+
+package packagemanager
+
+import (
+	"fmt"
+	"net/url"
+
+	"github.com/arduino/arduino-cli/arduino"
+	"github.com/arduino/arduino-cli/arduino/cores"
+	"github.com/arduino/arduino-cli/arduino/resources"
+	"github.com/arduino/arduino-cli/arduino/sketch"
+	"github.com/arduino/arduino-cli/cli/globals"
+	"github.com/arduino/arduino-cli/configuration"
+	rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
+	"github.com/arduino/go-paths-helper"
+	"github.com/sirupsen/logrus"
+)
+
+// LoadHardwareForProfile load the hardware platforms for the given profile.
+// If installMissing is true then possibly missing tools and platforms will be downloaded and installed.
+func (pm *PackageManager) LoadHardwareForProfile(p *sketch.Profile, installMissing bool, downloadCB rpc.DownloadProgressCB, taskCB rpc.TaskProgressCB) []error {
+	// Load required platforms
+	var merr []error
+	var platformReleases []*cores.PlatformRelease
+	indexURLs := map[string]*url.URL{}
+	for _, platformRef := range p.Platforms {
+		if platformRelease, err := pm.loadProfilePlatform(platformRef, installMissing, downloadCB, taskCB); err != nil {
+			merr = append(merr, fmt.Errorf("%s: %w", tr("loading required platform %s", platformRef), err))
+			logrus.WithField("platform", platformRef).WithError(err).Debugf("Error loading platform for profile")
+		} else {
+			platformReleases = append(platformReleases, platformRelease)
+			indexURLs[platformRelease.Platform.Name] = platformRef.PlatformIndexURL
+			logrus.WithField("platform", platformRef).Debugf("Loaded platform for profile")
+		}
+	}
+
+	// Load tools dependencies for the platforms
+	for _, platformRelease := range platformReleases {
+		// TODO: pm.FindPlatformReleaseDependencies(platformRelease)
+
+		for _, toolDep := range platformRelease.ToolDependencies {
+			indexURL := indexURLs[toolDep.ToolPackager]
+			if err := pm.loadProfileTool(toolDep, indexURL, installMissing, downloadCB, taskCB); err != nil {
+				merr = append(merr, fmt.Errorf("%s: %w", tr("loading required tool %s", toolDep), err))
+				logrus.WithField("tool", toolDep).WithField("index_url", indexURL).WithError(err).Debugf("Error loading tool for profile")
+			} else {
+				logrus.WithField("tool", toolDep).WithField("index_url", indexURL).Debugf("Loaded tool for profile")
+			}
+		}
+	}
+
+	return merr
+}
+
+func (pm *PackageManager) loadProfilePlatform(platformRef *sketch.ProfilePlatformReference, installMissing bool, downloadCB rpc.DownloadProgressCB, taskCB rpc.TaskProgressCB) (*cores.PlatformRelease, error) {
+	targetPackage := pm.Packages.GetOrCreatePackage(platformRef.Packager)
+	platform := targetPackage.GetOrCreatePlatform(platformRef.Architecture)
+	release := platform.GetOrCreateRelease(platformRef.Version)
+
+	uid := platformRef.InternalUniqueIdentifier()
+	destDir := configuration.ProfilesCacheDir(configuration.Settings).Join(uid)
+	if !destDir.IsDir() && installMissing {
+		// Try installing the missing platform
+		if err := pm.installMissingProfilePlatform(platformRef, destDir, downloadCB, taskCB); err != nil {
+			return nil, err
+		}
+	}
+	return release, pm.loadPlatformRelease(release, destDir)
+}
+
+func (pm *PackageManager) installMissingProfilePlatform(platformRef *sketch.ProfilePlatformReference, destDir *paths.Path, downloadCB rpc.DownloadProgressCB, taskCB rpc.TaskProgressCB) error {
+	// Instantiate a temporary package manager only for platform installation
+	_ = pm.TempDir.MkdirAll()
+	tmp, err := paths.MkTempDir(pm.TempDir.String(), "")
+	if err != nil {
+		return fmt.Errorf("installing missing platform: could not create temp dir %s", err)
+	}
+	tmpPm := NewPackageManager(tmp, tmp, pm.DownloadDir, tmp, pm.userAgent)
+	defer tmp.RemoveAll()
+
+	// Download the main index and parse it
+	taskCB(&rpc.TaskProgress{Name: tr("Downloading platform %s", platformRef)})
+	defaultIndexURL, _ := url.Parse(globals.DefaultIndexURL)
+	indexesToDownload := []*url.URL{defaultIndexURL}
+	if platformRef.PlatformIndexURL != nil {
+		indexesToDownload = append(indexesToDownload, platformRef.PlatformIndexURL)
+	}
+	for _, indexURL := range indexesToDownload {
+		if err != nil {
+			taskCB(&rpc.TaskProgress{Name: tr("Error downloading %s", indexURL)})
+			return &arduino.FailedDownloadError{Message: tr("Error downloading %s", indexURL), Cause: err}
+		}
+		indexResource := resources.IndexResource{URL: indexURL}
+		if err := indexResource.Download(tmpPm.IndexDir, downloadCB); err != nil {
+			taskCB(&rpc.TaskProgress{Name: tr("Error downloading %s", indexURL)})
+			return &arduino.FailedDownloadError{Message: tr("Error downloading %s", indexURL), Cause: err}
+		}
+		if err := tmpPm.LoadPackageIndex(indexURL); err != nil {
+			taskCB(&rpc.TaskProgress{Name: tr("Error loading index %s", indexURL)})
+			return &arduino.FailedInstallError{Message: tr("Error loading index %s", indexURL), Cause: err}
+		}
+	}
+
+	// Download the platform
+	tmpTargetPackage := tmpPm.Packages.GetOrCreatePackage(platformRef.Packager)
+	tmpPlatform := tmpTargetPackage.GetOrCreatePlatform(platformRef.Architecture)
+	tmpPlatformRelease := tmpPlatform.GetOrCreateRelease(platformRef.Version)
+	if err := tmpPm.DownloadPlatformRelease(tmpPlatformRelease, nil, downloadCB); err != nil {
+		taskCB(&rpc.TaskProgress{Name: tr("Error downloading platform %s", tmpPlatformRelease)})
+		return &arduino.FailedInstallError{Message: tr("Error downloading platform %s", tmpPlatformRelease), Cause: err}
+	}
+	taskCB(&rpc.TaskProgress{Completed: true})
+
+	// Perform install
+	taskCB(&rpc.TaskProgress{Name: tr("Installing platform %s", tmpPlatformRelease)})
+	if err := tmpPm.InstallPlatformInDirectory(tmpPlatformRelease, destDir); err != nil {
+		taskCB(&rpc.TaskProgress{Name: tr("Error installing platform %s", tmpPlatformRelease)})
+		return &arduino.FailedInstallError{Message: tr("Error installing platform %s", tmpPlatformRelease), Cause: err}
+	}
+	taskCB(&rpc.TaskProgress{Completed: true})
+	return nil
+}
+
+func (pm *PackageManager) loadProfileTool(toolRef *cores.ToolDependency, indexURL *url.URL, installMissing bool, downloadCB rpc.DownloadProgressCB, taskCB rpc.TaskProgressCB) error {
+	targetPackage := pm.Packages.GetOrCreatePackage(toolRef.ToolPackager)
+	tool := targetPackage.GetOrCreateTool(toolRef.ToolName)
+
+	uid := toolRef.InternalUniqueIdentifier(indexURL)
+	destDir := configuration.ProfilesCacheDir(configuration.Settings).Join(uid)
+
+	if !destDir.IsDir() && installMissing {
+		// Try installing the missing tool
+		toolRelease := tool.GetOrCreateRelease(toolRef.ToolVersion)
+		if toolRelease == nil {
+			return &arduino.InvalidVersionError{Cause: fmt.Errorf(tr("version %s not found", toolRef.ToolVersion))}
+		}
+		if err := pm.installMissingProfileTool(toolRelease, destDir, downloadCB, taskCB); err != nil {
+			return err
+		}
+	}
+
+	return pm.loadToolReleaseFromDirectory(tool, toolRef.ToolVersion, destDir)
+}
+
+func (pm *PackageManager) installMissingProfileTool(toolRelease *cores.ToolRelease, destDir *paths.Path, downloadCB rpc.DownloadProgressCB, taskCB rpc.TaskProgressCB) error {
+	// Instantiate a temporary package manager only for platform installation
+	tmp, err := paths.MkTempDir(destDir.Parent().String(), "")
+	if err != nil {
+		return fmt.Errorf("installing missing platform: could not create temp dir %s", err)
+	}
+	defer tmp.RemoveAll()
+
+	// Download the tool
+	toolResource := toolRelease.GetCompatibleFlavour()
+	if toolResource == nil {
+		return &arduino.InvalidVersionError{Cause: fmt.Errorf(tr("version %s not available for this operating system", toolRelease))}
+	}
+	taskCB(&rpc.TaskProgress{Name: tr("Downloading tool %s", toolRelease)})
+	if err := toolResource.Download(pm.DownloadDir, nil, toolRelease.String(), downloadCB); err != nil {
+		taskCB(&rpc.TaskProgress{Name: tr("Error downloading tool %s", toolRelease)})
+		return &arduino.FailedInstallError{Message: tr("Error installing tool %s", toolRelease), Cause: err}
+	}
+	taskCB(&rpc.TaskProgress{Completed: true})
+
+	// Install tool
+	taskCB(&rpc.TaskProgress{Name: tr("Installing tool %s", toolRelease)})
+	if err := toolResource.Install(pm.DownloadDir, tmp, destDir); err != nil {
+		taskCB(&rpc.TaskProgress{Name: tr("Error installing tool %s", toolRelease)})
+		return &arduino.FailedInstallError{Message: tr("Error installing tool %s", toolRelease), Cause: err}
+	}
+	taskCB(&rpc.TaskProgress{Completed: true})
+	return nil
+}

arduino/errors.go

@@ -193,6 +193,43 @@ func (e *UnknownFQBNError) ToRPCStatus() *status.Status {
 	return status.New(codes.NotFound, e.Error())
 }
 
+// UnknownProfileError is returned when the profile is not found
+type UnknownProfileError struct {
+	Profile string
+	Cause   error
+}
+
+func (e *UnknownProfileError) Error() string {
+	return composeErrorMsg(tr("Profile '%s' not found", e.Profile), e.Cause)
+}
+
+func (e *UnknownProfileError) Unwrap() error {
+	return e.Cause
+}
+
+// ToRPCStatus converts the error into a *status.Status
+func (e *UnknownProfileError) ToRPCStatus() *status.Status {
+	return status.New(codes.NotFound, e.Error())
+}
+
+// InvalidProfileError is returned when the profile has errors
+type InvalidProfileError struct {
+	Cause error
+}
+
+func (e *InvalidProfileError) Error() string {
+	return composeErrorMsg(tr("Invalid profile"), e.Cause)
+}
+
+func (e *InvalidProfileError) Unwrap() error {
+	return e.Cause
+}
+
+// ToRPCStatus converts the error into a *status.Status
+func (e *InvalidProfileError) ToRPCStatus() *status.Status {
+	return status.New(codes.FailedPrecondition, e.Error())
+}
+
 // MissingPortAddressError is returned when the port protocol is mandatory and not specified
 type MissingPortAddressError struct{}