Skip to content

Commit 7e9e4ca

Browse files
cmaglieper1234
andauthored
[breaking] Implementation of sketch profiles (#1713)
* cosmetic: renamed import * Simplify function pm.DownloadPlatformRelease * Implementation of the Profiles parser * Added methods to get profiles from sketch * Added gRPC parameters to support profiles * Added function to load packages for profiles * Added support for profiles in compile command * Added progress callback and installMissing flag (stubs) in pm.PrepareLibrariesAndPackageManagersForProfile * Added auto-install procedures for profiles * Handle platform not found errors * Draft implementation of upload with profiles * Made packagemamager.loadToolsFromPackage public * Simplified callbacks in commands.Init * cosmetic: added shortcut variable for library manager * cosmetic: added shortcut variable for package manager; small readability improvement * Wiring profiles into arduino-cli and gRPC implementation * Made gRPC Init return a full Profile structure Makes it more future-proof in case of further expasion * (tech debt) Disable profiles if compiling with --libraries/--library * Fixed some linter warnings * Apply suggestions from code review Co-authored-by: per1234 <[email protected]> * Added profiles specification docs * Allow both sketch.yaml and .yml (with priority for .yaml) * Correctly handle nil return value * Apply suggestions from code review Co-authored-by: per1234 <[email protected]> * Apply suggestions from code review * Provide `core install` suggestions only when compiling without profiles * Remove stray comment * Fixed some comments in protoc files * Apply suggestions from code review * Apply suggestions from code review Co-authored-by: per1234 <[email protected]> * Implemented missing AsYaml methods and added tests * Apply suggestions from code review * run of prettier formatter * Preserve profiles ordering in profiles.yaml * Apply suggestions from code review Co-authored-by: per1234 <[email protected]> Co-authored-by: per1234 <[email protected]>
1 parent 48dd5c7 commit 7e9e4ca

File tree

25 files changed

+1793
-750
lines changed

25 files changed

+1793
-750
lines changed

Diff for: arduino/cores/cores.go

+15
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,16 @@
1616
package cores
1717

1818
import (
19+
"crypto/sha256"
20+
"encoding/hex"
1921
"encoding/json"
2022
"fmt"
23+
"net/url"
2124
"sort"
2225
"strings"
2326

2427
"github.com/arduino/arduino-cli/arduino/resources"
28+
"github.com/arduino/arduino-cli/arduino/utils"
2529
"github.com/arduino/arduino-cli/i18n"
2630
rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
2731
paths "github.com/arduino/go-paths-helper"
@@ -122,6 +126,17 @@ func (dep *ToolDependency) String() string {
122126
return dep.ToolPackager + ":" + dep.ToolName + "@" + dep.ToolVersion.String()
123127
}
124128

129+
// InternalUniqueIdentifier returns the unique identifier for this object
130+
func (dep *ToolDependency) InternalUniqueIdentifier(platformIndexURL *url.URL) string {
131+
h := sha256.New()
132+
h.Write([]byte(dep.String()))
133+
if platformIndexURL != nil {
134+
h.Write([]byte(platformIndexURL.String()))
135+
}
136+
res := dep.String() + "_" + hex.EncodeToString(h.Sum([]byte{}))[:16]
137+
return utils.SanitizeName(res)
138+
}
139+
125140
// DiscoveryDependencies is a list of DiscoveryDependency
126141
type DiscoveryDependencies []*DiscoveryDependency
127142

Diff for: arduino/cores/packagemanager/download.go

+6-2
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ package packagemanager
1818
import (
1919
"fmt"
2020

21+
"github.com/arduino/arduino-cli/arduino"
2122
"github.com/arduino/arduino-cli/arduino/cores"
2223
rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
2324
"go.bug.st/downloader/v2"
@@ -128,6 +129,9 @@ func (pm *PackageManager) DownloadToolRelease(tool *cores.ToolRelease, config *d
128129

129130
// DownloadPlatformRelease downloads a PlatformRelease. If the platform is already downloaded a
130131
// nil Downloader is returned.
131-
func (pm *PackageManager) DownloadPlatformRelease(platform *cores.PlatformRelease, config *downloader.Config, label string, progressCB rpc.DownloadProgressCB) error {
132-
return platform.Resource.Download(pm.DownloadDir, config, label, progressCB)
132+
func (pm *PackageManager) DownloadPlatformRelease(platform *cores.PlatformRelease, config *downloader.Config, progressCB rpc.DownloadProgressCB) error {
133+
if platform.Resource == nil {
134+
return &arduino.PlatformNotFoundError{Platform: platform.String()}
135+
}
136+
return platform.Resource.Download(pm.DownloadDir, config, platform.String(), progressCB)
133137
}

Diff for: arduino/cores/packagemanager/loader.go

+5-3
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,7 @@ func (pm *PackageManager) LoadHardwareFromDirectory(path *paths.Path) []error {
128128
toolsSubdirPath := packagerPath.Join("tools")
129129
if toolsSubdirPath.IsDir() {
130130
pm.Log.Infof("Checking existence of 'tools' path: %s", toolsSubdirPath)
131-
merr = append(merr, pm.loadToolsFromPackage(targetPackage, toolsSubdirPath)...)
131+
merr = append(merr, pm.LoadToolsFromPackageDir(targetPackage, toolsSubdirPath)...)
132132
}
133133
// If the Package does not contain Platforms or Tools we remove it since does not contain anything valuable
134134
if len(targetPackage.Platforms) == 0 && len(targetPackage.Tools) == 0 {
@@ -589,7 +589,9 @@ func convertUploadToolsToPluggableDiscovery(props *properties.Map) {
589589
props.Merge(propsToAdd)
590590
}
591591

592-
func (pm *PackageManager) loadToolsFromPackage(targetPackage *cores.Package, toolsPath *paths.Path) []error {
592+
// LoadToolsFromPackageDir loads a set of tools from the given toolsPath. The tools will be loaded
593+
// in the given *Package.
594+
func (pm *PackageManager) LoadToolsFromPackageDir(targetPackage *cores.Package, toolsPath *paths.Path) []error {
593595
pm.Log.Infof("Loading tools from dir: %s", toolsPath)
594596

595597
var merr []error
@@ -712,7 +714,7 @@ func (pm *PackageManager) LoadToolsFromBundleDirectory(toolsPath *paths.Path) er
712714
} else {
713715
// otherwise load the tools inside the unnamed package
714716
unnamedPackage := pm.Packages.GetOrCreatePackage("")
715-
pm.loadToolsFromPackage(unnamedPackage, toolsPath)
717+
pm.LoadToolsFromPackageDir(unnamedPackage, toolsPath)
716718
}
717719
return nil
718720
}

Diff for: arduino/cores/packagemanager/profiles.go

+187
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,187 @@
1+
// This file is part of arduino-cli.
2+
//
3+
// Copyright 2020-2022 ARDUINO SA (http://www.arduino.cc/)
4+
//
5+
// This software is released under the GNU General Public License version 3,
6+
// which covers the main part of arduino-cli.
7+
// The terms of this license can be found at:
8+
// https://www.gnu.org/licenses/gpl-3.0.en.html
9+
//
10+
// You can be released from the requirements of the above licenses by purchasing
11+
// a commercial license. Buying such a license is mandatory if you want to
12+
// modify or otherwise use the software for commercial activities involving the
13+
// Arduino software without disclosing the source code of your own applications.
14+
// To purchase a commercial license, send an email to [email protected].
15+
16+
package packagemanager
17+
18+
import (
19+
"fmt"
20+
"net/url"
21+
22+
"github.com/arduino/arduino-cli/arduino"
23+
"github.com/arduino/arduino-cli/arduino/cores"
24+
"github.com/arduino/arduino-cli/arduino/resources"
25+
"github.com/arduino/arduino-cli/arduino/sketch"
26+
"github.com/arduino/arduino-cli/cli/globals"
27+
"github.com/arduino/arduino-cli/configuration"
28+
rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
29+
"github.com/arduino/go-paths-helper"
30+
"github.com/sirupsen/logrus"
31+
)
32+
33+
// LoadHardwareForProfile load the hardware platforms for the given profile.
34+
// If installMissing is true then possibly missing tools and platforms will be downloaded and installed.
35+
func (pm *PackageManager) LoadHardwareForProfile(p *sketch.Profile, installMissing bool, downloadCB rpc.DownloadProgressCB, taskCB rpc.TaskProgressCB) []error {
36+
// Load required platforms
37+
var merr []error
38+
var platformReleases []*cores.PlatformRelease
39+
indexURLs := map[string]*url.URL{}
40+
for _, platformRef := range p.Platforms {
41+
if platformRelease, err := pm.loadProfilePlatform(platformRef, installMissing, downloadCB, taskCB); err != nil {
42+
merr = append(merr, fmt.Errorf("%s: %w", tr("loading required platform %s", platformRef), err))
43+
logrus.WithField("platform", platformRef).WithError(err).Debugf("Error loading platform for profile")
44+
} else {
45+
platformReleases = append(platformReleases, platformRelease)
46+
indexURLs[platformRelease.Platform.Name] = platformRef.PlatformIndexURL
47+
logrus.WithField("platform", platformRef).Debugf("Loaded platform for profile")
48+
}
49+
}
50+
51+
// Load tools dependencies for the platforms
52+
for _, platformRelease := range platformReleases {
53+
// TODO: pm.FindPlatformReleaseDependencies(platformRelease)
54+
55+
for _, toolDep := range platformRelease.ToolDependencies {
56+
indexURL := indexURLs[toolDep.ToolPackager]
57+
if err := pm.loadProfileTool(toolDep, indexURL, installMissing, downloadCB, taskCB); err != nil {
58+
merr = append(merr, fmt.Errorf("%s: %w", tr("loading required tool %s", toolDep), err))
59+
logrus.WithField("tool", toolDep).WithField("index_url", indexURL).WithError(err).Debugf("Error loading tool for profile")
60+
} else {
61+
logrus.WithField("tool", toolDep).WithField("index_url", indexURL).Debugf("Loaded tool for profile")
62+
}
63+
}
64+
}
65+
66+
return merr
67+
}
68+
69+
func (pm *PackageManager) loadProfilePlatform(platformRef *sketch.ProfilePlatformReference, installMissing bool, downloadCB rpc.DownloadProgressCB, taskCB rpc.TaskProgressCB) (*cores.PlatformRelease, error) {
70+
targetPackage := pm.Packages.GetOrCreatePackage(platformRef.Packager)
71+
platform := targetPackage.GetOrCreatePlatform(platformRef.Architecture)
72+
release := platform.GetOrCreateRelease(platformRef.Version)
73+
74+
uid := platformRef.InternalUniqueIdentifier()
75+
destDir := configuration.ProfilesCacheDir(configuration.Settings).Join(uid)
76+
if !destDir.IsDir() && installMissing {
77+
// Try installing the missing platform
78+
if err := pm.installMissingProfilePlatform(platformRef, destDir, downloadCB, taskCB); err != nil {
79+
return nil, err
80+
}
81+
}
82+
return release, pm.loadPlatformRelease(release, destDir)
83+
}
84+
85+
func (pm *PackageManager) installMissingProfilePlatform(platformRef *sketch.ProfilePlatformReference, destDir *paths.Path, downloadCB rpc.DownloadProgressCB, taskCB rpc.TaskProgressCB) error {
86+
// Instantiate a temporary package manager only for platform installation
87+
_ = pm.TempDir.MkdirAll()
88+
tmp, err := paths.MkTempDir(pm.TempDir.String(), "")
89+
if err != nil {
90+
return fmt.Errorf("installing missing platform: could not create temp dir %s", err)
91+
}
92+
tmpPm := NewPackageManager(tmp, tmp, pm.DownloadDir, tmp, pm.userAgent)
93+
defer tmp.RemoveAll()
94+
95+
// Download the main index and parse it
96+
taskCB(&rpc.TaskProgress{Name: tr("Downloading platform %s", platformRef)})
97+
defaultIndexURL, _ := url.Parse(globals.DefaultIndexURL)
98+
indexesToDownload := []*url.URL{defaultIndexURL}
99+
if platformRef.PlatformIndexURL != nil {
100+
indexesToDownload = append(indexesToDownload, platformRef.PlatformIndexURL)
101+
}
102+
for _, indexURL := range indexesToDownload {
103+
if err != nil {
104+
taskCB(&rpc.TaskProgress{Name: tr("Error downloading %s", indexURL)})
105+
return &arduino.FailedDownloadError{Message: tr("Error downloading %s", indexURL), Cause: err}
106+
}
107+
indexResource := resources.IndexResource{URL: indexURL}
108+
if err := indexResource.Download(tmpPm.IndexDir, downloadCB); err != nil {
109+
taskCB(&rpc.TaskProgress{Name: tr("Error downloading %s", indexURL)})
110+
return &arduino.FailedDownloadError{Message: tr("Error downloading %s", indexURL), Cause: err}
111+
}
112+
if err := tmpPm.LoadPackageIndex(indexURL); err != nil {
113+
taskCB(&rpc.TaskProgress{Name: tr("Error loading index %s", indexURL)})
114+
return &arduino.FailedInstallError{Message: tr("Error loading index %s", indexURL), Cause: err}
115+
}
116+
}
117+
118+
// Download the platform
119+
tmpTargetPackage := tmpPm.Packages.GetOrCreatePackage(platformRef.Packager)
120+
tmpPlatform := tmpTargetPackage.GetOrCreatePlatform(platformRef.Architecture)
121+
tmpPlatformRelease := tmpPlatform.GetOrCreateRelease(platformRef.Version)
122+
if err := tmpPm.DownloadPlatformRelease(tmpPlatformRelease, nil, downloadCB); err != nil {
123+
taskCB(&rpc.TaskProgress{Name: tr("Error downloading platform %s", tmpPlatformRelease)})
124+
return &arduino.FailedInstallError{Message: tr("Error downloading platform %s", tmpPlatformRelease), Cause: err}
125+
}
126+
taskCB(&rpc.TaskProgress{Completed: true})
127+
128+
// Perform install
129+
taskCB(&rpc.TaskProgress{Name: tr("Installing platform %s", tmpPlatformRelease)})
130+
if err := tmpPm.InstallPlatformInDirectory(tmpPlatformRelease, destDir); err != nil {
131+
taskCB(&rpc.TaskProgress{Name: tr("Error installing platform %s", tmpPlatformRelease)})
132+
return &arduino.FailedInstallError{Message: tr("Error installing platform %s", tmpPlatformRelease), Cause: err}
133+
}
134+
taskCB(&rpc.TaskProgress{Completed: true})
135+
return nil
136+
}
137+
138+
func (pm *PackageManager) loadProfileTool(toolRef *cores.ToolDependency, indexURL *url.URL, installMissing bool, downloadCB rpc.DownloadProgressCB, taskCB rpc.TaskProgressCB) error {
139+
targetPackage := pm.Packages.GetOrCreatePackage(toolRef.ToolPackager)
140+
tool := targetPackage.GetOrCreateTool(toolRef.ToolName)
141+
142+
uid := toolRef.InternalUniqueIdentifier(indexURL)
143+
destDir := configuration.ProfilesCacheDir(configuration.Settings).Join(uid)
144+
145+
if !destDir.IsDir() && installMissing {
146+
// Try installing the missing tool
147+
toolRelease := tool.GetOrCreateRelease(toolRef.ToolVersion)
148+
if toolRelease == nil {
149+
return &arduino.InvalidVersionError{Cause: fmt.Errorf(tr("version %s not found", toolRef.ToolVersion))}
150+
}
151+
if err := pm.installMissingProfileTool(toolRelease, destDir, downloadCB, taskCB); err != nil {
152+
return err
153+
}
154+
}
155+
156+
return pm.loadToolReleaseFromDirectory(tool, toolRef.ToolVersion, destDir)
157+
}
158+
159+
func (pm *PackageManager) installMissingProfileTool(toolRelease *cores.ToolRelease, destDir *paths.Path, downloadCB rpc.DownloadProgressCB, taskCB rpc.TaskProgressCB) error {
160+
// Instantiate a temporary package manager only for platform installation
161+
tmp, err := paths.MkTempDir(destDir.Parent().String(), "")
162+
if err != nil {
163+
return fmt.Errorf("installing missing platform: could not create temp dir %s", err)
164+
}
165+
defer tmp.RemoveAll()
166+
167+
// Download the tool
168+
toolResource := toolRelease.GetCompatibleFlavour()
169+
if toolResource == nil {
170+
return &arduino.InvalidVersionError{Cause: fmt.Errorf(tr("version %s not available for this operating system", toolRelease))}
171+
}
172+
taskCB(&rpc.TaskProgress{Name: tr("Downloading tool %s", toolRelease)})
173+
if err := toolResource.Download(pm.DownloadDir, nil, toolRelease.String(), downloadCB); err != nil {
174+
taskCB(&rpc.TaskProgress{Name: tr("Error downloading tool %s", toolRelease)})
175+
return &arduino.FailedInstallError{Message: tr("Error installing tool %s", toolRelease), Cause: err}
176+
}
177+
taskCB(&rpc.TaskProgress{Completed: true})
178+
179+
// Install tool
180+
taskCB(&rpc.TaskProgress{Name: tr("Installing tool %s", toolRelease)})
181+
if err := toolResource.Install(pm.DownloadDir, tmp, destDir); err != nil {
182+
taskCB(&rpc.TaskProgress{Name: tr("Error installing tool %s", toolRelease)})
183+
return &arduino.FailedInstallError{Message: tr("Error installing tool %s", toolRelease), Cause: err}
184+
}
185+
taskCB(&rpc.TaskProgress{Completed: true})
186+
return nil
187+
}

Diff for: arduino/errors.go

+37
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,43 @@ func (e *UnknownFQBNError) ToRPCStatus() *status.Status {
193193
return status.New(codes.NotFound, e.Error())
194194
}
195195

196+
// UnknownProfileError is returned when the profile is not found
197+
type UnknownProfileError struct {
198+
Profile string
199+
Cause error
200+
}
201+
202+
func (e *UnknownProfileError) Error() string {
203+
return composeErrorMsg(tr("Profile '%s' not found", e.Profile), e.Cause)
204+
}
205+
206+
func (e *UnknownProfileError) Unwrap() error {
207+
return e.Cause
208+
}
209+
210+
// ToRPCStatus converts the error into a *status.Status
211+
func (e *UnknownProfileError) ToRPCStatus() *status.Status {
212+
return status.New(codes.NotFound, e.Error())
213+
}
214+
215+
// InvalidProfileError is returned when the profile has errors
216+
type InvalidProfileError struct {
217+
Cause error
218+
}
219+
220+
func (e *InvalidProfileError) Error() string {
221+
return composeErrorMsg(tr("Invalid profile"), e.Cause)
222+
}
223+
224+
func (e *InvalidProfileError) Unwrap() error {
225+
return e.Cause
226+
}
227+
228+
// ToRPCStatus converts the error into a *status.Status
229+
func (e *InvalidProfileError) ToRPCStatus() *status.Status {
230+
return status.New(codes.FailedPrecondition, e.Error())
231+
}
232+
196233
// MissingPortAddressError is returned when the port protocol is mandatory and not specified
197234
type MissingPortAddressError struct{}
198235

0 commit comments

Comments
 (0)