From 55b8f6a5baccfcbb88b18ca2c0b58c8100ffb037 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 22 Dec 2023 16:52:04 +0100 Subject: [PATCH 1/6] Factored some library_index.json URL globals --- commands/instances.go | 6 +--- internal/arduino/globals/globals.go | 15 +++++++++ .../libraries/librariesmanager/download.go | 32 ------------------- 3 files changed, 16 insertions(+), 37 deletions(-) delete mode 100644 internal/arduino/libraries/librariesmanager/download.go diff --git a/commands/instances.go b/commands/instances.go index 41463cba257..4303776c395 100644 --- a/commands/instances.go +++ b/commands/instances.go @@ -412,11 +412,7 @@ func UpdateLibrariesIndex(ctx context.Context, req *rpc.UpdateLibrariesIndexRequ } defer tmp.RemoveAll() - indexResource := resources.IndexResource{ - URL: librariesmanager.LibraryIndexWithSignatureArchiveURL, - EnforceSignatureVerification: true, - } - if err := indexResource.Download(lm.IndexFile.Parent(), downloadCB); err != nil { + if err := globals.LibrariesIndexResource.Download(lm.IndexFile.Parent(), downloadCB); err != nil { return err } diff --git a/internal/arduino/globals/globals.go b/internal/arduino/globals/globals.go index 68b5dd577c9..9f91213c492 100644 --- a/internal/arduino/globals/globals.go +++ b/internal/arduino/globals/globals.go @@ -15,6 +15,12 @@ package globals +import ( + "net/url" + + "github.com/arduino/arduino-cli/internal/arduino/resources" +) + var ( // MainFileValidExtension is the extension that must be used for files in new sketches MainFileValidExtension = ".ino" @@ -63,4 +69,13 @@ var ( // DefaultIndexURL is the default index url DefaultIndexURL = "https://downloads.arduino.cc/packages/package_index.tar.bz2" + + // LibrariesIndexURL is the URL where to get the libraries index. + LibrariesIndexURL, _ = url.Parse("https://downloads.arduino.cc/libraries/library_index.tar.bz2") + + // LibrariesIndexResource is the IndexResource to get the libraries index. + LibrariesIndexResource = resources.IndexResource{ + URL: LibrariesIndexURL, + EnforceSignatureVerification: true, + } ) diff --git a/internal/arduino/libraries/librariesmanager/download.go b/internal/arduino/libraries/librariesmanager/download.go deleted file mode 100644 index 254f88a981d..00000000000 --- a/internal/arduino/libraries/librariesmanager/download.go +++ /dev/null @@ -1,32 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package librariesmanager - -import ( - "net/url" -) - -// LibraryIndexURL is the URL where to get the library index. -var LibraryIndexURL, _ = url.Parse("https://downloads.arduino.cc/libraries/library_index.json") - -// LibraryIndexGZURL is the URL where to get the gzipped library index. -var LibraryIndexGZURL, _ = url.Parse("https://downloads.arduino.cc/libraries/library_index.json.gz") - -// LibraryIndexSignature is the URL where to get the library index signature. -var LibraryIndexSignature, _ = url.Parse("https://downloads.arduino.cc/libraries/library_index.json.sig") - -// LibraryIndexWithSignatureArchiveURL is the URL where to get the library index. -var LibraryIndexWithSignatureArchiveURL, _ = url.Parse("https://downloads.arduino.cc/libraries/library_index.tar.bz2") From 9df2fd543d073137839e4e853140d8fcab3d4a2e Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 22 Dec 2023 16:54:53 +0100 Subject: [PATCH 2/6] Removed unused parameter --- commands/lib/uninstall.go | 2 +- commands/lib/utils.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/commands/lib/uninstall.go b/commands/lib/uninstall.go index a7ee8278c0b..28f10153459 100644 --- a/commands/lib/uninstall.go +++ b/commands/lib/uninstall.go @@ -32,7 +32,7 @@ func LibraryUninstall(ctx context.Context, req *rpc.LibraryUninstallRequest, tas return err } - ref, err := createLibIndexReference(lm, req) + ref, err := createLibIndexReference(req) if err != nil { return &cmderrors.InvalidLibraryError{Cause: err} } diff --git a/commands/lib/utils.go b/commands/lib/utils.go index cab8d85ec0c..fbb5d52208c 100644 --- a/commands/lib/utils.go +++ b/commands/lib/utils.go @@ -27,7 +27,7 @@ type libraryReferencer interface { GetName() string } -func createLibIndexReference(lm *librariesmanager.LibrariesManager, req libraryReferencer) (*librariesindex.Reference, error) { +func createLibIndexReference(req libraryReferencer) (*librariesindex.Reference, error) { version, err := commands.ParseVersion(req) if err != nil { return nil, &cmderrors.InvalidVersionError{Cause: err} @@ -37,7 +37,7 @@ func createLibIndexReference(lm *librariesmanager.LibrariesManager, req libraryR } func findLibraryIndexRelease(lm *librariesmanager.LibrariesManager, req libraryReferencer) (*librariesindex.Release, error) { - ref, err := createLibIndexReference(lm, req) + ref, err := createLibIndexReference(req) if err != nil { return nil, err } From e9546fc26796ce08f1e86871e9d46f1c831ae9ac Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 22 Dec 2023 16:57:13 +0100 Subject: [PATCH 3/6] findLibraryIndexRelease requires libraries index instead of libraries manager --- commands/lib/download.go | 2 +- commands/lib/install.go | 2 +- commands/lib/resolve_deps.go | 2 +- commands/lib/utils.go | 5 ++--- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/commands/lib/download.go b/commands/lib/download.go index 47b39c9c1fd..b8019aff488 100644 --- a/commands/lib/download.go +++ b/commands/lib/download.go @@ -42,7 +42,7 @@ func LibraryDownload(ctx context.Context, req *rpc.LibraryDownloadRequest, downl logrus.Info("Preparing download") - lib, err := findLibraryIndexRelease(lm, req) + lib, err := findLibraryIndexRelease(lm.Index, req) if err != nil { return nil, err } diff --git a/commands/lib/install.go b/commands/lib/install.go index d0dcd846c67..4f57741456d 100644 --- a/commands/lib/install.go +++ b/commands/lib/install.go @@ -72,7 +72,7 @@ func LibraryInstall(ctx context.Context, req *rpc.LibraryInstallRequest, downloa // Find the libReleasesToInstall to install libReleasesToInstall := map[*librariesindex.Release]*librariesmanager.LibraryInstallPlan{} for _, lib := range toInstall { - libRelease, err := findLibraryIndexRelease(lm, &rpc.LibraryInstallRequest{ + libRelease, err := findLibraryIndexRelease(lm.Index, &rpc.LibraryInstallRequest{ Name: lib.GetName(), Version: lib.GetVersionRequired(), }) diff --git a/commands/lib/resolve_deps.go b/commands/lib/resolve_deps.go index b1d17bbc42c..b25073424d2 100644 --- a/commands/lib/resolve_deps.go +++ b/commands/lib/resolve_deps.go @@ -36,7 +36,7 @@ func LibraryResolveDependencies(ctx context.Context, req *rpc.LibraryResolveDepe } // Search the requested lib - reqLibRelease, err := findLibraryIndexRelease(lm, req) + reqLibRelease, err := findLibraryIndexRelease(lm.Index, req) if err != nil { return nil, err } diff --git a/commands/lib/utils.go b/commands/lib/utils.go index fbb5d52208c..25e3364cdb1 100644 --- a/commands/lib/utils.go +++ b/commands/lib/utils.go @@ -19,7 +19,6 @@ import ( "github.com/arduino/arduino-cli/commands" "github.com/arduino/arduino-cli/commands/cmderrors" "github.com/arduino/arduino-cli/internal/arduino/libraries/librariesindex" - "github.com/arduino/arduino-cli/internal/arduino/libraries/librariesmanager" ) type libraryReferencer interface { @@ -36,12 +35,12 @@ func createLibIndexReference(req libraryReferencer) (*librariesindex.Reference, return &librariesindex.Reference{Name: req.GetName(), Version: version}, nil } -func findLibraryIndexRelease(lm *librariesmanager.LibrariesManager, req libraryReferencer) (*librariesindex.Release, error) { +func findLibraryIndexRelease(li *librariesindex.Index, req libraryReferencer) (*librariesindex.Release, error) { ref, err := createLibIndexReference(req) if err != nil { return nil, err } - lib := lm.Index.FindRelease(ref) + lib := li.FindRelease(ref) if lib == nil { return nil, &cmderrors.LibraryNotFoundError{Library: ref.String()} } From debf8e4d64d201ebbbd7590e762a28705af39f5d Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 22 Dec 2023 17:20:59 +0100 Subject: [PATCH 4/6] LibrariesManager.AddLibrariesDir now accepts directly a LibraryDir struct This is preparatory for the next commit --- commands/instances.go | 21 +++++++++--- .../builder/internal/detector/detector.go | 22 ++++++++++--- .../librariesmanager/librariesmanager.go | 33 ++++--------------- 3 files changed, 42 insertions(+), 34 deletions(-) diff --git a/commands/instances.go b/commands/instances.go index 4303776c395..c06d81e93eb 100644 --- a/commands/instances.go +++ b/commands/instances.go @@ -311,7 +311,11 @@ func Init(req *rpc.InitRequest, responseCallback func(r *rpc.InitResponse)) erro for _, pack := range pme.GetPackages() { for _, platform := range pack.Platforms { if platformRelease := pme.GetInstalledPlatformRelease(platform); platformRelease != nil { - lm.AddPlatformReleaseLibrariesDir(platformRelease, libraries.PlatformBuiltIn) + lm.AddLibrariesDir(&librariesmanager.LibrariesDir{ + PlatformRelease: platformRelease, + Path: platformRelease.GetLibrariesDir(), + Location: libraries.PlatformBuiltIn, + }) } } } @@ -324,11 +328,17 @@ func Init(req *rpc.InitRequest, responseCallback func(r *rpc.InitResponse)) erro if profile == nil { // Add directories of libraries bundled with IDE if bundledLibsDir := configuration.IDEBuiltinLibrariesDir(configuration.Settings); bundledLibsDir != nil { - lm.AddLibrariesDir(bundledLibsDir, libraries.IDEBuiltIn) + lm.AddLibrariesDir(&librariesmanager.LibrariesDir{ + Path: bundledLibsDir, + Location: libraries.IDEBuiltIn, + }) } // Add libraries directory from config file - lm.AddLibrariesDir(configuration.LibrariesDir(configuration.Settings), libraries.User) + lm.AddLibrariesDir(&librariesmanager.LibrariesDir{ + Path: configuration.LibrariesDir(configuration.Settings), + Location: libraries.User, + }) } else { // Load libraries required for profile for _, libraryRef := range profile.Libraries { @@ -368,7 +378,10 @@ func Init(req *rpc.InitRequest, responseCallback func(r *rpc.InitResponse)) erro taskCallback(&rpc.TaskProgress{Completed: true}) } - lm.AddLibrariesDir(libRoot, libraries.User) + lm.AddLibrariesDir(&librariesmanager.LibrariesDir{ + Path: libRoot, + Location: libraries.User, + }) } } diff --git a/internal/arduino/builder/internal/detector/detector.go b/internal/arduino/builder/internal/detector/detector.go index a8cf2d3c077..ae7fdd59696 100644 --- a/internal/arduino/builder/internal/detector/detector.go +++ b/internal/arduino/builder/internal/detector/detector.go @@ -608,20 +608,34 @@ func LibrariesLoader( if err := builtInLibrariesFolders.ToAbs(); err != nil { return nil, nil, nil, err } - lm.AddLibrariesDir(builtInLibrariesFolders, libraries.IDEBuiltIn) + lm.AddLibrariesDir(&librariesmanager.LibrariesDir{ + Path: builtInLibrariesFolders, + Location: libraries.IDEBuiltIn, + }) } if actualPlatform != targetPlatform { - lm.AddPlatformReleaseLibrariesDir(actualPlatform, libraries.ReferencedPlatformBuiltIn) + lm.AddLibrariesDir(&librariesmanager.LibrariesDir{ + PlatformRelease: actualPlatform, + Path: actualPlatform.GetLibrariesDir(), + Location: libraries.ReferencedPlatformBuiltIn, + }) } - lm.AddPlatformReleaseLibrariesDir(targetPlatform, libraries.PlatformBuiltIn) + lm.AddLibrariesDir(&librariesmanager.LibrariesDir{ + PlatformRelease: targetPlatform, + Path: targetPlatform.GetLibrariesDir(), + Location: libraries.PlatformBuiltIn, + }) librariesFolders := otherLibrariesDirs if err := librariesFolders.ToAbs(); err != nil { return nil, nil, nil, err } for _, folder := range librariesFolders { - lm.AddLibrariesDir(folder, libraries.User) + lm.AddLibrariesDir(&librariesmanager.LibrariesDir{ + Path: folder, + Location: libraries.User, // XXX: Should be libraries.Unmanaged? + }) } for _, status := range lm.RescanLibraries() { diff --git a/internal/arduino/libraries/librariesmanager/librariesmanager.go b/internal/arduino/libraries/librariesmanager/librariesmanager.go index a73a6025c72..df0dc874376 100644 --- a/internal/arduino/libraries/librariesmanager/librariesmanager.go +++ b/internal/arduino/libraries/librariesmanager/librariesmanager.go @@ -102,38 +102,19 @@ func (lm *LibrariesManager) LoadIndex() error { // AddLibrariesDir adds path to the list of directories // to scan when searching for libraries. If a path is already // in the list it is ignored. -func (lm *LibrariesManager) AddLibrariesDir(path *paths.Path, location libraries.LibraryLocation) { - for _, dir := range lm.LibrariesDir { - if dir.Path.EquivalentTo(path) { - return - } - } - logrus.WithField("dir", path).WithField("location", location.String()).Info("Adding libraries dir") - lm.LibrariesDir = append(lm.LibrariesDir, &LibrariesDir{ - Path: path, - Location: location, - }) -} - -// AddPlatformReleaseLibrariesDir add the libraries directory in the -// specified PlatformRelease to the list of directories to scan when -// searching for libraries. -func (lm *LibrariesManager) AddPlatformReleaseLibrariesDir(plaftormRelease *cores.PlatformRelease, location libraries.LibraryLocation) { - path := plaftormRelease.GetLibrariesDir() - if path == nil { +func (lm *LibrariesManager) AddLibrariesDir(libDir *LibrariesDir) { + if libDir.Path == nil { return } for _, dir := range lm.LibrariesDir { - if dir.Path.EquivalentTo(path) { + if dir.Path.EquivalentTo(libDir.Path) { return } } - logrus.WithField("dir", path).WithField("location", location.String()).Info("Adding libraries dir") - lm.LibrariesDir = append(lm.LibrariesDir, &LibrariesDir{ - Path: path, - Location: location, - PlatformRelease: plaftormRelease, - }) + logrus.WithField("dir", libDir.Path). + WithField("location", libDir.Location.String()). + Info("Adding libraries dir") + lm.LibrariesDir = append(lm.LibrariesDir, libDir) } // RescanLibraries reload all installed libraries in the system. From 17cbf4f596e342cbcfaf887b8ee329ce3b657f54 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 22 Dec 2023 17:22:44 +0100 Subject: [PATCH 5/6] AddLibrariesDir now can handle also single-lib directories --- .../librariesmanager/librariesmanager.go | 35 ++++++++++++------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/internal/arduino/libraries/librariesmanager/librariesmanager.go b/internal/arduino/libraries/librariesmanager/librariesmanager.go index df0dc874376..9f0bd168ec6 100644 --- a/internal/arduino/libraries/librariesmanager/librariesmanager.go +++ b/internal/arduino/libraries/librariesmanager/librariesmanager.go @@ -49,6 +49,7 @@ type LibrariesDir struct { Path *paths.Path Location libraries.LibraryLocation PlatformRelease *cores.PlatformRelease + IsSingleLibrary bool // true if Path points directly to a library instad of a dir of libraries } var tr = i18n.Tr @@ -113,6 +114,7 @@ func (lm *LibrariesManager) AddLibrariesDir(libDir *LibrariesDir) { } logrus.WithField("dir", libDir.Path). WithField("location", libDir.Location.String()). + WithField("isSingleLibrary", libDir.IsSingleLibrary). Info("Adding libraries dir") lm.LibrariesDir = append(lm.LibrariesDir, libDir) } @@ -122,7 +124,7 @@ func (lm *LibrariesManager) RescanLibraries() []*status.Status { lm.clearLibraries() statuses := []*status.Status{} for _, dir := range lm.LibrariesDir { - if errs := lm.LoadLibrariesFromDir(dir); len(errs) > 0 { + if errs := lm.loadLibrariesFromDir(dir); len(errs) > 0 { statuses = append(statuses, errs...) } } @@ -145,22 +147,29 @@ func (lm *LibrariesManager) getLibrariesDir(installLocation libraries.LibraryLoc } } -// LoadLibrariesFromDir loads all libraries in the given directory. Returns +// loadLibrariesFromDir loads all libraries in the given directory. Returns // nil if the directory doesn't exists. -func (lm *LibrariesManager) LoadLibrariesFromDir(librariesDir *LibrariesDir) []*status.Status { +func (lm *LibrariesManager) loadLibrariesFromDir(librariesDir *LibrariesDir) []*status.Status { statuses := []*status.Status{} - subDirs, err := librariesDir.Path.ReadDir() - if os.IsNotExist(err) { - return statuses - } - if err != nil { - s := status.Newf(codes.FailedPrecondition, tr("reading dir %[1]s: %[2]s"), librariesDir.Path, err) - return append(statuses, s) + + var libDirs paths.PathList + if librariesDir.IsSingleLibrary { + libDirs.Add(librariesDir.Path) + } else { + d, err := librariesDir.Path.ReadDir() + if os.IsNotExist(err) { + return statuses + } + if err != nil { + s := status.Newf(codes.FailedPrecondition, tr("reading dir %[1]s: %[2]s"), librariesDir.Path, err) + return append(statuses, s) + } + d.FilterDirs() + d.FilterOutHiddenFiles() + libDirs = d } - subDirs.FilterDirs() - subDirs.FilterOutHiddenFiles() - for _, subDir := range subDirs { + for _, subDir := range libDirs { library, err := libraries.Load(subDir, librariesDir.Location) if err != nil { s := status.Newf(codes.Internal, tr("loading library from %[1]s: %[2]s"), subDir, err) From 2728183d75746c5fbeeb9ec60b93ef082dfa4fb2 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 22 Dec 2023 17:25:12 +0100 Subject: [PATCH 6/6] Removed weird hack in the library detector Previously the libraries detector had to load the single librares provided via '--library' flag after a Rescan, because the library manager wasn't able to handle a single folder library. Now it can so we removed all the tricky machinery. --- .../builder/internal/detector/detector.go | 15 ++++++++------- .../librariesmanager/librariesmanager.go | 19 ------------------- 2 files changed, 8 insertions(+), 26 deletions(-) diff --git a/internal/arduino/builder/internal/detector/detector.go b/internal/arduino/builder/internal/detector/detector.go index ae7fdd59696..efc0bd9b970 100644 --- a/internal/arduino/builder/internal/detector/detector.go +++ b/internal/arduino/builder/internal/detector/detector.go @@ -638,6 +638,14 @@ func LibrariesLoader( }) } + for _, dir := range libraryDirs { + lm.AddLibrariesDir(&librariesmanager.LibrariesDir{ + Path: dir, + Location: libraries.Unmanaged, + IsSingleLibrary: true, + }) + } + for _, status := range lm.RescanLibraries() { // With the refactoring of the initialization step of the CLI we changed how // errors are returned when loading platforms and libraries, that meant returning a list of @@ -647,13 +655,6 @@ func LibrariesLoader( // When we're gonna refactor the legacy package this will be gone. verboseOut.Write([]byte(status.Message())) } - - for _, dir := range libraryDirs { - // Libraries specified this way have top priority - if err := lm.LoadLibraryFromDir(dir, libraries.Unmanaged); err != nil { - return nil, nil, nil, err - } - } } resolver := librariesresolver.NewCppResolver() diff --git a/internal/arduino/libraries/librariesmanager/librariesmanager.go b/internal/arduino/libraries/librariesmanager/librariesmanager.go index 9f0bd168ec6..db7f3b2f586 100644 --- a/internal/arduino/libraries/librariesmanager/librariesmanager.go +++ b/internal/arduino/libraries/librariesmanager/librariesmanager.go @@ -185,25 +185,6 @@ func (lm *LibrariesManager) loadLibrariesFromDir(librariesDir *LibrariesDir) []* return statuses } -// LoadLibraryFromDir loads one single library from the libRootDir. -// libRootDir must point to the root of a valid library. -// An error is returned if the path doesn't exist or loading of the library fails. -func (lm *LibrariesManager) LoadLibraryFromDir(libRootDir *paths.Path, location libraries.LibraryLocation) error { - if libRootDir.NotExist() { - return fmt.Errorf(tr("library path does not exist: %s"), libRootDir) - } - - library, err := libraries.Load(libRootDir, location) - if err != nil { - return fmt.Errorf(tr("loading library from %[1]s: %[2]s"), libRootDir, err) - } - - alternatives := lm.Libraries[library.Name] - alternatives.Add(library) - lm.Libraries[library.Name] = alternatives - return nil -} - // FindByReference return the installed libraries matching the Reference // name and version or, if the version is nil, the libraries installed // in the installLocation.