From aa53e09522afc152c5e120218673d85401161da4 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Wed, 27 Dec 2023 16:52:04 +0100 Subject: [PATCH 01/14] Removed libraries index fields from LibraryManager The presence of the Index* fileds inside LibraryManager didn't give any functional benefit. --- commands/instances.go | 22 +++++++++--- commands/internal/instances/instances.go | 35 ++++++++++++++++--- commands/lib/search.go | 9 +++-- commands/lib/search_test.go | 34 +++++++++--------- .../builder/internal/detector/detector.go | 4 +-- .../librariesmanager/librariesmanager.go | 35 ++++--------------- .../librariesmanager/librariesmanager_test.go | 2 +- 7 files changed, 79 insertions(+), 62 deletions(-) diff --git a/commands/instances.go b/commands/instances.go index c06d81e93eb..9a0448449ac 100644 --- a/commands/instances.go +++ b/commands/instances.go @@ -302,7 +302,6 @@ func Init(req *rpc.InitRequest, responseCallback func(r *rpc.InitResponse)) erro // Create library manager and add libraries directories lm := librariesmanager.NewLibraryManager( - pme.IndexDir, pme.DownloadDir, ) _ = instances.SetLibraryManager(instance, lm) // should never fail @@ -320,10 +319,21 @@ func Init(req *rpc.InitRequest, responseCallback func(r *rpc.InitResponse)) erro } } - if err := lm.LoadIndex(); err != nil { + indexFileName, err := globals.LibrariesIndexResource.IndexFileName() + if err != nil { + // should never happen + panic("failed getting libraries index file name: " + err.Error()) + } + indexFile := pme.IndexDir.Join(indexFileName) + + logrus.WithField("index", indexFile).Info("Loading libraries index file") + li, err := librariesindex.LoadIndex(indexFile) + if err != nil { s := status.Newf(codes.FailedPrecondition, tr("Loading index file: %v"), err) responseError(s) + li = librariesindex.EmptyIndex } + instances.SetLibrariesIndex(instance, li) if profile == nil { // Add directories of libraries bundled with IDE @@ -409,12 +419,14 @@ func Destroy(ctx context.Context, req *rpc.DestroyRequest) (*rpc.DestroyResponse // UpdateLibrariesIndex updates the library_index.json func UpdateLibrariesIndex(ctx context.Context, req *rpc.UpdateLibrariesIndexRequest, downloadCB rpc.DownloadProgressCB) error { logrus.Info("Updating libraries index") - lm, err := instances.GetLibraryManager(req.GetInstance()) + pme, release, err := instances.GetPackageManagerExplorer(req.GetInstance()) if err != nil { return err } + indexDir := pme.IndexDir + release() - if err := lm.IndexFile.Parent().MkdirAll(); err != nil { + if err := indexDir.MkdirAll(); err != nil { return &cmderrors.PermissionDeniedError{Message: tr("Could not create index directory"), Cause: err} } @@ -425,7 +437,7 @@ func UpdateLibrariesIndex(ctx context.Context, req *rpc.UpdateLibrariesIndexRequ } defer tmp.RemoveAll() - if err := globals.LibrariesIndexResource.Download(lm.IndexFile.Parent(), downloadCB); err != nil { + if err := globals.LibrariesIndexResource.Download(indexDir, downloadCB); err != nil { return err } diff --git a/commands/internal/instances/instances.go b/commands/internal/instances/instances.go index 0621047f4bd..43233585bed 100644 --- a/commands/internal/instances/instances.go +++ b/commands/internal/instances/instances.go @@ -5,6 +5,7 @@ import ( "github.com/arduino/arduino-cli/commands/cmderrors" "github.com/arduino/arduino-cli/internal/arduino/cores/packagemanager" + "github.com/arduino/arduino-cli/internal/arduino/libraries/librariesindex" "github.com/arduino/arduino-cli/internal/arduino/libraries/librariesmanager" rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" "github.com/arduino/arduino-cli/version" @@ -17,6 +18,7 @@ import ( type coreInstance struct { pm *packagemanager.PackageManager lm *librariesmanager.LibrariesManager + li *librariesindex.Index } // instances contains all the running Arduino Core Services instances @@ -60,6 +62,29 @@ func GetLibraryManager(inst *rpc.Instance) (*librariesmanager.LibrariesManager, return i.lm, nil } +// GetLibrariesIndex returns the library index for the given instance. +func GetLibrariesIndex(inst *rpc.Instance) (*librariesindex.Index, error) { + instancesMux.Lock() + defer instancesMux.Unlock() + i := instances[inst.GetId()] + if i == nil { + return nil, &cmderrors.InvalidInstanceError{} + } + return i.li, nil +} + +// SetLibrariesIndex sets the library index for the given instance. +func SetLibrariesIndex(inst *rpc.Instance, li *librariesindex.Index) error { + instancesMux.Lock() + defer instancesMux.Unlock() + i := instances[inst.GetId()] + if i == nil { + return &cmderrors.InvalidInstanceError{} + } + i.li = li + return nil +} + // SetLibraryManager sets the library manager for the given instance. func SetLibraryManager(inst *rpc.Instance, lm *librariesmanager.LibrariesManager) bool { instancesMux.Lock() @@ -74,16 +99,18 @@ func SetLibraryManager(inst *rpc.Instance, lm *librariesmanager.LibrariesManager // Create a new *rpc.Instance ready to be initialized func Create(dataDir, packagesDir, downloadsDir *paths.Path, extraUserAgent ...string) (*rpc.Instance, error) { - instance := &coreInstance{} - // Create package manager userAgent := "arduino-cli/" + version.VersionInfo.VersionString for _, ua := range extraUserAgent { userAgent += " " + ua } tempDir := dataDir.Join("tmp") - instance.pm = packagemanager.NewBuilder(dataDir, packagesDir, downloadsDir, tempDir, userAgent).Build() - instance.lm = librariesmanager.NewLibraryManager(dataDir, downloadsDir) + + instance := &coreInstance{ + pm: packagemanager.NewBuilder(dataDir, packagesDir, downloadsDir, tempDir, userAgent).Build(), + lm: librariesmanager.NewLibraryManager(downloadsDir), + li: librariesindex.EmptyIndex, + } // Save instance instancesMux.Lock() diff --git a/commands/lib/search.go b/commands/lib/search.go index cbd9bf4901a..7c2d21cdd70 100644 --- a/commands/lib/search.go +++ b/commands/lib/search.go @@ -22,26 +22,25 @@ import ( "github.com/arduino/arduino-cli/commands/internal/instances" "github.com/arduino/arduino-cli/internal/arduino/libraries/librariesindex" - "github.com/arduino/arduino-cli/internal/arduino/libraries/librariesmanager" rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" semver "go.bug.st/relaxed-semver" ) // LibrarySearch FIXMEDOC func LibrarySearch(ctx context.Context, req *rpc.LibrarySearchRequest) (*rpc.LibrarySearchResponse, error) { - lm, err := instances.GetLibraryManager(req.GetInstance()) + li, err := instances.GetLibrariesIndex(req.GetInstance()) if err != nil { return nil, err } - return searchLibrary(req, lm), nil + return searchLibrary(req, li), nil } -func searchLibrary(req *rpc.LibrarySearchRequest, lm *librariesmanager.LibrariesManager) *rpc.LibrarySearchResponse { +func searchLibrary(req *rpc.LibrarySearchRequest, li *librariesindex.Index) *rpc.LibrarySearchResponse { res := []*rpc.SearchedLibrary{} query := req.GetSearchArgs() matcher := MatcherFromQueryString(query) - for _, lib := range lm.Index.Libraries { + for _, lib := range li.Libraries { if matcher(lib) { res = append(res, indexLibraryToRPCSearchLibrary(lib, req.GetOmitReleasesDetails())) } diff --git a/commands/lib/search_test.go b/commands/lib/search_test.go index e1e0b3157f2..faae69ef1a9 100644 --- a/commands/lib/search_test.go +++ b/commands/lib/search_test.go @@ -19,22 +19,24 @@ import ( "strings" "testing" - "github.com/arduino/arduino-cli/internal/arduino/libraries/librariesmanager" + "github.com/arduino/arduino-cli/internal/arduino/globals" + "github.com/arduino/arduino-cli/internal/arduino/libraries/librariesindex" rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" paths "github.com/arduino/go-paths-helper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -var customIndexPath = paths.New("testdata", "test1") -var fullIndexPath = paths.New("testdata", "full") -var qualifiedSearchIndexPath = paths.New("testdata", "qualified_search") +var indexFilename, _ = globals.LibrariesIndexResource.IndexFileName() +var customIndexPath = paths.New("testdata", "test1", indexFilename) +var fullIndexPath = paths.New("testdata", "full", indexFilename) +var qualifiedSearchIndexPath = paths.New("testdata", "qualified_search", indexFilename) func TestSearchLibrary(t *testing.T) { - lm := librariesmanager.NewLibraryManager(customIndexPath, nil) - lm.LoadIndex() + li, err := librariesindex.LoadIndex(customIndexPath) + require.NoError(t, err) - resp := searchLibrary(&rpc.LibrarySearchRequest{SearchArgs: "test"}, lm) + resp := searchLibrary(&rpc.LibrarySearchRequest{SearchArgs: "test"}, li) assert := assert.New(t) assert.Equal(resp.GetStatus(), rpc.LibrarySearchStatus_LIBRARY_SEARCH_STATUS_SUCCESS) assert.Equal(len(resp.GetLibraries()), 2) @@ -43,10 +45,10 @@ func TestSearchLibrary(t *testing.T) { } func TestSearchLibrarySimilar(t *testing.T) { - lm := librariesmanager.NewLibraryManager(customIndexPath, nil) - lm.LoadIndex() + li, err := librariesindex.LoadIndex(customIndexPath) + require.NoError(t, err) - resp := searchLibrary(&rpc.LibrarySearchRequest{SearchArgs: "arduino"}, lm) + resp := searchLibrary(&rpc.LibrarySearchRequest{SearchArgs: "arduino"}, li) assert := assert.New(t) assert.Equal(resp.GetStatus(), rpc.LibrarySearchStatus_LIBRARY_SEARCH_STATUS_SUCCESS) assert.Equal(len(resp.GetLibraries()), 2) @@ -59,12 +61,12 @@ func TestSearchLibrarySimilar(t *testing.T) { } func TestSearchLibraryFields(t *testing.T) { - lm := librariesmanager.NewLibraryManager(fullIndexPath, nil) - lm.LoadIndex() + li, err := librariesindex.LoadIndex(fullIndexPath) + require.NoError(t, err) query := func(q string) []string { libs := []string{} - for _, lib := range searchLibrary(&rpc.LibrarySearchRequest{SearchArgs: q}, lm).GetLibraries() { + for _, lib := range searchLibrary(&rpc.LibrarySearchRequest{SearchArgs: q}, li).GetLibraries() { libs = append(libs, lib.GetName()) } return libs @@ -97,12 +99,12 @@ func TestSearchLibraryFields(t *testing.T) { } func TestSearchLibraryWithQualifiers(t *testing.T) { - lm := librariesmanager.NewLibraryManager(qualifiedSearchIndexPath, nil) - lm.LoadIndex() + li, err := librariesindex.LoadIndex(qualifiedSearchIndexPath) + require.NoError(t, err) query := func(q string) []string { libs := []string{} - for _, lib := range searchLibrary(&rpc.LibrarySearchRequest{SearchArgs: q}, lm).GetLibraries() { + for _, lib := range searchLibrary(&rpc.LibrarySearchRequest{SearchArgs: q}, li).GetLibraries() { libs = append(libs, lib.GetName()) } return libs diff --git a/internal/arduino/builder/internal/detector/detector.go b/internal/arduino/builder/internal/detector/detector.go index efc0bd9b970..24d131bad15 100644 --- a/internal/arduino/builder/internal/detector/detector.go +++ b/internal/arduino/builder/internal/detector/detector.go @@ -598,10 +598,10 @@ func LibrariesLoader( if useCachedLibrariesResolution { // Since we are using the cached libraries resolution // the library manager is not needed. - lm = librariesmanager.NewLibraryManager(nil, nil) + lm = librariesmanager.NewLibraryManager(nil) } if librariesManager == nil { - lm = librariesmanager.NewLibraryManager(nil, nil) + lm = librariesmanager.NewLibraryManager(nil) builtInLibrariesFolders := builtInLibrariesDirs if builtInLibrariesFolders != nil { diff --git a/internal/arduino/libraries/librariesmanager/librariesmanager.go b/internal/arduino/libraries/librariesmanager/librariesmanager.go index db7f3b2f586..d47ae18550b 100644 --- a/internal/arduino/libraries/librariesmanager/librariesmanager.go +++ b/internal/arduino/libraries/librariesmanager/librariesmanager.go @@ -37,11 +37,8 @@ import ( type LibrariesManager struct { LibrariesDir []*LibrariesDir Libraries map[string]libraries.List `json:"libraries"` - - Index *librariesindex.Index - IndexFile *paths.Path - IndexFileSignature *paths.Path - DownloadsDir *paths.Path + // TODO: Fix all access to lm.Index left around + DownloadsDir *paths.Path } // LibrariesDir is a directory containing libraries @@ -72,32 +69,12 @@ func (lm LibrariesManager) Names() []string { } // NewLibraryManager creates a new library manager -func NewLibraryManager(indexDir *paths.Path, downloadsDir *paths.Path) *LibrariesManager { - var indexFile, indexFileSignature *paths.Path - if indexDir != nil { - indexFile = indexDir.Join("library_index.json") - indexFileSignature = indexDir.Join("library_index.json.sig") - } +func NewLibraryManager(downloadsDir *paths.Path) *LibrariesManager { return &LibrariesManager{ - Libraries: map[string]libraries.List{}, - IndexFile: indexFile, - IndexFileSignature: indexFileSignature, - DownloadsDir: downloadsDir, - Index: librariesindex.EmptyIndex, - } -} - -// LoadIndex reads a library_index.json from a file and returns -// the corresponding Index structure. -func (lm *LibrariesManager) LoadIndex() error { - logrus.WithField("index", lm.IndexFile).Info("Loading libraries index file") - index, err := librariesindex.LoadIndex(lm.IndexFile) - if err != nil { - lm.Index = librariesindex.EmptyIndex - return err + Libraries: map[string]libraries.List{}, + DownloadsDir: downloadsDir, + Index: librariesindex.EmptyIndex, } - lm.Index = index - return nil } // AddLibrariesDir adds path to the list of directories diff --git a/internal/arduino/libraries/librariesmanager/librariesmanager_test.go b/internal/arduino/libraries/librariesmanager/librariesmanager_test.go index 54475cd1fd3..22a3a1ce3f4 100644 --- a/internal/arduino/libraries/librariesmanager/librariesmanager_test.go +++ b/internal/arduino/libraries/librariesmanager/librariesmanager_test.go @@ -24,7 +24,7 @@ import ( func Test_RescanLibrariesCallClear(t *testing.T) { baseDir := paths.New(t.TempDir()) - lm := NewLibraryManager(baseDir.Join("index_dir"), baseDir.Join("downloads_dir")) + lm := NewLibraryManager(baseDir.Join("downloads_dir")) lm.Libraries["testLibA"] = libraries.List{} lm.Libraries["testLibB"] = libraries.List{} lm.RescanLibraries() From f3b52c60ec25bf407ec2bb757d9a2bbea344f6a9 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Wed, 27 Dec 2023 17:22:37 +0100 Subject: [PATCH 02/14] Removed all references to LibraryManger.Index --- commands/instances.go | 2 +- commands/lib/download.go | 7 ++++++- commands/lib/install.go | 7 ++++++- commands/lib/list.go | 13 +++++++++---- commands/lib/resolve_deps.go | 15 ++++++++++----- commands/lib/upgrade.go | 14 ++++++++++++-- .../librariesmanager/librariesmanager.go | 2 -- 7 files changed, 44 insertions(+), 16 deletions(-) diff --git a/commands/instances.go b/commands/instances.go index 9a0448449ac..97cd38e8b63 100644 --- a/commands/instances.go +++ b/commands/instances.go @@ -359,7 +359,7 @@ func Init(req *rpc.InitRequest, responseCallback func(r *rpc.InitResponse)) erro if !libDir.IsDir() { // Download library taskCallback(&rpc.TaskProgress{Name: tr("Downloading library %s", libraryRef)}) - libRelease := lm.Index.FindRelease(&librariesindex.Reference{ + libRelease := li.FindRelease(&librariesindex.Reference{ Name: libraryRef.Library, Version: libraryRef.Version, }) diff --git a/commands/lib/download.go b/commands/lib/download.go index b8019aff488..3d8162e127b 100644 --- a/commands/lib/download.go +++ b/commands/lib/download.go @@ -40,9 +40,14 @@ func LibraryDownload(ctx context.Context, req *rpc.LibraryDownloadRequest, downl return nil, err } + li, err := instances.GetLibrariesIndex(req.GetInstance()) + if err != nil { + return nil, err + } + logrus.Info("Preparing download") - lib, err := findLibraryIndexRelease(lm.Index, req) + lib, err := findLibraryIndexRelease(li, req) if err != nil { return nil, err } diff --git a/commands/lib/install.go b/commands/lib/install.go index 4f57741456d..d4b6dd8d55f 100644 --- a/commands/lib/install.go +++ b/commands/lib/install.go @@ -33,6 +33,11 @@ import ( // LibraryInstall resolves the library dependencies, then downloads and installs the libraries into the install location. func LibraryInstall(ctx context.Context, req *rpc.LibraryInstallRequest, downloadCB rpc.DownloadProgressCB, taskCB rpc.TaskProgressCB) error { + li, err := instances.GetLibrariesIndex(req.GetInstance()) + if err != nil { + return err + } + lm, err := instances.GetLibraryManager(req.GetInstance()) if err != nil { return err @@ -72,7 +77,7 @@ func LibraryInstall(ctx context.Context, req *rpc.LibraryInstallRequest, downloa // Find the libReleasesToInstall to install libReleasesToInstall := map[*librariesindex.Release]*librariesmanager.LibraryInstallPlan{} for _, lib := range toInstall { - libRelease, err := findLibraryIndexRelease(lm.Index, &rpc.LibraryInstallRequest{ + libRelease, err := findLibraryIndexRelease(li, &rpc.LibraryInstallRequest{ Name: lib.GetName(), Version: lib.GetVersionRequired(), }) diff --git a/commands/lib/list.go b/commands/lib/list.go index 0f2f7ec1339..490ee575ed2 100644 --- a/commands/lib/list.go +++ b/commands/lib/list.go @@ -42,6 +42,11 @@ func LibraryList(ctx context.Context, req *rpc.LibraryListRequest) (*rpc.Library } defer release() + li, err := instances.GetLibrariesIndex(req.GetInstance()) + if err != nil { + return nil, err + } + lm, err := instances.GetLibraryManager(req.GetInstance()) if err != nil { return nil, err @@ -51,7 +56,7 @@ func LibraryList(ctx context.Context, req *rpc.LibraryListRequest) (*rpc.Library var allLibs []*installedLib if fqbnString := req.GetFqbn(); fqbnString != "" { - allLibs = listLibraries(lm, req.GetUpdatable(), true) + allLibs = listLibraries(lm, li, req.GetUpdatable(), true) fqbn, err := cores.ParseFQBN(req.GetFqbn()) if err != nil { return nil, &cmderrors.InvalidFQBNError{Cause: err} @@ -91,7 +96,7 @@ func LibraryList(ctx context.Context, req *rpc.LibraryListRequest) (*rpc.Library allLibs = append(allLibs, lib) } } else { - allLibs = listLibraries(lm, req.GetUpdatable(), req.GetAll()) + allLibs = listLibraries(lm, li, req.GetUpdatable(), req.GetAll()) } installedLibs := []*rpc.InstalledLibrary{} @@ -118,7 +123,7 @@ func LibraryList(ctx context.Context, req *rpc.LibraryListRequest) (*rpc.Library // listLibraries returns the list of installed libraries. If updatable is true it // returns only the libraries that may be updated. -func listLibraries(lm *librariesmanager.LibrariesManager, updatable bool, all bool) []*installedLib { +func listLibraries(lm *librariesmanager.LibrariesManager, li *librariesindex.Index, updatable bool, all bool) []*installedLib { res := []*installedLib{} for _, libAlternatives := range lm.Libraries { for _, lib := range libAlternatives { @@ -127,7 +132,7 @@ func listLibraries(lm *librariesmanager.LibrariesManager, updatable bool, all bo continue } } - available := lm.Index.FindLibraryUpdate(lib) + available := li.FindLibraryUpdate(lib) if updatable && available == nil { continue } diff --git a/commands/lib/resolve_deps.go b/commands/lib/resolve_deps.go index b25073424d2..f721da92e50 100644 --- a/commands/lib/resolve_deps.go +++ b/commands/lib/resolve_deps.go @@ -36,14 +36,19 @@ func LibraryResolveDependencies(ctx context.Context, req *rpc.LibraryResolveDepe } // Search the requested lib - reqLibRelease, err := findLibraryIndexRelease(lm.Index, req) + li, err := instances.GetLibrariesIndex(req.GetInstance()) + if err != nil { + return nil, err + } + + reqLibRelease, err := findLibraryIndexRelease(li, req) if err != nil { return nil, err } // Extract all installed libraries installedLibs := map[string]*libraries.Library{} - for _, lib := range listLibraries(lm, false, false) { + for _, lib := range listLibraries(lm, li, false, false) { installedLibs[lib.Library.Name] = lib.Library } @@ -53,7 +58,7 @@ func LibraryResolveDependencies(ctx context.Context, req *rpc.LibraryResolveDepe libs := lm.FindAllInstalled() libs = libs.FilterByVersionAndInstallLocation(nil, libraries.User) for _, lib := range libs { - release := lm.Index.FindRelease(&librariesindex.Reference{ + release := li.FindRelease(&librariesindex.Reference{ Name: lib.Name, Version: lib.Version, }) @@ -62,13 +67,13 @@ func LibraryResolveDependencies(ctx context.Context, req *rpc.LibraryResolveDepe } } } - deps := lm.Index.ResolveDependencies(reqLibRelease, overrides) + deps := li.ResolveDependencies(reqLibRelease, overrides) // If no solution has been found if len(deps) == 0 { // Check if there is a problem with the first level deps for _, directDep := range reqLibRelease.GetDependencies() { - if _, ok := lm.Index.Libraries[directDep.GetName()]; !ok { + if _, ok := li.Libraries[directDep.GetName()]; !ok { err := errors.New(tr("dependency '%s' is not available", directDep.GetName())) return nil, &cmderrors.LibraryDependenciesResolutionFailedError{Cause: err} } diff --git a/commands/lib/upgrade.go b/commands/lib/upgrade.go index 824d645d648..a5b8150e538 100644 --- a/commands/lib/upgrade.go +++ b/commands/lib/upgrade.go @@ -26,12 +26,17 @@ import ( // LibraryUpgradeAll upgrades all the available libraries func LibraryUpgradeAll(req *rpc.LibraryUpgradeAllRequest, downloadCB rpc.DownloadProgressCB, taskCB rpc.TaskProgressCB) error { + li, err := instances.GetLibrariesIndex(req.GetInstance()) + if err != nil { + return err + } + lm, err := instances.GetLibraryManager(req.GetInstance()) if err != nil { return err } - if err := upgrade(req.GetInstance(), listLibraries(lm, true, false), downloadCB, taskCB); err != nil { + if err := upgrade(req.GetInstance(), listLibraries(lm, li, true, false), downloadCB, taskCB); err != nil { return err } @@ -44,6 +49,11 @@ func LibraryUpgradeAll(req *rpc.LibraryUpgradeAllRequest, downloadCB rpc.Downloa // LibraryUpgrade upgrades a library func LibraryUpgrade(ctx context.Context, req *rpc.LibraryUpgradeRequest, downloadCB rpc.DownloadProgressCB, taskCB rpc.TaskProgressCB) error { + li, err := instances.GetLibrariesIndex(req.GetInstance()) + if err != nil { + return err + } + lm, err := instances.GetLibraryManager(req.GetInstance()) if err != nil { return err @@ -51,7 +61,7 @@ func LibraryUpgrade(ctx context.Context, req *rpc.LibraryUpgradeRequest, downloa // Get the library to upgrade name := req.GetName() - lib := filterByName(listLibraries(lm, false, false), name) + lib := filterByName(listLibraries(lm, li, false, false), name) if lib == nil { // library not installed... return &cmderrors.LibraryNotFoundError{Library: name} diff --git a/internal/arduino/libraries/librariesmanager/librariesmanager.go b/internal/arduino/libraries/librariesmanager/librariesmanager.go index d47ae18550b..62d47301ebf 100644 --- a/internal/arduino/libraries/librariesmanager/librariesmanager.go +++ b/internal/arduino/libraries/librariesmanager/librariesmanager.go @@ -37,7 +37,6 @@ import ( type LibrariesManager struct { LibrariesDir []*LibrariesDir Libraries map[string]libraries.List `json:"libraries"` - // TODO: Fix all access to lm.Index left around DownloadsDir *paths.Path } @@ -73,7 +72,6 @@ func NewLibraryManager(downloadsDir *paths.Path) *LibrariesManager { return &LibrariesManager{ Libraries: map[string]libraries.List{}, DownloadsDir: downloadsDir, - Index: librariesindex.EmptyIndex, } } From efef4fabf5e3b1713d504ebd062db3b52fd4bc35 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Wed, 27 Dec 2023 17:43:36 +0100 Subject: [PATCH 03/14] Removed unused tmp file creation --- commands/instances.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/commands/instances.go b/commands/instances.go index 97cd38e8b63..0c60a2cd36f 100644 --- a/commands/instances.go +++ b/commands/instances.go @@ -430,13 +430,6 @@ func UpdateLibrariesIndex(ctx context.Context, req *rpc.UpdateLibrariesIndexRequ return &cmderrors.PermissionDeniedError{Message: tr("Could not create index directory"), Cause: err} } - // Create a temp dir to stage all downloads - tmp, err := paths.MkTempDir("", "library_index_download") - if err != nil { - return &cmderrors.TempDirCreationFailedError{Cause: err} - } - defer tmp.RemoveAll() - if err := globals.LibrariesIndexResource.Download(indexDir, downloadCB); err != nil { return err } From 39f156748c3a7ebc5438aae916a0d769f564b7e9 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Wed, 27 Dec 2023 22:33:23 +0100 Subject: [PATCH 04/14] downloadLibrary do not require a libmanager but just the downloadDir --- commands/lib/download.go | 15 +++++++++------ commands/lib/install.go | 11 ++++++++++- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/commands/lib/download.go b/commands/lib/download.go index 3d8162e127b..0e8dd78b099 100644 --- a/commands/lib/download.go +++ b/commands/lib/download.go @@ -22,9 +22,9 @@ import ( "github.com/arduino/arduino-cli/commands/internal/instances" "github.com/arduino/arduino-cli/internal/arduino/httpclient" "github.com/arduino/arduino-cli/internal/arduino/libraries/librariesindex" - "github.com/arduino/arduino-cli/internal/arduino/libraries/librariesmanager" "github.com/arduino/arduino-cli/internal/i18n" rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" + "github.com/arduino/go-paths-helper" "github.com/sirupsen/logrus" ) @@ -35,9 +35,12 @@ var tr = i18n.Tr func LibraryDownload(ctx context.Context, req *rpc.LibraryDownloadRequest, downloadCB rpc.DownloadProgressCB) (*rpc.LibraryDownloadResponse, error) { logrus.Info("Executing `arduino-cli lib download`") - lm, err := instances.GetLibraryManager(req.GetInstance()) - if err != nil { + var downloadsDir *paths.Path + if pme, release, err := instances.GetPackageManagerExplorer(req.GetInstance()); err != nil { return nil, err + } else { + downloadsDir = pme.DownloadDir + release() } li, err := instances.GetLibrariesIndex(req.GetInstance()) @@ -52,14 +55,14 @@ func LibraryDownload(ctx context.Context, req *rpc.LibraryDownloadRequest, downl return nil, err } - if err := downloadLibrary(lm, lib, downloadCB, func(*rpc.TaskProgress) {}, "download"); err != nil { + if err := downloadLibrary(downloadsDir, lib, downloadCB, func(*rpc.TaskProgress) {}, "download"); err != nil { return nil, err } return &rpc.LibraryDownloadResponse{}, nil } -func downloadLibrary(lm *librariesmanager.LibrariesManager, libRelease *librariesindex.Release, +func downloadLibrary(downloadsDir *paths.Path, libRelease *librariesindex.Release, downloadCB rpc.DownloadProgressCB, taskCB rpc.TaskProgressCB, queryParameter string) error { taskCB(&rpc.TaskProgress{Name: tr("Downloading %s", libRelease)}) @@ -67,7 +70,7 @@ func downloadLibrary(lm *librariesmanager.LibrariesManager, libRelease *librarie if err != nil { return &cmderrors.FailedDownloadError{Message: tr("Can't download library"), Cause: err} } - if err := libRelease.Resource.Download(lm.DownloadsDir, config, libRelease.String(), downloadCB, queryParameter); err != nil { + if err := libRelease.Resource.Download(downloadsDir, config, libRelease.String(), downloadCB, queryParameter); err != nil { return &cmderrors.FailedDownloadError{Message: tr("Can't download library"), Cause: err} } taskCB(&rpc.TaskProgress{Completed: true}) diff --git a/commands/lib/install.go b/commands/lib/install.go index d4b6dd8d55f..1de9513312f 100644 --- a/commands/lib/install.go +++ b/commands/lib/install.go @@ -74,6 +74,15 @@ func LibraryInstall(ctx context.Context, req *rpc.LibraryInstallRequest, downloa } } + // Obtain the download directory + var downloadsDir *paths.Path + if pme, releasePme, err := instances.GetPackageManagerExplorer(req.GetInstance()); err != nil { + return err + } else { + downloadsDir = pme.DownloadDir + releasePme() + } + // Find the libReleasesToInstall to install libReleasesToInstall := map[*librariesindex.Release]*librariesmanager.LibraryInstallPlan{} for _, lib := range toInstall { @@ -114,7 +123,7 @@ func LibraryInstall(ctx context.Context, req *rpc.LibraryInstallRequest, downloa downloadReason += "-builtin" } } - if err := downloadLibrary(lm, libRelease, downloadCB, taskCB, downloadReason); err != nil { + if err := downloadLibrary(downloadsDir, libRelease, downloadCB, taskCB, downloadReason); err != nil { return err } if err := installLibrary(lm, libRelease, installTask, taskCB); err != nil { From 76e209a3cad6fb401a8bbefde20a8bf2c749ae30 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Wed, 27 Dec 2023 22:45:12 +0100 Subject: [PATCH 05/14] Inline method LibrariesManager.Install --- commands/lib/install.go | 5 ++++- internal/arduino/libraries/librariesmanager/install.go | 5 ----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/commands/lib/install.go b/commands/lib/install.go index 1de9513312f..442fbffb777 100644 --- a/commands/lib/install.go +++ b/commands/lib/install.go @@ -149,7 +149,10 @@ func installLibrary(lm *librariesmanager.LibrariesManager, libRelease *libraries Cause: fmt.Errorf("%s: %s", tr("could not remove old library"), err)} } } - if err := lm.Install(libRelease, installTask.TargetPath); err != nil { + + installPath := installTask.TargetPath + tmpDirPath := installPath.Parent() + if err := libRelease.Resource.Install(lm.DownloadsDir, tmpDirPath, installPath); err != nil { return &cmderrors.FailedLibraryInstallError{Cause: err} } diff --git a/internal/arduino/libraries/librariesmanager/install.go b/internal/arduino/libraries/librariesmanager/install.go index 39af1f4b704..1b51e66c89f 100644 --- a/internal/arduino/libraries/librariesmanager/install.go +++ b/internal/arduino/libraries/librariesmanager/install.go @@ -103,11 +103,6 @@ func (lm *LibrariesManager) InstallPrerequisiteCheck(name string, version *semve }, nil } -// Install installs a library on the specified path. -func (lm *LibrariesManager) Install(indexLibrary *librariesindex.Release, installPath *paths.Path) error { - return indexLibrary.Resource.Install(lm.DownloadsDir, installPath.Parent(), installPath) -} - // importLibraryFromDirectory installs a library by copying it from the given directory. func (lm *LibrariesManager) importLibraryFromDirectory(libPath *paths.Path, overwrite bool) error { // Check if the library is valid and load metatada From 9a876f8f4cab520d001ed82ec21dae8c5a192134 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Wed, 27 Dec 2023 22:45:53 +0100 Subject: [PATCH 06/14] Move initializations closer to usage --- commands/lib/install.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/commands/lib/install.go b/commands/lib/install.go index 442fbffb777..5eee82b4be5 100644 --- a/commands/lib/install.go +++ b/commands/lib/install.go @@ -38,13 +38,7 @@ func LibraryInstall(ctx context.Context, req *rpc.LibraryInstallRequest, downloa return err } - lm, err := instances.GetLibraryManager(req.GetInstance()) - if err != nil { - return err - } - toInstall := map[string]*rpc.LibraryDependencyStatus{} - installLocation := libraries.FromRPCLibraryInstallLocation(req.GetInstallLocation()) if req.GetNoDeps() { toInstall[req.GetName()] = &rpc.LibraryDependencyStatus{ Name: req.GetName(), @@ -83,8 +77,14 @@ func LibraryInstall(ctx context.Context, req *rpc.LibraryInstallRequest, downloa releasePme() } + lm, err := instances.GetLibraryManager(req.GetInstance()) + if err != nil { + return err + } + // Find the libReleasesToInstall to install libReleasesToInstall := map[*librariesindex.Release]*librariesmanager.LibraryInstallPlan{} + installLocation := libraries.FromRPCLibraryInstallLocation(req.GetInstallLocation()) for _, lib := range toInstall { libRelease, err := findLibraryIndexRelease(li, &rpc.LibraryInstallRequest{ Name: lib.GetName(), From a547daf8ffa8d30bc335a2c5a8667dc20f6082cf Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Wed, 27 Dec 2023 22:53:14 +0100 Subject: [PATCH 07/14] Use LibraryManager.FindAllInstalled instead of direct access to field --- commands/lib/list.go | 24 ++++++------ .../libraries/librariesresolver/cpp.go | 38 ++++++++----------- 2 files changed, 26 insertions(+), 36 deletions(-) diff --git a/commands/lib/list.go b/commands/lib/list.go index 490ee575ed2..31610512154 100644 --- a/commands/lib/list.go +++ b/commands/lib/list.go @@ -125,22 +125,20 @@ func LibraryList(ctx context.Context, req *rpc.LibraryListRequest) (*rpc.Library // returns only the libraries that may be updated. func listLibraries(lm *librariesmanager.LibrariesManager, li *librariesindex.Index, updatable bool, all bool) []*installedLib { res := []*installedLib{} - for _, libAlternatives := range lm.Libraries { - for _, lib := range libAlternatives { - if !all { - if lib.Location != libraries.User { - continue - } - } - available := li.FindLibraryUpdate(lib) - if updatable && available == nil { + for _, lib := range lm.FindAllInstalled() { + if !all { + if lib.Location != libraries.User { continue } - res = append(res, &installedLib{ - Library: lib, - Available: available, - }) } + available := li.FindLibraryUpdate(lib) + if updatable && available == nil { + continue + } + res = append(res, &installedLib{ + Library: lib, + Available: available, + }) } return res } diff --git a/internal/arduino/libraries/librariesresolver/cpp.go b/internal/arduino/libraries/librariesresolver/cpp.go index deee71545d3..1aad85758b2 100644 --- a/internal/arduino/libraries/librariesresolver/cpp.go +++ b/internal/arduino/libraries/librariesresolver/cpp.go @@ -46,10 +46,8 @@ func NewCppResolver() *Cpp { // ScanFromLibrariesManager reads all librariers loaded in the LibrariesManager to find // and cache all C++ headers for later retrieval func (resolver *Cpp) ScanFromLibrariesManager(lm *librariesmanager.LibrariesManager) error { - for _, libAlternatives := range lm.Libraries { - for _, lib := range libAlternatives { - resolver.ScanLibrary(lib) - } + for _, lib := range lm.FindAllInstalled() { + resolver.ScanLibrary(lib) } return nil } @@ -57,11 +55,9 @@ func (resolver *Cpp) ScanFromLibrariesManager(lm *librariesmanager.LibrariesMana // ScanIDEBuiltinLibraries reads ide-builtin librariers loaded in the LibrariesManager to find // and cache all C++ headers for later retrieval. func (resolver *Cpp) ScanIDEBuiltinLibraries(lm *librariesmanager.LibrariesManager) error { - for _, libAlternatives := range lm.Libraries { - for _, lib := range libAlternatives { - if lib.Location == libraries.IDEBuiltIn { - resolver.ScanLibrary(lib) - } + for _, lib := range lm.FindAllInstalled() { + if lib.Location == libraries.IDEBuiltIn { + resolver.ScanLibrary(lib) } } return nil @@ -70,11 +66,9 @@ func (resolver *Cpp) ScanIDEBuiltinLibraries(lm *librariesmanager.LibrariesManag // ScanUserAndUnmanagedLibraries reads user/unmanaged librariers loaded in the LibrariesManager to find // and cache all C++ headers for later retrieval. func (resolver *Cpp) ScanUserAndUnmanagedLibraries(lm *librariesmanager.LibrariesManager) error { - for _, libAlternatives := range lm.Libraries { - for _, lib := range libAlternatives { - if lib.Location == libraries.User || lib.Location == libraries.Unmanaged { - resolver.ScanLibrary(lib) - } + for _, lib := range lm.FindAllInstalled() { + if lib.Location == libraries.User || lib.Location == libraries.Unmanaged { + resolver.ScanLibrary(lib) } } return nil @@ -83,16 +77,14 @@ func (resolver *Cpp) ScanUserAndUnmanagedLibraries(lm *librariesmanager.Librarie // ScanPlatformLibraries reads platform-bundled libraries for a specific platform loaded in the LibrariesManager // to find and cache all C++ headers for later retrieval. func (resolver *Cpp) ScanPlatformLibraries(lm *librariesmanager.LibrariesManager, platform *cores.PlatformRelease) error { - for _, libAlternatives := range lm.Libraries { - for _, lib := range libAlternatives { - if lib.Location != libraries.PlatformBuiltIn && lib.Location != libraries.ReferencedPlatformBuiltIn { - continue - } - if lib.ContainerPlatform != platform { - continue - } - resolver.ScanLibrary(lib) + for _, lib := range lm.FindAllInstalled() { + if lib.Location != libraries.PlatformBuiltIn && lib.Location != libraries.ReferencedPlatformBuiltIn { + continue + } + if lib.ContainerPlatform != platform { + continue } + resolver.ScanLibrary(lib) } return nil } From 42d96a9de47b66fe6a88880c0913d5bffa2b25bd Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Wed, 27 Dec 2023 22:55:46 +0100 Subject: [PATCH 08/14] Made LibraryManager.Libraries private --- .../libraries/librariesmanager/install.go | 4 ++-- .../librariesmanager/librariesmanager.go | 20 +++++++++---------- .../librariesmanager/librariesmanager_test.go | 6 +++--- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/internal/arduino/libraries/librariesmanager/install.go b/internal/arduino/libraries/librariesmanager/install.go index 1b51e66c89f..a6d73d919b4 100644 --- a/internal/arduino/libraries/librariesmanager/install.go +++ b/internal/arduino/libraries/librariesmanager/install.go @@ -151,9 +151,9 @@ func (lm *LibrariesManager) Uninstall(lib *libraries.Library) error { return fmt.Errorf(tr("removing library directory: %s"), err) } - alternatives := lm.Libraries[lib.Name] + alternatives := lm.libraries[lib.Name] alternatives.Remove(lib) - lm.Libraries[lib.Name] = alternatives + lm.libraries[lib.Name] = alternatives return nil } diff --git a/internal/arduino/libraries/librariesmanager/librariesmanager.go b/internal/arduino/libraries/librariesmanager/librariesmanager.go index 62d47301ebf..5dc5185198d 100644 --- a/internal/arduino/libraries/librariesmanager/librariesmanager.go +++ b/internal/arduino/libraries/librariesmanager/librariesmanager.go @@ -36,7 +36,7 @@ import ( // (the list of libraries, revisions, installed paths, etc.) type LibrariesManager struct { LibrariesDir []*LibrariesDir - Libraries map[string]libraries.List `json:"libraries"` + libraries map[string]libraries.List DownloadsDir *paths.Path } @@ -52,9 +52,9 @@ var tr = i18n.Tr // Names returns an array with all the names of the installed libraries. func (lm LibrariesManager) Names() []string { - res := make([]string, len(lm.Libraries)) + res := make([]string, len(lm.libraries)) i := 0 - for n := range lm.Libraries { + for n := range lm.libraries { res[i] = n i++ } @@ -70,7 +70,7 @@ func (lm LibrariesManager) Names() []string { // NewLibraryManager creates a new library manager func NewLibraryManager(downloadsDir *paths.Path) *LibrariesManager { return &LibrariesManager{ - Libraries: map[string]libraries.List{}, + libraries: map[string]libraries.List{}, DownloadsDir: downloadsDir, } } @@ -152,9 +152,9 @@ func (lm *LibrariesManager) loadLibrariesFromDir(librariesDir *LibrariesDir) []* continue } library.ContainerPlatform = librariesDir.PlatformRelease - alternatives := lm.Libraries[library.Name] + alternatives := lm.libraries[library.Name] alternatives.Add(library) - lm.Libraries[library.Name] = alternatives + lm.libraries[library.Name] = alternatives } return statuses @@ -164,7 +164,7 @@ func (lm *LibrariesManager) loadLibrariesFromDir(librariesDir *LibrariesDir) []* // name and version or, if the version is nil, the libraries installed // in the installLocation. func (lm *LibrariesManager) FindByReference(libRef *librariesindex.Reference, installLocation libraries.LibraryLocation) libraries.List { - alternatives := lm.Libraries[libRef.Name] + alternatives := lm.libraries[libRef.Name] if alternatives == nil { return nil } @@ -174,7 +174,7 @@ func (lm *LibrariesManager) FindByReference(libRef *librariesindex.Reference, in // FindAllInstalled returns all the installed libraries func (lm *LibrariesManager) FindAllInstalled() libraries.List { var res libraries.List - for _, libAlternatives := range lm.Libraries { + for _, libAlternatives := range lm.libraries { for _, libRelease := range libAlternatives { if libRelease.InstallDir == nil { continue @@ -186,7 +186,7 @@ func (lm *LibrariesManager) FindAllInstalled() libraries.List { } func (lm *LibrariesManager) clearLibraries() { - for k := range lm.Libraries { - delete(lm.Libraries, k) + for k := range lm.libraries { + delete(lm.libraries, k) } } diff --git a/internal/arduino/libraries/librariesmanager/librariesmanager_test.go b/internal/arduino/libraries/librariesmanager/librariesmanager_test.go index 22a3a1ce3f4..f5e56bb8775 100644 --- a/internal/arduino/libraries/librariesmanager/librariesmanager_test.go +++ b/internal/arduino/libraries/librariesmanager/librariesmanager_test.go @@ -25,8 +25,8 @@ import ( func Test_RescanLibrariesCallClear(t *testing.T) { baseDir := paths.New(t.TempDir()) lm := NewLibraryManager(baseDir.Join("downloads_dir")) - lm.Libraries["testLibA"] = libraries.List{} - lm.Libraries["testLibB"] = libraries.List{} + lm.libraries["testLibA"] = libraries.List{} + lm.libraries["testLibB"] = libraries.List{} lm.RescanLibraries() - require.Len(t, lm.Libraries, 0) + require.Len(t, lm.libraries, 0) } From 7c2d59531d43e59c90935678134bf255f82ad7ea Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Wed, 27 Dec 2023 23:09:45 +0100 Subject: [PATCH 09/14] Simplified libraries resolver init --- .../builder/internal/detector/detector.go | 17 +------- .../libraries/librariesresolver/cpp.go | 41 ++++++++----------- .../libraries/librariesresolver/cpp_test.go | 10 ++--- 3 files changed, 25 insertions(+), 43 deletions(-) diff --git a/internal/arduino/builder/internal/detector/detector.go b/internal/arduino/builder/internal/detector/detector.go index 24d131bad15..30b54837dbf 100644 --- a/internal/arduino/builder/internal/detector/detector.go +++ b/internal/arduino/builder/internal/detector/detector.go @@ -657,21 +657,8 @@ func LibrariesLoader( } } - resolver := librariesresolver.NewCppResolver() - if err := resolver.ScanIDEBuiltinLibraries(lm); err != nil { - return nil, nil, nil, err - } - if err := resolver.ScanUserAndUnmanagedLibraries(lm); err != nil { - return nil, nil, nil, err - } - if err := resolver.ScanPlatformLibraries(lm, targetPlatform); err != nil { - return nil, nil, nil, err - } - if actualPlatform != targetPlatform { - if err := resolver.ScanPlatformLibraries(lm, actualPlatform); err != nil { - return nil, nil, nil, err - } - } + allLibs := lm.FindAllInstalled() + resolver := librariesresolver.NewCppResolver(allLibs, targetPlatform, actualPlatform) return lm, resolver, verboseOut.Bytes(), nil } diff --git a/internal/arduino/libraries/librariesresolver/cpp.go b/internal/arduino/libraries/librariesresolver/cpp.go index 1aad85758b2..7f11f8a66c8 100644 --- a/internal/arduino/libraries/librariesresolver/cpp.go +++ b/internal/arduino/libraries/librariesresolver/cpp.go @@ -22,7 +22,6 @@ import ( "github.com/arduino/arduino-cli/internal/arduino/cores" "github.com/arduino/arduino-cli/internal/arduino/libraries" - "github.com/arduino/arduino-cli/internal/arduino/libraries/librariesmanager" "github.com/arduino/arduino-cli/internal/arduino/utils" "github.com/arduino/arduino-cli/internal/i18n" "github.com/schollz/closestmatch" @@ -37,56 +36,52 @@ type Cpp struct { var tr = i18n.Tr // NewCppResolver creates a new Cpp resolver -func NewCppResolver() *Cpp { - return &Cpp{ +func NewCppResolver(allLibs []*libraries.Library, targetPlatform, actualPlatform *cores.PlatformRelease) *Cpp { + resolver := &Cpp{ headers: map[string]libraries.List{}, } -} - -// ScanFromLibrariesManager reads all librariers loaded in the LibrariesManager to find -// and cache all C++ headers for later retrieval -func (resolver *Cpp) ScanFromLibrariesManager(lm *librariesmanager.LibrariesManager) error { - for _, lib := range lm.FindAllInstalled() { - resolver.ScanLibrary(lib) + resolver.ScanIDEBuiltinLibraries(allLibs) + resolver.ScanUserAndUnmanagedLibraries(allLibs) + resolver.ScanPlatformLibraries(allLibs, targetPlatform) + if actualPlatform != targetPlatform { + resolver.ScanPlatformLibraries(allLibs, actualPlatform) } - return nil + + return resolver } // ScanIDEBuiltinLibraries reads ide-builtin librariers loaded in the LibrariesManager to find // and cache all C++ headers for later retrieval. -func (resolver *Cpp) ScanIDEBuiltinLibraries(lm *librariesmanager.LibrariesManager) error { - for _, lib := range lm.FindAllInstalled() { +func (resolver *Cpp) ScanIDEBuiltinLibraries(allLibs []*libraries.Library) { + for _, lib := range allLibs { if lib.Location == libraries.IDEBuiltIn { - resolver.ScanLibrary(lib) + _ = resolver.ScanLibrary(lib) } } - return nil } // ScanUserAndUnmanagedLibraries reads user/unmanaged librariers loaded in the LibrariesManager to find // and cache all C++ headers for later retrieval. -func (resolver *Cpp) ScanUserAndUnmanagedLibraries(lm *librariesmanager.LibrariesManager) error { - for _, lib := range lm.FindAllInstalled() { +func (resolver *Cpp) ScanUserAndUnmanagedLibraries(allLibs []*libraries.Library) { + for _, lib := range allLibs { if lib.Location == libraries.User || lib.Location == libraries.Unmanaged { - resolver.ScanLibrary(lib) + _ = resolver.ScanLibrary(lib) } } - return nil } // ScanPlatformLibraries reads platform-bundled libraries for a specific platform loaded in the LibrariesManager // to find and cache all C++ headers for later retrieval. -func (resolver *Cpp) ScanPlatformLibraries(lm *librariesmanager.LibrariesManager, platform *cores.PlatformRelease) error { - for _, lib := range lm.FindAllInstalled() { +func (resolver *Cpp) ScanPlatformLibraries(allLibs []*libraries.Library, platform *cores.PlatformRelease) { + for _, lib := range allLibs { if lib.Location != libraries.PlatformBuiltIn && lib.Location != libraries.ReferencedPlatformBuiltIn { continue } if lib.ContainerPlatform != platform { continue } - resolver.ScanLibrary(lib) + _ = resolver.ScanLibrary(lib) } - return nil } // ScanLibrary reads a library to find and cache C++ headers for later retrieval diff --git a/internal/arduino/libraries/librariesresolver/cpp_test.go b/internal/arduino/libraries/librariesresolver/cpp_test.go index f65bc195769..a82da18f465 100644 --- a/internal/arduino/libraries/librariesresolver/cpp_test.go +++ b/internal/arduino/libraries/librariesresolver/cpp_test.go @@ -35,7 +35,7 @@ var bundleServo = &libraries.Library{Name: "Servo", Location: libraries.IDEBuilt func runResolver(include string, arch string, libs ...*libraries.Library) *libraries.Library { libraryList := libraries.List{} libraryList.Add(libs...) - resolver := NewCppResolver() + resolver := &Cpp{headers: make(map[string]libraries.List)} resolver.headers[include] = libraryList return resolver.ResolveFor(include, arch) } @@ -95,7 +95,7 @@ func TestClosestMatchWithTotallyDifferentNames(t *testing.T) { libraryList.Add(l6) libraryList.Add(l7) libraryList.Add(l8) - resolver := NewCppResolver() + resolver := &Cpp{headers: make(map[string]libraries.List)} resolver.headers["XYZ.h"] = libraryList res := resolver.ResolveFor("XYZ.h", "xyz") require.NotNil(t, res) @@ -121,7 +121,7 @@ func TestCppHeaderPriority(t *testing.T) { } func TestCppHeaderResolverWithNilResult(t *testing.T) { - resolver := NewCppResolver() + resolver := &Cpp{headers: make(map[string]libraries.List)} libraryList := libraries.List{} libraryList.Add(l1) resolver.headers["aaa.h"] = libraryList @@ -130,7 +130,7 @@ func TestCppHeaderResolverWithNilResult(t *testing.T) { func TestCppHeaderResolver(t *testing.T) { resolve := func(header string, libs ...*libraries.Library) string { - resolver := NewCppResolver() + resolver := &Cpp{headers: make(map[string]libraries.List)} librarylist := libraries.List{} for _, lib := range libs { librarylist.Add(lib) @@ -149,7 +149,7 @@ func TestCppHeaderResolver(t *testing.T) { } func TestCppHeaderResolverWithLibrariesInStrangeDirectoryNames(t *testing.T) { - resolver := NewCppResolver() + resolver := &Cpp{headers: make(map[string]libraries.List)} librarylist := libraries.List{} librarylist.Add(&libraries.Library{DirName: "onewire_2_3_4", Name: "OneWire", Architectures: []string{"*"}}) librarylist.Add(&libraries.Library{DirName: "onewireng_2_3_4", Name: "OneWireNg", Architectures: []string{"avr"}}) From 2d70be44edbaf85476b23d4f7dc3b851106a7902 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Wed, 27 Dec 2023 23:20:58 +0100 Subject: [PATCH 10/14] Made LibrariesManager.LibrariesDir private --- .../libraries/librariesmanager/librariesmanager.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/arduino/libraries/librariesmanager/librariesmanager.go b/internal/arduino/libraries/librariesmanager/librariesmanager.go index 5dc5185198d..c0619d73e7a 100644 --- a/internal/arduino/libraries/librariesmanager/librariesmanager.go +++ b/internal/arduino/libraries/librariesmanager/librariesmanager.go @@ -35,7 +35,7 @@ import ( // LibrariesManager keeps the current status of the libraries in the system // (the list of libraries, revisions, installed paths, etc.) type LibrariesManager struct { - LibrariesDir []*LibrariesDir + librariesDir []*LibrariesDir libraries map[string]libraries.List DownloadsDir *paths.Path } @@ -82,7 +82,7 @@ func (lm *LibrariesManager) AddLibrariesDir(libDir *LibrariesDir) { if libDir.Path == nil { return } - for _, dir := range lm.LibrariesDir { + for _, dir := range lm.librariesDir { if dir.Path.EquivalentTo(libDir.Path) { return } @@ -91,14 +91,14 @@ func (lm *LibrariesManager) AddLibrariesDir(libDir *LibrariesDir) { WithField("location", libDir.Location.String()). WithField("isSingleLibrary", libDir.IsSingleLibrary). Info("Adding libraries dir") - lm.LibrariesDir = append(lm.LibrariesDir, libDir) + lm.librariesDir = append(lm.librariesDir, libDir) } // RescanLibraries reload all installed libraries in the system. func (lm *LibrariesManager) RescanLibraries() []*status.Status { lm.clearLibraries() statuses := []*status.Status{} - for _, dir := range lm.LibrariesDir { + for _, dir := range lm.librariesDir { if errs := lm.loadLibrariesFromDir(dir); len(errs) > 0 { statuses = append(statuses, errs...) } @@ -107,7 +107,7 @@ func (lm *LibrariesManager) RescanLibraries() []*status.Status { } func (lm *LibrariesManager) getLibrariesDir(installLocation libraries.LibraryLocation) (*paths.Path, error) { - for _, dir := range lm.LibrariesDir { + for _, dir := range lm.librariesDir { if dir.Location == installLocation { return dir.Path, nil } From 2bf50c8e8cc473de541003dbd876db9114673ff9 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Wed, 27 Dec 2023 23:25:27 +0100 Subject: [PATCH 11/14] Removed DownloadsDir field from LibraryManager --- commands/instances.go | 8 +++----- commands/internal/instances/instances.go | 2 +- commands/lib/install.go | 6 +++--- internal/arduino/builder/internal/detector/detector.go | 4 ++-- .../libraries/librariesmanager/librariesmanager.go | 6 ++---- .../libraries/librariesmanager/librariesmanager_test.go | 4 +--- 6 files changed, 12 insertions(+), 18 deletions(-) diff --git a/commands/instances.go b/commands/instances.go index 0c60a2cd36f..e91f7d0caba 100644 --- a/commands/instances.go +++ b/commands/instances.go @@ -301,9 +301,7 @@ func Init(req *rpc.InitRequest, responseCallback func(r *rpc.InitResponse)) erro } // Create library manager and add libraries directories - lm := librariesmanager.NewLibraryManager( - pme.DownloadDir, - ) + lm := librariesmanager.NewLibraryManager() _ = instances.SetLibraryManager(instance, lm) // should never fail // Load libraries @@ -369,7 +367,7 @@ func Init(req *rpc.InitRequest, responseCallback func(r *rpc.InitResponse)) erro responseError(err.ToRPCStatus()) continue } - if err := libRelease.Resource.Download(lm.DownloadsDir, nil, libRelease.String(), downloadCallback, ""); err != nil { + if err := libRelease.Resource.Download(pme.DownloadDir, nil, libRelease.String(), downloadCallback, ""); err != nil { taskCallback(&rpc.TaskProgress{Name: tr("Error downloading library %s", libraryRef)}) e := &cmderrors.FailedLibraryInstallError{Cause: err} responseError(e.ToRPCStatus()) @@ -379,7 +377,7 @@ func Init(req *rpc.InitRequest, responseCallback func(r *rpc.InitResponse)) erro // Install library taskCallback(&rpc.TaskProgress{Name: tr("Installing library %s", libraryRef)}) - if err := libRelease.Resource.Install(lm.DownloadsDir, libRoot, libDir); err != nil { + if err := libRelease.Resource.Install(pme.DownloadDir, libRoot, libDir); err != nil { taskCallback(&rpc.TaskProgress{Name: tr("Error installing library %s", libraryRef)}) e := &cmderrors.FailedLibraryInstallError{Cause: err} responseError(e.ToRPCStatus()) diff --git a/commands/internal/instances/instances.go b/commands/internal/instances/instances.go index 43233585bed..0dd1da1ef96 100644 --- a/commands/internal/instances/instances.go +++ b/commands/internal/instances/instances.go @@ -108,7 +108,7 @@ func Create(dataDir, packagesDir, downloadsDir *paths.Path, extraUserAgent ...st instance := &coreInstance{ pm: packagemanager.NewBuilder(dataDir, packagesDir, downloadsDir, tempDir, userAgent).Build(), - lm: librariesmanager.NewLibraryManager(downloadsDir), + lm: librariesmanager.NewLibraryManager(), li: librariesindex.EmptyIndex, } diff --git a/commands/lib/install.go b/commands/lib/install.go index 5eee82b4be5..9b57aaeef0b 100644 --- a/commands/lib/install.go +++ b/commands/lib/install.go @@ -126,7 +126,7 @@ func LibraryInstall(ctx context.Context, req *rpc.LibraryInstallRequest, downloa if err := downloadLibrary(downloadsDir, libRelease, downloadCB, taskCB, downloadReason); err != nil { return err } - if err := installLibrary(lm, libRelease, installTask, taskCB); err != nil { + if err := installLibrary(lm, downloadsDir, libRelease, installTask, taskCB); err != nil { return err } } @@ -138,7 +138,7 @@ func LibraryInstall(ctx context.Context, req *rpc.LibraryInstallRequest, downloa return nil } -func installLibrary(lm *librariesmanager.LibrariesManager, libRelease *librariesindex.Release, installTask *librariesmanager.LibraryInstallPlan, taskCB rpc.TaskProgressCB) error { +func installLibrary(lm *librariesmanager.LibrariesManager, downloadsDir *paths.Path, libRelease *librariesindex.Release, installTask *librariesmanager.LibraryInstallPlan, taskCB rpc.TaskProgressCB) error { taskCB(&rpc.TaskProgress{Name: tr("Installing %s", libRelease)}) logrus.WithField("library", libRelease).Info("Installing library") @@ -152,7 +152,7 @@ func installLibrary(lm *librariesmanager.LibrariesManager, libRelease *libraries installPath := installTask.TargetPath tmpDirPath := installPath.Parent() - if err := libRelease.Resource.Install(lm.DownloadsDir, tmpDirPath, installPath); err != nil { + if err := libRelease.Resource.Install(downloadsDir, tmpDirPath, installPath); err != nil { return &cmderrors.FailedLibraryInstallError{Cause: err} } diff --git a/internal/arduino/builder/internal/detector/detector.go b/internal/arduino/builder/internal/detector/detector.go index 30b54837dbf..2a7a71d78d5 100644 --- a/internal/arduino/builder/internal/detector/detector.go +++ b/internal/arduino/builder/internal/detector/detector.go @@ -598,10 +598,10 @@ func LibrariesLoader( if useCachedLibrariesResolution { // Since we are using the cached libraries resolution // the library manager is not needed. - lm = librariesmanager.NewLibraryManager(nil) + lm = librariesmanager.NewLibraryManager() } if librariesManager == nil { - lm = librariesmanager.NewLibraryManager(nil) + lm = librariesmanager.NewLibraryManager() builtInLibrariesFolders := builtInLibrariesDirs if builtInLibrariesFolders != nil { diff --git a/internal/arduino/libraries/librariesmanager/librariesmanager.go b/internal/arduino/libraries/librariesmanager/librariesmanager.go index c0619d73e7a..80eee0d9d02 100644 --- a/internal/arduino/libraries/librariesmanager/librariesmanager.go +++ b/internal/arduino/libraries/librariesmanager/librariesmanager.go @@ -37,7 +37,6 @@ import ( type LibrariesManager struct { librariesDir []*LibrariesDir libraries map[string]libraries.List - DownloadsDir *paths.Path } // LibrariesDir is a directory containing libraries @@ -68,10 +67,9 @@ func (lm LibrariesManager) Names() []string { } // NewLibraryManager creates a new library manager -func NewLibraryManager(downloadsDir *paths.Path) *LibrariesManager { +func NewLibraryManager() *LibrariesManager { return &LibrariesManager{ - libraries: map[string]libraries.List{}, - DownloadsDir: downloadsDir, + libraries: map[string]libraries.List{}, } } diff --git a/internal/arduino/libraries/librariesmanager/librariesmanager_test.go b/internal/arduino/libraries/librariesmanager/librariesmanager_test.go index f5e56bb8775..15a367c7988 100644 --- a/internal/arduino/libraries/librariesmanager/librariesmanager_test.go +++ b/internal/arduino/libraries/librariesmanager/librariesmanager_test.go @@ -18,13 +18,11 @@ import ( "testing" "github.com/arduino/arduino-cli/internal/arduino/libraries" - "github.com/arduino/go-paths-helper" "github.com/stretchr/testify/require" ) func Test_RescanLibrariesCallClear(t *testing.T) { - baseDir := paths.New(t.TempDir()) - lm := NewLibraryManager(baseDir.Join("downloads_dir")) + lm := NewLibraryManager() lm.libraries["testLibA"] = libraries.List{} lm.libraries["testLibB"] = libraries.List{} lm.RescanLibraries() From ddf25e52f74d1b4f9077e2560e919834d17493b3 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 28 Dec 2023 11:58:09 +0100 Subject: [PATCH 12/14] Removed librariesindex.Reference structure and related helpers It does make things more complicated without any actual benefit. --- commands/core/download.go | 2 +- commands/core/install.go | 2 +- commands/instances.go | 7 +-- commands/lib/download.go | 8 +++- commands/lib/install.go | 9 ++-- commands/lib/resolve_deps.go | 14 +++--- commands/lib/uninstall.go | 8 ++-- commands/lib/utils.go | 48 ------------------- commands/version.go | 23 ++++----- .../arduino/libraries/librariesindex/index.go | 16 ++++--- .../libraries/librariesindex/index_test.go | 33 +++++++------ .../libraries/librariesindex/reference.go | 33 ------------- .../libraries/librariesmanager/install.go | 3 +- .../librariesmanager/librariesmanager.go | 8 ++-- 14 files changed, 71 insertions(+), 143 deletions(-) delete mode 100644 commands/lib/utils.go delete mode 100644 internal/arduino/libraries/librariesindex/reference.go diff --git a/commands/core/download.go b/commands/core/download.go index a64d35e4ae1..7c686f4aee9 100644 --- a/commands/core/download.go +++ b/commands/core/download.go @@ -36,7 +36,7 @@ func PlatformDownload(ctx context.Context, req *rpc.PlatformDownloadRequest, dow } defer release() - version, err := commands.ParseVersion(req) + version, err := commands.ParseVersion(req.GetVersion()) if err != nil { return nil, &cmderrors.InvalidVersionError{Cause: err} } diff --git a/commands/core/install.go b/commands/core/install.go index ce012081526..a568a9dcfba 100644 --- a/commands/core/install.go +++ b/commands/core/install.go @@ -35,7 +35,7 @@ func PlatformInstall(ctx context.Context, req *rpc.PlatformInstallRequest, downl } defer release() - version, err := commands.ParseVersion(req) + version, err := commands.ParseVersion(req.GetVersion()) if err != nil { return &cmderrors.InvalidVersionError{Cause: err} } diff --git a/commands/instances.go b/commands/instances.go index e91f7d0caba..f4b6bce22c0 100644 --- a/commands/instances.go +++ b/commands/instances.go @@ -357,11 +357,8 @@ func Init(req *rpc.InitRequest, responseCallback func(r *rpc.InitResponse)) erro if !libDir.IsDir() { // Download library taskCallback(&rpc.TaskProgress{Name: tr("Downloading library %s", libraryRef)}) - libRelease := li.FindRelease(&librariesindex.Reference{ - Name: libraryRef.Library, - Version: libraryRef.Version, - }) - if libRelease == nil { + libRelease, err := li.FindRelease(libraryRef.Library, libraryRef.Version) + if err != nil { taskCallback(&rpc.TaskProgress{Name: tr("Library %s not found", libraryRef)}) err := &cmderrors.LibraryNotFoundError{Library: libraryRef.Library} responseError(err.ToRPCStatus()) diff --git a/commands/lib/download.go b/commands/lib/download.go index 0e8dd78b099..9d174ae1927 100644 --- a/commands/lib/download.go +++ b/commands/lib/download.go @@ -18,6 +18,7 @@ package lib import ( "context" + "github.com/arduino/arduino-cli/commands" "github.com/arduino/arduino-cli/commands/cmderrors" "github.com/arduino/arduino-cli/commands/internal/instances" "github.com/arduino/arduino-cli/internal/arduino/httpclient" @@ -50,7 +51,12 @@ func LibraryDownload(ctx context.Context, req *rpc.LibraryDownloadRequest, downl logrus.Info("Preparing download") - lib, err := findLibraryIndexRelease(li, req) + version, err := commands.ParseVersion(req.GetVersion()) + if err != nil { + return nil, err + } + + lib, err := li.FindRelease(req.GetName(), version) if err != nil { return nil, err } diff --git a/commands/lib/install.go b/commands/lib/install.go index 9b57aaeef0b..f406cb02c0f 100644 --- a/commands/lib/install.go +++ b/commands/lib/install.go @@ -86,10 +86,11 @@ func LibraryInstall(ctx context.Context, req *rpc.LibraryInstallRequest, downloa libReleasesToInstall := map[*librariesindex.Release]*librariesmanager.LibraryInstallPlan{} installLocation := libraries.FromRPCLibraryInstallLocation(req.GetInstallLocation()) for _, lib := range toInstall { - libRelease, err := findLibraryIndexRelease(li, &rpc.LibraryInstallRequest{ - Name: lib.GetName(), - Version: lib.GetVersionRequired(), - }) + version, err := commands.ParseVersion(lib.GetVersionRequired()) + if err != nil { + return err + } + libRelease, err := li.FindRelease(lib.GetName(), version) if err != nil { return err } diff --git a/commands/lib/resolve_deps.go b/commands/lib/resolve_deps.go index f721da92e50..304de1e7850 100644 --- a/commands/lib/resolve_deps.go +++ b/commands/lib/resolve_deps.go @@ -20,6 +20,7 @@ import ( "errors" "sort" + "github.com/arduino/arduino-cli/commands" "github.com/arduino/arduino-cli/commands/cmderrors" "github.com/arduino/arduino-cli/commands/internal/instances" "github.com/arduino/arduino-cli/internal/arduino/libraries" @@ -41,7 +42,12 @@ func LibraryResolveDependencies(ctx context.Context, req *rpc.LibraryResolveDepe return nil, err } - reqLibRelease, err := findLibraryIndexRelease(li, req) + version, err := commands.ParseVersion(req.GetVersion()) + if err != nil { + return nil, err + } + + reqLibRelease, err := li.FindRelease(req.GetName(), version) if err != nil { return nil, err } @@ -58,11 +64,7 @@ func LibraryResolveDependencies(ctx context.Context, req *rpc.LibraryResolveDepe libs := lm.FindAllInstalled() libs = libs.FilterByVersionAndInstallLocation(nil, libraries.User) for _, lib := range libs { - release := li.FindRelease(&librariesindex.Reference{ - Name: lib.Name, - Version: lib.Version, - }) - if release != nil { + if release, err := li.FindRelease(lib.Name, lib.Version); err == nil { overrides = append(overrides, release) } } diff --git a/commands/lib/uninstall.go b/commands/lib/uninstall.go index 28f10153459..e02182dffe3 100644 --- a/commands/lib/uninstall.go +++ b/commands/lib/uninstall.go @@ -18,6 +18,7 @@ package lib import ( "context" + "github.com/arduino/arduino-cli/commands" "github.com/arduino/arduino-cli/commands/cmderrors" "github.com/arduino/arduino-cli/commands/internal/instances" "github.com/arduino/arduino-cli/internal/arduino/libraries" @@ -32,13 +33,12 @@ func LibraryUninstall(ctx context.Context, req *rpc.LibraryUninstallRequest, tas return err } - ref, err := createLibIndexReference(req) + version, err := commands.ParseVersion(req.GetVersion()) if err != nil { - return &cmderrors.InvalidLibraryError{Cause: err} + return err } - libs := lm.FindByReference(ref, libraries.User) - + libs := lm.FindByReference(req.GetName(), version, libraries.User) if len(libs) == 0 { taskCB(&rpc.TaskProgress{Message: tr("Library %s is not installed", req.GetName()), Completed: true}) return nil diff --git a/commands/lib/utils.go b/commands/lib/utils.go deleted file mode 100644 index 25e3364cdb1..00000000000 --- a/commands/lib/utils.go +++ /dev/null @@ -1,48 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package lib - -import ( - "github.com/arduino/arduino-cli/commands" - "github.com/arduino/arduino-cli/commands/cmderrors" - "github.com/arduino/arduino-cli/internal/arduino/libraries/librariesindex" -) - -type libraryReferencer interface { - commands.Versioned - GetName() string -} - -func createLibIndexReference(req libraryReferencer) (*librariesindex.Reference, error) { - version, err := commands.ParseVersion(req) - if err != nil { - return nil, &cmderrors.InvalidVersionError{Cause: err} - } - - return &librariesindex.Reference{Name: req.GetName(), Version: version}, nil -} - -func findLibraryIndexRelease(li *librariesindex.Index, req libraryReferencer) (*librariesindex.Release, error) { - ref, err := createLibIndexReference(req) - if err != nil { - return nil, err - } - lib := li.FindRelease(ref) - if lib == nil { - return nil, &cmderrors.LibraryNotFoundError{Library: ref.String()} - } - return lib, nil -} diff --git a/commands/version.go b/commands/version.go index fe9d36ca727..2fd87215a2f 100644 --- a/commands/version.go +++ b/commands/version.go @@ -16,19 +16,20 @@ package commands import ( + "github.com/arduino/arduino-cli/commands/cmderrors" semver "go.bug.st/relaxed-semver" ) -// Versioned is an object that provides a GetVersion() method -type Versioned interface { - GetVersion() string -} - -// ParseVersion returns the version parsed from an interface that provides -// the GetVersion() method (interface Versioned) -func ParseVersion(req Versioned) (*semver.Version, error) { - if req.GetVersion() != "" { - return semver.Parse(req.GetVersion()) +// ParseVersion returns the parsed version or nil if the version is +// the empty string. An error is returned if the version is not valid +// semver. +func ParseVersion(version string) (*semver.Version, error) { + if version == "" { + return nil, nil + } + res, err := semver.Parse(version) + if err != nil { + return nil, &cmderrors.InvalidVersionError{Cause: err} } - return nil, nil + return res, nil } diff --git a/internal/arduino/libraries/librariesindex/index.go b/internal/arduino/libraries/librariesindex/index.go index 92226146434..533e20706bc 100644 --- a/internal/arduino/libraries/librariesindex/index.go +++ b/internal/arduino/libraries/librariesindex/index.go @@ -18,6 +18,7 @@ package librariesindex import ( "sort" + "github.com/arduino/arduino-cli/commands/cmderrors" "github.com/arduino/arduino-cli/internal/arduino/libraries" "github.com/arduino/arduino-cli/internal/arduino/resources" rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" @@ -112,14 +113,17 @@ func (r *Release) String() string { // FindRelease search a library Release in the index. Returns nil if the // release is not found. If the version is not specified returns the latest // version available. -func (idx *Index) FindRelease(ref *Reference) *Release { - if library, exists := idx.Libraries[ref.Name]; exists { - if ref.Version == nil { - return library.Latest +func (idx *Index) FindRelease(name string, version *semver.Version) (*Release, error) { + if library, exists := idx.Libraries[name]; exists { + if version == nil { + return library.Latest, nil } - return library.Releases[ref.Version.NormalizedString()] + return library.Releases[version.NormalizedString()], nil } - return nil + if version == nil { + return nil, &cmderrors.LibraryNotFoundError{Library: name + "@latest"} + } + return nil, &cmderrors.LibraryNotFoundError{Library: name + "@" + version.String()} } // FindIndexedLibrary search an indexed library that matches the provided diff --git a/internal/arduino/libraries/librariesindex/index_test.go b/internal/arduino/libraries/librariesindex/index_test.go index b347d2bc664..2340d550b85 100644 --- a/internal/arduino/libraries/librariesindex/index_test.go +++ b/internal/arduino/libraries/librariesindex/index_test.go @@ -49,27 +49,22 @@ func TestIndexer(t *testing.T) { require.Equal(t, "", alp.Latest.Dependencies[0].GetConstraint().String()) require.Equal(t, "[1.0.0 1.1.0 1.2.0 1.2.1 1.2.2]", fmt.Sprintf("%v", alp.Versions())) - rtc100ref := &Reference{Name: "RTCZero", Version: semver.MustParse("1.0.0")} - require.Equal(t, "RTCZero@1.0.0", rtc100ref.String()) - rtc100 := index.FindRelease(rtc100ref) + rtc100, err := index.FindRelease("RTCZero", semver.MustParse("1.0.0")) + require.NoError(t, err) require.NotNil(t, rtc100) require.Equal(t, "RTCZero@1.0.0", rtc100.String()) - rtcLatestRef := &Reference{Name: "RTCZero"} - require.Equal(t, "RTCZero", rtcLatestRef.String()) - rtcLatest := index.FindRelease(rtcLatestRef) + rtcLatest, err := index.FindRelease("RTCZero", nil) + require.NoError(t, err) require.NotNil(t, rtcLatest) require.Equal(t, "RTCZero@1.6.0", rtcLatest.String()) - rtcInexistent := index.FindRelease(&Reference{ - Name: "RTCZero", - Version: semver.MustParse("0.0.0-blah"), - }) + rtcInexistent, err := index.FindRelease("RTCZero", semver.MustParse("0.0.0-blah")) + require.Error(t, err) require.Nil(t, rtcInexistent) - rtcInexistent = index.FindRelease(&Reference{ - Name: "RTCZero-blah", - }) + rtcInexistent, err = index.FindRelease("RTCZero-blah", nil) + require.Error(t, err) require.Nil(t, rtcInexistent) rtc := index.FindIndexedLibrary(&libraries.Library{Name: "RTCZero"}) @@ -95,16 +90,20 @@ func TestIndexer(t *testing.T) { require.Contains(t, resolve1, alp.Releases["1.2.1"]) require.Contains(t, resolve1, rtc.Releases["1.6.0"]) - oauth010 := index.FindRelease(&Reference{Name: "Arduino_OAuth", Version: semver.MustParse("0.1.0")}) + oauth010, err := index.FindRelease("Arduino_OAuth", semver.MustParse("0.1.0")) + require.NoError(t, err) require.NotNil(t, oauth010) require.Equal(t, "Arduino_OAuth@0.1.0", oauth010.String()) - eccx135 := index.FindRelease(&Reference{Name: "ArduinoECCX08", Version: semver.MustParse("1.3.5")}) + eccx135, err := index.FindRelease("ArduinoECCX08", semver.MustParse("1.3.5")) + require.NoError(t, err) require.NotNil(t, eccx135) require.Equal(t, "ArduinoECCX08@1.3.5", eccx135.String()) - bear172 := index.FindRelease(&Reference{Name: "ArduinoBearSSL", Version: semver.MustParse("1.7.2")}) + bear172, err := index.FindRelease("ArduinoBearSSL", semver.MustParse("1.7.2")) + require.NoError(t, err) require.NotNil(t, bear172) require.Equal(t, "ArduinoBearSSL@1.7.2", bear172.String()) - http040 := index.FindRelease(&Reference{Name: "ArduinoHttpClient", Version: semver.MustParse("0.4.0")}) + http040, err := index.FindRelease("ArduinoHttpClient", semver.MustParse("0.4.0")) + require.NoError(t, err) require.NotNil(t, http040) require.Equal(t, "ArduinoHttpClient@0.4.0", http040.String()) diff --git a/internal/arduino/libraries/librariesindex/reference.go b/internal/arduino/libraries/librariesindex/reference.go deleted file mode 100644 index a69663ee8f5..00000000000 --- a/internal/arduino/libraries/librariesindex/reference.go +++ /dev/null @@ -1,33 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package librariesindex - -import ( - semver "go.bug.st/relaxed-semver" -) - -// Reference uniquely identify a Library in the library index -type Reference struct { - Name string // The name of the parsed item. - Version *semver.Version // The Version of the parsed item. -} - -func (r *Reference) String() string { - if r.Version == nil { - return r.Name - } - return r.Name + "@" + r.Version.String() -} diff --git a/internal/arduino/libraries/librariesmanager/install.go b/internal/arduino/libraries/librariesmanager/install.go index a6d73d919b4..7151b127aae 100644 --- a/internal/arduino/libraries/librariesmanager/install.go +++ b/internal/arduino/libraries/librariesmanager/install.go @@ -25,7 +25,6 @@ import ( "github.com/arduino/arduino-cli/commands/cmderrors" "github.com/arduino/arduino-cli/internal/arduino/globals" "github.com/arduino/arduino-cli/internal/arduino/libraries" - "github.com/arduino/arduino-cli/internal/arduino/libraries/librariesindex" "github.com/arduino/arduino-cli/internal/arduino/utils" paths "github.com/arduino/go-paths-helper" "github.com/codeclysm/extract/v3" @@ -65,8 +64,8 @@ func (lm *LibrariesManager) InstallPrerequisiteCheck(name string, version *semve } lm.RescanLibraries() - libs := lm.FindByReference(&librariesindex.Reference{Name: name}, installLocation) + libs := lm.FindByReference(name, nil, installLocation) if len(libs) > 1 { libsDir := paths.NewPathList() for _, lib := range libs { diff --git a/internal/arduino/libraries/librariesmanager/librariesmanager.go b/internal/arduino/libraries/librariesmanager/librariesmanager.go index 80eee0d9d02..9e10b51383a 100644 --- a/internal/arduino/libraries/librariesmanager/librariesmanager.go +++ b/internal/arduino/libraries/librariesmanager/librariesmanager.go @@ -24,10 +24,10 @@ import ( "github.com/arduino/arduino-cli/internal/arduino/cores" "github.com/arduino/arduino-cli/internal/arduino/libraries" - "github.com/arduino/arduino-cli/internal/arduino/libraries/librariesindex" "github.com/arduino/arduino-cli/internal/i18n" paths "github.com/arduino/go-paths-helper" "github.com/sirupsen/logrus" + semver "go.bug.st/relaxed-semver" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -161,12 +161,12 @@ func (lm *LibrariesManager) loadLibrariesFromDir(librariesDir *LibrariesDir) []* // FindByReference return the installed libraries matching the Reference // name and version or, if the version is nil, the libraries installed // in the installLocation. -func (lm *LibrariesManager) FindByReference(libRef *librariesindex.Reference, installLocation libraries.LibraryLocation) libraries.List { - alternatives := lm.libraries[libRef.Name] +func (lm *LibrariesManager) FindByReference(name string, version *semver.Version, installLocation libraries.LibraryLocation) libraries.List { + alternatives := lm.libraries[name] if alternatives == nil { return nil } - return alternatives.FilterByVersionAndInstallLocation(libRef.Version, installLocation) + return alternatives.FilterByVersionAndInstallLocation(version, installLocation) } // FindAllInstalled returns all the installed libraries From 4d098006ff821b51806494182ad496b5e9662477 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Wed, 13 Dec 2023 19:53:33 +0100 Subject: [PATCH 13/14] Removed state-altering methods from LibrariesManager The original LibrariresManager has been split into three objects with specific goals: * The Builder object is used to construct a new LibrariesManager. It has methods to add librarires directories and to build the LibrariesManager. * The Explorer object is used to query the LibrariesManager about installed libraries. * The Installer object is used to rescan installed libraries and to install and uninstall. --- commands/instances.go | 24 ++-- commands/internal/instances/instances.go | 22 +++- commands/lib/install.go | 35 ++++-- commands/lib/list.go | 15 ++- commands/lib/resolve_deps.go | 21 ++-- commands/lib/uninstall.go | 6 +- commands/lib/upgrade.go | 12 +- .../builder/internal/detector/detector.go | 36 +++--- .../libraries/librariesmanager/install.go | 28 ++--- .../librariesmanager/librariesmanager.go | 113 +++++++++++++----- .../librariesmanager/librariesmanager_test.go | 15 ++- .../completion/completion_test.go | 2 +- 12 files changed, 225 insertions(+), 104 deletions(-) diff --git a/commands/instances.go b/commands/instances.go index f4b6bce22c0..e6d9a2ebd68 100644 --- a/commands/instances.go +++ b/commands/instances.go @@ -301,14 +301,13 @@ func Init(req *rpc.InitRequest, responseCallback func(r *rpc.InitResponse)) erro } // Create library manager and add libraries directories - lm := librariesmanager.NewLibraryManager() - _ = instances.SetLibraryManager(instance, lm) // should never fail + lmb := librariesmanager.NewBuilder() // Load libraries for _, pack := range pme.GetPackages() { for _, platform := range pack.Platforms { if platformRelease := pme.GetInstalledPlatformRelease(platform); platformRelease != nil { - lm.AddLibrariesDir(&librariesmanager.LibrariesDir{ + lmb.AddLibrariesDir(&librariesmanager.LibrariesDir{ PlatformRelease: platformRelease, Path: platformRelease.GetLibrariesDir(), Location: libraries.PlatformBuiltIn, @@ -336,14 +335,14 @@ func Init(req *rpc.InitRequest, responseCallback func(r *rpc.InitResponse)) erro if profile == nil { // Add directories of libraries bundled with IDE if bundledLibsDir := configuration.IDEBuiltinLibrariesDir(configuration.Settings); bundledLibsDir != nil { - lm.AddLibrariesDir(&librariesmanager.LibrariesDir{ + lmb.AddLibrariesDir(&librariesmanager.LibrariesDir{ Path: bundledLibsDir, Location: libraries.IDEBuiltIn, }) } // Add libraries directory from config file - lm.AddLibrariesDir(&librariesmanager.LibrariesDir{ + lmb.AddLibrariesDir(&librariesmanager.LibrariesDir{ Path: configuration.LibrariesDir(configuration.Settings), Location: libraries.User, }) @@ -383,16 +382,23 @@ func Init(req *rpc.InitRequest, responseCallback func(r *rpc.InitResponse)) erro taskCallback(&rpc.TaskProgress{Completed: true}) } - lm.AddLibrariesDir(&librariesmanager.LibrariesDir{ + lmb.AddLibrariesDir(&librariesmanager.LibrariesDir{ Path: libRoot, Location: libraries.User, }) } } - for _, status := range lm.RescanLibraries() { - logrus.WithError(status.Err()).Warnf("Error loading library") - // TODO: report as warning: responseError(err) + lm := lmb.Build() + _ = instances.SetLibraryManager(instance, lm) // should never fail + + { + lmi, release := lm.NewInstaller() + for _, status := range lmi.RescanLibraries() { + logrus.WithError(status.Err()).Warnf("Error loading library") + // TODO: report as warning: responseError(err) + } + release() } // Refreshes the locale used, this will change the diff --git a/commands/internal/instances/instances.go b/commands/internal/instances/instances.go index 0dd1da1ef96..a26a2dee1f8 100644 --- a/commands/internal/instances/instances.go +++ b/commands/internal/instances/instances.go @@ -62,6 +62,26 @@ func GetLibraryManager(inst *rpc.Instance) (*librariesmanager.LibrariesManager, return i.lm, nil } +// GetLibraryManagerExplorer returns the library manager Explorer for the given instance. +func GetLibraryManagerExplorer(inst *rpc.Instance) (*librariesmanager.Explorer, func(), error) { + lm, err := GetLibraryManager(inst) + if err != nil { + return nil, nil, err + } + lmi, release := lm.NewExplorer() + return lmi, release, nil +} + +// GetLibraryManagerInstaller returns the library manager Installer for the given instance. +func GetLibraryManagerInstaller(inst *rpc.Instance) (*librariesmanager.Installer, func(), error) { + lm, err := GetLibraryManager(inst) + if err != nil { + return nil, nil, err + } + lmi, release := lm.NewInstaller() + return lmi, release, nil +} + // GetLibrariesIndex returns the library index for the given instance. func GetLibrariesIndex(inst *rpc.Instance) (*librariesindex.Index, error) { instancesMux.Lock() @@ -108,7 +128,7 @@ func Create(dataDir, packagesDir, downloadsDir *paths.Path, extraUserAgent ...st instance := &coreInstance{ pm: packagemanager.NewBuilder(dataDir, packagesDir, downloadsDir, tempDir, userAgent).Build(), - lm: librariesmanager.NewLibraryManager(), + lm: librariesmanager.NewBuilder().Build(), li: librariesindex.EmptyIndex, } diff --git a/commands/lib/install.go b/commands/lib/install.go index f406cb02c0f..873e442e4a0 100644 --- a/commands/lib/install.go +++ b/commands/lib/install.go @@ -33,6 +33,7 @@ import ( // LibraryInstall resolves the library dependencies, then downloads and installs the libraries into the install location. func LibraryInstall(ctx context.Context, req *rpc.LibraryInstallRequest, downloadCB rpc.DownloadProgressCB, taskCB rpc.TaskProgressCB) error { + // Obtain the library index from the manager li, err := instances.GetLibrariesIndex(req.GetInstance()) if err != nil { return err @@ -45,12 +46,14 @@ func LibraryInstall(ctx context.Context, req *rpc.LibraryInstallRequest, downloa VersionRequired: req.GetVersion(), } } else { - res, err := LibraryResolveDependencies(ctx, &rpc.LibraryResolveDependenciesRequest{ - Instance: req.GetInstance(), - Name: req.GetName(), - Version: req.GetVersion(), - DoNotUpdateInstalledLibraries: req.GetNoOverwrite(), - }) + // Obtain the library explorer from the instance + lme, releaseLme, err := instances.GetLibraryManagerExplorer(req.GetInstance()) + if err != nil { + return err + } + + res, err := libraryResolveDependencies(ctx, lme, li, req.GetName(), req.GetVersion(), req.GetNoOverwrite()) + releaseLme() if err != nil { return err } @@ -77,10 +80,12 @@ func LibraryInstall(ctx context.Context, req *rpc.LibraryInstallRequest, downloa releasePme() } - lm, err := instances.GetLibraryManager(req.GetInstance()) + // Obtain the library installer from the manager + lmi, releaseLmi, err := instances.GetLibraryManagerInstaller(req.GetInstance()) if err != nil { return err } + defer releaseLmi() // Find the libReleasesToInstall to install libReleasesToInstall := map[*librariesindex.Release]*librariesmanager.LibraryInstallPlan{} @@ -95,7 +100,7 @@ func LibraryInstall(ctx context.Context, req *rpc.LibraryInstallRequest, downloa return err } - installTask, err := lm.InstallPrerequisiteCheck(libRelease.Library.Name, libRelease.Version, installLocation) + installTask, err := lmi.InstallPrerequisiteCheck(libRelease.Library.Name, libRelease.Version, installLocation) if err != nil { return err } @@ -127,7 +132,7 @@ func LibraryInstall(ctx context.Context, req *rpc.LibraryInstallRequest, downloa if err := downloadLibrary(downloadsDir, libRelease, downloadCB, taskCB, downloadReason); err != nil { return err } - if err := installLibrary(lm, downloadsDir, libRelease, installTask, taskCB); err != nil { + if err := installLibrary(lmi, downloadsDir, libRelease, installTask, taskCB); err != nil { return err } } @@ -139,13 +144,13 @@ func LibraryInstall(ctx context.Context, req *rpc.LibraryInstallRequest, downloa return nil } -func installLibrary(lm *librariesmanager.LibrariesManager, downloadsDir *paths.Path, libRelease *librariesindex.Release, installTask *librariesmanager.LibraryInstallPlan, taskCB rpc.TaskProgressCB) error { +func installLibrary(lmi *librariesmanager.Installer, downloadsDir *paths.Path, libRelease *librariesindex.Release, installTask *librariesmanager.LibraryInstallPlan, taskCB rpc.TaskProgressCB) error { taskCB(&rpc.TaskProgress{Name: tr("Installing %s", libRelease)}) logrus.WithField("library", libRelease).Info("Installing library") if libReplaced := installTask.ReplacedLib; libReplaced != nil { taskCB(&rpc.TaskProgress{Message: tr("Replacing %[1]s with %[2]s", libReplaced, libRelease)}) - if err := lm.Uninstall(libReplaced); err != nil { + if err := lmi.Uninstall(libReplaced); err != nil { return &cmderrors.FailedLibraryInstallError{ Cause: fmt.Errorf("%s: %s", tr("could not remove old library"), err)} } @@ -167,7 +172,9 @@ func ZipLibraryInstall(ctx context.Context, req *rpc.ZipLibraryInstallRequest, t if err != nil { return err } - if err := lm.InstallZipLib(ctx, paths.New(req.GetPath()), req.GetOverwrite()); err != nil { + lmi, release := lm.NewInstaller() + defer release() + if err := lmi.InstallZipLib(ctx, paths.New(req.GetPath()), req.GetOverwrite()); err != nil { return &cmderrors.FailedLibraryInstallError{Cause: err} } taskCB(&rpc.TaskProgress{Message: tr("Library installed"), Completed: true}) @@ -180,7 +187,9 @@ func GitLibraryInstall(ctx context.Context, req *rpc.GitLibraryInstallRequest, t if err != nil { return err } - if err := lm.InstallGitLib(req.GetUrl(), req.GetOverwrite()); err != nil { + lmi, release := lm.NewInstaller() + defer release() + if err := lmi.InstallGitLib(req.GetUrl(), req.GetOverwrite()); err != nil { return &cmderrors.FailedLibraryInstallError{Cause: err} } taskCB(&rpc.TaskProgress{Message: tr("Library installed"), Completed: true}) diff --git a/commands/lib/list.go b/commands/lib/list.go index 31610512154..f5b28bdc426 100644 --- a/commands/lib/list.go +++ b/commands/lib/list.go @@ -47,16 +47,17 @@ func LibraryList(ctx context.Context, req *rpc.LibraryListRequest) (*rpc.Library return nil, err } - lm, err := instances.GetLibraryManager(req.GetInstance()) + lme, release, err := instances.GetLibraryManagerExplorer(req.GetInstance()) if err != nil { return nil, err } + defer release() nameFilter := strings.ToLower(req.GetName()) var allLibs []*installedLib if fqbnString := req.GetFqbn(); fqbnString != "" { - allLibs = listLibraries(lm, li, req.GetUpdatable(), true) + allLibs = listLibraries(lme, li, req.GetUpdatable(), true) fqbn, err := cores.ParseFQBN(req.GetFqbn()) if err != nil { return nil, &cmderrors.InvalidFQBNError{Cause: err} @@ -96,7 +97,7 @@ func LibraryList(ctx context.Context, req *rpc.LibraryListRequest) (*rpc.Library allLibs = append(allLibs, lib) } } else { - allLibs = listLibraries(lm, li, req.GetUpdatable(), req.GetAll()) + allLibs = listLibraries(lme, li, req.GetUpdatable(), req.GetAll()) } installedLibs := []*rpc.InstalledLibrary{} @@ -122,10 +123,12 @@ func LibraryList(ctx context.Context, req *rpc.LibraryListRequest) (*rpc.Library } // listLibraries returns the list of installed libraries. If updatable is true it -// returns only the libraries that may be updated. -func listLibraries(lm *librariesmanager.LibrariesManager, li *librariesindex.Index, updatable bool, all bool) []*installedLib { +// returns only the libraries that may be updated by looking at the index for updates. +// If all is true, it returns all the libraries (including the libraries builtin in the +// platforms), otherwise only the user installed libraries. +func listLibraries(lme *librariesmanager.Explorer, li *librariesindex.Index, updatable bool, all bool) []*installedLib { res := []*installedLib{} - for _, lib := range lm.FindAllInstalled() { + for _, lib := range lme.FindAllInstalled() { if !all { if lib.Location != libraries.User { continue diff --git a/commands/lib/resolve_deps.go b/commands/lib/resolve_deps.go index 304de1e7850..b734e3d4ec7 100644 --- a/commands/lib/resolve_deps.go +++ b/commands/lib/resolve_deps.go @@ -25,43 +25,50 @@ import ( "github.com/arduino/arduino-cli/commands/internal/instances" "github.com/arduino/arduino-cli/internal/arduino/libraries" "github.com/arduino/arduino-cli/internal/arduino/libraries/librariesindex" + "github.com/arduino/arduino-cli/internal/arduino/libraries/librariesmanager" rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" semver "go.bug.st/relaxed-semver" ) // LibraryResolveDependencies FIXMEDOC func LibraryResolveDependencies(ctx context.Context, req *rpc.LibraryResolveDependenciesRequest) (*rpc.LibraryResolveDependenciesResponse, error) { - lm, err := instances.GetLibraryManager(req.GetInstance()) + lme, release, err := instances.GetLibraryManagerExplorer(req.GetInstance()) if err != nil { return nil, err } + defer release() - // Search the requested lib li, err := instances.GetLibrariesIndex(req.GetInstance()) if err != nil { return nil, err } - version, err := commands.ParseVersion(req.GetVersion()) + return libraryResolveDependencies(ctx, lme, li, req.GetName(), req.GetVersion(), req.GetDoNotUpdateInstalledLibraries()) +} + +func libraryResolveDependencies(ctx context.Context, lme *librariesmanager.Explorer, li *librariesindex.Index, + reqName, reqVersion string, noOverwrite bool) (*rpc.LibraryResolveDependenciesResponse, error) { + version, err := commands.ParseVersion(reqVersion) if err != nil { return nil, err } - reqLibRelease, err := li.FindRelease(req.GetName(), version) + // Search the requested lib + reqLibRelease, err := li.FindRelease(reqName, version) if err != nil { return nil, err } // Extract all installed libraries installedLibs := map[string]*libraries.Library{} - for _, lib := range listLibraries(lm, li, false, false) { + for _, lib := range listLibraries(lme, li, false, false) { installedLibs[lib.Library.Name] = lib.Library } // Resolve all dependencies... var overrides []*librariesindex.Release - if req.GetDoNotUpdateInstalledLibraries() { - libs := lm.FindAllInstalled() + if noOverwrite { + libs := lme.FindAllInstalled() libs = libs.FilterByVersionAndInstallLocation(nil, libraries.User) for _, lib := range libs { if release, err := li.FindRelease(lib.Name, lib.Version); err == nil { diff --git a/commands/lib/uninstall.go b/commands/lib/uninstall.go index e02182dffe3..aee4ea68143 100644 --- a/commands/lib/uninstall.go +++ b/commands/lib/uninstall.go @@ -37,8 +37,10 @@ func LibraryUninstall(ctx context.Context, req *rpc.LibraryUninstallRequest, tas if err != nil { return err } + lmi, release := lm.NewInstaller() + defer release() - libs := lm.FindByReference(req.GetName(), version, libraries.User) + libs := lmi.FindByReference(req.GetName(), version, libraries.User) if len(libs) == 0 { taskCB(&rpc.TaskProgress{Message: tr("Library %s is not installed", req.GetName()), Completed: true}) return nil @@ -46,7 +48,7 @@ func LibraryUninstall(ctx context.Context, req *rpc.LibraryUninstallRequest, tas if len(libs) == 1 { taskCB(&rpc.TaskProgress{Name: tr("Uninstalling %s", libs)}) - lm.Uninstall(libs[0]) + lmi.Uninstall(libs[0]) taskCB(&rpc.TaskProgress{Completed: true}) return nil } diff --git a/commands/lib/upgrade.go b/commands/lib/upgrade.go index a5b8150e538..0377744b972 100644 --- a/commands/lib/upgrade.go +++ b/commands/lib/upgrade.go @@ -31,12 +31,14 @@ func LibraryUpgradeAll(req *rpc.LibraryUpgradeAllRequest, downloadCB rpc.Downloa return err } - lm, err := instances.GetLibraryManager(req.GetInstance()) + lme, release, err := instances.GetLibraryManagerExplorer(req.GetInstance()) if err != nil { return err } + libsToUpgrade := listLibraries(lme, li, true, false) + release() - if err := upgrade(req.GetInstance(), listLibraries(lm, li, true, false), downloadCB, taskCB); err != nil { + if err := upgrade(req.GetInstance(), libsToUpgrade, downloadCB, taskCB); err != nil { return err } @@ -54,14 +56,16 @@ func LibraryUpgrade(ctx context.Context, req *rpc.LibraryUpgradeRequest, downloa return err } - lm, err := instances.GetLibraryManager(req.GetInstance()) + lme, release, err := instances.GetLibraryManagerExplorer(req.GetInstance()) if err != nil { return err } + libs := listLibraries(lme, li, false, false) + release() // Get the library to upgrade name := req.GetName() - lib := filterByName(listLibraries(lm, li, false, false), name) + lib := filterByName(libs, name) if lib == nil { // library not installed... return &cmderrors.LibraryNotFoundError{Library: name} diff --git a/internal/arduino/builder/internal/detector/detector.go b/internal/arduino/builder/internal/detector/detector.go index 2a7a71d78d5..7d0a67f7cdd 100644 --- a/internal/arduino/builder/internal/detector/detector.go +++ b/internal/arduino/builder/internal/detector/detector.go @@ -598,30 +598,30 @@ func LibrariesLoader( if useCachedLibrariesResolution { // Since we are using the cached libraries resolution // the library manager is not needed. - lm = librariesmanager.NewLibraryManager() + lm = librariesmanager.NewBuilder().Build() } if librariesManager == nil { - lm = librariesmanager.NewLibraryManager() + lmb := librariesmanager.NewBuilder() builtInLibrariesFolders := builtInLibrariesDirs if builtInLibrariesFolders != nil { if err := builtInLibrariesFolders.ToAbs(); err != nil { return nil, nil, nil, err } - lm.AddLibrariesDir(&librariesmanager.LibrariesDir{ + lmb.AddLibrariesDir(&librariesmanager.LibrariesDir{ Path: builtInLibrariesFolders, Location: libraries.IDEBuiltIn, }) } if actualPlatform != targetPlatform { - lm.AddLibrariesDir(&librariesmanager.LibrariesDir{ + lmb.AddLibrariesDir(&librariesmanager.LibrariesDir{ PlatformRelease: actualPlatform, Path: actualPlatform.GetLibrariesDir(), Location: libraries.ReferencedPlatformBuiltIn, }) } - lm.AddLibrariesDir(&librariesmanager.LibrariesDir{ + lmb.AddLibrariesDir(&librariesmanager.LibrariesDir{ PlatformRelease: targetPlatform, Path: targetPlatform.GetLibrariesDir(), Location: libraries.PlatformBuiltIn, @@ -632,28 +632,34 @@ func LibrariesLoader( return nil, nil, nil, err } for _, folder := range librariesFolders { - lm.AddLibrariesDir(&librariesmanager.LibrariesDir{ + lmb.AddLibrariesDir(&librariesmanager.LibrariesDir{ Path: folder, Location: libraries.User, // XXX: Should be libraries.Unmanaged? }) } for _, dir := range libraryDirs { - lm.AddLibrariesDir(&librariesmanager.LibrariesDir{ + lmb.AddLibrariesDir(&librariesmanager.LibrariesDir{ Path: dir, Location: libraries.Unmanaged, IsSingleLibrary: true, }) } - for _, status := range lm.RescanLibraries() { - // With the refactoring of the initialization step of the CLI we changed how - // errors are returned when loading platforms and libraries, that meant returning a list of - // errors instead of a single one to enhance the experience for the user. - // I have no intention right now to start a refactoring of the legacy package too, so - // here's this shitty solution for now. - // When we're gonna refactor the legacy package this will be gone. - verboseOut.Write([]byte(status.Message())) + lm = lmb.Build() + + { + lmi, release := lm.NewInstaller() + for _, status := range lmi.RescanLibraries() { + // With the refactoring of the initialization step of the CLI we changed how + // errors are returned when loading platforms and libraries, that meant returning a list of + // errors instead of a single one to enhance the experience for the user. + // I have no intention right now to start a refactoring of the legacy package too, so + // here's this shitty solution for now. + // When we're gonna refactor the legacy package this will be gone. + verboseOut.Write([]byte(status.Message())) + } + release() } } diff --git a/internal/arduino/libraries/librariesmanager/install.go b/internal/arduino/libraries/librariesmanager/install.go index 7151b127aae..4517756e3e7 100644 --- a/internal/arduino/libraries/librariesmanager/install.go +++ b/internal/arduino/libraries/librariesmanager/install.go @@ -57,15 +57,15 @@ type LibraryInstallPlan struct { // InstallPrerequisiteCheck performs prequisite checks to install a library. It returns the // install path, where the library should be installed and the possible library that is already // installed on the same folder and it's going to be replaced by the new one. -func (lm *LibrariesManager) InstallPrerequisiteCheck(name string, version *semver.Version, installLocation libraries.LibraryLocation) (*LibraryInstallPlan, error) { - installDir, err := lm.getLibrariesDir(installLocation) +func (lmi *Installer) InstallPrerequisiteCheck(name string, version *semver.Version, installLocation libraries.LibraryLocation) (*LibraryInstallPlan, error) { + installDir, err := lmi.getLibrariesDir(installLocation) if err != nil { return nil, err } - lm.RescanLibraries() + lmi.RescanLibraries() - libs := lm.FindByReference(name, nil, installLocation) + libs := lmi.FindByReference(name, nil, installLocation) if len(libs) > 1 { libsDir := paths.NewPathList() for _, lib := range libs { @@ -103,7 +103,7 @@ func (lm *LibrariesManager) InstallPrerequisiteCheck(name string, version *semve } // importLibraryFromDirectory installs a library by copying it from the given directory. -func (lm *LibrariesManager) importLibraryFromDirectory(libPath *paths.Path, overwrite bool) error { +func (lmi *Installer) importLibraryFromDirectory(libPath *paths.Path, overwrite bool) error { // Check if the library is valid and load metatada if err := validateLibrary(libPath); err != nil { return err @@ -114,7 +114,7 @@ func (lm *LibrariesManager) importLibraryFromDirectory(libPath *paths.Path, over } // Check if the library is already installed and determine install path - installPlan, err := lm.InstallPrerequisiteCheck(library.Name, library.Version, libraries.User) + installPlan, err := lmi.InstallPrerequisiteCheck(library.Name, library.Version, libraries.User) if err != nil { return err } @@ -128,7 +128,7 @@ func (lm *LibrariesManager) importLibraryFromDirectory(libPath *paths.Path, over if !overwrite { return fmt.Errorf(tr("Library %[1]s is already installed, but with a different version: %[2]s", installPlan.Name, installPlan.ReplacedLib)) } - if err := lm.Uninstall(installPlan.ReplacedLib); err != nil { + if err := lmi.Uninstall(installPlan.ReplacedLib); err != nil { return err } } @@ -142,7 +142,7 @@ func (lm *LibrariesManager) importLibraryFromDirectory(libPath *paths.Path, over } // Uninstall removes a Library -func (lm *LibrariesManager) Uninstall(lib *libraries.Library) error { +func (lmi *Installer) Uninstall(lib *libraries.Library) error { if lib == nil || lib.InstallDir == nil { return fmt.Errorf(tr("install directory not set")) } @@ -150,14 +150,14 @@ func (lm *LibrariesManager) Uninstall(lib *libraries.Library) error { return fmt.Errorf(tr("removing library directory: %s"), err) } - alternatives := lm.libraries[lib.Name] + alternatives := lmi.libraries[lib.Name] alternatives.Remove(lib) - lm.libraries[lib.Name] = alternatives + lmi.libraries[lib.Name] = alternatives return nil } // InstallZipLib installs a Zip library on the specified path. -func (lm *LibrariesManager) InstallZipLib(ctx context.Context, archivePath *paths.Path, overwrite bool) error { +func (lmi *Installer) InstallZipLib(ctx context.Context, archivePath *paths.Path, overwrite bool) error { // Clone library in a temporary directory tmpDir, err := paths.MkTempDir("", "") if err != nil { @@ -191,7 +191,7 @@ func (lm *LibrariesManager) InstallZipLib(ctx context.Context, archivePath *path tmpInstallPath := libRootFiles[0] // Install extracted library in the destination directory - if err := lm.importLibraryFromDirectory(tmpInstallPath, overwrite); err != nil { + if err := lmi.importLibraryFromDirectory(tmpInstallPath, overwrite); err != nil { return fmt.Errorf(tr("moving extracted archive to destination dir: %s"), err) } @@ -199,7 +199,7 @@ func (lm *LibrariesManager) InstallZipLib(ctx context.Context, archivePath *path } // InstallGitLib installs a library hosted on a git repository on the specified path. -func (lm *LibrariesManager) InstallGitLib(gitURL string, overwrite bool) error { +func (lmi *Installer) InstallGitLib(gitURL string, overwrite bool) error { gitLibraryName, ref, err := parseGitURL(gitURL) if err != nil { return err @@ -240,7 +240,7 @@ func (lm *LibrariesManager) InstallGitLib(gitURL string, overwrite bool) error { tmpInstallPath.Join(".git").RemoveAll() // Install extracted library in the destination directory - if err := lm.importLibraryFromDirectory(tmpInstallPath, overwrite); err != nil { + if err := lmi.importLibraryFromDirectory(tmpInstallPath, overwrite); err != nil { return fmt.Errorf(tr("moving extracted archive to destination dir: %s"), err) } diff --git a/internal/arduino/libraries/librariesmanager/librariesmanager.go b/internal/arduino/libraries/librariesmanager/librariesmanager.go index 9e10b51383a..64102cf46f9 100644 --- a/internal/arduino/libraries/librariesmanager/librariesmanager.go +++ b/internal/arduino/libraries/librariesmanager/librariesmanager.go @@ -21,6 +21,7 @@ import ( "os" "slices" "strings" + "sync" "github.com/arduino/arduino-cli/internal/arduino/cores" "github.com/arduino/arduino-cli/internal/arduino/libraries" @@ -32,11 +33,30 @@ import ( "google.golang.org/grpc/status" ) +// Builder is used to create a new LibrariesManager. The builder has +// methods to load and parse libraries and to actually build the +// LibraryManager. Once the LibrariesManager is built, it cannot be +// altered anymore. +type Builder struct { + *LibrariesManager +} + +// Explorer is used to query the library manager about the installed libraries. +type Explorer struct { + *LibrariesManager +} + +// Installer is used to rescan installed libraries after install/uninstall. +type Installer struct { + *LibrariesManager +} + // LibrariesManager keeps the current status of the libraries in the system // (the list of libraries, revisions, installed paths, etc.) type LibrariesManager struct { - librariesDir []*LibrariesDir - libraries map[string]libraries.List + librariesLock sync.RWMutex + librariesDir []*LibrariesDir + libraries map[string]libraries.List } // LibrariesDir is a directory containing libraries @@ -50,7 +70,7 @@ type LibrariesDir struct { var tr = i18n.Tr // Names returns an array with all the names of the installed libraries. -func (lm LibrariesManager) Names() []string { +func (lm *Explorer) Names() []string { res := make([]string, len(lm.libraries)) i := 0 for n := range lm.libraries { @@ -66,21 +86,63 @@ func (lm LibrariesManager) Names() []string { return res } -// NewLibraryManager creates a new library manager -func NewLibraryManager() *LibrariesManager { - return &LibrariesManager{ - libraries: map[string]libraries.List{}, +// NewBuilder creates a new library manager builder. +func NewBuilder() *Builder { + return &Builder{ + &LibrariesManager{ + libraries: map[string]libraries.List{}, + }, } } +// NewBuilder creates a Builder with the same configuration of this +// LibrariesManager. A "commit" function callback is returned: calling +// this function will write the new configuration into this LibrariesManager. +func (lm *LibrariesManager) NewBuilder() (*Builder, func()) { + lmb := NewBuilder() + return lmb, func() { + lmb.BuildIntoExistingLibrariesManager(lm) + } +} + +// NewExplorer returns a new Explorer. The returned function must be called +// to release the lock on the LibrariesManager. +func (lm *LibrariesManager) NewExplorer() (*Explorer, func()) { + lm.librariesLock.RLock() + return &Explorer{lm}, lm.librariesLock.RUnlock +} + +// NewInstaller returns a new Installer. The returned function must be called +// to release the lock on the LibrariesManager. +func (lm *LibrariesManager) NewInstaller() (*Installer, func()) { + lm.librariesLock.Lock() + return &Installer{lm}, lm.librariesLock.Unlock +} + +// Build builds a new LibrariesManager. +func (lmb *Builder) Build() *LibrariesManager { + res := &LibrariesManager{} + lmb.BuildIntoExistingLibrariesManager(res) + return res +} + +// BuildIntoExistingLibrariesManager will overwrite the given LibrariesManager instead +// of building a new one. +func (lmb *Builder) BuildIntoExistingLibrariesManager(old *LibrariesManager) { + old.librariesLock.Lock() + old.librariesDir = lmb.librariesDir + old.libraries = lmb.libraries + old.librariesLock.Unlock() +} + // AddLibrariesDir adds path to the list of directories // to scan when searching for libraries. If a path is already // in the list it is ignored. -func (lm *LibrariesManager) AddLibrariesDir(libDir *LibrariesDir) { +func (lmb *Builder) AddLibrariesDir(libDir *LibrariesDir) { if libDir.Path == nil { return } - for _, dir := range lm.librariesDir { + for _, dir := range lmb.librariesDir { if dir.Path.EquivalentTo(libDir.Path) { return } @@ -89,15 +151,15 @@ func (lm *LibrariesManager) AddLibrariesDir(libDir *LibrariesDir) { WithField("location", libDir.Location.String()). WithField("isSingleLibrary", libDir.IsSingleLibrary). Info("Adding libraries dir") - lm.librariesDir = append(lm.librariesDir, libDir) + lmb.librariesDir = append(lmb.librariesDir, libDir) } // RescanLibraries reload all installed libraries in the system. -func (lm *LibrariesManager) RescanLibraries() []*status.Status { - lm.clearLibraries() +func (lmi *Installer) RescanLibraries() []*status.Status { + lmi.libraries = map[string]libraries.List{} statuses := []*status.Status{} - for _, dir := range lm.librariesDir { - if errs := lm.loadLibrariesFromDir(dir); len(errs) > 0 { + for _, dir := range lmi.librariesDir { + if errs := lmi.loadLibrariesFromDir(dir); len(errs) > 0 { statuses = append(statuses, errs...) } } @@ -122,7 +184,7 @@ func (lm *LibrariesManager) getLibrariesDir(installLocation libraries.LibraryLoc // loadLibrariesFromDir loads all libraries in the given directory. Returns // nil if the directory doesn't exists. -func (lm *LibrariesManager) loadLibrariesFromDir(librariesDir *LibrariesDir) []*status.Status { +func (lmi *Installer) loadLibrariesFromDir(librariesDir *LibrariesDir) []*status.Status { statuses := []*status.Status{} var libDirs paths.PathList @@ -142,17 +204,17 @@ func (lm *LibrariesManager) loadLibrariesFromDir(librariesDir *LibrariesDir) []* libDirs = d } - for _, subDir := range libDirs { - library, err := libraries.Load(subDir, librariesDir.Location) + for _, libDir := range libDirs { + library, err := libraries.Load(libDir, librariesDir.Location) if err != nil { - s := status.Newf(codes.Internal, tr("loading library from %[1]s: %[2]s"), subDir, err) + s := status.Newf(codes.Internal, tr("loading library from %[1]s: %[2]s"), libDir, err) statuses = append(statuses, s) continue } library.ContainerPlatform = librariesDir.PlatformRelease - alternatives := lm.libraries[library.Name] + alternatives := lmi.libraries[library.Name] alternatives.Add(library) - lm.libraries[library.Name] = alternatives + lmi.libraries[library.Name] = alternatives } return statuses @@ -161,8 +223,8 @@ func (lm *LibrariesManager) loadLibrariesFromDir(librariesDir *LibrariesDir) []* // FindByReference return the installed libraries matching the Reference // name and version or, if the version is nil, the libraries installed // in the installLocation. -func (lm *LibrariesManager) FindByReference(name string, version *semver.Version, installLocation libraries.LibraryLocation) libraries.List { - alternatives := lm.libraries[name] +func (lmi *Installer) FindByReference(name string, version *semver.Version, installLocation libraries.LibraryLocation) libraries.List { + alternatives := lmi.libraries[name] if alternatives == nil { return nil } @@ -174,6 +236,7 @@ func (lm *LibrariesManager) FindAllInstalled() libraries.List { var res libraries.List for _, libAlternatives := range lm.libraries { for _, libRelease := range libAlternatives { + // TODO: is this check redundant? if libRelease.InstallDir == nil { continue } @@ -182,9 +245,3 @@ func (lm *LibrariesManager) FindAllInstalled() libraries.List { } return res } - -func (lm *LibrariesManager) clearLibraries() { - for k := range lm.libraries { - delete(lm.libraries, k) - } -} diff --git a/internal/arduino/libraries/librariesmanager/librariesmanager_test.go b/internal/arduino/libraries/librariesmanager/librariesmanager_test.go index 15a367c7988..2393868999e 100644 --- a/internal/arduino/libraries/librariesmanager/librariesmanager_test.go +++ b/internal/arduino/libraries/librariesmanager/librariesmanager_test.go @@ -22,9 +22,16 @@ import ( ) func Test_RescanLibrariesCallClear(t *testing.T) { - lm := NewLibraryManager() - lm.libraries["testLibA"] = libraries.List{} - lm.libraries["testLibB"] = libraries.List{} - lm.RescanLibraries() + lmb := NewBuilder() + lmb.libraries["testLibA"] = libraries.List{} + lmb.libraries["testLibB"] = libraries.List{} + lm := lmb.Build() + + { + lmi, release := lm.NewInstaller() + lmi.RescanLibraries() + release() + } + require.Len(t, lm.libraries, 0) } diff --git a/internal/integrationtest/completion/completion_test.go b/internal/integrationtest/completion/completion_test.go index a216eb0f6ff..678f2f0139e 100644 --- a/internal/integrationtest/completion/completion_test.go +++ b/internal/integrationtest/completion/completion_test.go @@ -247,7 +247,7 @@ func TestProfileCompletion(t *testing.T) { stdout, _, _ = cli.Run("__complete", "upload", sketchWithProfilesPath.String(), "--profile", "") require.Contains(t, string(stdout), "profile1") - // The cli is running in the sketch folder, so need the explictly specify the path in the cli + // The cli is running in the sketch folder, so need the explicitly specify the path in the cli cli.SetWorkingDir(sketchWithProfilesPath) stdout, _, _ = cli.Run("__complete", "compile", "--profile", "") require.Contains(t, string(stdout), "profile1") From e5180b1e64d9287214e8ed2a0395246c8dd98ab1 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 28 Dec 2023 12:37:58 +0100 Subject: [PATCH 14/14] Fixed test --- internal/arduino/libraries/librariesindex/index.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/arduino/libraries/librariesindex/index.go b/internal/arduino/libraries/librariesindex/index.go index 533e20706bc..033630f910d 100644 --- a/internal/arduino/libraries/librariesindex/index.go +++ b/internal/arduino/libraries/librariesindex/index.go @@ -118,7 +118,9 @@ func (idx *Index) FindRelease(name string, version *semver.Version) (*Release, e if version == nil { return library.Latest, nil } - return library.Releases[version.NormalizedString()], nil + if release, exists := library.Releases[version.NormalizedString()]; exists { + return release, nil + } } if version == nil { return nil, &cmderrors.LibraryNotFoundError{Library: name + "@latest"}