diff --git a/arduino/libraries/librariesmanager/download.go b/arduino/libraries/librariesmanager/download.go index d8ec7dae1c9..b3920fca756 100644 --- a/arduino/libraries/librariesmanager/download.go +++ b/arduino/libraries/librariesmanager/download.go @@ -17,16 +17,13 @@ package librariesmanager import ( "net/url" - - "go.bug.st/downloader/v2" ) -// LibraryIndexURL is the URL where to get library index. +// LibraryIndexURL is the URL where to get the library index. var LibraryIndexURL, _ = url.Parse("https://downloads.arduino.cc/libraries/library_index.json") -// UpdateIndex downloads the libraries index file from Arduino repository. -func (lm *LibrariesManager) UpdateIndex(config *downloader.Config) (*downloader.Downloader, error) { - lm.IndexFile.Parent().MkdirAll() - // TODO: Download from gzipped URL index - return downloader.DownloadWithConfig(lm.IndexFile.String(), LibraryIndexURL.String(), *config, downloader.NoResume) -} +// LibraryIndexGZURL is the URL where to get the gzipped library index. +var LibraryIndexGZURL, _ = url.Parse("https://downloads.arduino.cc/libraries/library_index.json.gz") + +// LibraryIndexSignature is the URL where to get the library index signature. +var LibraryIndexSignature, _ = url.Parse("https://downloads.arduino.cc/libraries/library_index.json.sig") diff --git a/arduino/libraries/librariesmanager/librariesmanager.go b/arduino/libraries/librariesmanager/librariesmanager.go index a33e69a8675..06496c4a449 100644 --- a/arduino/libraries/librariesmanager/librariesmanager.go +++ b/arduino/libraries/librariesmanager/librariesmanager.go @@ -35,9 +35,10 @@ type LibrariesManager struct { LibrariesDir []*LibrariesDir Libraries map[string]*LibraryAlternatives `json:"libraries"` - Index *librariesindex.Index - IndexFile *paths.Path - DownloadsDir *paths.Path + Index *librariesindex.Index + IndexFile *paths.Path + IndexFileSignature *paths.Path + DownloadsDir *paths.Path } // LibrariesDir is a directory containing libraries @@ -95,15 +96,17 @@ func (lm LibrariesManager) Names() []string { // NewLibraryManager creates a new library manager func NewLibraryManager(indexDir *paths.Path, downloadsDir *paths.Path) *LibrariesManager { - var indexFile *paths.Path + var indexFile, indexFileSignature *paths.Path if indexDir != nil { indexFile = indexDir.Join("library_index.json") + indexFileSignature = indexDir.Join("library_index.json.sig") } return &LibrariesManager{ - Libraries: map[string]*LibraryAlternatives{}, - IndexFile: indexFile, - DownloadsDir: downloadsDir, - Index: librariesindex.EmptyIndex, + Libraries: map[string]*LibraryAlternatives{}, + IndexFile: indexFile, + IndexFileSignature: indexFileSignature, + DownloadsDir: downloadsDir, + Index: librariesindex.EmptyIndex, } } diff --git a/commands/instances.go b/commands/instances.go index a6650658759..63c4f7acf1e 100644 --- a/commands/instances.go +++ b/commands/instances.go @@ -17,7 +17,6 @@ package commands import ( "context" - "errors" "fmt" "io/ioutil" "net/url" @@ -37,6 +36,7 @@ import ( "github.com/arduino/arduino-cli/configuration" rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" paths "github.com/arduino/go-paths-helper" + "github.com/pkg/errors" "github.com/sirupsen/logrus" "go.bug.st/downloader/v2" ) @@ -179,14 +179,62 @@ func UpdateLibrariesIndex(ctx context.Context, req *rpc.UpdateLibrariesIndexRequ if err != nil { return err } - d, err := lm.UpdateIndex(config) + + if err := lm.IndexFile.Parent().MkdirAll(); err != nil { + return err + } + + // Create a temp dir to stage all downloads + tmp, err := paths.MkTempDir("", "library_index_download") if err != nil { return err } - Download(d, "Updating index: library_index.json", downloadCB) - if d.Error() != nil { - return d.Error() + defer tmp.RemoveAll() + + // Download gzipped library_index + tmpIndexGz := tmp.Join("library_index.json.gz") + if d, err := downloader.DownloadWithConfig(tmpIndexGz.String(), librariesmanager.LibraryIndexGZURL.String(), *config, downloader.NoResume); err == nil { + if err := Download(d, "Updating index: library_index.json.gz", downloadCB); err != nil { + return errors.Wrap(err, "downloading library_index.json.gz") + } + } else { + return err } + + // Download signature + tmpSignature := tmp.Join("library_index.json.sig") + if d, err := downloader.DownloadWithConfig(tmpSignature.String(), librariesmanager.LibraryIndexSignature.String(), *config, downloader.NoResume); err == nil { + if err := Download(d, "Updating index: library_index.json.sig", downloadCB); err != nil { + return errors.Wrap(err, "downloading library_index.json.sig") + } + } else { + return err + } + + // Extract the real library_index + tmpIndex := tmp.Join("library_index.json") + if err := paths.GUnzip(tmpIndexGz, tmpIndex); err != nil { + return errors.Wrap(err, "unzipping library_index.json.gz") + } + + // Check signature + if ok, _, err := security.VerifyArduinoDetachedSignature(tmpIndex, tmpSignature); err != nil { + return errors.Wrap(err, "verifying signature") + } else if !ok { + return errors.New("library_index.json has an invalid signature") + } + + // Copy extracted library_index and signature to final destination + lm.IndexFile.Remove() + lm.IndexFileSignature.Remove() + if err := tmpIndex.CopyTo(lm.IndexFile); err != nil { + return errors.Wrap(err, "writing library_index.json") + } + if err := tmpSignature.CopyTo(lm.IndexFileSignature); err != nil { + return errors.Wrap(err, "writing library_index.json.sig") + } + + // Rescan libraries if _, err := Rescan(req.GetInstance().GetId()); err != nil { return fmt.Errorf("rescanning filesystem: %s", err) } diff --git a/docs/UPGRADING.md b/docs/UPGRADING.md index ba3c8ce5867..46ccb8b390b 100644 --- a/docs/UPGRADING.md +++ b/docs/UPGRADING.md @@ -2,6 +2,18 @@ Here you can find a list of migration guides to handle breaking changes between releases of the CLI. +## Unreleased + +### Removed rarely used golang API + +The following function from the `github.com/arduino/arduino-cli/arduino/libraries` module is no longer available: + +```go +func (lm *LibrariesManager) UpdateIndex(config *downloader.Config) (*downloader.Downloader, error) { +``` + +We recommend using the equivalent gRPC API to perform the update of the index. + ## 0.18.0 ### Breaking changes in gRPC API and CLI JSON output. diff --git a/test/test_lib.py b/test/test_lib.py index 4d4dfc37c6b..8b8334a4fd0 100644 --- a/test/test_lib.py +++ b/test/test_lib.py @@ -404,7 +404,9 @@ def test_install_with_zip_path(run_command, data_dir, downloads_dir): def test_update_index(run_command): result = run_command("lib update-index") assert result.ok - assert "Updating index: library_index.json downloaded" == result.stdout.splitlines()[-1].strip() + lines = [l.strip() for l in result.stdout.splitlines()] + assert "Updating index: library_index.json.gz downloaded" in lines + assert "Updating index: library_index.json.sig downloaded" in lines def test_uninstall(run_command): @@ -454,7 +456,8 @@ def test_search(run_command): result = run_command("lib search --names") assert result.ok lines = [l.strip() for l in result.stdout.strip().splitlines()] - assert "Updating index: library_index.json downloaded" in lines + assert "Updating index: library_index.json.gz downloaded" in lines + assert "Updating index: library_index.json.sig downloaded" in lines libs = [l[6:].strip('"') for l in lines if "Name:" in l] expected = {"WiFi101", "WiFi101OTA", "Firebase Arduino based on WiFi101"} diff --git a/test/test_update.py b/test/test_update.py index d35f7831930..582ef51aa6d 100644 --- a/test/test_update.py +++ b/test/test_update.py @@ -23,7 +23,8 @@ def test_update(run_command): assert "Updating index: package_index.json downloaded" in lines assert "Updating index: package_index.json.sig downloaded" in lines - assert "Updating index: library_index.json downloaded" in lines + assert "Updating index: library_index.json.gz downloaded" in lines + assert "Updating index: library_index.json.sig downloaded" in lines def test_update_showing_outdated(run_command): @@ -46,7 +47,8 @@ def test_update_showing_outdated(run_command): assert "Updating index: package_index.json downloaded" in lines assert "Updating index: package_index.json.sig downloaded" in lines - assert "Updating index: library_index.json downloaded" in lines + assert "Updating index: library_index.json.gz downloaded" in lines + assert "Updating index: library_index.json.sig downloaded" in lines assert lines[-5].startswith("Arduino AVR Boards") assert lines[-2].startswith("USBHost")