From 6d55f37c6121a80ba74477df1c44c12fdd0e3f53 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Tue, 1 Jun 2021 18:43:03 +0200 Subject: [PATCH 1/6] Download gzipped version of the library index --- .../libraries/librariesmanager/download.go | 12 ++----- commands/instances.go | 36 ++++++++++++++++--- 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/arduino/libraries/librariesmanager/download.go b/arduino/libraries/librariesmanager/download.go index d8ec7dae1c9..fabdc55e57d 100644 --- a/arduino/libraries/librariesmanager/download.go +++ b/arduino/libraries/librariesmanager/download.go @@ -17,16 +17,10 @@ package librariesmanager import ( "net/url" - - "go.bug.st/downloader/v2" ) -// LibraryIndexURL is the URL where to get library index. +// LibraryIndexURL is the URL where to get the library index. var LibraryIndexURL, _ = url.Parse("https://downloads.arduino.cc/libraries/library_index.json") -// UpdateIndex downloads the libraries index file from Arduino repository. -func (lm *LibrariesManager) UpdateIndex(config *downloader.Config) (*downloader.Downloader, error) { - lm.IndexFile.Parent().MkdirAll() - // TODO: Download from gzipped URL index - return downloader.DownloadWithConfig(lm.IndexFile.String(), LibraryIndexURL.String(), *config, downloader.NoResume) -} +// LibraryIndexGZURL is the URL where to get the gzipped library index. +var LibraryIndexGZURL, _ = url.Parse("https://downloads.arduino.cc/libraries/library_index.json.gz") diff --git a/commands/instances.go b/commands/instances.go index a6650658759..6ea4c5b840a 100644 --- a/commands/instances.go +++ b/commands/instances.go @@ -17,7 +17,6 @@ package commands import ( "context" - "errors" "fmt" "io/ioutil" "net/url" @@ -37,6 +36,7 @@ import ( "github.com/arduino/arduino-cli/configuration" rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" paths "github.com/arduino/go-paths-helper" + "github.com/pkg/errors" "github.com/sirupsen/logrus" "go.bug.st/downloader/v2" ) @@ -179,14 +179,40 @@ func UpdateLibrariesIndex(ctx context.Context, req *rpc.UpdateLibrariesIndexRequ if err != nil { return err } - d, err := lm.UpdateIndex(config) + + if err := lm.IndexFile.Parent().MkdirAll(); err != nil { + return err + } + + // Create a temp dir to stage all downloads + tmp, err := paths.MkTempDir("", "library_index_download") if err != nil { return err } - Download(d, "Updating index: library_index.json", downloadCB) - if d.Error() != nil { - return d.Error() + defer tmp.RemoveAll() + + // Download gzipped library_index + tmpIndexGz := tmp.Join("library_index.json.gz") + if d, err := downloader.DownloadWithConfig(tmpIndexGz.String(), librariesmanager.LibraryIndexGZURL.String(), *config, downloader.NoResume); err != nil { + return err + } else { + if err := Download(d, "Updating index: library_index.json.gz", downloadCB); err != nil { + return errors.Wrap(err, "downloading library_index.json.gz") + } + } + + // Extract the real library_index + tmpIndex := tmp.Join("library_index.json") + if err := paths.GUnzip(tmpIndexGz, tmpIndex); err != nil { + return errors.Wrap(err, "unzipping library_index.json.gz") + } + + // Copy extracted library_index to final destination + lm.IndexFile.Remove() + if err := tmpIndex.CopyTo(lm.IndexFile); err != nil { + return errors.Wrap(err, "writing library_index.json") } + if _, err := Rescan(req.GetInstance().GetId()); err != nil { return fmt.Errorf("rescanning filesystem: %s", err) } From b0f110b2273fda5bf83286aba927b2454cfb02bf Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 3 Jun 2021 09:53:50 +0200 Subject: [PATCH 2/6] Check gpg signature for library_index.json --- .../libraries/librariesmanager/download.go | 3 +++ .../librariesmanager/librariesmanager.go | 19 ++++++++------- commands/instances.go | 24 ++++++++++++++++++- 3 files changed, 37 insertions(+), 9 deletions(-) diff --git a/arduino/libraries/librariesmanager/download.go b/arduino/libraries/librariesmanager/download.go index fabdc55e57d..b3920fca756 100644 --- a/arduino/libraries/librariesmanager/download.go +++ b/arduino/libraries/librariesmanager/download.go @@ -24,3 +24,6 @@ var LibraryIndexURL, _ = url.Parse("https://downloads.arduino.cc/libraries/libra // LibraryIndexGZURL is the URL where to get the gzipped library index. var LibraryIndexGZURL, _ = url.Parse("https://downloads.arduino.cc/libraries/library_index.json.gz") + +// LibraryIndexSignature is the URL where to get the library index signature. +var LibraryIndexSignature, _ = url.Parse("https://downloads.arduino.cc/libraries/library_index.json.sig") diff --git a/arduino/libraries/librariesmanager/librariesmanager.go b/arduino/libraries/librariesmanager/librariesmanager.go index a33e69a8675..06496c4a449 100644 --- a/arduino/libraries/librariesmanager/librariesmanager.go +++ b/arduino/libraries/librariesmanager/librariesmanager.go @@ -35,9 +35,10 @@ type LibrariesManager struct { LibrariesDir []*LibrariesDir Libraries map[string]*LibraryAlternatives `json:"libraries"` - Index *librariesindex.Index - IndexFile *paths.Path - DownloadsDir *paths.Path + Index *librariesindex.Index + IndexFile *paths.Path + IndexFileSignature *paths.Path + DownloadsDir *paths.Path } // LibrariesDir is a directory containing libraries @@ -95,15 +96,17 @@ func (lm LibrariesManager) Names() []string { // NewLibraryManager creates a new library manager func NewLibraryManager(indexDir *paths.Path, downloadsDir *paths.Path) *LibrariesManager { - var indexFile *paths.Path + var indexFile, indexFileSignature *paths.Path if indexDir != nil { indexFile = indexDir.Join("library_index.json") + indexFileSignature = indexDir.Join("library_index.json.sig") } return &LibrariesManager{ - Libraries: map[string]*LibraryAlternatives{}, - IndexFile: indexFile, - DownloadsDir: downloadsDir, - Index: librariesindex.EmptyIndex, + Libraries: map[string]*LibraryAlternatives{}, + IndexFile: indexFile, + IndexFileSignature: indexFileSignature, + DownloadsDir: downloadsDir, + Index: librariesindex.EmptyIndex, } } diff --git a/commands/instances.go b/commands/instances.go index 6ea4c5b840a..12164e22d2b 100644 --- a/commands/instances.go +++ b/commands/instances.go @@ -201,18 +201,40 @@ func UpdateLibrariesIndex(ctx context.Context, req *rpc.UpdateLibrariesIndexRequ } } + // Download signature + tmpSignature := tmp.Join("library_index.json.sig") + if d, err := downloader.DownloadWithConfig(tmpSignature.String(), librariesmanager.LibraryIndexSignature.String(), *config, downloader.NoResume); err != nil { + return err + } else { + if err := Download(d, "Updating index: library_index.json.sig", downloadCB); err != nil { + return errors.Wrap(err, "downloading library_index.json.sig") + } + } + // Extract the real library_index tmpIndex := tmp.Join("library_index.json") if err := paths.GUnzip(tmpIndexGz, tmpIndex); err != nil { return errors.Wrap(err, "unzipping library_index.json.gz") } - // Copy extracted library_index to final destination + // Check signature + if ok, _, err := security.VerifyArduinoDetachedSignature(tmpIndex, tmpSignature); err != nil { + return errors.Wrap(err, "verifying signature") + } else if !ok { + return errors.New("library_index.json has an invalid signature!") + } + + // Copy extracted library_index and signature to final destination lm.IndexFile.Remove() + lm.IndexFileSignature.Remove() if err := tmpIndex.CopyTo(lm.IndexFile); err != nil { return errors.Wrap(err, "writing library_index.json") } + if err := tmpSignature.CopyTo(lm.IndexFileSignature); err != nil { + return errors.Wrap(err, "writing library_index.json.sig") + } + // Rescan libraries if _, err := Rescan(req.GetInstance().GetId()); err != nil { return fmt.Errorf("rescanning filesystem: %s", err) } From 64fda78109739b73e8c2642b1e77327ad4dfc325 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 4 Jun 2021 10:39:56 +0200 Subject: [PATCH 3/6] Make linter happy --- commands/instances.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/commands/instances.go b/commands/instances.go index 12164e22d2b..63c4f7acf1e 100644 --- a/commands/instances.go +++ b/commands/instances.go @@ -193,22 +193,22 @@ func UpdateLibrariesIndex(ctx context.Context, req *rpc.UpdateLibrariesIndexRequ // Download gzipped library_index tmpIndexGz := tmp.Join("library_index.json.gz") - if d, err := downloader.DownloadWithConfig(tmpIndexGz.String(), librariesmanager.LibraryIndexGZURL.String(), *config, downloader.NoResume); err != nil { - return err - } else { + if d, err := downloader.DownloadWithConfig(tmpIndexGz.String(), librariesmanager.LibraryIndexGZURL.String(), *config, downloader.NoResume); err == nil { if err := Download(d, "Updating index: library_index.json.gz", downloadCB); err != nil { return errors.Wrap(err, "downloading library_index.json.gz") } + } else { + return err } // Download signature tmpSignature := tmp.Join("library_index.json.sig") - if d, err := downloader.DownloadWithConfig(tmpSignature.String(), librariesmanager.LibraryIndexSignature.String(), *config, downloader.NoResume); err != nil { - return err - } else { + if d, err := downloader.DownloadWithConfig(tmpSignature.String(), librariesmanager.LibraryIndexSignature.String(), *config, downloader.NoResume); err == nil { if err := Download(d, "Updating index: library_index.json.sig", downloadCB); err != nil { return errors.Wrap(err, "downloading library_index.json.sig") } + } else { + return err } // Extract the real library_index @@ -221,7 +221,7 @@ func UpdateLibrariesIndex(ctx context.Context, req *rpc.UpdateLibrariesIndexRequ if ok, _, err := security.VerifyArduinoDetachedSignature(tmpIndex, tmpSignature); err != nil { return errors.Wrap(err, "verifying signature") } else if !ok { - return errors.New("library_index.json has an invalid signature!") + return errors.New("library_index.json has an invalid signature") } // Copy extracted library_index and signature to final destination From d216ebbe15391cb3c672f342cb3cd993bcc025a6 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 4 Jun 2021 15:59:56 +0200 Subject: [PATCH 4/6] Updated readme --- docs/UPGRADING.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/UPGRADING.md b/docs/UPGRADING.md index ba3c8ce5867..05e1c3fdfa6 100644 --- a/docs/UPGRADING.md +++ b/docs/UPGRADING.md @@ -2,6 +2,18 @@ Here you can find a list of migration guides to handle breaking changes between releases of the CLI. +## 0.19.0 + +### Removed rarely used golang API + +The following function from the `github.com/arduino/arduino-cli/arduino/libraries` module is no longer available: + +```go +func (lm *LibrariesManager) UpdateIndex(config *downloader.Config) (*downloader.Downloader, error) { +``` + +We recommend using the equivalent gRPC API to perform the update of the index. + ## 0.18.0 ### Breaking changes in gRPC API and CLI JSON output. From 74fe2bd6a0be3add306ab5979507fd1de1c5d025 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 4 Jun 2021 17:57:47 +0200 Subject: [PATCH 5/6] Fixed integration tests --- test/test_lib.py | 7 +++++-- test/test_update.py | 6 ++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/test/test_lib.py b/test/test_lib.py index 4d4dfc37c6b..8b8334a4fd0 100644 --- a/test/test_lib.py +++ b/test/test_lib.py @@ -404,7 +404,9 @@ def test_install_with_zip_path(run_command, data_dir, downloads_dir): def test_update_index(run_command): result = run_command("lib update-index") assert result.ok - assert "Updating index: library_index.json downloaded" == result.stdout.splitlines()[-1].strip() + lines = [l.strip() for l in result.stdout.splitlines()] + assert "Updating index: library_index.json.gz downloaded" in lines + assert "Updating index: library_index.json.sig downloaded" in lines def test_uninstall(run_command): @@ -454,7 +456,8 @@ def test_search(run_command): result = run_command("lib search --names") assert result.ok lines = [l.strip() for l in result.stdout.strip().splitlines()] - assert "Updating index: library_index.json downloaded" in lines + assert "Updating index: library_index.json.gz downloaded" in lines + assert "Updating index: library_index.json.sig downloaded" in lines libs = [l[6:].strip('"') for l in lines if "Name:" in l] expected = {"WiFi101", "WiFi101OTA", "Firebase Arduino based on WiFi101"} diff --git a/test/test_update.py b/test/test_update.py index d35f7831930..582ef51aa6d 100644 --- a/test/test_update.py +++ b/test/test_update.py @@ -23,7 +23,8 @@ def test_update(run_command): assert "Updating index: package_index.json downloaded" in lines assert "Updating index: package_index.json.sig downloaded" in lines - assert "Updating index: library_index.json downloaded" in lines + assert "Updating index: library_index.json.gz downloaded" in lines + assert "Updating index: library_index.json.sig downloaded" in lines def test_update_showing_outdated(run_command): @@ -46,7 +47,8 @@ def test_update_showing_outdated(run_command): assert "Updating index: package_index.json downloaded" in lines assert "Updating index: package_index.json.sig downloaded" in lines - assert "Updating index: library_index.json downloaded" in lines + assert "Updating index: library_index.json.gz downloaded" in lines + assert "Updating index: library_index.json.sig downloaded" in lines assert lines[-5].startswith("Arduino AVR Boards") assert lines[-2].startswith("USBHost") From 99f95918e1345cd6980f61ab993cd7ce55d0f668 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Mon, 7 Jun 2021 09:37:09 +0200 Subject: [PATCH 6/6] Update docs/UPGRADING.md Co-authored-by: Silvano Cerza <3314350+silvanocerza@users.noreply.github.com> --- docs/UPGRADING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/UPGRADING.md b/docs/UPGRADING.md index 05e1c3fdfa6..46ccb8b390b 100644 --- a/docs/UPGRADING.md +++ b/docs/UPGRADING.md @@ -2,7 +2,7 @@ Here you can find a list of migration guides to handle breaking changes between releases of the CLI. -## 0.19.0 +## Unreleased ### Removed rarely used golang API