Check gpg signature for library_index.json (#1307)

* Download gzipped version of the library index

* Check gpg signature for library_index.json

* Update docs/UPGRADING.md

Author: cmaglie

Diff for: arduino/libraries/librariesmanager/download.go

@@ -17,16 +17,13 @@ package librariesmanager
 
 import (
 	"net/url"
-
-	"go.bug.st/downloader/v2"
 )
 
-// LibraryIndexURL is the URL where to get library index.
+// LibraryIndexURL is the URL where to get the library index.
 var LibraryIndexURL, _ = url.Parse("https://downloads.arduino.cc/libraries/library_index.json")
 
-// UpdateIndex downloads the libraries index file from Arduino repository.
-func (lm *LibrariesManager) UpdateIndex(config *downloader.Config) (*downloader.Downloader, error) {
-	lm.IndexFile.Parent().MkdirAll()
-	// TODO: Download from gzipped URL index
-	return downloader.DownloadWithConfig(lm.IndexFile.String(), LibraryIndexURL.String(), *config, downloader.NoResume)
-}
+// LibraryIndexGZURL is the URL where to get the gzipped library index.
+var LibraryIndexGZURL, _ = url.Parse("https://downloads.arduino.cc/libraries/library_index.json.gz")
+
+// LibraryIndexSignature is the URL where to get the library index signature.
+var LibraryIndexSignature, _ = url.Parse("https://downloads.arduino.cc/libraries/library_index.json.sig")

Diff for: arduino/libraries/librariesmanager/librariesmanager.go

@@ -35,9 +35,10 @@ type LibrariesManager struct {
 	LibrariesDir []*LibrariesDir
 	Libraries    map[string]*LibraryAlternatives `json:"libraries"`
 
-	Index        *librariesindex.Index
-	IndexFile    *paths.Path
-	DownloadsDir *paths.Path
+	Index              *librariesindex.Index
+	IndexFile          *paths.Path
+	IndexFileSignature *paths.Path
+	DownloadsDir       *paths.Path
 }
 
 // LibrariesDir is a directory containing libraries
@@ -95,15 +96,17 @@ func (lm LibrariesManager) Names() []string {
 
 // NewLibraryManager creates a new library manager
 func NewLibraryManager(indexDir *paths.Path, downloadsDir *paths.Path) *LibrariesManager {
-	var indexFile *paths.Path
+	var indexFile, indexFileSignature *paths.Path
 	if indexDir != nil {
 		indexFile = indexDir.Join("library_index.json")
+		indexFileSignature = indexDir.Join("library_index.json.sig")
 	}
 	return &LibrariesManager{
-		Libraries:    map[string]*LibraryAlternatives{},
-		IndexFile:    indexFile,
-		DownloadsDir: downloadsDir,
-		Index:        librariesindex.EmptyIndex,
+		Libraries:          map[string]*LibraryAlternatives{},
+		IndexFile:          indexFile,
+		IndexFileSignature: indexFileSignature,
+		DownloadsDir:       downloadsDir,
+		Index:              librariesindex.EmptyIndex,
 	}
 }
 

Diff for: commands/instances.go

@@ -17,7 +17,6 @@ package commands
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"io/ioutil"
 	"net/url"
@@ -37,6 +36,7 @@ import (
 	"github.com/arduino/arduino-cli/configuration"
 	rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
 	paths "github.com/arduino/go-paths-helper"
+	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"go.bug.st/downloader/v2"
 )
@@ -179,14 +179,62 @@ func UpdateLibrariesIndex(ctx context.Context, req *rpc.UpdateLibrariesIndexRequ
 	if err != nil {
 		return err
 	}
-	d, err := lm.UpdateIndex(config)
+
+	if err := lm.IndexFile.Parent().MkdirAll(); err != nil {
+		return err
+	}
+
+	// Create a temp dir to stage all downloads
+	tmp, err := paths.MkTempDir("", "library_index_download")
 	if err != nil {
 		return err
 	}
-	Download(d, "Updating index: library_index.json", downloadCB)
-	if d.Error() != nil {
-		return d.Error()
+	defer tmp.RemoveAll()
+
+	// Download gzipped library_index
+	tmpIndexGz := tmp.Join("library_index.json.gz")
+	if d, err := downloader.DownloadWithConfig(tmpIndexGz.String(), librariesmanager.LibraryIndexGZURL.String(), *config, downloader.NoResume); err == nil {
+		if err := Download(d, "Updating index: library_index.json.gz", downloadCB); err != nil {
+			return errors.Wrap(err, "downloading library_index.json.gz")
+		}
+	} else {
+		return err
 	}
+
+	// Download signature
+	tmpSignature := tmp.Join("library_index.json.sig")
+	if d, err := downloader.DownloadWithConfig(tmpSignature.String(), librariesmanager.LibraryIndexSignature.String(), *config, downloader.NoResume); err == nil {
+		if err := Download(d, "Updating index: library_index.json.sig", downloadCB); err != nil {
+			return errors.Wrap(err, "downloading library_index.json.sig")
+		}
+	} else {
+		return err
+	}
+
+	// Extract the real library_index
+	tmpIndex := tmp.Join("library_index.json")
+	if err := paths.GUnzip(tmpIndexGz, tmpIndex); err != nil {
+		return errors.Wrap(err, "unzipping library_index.json.gz")
+	}
+
+	// Check signature
+	if ok, _, err := security.VerifyArduinoDetachedSignature(tmpIndex, tmpSignature); err != nil {
+		return errors.Wrap(err, "verifying signature")
+	} else if !ok {
+		return errors.New("library_index.json has an invalid signature")
+	}
+
+	// Copy extracted library_index and signature to final destination
+	lm.IndexFile.Remove()
+	lm.IndexFileSignature.Remove()
+	if err := tmpIndex.CopyTo(lm.IndexFile); err != nil {
+		return errors.Wrap(err, "writing library_index.json")
+	}
+	if err := tmpSignature.CopyTo(lm.IndexFileSignature); err != nil {
+		return errors.Wrap(err, "writing library_index.json.sig")
+	}
+
+	// Rescan libraries
 	if _, err := Rescan(req.GetInstance().GetId()); err != nil {
 		return fmt.Errorf("rescanning filesystem: %s", err)
 	}

Diff for: docs/UPGRADING.md

@@ -2,6 +2,18 @@
 
 Here you can find a list of migration guides to handle breaking changes between releases of the CLI.
 
+## Unreleased
+
+### Removed rarely used golang API
+
+The following function from the `github.com/arduino/arduino-cli/arduino/libraries` module is no longer available:
+
+```go
+func (lm *LibrariesManager) UpdateIndex(config *downloader.Config) (*downloader.Downloader, error) {
+```
+
+We recommend using the equivalent gRPC API to perform the update of the index.
+
 ## 0.18.0
 
 ### Breaking changes in gRPC API and CLI JSON output.

Diff for: test/test_lib.py

@@ -404,7 +404,9 @@ def test_install_with_zip_path(run_command, data_dir, downloads_dir):
 def test_update_index(run_command):
     result = run_command("lib update-index")
     assert result.ok
-    assert "Updating index: library_index.json downloaded" == result.stdout.splitlines()[-1].strip()
+    lines = [l.strip() for l in result.stdout.splitlines()]
+    assert "Updating index: library_index.json.gz downloaded" in lines
+    assert "Updating index: library_index.json.sig downloaded" in lines
 
 
 def test_uninstall(run_command):
@@ -454,7 +456,8 @@ def test_search(run_command):
     result = run_command("lib search --names")
     assert result.ok
     lines = [l.strip() for l in result.stdout.strip().splitlines()]
-    assert "Updating index: library_index.json downloaded" in lines
+    assert "Updating index: library_index.json.gz downloaded" in lines
+    assert "Updating index: library_index.json.sig downloaded" in lines
     libs = [l[6:].strip('"') for l in lines if "Name:" in l]
 
     expected = {"WiFi101", "WiFi101OTA", "Firebase Arduino based on WiFi101"}

Diff for: test/test_update.py

@@ -23,7 +23,8 @@ def test_update(run_command):
 
     assert "Updating index: package_index.json downloaded" in lines
     assert "Updating index: package_index.json.sig downloaded" in lines
-    assert "Updating index: library_index.json downloaded" in lines
+    assert "Updating index: library_index.json.gz downloaded" in lines
+    assert "Updating index: library_index.json.sig downloaded" in lines
 
 
 def test_update_showing_outdated(run_command):
@@ -46,7 +47,8 @@ def test_update_showing_outdated(run_command):
 
     assert "Updating index: package_index.json downloaded" in lines
     assert "Updating index: package_index.json.sig downloaded" in lines
-    assert "Updating index: library_index.json downloaded" in lines
+    assert "Updating index: library_index.json.gz downloaded" in lines
+    assert "Updating index: library_index.json.sig downloaded" in lines
     assert lines[-5].startswith("Arduino AVR Boards")
     assert lines[-2].startswith("USBHost")