Download gzipped version of the library index

Author: cmaglie

Diff for: arduino/libraries/librariesmanager/download.go

@@ -17,16 +17,10 @@ package librariesmanager
 
 import (
 	"net/url"
-
-	"go.bug.st/downloader/v2"
 )
 
-// LibraryIndexURL is the URL where to get library index.
+// LibraryIndexURL is the URL where to get the library index.
 var LibraryIndexURL, _ = url.Parse("https://downloads.arduino.cc/libraries/library_index.json")
 
-// UpdateIndex downloads the libraries index file from Arduino repository.
-func (lm *LibrariesManager) UpdateIndex(config *downloader.Config) (*downloader.Downloader, error) {
-	lm.IndexFile.Parent().MkdirAll()
-	// TODO: Download from gzipped URL index
-	return downloader.DownloadWithConfig(lm.IndexFile.String(), LibraryIndexURL.String(), *config, downloader.NoResume)
-}
+// LibraryIndexGZURL is the URL where to get the gzipped library index.
+var LibraryIndexGZURL, _ = url.Parse("https://downloads.arduino.cc/libraries/library_index.json.gz")

Diff for: commands/instances.go

@@ -17,7 +17,6 @@ package commands
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"io/ioutil"
 	"net/url"
@@ -37,6 +36,7 @@ import (
 	"github.com/arduino/arduino-cli/configuration"
 	rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
 	paths "github.com/arduino/go-paths-helper"
+	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"go.bug.st/downloader/v2"
 )
@@ -179,14 +179,40 @@ func UpdateLibrariesIndex(ctx context.Context, req *rpc.UpdateLibrariesIndexRequ
 	if err != nil {
 		return err
 	}
-	d, err := lm.UpdateIndex(config)
+
+	if err := lm.IndexFile.Parent().MkdirAll(); err != nil {
+		return err
+	}
+
+	// Create a temp dir to stage all downloads
+	tmp, err := paths.MkTempDir("", "library_index_download")
 	if err != nil {
 		return err
 	}
-	Download(d, "Updating index: library_index.json", downloadCB)
-	if d.Error() != nil {
-		return d.Error()
+	defer tmp.RemoveAll()
+
+	// Download gzipped library_index
+	tmpIndexGz := tmp.Join("library_index.json.gz")
+	if d, err := downloader.DownloadWithConfig(tmpIndexGz.String(), librariesmanager.LibraryIndexGZURL.String(), *config, downloader.NoResume); err != nil {
+		return err
+	} else {
+		if err := Download(d, "Updating index: library_index.json.gz", downloadCB); err != nil {
+			return errors.Wrap(err, "downloading library_index.json.gz")
+		}
+	}
+
+	// Extract the real library_index
+	tmpIndex := tmp.Join("library_index.json")
+	if err := paths.GUnzip(tmpIndexGz, tmpIndex); err != nil {
+		return errors.Wrap(err, "unzipping library_index.json.gz")
+	}
+
+	// Copy extracted library_index to final destination
+	lm.IndexFile.Remove()
+	if err := tmpIndex.CopyTo(lm.IndexFile); err != nil {
+		return errors.Wrap(err, "writing library_index.json")
 	}
+
 	if _, err := Rescan(req.GetInstance().GetId()); err != nil {
 		return fmt.Errorf("rescanning filesystem: %s", err)
 	}