Download gzipped version of the library index

Author: cmaglie

Diff for: arduino/libraries/librariesmanager/download.go

@@ -17,16 +17,10 @@ package librariesmanager
 
 import (
 	"net/url"
-
-	"go.bug.st/downloader/v2"
 )
 
-// LibraryIndexURL is the URL where to get library index.
+// LibraryIndexURL is the URL where to get the library index.
 var LibraryIndexURL, _ = url.Parse("https://downloads.arduino.cc/libraries/library_index.json")
 
-// UpdateIndex downloads the libraries index file from Arduino repository.
-func (lm *LibrariesManager) UpdateIndex(config *downloader.Config) (*downloader.Downloader, error) {
-	lm.IndexFile.Parent().MkdirAll()
-	// TODO: Download from gzipped URL index
-	return downloader.DownloadWithConfig(lm.IndexFile.String(), LibraryIndexURL.String(), *config, downloader.NoResume)
-}
+// LibraryIndexGZURL is the URL where to get the gzipped library index.
+var LibraryIndexGZURL, _ = url.Parse("https://downloads.arduino.cc/libraries/library_index.json.gz")

Diff for: commands/instances.go

@@ -17,7 +17,6 @@ package commands
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"io/ioutil"
 	"net/url"
@@ -37,6 +36,7 @@ import (
 	"github.com/arduino/arduino-cli/configuration"
 	rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
 	paths "github.com/arduino/go-paths-helper"
+	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"go.bug.st/downloader/v2"
 )
@@ -179,14 +179,40 @@ func UpdateLibrariesIndex(ctx context.Context, req *rpc.UpdateLibrariesIndexRequ
 	if err != nil {
 		return err
 	}
-	d, err := lm.UpdateIndex(config)
+
+	if err := lm.IndexFile.Parent().MkdirAll(); err != nil {
+		return err
+	}
+
+	// Create a temp dir to stage all downloads
+	tmp, err := paths.MkTempDir("", "library_index_download")
 	if err != nil {
 		return err
 	}
-	Download(d, "Updating index: library_index.json", downloadCB)
-	if d.Error() != nil {
-		return d.Error()
+	defer tmp.RemoveAll()
+
+	// Download gzipped library_index
+	tmpIndexGz := tmp.Join("library_index.json.gz")
+	if d, err := downloader.DownloadWithConfig(tmpIndexGz.String(), librariesmanager.LibraryIndexGZURL.String(), *config, downloader.NoResume); err != nil {
+		return err
+	} else {
+		if err := Download(d, "Updating index: library_index.json.gz", downloadCB); err != nil {
+			return errors.Wrap(err, "downloading library_index.json.gz")
+		}
+	}
+
+	// Extract the real library_index
+	tmpIndex := tmp.Join("library_index.json")
+	if err := paths.GUnzip(tmpIndexGz, tmpIndex); err != nil {
+		return errors.Wrap(err, "unzipping library_index.json.gz")
+	}
+
+	// Copy extracted library_index to final destination
+	lm.IndexFile.Remove()
+	if err := tmpIndex.CopyTo(lm.IndexFile); err != nil {
+		return errors.Wrap(err, "writing library_index.json")
 	}
+
 	if _, err := Rescan(req.GetInstance().GetId()); err != nil {
 		return fmt.Errorf("rescanning filesystem: %s", err)
 	}

Diff for: go.mod

@@ -4,7 +4,7 @@ go 1.14
 
 require (
 	github.com/arduino/board-discovery v0.0.0-20180823133458-1ba29327fb0c
-	github.com/arduino/go-paths-helper v1.5.0
+	github.com/arduino/go-paths-helper v1.6.0
 	github.com/arduino/go-properties-orderedmap v1.3.0
 	github.com/arduino/go-timeutils v0.0.0-20171220113728-d1dd9e313b1b
 	github.com/arduino/go-win32-utils v0.0.0-20180330194947-ed041402e83b

Diff for: go.sum

@@ -18,6 +18,8 @@ github.com/arduino/go-paths-helper v1.4.0 h1:ilnseAdxmN1bFnLxxXHRtcdmt9jBf3O4jtY
 github.com/arduino/go-paths-helper v1.4.0/go.mod h1:V82BWgAAp4IbmlybxQdk9Bpkz8M4Qyx+RAFKaG9NuvU=
 github.com/arduino/go-paths-helper v1.5.0 h1:RVo189hD+GhUS1rQ3gixwK1nSbvVR8MGIGa7Gxv2bdM=
 github.com/arduino/go-paths-helper v1.5.0/go.mod h1:V82BWgAAp4IbmlybxQdk9Bpkz8M4Qyx+RAFKaG9NuvU=
+github.com/arduino/go-paths-helper v1.6.0 h1:S7/d7DqB9XlnvF9KrgSiGmo2oWKmYW6O/DTjj3Bijx4=
+github.com/arduino/go-paths-helper v1.6.0/go.mod h1:V82BWgAAp4IbmlybxQdk9Bpkz8M4Qyx+RAFKaG9NuvU=
 github.com/arduino/go-properties-orderedmap v1.3.0 h1:4No/vQopB36e7WUIk6H6TxiSEJPiMrVOCZylYmua39o=
 github.com/arduino/go-properties-orderedmap v1.3.0/go.mod h1:DKjD2VXY/NZmlingh4lSFMEYCVubfeArCsGPGDwb2yk=
 github.com/arduino/go-timeutils v0.0.0-20171220113728-d1dd9e313b1b h1:9hDi4F2st6dbLC3y4i02zFT5quS4X6iioWifGlVwfy4=