From 2237ca251010fdea922e14bd36e6005ffbeef6e9 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 6 Mar 2020 12:16:37 +0100 Subject: [PATCH 1/7] Update go-downloader to 1.2.0 --- go.mod | 2 +- go.sum | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8add00a9c98..9589dbe49b9 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,7 @@ require ( github.com/spf13/viper v1.6.2 github.com/stretchr/testify v1.4.0 go.bug.st/cleanup v1.0.0 - go.bug.st/downloader v1.1.0 + go.bug.st/downloader v1.2.0 go.bug.st/relaxed-semver v0.0.0-20190922224835-391e10178d18 go.bug.st/serial v1.0.0 go.bug.st/serial.v1 v0.0.0-20180827123349-5f7892a7bb45 // indirect diff --git a/go.sum b/go.sum index 46bfc061b4c..2f84109f8f6 100644 --- a/go.sum +++ b/go.sum @@ -189,6 +189,7 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM github.com/spf13/viper v1.6.2 h1:7aKfF+e8/k68gda3LOjo5RxiUqddoFxVq4BKBPrxk5E= github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= @@ -204,8 +205,8 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.bug.st/cleanup v1.0.0 h1:XVj1HZxkBXeq3gMT7ijWUpHyIC1j8XAoNSyQ06CskgA= go.bug.st/cleanup v1.0.0/go.mod h1:EqVmTg2IBk4znLbPD28xne3abjsJftMdqqJEjhn70bk= -go.bug.st/downloader v1.1.0 h1:LipC9rqRCe8kwa+ah3ZDfCqneVaf34cB/TKjXZiZt54= -go.bug.st/downloader v1.1.0/go.mod h1:l+RPbNbrTB+MoAIp8nrZsP22nRPDy26XJZQqmm4gNT4= +go.bug.st/downloader v1.2.0 h1:YmXFTcTnm0v8WzAWHn2DyV46c/Izlc/gReXubc2oBho= +go.bug.st/downloader v1.2.0/go.mod h1:l+RPbNbrTB+MoAIp8nrZsP22nRPDy26XJZQqmm4gNT4= go.bug.st/relaxed-semver v0.0.0-20190922224835-391e10178d18 h1:F1qxtaFuewctYc/SsHRn+Q7Dtwi+yJGPgVq8YLtQz98= go.bug.st/relaxed-semver v0.0.0-20190922224835-391e10178d18/go.mod h1:Cx1VqMtEhE9pIkEyUj3LVVVPkv89dgW8aCKrRPDR/uE= go.bug.st/serial v1.0.0 h1:ogEPzrllCsnG00EqKRjeYvPRsO7NJW6DqykzkdD6E/k= From 464dbb5e905c216d744734a3df2fd247e0b07f37 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 6 Mar 2020 11:47:24 +0100 Subject: [PATCH 2/7] Use downloader.SetDefaultConfig() to set user-agent for the arduino-cli This change allows to not pass-trough the downloader configuration from function to function everywhere (and sometime we forget to pass it for example in the "core update-index" command). --- arduino/cores/packagemanager/download.go | 9 ++++---- arduino/resources/helpers.go | 7 ++---- arduino/resources/helpers_test.go | 6 ++++- arduino/resources/resources_test.go | 5 ++-- cli/cli.go | 10 ++++++++ cli/core/download.go | 3 +-- cli/core/install.go | 3 +-- cli/core/upgrade.go | 2 +- cli/daemon/daemon.go | 29 +++++++++++------------- cli/instance/instance.go | 4 +--- cli/lib/download.go | 4 +--- cli/lib/install.go | 4 +--- cli/lib/upgrade.go | 5 ++-- commands/bundled_tools.go | 6 ++--- commands/core/download.go | 18 ++++++--------- commands/core/install.go | 11 ++++----- commands/core/upgrade.go | 9 ++++---- commands/daemon/daemon.go | 11 +-------- commands/instances.go | 19 ++++++---------- commands/lib/download.go | 10 ++++---- commands/lib/install.go | 5 ++-- commands/lib/upgrade.go | 13 +++++------ 22 files changed, 82 insertions(+), 111 deletions(-) diff --git a/arduino/cores/packagemanager/download.go b/arduino/cores/packagemanager/download.go index a3305fa3d4c..8e843795dd4 100644 --- a/arduino/cores/packagemanager/download.go +++ b/arduino/cores/packagemanager/download.go @@ -17,7 +17,6 @@ package packagemanager import ( "fmt" - "net/http" "github.com/arduino/arduino-cli/arduino/cores" "go.bug.st/downloader" @@ -101,16 +100,16 @@ func (pm *PackageManager) FindPlatformReleaseDependencies(item *PlatformReferenc // DownloadToolRelease downloads a ToolRelease. If the tool is already downloaded a nil Downloader // is returned. -func (pm *PackageManager) DownloadToolRelease(tool *cores.ToolRelease, downloaderHeaders http.Header) (*downloader.Downloader, error) { +func (pm *PackageManager) DownloadToolRelease(tool *cores.ToolRelease) (*downloader.Downloader, error) { resource := tool.GetCompatibleFlavour() if resource == nil { return nil, fmt.Errorf("tool not available for your OS") } - return resource.Download(pm.DownloadDir, downloaderHeaders) + return resource.Download(pm.DownloadDir) } // DownloadPlatformRelease downloads a PlatformRelease. If the platform is already downloaded a // nil Downloader is returned. -func (pm *PackageManager) DownloadPlatformRelease(platform *cores.PlatformRelease, downloaderHeaders http.Header) (*downloader.Downloader, error) { - return platform.Resource.Download(pm.DownloadDir, downloaderHeaders) +func (pm *PackageManager) DownloadPlatformRelease(platform *cores.PlatformRelease) (*downloader.Downloader, error) { + return platform.Resource.Download(pm.DownloadDir) } diff --git a/arduino/resources/helpers.go b/arduino/resources/helpers.go index c4728465397..d568da14be9 100644 --- a/arduino/resources/helpers.go +++ b/arduino/resources/helpers.go @@ -17,7 +17,6 @@ package resources import ( "fmt" - "net/http" "os" "github.com/arduino/go-paths-helper" @@ -44,7 +43,7 @@ func (r *DownloadResource) IsCached(downloadDir *paths.Path) (bool, error) { } // Download a DownloadResource. -func (r *DownloadResource) Download(downloadDir *paths.Path, downloaderHeaders http.Header) (*downloader.Downloader, error) { +func (r *DownloadResource) Download(downloadDir *paths.Path) (*downloader.Downloader, error) { cached, err := r.TestLocalArchiveIntegrity(downloadDir) if err != nil { return nil, fmt.Errorf("testing local archive integrity: %s", err) @@ -72,7 +71,5 @@ func (r *DownloadResource) Download(downloadDir *paths.Path, downloaderHeaders h return nil, fmt.Errorf("getting archive file info: %s", err) } - downloadConfig := downloader.Config{ - RequestHeaders: downloaderHeaders} - return downloader.DownloadWithConfig(path.String(), r.URL, downloadConfig) + return downloader.Download(path.String(), r.URL) } diff --git a/arduino/resources/helpers_test.go b/arduino/resources/helpers_test.go index 1e9e872099d..f807a1bc841 100644 --- a/arduino/resources/helpers_test.go +++ b/arduino/resources/helpers_test.go @@ -25,6 +25,7 @@ import ( "github.com/arduino/go-paths-helper" "github.com/stretchr/testify/require" + "go.bug.st/downloader" ) type EchoHandler struct{} @@ -52,7 +53,10 @@ func TestDownloadApplyUserAgentHeaderUsingConfig(t *testing.T) { URL: srv.URL, } - d, err := r.Download(tmp, http.Header{"User-Agent": []string{goldUserAgentValue}}) + prev := downloader.GetDefaultConfig() + downloader.SetDefaultConfig(downloader.Config{RequestHeaders: http.Header{"User-Agent": []string{goldUserAgentValue}}}) + d, err := r.Download(tmp) + downloader.SetDefaultConfig(prev) require.NoError(t, err) err = d.Run() require.NoError(t, err) diff --git a/arduino/resources/resources_test.go b/arduino/resources/resources_test.go index d6f59f75227..8b37a53f72b 100644 --- a/arduino/resources/resources_test.go +++ b/arduino/resources/resources_test.go @@ -18,7 +18,6 @@ package resources import ( "crypto" "encoding/hex" - "net/http" "testing" "github.com/arduino/go-paths-helper" @@ -42,7 +41,7 @@ func TestDownloadAndChecksums(t *testing.T) { require.NoError(t, err) downloadAndTestChecksum := func() { - d, err := r.Download(tmp, http.Header{}) + d, err := r.Download(tmp) require.NoError(t, err) err = d.Run() require.NoError(t, err) @@ -58,7 +57,7 @@ func TestDownloadAndChecksums(t *testing.T) { downloadAndTestChecksum() // Download with cached file - d, err := r.Download(tmp, http.Header{}) + d, err := r.Download(tmp) require.NoError(t, err) require.Nil(t, d) diff --git a/cli/cli.go b/cli/cli.go index 653aaff6d04..068819bf245 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -18,6 +18,7 @@ package cli import ( "fmt" "io/ioutil" + "net/url" "os" "path/filepath" "strings" @@ -45,6 +46,7 @@ import ( "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/viper" + "go.bug.st/downloader" ) var ( @@ -254,4 +256,12 @@ func preRun(cmd *cobra.Command, args []string) { os.Exit(errorcodes.ErrBadCall) }) } + + // + // Configure network + // + netConf := downloader.Config{ + RequestHeaders: globals.NewHTTPClientHeader(), + } + downloader.SetDefaultConfig(netConf) } diff --git a/cli/core/download.go b/cli/core/download.go index 0c7b4227710..273c66b48df 100644 --- a/cli/core/download.go +++ b/cli/core/download.go @@ -66,8 +66,7 @@ func runDownloadCommand(cmd *cobra.Command, args []string) { Architecture: platformRef.Architecture, Version: platformRef.Version, } - _, err := core.PlatformDownload(context.Background(), platformDownloadreq, output.ProgressBar(), - globals.NewHTTPClientHeader()) + _, err := core.PlatformDownload(context.Background(), platformDownloadreq, output.ProgressBar()) if err != nil { feedback.Errorf("Error downloading %s: %v", args[i], err) os.Exit(errorcodes.ErrNetwork) diff --git a/cli/core/install.go b/cli/core/install.go index 62a8eb6ed7b..011ad7679bd 100644 --- a/cli/core/install.go +++ b/cli/core/install.go @@ -67,8 +67,7 @@ func runInstallCommand(cmd *cobra.Command, args []string) { Architecture: platformRef.Architecture, Version: platformRef.Version, } - _, err := core.PlatformInstall(context.Background(), plattformInstallReq, output.ProgressBar(), - output.TaskProgress(), globals.NewHTTPClientHeader()) + _, err := core.PlatformInstall(context.Background(), plattformInstallReq, output.ProgressBar(), output.TaskProgress()) if err != nil { feedback.Errorf("Error during install: %v", err) os.Exit(errorcodes.ErrGeneric) diff --git a/cli/core/upgrade.go b/cli/core/upgrade.go index 9c7cfdfcfa8..febc1bcebcf 100644 --- a/cli/core/upgrade.go +++ b/cli/core/upgrade.go @@ -93,7 +93,7 @@ func runUpgradeCommand(cmd *cobra.Command, args []string) { Architecture: platformRef.Architecture, } - _, err := core.PlatformUpgrade(context.Background(), r, output.ProgressBar(), output.TaskProgress(), globals.NewHTTPClientHeader()) + _, err := core.PlatformUpgrade(context.Background(), r, output.ProgressBar(), output.TaskProgress()) if err == core.ErrAlreadyLatest { feedback.Printf("Platform %s is already at the latest version", platformRef) } else if err != nil { diff --git a/cli/daemon/daemon.go b/cli/daemon/daemon.go index 5fa00ca20ca..fbac9207e45 100644 --- a/cli/daemon/daemon.go +++ b/cli/daemon/daemon.go @@ -21,7 +21,6 @@ import ( "io" "io/ioutil" "net" - "net/http" "os" "runtime" "syscall" @@ -39,6 +38,7 @@ import ( "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/viper" + "go.bug.st/downloader" "google.golang.org/grpc" ) @@ -71,22 +71,19 @@ func runDaemonCommand(cmd *cobra.Command, args []string) { port := viper.GetString("daemon.port") s := grpc.NewServer() - // Compose user agent header - headers := http.Header{ - "User-Agent": []string{ - fmt.Sprintf("%s/%s daemon (%s; %s; %s) Commit:%s", - globals.VersionInfo.Application, - globals.VersionInfo.VersionString, - runtime.GOARCH, - runtime.GOOS, - runtime.Version(), - globals.VersionInfo.Commit), - }, - } - // Register the commands service + // Set specific user-agent for the daemon + netConf := downloader.GetDefaultConfig() + netConf.RequestHeaders.Set("User-Agent", + fmt.Sprintf("%s/%s daemon (%s; %s; %s) Commit:%s", + globals.VersionInfo.Application, + globals.VersionInfo.VersionString, + runtime.GOARCH, runtime.GOOS, + runtime.Version(), globals.VersionInfo.Commit)) + downloader.SetDefaultConfig(netConf) + + // register the commands service srv_commands.RegisterArduinoCoreServer(s, &daemon.ArduinoCoreServerImpl{ - DownloaderHeaders: headers, - VersionString: globals.VersionInfo.VersionString, + VersionString: globals.VersionInfo.VersionString, }) // Register the monitors service diff --git a/cli/instance/instance.go b/cli/instance/instance.go index 06f8e94ce11..37c0d355323 100644 --- a/cli/instance/instance.go +++ b/cli/instance/instance.go @@ -18,7 +18,6 @@ package instance import ( "context" - "github.com/arduino/arduino-cli/cli/globals" "github.com/arduino/arduino-cli/cli/output" "github.com/arduino/arduino-cli/commands" rpc "github.com/arduino/arduino-cli/rpc/commands" @@ -45,8 +44,7 @@ func CreateInstance() (*rpc.Instance, error) { func getInitResponse() (*rpc.InitResp, error) { // invoke Init() - resp, err := commands.Init(context.Background(), &rpc.InitReq{}, - output.ProgressBar(), output.TaskProgress(), globals.NewHTTPClientHeader()) + resp, err := commands.Init(context.Background(), &rpc.InitReq{}, output.ProgressBar(), output.TaskProgress()) // Init() failed if err != nil { diff --git a/cli/lib/download.go b/cli/lib/download.go index b1d1e012f5b..2419df395b4 100644 --- a/cli/lib/download.go +++ b/cli/lib/download.go @@ -21,7 +21,6 @@ import ( "github.com/arduino/arduino-cli/cli/errorcodes" "github.com/arduino/arduino-cli/cli/feedback" - "github.com/arduino/arduino-cli/cli/globals" "github.com/arduino/arduino-cli/cli/instance" "github.com/arduino/arduino-cli/cli/output" "github.com/arduino/arduino-cli/commands/lib" @@ -57,8 +56,7 @@ func runDownloadCommand(cmd *cobra.Command, args []string) { Name: library.Name, Version: library.Version, } - _, err := lib.LibraryDownload(context.Background(), libraryDownloadReq, output.ProgressBar(), - globals.NewHTTPClientHeader()) + _, err := lib.LibraryDownload(context.Background(), libraryDownloadReq, output.ProgressBar()) if err != nil { feedback.Errorf("Error downloading %s: %v", library, err) os.Exit(errorcodes.ErrNetwork) diff --git a/cli/lib/install.go b/cli/lib/install.go index c9a0732d96b..8ed0bf64ab6 100644 --- a/cli/lib/install.go +++ b/cli/lib/install.go @@ -21,7 +21,6 @@ import ( "github.com/arduino/arduino-cli/cli/errorcodes" "github.com/arduino/arduino-cli/cli/feedback" - "github.com/arduino/arduino-cli/cli/globals" "github.com/arduino/arduino-cli/cli/instance" "github.com/arduino/arduino-cli/cli/output" "github.com/arduino/arduino-cli/commands/lib" @@ -95,8 +94,7 @@ func runInstallCommand(cmd *cobra.Command, args []string) { Name: library.Name, Version: library.VersionRequired, } - err := lib.LibraryInstall(context.Background(), libraryInstallReq, output.ProgressBar(), - output.TaskProgress(), globals.NewHTTPClientHeader()) + err := lib.LibraryInstall(context.Background(), libraryInstallReq, output.ProgressBar(), output.TaskProgress()) if err != nil { feedback.Errorf("Error installing %s: %v", library, err) os.Exit(errorcodes.ErrGeneric) diff --git a/cli/lib/upgrade.go b/cli/lib/upgrade.go index 584fa711c5a..7da8fe87856 100644 --- a/cli/lib/upgrade.go +++ b/cli/lib/upgrade.go @@ -20,7 +20,6 @@ import ( "github.com/arduino/arduino-cli/cli/errorcodes" "github.com/arduino/arduino-cli/cli/feedback" - "github.com/arduino/arduino-cli/cli/globals" "github.com/arduino/arduino-cli/cli/instance" "github.com/arduino/arduino-cli/cli/output" "github.com/arduino/arduino-cli/commands/lib" @@ -48,13 +47,13 @@ func runUpgradeCommand(cmd *cobra.Command, args []string) { instance := instance.CreateInstanceIgnorePlatformIndexErrors() if len(args) == 0 { - err := lib.LibraryUpgradeAll(instance.Id, output.ProgressBar(), output.TaskProgress(), globals.NewHTTPClientHeader()) + err := lib.LibraryUpgradeAll(instance.Id, output.ProgressBar(), output.TaskProgress()) if err != nil { feedback.Errorf("Error upgrading libraries: %v", err) os.Exit(errorcodes.ErrGeneric) } } else { - err := lib.LibraryUpgrade(instance.Id, args, output.ProgressBar(), output.TaskProgress(), globals.NewHTTPClientHeader()) + err := lib.LibraryUpgrade(instance.Id, args, output.ProgressBar(), output.TaskProgress()) if err != nil { feedback.Errorf("Error upgrading libraries: %v", err) os.Exit(errorcodes.ErrGeneric) diff --git a/commands/bundled_tools.go b/commands/bundled_tools.go index 8e6398762be..ec0a350ff61 100644 --- a/commands/bundled_tools.go +++ b/commands/bundled_tools.go @@ -17,7 +17,6 @@ package commands import ( "fmt" - "net/http" "github.com/arduino/arduino-cli/arduino/cores" "github.com/arduino/arduino-cli/arduino/cores/packagemanager" @@ -25,9 +24,8 @@ import ( ) // DownloadToolRelease downloads a ToolRelease -func DownloadToolRelease(pm *packagemanager.PackageManager, toolRelease *cores.ToolRelease, - downloadCB DownloadProgressCB, downloaderHeaders http.Header) error { - resp, err := pm.DownloadToolRelease(toolRelease, downloaderHeaders) +func DownloadToolRelease(pm *packagemanager.PackageManager, toolRelease *cores.ToolRelease, downloadCB DownloadProgressCB) error { + resp, err := pm.DownloadToolRelease(toolRelease) if err != nil { return err } diff --git a/commands/core/download.go b/commands/core/download.go index 2bc848cae7e..4d691bb2b60 100644 --- a/commands/core/download.go +++ b/commands/core/download.go @@ -19,7 +19,6 @@ import ( "context" "errors" "fmt" - "net/http" "github.com/arduino/arduino-cli/arduino/cores" "github.com/arduino/arduino-cli/arduino/cores/packagemanager" @@ -28,8 +27,7 @@ import ( ) // PlatformDownload FIXMEDOC -func PlatformDownload(ctx context.Context, req *rpc.PlatformDownloadReq, downloadCB commands.DownloadProgressCB, - downloaderHeaders http.Header) (*rpc.PlatformDownloadResp, error) { +func PlatformDownload(ctx context.Context, req *rpc.PlatformDownloadReq, downloadCB commands.DownloadProgressCB) (*rpc.PlatformDownloadResp, error) { pm := commands.GetPackageManager(req.GetInstance().GetId()) if pm == nil { return nil, errors.New("invalid instance") @@ -49,13 +47,13 @@ func PlatformDownload(ctx context.Context, req *rpc.PlatformDownloadReq, downloa return nil, fmt.Errorf("find platform dependencies: %s", err) } - err = downloadPlatform(pm, platform, downloadCB, downloaderHeaders) + err = downloadPlatform(pm, platform, downloadCB) if err != nil { return nil, err } for _, tool := range tools { - err := downloadTool(pm, tool, downloadCB, downloaderHeaders) + err := downloadTool(pm, tool, downloadCB) if err != nil { return nil, fmt.Errorf("downloading tool %s: %s", tool, err) } @@ -64,22 +62,20 @@ func PlatformDownload(ctx context.Context, req *rpc.PlatformDownloadReq, downloa return &rpc.PlatformDownloadResp{}, nil } -func downloadPlatform(pm *packagemanager.PackageManager, platformRelease *cores.PlatformRelease, - downloadCB commands.DownloadProgressCB, downloaderHeaders http.Header) error { +func downloadPlatform(pm *packagemanager.PackageManager, platformRelease *cores.PlatformRelease, downloadCB commands.DownloadProgressCB) error { // Download platform - resp, err := pm.DownloadPlatformRelease(platformRelease, downloaderHeaders) + resp, err := pm.DownloadPlatformRelease(platformRelease) if err != nil { return err } return commands.Download(resp, platformRelease.String(), downloadCB) } -func downloadTool(pm *packagemanager.PackageManager, tool *cores.ToolRelease, downloadCB commands.DownloadProgressCB, - downloaderHeaders http.Header) error { +func downloadTool(pm *packagemanager.PackageManager, tool *cores.ToolRelease, downloadCB commands.DownloadProgressCB) error { // Check if tool has a flavor available for the current OS if tool.GetCompatibleFlavour() == nil { return fmt.Errorf("tool %s not available for the current OS", tool) } - return commands.DownloadToolRelease(pm, tool, downloadCB, downloaderHeaders) + return commands.DownloadToolRelease(pm, tool, downloadCB) } diff --git a/commands/core/install.go b/commands/core/install.go index bf01129effc..0d2e49deb73 100644 --- a/commands/core/install.go +++ b/commands/core/install.go @@ -19,7 +19,6 @@ import ( "context" "errors" "fmt" - "net/http" "github.com/arduino/arduino-cli/arduino/cores" "github.com/arduino/arduino-cli/arduino/cores/packagemanager" @@ -29,7 +28,7 @@ import ( // PlatformInstall FIXMEDOC func PlatformInstall(ctx context.Context, req *rpc.PlatformInstallReq, - downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB, downloaderHeaders http.Header) (*rpc.PlatformInstallResp, error) { + downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB) (*rpc.PlatformInstallResp, error) { pm := commands.GetPackageManager(req.GetInstance().GetId()) if pm == nil { @@ -50,7 +49,7 @@ func PlatformInstall(ctx context.Context, req *rpc.PlatformInstallReq, return nil, fmt.Errorf("finding platform dependencies: %s", err) } - err = installPlatform(pm, platform, tools, downloadCB, taskCB, downloaderHeaders) + err = installPlatform(pm, platform, tools, downloadCB, taskCB) if err != nil { return nil, err } @@ -65,7 +64,7 @@ func PlatformInstall(ctx context.Context, req *rpc.PlatformInstallReq, func installPlatform(pm *packagemanager.PackageManager, platformRelease *cores.PlatformRelease, requiredTools []*cores.ToolRelease, - downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB, downloaderHeaders http.Header) error { + downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB) error { log := pm.Log.WithField("platform", platformRelease) // Prerequisite checks before install @@ -87,9 +86,9 @@ func installPlatform(pm *packagemanager.PackageManager, // Package download taskCB(&rpc.TaskProgress{Name: "Downloading packages"}) for _, tool := range toolsToInstall { - downloadTool(pm, tool, downloadCB, downloaderHeaders) + downloadTool(pm, tool, downloadCB) } - downloadPlatform(pm, platformRelease, downloadCB, downloaderHeaders) + downloadPlatform(pm, platformRelease, downloadCB) taskCB(&rpc.TaskProgress{Completed: true}) // Install tools first diff --git a/commands/core/upgrade.go b/commands/core/upgrade.go index 392e8bf4454..1297496da1c 100644 --- a/commands/core/upgrade.go +++ b/commands/core/upgrade.go @@ -19,7 +19,6 @@ import ( "context" "errors" "fmt" - "net/http" "github.com/arduino/arduino-cli/arduino/cores/packagemanager" "github.com/arduino/arduino-cli/commands" @@ -34,7 +33,7 @@ var ( // PlatformUpgrade FIXMEDOC func PlatformUpgrade(ctx context.Context, req *rpc.PlatformUpgradeReq, - downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB, downloaderHeaders http.Header) (*rpc.PlatformUpgradeResp, error) { + downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB) (*rpc.PlatformUpgradeResp, error) { pm := commands.GetPackageManager(req.GetInstance().GetId()) if pm == nil { @@ -46,7 +45,7 @@ func PlatformUpgrade(ctx context.Context, req *rpc.PlatformUpgradeReq, Package: req.PlatformPackage, PlatformArchitecture: req.Architecture, } - if err := upgradePlatform(pm, ref, downloadCB, taskCB, downloaderHeaders); err != nil { + if err := upgradePlatform(pm, ref, downloadCB, taskCB); err != nil { return nil, err } @@ -58,7 +57,7 @@ func PlatformUpgrade(ctx context.Context, req *rpc.PlatformUpgradeReq, } func upgradePlatform(pm *packagemanager.PackageManager, platformRef *packagemanager.PlatformReference, - downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB, downloaderHeaders http.Header) error { + downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB) error { if platformRef.PlatformVersion != nil { return fmt.Errorf("upgrade doesn't accept parameters with version") } @@ -85,7 +84,7 @@ func upgradePlatform(pm *packagemanager.PackageManager, platformRef *packagemana if err != nil { return fmt.Errorf("platform %s is not installed", platformRef) } - err = installPlatform(pm, platform, tools, downloadCB, taskCB, downloaderHeaders) + err = installPlatform(pm, platform, tools, downloadCB, taskCB) if err != nil { return err } diff --git a/commands/daemon/daemon.go b/commands/daemon/daemon.go index 579e4abd0f2..2dd088beff9 100644 --- a/commands/daemon/daemon.go +++ b/commands/daemon/daemon.go @@ -19,7 +19,6 @@ package daemon import ( "context" - "net/http" "github.com/arduino/arduino-cli/arduino/utils" "github.com/arduino/arduino-cli/commands" @@ -33,8 +32,7 @@ import ( // ArduinoCoreServerImpl FIXMEDOC type ArduinoCoreServerImpl struct { - DownloaderHeaders http.Header - VersionString string + VersionString string } // BoardDetails FIXMEDOC @@ -108,7 +106,6 @@ func (s *ArduinoCoreServerImpl) Init(req *rpc.InitReq, stream rpc.ArduinoCore_In resp, err := commands.Init(stream.Context(), req, func(p *rpc.DownloadProgress) { stream.Send(&rpc.InitResp{DownloadProgress: p}) }, func(p *rpc.TaskProgress) { stream.Send(&rpc.InitResp{TaskProgress: p}) }, - s.DownloaderHeaders, ) if err != nil { return err @@ -140,7 +137,6 @@ func (s *ArduinoCoreServerImpl) PlatformInstall(req *rpc.PlatformInstallReq, str stream.Context(), req, func(p *rpc.DownloadProgress) { stream.Send(&rpc.PlatformInstallResp{Progress: p}) }, func(p *rpc.TaskProgress) { stream.Send(&rpc.PlatformInstallResp{TaskProgress: p}) }, - s.DownloaderHeaders, ) if err != nil { return err @@ -153,7 +149,6 @@ func (s *ArduinoCoreServerImpl) PlatformDownload(req *rpc.PlatformDownloadReq, s resp, err := core.PlatformDownload( stream.Context(), req, func(p *rpc.DownloadProgress) { stream.Send(&rpc.PlatformDownloadResp{Progress: p}) }, - s.DownloaderHeaders, ) if err != nil { return err @@ -179,7 +174,6 @@ func (s *ArduinoCoreServerImpl) PlatformUpgrade(req *rpc.PlatformUpgradeReq, str stream.Context(), req, func(p *rpc.DownloadProgress) { stream.Send(&rpc.PlatformUpgradeResp{Progress: p}) }, func(p *rpc.TaskProgress) { stream.Send(&rpc.PlatformUpgradeResp{TaskProgress: p}) }, - s.DownloaderHeaders, ) if err != nil { return err @@ -227,7 +221,6 @@ func (s *ArduinoCoreServerImpl) LibraryDownload(req *rpc.LibraryDownloadReq, str resp, err := lib.LibraryDownload( stream.Context(), req, func(p *rpc.DownloadProgress) { stream.Send(&rpc.LibraryDownloadResp{Progress: p}) }, - s.DownloaderHeaders, ) if err != nil { return err @@ -241,7 +234,6 @@ func (s *ArduinoCoreServerImpl) LibraryInstall(req *rpc.LibraryInstallReq, strea stream.Context(), req, func(p *rpc.DownloadProgress) { stream.Send(&rpc.LibraryInstallResp{Progress: p}) }, func(p *rpc.TaskProgress) { stream.Send(&rpc.LibraryInstallResp{TaskProgress: p}) }, - s.DownloaderHeaders, ) if err != nil { return err @@ -265,7 +257,6 @@ func (s *ArduinoCoreServerImpl) LibraryUpgradeAll(req *rpc.LibraryUpgradeAllReq, err := lib.LibraryUpgradeAll(req.GetInstance().GetId(), func(p *rpc.DownloadProgress) { stream.Send(&rpc.LibraryUpgradeAllResp{Progress: p}) }, func(p *rpc.TaskProgress) { stream.Send(&rpc.LibraryUpgradeAllResp{TaskProgress: p}) }, - s.DownloaderHeaders, ) if err != nil { return err diff --git a/commands/instances.go b/commands/instances.go index a995a525164..77a1cc91606 100644 --- a/commands/instances.go +++ b/commands/instances.go @@ -19,7 +19,6 @@ import ( "context" "fmt" "io/ioutil" - "net/http" "net/url" "path" "time" @@ -89,13 +88,12 @@ func GetLibraryManager(instanceID int32) *librariesmanager.LibrariesManager { return i.lm } -func (instance *CoreInstance) installToolIfMissing(tool *cores.ToolRelease, downloadCB DownloadProgressCB, - taskCB TaskProgressCB, downloaderHeaders http.Header) (bool, error) { +func (instance *CoreInstance) installToolIfMissing(tool *cores.ToolRelease, downloadCB DownloadProgressCB, taskCB TaskProgressCB) (bool, error) { if tool.IsInstalled() { return false, nil } taskCB(&rpc.TaskProgress{Name: "Downloading missing tool " + tool.String()}) - if err := DownloadToolRelease(instance.PackageManager, tool, downloadCB, downloaderHeaders); err != nil { + if err := DownloadToolRelease(instance.PackageManager, tool, downloadCB); err != nil { return false, fmt.Errorf("downloading %s tool: %s", tool, err) } taskCB(&rpc.TaskProgress{Completed: true}) @@ -105,18 +103,17 @@ func (instance *CoreInstance) installToolIfMissing(tool *cores.ToolRelease, down return true, nil } -func (instance *CoreInstance) checkForBuiltinTools(downloadCB DownloadProgressCB, taskCB TaskProgressCB, - downloaderHeaders http.Header) error { +func (instance *CoreInstance) checkForBuiltinTools(downloadCB DownloadProgressCB, taskCB TaskProgressCB) error { // Check for ctags tool ctags, _ := getBuiltinCtagsTool(instance.PackageManager) - ctagsInstalled, err := instance.installToolIfMissing(ctags, downloadCB, taskCB, downloaderHeaders) + ctagsInstalled, err := instance.installToolIfMissing(ctags, downloadCB, taskCB) if err != nil { return err } // Check for bultin serial-discovery tool serialDiscoveryTool, _ := getBuiltinSerialDiscoveryTool(instance.PackageManager) - serialDiscoveryInstalled, err := instance.installToolIfMissing(serialDiscoveryTool, downloadCB, taskCB, downloaderHeaders) + serialDiscoveryInstalled, err := instance.installToolIfMissing(serialDiscoveryTool, downloadCB, taskCB) if err != nil { return err } @@ -130,9 +127,7 @@ func (instance *CoreInstance) checkForBuiltinTools(downloadCB DownloadProgressCB } // Init FIXMEDOC -func Init(ctx context.Context, req *rpc.InitReq, downloadCB DownloadProgressCB, taskCB TaskProgressCB, - downloaderHeaders http.Header) (*rpc.InitResp, error) { - +func Init(ctx context.Context, req *rpc.InitReq, downloadCB DownloadProgressCB, taskCB TaskProgressCB) (*rpc.InitResp, error) { res, err := createInstance(ctx, req.GetLibraryManagerOnly()) if err != nil { return nil, fmt.Errorf("cannot initialize package manager: %s", err) @@ -147,7 +142,7 @@ func Init(ctx context.Context, req *rpc.InitReq, downloadCB DownloadProgressCB, instancesCount++ instances[handle] = instance - if err := instance.checkForBuiltinTools(downloadCB, taskCB, downloaderHeaders); err != nil { + if err := instance.checkForBuiltinTools(downloadCB, taskCB); err != nil { return nil, err } diff --git a/commands/lib/download.go b/commands/lib/download.go index a133c1fe886..2e9df47f1a0 100644 --- a/commands/lib/download.go +++ b/commands/lib/download.go @@ -18,7 +18,6 @@ package lib import ( "context" "fmt" - "net/http" "github.com/arduino/arduino-cli/arduino/libraries/librariesindex" "github.com/arduino/arduino-cli/arduino/libraries/librariesmanager" @@ -28,8 +27,7 @@ import ( ) // LibraryDownload FIXMEDOC -func LibraryDownload(ctx context.Context, req *rpc.LibraryDownloadReq, downloadCB commands.DownloadProgressCB, - downloaderHeaders http.Header) (*rpc.LibraryDownloadResp, error) { +func LibraryDownload(ctx context.Context, req *rpc.LibraryDownloadReq, downloadCB commands.DownloadProgressCB) (*rpc.LibraryDownloadResp, error) { logrus.Info("Executing `arduino lib download`") lm := commands.GetLibraryManager(req.GetInstance().GetId()) @@ -41,7 +39,7 @@ func LibraryDownload(ctx context.Context, req *rpc.LibraryDownloadReq, downloadC return nil, fmt.Errorf("looking for library: %s", err) } - if err := downloadLibrary(lm, lib, downloadCB, func(*rpc.TaskProgress) {}, downloaderHeaders); err != nil { + if err := downloadLibrary(lm, lib, downloadCB, func(*rpc.TaskProgress) {}); err != nil { return nil, err } @@ -49,10 +47,10 @@ func LibraryDownload(ctx context.Context, req *rpc.LibraryDownloadReq, downloadC } func downloadLibrary(lm *librariesmanager.LibrariesManager, libRelease *librariesindex.Release, - downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB, downloaderHeaders http.Header) error { + downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB) error { taskCB(&rpc.TaskProgress{Name: "Downloading " + libRelease.String()}) - if d, err := libRelease.Resource.Download(lm.DownloadsDir, downloaderHeaders); err != nil { + if d, err := libRelease.Resource.Download(lm.DownloadsDir); err != nil { return err } else if err := commands.Download(d, libRelease.String(), downloadCB); err != nil { return err diff --git a/commands/lib/install.go b/commands/lib/install.go index 714c8e352c8..93fad6f8664 100644 --- a/commands/lib/install.go +++ b/commands/lib/install.go @@ -18,7 +18,6 @@ package lib import ( "context" "fmt" - "net/http" "github.com/arduino/arduino-cli/arduino/libraries/librariesindex" "github.com/arduino/arduino-cli/arduino/libraries/librariesmanager" @@ -29,7 +28,7 @@ import ( // LibraryInstall FIXMEDOC func LibraryInstall(ctx context.Context, req *rpc.LibraryInstallReq, - downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB, downloaderHeaders http.Header) error { + downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB) error { lm := commands.GetLibraryManager(req.GetInstance().GetId()) @@ -38,7 +37,7 @@ func LibraryInstall(ctx context.Context, req *rpc.LibraryInstallReq, return fmt.Errorf("looking for library: %s", err) } - if err := downloadLibrary(lm, libRelease, downloadCB, taskCB, downloaderHeaders); err != nil { + if err := downloadLibrary(lm, libRelease, downloadCB, taskCB); err != nil { return fmt.Errorf("downloading library: %s", err) } diff --git a/commands/lib/upgrade.go b/commands/lib/upgrade.go index c991cee1399..d85526a5fab 100644 --- a/commands/lib/upgrade.go +++ b/commands/lib/upgrade.go @@ -17,7 +17,6 @@ package lib import ( "fmt" - "net/http" "github.com/arduino/arduino-cli/arduino/libraries/librariesmanager" "github.com/arduino/arduino-cli/commands" @@ -25,11 +24,11 @@ import ( // LibraryUpgradeAll upgrades all the available libraries func LibraryUpgradeAll(instanceID int32, downloadCB commands.DownloadProgressCB, - taskCB commands.TaskProgressCB, headers http.Header) error { + taskCB commands.TaskProgressCB) error { // get the library manager lm := commands.GetLibraryManager(instanceID) - if err := upgrade(lm, listLibraries(lm, true, true), downloadCB, taskCB, headers); err != nil { + if err := upgrade(lm, listLibraries(lm, true, true), downloadCB, taskCB); err != nil { return err } @@ -42,7 +41,7 @@ func LibraryUpgradeAll(instanceID int32, downloadCB commands.DownloadProgressCB, // LibraryUpgrade upgrades only the given libraries func LibraryUpgrade(instanceID int32, libraryNames []string, downloadCB commands.DownloadProgressCB, - taskCB commands.TaskProgressCB, headers http.Header) error { + taskCB commands.TaskProgressCB) error { // get the library manager lm := commands.GetLibraryManager(instanceID) @@ -50,16 +49,16 @@ func LibraryUpgrade(instanceID int32, libraryNames []string, downloadCB commands libs := filterByName(listLibraries(lm, true, true), libraryNames) // do it - return upgrade(lm, libs, downloadCB, taskCB, headers) + return upgrade(lm, libs, downloadCB, taskCB) } func upgrade(lm *librariesmanager.LibrariesManager, libs []*installedLib, downloadCB commands.DownloadProgressCB, - taskCB commands.TaskProgressCB, downloaderHeaders http.Header) error { + taskCB commands.TaskProgressCB) error { // Go through the list and download them for _, lib := range libs { - if err := downloadLibrary(lm, lib.Available, downloadCB, taskCB, downloaderHeaders); err != nil { + if err := downloadLibrary(lm, lib.Available, downloadCB, taskCB); err != nil { return err } } From 3b4c4e2f70cfeb585e9260dd3a7360a6bbbd71f1 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 6 Mar 2020 12:03:22 +0100 Subject: [PATCH 3/7] Add support for network proxy configuration --- cli/cli.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cli/cli.go b/cli/cli.go index 068819bf245..0b3e1c9ebb6 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -263,5 +263,14 @@ func preRun(cmd *cobra.Command, args []string) { netConf := downloader.Config{ RequestHeaders: globals.NewHTTPClientHeader(), } + if viper.IsSet("network.proxy") { + proxy := viper.GetString("network.proxy") + if _, err := url.Parse(proxy); err != nil { + feedback.Error("Invalid network.proxy '" + proxy + "': " + err.Error()) + os.Exit(errorcodes.ErrBadArgument) + } + netConf.ProxyURL = proxy + logrus.Infof("Using proxy %s", proxy) + } downloader.SetDefaultConfig(netConf) } From cbf387af763f1edca172b079fedb498213706ae7 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 6 Mar 2020 12:03:57 +0100 Subject: [PATCH 4/7] Cosmetic: added blank space in configuration/defaults.go --- configuration/defaults.go | 1 + 1 file changed, 1 insertion(+) diff --git a/configuration/defaults.go b/configuration/defaults.go index 262edca08d8..4ccf1c380b1 100644 --- a/configuration/defaults.go +++ b/configuration/defaults.go @@ -25,6 +25,7 @@ func setDefaults(dataDir, userDir string) { // logging viper.SetDefault("logging.level", "info") viper.SetDefault("logging.format", "text") + // board manager viper.SetDefault("board_manager.additional_urls", []string{}) From 0b8cd8d4865a3c1df08464ada9a0ecc9ccffbb7e Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 6 Mar 2020 16:37:35 +0100 Subject: [PATCH 5/7] Refactored function to generate user-agent --- cli/cli.go | 2 +- cli/daemon/daemon.go | 8 +------- cli/globals/globals.go | 14 +++++++++++--- commands/board/list.go | 2 +- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/cli/cli.go b/cli/cli.go index 0b3e1c9ebb6..bf9fefe79a5 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -261,7 +261,7 @@ func preRun(cmd *cobra.Command, args []string) { // Configure network // netConf := downloader.Config{ - RequestHeaders: globals.NewHTTPClientHeader(), + RequestHeaders: globals.NewHTTPClientHeader(""), } if viper.IsSet("network.proxy") { proxy := viper.GetString("network.proxy") diff --git a/cli/daemon/daemon.go b/cli/daemon/daemon.go index fbac9207e45..038f6865d2e 100644 --- a/cli/daemon/daemon.go +++ b/cli/daemon/daemon.go @@ -22,7 +22,6 @@ import ( "io/ioutil" "net" "os" - "runtime" "syscall" "github.com/arduino/arduino-cli/cli/errorcodes" @@ -73,12 +72,7 @@ func runDaemonCommand(cmd *cobra.Command, args []string) { // Set specific user-agent for the daemon netConf := downloader.GetDefaultConfig() - netConf.RequestHeaders.Set("User-Agent", - fmt.Sprintf("%s/%s daemon (%s; %s; %s) Commit:%s", - globals.VersionInfo.Application, - globals.VersionInfo.VersionString, - runtime.GOARCH, runtime.GOOS, - runtime.Version(), globals.VersionInfo.Commit)) + netConf.RequestHeaders = globals.NewHTTPClientHeader("daemon") downloader.SetDefaultConfig(netConf) // register the commands service diff --git a/cli/globals/globals.go b/cli/globals/globals.go index 0d20aac245e..5222590dc85 100644 --- a/cli/globals/globals.go +++ b/cli/globals/globals.go @@ -33,8 +33,16 @@ var ( ) // NewHTTPClientHeader returns the http.Header object that must be used by the clients inside the downloaders -func NewHTTPClientHeader() http.Header { - userAgentValue := fmt.Sprintf("%s/%s (%s; %s; %s) Commit:%s", VersionInfo.Application, - VersionInfo.VersionString, runtime.GOARCH, runtime.GOOS, runtime.Version(), VersionInfo.Commit) +// and adds the subComponent if specified +func NewHTTPClientHeader(subComponent string) http.Header { + if subComponent != "" { + subComponent = " " + subComponent + } + userAgentValue := fmt.Sprintf("%s/%s%s (%s; %s; %s) Commit:%s", + VersionInfo.Application, + VersionInfo.VersionString, + subComponent, + runtime.GOARCH, runtime.GOOS, runtime.Version(), + VersionInfo.Commit) return http.Header{"User-Agent": []string{userAgentValue}} } diff --git a/commands/board/list.go b/commands/board/list.go index 0ec7ba6ec64..de44e8b9da4 100644 --- a/commands/board/list.go +++ b/commands/board/list.go @@ -51,7 +51,7 @@ func apiByVidPid(vid, pid string) ([]*rpc.BoardListItem, error) { url := fmt.Sprintf("%s/%s/%s", vidPidURL, vid, pid) retVal := []*rpc.BoardListItem{} req, _ := http.NewRequest("GET", url, nil) - req.Header = globals.NewHTTPClientHeader() + req.Header = globals.NewHTTPClientHeader("") req.Header.Set("Content-Type", "application/json") if res, err := http.DefaultClient.Do(req); err == nil { From 0f0dd7504b81c05f8c1ad6f46c19554d68269868 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 6 Mar 2020 16:37:54 +0100 Subject: [PATCH 6/7] Added a TODO for missing proxy usage in board list API query --- commands/board/list.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/commands/board/list.go b/commands/board/list.go index de44e8b9da4..b77b743a654 100644 --- a/commands/board/list.go +++ b/commands/board/list.go @@ -54,6 +54,8 @@ func apiByVidPid(vid, pid string) ([]*rpc.BoardListItem, error) { req.Header = globals.NewHTTPClientHeader("") req.Header.Set("Content-Type", "application/json") + // TODO: use proxy if set + if res, err := http.DefaultClient.Do(req); err == nil { if res.StatusCode >= 400 { if res.StatusCode == 404 { From 16e0871fc676a0489adf1c9d3b2c4b9829658cdc Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Sun, 12 Apr 2020 23:52:04 +0200 Subject: [PATCH 7/7] Partially revert global network configuration. It's better to read the network configuration before each http request so in case it is changed using "Settings" functions in daemon mode the changes are immediately applied. --- arduino/cores/packagemanager/download.go | 8 +-- .../libraries/librariesmanager/download.go | 4 +- arduino/resources/helpers.go | 4 +- arduino/resources/helpers_test.go | 5 +- arduino/resources/resources_test.go | 5 +- cli/cli.go | 19 ----- cli/daemon/daemon.go | 6 +- commands/bundled_tools.go | 6 +- commands/core/download.go | 6 +- commands/download.go | 71 +++++++++++++++++++ commands/instances.go | 38 +++------- commands/lib/download.go | 6 +- 12 files changed, 109 insertions(+), 69 deletions(-) create mode 100644 commands/download.go diff --git a/arduino/cores/packagemanager/download.go b/arduino/cores/packagemanager/download.go index 8e843795dd4..45da56ecb39 100644 --- a/arduino/cores/packagemanager/download.go +++ b/arduino/cores/packagemanager/download.go @@ -100,16 +100,16 @@ func (pm *PackageManager) FindPlatformReleaseDependencies(item *PlatformReferenc // DownloadToolRelease downloads a ToolRelease. If the tool is already downloaded a nil Downloader // is returned. -func (pm *PackageManager) DownloadToolRelease(tool *cores.ToolRelease) (*downloader.Downloader, error) { +func (pm *PackageManager) DownloadToolRelease(tool *cores.ToolRelease, config *downloader.Config) (*downloader.Downloader, error) { resource := tool.GetCompatibleFlavour() if resource == nil { return nil, fmt.Errorf("tool not available for your OS") } - return resource.Download(pm.DownloadDir) + return resource.Download(pm.DownloadDir, config) } // DownloadPlatformRelease downloads a PlatformRelease. If the platform is already downloaded a // nil Downloader is returned. -func (pm *PackageManager) DownloadPlatformRelease(platform *cores.PlatformRelease) (*downloader.Downloader, error) { - return platform.Resource.Download(pm.DownloadDir) +func (pm *PackageManager) DownloadPlatformRelease(platform *cores.PlatformRelease, config *downloader.Config) (*downloader.Downloader, error) { + return platform.Resource.Download(pm.DownloadDir, config) } diff --git a/arduino/libraries/librariesmanager/download.go b/arduino/libraries/librariesmanager/download.go index 8b4c07feb31..e64a5891991 100644 --- a/arduino/libraries/librariesmanager/download.go +++ b/arduino/libraries/librariesmanager/download.go @@ -25,8 +25,8 @@ import ( var LibraryIndexURL, _ = url.Parse("https://downloads.arduino.cc/libraries/library_index.json") // UpdateIndex downloads the libraries index file from Arduino repository. -func (lm *LibrariesManager) UpdateIndex() (*downloader.Downloader, error) { +func (lm *LibrariesManager) UpdateIndex(config *downloader.Config) (*downloader.Downloader, error) { lm.IndexFile.Parent().MkdirAll() // TODO: Download from gzipped URL index - return downloader.Download(lm.IndexFile.String(), LibraryIndexURL.String(), downloader.NoResume) + return downloader.DownloadWithConfig(lm.IndexFile.String(), LibraryIndexURL.String(), *config, downloader.NoResume) } diff --git a/arduino/resources/helpers.go b/arduino/resources/helpers.go index d568da14be9..485f9410d97 100644 --- a/arduino/resources/helpers.go +++ b/arduino/resources/helpers.go @@ -43,7 +43,7 @@ func (r *DownloadResource) IsCached(downloadDir *paths.Path) (bool, error) { } // Download a DownloadResource. -func (r *DownloadResource) Download(downloadDir *paths.Path) (*downloader.Downloader, error) { +func (r *DownloadResource) Download(downloadDir *paths.Path, config *downloader.Config) (*downloader.Downloader, error) { cached, err := r.TestLocalArchiveIntegrity(downloadDir) if err != nil { return nil, fmt.Errorf("testing local archive integrity: %s", err) @@ -71,5 +71,5 @@ func (r *DownloadResource) Download(downloadDir *paths.Path) (*downloader.Downlo return nil, fmt.Errorf("getting archive file info: %s", err) } - return downloader.Download(path.String(), r.URL) + return downloader.DownloadWithConfig(path.String(), r.URL, *config) } diff --git a/arduino/resources/helpers_test.go b/arduino/resources/helpers_test.go index f807a1bc841..20733cdb0e0 100644 --- a/arduino/resources/helpers_test.go +++ b/arduino/resources/helpers_test.go @@ -53,10 +53,7 @@ func TestDownloadApplyUserAgentHeaderUsingConfig(t *testing.T) { URL: srv.URL, } - prev := downloader.GetDefaultConfig() - downloader.SetDefaultConfig(downloader.Config{RequestHeaders: http.Header{"User-Agent": []string{goldUserAgentValue}}}) - d, err := r.Download(tmp) - downloader.SetDefaultConfig(prev) + d, err := r.Download(tmp, &downloader.Config{RequestHeaders: http.Header{"User-Agent": []string{goldUserAgentValue}}}) require.NoError(t, err) err = d.Run() require.NoError(t, err) diff --git a/arduino/resources/resources_test.go b/arduino/resources/resources_test.go index 8b37a53f72b..5fcdfd43570 100644 --- a/arduino/resources/resources_test.go +++ b/arduino/resources/resources_test.go @@ -22,6 +22,7 @@ import ( "github.com/arduino/go-paths-helper" "github.com/stretchr/testify/require" + "go.bug.st/downloader" ) func TestDownloadAndChecksums(t *testing.T) { @@ -41,7 +42,7 @@ func TestDownloadAndChecksums(t *testing.T) { require.NoError(t, err) downloadAndTestChecksum := func() { - d, err := r.Download(tmp) + d, err := r.Download(tmp, &downloader.Config{}) require.NoError(t, err) err = d.Run() require.NoError(t, err) @@ -57,7 +58,7 @@ func TestDownloadAndChecksums(t *testing.T) { downloadAndTestChecksum() // Download with cached file - d, err := r.Download(tmp) + d, err := r.Download(tmp, &downloader.Config{}) require.NoError(t, err) require.Nil(t, d) diff --git a/cli/cli.go b/cli/cli.go index bf9fefe79a5..653aaff6d04 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -18,7 +18,6 @@ package cli import ( "fmt" "io/ioutil" - "net/url" "os" "path/filepath" "strings" @@ -46,7 +45,6 @@ import ( "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/viper" - "go.bug.st/downloader" ) var ( @@ -256,21 +254,4 @@ func preRun(cmd *cobra.Command, args []string) { os.Exit(errorcodes.ErrBadCall) }) } - - // - // Configure network - // - netConf := downloader.Config{ - RequestHeaders: globals.NewHTTPClientHeader(""), - } - if viper.IsSet("network.proxy") { - proxy := viper.GetString("network.proxy") - if _, err := url.Parse(proxy); err != nil { - feedback.Error("Invalid network.proxy '" + proxy + "': " + err.Error()) - os.Exit(errorcodes.ErrBadArgument) - } - netConf.ProxyURL = proxy - logrus.Infof("Using proxy %s", proxy) - } - downloader.SetDefaultConfig(netConf) } diff --git a/cli/daemon/daemon.go b/cli/daemon/daemon.go index 038f6865d2e..58d23385fea 100644 --- a/cli/daemon/daemon.go +++ b/cli/daemon/daemon.go @@ -37,7 +37,6 @@ import ( "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/viper" - "go.bug.st/downloader" "google.golang.org/grpc" ) @@ -66,14 +65,11 @@ func runDaemonCommand(cmd *cobra.Command, args []string) { stats.Incr("daemon", stats.T("success", "true")) defer stats.Flush() } - port := viper.GetString("daemon.port") s := grpc.NewServer() // Set specific user-agent for the daemon - netConf := downloader.GetDefaultConfig() - netConf.RequestHeaders = globals.NewHTTPClientHeader("daemon") - downloader.SetDefaultConfig(netConf) + viper.Set("network.user_agent_ext", "daemon") // register the commands service srv_commands.RegisterArduinoCoreServer(s, &daemon.ArduinoCoreServerImpl{ diff --git a/commands/bundled_tools.go b/commands/bundled_tools.go index ec0a350ff61..b8ba834b8a0 100644 --- a/commands/bundled_tools.go +++ b/commands/bundled_tools.go @@ -25,7 +25,11 @@ import ( // DownloadToolRelease downloads a ToolRelease func DownloadToolRelease(pm *packagemanager.PackageManager, toolRelease *cores.ToolRelease, downloadCB DownloadProgressCB) error { - resp, err := pm.DownloadToolRelease(toolRelease) + config, err := GetDownloaderConfig() + if err != nil { + return err + } + resp, err := pm.DownloadToolRelease(toolRelease, config) if err != nil { return err } diff --git a/commands/core/download.go b/commands/core/download.go index 4d691bb2b60..2de0232d008 100644 --- a/commands/core/download.go +++ b/commands/core/download.go @@ -64,7 +64,11 @@ func PlatformDownload(ctx context.Context, req *rpc.PlatformDownloadReq, downloa func downloadPlatform(pm *packagemanager.PackageManager, platformRelease *cores.PlatformRelease, downloadCB commands.DownloadProgressCB) error { // Download platform - resp, err := pm.DownloadPlatformRelease(platformRelease) + config, err := commands.GetDownloaderConfig() + if err != nil { + return err + } + resp, err := pm.DownloadPlatformRelease(platformRelease, config) if err != nil { return err } diff --git a/commands/download.go b/commands/download.go new file mode 100644 index 00000000000..e049faa76ee --- /dev/null +++ b/commands/download.go @@ -0,0 +1,71 @@ +// This file is part of arduino-cli. +// +// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +package commands + +import ( + "net/url" + "time" + + "github.com/arduino/arduino-cli/cli/globals" + rpc "github.com/arduino/arduino-cli/rpc/commands" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "github.com/spf13/viper" + "go.bug.st/downloader" +) + +// GetDownloaderConfig returns the downloader configuration based on +// current settings. +func GetDownloaderConfig() (*downloader.Config, error) { + res := &downloader.Config{ + RequestHeaders: globals.NewHTTPClientHeader(viper.GetString("network.user_agent_ext")), + } + if viper.IsSet("network.proxy") { + proxy := viper.GetString("network.proxy") + if _, err := url.Parse(proxy); err != nil { + return nil, errors.New("Invalid network.proxy '" + proxy + "': " + err.Error()) + } + res.ProxyURL = proxy + logrus.Infof("Using proxy %s", proxy) + } + return res, nil +} + +// Download performs a download loop using the provided downloader.Downloader. +// Messages are passed back to the DownloadProgressCB using label as text for the File field. +func Download(d *downloader.Downloader, label string, downloadCB DownloadProgressCB) error { + if d == nil { + // This signal means that the file is already downloaded + downloadCB(&rpc.DownloadProgress{ + File: label, + Completed: true, + }) + return nil + } + downloadCB(&rpc.DownloadProgress{ + File: label, + Url: d.URL, + TotalSize: d.Size(), + }) + d.RunAndPoll(func(downloaded int64) { + downloadCB(&rpc.DownloadProgress{Downloaded: downloaded}) + }, 250*time.Millisecond) + if d.Error() != nil { + return d.Error() + } + downloadCB(&rpc.DownloadProgress{Completed: true}) + return nil +} diff --git a/commands/instances.go b/commands/instances.go index 77a1cc91606..277139a1ca1 100644 --- a/commands/instances.go +++ b/commands/instances.go @@ -21,7 +21,6 @@ import ( "io/ioutil" "net/url" "path" - "time" "github.com/arduino/arduino-cli/arduino/cores" "github.com/arduino/arduino-cli/arduino/cores/packageindex" @@ -171,7 +170,11 @@ func UpdateLibrariesIndex(ctx context.Context, req *rpc.UpdateLibrariesIndexReq, if lm == nil { return fmt.Errorf("invalid handle") } - d, err := lm.UpdateIndex() + config, err := GetDownloaderConfig() + if err != nil { + return err + } + d, err := lm.UpdateIndex(config) if err != nil { return err } @@ -215,7 +218,11 @@ func UpdateIndex(ctx context.Context, req *rpc.UpdateIndexReq, downloadCB Downlo tmp := paths.New(tmpFile.Name()) defer tmp.Remove() - d, err := downloader.Download(tmp.String(), URL.String()) + config, err := GetDownloaderConfig() + if err != nil { + return nil, fmt.Errorf("downloading index %s: %s", URL, err) + } + d, err := downloader.DownloadWithConfig(tmp.String(), URL.String(), *config) if err != nil { return nil, fmt.Errorf("downloading index %s: %s", URL, err) } @@ -348,28 +355,3 @@ func createInstance(ctx context.Context, getLibOnly bool) (*createInstanceResult return res, nil } - -// Download FIXMEDOC -func Download(d *downloader.Downloader, label string, downloadCB DownloadProgressCB) error { - if d == nil { - // This signal means that the file is already downloaded - downloadCB(&rpc.DownloadProgress{ - File: label, - Completed: true, - }) - return nil - } - downloadCB(&rpc.DownloadProgress{ - File: label, - Url: d.URL, - TotalSize: d.Size(), - }) - d.RunAndPoll(func(downloaded int64) { - downloadCB(&rpc.DownloadProgress{Downloaded: downloaded}) - }, 250*time.Millisecond) - if d.Error() != nil { - return d.Error() - } - downloadCB(&rpc.DownloadProgress{Completed: true}) - return nil -} diff --git a/commands/lib/download.go b/commands/lib/download.go index 2e9df47f1a0..117efb3aeeb 100644 --- a/commands/lib/download.go +++ b/commands/lib/download.go @@ -50,7 +50,11 @@ func downloadLibrary(lm *librariesmanager.LibrariesManager, libRelease *librarie downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB) error { taskCB(&rpc.TaskProgress{Name: "Downloading " + libRelease.String()}) - if d, err := libRelease.Resource.Download(lm.DownloadsDir); err != nil { + config, err := commands.GetDownloaderConfig() + if err != nil { + return err + } + if d, err := libRelease.Resource.Download(lm.DownloadsDir, config); err != nil { return err } else if err := commands.Download(d, libRelease.String(), downloadCB); err != nil { return err