diff --git a/arduino/cores/packagemanager/download.go b/arduino/cores/packagemanager/download.go index a3305fa3d4c..45da56ecb39 100644 --- a/arduino/cores/packagemanager/download.go +++ b/arduino/cores/packagemanager/download.go @@ -17,7 +17,6 @@ package packagemanager import ( "fmt" - "net/http" "github.com/arduino/arduino-cli/arduino/cores" "go.bug.st/downloader" @@ -101,16 +100,16 @@ func (pm *PackageManager) FindPlatformReleaseDependencies(item *PlatformReferenc // DownloadToolRelease downloads a ToolRelease. If the tool is already downloaded a nil Downloader // is returned. -func (pm *PackageManager) DownloadToolRelease(tool *cores.ToolRelease, downloaderHeaders http.Header) (*downloader.Downloader, error) { +func (pm *PackageManager) DownloadToolRelease(tool *cores.ToolRelease, config *downloader.Config) (*downloader.Downloader, error) { resource := tool.GetCompatibleFlavour() if resource == nil { return nil, fmt.Errorf("tool not available for your OS") } - return resource.Download(pm.DownloadDir, downloaderHeaders) + return resource.Download(pm.DownloadDir, config) } // DownloadPlatformRelease downloads a PlatformRelease. If the platform is already downloaded a // nil Downloader is returned. -func (pm *PackageManager) DownloadPlatformRelease(platform *cores.PlatformRelease, downloaderHeaders http.Header) (*downloader.Downloader, error) { - return platform.Resource.Download(pm.DownloadDir, downloaderHeaders) +func (pm *PackageManager) DownloadPlatformRelease(platform *cores.PlatformRelease, config *downloader.Config) (*downloader.Downloader, error) { + return platform.Resource.Download(pm.DownloadDir, config) } diff --git a/arduino/libraries/librariesmanager/download.go b/arduino/libraries/librariesmanager/download.go index 8b4c07feb31..e64a5891991 100644 --- a/arduino/libraries/librariesmanager/download.go +++ b/arduino/libraries/librariesmanager/download.go @@ -25,8 +25,8 @@ import ( var LibraryIndexURL, _ = url.Parse("https://downloads.arduino.cc/libraries/library_index.json") // UpdateIndex downloads the libraries index file from Arduino repository. -func (lm *LibrariesManager) UpdateIndex() (*downloader.Downloader, error) { +func (lm *LibrariesManager) UpdateIndex(config *downloader.Config) (*downloader.Downloader, error) { lm.IndexFile.Parent().MkdirAll() // TODO: Download from gzipped URL index - return downloader.Download(lm.IndexFile.String(), LibraryIndexURL.String(), downloader.NoResume) + return downloader.DownloadWithConfig(lm.IndexFile.String(), LibraryIndexURL.String(), *config, downloader.NoResume) } diff --git a/arduino/resources/helpers.go b/arduino/resources/helpers.go index c4728465397..485f9410d97 100644 --- a/arduino/resources/helpers.go +++ b/arduino/resources/helpers.go @@ -17,7 +17,6 @@ package resources import ( "fmt" - "net/http" "os" "github.com/arduino/go-paths-helper" @@ -44,7 +43,7 @@ func (r *DownloadResource) IsCached(downloadDir *paths.Path) (bool, error) { } // Download a DownloadResource. -func (r *DownloadResource) Download(downloadDir *paths.Path, downloaderHeaders http.Header) (*downloader.Downloader, error) { +func (r *DownloadResource) Download(downloadDir *paths.Path, config *downloader.Config) (*downloader.Downloader, error) { cached, err := r.TestLocalArchiveIntegrity(downloadDir) if err != nil { return nil, fmt.Errorf("testing local archive integrity: %s", err) @@ -72,7 +71,5 @@ func (r *DownloadResource) Download(downloadDir *paths.Path, downloaderHeaders h return nil, fmt.Errorf("getting archive file info: %s", err) } - downloadConfig := downloader.Config{ - RequestHeaders: downloaderHeaders} - return downloader.DownloadWithConfig(path.String(), r.URL, downloadConfig) + return downloader.DownloadWithConfig(path.String(), r.URL, *config) } diff --git a/arduino/resources/helpers_test.go b/arduino/resources/helpers_test.go index 1e9e872099d..20733cdb0e0 100644 --- a/arduino/resources/helpers_test.go +++ b/arduino/resources/helpers_test.go @@ -25,6 +25,7 @@ import ( "github.com/arduino/go-paths-helper" "github.com/stretchr/testify/require" + "go.bug.st/downloader" ) type EchoHandler struct{} @@ -52,7 +53,7 @@ func TestDownloadApplyUserAgentHeaderUsingConfig(t *testing.T) { URL: srv.URL, } - d, err := r.Download(tmp, http.Header{"User-Agent": []string{goldUserAgentValue}}) + d, err := r.Download(tmp, &downloader.Config{RequestHeaders: http.Header{"User-Agent": []string{goldUserAgentValue}}}) require.NoError(t, err) err = d.Run() require.NoError(t, err) diff --git a/arduino/resources/resources_test.go b/arduino/resources/resources_test.go index d6f59f75227..5fcdfd43570 100644 --- a/arduino/resources/resources_test.go +++ b/arduino/resources/resources_test.go @@ -18,11 +18,11 @@ package resources import ( "crypto" "encoding/hex" - "net/http" "testing" "github.com/arduino/go-paths-helper" "github.com/stretchr/testify/require" + "go.bug.st/downloader" ) func TestDownloadAndChecksums(t *testing.T) { @@ -42,7 +42,7 @@ func TestDownloadAndChecksums(t *testing.T) { require.NoError(t, err) downloadAndTestChecksum := func() { - d, err := r.Download(tmp, http.Header{}) + d, err := r.Download(tmp, &downloader.Config{}) require.NoError(t, err) err = d.Run() require.NoError(t, err) @@ -58,7 +58,7 @@ func TestDownloadAndChecksums(t *testing.T) { downloadAndTestChecksum() // Download with cached file - d, err := r.Download(tmp, http.Header{}) + d, err := r.Download(tmp, &downloader.Config{}) require.NoError(t, err) require.Nil(t, d) diff --git a/cli/core/download.go b/cli/core/download.go index 0c7b4227710..273c66b48df 100644 --- a/cli/core/download.go +++ b/cli/core/download.go @@ -66,8 +66,7 @@ func runDownloadCommand(cmd *cobra.Command, args []string) { Architecture: platformRef.Architecture, Version: platformRef.Version, } - _, err := core.PlatformDownload(context.Background(), platformDownloadreq, output.ProgressBar(), - globals.NewHTTPClientHeader()) + _, err := core.PlatformDownload(context.Background(), platformDownloadreq, output.ProgressBar()) if err != nil { feedback.Errorf("Error downloading %s: %v", args[i], err) os.Exit(errorcodes.ErrNetwork) diff --git a/cli/core/install.go b/cli/core/install.go index 62a8eb6ed7b..011ad7679bd 100644 --- a/cli/core/install.go +++ b/cli/core/install.go @@ -67,8 +67,7 @@ func runInstallCommand(cmd *cobra.Command, args []string) { Architecture: platformRef.Architecture, Version: platformRef.Version, } - _, err := core.PlatformInstall(context.Background(), plattformInstallReq, output.ProgressBar(), - output.TaskProgress(), globals.NewHTTPClientHeader()) + _, err := core.PlatformInstall(context.Background(), plattformInstallReq, output.ProgressBar(), output.TaskProgress()) if err != nil { feedback.Errorf("Error during install: %v", err) os.Exit(errorcodes.ErrGeneric) diff --git a/cli/core/upgrade.go b/cli/core/upgrade.go index 9c7cfdfcfa8..febc1bcebcf 100644 --- a/cli/core/upgrade.go +++ b/cli/core/upgrade.go @@ -93,7 +93,7 @@ func runUpgradeCommand(cmd *cobra.Command, args []string) { Architecture: platformRef.Architecture, } - _, err := core.PlatformUpgrade(context.Background(), r, output.ProgressBar(), output.TaskProgress(), globals.NewHTTPClientHeader()) + _, err := core.PlatformUpgrade(context.Background(), r, output.ProgressBar(), output.TaskProgress()) if err == core.ErrAlreadyLatest { feedback.Printf("Platform %s is already at the latest version", platformRef) } else if err != nil { diff --git a/cli/daemon/daemon.go b/cli/daemon/daemon.go index 5fa00ca20ca..58d23385fea 100644 --- a/cli/daemon/daemon.go +++ b/cli/daemon/daemon.go @@ -21,9 +21,7 @@ import ( "io" "io/ioutil" "net" - "net/http" "os" - "runtime" "syscall" "github.com/arduino/arduino-cli/cli/errorcodes" @@ -67,26 +65,15 @@ func runDaemonCommand(cmd *cobra.Command, args []string) { stats.Incr("daemon", stats.T("success", "true")) defer stats.Flush() } - port := viper.GetString("daemon.port") s := grpc.NewServer() - // Compose user agent header - headers := http.Header{ - "User-Agent": []string{ - fmt.Sprintf("%s/%s daemon (%s; %s; %s) Commit:%s", - globals.VersionInfo.Application, - globals.VersionInfo.VersionString, - runtime.GOARCH, - runtime.GOOS, - runtime.Version(), - globals.VersionInfo.Commit), - }, - } - // Register the commands service + // Set specific user-agent for the daemon + viper.Set("network.user_agent_ext", "daemon") + + // register the commands service srv_commands.RegisterArduinoCoreServer(s, &daemon.ArduinoCoreServerImpl{ - DownloaderHeaders: headers, - VersionString: globals.VersionInfo.VersionString, + VersionString: globals.VersionInfo.VersionString, }) // Register the monitors service diff --git a/cli/globals/globals.go b/cli/globals/globals.go index 0d20aac245e..5222590dc85 100644 --- a/cli/globals/globals.go +++ b/cli/globals/globals.go @@ -33,8 +33,16 @@ var ( ) // NewHTTPClientHeader returns the http.Header object that must be used by the clients inside the downloaders -func NewHTTPClientHeader() http.Header { - userAgentValue := fmt.Sprintf("%s/%s (%s; %s; %s) Commit:%s", VersionInfo.Application, - VersionInfo.VersionString, runtime.GOARCH, runtime.GOOS, runtime.Version(), VersionInfo.Commit) +// and adds the subComponent if specified +func NewHTTPClientHeader(subComponent string) http.Header { + if subComponent != "" { + subComponent = " " + subComponent + } + userAgentValue := fmt.Sprintf("%s/%s%s (%s; %s; %s) Commit:%s", + VersionInfo.Application, + VersionInfo.VersionString, + subComponent, + runtime.GOARCH, runtime.GOOS, runtime.Version(), + VersionInfo.Commit) return http.Header{"User-Agent": []string{userAgentValue}} } diff --git a/cli/instance/instance.go b/cli/instance/instance.go index 06f8e94ce11..37c0d355323 100644 --- a/cli/instance/instance.go +++ b/cli/instance/instance.go @@ -18,7 +18,6 @@ package instance import ( "context" - "github.com/arduino/arduino-cli/cli/globals" "github.com/arduino/arduino-cli/cli/output" "github.com/arduino/arduino-cli/commands" rpc "github.com/arduino/arduino-cli/rpc/commands" @@ -45,8 +44,7 @@ func CreateInstance() (*rpc.Instance, error) { func getInitResponse() (*rpc.InitResp, error) { // invoke Init() - resp, err := commands.Init(context.Background(), &rpc.InitReq{}, - output.ProgressBar(), output.TaskProgress(), globals.NewHTTPClientHeader()) + resp, err := commands.Init(context.Background(), &rpc.InitReq{}, output.ProgressBar(), output.TaskProgress()) // Init() failed if err != nil { diff --git a/cli/lib/download.go b/cli/lib/download.go index b1d1e012f5b..2419df395b4 100644 --- a/cli/lib/download.go +++ b/cli/lib/download.go @@ -21,7 +21,6 @@ import ( "github.com/arduino/arduino-cli/cli/errorcodes" "github.com/arduino/arduino-cli/cli/feedback" - "github.com/arduino/arduino-cli/cli/globals" "github.com/arduino/arduino-cli/cli/instance" "github.com/arduino/arduino-cli/cli/output" "github.com/arduino/arduino-cli/commands/lib" @@ -57,8 +56,7 @@ func runDownloadCommand(cmd *cobra.Command, args []string) { Name: library.Name, Version: library.Version, } - _, err := lib.LibraryDownload(context.Background(), libraryDownloadReq, output.ProgressBar(), - globals.NewHTTPClientHeader()) + _, err := lib.LibraryDownload(context.Background(), libraryDownloadReq, output.ProgressBar()) if err != nil { feedback.Errorf("Error downloading %s: %v", library, err) os.Exit(errorcodes.ErrNetwork) diff --git a/cli/lib/install.go b/cli/lib/install.go index c9a0732d96b..8ed0bf64ab6 100644 --- a/cli/lib/install.go +++ b/cli/lib/install.go @@ -21,7 +21,6 @@ import ( "github.com/arduino/arduino-cli/cli/errorcodes" "github.com/arduino/arduino-cli/cli/feedback" - "github.com/arduino/arduino-cli/cli/globals" "github.com/arduino/arduino-cli/cli/instance" "github.com/arduino/arduino-cli/cli/output" "github.com/arduino/arduino-cli/commands/lib" @@ -95,8 +94,7 @@ func runInstallCommand(cmd *cobra.Command, args []string) { Name: library.Name, Version: library.VersionRequired, } - err := lib.LibraryInstall(context.Background(), libraryInstallReq, output.ProgressBar(), - output.TaskProgress(), globals.NewHTTPClientHeader()) + err := lib.LibraryInstall(context.Background(), libraryInstallReq, output.ProgressBar(), output.TaskProgress()) if err != nil { feedback.Errorf("Error installing %s: %v", library, err) os.Exit(errorcodes.ErrGeneric) diff --git a/cli/lib/upgrade.go b/cli/lib/upgrade.go index 584fa711c5a..7da8fe87856 100644 --- a/cli/lib/upgrade.go +++ b/cli/lib/upgrade.go @@ -20,7 +20,6 @@ import ( "github.com/arduino/arduino-cli/cli/errorcodes" "github.com/arduino/arduino-cli/cli/feedback" - "github.com/arduino/arduino-cli/cli/globals" "github.com/arduino/arduino-cli/cli/instance" "github.com/arduino/arduino-cli/cli/output" "github.com/arduino/arduino-cli/commands/lib" @@ -48,13 +47,13 @@ func runUpgradeCommand(cmd *cobra.Command, args []string) { instance := instance.CreateInstanceIgnorePlatformIndexErrors() if len(args) == 0 { - err := lib.LibraryUpgradeAll(instance.Id, output.ProgressBar(), output.TaskProgress(), globals.NewHTTPClientHeader()) + err := lib.LibraryUpgradeAll(instance.Id, output.ProgressBar(), output.TaskProgress()) if err != nil { feedback.Errorf("Error upgrading libraries: %v", err) os.Exit(errorcodes.ErrGeneric) } } else { - err := lib.LibraryUpgrade(instance.Id, args, output.ProgressBar(), output.TaskProgress(), globals.NewHTTPClientHeader()) + err := lib.LibraryUpgrade(instance.Id, args, output.ProgressBar(), output.TaskProgress()) if err != nil { feedback.Errorf("Error upgrading libraries: %v", err) os.Exit(errorcodes.ErrGeneric) diff --git a/commands/board/list.go b/commands/board/list.go index 0ec7ba6ec64..b77b743a654 100644 --- a/commands/board/list.go +++ b/commands/board/list.go @@ -51,9 +51,11 @@ func apiByVidPid(vid, pid string) ([]*rpc.BoardListItem, error) { url := fmt.Sprintf("%s/%s/%s", vidPidURL, vid, pid) retVal := []*rpc.BoardListItem{} req, _ := http.NewRequest("GET", url, nil) - req.Header = globals.NewHTTPClientHeader() + req.Header = globals.NewHTTPClientHeader("") req.Header.Set("Content-Type", "application/json") + // TODO: use proxy if set + if res, err := http.DefaultClient.Do(req); err == nil { if res.StatusCode >= 400 { if res.StatusCode == 404 { diff --git a/commands/bundled_tools.go b/commands/bundled_tools.go index 8e6398762be..b8ba834b8a0 100644 --- a/commands/bundled_tools.go +++ b/commands/bundled_tools.go @@ -17,7 +17,6 @@ package commands import ( "fmt" - "net/http" "github.com/arduino/arduino-cli/arduino/cores" "github.com/arduino/arduino-cli/arduino/cores/packagemanager" @@ -25,9 +24,12 @@ import ( ) // DownloadToolRelease downloads a ToolRelease -func DownloadToolRelease(pm *packagemanager.PackageManager, toolRelease *cores.ToolRelease, - downloadCB DownloadProgressCB, downloaderHeaders http.Header) error { - resp, err := pm.DownloadToolRelease(toolRelease, downloaderHeaders) +func DownloadToolRelease(pm *packagemanager.PackageManager, toolRelease *cores.ToolRelease, downloadCB DownloadProgressCB) error { + config, err := GetDownloaderConfig() + if err != nil { + return err + } + resp, err := pm.DownloadToolRelease(toolRelease, config) if err != nil { return err } diff --git a/commands/core/download.go b/commands/core/download.go index 2bc848cae7e..2de0232d008 100644 --- a/commands/core/download.go +++ b/commands/core/download.go @@ -19,7 +19,6 @@ import ( "context" "errors" "fmt" - "net/http" "github.com/arduino/arduino-cli/arduino/cores" "github.com/arduino/arduino-cli/arduino/cores/packagemanager" @@ -28,8 +27,7 @@ import ( ) // PlatformDownload FIXMEDOC -func PlatformDownload(ctx context.Context, req *rpc.PlatformDownloadReq, downloadCB commands.DownloadProgressCB, - downloaderHeaders http.Header) (*rpc.PlatformDownloadResp, error) { +func PlatformDownload(ctx context.Context, req *rpc.PlatformDownloadReq, downloadCB commands.DownloadProgressCB) (*rpc.PlatformDownloadResp, error) { pm := commands.GetPackageManager(req.GetInstance().GetId()) if pm == nil { return nil, errors.New("invalid instance") @@ -49,13 +47,13 @@ func PlatformDownload(ctx context.Context, req *rpc.PlatformDownloadReq, downloa return nil, fmt.Errorf("find platform dependencies: %s", err) } - err = downloadPlatform(pm, platform, downloadCB, downloaderHeaders) + err = downloadPlatform(pm, platform, downloadCB) if err != nil { return nil, err } for _, tool := range tools { - err := downloadTool(pm, tool, downloadCB, downloaderHeaders) + err := downloadTool(pm, tool, downloadCB) if err != nil { return nil, fmt.Errorf("downloading tool %s: %s", tool, err) } @@ -64,22 +62,24 @@ func PlatformDownload(ctx context.Context, req *rpc.PlatformDownloadReq, downloa return &rpc.PlatformDownloadResp{}, nil } -func downloadPlatform(pm *packagemanager.PackageManager, platformRelease *cores.PlatformRelease, - downloadCB commands.DownloadProgressCB, downloaderHeaders http.Header) error { +func downloadPlatform(pm *packagemanager.PackageManager, platformRelease *cores.PlatformRelease, downloadCB commands.DownloadProgressCB) error { // Download platform - resp, err := pm.DownloadPlatformRelease(platformRelease, downloaderHeaders) + config, err := commands.GetDownloaderConfig() + if err != nil { + return err + } + resp, err := pm.DownloadPlatformRelease(platformRelease, config) if err != nil { return err } return commands.Download(resp, platformRelease.String(), downloadCB) } -func downloadTool(pm *packagemanager.PackageManager, tool *cores.ToolRelease, downloadCB commands.DownloadProgressCB, - downloaderHeaders http.Header) error { +func downloadTool(pm *packagemanager.PackageManager, tool *cores.ToolRelease, downloadCB commands.DownloadProgressCB) error { // Check if tool has a flavor available for the current OS if tool.GetCompatibleFlavour() == nil { return fmt.Errorf("tool %s not available for the current OS", tool) } - return commands.DownloadToolRelease(pm, tool, downloadCB, downloaderHeaders) + return commands.DownloadToolRelease(pm, tool, downloadCB) } diff --git a/commands/core/install.go b/commands/core/install.go index bf01129effc..0d2e49deb73 100644 --- a/commands/core/install.go +++ b/commands/core/install.go @@ -19,7 +19,6 @@ import ( "context" "errors" "fmt" - "net/http" "github.com/arduino/arduino-cli/arduino/cores" "github.com/arduino/arduino-cli/arduino/cores/packagemanager" @@ -29,7 +28,7 @@ import ( // PlatformInstall FIXMEDOC func PlatformInstall(ctx context.Context, req *rpc.PlatformInstallReq, - downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB, downloaderHeaders http.Header) (*rpc.PlatformInstallResp, error) { + downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB) (*rpc.PlatformInstallResp, error) { pm := commands.GetPackageManager(req.GetInstance().GetId()) if pm == nil { @@ -50,7 +49,7 @@ func PlatformInstall(ctx context.Context, req *rpc.PlatformInstallReq, return nil, fmt.Errorf("finding platform dependencies: %s", err) } - err = installPlatform(pm, platform, tools, downloadCB, taskCB, downloaderHeaders) + err = installPlatform(pm, platform, tools, downloadCB, taskCB) if err != nil { return nil, err } @@ -65,7 +64,7 @@ func PlatformInstall(ctx context.Context, req *rpc.PlatformInstallReq, func installPlatform(pm *packagemanager.PackageManager, platformRelease *cores.PlatformRelease, requiredTools []*cores.ToolRelease, - downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB, downloaderHeaders http.Header) error { + downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB) error { log := pm.Log.WithField("platform", platformRelease) // Prerequisite checks before install @@ -87,9 +86,9 @@ func installPlatform(pm *packagemanager.PackageManager, // Package download taskCB(&rpc.TaskProgress{Name: "Downloading packages"}) for _, tool := range toolsToInstall { - downloadTool(pm, tool, downloadCB, downloaderHeaders) + downloadTool(pm, tool, downloadCB) } - downloadPlatform(pm, platformRelease, downloadCB, downloaderHeaders) + downloadPlatform(pm, platformRelease, downloadCB) taskCB(&rpc.TaskProgress{Completed: true}) // Install tools first diff --git a/commands/core/upgrade.go b/commands/core/upgrade.go index 392e8bf4454..1297496da1c 100644 --- a/commands/core/upgrade.go +++ b/commands/core/upgrade.go @@ -19,7 +19,6 @@ import ( "context" "errors" "fmt" - "net/http" "github.com/arduino/arduino-cli/arduino/cores/packagemanager" "github.com/arduino/arduino-cli/commands" @@ -34,7 +33,7 @@ var ( // PlatformUpgrade FIXMEDOC func PlatformUpgrade(ctx context.Context, req *rpc.PlatformUpgradeReq, - downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB, downloaderHeaders http.Header) (*rpc.PlatformUpgradeResp, error) { + downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB) (*rpc.PlatformUpgradeResp, error) { pm := commands.GetPackageManager(req.GetInstance().GetId()) if pm == nil { @@ -46,7 +45,7 @@ func PlatformUpgrade(ctx context.Context, req *rpc.PlatformUpgradeReq, Package: req.PlatformPackage, PlatformArchitecture: req.Architecture, } - if err := upgradePlatform(pm, ref, downloadCB, taskCB, downloaderHeaders); err != nil { + if err := upgradePlatform(pm, ref, downloadCB, taskCB); err != nil { return nil, err } @@ -58,7 +57,7 @@ func PlatformUpgrade(ctx context.Context, req *rpc.PlatformUpgradeReq, } func upgradePlatform(pm *packagemanager.PackageManager, platformRef *packagemanager.PlatformReference, - downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB, downloaderHeaders http.Header) error { + downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB) error { if platformRef.PlatformVersion != nil { return fmt.Errorf("upgrade doesn't accept parameters with version") } @@ -85,7 +84,7 @@ func upgradePlatform(pm *packagemanager.PackageManager, platformRef *packagemana if err != nil { return fmt.Errorf("platform %s is not installed", platformRef) } - err = installPlatform(pm, platform, tools, downloadCB, taskCB, downloaderHeaders) + err = installPlatform(pm, platform, tools, downloadCB, taskCB) if err != nil { return err } diff --git a/commands/daemon/daemon.go b/commands/daemon/daemon.go index 579e4abd0f2..2dd088beff9 100644 --- a/commands/daemon/daemon.go +++ b/commands/daemon/daemon.go @@ -19,7 +19,6 @@ package daemon import ( "context" - "net/http" "github.com/arduino/arduino-cli/arduino/utils" "github.com/arduino/arduino-cli/commands" @@ -33,8 +32,7 @@ import ( // ArduinoCoreServerImpl FIXMEDOC type ArduinoCoreServerImpl struct { - DownloaderHeaders http.Header - VersionString string + VersionString string } // BoardDetails FIXMEDOC @@ -108,7 +106,6 @@ func (s *ArduinoCoreServerImpl) Init(req *rpc.InitReq, stream rpc.ArduinoCore_In resp, err := commands.Init(stream.Context(), req, func(p *rpc.DownloadProgress) { stream.Send(&rpc.InitResp{DownloadProgress: p}) }, func(p *rpc.TaskProgress) { stream.Send(&rpc.InitResp{TaskProgress: p}) }, - s.DownloaderHeaders, ) if err != nil { return err @@ -140,7 +137,6 @@ func (s *ArduinoCoreServerImpl) PlatformInstall(req *rpc.PlatformInstallReq, str stream.Context(), req, func(p *rpc.DownloadProgress) { stream.Send(&rpc.PlatformInstallResp{Progress: p}) }, func(p *rpc.TaskProgress) { stream.Send(&rpc.PlatformInstallResp{TaskProgress: p}) }, - s.DownloaderHeaders, ) if err != nil { return err @@ -153,7 +149,6 @@ func (s *ArduinoCoreServerImpl) PlatformDownload(req *rpc.PlatformDownloadReq, s resp, err := core.PlatformDownload( stream.Context(), req, func(p *rpc.DownloadProgress) { stream.Send(&rpc.PlatformDownloadResp{Progress: p}) }, - s.DownloaderHeaders, ) if err != nil { return err @@ -179,7 +174,6 @@ func (s *ArduinoCoreServerImpl) PlatformUpgrade(req *rpc.PlatformUpgradeReq, str stream.Context(), req, func(p *rpc.DownloadProgress) { stream.Send(&rpc.PlatformUpgradeResp{Progress: p}) }, func(p *rpc.TaskProgress) { stream.Send(&rpc.PlatformUpgradeResp{TaskProgress: p}) }, - s.DownloaderHeaders, ) if err != nil { return err @@ -227,7 +221,6 @@ func (s *ArduinoCoreServerImpl) LibraryDownload(req *rpc.LibraryDownloadReq, str resp, err := lib.LibraryDownload( stream.Context(), req, func(p *rpc.DownloadProgress) { stream.Send(&rpc.LibraryDownloadResp{Progress: p}) }, - s.DownloaderHeaders, ) if err != nil { return err @@ -241,7 +234,6 @@ func (s *ArduinoCoreServerImpl) LibraryInstall(req *rpc.LibraryInstallReq, strea stream.Context(), req, func(p *rpc.DownloadProgress) { stream.Send(&rpc.LibraryInstallResp{Progress: p}) }, func(p *rpc.TaskProgress) { stream.Send(&rpc.LibraryInstallResp{TaskProgress: p}) }, - s.DownloaderHeaders, ) if err != nil { return err @@ -265,7 +257,6 @@ func (s *ArduinoCoreServerImpl) LibraryUpgradeAll(req *rpc.LibraryUpgradeAllReq, err := lib.LibraryUpgradeAll(req.GetInstance().GetId(), func(p *rpc.DownloadProgress) { stream.Send(&rpc.LibraryUpgradeAllResp{Progress: p}) }, func(p *rpc.TaskProgress) { stream.Send(&rpc.LibraryUpgradeAllResp{TaskProgress: p}) }, - s.DownloaderHeaders, ) if err != nil { return err diff --git a/commands/download.go b/commands/download.go new file mode 100644 index 00000000000..e049faa76ee --- /dev/null +++ b/commands/download.go @@ -0,0 +1,71 @@ +// This file is part of arduino-cli. +// +// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +package commands + +import ( + "net/url" + "time" + + "github.com/arduino/arduino-cli/cli/globals" + rpc "github.com/arduino/arduino-cli/rpc/commands" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "github.com/spf13/viper" + "go.bug.st/downloader" +) + +// GetDownloaderConfig returns the downloader configuration based on +// current settings. +func GetDownloaderConfig() (*downloader.Config, error) { + res := &downloader.Config{ + RequestHeaders: globals.NewHTTPClientHeader(viper.GetString("network.user_agent_ext")), + } + if viper.IsSet("network.proxy") { + proxy := viper.GetString("network.proxy") + if _, err := url.Parse(proxy); err != nil { + return nil, errors.New("Invalid network.proxy '" + proxy + "': " + err.Error()) + } + res.ProxyURL = proxy + logrus.Infof("Using proxy %s", proxy) + } + return res, nil +} + +// Download performs a download loop using the provided downloader.Downloader. +// Messages are passed back to the DownloadProgressCB using label as text for the File field. +func Download(d *downloader.Downloader, label string, downloadCB DownloadProgressCB) error { + if d == nil { + // This signal means that the file is already downloaded + downloadCB(&rpc.DownloadProgress{ + File: label, + Completed: true, + }) + return nil + } + downloadCB(&rpc.DownloadProgress{ + File: label, + Url: d.URL, + TotalSize: d.Size(), + }) + d.RunAndPoll(func(downloaded int64) { + downloadCB(&rpc.DownloadProgress{Downloaded: downloaded}) + }, 250*time.Millisecond) + if d.Error() != nil { + return d.Error() + } + downloadCB(&rpc.DownloadProgress{Completed: true}) + return nil +} diff --git a/commands/instances.go b/commands/instances.go index a995a525164..277139a1ca1 100644 --- a/commands/instances.go +++ b/commands/instances.go @@ -19,10 +19,8 @@ import ( "context" "fmt" "io/ioutil" - "net/http" "net/url" "path" - "time" "github.com/arduino/arduino-cli/arduino/cores" "github.com/arduino/arduino-cli/arduino/cores/packageindex" @@ -89,13 +87,12 @@ func GetLibraryManager(instanceID int32) *librariesmanager.LibrariesManager { return i.lm } -func (instance *CoreInstance) installToolIfMissing(tool *cores.ToolRelease, downloadCB DownloadProgressCB, - taskCB TaskProgressCB, downloaderHeaders http.Header) (bool, error) { +func (instance *CoreInstance) installToolIfMissing(tool *cores.ToolRelease, downloadCB DownloadProgressCB, taskCB TaskProgressCB) (bool, error) { if tool.IsInstalled() { return false, nil } taskCB(&rpc.TaskProgress{Name: "Downloading missing tool " + tool.String()}) - if err := DownloadToolRelease(instance.PackageManager, tool, downloadCB, downloaderHeaders); err != nil { + if err := DownloadToolRelease(instance.PackageManager, tool, downloadCB); err != nil { return false, fmt.Errorf("downloading %s tool: %s", tool, err) } taskCB(&rpc.TaskProgress{Completed: true}) @@ -105,18 +102,17 @@ func (instance *CoreInstance) installToolIfMissing(tool *cores.ToolRelease, down return true, nil } -func (instance *CoreInstance) checkForBuiltinTools(downloadCB DownloadProgressCB, taskCB TaskProgressCB, - downloaderHeaders http.Header) error { +func (instance *CoreInstance) checkForBuiltinTools(downloadCB DownloadProgressCB, taskCB TaskProgressCB) error { // Check for ctags tool ctags, _ := getBuiltinCtagsTool(instance.PackageManager) - ctagsInstalled, err := instance.installToolIfMissing(ctags, downloadCB, taskCB, downloaderHeaders) + ctagsInstalled, err := instance.installToolIfMissing(ctags, downloadCB, taskCB) if err != nil { return err } // Check for bultin serial-discovery tool serialDiscoveryTool, _ := getBuiltinSerialDiscoveryTool(instance.PackageManager) - serialDiscoveryInstalled, err := instance.installToolIfMissing(serialDiscoveryTool, downloadCB, taskCB, downloaderHeaders) + serialDiscoveryInstalled, err := instance.installToolIfMissing(serialDiscoveryTool, downloadCB, taskCB) if err != nil { return err } @@ -130,9 +126,7 @@ func (instance *CoreInstance) checkForBuiltinTools(downloadCB DownloadProgressCB } // Init FIXMEDOC -func Init(ctx context.Context, req *rpc.InitReq, downloadCB DownloadProgressCB, taskCB TaskProgressCB, - downloaderHeaders http.Header) (*rpc.InitResp, error) { - +func Init(ctx context.Context, req *rpc.InitReq, downloadCB DownloadProgressCB, taskCB TaskProgressCB) (*rpc.InitResp, error) { res, err := createInstance(ctx, req.GetLibraryManagerOnly()) if err != nil { return nil, fmt.Errorf("cannot initialize package manager: %s", err) @@ -147,7 +141,7 @@ func Init(ctx context.Context, req *rpc.InitReq, downloadCB DownloadProgressCB, instancesCount++ instances[handle] = instance - if err := instance.checkForBuiltinTools(downloadCB, taskCB, downloaderHeaders); err != nil { + if err := instance.checkForBuiltinTools(downloadCB, taskCB); err != nil { return nil, err } @@ -176,7 +170,11 @@ func UpdateLibrariesIndex(ctx context.Context, req *rpc.UpdateLibrariesIndexReq, if lm == nil { return fmt.Errorf("invalid handle") } - d, err := lm.UpdateIndex() + config, err := GetDownloaderConfig() + if err != nil { + return err + } + d, err := lm.UpdateIndex(config) if err != nil { return err } @@ -220,7 +218,11 @@ func UpdateIndex(ctx context.Context, req *rpc.UpdateIndexReq, downloadCB Downlo tmp := paths.New(tmpFile.Name()) defer tmp.Remove() - d, err := downloader.Download(tmp.String(), URL.String()) + config, err := GetDownloaderConfig() + if err != nil { + return nil, fmt.Errorf("downloading index %s: %s", URL, err) + } + d, err := downloader.DownloadWithConfig(tmp.String(), URL.String(), *config) if err != nil { return nil, fmt.Errorf("downloading index %s: %s", URL, err) } @@ -353,28 +355,3 @@ func createInstance(ctx context.Context, getLibOnly bool) (*createInstanceResult return res, nil } - -// Download FIXMEDOC -func Download(d *downloader.Downloader, label string, downloadCB DownloadProgressCB) error { - if d == nil { - // This signal means that the file is already downloaded - downloadCB(&rpc.DownloadProgress{ - File: label, - Completed: true, - }) - return nil - } - downloadCB(&rpc.DownloadProgress{ - File: label, - Url: d.URL, - TotalSize: d.Size(), - }) - d.RunAndPoll(func(downloaded int64) { - downloadCB(&rpc.DownloadProgress{Downloaded: downloaded}) - }, 250*time.Millisecond) - if d.Error() != nil { - return d.Error() - } - downloadCB(&rpc.DownloadProgress{Completed: true}) - return nil -} diff --git a/commands/lib/download.go b/commands/lib/download.go index a133c1fe886..117efb3aeeb 100644 --- a/commands/lib/download.go +++ b/commands/lib/download.go @@ -18,7 +18,6 @@ package lib import ( "context" "fmt" - "net/http" "github.com/arduino/arduino-cli/arduino/libraries/librariesindex" "github.com/arduino/arduino-cli/arduino/libraries/librariesmanager" @@ -28,8 +27,7 @@ import ( ) // LibraryDownload FIXMEDOC -func LibraryDownload(ctx context.Context, req *rpc.LibraryDownloadReq, downloadCB commands.DownloadProgressCB, - downloaderHeaders http.Header) (*rpc.LibraryDownloadResp, error) { +func LibraryDownload(ctx context.Context, req *rpc.LibraryDownloadReq, downloadCB commands.DownloadProgressCB) (*rpc.LibraryDownloadResp, error) { logrus.Info("Executing `arduino lib download`") lm := commands.GetLibraryManager(req.GetInstance().GetId()) @@ -41,7 +39,7 @@ func LibraryDownload(ctx context.Context, req *rpc.LibraryDownloadReq, downloadC return nil, fmt.Errorf("looking for library: %s", err) } - if err := downloadLibrary(lm, lib, downloadCB, func(*rpc.TaskProgress) {}, downloaderHeaders); err != nil { + if err := downloadLibrary(lm, lib, downloadCB, func(*rpc.TaskProgress) {}); err != nil { return nil, err } @@ -49,10 +47,14 @@ func LibraryDownload(ctx context.Context, req *rpc.LibraryDownloadReq, downloadC } func downloadLibrary(lm *librariesmanager.LibrariesManager, libRelease *librariesindex.Release, - downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB, downloaderHeaders http.Header) error { + downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB) error { taskCB(&rpc.TaskProgress{Name: "Downloading " + libRelease.String()}) - if d, err := libRelease.Resource.Download(lm.DownloadsDir, downloaderHeaders); err != nil { + config, err := commands.GetDownloaderConfig() + if err != nil { + return err + } + if d, err := libRelease.Resource.Download(lm.DownloadsDir, config); err != nil { return err } else if err := commands.Download(d, libRelease.String(), downloadCB); err != nil { return err diff --git a/commands/lib/install.go b/commands/lib/install.go index 714c8e352c8..93fad6f8664 100644 --- a/commands/lib/install.go +++ b/commands/lib/install.go @@ -18,7 +18,6 @@ package lib import ( "context" "fmt" - "net/http" "github.com/arduino/arduino-cli/arduino/libraries/librariesindex" "github.com/arduino/arduino-cli/arduino/libraries/librariesmanager" @@ -29,7 +28,7 @@ import ( // LibraryInstall FIXMEDOC func LibraryInstall(ctx context.Context, req *rpc.LibraryInstallReq, - downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB, downloaderHeaders http.Header) error { + downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB) error { lm := commands.GetLibraryManager(req.GetInstance().GetId()) @@ -38,7 +37,7 @@ func LibraryInstall(ctx context.Context, req *rpc.LibraryInstallReq, return fmt.Errorf("looking for library: %s", err) } - if err := downloadLibrary(lm, libRelease, downloadCB, taskCB, downloaderHeaders); err != nil { + if err := downloadLibrary(lm, libRelease, downloadCB, taskCB); err != nil { return fmt.Errorf("downloading library: %s", err) } diff --git a/commands/lib/upgrade.go b/commands/lib/upgrade.go index c991cee1399..d85526a5fab 100644 --- a/commands/lib/upgrade.go +++ b/commands/lib/upgrade.go @@ -17,7 +17,6 @@ package lib import ( "fmt" - "net/http" "github.com/arduino/arduino-cli/arduino/libraries/librariesmanager" "github.com/arduino/arduino-cli/commands" @@ -25,11 +24,11 @@ import ( // LibraryUpgradeAll upgrades all the available libraries func LibraryUpgradeAll(instanceID int32, downloadCB commands.DownloadProgressCB, - taskCB commands.TaskProgressCB, headers http.Header) error { + taskCB commands.TaskProgressCB) error { // get the library manager lm := commands.GetLibraryManager(instanceID) - if err := upgrade(lm, listLibraries(lm, true, true), downloadCB, taskCB, headers); err != nil { + if err := upgrade(lm, listLibraries(lm, true, true), downloadCB, taskCB); err != nil { return err } @@ -42,7 +41,7 @@ func LibraryUpgradeAll(instanceID int32, downloadCB commands.DownloadProgressCB, // LibraryUpgrade upgrades only the given libraries func LibraryUpgrade(instanceID int32, libraryNames []string, downloadCB commands.DownloadProgressCB, - taskCB commands.TaskProgressCB, headers http.Header) error { + taskCB commands.TaskProgressCB) error { // get the library manager lm := commands.GetLibraryManager(instanceID) @@ -50,16 +49,16 @@ func LibraryUpgrade(instanceID int32, libraryNames []string, downloadCB commands libs := filterByName(listLibraries(lm, true, true), libraryNames) // do it - return upgrade(lm, libs, downloadCB, taskCB, headers) + return upgrade(lm, libs, downloadCB, taskCB) } func upgrade(lm *librariesmanager.LibrariesManager, libs []*installedLib, downloadCB commands.DownloadProgressCB, - taskCB commands.TaskProgressCB, downloaderHeaders http.Header) error { + taskCB commands.TaskProgressCB) error { // Go through the list and download them for _, lib := range libs { - if err := downloadLibrary(lm, lib.Available, downloadCB, taskCB, downloaderHeaders); err != nil { + if err := downloadLibrary(lm, lib.Available, downloadCB, taskCB); err != nil { return err } } diff --git a/configuration/defaults.go b/configuration/defaults.go index 262edca08d8..4ccf1c380b1 100644 --- a/configuration/defaults.go +++ b/configuration/defaults.go @@ -25,6 +25,7 @@ func setDefaults(dataDir, userDir string) { // logging viper.SetDefault("logging.level", "info") viper.SetDefault("logging.format", "text") + // board manager viper.SetDefault("board_manager.additional_urls", []string{}) diff --git a/go.mod b/go.mod index 8add00a9c98..9589dbe49b9 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,7 @@ require ( github.com/spf13/viper v1.6.2 github.com/stretchr/testify v1.4.0 go.bug.st/cleanup v1.0.0 - go.bug.st/downloader v1.1.0 + go.bug.st/downloader v1.2.0 go.bug.st/relaxed-semver v0.0.0-20190922224835-391e10178d18 go.bug.st/serial v1.0.0 go.bug.st/serial.v1 v0.0.0-20180827123349-5f7892a7bb45 // indirect diff --git a/go.sum b/go.sum index 46bfc061b4c..2f84109f8f6 100644 --- a/go.sum +++ b/go.sum @@ -189,6 +189,7 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM github.com/spf13/viper v1.6.2 h1:7aKfF+e8/k68gda3LOjo5RxiUqddoFxVq4BKBPrxk5E= github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= @@ -204,8 +205,8 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.bug.st/cleanup v1.0.0 h1:XVj1HZxkBXeq3gMT7ijWUpHyIC1j8XAoNSyQ06CskgA= go.bug.st/cleanup v1.0.0/go.mod h1:EqVmTg2IBk4znLbPD28xne3abjsJftMdqqJEjhn70bk= -go.bug.st/downloader v1.1.0 h1:LipC9rqRCe8kwa+ah3ZDfCqneVaf34cB/TKjXZiZt54= -go.bug.st/downloader v1.1.0/go.mod h1:l+RPbNbrTB+MoAIp8nrZsP22nRPDy26XJZQqmm4gNT4= +go.bug.st/downloader v1.2.0 h1:YmXFTcTnm0v8WzAWHn2DyV46c/Izlc/gReXubc2oBho= +go.bug.st/downloader v1.2.0/go.mod h1:l+RPbNbrTB+MoAIp8nrZsP22nRPDy26XJZQqmm4gNT4= go.bug.st/relaxed-semver v0.0.0-20190922224835-391e10178d18 h1:F1qxtaFuewctYc/SsHRn+Q7Dtwi+yJGPgVq8YLtQz98= go.bug.st/relaxed-semver v0.0.0-20190922224835-391e10178d18/go.mod h1:Cx1VqMtEhE9pIkEyUj3LVVVPkv89dgW8aCKrRPDR/uE= go.bug.st/serial v1.0.0 h1:ogEPzrllCsnG00EqKRjeYvPRsO7NJW6DqykzkdD6E/k=