diff --git a/arduino/cores/packagemanager/download.go b/arduino/cores/packagemanager/download.go index 45da56ecb39..8e6a9e7c4b6 100644 --- a/arduino/cores/packagemanager/download.go +++ b/arduino/cores/packagemanager/download.go @@ -19,7 +19,7 @@ import ( "fmt" "github.com/arduino/arduino-cli/arduino/cores" - "go.bug.st/downloader" + "go.bug.st/downloader/v2" semver "go.bug.st/relaxed-semver" ) diff --git a/arduino/libraries/librariesmanager/download.go b/arduino/libraries/librariesmanager/download.go index e64a5891991..d8ec7dae1c9 100644 --- a/arduino/libraries/librariesmanager/download.go +++ b/arduino/libraries/librariesmanager/download.go @@ -18,7 +18,7 @@ package librariesmanager import ( "net/url" - "go.bug.st/downloader" + "go.bug.st/downloader/v2" ) // LibraryIndexURL is the URL where to get library index. diff --git a/arduino/resources/helpers.go b/arduino/resources/helpers.go index 485f9410d97..c735ff36559 100644 --- a/arduino/resources/helpers.go +++ b/arduino/resources/helpers.go @@ -20,7 +20,7 @@ import ( "os" "github.com/arduino/go-paths-helper" - "go.bug.st/downloader" + "go.bug.st/downloader/v2" ) // ArchivePath returns the path of the Archive of the specified DownloadResource relative diff --git a/arduino/resources/helpers_test.go b/arduino/resources/helpers_test.go index 20733cdb0e0..21445d25aab 100644 --- a/arduino/resources/helpers_test.go +++ b/arduino/resources/helpers_test.go @@ -23,9 +23,10 @@ import ( "strings" "testing" + "github.com/arduino/arduino-cli/httpclient" "github.com/arduino/go-paths-helper" "github.com/stretchr/testify/require" - "go.bug.st/downloader" + "go.bug.st/downloader/v2" ) type EchoHandler struct{} @@ -53,7 +54,9 @@ func TestDownloadApplyUserAgentHeaderUsingConfig(t *testing.T) { URL: srv.URL, } - d, err := r.Download(tmp, &downloader.Config{RequestHeaders: http.Header{"User-Agent": []string{goldUserAgentValue}}}) + httpClient := httpclient.NewWithConfig(&httpclient.Config{UserAgent: goldUserAgentValue}) + + d, err := r.Download(tmp, &downloader.Config{HttpClient: *httpClient}) require.NoError(t, err) err = d.Run() require.NoError(t, err) diff --git a/arduino/resources/resources_test.go b/arduino/resources/resources_test.go index 5fcdfd43570..326f34c11f3 100644 --- a/arduino/resources/resources_test.go +++ b/arduino/resources/resources_test.go @@ -22,7 +22,7 @@ import ( "github.com/arduino/go-paths-helper" "github.com/stretchr/testify/require" - "go.bug.st/downloader" + "go.bug.st/downloader/v2" ) func TestDownloadAndChecksums(t *testing.T) { diff --git a/cli/globals/globals.go b/cli/globals/globals.go index 5222590dc85..8a73ba08745 100644 --- a/cli/globals/globals.go +++ b/cli/globals/globals.go @@ -16,11 +16,8 @@ package globals import ( - "fmt" - "net/http" "os" "path/filepath" - "runtime" "github.com/arduino/arduino-cli/version" ) @@ -31,18 +28,3 @@ var ( // DefaultIndexURL is the default index url DefaultIndexURL = "https://downloads.arduino.cc/packages/package_index.json" ) - -// NewHTTPClientHeader returns the http.Header object that must be used by the clients inside the downloaders -// and adds the subComponent if specified -func NewHTTPClientHeader(subComponent string) http.Header { - if subComponent != "" { - subComponent = " " + subComponent - } - userAgentValue := fmt.Sprintf("%s/%s%s (%s; %s; %s) Commit:%s", - VersionInfo.Application, - VersionInfo.VersionString, - subComponent, - runtime.GOARCH, runtime.GOOS, runtime.Version(), - VersionInfo.Commit) - return http.Header{"User-Agent": []string{userAgentValue}} -} diff --git a/commands/board/list.go b/commands/board/list.go index b77b743a654..c47e7d6c474 100644 --- a/commands/board/list.go +++ b/commands/board/list.go @@ -23,8 +23,8 @@ import ( "regexp" "sync" - "github.com/arduino/arduino-cli/cli/globals" "github.com/arduino/arduino-cli/commands" + "github.com/arduino/arduino-cli/httpclient" rpc "github.com/arduino/arduino-cli/rpc/commands" "github.com/pkg/errors" "github.com/segmentio/stats/v4" @@ -51,12 +51,17 @@ func apiByVidPid(vid, pid string) ([]*rpc.BoardListItem, error) { url := fmt.Sprintf("%s/%s/%s", vidPidURL, vid, pid) retVal := []*rpc.BoardListItem{} req, _ := http.NewRequest("GET", url, nil) - req.Header = globals.NewHTTPClientHeader("") req.Header.Set("Content-Type", "application/json") // TODO: use proxy if set - if res, err := http.DefaultClient.Do(req); err == nil { + httpClient, err := httpclient.New() + + if err != nil { + return nil, errors.Wrap(err, "failed to initialize http client") + } + + if res, err := httpClient.Do(req); err == nil { if res.StatusCode >= 400 { if res.StatusCode == 404 { return nil, ErrNotFound diff --git a/commands/download.go b/commands/download.go index e049faa76ee..7f3512d5277 100644 --- a/commands/download.go +++ b/commands/download.go @@ -16,30 +16,24 @@ package commands import ( - "net/url" "time" - "github.com/arduino/arduino-cli/cli/globals" + "github.com/arduino/arduino-cli/httpclient" rpc "github.com/arduino/arduino-cli/rpc/commands" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/spf13/viper" - "go.bug.st/downloader" + "go.bug.st/downloader/v2" ) // GetDownloaderConfig returns the downloader configuration based on // current settings. func GetDownloaderConfig() (*downloader.Config, error) { - res := &downloader.Config{ - RequestHeaders: globals.NewHTTPClientHeader(viper.GetString("network.user_agent_ext")), + + httpClient, err := httpclient.New() + if err != nil { + return nil, err } - if viper.IsSet("network.proxy") { - proxy := viper.GetString("network.proxy") - if _, err := url.Parse(proxy); err != nil { - return nil, errors.New("Invalid network.proxy '" + proxy + "': " + err.Error()) - } - res.ProxyURL = proxy - logrus.Infof("Using proxy %s", proxy) + + res := &downloader.Config{ + HttpClient: *httpClient, } return res, nil } diff --git a/commands/instances.go b/commands/instances.go index 277139a1ca1..1cdc1459e86 100644 --- a/commands/instances.go +++ b/commands/instances.go @@ -33,7 +33,7 @@ import ( paths "github.com/arduino/go-paths-helper" "github.com/sirupsen/logrus" "github.com/spf13/viper" - "go.bug.st/downloader" + "go.bug.st/downloader/v2" ) // this map contains all the running Arduino Core Services instances diff --git a/go.mod b/go.mod index 9589dbe49b9..758841d14bd 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,7 @@ require ( github.com/spf13/viper v1.6.2 github.com/stretchr/testify v1.4.0 go.bug.st/cleanup v1.0.0 - go.bug.st/downloader v1.2.0 + go.bug.st/downloader/v2 v2.0.1 go.bug.st/relaxed-semver v0.0.0-20190922224835-391e10178d18 go.bug.st/serial v1.0.0 go.bug.st/serial.v1 v0.0.0-20180827123349-5f7892a7bb45 // indirect diff --git a/go.sum b/go.sum index 2f84109f8f6..d32bee8d0ec 100644 --- a/go.sum +++ b/go.sum @@ -205,8 +205,8 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.bug.st/cleanup v1.0.0 h1:XVj1HZxkBXeq3gMT7ijWUpHyIC1j8XAoNSyQ06CskgA= go.bug.st/cleanup v1.0.0/go.mod h1:EqVmTg2IBk4znLbPD28xne3abjsJftMdqqJEjhn70bk= -go.bug.st/downloader v1.2.0 h1:YmXFTcTnm0v8WzAWHn2DyV46c/Izlc/gReXubc2oBho= -go.bug.st/downloader v1.2.0/go.mod h1:l+RPbNbrTB+MoAIp8nrZsP22nRPDy26XJZQqmm4gNT4= +go.bug.st/downloader/v2 v2.0.1 h1:F/ZgVDrUHo2tHIeOVbcl4rgGaMmQgAEjn5c8yi5t+Iw= +go.bug.st/downloader/v2 v2.0.1/go.mod h1:VZW2V1iGKV8rJL2ZEGIDzzBeKowYv34AedJz13RzVII= go.bug.st/relaxed-semver v0.0.0-20190922224835-391e10178d18 h1:F1qxtaFuewctYc/SsHRn+Q7Dtwi+yJGPgVq8YLtQz98= go.bug.st/relaxed-semver v0.0.0-20190922224835-391e10178d18/go.mod h1:Cx1VqMtEhE9pIkEyUj3LVVVPkv89dgW8aCKrRPDR/uE= go.bug.st/serial v1.0.0 h1:ogEPzrllCsnG00EqKRjeYvPRsO7NJW6DqykzkdD6E/k= diff --git a/httpclient/httpclient.go b/httpclient/httpclient.go new file mode 100644 index 00000000000..4d097eb6552 --- /dev/null +++ b/httpclient/httpclient.go @@ -0,0 +1,40 @@ +// This file is part of arduino-cli. +// +// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +package httpclient + +import ( + "net/http" +) + +// New returns a default http client for use in the cli API calls +func New() (*http.Client, error) { + config, err := DefaultConfig() + + if err != nil { + return nil, err + } + + return NewWithConfig(config), nil +} + +// NewWithConfig creates a http client for use in the cli API calls with a given configuration +func NewWithConfig(config *Config) *http.Client { + transport := newHTTPClientTransport(config) + + return &http.Client{ + Transport: transport, + } +} diff --git a/httpclient/httpclient_config.go b/httpclient/httpclient_config.go new file mode 100644 index 00000000000..fe44d0a12da --- /dev/null +++ b/httpclient/httpclient_config.go @@ -0,0 +1,64 @@ +// This file is part of arduino-cli. +// +// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +package httpclient + +import ( + "errors" + "fmt" + "net/url" + "runtime" + + "github.com/arduino/arduino-cli/cli/globals" + "github.com/spf13/viper" +) + +// Config is the configuration of the http client +type Config struct { + UserAgent string + Proxy *url.URL +} + +// DefaultConfig returns the default http client config +func DefaultConfig() (*Config, error) { + var proxy *url.URL + var err error + if viper.IsSet("network.proxy") { + proxyConfig := viper.GetString("network.proxy") + if proxy, err = url.Parse(proxyConfig); err != nil { + return nil, errors.New("Invalid network.proxy '" + proxyConfig + "': " + err.Error()) + } + } + + return &Config{ + UserAgent: UserAgent(), + Proxy: proxy, + }, nil +} + +// UserAgent returns the user agent for the cli http client +func UserAgent() string { + subComponent := viper.GetString("network.user_agent_ext") + if subComponent != "" { + subComponent = " " + subComponent + } + + return fmt.Sprintf("%s/%s%s (%s; %s; %s) Commit:%s", + globals.VersionInfo.Application, + globals.VersionInfo.VersionString, + subComponent, + runtime.GOARCH, runtime.GOOS, runtime.Version(), + globals.VersionInfo.Commit) +} diff --git a/httpclient/httpclient_test.go b/httpclient/httpclient_test.go new file mode 100644 index 00000000000..010363a652d --- /dev/null +++ b/httpclient/httpclient_test.go @@ -0,0 +1,70 @@ +// This file is part of arduino-cli. +// +// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +package httpclient + +import ( + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestUserAgentHeader(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, r.Header.Get("User-Agent")) + })) + defer ts.Close() + + client := NewWithConfig(&Config{ + UserAgent: "test-user-agent", + }) + + request, err := http.NewRequest("GET", ts.URL, nil) + require.NoError(t, err) + + response, err := client.Do(request) + require.NoError(t, err) + + b, err := ioutil.ReadAll(response.Body) + require.NoError(t, err) + + require.Equal(t, "test-user-agent", string(b)) +} + +func TestProxy(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNoContent) + })) + defer ts.Close() + + proxyURL, err := url.Parse(ts.URL) + require.NoError(t, err) + + client := NewWithConfig(&Config{ + Proxy: proxyURL, + }) + + request, err := http.NewRequest("GET", "http://arduino.cc", nil) + require.NoError(t, err) + + response, err := client.Do(request) + require.NoError(t, err) + require.Equal(t, http.StatusNoContent, response.StatusCode) +} diff --git a/httpclient/httpclient_transport.go b/httpclient/httpclient_transport.go new file mode 100644 index 00000000000..45cba70f016 --- /dev/null +++ b/httpclient/httpclient_transport.go @@ -0,0 +1,41 @@ +// This file is part of arduino-cli. +// +// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +package httpclient + +import "net/http" + +type httpClientRoundTripper struct { + transport http.RoundTripper + config *Config +} + +func newHTTPClientTransport(config *Config) http.RoundTripper { + proxy := http.ProxyURL(config.Proxy) + + transport := &http.Transport{ + Proxy: proxy, + } + + return &httpClientRoundTripper{ + transport: transport, + config: config, + } +} + +func (h *httpClientRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + req.Header.Add("User-Agent", h.config.UserAgent) + return h.transport.RoundTrip(req) +}