add cli httpclient with support for proxy configuration

arduino/cores/packagemanager/download.go

@@ -19,7 +19,7 @@ import (
 	"fmt"
 
 	"github.com/arduino/arduino-cli/arduino/cores"
-	"go.bug.st/downloader"
+	"go.bug.st/downloader/v2"
 	semver "go.bug.st/relaxed-semver"
 )
 

arduino/libraries/librariesmanager/download.go

@@ -18,7 +18,7 @@ package librariesmanager
 import (
 	"net/url"
 
-	"go.bug.st/downloader"
+	"go.bug.st/downloader/v2"
 )
 
 // LibraryIndexURL is the URL where to get library index.

arduino/resources/helpers.go

@@ -20,7 +20,7 @@ import (
 	"os"
 
 	"github.com/arduino/go-paths-helper"
-	"go.bug.st/downloader"
+	"go.bug.st/downloader/v2"
 )
 
 // ArchivePath returns the path of the Archive of the specified DownloadResource relative

arduino/resources/helpers_test.go

@@ -23,9 +23,10 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/arduino/arduino-cli/httpclient"
 	"github.com/arduino/go-paths-helper"
 	"github.com/stretchr/testify/require"
-	"go.bug.st/downloader"
+	"go.bug.st/downloader/v2"
 )
 
 type EchoHandler struct{}
@@ -53,7 +54,9 @@ func TestDownloadApplyUserAgentHeaderUsingConfig(t *testing.T) {
 		URL:             srv.URL,
 	}
 
-	d, err := r.Download(tmp, &downloader.Config{RequestHeaders: http.Header{"User-Agent": []string{goldUserAgentValue}}})
+	httpClient := httpclient.NewWithConfig(&httpclient.Config{UserAgent: goldUserAgentValue})
+
+	d, err := r.Download(tmp, &downloader.Config{HttpClient: *httpClient})
 	require.NoError(t, err)
 	err = d.Run()
 	require.NoError(t, err)

arduino/resources/resources_test.go

@@ -22,7 +22,7 @@ import (
 
 	"github.com/arduino/go-paths-helper"
 	"github.com/stretchr/testify/require"
-	"go.bug.st/downloader"
+	"go.bug.st/downloader/v2"
 )
 
 func TestDownloadAndChecksums(t *testing.T) {

cli/globals/globals.go

@@ -16,11 +16,8 @@
 package globals
 
 import (
-	"fmt"
-	"net/http"
 	"os"
 	"path/filepath"
-	"runtime"
 
 	"github.com/arduino/arduino-cli/version"
 )
@@ -31,18 +28,3 @@ var (
 	// DefaultIndexURL is the default index url
 	DefaultIndexURL = "https://downloads.arduino.cc/packages/package_index.json"
 )
-
-// NewHTTPClientHeader returns the http.Header object that must be used by the clients inside the downloaders
-// and adds the subComponent if specified
-func NewHTTPClientHeader(subComponent string) http.Header {
-	if subComponent != "" {
-		subComponent = " " + subComponent
-	}
-	userAgentValue := fmt.Sprintf("%s/%s%s (%s; %s; %s) Commit:%s",
-		VersionInfo.Application,
-		VersionInfo.VersionString,
-		subComponent,
-		runtime.GOARCH, runtime.GOOS, runtime.Version(),
-		VersionInfo.Commit)
-	return http.Header{"User-Agent": []string{userAgentValue}}
-}

commands/board/list.go

@@ -23,8 +23,8 @@ import (
 	"regexp"
 	"sync"
 
-	"github.com/arduino/arduino-cli/cli/globals"
 	"github.com/arduino/arduino-cli/commands"
+	"github.com/arduino/arduino-cli/httpclient"
 	rpc "github.com/arduino/arduino-cli/rpc/commands"
 	"github.com/pkg/errors"
 	"github.com/segmentio/stats/v4"
@@ -51,12 +51,17 @@ func apiByVidPid(vid, pid string) ([]*rpc.BoardListItem, error) {
 	url := fmt.Sprintf("%s/%s/%s", vidPidURL, vid, pid)
 	retVal := []*rpc.BoardListItem{}
 	req, _ := http.NewRequest("GET", url, nil)
-	req.Header = globals.NewHTTPClientHeader("")
 	req.Header.Set("Content-Type", "application/json")
 
 	// TODO: use proxy if set
 
-	if res, err := http.DefaultClient.Do(req); err == nil {
+	httpClient, err := httpclient.New()
+
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to initialize http client")
+	}
+
+	if res, err := httpClient.Do(req); err == nil {
 		if res.StatusCode >= 400 {
 			if res.StatusCode == 404 {
 				return nil, ErrNotFound

commands/download.go

@@ -16,30 +16,24 @@
 package commands
 
 import (
-	"net/url"
 	"time"
 
-	"github.com/arduino/arduino-cli/cli/globals"
+	"github.com/arduino/arduino-cli/httpclient"
 	rpc "github.com/arduino/arduino-cli/rpc/commands"
-	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
-	"github.com/spf13/viper"
-	"go.bug.st/downloader"
+	"go.bug.st/downloader/v2"
 )
 
 // GetDownloaderConfig returns the downloader configuration based on
 // current settings.
 func GetDownloaderConfig() (*downloader.Config, error) {
-	res := &downloader.Config{
-		RequestHeaders: globals.NewHTTPClientHeader(viper.GetString("network.user_agent_ext")),
+
+	httpClient, err := httpclient.New()
+	if err != nil {
+		return nil, err
 	}
-	if viper.IsSet("network.proxy") {
-		proxy := viper.GetString("network.proxy")
-		if _, err := url.Parse(proxy); err != nil {
-			return nil, errors.New("Invalid network.proxy '" + proxy + "': " + err.Error())
-		}
-		res.ProxyURL = proxy
-		logrus.Infof("Using proxy %s", proxy)
+
+	res := &downloader.Config{
+		HttpClient: *httpClient,
 	}
 	return res, nil
 }

commands/instances.go

@@ -33,7 +33,7 @@ import (
 	paths "github.com/arduino/go-paths-helper"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/viper"
-	"go.bug.st/downloader"
+	"go.bug.st/downloader/v2"
 )
 
 // this map contains all the running Arduino Core Services instances

go.mod

@@ -38,7 +38,7 @@ require (
 	github.com/spf13/viper v1.6.2
 	github.com/stretchr/testify v1.4.0
 	go.bug.st/cleanup v1.0.0
-	go.bug.st/downloader v1.2.0
+	go.bug.st/downloader/v2 v2.0.1
 	go.bug.st/relaxed-semver v0.0.0-20190922224835-391e10178d18
 	go.bug.st/serial v1.0.0
 	go.bug.st/serial.v1 v0.0.0-20180827123349-5f7892a7bb45 // indirect

go.sum

@@ -205,8 +205,8 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 go.bug.st/cleanup v1.0.0 h1:XVj1HZxkBXeq3gMT7ijWUpHyIC1j8XAoNSyQ06CskgA=
 go.bug.st/cleanup v1.0.0/go.mod h1:EqVmTg2IBk4znLbPD28xne3abjsJftMdqqJEjhn70bk=
-go.bug.st/downloader v1.2.0 h1:YmXFTcTnm0v8WzAWHn2DyV46c/Izlc/gReXubc2oBho=
-go.bug.st/downloader v1.2.0/go.mod h1:l+RPbNbrTB+MoAIp8nrZsP22nRPDy26XJZQqmm4gNT4=
+go.bug.st/downloader/v2 v2.0.1 h1:F/ZgVDrUHo2tHIeOVbcl4rgGaMmQgAEjn5c8yi5t+Iw=
+go.bug.st/downloader/v2 v2.0.1/go.mod h1:VZW2V1iGKV8rJL2ZEGIDzzBeKowYv34AedJz13RzVII=
 go.bug.st/relaxed-semver v0.0.0-20190922224835-391e10178d18 h1:F1qxtaFuewctYc/SsHRn+Q7Dtwi+yJGPgVq8YLtQz98=
 go.bug.st/relaxed-semver v0.0.0-20190922224835-391e10178d18/go.mod h1:Cx1VqMtEhE9pIkEyUj3LVVVPkv89dgW8aCKrRPDR/uE=
 go.bug.st/serial v1.0.0 h1:ogEPzrllCsnG00EqKRjeYvPRsO7NJW6DqykzkdD6E/k=

httpclient/httpclient.go

@@ -0,0 +1,40 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to [email protected].
+
+package httpclient
+
+import (
+	"net/http"
+)
+
+// New returns a default http client for use in the cli API calls
+func New() (*http.Client, error) {
+	config, err := DefaultConfig()
+
+	if err != nil {
+		return nil, err
+	}
+
+	return NewWithConfig(config), nil
+}
+
+// NewWithConfig creates a http client for use in the cli API calls with a given configuration
+func NewWithConfig(config *Config) *http.Client {
+	transport := newHTTPClientTransport(config)
+
+	return &http.Client{
+		Transport: transport,
+	}
+}

httpclient/httpclient_config.go

@@ -0,0 +1,64 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to [email protected].
+
+package httpclient
+
+import (
+	"errors"
+	"fmt"
+	"net/url"
+	"runtime"
+
+	"github.com/arduino/arduino-cli/cli/globals"
+	"github.com/spf13/viper"
+)
+
+// Config is the configuration of the http client
+type Config struct {
+	UserAgent string
+	Proxy     *url.URL
+}
+
+// DefaultConfig returns the default http client config
+func DefaultConfig() (*Config, error) {
+	var proxy *url.URL
+	var err error
+	if viper.IsSet("network.proxy") {
+		proxyConfig := viper.GetString("network.proxy")
+		if proxy, err = url.Parse(proxyConfig); err != nil {
+			return nil, errors.New("Invalid network.proxy '" + proxyConfig + "': " + err.Error())
+		}
+	}
+
+	return &Config{
+		UserAgent: UserAgent(),
+		Proxy:     proxy,
+	}, nil
+}
+
+// UserAgent returns the user agent for the cli http client
+func UserAgent() string {
+	subComponent := viper.GetString("network.user_agent_ext")
+	if subComponent != "" {
+		subComponent = " " + subComponent
+	}
+
+	return fmt.Sprintf("%s/%s%s (%s; %s; %s) Commit:%s",
+		globals.VersionInfo.Application,
+		globals.VersionInfo.VersionString,
+		subComponent,
+		runtime.GOARCH, runtime.GOOS, runtime.Version(),
+		globals.VersionInfo.Commit)
+}

httpclient/httpclient_test.go

@@ -0,0 +1,70 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to [email protected].
+
+package httpclient
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestUserAgentHeader(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprint(w, r.Header.Get("User-Agent"))
+	}))
+	defer ts.Close()
+
+	client := NewWithConfig(&Config{
+		UserAgent: "test-user-agent",
+	})
+
+	request, err := http.NewRequest("GET", ts.URL, nil)
+	require.NoError(t, err)
+
+	response, err := client.Do(request)
+	require.NoError(t, err)
+
+	b, err := ioutil.ReadAll(response.Body)
+	require.NoError(t, err)
+
+	require.Equal(t, "test-user-agent", string(b))
+}
+
+func TestProxy(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusNoContent)
+	}))
+	defer ts.Close()
+
+	proxyURL, err := url.Parse(ts.URL)
+	require.NoError(t, err)
+
+	client := NewWithConfig(&Config{
+		Proxy: proxyURL,
+	})
+
+	request, err := http.NewRequest("GET", "http://arduino.cc", nil)
+	require.NoError(t, err)
+
+	response, err := client.Do(request)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusNoContent, response.StatusCode)
+}