Downloader helper user agent (#227)

* add global variables and Info struct in order to inject build time values via ldflags

* replace cli.AppName with global variable global.GetAppName() getter

* replaced cli.Version var with global getter

* add .idea to .gitignore

* re organize imports

* add license to global.go

* add hardcoded fallback version string

* organize imports step2

* organize imports step3

* replace cli.AppName with global variable global.GetAppName() getter

* add User-Agent header to downloader helper configuration

* add testing for user agent-string generation

* re organize imports

* refactor global package into version package and rename and replace having cli login,logout,validate removed

* refactor versioning variables in version package and implement Info struct wihth String method

* refactor Info cosntructor to use directly package vars and renamed package from global to version

* replace package name from global to version

* solve package name clash for version and cli/version

* replace application name getter with VersionInfo.Application field
having removed cli login,logout,validate

* search and replace version string getter with VersionInfo.VersionString field

* implement empty http.Header struct propagation from cli to resources

* align downloader helper to use propagated http.Header from upper layers and modify test accordingly

* align daemon to use empty http.Header as struct field in ArduinoCoreServerImpl

* clean inside cobra commands for empty http.Header struct and inject headers build from VersionInfo struct in cli InitInstance func

* remove unused runDaemonCommand in daemon

* add user agent specific for daemon mode via DownloaderHeaders property for ArduinoCoreServerImpl

* add testing information in README.md

* removed getters and related usage for version package

* update dependencies

* finalize helpers_test User-Agent header value

* add integration test for version info injection via vars

* tidy go mod and reorganize imports and tidy long lines

* inject ldflags variables in build task in Taskfile.yml

* replace commands in /travis.yml with tasks properly merging flags

* fix install command for go-task to solve "undefined: interp.EnvFromList" error

* add coverage files to .gitignore

* tidy dependencies in go.mod

* update task executable path in tasks due to install path generated by go-task install script

* implement exported variable cli.HTTPClientHeader in order to have it available in all cli subpackages and not onli in cli

* fix comment replacing "version config" with "global config"

* leverage cli.VersionInfo to present a better terminal and json data for version info

Author: rsora

Diff for: .gitignore

@@ -5,3 +5,5 @@
 /cmd/formatter/debug.test
 /arduino-cli.yaml
 /wiki
+.idea
+coverage_*.txt

Diff for: .travis.yml

@@ -11,6 +11,7 @@ env:
 # Make sure golangci-lint is vendored.
 before_install:
   - curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | bash -s -- -b $GOPATH/bin v1.16.0
+  - curl -sL https://taskfile.dev/install.sh | sh
 
 install: true
 
@@ -20,9 +21,10 @@ script:
   # Run linter
   - golangci-lint run
   # Build and test
-  - go build
-  - go test -timeout 20m -v -coverpkg=./... -coverprofile=coverage.txt -covermode=atomic ./...
+  - ./bin/task build
+  - ./bin/task test
 
 after_success:
-  - bash <(curl -s https://codecov.io/bash)
+  - bash <(curl -s https://codecov.io/bash) -cF unittests,integration
+
 

Diff for: README.md

@@ -333,3 +333,20 @@ Because:
 #### How can I find the core/FQBN for a board?
 
 See: https://github.com/arduino/arduino-cli#step-4-find-and-install-the-right-core
+
+# Testing
+
+Currently Unit and Integration test are available for launch in 2 ways:
+
+1. classic `go test ./...` to launch both unit and integration test
+
+2. via [task](https://taskfile.dev) that includes the following options:
+
+```
+* build:                Build the project
+* test:                 Run the full testsuite
+* test-integration:     Run integration tests only
+* test-unit:            Run unit tests only
+```
+
+For Example to launch unit tests only run: `task test-unit` 

Diff for: Taskfile.yml

@@ -4,7 +4,7 @@ tasks:
   build:
     desc: Build the project
     cmds:
-      - go build -v -i
+      - go build -v -i {{.LDFLAGS}}
 
   test:
     desc: Run the full testsuite
@@ -15,9 +15,26 @@ tasks:
   test-unit:
     desc: Run unit tests only
     cmds:
-      - go test -short {{ default "-v" .GOFLAGS }} {{ default "./..." .TARGETS }}
+      - go test -short {{ default "-v" .GOFLAGS }} -coverprofile=coverage_unit.txt {{ default "./..." .TARGETS }}
 
   test-integration:
     desc: Run integration tests only
     cmds:
-      - go test -run Integration {{ default "-v" .GOFLAGS }} {{ default "./..." .TARGETS }}
+      - go test -run Integration {{ default "-v" .GOFLAGS }} -coverprofile=coverage_integ.txt {{ default "./..." .TARGETS }} {{.TEST_LDFLAGS}}
+
+vars:
+  # build vars
+  VERSIONSTRING: "0.3.6-alpha.preview"
+  COMMIT:
+    sh: echo ${TRAVIS_COMMIT:-`git log -n 1 --format=%h`}
+  LDFLAGS: >
+      -ldflags '-X github.com/arduino/arduino-cli/version.versionString={{.VERSIONSTRING}}
+      -X github.com/arduino/arduino-cli/version.commit={{.COMMIT}}'
+
+  # test vars
+  GOFLAGS: "-timeout 5m -v -coverpkg=./... -covermode=atomic"
+  TEST_VERSIONSTRING: "0.0.0-test.preview"
+  TEST_COMMIT: "deadbeef"
+  TEST_LDFLAGS: >
+    -ldflags  '-X github.com/arduino/arduino-cli/version.versionString={{.TEST_VERSIONSTRING}}
+    -X github.com/arduino/arduino-cli/version.commit={{.TEST_COMMIT}}'

Diff for: arduino/cores/packagemanager/download.go

@@ -19,6 +19,7 @@ package packagemanager
 
 import (
 	"fmt"
+	"net/http"
 
 	"github.com/arduino/arduino-cli/arduino/cores"
 	"go.bug.st/downloader"
@@ -102,16 +103,16 @@ func (pm *PackageManager) FindPlatformReleaseDependencies(item *PlatformReferenc
 
 // DownloadToolRelease downloads a ToolRelease. If the tool is already downloaded a nil Downloader
 // is returned.
-func (pm *PackageManager) DownloadToolRelease(tool *cores.ToolRelease) (*downloader.Downloader, error) {
+func (pm *PackageManager) DownloadToolRelease(tool *cores.ToolRelease, downloaderHeaders http.Header) (*downloader.Downloader, error) {
 	resource := tool.GetCompatibleFlavour()
 	if resource == nil {
 		return nil, fmt.Errorf("tool not available for your OS")
 	}
-	return resource.Download(pm.DownloadDir)
+	return resource.Download(pm.DownloadDir, downloaderHeaders)
 }
 
 // DownloadPlatformRelease downloads a PlatformRelease. If the platform is already downloaded a
 // nil Downloader is returned.
-func (pm *PackageManager) DownloadPlatformRelease(platform *cores.PlatformRelease) (*downloader.Downloader, error) {
-	return platform.Resource.Download(pm.DownloadDir)
+func (pm *PackageManager) DownloadPlatformRelease(platform *cores.PlatformRelease, downloaderHeaders http.Header) (*downloader.Downloader, error) {
+	return platform.Resource.Download(pm.DownloadDir, downloaderHeaders)
 }

Diff for: arduino/resources/helpers.go

@@ -19,9 +19,10 @@ package resources
 
 import (
 	"fmt"
+	"net/http"
 	"os"
 
-	paths "github.com/arduino/go-paths-helper"
+	"github.com/arduino/go-paths-helper"
 	"go.bug.st/downloader"
 )
 
@@ -45,7 +46,7 @@ func (r *DownloadResource) IsCached(downloadDir *paths.Path) (bool, error) {
 }
 
 // Download a DownloadResource.
-func (r *DownloadResource) Download(downloadDir *paths.Path) (*downloader.Downloader, error) {
+func (r *DownloadResource) Download(downloadDir *paths.Path, downloaderHeaders http.Header) (*downloader.Downloader, error) {
 	cached, err := r.TestLocalArchiveIntegrity(downloadDir)
 	if err != nil {
 		return nil, fmt.Errorf("testing local archive integrity: %s", err)
@@ -73,5 +74,7 @@ func (r *DownloadResource) Download(downloadDir *paths.Path) (*downloader.Downlo
 		return nil, fmt.Errorf("getting archive file info: %s", err)
 	}
 
-	return downloader.Download(path.String(), r.URL)
+	downloadConfig := downloader.Config{
+		RequestHeaders: downloaderHeaders}
+	return downloader.DownloadWithConfig(path.String(), r.URL, downloadConfig)
 }

Diff for: arduino/resources/helpers_test.go

@@ -0,0 +1,82 @@
+/*
+ * This file is part of arduino-cli.
+ *
+ * Copyright 2018 ARDUINO SA (http://www.arduino.cc/)
+ *
+ * This software is released under the GNU General Public License version 3,
+ * which covers the main part of arduino-cli.
+ * The terms of this license can be found at:
+ * https://www.gnu.org/licenses/gpl-3.0.en.html
+ *
+ * You can be released from the requirements of the above licenses by purchasing
+ * a commercial license. Buying such a license is mandatory if you want to modify or
+ * otherwise use the software for commercial activities involving the Arduino
+ * software without disclosing the source code of your own applications. To purchase
+ * a commercial license, send an email to [email protected].
+ */
+
+package resources
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/arduino/go-paths-helper"
+	"github.com/stretchr/testify/require"
+)
+
+type EchoHandler struct{}
+
+// EchoHandler echos back the request as a response if used as http handler
+func (h *EchoHandler) ServeHTTP(writer http.ResponseWriter, request *http.Request) {
+	request.Write(writer)
+}
+
+func TestDownloadApplyUserAgentHeaderUsingConfig(t *testing.T) {
+	goldUserAgentValue := fmt.Sprintf("arduino-cli/0.0.0-test.preview (amd64; linux; go1.12.4) Commit:deadbeef/Build:2019-06-12 11:11:11.111")
+	goldUserAgentString := "User-Agent: " + goldUserAgentValue
+
+	tmp, err := paths.MkTempDir("", "")
+	require.NoError(t, err)
+	defer tmp.RemoveAll()
+
+	// startup echo server
+	srv := httptest.NewServer(&EchoHandler{})
+	defer srv.Close()
+
+	r := &DownloadResource{
+		ArchiveFileName: "echo.txt",
+		CachePath:       "cache",
+		URL:             srv.URL,
+	}
+
+	d, err := r.Download(tmp, http.Header{"User-Agent": []string{goldUserAgentValue}})
+	require.NoError(t, err)
+	err = d.Run()
+	require.NoError(t, err)
+
+	// leverage the download helper to download the echo for the request made by the downloader itself
+	//
+	// expect something like:
+	//    GET /echo HTTP/1.1
+	//    Host: 127.0.0.1:64999
+	//    User-Agent: arduino-cli/0.0.0-test.preview  (amd64; linux; go1.12.4) Commit:deadbeef/Build:2019-06-12 11:11:11.111
+	//    Accept-Encoding: gzip
+
+	b, err := ioutil.ReadFile(tmp.String() + "/cache/echo.txt") // just pass the file name
+	require.NoError(t, err)
+
+	requestLines := strings.Split(string(b), "\r\n")
+	userAgentHeaderString := ""
+	for _, line := range requestLines {
+		if strings.Contains(line, "User-Agent: ") {
+			userAgentHeaderString = line
+		}
+	}
+	require.Equal(t, goldUserAgentString, userAgentHeaderString)
+
+}

Diff for: arduino/resources/resources_test.go

@@ -20,6 +20,7 @@ package resources
 import (
 	"crypto"
 	"encoding/hex"
+	"net/http"
 	"testing"
 
 	paths "github.com/arduino/go-paths-helper"
@@ -43,7 +44,7 @@ func TestDownloadAndChecksums(t *testing.T) {
 	require.NoError(t, err)
 
 	downloadAndTestChecksum := func() {
-		d, err := r.Download(tmp)
+		d, err := r.Download(tmp, http.Header{})
 		require.NoError(t, err)
 		err = d.Run()
 		require.NoError(t, err)
@@ -59,7 +60,7 @@ func TestDownloadAndChecksums(t *testing.T) {
 	downloadAndTestChecksum()
 
 	// Download with cached file
-	d, err := r.Download(tmp)
+	d, err := r.Download(tmp, http.Header{})
 	require.NoError(t, err)
 	require.Nil(t, d)
 

Diff for: cli/board/attach.go

@@ -33,9 +33,9 @@ func initAttachCommand() *cobra.Command {
 		Use:   "attach <port>|<FQBN> [sketchPath]",
 		Short: "Attaches a sketch to a board.",
 		Long:  "Attaches a sketch to a board.",
-		Example: "  " + cli.AppName + " board attach serial:///dev/tty/ACM0\n" +
-			"  " + cli.AppName + " board attach serial:///dev/tty/ACM0 HelloWorld\n" +
-			"  " + cli.AppName + " board attach arduino:samd:mkr1000",
+		Example: "  " + cli.VersionInfo.Application + " board attach serial:///dev/tty/ACM0\n" +
+			"  " + cli.VersionInfo.Application + " board attach serial:///dev/tty/ACM0 HelloWorld\n" +
+			"  " + cli.VersionInfo.Application + " board attach arduino:samd:mkr1000",
 		Args: cobra.RangeArgs(1, 2),
 		Run:  runAttachCommand,
 	}

Diff for: cli/board/board.go

@@ -29,9 +29,9 @@ func InitCommand() *cobra.Command {
 		Short: "Arduino board commands.",
 		Long:  "Arduino board commands.",
 		Example: "  # Lists all connected boards.\n" +
-			"  " + cli.AppName + " board list\n\n" +
+			"  " + cli.VersionInfo.Application + " board list\n\n" +
 			"  # Attaches a sketch to a board.\n" +
-			"  " + cli.AppName + " board attach serial:///dev/tty/ACM0 mySketch",
+			"  " + cli.VersionInfo.Application + " board attach serial:///dev/tty/ACM0 mySketch",
 	}
 	boardCommand.AddCommand(initAttachCommand())
 	boardCommand.AddCommand(initDetailsCommand())

Diff for: cli/board/details.go

@@ -35,7 +35,7 @@ func initDetailsCommand() *cobra.Command {
 		Use:     "details <FQBN>",
 		Short:   "Print details about a board.",
 		Long:    "Show information about a board, in particular if the board has options to be specified in the FQBN.",
-		Example: "  " + cli.AppName + " board details arduino:avr:nano",
+		Example: "  " + cli.VersionInfo.Application + " board details arduino:avr:nano",
 		Args:    cobra.ExactArgs(1),
 		Run:     runDetailsCommand,
 	}

Diff for: cli/board/list.go

@@ -37,7 +37,7 @@ func initListCommand() *cobra.Command {
 		Use:     "list",
 		Short:   "List connected boards.",
 		Long:    "Detects and displays a list of connected boards to the current computer.",
-		Example: "  " + cli.AppName + " board list --timeout 10s",
+		Example: "  " + cli.VersionInfo.Application + " board list --timeout 10s",
 		Args:    cobra.NoArgs,
 		Run:     runListCommand,
 	}

Diff for: cli/board/listall.go

@@ -39,8 +39,8 @@ func initListAllCommand() *cobra.Command {
 			"List all boards that have the support platform installed. You can search\n" +
 			"for a specific board if you specify the board name",
 		Example: "" +
-			"  " + cli.AppName + " board listall\n" +
-			"  " + cli.AppName + " board listall zero",
+			"  " + cli.VersionInfo.Application + " board listall\n" +
+			"  " + cli.VersionInfo.Application + " board listall zero",
 		Args: cobra.ArbitraryArgs,
 		Run:  runListAllCommand,
 	}

Diff for: cli/cli.go

@@ -20,16 +20,20 @@ package cli
 import (
 	"context"
 	"errors"
+	"fmt"
+	"net/http"
 	"os"
 	"path/filepath"
+	"runtime"
 
 	"github.com/arduino/arduino-cli/arduino/cores/packagemanager"
 	"github.com/arduino/arduino-cli/arduino/libraries/librariesmanager"
 	"github.com/arduino/arduino-cli/commands"
 	"github.com/arduino/arduino-cli/common/formatter"
 	"github.com/arduino/arduino-cli/configs"
 	"github.com/arduino/arduino-cli/rpc"
-	paths "github.com/arduino/go-paths-helper"
+	"github.com/arduino/arduino-cli/version"
+	"github.com/arduino/go-paths-helper"
 	"github.com/sirupsen/logrus"
 )
 
@@ -48,8 +52,13 @@ const (
 	ErrBadArgument
 )
 
-// Version is the current CLI version
-var Version = "0.3.6-alpha.preview"
+// appName is the command line name of the Arduino CLI executable on the user system (users may change it)
+var appName = filepath.Base(os.Args[0])
+
+// VersionInfo contains all info injected during build
+var VersionInfo = version.NewInfo(appName)
+
+var HTTPClientHeader = getHTTPClientHeader()
 
 // ErrLogrus represents the logrus instance, which has the role to
 // log all non info messages.
@@ -61,9 +70,6 @@ var GlobalFlags struct {
 	OutputJSON bool // true output in JSON, false output as Text
 }
 
-// AppName is the command line name of the Arduino CLI executable
-var AppName = filepath.Base(os.Args[0])
-
 var Config *configs.Configuration
 
 func packageManagerInitReq() *rpc.InitReq {
@@ -83,10 +89,18 @@ func packageManagerInitReq() *rpc.InitReq {
 	return &rpc.InitReq{Configuration: conf}
 }
 
+func getHTTPClientHeader() http.Header {
+	userAgentValue := fmt.Sprintf("%s/%s (%s; %s; %s) Commit:%s/Build:%s", VersionInfo.Application,
+		VersionInfo.VersionString, runtime.GOARCH, runtime.GOOS, runtime.Version(), VersionInfo.Commit, VersionInfo.BuildDate)
+	downloaderHeaders := http.Header{"User-Agent": []string{userAgentValue}}
+	return downloaderHeaders
+}
+
 func InitInstance() *rpc.InitResp {
 	logrus.Info("Initializing package manager")
 	req := packageManagerInitReq()
-	resp, err := commands.Init(context.Background(), req, OutputProgressBar(), OutputTaskProgress())
+
+	resp, err := commands.Init(context.Background(), req, OutputProgressBar(), OutputTaskProgress(), HTTPClientHeader)
 	if err != nil {
 		formatter.PrintError(err, "Error initializing package manager")
 		os.Exit(ErrGeneric)
@@ -116,7 +130,7 @@ func CreateInstance() *rpc.Instance {
 		for _, err := range resp.GetPlatformsIndexErrors() {
 			formatter.PrintError(errors.New(err), "Error loading index")
 		}
-		formatter.PrintErrorMessage("Launch '" + AppName + " core update-index' to fix or download indexes.")
+		formatter.PrintErrorMessage("Launch '" + VersionInfo.Application + " core update-index' to fix or download indexes.")
 		os.Exit(ErrGeneric)
 	}
 	return resp.GetInstance()

Diff for: cli/compile/compile.go

@@ -35,7 +35,7 @@ func InitCommand() *cobra.Command {
 		Use:     "compile",
 		Short:   "Compiles Arduino sketches.",
 		Long:    "Compiles Arduino sketches.",
-		Example: "  " + cli.AppName + " compile -b arduino:avr:uno /home/user/Arduino/MySketch",
+		Example: "  " + cli.VersionInfo.Application + " compile -b arduino:avr:uno /home/user/Arduino/MySketch",
 		Args:    cobra.MaximumNArgs(1),
 		Run:     run,
 	}