Merge branch 'master' into massi/output-reprise

Author: Massimiliano Pippi

.github/workflows/test.yaml

@@ -58,11 +58,6 @@ jobs:
           architecture: 'x64'
 
       - name: Run integration tests
-        # Since we are investigating issues emerged
-        # with python integration tests on windows
-        # this step is disabled for windows-2019
-        # platform ATM
-        if: matrix.operating-system != 'windows-2019'
         run: |
           pip install -r test/requirements.txt
           task test-integration

README.md

@@ -20,6 +20,8 @@ build the source code, run the tests, and contribute your changes to the project
 
 ### Get the latest package
 
+#### Using the install script
+
 The easiest way to get the latest version of `arduino-cli` on any supported platform is using the
 `install.sh` script:
 
@@ -34,9 +36,26 @@ for example `~/local/bin`, set the `BINDIR` environment variable like this:
 curl -fsSL https://raw.githubusercontent.com/arduino/arduino-cli/master/install.sh | BINDIR=~/local/bin sh
 ```
 
+#### Using the latest release links
+
+In order to get the latest stable release for your platform use the following links:
+
+- [Linux 64 bit](https://downloads.arduino.cc/arduino-cli/arduino-cli_latest_Linux_64bit.tar.gz)
+- [Linux 32 bit](https://downloads.arduino.cc/arduino-cli/arduino-cli_latest_Linux_32bit.tar.gz)
+- [Linux ARM 64 bit](https://downloads.arduino.cc/arduino-cli/arduino-cli_latest_Linux_ARM64.tar.gz)
+- [Linux ARM 32 bit](https://downloads.arduino.cc/arduino-cli/arduino-cli_latest_Linux_ARMv7.tar.gz)
+- [Windows 64 bit](https://downloads.arduino.cc/arduino-cli/arduino-cli_latest_Windows_64bit.zip)
+- [Windows 32 bit](https://downloads.arduino.cc/arduino-cli/arduino-cli_latest_Windows_32bit.zip)
+- [Mac OSX](https://downloads.arduino.cc/arduino-cli/arduino-cli_latest_macOS_64bit.tar.gz)
+
+These links return a `302: Found` response, redirecting to latest generated builds by replacing `latest` with the latest
+available stable release. Once downloaded, place the executable `arduino-cli` into a directory which is in your `PATH`.
+
 **Deprecation notice:** Links in the form `http://downloads.arduino.cc/arduino-cli/arduino-cli-latest-<platform>.tar.bz2`
 won't be further updated. That URL will provide arduino-cli 0.3.7-alpha.preview, regardless of further releases.
 
+#### Using the Github release page
+
 Alternatively you can download one of the pre-built binaries for the supported platforms from the
 [release page](https://github.com/arduino/arduino-cli/releases). Once downloaded, place the executable
 `arduino-cli` into a directory which is in your `PATH`.

Taskfile.yml

@@ -63,12 +63,10 @@ vars:
     sh: echo `go list ./... | grep -v legacy | tr '\n' ' '`
 
   # build vars
-  VERSIONSTRING: "0.3.7-alpha.preview"
   COMMIT:
     sh: echo ${TRAVIS_COMMIT:-`git log -n 1 --format=%h`}
   LDFLAGS: >
-      -ldflags '-X github.com/arduino/arduino-cli/version.versionString={{.VERSIONSTRING}}
-      -X github.com/arduino/arduino-cli/version.commit={{.COMMIT}}'
+      -ldflags '-X github.com/arduino/arduino-cli/version.commit={{.COMMIT}}'
 
   # test vars
   GOFLAGS: "-timeout 10m -v -coverpkg=./... -covermode=atomic"

arduino/builder/builder.go

@@ -19,18 +19,22 @@ import (
 	"crypto/md5"
 	"encoding/hex"
 	"os"
-	"path/filepath"
 	"strings"
 
+	"github.com/arduino/go-paths-helper"
 	"github.com/pkg/errors"
 )
 
-// GenBuildPath generates a suitable name for the build folder
-func GenBuildPath(sketchPath string) string {
-	md5SumBytes := md5.Sum([]byte(sketchPath))
+// GenBuildPath generates a suitable name for the build folder.
+// The sketchPath, if not nil, is also used to furhter differentiate build paths.
+func GenBuildPath(sketchPath *paths.Path) *paths.Path {
+	path := ""
+	if sketchPath != nil {
+		path = sketchPath.String()
+	}
+	md5SumBytes := md5.Sum([]byte(path))
 	md5Sum := strings.ToUpper(hex.EncodeToString(md5SumBytes[:]))
-
-	return filepath.Join(os.TempDir(), "arduino-sketch-"+md5Sum)
+	return paths.TempDir().Join("arduino-sketch-" + md5Sum)
 }
 
 // EnsureBuildPathExists creates the build path if doesn't already exists.

arduino/builder/builder_test.go

@@ -23,6 +23,7 @@ import (
 	"testing"
 
 	"github.com/arduino/arduino-cli/arduino/builder"
+	"github.com/arduino/go-paths-helper"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -36,7 +37,10 @@ func tmpDirOrDie() string {
 
 func TestGenBuildPath(t *testing.T) {
 	want := filepath.Join(os.TempDir(), "arduino-sketch-ACBD18DB4CC2F85CEDEF654FCCC4A4D8")
-	assert.Equal(t, want, builder.GenBuildPath("foo"))
+	assert.Equal(t, want, builder.GenBuildPath(paths.New("foo")).String())
+
+	want = filepath.Join(os.TempDir(), "arduino-sketch-D41D8CD98F00B204E9800998ECF8427E")
+	assert.Equal(t, want, builder.GenBuildPath(nil).String())
 }
 
 func TestEnsureBuildPathExists(t *testing.T) {

cli/board/attach.go

@@ -27,6 +27,8 @@ import (
 	"github.com/arduino/arduino-cli/cli/output"
 	"github.com/arduino/arduino-cli/commands/board"
 	rpc "github.com/arduino/arduino-cli/rpc/commands"
+	"github.com/arduino/go-paths-helper"
+	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
 
@@ -52,18 +54,37 @@ var attachFlags struct {
 
 func runAttachCommand(cmd *cobra.Command, args []string) {
 	instance := instance.CreateInstance()
-	var path string
-	if len(args) > 0 {
-		path = args[1]
+
+	var path *paths.Path
+	if len(args) > 1 {
+		path = paths.New(args[1])
+	} else {
+		path = initSketchPath(path)
 	}
+
 	_, err := board.Attach(context.Background(), &rpc.BoardAttachReq{
 		Instance:      instance,
 		BoardUri:      args[0],
-		SketchPath:    path,
+		SketchPath:    path.String(),
 		SearchTimeout: attachFlags.searchTimeout,
 	}, output.TaskProgress())
 	if err != nil {
 		feedback.Errorf("Attach board error: %v", err)
 		os.Exit(errorcodes.ErrGeneric)
 	}
 }
+
+// initSketchPath returns the current working directory
+func initSketchPath(sketchPath *paths.Path) *paths.Path {
+	if sketchPath != nil {
+		return sketchPath
+	}
+
+	wd, err := paths.Getwd()
+	if err != nil {
+		feedback.Errorf("Couldn't get current working directory: %v", err)
+		os.Exit(errorcodes.ErrGeneric)
+	}
+	logrus.Infof("Reading sketch from dir: %s", wd)
+	return wd
+}

cli/cli.go

@@ -82,7 +82,7 @@ func createCliCommandTree(cmd *cobra.Command) {
 	cmd.AddCommand(version.NewCommand())
 
 	cmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "Print the logs on the standard output.")
-	cmd.PersistentFlags().StringVar(&globals.LogLevel, "log-level", defaultLogLevel, "Messages with this level and above will be logged (default: warn).")
+	cmd.PersistentFlags().StringVar(&globals.LogLevel, "log-level", defaultLogLevel, "Messages with this level and above will be logged.")
 	cmd.PersistentFlags().StringVar(&logFile, "log-file", "", "Path to the file where logs will be written.")
 	cmd.PersistentFlags().StringVar(&logFormat, "log-format", "text", "The output format for the logs, can be [text|json].")
 	cmd.PersistentFlags().StringVar(&outputFormat, "format", "text", "The output format, can be [text|json].")

cli/core/download.go

@@ -64,7 +64,7 @@ func runDownloadCommand(cmd *cobra.Command, args []string) {
 			Version:         platformRef.Version,
 		}
 		_, err := core.PlatformDownload(context.Background(), platformDownloadreq, output.ProgressBar(),
-			globals.HTTPClientHeader)
+			globals.NewHTTPClientHeader())
 		if err != nil {
 			feedback.Errorf("Error downloading %s: %v", args[i], err)
 			os.Exit(errorcodes.ErrNetwork)

cli/core/install.go

@@ -65,7 +65,7 @@ func runInstallCommand(cmd *cobra.Command, args []string) {
 			Version:         platformRef.Version,
 		}
 		_, err := core.PlatformInstall(context.Background(), plattformInstallReq, output.ProgressBar(),
-			output.TaskProgress(), globals.HTTPClientHeader)
+			output.TaskProgress(), globals.NewHTTPClientHeader())
 		if err != nil {
 			feedback.Errorf("Error during install: %v", err)
 			os.Exit(errorcodes.ErrGeneric)

cli/core/upgrade.go

@@ -90,7 +90,7 @@ func runUpgradeCommand(cmd *cobra.Command, args []string) {
 			Architecture:    platformRef.Architecture,
 		}
 
-		_, err := core.PlatformUpgrade(context.Background(), r, output.ProgressBar(), output.TaskProgress(), globals.HTTPClientHeader)
+		_, err := core.PlatformUpgrade(context.Background(), r, output.ProgressBar(), output.TaskProgress(), globals.NewHTTPClientHeader())
 		if err == core.ErrAlreadyLatest {
 			feedback.Printf("Platform %s is already at the latest version", platformRef)
 		} else if err != nil {

cli/daemon/daemon.go

@@ -41,7 +41,7 @@ const (
 func NewCommand() *cobra.Command {
 	return &cobra.Command{
 		Use:     "daemon",
-		Short:   "Run as a daemon",
+		Short:   fmt.Sprintf("Run as a daemon on port %s", port),
 		Long:    "Running as a daemon the initialization of cores and libraries is done only once.",
 		Example: "  " + os.Args[0] + " daemon",
 		Args:    cobra.NoArgs,

cli/globals/globals.go

@@ -29,8 +29,6 @@ import (
 var (
 	// Debug determines whether to dump debug output to stderr or not
 	Debug bool
-	// HTTPClientHeader is the object that will be propagated to configure the clients inside the downloaders
-	HTTPClientHeader = getHTTPClientHeader()
 	// VersionInfo contains all info injected during build
 	VersionInfo = version.NewInfo(filepath.Base(os.Args[0]))
 	// Config FIXMEDOC
@@ -44,9 +42,9 @@ var (
 	LogLevel string
 )
 
-func getHTTPClientHeader() http.Header {
+// NewHTTPClientHeader returns the http.Header object that must be used by the clients inside the downloaders
+func NewHTTPClientHeader() http.Header {
 	userAgentValue := fmt.Sprintf("%s/%s (%s; %s; %s) Commit:%s", VersionInfo.Application,
 		VersionInfo.VersionString, runtime.GOARCH, runtime.GOOS, runtime.Version(), VersionInfo.Commit)
-	downloaderHeaders := http.Header{"User-Agent": []string{userAgentValue}}
-	return downloaderHeaders
+	return http.Header{"User-Agent": []string{userAgentValue}}
 }

cli/instance/instance.go

@@ -36,7 +36,7 @@ func initInstance() *rpc.InitResp {
 	logrus.Info("Initializing package manager")
 	req := packageManagerInitReq()
 
-	resp, err := commands.Init(context.Background(), req, output.ProgressBar(), output.TaskProgress(), globals.HTTPClientHeader)
+	resp, err := commands.Init(context.Background(), req, output.ProgressBar(), output.TaskProgress(), globals.NewHTTPClientHeader())
 	if err != nil {
 		feedback.Errorf("Error initializing package manager: %v", err)
 		os.Exit(errorcodes.ErrGeneric)

cli/lib/download.go

@@ -60,7 +60,7 @@ func runDownloadCommand(cmd *cobra.Command, args []string) {
 			Version:  library.Version,
 		}
 		_, err := lib.LibraryDownload(context.Background(), libraryDownloadReq, output.ProgressBar(),
-			globals.HTTPClientHeader)
+			globals.NewHTTPClientHeader())
 		if err != nil {
 			feedback.Errorf("Error downloading %s: %v", library, err)
 			os.Exit(errorcodes.ErrNetwork)

cli/lib/install.go

@@ -60,7 +60,7 @@ func runInstallCommand(cmd *cobra.Command, args []string) {
 			Version:  library.Version,
 		}
 		err := lib.LibraryInstall(context.Background(), libraryInstallReq, output.ProgressBar(),
-			output.TaskProgress(), globals.HTTPClientHeader)
+			output.TaskProgress(), globals.NewHTTPClientHeader())
 		if err != nil {
 			feedback.Errorf("Error installing %s: %v", library, err)
 			os.Exit(errorcodes.ErrGeneric)

cli/lib/upgrade.go

@@ -50,13 +50,13 @@ func runUpgradeCommand(cmd *cobra.Command, args []string) {
 	instance := instance.CreateInstaceIgnorePlatformIndexErrors()
 
 	if len(args) == 0 {
-		err := lib.LibraryUpgradeAll(instance.Id, output.ProgressBar(), output.TaskProgress(), globals.HTTPClientHeader)
+		err := lib.LibraryUpgradeAll(instance.Id, output.ProgressBar(), output.TaskProgress(), globals.NewHTTPClientHeader())
 		if err != nil {
 			feedback.Errorf("Error upgrading libraries: %v", err)
 			os.Exit(errorcodes.ErrGeneric)
 		}
 	} else {
-		err := lib.LibraryUpgrade(instance.Id, args, output.ProgressBar(), output.TaskProgress(), globals.HTTPClientHeader)
+		err := lib.LibraryUpgrade(instance.Id, args, output.ProgressBar(), output.TaskProgress(), globals.NewHTTPClientHeader())
 		if err != nil {
 			feedback.Errorf("Error upgrading libraries: %v", err)
 			os.Exit(errorcodes.ErrGeneric)

client_example/go.mod

@@ -1,6 +1,6 @@
 module github.com/arduino/arduino-cli/client_example
 
-go 1.12
+go 1.13
 
 require (
 	github.com/arduino/arduino-cli v0.0.0-20190826141027-35722fda467d

client_example/main.go

@@ -167,7 +167,9 @@ func initInstance(client rpc.ArduinoCoreClient) *rpc.Instance {
 	// the data folder.
 	initRespStream, err := client.Init(context.Background(), &rpc.InitReq{
 		Configuration: &rpc.Configuration{
-			DataDir: dataDir,
+			DataDir:       dataDir,
+			SketchbookDir: filepath.Join(dataDir, "sketchbook"),
+			DownloadsDir:  filepath.Join(dataDir, "staging"),
 		},
 	})
 	if err != nil {
@@ -368,11 +370,12 @@ func callBoardsDetails(client rpc.ArduinoCoreClient, instance *rpc.Instance) {
 }
 
 func callBoardAttach(client rpc.ArduinoCoreClient, instance *rpc.Instance) {
+	currDir, _ := os.Getwd()
 	boardattachresp, err := client.BoardAttach(context.Background(),
 		&rpc.BoardAttachReq{
 			Instance:   instance,
 			BoardUri:   "/dev/ttyACM0",
-			SketchPath: filepath.Join(dataDir, "hello.ino"),
+			SketchPath: filepath.Join(currDir, "hello.ino"),
 		})
 
 	if err != nil {
@@ -402,11 +405,12 @@ func callBoardAttach(client rpc.ArduinoCoreClient, instance *rpc.Instance) {
 }
 
 func callCompile(client rpc.ArduinoCoreClient, instance *rpc.Instance) {
+	currDir, _ := os.Getwd()
 	compRespStream, err := client.Compile(context.Background(),
 		&rpc.CompileReq{
 			Instance:   instance,
 			Fqbn:       "arduino:samd:mkr1000",
-			SketchPath: "hello.ino",
+			SketchPath: filepath.Join(currDir, "hello.ino"),
 			Verbose:    true,
 		})
 
@@ -440,11 +444,12 @@ func callCompile(client rpc.ArduinoCoreClient, instance *rpc.Instance) {
 }
 
 func callUpload(client rpc.ArduinoCoreClient, instance *rpc.Instance) {
+	currDir, _ := os.Getwd()
 	uplRespStream, err := client.Upload(context.Background(),
 		&rpc.UploadReq{
 			Instance:   instance,
 			Fqbn:       "arduino:samd:mkr1000",
-			SketchPath: "hello.ino",
+			SketchPath: filepath.Join(currDir, "hello.ino"),
 			Port:       "/dev/ttyACM0",
 			Verbose:    true,
 		})

commands/board/list.go

@@ -40,7 +40,7 @@ var (
 func apiByVidPid(url string) ([]*rpc.BoardListItem, error) {
 	retVal := []*rpc.BoardListItem{}
 	req, _ := http.NewRequest("GET", url, nil)
-	req.Header = globals.HTTPClientHeader
+	req.Header = globals.NewHTTPClientHeader()
 	req.Header.Set("Content-Type", "application/json")
 
 	if res, err := http.DefaultClient.Do(req); err == nil {
@@ -115,14 +115,16 @@ func List(instanceID int32) ([]*rpc.DetectedPort, error) {
 				port.IdentificationPrefs.Get("pid"))
 			items, err := apiByVidPid(url)
 			if err == ErrNotFound {
-				// the board couldn't be detected, keep going with the next port
+				// the board couldn't be detected, print a warning
 				logrus.Debug("Board not recognized")
-				continue
 			} else if err != nil {
 				// this is bad, bail out
 				return nil, errors.Wrap(err, "error getting board info from Arduino Cloud")
 			}
 
+			// add a DetectedPort entry in any case: the `Boards` field will
+			// be empty but the port will be shown anyways (useful for 3rd party
+			// boards)
 			b = items
 		}
 

commands/compile/compile.go

@@ -95,7 +95,7 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W
 	}
 
 	if toolsDir, err := config.BundleToolsDirectories(); err == nil {
-		builderCtx.ToolsDirs = toolsDir
+		builderCtx.BuiltInToolsDirs = toolsDir
 	} else {
 		return nil, fmt.Errorf("cannot get bundled tools directories: %s", err)
 	}
@@ -209,10 +209,12 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W
 
 	// Copy .elf file to sketch directory
 	srcElf := paths.New(outputPath[:len(outputPath)-3] + "elf")
-	dstElf := exportPath.Join(exportFile + ".elf")
-	logrus.WithField("from", srcElf).WithField("to", dstElf).Debug("copying sketch build output")
-	if err = srcElf.CopyTo(dstElf); err != nil {
-		return nil, fmt.Errorf("copying elf file: %s", err)
+	if srcElf.Exist() {
+		dstElf := exportPath.Join(exportFile + ".elf")
+		logrus.WithField("from", srcElf).WithField("to", dstElf).Debug("copying sketch build output")
+		if err = srcElf.CopyTo(dstElf); err != nil {
+			return nil, fmt.Errorf("copying elf file: %s", err)
+		}
 	}
 
 	logrus.Tracef("Compile %s for %s successful", sketch.Name, fqbnIn)

go.mod

@@ -5,7 +5,7 @@ go 1.13
 require (
 	bou.ke/monkey v1.0.1
 	github.com/arduino/board-discovery v0.0.0-20180823133458-1ba29327fb0c
-	github.com/arduino/go-paths-helper v0.0.0-20190214132331-c3c98d1bf2e1
+	github.com/arduino/go-paths-helper v1.0.1
 	github.com/arduino/go-properties-orderedmap v0.0.0-20190828172252-05018b28ff6c
 	github.com/arduino/go-timeutils v0.0.0-20171220113728-d1dd9e313b1b
 	github.com/arduino/go-win32-utils v0.0.0-20180330194947-ed041402e83b