Merge branch 'master' into massi/rpc-monitor

Author: masci

Diff for: .dependabot/config.yml

@@ -5,4 +5,6 @@ update_configs:
     update_schedule: "daily"
     allowed_updates:
       - match:
-          update_type: "security"
+          update_type: "security"
+    default_labels:
+      - "component/dependencies"

Diff for: .drone.yml

@@ -3,29 +3,36 @@ name: default
 
 steps:
 - name: lint
-  image: arduino/arduino-cli:drone-0.2.1
+  image: arduino/arduino-cli:drone-1.0
   commands:
     # Check if the Go code is properly formatted and run the linter
     - task check
     # Ensure protobufs compile
     - task protoc
 
 - name: build
-  image: arduino/arduino-cli:drone-0.2.1
+  image: arduino/arduino-cli:drone-1.0
   commands:
     - task build
 
 - name: test
-  image: arduino/arduino-cli:drone-0.2.1
+  image: arduino/arduino-cli:drone-1.0
   commands:
-    - task test
+    - task test-unit
     - task test-legacy
 
+- name: integration
+  image: arduino/arduino-cli:drone-1.0
+  failure: ignore  # work in progress, we know some tests will fail at the moment
+  commands:
+    - pip install -r test/requirements.txt
+    - task test-integration
+
 # Contrary to other CI platforms, uploading reports to Codecov requires Drone to provide a token.
 # To avoid exposing the Codecov token to external PRs, we only upload coverage when we merge on
 # `master`.
 - name: coverage
-  image: arduino/arduino-cli:drone-0.2.1
+  image: arduino/arduino-cli:drone-1.0
   environment:
     CODECOV_TOKEN:
       from_secret: codecov_token

Diff for: .gitignore

@@ -1,9 +1,12 @@
 /debug
 /arduino-cli
 /main
-/.vscode/settings.json
+/.vscode/
 /cmd/formatter/debug.test
 /arduino-cli.yaml
 /wiki
 .idea
-coverage_*.txt
+coverage_*.txt
+__pycache__
+venv
+.pytest_cache

Diff for: Dockerfiles/CI/Dockerfile

@@ -1,11 +1,12 @@
 FROM golang:1.12
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
-		bzip2 \
-        unzip \
+    bzip2 \
+    unzip \
     && rm -rf /var/lib/apt/lists/*
 
 ENV PROTOBUF_VER 3.8.0
+ENV PATH="/miniconda/bin:${PATH}"
 
 # NOTE: most of the following assume WORDKIR is '/'
 RUN set -ex \
@@ -20,5 +21,11 @@ RUN set -ex \
     && go get github.com/golang/protobuf/protoc-gen-go \
     && mkdir protobuf && cd protobuf \
     && curl -LO https://github.com/google/protobuf/releases/download/v$PROTOBUF_VER/protoc-$PROTOBUF_VER-linux-x86_64.zip \
-    && unzip protoc-$PROTOBUF_VER-linux-x86_64.zip && cp ./bin/* /bin/ \
-    && cd .. &&  rm -rf protobuf
+    && unzip protoc-$PROTOBUF_VER-linux-x86_64.zip \
+    && cp ./bin/* /bin/  \
+    # protoc will search for default includes in the path of the binary
+    && cp -r ./include /bin/ \
+    && cd .. &&  rm -rf protobuf \
+    # Install a recent version of Python
+    && curl -o $HOME/miniconda.sh -LO https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \
+    && chmod +x $HOME/miniconda.sh  && bash ~/miniconda.sh -b -p /miniconda && rm -f $HOME/miniconda.sh

Diff for: Taskfile.yml

@@ -21,17 +21,18 @@ tasks:
   test-unit:
     desc: Run unit tests only
     cmds:
-      - go test -short {{ default "-v" .GOFLAGS }} -coverprofile=coverage_unit.txt {{ default .DEFAULT_TARGETS .TARGETS }}
+      - go test -short {{ default "-v" .GOFLAGS }} -coverprofile=coverage_unit.txt {{ default .DEFAULT_TARGETS .TARGETS }} {{.TEST_LDFLAGS}}
 
   test-integration:
     desc: Run integration tests only
     cmds:
       - go test -run Integration {{ default "-v" .GOFLAGS }} -coverprofile=coverage_integ.txt {{ default .DEFAULT_TARGETS .TARGETS }} {{.TEST_LDFLAGS}}
+      - pytest test/
 
   test-legacy:
     desc: Run tests for the `legacy` package
     cmds:
-      - go test {{ default "-v" .GOFLAGS }} ./legacy/...
+      - go test {{ default "-v -failfast" .GOFLAGS }} ./legacy/...
 
   test-unit-race:
     desc: Run unit tests only with race condition detection

Diff for: arduino/cores/cores.go

@@ -18,6 +18,7 @@
 package cores
 
 import (
+	"encoding/json"
 	"strings"
 
 	paths "github.com/arduino/go-paths-helper"
@@ -41,14 +42,13 @@ type PlatformRelease struct {
 	Resource       *resources.DownloadResource
 	Version        *semver.Version
 	BoardsManifest []*BoardManifest
-	Dependencies   ToolDependencies // The Dependency entries to load tools.
-	Platform       *Platform        `json:"-"`
-
-	Properties  *properties.Map            `json:"-"`
-	Boards      map[string]*Board          `json:"-"`
-	Programmers map[string]*properties.Map `json:"-"`
-	Menus       *properties.Map            `json:"-"`
-	InstallDir  *paths.Path                `json:"-"`
+	Dependencies   ToolDependencies           // The Dependency entries to load tools.
+	Platform       *Platform                  `json:"-"`
+	Properties     *properties.Map            `json:"-"`
+	Boards         map[string]*Board          `json:"-"`
+	Programmers    map[string]*properties.Map `json:"-"`
+	Menus          *properties.Map            `json:"-"`
+	InstallDir     *paths.Path                `json:"-"`
 }
 
 // BoardManifest contains information about a board. These metadata are usually
@@ -226,3 +226,25 @@ func (release *PlatformRelease) String() string {
 	}
 	return release.Platform.String() + "@" + version
 }
+
+// MarshalJSON provides a more user friendly serialization for
+// PlatformRelease objects.
+func (release *PlatformRelease) MarshalJSON() ([]byte, error) {
+	latestStr := ""
+	latest := release.Platform.GetLatestRelease()
+	if latest != nil {
+		latestStr = latest.Version.String()
+	}
+
+	return json.Marshal(&struct {
+		ID        string `json:"ID,omitempty"`
+		Installed string `json:"Installed,omitempty"`
+		Latest    string `json:"Latest,omitempty"`
+		Name      string `json:"Name,omitempty"`
+	}{
+		ID:        release.Platform.String(),
+		Installed: release.Version.String(),
+		Latest:    latestStr,
+		Name:      release.Platform.Name,
+	})
+}

Diff for: arduino/cores/packageindex/index.go

@@ -24,7 +24,7 @@ import (
 	"github.com/arduino/arduino-cli/arduino/cores"
 	"github.com/arduino/arduino-cli/arduino/resources"
 	"github.com/arduino/go-paths-helper"
-	"go.bug.st/relaxed-semver"
+	semver "go.bug.st/relaxed-semver"
 )
 
 // Index represents Cores and Tools struct as seen from package_index.json file.
@@ -97,13 +97,13 @@ type indexHelp struct {
 
 // MergeIntoPackages converts the Index data into a cores.Packages and merge them
 // with the existing conents of the cores.Packages passed as parameter.
-func (index Index) MergeIntoPackages(outPackages *cores.Packages) {
+func (index Index) MergeIntoPackages(outPackages cores.Packages) {
 	for _, inPackage := range index.Packages {
 		inPackage.extractPackageIn(outPackages)
 	}
 }
 
-func (inPackage indexPackage) extractPackageIn(outPackages *cores.Packages) {
+func (inPackage indexPackage) extractPackageIn(outPackages cores.Packages) {
 	outPackage := outPackages.GetOrCreatePackage(inPackage.Name)
 	outPackage.Maintainer = inPackage.Maintainer
 	outPackage.WebsiteURL = inPackage.WebsiteURL

Diff for: arduino/cores/packagemanager/download.go

@@ -23,7 +23,7 @@ import (
 
 	"github.com/arduino/arduino-cli/arduino/cores"
 	"go.bug.st/downloader"
-	"go.bug.st/relaxed-semver"
+	semver "go.bug.st/relaxed-semver"
 )
 
 // PlatformReference represents a tuple to identify a Platform
@@ -44,7 +44,7 @@ func (platform *PlatformReference) String() string {
 // FindPlatform returns the Platform matching the PlatformReference or nil if not found.
 // The PlatformVersion field of the reference is ignored.
 func (pm *PackageManager) FindPlatform(ref *PlatformReference) *cores.Platform {
-	targetPackage, ok := pm.GetPackages().Packages[ref.Package]
+	targetPackage, ok := pm.Packages[ref.Package]
 	if !ok {
 		return nil
 	}
@@ -71,7 +71,7 @@ func (pm *PackageManager) FindPlatformRelease(ref *PlatformReference) *cores.Pla
 // FindPlatformReleaseDependencies takes a PlatformReference and returns a set of items to download and
 // a set of outputs for non existing platforms.
 func (pm *PackageManager) FindPlatformReleaseDependencies(item *PlatformReference) (*cores.PlatformRelease, []*cores.ToolRelease, error) {
-	targetPackage, exists := pm.packages.Packages[item.Package]
+	targetPackage, exists := pm.Packages[item.Package]
 	if !exists {
 		return nil, nil, fmt.Errorf("package %s not found", item.Package)
 	}
@@ -94,7 +94,7 @@ func (pm *PackageManager) FindPlatformReleaseDependencies(item *PlatformReferenc
 	}
 
 	// replaces "latest" with latest version too
-	toolDeps, err := pm.packages.GetDepsOfPlatformRelease(release)
+	toolDeps, err := pm.Packages.GetDepsOfPlatformRelease(release)
 	if err != nil {
 		return nil, nil, fmt.Errorf("getting tool dependencies for platform %s: %s", release.String(), err)
 	}

Diff for: arduino/cores/packagemanager/install_uninstall.go

@@ -127,7 +127,7 @@ func (pm *PackageManager) UninstallTool(toolRelease *cores.ToolRelease) error {
 // passed as parameter
 func (pm *PackageManager) IsToolRequired(toolRelease *cores.ToolRelease) bool {
 	// Search in all installed platforms
-	for _, targetPackage := range pm.packages.Packages {
+	for _, targetPackage := range pm.Packages {
 		for _, platform := range targetPackage.Platforms {
 			if platformRelease := pm.GetInstalledPlatformRelease(platform); platformRelease != nil {
 				if platformRelease.RequiresToolRelease(toolRelease) {

Diff for: arduino/cores/packagemanager/loader.go

@@ -112,7 +112,7 @@ func (pm *PackageManager) LoadHardwareFromDirectory(path *paths.Path) error {
 			continue
 		}
 
-		targetPackage := pm.packages.GetOrCreatePackage(packager)
+		targetPackage := pm.Packages.GetOrCreatePackage(packager)
 		if err := pm.loadPlatforms(targetPackage, architectureParentPath); err != nil {
 			return fmt.Errorf("loading package %s: %s", packager, err)
 		}
@@ -419,7 +419,7 @@ func (pm *PackageManager) LoadToolsFromBundleDirectory(toolsPath *paths.Path) er
 		}
 
 		for packager, toolsData := range all.FirstLevelOf() {
-			targetPackage := pm.packages.GetOrCreatePackage(packager)
+			targetPackage := pm.Packages.GetOrCreatePackage(packager)
 
 			for toolName, toolVersion := range toolsData.AsMap() {
 				tool := targetPackage.GetOrCreateTool(toolName)
@@ -431,7 +431,7 @@ func (pm *PackageManager) LoadToolsFromBundleDirectory(toolsPath *paths.Path) er
 		}
 	} else {
 		// otherwise load the tools inside the unnamed package
-		unnamedPackage := pm.packages.GetOrCreatePackage("")
+		unnamedPackage := pm.Packages.GetOrCreatePackage("")
 		pm.loadToolsFromPackage(unnamedPackage, toolsPath)
 	}
 	return nil