feature: Detect board port change after upload (#2253)

* UploadResponse now has 'oneof' clause for better API design

* Added scaffolding to return updated-port after upload

* Upload port change detection (first draft)

* Simplified port detection using a Future-style abstraction

* Perform watcher-flush higher in the call tree

* Do not infer upload port if 'upload.wait_for_upload_port' is false

* Further simplified port detection subroutine structure

* fixed linter issue

* Always return an updatedUploadPort.

Arduino CLI should always return the port after an upload, even in the case
where no port change is expected. The consumer shouldn't be required to
implement "if not updated_upload_port, use original port" logic.

The whole point is that all the logic for determining which port should be
selected after an upload should be implemented in Arduino CLI. The consumer
should be able to simply select the port Arduino CLI tells it to select in
all cases.

* Updated docs

* Perform a deep-copy of upload ports where needed.

Previously only the pointer was copied, thus making changes in
`actualPort` to be reflected also to `port`. This lead to some weird
result in the `updatedUploadPort` result:

{
  "stdout": "Verify 11344 bytes of flash with checksum.\nVerify successful\ndone in 0.010 seconds\nCPU reset.\n",
  "stderr": "",
  "updated_upload_port": {
    "address": "/dev/tty.usbmodem14101",     <------- this address...
    "label": "/dev/cu.usbmodem14101",        <------- ...is different from the label
    "protocol": "serial",
    "protocol_label": "Serial Port (USB)",
    "properties": {
      "pid": "0x804E",
      "serialNumber": "94A3397C5150435437202020FF150838",
      "vid": "0x2341"
    },
    "hardware_id": "94A3397C5150435437202020FF150838"
  }
}

* When updating `actualPort` address, update also the address label.

* Fixed some potential nil pointer exceptions

* Further simplified board watcher

We must acesss the gRPC API only until we cross the `command` package
border. Once we are inside the `command` package we should use the
internal API only.

* Before returning from upload, check if the port is still alive

Now the upload detects cases when the upload port is "unstable", i.e.
the port changes even if it shouldn't (because the wait_for_upload_port
property in boards.txt is set to false).

This change should make the upload process more resilient.

* Apply suggestions from code review

Co-authored-by: per1234 <[email protected]>

* Fixed nil exception

* Improved tracking algorithm for upload-port reconnection

The new algorithm takes into account the case where a single board may
expose multiple ports, in this case the selection will increase priority
to ports that:

  1. have the same HW id as the user specified port for upload
  2. have the same protocol as the user specified port for upload
  3. have the same address as the user specified port for upload

---------

Co-authored-by: per1234 <[email protected]>

Author: cmaglie

Diff for: arduino/discovery/discovery.go

@@ -97,6 +97,12 @@ type Port struct {
 
 var tr = i18n.Tr
 
+// Equals returns true if the given port has the same address and protocol
+// of the current port.
+func (p *Port) Equals(o *Port) bool {
+	return p.Address == o.Address && p.Protocol == o.Protocol
+}
+
 // ToRPC converts Port into rpc.Port
 func (p *Port) ToRPC() *rpc.Port {
 	props := p.Properties
@@ -113,13 +119,43 @@ func (p *Port) ToRPC() *rpc.Port {
 	}
 }
 
+// PortFromRPCPort converts an *rpc.Port to a *Port
+func PortFromRPCPort(o *rpc.Port) (p *Port) {
+	if o == nil {
+		return nil
+	}
+	res := &Port{
+		Address:       o.Address,
+		AddressLabel:  o.Label,
+		Protocol:      o.Protocol,
+		ProtocolLabel: o.ProtocolLabel,
+		HardwareID:    o.HardwareId,
+	}
+	if o.Properties != nil {
+		res.Properties = properties.NewFromHashmap(o.Properties)
+	}
+	return res
+}
+
 func (p *Port) String() string {
 	if p == nil {
 		return "none"
 	}
 	return p.Address
 }
 
+// Clone creates a copy of this Port
+func (p *Port) Clone() *Port {
+	if p == nil {
+		return nil
+	}
+	var res Port = *p
+	if p.Properties != nil {
+		res.Properties = p.Properties.Clone()
+	}
+	return &res
+}
+
 // Event is a pluggable discovery event
 type Event struct {
 	Type        string

Diff for: arduino/discovery/discovery_client/main.go

@@ -51,10 +51,12 @@ func main() {
 		fmt.Printf("   Address: %s\n", port.Address)
 		fmt.Printf("  Protocol: %s\n", port.Protocol)
 		if ev.Type == "add" {
-			keys := port.Properties.Keys()
-			sort.Strings(keys)
-			for _, k := range keys {
-				fmt.Printf("            %s=%s\n", k, port.Properties.Get(k))
+			if port.Properties != nil {
+				keys := port.Properties.Keys()
+				sort.Strings(keys)
+				for _, k := range keys {
+					fmt.Printf("            %s=%s\n", k, port.Properties.Get(k))
+				}
 			}
 		}
 		fmt.Println()

Diff for: commands/board/list.go

@@ -34,6 +34,7 @@ import (
 	"github.com/arduino/arduino-cli/commands"
 	"github.com/arduino/arduino-cli/internal/inventory"
 	rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
+	"github.com/arduino/go-properties-orderedmap"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
@@ -128,20 +129,22 @@ func apiByVidPid(vid, pid string) ([]*rpc.BoardListItem, error) {
 	}, nil
 }
 
-func identifyViaCloudAPI(port *discovery.Port) ([]*rpc.BoardListItem, error) {
+func identifyViaCloudAPI(props *properties.Map) ([]*rpc.BoardListItem, error) {
 	// If the port is not USB do not try identification via cloud
-	id := port.Properties
-	if !id.ContainsKey("vid") || !id.ContainsKey("pid") {
+	if !props.ContainsKey("vid") || !props.ContainsKey("pid") {
 		return nil, nil
 	}
 
 	logrus.Debug("Querying builder API for board identification...")
-	return cachedAPIByVidPid(id.Get("vid"), id.Get("pid"))
+	return cachedAPIByVidPid(props.Get("vid"), props.Get("pid"))
 }
 
 // identify returns a list of boards checking first the installed platforms or the Cloud API
 func identify(pme *packagemanager.Explorer, port *discovery.Port) ([]*rpc.BoardListItem, error) {
 	boards := []*rpc.BoardListItem{}
+	if port.Properties == nil {
+		return boards, nil
+	}
 
 	// first query installed cores through the Package Manager
 	logrus.Debug("Querying installed cores for board identification...")
@@ -167,7 +170,7 @@ func identify(pme *packagemanager.Explorer, port *discovery.Port) ([]*rpc.BoardL
 	// if installed cores didn't recognize the board, try querying
 	// the builder API if the board is a USB device port
 	if len(boards) == 0 {
-		items, err := identifyViaCloudAPI(port)
+		items, err := identifyViaCloudAPI(port.Properties)
 		if err != nil {
 			// this is bad, but keep going
 			logrus.WithError(err).Debug("Error querying builder API")

Diff for: commands/board/list_test.go

@@ -103,10 +103,7 @@ func TestGetByVidPidMalformedResponse(t *testing.T) {
 }
 
 func TestBoardDetectionViaAPIWithNonUSBPort(t *testing.T) {
-	port := &discovery.Port{
-		Properties: properties.NewMap(),
-	}
-	items, err := identifyViaCloudAPI(port)
+	items, err := identifyViaCloudAPI(properties.NewMap())
 	require.NoError(t, err)
 	require.Empty(t, items)
 }

Diff for: commands/daemon/daemon.go

@@ -297,15 +297,27 @@ func (s *ArduinoCoreServerImpl) PlatformList(ctx context.Context, req *rpc.Platf
 // Upload FIXMEDOC
 func (s *ArduinoCoreServerImpl) Upload(req *rpc.UploadRequest, stream rpc.ArduinoCoreService_UploadServer) error {
 	syncSend := NewSynchronizedSend(stream.Send)
-	outStream := feedStreamTo(func(data []byte) { syncSend.Send(&rpc.UploadResponse{OutStream: data}) })
-	errStream := feedStreamTo(func(data []byte) { syncSend.Send(&rpc.UploadResponse{ErrStream: data}) })
-	err := upload.Upload(stream.Context(), req, outStream, errStream)
+	outStream := feedStreamTo(func(data []byte) {
+		syncSend.Send(&rpc.UploadResponse{
+			Message: &rpc.UploadResponse_OutStream{OutStream: data},
+		})
+	})
+	errStream := feedStreamTo(func(data []byte) {
+		syncSend.Send(&rpc.UploadResponse{
+			Message: &rpc.UploadResponse_ErrStream{ErrStream: data},
+		})
+	})
+	res, err := upload.Upload(stream.Context(), req, outStream, errStream)
 	outStream.Close()
 	errStream.Close()
-	if err != nil {
-		return convertErrorToRPCStatus(err)
+	if res != nil {
+		syncSend.Send(&rpc.UploadResponse{
+			Message: &rpc.UploadResponse_Result{
+				Result: res,
+			},
+		})
 	}
-	return nil
+	return convertErrorToRPCStatus(err)
 }
 
 // UploadUsingProgrammer FIXMEDOC

Diff for: commands/upload/burnbootloader.go

@@ -39,7 +39,7 @@ func BurnBootloader(ctx context.Context, req *rpc.BurnBootloaderRequest, outStre
 	}
 	defer release()
 
-	err := runProgramAction(
+	_, err := runProgramAction(
 		pme,
 		nil, // sketch
 		"",  // importFile