OTA on multiple devices

cli/ota/massupload.go

@@ -0,0 +1,94 @@
+// This file is part of arduino-cloud-cli.
+//
+// Copyright (C) 2021 ARDUINO SA (http://www.arduino.cc/)
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published
+// by the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+package ota
+
+import (
+	"os"
+	"strings"
+
+	"github.com/arduino/arduino-cli/cli/errorcodes"
+	"github.com/arduino/arduino-cli/cli/feedback"
+	"github.com/arduino/arduino-cloud-cli/command/ota"
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+var massUploadFlags struct {
+	deviceIDs []string
+	tags      map[string]string
+	file      string
+	deferred  bool
+	fqbn      string
+}
+
+func initMassUploadCommand() *cobra.Command {
+	massUploadCommand := &cobra.Command{
+		Use:   "mass-upload",
+		Short: "Mass OTA upload",
+		Long:  "Mass OTA upload on devices of Arduino IoT Cloud",
+		Run:   runMassUploadCommand,
+	}
+
+	massUploadCommand.Flags().StringSliceVarP(&massUploadFlags.deviceIDs, "device-ids", "d", nil,
+		"Comma-separated list of device IDs to update")
+	massUploadCommand.Flags().StringToStringVar(&massUploadFlags.tags, "tags", nil,
+		"Comma-separated list of tags with format <key>=<value>.\n"+
+			"Perform and OTA upload on all devices that match the provided tags.\n"+
+			"Mutually exclusive with `--device-id`.",
+	)
+	massUploadCommand.Flags().StringVarP(&massUploadFlags.file, "file", "", "", "Binary file (.bin) to be uploaded")
+	massUploadCommand.Flags().BoolVar(&massUploadFlags.deferred, "deferred", false, "Perform a deferred OTA. It can take up to 1 week.")
+	massUploadCommand.Flags().StringVarP(&massUploadFlags.fqbn, "fqbn", "b", "", "FQBN of the devices to update")
+
+	massUploadCommand.MarkFlagRequired("file")
+	massUploadCommand.MarkFlagRequired("fqbn")
+	return massUploadCommand
+}
+
+func runMassUploadCommand(cmd *cobra.Command, args []string) {
+	logrus.Infof("Uploading binary %s", massUploadFlags.file)
+
+	params := &ota.MassUploadParams{
+		DeviceIDs: massUploadFlags.deviceIDs,
+		Tags:      massUploadFlags.tags,
+		File:      massUploadFlags.file,
+		Deferred:  massUploadFlags.deferred,
+		FQBN:      massUploadFlags.fqbn,
+	}
+
+	resp, err := ota.MassUpload(params)
+	if err != nil {
+		feedback.Errorf("Error during ota upload: %v", err)
+		os.Exit(errorcodes.ErrGeneric)
+	}
+
+	success := strings.Join(resp.Updated, ",")
+	success = strings.TrimRight(success, ",")
+	feedback.Printf("\nSuccessfully sent OTA request to: %s", success)
+
+	invalid := strings.Join(resp.Invalid, ",")
+	invalid = strings.TrimRight(invalid, ",")
+	feedback.Printf("Cannot send OTA request to: %s", invalid)
+
+	fail := strings.Join(resp.Failed, ",")
+	fail = strings.TrimRight(fail, ",")
+	feedback.Printf("Failed to send OTA request to: %s", fail)
+
+	det := strings.Join(resp.Errors, "\n")
+	feedback.Printf("\nDetails:\n%s", det)
+}

cli/ota/ota.go

@@ -29,6 +29,7 @@ func NewCommand() *cobra.Command {
 	}
 
 	otaCommand.AddCommand(initUploadCommand())
+	otaCommand.AddCommand(initMassUploadCommand())
 
 	return otaCommand
 }

command/ota/massupload.go

@@ -0,0 +1,202 @@
+// This file is part of arduino-cloud-cli.
+//
+// Copyright (C) 2021 ARDUINO SA (http://www.arduino.cc/)
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published
+// by the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+package ota
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/arduino/arduino-cloud-cli/internal/config"
+	"github.com/arduino/arduino-cloud-cli/internal/iot"
+	iotclient "github.com/arduino/iot-client-go"
+)
+
+const (
+	numConcurrentUploads = 10
+)
+
+// MassUploadParams contains the parameters needed to
+// perform a Mass OTA upload.
+type MassUploadParams struct {
+	DeviceIDs []string
+	Tags      map[string]string
+	File      string
+	Deferred  bool
+	FQBN      string
+}
+
+// MassUploadResp contains the results of the mass ota upload
+type MassUploadResp struct {
+	Updated []string // Ids of devices updated
+	Invalid []string // Ids of device not valid (mismatched fqbn)
+	Failed  []string // Ids of device failed
+	Errors  []string // Contains detailed errors for each failure
+}
+
+// MassUpload command is used to mass upload a firmware OTA,
+// on devices of Arduino IoT Cloud.
+func MassUpload(params *MassUploadParams) (*MassUploadResp, error) {
+	if params.DeviceIDs == nil && params.Tags == nil {
+		return nil, errors.New("provide either DeviceIDs or Tags")
+	} else if params.DeviceIDs != nil && params.Tags != nil {
+		return nil, errors.New("cannot use both DeviceIDs and Tags. only one of them should be not nil")
+	}
+
+	// Generate .ota file
+	otaDir, err := ioutil.TempDir("", "")
+	if err != nil {
+		return nil, fmt.Errorf("%s: %w", "cannot create temporary folder", err)
+	}
+	otaFile := filepath.Join(otaDir, "temp.ota")
+	defer os.RemoveAll(otaDir)
+
+	err = Generate(params.File, otaFile, params.FQBN)
+	if err != nil {
+		return nil, fmt.Errorf("%s: %w", "cannot generate .ota file", err)
+	}
+
+	conf, err := config.Retrieve()
+	if err != nil {
+		return nil, err
+	}
+	iotClient, err := iot.NewClient(conf.Client, conf.Secret)
+	if err != nil {
+		return nil, err
+	}
+
+	d, err := idsGivenTags(iotClient, params.Tags)
+	if err != nil {
+		return nil, err
+	}
+	d = append(params.DeviceIDs, d...)
+	valid, invalid, details, err := validateDevices(iotClient, d, params.FQBN)
+	if err != nil {
+		return nil, fmt.Errorf("failed to validate devices: %w", err)
+	}
+	if len(valid) == 0 {
+		return &MassUploadResp{Invalid: invalid}, nil
+	}
+
+	expiration := otaExpirationMins
+	if params.Deferred {
+		expiration = otaDeferredExpirationMins
+	}
+
+	good, fail, ers := run(iotClient, valid, otaFile, expiration)
+	if err != nil {
+		return nil, err
+	}
+
+	// Merge the failure details with the details of invalid devices
+	ers = append(details, ers...)
+
+	return &MassUploadResp{Updated: good, Invalid: invalid, Failed: fail, Errors: ers}, nil
+}
+
+func idsGivenTags(iotClient iot.Client, tags map[string]string) ([]string, error) {
+	if tags == nil {
+		return nil, nil
+	}
+	devs, err := iotClient.DeviceList(tags)
+	if err != nil {
+		return nil, fmt.Errorf("%s: %w", "cannot retrieve devices from cloud", err)
+	}
+	devices := make([]string, 0, len(devs))
+	for _, d := range devs {
+		devices = append(devices, d.Id)
+	}
+	return devices, nil
+}
+
+func validateDevices(iotClient iot.Client, ids []string, fqbn string) (valid, invalid, details []string, err error) {
+	devs, err := iotClient.DeviceList(nil)
+	if err != nil {
+		return nil, nil, nil, fmt.Errorf("%s: %w", "cannot retrieve devices from cloud", err)
+	}
+
+	for _, id := range ids {
+		var found *iotclient.ArduinoDevicev2
+		for _, d := range devs {
+			if d.Id == id {
+				found = &d
+				break
+			}
+		}
+		// Device not found on the cloud
+		if found == nil {
+			invalid = append(invalid, id)
+			details = append(details, fmt.Sprintf("%s : not found", id))
+			continue
+		}
+		// Device FQBN doesn't match the passed one
+		if found.Fqbn != fqbn {
+			invalid = append(invalid, id)
+			details = append(details, fmt.Sprintf("%s : has FQBN `%s` instead of `%s`", found.Id, found.Fqbn, fqbn))
+			continue
+		}
+		valid = append(valid, id)
+	}
+	return valid, invalid, details, nil
+}
+
+func run(iotClient iot.Client, ids []string, otaFile string, expiration int) (updated, failed, errors []string) {
+	type job struct {
+		id   string
+		file *os.File
+	}
+	jobs := make(chan job, len(ids))
+
+	type result struct {
+		id  string
+		err error
+	}
+	results := make(chan result, len(ids))
+
+	for _, id := range ids {
+		file, err := os.Open(otaFile)
+		if err != nil {
+			failed = append(failed, id)
+			errors = append(errors, fmt.Sprintf("%s: cannot open ota file", id))
+		}
+		jobs <- job{id: id, file: file}
+	}
+	close(jobs)
+
+	for i := 0; i < numConcurrentUploads; i++ {
+		go func() {
+			for job := range jobs {
+				err := iotClient.DeviceOTA(job.id, job.file, expiration)
+				results <- result{id: job.id, err: err}
+			}
+		}()
+	}
+
+	for range ids {
+		r := <-results
+		if r.err != nil {
+			failed = append(failed, r.id)
+			errors = append(errors, fmt.Sprintf("%s: %s", r.id, r.err.Error()))
+		} else {
+			updated = append(updated, r.id)
+		}
+	}
+	return
+}

command/ota/massupload_test.go

@@ -0,0 +1,87 @@
+package ota
+
+import (
+	"errors"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/arduino/arduino-cloud-cli/internal/iot/mocks"
+	iotclient "github.com/arduino/iot-client-go"
+	"github.com/stretchr/testify/mock"
+)
+
+const testFilename = "testdata/empty.bin"
+
+func TestRun(t *testing.T) {
+	var (
+		failID1 = "00000000-b39d-47a2-adf3-d26cdf474707"
+		failID2 = "00000000-9efd-4670-a478-df76ebdeeb4f"
+		okID1   = "11111111-4838-4f46-8930-d735c5b76cd1"
+		okID2   = "11111111-003f-42f9-a80c-85a1de36753b"
+		okID3   = "11111111-dac4-4a6a-80a4-698062fe2af5"
+	)
+	mockClient := &mocks.Client{}
+	mockDeviceOTA := func(id string, file *os.File, expireMins int) error {
+		time.Sleep(100 * time.Millisecond)
+		if strings.Split(id, "-")[0] == "00000000" {
+			return errors.New("err")
+		}
+		return nil
+	}
+	mockClient.On("DeviceOTA", mock.Anything, mock.Anything, mock.Anything).Return(mockDeviceOTA, nil)
+
+	good, fail, err := run(mockClient, []string{okID1, failID1, okID2, failID2, okID3}, testFilename, 0)
+	if len(err) != 2 {
+		t.Errorf("two errors should have been returned, got %d: %v", len(err), err)
+	}
+	if len(fail) != 2 {
+		t.Errorf("two updates should have failed, got %d: %v", len(fail), fail)
+	}
+	if len(good) != 3 {
+		t.Errorf("two updates should have succeded, got %d: %v", len(good), good)
+	}
+}
+
+func TestValidateDevices(t *testing.T) {
+	var (
+		correctFQBN = "arduino:samd:nano_33_iot"
+		wrongFQBN   = "arduino:samd:mkrwifi1010"
+
+		idCorrect1 = "88d683a4-525e-423d-bad2-66a54d3585df"
+		idCorrect2 = "84b593fa-86dd-4954-904d-60f657158715"
+		idNotValid = "e3a3a667-a859-4317-be97-a61fb6f63487"
+		idNotFound = "deb17b7f-b39d-47a2-adf3-d26cdf474707"
+	)
+
+	mockClient := &mocks.Client{}
+	mockDeviceList := func(tags map[string]string) []iotclient.ArduinoDevicev2 {
+		return []iotclient.ArduinoDevicev2{
+			{Id: idCorrect1, Fqbn: correctFQBN},
+			{Id: idCorrect2, Fqbn: correctFQBN},
+			{Id: idNotValid, Fqbn: wrongFQBN},
+		}
+	}
+	mockClient.On("DeviceList", mock.Anything).Return(mockDeviceList, nil)
+
+	ids := []string{
+		idCorrect1,
+		idNotFound,
+		idCorrect2,
+		idNotValid,
+	}
+	v, i, d, _ := validateDevices(mockClient, ids, correctFQBN)
+
+	if len(v) != 2 {
+		t.Errorf("expected 2 valid devices, but found %d: %v", len(v), v)
+	}
+
+	if len(i) != 2 {
+		t.Errorf("expected 2 invalid devices, but found %d: %v", len(i), i)
+	}
+
+	if len(d) != 2 {
+		t.Errorf("expected 2 error details, but found %d: %v", len(d), d)
+	}
+}