diff --git a/README.md b/README.md index c2422bb6..e10a57bf 100644 --- a/README.md +++ b/README.md @@ -156,6 +156,14 @@ The default OTA upload should complete in 10 minutes. Use `--deferred` flag to e `$ arduino-cloud-cli ota upload --device-id --file --deferred` +It is also possible to perform a mass ota upload through a specific command. +The fqbn is mandatory. +To select the devices to update you can either provide a list of device ids or device tags. + +`$ arduino-cloud-cli ota mass-upload --fqbn --device-ids --file ` + +`$ arduino-cloud-cli ota mass-upload --fqbn --device-tags =,= --file ` + ## Dashboard commands Print a list of available dashboards and their widgets by using this command: diff --git a/cli/ota/massupload.go b/cli/ota/massupload.go new file mode 100644 index 00000000..e6dc1fd0 --- /dev/null +++ b/cli/ota/massupload.go @@ -0,0 +1,131 @@ +// This file is part of arduino-cloud-cli. +// +// Copyright (C) 2021 ARDUINO SA (http://www.arduino.cc/) +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published +// by the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ota + +import ( + "os" + "sort" + "strings" + + "github.com/arduino/arduino-cli/cli/errorcodes" + "github.com/arduino/arduino-cli/cli/feedback" + "github.com/arduino/arduino-cli/table" + "github.com/arduino/arduino-cloud-cli/command/ota" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +var massUploadFlags struct { + deviceIDs []string + tags map[string]string + file string + deferred bool + fqbn string +} + +func initMassUploadCommand() *cobra.Command { + massUploadCommand := &cobra.Command{ + Use: "mass-upload", + Short: "Mass OTA upload", + Long: "Mass OTA upload on devices of Arduino IoT Cloud", + Run: runMassUploadCommand, + } + + massUploadCommand.Flags().StringSliceVarP(&massUploadFlags.deviceIDs, "device-ids", "d", nil, + "Comma-separated list of device IDs to update") + massUploadCommand.Flags().StringToStringVar(&massUploadFlags.tags, "device-tags", nil, + "Comma-separated list of tags with format =.\n"+ + "Perform an OTA upload on all devices that match the provided tags.\n"+ + "Mutually exclusive with `--device-ids`.", + ) + massUploadCommand.Flags().StringVarP(&massUploadFlags.file, "file", "", "", "Binary file (.bin) to be uploaded") + massUploadCommand.Flags().BoolVar(&massUploadFlags.deferred, "deferred", false, "Perform a deferred OTA. It can take up to 1 week.") + massUploadCommand.Flags().StringVarP(&massUploadFlags.fqbn, "fqbn", "b", "", "FQBN of the devices to update") + + massUploadCommand.MarkFlagRequired("file") + massUploadCommand.MarkFlagRequired("fqbn") + return massUploadCommand +} + +func runMassUploadCommand(cmd *cobra.Command, args []string) { + logrus.Infof("Uploading binary %s", massUploadFlags.file) + + params := &ota.MassUploadParams{ + DeviceIDs: massUploadFlags.deviceIDs, + Tags: massUploadFlags.tags, + File: massUploadFlags.file, + Deferred: massUploadFlags.deferred, + FQBN: massUploadFlags.fqbn, + } + + resp, err := ota.MassUpload(params) + if err != nil { + feedback.Errorf("Error during ota upload: %v", err) + os.Exit(errorcodes.ErrGeneric) + } + + // Put successful devices ahead + sort.SliceStable(resp, func(i, j int) bool { + return resp[i].Err == nil + }) + + feedback.PrintResult(massUploadResult{resp}) + + var failed []string + for _, r := range resp { + if r.Err != nil { + failed = append(failed, r.ID) + } + } + if len(failed) == 0 { + return + } + failStr := strings.Join(failed, ",") + feedback.Printf( + "You can try to perform the OTA again on the failed devices using the following command:\n"+ + "$ arduino-cloud-cli ota upload --file %s -d %s", params.File, failStr, + ) +} + +type massUploadResult struct { + res []ota.Result +} + +func (r massUploadResult) Data() interface{} { + return r.res +} + +func (r massUploadResult) String() string { + if len(r.res) == 0 { + return "No OTA done." + } + t := table.New() + t.SetHeader("ID", "Result") + for _, r := range r.res { + outcome := "Success" + if r.Err != nil { + outcome = r.Err.Error() + } + + t.AddRow( + r.ID, + outcome, + ) + } + return t.Render() +} diff --git a/cli/ota/ota.go b/cli/ota/ota.go index 0a34125c..757c8272 100644 --- a/cli/ota/ota.go +++ b/cli/ota/ota.go @@ -29,6 +29,7 @@ func NewCommand() *cobra.Command { } otaCommand.AddCommand(initUploadCommand()) + otaCommand.AddCommand(initMassUploadCommand()) return otaCommand } diff --git a/command/ota/massupload.go b/command/ota/massupload.go new file mode 100644 index 00000000..85e2d539 --- /dev/null +++ b/command/ota/massupload.go @@ -0,0 +1,188 @@ +// This file is part of arduino-cloud-cli. +// +// Copyright (C) 2021 ARDUINO SA (http://www.arduino.cc/) +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published +// by the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ota + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + "path/filepath" + + "github.com/arduino/arduino-cloud-cli/internal/config" + "github.com/arduino/arduino-cloud-cli/internal/iot" + iotclient "github.com/arduino/iot-client-go" +) + +const ( + numConcurrentUploads = 10 +) + +// MassUploadParams contains the parameters needed to +// perform a Mass OTA upload. +type MassUploadParams struct { + DeviceIDs []string + Tags map[string]string + File string + Deferred bool + FQBN string +} + +// Result of an ota upload on a device +type Result struct { + ID string + Err error +} + +// MassUpload command is used to mass upload a firmware OTA, +// on devices of Arduino IoT Cloud. +func MassUpload(params *MassUploadParams) ([]Result, error) { + if params.DeviceIDs == nil && params.Tags == nil { + return nil, errors.New("provide either DeviceIDs or Tags") + } else if params.DeviceIDs != nil && params.Tags != nil { + return nil, errors.New("cannot use both DeviceIDs and Tags. only one of them should be not nil") + } + + // Generate .ota file + otaDir, err := ioutil.TempDir("", "") + if err != nil { + return nil, fmt.Errorf("%s: %w", "cannot create temporary folder", err) + } + otaFile := filepath.Join(otaDir, "temp.ota") + defer os.RemoveAll(otaDir) + + err = Generate(params.File, otaFile, params.FQBN) + if err != nil { + return nil, fmt.Errorf("%s: %w", "cannot generate .ota file", err) + } + + conf, err := config.Retrieve() + if err != nil { + return nil, err + } + iotClient, err := iot.NewClient(conf.Client, conf.Secret) + if err != nil { + return nil, err + } + + // Prepare the list of device-ids to update + d, err := idsGivenTags(iotClient, params.Tags) + if err != nil { + return nil, err + } + d = append(params.DeviceIDs, d...) + valid, invalid, err := validateDevices(iotClient, d, params.FQBN) + if err != nil { + return nil, fmt.Errorf("failed to validate devices: %w", err) + } + if len(valid) == 0 { + return invalid, nil + } + + expiration := otaExpirationMins + if params.Deferred { + expiration = otaDeferredExpirationMins + } + + res := run(iotClient, valid, otaFile, expiration) + res = append(res, invalid...) + return res, nil +} + +func idsGivenTags(iotClient iot.Client, tags map[string]string) ([]string, error) { + if tags == nil { + return nil, nil + } + devs, err := iotClient.DeviceList(tags) + if err != nil { + return nil, fmt.Errorf("%s: %w", "cannot retrieve devices from cloud", err) + } + devices := make([]string, 0, len(devs)) + for _, d := range devs { + devices = append(devices, d.Id) + } + return devices, nil +} + +func validateDevices(iotClient iot.Client, ids []string, fqbn string) (valid []string, invalid []Result, err error) { + devs, err := iotClient.DeviceList(nil) + if err != nil { + return nil, nil, fmt.Errorf("%s: %w", "cannot retrieve devices from cloud", err) + } + + for _, id := range ids { + var found *iotclient.ArduinoDevicev2 + for _, d := range devs { + if d.Id == id { + found = &d + break + } + } + // Device not found on the cloud + if found == nil { + inv := Result{ID: id, Err: fmt.Errorf("not found")} + invalid = append(invalid, inv) + continue + } + // Device FQBN doesn't match the passed one + if found.Fqbn != fqbn { + inv := Result{ID: id, Err: fmt.Errorf("has FQBN '%s' instead of '%s'", found.Fqbn, fqbn)} + invalid = append(invalid, inv) + continue + } + valid = append(valid, id) + } + return valid, invalid, nil +} + +func run(iotClient iot.Client, ids []string, otaFile string, expiration int) []Result { + type job struct { + id string + file *os.File + } + jobs := make(chan job, len(ids)) + + resCh := make(chan Result, len(ids)) + results := make([]Result, 0, len(ids)) + + for _, id := range ids { + file, err := os.Open(otaFile) + if err != nil { + r := Result{ID: id, Err: fmt.Errorf("cannot open ota file")} + results = append(results, r) + continue + } + jobs <- job{id: id, file: file} + } + close(jobs) + + for i := 0; i < numConcurrentUploads; i++ { + go func() { + for job := range jobs { + err := iotClient.DeviceOTA(job.id, job.file, expiration) + resCh <- Result{ID: job.id, Err: err} + } + }() + } + + for range ids { + r := <-resCh + results = append(results, r) + } + return results +} diff --git a/command/ota/massupload_test.go b/command/ota/massupload_test.go new file mode 100644 index 00000000..4c8b6f73 --- /dev/null +++ b/command/ota/massupload_test.go @@ -0,0 +1,91 @@ +package ota + +import ( + "errors" + "os" + "strings" + "testing" + + "github.com/arduino/arduino-cloud-cli/internal/iot/mocks" + iotclient "github.com/arduino/iot-client-go" + "github.com/stretchr/testify/mock" +) + +const testFilename = "testdata/empty.bin" + +func TestRun(t *testing.T) { + var ( + failPrefix = "00000000" + failID1 = failPrefix + "-b39d-47a2-adf3-d26cdf474707" + failID2 = failPrefix + "-9efd-4670-a478-df76ebdeeb4f" + okPrefix = "11111111" + okID1 = okPrefix + "-4838-4f46-8930-d735c5b76cd1" + okID2 = okPrefix + "-003f-42f9-a80c-85a1de36753b" + okID3 = okPrefix + "-dac4-4a6a-80a4-698062fe2af5" + ) + mockClient := &mocks.Client{} + mockDeviceOTA := func(id string, file *os.File, expireMins int) error { + if strings.Split(id, "-")[0] == failPrefix { + return errors.New("err") + } + return nil + } + mockClient.On("DeviceOTA", mock.Anything, mock.Anything, mock.Anything).Return(mockDeviceOTA, nil) + + devs := []string{okID1, failID1, okID2, failID2, okID3} + res := run(mockClient, devs, testFilename, 0) + if len(res) != len(devs) { + t.Errorf("expected %d results, got %d", len(devs), len(res)) + } + + for _, r := range res { + pre := strings.Split(r.ID, "-")[0] + if pre == okPrefix && r.Err != nil { + t.Errorf("device %s expected to succeed, but got error %s", r.ID, r.Err.Error()) + } + if pre == failPrefix && r.Err == nil { + t.Errorf("device %s expected to fail, but got no error", r.ID) + } + } +} + +func TestValidateDevices(t *testing.T) { + var ( + correctFQBN = "arduino:samd:nano_33_iot" + wrongFQBN = "arduino:samd:mkrwifi1010" + + idCorrect1 = "88d683a4-525e-423d-bad2-66a54d3585df" + idCorrect2 = "84b593fa-86dd-4954-904d-60f657158715" + idNotValid = "e3a3a667-a859-4317-be97-a61fb6f63487" + idNotFound = "deb17b7f-b39d-47a2-adf3-d26cdf474707" + ) + + mockClient := &mocks.Client{} + mockDeviceList := func(tags map[string]string) []iotclient.ArduinoDevicev2 { + return []iotclient.ArduinoDevicev2{ + {Id: idCorrect1, Fqbn: correctFQBN}, + {Id: idCorrect2, Fqbn: correctFQBN}, + {Id: idNotValid, Fqbn: wrongFQBN}, + } + } + mockClient.On("DeviceList", mock.Anything).Return(mockDeviceList, nil) + + ids := []string{ + idCorrect1, + idNotFound, + idCorrect2, + idNotValid, + } + v, i, err := validateDevices(mockClient, ids, correctFQBN) + if err != nil { + t.Errorf("unexpected error: %s", err.Error()) + } + + if len(v) != 2 { + t.Errorf("expected 2 valid devices, but found %d: %v", len(v), v) + } + + if len(i) != 2 { + t.Errorf("expected 2 invalid devices, but found %d: %v", len(i), i) + } +} diff --git a/command/ota/testdata/empty.bin b/command/ota/testdata/empty.bin new file mode 100644 index 00000000..e69de29b diff --git a/go.sum b/go.sum index 97016506..d79361d5 100644 --- a/go.sum +++ b/go.sum @@ -60,8 +60,6 @@ github.com/arduino/go-properties-orderedmap v1.3.0/go.mod h1:DKjD2VXY/NZmlingh4l github.com/arduino/go-timeutils v0.0.0-20171220113728-d1dd9e313b1b/go.mod h1:uwGy5PpN4lqW97FiLnbcx+xx8jly5YuPMJWfVwwjJiQ= github.com/arduino/go-win32-utils v0.0.0-20180330194947-ed041402e83b h1:3PjgYG5gVPA7cipp7vIR2lF96KkEJIFBJ+ANnuv6J20= github.com/arduino/go-win32-utils v0.0.0-20180330194947-ed041402e83b/go.mod h1:iIPnclBMYm1g32Q5kXoqng4jLhMStReIP7ZxaoUC2y8= -github.com/arduino/iot-client-go v1.3.4-0.20210930122852-04551f4cb061 h1:uQeaIHzj0tOlqnHaRskSy6UwMgQ6LIOECySpaYBCt5M= -github.com/arduino/iot-client-go v1.3.4-0.20210930122852-04551f4cb061/go.mod h1:gYvpMt7Qw+OSScTLyIlCnpbvy9y96ey/2zhB4w6FoK0= github.com/arduino/iot-client-go v1.3.4-0.20211103115604-d4d372164262 h1:qVq8cdkaRPaLc9DAjY/6rH3ocs6ZvnEJtD26f5++/RU= github.com/arduino/iot-client-go v1.3.4-0.20211103115604-d4d372164262/go.mod h1:gYvpMt7Qw+OSScTLyIlCnpbvy9y96ey/2zhB4w6FoK0= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= @@ -499,8 +497,6 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= -golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d h1:LO7XpTYMwTqxjLcGWPijK3vRXg1aWdlNOVOHRq45d7c= -golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211101193420-4a448f8816b3 h1:VrJZAjbekhoRn7n5FBujY31gboH+iB3pdLxn3gE9FjU= golang.org/x/net v0.0.0-20211101193420-4a448f8816b3/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -508,8 +504,6 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f h1:Qmd2pbz05z7z6lm0DrgQVVPuBm92jqujBKMHMOlOQEw= -golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211028175245-ba495a64dcb5 h1:v79phzBz03tsVCUTbvTBmmC3CUXF5mKYt7DA4ZVldpM= golang.org/x/oauth2 v0.0.0-20211028175245-ba495a64dcb5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=