Skip to content

Commit c2bf718

Browse files
cmagliecmaglie
authored
[breaking] core update-index do not stop at the first download error (#1866)
* Added flag to skip 3rd party package_index in UpdateIndex gRPC API Fix #1788 * UpdateIndex gRPC call does not stop at the first error * Updated docs * Fixed integration tests * Ensure that a 'Completed' download progress is always sent * UpdateIndex: Fixed a wrong success report and added test. * Improved test error messages
1 parent 3cd782d commit c2bf718

File tree

15 files changed

+1086
-715
lines changed

15 files changed

+1086
-715
lines changed

Diff for: ‎arduino/httpclient/httpclient.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ func DownloadFile(path *paths.Path, URL string, label string, downloadCB rpc.Dow
5050
Url: d.URL,
5151
TotalSize: d.Size(),
5252
})
53+
defer downloadCB(&rpc.DownloadProgress{Completed: true})
5354

5455
err = d.RunAndPoll(func(downloaded int64) {
5556
downloadCB(&rpc.DownloadProgress{Downloaded: downloaded})
@@ -63,7 +64,6 @@ func DownloadFile(path *paths.Path, URL string, label string, downloadCB rpc.Dow
6364
return &arduino.FailedDownloadError{Message: tr("Server responded with: %s", d.Resp.Status)}
6465
}
6566

66-
downloadCB(&rpc.DownloadProgress{Completed: true})
6767
return nil
6868
}
6969

Diff for: ‎cli/core/search.go

+3-4
Original file line numberDiff line numberDiff line change
@@ -67,11 +67,10 @@ func runSearchCommand(cmd *cobra.Command, args []string) {
6767
}
6868

6969
if indexesNeedUpdating(indexUpdateInterval) {
70-
_, err := commands.UpdateIndex(context.Background(), &rpc.UpdateIndexRequest{
71-
Instance: inst,
72-
}, output.ProgressBar())
70+
err := commands.UpdateIndex(context.Background(), &rpc.UpdateIndexRequest{Instance: inst},
71+
output.ProgressBar(),
72+
output.PrintErrorFromDownloadResult(tr("Error updating index")))
7373
if err != nil {
74-
feedback.Errorf(tr("Error updating index: %v"), err)
7574
os.Exit(errorcodes.ErrGeneric)
7675
}
7776
}

Diff for: ‎cli/core/update_index.go

+3-5
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ import (
2020
"os"
2121

2222
"github.com/arduino/arduino-cli/cli/errorcodes"
23-
"github.com/arduino/arduino-cli/cli/feedback"
2423
"github.com/arduino/arduino-cli/cli/instance"
2524
"github.com/arduino/arduino-cli/cli/output"
2625
"github.com/arduino/arduino-cli/commands"
@@ -45,11 +44,10 @@ func runUpdateIndexCommand(cmd *cobra.Command, args []string) {
4544
inst := instance.CreateInstanceAndRunFirstUpdate()
4645
logrus.Info("Executing `arduino-cli core update-index`")
4746

48-
_, err := commands.UpdateIndex(context.Background(), &rpc.UpdateIndexRequest{
49-
Instance: inst,
50-
}, output.ProgressBar())
47+
err := commands.UpdateIndex(context.Background(), &rpc.UpdateIndexRequest{Instance: inst},
48+
output.ProgressBar(),
49+
output.PrintErrorFromDownloadResult(tr("Error updating index")))
5150
if err != nil {
52-
feedback.Errorf(tr("Error updating index: %v"), err)
5351
os.Exit(errorcodes.ErrGeneric)
5452
}
5553
}

Diff for: ‎cli/instance/instance.go

+4-3
Original file line numberDiff line numberDiff line change
@@ -152,12 +152,13 @@ func FirstUpdate(instance *rpc.Instance) error {
152152
// similarly to the library update we download that file and all the other package indexes
153153
// from additional_urls
154154
if packageIndex.NotExist() {
155-
_, err := commands.UpdateIndex(context.Background(),
155+
err := commands.UpdateIndex(context.Background(),
156156
&rpc.UpdateIndexRequest{
157-
Instance: instance,
157+
Instance: instance,
158+
IgnoreCustomPackageIndexes: true,
158159
},
159160
output.ProgressBar(),
160-
)
161+
output.PrintErrorFromDownloadResult(tr("Error updating index")))
161162
if err != nil {
162163
return err
163164
}

Diff for: ‎cli/output/rpc_progress.go

+16
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ package output
1818
import (
1919
"fmt"
2020

21+
"github.com/arduino/arduino-cli/cli/feedback"
2122
"github.com/arduino/arduino-cli/i18n"
2223
rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
2324
"github.com/cmaglie/pb"
@@ -40,6 +41,21 @@ func ProgressBar() rpc.DownloadProgressCB {
4041
}
4142
}
4243

44+
// PrintErrorFromDownloadResult returns a DownloadResultCB that only prints
45+
// the errors with the give message prefixed.
46+
func PrintErrorFromDownloadResult(msg string) rpc.DownloadResultCB {
47+
if OutputFormat != "json" {
48+
return func(res *rpc.DownloadResult) {
49+
if !res.GetSuccessful() {
50+
feedback.Errorf("%s: %s", msg, res.GetError())
51+
}
52+
}
53+
}
54+
return func(res *rpc.DownloadResult) {
55+
// XXX: Output result in JSON?
56+
}
57+
}
58+
4359
// TaskProgress returns a TaskProgressCB that prints the task progress.
4460
// If JSON output format has been selected, the callback outputs nothing.
4561
func TaskProgress() rpc.TaskProgressCB {

Diff for: ‎cli/update/update.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -56,9 +56,9 @@ func runUpdateCommand(cmd *cobra.Command, args []string) {
5656
inst := instance.CreateInstanceAndRunFirstUpdate()
5757
logrus.Info("Executing `arduino-cli update`")
5858

59-
err := commands.UpdateCoreLibrariesIndex(context.Background(), &rpc.UpdateCoreLibrariesIndexRequest{
60-
Instance: inst,
61-
}, output.ProgressBar())
59+
err := commands.UpdateCoreLibrariesIndex(context.Background(), &rpc.UpdateCoreLibrariesIndexRequest{Instance: inst},
60+
output.ProgressBar(),
61+
output.PrintErrorFromDownloadResult(tr("Error updating index")))
6262
if err != nil {
6363
feedback.Errorf(tr("Error updating core and libraries index: %v"), err)
6464
os.Exit(errorcodes.ErrGeneric)

Diff for: ‎commands/daemon/daemon.go

+9-6
Original file line numberDiff line numberDiff line change
@@ -161,13 +161,15 @@ func (s *ArduinoCoreServerImpl) Destroy(ctx context.Context, req *rpc.DestroyReq
161161

162162
// UpdateIndex FIXMEDOC
163163
func (s *ArduinoCoreServerImpl) UpdateIndex(req *rpc.UpdateIndexRequest, stream rpc.ArduinoCoreService_UpdateIndexServer) error {
164-
resp, err := commands.UpdateIndex(stream.Context(), req,
165-
func(p *rpc.DownloadProgress) { stream.Send(&rpc.UpdateIndexResponse{DownloadProgress: p}) },
164+
err := commands.UpdateIndex(stream.Context(), req,
165+
func(p *rpc.DownloadProgress) {
166+
stream.Send(&rpc.UpdateIndexResponse{Message: &rpc.UpdateIndexResponse_DownloadProgress{DownloadProgress: p}})
167+
},
168+
func(r *rpc.DownloadResult) {
169+
stream.Send(&rpc.UpdateIndexResponse{Message: &rpc.UpdateIndexResponse_DownloadResult{DownloadResult: r}})
170+
},
166171
)
167-
if err != nil {
168-
return convertErrorToRPCStatus(err)
169-
}
170-
return stream.Send(resp)
172+
return convertErrorToRPCStatus(err)
171173
}
172174

173175
// UpdateLibrariesIndex FIXMEDOC
@@ -185,6 +187,7 @@ func (s *ArduinoCoreServerImpl) UpdateLibrariesIndex(req *rpc.UpdateLibrariesInd
185187
func (s *ArduinoCoreServerImpl) UpdateCoreLibrariesIndex(req *rpc.UpdateCoreLibrariesIndexRequest, stream rpc.ArduinoCoreService_UpdateCoreLibrariesIndexServer) error {
186188
err := commands.UpdateCoreLibrariesIndex(stream.Context(), req,
187189
func(p *rpc.DownloadProgress) { stream.Send(&rpc.UpdateCoreLibrariesIndexResponse{DownloadProgress: p}) },
190+
func(res *rpc.DownloadResult) { /* ignore */ },
188191
)
189192
if err != nil {
190193
return convertErrorToRPCStatus(err)

Diff for: ‎commands/instances.go

+40-9
Original file line numberDiff line numberDiff line change
@@ -490,20 +490,29 @@ func UpdateLibrariesIndex(ctx context.Context, req *rpc.UpdateLibrariesIndexRequ
490490
}
491491

492492
// UpdateIndex FIXMEDOC
493-
func UpdateIndex(ctx context.Context, req *rpc.UpdateIndexRequest, downloadCB rpc.DownloadProgressCB) (*rpc.UpdateIndexResponse, error) {
493+
func UpdateIndex(ctx context.Context, req *rpc.UpdateIndexRequest, downloadCB rpc.DownloadProgressCB, downloadResultCB rpc.DownloadResultCB) error {
494494
if instances.GetInstance(req.GetInstance().GetId()) == nil {
495-
return nil, &arduino.InvalidInstanceError{}
495+
return &arduino.InvalidInstanceError{}
496496
}
497497

498498
indexpath := configuration.DataDir(configuration.Settings)
499499

500500
urls := []string{globals.DefaultIndexURL}
501-
urls = append(urls, configuration.Settings.GetStringSlice("board_manager.additional_urls")...)
501+
if !req.GetIgnoreCustomPackageIndexes() {
502+
urls = append(urls, configuration.Settings.GetStringSlice("board_manager.additional_urls")...)
503+
}
504+
505+
failed := false
502506
for _, u := range urls {
503507
logrus.Info("URL: ", u)
504508
URL, err := utils.URLParse(u)
505509
if err != nil {
506510
logrus.Warnf("unable to parse additional URL: %s", u)
511+
downloadResultCB(&rpc.DownloadResult{
512+
Url: u,
513+
Error: fmt.Sprintf("%s: %v", tr("Unable to parse URL"), err),
514+
})
515+
failed = true
507516
continue
508517
}
509518

@@ -512,7 +521,12 @@ func UpdateIndex(ctx context.Context, req *rpc.UpdateIndexRequest, downloadCB rp
512521
if URL.Scheme == "file" {
513522
path := paths.New(URL.Path)
514523
if _, err := packageindex.LoadIndexNoSign(path); err != nil {
515-
return nil, &arduino.InvalidArgumentError{Message: tr("Invalid package index in %s", path), Cause: err}
524+
downloadResultCB(&rpc.DownloadResult{
525+
Url: u,
526+
Error: fmt.Sprintf("%s: %v", tr("Invalid package index in %s", path), err),
527+
})
528+
failed = true
529+
continue
516530
}
517531

518532
fi, _ := os.Stat(path.String())
@@ -521,6 +535,10 @@ func UpdateIndex(ctx context.Context, req *rpc.UpdateIndexRequest, downloadCB rp
521535
TotalSize: fi.Size(),
522536
})
523537
downloadCB(&rpc.DownloadProgress{Completed: true})
538+
downloadResultCB(&rpc.DownloadResult{
539+
Url: u,
540+
Successful: true,
541+
})
524542
continue
525543
}
526544

@@ -532,18 +550,31 @@ func UpdateIndex(ctx context.Context, req *rpc.UpdateIndexRequest, downloadCB rp
532550
indexResource.SignatureURL.Path += ".sig"
533551
}
534552
if err := indexResource.Download(indexpath, downloadCB); err != nil {
535-
return nil, err
553+
downloadResultCB(&rpc.DownloadResult{
554+
Url: u,
555+
Error: err.Error(),
556+
})
557+
failed = true
558+
continue
536559
}
560+
561+
downloadResultCB(&rpc.DownloadResult{
562+
Url: u,
563+
Successful: true,
564+
})
537565
}
538566

539-
return &rpc.UpdateIndexResponse{}, nil
567+
if failed {
568+
return &arduino.FailedDownloadError{Message: tr("Some indexes could not be updated.")}
569+
}
570+
return nil
540571
}
541572

542573
// UpdateCoreLibrariesIndex updates both Cores and Libraries indexes
543-
func UpdateCoreLibrariesIndex(ctx context.Context, req *rpc.UpdateCoreLibrariesIndexRequest, downloadCB rpc.DownloadProgressCB) error {
544-
_, err := UpdateIndex(ctx, &rpc.UpdateIndexRequest{
574+
func UpdateCoreLibrariesIndex(ctx context.Context, req *rpc.UpdateCoreLibrariesIndexRequest, downloadCB rpc.DownloadProgressCB, downloadResultCB rpc.DownloadResultCB) error {
575+
err := UpdateIndex(ctx, &rpc.UpdateIndexRequest{
545576
Instance: req.Instance,
546-
}, downloadCB)
577+
}, downloadCB, downloadResultCB)
547578
if err != nil {
548579
return err
549580
}

Diff for: ‎docs/UPGRADING.md

+54
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,60 @@
22

33
Here you can find a list of migration guides to handle breaking changes between releases of the CLI.
44

5+
## 0.28.0
6+
7+
### Breaking changes in UpdateIndex API (both gRPC and go-lang)
8+
9+
The gRPC message `cc.arduino.cli.commands.v1.UpdateIndexResponse` has been changed from:
10+
11+
```
12+
message UpdateIndexResponse {
13+
// Progress of the platforms index download.
14+
DownloadProgress download_progress = 1;
15+
}
16+
```
17+
18+
to
19+
20+
```
21+
message UpdateIndexResponse {
22+
oneof message {
23+
// Progress of the platforms index download.
24+
DownloadProgress download_progress = 1;
25+
// Report of the index update downloads.
26+
DownloadResult download_result = 2;
27+
}
28+
}
29+
30+
message DownloadResult {
31+
// Index URL.
32+
string url = 1;
33+
// Download result: true if successful, false if an error occurred.
34+
bool successful = 2;
35+
// Download error details.
36+
string error = 3;
37+
}
38+
```
39+
40+
even if not strictly a breaking change it's worth noting that the detailed error message is now streamed in the
41+
response. About the go-lang API the following functions in `github.com/arduino/arduino-cli/commands`:
42+
43+
```go
44+
func UpdateIndex(ctx context.Context, req *rpc.UpdateIndexRequest, downloadCB rpc.DownloadProgressCB) (*rpc.UpdateIndexResponse, error) { ... }
45+
func UpdateCoreLibrariesIndex(ctx context.Context, req *rpc.UpdateCoreLibrariesIndexRequest, downloadCB rpc.DownloadProgressCB) error { ... }
46+
```
47+
48+
have changed their signature to:
49+
50+
```go
51+
func UpdateIndex(ctx context.Context, req *rpc.UpdateIndexRequest, downloadCB rpc.DownloadProgressCB, downloadResultCB rpc.DownloadResultCB) error { ... }
52+
func UpdateCoreLibrariesIndex(ctx context.Context, req *rpc.UpdateCoreLibrariesIndexRequest, downloadCB rpc.DownloadProgressCB, downloadResultCB rpc.DownloadResultCB) error { ... }
53+
```
54+
55+
`UpdateIndex` do not return anymore the latest `UpdateIndexResponse` (it was always empty). Both `UpdateIndex` and
56+
`UpdateCoreLibrariesIndex` now accepts an `rpc.DownloadResultCB` to get download results, you can pass an empty callback
57+
if you're not interested in the error details.
58+
559
## 0.27.0
660

761
### Breaking changes in golang API `github.com/arduino/arduino-cli/arduino/cores/packagemanager.PackageManager`

Diff for: ‎internal/integrationtest/arduino-cli.go

+11
Original file line numberDiff line numberDiff line change
@@ -367,3 +367,14 @@ func (inst *ArduinoCLIInstance) LibraryUninstall(ctx context.Context, name, vers
367367
logCallf(">>> LibraryUninstall(%+v)\n", req)
368368
return installCl, err
369369
}
370+
371+
// UpdateIndex calls the "UpdateIndex" gRPC method.
372+
func (inst *ArduinoCLIInstance) UpdateIndex(ctx context.Context, ignoreCustomPackages bool) (commands.ArduinoCoreService_UpdateIndexClient, error) {
373+
req := &commands.UpdateIndexRequest{
374+
Instance: inst.instance,
375+
IgnoreCustomPackageIndexes: ignoreCustomPackages,
376+
}
377+
updCl, err := inst.cli.daemonClient.UpdateIndex(ctx, req)
378+
logCallf(">>> UpdateIndex(%+v)\n", req)
379+
return updCl, err
380+
}

0 commit comments

Comments
 (0)