Skip to content

[breaking] gRPC UpdateIndex and UpdateLibrariesIndex improvements #2569

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Mar 22, 2024
Merged

[breaking] gRPC UpdateIndex and UpdateLibrariesIndex improvements #2569

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
2a634d7
gRPC UpdateIndex and UpdateLibrariesIndex improvements
cmaglie Mar 22, 2024
7955539
Do not make any output in case of skipped/already-up-to-date
cmaglie Mar 22, 2024
e387859
Added json output to 'core update index'
cmaglie Mar 22, 2024
55c9b06
Added json output to 'lib update index'
cmaglie Mar 22, 2024
88d3207
Removed unused function
cmaglie Mar 22, 2024
e92a812
Workaround for Windows paths in URI
cmaglie Mar 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 22 additions & 4 deletions commands/daemon/daemon.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,18 +121,36 @@ func (s *ArduinoCoreServerImpl) Destroy(ctx context.Context, req *rpc.DestroyReq
// UpdateIndex FIXMEDOC
func (s *ArduinoCoreServerImpl) UpdateIndex(req *rpc.UpdateIndexRequest, stream rpc.ArduinoCoreService_UpdateIndexServer) error {
syncSend := NewSynchronizedSend(stream.Send)
err := commands.UpdateIndex(stream.Context(), req,
func(p *rpc.DownloadProgress) { syncSend.Send(&rpc.UpdateIndexResponse{DownloadProgress: p}) },
res, err := commands.UpdateIndex(stream.Context(), req,
func(p *rpc.DownloadProgress) {
syncSend.Send(&rpc.UpdateIndexResponse{
Message: &rpc.UpdateIndexResponse_DownloadProgress{DownloadProgress: p},
})
},
)
if res != nil {
syncSend.Send(&rpc.UpdateIndexResponse{
Message: &rpc.UpdateIndexResponse_Result_{Result: res},
})
}
return convertErrorToRPCStatus(err)
}

// UpdateLibrariesIndex FIXMEDOC
func (s *ArduinoCoreServerImpl) UpdateLibrariesIndex(req *rpc.UpdateLibrariesIndexRequest, stream rpc.ArduinoCoreService_UpdateLibrariesIndexServer) error {
syncSend := NewSynchronizedSend(stream.Send)
err := commands.UpdateLibrariesIndex(stream.Context(), req,
func(p *rpc.DownloadProgress) { syncSend.Send(&rpc.UpdateLibrariesIndexResponse{DownloadProgress: p}) },
res, err := commands.UpdateLibrariesIndex(stream.Context(), req,
func(p *rpc.DownloadProgress) {
syncSend.Send(&rpc.UpdateLibrariesIndexResponse{
Message: &rpc.UpdateLibrariesIndexResponse_DownloadProgress{DownloadProgress: p},
})
},
)
if res != nil {
syncSend.Send(&rpc.UpdateLibrariesIndexResponse{
Message: &rpc.UpdateLibrariesIndexResponse_Result_{Result: res},
})
}
return convertErrorToRPCStatus(err)
}

Expand Down
88 changes: 74 additions & 14 deletions commands/instances.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,9 @@ import (
"fmt"
"net/url"
"path/filepath"
"runtime"
"strings"
"time"

"github.com/arduino/arduino-cli/commands/cmderrors"
"github.com/arduino/arduino-cli/commands/internal/instances"
Expand Down Expand Up @@ -406,30 +408,59 @@ func Destroy(ctx context.Context, req *rpc.DestroyRequest) (*rpc.DestroyResponse
}

// UpdateLibrariesIndex updates the library_index.json
func UpdateLibrariesIndex(ctx context.Context, req *rpc.UpdateLibrariesIndexRequest, downloadCB rpc.DownloadProgressCB) error {
func UpdateLibrariesIndex(ctx context.Context, req *rpc.UpdateLibrariesIndexRequest, downloadCB rpc.DownloadProgressCB) (*rpc.UpdateLibrariesIndexResponse_Result, error) {
logrus.Info("Updating libraries index")

pme, release, err := instances.GetPackageManagerExplorer(req.GetInstance())
if err != nil {
return err
return nil, err
}
indexDir := pme.IndexDir
release()

index := globals.LibrariesIndexResource
result := func(status rpc.IndexUpdateReport_Status) *rpc.UpdateLibrariesIndexResponse_Result {
return &rpc.UpdateLibrariesIndexResponse_Result{
LibrariesIndex: &rpc.IndexUpdateReport{
IndexUrl: globals.LibrariesIndexResource.URL.String(),
Status: status,
},
}
}

// Create the index directory if it doesn't exist
if err := indexDir.MkdirAll(); err != nil {
return &cmderrors.PermissionDeniedError{Message: tr("Could not create index directory"), Cause: err}
return result(rpc.IndexUpdateReport_STATUS_FAILED), &cmderrors.PermissionDeniedError{Message: tr("Could not create index directory"), Cause: err}
}

// Check if the index file is already up-to-date
indexFileName, _ := index.IndexFileName()
if info, err := indexDir.Join(indexFileName).Stat(); err == nil {
ageSecs := int64(time.Since(info.ModTime()).Seconds())
if ageSecs < req.GetUpdateIfOlderThanSecs() {
return result(rpc.IndexUpdateReport_STATUS_ALREADY_UP_TO_DATE), nil
}
}

// Perform index update
if err := globals.LibrariesIndexResource.Download(indexDir, downloadCB); err != nil {
return err
return nil, err
}

return nil
return result(rpc.IndexUpdateReport_STATUS_UPDATED), nil
}

// UpdateIndex FIXMEDOC
func UpdateIndex(ctx context.Context, req *rpc.UpdateIndexRequest, downloadCB rpc.DownloadProgressCB) error {
func UpdateIndex(ctx context.Context, req *rpc.UpdateIndexRequest, downloadCB rpc.DownloadProgressCB) (*rpc.UpdateIndexResponse_Result, error) {
if !instances.IsValid(req.GetInstance()) {
return &cmderrors.InvalidInstanceError{}
return nil, &cmderrors.InvalidInstanceError{}
}

report := func(indexURL *url.URL, status rpc.IndexUpdateReport_Status) *rpc.IndexUpdateReport {
return &rpc.IndexUpdateReport{
IndexUrl: indexURL.String(),
Status: status,
}
}

indexpath := configuration.DataDir(configuration.Settings)
Expand All @@ -440,46 +471,75 @@ func UpdateIndex(ctx context.Context, req *rpc.UpdateIndexRequest, downloadCB rp
}

failed := false
result := &rpc.UpdateIndexResponse_Result{}
for _, u := range urls {
URL, err := utils.URLParse(u)
URL, err := url.Parse(u)
if err != nil {
logrus.Warnf("unable to parse additional URL: %s", u)
msg := fmt.Sprintf("%s: %v", tr("Unable to parse URL"), err)
downloadCB.Start(u, tr("Downloading index: %s", u))
downloadCB.End(false, msg)
failed = true
result.UpdatedIndexes = append(result.UpdatedIndexes, report(URL, rpc.IndexUpdateReport_STATUS_FAILED))
continue
}

logrus.WithField("url", URL).Print("Updating index")

if URL.Scheme == "file" {
downloadCB.Start(u, tr("Downloading index: %s", filepath.Base(URL.Path)))
path := paths.New(URL.Path)
if URL.Scheme == "file" && runtime.GOOS == "windows" && len(URL.Path) > 1 {
// https://github.com/golang/go/issues/32456
// Parsed local file URLs on Windows are returned with a leading / so we remove it
path = paths.New(URL.Path[1:])
}
if _, err := packageindex.LoadIndexNoSign(path); err != nil {
msg := fmt.Sprintf("%s: %v", tr("Invalid package index in %s", path), err)
downloadCB.Start(u, tr("Downloading index: %s", filepath.Base(URL.Path)))
downloadCB.End(false, msg)
failed = true
result.UpdatedIndexes = append(result.UpdatedIndexes, report(URL, rpc.IndexUpdateReport_STATUS_FAILED))
} else {
downloadCB.End(true, "")
result.UpdatedIndexes = append(result.UpdatedIndexes, report(URL, rpc.IndexUpdateReport_STATUS_SKIPPED))
}
continue
}

// Check if the index is up-to-date
indexResource := resources.IndexResource{URL: URL}
indexFileName, err := indexResource.IndexFileName()
if err != nil {
downloadCB.Start(u, tr("Downloading index: %s", filepath.Base(URL.Path)))
downloadCB.End(false, tr("Invalid index URL: %s", err))
failed = true
result.UpdatedIndexes = append(result.UpdatedIndexes, report(URL, rpc.IndexUpdateReport_STATUS_FAILED))
continue
}
indexFile := indexpath.Join(indexFileName)
if info, err := indexFile.Stat(); err == nil {
ageSecs := int64(time.Since(info.ModTime()).Seconds())
if ageSecs < req.GetUpdateIfOlderThanSecs() {
result.UpdatedIndexes = append(result.UpdatedIndexes, report(URL, rpc.IndexUpdateReport_STATUS_ALREADY_UP_TO_DATE))
continue
}
}

if strings.HasSuffix(URL.Host, "arduino.cc") && strings.HasSuffix(URL.Path, ".json") {
indexResource.SignatureURL, _ = url.Parse(u) // should not fail because we already parsed it
indexResource.SignatureURL.Path += ".sig"
}
if err := indexResource.Download(indexpath, downloadCB); err != nil {
failed = true
result.UpdatedIndexes = append(result.UpdatedIndexes, report(URL, rpc.IndexUpdateReport_STATUS_FAILED))
} else {
result.UpdatedIndexes = append(result.UpdatedIndexes, report(URL, rpc.IndexUpdateReport_STATUS_UPDATED))
}
}

if failed {
return &cmderrors.FailedDownloadError{Message: tr("Some indexes could not be updated.")}
return result, &cmderrors.FailedDownloadError{Message: tr("Some indexes could not be updated.")}
}
return nil
return result, nil
}

// firstUpdate downloads libraries and packages indexes if they don't exist.
Expand All @@ -493,7 +553,7 @@ func firstUpdate(ctx context.Context, instance *rpc.Instance, downloadCb func(ms
// The library_index.json file doesn't exists, that means the CLI is run for the first time
// so we proceed with the first update that downloads the file
req := &rpc.UpdateLibrariesIndexRequest{Instance: instance}
if err := UpdateLibrariesIndex(ctx, req, downloadCb); err != nil {
if _, err := UpdateLibrariesIndex(ctx, req, downloadCb); err != nil {
return err
}
}
Expand All @@ -515,7 +575,7 @@ func firstUpdate(ctx context.Context, instance *rpc.Instance, downloadCb func(ms
// library update we download that file and all the other package indexes from
// additional_urls
req := &rpc.UpdateIndexRequest{Instance: instance}
if err := UpdateIndex(ctx, req, downloadCb); err != nil {
if _, err := UpdateIndex(ctx, req, downloadCb); err != nil {
return err
}
break
Expand Down
70 changes: 70 additions & 0 deletions docs/UPGRADING.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,76 @@ Here you can find a list of migration guides to handle breaking changes between

## 0.36.0

### The gRPC `cc.arduino.cli.commands.v1.UpdateIndexResponse` and `UpdateLibrariesIndexResponse` have changed.

The responses coming from the update index commands:

```proto
message UpdateIndexResponse {
// Progress of the package index download.
DownloadProgress download_progress = 1;
}

message UpdateLibrariesIndexResponse {
// Progress of the libraries index download.
DownloadProgress download_progress = 1;
}
```

are now more explicit and contains details about the result of the operation:

```proto
message UpdateIndexResponse {
message Result {
// The result of the packages index update.
repeated IndexUpdateReport updated_indexes = 1;
}
oneof message {
// Progress of the package index download.
DownloadProgress download_progress = 1;
// The result of the index update.
Result result = 2;
}
}

message UpdateLibrariesIndexResponse {
message Result {
// The result of the libraries index update.
IndexUpdateReport libraries_index = 1;
}
oneof message {
// Progress of the libraries index download.
DownloadProgress download_progress = 1;
// The result of the index update.
Result result = 2;
}
}
```

The `IndexUpdateReport` message contains details for each index update operation performed:

```proto
message IndexUpdateReport {
enum Status {
// The status of the index update is unspecified.
STATUS_UNSPECIFIED = 0;
// The index has been successfully updated.
STATUS_UPDATED = 1;
// The index was already up to date.
STATUS_ALREADY_UP_TO_DATE = 2;
// The index update failed.
STATUS_FAILED = 3;
// The index update was skipped.
STATUS_SKIPPED = 4;
}

// The URL of the index that was updated.
string index_url = 1;
// The result of the index update.
Status status = 2;
}
```

### The gRPC `cc.arduino.cli.commands.v1.Profile` message has been removed in favor of `SketchProfile`

The message `Profile` has been replaced with `SketchProfile` in the `InitResponse.profile` field:
Expand Down
78 changes: 13 additions & 65 deletions internal/cli/core/search.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,11 @@ import (
"context"
"fmt"
"os"
"path"
"strings"
"time"

"github.com/arduino/arduino-cli/commands"
"github.com/arduino/arduino-cli/commands/core"
"github.com/arduino/arduino-cli/internal/arduino/globals"
"github.com/arduino/arduino-cli/internal/arduino/utils"
"github.com/arduino/arduino-cli/internal/cli/configuration"
"github.com/arduino/arduino-cli/internal/cli/feedback"
"github.com/arduino/arduino-cli/internal/cli/feedback/result"
"github.com/arduino/arduino-cli/internal/cli/feedback/table"
Expand Down Expand Up @@ -55,17 +51,24 @@ func initSearchCommand() *cobra.Command {
}

// indexUpdateInterval specifies the time threshold over which indexes are updated
const indexUpdateInterval = "24h"
const indexUpdateInterval = 24 * time.Hour

func runSearchCommand(cmd *cobra.Command, args []string, allVersions bool) {
inst := instance.CreateAndInit()

if indexesNeedUpdating(indexUpdateInterval) {
err := commands.UpdateIndex(context.Background(), &rpc.UpdateIndexRequest{Instance: inst}, feedback.ProgressBar())
if err != nil {
feedback.FatalError(err, feedback.ErrGeneric)
res, err := commands.UpdateIndex(
context.Background(),
&rpc.UpdateIndexRequest{Instance: inst, UpdateIfOlderThanSecs: int64(indexUpdateInterval.Seconds())},
feedback.ProgressBar())
if err != nil {
feedback.FatalError(err, feedback.ErrGeneric)
}
for _, idxRes := range res.GetUpdatedIndexes() {
if idxRes.GetStatus() == rpc.IndexUpdateReport_STATUS_UPDATED {
// At least one index has been updated, reinitialize the instance
instance.Init(inst)
break
}
instance.Init(inst)
}

arguments := strings.ToLower(strings.Join(args, " "))
Expand Down Expand Up @@ -134,58 +137,3 @@ func (sr searchResults) String() string {
}
return t.Render()
}

// indexesNeedUpdating returns whether one or more index files need updating.
// A duration string must be provided to calculate the time threshold
// used to update the indexes, if the duration is not valid a default
// of 24 hours is used.
// Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
func indexesNeedUpdating(duration string) bool {
indexpath := configuration.DataDir(configuration.Settings)

now := time.Now()
modTimeThreshold, err := time.ParseDuration(duration)
if err != nil {
feedback.Fatal(tr("Invalid timeout: %s", err), feedback.ErrBadArgument)
}

urls := []string{globals.DefaultIndexURL}
urls = append(urls, configuration.Settings.GetStringSlice("board_manager.additional_urls")...)
for _, u := range urls {
URL, err := utils.URLParse(u)
if err != nil {
continue
}

if URL.Scheme == "file" {
// No need to update local files
continue
}

// should handle:
// - package_index.json
// - package_index.json.sig
// - package_index.json.gz
// - package_index.tar.bz2
indexFileName := path.Base(URL.Path)
indexFileName = strings.TrimSuffix(indexFileName, ".tar.bz2")
indexFileName = strings.TrimSuffix(indexFileName, ".gz")
indexFileName = strings.TrimSuffix(indexFileName, ".sig")
indexFileName = strings.TrimSuffix(indexFileName, ".json")
// and obtain package_index.json as result
coreIndexPath := indexpath.Join(indexFileName + ".json")
if coreIndexPath.NotExist() {
return true
}

info, err := coreIndexPath.Stat()
if err != nil {
return true
}

if now.After(info.ModTime().Add(modTimeThreshold)) {
return true
}
}
return false
}
Loading
Loading