Skip to content

Commit 2791756

Browse files
author
Roberto Sora
authored
Export Prometheus telemetry in daemon mode (arduino#573)
* Apply cosmetics * Implement ugly telemetry POC * Added prefix and moved Insrumentation inside the command package * Refactor the telemetry module * Implement configuration via Viper * Add stats flush in case of a not daemonized cli daemon proces * Add repertory to store installation id and secret * Repertory force write * Cosmetics * Use viper config for repertory dir * Add test for repertory file creation * Add testing for telemetry deaemon and repertory * Wait for repertory and kill daemon * Updated pyinvoke to use async feature to test the daemon * Updated daemon test timeout * Cosmetics * Set getDefaultArduinoDataDir as unexported * Cosmetics * Cosmetics * Cosmetics * Lint on repertory module * Set SIGTERM as kill signal in case of win platform to overcome pyinvoke bug * Import platform as a module * Reverse platform if for signal value * Extract pid value * Remove print leftover * Add better error handling in repertory creation * Update docs with old README extract * Remove telemetry.pattern setting from docs * Remove serverPattern config option for telemetry * Upgrade viper to 1.6.2 * Defer stats Increment in compile command and explicit set for success/failure * Fix board list help message * Implement stats flush anyway to leverage module no-op in case of no handler configured * Rename "repertory" module in "inventory" and refactor Sanitize function * Sanitize ExportFile in command/compile * Refactor daemon start fixture to include daemon process cleanup * Use defer function to push success tag correctly updated * Use named return parameters to handle success tagging for a command stat
1 parent 8483cb2 commit 2791756

File tree

18 files changed

+479
-28
lines changed

18 files changed

+479
-28
lines changed

Diff for: ‎cli/board/list.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ func initListCommand() *cobra.Command {
4242
}
4343

4444
listCommand.Flags().StringVar(&listFlags.timeout, "timeout", "0s",
45-
"The timeout of the search of connected devices, try to increase it if your board is not found (e.g. to 10s).")
45+
"The connected devices search timeout, raise it if your board doesn't show up (e.g. to 10s).")
4646
return listCommand
4747
}
4848

Diff for: ‎cli/cli.go

+4
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ import (
3939
"github.com/arduino/arduino-cli/cli/upload"
4040
"github.com/arduino/arduino-cli/cli/version"
4141
"github.com/arduino/arduino-cli/configuration"
42+
"github.com/arduino/arduino-cli/inventory"
4243
"github.com/mattn/go-colorable"
4344
"github.com/rifflock/lfshook"
4445
"github.com/sirupsen/logrus"
@@ -167,6 +168,9 @@ func preRun(cmd *cobra.Command, args []string) {
167168
configuration.Init(configPath)
168169
configFile := viper.ConfigFileUsed()
169170

171+
// initialize inventory
172+
inventory.Init(viper.GetString("directories.Data"))
173+
170174
//
171175
// Prepare logging
172176
//

Diff for: ‎cli/daemon/daemon.go

+24-7
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ import (
3434
srv_debug "github.com/arduino/arduino-cli/rpc/debug"
3535
srv_monitor "github.com/arduino/arduino-cli/rpc/monitor"
3636
srv_settings "github.com/arduino/arduino-cli/rpc/settings"
37+
"github.com/arduino/arduino-cli/telemetry"
38+
"github.com/segmentio/stats/v4"
3739
"github.com/sirupsen/logrus"
3840
"github.com/spf13/cobra"
3941
"github.com/spf13/viper"
@@ -59,16 +61,29 @@ func NewCommand() *cobra.Command {
5961
var daemonize bool
6062

6163
func runDaemonCommand(cmd *cobra.Command, args []string) {
64+
65+
if viper.GetBool("telemetry.enabled") {
66+
telemetry.Activate("daemon")
67+
stats.Incr("daemon", stats.T("success", "true"))
68+
defer stats.Flush()
69+
}
70+
6271
port := viper.GetString("daemon.port")
6372
s := grpc.NewServer()
6473

65-
// register the commands service
66-
headers := http.Header{"User-Agent": []string{
67-
fmt.Sprintf("%s/%s daemon (%s; %s; %s) Commit:%s",
68-
globals.VersionInfo.Application,
69-
globals.VersionInfo.VersionString,
70-
runtime.GOARCH, runtime.GOOS,
71-
runtime.Version(), globals.VersionInfo.Commit)}}
74+
// Compose user agent header
75+
headers := http.Header{
76+
"User-Agent": []string{
77+
fmt.Sprintf("%s/%s daemon (%s; %s; %s) Commit:%s",
78+
globals.VersionInfo.Application,
79+
globals.VersionInfo.VersionString,
80+
runtime.GOARCH,
81+
runtime.GOOS,
82+
runtime.Version(),
83+
globals.VersionInfo.Commit),
84+
},
85+
}
86+
// Register the commands service
7287
srv_commands.RegisterArduinoCoreServer(s, &daemon.ArduinoCoreServerImpl{
7388
DownloaderHeaders: headers,
7489
VersionString: globals.VersionInfo.VersionString,
@@ -88,6 +103,8 @@ func runDaemonCommand(cmd *cobra.Command, args []string) {
88103
go func() {
89104
// Stdin is closed when the controlling parent process ends
90105
_, _ = io.Copy(ioutil.Discard, os.Stdin)
106+
// Flush telemetry stats (this is a no-op if telemetry is disabled)
107+
stats.Flush()
91108
os.Exit(0)
92109
}()
93110
}

Diff for: ‎commands/board/attach.go

+2-3
Original file line numberDiff line numberDiff line change
@@ -28,13 +28,12 @@ import (
2828
"github.com/arduino/arduino-cli/arduino/sketches"
2929
"github.com/arduino/arduino-cli/commands"
3030
rpc "github.com/arduino/arduino-cli/rpc/commands"
31-
discovery "github.com/arduino/board-discovery"
32-
paths "github.com/arduino/go-paths-helper"
31+
"github.com/arduino/board-discovery"
32+
"github.com/arduino/go-paths-helper"
3333
)
3434

3535
// Attach FIXMEDOC
3636
func Attach(ctx context.Context, req *rpc.BoardAttachReq, taskCB commands.TaskProgressCB) (*rpc.BoardAttachResp, error) {
37-
3837
pm := commands.GetPackageManager(req.GetInstance().GetId())
3938
if pm == nil {
4039
return nil, errors.New("invalid instance")

Diff for: ‎commands/board/list.go

+13-1
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import (
2727
"github.com/arduino/arduino-cli/commands"
2828
rpc "github.com/arduino/arduino-cli/rpc/commands"
2929
"github.com/pkg/errors"
30+
"github.com/segmentio/stats/v4"
3031
"github.com/sirupsen/logrus"
3132
)
3233

@@ -100,10 +101,21 @@ func identifyViaCloudAPI(port *commands.BoardPort) ([]*rpc.BoardListItem, error)
100101
}
101102

102103
// List FIXMEDOC
103-
func List(instanceID int32) ([]*rpc.DetectedPort, error) {
104+
func List(instanceID int32) (r []*rpc.DetectedPort, e error) {
104105
m.Lock()
105106
defer m.Unlock()
106107

108+
tags := map[string]string{}
109+
// Use defer func() to evaluate tags map when function returns
110+
// and set success flag inspecting the error named return parameter
111+
defer func() {
112+
tags["success"] = "true"
113+
if e != nil {
114+
tags["success"] = "false"
115+
}
116+
stats.Incr("compile", stats.M(tags)...)
117+
}()
118+
107119
pm := commands.GetPackageManager(instanceID)
108120
if pm == nil {
109121
return nil, errors.New("invalid instance")

Diff for: ‎commands/compile/compile.go

+30-2
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
"io"
2323
"path/filepath"
2424
"sort"
25+
"strconv"
2526
"strings"
2627

2728
"github.com/arduino/arduino-cli/arduino/cores"
@@ -33,14 +34,42 @@ import (
3334
"github.com/arduino/arduino-cli/legacy/builder/i18n"
3435
"github.com/arduino/arduino-cli/legacy/builder/types"
3536
rpc "github.com/arduino/arduino-cli/rpc/commands"
37+
"github.com/arduino/arduino-cli/telemetry"
3638
paths "github.com/arduino/go-paths-helper"
3739
properties "github.com/arduino/go-properties-orderedmap"
40+
"github.com/segmentio/stats/v4"
3841
"github.com/sirupsen/logrus"
3942
"github.com/spf13/viper"
4043
)
4144

4245
// Compile FIXMEDOC
43-
func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.Writer, debug bool) (*rpc.CompileResp, error) {
46+
func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.Writer, debug bool) (r *rpc.CompileResp, e error) {
47+
48+
tags := map[string]string{
49+
"fqbn": req.Fqbn,
50+
"sketchPath": telemetry.Sanitize(req.SketchPath),
51+
"showProperties": strconv.FormatBool(req.ShowProperties),
52+
"preprocess": strconv.FormatBool(req.Preprocess),
53+
"buildProperties": strings.Join(req.BuildProperties, ","),
54+
"warnings": req.Warnings,
55+
"verbose": strconv.FormatBool(req.Verbose),
56+
"quiet": strconv.FormatBool(req.Quiet),
57+
"vidPid": req.VidPid,
58+
"exportFile": telemetry.Sanitize(req.ExportFile),
59+
"jobs": strconv.FormatInt(int64(req.Jobs), 10),
60+
"libraries": strings.Join(req.Libraries, ","),
61+
}
62+
63+
// Use defer func() to evaluate tags map when function returns
64+
// and set success flag inspecting the error named return parameter
65+
defer func() {
66+
tags["success"] = "true"
67+
if e != nil {
68+
tags["success"] = "false"
69+
}
70+
stats.Incr("compile", stats.M(tags)...)
71+
}()
72+
4473
pm := commands.GetPackageManager(req.GetInstance().GetId())
4574
if pm == nil {
4675
return nil, errors.New("invalid instance")
@@ -224,6 +253,5 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W
224253
}
225254

226255
logrus.Tracef("Compile %s for %s successful", sketch.Name, fqbnIn)
227-
228256
return &rpc.CompileResp{}, nil
229257
}

Diff for: ‎configuration/defaults.go

+4
Original file line numberDiff line numberDiff line change
@@ -35,4 +35,8 @@ func setDefaults(dataDir, userDir string) {
3535

3636
// daemon settings
3737
viper.SetDefault("daemon.port", "50051")
38+
39+
//telemetry settings
40+
viper.SetDefault("telemetry.enabled", true)
41+
viper.SetDefault("telemetry.addr", ":9090")
3842
}

Diff for: ‎docs/getting-started.md

+37
Original file line numberDiff line numberDiff line change
@@ -309,3 +309,40 @@ [email protected] downloaded
309309
Installing [email protected]...
310310
Installed [email protected]
311311
```
312+
313+
314+
Using the ``daemon`` mode and the gRPC interface
315+
------------------------------------------------
316+
317+
Arduino CLI can be launched as a gRPC server via the `daemon` command.
318+
319+
The [client_example] folder contains a sample client code that shows how to
320+
interact with the gRPC server. Available services and messages are detailed
321+
in the [gRPC reference] pages.
322+
323+
324+
To provide observability for the gRPC server activities besides logs,
325+
the `daemon` mode activates and exposes by default a [Prometheus](https://prometheus.io/)
326+
endpoint (http://localhost:9090/metrics) that can be fetched for
327+
telemetry data like:
328+
329+
```text
330+
# TYPE daemon_compile counter
331+
daemon_compile{buildProperties="",exportFile="",fqbn="arduino:samd:mkr1000",installationID="ed6f1f22-1fbe-4b1f-84be-84d035b6369c",jobs="0",libraries="",preprocess="false",quiet="false",showProperties="false",sketchPath="5ff767c6fa5a91230f5cb4e267c889aa61489ab2c4f70f35f921f934c1462cb6",success="true",verbose="true",vidPid="",warnings=""} 1 1580385724726
332+
333+
# TYPE daemon_board_list counter
334+
daemon_board_list{installationID="ed6f1f22-1fbe-4b1f-84be-84d035b6369c",success="true"} 1 1580385724833
335+
```
336+
337+
The telemetry settings are exposed via the ``telemetry`` section
338+
in the CLI configuration:
339+
340+
```yaml
341+
telemetry:
342+
enabled: true
343+
addr: :9090
344+
```
345+
346+
[client_example]: https://github.com/arduino/arduino-cli/blob/master/client_example
347+
[gRPC reference]: /rpc/commands
348+
[Prometheus]: https://prometheus.io/

Diff for: ‎go.mod

+4-4
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,12 @@ require (
1717
github.com/fluxio/multierror v0.0.0-20160419044231-9c68d39025e5 // indirect
1818
github.com/fsnotify/fsnotify v1.4.7
1919
github.com/go-errors/errors v1.0.1
20+
github.com/gofrs/uuid v3.2.0+incompatible
2021
github.com/golang/protobuf v1.3.3
2122
github.com/h2non/filetype v1.0.8 // indirect
2223
github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5 // indirect
2324
github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8 // indirect
2425
github.com/juju/testing v0.0.0-20190429233213-dfc56b8c09fc // indirect
25-
github.com/kr/pretty v0.1.0 // indirect
2626
github.com/mattn/go-colorable v0.1.2
2727
github.com/mattn/go-runewidth v0.0.2 // indirect
2828
github.com/miekg/dns v1.0.5 // indirect
@@ -31,11 +31,11 @@ require (
3131
github.com/pmylund/sortutil v0.0.0-20120526081524-abeda66eb583
3232
github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5
3333
github.com/schollz/closestmatch v2.1.0+incompatible
34+
github.com/segmentio/stats/v4 v4.5.3
3435
github.com/sirupsen/logrus v1.4.2
35-
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect
3636
github.com/spf13/cobra v0.0.5
3737
github.com/spf13/jwalterweatherman v1.0.0
38-
github.com/spf13/viper v1.3.2
38+
github.com/spf13/viper v1.6.2
3939
github.com/stretchr/testify v1.4.0
4040
go.bug.st/cleanup v1.0.0
4141
go.bug.st/downloader v1.1.0
@@ -49,5 +49,5 @@ require (
4949
google.golang.org/grpc v1.27.0
5050
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
5151
gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce // indirect
52-
gopkg.in/yaml.v2 v2.2.2
52+
gopkg.in/yaml.v2 v2.2.4
5353
)

0 commit comments

Comments
 (0)