From ba172c34b7d41ae92639a9a4be0b7a3cabc9a97c Mon Sep 17 00:00:00 2001 From: Roberto Sora Date: Wed, 22 Jan 2020 16:37:54 +0100 Subject: [PATCH 01/39] Apply cosmetics --- cli/daemon/daemon.go | 41 ++++++++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/cli/daemon/daemon.go b/cli/daemon/daemon.go index fc38dd11d38..3943d880605 100644 --- a/cli/daemon/daemon.go +++ b/cli/daemon/daemon.go @@ -34,6 +34,8 @@ import ( srv_debug "github.com/arduino/arduino-cli/rpc/debug" srv_monitor "github.com/arduino/arduino-cli/rpc/monitor" srv_settings "github.com/arduino/arduino-cli/rpc/settings" + stats "github.com/segmentio/stats/v4" + "github.com/segmentio/stats/v4/prometheus" "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -59,16 +61,40 @@ func NewCommand() *cobra.Command { var daemonize bool func runDaemonCommand(cmd *cobra.Command, args []string) { + + logrus.Infof("configure telemetry") + + // Configure telemetry engine + ph := prometheus.DefaultHandler + + // Register the client so it receives metrics from the default engine. + stats.Register(ph) + + // Flush the default stats engine on return to ensure all buffered + // metrics are sent to the server. + defer stats.Flush() + http.Handle("/metrics", ph) + go http.ListenAndServe(":2112", nil) + logrus.Infof("Prometheus telemetry is available on /metrics, TCP port 2112") + // Set a tag on a counter increment. + stats.Incr("user.login", stats.Tag{"user", "luke"}) + port := viper.GetString("daemon.port") s := grpc.NewServer() - // register the commands service - headers := http.Header{"User-Agent": []string{ - fmt.Sprintf("%s/%s daemon (%s; %s; %s) Commit:%s", - globals.VersionInfo.Application, - globals.VersionInfo.VersionString, - runtime.GOARCH, runtime.GOOS, - runtime.Version(), globals.VersionInfo.Commit)}} + // Compose user agent header + headers := http.Header{ + "User-Agent": []string{ + fmt.Sprintf("%s/%s daemon (%s; %s; %s) Commit:%s", + globals.VersionInfo.Application, + globals.VersionInfo.VersionString, + runtime.GOARCH, + runtime.GOOS, + runtime.Version(), + globals.VersionInfo.Commit), + }, + } + // Register the commands service srv_commands.RegisterArduinoCoreServer(s, &daemon.ArduinoCoreServerImpl{ DownloaderHeaders: headers, VersionString: globals.VersionInfo.VersionString, @@ -121,4 +147,5 @@ func runDaemonCommand(cmd *cobra.Command, args []string) { if err := s.Serve(lis); err != nil { logrus.Fatalf("Failed to serve: %v", err) } + } From 2c6634d8553d3869e51a544e3bfbdaf63d80dcfd Mon Sep 17 00:00:00 2001 From: Roberto Sora Date: Thu, 23 Jan 2020 15:05:06 +0100 Subject: [PATCH 02/39] Implement ugly telemetry POC --- cli/daemon/daemon.go | 20 ++++++++++++-------- commands/daemon/daemon.go | 4 ++++ go.mod | 1 + go.sum | 10 ++++++++++ main.go | 1 + telemetry/telemetry.go | 1 + 6 files changed, 29 insertions(+), 8 deletions(-) create mode 100644 telemetry/telemetry.go diff --git a/cli/daemon/daemon.go b/cli/daemon/daemon.go index 3943d880605..7c4b2227d99 100644 --- a/cli/daemon/daemon.go +++ b/cli/daemon/daemon.go @@ -62,23 +62,27 @@ var daemonize bool func runDaemonCommand(cmd *cobra.Command, args []string) { - logrus.Infof("configure telemetry") - // Configure telemetry engine ph := prometheus.DefaultHandler // Register the client so it receives metrics from the default engine. - stats.Register(ph) + engine := stats.WithPrefix("daemon") + engine.Register(ph) // Flush the default stats engine on return to ensure all buffered // metrics are sent to the server. defer stats.Flush() - http.Handle("/metrics", ph) - go http.ListenAndServe(":2112", nil) - logrus.Infof("Prometheus telemetry is available on /metrics, TCP port 2112") - // Set a tag on a counter increment. - stats.Incr("user.login", stats.Tag{"user", "luke"}) + // move everything inside commands and search for setting up a common prefix for all metrics sent! + logrus.Infof("Setting up Prometheus telemetry on /metrics, TCP port 2112") + go func() { + http.Handle("/metrics", ph) + logrus.Error(http.ListenAndServe(":2112", nil)) + }() + + engine.Incr("board.yes", stats.Tag{"success", "false"}) + stats.Flush() + engine.Flush() port := viper.GetString("daemon.port") s := grpc.NewServer() diff --git a/commands/daemon/daemon.go b/commands/daemon/daemon.go index 579e4abd0f2..ebaf4cb8d6b 100644 --- a/commands/daemon/daemon.go +++ b/commands/daemon/daemon.go @@ -29,6 +29,7 @@ import ( "github.com/arduino/arduino-cli/commands/lib" "github.com/arduino/arduino-cli/commands/upload" rpc "github.com/arduino/arduino-cli/rpc/commands" + "github.com/segmentio/stats/v4" ) // ArduinoCoreServerImpl FIXMEDOC @@ -46,9 +47,12 @@ func (s *ArduinoCoreServerImpl) BoardDetails(ctx context.Context, req *rpc.Board func (s *ArduinoCoreServerImpl) BoardList(ctx context.Context, req *rpc.BoardListReq) (*rpc.BoardListResp, error) { ports, err := board.List(req.GetInstance().GetId()) if err != nil { + // helper function from grpc req to tags + stats.Incr("board.list", stats.Tag{"success", "false"}) return nil, err } + stats.Incr("board.list", stats.Tag{"success", "true"}) return &rpc.BoardListResp{ Ports: ports, }, nil diff --git a/go.mod b/go.mod index f81e291f0da..38706d780ce 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,7 @@ require ( github.com/pmylund/sortutil v0.0.0-20120526081524-abeda66eb583 github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 github.com/schollz/closestmatch v2.1.0+incompatible + github.com/segmentio/stats/v4 v4.5.3 github.com/sirupsen/logrus v1.4.2 github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect github.com/spf13/cobra v0.0.5 diff --git a/go.sum b/go.sum index a07a80b85a8..99ccf4d184a 100644 --- a/go.sum +++ b/go.sum @@ -85,6 +85,9 @@ github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-runewidth v0.0.2 h1:UnlwIPBGaTZfPQ6T1IGzPI0EkYAQmT9fAEJ/poFC63o= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mdlayher/genetlink v0.0.0-20190313224034-60417448a851/go.mod h1:EsbsAEUEs15qC1cosAwxgCWV0Qhd8TmkxnA9Kw1Vhl4= +github.com/mdlayher/netlink v0.0.0-20190313131330-258ea9dff42c/go.mod h1:eQB3mZE4aiYnlUsyGGCOpPETfdQq4Jhsgf1fk3cwQaA= +github.com/mdlayher/taskstats v0.0.0-20190313225729-7cbba52ee072/go.mod h1:sGdS7A6CAETR53zkdjGkgoFlh1vSm7MtX+i8XfEsTMA= github.com/miekg/dns v1.0.5 h1:MQBGf2JEJDu0rg9WOpQZzeO+zW8UKwgkvP3R1dUU1Yw= github.com/miekg/dns v1.0.5/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -108,6 +111,12 @@ github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNue github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/schollz/closestmatch v2.1.0+incompatible h1:Uel2GXEpJqOWBrlyI+oY9LTiyyjYS17cCYRqP13/SHk= github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g= +github.com/segmentio/fasthash v0.0.0-20180216231524-a72b379d632e h1:uO75wNGioszjmIzcY/tvdDYKRLVvzggtAmmJkn9j4GQ= +github.com/segmentio/fasthash v0.0.0-20180216231524-a72b379d632e/go.mod h1:tm/wZFQ8e24NYaBGIlnO2WGCAi67re4HHuOm0sftE/M= +github.com/segmentio/objconv v1.0.1/go.mod h1:auayaH5k3137Cl4SoXTgrzQcuQDmvuVtZgS0fb1Ahys= +github.com/segmentio/stats v4.1.0+incompatible h1:Sz7ypAORgiXoR8x8VNIOKYoVpJUIeHGt/2UDMjjRFuo= +github.com/segmentio/stats/v4 v4.5.3 h1:Y/DSUWZ4c8ICgqJ9rQohzKvGqGWbLPWad5zmxVoKN+Y= +github.com/segmentio/stats/v4 v4.5.3/go.mod h1:LsaahUJR7iiSs8mnkvQvdQ/RLHAS5adGLxuntg0ydGo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= @@ -168,6 +177,7 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9 h1:ZBzSG/7F4eNKz2L3GE9o300RX0Az1Bw5HF7PDraD+qU= diff --git a/main.go b/main.go index 4ab55ba9d8e..73fa22ed7fe 100644 --- a/main.go +++ b/main.go @@ -23,6 +23,7 @@ import ( ) func main() { + if err := cli.ArduinoCli.Execute(); err != nil { os.Exit(errorcodes.ErrGeneric) } diff --git a/telemetry/telemetry.go b/telemetry/telemetry.go new file mode 100644 index 00000000000..073f40d23f2 --- /dev/null +++ b/telemetry/telemetry.go @@ -0,0 +1 @@ +package telemetry From 11d1bdb4662bc97cc9bfeeb38469338963bada27 Mon Sep 17 00:00:00 2001 From: Roberto Sora Date: Thu, 23 Jan 2020 19:05:29 +0100 Subject: [PATCH 03/39] Added prefix and moved Insrumentation inside the command package --- cli/daemon/daemon.go | 14 +++++--------- commands/board/list.go | 5 +++++ commands/daemon/daemon.go | 4 ---- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/cli/daemon/daemon.go b/cli/daemon/daemon.go index 7c4b2227d99..c961dfc342d 100644 --- a/cli/daemon/daemon.go +++ b/cli/daemon/daemon.go @@ -63,16 +63,15 @@ var daemonize bool func runDaemonCommand(cmd *cobra.Command, args []string) { // Configure telemetry engine + // Create a Prometheus default handler ph := prometheus.DefaultHandler - - // Register the client so it receives metrics from the default engine. - engine := stats.WithPrefix("daemon") - engine.Register(ph) - + // Replace the default stats engine with an engine that prepends the "daemon" prefix to all metrics + stats.DefaultEngine = stats.WithPrefix("daemon") + // Register the handler so it receives metrics from the default engine. + stats.Register(ph) // Flush the default stats engine on return to ensure all buffered // metrics are sent to the server. defer stats.Flush() - // move everything inside commands and search for setting up a common prefix for all metrics sent! logrus.Infof("Setting up Prometheus telemetry on /metrics, TCP port 2112") go func() { @@ -80,9 +79,6 @@ func runDaemonCommand(cmd *cobra.Command, args []string) { logrus.Error(http.ListenAndServe(":2112", nil)) }() - engine.Incr("board.yes", stats.Tag{"success", "false"}) - stats.Flush() - engine.Flush() port := viper.GetString("daemon.port") s := grpc.NewServer() diff --git a/commands/board/list.go b/commands/board/list.go index 283697161e7..d3ed1d3f919 100644 --- a/commands/board/list.go +++ b/commands/board/list.go @@ -18,6 +18,7 @@ package board import ( "encoding/json" "fmt" + "github.com/segmentio/stats/v4" "io/ioutil" "net/http" "regexp" @@ -106,11 +107,13 @@ func List(instanceID int32) ([]*rpc.DetectedPort, error) { pm := commands.GetPackageManager(instanceID) if pm == nil { + stats.Incr("board.list", stats.Tag{"success", "false"}) return nil, errors.New("invalid instance") } ports, err := commands.ListBoards(pm) if err != nil { + stats.Incr("board.list", stats.Tag{"success", "false"}) return nil, errors.Wrap(err, "error getting port list from serial-discovery") } @@ -136,6 +139,7 @@ func List(instanceID int32) ([]*rpc.DetectedPort, error) { logrus.Debug("Board not recognized") } else if err != nil { // this is bad, bail out + stats.Incr("board.list", stats.Tag{"success", "false"}) return nil, errors.Wrap(err, "error getting board info from Arduino Cloud") } @@ -156,5 +160,6 @@ func List(instanceID int32) ([]*rpc.DetectedPort, error) { retVal = append(retVal, p) } + stats.Incr("board.list", stats.Tag{"success", "true"}) return retVal, nil } diff --git a/commands/daemon/daemon.go b/commands/daemon/daemon.go index ebaf4cb8d6b..579e4abd0f2 100644 --- a/commands/daemon/daemon.go +++ b/commands/daemon/daemon.go @@ -29,7 +29,6 @@ import ( "github.com/arduino/arduino-cli/commands/lib" "github.com/arduino/arduino-cli/commands/upload" rpc "github.com/arduino/arduino-cli/rpc/commands" - "github.com/segmentio/stats/v4" ) // ArduinoCoreServerImpl FIXMEDOC @@ -47,12 +46,9 @@ func (s *ArduinoCoreServerImpl) BoardDetails(ctx context.Context, req *rpc.Board func (s *ArduinoCoreServerImpl) BoardList(ctx context.Context, req *rpc.BoardListReq) (*rpc.BoardListResp, error) { ports, err := board.List(req.GetInstance().GetId()) if err != nil { - // helper function from grpc req to tags - stats.Incr("board.list", stats.Tag{"success", "false"}) return nil, err } - stats.Incr("board.list", stats.Tag{"success", "true"}) return &rpc.BoardListResp{ Ports: ports, }, nil From bb6a490ed5adbba8636ecb135805cde1ee0fefbd Mon Sep 17 00:00:00 2001 From: Roberto Sora Date: Fri, 24 Jan 2020 16:46:13 +0100 Subject: [PATCH 04/39] Refactor the telemetry module --- cli/daemon/daemon.go | 23 ++++------------------ commands/board/attach.go | 5 ++--- commands/board/list.go | 9 +++++---- commands/compile/compile.go | 39 ++++++++++++++++++++++++++++++++++++- telemetry/telemetry.go | 35 +++++++++++++++++++++++++++++++++ 5 files changed, 84 insertions(+), 27 deletions(-) diff --git a/cli/daemon/daemon.go b/cli/daemon/daemon.go index c961dfc342d..78dec9128a3 100644 --- a/cli/daemon/daemon.go +++ b/cli/daemon/daemon.go @@ -34,8 +34,7 @@ import ( srv_debug "github.com/arduino/arduino-cli/rpc/debug" srv_monitor "github.com/arduino/arduino-cli/rpc/monitor" srv_settings "github.com/arduino/arduino-cli/rpc/settings" - stats "github.com/segmentio/stats/v4" - "github.com/segmentio/stats/v4/prometheus" + "github.com/arduino/arduino-cli/telemetry" "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -62,22 +61,8 @@ var daemonize bool func runDaemonCommand(cmd *cobra.Command, args []string) { - // Configure telemetry engine - // Create a Prometheus default handler - ph := prometheus.DefaultHandler - // Replace the default stats engine with an engine that prepends the "daemon" prefix to all metrics - stats.DefaultEngine = stats.WithPrefix("daemon") - // Register the handler so it receives metrics from the default engine. - stats.Register(ph) - // Flush the default stats engine on return to ensure all buffered - // metrics are sent to the server. - defer stats.Flush() - // move everything inside commands and search for setting up a common prefix for all metrics sent! - logrus.Infof("Setting up Prometheus telemetry on /metrics, TCP port 2112") - go func() { - http.Handle("/metrics", ph) - logrus.Error(http.ListenAndServe(":2112", nil)) - }() + telemetry.Activate("daemon") + defer telemetry.Engine.Flush() port := viper.GetString("daemon.port") s := grpc.NewServer() @@ -114,6 +99,7 @@ func runDaemonCommand(cmd *cobra.Command, args []string) { go func() { // Stdin is closed when the controlling parent process ends _, _ = io.Copy(ioutil.Discard, os.Stdin) + telemetry.Engine.Flush() os.Exit(0) }() } @@ -147,5 +133,4 @@ func runDaemonCommand(cmd *cobra.Command, args []string) { if err := s.Serve(lis); err != nil { logrus.Fatalf("Failed to serve: %v", err) } - } diff --git a/commands/board/attach.go b/commands/board/attach.go index 77906062bae..488779e81d5 100644 --- a/commands/board/attach.go +++ b/commands/board/attach.go @@ -28,13 +28,12 @@ import ( "github.com/arduino/arduino-cli/arduino/sketches" "github.com/arduino/arduino-cli/commands" rpc "github.com/arduino/arduino-cli/rpc/commands" - discovery "github.com/arduino/board-discovery" - paths "github.com/arduino/go-paths-helper" + "github.com/arduino/board-discovery" + "github.com/arduino/go-paths-helper" ) // Attach FIXMEDOC func Attach(ctx context.Context, req *rpc.BoardAttachReq, taskCB commands.TaskProgressCB) (*rpc.BoardAttachResp, error) { - pm := commands.GetPackageManager(req.GetInstance().GetId()) if pm == nil { return nil, errors.New("invalid instance") diff --git a/commands/board/list.go b/commands/board/list.go index d3ed1d3f919..1baefb8efe5 100644 --- a/commands/board/list.go +++ b/commands/board/list.go @@ -18,6 +18,7 @@ package board import ( "encoding/json" "fmt" + "github.com/arduino/arduino-cli/telemetry" "github.com/segmentio/stats/v4" "io/ioutil" "net/http" @@ -107,13 +108,13 @@ func List(instanceID int32) ([]*rpc.DetectedPort, error) { pm := commands.GetPackageManager(instanceID) if pm == nil { - stats.Incr("board.list", stats.Tag{"success", "false"}) + telemetry.Engine.Incr("board.list", stats.T("success", "false")) return nil, errors.New("invalid instance") } ports, err := commands.ListBoards(pm) if err != nil { - stats.Incr("board.list", stats.Tag{"success", "false"}) + telemetry.Engine.Incr("board.list", stats.T("success", "false")) return nil, errors.Wrap(err, "error getting port list from serial-discovery") } @@ -139,7 +140,7 @@ func List(instanceID int32) ([]*rpc.DetectedPort, error) { logrus.Debug("Board not recognized") } else if err != nil { // this is bad, bail out - stats.Incr("board.list", stats.Tag{"success", "false"}) + telemetry.Engine.Incr("board.list", stats.T("success", "false")) return nil, errors.Wrap(err, "error getting board info from Arduino Cloud") } @@ -160,6 +161,6 @@ func List(instanceID int32) ([]*rpc.DetectedPort, error) { retVal = append(retVal, p) } - stats.Incr("board.list", stats.Tag{"success", "true"}) + telemetry.Engine.Incr("board.list", stats.T("success", "true")) return retVal, nil } diff --git a/commands/compile/compile.go b/commands/compile/compile.go index e00398abed0..c31e62da332 100644 --- a/commands/compile/compile.go +++ b/commands/compile/compile.go @@ -22,6 +22,7 @@ import ( "io" "path/filepath" "sort" + "strconv" "strings" "github.com/arduino/arduino-cli/arduino/cores" @@ -33,26 +34,48 @@ import ( "github.com/arduino/arduino-cli/legacy/builder/i18n" "github.com/arduino/arduino-cli/legacy/builder/types" rpc "github.com/arduino/arduino-cli/rpc/commands" + "github.com/arduino/arduino-cli/telemetry" paths "github.com/arduino/go-paths-helper" properties "github.com/arduino/go-properties-orderedmap" + "github.com/segmentio/stats/v4" "github.com/sirupsen/logrus" "github.com/spf13/viper" ) // Compile FIXMEDOC func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.Writer, debug bool) (*rpc.CompileResp, error) { + + tags := map[string]string{ + "fqbn": req.Fqbn, + "sketchPath": req.SketchPath, + "showProperties": strconv.FormatBool(req.ShowProperties), + "preprocess": strconv.FormatBool(req.Preprocess), + "buildProperties": strings.Join(req.BuildProperties, ","), + "warnings": req.Warnings, + "verbose": strconv.FormatBool(req.Verbose), + "quiet": strconv.FormatBool(req.Quiet), + "vidPid": req.VidPid, + "exportFile": req.ExportFile, + "jobs": strconv.FormatInt(int64(req.Jobs), 10), + "libraries": strings.Join(req.Libraries, ","), + "success": "false", + } + pm := commands.GetPackageManager(req.GetInstance().GetId()) if pm == nil { + telemetry.Engine.Incr("compile", stats.M(tags)...) return nil, errors.New("invalid instance") } logrus.Tracef("Compile %s for %s started", req.GetSketchPath(), req.GetFqbn()) if req.GetSketchPath() == "" { + telemetry.Engine.Incr("compile", stats.M(tags)...) return nil, fmt.Errorf("missing sketchPath") } sketchPath := paths.New(req.GetSketchPath()) sketch, err := sketches.NewSketchFromPath(sketchPath) if err != nil { + telemetry.Engine.Incr("compile", stats.M(tags)...) return nil, fmt.Errorf("opening sketch: %s", err) } @@ -61,10 +84,12 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W fqbnIn = sketch.Metadata.CPU.Fqbn } if fqbnIn == "" { + telemetry.Engine.Incr("compile", stats.M(tags)...) return nil, fmt.Errorf("no FQBN provided") } fqbn, err := cores.ParseFQBN(fqbnIn) if err != nil { + telemetry.Engine.Incr("compile", stats.M(tags)...) return nil, fmt.Errorf("incorrect FQBN: %s", err) } @@ -78,6 +103,7 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W // "\"%[1]s:%[2]s\" platform is not installed, please install it by running \""+ // version.GetAppName()+" core install %[1]s:%[2]s\".", fqbn.Package, fqbn.PlatformArch) // feedback.Error(errorMessage) + telemetry.Engine.Incr("compile", stats.M(tags)...) return nil, fmt.Errorf("platform not installed") } @@ -97,6 +123,7 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W builderCtx.BuildPath = paths.New(req.GetBuildPath()) err = builderCtx.BuildPath.MkdirAll() if err != nil { + telemetry.Engine.Incr("compile", stats.M(tags)...) return nil, fmt.Errorf("cannot create build directory: %s", err) } } @@ -125,6 +152,7 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W builderCtx.BuildCachePath = paths.New(req.GetBuildCachePath()) err = builderCtx.BuildCachePath.MkdirAll() if err != nil { + telemetry.Engine.Incr("compile", stats.M(tags)...) return nil, fmt.Errorf("cannot create build cache directory: %s", err) } } @@ -157,13 +185,18 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W // if --preprocess or --show-properties were passed, we can stop here if req.GetShowProperties() { + tags["success"] = "true" + telemetry.Engine.Incr("compile", stats.M(tags)...) return &rpc.CompileResp{}, builder.RunParseHardwareAndDumpBuildProperties(builderCtx) } else if req.GetPreprocess() { + tags["success"] = "true" + telemetry.Engine.Incr("compile", stats.M(tags)...) return &rpc.CompileResp{}, builder.RunPreprocess(builderCtx) } // if it's a regular build, go on... if err := builder.RunBuilder(builderCtx); err != nil { + telemetry.Engine.Incr("compile", stats.M(tags)...) return nil, err } @@ -199,6 +232,7 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W // Copy "sketch.ino.*.hex" / "sketch.ino.*.bin" artifacts to sketch directory srcDir, err := outputPath.Parent().ReadDir() // read "/build/path/*" if err != nil { + telemetry.Engine.Incr("compile", stats.M(tags)...) return nil, fmt.Errorf("reading build directory: %s", err) } srcDir.FilterPrefix(base + ".") @@ -209,6 +243,7 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W dstOutput := exportPath.Join(exportFile + srcFilename) logrus.WithField("from", srcOutput).WithField("to", dstOutput).Debug("copying sketch build output") if err = srcOutput.CopyTo(dstOutput); err != nil { + telemetry.Engine.Incr("compile", stats.M(tags)...) return nil, fmt.Errorf("copying output file: %s", err) } } @@ -219,11 +254,13 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W dstElf := exportPath.Join(exportFile + ".elf") logrus.WithField("from", srcElf).WithField("to", dstElf).Debug("copying sketch build output") if err = srcElf.CopyTo(dstElf); err != nil { + telemetry.Engine.Incr("compile", stats.M(tags)...) return nil, fmt.Errorf("copying elf file: %s", err) } } logrus.Tracef("Compile %s for %s successful", sketch.Name, fqbnIn) - + tags["success"] = "true" + telemetry.Engine.Incr("compile", stats.M(tags)...) return &rpc.CompileResp{}, nil } diff --git a/telemetry/telemetry.go b/telemetry/telemetry.go index 073f40d23f2..092854f1ec4 100644 --- a/telemetry/telemetry.go +++ b/telemetry/telemetry.go @@ -1 +1,36 @@ package telemetry + +import ( + "net/http" + + "github.com/segmentio/stats/v4" + "github.com/segmentio/stats/v4/prometheus" + "github.com/sirupsen/logrus" +) + +// Engine is the engine used by global helper functions for this module. +var Engine = stats.DefaultEngine + +var serverAddr = ":2112" +var serverPattern = "/metrics" + +//Activate configure and starts the telemetry server exposing a Prometheus resource +func Activate(metricPrefix string) { + // Configure telemetry engine + // Create a Prometheus default handler + ph := prometheus.DefaultHandler + // Replace the default stats engine with an engine that prepends the "daemon" prefix to all metrics + Engine = stats.WithPrefix(metricPrefix) + // Register the handler so it receives metrics from the default engine. + Engine.Register(ph) + // Flush the default stats engine on return to ensure all buffered + // metrics are sent to the server. + defer Engine.Flush() + // move everything inside commands and search for setting up a common prefix for all metrics sent! + logrus.Infof("Setting up Prometheus telemetry on %s%s", serverAddr, serverPattern) + go func() { + http.Handle(serverPattern, ph) + logrus.Error(http.ListenAndServe(serverAddr, nil)) + }() + +} From 18cdce8a2895f3dfc0d1f8390ec5a0c4130a07f5 Mon Sep 17 00:00:00 2001 From: Roberto Sora Date: Fri, 24 Jan 2020 21:02:31 +0100 Subject: [PATCH 05/39] Implement configuration via Viper --- cli/daemon/daemon.go | 7 ++++--- commands/compile/compile.go | 2 +- configuration/defaults.go | 14 ++++++++++++++ go.mod | 1 + go.sum | 6 +++++- telemetry/telemetry.go | 38 ++++++++++++++++++++++++++----------- 6 files changed, 52 insertions(+), 16 deletions(-) diff --git a/cli/daemon/daemon.go b/cli/daemon/daemon.go index 78dec9128a3..f2df9a16487 100644 --- a/cli/daemon/daemon.go +++ b/cli/daemon/daemon.go @@ -61,8 +61,10 @@ var daemonize bool func runDaemonCommand(cmd *cobra.Command, args []string) { - telemetry.Activate("daemon") - defer telemetry.Engine.Flush() + if viper.GetBool("telemetry.enabled") { + telemetry.Activate("daemon", viper.GetString("installation.id")) + defer telemetry.Engine.Flush() + } port := viper.GetString("daemon.port") s := grpc.NewServer() @@ -99,7 +101,6 @@ func runDaemonCommand(cmd *cobra.Command, args []string) { go func() { // Stdin is closed when the controlling parent process ends _, _ = io.Copy(ioutil.Discard, os.Stdin) - telemetry.Engine.Flush() os.Exit(0) }() } diff --git a/commands/compile/compile.go b/commands/compile/compile.go index c31e62da332..dc7e2283097 100644 --- a/commands/compile/compile.go +++ b/commands/compile/compile.go @@ -47,7 +47,7 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W tags := map[string]string{ "fqbn": req.Fqbn, - "sketchPath": req.SketchPath, + "sketchPath": telemetry.SanitizeSketchPath(req.SketchPath), "showProperties": strconv.FormatBool(req.ShowProperties), "preprocess": strconv.FormatBool(req.Preprocess), "buildProperties": strings.Join(req.BuildProperties, ","), diff --git a/configuration/defaults.go b/configuration/defaults.go index 89f12cdfc77..dca16777ea3 100644 --- a/configuration/defaults.go +++ b/configuration/defaults.go @@ -18,6 +18,7 @@ package configuration import ( "path/filepath" + "github.com/gofrs/uuid" "github.com/spf13/viper" ) @@ -35,4 +36,17 @@ func setDefaults(dataDir, userDir string) { // daemon settings viper.SetDefault("daemon.port", "50051") + + //telemetry settings + viper.SetDefault("telemetry.enabled", true) + viper.SetDefault("telemetry.addr", ":2112") + viper.SetDefault("telemetry.pattern", "/metrics") + + //Installation ID + // FIXME: how should I treat this error? + installationID, _ := uuid.NewV4() + installationSecret, _ := uuid.NewV4() + viper.SetDefault("installation.id", installationID.String()) + viper.SetDefault("installation.secret", installationSecret.String()) + } diff --git a/go.mod b/go.mod index 38706d780ce..b23803c7105 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/fluxio/multierror v0.0.0-20160419044231-9c68d39025e5 // indirect github.com/fsnotify/fsnotify v1.4.7 github.com/go-errors/errors v1.0.1 + github.com/gofrs/uuid v3.2.0+incompatible github.com/golang/protobuf v1.3.3 github.com/h2non/filetype v1.0.8 // indirect github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5 // indirect diff --git a/go.sum b/go.sum index 99ccf4d184a..14573ece853 100644 --- a/go.sum +++ b/go.sum @@ -44,6 +44,8 @@ github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= +github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -113,8 +115,8 @@ github.com/schollz/closestmatch v2.1.0+incompatible h1:Uel2GXEpJqOWBrlyI+oY9LTiy github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g= github.com/segmentio/fasthash v0.0.0-20180216231524-a72b379d632e h1:uO75wNGioszjmIzcY/tvdDYKRLVvzggtAmmJkn9j4GQ= github.com/segmentio/fasthash v0.0.0-20180216231524-a72b379d632e/go.mod h1:tm/wZFQ8e24NYaBGIlnO2WGCAi67re4HHuOm0sftE/M= +github.com/segmentio/objconv v1.0.1 h1:QjfLzwriJj40JibCV3MGSEiAoXixbp4ybhwfTB8RXOM= github.com/segmentio/objconv v1.0.1/go.mod h1:auayaH5k3137Cl4SoXTgrzQcuQDmvuVtZgS0fb1Ahys= -github.com/segmentio/stats v4.1.0+incompatible h1:Sz7ypAORgiXoR8x8VNIOKYoVpJUIeHGt/2UDMjjRFuo= github.com/segmentio/stats/v4 v4.5.3 h1:Y/DSUWZ4c8ICgqJ9rQohzKvGqGWbLPWad5zmxVoKN+Y= github.com/segmentio/stats/v4 v4.5.3/go.mod h1:LsaahUJR7iiSs8mnkvQvdQ/RLHAS5adGLxuntg0ydGo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= @@ -167,6 +169,7 @@ golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -213,6 +216,7 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogR gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce h1:xcEWjVhvbDy+nHP67nPDDpbYrY+ILlfndk4bRioVHaU= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/telemetry/telemetry.go b/telemetry/telemetry.go index 092854f1ec4..b8586a022e9 100644 --- a/telemetry/telemetry.go +++ b/telemetry/telemetry.go @@ -1,32 +1,35 @@ package telemetry import ( + "crypto/hmac" + "crypto/sha256" + "encoding/hex" "net/http" + "path/filepath" "github.com/segmentio/stats/v4" "github.com/segmentio/stats/v4/prometheus" "github.com/sirupsen/logrus" + "github.com/spf13/viper" ) // Engine is the engine used by global helper functions for this module. var Engine = stats.DefaultEngine -var serverAddr = ":2112" -var serverPattern = "/metrics" - -//Activate configure and starts the telemetry server exposing a Prometheus resource -func Activate(metricPrefix string) { +// Activate configure and starts the telemetry server exposing a Prometheus resource +func Activate(metricPrefix, installationID string) { // Configure telemetry engine // Create a Prometheus default handler ph := prometheus.DefaultHandler - // Replace the default stats engine with an engine that prepends the "daemon" prefix to all metrics - Engine = stats.WithPrefix(metricPrefix) + // Create a new stats engine with an engine that prepends the "daemon" prefix to all metrics + // and includes the installationID as a tag + Engine = stats.WithPrefix(metricPrefix, stats.T("installationID", installationID)) // Register the handler so it receives metrics from the default engine. Engine.Register(ph) - // Flush the default stats engine on return to ensure all buffered - // metrics are sent to the server. - defer Engine.Flush() - // move everything inside commands and search for setting up a common prefix for all metrics sent! + + // Configure using viper settings + serverAddr := viper.GetString("telemetry.addr") + serverPattern := viper.GetString("telemetry.pattern") logrus.Infof("Setting up Prometheus telemetry on %s%s", serverAddr, serverPattern) go func() { http.Handle(serverPattern, ph) @@ -34,3 +37,16 @@ func Activate(metricPrefix string) { }() } + +// SanitizeSketchPath uses config generated UUID (installation.secret) as an HMAC secret to sanitize and anonymize the sketch +// name maintaining it distinguishable from a different sketch from the same Installation +func SanitizeSketchPath(sketchPath string) string { + installationSecret := viper.GetString("installation.secret") + sketchName := filepath.Base(sketchPath) + // Create a new HMAC by defining the hash type and the key (as byte array) + h := hmac.New(sha256.New, []byte(installationSecret)) + // Write Data to it + h.Write([]byte(sketchName)) + // Get result and encode as hexadecimal string + return hex.EncodeToString(h.Sum(nil)) +} From 1262ba62df5060662ceb1aeadf9fb0b99c9b17e1 Mon Sep 17 00:00:00 2001 From: Roberto Sora Date: Fri, 24 Jan 2020 21:09:12 +0100 Subject: [PATCH 06/39] Add stats flush in case of a not daemonized cli daemon proces --- cli/daemon/daemon.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cli/daemon/daemon.go b/cli/daemon/daemon.go index f2df9a16487..bb257280cbc 100644 --- a/cli/daemon/daemon.go +++ b/cli/daemon/daemon.go @@ -101,6 +101,9 @@ func runDaemonCommand(cmd *cobra.Command, args []string) { go func() { // Stdin is closed when the controlling parent process ends _, _ = io.Copy(ioutil.Discard, os.Stdin) + if viper.GetBool("telemetry.enabled") { + telemetry.Engine.Flush() + } os.Exit(0) }() } From a9d1b3126319df941ab6f92f358b880b05307a2b Mon Sep 17 00:00:00 2001 From: Roberto Sora Date: Fri, 24 Jan 2020 22:52:40 +0100 Subject: [PATCH 07/39] Add repertory to store installation id and secret --- cli/cli.go | 4 ++++ cli/daemon/daemon.go | 7 +++--- commands/board/list.go | 9 ++++---- commands/compile/compile.go | 30 ++++++++++++------------- configuration/configuration.go | 9 ++++---- configuration/defaults.go | 8 ------- repertory/repertory.go | 40 ++++++++++++++++++++++++++++++++++ telemetry/telemetry.go | 19 ++++++++-------- 8 files changed, 81 insertions(+), 45 deletions(-) create mode 100644 repertory/repertory.go diff --git a/cli/cli.go b/cli/cli.go index c5a061d085b..e96ee5b7f03 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -17,6 +17,7 @@ package cli import ( "fmt" + "github.com/arduino/arduino-cli/repertory" "io/ioutil" "os" "path/filepath" @@ -167,6 +168,9 @@ func preRun(cmd *cobra.Command, args []string) { configuration.Init(configPath) configFile := viper.ConfigFileUsed() + // initialize repertory + repertory.Init() + // // Prepare logging // diff --git a/cli/daemon/daemon.go b/cli/daemon/daemon.go index bb257280cbc..60884b9ffa7 100644 --- a/cli/daemon/daemon.go +++ b/cli/daemon/daemon.go @@ -18,6 +18,7 @@ package daemon import ( "errors" "fmt" + "github.com/segmentio/stats/v4" "io" "io/ioutil" "net" @@ -62,8 +63,8 @@ var daemonize bool func runDaemonCommand(cmd *cobra.Command, args []string) { if viper.GetBool("telemetry.enabled") { - telemetry.Activate("daemon", viper.GetString("installation.id")) - defer telemetry.Engine.Flush() + telemetry.Activate("daemon") + defer stats.Flush() } port := viper.GetString("daemon.port") @@ -102,7 +103,7 @@ func runDaemonCommand(cmd *cobra.Command, args []string) { // Stdin is closed when the controlling parent process ends _, _ = io.Copy(ioutil.Discard, os.Stdin) if viper.GetBool("telemetry.enabled") { - telemetry.Engine.Flush() + stats.Flush() } os.Exit(0) }() diff --git a/commands/board/list.go b/commands/board/list.go index 1baefb8efe5..2196c2296d6 100644 --- a/commands/board/list.go +++ b/commands/board/list.go @@ -18,7 +18,6 @@ package board import ( "encoding/json" "fmt" - "github.com/arduino/arduino-cli/telemetry" "github.com/segmentio/stats/v4" "io/ioutil" "net/http" @@ -108,13 +107,13 @@ func List(instanceID int32) ([]*rpc.DetectedPort, error) { pm := commands.GetPackageManager(instanceID) if pm == nil { - telemetry.Engine.Incr("board.list", stats.T("success", "false")) + stats.Incr("board.list", stats.T("success", "false")) return nil, errors.New("invalid instance") } ports, err := commands.ListBoards(pm) if err != nil { - telemetry.Engine.Incr("board.list", stats.T("success", "false")) + stats.Incr("board.list", stats.T("success", "false")) return nil, errors.Wrap(err, "error getting port list from serial-discovery") } @@ -140,7 +139,7 @@ func List(instanceID int32) ([]*rpc.DetectedPort, error) { logrus.Debug("Board not recognized") } else if err != nil { // this is bad, bail out - telemetry.Engine.Incr("board.list", stats.T("success", "false")) + stats.Incr("board.list", stats.T("success", "false")) return nil, errors.Wrap(err, "error getting board info from Arduino Cloud") } @@ -161,6 +160,6 @@ func List(instanceID int32) ([]*rpc.DetectedPort, error) { retVal = append(retVal, p) } - telemetry.Engine.Incr("board.list", stats.T("success", "true")) + stats.Incr("board.list", stats.T("success", "true")) return retVal, nil } diff --git a/commands/compile/compile.go b/commands/compile/compile.go index dc7e2283097..f321ada49f5 100644 --- a/commands/compile/compile.go +++ b/commands/compile/compile.go @@ -63,19 +63,19 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W pm := commands.GetPackageManager(req.GetInstance().GetId()) if pm == nil { - telemetry.Engine.Incr("compile", stats.M(tags)...) + stats.Incr("compile", stats.M(tags)...) return nil, errors.New("invalid instance") } logrus.Tracef("Compile %s for %s started", req.GetSketchPath(), req.GetFqbn()) if req.GetSketchPath() == "" { - telemetry.Engine.Incr("compile", stats.M(tags)...) + stats.Incr("compile", stats.M(tags)...) return nil, fmt.Errorf("missing sketchPath") } sketchPath := paths.New(req.GetSketchPath()) sketch, err := sketches.NewSketchFromPath(sketchPath) if err != nil { - telemetry.Engine.Incr("compile", stats.M(tags)...) + stats.Incr("compile", stats.M(tags)...) return nil, fmt.Errorf("opening sketch: %s", err) } @@ -84,12 +84,12 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W fqbnIn = sketch.Metadata.CPU.Fqbn } if fqbnIn == "" { - telemetry.Engine.Incr("compile", stats.M(tags)...) + stats.Incr("compile", stats.M(tags)...) return nil, fmt.Errorf("no FQBN provided") } fqbn, err := cores.ParseFQBN(fqbnIn) if err != nil { - telemetry.Engine.Incr("compile", stats.M(tags)...) + stats.Incr("compile", stats.M(tags)...) return nil, fmt.Errorf("incorrect FQBN: %s", err) } @@ -103,7 +103,7 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W // "\"%[1]s:%[2]s\" platform is not installed, please install it by running \""+ // version.GetAppName()+" core install %[1]s:%[2]s\".", fqbn.Package, fqbn.PlatformArch) // feedback.Error(errorMessage) - telemetry.Engine.Incr("compile", stats.M(tags)...) + stats.Incr("compile", stats.M(tags)...) return nil, fmt.Errorf("platform not installed") } @@ -123,7 +123,7 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W builderCtx.BuildPath = paths.New(req.GetBuildPath()) err = builderCtx.BuildPath.MkdirAll() if err != nil { - telemetry.Engine.Incr("compile", stats.M(tags)...) + stats.Incr("compile", stats.M(tags)...) return nil, fmt.Errorf("cannot create build directory: %s", err) } } @@ -152,7 +152,7 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W builderCtx.BuildCachePath = paths.New(req.GetBuildCachePath()) err = builderCtx.BuildCachePath.MkdirAll() if err != nil { - telemetry.Engine.Incr("compile", stats.M(tags)...) + stats.Incr("compile", stats.M(tags)...) return nil, fmt.Errorf("cannot create build cache directory: %s", err) } } @@ -186,17 +186,17 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W // if --preprocess or --show-properties were passed, we can stop here if req.GetShowProperties() { tags["success"] = "true" - telemetry.Engine.Incr("compile", stats.M(tags)...) + stats.Incr("compile", stats.M(tags)...) return &rpc.CompileResp{}, builder.RunParseHardwareAndDumpBuildProperties(builderCtx) } else if req.GetPreprocess() { tags["success"] = "true" - telemetry.Engine.Incr("compile", stats.M(tags)...) + stats.Incr("compile", stats.M(tags)...) return &rpc.CompileResp{}, builder.RunPreprocess(builderCtx) } // if it's a regular build, go on... if err := builder.RunBuilder(builderCtx); err != nil { - telemetry.Engine.Incr("compile", stats.M(tags)...) + stats.Incr("compile", stats.M(tags)...) return nil, err } @@ -232,7 +232,7 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W // Copy "sketch.ino.*.hex" / "sketch.ino.*.bin" artifacts to sketch directory srcDir, err := outputPath.Parent().ReadDir() // read "/build/path/*" if err != nil { - telemetry.Engine.Incr("compile", stats.M(tags)...) + stats.Incr("compile", stats.M(tags)...) return nil, fmt.Errorf("reading build directory: %s", err) } srcDir.FilterPrefix(base + ".") @@ -243,7 +243,7 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W dstOutput := exportPath.Join(exportFile + srcFilename) logrus.WithField("from", srcOutput).WithField("to", dstOutput).Debug("copying sketch build output") if err = srcOutput.CopyTo(dstOutput); err != nil { - telemetry.Engine.Incr("compile", stats.M(tags)...) + stats.Incr("compile", stats.M(tags)...) return nil, fmt.Errorf("copying output file: %s", err) } } @@ -254,13 +254,13 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W dstElf := exportPath.Join(exportFile + ".elf") logrus.WithField("from", srcElf).WithField("to", dstElf).Debug("copying sketch build output") if err = srcElf.CopyTo(dstElf); err != nil { - telemetry.Engine.Incr("compile", stats.M(tags)...) + stats.Incr("compile", stats.M(tags)...) return nil, fmt.Errorf("copying elf file: %s", err) } } logrus.Tracef("Compile %s for %s successful", sketch.Name, fqbnIn) tags["success"] = "true" - telemetry.Engine.Incr("compile", stats.M(tags)...) + stats.Incr("compile", stats.M(tags)...) return &rpc.CompileResp{}, nil } diff --git a/configuration/configuration.go b/configuration/configuration.go index 078bdf3743b..1078c22e2c8 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -38,7 +38,7 @@ func Init(configPath string) { // Get default data path if none was provided if configPath == "" { - configPath = getDefaultArduinoDataDir() + configPath = GetDefaultArduinoDataDir() } // Add paths where to search for a config file @@ -58,7 +58,7 @@ func Init(configPath string) { // those were set through env vars or cli flags dataDir := viper.GetString("directories.Data") if dataDir == "" { - dataDir = getDefaultArduinoDataDir() + dataDir = GetDefaultArduinoDataDir() } userDir := viper.GetString("directories.User") if userDir == "" { @@ -76,10 +76,11 @@ func Init(configPath string) { feedback.Errorf("Error reading config file: %v", err) } } + } -// getDefaultArduinoDataDir returns the full path to the default arduino folder -func getDefaultArduinoDataDir() string { +// GetDefaultArduinoDataDir returns the full path to the default arduino folder +func GetDefaultArduinoDataDir() string { userHomeDir, err := os.UserHomeDir() if err != nil { feedback.Errorf("Unable to get user home dir: %v", err) diff --git a/configuration/defaults.go b/configuration/defaults.go index dca16777ea3..2c5f30d8a11 100644 --- a/configuration/defaults.go +++ b/configuration/defaults.go @@ -18,7 +18,6 @@ package configuration import ( "path/filepath" - "github.com/gofrs/uuid" "github.com/spf13/viper" ) @@ -42,11 +41,4 @@ func setDefaults(dataDir, userDir string) { viper.SetDefault("telemetry.addr", ":2112") viper.SetDefault("telemetry.pattern", "/metrics") - //Installation ID - // FIXME: how should I treat this error? - installationID, _ := uuid.NewV4() - installationSecret, _ := uuid.NewV4() - viper.SetDefault("installation.id", installationID.String()) - viper.SetDefault("installation.secret", installationSecret.String()) - } diff --git a/repertory/repertory.go b/repertory/repertory.go new file mode 100644 index 00000000000..cb231b3fecf --- /dev/null +++ b/repertory/repertory.go @@ -0,0 +1,40 @@ +package repertory + +import ( + "github.com/arduino/arduino-cli/cli/feedback" + "github.com/arduino/arduino-cli/configuration" + "github.com/gofrs/uuid" + "github.com/spf13/viper" + "os" +) + +// Store is the Read Only config storage +var Store = viper.New() + +// Configure configures the Read Only config storage +func Init() { + configPath := configuration.GetDefaultArduinoDataDir() + Store.SetConfigType("yaml") + Store.SetConfigName("repertory.yaml") + Store.AddConfigPath(configPath) + // Attempt to read config file + if err := Store.ReadInConfig(); err != nil { + // ConfigFileNotFoundError is acceptable, anything else + // should be reported to the user + if _, ok := err.(viper.ConfigFileNotFoundError); ok { + // FIXME: how should I treat this error? + installationID, _ := uuid.NewV4() + Store.SetDefault("installation.id", installationID.String()) + installationSecret, _ := uuid.NewV4() + Store.SetDefault("installation.secret", installationSecret.String()) + if err = Store.SafeWriteConfigAs(configPath); err != nil { + if os.IsNotExist(err) { + err = Store.WriteConfigAs(configPath) + } + } + } else { + feedback.Errorf("Error reading repertory file: %v", err) + + } + } +} diff --git a/telemetry/telemetry.go b/telemetry/telemetry.go index b8586a022e9..01a34c5d03a 100644 --- a/telemetry/telemetry.go +++ b/telemetry/telemetry.go @@ -4,6 +4,7 @@ import ( "crypto/hmac" "crypto/sha256" "encoding/hex" + "github.com/arduino/arduino-cli/repertory" "net/http" "path/filepath" @@ -13,19 +14,16 @@ import ( "github.com/spf13/viper" ) -// Engine is the engine used by global helper functions for this module. -var Engine = stats.DefaultEngine - -// Activate configure and starts the telemetry server exposing a Prometheus resource -func Activate(metricPrefix, installationID string) { - // Configure telemetry engine +// Activate configures and starts the telemetry server exposing a Prometheus resource +func Activate(metricPrefix string) { // Create a Prometheus default handler ph := prometheus.DefaultHandler // Create a new stats engine with an engine that prepends the "daemon" prefix to all metrics - // and includes the installationID as a tag - Engine = stats.WithPrefix(metricPrefix, stats.T("installationID", installationID)) + // and includes the installationID as a tag, then replace the default stats engine + stats.DefaultEngine = stats.WithPrefix(metricPrefix, stats.T("installationID", + repertory.Store.GetString("installation.id"))) // Register the handler so it receives metrics from the default engine. - Engine.Register(ph) + stats.Register(ph) // Configure using viper settings serverAddr := viper.GetString("telemetry.addr") @@ -41,7 +39,8 @@ func Activate(metricPrefix, installationID string) { // SanitizeSketchPath uses config generated UUID (installation.secret) as an HMAC secret to sanitize and anonymize the sketch // name maintaining it distinguishable from a different sketch from the same Installation func SanitizeSketchPath(sketchPath string) string { - installationSecret := viper.GetString("installation.secret") + logrus.Infof("repertory.Store.ConfigFileUsed() %s", repertory.Store.ConfigFileUsed()) + installationSecret := repertory.Store.GetString("installation.secret") sketchName := filepath.Base(sketchPath) // Create a new HMAC by defining the hash type and the key (as byte array) h := hmac.New(sha256.New, []byte(installationSecret)) From 74fe85219738cda0d60718a183ab0a3fc4e52499 Mon Sep 17 00:00:00 2001 From: Roberto Sora Date: Thu, 30 Jan 2020 12:34:46 +0100 Subject: [PATCH 08/39] Repertory force write --- telemetry/telemetry.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/telemetry/telemetry.go b/telemetry/telemetry.go index 01a34c5d03a..3f219ca23cf 100644 --- a/telemetry/telemetry.go +++ b/telemetry/telemetry.go @@ -1,3 +1,18 @@ +// This file is part of arduino-cli. +// +// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + package telemetry import ( From ab0a98ca723e9e9986030e8efeea539e2e6721c1 Mon Sep 17 00:00:00 2001 From: Roberto Sora Date: Thu, 30 Jan 2020 15:07:38 +0100 Subject: [PATCH 09/39] Cosmetics --- repertory/repertory.go | 57 ++++++++++++++++++++++++++++++++---------- telemetry/telemetry.go | 6 ++--- 2 files changed, 47 insertions(+), 16 deletions(-) diff --git a/repertory/repertory.go b/repertory/repertory.go index cb231b3fecf..ee552cca67f 100644 --- a/repertory/repertory.go +++ b/repertory/repertory.go @@ -1,40 +1,71 @@ +// This file is part of arduino-cli. +// +// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + package repertory import ( + "path/filepath" + "github.com/arduino/arduino-cli/cli/feedback" "github.com/arduino/arduino-cli/configuration" "github.com/gofrs/uuid" "github.com/spf13/viper" - "os" ) // Store is the Read Only config storage var Store = viper.New() +var ( + Type = "yaml" + Name = "repertory" + "." + Type +) + // Configure configures the Read Only config storage func Init() { configPath := configuration.GetDefaultArduinoDataDir() - Store.SetConfigType("yaml") - Store.SetConfigName("repertory.yaml") + Store.SetConfigName(Name) + Store.SetConfigType(Type) Store.AddConfigPath(configPath) + configFilePath := filepath.Join(configPath, Name) // Attempt to read config file if err := Store.ReadInConfig(); err != nil { // ConfigFileNotFoundError is acceptable, anything else // should be reported to the user if _, ok := err.(viper.ConfigFileNotFoundError); ok { - // FIXME: how should I treat this error? - installationID, _ := uuid.NewV4() - Store.SetDefault("installation.id", installationID.String()) - installationSecret, _ := uuid.NewV4() - Store.SetDefault("installation.secret", installationSecret.String()) - if err = Store.SafeWriteConfigAs(configPath); err != nil { - if os.IsNotExist(err) { - err = Store.WriteConfigAs(configPath) - } + generateInstallationData() + err := Store.WriteConfigAs(configFilePath) + if err != nil { + feedback.Errorf("Error writing repertory file: %v", err) } } else { feedback.Errorf("Error reading repertory file: %v", err) - } } } + +func generateInstallationData() { + + installationID, err := uuid.NewV4() + if err != nil { + feedback.Errorf("Error generating installation.id: %v", err) + } + Store.Set("installation.id", installationID.String()) + + installationSecret, err := uuid.NewV4() + if err != nil { + feedback.Errorf("Error generating installation.secret: %v", err) + } + Store.Set("installation.secret", installationSecret.String()) +} diff --git a/telemetry/telemetry.go b/telemetry/telemetry.go index 3f219ca23cf..2529031ac45 100644 --- a/telemetry/telemetry.go +++ b/telemetry/telemetry.go @@ -19,10 +19,10 @@ import ( "crypto/hmac" "crypto/sha256" "encoding/hex" - "github.com/arduino/arduino-cli/repertory" "net/http" "path/filepath" + "github.com/arduino/arduino-cli/repertory" "github.com/segmentio/stats/v4" "github.com/segmentio/stats/v4/prometheus" "github.com/sirupsen/logrus" @@ -51,8 +51,8 @@ func Activate(metricPrefix string) { } -// SanitizeSketchPath uses config generated UUID (installation.secret) as an HMAC secret to sanitize and anonymize the sketch -// name maintaining it distinguishable from a different sketch from the same Installation +// SanitizeSketchPath uses config generated UUID (installation.secret) as an HMAC secret to sanitize and anonymize +// the sketch name maintaining it distinguishable from a different sketch from the same Installation func SanitizeSketchPath(sketchPath string) string { logrus.Infof("repertory.Store.ConfigFileUsed() %s", repertory.Store.ConfigFileUsed()) installationSecret := repertory.Store.GetString("installation.secret") From 315cf006a9c35b61820efa03d5797aab1c12da99 Mon Sep 17 00:00:00 2001 From: Roberto Sora Date: Thu, 30 Jan 2020 16:13:10 +0100 Subject: [PATCH 10/39] Use viper config for repertory dir --- cli/cli.go | 2 +- repertory/repertory.go | 25 +++++++++++++++++-------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/cli/cli.go b/cli/cli.go index e96ee5b7f03..55b7c5c1429 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -169,7 +169,7 @@ func preRun(cmd *cobra.Command, args []string) { configFile := viper.ConfigFileUsed() // initialize repertory - repertory.Init() + repertory.Init(viper.GetString("directories.Data")) // // Prepare logging diff --git a/repertory/repertory.go b/repertory/repertory.go index ee552cca67f..3433dd151e4 100644 --- a/repertory/repertory.go +++ b/repertory/repertory.go @@ -16,10 +16,10 @@ package repertory import ( + "os" "path/filepath" "github.com/arduino/arduino-cli/cli/feedback" - "github.com/arduino/arduino-cli/configuration" "github.com/gofrs/uuid" "github.com/spf13/viper" ) @@ -33,22 +33,18 @@ var ( ) // Configure configures the Read Only config storage -func Init() { - configPath := configuration.GetDefaultArduinoDataDir() +func Init(configPath string) { + configFilePath := filepath.Join(configPath, Name) Store.SetConfigName(Name) Store.SetConfigType(Type) Store.AddConfigPath(configPath) - configFilePath := filepath.Join(configPath, Name) // Attempt to read config file if err := Store.ReadInConfig(); err != nil { // ConfigFileNotFoundError is acceptable, anything else // should be reported to the user if _, ok := err.(viper.ConfigFileNotFoundError); ok { generateInstallationData() - err := Store.WriteConfigAs(configFilePath) - if err != nil { - feedback.Errorf("Error writing repertory file: %v", err) - } + writeStore(configFilePath) } else { feedback.Errorf("Error reading repertory file: %v", err) } @@ -69,3 +65,16 @@ func generateInstallationData() { } Store.Set("installation.secret", installationSecret.String()) } + +func writeStore(configFilePath string) { + configPath := filepath.Dir(configFilePath) + // Create dir if not present + if _, err := os.Stat(configPath); os.IsNotExist(err) { + os.MkdirAll(configPath, 0755) + } + // Create file if not present + err := Store.WriteConfigAs(configFilePath) + if err != nil { + feedback.Errorf("Error writing repertory file: %v", err) + } +} From b961a4861da9e969db922c989a86cd19871d543c Mon Sep 17 00:00:00 2001 From: Roberto Sora Date: Thu, 30 Jan 2020 16:43:51 +0100 Subject: [PATCH 11/39] Add test for repertory file creation --- test/requirements.in | 1 + test/requirements.txt | 5 +- test/test_daemon.py | 276 ++++++++++++++++++++++++++++++++++++++++++ test/test_main.py | 19 ++- 4 files changed, 299 insertions(+), 2 deletions(-) create mode 100644 test/test_daemon.py diff --git a/test/requirements.in b/test/requirements.in index bf965b6a774..8e1078f7f49 100644 --- a/test/requirements.in +++ b/test/requirements.in @@ -2,5 +2,6 @@ pytest==5.3.4 simplejson==3.17.0 semver==2.9.0 pyserial==3.4 +pyyaml==5.3 # temporary, replaces invoke==1.3.0 in favour of https://github.com/pyinvoke/invoke/pull/661 git+https://github.com/flazzarini/invoke.git diff --git a/test/requirements.txt b/test/requirements.txt index 12155b5fe1e..0b8c8fefedc 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,9 +2,10 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile +# pip-compile test/requirements.in # attrs==19.1.0 # via pytest +importlib-metadata==1.5.0 # via pluggy, pytest git+https://github.com/flazzarini/invoke.git more-itertools==7.1.0 # via pytest packaging==19.0 # via pytest @@ -13,7 +14,9 @@ py==1.8.0 # via pytest pyparsing==2.4.0 # via packaging pyserial==3.4 pytest==5.3.4 +pyyaml==5.3 semver==2.9.0 simplejson==3.17.0 six==1.12.0 # via packaging wcwidth==0.1.7 # via pytest +zipp==2.1.0 # via importlib-metadata diff --git a/test/test_daemon.py b/test/test_daemon.py new file mode 100644 index 00000000000..9a05a191124 --- /dev/null +++ b/test/test_daemon.py @@ -0,0 +1,276 @@ +# This file is part of arduino-cli. +# +# Copyright 2020 ARDUINO SA (http://www.arduino.cc/) +# +# This software is released under the GNU General Public License version 3, +# which covers the main part of arduino-cli. +# The terms of this license can be found at: +# https://www.gnu.org/licenses/gpl-3.0.en.html +# +# You can be released from the requirements of the above licenses by purchasing +# a commercial license. Buying such a license is mandatory if you want to modify or +# otherwise use the software for commercial activities involving the Arduino +# software without disclosing the source code of your own applications. To purchase +# a commercial license, send an email to license@arduino.cc. +import json +import os +import platform + +import pytest + +from .common import running_on_ci + + +def test_compile_without_fqbn(run_command): + # Init the environment explicitly + result = run_command("core update-index") + assert result.ok + + # Download latest AVR + result = run_command("core install arduino:avr") + assert result.ok + + # Build sketch without FQBN + result = run_command("compile") + assert result.failed + + +def test_compile_with_simple_sketch(run_command, data_dir, working_dir): + # Init the environment explicitly + result = run_command("core update-index") + assert result.ok + + # Download latest AVR + result = run_command("core install arduino:avr") + assert result.ok + + sketch_name = "CompileIntegrationTest" + sketch_path = os.path.join(data_dir, sketch_name) + fqbn = "arduino:avr:uno" + + # Create a test sketch + result = run_command("sketch new {}".format(sketch_path)) + assert result.ok + assert "Sketch created in: {}".format(sketch_path) in result.stdout + + # Build sketch for arduino:avr:uno + log_file_name = "compile.log" + log_file_path = os.path.join(data_dir, log_file_name) + result = run_command( + "compile -b {fqbn} {sketch_path} --log-format json --log-file {log_file} --log-level trace".format( + fqbn=fqbn, sketch_path=sketch_path, log_file=log_file_path + ) + ) + assert result.ok + + # let's test from the logs if the hex file produced by successful compile is moved to our sketch folder + log_json = open(log_file_path, "r") + json_log_lines = log_json.readlines() + expected_trace_sequence = [ + "Compile {sketch} for {fqbn} started".format(sketch=sketch_path, fqbn=fqbn), + "Compile {sketch} for {fqbn} successful".format(sketch=sketch_name, fqbn=fqbn), + ] + assert is_message_sequence_in_json_log_traces( + expected_trace_sequence, json_log_lines + ) + + # Test the --output flag with absolute path + target = os.path.join(data_dir, "test.hex") + result = run_command( + "compile -b {fqbn} {sketch_path} -o {target}".format( + fqbn=fqbn, sketch_path=sketch_path, target=target + ) + ) + assert result.ok + assert os.path.exists(target) + + +@pytest.mark.skipif( + running_on_ci() and platform.system() == "Windows", + reason="Test disabled on Github Actions Win VM until tmpdir inconsistent behavior bug is fixed", +) +def test_output_flag_default_path(run_command, data_dir, working_dir): + # Init the environment explicitly + result = run_command("core update-index") + assert result.ok + + # Download latest AVR + result = run_command("core install arduino:avr") + assert result.ok + + # Create a test sketch + sketch_path = os.path.join(data_dir, "test_output_flag_default_path") + fqbn = "arduino:avr:uno" + result = run_command("sketch new {}".format(sketch_path)) + assert result.ok + + # Test the --output flag defaulting to current working dir + result = run_command( + "compile -b {fqbn} {sketch_path} -o test".format( + fqbn=fqbn, sketch_path=sketch_path + ) + ) + assert result.ok + assert os.path.exists(os.path.join(working_dir, "test.hex")) + + # Test extension won't be added if already present + result = run_command( + "compile -b {fqbn} {sketch_path} -o test2.hex".format( + fqbn=fqbn, sketch_path=sketch_path + ) + ) + assert result.ok + assert os.path.exists(os.path.join(working_dir, "test2.hex")) + + +def test_compile_with_sketch_with_symlink_selfloop(run_command, data_dir): + # Init the environment explicitly + result = run_command("core update-index") + assert result.ok + + # Download latest AVR + result = run_command("core install arduino:avr") + assert result.ok + + sketch_name = "CompileIntegrationTestSymlinkSelfLoop" + sketch_path = os.path.join(data_dir, sketch_name) + fqbn = "arduino:avr:uno" + + # Create a test sketch + result = run_command("sketch new {}".format(sketch_path)) + assert result.ok + assert "Sketch created in: {}".format(sketch_path) in result.stdout + + # create a symlink that loops on himself + loop_file_path = os.path.join(sketch_path, "loop") + os.symlink(loop_file_path, loop_file_path) + + # Build sketch for arduino:avr:uno + result = run_command( + "compile -b {fqbn} {sketch_path}".format(fqbn=fqbn, sketch_path=sketch_path) + ) + # The assertion is a bit relaxed in this case because win behaves differently from macOs and linux + # returning a different error detailed message + assert "Error during sketch processing" in result.stderr + assert not result.ok + + sketch_name = "CompileIntegrationTestSymlinkDirLoop" + sketch_path = os.path.join(data_dir, sketch_name) + fqbn = "arduino:avr:uno" + + # Create a test sketch + result = run_command("sketch new {}".format(sketch_path)) + assert result.ok + assert "Sketch created in: {}".format(sketch_path) in result.stdout + + # create a symlink that loops on the upper level + loop_dir_path = os.path.join(sketch_path, "loop_dir") + os.mkdir(loop_dir_path) + loop_dir_symlink_path = os.path.join(loop_dir_path, "loop_dir_symlink") + os.symlink(loop_dir_path, loop_dir_symlink_path) + + # Build sketch for arduino:avr:uno + result = run_command( + "compile -b {fqbn} {sketch_path}".format(fqbn=fqbn, sketch_path=sketch_path) + ) + # The assertion is a bit relaxed also in this case because macOS behaves differently from win and linux: + # the cli does not follow recursively the symlink til breaking + assert "Error during sketch processing" in result.stderr + assert not result.ok + + +@pytest.mark.skipif(running_on_ci(), reason="VMs have no serial ports") +def test_compile_and_compile_combo(run_command, data_dir, detected_boards): + # Init the environment explicitly + result = run_command("core update-index") + assert result.ok + + # Install required core(s) + result = run_command("core install arduino:avr") + result = run_command("core install arduino:samd") + assert result.ok + + # Create a test sketch + sketch_name = "CompileAndUploadIntegrationTest" + sketch_path = os.path.join(data_dir, sketch_name) + result = run_command("sketch new {}".format(sketch_path)) + assert result.ok + assert "Sketch created in: {}".format(sketch_path) in result.stdout + + # Build sketch for each detected board + for board in detected_boards: + log_file_name = "{fqbn}-compile.log".format(fqbn=board.fqbn.replace(":", "-")) + log_file_path = os.path.join(data_dir, log_file_name) + command_log_flags = "--log-format json --log-file {} --log-level trace".format( + log_file_path + ) + result = run_command( + "compile -b {fqbn} --upload -p {address} {sketch_path} {log_flags}".format( + fqbn=board.fqbn, + address=board.address, + sketch_path=sketch_path, + log_flags=command_log_flags, + ) + ) + assert result.ok + # check from the logs if the bin file were uploaded on the current board + log_json = open(log_file_path, "r") + json_log_lines = log_json.readlines() + expected_trace_sequence = [ + "Compile {sketch} for {fqbn} started".format( + sketch=sketch_path, fqbn=board.fqbn + ), + "Compile {sketch} for {fqbn} successful".format( + sketch=sketch_name, fqbn=board.fqbn + ), + "Upload {sketch} on {fqbn} started".format( + sketch=sketch_path, fqbn=board.fqbn + ), + "Upload {sketch} on {fqbn} successful".format( + sketch=sketch_name, fqbn=board.fqbn + ), + ] + assert is_message_sequence_in_json_log_traces( + expected_trace_sequence, json_log_lines + ) + + +def is_message_sequence_in_json_log_traces(message_sequence, log_json_lines): + trace_entries = [] + for entry in log_json_lines: + entry = json.loads(entry) + if entry.get("level") == "trace": + if entry.get("msg") in message_sequence: + trace_entries.append(entry.get("msg")) + return message_sequence == trace_entries + + +def test_compile_blacklisted_sketchname(run_command, data_dir): + """ + Compile should ignore folders named `RCS`, `.git` and the likes, but + it should be ok for a sketch to be named like RCS.ino + """ + # Init the environment explicitly + result = run_command("core update-index") + assert result.ok + + # Download latest AVR + result = run_command("core install arduino:avr") + assert result.ok + + sketch_name = "RCS" + sketch_path = os.path.join(data_dir, sketch_name) + fqbn = "arduino:avr:uno" + + # Create a test sketch + result = run_command("sketch new {}".format(sketch_path)) + assert result.ok + assert "Sketch created in: {}".format(sketch_path) in result.stdout + + # Build sketch for arduino:avr:uno + log_file_name = "compile.log" + log_file_path = os.path.join(data_dir, log_file_name) + result = run_command( + "compile -b {fqbn} {sketch_path}".format(fqbn=fqbn, sketch_path=sketch_path) + ) + assert result.ok diff --git a/test/test_main.py b/test/test_main.py index 368660e5b75..0cdb75807c2 100644 --- a/test/test_main.py +++ b/test/test_main.py @@ -12,10 +12,11 @@ # otherwise use the software for commercial activities involving the Arduino # software without disclosing the source code of your own applications. To purchase # a commercial license, send an email to license@arduino.cc. -import os import json +import os import semver +import yaml def test_help(run_command): @@ -72,3 +73,19 @@ def test_log_options(run_command, data_dir): with open(log_file) as f: for line in f.readlines(): json.loads(line) + + +def test_repertory_creation(run_command, data_dir): + """ + using `version` as a test command + """ + + # no logs + out_lines = run_command("version").stdout.strip().split("\n") + assert len(out_lines) == 1 + + # json on file + repertory_file = os.path.join(data_dir, "repertory.yaml") + with open(repertory_file, 'r') as stream: + repertory = yaml.safe_load(stream) + assert "installation" in repertory From e1ea063a3c7846647a5efbea213a7d40856fef3d Mon Sep 17 00:00:00 2001 From: Roberto Sora Date: Thu, 30 Jan 2020 19:03:34 +0100 Subject: [PATCH 12/39] Add testing for telemetry deaemon and repertory --- cli/daemon/daemon.go | 3 +- configuration/defaults.go | 2 +- test/requirements.in | 2 + test/requirements.txt | 6 + test/test_daemon.py | 285 ++++---------------------------------- test/test_main.py | 2 +- 6 files changed, 40 insertions(+), 260 deletions(-) diff --git a/cli/daemon/daemon.go b/cli/daemon/daemon.go index 60884b9ffa7..38b7be2a445 100644 --- a/cli/daemon/daemon.go +++ b/cli/daemon/daemon.go @@ -18,6 +18,7 @@ package daemon import ( "errors" "fmt" + "github.com/arduino/arduino-cli/telemetry" "github.com/segmentio/stats/v4" "io" "io/ioutil" @@ -35,7 +36,6 @@ import ( srv_debug "github.com/arduino/arduino-cli/rpc/debug" srv_monitor "github.com/arduino/arduino-cli/rpc/monitor" srv_settings "github.com/arduino/arduino-cli/rpc/settings" - "github.com/arduino/arduino-cli/telemetry" "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -64,6 +64,7 @@ func runDaemonCommand(cmd *cobra.Command, args []string) { if viper.GetBool("telemetry.enabled") { telemetry.Activate("daemon") + stats.Incr("daemon", stats.T("success", "true")) defer stats.Flush() } diff --git a/configuration/defaults.go b/configuration/defaults.go index 2c5f30d8a11..e3e4ae3f9e7 100644 --- a/configuration/defaults.go +++ b/configuration/defaults.go @@ -38,7 +38,7 @@ func setDefaults(dataDir, userDir string) { //telemetry settings viper.SetDefault("telemetry.enabled", true) - viper.SetDefault("telemetry.addr", ":2112") + viper.SetDefault("telemetry.addr", ":9090") viper.SetDefault("telemetry.pattern", "/metrics") } diff --git a/test/requirements.in b/test/requirements.in index 8e1078f7f49..9441fb27dce 100644 --- a/test/requirements.in +++ b/test/requirements.in @@ -3,5 +3,7 @@ simplejson==3.17.0 semver==2.9.0 pyserial==3.4 pyyaml==5.3 +prometheus-client==0.7.1 +requests==2.22.0 # temporary, replaces invoke==1.3.0 in favour of https://github.com/pyinvoke/invoke/pull/661 git+https://github.com/flazzarini/invoke.git diff --git a/test/requirements.txt b/test/requirements.txt index 0b8c8fefedc..2ca080add12 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -5,18 +5,24 @@ # pip-compile test/requirements.in # attrs==19.1.0 # via pytest +certifi==2019.11.28 # via requests +chardet==3.0.4 # via requests +idna==2.8 # via requests importlib-metadata==1.5.0 # via pluggy, pytest git+https://github.com/flazzarini/invoke.git more-itertools==7.1.0 # via pytest packaging==19.0 # via pytest pluggy==0.13.1 # via pytest +prometheus-client==0.7.1 py==1.8.0 # via pytest pyparsing==2.4.0 # via packaging pyserial==3.4 pytest==5.3.4 pyyaml==5.3 +requests==2.22.0 semver==2.9.0 simplejson==3.17.0 six==1.12.0 # via packaging +urllib3==1.25.8 # via requests wcwidth==0.1.7 # via pytest zipp==2.1.0 # via importlib-metadata diff --git a/test/test_daemon.py b/test/test_daemon.py index 9a05a191124..4a95cc24ea3 100644 --- a/test/test_daemon.py +++ b/test/test_daemon.py @@ -12,265 +12,36 @@ # otherwise use the software for commercial activities involving the Arduino # software without disclosing the source code of your own applications. To purchase # a commercial license, send an email to license@arduino.cc. -import json -import os -import platform - -import pytest - -from .common import running_on_ci - - -def test_compile_without_fqbn(run_command): - # Init the environment explicitly - result = run_command("core update-index") - assert result.ok - - # Download latest AVR - result = run_command("core install arduino:avr") - assert result.ok - - # Build sketch without FQBN - result = run_command("compile") - assert result.failed - - -def test_compile_with_simple_sketch(run_command, data_dir, working_dir): - # Init the environment explicitly - result = run_command("core update-index") - assert result.ok - - # Download latest AVR - result = run_command("core install arduino:avr") - assert result.ok - - sketch_name = "CompileIntegrationTest" - sketch_path = os.path.join(data_dir, sketch_name) - fqbn = "arduino:avr:uno" - - # Create a test sketch - result = run_command("sketch new {}".format(sketch_path)) - assert result.ok - assert "Sketch created in: {}".format(sketch_path) in result.stdout - - # Build sketch for arduino:avr:uno - log_file_name = "compile.log" - log_file_path = os.path.join(data_dir, log_file_name) - result = run_command( - "compile -b {fqbn} {sketch_path} --log-format json --log-file {log_file} --log-level trace".format( - fqbn=fqbn, sketch_path=sketch_path, log_file=log_file_path - ) - ) - assert result.ok - - # let's test from the logs if the hex file produced by successful compile is moved to our sketch folder - log_json = open(log_file_path, "r") - json_log_lines = log_json.readlines() - expected_trace_sequence = [ - "Compile {sketch} for {fqbn} started".format(sketch=sketch_path, fqbn=fqbn), - "Compile {sketch} for {fqbn} successful".format(sketch=sketch_name, fqbn=fqbn), - ] - assert is_message_sequence_in_json_log_traces( - expected_trace_sequence, json_log_lines - ) - - # Test the --output flag with absolute path - target = os.path.join(data_dir, "test.hex") - result = run_command( - "compile -b {fqbn} {sketch_path} -o {target}".format( - fqbn=fqbn, sketch_path=sketch_path, target=target - ) - ) - assert result.ok - assert os.path.exists(target) - - -@pytest.mark.skipif( - running_on_ci() and platform.system() == "Windows", - reason="Test disabled on Github Actions Win VM until tmpdir inconsistent behavior bug is fixed", -) -def test_output_flag_default_path(run_command, data_dir, working_dir): - # Init the environment explicitly - result = run_command("core update-index") - assert result.ok - - # Download latest AVR - result = run_command("core install arduino:avr") - assert result.ok - - # Create a test sketch - sketch_path = os.path.join(data_dir, "test_output_flag_default_path") - fqbn = "arduino:avr:uno" - result = run_command("sketch new {}".format(sketch_path)) - assert result.ok - - # Test the --output flag defaulting to current working dir - result = run_command( - "compile -b {fqbn} {sketch_path} -o test".format( - fqbn=fqbn, sketch_path=sketch_path - ) - ) - assert result.ok - assert os.path.exists(os.path.join(working_dir, "test.hex")) - - # Test extension won't be added if already present - result = run_command( - "compile -b {fqbn} {sketch_path} -o test2.hex".format( - fqbn=fqbn, sketch_path=sketch_path - ) - ) - assert result.ok - assert os.path.exists(os.path.join(working_dir, "test2.hex")) - - -def test_compile_with_sketch_with_symlink_selfloop(run_command, data_dir): - # Init the environment explicitly - result = run_command("core update-index") - assert result.ok - - # Download latest AVR - result = run_command("core install arduino:avr") - assert result.ok - - sketch_name = "CompileIntegrationTestSymlinkSelfLoop" - sketch_path = os.path.join(data_dir, sketch_name) - fqbn = "arduino:avr:uno" - - # Create a test sketch - result = run_command("sketch new {}".format(sketch_path)) - assert result.ok - assert "Sketch created in: {}".format(sketch_path) in result.stdout - - # create a symlink that loops on himself - loop_file_path = os.path.join(sketch_path, "loop") - os.symlink(loop_file_path, loop_file_path) - - # Build sketch for arduino:avr:uno - result = run_command( - "compile -b {fqbn} {sketch_path}".format(fqbn=fqbn, sketch_path=sketch_path) - ) - # The assertion is a bit relaxed in this case because win behaves differently from macOs and linux - # returning a different error detailed message - assert "Error during sketch processing" in result.stderr - assert not result.ok - - sketch_name = "CompileIntegrationTestSymlinkDirLoop" - sketch_path = os.path.join(data_dir, sketch_name) - fqbn = "arduino:avr:uno" - - # Create a test sketch - result = run_command("sketch new {}".format(sketch_path)) - assert result.ok - assert "Sketch created in: {}".format(sketch_path) in result.stdout - - # create a symlink that loops on the upper level - loop_dir_path = os.path.join(sketch_path, "loop_dir") - os.mkdir(loop_dir_path) - loop_dir_symlink_path = os.path.join(loop_dir_path, "loop_dir_symlink") - os.symlink(loop_dir_path, loop_dir_symlink_path) - - # Build sketch for arduino:avr:uno - result = run_command( - "compile -b {fqbn} {sketch_path}".format(fqbn=fqbn, sketch_path=sketch_path) - ) - # The assertion is a bit relaxed also in this case because macOS behaves differently from win and linux: - # the cli does not follow recursively the symlink til breaking - assert "Error during sketch processing" in result.stderr - assert not result.ok - - -@pytest.mark.skipif(running_on_ci(), reason="VMs have no serial ports") -def test_compile_and_compile_combo(run_command, data_dir, detected_boards): - # Init the environment explicitly - result = run_command("core update-index") - assert result.ok - - # Install required core(s) - result = run_command("core install arduino:avr") - result = run_command("core install arduino:samd") - assert result.ok - - # Create a test sketch - sketch_name = "CompileAndUploadIntegrationTest" - sketch_path = os.path.join(data_dir, sketch_name) - result = run_command("sketch new {}".format(sketch_path)) - assert result.ok - assert "Sketch created in: {}".format(sketch_path) in result.stdout - - # Build sketch for each detected board - for board in detected_boards: - log_file_name = "{fqbn}-compile.log".format(fqbn=board.fqbn.replace(":", "-")) - log_file_path = os.path.join(data_dir, log_file_name) - command_log_flags = "--log-format json --log-file {} --log-level trace".format( - log_file_path - ) - result = run_command( - "compile -b {fqbn} --upload -p {address} {sketch_path} {log_flags}".format( - fqbn=board.fqbn, - address=board.address, - sketch_path=sketch_path, - log_flags=command_log_flags, - ) - ) - assert result.ok - # check from the logs if the bin file were uploaded on the current board - log_json = open(log_file_path, "r") - json_log_lines = log_json.readlines() - expected_trace_sequence = [ - "Compile {sketch} for {fqbn} started".format( - sketch=sketch_path, fqbn=board.fqbn - ), - "Compile {sketch} for {fqbn} successful".format( - sketch=sketch_name, fqbn=board.fqbn - ), - "Upload {sketch} on {fqbn} started".format( - sketch=sketch_path, fqbn=board.fqbn - ), - "Upload {sketch} on {fqbn} successful".format( - sketch=sketch_name, fqbn=board.fqbn - ), - ] - assert is_message_sequence_in_json_log_traces( - expected_trace_sequence, json_log_lines - ) +import os +import subprocess +import time -def is_message_sequence_in_json_log_traces(message_sequence, log_json_lines): - trace_entries = [] - for entry in log_json_lines: - entry = json.loads(entry) - if entry.get("level") == "trace": - if entry.get("msg") in message_sequence: - trace_entries.append(entry.get("msg")) - return message_sequence == trace_entries +import requests +import yaml +from prometheus_client.parser import text_string_to_metric_families -def test_compile_blacklisted_sketchname(run_command, data_dir): - """ - Compile should ignore folders named `RCS`, `.git` and the likes, but - it should be ok for a sketch to be named like RCS.ino - """ +def test_telemetry_prometheus_endpoint(pytestconfig, data_dir, downloads_dir): # Init the environment explicitly - result = run_command("core update-index") - assert result.ok - - # Download latest AVR - result = run_command("core install arduino:avr") - assert result.ok - - sketch_name = "RCS" - sketch_path = os.path.join(data_dir, sketch_name) - fqbn = "arduino:avr:uno" - - # Create a test sketch - result = run_command("sketch new {}".format(sketch_path)) - assert result.ok - assert "Sketch created in: {}".format(sketch_path) in result.stdout - - # Build sketch for arduino:avr:uno - log_file_name = "compile.log" - log_file_path = os.path.join(data_dir, log_file_name) - result = run_command( - "compile -b {fqbn} {sketch_path}".format(fqbn=fqbn, sketch_path=sketch_path) - ) - assert result.ok + cli_path = os.path.join(str(pytestconfig.rootdir), "..", "arduino-cli") + env = os.environ.copy() + env["ARDUINO_DATA_DIR"] = data_dir + env["ARDUINO_DOWNLOADS_DIR"] = downloads_dir + env["ARDUINO_SKETCHBOOK_DIR"] = data_dir + + subprocess.Popen([cli_path, "daemon"], env=env) + + time.sleep(5) + + # parse repertory file + repertory_file = os.path.join(data_dir, "repertory.yaml") + with open(repertory_file, 'r') as stream: + repertory = yaml.safe_load(stream) + + # Check if :9090/metrics endpoint is alive, + # telemetry is enabled by default in daemon mode + metrics = requests.get("http://localhost:9090/metrics").text + family = next(text_string_to_metric_families(metrics)) + sample = family.samples[0] + assert repertory["installation"]["id"] == sample.labels["installationID"] diff --git a/test/test_main.py b/test/test_main.py index 0cdb75807c2..826ccf19743 100644 --- a/test/test_main.py +++ b/test/test_main.py @@ -84,7 +84,7 @@ def test_repertory_creation(run_command, data_dir): out_lines = run_command("version").stdout.strip().split("\n") assert len(out_lines) == 1 - # json on file + # parse repertory file repertory_file = os.path.join(data_dir, "repertory.yaml") with open(repertory_file, 'r') as stream: repertory = yaml.safe_load(stream) From 30e7a2b0f597007c25d0f3348927039a373a0053 Mon Sep 17 00:00:00 2001 From: Roberto Sora Date: Thu, 30 Jan 2020 19:19:53 +0100 Subject: [PATCH 13/39] Wait for repertory and kill daemon --- test/requirements.in | 1 + test/requirements.txt | 1 + test/test_daemon.py | 18 +++++++++++------- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/test/requirements.in b/test/requirements.in index 9441fb27dce..e3894b5811f 100644 --- a/test/requirements.in +++ b/test/requirements.in @@ -5,5 +5,6 @@ pyserial==3.4 pyyaml==5.3 prometheus-client==0.7.1 requests==2.22.0 +pytest-timeout==1.3.4 # temporary, replaces invoke==1.3.0 in favour of https://github.com/pyinvoke/invoke/pull/661 git+https://github.com/flazzarini/invoke.git diff --git a/test/requirements.txt b/test/requirements.txt index 2ca080add12..6c0f9fa88df 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -17,6 +17,7 @@ prometheus-client==0.7.1 py==1.8.0 # via pytest pyparsing==2.4.0 # via packaging pyserial==3.4 +pytest-timeout==1.3.4 pytest==5.3.4 pyyaml==5.3 requests==2.22.0 diff --git a/test/test_daemon.py b/test/test_daemon.py index 4a95cc24ea3..794e31ef276 100644 --- a/test/test_daemon.py +++ b/test/test_daemon.py @@ -16,26 +16,28 @@ import os import subprocess import time - +import pytest import requests import yaml from prometheus_client.parser import text_string_to_metric_families - +@pytest.mark.timeout(30) def test_telemetry_prometheus_endpoint(pytestconfig, data_dir, downloads_dir): - # Init the environment explicitly + + # Use raw subprocess here due to a missing functionality in pinned pyinvoke ver, + # in order to launch and detach the cli process in daemon mode cli_path = os.path.join(str(pytestconfig.rootdir), "..", "arduino-cli") env = os.environ.copy() env["ARDUINO_DATA_DIR"] = data_dir env["ARDUINO_DOWNLOADS_DIR"] = downloads_dir env["ARDUINO_SKETCHBOOK_DIR"] = data_dir + daemon = subprocess.Popen([cli_path, "daemon"], env=env) - subprocess.Popen([cli_path, "daemon"], env=env) - - time.sleep(5) - # parse repertory file + # wait for and then parse repertory file repertory_file = os.path.join(data_dir, "repertory.yaml") + while not os.path.exists(repertory_file): + time.sleep(1) with open(repertory_file, 'r') as stream: repertory = yaml.safe_load(stream) @@ -45,3 +47,5 @@ def test_telemetry_prometheus_endpoint(pytestconfig, data_dir, downloads_dir): family = next(text_string_to_metric_families(metrics)) sample = family.samples[0] assert repertory["installation"]["id"] == sample.labels["installationID"] + #add a fixture here! + daemon.kill() From eb3358883fca0ad2d1e529bf592c9f1da84ec3d4 Mon Sep 17 00:00:00 2001 From: Roberto Sora Date: Fri, 31 Jan 2020 14:46:46 +0100 Subject: [PATCH 14/39] Updated pyinvoke to use async feature to test the daemon --- test/conftest.py | 36 +++++++++++++++++++++++++++++++++--- test/requirements.in | 3 +-- test/requirements.txt | 2 +- test/test_daemon.py | 24 ++++++++---------------- 4 files changed, 43 insertions(+), 22 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 9007eed3488..04aef580a90 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -15,8 +15,9 @@ import os import pytest -from invoke.context import Context import simplejson as json +from invoke import Local +from invoke.context import Context from .common import Board @@ -57,7 +58,7 @@ def run_command(pytestconfig, data_dir, downloads_dir, working_dir): will work in the same temporary folder. Useful reference: - http://docs.pyinvoke.org/en/1.2/api/runners.html#invoke.runners.Result + http://docs.pyinvoke.org/en/1.4/api/runners.html#invoke.runners.Result """ cli_path = os.path.join(str(pytestconfig.rootdir), "..", "arduino-cli") env = { @@ -67,7 +68,7 @@ def run_command(pytestconfig, data_dir, downloads_dir, working_dir): } os.makedirs(os.path.join(data_dir, "packages")) - def _run(cmd_string): + def _run(cmd_string, asynchronous=False): cli_full_line = "{} {}".format(cli_path, cmd_string) run_context = Context() with run_context.cd(working_dir): @@ -78,6 +79,35 @@ def _run(cmd_string): return _run +@pytest.fixture(scope="function") +def daemon_runner(pytestconfig, data_dir, downloads_dir, working_dir): + """ + Provide an invoke's `Local` object that has started the arduino-cli in daemon mode. + This way is simple to start and kill the daemon when the test is finished + via the kill() function + + Useful reference: + http://docs.pyinvoke.org/en/1.4/api/runners.html#invoke.runners.Local + http://docs.pyinvoke.org/en/1.4/api/runners.html + """ + cli_full_line = os.path.join(str(pytestconfig.rootdir), "..", "arduino-cli daemon") + env = { + "ARDUINO_DATA_DIR": data_dir, + "ARDUINO_DOWNLOADS_DIR": downloads_dir, + "ARDUINO_SKETCHBOOK_DIR": data_dir, + } + os.makedirs(os.path.join(data_dir, "packages")) + run_context = Context() + run_context.cd(working_dir) + # Local Class is the implementation of a Runner abstract class + runner = Local(run_context) + runner.run( + cli_full_line, echo=False, hide=True, warn=True, env=env, asynchronous=True + ) + + return runner + + @pytest.fixture(scope="function") def detected_boards(run_command): """ diff --git a/test/requirements.in b/test/requirements.in index e3894b5811f..933d0c66c3f 100644 --- a/test/requirements.in +++ b/test/requirements.in @@ -6,5 +6,4 @@ pyyaml==5.3 prometheus-client==0.7.1 requests==2.22.0 pytest-timeout==1.3.4 -# temporary, replaces invoke==1.3.0 in favour of https://github.com/pyinvoke/invoke/pull/661 -git+https://github.com/flazzarini/invoke.git +invoke==1.4.1 diff --git a/test/requirements.txt b/test/requirements.txt index 6c0f9fa88df..63ccf537141 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -9,7 +9,7 @@ certifi==2019.11.28 # via requests chardet==3.0.4 # via requests idna==2.8 # via requests importlib-metadata==1.5.0 # via pluggy, pytest -git+https://github.com/flazzarini/invoke.git +invoke==1.4.1 more-itertools==7.1.0 # via pytest packaging==19.0 # via pytest pluggy==0.13.1 # via pytest diff --git a/test/test_daemon.py b/test/test_daemon.py index 794e31ef276..e2db3fd566d 100644 --- a/test/test_daemon.py +++ b/test/test_daemon.py @@ -14,27 +14,18 @@ # a commercial license, send an email to license@arduino.cc. import os -import subprocess import time + import pytest import requests import yaml from prometheus_client.parser import text_string_to_metric_families -@pytest.mark.timeout(30) -def test_telemetry_prometheus_endpoint(pytestconfig, data_dir, downloads_dir): - - # Use raw subprocess here due to a missing functionality in pinned pyinvoke ver, - # in order to launch and detach the cli process in daemon mode - cli_path = os.path.join(str(pytestconfig.rootdir), "..", "arduino-cli") - env = os.environ.copy() - env["ARDUINO_DATA_DIR"] = data_dir - env["ARDUINO_DOWNLOADS_DIR"] = downloads_dir - env["ARDUINO_SKETCHBOOK_DIR"] = data_dir - daemon = subprocess.Popen([cli_path, "daemon"], env=env) - - # wait for and then parse repertory file +@pytest.mark.timeout(100) +def test_telemetry_prometheus_endpoint(daemon_runner, data_dir): + # Wait for the repertory file to be created and then parse it + # in order to check the generated ids repertory_file = os.path.join(data_dir, "repertory.yaml") while not os.path.exists(repertory_file): time.sleep(1) @@ -47,5 +38,6 @@ def test_telemetry_prometheus_endpoint(pytestconfig, data_dir, downloads_dir): family = next(text_string_to_metric_families(metrics)) sample = family.samples[0] assert repertory["installation"]["id"] == sample.labels["installationID"] - #add a fixture here! - daemon.kill() + + # Kill the runner's process as we finished our test + daemon_runner.kill() From 897452315b89400e8e8cd29964589b5c324ac5d5 Mon Sep 17 00:00:00 2001 From: Roberto Sora Date: Fri, 31 Jan 2020 15:33:33 +0100 Subject: [PATCH 15/39] Updated daemon test timeout --- test/test_daemon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_daemon.py b/test/test_daemon.py index e2db3fd566d..3c56dfe0ce7 100644 --- a/test/test_daemon.py +++ b/test/test_daemon.py @@ -22,7 +22,7 @@ from prometheus_client.parser import text_string_to_metric_families -@pytest.mark.timeout(100) +@pytest.mark.timeout(60) def test_telemetry_prometheus_endpoint(daemon_runner, data_dir): # Wait for the repertory file to be created and then parse it # in order to check the generated ids From e98c271a44fbfefaa2a2e373bf9c0f4ff94313b5 Mon Sep 17 00:00:00 2001 From: Roberto Sora Date: Fri, 31 Jan 2020 15:46:20 +0100 Subject: [PATCH 16/39] Cosmetics --- cli/cli.go | 2 +- cli/daemon/daemon.go | 4 ++-- commands/board/list.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cli/cli.go b/cli/cli.go index 55b7c5c1429..6b34ea90af0 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -17,7 +17,6 @@ package cli import ( "fmt" - "github.com/arduino/arduino-cli/repertory" "io/ioutil" "os" "path/filepath" @@ -40,6 +39,7 @@ import ( "github.com/arduino/arduino-cli/cli/upload" "github.com/arduino/arduino-cli/cli/version" "github.com/arduino/arduino-cli/configuration" + "github.com/arduino/arduino-cli/repertory" "github.com/mattn/go-colorable" "github.com/rifflock/lfshook" "github.com/sirupsen/logrus" diff --git a/cli/daemon/daemon.go b/cli/daemon/daemon.go index 38b7be2a445..70ff94ae2c2 100644 --- a/cli/daemon/daemon.go +++ b/cli/daemon/daemon.go @@ -18,8 +18,6 @@ package daemon import ( "errors" "fmt" - "github.com/arduino/arduino-cli/telemetry" - "github.com/segmentio/stats/v4" "io" "io/ioutil" "net" @@ -36,6 +34,8 @@ import ( srv_debug "github.com/arduino/arduino-cli/rpc/debug" srv_monitor "github.com/arduino/arduino-cli/rpc/monitor" srv_settings "github.com/arduino/arduino-cli/rpc/settings" + "github.com/arduino/arduino-cli/telemetry" + "github.com/segmentio/stats/v4" "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/viper" diff --git a/commands/board/list.go b/commands/board/list.go index 2196c2296d6..b78494c4402 100644 --- a/commands/board/list.go +++ b/commands/board/list.go @@ -18,7 +18,6 @@ package board import ( "encoding/json" "fmt" - "github.com/segmentio/stats/v4" "io/ioutil" "net/http" "regexp" @@ -28,6 +27,7 @@ import ( "github.com/arduino/arduino-cli/commands" rpc "github.com/arduino/arduino-cli/rpc/commands" "github.com/pkg/errors" + "github.com/segmentio/stats/v4" "github.com/sirupsen/logrus" ) From ead55a3b3078216d56c15050db842ffa2cb15850 Mon Sep 17 00:00:00 2001 From: Roberto Sora Date: Fri, 31 Jan 2020 15:47:38 +0100 Subject: [PATCH 17/39] Set getDefaultArduinoDataDir as unexported --- configuration/configuration.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/configuration/configuration.go b/configuration/configuration.go index 1078c22e2c8..ed1cf70c3e6 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -38,7 +38,7 @@ func Init(configPath string) { // Get default data path if none was provided if configPath == "" { - configPath = GetDefaultArduinoDataDir() + configPath = getDefaultArduinoDataDir() } // Add paths where to search for a config file @@ -58,7 +58,7 @@ func Init(configPath string) { // those were set through env vars or cli flags dataDir := viper.GetString("directories.Data") if dataDir == "" { - dataDir = GetDefaultArduinoDataDir() + dataDir = getDefaultArduinoDataDir() } userDir := viper.GetString("directories.User") if userDir == "" { @@ -79,8 +79,8 @@ func Init(configPath string) { } -// GetDefaultArduinoDataDir returns the full path to the default arduino folder -func GetDefaultArduinoDataDir() string { +// getDefaultArduinoDataDir returns the full path to the default arduino folder +func getDefaultArduinoDataDir() string { userHomeDir, err := os.UserHomeDir() if err != nil { feedback.Errorf("Unable to get user home dir: %v", err) From 216303f1afb0c7c0c625a59c4272f8e53337c7d5 Mon Sep 17 00:00:00 2001 From: Roberto Sora Date: Fri, 31 Jan 2020 15:50:56 +0100 Subject: [PATCH 18/39] Cosmetics --- configuration/configuration.go | 1 - 1 file changed, 1 deletion(-) diff --git a/configuration/configuration.go b/configuration/configuration.go index ed1cf70c3e6..078bdf3743b 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -76,7 +76,6 @@ func Init(configPath string) { feedback.Errorf("Error reading config file: %v", err) } } - } // getDefaultArduinoDataDir returns the full path to the default arduino folder From 18481c00427f9cae2a56fec017d5aab2145c85f2 Mon Sep 17 00:00:00 2001 From: Roberto Sora Date: Fri, 31 Jan 2020 15:51:33 +0100 Subject: [PATCH 19/39] Cosmetics --- main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/main.go b/main.go index 73fa22ed7fe..4ab55ba9d8e 100644 --- a/main.go +++ b/main.go @@ -23,7 +23,6 @@ import ( ) func main() { - if err := cli.ArduinoCli.Execute(); err != nil { os.Exit(errorcodes.ErrGeneric) } From c44870413a2deca51167b91576b1459ac31bb28a Mon Sep 17 00:00:00 2001 From: Roberto Sora Date: Fri, 31 Jan 2020 15:54:46 +0100 Subject: [PATCH 20/39] Cosmetics --- repertory/repertory.go | 1 - test/conftest.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/repertory/repertory.go b/repertory/repertory.go index 3433dd151e4..3b52bafba21 100644 --- a/repertory/repertory.go +++ b/repertory/repertory.go @@ -52,7 +52,6 @@ func Init(configPath string) { } func generateInstallationData() { - installationID, err := uuid.NewV4() if err != nil { feedback.Errorf("Error generating installation.id: %v", err) diff --git a/test/conftest.py b/test/conftest.py index 04aef580a90..0db5244c1a0 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -68,7 +68,7 @@ def run_command(pytestconfig, data_dir, downloads_dir, working_dir): } os.makedirs(os.path.join(data_dir, "packages")) - def _run(cmd_string, asynchronous=False): + def _run(cmd_string): cli_full_line = "{} {}".format(cli_path, cmd_string) run_context = Context() with run_context.cd(working_dir): From 0aea65c9a3196790a5463047c6dd04d89fcb0b11 Mon Sep 17 00:00:00 2001 From: Roberto Sora Date: Fri, 31 Jan 2020 16:59:25 +0100 Subject: [PATCH 21/39] Lint on repertory module --- repertory/repertory.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/repertory/repertory.go b/repertory/repertory.go index 3b52bafba21..6d9d8ae97c5 100644 --- a/repertory/repertory.go +++ b/repertory/repertory.go @@ -28,11 +28,13 @@ import ( var Store = viper.New() var ( + // Type is the repertory file type Type = "yaml" + // Name is the repertory file Name with Type as extension Name = "repertory" + "." + Type ) -// Configure configures the Read Only config storage +// Init configures the Read Only config storage func Init(configPath string) { configFilePath := filepath.Join(configPath, Name) Store.SetConfigName(Name) From 855b37700439d529654407547490a53c9f9ea125 Mon Sep 17 00:00:00 2001 From: rsora Date: Wed, 5 Feb 2020 14:57:50 +0100 Subject: [PATCH 22/39] Set SIGTERM as kill signal in case of win platform to overcome pyinvoke bug --- test/test_daemon.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/test_daemon.py b/test/test_daemon.py index 3c56dfe0ce7..14b9586a2cb 100644 --- a/test/test_daemon.py +++ b/test/test_daemon.py @@ -14,7 +14,9 @@ # a commercial license, send an email to license@arduino.cc. import os +import signal import time +from sys import platform import pytest import requests @@ -39,5 +41,8 @@ def test_telemetry_prometheus_endpoint(daemon_runner, data_dir): sample = family.samples[0] assert repertory["installation"]["id"] == sample.labels["installationID"] - # Kill the runner's process as we finished our test - daemon_runner.kill() + # Kill the runner's process as we finished our test (platform dependent) + os_signal = signal.SIGKILL + if platform.system() == "Windows": + os_signal = signal.SIGTERM + os.kill(daemon_runner.pid, os_signal) From b203b7e2755d712ea6d085395248a07de86b4dbe Mon Sep 17 00:00:00 2001 From: rsora Date: Wed, 5 Feb 2020 15:04:54 +0100 Subject: [PATCH 23/39] Import platform as a module --- test/test_daemon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_daemon.py b/test/test_daemon.py index 14b9586a2cb..ea1a6b763f8 100644 --- a/test/test_daemon.py +++ b/test/test_daemon.py @@ -14,9 +14,9 @@ # a commercial license, send an email to license@arduino.cc. import os +import platform import signal import time -from sys import platform import pytest import requests From d0f448c2817adf9d1e8cf98a6f9efe5031b0c1cc Mon Sep 17 00:00:00 2001 From: rsora Date: Wed, 5 Feb 2020 15:15:17 +0100 Subject: [PATCH 24/39] Reverse platform if for signal value --- test/test_daemon.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_daemon.py b/test/test_daemon.py index ea1a6b763f8..1f41a6ca6a6 100644 --- a/test/test_daemon.py +++ b/test/test_daemon.py @@ -42,7 +42,7 @@ def test_telemetry_prometheus_endpoint(daemon_runner, data_dir): assert repertory["installation"]["id"] == sample.labels["installationID"] # Kill the runner's process as we finished our test (platform dependent) - os_signal = signal.SIGKILL - if platform.system() == "Windows": - os_signal = signal.SIGTERM + os_signal = signal.SIGTERM + if platform.system() != "Windows": + os_signal = signal.SIGKILL os.kill(daemon_runner.pid, os_signal) From dc61b78131c9de437acc81320cd319fcaf1bfc48 Mon Sep 17 00:00:00 2001 From: rsora Date: Wed, 5 Feb 2020 15:36:13 +0100 Subject: [PATCH 25/39] Extract pid value --- test/test_daemon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_daemon.py b/test/test_daemon.py index 1f41a6ca6a6..d56ad88124a 100644 --- a/test/test_daemon.py +++ b/test/test_daemon.py @@ -45,4 +45,4 @@ def test_telemetry_prometheus_endpoint(daemon_runner, data_dir): os_signal = signal.SIGTERM if platform.system() != "Windows": os_signal = signal.SIGKILL - os.kill(daemon_runner.pid, os_signal) + os.kill(daemon_runner.process.pid, os_signal) From 040dc8570b9af6040d6272f012e0b36473244840 Mon Sep 17 00:00:00 2001 From: rsora Date: Tue, 3 Mar 2020 19:27:21 +0100 Subject: [PATCH 26/39] Remove print leftover --- test/test_board.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/test_board.py b/test/test_board.py index 1adf5d1fd21..8432d96673a 100644 --- a/test/test_board.py +++ b/test/test_board.py @@ -36,6 +36,5 @@ def test_board_list(run_command): def test_board_listall(run_command): assert run_command("core update-index") result = run_command("board listall") - print(result.stderr, result.stdout) assert result.ok assert ["Board", "Name", "FQBN"] == result.stdout.splitlines()[0].strip().split() From 2d0808a9875cdb63320d6433e888b9e4110cfeb2 Mon Sep 17 00:00:00 2001 From: rsora Date: Thu, 5 Mar 2020 14:01:50 +0100 Subject: [PATCH 27/39] Add better error handling in repertory creation --- repertory/repertory.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/repertory/repertory.go b/repertory/repertory.go index 6d9d8ae97c5..6edad6ab923 100644 --- a/repertory/repertory.go +++ b/repertory/repertory.go @@ -69,10 +69,13 @@ func generateInstallationData() { func writeStore(configFilePath string) { configPath := filepath.Dir(configFilePath) - // Create dir if not present - if _, err := os.Stat(configPath); os.IsNotExist(err) { - os.MkdirAll(configPath, 0755) + + // Create config dir if not present, + // MkdirAll will retrun no error if the path already exists + if err := os.MkdirAll(configPath, os.FileMode(0755)); err != nil { + feedback.Errorf("Error creating repertory dir: %v", err) } + // Create file if not present err := Store.WriteConfigAs(configFilePath) if err != nil { From b3bbdc6f8c17d511139004c90bbb58fe5a44f91d Mon Sep 17 00:00:00 2001 From: rsora Date: Thu, 5 Mar 2020 19:31:08 +0100 Subject: [PATCH 28/39] Update docs with old README extract --- docs/getting-started.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/docs/getting-started.md b/docs/getting-started.md index 90a9a0df280..568014a8cfe 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -309,3 +309,41 @@ FTDebouncer@1.3.0 downloaded Installing FTDebouncer@1.3.0... Installed FTDebouncer@1.3.0 ``` + + +Using the ``daemon`` mode and the gRPC interface +------------------------------------------------ + +Arduino CLI can be launched as a gRPC server via the `daemon` command. + +The [client_example] folder contains a sample client code that shows how to +interact with the gRPC server. Available services and messages are detailed +in the [gRPC reference] pages. + + +To provide observability for the gRPC server activities besides logs, +the `daemon` mode activates and exposes by default a [Prometheus](https://prometheus.io/) +endpoint (http://localhost:9090/metrics) that can be fetched for +telemetry data like: + +```text +# TYPE daemon_compile counter +daemon_compile{buildProperties="",exportFile="",fqbn="arduino:samd:mkr1000",installationID="ed6f1f22-1fbe-4b1f-84be-84d035b6369c",jobs="0",libraries="",preprocess="false",quiet="false",showProperties="false",sketchPath="5ff767c6fa5a91230f5cb4e267c889aa61489ab2c4f70f35f921f934c1462cb6",success="true",verbose="true",vidPid="",warnings=""} 1 1580385724726 + +# TYPE daemon_board_list counter +daemon_board_list{installationID="ed6f1f22-1fbe-4b1f-84be-84d035b6369c",success="true"} 1 1580385724833 +``` + +The telemetry settings are exposed via the ``telemetry`` section +in the CLI configuration: + +```yaml +telemetry: + enabled: true + addr: :9090 + pattern: /metrics +``` + +[client_example]: https://github.com/arduino/arduino-cli/blob/master/client_example +[gRPC reference]: /rpc/commands +[Prometheus]: https://prometheus.io/ From 90477d5e433295fae447fb46bbc8b3708b15c643 Mon Sep 17 00:00:00 2001 From: rsora Date: Thu, 5 Mar 2020 19:45:15 +0100 Subject: [PATCH 29/39] Remove telemetry.pattern setting from docs --- docs/getting-started.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index 568014a8cfe..20c6bf8a0f2 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -341,7 +341,6 @@ in the CLI configuration: telemetry: enabled: true addr: :9090 - pattern: /metrics ``` [client_example]: https://github.com/arduino/arduino-cli/blob/master/client_example From 4d5af7a04d6c7c9a349be230a2494efe33faaeba Mon Sep 17 00:00:00 2001 From: rsora Date: Fri, 6 Mar 2020 19:19:07 +0100 Subject: [PATCH 30/39] Remove serverPattern config option for telemetry --- configuration/defaults.go | 2 -- telemetry/telemetry.go | 4 +++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/configuration/defaults.go b/configuration/defaults.go index e3e4ae3f9e7..262edca08d8 100644 --- a/configuration/defaults.go +++ b/configuration/defaults.go @@ -39,6 +39,4 @@ func setDefaults(dataDir, userDir string) { //telemetry settings viper.SetDefault("telemetry.enabled", true) viper.SetDefault("telemetry.addr", ":9090") - viper.SetDefault("telemetry.pattern", "/metrics") - } diff --git a/telemetry/telemetry.go b/telemetry/telemetry.go index 2529031ac45..72b29b5481d 100644 --- a/telemetry/telemetry.go +++ b/telemetry/telemetry.go @@ -29,6 +29,9 @@ import ( "github.com/spf13/viper" ) +// serverPattern is the telemetry endpoint resource path for consume metrics +var serverPattern = "/metrics" + // Activate configures and starts the telemetry server exposing a Prometheus resource func Activate(metricPrefix string) { // Create a Prometheus default handler @@ -42,7 +45,6 @@ func Activate(metricPrefix string) { // Configure using viper settings serverAddr := viper.GetString("telemetry.addr") - serverPattern := viper.GetString("telemetry.pattern") logrus.Infof("Setting up Prometheus telemetry on %s%s", serverAddr, serverPattern) go func() { http.Handle(serverPattern, ph) From 0fdc125802820908f8a856668d7f4928f10e39e0 Mon Sep 17 00:00:00 2001 From: rsora Date: Fri, 6 Mar 2020 19:38:25 +0100 Subject: [PATCH 31/39] Upgrade viper to 1.6.2 --- go.mod | 6 ++--- go.sum | 83 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 83 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index b23803c7105..f8371f2b4b8 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,6 @@ require ( github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5 // indirect github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8 // indirect github.com/juju/testing v0.0.0-20190429233213-dfc56b8c09fc // indirect - github.com/kr/pretty v0.1.0 // indirect github.com/mattn/go-colorable v0.1.2 github.com/mattn/go-runewidth v0.0.2 // indirect github.com/miekg/dns v1.0.5 // indirect @@ -34,10 +33,9 @@ require ( github.com/schollz/closestmatch v2.1.0+incompatible github.com/segmentio/stats/v4 v4.5.3 github.com/sirupsen/logrus v1.4.2 - github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect github.com/spf13/cobra v0.0.5 github.com/spf13/jwalterweatherman v1.0.0 - github.com/spf13/viper v1.3.2 + github.com/spf13/viper v1.6.2 github.com/stretchr/testify v1.4.0 go.bug.st/cleanup v1.0.0 go.bug.st/downloader v1.1.0 @@ -51,5 +49,5 @@ require ( google.golang.org/grpc v1.27.0 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce // indirect - gopkg.in/yaml.v2 v2.2.2 + gopkg.in/yaml.v2 v2.2.4 ) diff --git a/go.sum b/go.sum index 14573ece853..9bd0b267f71 100644 --- a/go.sum +++ b/go.sum @@ -3,6 +3,9 @@ bou.ke/monkey v1.0.1/go.mod h1:FgHuK96Rv2Nlf+0u1OOVDpCMdsWyOFmeeketDHE7LIg= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/arduino/board-discovery v0.0.0-20180823133458-1ba29327fb0c h1:agh2JT96G8egU7FEb13L4dq3fnCN7lxXhJ86t69+W7s= github.com/arduino/board-discovery v0.0.0-20180823133458-1ba29327fb0c/go.mod h1:HK7SpkEax/3P+0w78iRQx1sz1vCDYYw9RXwHjQTB5i8= github.com/arduino/go-paths-helper v1.0.1 h1:utYXLM2RfFlc9qp/MJTIYp3t6ux/xM6mWjeEb/WLK4Q= @@ -14,7 +17,10 @@ github.com/arduino/go-timeutils v0.0.0-20171220113728-d1dd9e313b1b/go.mod h1:uwG github.com/arduino/go-win32-utils v0.0.0-20180330194947-ed041402e83b h1:3PjgYG5gVPA7cipp7vIR2lF96KkEJIFBJ+ANnuv6J20= github.com/arduino/go-win32-utils v0.0.0-20180330194947-ed041402e83b/go.mod h1:iIPnclBMYm1g32Q5kXoqng4jLhMStReIP7ZxaoUC2y8= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cmaglie/pb v1.0.27 h1:ynGj8vBXR+dtj4B7Q/W/qGt31771Ux5iFfRQBnwdQiA= github.com/cmaglie/pb v1.0.27/go.mod h1:GilkKZMXYjBA4NxItWFfO+lwkp59PLHQ+IOW/b/kmZI= @@ -22,9 +28,12 @@ github.com/codeclysm/cc v1.2.2 h1:1ChS4EvWTjw6bH2sd6QiMcmih0itVVrWdh9MmOliX/I= github.com/codeclysm/cc v1.2.2/go.mod h1:XtW4ArCNgQwFphcRGG9+sPX5WM1J6/u0gMy5ZdV3obA= github.com/codeclysm/extract v2.2.0+incompatible h1:q3wyckoA30bhUSiwdQezMqVhwd8+WGE64/GL//LtUhI= github.com/codeclysm/extract v2.2.0+incompatible/go.mod h1:2nhFMPHiU9At61hz+12bfrlpXSUrOnK+wR+KlGO4Uks= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/creack/goselect v0.1.1 h1:tiSSgKE1eJtxs1h/VgGQWuXUP0YS4CDIFMp6vaI1ls0= @@ -32,6 +41,8 @@ github.com/creack/goselect v0.1.1/go.mod h1:a/NhLweNvqIYMuxcMOuWY516Cimucms3DglD github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= @@ -42,28 +53,43 @@ github.com/fluxio/multierror v0.0.0-20160419044231-9c68d39025e5 h1:R8jFW6G/bjoXj github.com/fluxio/multierror v0.0.0-20160419044231-9c68d39025e5/go.mod h1:BEUDl7FG1cc76sM0J0x8dqr6RhiL4uqvk6oFkwuNyuM= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/h2non/filetype v1.0.8 h1:le8gpf+FQA0/DlDABbtisA1KiTS0Xi+YSC/E8yY3Y14= github.com/h2non/filetype v1.0.8/go.mod h1:isekKqOuhMj+s/7r3rIeTErIRy4Rub5uBWHfvMusLMU= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5 h1:rhqTjzJlm7EbkELJDKMTU7udov+Se0xZkWmugr6zGok= @@ -72,8 +98,12 @@ github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8 h1:UUHMLvzt/31azWTN/ifG github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= github.com/juju/testing v0.0.0-20190429233213-dfc56b8c09fc h1:5xUWujf6ES9tEpFHFzI34vcHm8U07lGjxAuJML3qwqM= github.com/juju/testing v0.0.0-20190429233213-dfc56b8c09fc/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -81,12 +111,15 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-runewidth v0.0.2 h1:UnlwIPBGaTZfPQ6T1IGzPI0EkYAQmT9fAEJ/poFC63o= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mdlayher/genetlink v0.0.0-20190313224034-60417448a851/go.mod h1:EsbsAEUEs15qC1cosAwxgCWV0Qhd8TmkxnA9Kw1Vhl4= github.com/mdlayher/netlink v0.0.0-20190313131330-258ea9dff42c/go.mod h1:eQB3mZE4aiYnlUsyGGCOpPETfdQq4Jhsgf1fk3cwQaA= github.com/mdlayher/taskstats v0.0.0-20190313225729-7cbba52ee072/go.mod h1:sGdS7A6CAETR53zkdjGkgoFlh1vSm7MtX+i8XfEsTMA= @@ -96,19 +129,32 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/oleksandr/bonjour v0.0.0-20160508152359-5dcf00d8b228 h1:Cvfd2dOlXIPTeEkOT/h8PyK4phBngOM4at9/jlgy7d4= github.com/oleksandr/bonjour v0.0.0-20160508152359-5dcf00d8b228/go.mod h1:MGuVJ1+5TX1SCoO2Sx0eAnjpdRytYla2uC1YIZfkC9c= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmylund/sortutil v0.0.0-20120526081524-abeda66eb583 h1:ogHi8YLNeIxABOaH6UgtbwkODheuAK+ErP8gWXYQVj0= github.com/pmylund/sortutil v0.0.0-20120526081524-abeda66eb583/go.mod h1:sFPiU/UgDcsQVu3vkqpZLCXWFwUoQRpHGu9ATihPAl0= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 h1:mZHayPoR0lNmnHyvtYjDeq0zlVHn9K/ZXoy17ylucdo= github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5/go.mod h1:GEXHk5HgEKCvEIIrSpFI3ozzG5xOKA2DVlEX/gGnewM= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/schollz/closestmatch v2.1.0+incompatible h1:Uel2GXEpJqOWBrlyI+oY9LTiyyjYS17cCYRqP13/SHk= @@ -119,12 +165,15 @@ github.com/segmentio/objconv v1.0.1 h1:QjfLzwriJj40JibCV3MGSEiAoXixbp4ybhwfTB8RX github.com/segmentio/objconv v1.0.1/go.mod h1:auayaH5k3137Cl4SoXTgrzQcuQDmvuVtZgS0fb1Ahys= github.com/segmentio/stats/v4 v4.5.3 h1:Y/DSUWZ4c8ICgqJ9rQohzKvGqGWbLPWad5zmxVoKN+Y= github.com/segmentio/stats/v4 v4.5.3/go.mod h1:LsaahUJR7iiSs8mnkvQvdQ/RLHAS5adGLxuntg0ydGo= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= -github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= @@ -137,6 +186,8 @@ github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/spf13/viper v1.6.2 h1:7aKfF+e8/k68gda3LOjo5RxiUqddoFxVq4BKBPrxk5E= +github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -144,7 +195,12 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.bug.st/cleanup v1.0.0 h1:XVj1HZxkBXeq3gMT7ijWUpHyIC1j8XAoNSyQ06CskgA= go.bug.st/cleanup v1.0.0/go.mod h1:EqVmTg2IBk4znLbPD28xne3abjsJftMdqqJEjhn70bk= @@ -156,6 +212,11 @@ go.bug.st/serial v1.0.0 h1:ogEPzrllCsnG00EqKRjeYvPRsO7NJW6DqykzkdD6E/k= go.bug.st/serial v1.0.0/go.mod h1:rpXPISGjuNjPTRTcMlxi9lN6LoIPxd1ixVjBd8aSk/Q= go.bug.st/serial.v1 v0.0.0-20180827123349-5f7892a7bb45 h1:mACY1anK6HNCZtm/DK2Rf2ZPHggVqeB0+7rY9Gl6wyI= go.bug.st/serial.v1 v0.0.0-20180827123349-5f7892a7bb45/go.mod h1:dRSl/CVCTf56CkXgJMDOdSwNfo2g1orOGE/gBGdvjZw= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -166,17 +227,24 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 h1:XQyxROzUlZH+WIQwySDgnISg golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -191,6 +259,8 @@ golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -207,17 +277,26 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90 h1:7THRSvPuzF1bql5kyFzX0JM0vpGhwuhskgJrJsbZ80Y= google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.27.0 h1:rRYRFMVgRv6E0D70Skyfsr28tDXIuuPZyWGMPdMcnXg= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce h1:xcEWjVhvbDy+nHP67nPDDpbYrY+ILlfndk4bRioVHaU= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From 965a70f129179e28fb0489f0f06226fdabf48102 Mon Sep 17 00:00:00 2001 From: rsora Date: Fri, 6 Mar 2020 19:49:18 +0100 Subject: [PATCH 32/39] Defer stats Increment in compile command and explicit set for success/failure --- commands/compile/compile.go | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/commands/compile/compile.go b/commands/compile/compile.go index f321ada49f5..719569f42a2 100644 --- a/commands/compile/compile.go +++ b/commands/compile/compile.go @@ -58,24 +58,24 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W "exportFile": req.ExportFile, "jobs": strconv.FormatInt(int64(req.Jobs), 10), "libraries": strings.Join(req.Libraries, ","), - "success": "false", } + defer stats.Incr("compile", stats.M(tags)...) + pm := commands.GetPackageManager(req.GetInstance().GetId()) if pm == nil { - stats.Incr("compile", stats.M(tags)...) return nil, errors.New("invalid instance") } logrus.Tracef("Compile %s for %s started", req.GetSketchPath(), req.GetFqbn()) if req.GetSketchPath() == "" { - stats.Incr("compile", stats.M(tags)...) + tags["success"] = "false" return nil, fmt.Errorf("missing sketchPath") } sketchPath := paths.New(req.GetSketchPath()) sketch, err := sketches.NewSketchFromPath(sketchPath) if err != nil { - stats.Incr("compile", stats.M(tags)...) + tags["success"] = "false" return nil, fmt.Errorf("opening sketch: %s", err) } @@ -84,12 +84,12 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W fqbnIn = sketch.Metadata.CPU.Fqbn } if fqbnIn == "" { - stats.Incr("compile", stats.M(tags)...) + tags["success"] = "false" return nil, fmt.Errorf("no FQBN provided") } fqbn, err := cores.ParseFQBN(fqbnIn) if err != nil { - stats.Incr("compile", stats.M(tags)...) + tags["success"] = "false" return nil, fmt.Errorf("incorrect FQBN: %s", err) } @@ -103,7 +103,7 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W // "\"%[1]s:%[2]s\" platform is not installed, please install it by running \""+ // version.GetAppName()+" core install %[1]s:%[2]s\".", fqbn.Package, fqbn.PlatformArch) // feedback.Error(errorMessage) - stats.Incr("compile", stats.M(tags)...) + tags["success"] = "false" return nil, fmt.Errorf("platform not installed") } @@ -123,7 +123,7 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W builderCtx.BuildPath = paths.New(req.GetBuildPath()) err = builderCtx.BuildPath.MkdirAll() if err != nil { - stats.Incr("compile", stats.M(tags)...) + tags["success"] = "false" return nil, fmt.Errorf("cannot create build directory: %s", err) } } @@ -152,7 +152,7 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W builderCtx.BuildCachePath = paths.New(req.GetBuildCachePath()) err = builderCtx.BuildCachePath.MkdirAll() if err != nil { - stats.Incr("compile", stats.M(tags)...) + tags["success"] = "false" return nil, fmt.Errorf("cannot create build cache directory: %s", err) } } @@ -186,17 +186,15 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W // if --preprocess or --show-properties were passed, we can stop here if req.GetShowProperties() { tags["success"] = "true" - stats.Incr("compile", stats.M(tags)...) return &rpc.CompileResp{}, builder.RunParseHardwareAndDumpBuildProperties(builderCtx) } else if req.GetPreprocess() { tags["success"] = "true" - stats.Incr("compile", stats.M(tags)...) return &rpc.CompileResp{}, builder.RunPreprocess(builderCtx) } // if it's a regular build, go on... if err := builder.RunBuilder(builderCtx); err != nil { - stats.Incr("compile", stats.M(tags)...) + tags["success"] = "false" return nil, err } @@ -232,7 +230,7 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W // Copy "sketch.ino.*.hex" / "sketch.ino.*.bin" artifacts to sketch directory srcDir, err := outputPath.Parent().ReadDir() // read "/build/path/*" if err != nil { - stats.Incr("compile", stats.M(tags)...) + tags["success"] = "false" return nil, fmt.Errorf("reading build directory: %s", err) } srcDir.FilterPrefix(base + ".") @@ -243,7 +241,7 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W dstOutput := exportPath.Join(exportFile + srcFilename) logrus.WithField("from", srcOutput).WithField("to", dstOutput).Debug("copying sketch build output") if err = srcOutput.CopyTo(dstOutput); err != nil { - stats.Incr("compile", stats.M(tags)...) + tags["success"] = "false" return nil, fmt.Errorf("copying output file: %s", err) } } @@ -254,13 +252,12 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W dstElf := exportPath.Join(exportFile + ".elf") logrus.WithField("from", srcElf).WithField("to", dstElf).Debug("copying sketch build output") if err = srcElf.CopyTo(dstElf); err != nil { - stats.Incr("compile", stats.M(tags)...) + tags["success"] = "false" return nil, fmt.Errorf("copying elf file: %s", err) } } logrus.Tracef("Compile %s for %s successful", sketch.Name, fqbnIn) tags["success"] = "true" - stats.Incr("compile", stats.M(tags)...) return &rpc.CompileResp{}, nil } From e1582e76975fc7cc4cc1cb7fab8356e9df5fc194 Mon Sep 17 00:00:00 2001 From: rsora Date: Fri, 6 Mar 2020 19:53:09 +0100 Subject: [PATCH 33/39] Fix board list help message --- cli/board/list.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/board/list.go b/cli/board/list.go index 8aed4babd3c..16b3b2ea444 100644 --- a/cli/board/list.go +++ b/cli/board/list.go @@ -42,7 +42,7 @@ func initListCommand() *cobra.Command { } listCommand.Flags().StringVar(&listFlags.timeout, "timeout", "0s", - "The timeout of the search of connected devices, try to increase it if your board is not found (e.g. to 10s).") + "The connected devices search timeout, raise it if your board doesn't show up (e.g. to 10s).") return listCommand } From ce6f4fa8321b82b26673a36c899167a467d171c5 Mon Sep 17 00:00:00 2001 From: rsora Date: Fri, 6 Mar 2020 19:59:30 +0100 Subject: [PATCH 34/39] Implement stats flush anyway to leverage module no-op in case of no handler configured --- cli/daemon/daemon.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cli/daemon/daemon.go b/cli/daemon/daemon.go index 70ff94ae2c2..5fa00ca20ca 100644 --- a/cli/daemon/daemon.go +++ b/cli/daemon/daemon.go @@ -103,9 +103,8 @@ func runDaemonCommand(cmd *cobra.Command, args []string) { go func() { // Stdin is closed when the controlling parent process ends _, _ = io.Copy(ioutil.Discard, os.Stdin) - if viper.GetBool("telemetry.enabled") { - stats.Flush() - } + // Flush telemetry stats (this is a no-op if telemetry is disabled) + stats.Flush() os.Exit(0) }() } From 4e7b4cf4770d8ea9c82e614e83a88aa675f450c5 Mon Sep 17 00:00:00 2001 From: rsora Date: Mon, 9 Mar 2020 13:49:40 +0100 Subject: [PATCH 35/39] Rename "repertory" module in "inventory" and refactor Sanitize function --- cli/cli.go | 6 +++--- commands/compile/compile.go | 2 +- .../repertory.go => inventory/inventory.go | 14 ++++++------- telemetry/telemetry.go | 21 ++++++++----------- test/test_daemon.py | 12 +++++------ test/test_main.py | 12 +++++------ 6 files changed, 32 insertions(+), 35 deletions(-) rename repertory/repertory.go => inventory/inventory.go (87%) diff --git a/cli/cli.go b/cli/cli.go index 6b34ea90af0..76e18f08397 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -39,7 +39,7 @@ import ( "github.com/arduino/arduino-cli/cli/upload" "github.com/arduino/arduino-cli/cli/version" "github.com/arduino/arduino-cli/configuration" - "github.com/arduino/arduino-cli/repertory" + "github.com/arduino/arduino-cli/inventory" "github.com/mattn/go-colorable" "github.com/rifflock/lfshook" "github.com/sirupsen/logrus" @@ -168,8 +168,8 @@ func preRun(cmd *cobra.Command, args []string) { configuration.Init(configPath) configFile := viper.ConfigFileUsed() - // initialize repertory - repertory.Init(viper.GetString("directories.Data")) + // initialize inventory + inventory.Init(viper.GetString("directories.Data")) // // Prepare logging diff --git a/commands/compile/compile.go b/commands/compile/compile.go index 719569f42a2..6da14be0ff5 100644 --- a/commands/compile/compile.go +++ b/commands/compile/compile.go @@ -47,7 +47,7 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W tags := map[string]string{ "fqbn": req.Fqbn, - "sketchPath": telemetry.SanitizeSketchPath(req.SketchPath), + "sketchPath": telemetry.Sanitize(req.SketchPath), "showProperties": strconv.FormatBool(req.ShowProperties), "preprocess": strconv.FormatBool(req.Preprocess), "buildProperties": strings.Join(req.BuildProperties, ","), diff --git a/repertory/repertory.go b/inventory/inventory.go similarity index 87% rename from repertory/repertory.go rename to inventory/inventory.go index 6edad6ab923..205f96eb244 100644 --- a/repertory/repertory.go +++ b/inventory/inventory.go @@ -13,7 +13,7 @@ // Arduino software without disclosing the source code of your own applications. // To purchase a commercial license, send an email to license@arduino.cc. -package repertory +package inventory import ( "os" @@ -28,10 +28,10 @@ import ( var Store = viper.New() var ( - // Type is the repertory file type + // Type is the inventory file type Type = "yaml" - // Name is the repertory file Name with Type as extension - Name = "repertory" + "." + Type + // Name is the inventory file Name with Type as extension + Name = "inventory" + "." + Type ) // Init configures the Read Only config storage @@ -48,7 +48,7 @@ func Init(configPath string) { generateInstallationData() writeStore(configFilePath) } else { - feedback.Errorf("Error reading repertory file: %v", err) + feedback.Errorf("Error reading inventory file: %v", err) } } } @@ -73,12 +73,12 @@ func writeStore(configFilePath string) { // Create config dir if not present, // MkdirAll will retrun no error if the path already exists if err := os.MkdirAll(configPath, os.FileMode(0755)); err != nil { - feedback.Errorf("Error creating repertory dir: %v", err) + feedback.Errorf("Error creating inventory dir: %v", err) } // Create file if not present err := Store.WriteConfigAs(configFilePath) if err != nil { - feedback.Errorf("Error writing repertory file: %v", err) + feedback.Errorf("Error writing inventory file: %v", err) } } diff --git a/telemetry/telemetry.go b/telemetry/telemetry.go index 72b29b5481d..b2180f6b7ce 100644 --- a/telemetry/telemetry.go +++ b/telemetry/telemetry.go @@ -19,14 +19,12 @@ import ( "crypto/hmac" "crypto/sha256" "encoding/hex" - "net/http" - "path/filepath" - - "github.com/arduino/arduino-cli/repertory" + "github.com/arduino/arduino-cli/inventory" "github.com/segmentio/stats/v4" "github.com/segmentio/stats/v4/prometheus" "github.com/sirupsen/logrus" "github.com/spf13/viper" + "net/http" ) // serverPattern is the telemetry endpoint resource path for consume metrics @@ -39,7 +37,7 @@ func Activate(metricPrefix string) { // Create a new stats engine with an engine that prepends the "daemon" prefix to all metrics // and includes the installationID as a tag, then replace the default stats engine stats.DefaultEngine = stats.WithPrefix(metricPrefix, stats.T("installationID", - repertory.Store.GetString("installation.id"))) + inventory.Store.GetString("installation.id"))) // Register the handler so it receives metrics from the default engine. stats.Register(ph) @@ -53,16 +51,15 @@ func Activate(metricPrefix string) { } -// SanitizeSketchPath uses config generated UUID (installation.secret) as an HMAC secret to sanitize and anonymize -// the sketch name maintaining it distinguishable from a different sketch from the same Installation -func SanitizeSketchPath(sketchPath string) string { - logrus.Infof("repertory.Store.ConfigFileUsed() %s", repertory.Store.ConfigFileUsed()) - installationSecret := repertory.Store.GetString("installation.secret") - sketchName := filepath.Base(sketchPath) +// Sanitize uses config generated UUID (installation.secret) as an HMAC secret to sanitize and anonymize +// a string, maintaining it distinguishable from a different string from the same Installation +func Sanitize(s string) string { + logrus.Infof("inventory.Store.ConfigFileUsed() %s", inventory.Store.ConfigFileUsed()) + installationSecret := inventory.Store.GetString("installation.secret") // Create a new HMAC by defining the hash type and the key (as byte array) h := hmac.New(sha256.New, []byte(installationSecret)) // Write Data to it - h.Write([]byte(sketchName)) + h.Write([]byte(s)) // Get result and encode as hexadecimal string return hex.EncodeToString(h.Sum(nil)) } diff --git a/test/test_daemon.py b/test/test_daemon.py index d56ad88124a..ef6db9aa0fa 100644 --- a/test/test_daemon.py +++ b/test/test_daemon.py @@ -26,20 +26,20 @@ @pytest.mark.timeout(60) def test_telemetry_prometheus_endpoint(daemon_runner, data_dir): - # Wait for the repertory file to be created and then parse it + # Wait for the inventory file to be created and then parse it # in order to check the generated ids - repertory_file = os.path.join(data_dir, "repertory.yaml") - while not os.path.exists(repertory_file): + inventory_file = os.path.join(data_dir, "inventory.yaml") + while not os.path.exists(inventory_file): time.sleep(1) - with open(repertory_file, 'r') as stream: - repertory = yaml.safe_load(stream) + with open(inventory_file, 'r') as stream: + inventory = yaml.safe_load(stream) # Check if :9090/metrics endpoint is alive, # telemetry is enabled by default in daemon mode metrics = requests.get("http://localhost:9090/metrics").text family = next(text_string_to_metric_families(metrics)) sample = family.samples[0] - assert repertory["installation"]["id"] == sample.labels["installationID"] + assert inventory["installation"]["id"] == sample.labels["installationID"] # Kill the runner's process as we finished our test (platform dependent) os_signal = signal.SIGTERM diff --git a/test/test_main.py b/test/test_main.py index 826ccf19743..2ff2d4839a7 100644 --- a/test/test_main.py +++ b/test/test_main.py @@ -75,7 +75,7 @@ def test_log_options(run_command, data_dir): json.loads(line) -def test_repertory_creation(run_command, data_dir): +def test_inventory_creation(run_command, data_dir): """ using `version` as a test command """ @@ -84,8 +84,8 @@ def test_repertory_creation(run_command, data_dir): out_lines = run_command("version").stdout.strip().split("\n") assert len(out_lines) == 1 - # parse repertory file - repertory_file = os.path.join(data_dir, "repertory.yaml") - with open(repertory_file, 'r') as stream: - repertory = yaml.safe_load(stream) - assert "installation" in repertory + # parse inventory file + inventory_file = os.path.join(data_dir, "inventory.yaml") + with open(inventory_file, 'r') as stream: + inventory = yaml.safe_load(stream) + assert "installation" in inventory From aaed29d7d8d38641c0629dc29e885dd21ac4bef9 Mon Sep 17 00:00:00 2001 From: rsora Date: Mon, 9 Mar 2020 14:17:48 +0100 Subject: [PATCH 36/39] Sanitize ExportFile in command/compile --- commands/compile/compile.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commands/compile/compile.go b/commands/compile/compile.go index 6da14be0ff5..50b9f076bf8 100644 --- a/commands/compile/compile.go +++ b/commands/compile/compile.go @@ -55,7 +55,7 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W "verbose": strconv.FormatBool(req.Verbose), "quiet": strconv.FormatBool(req.Quiet), "vidPid": req.VidPid, - "exportFile": req.ExportFile, + "exportFile": telemetry.Sanitize(req.ExportFile), "jobs": strconv.FormatInt(int64(req.Jobs), 10), "libraries": strings.Join(req.Libraries, ","), } From 8b425a7f525cc660b03cf3a925e18e17951bfa18 Mon Sep 17 00:00:00 2001 From: rsora Date: Mon, 9 Mar 2020 14:18:46 +0100 Subject: [PATCH 37/39] Refactor daemon start fixture to include daemon process cleanup --- test/conftest.py | 11 ++++++++++- test/test_daemon.py | 8 -------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 0db5244c1a0..8e81250c20e 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -13,6 +13,8 @@ # software without disclosing the source code of your own applications. To purchase # a commercial license, send an email to license@arduino.cc. import os +import platform +import signal import pytest import simplejson as json @@ -105,7 +107,14 @@ def daemon_runner(pytestconfig, data_dir, downloads_dir, working_dir): cli_full_line, echo=False, hide=True, warn=True, env=env, asynchronous=True ) - return runner + # we block here until the test function using this fixture has returned + yield runner + + # Kill the runner's process as we finished our test (platform dependent) + os_signal = signal.SIGTERM + if platform.system() != "Windows": + os_signal = signal.SIGKILL + os.kill(runner.process.pid, os_signal) @pytest.fixture(scope="function") diff --git a/test/test_daemon.py b/test/test_daemon.py index ef6db9aa0fa..531ac975a71 100644 --- a/test/test_daemon.py +++ b/test/test_daemon.py @@ -14,8 +14,6 @@ # a commercial license, send an email to license@arduino.cc. import os -import platform -import signal import time import pytest @@ -40,9 +38,3 @@ def test_telemetry_prometheus_endpoint(daemon_runner, data_dir): family = next(text_string_to_metric_families(metrics)) sample = family.samples[0] assert inventory["installation"]["id"] == sample.labels["installationID"] - - # Kill the runner's process as we finished our test (platform dependent) - os_signal = signal.SIGTERM - if platform.system() != "Windows": - os_signal = signal.SIGKILL - os.kill(daemon_runner.process.pid, os_signal) From 0625b397a376518df2cd127804203e5a83373491 Mon Sep 17 00:00:00 2001 From: rsora Date: Mon, 9 Mar 2020 20:24:51 +0100 Subject: [PATCH 38/39] Use defer function to push success tag correctly updated --- commands/board/list.go | 12 ++++++++---- commands/compile/compile.go | 4 +++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/commands/board/list.go b/commands/board/list.go index b78494c4402..345fbd7aa15 100644 --- a/commands/board/list.go +++ b/commands/board/list.go @@ -105,15 +105,19 @@ func List(instanceID int32) ([]*rpc.DetectedPort, error) { m.Lock() defer m.Unlock() + tags := map[string]string{} + // Use defer func() to evaluate tags map when function returns + defer func() { stats.Incr("board.list", stats.M(tags)...) }() + pm := commands.GetPackageManager(instanceID) if pm == nil { - stats.Incr("board.list", stats.T("success", "false")) + tags["success"] = "false" return nil, errors.New("invalid instance") } ports, err := commands.ListBoards(pm) if err != nil { - stats.Incr("board.list", stats.T("success", "false")) + tags["success"] = "false" return nil, errors.Wrap(err, "error getting port list from serial-discovery") } @@ -139,7 +143,7 @@ func List(instanceID int32) ([]*rpc.DetectedPort, error) { logrus.Debug("Board not recognized") } else if err != nil { // this is bad, bail out - stats.Incr("board.list", stats.T("success", "false")) + tags["success"] = "false" return nil, errors.Wrap(err, "error getting board info from Arduino Cloud") } @@ -160,6 +164,6 @@ func List(instanceID int32) ([]*rpc.DetectedPort, error) { retVal = append(retVal, p) } - stats.Incr("board.list", stats.T("success", "true")) + tags["success"] = "true" return retVal, nil } diff --git a/commands/compile/compile.go b/commands/compile/compile.go index 50b9f076bf8..ae8815c00e0 100644 --- a/commands/compile/compile.go +++ b/commands/compile/compile.go @@ -60,10 +60,12 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W "libraries": strings.Join(req.Libraries, ","), } - defer stats.Incr("compile", stats.M(tags)...) + // Use defer func() to evaluate tags map when function returns + defer func() { stats.Incr("compile", stats.M(tags)...) }() pm := commands.GetPackageManager(req.GetInstance().GetId()) if pm == nil { + tags["success"] = "false" return nil, errors.New("invalid instance") } From 3300d4d7da96c8790a0af036e1fd5dbcf6cf354e Mon Sep 17 00:00:00 2001 From: rsora Date: Tue, 10 Mar 2020 18:45:56 +0100 Subject: [PATCH 39/39] Use named return parameters to handle success tagging for a command stat --- commands/board/list.go | 15 +++++++++------ commands/compile/compile.go | 26 +++++++++----------------- 2 files changed, 18 insertions(+), 23 deletions(-) diff --git a/commands/board/list.go b/commands/board/list.go index 345fbd7aa15..0ec7ba6ec64 100644 --- a/commands/board/list.go +++ b/commands/board/list.go @@ -101,23 +101,28 @@ func identifyViaCloudAPI(port *commands.BoardPort) ([]*rpc.BoardListItem, error) } // List FIXMEDOC -func List(instanceID int32) ([]*rpc.DetectedPort, error) { +func List(instanceID int32) (r []*rpc.DetectedPort, e error) { m.Lock() defer m.Unlock() tags := map[string]string{} // Use defer func() to evaluate tags map when function returns - defer func() { stats.Incr("board.list", stats.M(tags)...) }() + // and set success flag inspecting the error named return parameter + defer func() { + tags["success"] = "true" + if e != nil { + tags["success"] = "false" + } + stats.Incr("compile", stats.M(tags)...) + }() pm := commands.GetPackageManager(instanceID) if pm == nil { - tags["success"] = "false" return nil, errors.New("invalid instance") } ports, err := commands.ListBoards(pm) if err != nil { - tags["success"] = "false" return nil, errors.Wrap(err, "error getting port list from serial-discovery") } @@ -143,7 +148,6 @@ func List(instanceID int32) ([]*rpc.DetectedPort, error) { logrus.Debug("Board not recognized") } else if err != nil { // this is bad, bail out - tags["success"] = "false" return nil, errors.Wrap(err, "error getting board info from Arduino Cloud") } @@ -164,6 +168,5 @@ func List(instanceID int32) ([]*rpc.DetectedPort, error) { retVal = append(retVal, p) } - tags["success"] = "true" return retVal, nil } diff --git a/commands/compile/compile.go b/commands/compile/compile.go index ae8815c00e0..9a688ad2a8b 100644 --- a/commands/compile/compile.go +++ b/commands/compile/compile.go @@ -43,7 +43,7 @@ import ( ) // Compile FIXMEDOC -func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.Writer, debug bool) (*rpc.CompileResp, error) { +func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.Writer, debug bool) (r *rpc.CompileResp, e error) { tags := map[string]string{ "fqbn": req.Fqbn, @@ -61,23 +61,27 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W } // Use defer func() to evaluate tags map when function returns - defer func() { stats.Incr("compile", stats.M(tags)...) }() + // and set success flag inspecting the error named return parameter + defer func() { + tags["success"] = "true" + if e != nil { + tags["success"] = "false" + } + stats.Incr("compile", stats.M(tags)...) + }() pm := commands.GetPackageManager(req.GetInstance().GetId()) if pm == nil { - tags["success"] = "false" return nil, errors.New("invalid instance") } logrus.Tracef("Compile %s for %s started", req.GetSketchPath(), req.GetFqbn()) if req.GetSketchPath() == "" { - tags["success"] = "false" return nil, fmt.Errorf("missing sketchPath") } sketchPath := paths.New(req.GetSketchPath()) sketch, err := sketches.NewSketchFromPath(sketchPath) if err != nil { - tags["success"] = "false" return nil, fmt.Errorf("opening sketch: %s", err) } @@ -86,12 +90,10 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W fqbnIn = sketch.Metadata.CPU.Fqbn } if fqbnIn == "" { - tags["success"] = "false" return nil, fmt.Errorf("no FQBN provided") } fqbn, err := cores.ParseFQBN(fqbnIn) if err != nil { - tags["success"] = "false" return nil, fmt.Errorf("incorrect FQBN: %s", err) } @@ -105,7 +107,6 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W // "\"%[1]s:%[2]s\" platform is not installed, please install it by running \""+ // version.GetAppName()+" core install %[1]s:%[2]s\".", fqbn.Package, fqbn.PlatformArch) // feedback.Error(errorMessage) - tags["success"] = "false" return nil, fmt.Errorf("platform not installed") } @@ -125,7 +126,6 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W builderCtx.BuildPath = paths.New(req.GetBuildPath()) err = builderCtx.BuildPath.MkdirAll() if err != nil { - tags["success"] = "false" return nil, fmt.Errorf("cannot create build directory: %s", err) } } @@ -154,7 +154,6 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W builderCtx.BuildCachePath = paths.New(req.GetBuildCachePath()) err = builderCtx.BuildCachePath.MkdirAll() if err != nil { - tags["success"] = "false" return nil, fmt.Errorf("cannot create build cache directory: %s", err) } } @@ -187,16 +186,13 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W // if --preprocess or --show-properties were passed, we can stop here if req.GetShowProperties() { - tags["success"] = "true" return &rpc.CompileResp{}, builder.RunParseHardwareAndDumpBuildProperties(builderCtx) } else if req.GetPreprocess() { - tags["success"] = "true" return &rpc.CompileResp{}, builder.RunPreprocess(builderCtx) } // if it's a regular build, go on... if err := builder.RunBuilder(builderCtx); err != nil { - tags["success"] = "false" return nil, err } @@ -232,7 +228,6 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W // Copy "sketch.ino.*.hex" / "sketch.ino.*.bin" artifacts to sketch directory srcDir, err := outputPath.Parent().ReadDir() // read "/build/path/*" if err != nil { - tags["success"] = "false" return nil, fmt.Errorf("reading build directory: %s", err) } srcDir.FilterPrefix(base + ".") @@ -243,7 +238,6 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W dstOutput := exportPath.Join(exportFile + srcFilename) logrus.WithField("from", srcOutput).WithField("to", dstOutput).Debug("copying sketch build output") if err = srcOutput.CopyTo(dstOutput); err != nil { - tags["success"] = "false" return nil, fmt.Errorf("copying output file: %s", err) } } @@ -254,12 +248,10 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W dstElf := exportPath.Join(exportFile + ".elf") logrus.WithField("from", srcElf).WithField("to", dstElf).Debug("copying sketch build output") if err = srcElf.CopyTo(dstElf); err != nil { - tags["success"] = "false" return nil, fmt.Errorf("copying elf file: %s", err) } } logrus.Tracef("Compile %s for %s successful", sketch.Name, fqbnIn) - tags["success"] = "true" return &rpc.CompileResp{}, nil }