From d6e7189e02d6be409c91f88067a508a2c8d6c047 Mon Sep 17 00:00:00 2001 From: Matteo Suppo Date: Tue, 1 Mar 2016 15:07:21 +0100 Subject: [PATCH 1/7] Move utilities to a package to keep things clean --- conn.go | 3 +- discovery.go | 16 ++++- killbrowser_linux.go | 4 +- seriallist_linux.go | 4 +- utilities.go | 145 ----------------------------------------- utilities/utilities.go | 89 +++++++++++++++++++++++++ 6 files changed, 111 insertions(+), 150 deletions(-) delete mode 100644 utilities.go create mode 100644 utilities/utilities.go diff --git a/conn.go b/conn.go index 50a552cd0..18a6ebe2c 100644 --- a/conn.go +++ b/conn.go @@ -14,6 +14,7 @@ import ( "net/http" log "github.com/Sirupsen/logrus" + "github.com/arduino/arduino-create-agent/utilities" "github.com/gin-gonic/gin" "github.com/googollee/go-socket.io" ) @@ -95,7 +96,7 @@ func uploadHandler(c *gin.Context) { buffer := bytes.NewBuffer(data.Hex) - path, err := saveFileonTempDir(data.Filename, buffer) + path, err := utilities.SaveFileonTempDir(data.Filename, buffer) if err != nil { c.String(http.StatusBadRequest, err.Error()) } diff --git a/discovery.go b/discovery.go index 88becff59..1e857a4b2 100644 --- a/discovery.go +++ b/discovery.go @@ -29,11 +29,12 @@ package main import ( - log "github.com/Sirupsen/logrus" - "github.com/oleksandr/bonjour" "net" "strings" "time" + + log "github.com/Sirupsen/logrus" + "github.com/oleksandr/bonjour" ) const timeoutConst = 2 @@ -143,3 +144,14 @@ func getPorts() ([]OsSerialPort, error) { return arrPorts, nil } } + +// Filter returns a new slice containing all OsSerialPort in the slice that satisfy the predicate f. +func Filter(vs []OsSerialPort, f func(OsSerialPort) bool) []OsSerialPort { + var vsf []OsSerialPort + for _, v := range vs { + if f(v) { + vsf = append(vsf, v) + } + } + return vsf +} diff --git a/killbrowser_linux.go b/killbrowser_linux.go index b7b12f13c..a6c0560e5 100644 --- a/killbrowser_linux.go +++ b/killbrowser_linux.go @@ -3,6 +3,8 @@ package main import ( "os/exec" "strings" + + "github.com/arduino/arduino-create-agent/utilities" ) func findBrowser(process string) ([]byte, error) { @@ -10,7 +12,7 @@ func findBrowser(process string) ([]byte, error) { grep := exec.Command("grep", process) head := exec.Command("head", "-n", "1") - return pipe_commands(ps, grep, head) + return utilities.PipeCommands(ps, grep, head) } func killBrowser(process string) ([]byte, error) { diff --git a/seriallist_linux.go b/seriallist_linux.go index 925b9fe7d..b6e3ec559 100755 --- a/seriallist_linux.go +++ b/seriallist_linux.go @@ -6,6 +6,8 @@ import ( "path/filepath" "strconv" "strings" + + "github.com/arduino/arduino-create-agent/utilities" ) func associateVidPidWithPort(ports []OsSerialPort) []OsSerialPort { @@ -14,7 +16,7 @@ func associateVidPidWithPort(ports []OsSerialPort) []OsSerialPort { ueventcmd := exec.Command("cat", "/sys/class/tty/"+filepath.Base(ports[index].Name)+"/device/uevent") grep3cmd := exec.Command("grep", "PRODUCT=") - cmdOutput2, _ := pipe_commands(ueventcmd, grep3cmd) + cmdOutput2, _ := utilities.PipeCommands(ueventcmd, grep3cmd) cmdOutput2S := string(cmdOutput2) if len(cmdOutput2S) == 0 { diff --git a/utilities.go b/utilities.go deleted file mode 100644 index 198473ead..000000000 --- a/utilities.go +++ /dev/null @@ -1,145 +0,0 @@ -// utilities.go - -package main - -import ( - "archive/zip" - "bytes" - "crypto/md5" - "io" - "os" - "os/exec" - "path" - "path/filepath" - - "github.com/pivotal-golang/archiver/extractor" -) - -func computeMd5(filePath string) ([]byte, error) { - var result []byte - file, err := os.Open(filePath) - if err != nil { - return result, err - } - defer file.Close() - - hash := md5.New() - if _, err := io.Copy(hash, file); err != nil { - return result, err - } - - return hash.Sum(result), nil -} - -func call(stack []*exec.Cmd, pipes []*io.PipeWriter) (err error) { - if stack[0].Process == nil { - if err = stack[0].Start(); err != nil { - return err - } - } - if len(stack) > 1 { - if err = stack[1].Start(); err != nil { - return err - } - defer func() { - pipes[0].Close() - err = call(stack[1:], pipes[1:]) - }() - } - return stack[0].Wait() -} - -// code inspired by https://gist.github.com/tyndyll/89fbb2c2273f83a074dc -func pipe_commands(commands ...*exec.Cmd) ([]byte, error) { - var errorBuffer, outputBuffer bytes.Buffer - pipeStack := make([]*io.PipeWriter, len(commands)-1) - i := 0 - for ; i < len(commands)-1; i++ { - stdinPipe, stdoutPipe := io.Pipe() - commands[i].Stdout = stdoutPipe - commands[i].Stderr = &errorBuffer - commands[i+1].Stdin = stdinPipe - pipeStack[i] = stdoutPipe - } - commands[i].Stdout = &outputBuffer - commands[i].Stderr = &errorBuffer - - if err := call(commands, pipeStack); err != nil { - return nil, err - } - - return outputBuffer.Bytes(), nil -} - -// Filter returns a new slice containing all OsSerialPort in the slice that satisfy the predicate f. -func Filter(vs []OsSerialPort, f func(OsSerialPort) bool) []OsSerialPort { - var vsf []OsSerialPort - for _, v := range vs { - if f(v) { - vsf = append(vsf, v) - } - } - return vsf -} - -func UnzipWrapper(src, dest string) error { - e := extractor.NewDetectable() - return e.Extract(src, dest) -} - -func IsZip(path string) bool { - r, err := zip.OpenReader(path) - if err == nil { - r.Close() - return true - } - return false -} - -func Unzip(zippath string, destination string) (err error) { - r, err := zip.OpenReader(zippath) - if err != nil { - return err - } - for _, f := range r.File { - fullname := path.Join(destination, f.Name) - if f.FileInfo().IsDir() { - os.MkdirAll(fullname, f.FileInfo().Mode().Perm()) - } else { - os.MkdirAll(filepath.Dir(fullname), 0755) - perms := f.FileInfo().Mode().Perm() - out, err := os.OpenFile(fullname, os.O_CREATE|os.O_RDWR, perms) - if err != nil { - return err - } - rc, err := f.Open() - if err != nil { - return err - } - _, err = io.CopyN(out, rc, f.FileInfo().Size()) - if err != nil { - return err - } - rc.Close() - out.Close() - - mtime := f.FileInfo().ModTime() - err = os.Chtimes(fullname, mtime, mtime) - if err != nil { - return err - } - } - } - return -} - -func UnzipList(path string) (list []string, err error) { - r, err := zip.OpenReader(path) - if err != nil { - return - } - for _, f := range r.File { - list = append(list, f.Name) - } - return -} diff --git a/utilities/utilities.go b/utilities/utilities.go new file mode 100644 index 000000000..2334ce295 --- /dev/null +++ b/utilities/utilities.go @@ -0,0 +1,89 @@ +package utilities + +import ( + "bytes" + "errors" + "io" + "io/ioutil" + "os" + "os/exec" + "path/filepath" +) + +// SaveFileonTempDir creates a temp directory and saves the file data as the +// filename in that directory. +// Returns an error if the agent doesn't have permission to create the folder. +// Returns an error if the filename doesn't form a valid path. +// +// Note that path could be defined and still there could be an error. +func SaveFileonTempDir(filename string, data io.Reader) (path string, err error) { + // Create Temp Directory + tmpdir, err := ioutil.TempDir("", "arduino-create-agent") + if err != nil { + return "", errors.New("Could not create temp directory to store downloaded file. Do you have permissions?") + } + + // Determine filename + filename, err = filepath.Abs(tmpdir + "/" + filename) + if err != nil { + return "", err + } + + // Touch file + output, err := os.Create(filename) + if err != nil { + return filename, err + } + defer output.Close() + + // Write file + _, err = io.Copy(output, data) + if err != nil { + return filename, err + } + return filename, nil +} + +// PipeCommands executes the commands received as input by feeding the output of +// one to the input of the other, exactly like Unix Pipe (|). +// Returns the output of the final command and the eventual error. +// +// code inspired by https://gist.github.com/tyndyll/89fbb2c2273f83a074dc +func PipeCommands(commands ...*exec.Cmd) ([]byte, error) { + var errorBuffer, outputBuffer bytes.Buffer + pipeStack := make([]*io.PipeWriter, len(commands)-1) + i := 0 + for ; i < len(commands)-1; i++ { + stdinPipe, stdoutPipe := io.Pipe() + commands[i].Stdout = stdoutPipe + commands[i].Stderr = &errorBuffer + commands[i+1].Stdin = stdinPipe + pipeStack[i] = stdoutPipe + } + commands[i].Stdout = &outputBuffer + commands[i].Stderr = &errorBuffer + + if err := call(commands, pipeStack); err != nil { + return nil, err + } + + return outputBuffer.Bytes(), nil +} + +func call(stack []*exec.Cmd, pipes []*io.PipeWriter) (err error) { + if stack[0].Process == nil { + if err = stack[0].Start(); err != nil { + return err + } + } + if len(stack) > 1 { + if err = stack[1].Start(); err != nil { + return err + } + defer func() { + pipes[0].Close() + err = call(stack[1:], pipes[1:]) + }() + } + return stack[0].Wait() +} From 4f8115c0df588741d0ccbb5ab03fd876d7ddae74 Mon Sep 17 00:00:00 2001 From: Matteo Suppo Date: Tue, 1 Mar 2016 15:17:36 +0100 Subject: [PATCH 2/7] Move initialization of tools in package tools --- hub.go | 4 +++- main.go | 15 ++++++--------- seriallist_darwin.go | 3 --- seriallist_linux.go | 3 --- seriallist_windows.go | 14 ++++---------- tools/hidefile_darwin.go | 5 +++++ tools/hidefile_linux.go | 5 +++++ tools/hidefile_windows.go | 10 ++++++++++ tools/tools.go | 24 ++++++++++++++++++++++++ 9 files changed, 57 insertions(+), 26 deletions(-) create mode 100644 tools/hidefile_darwin.go create mode 100644 tools/hidefile_linux.go create mode 100644 tools/hidefile_windows.go create mode 100644 tools/tools.go diff --git a/hub.go b/hub.go index 8d17c3ad3..968586688 100755 --- a/hub.go +++ b/hub.go @@ -2,7 +2,9 @@ package main import ( "fmt" + log "github.com/Sirupsen/logrus" + "github.com/arduino/arduino-create-agent/tools" "github.com/kardianos/osext" //"os" "os/exec" @@ -191,7 +193,7 @@ func checkCmd(m []byte) { } else if strings.HasPrefix(sl, "downloadtool") { args := strings.Split(s, " ") if len(args) > 2 { - go spDownloadTool(args[1], args[2]) + go tools.Download(args[1], args[2]) } } else if strings.HasPrefix(sl, "bufferalgorithm") { go spBufferAlgorithms() diff --git a/main.go b/main.go index 90958b360..6d7322335 100755 --- a/main.go +++ b/main.go @@ -6,7 +6,6 @@ package main import ( "flag" "os" - "os/user" "path/filepath" "runtime/debug" "strconv" @@ -14,6 +13,11 @@ import ( "time" log "github.com/Sirupsen/logrus" +<<<<<<< e73846650fde9b0955aa35e237100ec552af47fb +======= + "github.com/arduino/arduino-create-agent/tools" + "github.com/carlescere/scheduler" +>>>>>>> Move initialization of tools in package tools "github.com/gin-gonic/gin" "github.com/itsjamie/gin-cors" "github.com/kardianos/osext" @@ -39,7 +43,6 @@ var ( appName = flag.String("appName", "", "") genCert = flag.Bool("generateCert", false, "") globalToolsMap = make(map[string]string) - tempToolsPath = createToolsDir() port string portSSL string origins = flag.String("origins", "", "Allowed origin list for CORS") @@ -60,11 +63,6 @@ func (u *logWriter) Write(p []byte) (n int, err error) { var logger_ws logWriter -func createToolsDir() string { - usr, _ := user.Current() - return usr.HomeDir + "/.arduino-create" -} - func homeHandler(c *gin.Context) { homeTemplate.Execute(c.Writer, c.Request.Host) } @@ -92,8 +90,7 @@ func main() { src, _ := osext.Executable() dest := filepath.Dir(src) - os.Mkdir(tempToolsPath, 0777) - hideFile(tempToolsPath) + tools.CreateDir() if embedded_autoextract { // save the config.ini (if it exists) diff --git a/seriallist_darwin.go b/seriallist_darwin.go index be18cf2c3..940471aff 100644 --- a/seriallist_darwin.go +++ b/seriallist_darwin.go @@ -63,8 +63,5 @@ func associateVidPidWithPort(ports []OsSerialPort) []OsSerialPort { return ports } -func hideFile(path string) { -} - func tellCommandNotToSpawnShell(_ *exec.Cmd) { } diff --git a/seriallist_linux.go b/seriallist_linux.go index b6e3ec559..590d6ea39 100755 --- a/seriallist_linux.go +++ b/seriallist_linux.go @@ -35,8 +35,5 @@ func associateVidPidWithPort(ports []OsSerialPort) []OsSerialPort { return ports } -func hideFile(path string) { -} - func tellCommandNotToSpawnShell(_ *exec.Cmd) { } diff --git a/seriallist_windows.go b/seriallist_windows.go index 1e7329f0a..8c46b923a 100644 --- a/seriallist_windows.go +++ b/seriallist_windows.go @@ -1,15 +1,16 @@ package main import ( - log "github.com/Sirupsen/logrus" - "github.com/mattn/go-ole" - "github.com/mattn/go-ole/oleutil" "os" "os/exec" "regexp" "strings" "sync" "syscall" + + log "github.com/Sirupsen/logrus" + "github.com/mattn/go-ole" + "github.com/mattn/go-ole/oleutil" ) var ( @@ -62,13 +63,6 @@ func getListSynchronously() { } -func hideFile(path string) { - cpath, cpathErr := syscall.UTF16PtrFromString(path) - if cpathErr != nil { - } - syscall.SetFileAttributes(cpath, syscall.FILE_ATTRIBUTE_HIDDEN) -} - func getListViaWmiPnpEntity() ([]OsSerialPort, os.SyscallError) { //log.Println("Doing getListViaWmiPnpEntity()") diff --git a/tools/hidefile_darwin.go b/tools/hidefile_darwin.go new file mode 100644 index 000000000..ea3ec0b66 --- /dev/null +++ b/tools/hidefile_darwin.go @@ -0,0 +1,5 @@ +package tools + +func hideFile(path string) { + +} diff --git a/tools/hidefile_linux.go b/tools/hidefile_linux.go new file mode 100644 index 000000000..ea3ec0b66 --- /dev/null +++ b/tools/hidefile_linux.go @@ -0,0 +1,5 @@ +package tools + +func hideFile(path string) { + +} diff --git a/tools/hidefile_windows.go b/tools/hidefile_windows.go new file mode 100644 index 000000000..64fab3e61 --- /dev/null +++ b/tools/hidefile_windows.go @@ -0,0 +1,10 @@ +package tools + +import "syscall" + +func hideFile(path string) { + cpath, cpathErr := syscall.UTF16PtrFromString(path) + if cpathErr != nil { + } + syscall.SetFileAttributes(cpath, syscall.FILE_ATTRIBUTE_HIDDEN) +} diff --git a/tools/tools.go b/tools/tools.go new file mode 100644 index 000000000..3f53fe1dc --- /dev/null +++ b/tools/tools.go @@ -0,0 +1,24 @@ +package tools + +import ( + "os" + "os/user" +) + +func dir() string { + usr, _ := user.Current() + return usr.HomeDir + "/.arduino-create" +} + +// CreateDir creates the directory where the tools will be stored +func CreateDir() { + directory := dir() + os.Mkdir(directory, 0777) + hideFile(directory) +} + +// Download will parse the index at the indexURL for the tool to download +func Download(name, indexURL string) { + if _, err := os.Stat(dir() + "/" + name); err != nil { + } +} From c00e74039908772356ce8e16cdfaa04ddc8b0191 Mon Sep 17 00:00:00 2001 From: Matteo Suppo Date: Tue, 1 Mar 2016 17:37:31 +0100 Subject: [PATCH 3/7] Move the function to download tools in a package Download from http://downloads.arduino.cc/packages/package_index.json Update the Installed Map at startup --- docs/tools.md | 64 +++++++++++ download.go | 124 --------------------- hub.go | 16 ++- main.go | 34 +++--- programmer.go | 24 ++-- tools/download.go | 275 ++++++++++++++++++++++++++++++++++++++++++++++ tools/tools.go | 89 +++++++++++++-- 7 files changed, 457 insertions(+), 169 deletions(-) create mode 100644 docs/tools.md delete mode 100644 download.go create mode 100644 tools/download.go diff --git a/docs/tools.md b/docs/tools.md new file mode 100644 index 000000000..f6ffd2a09 --- /dev/null +++ b/docs/tools.md @@ -0,0 +1,64 @@ +# tools +-- + import "github.com/arduino/arduino-create-agent/tools" + + +## Usage + +#### type Tools + +```go +type Tools struct { + Directory string + IndexURL string + Logger log.StdLogger +} +``` + +Tools handle the tools necessary for an upload on a board. It provides a means +to download a tool from the arduino servers. + +- *Directory* contains the location where the tools are downloaded. +- *IndexURL* contains the url where the tools description is contained. +- *Logger* is a StdLogger used for reporting debug and info messages +- *installed* contains a map of the tools and their exact location + +Usage: You have to instantiate the struct by passing it the required parameters: + + _tools := tools.Tools{ + Directory: "/home/user/.arduino-create", + IndexURL: "http://downloads.arduino.cc/packages/package_index.json" + Logger: log.Logger + } + +#### func (*Tools) Download + +```go +func (t *Tools) Download(name, version, behaviour string) error +``` +Download will parse the index at the indexURL for the tool to download. It will +extract it in a folder in .arduino-create, and it will update the Installed map. + +name contains the name of the tool. version contains the version of the tool. +behaviour contains the strategy to use when there is already a tool installed + +If version is "latest" it will always download the latest version (regardless of +the value of behaviour) + +If version is not "latest" and behaviour is "replace", it will download the +version again. If instead behaviour is "keep" it will not download the version +if it already exists. + +#### func (*Tools) GetLocation + +```go +func (t *Tools) GetLocation(command string) (string, error) +``` +GetLocation extracts the toolname from a command like + +#### func (*Tools) Init + +```go +func (t *Tools) Init() +``` +Init creates the Installed map and populates it from a file in .arduino-create diff --git a/download.go b/download.go deleted file mode 100644 index ecce08aac..000000000 --- a/download.go +++ /dev/null @@ -1,124 +0,0 @@ -// download.go -package main - -import ( - "encoding/json" - "errors" - log "github.com/Sirupsen/logrus" - "io" - "io/ioutil" - "net/http" - "os" - "path/filepath" - "runtime" - "strings" -) - -func saveFileonTempDir(filename string, sketch io.Reader) (path string, err error) { - // create tmp dir - tmpdir, err := ioutil.TempDir("", "arduino-create-agent") - if err != nil { - return "", errors.New("Could not create temp directory to store downloaded file. Do you have permissions?") - } - - filename, _ = filepath.Abs(tmpdir + "/" + filename) - - output, err := os.Create(filename) - if err != nil { - log.Println("Error while creating", filename, "-", err) - return filename, err - } - defer output.Close() - - n, err := io.Copy(output, sketch) - if err != nil { - log.Println("Error while copying", err) - return filename, err - } - - log.Println(n, "bytes saved") - - return filename, nil - -} - -func downloadFromUrl(url string) (filename string, err error) { - - // clean up url - // remove newlines and space at end - url = strings.TrimSpace(url) - - // create tmp dir - tmpdir, err := ioutil.TempDir("", "arduino-create-agent") - if err != nil { - return "", errors.New("Could not create temp directory to store downloaded file. Do you have permissions?") - } - tokens := strings.Split(url, "/") - filePrefix := tokens[len(tokens)-1] - log.Println("The filePrefix is", filePrefix) - - fileName, _ := filepath.Abs(tmpdir + "/" + filePrefix) - log.Println("Downloading", url, "to", fileName) - - // TODO: check file existence first with io.IsExist - output, err := os.Create(fileName) - if err != nil { - log.Println("Error while creating", fileName, "-", err) - return fileName, err - } - defer output.Close() - - response, err := http.Get(url) - if err != nil { - log.Println("Error while downloading", url, "-", err) - return fileName, err - } - defer response.Body.Close() - - n, err := io.Copy(output, response.Body) - if err != nil { - log.Println("Error while downloading", url, "-", err) - return fileName, err - } - - log.Println(n, "bytes downloaded.") - - return fileName, nil -} - -func spDownloadTool(name string, url string) { - - if _, err := os.Stat(tempToolsPath + "/" + name); err != nil { - - fileName, err := downloadFromUrl(url + "/" + name + "-" + runtime.GOOS + "-" + runtime.GOARCH + ".zip") - if err != nil { - log.Error("Could not download flashing tools!") - mapD := map[string]string{"DownloadStatus": "Error", "Msg": err.Error()} - mapB, _ := json.Marshal(mapD) - h.broadcastSys <- mapB - return - } - err = UnzipWrapper(fileName, tempToolsPath) - if err != nil { - log.Error("Could not unzip flashing tools!") - mapD := map[string]string{"DownloadStatus": "Error", "Msg": err.Error()} - mapB, _ := json.Marshal(mapD) - h.broadcastSys <- mapB - return - } - } else { - log.Info("Tool already present, skipping download") - } - - folders, _ := ioutil.ReadDir(tempToolsPath) - for _, f := range folders { - globalToolsMap["{runtime.tools."+f.Name()+".path}"] = filepath.ToSlash(tempToolsPath + "/" + f.Name()) - } - - log.Info("Map Updated") - mapD := map[string]string{"DownloadStatus": "Success", "Msg": "Map Updated"} - mapB, _ := json.Marshal(mapD) - h.broadcastSys <- mapB - return - -} diff --git a/hub.go b/hub.go index 968586688..98bc7cd1c 100755 --- a/hub.go +++ b/hub.go @@ -4,7 +4,6 @@ import ( "fmt" log "github.com/Sirupsen/logrus" - "github.com/arduino/arduino-create-agent/tools" "github.com/kardianos/osext" //"os" "os/exec" @@ -192,8 +191,19 @@ func checkCmd(m []byte) { go spList(true) } else if strings.HasPrefix(sl, "downloadtool") { args := strings.Split(s, " ") - if len(args) > 2 { - go tools.Download(args[1], args[2]) + if len(args) > 1 { + go func() { + err := Tools.Download(args[1], "latest", "replace") + if err != nil { + mapD := map[string]string{"DownloadStatus": "Error", "Msg": err.Error()} + mapB, _ := json.Marshal(mapD) + h.broadcastSys <- mapB + } else { + mapD := map[string]string{"DownloadStatus": "Success", "Msg": "Map Updated"} + mapB, _ := json.Marshal(mapD) + h.broadcastSys <- mapB + } + }() } } else if strings.HasPrefix(sl, "bufferalgorithm") { go spBufferAlgorithms() diff --git a/main.go b/main.go index 6d7322335..7fc24b8dc 100755 --- a/main.go +++ b/main.go @@ -13,11 +13,7 @@ import ( "time" log "github.com/Sirupsen/logrus" -<<<<<<< e73846650fde9b0955aa35e237100ec552af47fb -======= "github.com/arduino/arduino-create-agent/tools" - "github.com/carlescere/scheduler" ->>>>>>> Move initialization of tools in package tools "github.com/gin-gonic/gin" "github.com/itsjamie/gin-cors" "github.com/kardianos/osext" @@ -38,16 +34,16 @@ var ( gcType = flag.String("gc", "std", "Type of garbage collection. std = Normal garbage collection allowing system to decide (this has been known to cause a stop the world in the middle of a CNC job which can cause lost responses from the CNC controller and thus stalled jobs. use max instead to solve.), off = let memory grow unbounded (you have to send in the gc command manually to garbage collect or you will run out of RAM eventually), max = Force garbage collection on each recv or send on a serial port (this minimizes stop the world events and thus lost serial responses, but increases CPU usage)") logDump = flag.String("log", "off", "off = (default)") // hostname. allow user to override, otherwise we look it up - hostname = flag.String("hostname", "unknown-hostname", "Override the hostname we get from the OS") - updateUrl = flag.String("updateUrl", "", "") - appName = flag.String("appName", "", "") - genCert = flag.Bool("generateCert", false, "") - globalToolsMap = make(map[string]string) - port string - portSSL string - origins = flag.String("origins", "", "Allowed origin list for CORS") - signatureKey = flag.String("signatureKey", "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvc0yZr1yUSen7qmE3cxF\nIE12rCksDnqR+Hp7o0nGi9123eCSFcJ7CkIRC8F+8JMhgI3zNqn4cUEn47I3RKD1\nZChPUCMiJCvbLbloxfdJrUi7gcSgUXrlKQStOKF5Iz7xv1M4XOP3JtjXLGo3EnJ1\npFgdWTOyoSrA8/w1rck4c/ISXZSinVAggPxmLwVEAAln6Itj6giIZHKvA2fL2o8z\nCeK057Lu8X6u2CG8tRWSQzVoKIQw/PKK6CNXCAy8vo4EkXudRutnEYHEJlPkVgPn\n2qP06GI+I+9zKE37iqj0k1/wFaCVXHXIvn06YrmjQw6I0dDj/60Wvi500FuRVpn9\ntwIDAQAB\n-----END PUBLIC KEY-----", "Pem-encoded public key to verify signed commandlines") - address = flag.String("address", "127.0.0.1", "The address where to listen. Defaults to localhost") + hostname = flag.String("hostname", "unknown-hostname", "Override the hostname we get from the OS") + updateUrl = flag.String("updateUrl", "", "") + appName = flag.String("appName", "", "") + genCert = flag.Bool("generateCert", false, "") + port string + portSSL string + origins = flag.String("origins", "", "Allowed origin list for CORS") + address = flag.String("address", "127.0.0.1", "The address where to listen. Defaults to localhost") + signatureKey = flag.String("signatureKey", "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvc0yZr1yUSen7qmE3cxF\nIE12rCksDnqR+Hp7o0nGi9123eCSFcJ7CkIRC8F+8JMhgI3zNqn4cUEn47I3RKD1\nZChPUCMiJCvbLbloxfdJrUi7gcSgUXrlKQStOKF5Iz7xv1M4XOP3JtjXLGo3EnJ1\npFgdWTOyoSrA8/w1rck4c/ISXZSinVAggPxmLwVEAAln6Itj6giIZHKvA2fL2o8z\nCeK057Lu8X6u2CG8tRWSQzVoKIQw/PKK6CNXCAy8vo4EkXudRutnEYHEJlPkVgPn\n2qP06GI+I+9zKE37iqj0k1/wFaCVXHXIvn06YrmjQw6I0dDj/60Wvi500FuRVpn9\ntwIDAQAB\n-----END PUBLIC KEY-----", "Pem-encoded public key to verify signed commandlines") + Tools tools.Tools ) type NullWriter int @@ -90,13 +86,19 @@ func main() { src, _ := osext.Executable() dest := filepath.Dir(src) - tools.CreateDir() + // Instantiate Tools + Tools = tools.Tools{ + Directory: "/home/user/.arduino-create", + IndexURL: "http://downloads.arduino.cc/packages/package_index.json", + Logger: log.New(), + } + Tools.Init() if embedded_autoextract { // save the config.ini (if it exists) if _, err := os.Stat(dest + "/" + *configIni); os.IsNotExist(err) { log.Println("First run, unzipping self") - err := Unzip(src, dest) + err := utilities.Unzip(src, dest) log.Println("Self extraction, err:", err) } diff --git a/programmer.go b/programmer.go index 36df71b6f..4ae9107e5 100644 --- a/programmer.go +++ b/programmer.go @@ -21,7 +21,6 @@ import ( "github.com/facchinm/go-serial" "github.com/mattn/go-shellwords" "github.com/sfreiberg/simplessh" - "github.com/xrash/smetrics" ) var compiling = false @@ -232,23 +231,16 @@ func spProgramLocal(portname string, boardname string, filePath string, commandl var runtimeRe = regexp.MustCompile("\\{(.*?)\\}") runtimeVars := runtimeRe.FindAllString(commandline, -1) - fmt.Println(runtimeVars) - for _, element := range runtimeVars { - // use string similarity to resolve a runtime var with a "similar" map element - if globalToolsMap[element] == "" { - max_similarity := 0.0 - for i, candidate := range globalToolsMap { - similarity := smetrics.Jaro(element, i) - if similarity > 0.8 && similarity > max_similarity { - max_similarity = similarity - globalToolsMap[element] = candidate - } - } + location, err := Tools.GetLocation(element) + if err != nil { + log.Printf("Command finished with error: %v", err) + mapD := map[string]string{"ProgrammerStatus": "Error", "Msg": "Could not find the upload tool"} + mapB, _ := json.Marshal(mapD) + h.broadcastSys <- mapB } - - commandline = strings.Replace(commandline, element, globalToolsMap[element], 1) + commandline = strings.Replace(commandline, element, location, 1) } z, _ := shellwords.Parse(commandline) @@ -292,7 +284,6 @@ func spProgramRW(portname string, boardname string, filePath string, commandline var oscmd *exec.Cmd func spHandlerProgram(flasher string, cmdString []string) error { - // if runtime.GOOS == "darwin" { // sh, _ := exec.LookPath("sh") // // prepend the flasher to run it via sh @@ -392,7 +383,6 @@ func formatCmdline(cmdline string, boardOptions map[string]string) (string, bool } } } - log.Println(cmdline) return cmdline, true } diff --git a/tools/download.go b/tools/download.go new file mode 100644 index 000000000..b8dbf20fb --- /dev/null +++ b/tools/download.go @@ -0,0 +1,275 @@ +package tools + +import ( + "archive/tar" + "bytes" + "compress/bzip2" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "io" + "io/ioutil" + "net/http" + "os" + "path" + "runtime" + + "github.com/arduino/arduino-create-agent/utilities" + "github.com/pivotal-golang/archiver/extractor" +) + +type system struct { + Host string `json:"host"` + URL string `json:"url"` + Name string `json:"archiveFileName"` + CheckSum string `json:"checksum"` +} + +type tool struct { + Name string `json:"name"` + Version string `json:"version"` + Systems []system `json:"systems"` + url string + destination string +} + +type index struct { + Packages []struct { + Name string `json:"name"` + Tools []tool `json:"tools"` + } `json:"packages"` +} + +var systems = map[string]string{ + "linuxamd64": "x86_64-linux-gnu", + "linux386": "i686-linux-gnu", + "darwinamd64": "i386-apple-darwin11", + "windows386": "i686-mingw32", +} + +// Download will parse the index at the indexURL for the tool to download. +// It will extract it in a folder in .arduino-create, and it will update the +// Installed map. +// +// name contains the name of the tool. +// version contains the version of the tool. +// behaviour contains the strategy to use when there is already a tool installed +// +// If version is "latest" it will always download the latest version (regardless +// of the value of behaviour) +// +// If version is not "latest" and behaviour is "replace", it will download the +// version again. If instead behaviour is "keep" it will not download the version +// if it already exists. +func (t *Tools) Download(name, version, behaviour string) error { + var key string + if version == "latest" { + key = name + } else { + key = name + "-" + version + } + + // Check if it already exists + if version != "latest" && behaviour == "keep" { + if _, ok := t.installed[key]; ok { + t.Logger.Println("The tool is already present on the system") + return nil + } + } + // Fetch the index + resp, err := http.Get(t.IndexURL) + if err != nil { + return err + } + defer resp.Body.Close() + + // Read the body + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + var data index + json.Unmarshal(body, &data) + + // Find the tool by name + correctTool := findTool(name, version, data) + + if correctTool.Name == "" { + return errors.New("We couldn't find a tool with the name " + name + " and version " + version) + } + + // Find the url based on system + var correctSystem system + + for _, s := range correctTool.Systems { + if s.Host == systems[runtime.GOOS+runtime.GOARCH] { + correctSystem = s + } + } + + // Download the tool + t.Logger.Println("Downloading tool " + name + " from " + correctSystem.URL) + resp, err = http.Get(correctSystem.URL) + if err != nil { + return err + } + defer resp.Body.Close() + + // Read the body + body, err = ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + // Checksum + checksum := sha256.Sum256(body) + checkSumString := "SHA-256:" + hex.EncodeToString(checksum[:sha256.Size]) + + if checkSumString != correctSystem.CheckSum { + return errors.New("Checksum doesn't match") + } + + // Decompress + t.Logger.Println("Unpacking tool " + name) + + location := path.Join(dir(), name, version) + err = os.RemoveAll(location) + + if err != nil { + return err + } + + switch path.Ext(correctSystem.URL) { + case ".zip": + location, err = extractZip(body, location) + case ".bz2": + location, err = extractBz2(body, location) + default: + return errors.New("Unknown extension for file " + correctSystem.URL) + } + + if err != nil { + return err + } + + // Ensure that the files are executable + t.Logger.Println("Ensure that the files are executable") + err = makeExecutable(location) + if err != nil { + return err + } + + // Update the tool map + t.Logger.Println("Updating map with location " + location) + + t.installed[key] = location + return t.writeMap() +} + +func findTool(name, version string, data index) tool { + var correctTool tool + + for _, p := range data.Packages { + for _, t := range p.Tools { + if version != "latest" { + if t.Name == name && t.Version == version { + correctTool = t + } + } else { + // Find latest + if t.Name == name && t.Version > correctTool.Version { + correctTool = t + } + } + } + } + return correctTool +} + +func extractZip(body []byte, location string) (string, error) { + path, err := utilities.SaveFileonTempDir("tooldownloaded.zip", bytes.NewReader(body)) + if err != nil { + return "", err + } + + e := extractor.NewZip() + err = e.Extract(path, location) + if err != nil { + return "", err + } + return "", nil +} + +func extractBz2(body []byte, location string) (string, error) { + tarFile := bzip2.NewReader(bytes.NewReader(body)) + tarReader := tar.NewReader(tarFile) + + var subfolder string + + i := 0 + for { + header, err := tarReader.Next() + + if err == io.EOF { + break + } + + if err != nil { + return "", err + } + + filePath := path.Join(location, header.Name) + + // We get the name of the subfolder + if i == 0 { + subfolder = filePath + } + + switch header.Typeflag { + case tar.TypeDir: + err = os.MkdirAll(filePath, os.FileMode(header.Mode)) + case tar.TypeReg: + f, err := os.Create(filePath) + if err != nil { + break + } + defer f.Close() + _, err = io.Copy(f, tarReader) + case tar.TypeRegA: + f, err := os.Create(filePath) + if err != nil { + break + } + defer f.Close() + _, err = io.Copy(f, tarReader) + case tar.TypeSymlink: + err = os.Symlink(header.Linkname, filePath) + default: + err = errors.New("Unknown header in tar.bz2 file") + } + + if err != nil { + return "", err + } + + i++ + } + return subfolder, nil +} + +func makeExecutable(location string) error { + location = path.Join(location, "bin") + files, err := ioutil.ReadDir(location) + if err != nil { + return err + } + + for _, file := range files { + err = os.Chmod(path.Join(location, file.Name()), 0755) + if err != nil { + return err + } + } + return nil +} diff --git a/tools/tools.go b/tools/tools.go index 3f53fe1dc..f9c7ab7a3 100644 --- a/tools/tools.go +++ b/tools/tools.go @@ -1,24 +1,95 @@ package tools import ( + "encoding/json" + "io/ioutil" "os" "os/user" + "path" + "strings" + + log "github.com/Sirupsen/logrus" + "github.com/xrash/smetrics" ) +// Tools handle the tools necessary for an upload on a board. +// It provides a means to download a tool from the arduino servers. +// +// - *Directory* contains the location where the tools are downloaded. +// - *IndexURL* contains the url where the tools description is contained. +// - *Logger* is a StdLogger used for reporting debug and info messages +// - *installed* contains a map of the tools and their exact location +// +// Usage: +// You have to instantiate the struct by passing it the required parameters: +// _tools := tools.Tools{ +// Directory: "/home/user/.arduino-create", +// IndexURL: "http://downloads.arduino.cc/packages/package_index.json" +// Logger: log.Logger +// } +type Tools struct { + Directory string + IndexURL string + Logger log.StdLogger + installed map[string]string +} + +// Init creates the Installed map and populates it from a file in .arduino-create +func (t *Tools) Init() { + createDir(t.Directory) + t.installed = make(map[string]string) + t.readMap() + t.Logger.Println(t.installed) +} + +// GetLocation extracts the toolname from a command like +func (t *Tools) GetLocation(command string) (string, error) { + command = strings.Replace(command, "{runtime.tools.", "", 1) + command = strings.Replace(command, ".path}", "", 1) + + var location string + var ok bool + + // use string similarity to resolve a runtime var with a "similar" map element + if location, ok = t.installed[command]; !ok { + maxSimilarity := 0.0 + for i, candidate := range t.installed { + similarity := smetrics.Jaro(command, i) + if similarity > 0.8 && similarity > maxSimilarity { + maxSimilarity = similarity + location = candidate + } + } + } + + return location, nil +} + +func (t *Tools) writeMap() error { + b, err := json.Marshal(t.installed) + if err != nil { + return err + } + filePath := path.Join(dir(), "installed.json") + return ioutil.WriteFile(filePath, b, 0644) +} + +func (t *Tools) readMap() error { + filePath := path.Join(dir(), "installed.json") + b, err := ioutil.ReadFile(filePath) + if err != nil { + return err + } + return json.Unmarshal(b, &t.installed) +} + func dir() string { usr, _ := user.Current() return usr.HomeDir + "/.arduino-create" } -// CreateDir creates the directory where the tools will be stored -func CreateDir() { - directory := dir() +// createDir creates the directory where the tools will be stored +func createDir(directory string) { os.Mkdir(directory, 0777) hideFile(directory) } - -// Download will parse the index at the indexURL for the tool to download -func Download(name, indexURL string) { - if _, err := os.Stat(dir() + "/" + name); err != nil { - } -} From c9c32a9fc4318e1f67c979cc254c602f213faffb Mon Sep 17 00:00:00 2001 From: Matteo Suppo Date: Thu, 3 Mar 2016 11:51:36 +0100 Subject: [PATCH 4/7] Make the indexurl configurable --- main.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index 7fc24b8dc..a93a72c2a 100755 --- a/main.go +++ b/main.go @@ -6,6 +6,7 @@ package main import ( "flag" "os" + "os/user" "path/filepath" "runtime/debug" "strconv" @@ -44,6 +45,7 @@ var ( address = flag.String("address", "127.0.0.1", "The address where to listen. Defaults to localhost") signatureKey = flag.String("signatureKey", "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvc0yZr1yUSen7qmE3cxF\nIE12rCksDnqR+Hp7o0nGi9123eCSFcJ7CkIRC8F+8JMhgI3zNqn4cUEn47I3RKD1\nZChPUCMiJCvbLbloxfdJrUi7gcSgUXrlKQStOKF5Iz7xv1M4XOP3JtjXLGo3EnJ1\npFgdWTOyoSrA8/w1rck4c/ISXZSinVAggPxmLwVEAAln6Itj6giIZHKvA2fL2o8z\nCeK057Lu8X6u2CG8tRWSQzVoKIQw/PKK6CNXCAy8vo4EkXudRutnEYHEJlPkVgPn\n2qP06GI+I+9zKE37iqj0k1/wFaCVXHXIvn06YrmjQw6I0dDj/60Wvi500FuRVpn9\ntwIDAQAB\n-----END PUBLIC KEY-----", "Pem-encoded public key to verify signed commandlines") Tools tools.Tools + indexURL = flag.String("indexURL", "http://downloads.arduino.cc/packages/package_index.json", "The address from where to download the index json containing the location of upload tools") ) type NullWriter int @@ -87,9 +89,11 @@ func main() { dest := filepath.Dir(src) // Instantiate Tools + usr, _ := user.Current() + directory := usr.HomeDir + "/.arduino-create" Tools = tools.Tools{ - Directory: "/home/user/.arduino-create", - IndexURL: "http://downloads.arduino.cc/packages/package_index.json", + Directory: directory, + IndexURL: *indexURL, Logger: log.New(), } Tools.Init() From 29f28bc260969dece7b69cdbb248c7144ad11958 Mon Sep 17 00:00:00 2001 From: Matteo Suppo Date: Thu, 3 Mar 2016 12:00:08 +0100 Subject: [PATCH 5/7] Fix a typo for darwin --- seriallist_darwin.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/seriallist_darwin.go b/seriallist_darwin.go index 940471aff..29ae510ec 100644 --- a/seriallist_darwin.go +++ b/seriallist_darwin.go @@ -28,12 +28,12 @@ func associateVidPidWithPort(ports []OsSerialPort) []OsSerialPort { usbcmd := exec.Command("system_profiler", "SPUSBDataType") grepcmd := exec.Command("grep", "Location ID: 0x"+port_hash[:len(port_hash)-1], "-B6") - cmdOutput, _ := pipe_commands(usbcmd, grepcmd) + cmdOutput, _ := utillities.PipeCommands(usbcmd, grepcmd) if len(cmdOutput) == 0 { usbcmd = exec.Command("system_profiler", "SPUSBDataType") grepcmd = exec.Command("grep" /*"Serial Number: "+*/, strings.Trim(port_hash, "0"), "-B3", "-A3") - cmdOutput, _ = pipe_commands(usbcmd, grepcmd) + cmdOutput, _ = utillities.PipeCommands(usbcmd, grepcmd) } if len(cmdOutput) == 0 { From 7ac5aa0652de9b628da7a58e403896984df8b2b4 Mon Sep 17 00:00:00 2001 From: Matteo Suppo Date: Thu, 3 Mar 2016 12:04:27 +0100 Subject: [PATCH 6/7] Fix stupid typo again --- seriallist_darwin.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/seriallist_darwin.go b/seriallist_darwin.go index 29ae510ec..56cc20f53 100644 --- a/seriallist_darwin.go +++ b/seriallist_darwin.go @@ -3,6 +3,8 @@ package main import ( "os/exec" "strings" + + "github.com/arduino/arduino-create-agent/utilities" ) // execute system_profiler SPUSBDataType | grep "Vendor ID: 0x2341" -A5 -B2 @@ -28,12 +30,12 @@ func associateVidPidWithPort(ports []OsSerialPort) []OsSerialPort { usbcmd := exec.Command("system_profiler", "SPUSBDataType") grepcmd := exec.Command("grep", "Location ID: 0x"+port_hash[:len(port_hash)-1], "-B6") - cmdOutput, _ := utillities.PipeCommands(usbcmd, grepcmd) + cmdOutput, _ := utilities.PipeCommands(usbcmd, grepcmd) if len(cmdOutput) == 0 { usbcmd = exec.Command("system_profiler", "SPUSBDataType") grepcmd = exec.Command("grep" /*"Serial Number: "+*/, strings.Trim(port_hash, "0"), "-B3", "-A3") - cmdOutput, _ = utillities.PipeCommands(usbcmd, grepcmd) + cmdOutput, _ = utilities.PipeCommands(usbcmd, grepcmd) } if len(cmdOutput) == 0 { From 473fdd6e2fa369c5a112fd98b42cbb069a4ac757 Mon Sep 17 00:00:00 2001 From: Matteo Suppo Date: Thu, 21 Apr 2016 16:30:31 +0200 Subject: [PATCH 7/7] Foxed conflicts in main --- main.go | 1 + utilities/utilities.go | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/main.go b/main.go index a93a72c2a..7cfebcf38 100755 --- a/main.go +++ b/main.go @@ -15,6 +15,7 @@ import ( log "github.com/Sirupsen/logrus" "github.com/arduino/arduino-create-agent/tools" + "github.com/arduino/arduino-create-agent/utilities" "github.com/gin-gonic/gin" "github.com/itsjamie/gin-cors" "github.com/kardianos/osext" diff --git a/utilities/utilities.go b/utilities/utilities.go index 2334ce295..617d4fafa 100644 --- a/utilities/utilities.go +++ b/utilities/utilities.go @@ -1,12 +1,14 @@ package utilities import ( + "archive/zip" "bytes" "errors" "io" "io/ioutil" "os" "os/exec" + "path" "path/filepath" ) @@ -87,3 +89,40 @@ func call(stack []*exec.Cmd, pipes []*io.PipeWriter) (err error) { } return stack[0].Wait() } + +func Unzip(zippath string, destination string) (err error) { + r, err := zip.OpenReader(zippath) + if err != nil { + return err + } + for _, f := range r.File { + fullname := path.Join(destination, f.Name) + if f.FileInfo().IsDir() { + os.MkdirAll(fullname, f.FileInfo().Mode().Perm()) + } else { + os.MkdirAll(filepath.Dir(fullname), 0755) + perms := f.FileInfo().Mode().Perm() + out, err := os.OpenFile(fullname, os.O_CREATE|os.O_RDWR, perms) + if err != nil { + return err + } + rc, err := f.Open() + if err != nil { + return err + } + _, err = io.CopyN(out, rc, f.FileInfo().Size()) + if err != nil { + return err + } + rc.Close() + out.Close() + + mtime := f.FileInfo().ModTime() + err = os.Chtimes(fullname, mtime, mtime) + if err != nil { + return err + } + } + } + return +}