Merge pull request #265 from arduino/massi/sketch

Turn Sketch commands into builder API functions

Author: mastrolinux

Diff for: arduino/builder/sketch.go

@@ -0,0 +1,157 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2019 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to modify or
+// otherwise use the software for commercial activities involving the Arduino
+// software without disclosing the source code of your own applications. To purchase
+// a commercial license, send an email to [email protected].
+
+package builder
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/arduino/arduino-cli/arduino/globals"
+	"github.com/arduino/arduino-cli/arduino/sketch"
+
+	"github.com/pkg/errors"
+)
+
+var includesArduinoH = regexp.MustCompile(`(?m)^\s*#\s*include\s*[<\"]Arduino\.h[>\"]`)
+
+// QuoteCppString returns the given string as a quoted string for use with the C
+// preprocessor. This adds double quotes around it and escapes any
+// double quotes and backslashes in the string.
+func QuoteCppString(str string) string {
+	str = strings.Replace(str, "\\", "\\\\", -1)
+	str = strings.Replace(str, "\"", "\\\"", -1)
+	return "\"" + str + "\""
+}
+
+// SaveSketchItemCpp saves a preprocessed .cpp sketch file on disk
+func SaveSketchItemCpp(item *sketch.Item, buildPath string) error {
+
+	sketchName := filepath.Base(item.Path)
+
+	if err := os.MkdirAll(buildPath, os.FileMode(0755)); err != nil {
+		return errors.Wrap(err, "unable to create a folder to save the sketch")
+	}
+
+	destFile := filepath.Join(buildPath, sketchName+".cpp")
+
+	if err := ioutil.WriteFile(destFile, item.Source, os.FileMode(0644)); err != nil {
+		return errors.Wrap(err, "unable to save the sketch on disk")
+	}
+
+	return nil
+}
+
+// LoadSketch collects all the files composing a sketch.
+// The parameter `sketchPath` holds a path pointing to a single sketch file or a sketch folder,
+// the path must be absolute.
+func LoadSketch(sketchPath, buildPath string) (*sketch.Sketch, error) {
+	stat, err := os.Stat(sketchPath)
+	if err != nil {
+		return nil, errors.Wrap(err, "unable to stat Sketch location")
+	}
+
+	var sketchFolder, mainSketchFile string
+
+	// if a sketch folder was passed, save the parent and point sketchPath to the main .ino file
+	if stat.IsDir() {
+		sketchFolder = sketchPath
+		mainSketchFile = filepath.Join(sketchPath, stat.Name()+".ino")
+		// in the case a dir was passed, ensure the main file exists and is readable
+		f, err := os.Open(mainSketchFile)
+		if err != nil {
+			return nil, errors.Wrap(err, "unable to find the main sketch file")
+		}
+		f.Close()
+	} else {
+		sketchFolder = filepath.Dir(sketchPath)
+		mainSketchFile = sketchPath
+	}
+
+	// collect all the sketch files
+	var files []string
+	err = filepath.Walk(sketchFolder, func(path string, info os.FileInfo, err error) error {
+		// ignore hidden files and skip hidden directories
+		if strings.HasPrefix(info.Name(), ".") {
+			if info.IsDir() {
+				return filepath.SkipDir
+			}
+			return nil
+		}
+
+		// skip legacy SCM directories
+		if info.IsDir() && strings.HasPrefix(info.Name(), "CVS") || strings.HasPrefix(info.Name(), "RCS") {
+			return filepath.SkipDir
+		}
+
+		// ignore directory entries
+		if info.IsDir() {
+			return nil
+		}
+
+		// ignore if file extension doesn't match
+		ext := strings.ToLower(filepath.Ext(path))
+		_, isMain := globals.MainFileValidExtensions[ext]
+		_, isAdditional := globals.AdditionalFileValidExtensions[ext]
+		if !(isMain || isAdditional) {
+			return nil
+		}
+
+		// check if file is readable
+		f, err := os.Open(path)
+		if err != nil {
+			return nil
+		}
+		f.Close()
+
+		// collect the file
+		files = append(files, path)
+
+		// done
+		return nil
+	})
+
+	if err != nil {
+		return nil, errors.Wrap(err, "there was an error while collecting the sketch files")
+	}
+
+	return sketch.New(sketchFolder, mainSketchFile, buildPath, files)
+}
+
+// MergeSketchSources merges all the source files included in a sketch
+func MergeSketchSources(sketch *sketch.Sketch) (int, string) {
+	lineOffset := 0
+	mergedSource := ""
+
+	// add Arduino.h inclusion directive if missing
+	if !includesArduinoH.MatchString(sketch.MainFile.GetSourceStr()) {
+		mergedSource += "#include <Arduino.h>\n"
+		lineOffset++
+	}
+
+	mergedSource += "#line 1 " + QuoteCppString(sketch.MainFile.Path) + "\n"
+	mergedSource += sketch.MainFile.GetSourceStr() + "\n"
+	lineOffset++
+
+	for _, item := range sketch.OtherSketchFiles {
+		mergedSource += "#line 1 " + QuoteCppString(item.Path) + "\n"
+		mergedSource += item.GetSourceStr() + "\n"
+	}
+
+	return lineOffset, mergedSource
+}

Diff for: arduino/builder/sketch_test.go

@@ -0,0 +1,120 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2019 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to modify or
+// otherwise use the software for commercial activities involving the Arduino
+// software without disclosing the source code of your own applications. To purchase
+// a commercial license, send an email to [email protected].
+
+package builder_test
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/arduino/arduino-cli/arduino/builder"
+	"github.com/arduino/arduino-cli/arduino/sketch"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSaveSketch(t *testing.T) {
+	sketchName := t.Name() + ".ino"
+	outName := sketchName + ".cpp"
+	sketchFile := filepath.Join("testdata", sketchName)
+	tmp := tmpDirOrDie()
+	defer os.RemoveAll(tmp)
+	source, err := ioutil.ReadFile(sketchFile)
+	if err != nil {
+		t.Fatalf("unable to read golden file %s: %v", sketchFile, err)
+	}
+
+	builder.SaveSketchItemCpp(&sketch.Item{Path: sketchName, Source: source}, tmp)
+
+	out, err := ioutil.ReadFile(filepath.Join(tmp, outName))
+	if err != nil {
+		t.Fatalf("unable to read output file %s: %v", outName, err)
+	}
+
+	assert.Equal(t, source, out)
+}
+
+func TestLoadSketchFolder(t *testing.T) {
+	// pass the path to the sketch folder
+	sketchPath := filepath.Join("testdata", t.Name())
+	mainFilePath := filepath.Join(sketchPath, t.Name()+".ino")
+	s, err := builder.LoadSketch(sketchPath, "")
+	assert.Nil(t, err)
+	assert.NotNil(t, s)
+	assert.Equal(t, mainFilePath, s.MainFile.Path)
+	assert.Equal(t, sketchPath, s.LocationPath)
+	assert.Len(t, s.OtherSketchFiles, 2)
+	assert.Equal(t, "old.pde", filepath.Base(s.OtherSketchFiles[0].Path))
+	assert.Equal(t, "other.ino", filepath.Base(s.OtherSketchFiles[1].Path))
+	assert.Len(t, s.AdditionalFiles, 3)
+	assert.Equal(t, "header.h", filepath.Base(s.AdditionalFiles[0].Path))
+	assert.Equal(t, "s_file.S", filepath.Base(s.AdditionalFiles[1].Path))
+	assert.Equal(t, "helper.h", filepath.Base(s.AdditionalFiles[2].Path))
+
+	// pass the path to the main file
+	sketchPath = mainFilePath
+	s, err = builder.LoadSketch(sketchPath, "")
+	assert.Nil(t, err)
+	assert.NotNil(t, s)
+	assert.Equal(t, mainFilePath, s.MainFile.Path)
+	assert.Len(t, s.OtherSketchFiles, 2)
+	assert.Equal(t, "old.pde", filepath.Base(s.OtherSketchFiles[0].Path))
+	assert.Equal(t, "other.ino", filepath.Base(s.OtherSketchFiles[1].Path))
+	assert.Len(t, s.AdditionalFiles, 3)
+	assert.Equal(t, "header.h", filepath.Base(s.AdditionalFiles[0].Path))
+	assert.Equal(t, "s_file.S", filepath.Base(s.AdditionalFiles[1].Path))
+	assert.Equal(t, "helper.h", filepath.Base(s.AdditionalFiles[2].Path))
+}
+
+func TestLoadSketchFolderWrongMain(t *testing.T) {
+	sketchPath := filepath.Join("testdata", t.Name())
+	_, err := builder.LoadSketch(sketchPath, "")
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "unable to find the main sketch file")
+
+	_, err = builder.LoadSketch("does/not/exist", "")
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "no such file or directory")
+}
+
+func TestMergeSketchSources(t *testing.T) {
+	// borrow the sketch from TestLoadSketchFolder to avoid boilerplate
+	s, err := builder.LoadSketch(filepath.Join("testdata", "TestLoadSketchFolder"), "")
+	assert.Nil(t, err)
+	assert.NotNil(t, s)
+
+	// load expected result
+	mergedPath := filepath.Join("testdata", t.Name()+".txt")
+	mergedBytes, err := ioutil.ReadFile(mergedPath)
+	if err != nil {
+		t.Fatalf("unable to read golden file %s: %v", mergedPath, err)
+	}
+
+	offset, source := builder.MergeSketchSources(s)
+	assert.Equal(t, 2, offset)
+	assert.Equal(t, string(mergedBytes), source)
+}
+
+func TestMergeSketchSourcesArduinoIncluded(t *testing.T) {
+	s, err := builder.LoadSketch(filepath.Join("testdata", t.Name()), "")
+	assert.Nil(t, err)
+	assert.NotNil(t, s)
+
+	// ensure not to include Arduino.h when it's already there
+	_, source := builder.MergeSketchSources(s)
+	assert.Equal(t, 1, strings.Count(source, "<Arduino.h>"))
+}

Diff for: arduino/builder/testdata/TestLoadSketchFolder/.#sketch.ino

@@ -0,0 +1,2 @@
+void setup()
+void loop) }

Diff for: arduino/builder/testdata/TestLoadSketchFolder/TestLoadSketchFolder.ino

@@ -0,0 +1,7 @@
+void setup() {
+
+}
+
+void loop() {
+
+}

Diff for: arduino/builder/testdata/TestLoadSketchFolder/doc.txt

@@ -0,0 +1 @@
+#define FOO "BAR"

Diff for: arduino/builder/testdata/TestLoadSketchFolder/header.h

@@ -0,0 +1,3 @@
+String hello() {
+  return "world";
+}

Diff for: arduino/builder/testdata/TestLoadSketchFolder/old.pde

@@ -0,0 +1,2 @@
+#include <testlib4.h>
+#error "Whattya looking at?"

Diff for: arduino/builder/testdata/TestLoadSketchFolder/other.ino

@@ -0,0 +1,15 @@
+#include <Arduino.h>
+#line 1 "testdata/TestLoadSketchFolder/TestLoadSketchFolder.ino"
+void setup() {
+
+}
+
+void loop() {
+
+}
+#line 1 "testdata/TestLoadSketchFolder/old.pde"
+
+#line 1 "testdata/TestLoadSketchFolder/other.ino"
+String hello() {
+  return "world";
+}

Diff for: arduino/builder/testdata/TestLoadSketchFolder/s_file.S

@@ -0,0 +1,9 @@
+// a comment
+
+   #    include		<Arduino.h>
+
+void setup() {
+}
+
+void loop() {
+}

Diff for: arduino/builder/testdata/TestLoadSketchFolder/src/dont_load_me.ino

@@ -0,0 +1,6 @@
+#include <Bridge.h>
+#include <IRremote.h>
+#include <IRremoteInt.h>
+
+void setup() {}
+void loop() {}

Diff for: arduino/builder/testdata/TestLoadSketchFolder/src/helper.h

@@ -0,0 +1,43 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2019 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to modify or
+// otherwise use the software for commercial activities involving the Arduino
+// software without disclosing the source code of your own applications. To purchase
+// a commercial license, send an email to [email protected].
+
+package globals
+
+var (
+	empty struct{}
+
+	// MainFileValidExtensions lists valid extensions for a sketch file
+	MainFileValidExtensions = map[string]struct{}{
+		".ino": empty,
+		".pde": empty,
+	}
+
+	// AdditionalFileValidExtensions lists any file extension the builder considers as valid
+	AdditionalFileValidExtensions = map[string]struct{}{
+		".h":   empty,
+		".c":   empty,
+		".hpp": empty,
+		".hh":  empty,
+		".cpp": empty,
+		".s":   empty,
+	}
+
+	// SourceFilesValidExtensions lists valid extensions for source files (no headers)
+	SourceFilesValidExtensions = map[string]struct{}{
+		".c":   empty,
+		".cpp": empty,
+		".s":   empty,
+	}
+)