Only pass sketch sources to ctags

src/arduino.cc/builder/container_add_prototypes.go

@@ -42,10 +42,10 @@ func (s *ContainerAddPrototypes) Run(ctx *types.Context) error {
 	commands := []types.Command{
 		&GCCPreprocRunner{TargetFileName: constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E},
 		&ReadFileAndStoreInContext{Target: &ctx.SourceGccMinusE},
+		&FilterSketchSource{Source: &ctx.SourceGccMinusE},
 		&CTagsTargetFileSaver{Source: &ctx.SourceGccMinusE, TargetFileName: constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E},
 		&ctags.CTagsRunner{},
 		&ctags.CTagsParser{},
-		&CollectCTagsFromSketchFiles{},
 		&ctags.CTagsToPrototypes{},
 		&PrototypesAdder{},
 		&SketchSaver{},

src/arduino.cc/builder/ctags/ctags_parser.go

@@ -229,7 +229,14 @@ func parseTag(row string) *types.CTag {
 	parts := strings.Split(row, "\t")
 
 	tag.FunctionName = parts[0]
-	tag.Filename = parts[1]
+	// This unescapes any backslashes in the filename. These
+	// filenames that ctags outputs originate from the line markers
+	// in the source, as generated by gcc. gcc escapes both
+	// backslashes and double quotes, but ctags ignores any escaping
+	// and just cuts off the filename at the first double quote it
+	// sees. This means any backslashes are still escaped, and need
+	// to be unescape, and any quotes will just break the build.
+	tag.Filename = strings.Replace(parts[1], "\\\\", "\\", -1)
 
 	parts = parts[2:]
 

src/arduino.cc/builder/ctags/ctags_to_prototypes.go

@@ -37,7 +37,7 @@ import (
 type CTagsToPrototypes struct{}
 
 func (s *CTagsToPrototypes) Run(ctx *types.Context) error {
-	tags := ctx.CTagsCollected
+	tags := ctx.CTagsOfPreprocessedSource
 
 	lineWhereToInsertPrototypes := findLineWhereToInsertPrototypes(tags)
 	if lineWhereToInsertPrototypes != -1 {

src/arduino.cc/builder/collect_ctags_from_sketch_files.go → src/arduino.cc/builder/filter_sketch_source.go

@@ -32,31 +32,68 @@ package builder
 import (
 	"arduino.cc/builder/types"
 	"arduino.cc/builder/utils"
+	"strconv"
 	"strings"
 )
 
-type CollectCTagsFromSketchFiles struct{}
+type FilterSketchSource struct {
+	Source         *string
+}
 
-func (s *CollectCTagsFromSketchFiles) Run(ctx *types.Context) error {
-	sketchFileNames := collectSketchFileNamesFrom(ctx.Sketch)
+func (s *FilterSketchSource) Run(ctx *types.Context) error {
+	lines := strings.Split(*s.Source, "\n")
 
-	allCtags := ctx.CTagsOfPreprocessedSource
-	ctagsOfSketch := []*types.CTag{}
-	for _, ctag := range allCtags {
-		if utils.SliceContains(sketchFileNames, strings.Replace(ctag.Filename, "\\\\", "\\", -1)) {
-			ctagsOfSketch = append(ctagsOfSketch, ctag)
-		}
+	fileNames := []string{ctx.Sketch.MainFile.Name}
+	for _, file := range ctx.Sketch.OtherSketchFiles {
+		fileNames = append(fileNames, file.Name)
 	}
 
-	ctx.CTagsCollected = ctagsOfSketch
+	inSketch := false
+	filtered := ""
+
+	for _, line := range lines {
+		filename := parseLineMarker(line)
+		if filename != "" {
+			inSketch = utils.SliceContains(fileNames, filename)
+		}
+
+		if inSketch {
+			filtered += line + "\n"
+		}
+	}
 
+	*s.Source = filtered
 	return nil
 }
 
-func collectSketchFileNamesFrom(sketch *types.Sketch) []string {
-	fileNames := []string{sketch.MainFile.Name}
-	for _, file := range sketch.OtherSketchFiles {
-		fileNames = append(fileNames, file.Name)
+// Parses the given line as a gcc line marker and returns the contained
+// filename.
+func parseLineMarker(line string) string {
+	// A line marker contains the line number and filename and looks like:
+	// # 123 /path/to/file.cpp
+	// It can be followed by zero or more flag number that indicate the
+	// preprocessor state and can be ignored.
+	// For exact details on this format, see:
+	// https://github.com/gcc-mirror/gcc/blob/edd716b6b1caa1a5cb320a8cd7f626f30198e098/gcc/c-family/c-ppoutput.c#L413-L415
+
+	split := strings.SplitN(line, " ", 3)
+	if len(split) < 2 || split[0] != "#" {
+		return ""
+	}
+
+	_, err := strconv.Atoi(split[1])
+	if err != nil {
+		return ""
 	}
-	return fileNames
+
+	// If we get here, we found a # followed by a line number, so
+	// assume this is a line marker and see if the rest of the line
+	// starts with a string containing the filename
+	str, rest, ok := utils.ParseCppString(split[2])
+
+	if ok && (rest == "" || rest[0] == ' ') {
+		return str
+	}
+	return ""
 }
+

src/arduino.cc/builder/prototypes_adder.go

@@ -32,6 +32,7 @@ package builder
 import (
 	"arduino.cc/builder/constants"
 	"arduino.cc/builder/types"
+	"arduino.cc/builder/utils"
 	"fmt"
 	"strconv"
 	"strings"
@@ -87,7 +88,7 @@ func composePrototypeSection(line int, prototypes []*types.Prototype) string {
 	str := joinPrototypes(prototypes)
 	str += "\n#line "
 	str += strconv.Itoa(line)
-	str += " \"" + prototypes[0].File + "\""
+	str += " " + utils.QuoteCppString(prototypes[0].File)
 	str += "\n"
 
 	return str
@@ -99,7 +100,7 @@ func joinPrototypes(prototypes []*types.Prototype) string {
 		if signatureContainsaDefaultArg(proto) {
 			continue
 		}
-		prototypesSlice = append(prototypesSlice, "#line "+strconv.Itoa(proto.Line)+" \""+proto.File+"\"")
+		prototypesSlice = append(prototypesSlice, "#line "+strconv.Itoa(proto.Line)+" "+utils.QuoteCppString(proto.File))
 		prototypeParts := []string{}
 		if proto.Modifiers != "" {
 			prototypeParts = append(prototypeParts, proto.Modifiers)

src/arduino.cc/builder/sketch_source_merger.go

@@ -32,8 +32,8 @@ package builder
 
 import (
 	"arduino.cc/builder/types"
+	"arduino.cc/builder/utils"
 	"regexp"
-	"strings"
 )
 
 type SketchSourceMerger struct{}
@@ -47,7 +47,7 @@ func (s *SketchSourceMerger) Run(ctx *types.Context) error {
 		includeSection += "#include <Arduino.h>\n"
 		lineOffset++
 	}
-	includeSection += "#line 1 \"" + strings.Replace((&sketch.MainFile).Name, "\\", "\\\\", -1) + "\"\n"
+	includeSection += "#line 1 " + utils.QuoteCppString(sketch.MainFile.Name) + "\n"
 	lineOffset++
 	ctx.IncludeSection = includeSection
 
@@ -73,7 +73,7 @@ func sketchIncludesArduinoH(sketch *types.SketchFile) bool {
 }
 
 func addSourceWrappedWithLineDirective(sketch *types.SketchFile) string {
-	source := "#line 1 \"" + strings.Replace(sketch.Name, "\\", "\\\\", -1) + "\"\n"
+	source := "#line 1 " + utils.QuoteCppString(sketch.Name) + "\n"
 	source += sketch.Source
 	source += "\n"
 

src/arduino.cc/builder/test/ctags_to_prototypes_test.go

@@ -38,13 +38,6 @@ import (
 	"testing"
 )
 
-type CollectCtagsFromPreprocSource struct{}
-
-func (*CollectCtagsFromPreprocSource) Run(ctx *types.Context) error {
-	ctx.CTagsCollected = ctx.CTagsOfPreprocessedSource
-	return nil
-}
-
 func TestCTagsToPrototypesShouldListPrototypes(t *testing.T) {
 	ctx := &types.Context{}
 
@@ -55,7 +48,6 @@ func TestCTagsToPrototypesShouldListPrototypes(t *testing.T) {
 
 	commands := []types.Command{
 		&ctags.CTagsParser{},
-		&CollectCtagsFromPreprocSource{},
 		&ctags.CTagsToPrototypes{},
 	}
 
@@ -87,7 +79,6 @@ func TestCTagsToPrototypesShouldListTemplates(t *testing.T) {
 
 	commands := []types.Command{
 		&ctags.CTagsParser{},
-		&CollectCtagsFromPreprocSource{},
 		&ctags.CTagsToPrototypes{},
 	}
 
@@ -117,7 +108,6 @@ func TestCTagsToPrototypesShouldListTemplates2(t *testing.T) {
 
 	commands := []types.Command{
 		&ctags.CTagsParser{},
-		&CollectCtagsFromPreprocSource{},
 		&ctags.CTagsToPrototypes{},
 	}
 
@@ -148,7 +138,6 @@ func TestCTagsToPrototypesShouldDealWithClasses(t *testing.T) {
 
 	commands := []types.Command{
 		&ctags.CTagsParser{},
-		&CollectCtagsFromPreprocSource{},
 		&ctags.CTagsToPrototypes{},
 	}
 
@@ -174,7 +163,6 @@ func TestCTagsToPrototypesShouldDealWithStructs(t *testing.T) {
 
 	commands := []types.Command{
 		&ctags.CTagsParser{},
-		&CollectCtagsFromPreprocSource{},
 		&ctags.CTagsToPrototypes{},
 	}
 
@@ -204,7 +192,6 @@ func TestCTagsToPrototypesShouldDealWithMacros(t *testing.T) {
 
 	commands := []types.Command{
 		&ctags.CTagsParser{},
-		&CollectCtagsFromPreprocSource{},
 		&ctags.CTagsToPrototypes{},
 	}
 
@@ -236,7 +223,6 @@ func TestCTagsToPrototypesShouldDealFunctionWithDifferentSignatures(t *testing.T
 
 	commands := []types.Command{
 		&ctags.CTagsParser{},
-		&CollectCtagsFromPreprocSource{},
 		&ctags.CTagsToPrototypes{},
 	}
 
@@ -264,7 +250,6 @@ func TestCTagsToPrototypesClassMembersAreFilteredOut(t *testing.T) {
 
 	commands := []types.Command{
 		&ctags.CTagsParser{},
-		&CollectCtagsFromPreprocSource{},
 		&ctags.CTagsToPrototypes{},
 	}
 
@@ -293,7 +278,6 @@ func TestCTagsToPrototypesStructWithFunctions(t *testing.T) {
 
 	commands := []types.Command{
 		&ctags.CTagsParser{},
-		&CollectCtagsFromPreprocSource{},
 		&ctags.CTagsToPrototypes{},
 	}
 
@@ -322,7 +306,6 @@ func TestCTagsToPrototypesDefaultArguments(t *testing.T) {
 
 	commands := []types.Command{
 		&ctags.CTagsParser{},
-		&CollectCtagsFromPreprocSource{},
 		&ctags.CTagsToPrototypes{},
 	}
 
@@ -352,7 +335,6 @@ func TestCTagsToPrototypesNamespace(t *testing.T) {
 
 	commands := []types.Command{
 		&ctags.CTagsParser{},
-		&CollectCtagsFromPreprocSource{},
 		&ctags.CTagsToPrototypes{},
 	}
 
@@ -381,7 +363,6 @@ func TestCTagsToPrototypesStatic(t *testing.T) {
 
 	commands := []types.Command{
 		&ctags.CTagsParser{},
-		&CollectCtagsFromPreprocSource{},
 		&ctags.CTagsToPrototypes{},
 	}
 
@@ -412,7 +393,6 @@ func TestCTagsToPrototypesFunctionPointer(t *testing.T) {
 
 	commands := []types.Command{
 		&ctags.CTagsParser{},
-		&CollectCtagsFromPreprocSource{},
 		&ctags.CTagsToPrototypes{},
 	}
 
@@ -442,7 +422,6 @@ func TestCTagsToPrototypesFunctionPointers(t *testing.T) {
 
 	commands := []types.Command{
 		&ctags.CTagsParser{},
-		&CollectCtagsFromPreprocSource{},
 		&ctags.CTagsToPrototypes{},
 	}
 

src/arduino.cc/builder/test/helper.go

@@ -33,22 +33,20 @@ package test
 import (
 	"arduino.cc/builder/constants"
 	"arduino.cc/builder/types"
+	"arduino.cc/builder/utils"
 	"bytes"
 	"fmt"
 	"github.com/go-errors/errors"
 	"github.com/stretchr/testify/assert"
 	"io/ioutil"
 	"path/filepath"
-	"strings"
 	"testing"
 	"text/template"
 )
 
 func LoadAndInterpolate(t *testing.T, filename string, ctx *types.Context) string {
 	funcsMap := template.FuncMap{
-		"EscapeBackSlashes": func(s string) string {
-			return strings.Replace(s, "\\", "\\\\", -1)
-		},
+		"QuoteCppString": utils.QuoteCppString,
 	}
 
 	tpl, err := template.New(filepath.Base(filename)).Funcs(funcsMap).ParseFiles(filename)