diff --git a/src/arduino.cc/builder/ctags/ctags_has_issues.go b/src/arduino.cc/builder/ctags/ctags_has_issues.go new file mode 100644 index 00000000..7049af64 --- /dev/null +++ b/src/arduino.cc/builder/ctags/ctags_has_issues.go @@ -0,0 +1,310 @@ +/* + * This file is part of Arduino Builder. + * + * Arduino Builder is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As a special exception, you may use this file as part of a free software + * library without restriction. Specifically, if other files instantiate + * templates or use macros or inline functions from this file, or you compile + * this file and link it with other files to produce an executable, this + * file does not by itself cause the resulting executable to be covered by + * the GNU General Public License. This exception does not however + * invalidate any other reasons why the executable file might be covered by + * the GNU General Public License. + * + * Copyright 2015 Arduino LLC (http://www.arduino.cc/) + */ + +package ctags + +import ( + "bufio" + "os" + "strings" + + "arduino.cc/builder/types" +) + +func (p *CTagsParser) FixCLinkageTagsDeclarations(tags []*types.CTag) { + + linesMap := p.FindCLinkageLines(tags) + for i, _ := range tags { + + if sliceContainsInt(linesMap[tags[i].Filename], tags[i].Line) && + !strings.Contains(tags[i].PrototypeModifiers, EXTERN) { + tags[i].PrototypeModifiers = tags[i].PrototypeModifiers + " " + EXTERN + } + } +} + +func sliceContainsInt(s []int, e int) bool { + for _, a := range s { + if a == e { + return true + } + } + return false +} + +func (p *CTagsParser) prototypeAndCodeDontMatch(tag *types.CTag) bool { + if tag.SkipMe { + return true + } + + code := removeSpacesAndTabs(tag.Code) + + if strings.Index(code, ")") == -1 { + // Add to code non-whitespace non-comments tokens until we find a closing round bracket + file, err := os.Open(tag.Filename) + if err == nil { + defer file.Close() + + scanner := bufio.NewScanner(file) + line := 1 + + // skip lines until we get to the start of this tag + for scanner.Scan() && line < tag.Line { + line++ + } + + // read up to 10 lines in search of a closing paren + multilinecomment := false + temp := "" + + code, multilinecomment = removeComments(scanner.Text(), multilinecomment) + for scanner.Scan() && line < (tag.Line+10) && strings.Index(temp, ")") == -1 { + temp, multilinecomment = removeComments(scanner.Text(), multilinecomment) + code += temp + } + } + } + + code = removeSpacesAndTabs(code) + + prototype := removeSpacesAndTabs(tag.Prototype) + prototype = removeTralingSemicolon(prototype) + + // Prototype matches exactly with the code? + ret := strings.Index(code, prototype) + + if ret == -1 { + // If the definition is multiline ctags uses the function name as line number + // Try to match functions in the form + // void + // foo() {} + + // Add to code n non-whitespace non-comments tokens before the code line + + code = removeEverythingAfterClosingRoundBracket(code) + // Get how many characters are "missing" + n := strings.Index(prototype, code) + line := 0 + // Add these characters to "code" string + code, line = getFunctionProtoWithNPreviousCharacters(tag, code, n) + // Check again for perfect matching + ret = strings.Index(code, prototype) + if ret != -1 { + tag.Line = line + } + } + + return ret == -1 +} + +func findTemplateMultiline(tag *types.CTag) string { + code, _ := getFunctionProtoUntilTemplateToken(tag, tag.Code) + return removeEverythingAfterClosingRoundBracket(code) +} + +func removeEverythingAfterClosingRoundBracket(s string) string { + n := strings.Index(s, ")") + return s[0 : n+1] +} + +func getFunctionProtoUntilTemplateToken(tag *types.CTag, code string) (string, int) { + + /* FIXME I'm ugly */ + line := 0 + + file, err := os.Open(tag.Filename) + if err == nil { + defer file.Close() + + scanner := bufio.NewScanner(file) + multilinecomment := false + var textBuffer []string + + // buffer lines until we get to the start of this tag + for scanner.Scan() && line < (tag.Line-1) { + line++ + text := scanner.Text() + textBuffer = append(textBuffer, text) + } + + for line > 0 && !strings.Contains(code, TEMPLATE) { + + line = line - 1 + text := textBuffer[line] + + text, multilinecomment = removeComments(text, multilinecomment) + + code = text + code + } + } + return code, line +} + +func getFunctionProtoWithNPreviousCharacters(tag *types.CTag, code string, n int) (string, int) { + + /* FIXME I'm ugly */ + expectedPrototypeLen := len(code) + n + line := 0 + + file, err := os.Open(tag.Filename) + if err == nil { + defer file.Close() + + scanner := bufio.NewScanner(file) + multilinecomment := false + var textBuffer []string + + // buffer lines until we get to the start of this tag + for scanner.Scan() && line < (tag.Line-1) { + line++ + text := scanner.Text() + textBuffer = append(textBuffer, text) + } + + for line > 0 && len(code) < expectedPrototypeLen { + + line = line - 1 + text := textBuffer[line] + + text, multilinecomment = removeComments(text, multilinecomment) + + code = text + code + code = removeSpacesAndTabs(code) + } + } + return code, line +} + +func removeComments(text string, multilinecomment bool) (string, bool) { + // Remove C++ style comments + if strings.Index(text, "//") != -1 { + text = text[0:strings.Index(text, "//")] + } + + // Remove C style comments + if strings.Index(text, "*/") != -1 { + if strings.Index(text, "/*") != -1 { + // C style comments on the same line + text = text[0:strings.Index(text, "/*")] + text[strings.Index(text, "*/")+1:len(text)-1] + } else { + text = text[strings.Index(text, "*/")+1 : len(text)-1] + multilinecomment = true + } + } + + if multilinecomment { + if strings.Index(text, "/*") != -1 { + text = text[0:strings.Index(text, "/*")] + multilinecomment = false + } else { + text = "" + } + } + return text, multilinecomment +} + +/* This function scans the source files searching for "extern C" context + * It save the line numbers in a map filename -> {lines...} + */ +func (p *CTagsParser) FindCLinkageLines(tags []*types.CTag) map[string][]int { + + lines := make(map[string][]int) + + for _, tag := range tags { + + if lines[tag.Filename] != nil { + break + } + + file, err := os.Open(tag.Filename) + if err == nil { + defer file.Close() + + lines[tag.Filename] = append(lines[tag.Filename], -1) + + scanner := bufio.NewScanner(file) + + // we can't remove the comments otherwise the line number will be wrong + // there are three cases: + // 1 - extern "C" void foo() + // 2 - extern "C" { + // void foo(); + // void bar(); + // } + // 3 - extern "C" + // { + // void foo(); + // void bar(); + // } + // case 1 and 2 can be simply recognized with string matching and indent level count + // case 3 needs specia attention: if the line ONLY contains `extern "C"` string, don't bail out on indent level = 0 + + inScope := false + enteringScope := false + indentLevels := 0 + line := 0 + + externCDecl := removeSpacesAndTabs(EXTERN) + + for scanner.Scan() { + line++ + str := removeSpacesAndTabs(scanner.Text()) + + if len(str) == 0 { + continue + } + + // check if we are on the first non empty line after externCDecl in case 3 + if enteringScope == true { + enteringScope = false + } + + // check if the line contains externCDecl + if strings.Contains(str, externCDecl) { + inScope = true + if len(str) == len(externCDecl) { + // case 3 + enteringScope = true + } + } + if inScope == true { + lines[tag.Filename] = append(lines[tag.Filename], line) + } + indentLevels += strings.Count(str, "{") - strings.Count(str, "}") + + // Bail out if indentLevel is zero and we are not in case 3 + if indentLevels == 0 && enteringScope == false { + inScope = false + } + } + } + + } + return lines +} diff --git a/src/arduino.cc/builder/ctags/ctags_parser.go b/src/arduino.cc/builder/ctags/ctags_parser.go index 79035819..d420b4e0 100644 --- a/src/arduino.cc/builder/ctags/ctags_parser.go +++ b/src/arduino.cc/builder/ctags/ctags_parser.go @@ -30,8 +30,6 @@ package ctags import ( - "bufio" - "os" "strconv" "strings" @@ -72,7 +70,7 @@ func (p *CTagsParser) Parse(ctagsOutput string, mainFile string) []*types.CTag { p.addPrototypes() p.removeDefinedProtypes() p.skipDuplicates() - p.skipTagsWhere(prototypeAndCodeDontMatch) + p.skipTagsWhere(p.prototypeAndCodeDontMatch) return p.tags } @@ -86,14 +84,20 @@ func (p *CTagsParser) addPrototypes() { } func addPrototype(tag *types.CTag) { - if strings.Index(tag.Prototype, TEMPLATE) == 0 || strings.Index(tag.Code, TEMPLATE) == 0 { - code := tag.Code - if strings.Contains(code, "{") { - code = code[:strings.Index(code, "{")] + if strings.Index(tag.Prototype, TEMPLATE) == 0 { + if strings.Index(tag.Code, TEMPLATE) == 0 { + code := tag.Code + if strings.Contains(code, "{") { + code = code[:strings.Index(code, "{")] + } else { + code = code[:strings.LastIndex(code, ")")+1] + } + tag.Prototype = code + ";" } else { - code = code[:strings.LastIndex(code, ")")+1] + //tag.Code is 99% multiline, recreate it + code := findTemplateMultiline(tag) + tag.Prototype = code + ";" } - tag.Prototype = code + ";" return } @@ -101,9 +105,9 @@ func addPrototype(tag *types.CTag) { if strings.Index(tag.Code, STATIC+" ") != -1 { tag.PrototypeModifiers = tag.PrototypeModifiers + " " + STATIC } - if strings.Index(tag.Code, EXTERN+" ") != -1 { - tag.PrototypeModifiers = tag.PrototypeModifiers + " " + EXTERN - } + + // Extern "C" modifier is now added in FixCLinkageTagsDeclarations + tag.PrototypeModifiers = strings.TrimSpace(tag.PrototypeModifiers) } @@ -151,47 +155,6 @@ func (p *CTagsParser) skipTagsWhere(skipFunc skipFuncType) { } } -func prototypeAndCodeDontMatch(tag *types.CTag) bool { - if tag.SkipMe { - return true - } - - code := removeSpacesAndTabs(tag.Code) - - // original code is multi-line, which tags doesn't have - could we find this code in the - // original source file, for purposes of checking here? - if strings.Index(code, ")") == -1 { - file, err := os.Open(tag.Filename) - if err == nil { - defer file.Close() - - scanner := bufio.NewScanner(file) - line := 1 - - // skip lines until we get to the start of this tag - for scanner.Scan() && line < tag.Line { - line++ - } - - // read up to 10 lines in search of a closing paren - newcode := scanner.Text() - for scanner.Scan() && line < (tag.Line+10) && strings.Index(newcode, ")") == -1 { - newcode += scanner.Text() - } - - // don't bother replacing the code text if we haven't found a closing paren - if strings.Index(newcode, ")") != -1 { - code = removeSpacesAndTabs(newcode) - } - } - } - - prototype := removeSpacesAndTabs(tag.Prototype) - prototype = removeTralingSemicolon(prototype) - - return strings.Index(code, prototype) == -1 -} - func removeTralingSemicolon(s string) string { return s[0 : len(s)-1] } diff --git a/src/arduino.cc/builder/ctags_runner.go b/src/arduino.cc/builder/ctags_runner.go index e2b6acbb..2ab75638 100644 --- a/src/arduino.cc/builder/ctags_runner.go +++ b/src/arduino.cc/builder/ctags_runner.go @@ -74,7 +74,9 @@ func (s *CTagsRunner) Run(ctx *types.Context) error { ctx.CTagsOutput = string(sourceBytes) parser := &ctags.CTagsParser{} + ctx.CTagsOfPreprocessedSource = parser.Parse(ctx.CTagsOutput, ctx.Sketch.MainFile.Name) + parser.FixCLinkageTagsDeclarations(ctx.CTagsOfPreprocessedSource) protos, line := parser.GeneratePrototypes() if line != -1 { diff --git a/src/arduino.cc/builder/test/sketch_with_externC_multiline/sketch_with_externC_multiline.ino b/src/arduino.cc/builder/test/sketch_with_externC_multiline/sketch_with_externC_multiline.ino new file mode 100644 index 00000000..e911a4b0 --- /dev/null +++ b/src/arduino.cc/builder/test/sketch_with_externC_multiline/sketch_with_externC_multiline.ino @@ -0,0 +1,34 @@ +void setup() { + // put your setup code here, to run once: + } + + void loop() { + // put your main code here, to run repeatedly: + test2(); + test4(); + test6(); + test7(); + test10(); + } + + extern "C" { + void test2() {} + } + + extern "C" + { + void test4() {} + } + + extern "C" + + { + void test6() {} + } + + // this function should not have C linkage + void test7() {} + + extern "C" void test10() { + + }; \ No newline at end of file diff --git a/src/arduino.cc/builder/test/sketch_with_multiline_prototypes/sketch_with_multiline_prototypes.ino b/src/arduino.cc/builder/test/sketch_with_multiline_prototypes/sketch_with_multiline_prototypes.ino index a4fdf58a..85f7fb6d 100644 --- a/src/arduino.cc/builder/test/sketch_with_multiline_prototypes/sketch_with_multiline_prototypes.ino +++ b/src/arduino.cc/builder/test/sketch_with_multiline_prototypes/sketch_with_multiline_prototypes.ino @@ -1,4 +1,13 @@ -void setup() { myctagstestfunc(1,2,3,4); } +void setup() { + myctagstestfunc(1,2,3,4); + test(); + test3(); + test5(1,2,3); + test7(); + test8(); + test9(42, 42); + test10(0,0,0); +} void myctagstestfunc(int a, int b, @@ -7,3 +16,38 @@ int d) { } void loop() {} +void +test() {} + +void +// comment +test3() {} + +void +test5(int a, + int b, + int c) +{ + +} + +void /* comment */ +test7() {} + +void +/* +multi +line +comment +*/ +test8() {} + + void + /* comment */ + test9(int a, + int b) {} + +void test10(int a, // this + int b, // doesn't + int c // work + ) {} \ No newline at end of file diff --git a/src/arduino.cc/builder/test/sketch_with_multiline_template/sketch_with_multiline_template.ino b/src/arduino.cc/builder/test/sketch_with_multiline_template/sketch_with_multiline_template.ino new file mode 100644 index 00000000..a3ddecdf --- /dev/null +++ b/src/arduino.cc/builder/test/sketch_with_multiline_template/sketch_with_multiline_template.ino @@ -0,0 +1,10 @@ +template< typename T > +T func(T t){ + return t * t; +} + +void setup() { + func( 12.34f ); +} + +void loop() {} diff --git a/src/arduino.cc/builder/test/try_build_of_problematic_sketch_test.go b/src/arduino.cc/builder/test/try_build_of_problematic_sketch_test.go index 160c81c5..f78f20a2 100644 --- a/src/arduino.cc/builder/test/try_build_of_problematic_sketch_test.go +++ b/src/arduino.cc/builder/test/try_build_of_problematic_sketch_test.go @@ -206,6 +206,14 @@ func TestTryBuild039(t *testing.T) { } func TestTryBuild040(t *testing.T) { + tryBuild(t, "sketch_with_externC_multiline", "sketch_with_externC_multiline.ino") +} + +func TestTryBuild041(t *testing.T) { + tryBuild(t, "sketch_with_multiline_template", "sketch_with_multiline_template.ino") +} + +func TestTryBuild042(t *testing.T) { tryBuild(t, "sketch_with_fake_function_pointer", "sketch_with_fake_function_pointer.ino") }