Skip to content

Commit f28fffb

Browse files
author
Federico Fissore
committed
Brand new way of discovering libraries and related -I gcc parameters
For each source file found, gcc -E is continuously run until it doesn't report any error. When it reports an error, it is in the form of Ethernet.h: No such file or directory #include <Ethernet.h> Ethernet.h is searched from known libraries, its -I param added to gcc -E and gcc -E is rerun If Ethernet.h is not found, preprocessing fails and reports an error If found, the collected list of -I params is used to preprocess another (sketch or library) source file, until all dependendent libraries are resolved Fixes #33 Signed-off-by: Federico Fissore <[email protected]>
1 parent d46c7ce commit f28fffb

23 files changed

+380
-131
lines changed

Diff for: ‎src/arduino.cc/builder/builder_utils/utils.go

+44-11
Original file line numberDiff line numberDiff line change
@@ -34,9 +34,10 @@ import (
3434
"arduino.cc/builder/i18n"
3535
"arduino.cc/builder/props"
3636
"arduino.cc/builder/utils"
37+
"bufio"
3738
"fmt"
38-
"io"
3939
"os"
40+
"os/exec"
4041
"path/filepath"
4142
"strings"
4243
)
@@ -279,10 +280,27 @@ func ArchiveCompiledFiles(buildPath string, archiveFile string, objectFiles []st
279280
}
280281

281282
func ExecRecipe(properties map[string]string, recipe string, removeUnsetProperties bool, echoCommandLine bool, echoOutput bool, logger i18n.Logger) ([]byte, error) {
282-
return ExecRecipeSpecifyStdOutStdErr(properties, recipe, removeUnsetProperties, echoCommandLine, echoOutput, logger, os.Stdout, os.Stderr)
283+
command, err := PrepareCommandForRecipe(properties, recipe, removeUnsetProperties, echoCommandLine, echoOutput, logger)
284+
if err != nil {
285+
return nil, utils.WrapError(err)
286+
}
287+
288+
if echoOutput {
289+
command.Stdout = os.Stdout
290+
}
291+
292+
command.Stderr = os.Stderr
293+
294+
if echoOutput {
295+
err := command.Run()
296+
return nil, utils.WrapError(err)
297+
}
298+
299+
bytes, err := command.Output()
300+
return bytes, utils.WrapError(err)
283301
}
284302

285-
func ExecRecipeSpecifyStdOutStdErr(properties map[string]string, recipe string, removeUnsetProperties bool, echoCommandLine bool, echoOutput bool, logger i18n.Logger, stdout io.Writer, stderr io.Writer) ([]byte, error) {
303+
func PrepareCommandForRecipe(properties map[string]string, recipe string, removeUnsetProperties bool, echoCommandLine bool, echoOutput bool, logger i18n.Logger) (*exec.Cmd, error) {
286304
pattern := properties[recipe]
287305
if pattern == constants.EMPTY_STRING {
288306
return nil, utils.ErrorfWithLogger(logger, constants.MSG_PATTERN_MISSING, recipe)
@@ -306,19 +324,34 @@ func ExecRecipeSpecifyStdOutStdErr(properties map[string]string, recipe string,
306324
fmt.Println(commandLine)
307325
}
308326

309-
if echoOutput {
310-
command.Stdout = stdout
327+
return command, nil
328+
}
329+
330+
func ExecRecipeCollectStdErr(properties map[string]string, recipe string, removeUnsetProperties bool, echoCommandLine bool, echoOutput bool, logger i18n.Logger) (string, error, error) {
331+
command, err := PrepareCommandForRecipe(properties, recipe, removeUnsetProperties, echoCommandLine, echoOutput, logger)
332+
if err != nil {
333+
return "", nil, utils.WrapError(err)
311334
}
312335

313-
command.Stderr = stderr
336+
stderr, err := command.StderrPipe()
337+
if err != nil {
338+
return "", nil, utils.WrapError(err)
339+
}
314340

315-
if echoOutput {
316-
err := command.Run()
317-
return nil, utils.WrapError(err)
341+
err = command.Start()
342+
if err != nil {
343+
return "", nil, utils.WrapError(err)
318344
}
319345

320-
bytes, err := command.Output()
321-
return bytes, utils.WrapError(err)
346+
collectedStdErr := ""
347+
sc := bufio.NewScanner(stderr)
348+
for sc.Scan() {
349+
collectedStdErr += sc.Text() + "\n"
350+
}
351+
352+
err = command.Wait()
353+
354+
return collectedStdErr, err, nil
322355
}
323356

324357
func RemoveHyphenMDDFlagFromGCCCommandLine(properties map[string]string) {

Diff for: ‎src/arduino.cc/builder/constants/constants.go

+2
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,7 @@ const CTX_DEBUG_LEVEL = "debugLevel"
9595
const CTX_FOLDERS_WITH_SOURCES_QUEUE = "foldersWithSourcesQueue"
9696
const CTX_FQBN = "fqbn"
9797
const CTX_GCC_MINUS_E_SOURCE = "gccMinusESource"
98+
const CTX_GCC_MINUS_E_STDERR = "gccMinusEOutput"
9899
const CTX_GCC_MINUS_M_OUTPUT = "gccMinusMOutput"
99100
const CTX_HARDWARE_FOLDERS = "hardwareFolders"
100101
const CTX_HARDWARE = "hardware"
@@ -103,6 +104,7 @@ const CTX_IMPORTED_LIBRARIES = "importedLibraries"
103104
const CTX_INCLUDE_FOLDERS = "includeFolders"
104105
const CTX_INCLUDE_SECTION = "includeSection"
105106
const CTX_INCLUDES = "includes"
107+
const CTX_INCLUDES_JUST_FOUND = "includesJustFound"
106108
const CTX_LIBRARIES_BUILD_PATH = "librariesBuildPath"
107109
const CTX_LIBRARIES_FOLDERS = "librariesFolders"
108110
const CTX_LIBRARIES = "libraries"

Diff for: ‎src/arduino.cc/builder/container_find_includes.go

+37-26
Original file line numberDiff line numberDiff line change
@@ -44,22 +44,11 @@ func (s *ContainerFindIncludes) Run(context map[string]interface{}) error {
4444
return utils.WrapError(err)
4545
}
4646

47-
sketch := context[constants.CTX_SKETCH].(*types.Sketch)
4847
sketchBuildPath := context[constants.CTX_SKETCH_BUILD_PATH].(string)
49-
wheelSpins := context[constants.CTX_LIBRARY_DISCOVERY_RECURSION_DEPTH].(int)
50-
for i := 0; i < wheelSpins; i++ {
51-
commands := []types.Command{
52-
&IncludesFinderWithGCC{SourceFile: filepath.Join(sketchBuildPath, filepath.Base(sketch.MainFile.Name)+".cpp")},
53-
&GCCMinusMOutputParser{},
54-
&IncludesToIncludeFolders{},
55-
}
56-
57-
for _, command := range commands {
58-
err := runCommand(context, command)
59-
if err != nil {
60-
return utils.WrapError(err)
61-
}
62-
}
48+
sketch := context[constants.CTX_SKETCH].(*types.Sketch)
49+
err = findIncludesUntilDone(context, filepath.Join(sketchBuildPath, filepath.Base(sketch.MainFile.Name)+".cpp"))
50+
if err != nil {
51+
return utils.WrapError(err)
6352
}
6453

6554
foldersWithSources := context[constants.CTX_FOLDERS_WITH_SOURCES_QUEUE].(*types.UniqueSourceFolderQueue)
@@ -81,18 +70,13 @@ func (s *ContainerFindIncludes) Run(context map[string]interface{}) error {
8170
sourceFiles := context[constants.CTX_COLLECTED_SOURCE_FILES_QUEUE].(*types.UniqueStringQueue)
8271

8372
for !sourceFiles.Empty() {
84-
commands := []types.Command{
85-
&IncludesFinderWithGCC{SourceFile: sourceFiles.Pop().(string)},
86-
&GCCMinusMOutputParser{},
87-
&IncludesToIncludeFolders{},
88-
&CollectAllSourceFilesFromFoldersWithSources{},
73+
err = findIncludesUntilDone(context, sourceFiles.Pop().(string))
74+
if err != nil {
75+
return utils.WrapError(err)
8976
}
90-
91-
for _, command := range commands {
92-
err := runCommand(context, command)
93-
if err != nil {
94-
return utils.WrapError(err)
95-
}
77+
err := runCommand(context, &CollectAllSourceFilesFromFoldersWithSources{})
78+
if err != nil {
79+
return utils.WrapError(err)
9680
}
9781
}
9882

@@ -107,3 +91,30 @@ func runCommand(context map[string]interface{}, command types.Command) error {
10791
}
10892
return nil
10993
}
94+
95+
func findIncludesUntilDone(context map[string]interface{}, sourceFile string) error {
96+
importedLibraries := context[constants.CTX_IMPORTED_LIBRARIES].([]*types.Library)
97+
done := false
98+
for !done {
99+
commands := []types.Command{
100+
&GCCPreprocRunnerForDiscoveringIncludes{SourceFile: sourceFile},
101+
&IncludesFinderWithRegExp{ContextField: constants.CTX_GCC_MINUS_E_SOURCE},
102+
&IncludesToIncludeFolders{},
103+
}
104+
for _, command := range commands {
105+
err := runCommand(context, command)
106+
if err != nil {
107+
return utils.WrapError(err)
108+
}
109+
}
110+
if len(context[constants.CTX_INCLUDES_JUST_FOUND].([]string)) == 0 {
111+
done = true
112+
} else if len(context[constants.CTX_IMPORTED_LIBRARIES].([]*types.Library)) == len(importedLibraries) {
113+
err := runCommand(context, &GCCPreprocRunner{})
114+
return utils.WrapError(err)
115+
}
116+
importedLibraries = context[constants.CTX_IMPORTED_LIBRARIES].([]*types.Library)
117+
context[constants.CTX_INCLUDES_JUST_FOUND] = []string{}
118+
}
119+
return nil
120+
}

Diff for: ‎src/arduino.cc/builder/gcc_minus_m_output_parser.go

+1-6
Original file line numberDiff line numberDiff line change
@@ -60,12 +60,7 @@ func (s *GCCMinusMOutputParser) Run(context map[string]interface{}) error {
6060
return nil
6161
}
6262

63-
previousIncludes := utils.SliceToMapStringBool(context[constants.CTX_INCLUDES].([]string), true)
64-
currentIncludes := utils.SliceToMapStringBool(includes, true)
65-
66-
mergedIncludes := utils.MergeMapsOfStringBool(previousIncludes, currentIncludes)
67-
68-
context[constants.CTX_INCLUDES] = utils.KeysOfMapOfStringBool(mergedIncludes)
63+
context[constants.CTX_INCLUDES] = utils.AddStringsToStringsSet(context[constants.CTX_INCLUDES].([]string), includes)
6964

7065
return nil
7166
}

Diff for: ‎src/arduino.cc/builder/gcc_preproc_runner.go

+34-9
Original file line numberDiff line numberDiff line change
@@ -42,17 +42,9 @@ import (
4242
type GCCPreprocRunner struct{}
4343

4444
func (s *GCCPreprocRunner) Run(context map[string]interface{}) error {
45-
buildProperties := utils.GetMapStringStringOrDefault(context, constants.CTX_BUILD_PROPERTIES)
46-
properties := utils.MergeMapsOfStrings(make(map[string]string), buildProperties)
47-
4845
sketchBuildPath := context[constants.CTX_SKETCH_BUILD_PATH].(string)
4946
sketch := context[constants.CTX_SKETCH].(*types.Sketch)
50-
properties[constants.BUILD_PROPERTIES_SOURCE_FILE] = filepath.Join(sketchBuildPath, filepath.Base(sketch.MainFile.Name)+".cpp")
51-
52-
includes := context[constants.CTX_INCLUDE_FOLDERS].([]string)
53-
includes = utils.Map(includes, utils.WrapWithHyphenI)
54-
properties[constants.BUILD_PROPERTIES_INCLUDES] = strings.Join(includes, constants.SPACE)
55-
builder_utils.RemoveHyphenMDDFlagFromGCCCommandLine(properties)
47+
properties := prepareGCCPreprocRecipeProperties(context, filepath.Join(sketchBuildPath, filepath.Base(sketch.MainFile.Name)+".cpp"))
5648

5749
verbose := context[constants.CTX_VERBOSE].(bool)
5850
logger := context[constants.CTX_LOGGER].(i18n.Logger)
@@ -65,3 +57,36 @@ func (s *GCCPreprocRunner) Run(context map[string]interface{}) error {
6557

6658
return nil
6759
}
60+
61+
type GCCPreprocRunnerForDiscoveringIncludes struct {
62+
SourceFile string
63+
}
64+
65+
func (s *GCCPreprocRunnerForDiscoveringIncludes) Run(context map[string]interface{}) error {
66+
properties := prepareGCCPreprocRecipeProperties(context, s.SourceFile)
67+
68+
verbose := context[constants.CTX_VERBOSE].(bool)
69+
logger := context[constants.CTX_LOGGER].(i18n.Logger)
70+
output, _, err := builder_utils.ExecRecipeCollectStdErr(properties, constants.RECIPE_PREPROC_MACROS, true, verbose, false, logger)
71+
if err != nil {
72+
return utils.WrapError(err)
73+
}
74+
75+
context[constants.CTX_GCC_MINUS_E_SOURCE] = string(output)
76+
77+
return nil
78+
}
79+
80+
func prepareGCCPreprocRecipeProperties(context map[string]interface{}, sourceFile string) map[string]string {
81+
buildProperties := utils.GetMapStringStringOrDefault(context, constants.CTX_BUILD_PROPERTIES)
82+
properties := utils.MergeMapsOfStrings(make(map[string]string), buildProperties)
83+
84+
properties[constants.BUILD_PROPERTIES_SOURCE_FILE] = sourceFile
85+
86+
includes := context[constants.CTX_INCLUDE_FOLDERS].([]string)
87+
includes = utils.Map(includes, utils.WrapWithHyphenI)
88+
properties[constants.BUILD_PROPERTIES_INCLUDES] = strings.Join(includes, constants.SPACE)
89+
builder_utils.RemoveHyphenMDDFlagFromGCCCommandLine(properties)
90+
91+
return properties
92+
}

Diff for: ‎src/arduino.cc/builder/hardware/platform.keys.rewrite.txt

+7
Original file line numberDiff line numberDiff line change
@@ -36,3 +36,10 @@ new.10.recipe.ar.pattern="{compiler.path}{compiler.ar.cmd}" {compiler.ar.flags}
3636

3737
old.11.recipe.c.combine.pattern="{compiler.path}{compiler.c.elf.cmd}" {compiler.c.elf.flags} -mmcu={build.mcu} -o "{build.path}/{build.project_name}.elf" {object_files} "{build.path}/{archive_file}" "-L{build.path}" -lm
3838
new.11.recipe.c.combine.pattern="{compiler.path}{compiler.c.elf.cmd}" {compiler.c.elf.flags} -mmcu={build.mcu} -o "{build.path}/{build.project_name}.elf" {object_files} "{archive_file_path}" "-L{build.path}" -lm
39+
40+
#generic again
41+
old.12.preproc.includes.flags=-w -x c++ -M -MG -MP
42+
new.12.preproc.includes.flags=-w -x c++ -M -MG -MP -include Arduino.h
43+
44+
old.13.preproc.macros.flags=-w -x c++ -E -CC
45+
new.13.preproc.macros.flags=-w -x c++ -E -CC -include Arduino.h

Diff for: ‎src/arduino.cc/builder/hardware/platform.txt

+4-3
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,10 @@ tools.ctags.pattern="{cmd.path}" -u --language-force=c++ -f - --c++-kinds=svpf -
77
# additional entries
88
tools.avrdude.path={runtime.tools.avrdude.path}
99

10-
preproc.includes.flags=-w -x c++ -M -MG -MP
11-
recipe.preproc.includes="{compiler.path}{compiler.cpp.cmd}" {preproc.includes.flags} -DF_CPU={build.f_cpu} -DARDUINO={runtime.ide.version} -DARDUINO_{build.board} -DARDUINO_ARCH_{build.arch} {compiler.cpp.extra_flags} {build.extra_flags} {includes} "{source_file}"
10+
preproc.includes.flags=-w -x c++ -M -MG -MP -include Arduino.h
11+
preproc.includes.compatibility_flags={build.mbed_api_include} {build.nRF51822_api_include} {build.ble_api_include} {compiler.libsam.c.flags} {compiler.arm.cmsis.path} {build.variant_system_include}
12+
recipe.preproc.includes="{compiler.path}{compiler.cpp.cmd}" {preproc.includes.flags} -DF_CPU={build.f_cpu} -DARDUINO={runtime.ide.version} -DARDUINO_{build.board} -DARDUINO_ARCH_{build.arch} {compiler.cpp.extra_flags} {build.extra_flags} {preproc.includes.compatibility_flags} {includes} "{source_file}"
1213

13-
preproc.macros.flags=-w -x c++ -E -CC
14+
preproc.macros.flags=-w -x c++ -E -CC -include Arduino.h
1415
preproc.macros.compatibility_flags={build.mbed_api_include} {build.nRF51822_api_include} {build.ble_api_include} {compiler.libsam.c.flags} {compiler.arm.cmsis.path} {build.variant_system_include}
1516
recipe.preproc.macros="{compiler.path}{compiler.cpp.cmd}" {compiler.cpreprocessor.flags} {compiler.cpp.flags} {preproc.macros.flags} -DF_CPU={build.f_cpu} -DARDUINO={runtime.ide.version} -DARDUINO_{build.board} -DARDUINO_ARCH_{build.arch} {compiler.cpp.extra_flags} {build.extra_flags} {preproc.macros.compatibility_flags} {includes} "{source_file}"

Diff for: ‎src/arduino.cc/builder/includes_finder_with_regexp.go

+14-4
Original file line numberDiff line numberDiff line change
@@ -31,24 +31,34 @@ package builder
3131

3232
import (
3333
"arduino.cc/builder/constants"
34+
"arduino.cc/builder/utils"
3435
"regexp"
3536
"strings"
3637
)
3738

3839
var INCLUDE_REGEXP = regexp.MustCompile("(?ms)^\\s*#include\\s*[<\"](\\S+)[\">]")
3940

40-
type IncludesFinderWithRegExp struct{}
41+
type IncludesFinderWithRegExp struct {
42+
ContextField string
43+
}
4144

4245
func (s *IncludesFinderWithRegExp) Run(context map[string]interface{}) error {
43-
source := context[constants.CTX_SOURCE].(string)
46+
source := context[s.ContextField].(string)
4447

4548
matches := INCLUDE_REGEXP.FindAllStringSubmatch(source, -1)
46-
var includes []string
49+
includes := []string{}
4750
for _, match := range matches {
4851
includes = append(includes, strings.TrimSpace(match[1]))
4952
}
5053

51-
context[constants.CTX_INCLUDES] = includes
54+
context[constants.CTX_INCLUDES_JUST_FOUND] = includes
55+
56+
if !utils.MapHas(context, constants.CTX_INCLUDES) {
57+
context[constants.CTX_INCLUDES] = includes
58+
return nil
59+
}
60+
61+
context[constants.CTX_INCLUDES] = utils.AddStringsToStringsSet(context[constants.CTX_INCLUDES].([]string), includes)
5262

5363
return nil
5464
}

Diff for: ‎src/arduino.cc/builder/test/builder_test.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -53,8 +53,8 @@ func TestBuilderEmptySketch(t *testing.T) {
5353
context[constants.CTX_BUILT_IN_LIBRARIES_FOLDERS] = []string{"downloaded_libraries"}
5454
context[constants.CTX_OTHER_LIBRARIES_FOLDERS] = []string{"libraries"}
5555
context[constants.CTX_BUILD_PROPERTIES_RUNTIME_IDE_VERSION] = "10600"
56-
// context[constants.CTX_VERBOSE] = true
57-
// context[constants.CTX_DEBUG_LEVEL] = 10
56+
context[constants.CTX_VERBOSE] = true
57+
context[constants.CTX_DEBUG_LEVEL] = 10
5858

5959
command := builder.Builder{}
6060
err := command.Run(context)

Diff for: ‎src/arduino.cc/builder/test/coan_runner_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ func TestCoanRunner(t *testing.T) {
5454
context[constants.CTX_FQBN] = "arduino:avr:leonardo"
5555
context[constants.CTX_SKETCH_LOCATION] = filepath.Join("sketch2", "SketchWithIfDef.ino")
5656
context[constants.CTX_BUILD_PROPERTIES_RUNTIME_IDE_VERSION] = "10600"
57-
context[constants.CTX_VERBOSE] = false
57+
context[constants.CTX_VERBOSE] = true
5858

5959
commands := []types.Command{
6060
&builder.SetupHumanLoggerIfMissing{},

Diff for: ‎src/arduino.cc/builder/test/ctags_runner_test.go

+4-4
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ func TestCTagsRunner(t *testing.T) {
5656
context[constants.CTX_BUILD_PROPERTIES_RUNTIME_IDE_VERSION] = "10600"
5757
context[constants.CTX_BUILT_IN_LIBRARIES_FOLDERS] = []string{"downloaded_libraries"}
5858
context[constants.CTX_OTHER_LIBRARIES_FOLDERS] = []string{"libraries"}
59-
context[constants.CTX_VERBOSE] = false
59+
context[constants.CTX_VERBOSE] = true
6060

6161
commands := []types.Command{
6262
&builder.SetupHumanLoggerIfMissing{},
@@ -106,7 +106,7 @@ func TestCTagsRunnerSketchWithClass(t *testing.T) {
106106
context[constants.CTX_BUILD_PROPERTIES_RUNTIME_IDE_VERSION] = "10600"
107107
context[constants.CTX_BUILT_IN_LIBRARIES_FOLDERS] = []string{"downloaded_libraries"}
108108
context[constants.CTX_OTHER_LIBRARIES_FOLDERS] = []string{"libraries"}
109-
context[constants.CTX_VERBOSE] = false
109+
context[constants.CTX_VERBOSE] = true
110110

111111
commands := []types.Command{
112112
&builder.SetupHumanLoggerIfMissing{},
@@ -154,7 +154,7 @@ func TestCTagsRunnerSketchWithTypename(t *testing.T) {
154154
context[constants.CTX_BUILD_PROPERTIES_RUNTIME_IDE_VERSION] = "10600"
155155
context[constants.CTX_BUILT_IN_LIBRARIES_FOLDERS] = []string{"downloaded_libraries"}
156156
context[constants.CTX_OTHER_LIBRARIES_FOLDERS] = []string{"libraries"}
157-
context[constants.CTX_VERBOSE] = false
157+
context[constants.CTX_VERBOSE] = true
158158

159159
commands := []types.Command{
160160
&builder.SetupHumanLoggerIfMissing{},
@@ -201,7 +201,7 @@ func TestCTagsRunnerSketchWithNamespace(t *testing.T) {
201201
context[constants.CTX_BUILD_PROPERTIES_RUNTIME_IDE_VERSION] = "10600"
202202
context[constants.CTX_BUILT_IN_LIBRARIES_FOLDERS] = []string{"downloaded_libraries"}
203203
context[constants.CTX_OTHER_LIBRARIES_FOLDERS] = []string{"libraries"}
204-
context[constants.CTX_VERBOSE] = false
204+
context[constants.CTX_VERBOSE] = true
205205

206206
commands := []types.Command{
207207
&builder.SetupHumanLoggerIfMissing{},

0 commit comments

Comments
 (0)