From cfe6033718fb9702e024338164eb359e7fc6bd99 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 30 Mar 2023 17:00:27 +0200 Subject: [PATCH 01/42] ctags parser.Parse signature change --- legacy/builder/ctags/ctags_parser.go | 4 ++-- legacy/builder/ctags/ctags_parser_test.go | 2 +- legacy/builder/ctags/ctags_to_prototypes_test.go | 2 +- legacy/builder/ctags_runner.go | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/legacy/builder/ctags/ctags_parser.go b/legacy/builder/ctags/ctags_parser.go index 4ced464ae1e..a2c49851338 100644 --- a/legacy/builder/ctags/ctags_parser.go +++ b/legacy/builder/ctags/ctags_parser.go @@ -43,8 +43,8 @@ type CTagsParser struct { mainFile *paths.Path } -func (p *CTagsParser) Parse(ctagsOutput string, mainFile *paths.Path) []*types.CTag { - rows := strings.Split(ctagsOutput, "\n") +func (p *CTagsParser) Parse(ctagsOutput []byte, mainFile *paths.Path) []*types.CTag { + rows := strings.Split(string(ctagsOutput), "\n") rows = removeEmpty(rows) p.mainFile = mainFile diff --git a/legacy/builder/ctags/ctags_parser_test.go b/legacy/builder/ctags/ctags_parser_test.go index d3bb03b23a0..49711c27c18 100644 --- a/legacy/builder/ctags/ctags_parser_test.go +++ b/legacy/builder/ctags/ctags_parser_test.go @@ -30,7 +30,7 @@ func produceTags(t *testing.T, filename string) []*types.CTag { require.NoError(t, err) parser := CTagsParser{} - return parser.Parse(string(bytes), nil) + return parser.Parse(bytes, nil) } func TestCTagsParserShouldListPrototypes(t *testing.T) { diff --git a/legacy/builder/ctags/ctags_to_prototypes_test.go b/legacy/builder/ctags/ctags_to_prototypes_test.go index e73a1312069..7dcea2a0ab1 100644 --- a/legacy/builder/ctags/ctags_to_prototypes_test.go +++ b/legacy/builder/ctags/ctags_to_prototypes_test.go @@ -30,7 +30,7 @@ func producePrototypes(t *testing.T, filename string, mainFile string) ([]*types require.NoError(t, err) parser := &CTagsParser{} - parser.Parse(string(bytes), paths.New(mainFile)) + parser.Parse(bytes, paths.New(mainFile)) return parser.GeneratePrototypes() } diff --git a/legacy/builder/ctags_runner.go b/legacy/builder/ctags_runner.go index 2e42bdca234..7dd91f578a0 100644 --- a/legacy/builder/ctags_runner.go +++ b/legacy/builder/ctags_runner.go @@ -61,7 +61,7 @@ func (s *CTagsRunner) Run(ctx *types.Context) error { parser := &ctags.CTagsParser{} - ctx.CTagsOfPreprocessedSource = parser.Parse(ctx.CTagsOutput, ctx.Sketch.MainFile) + ctx.CTagsOfPreprocessedSource = parser.Parse(sourceBytes, ctx.Sketch.MainFile) parser.FixCLinkageTagsDeclarations(ctx.CTagsOfPreprocessedSource) protos, line := parser.GeneratePrototypes() From 02d08b15b30b12032cac1162764996f08e2e2591 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 30 Mar 2023 17:04:53 +0200 Subject: [PATCH 02/42] removing ctags-related data from builder ctx (part 1) --- legacy/builder/ctags_runner.go | 16 ++++++++------- legacy/builder/test/ctags_runner_test.go | 26 ++++++++++++++---------- legacy/builder/types/context.go | 1 - 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/legacy/builder/ctags_runner.go b/legacy/builder/ctags_runner.go index 7dd91f578a0..edf46a3d69a 100644 --- a/legacy/builder/ctags_runner.go +++ b/legacy/builder/ctags_runner.go @@ -26,9 +26,12 @@ import ( "github.com/pkg/errors" ) -type CTagsRunner struct{} +type CTagsRunner struct { + // Needed for unit-testing + CtagsOutput []byte +} -func (s *CTagsRunner) Run(ctx *types.Context) error { +func (r *CTagsRunner) Run(ctx *types.Context) error { ctagsTargetFilePath := ctx.CTagsTargetFile buildProperties := properties.NewMap() @@ -52,16 +55,13 @@ func (s *CTagsRunner) Run(ctx *types.Context) error { command := exec.Command(parts[0], parts[1:]...) command.Env = append(os.Environ(), ctx.PackageManager.GetEnvVarsForSpawnedProcess()...) - sourceBytes, _, err := utils.ExecCommand(ctx, command, utils.Capture /* stdout */, utils.ShowIfVerbose /* stderr */) + ctagsOutput, _, err := utils.ExecCommand(ctx, command, utils.Capture /* stdout */, utils.ShowIfVerbose /* stderr */) if err != nil { return errors.WithStack(err) } - ctx.CTagsOutput = string(sourceBytes) - parser := &ctags.CTagsParser{} - - ctx.CTagsOfPreprocessedSource = parser.Parse(sourceBytes, ctx.Sketch.MainFile) + ctx.CTagsOfPreprocessedSource = parser.Parse(ctagsOutput, ctx.Sketch.MainFile) parser.FixCLinkageTagsDeclarations(ctx.CTagsOfPreprocessedSource) protos, line := parser.GeneratePrototypes() @@ -70,5 +70,7 @@ func (s *CTagsRunner) Run(ctx *types.Context) error { } ctx.Prototypes = protos + // Needed for unit-testing + r.CtagsOutput = ctagsOutput return nil } diff --git a/legacy/builder/test/ctags_runner_test.go b/legacy/builder/test/ctags_runner_test.go index 013499a461c..b5b9e823dc5 100644 --- a/legacy/builder/test/ctags_runner_test.go +++ b/legacy/builder/test/ctags_runner_test.go @@ -31,6 +31,7 @@ func TestCTagsRunner(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) ctx.Verbose = true + ctagsRunner := &builder.CTagsRunner{} commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, &builder.ContainerMergeCopySketchFiles{}, @@ -38,7 +39,7 @@ func TestCTagsRunner(t *testing.T) { &builder.PrintUsedLibrariesIfVerbose{}, &builder.WarnAboutArchIncompatibleLibraries{}, &builder.CTagsTargetFileSaver{Source: &ctx.Source, TargetFileName: "ctags_target.cpp"}, - &builder.CTagsRunner{}, + ctagsRunner, } for _, command := range commands { err := command.Run(ctx) @@ -53,8 +54,7 @@ func TestCTagsRunner(t *testing.T) { "digitalCommand " + quotedSketchLocation + " /^void digitalCommand(BridgeClient client) {$/;\" kind:function line:82 signature:(BridgeClient client) returntype:void\n" + "analogCommand " + quotedSketchLocation + " /^void analogCommand(BridgeClient client) {$/;\" kind:function line:109 signature:(BridgeClient client) returntype:void\n" + "modeCommand " + quotedSketchLocation + " /^void modeCommand(BridgeClient client) {$/;\" kind:function line:149 signature:(BridgeClient client) returntype:void\n" - - require.Equal(t, expectedOutput, strings.Replace(ctx.CTagsOutput, "\r\n", "\n", -1)) + require.Equal(t, expectedOutput, strings.Replace(string(ctagsRunner.CtagsOutput), "\r\n", "\n", -1)) } func TestCTagsRunnerSketchWithClass(t *testing.T) { @@ -63,6 +63,7 @@ func TestCTagsRunnerSketchWithClass(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) ctx.Verbose = true + ctagsRunner := &builder.CTagsRunner{} commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, &builder.ContainerMergeCopySketchFiles{}, @@ -70,7 +71,7 @@ func TestCTagsRunnerSketchWithClass(t *testing.T) { &builder.PrintUsedLibrariesIfVerbose{}, &builder.WarnAboutArchIncompatibleLibraries{}, &builder.CTagsTargetFileSaver{Source: &ctx.Source, TargetFileName: "ctags_target.cpp"}, - &builder.CTagsRunner{}, + ctagsRunner, } for _, command := range commands { err := command.Run(ctx) @@ -83,7 +84,7 @@ func TestCTagsRunnerSketchWithClass(t *testing.T) { "set_values\t" + quotedSketchLocation + "\t/^void Rectangle::set_values (int x, int y) {$/;\"\tkind:function\tline:8\tclass:Rectangle\tsignature:(int x, int y)\treturntype:void\n" + "setup\t" + quotedSketchLocation + "\t/^void setup() {$/;\"\tkind:function\tline:13\tsignature:()\treturntype:void\n" + "loop\t" + quotedSketchLocation + "\t/^void loop() {$/;\"\tkind:function\tline:17\tsignature:()\treturntype:void\n" - require.Equal(t, expectedOutput, strings.Replace(ctx.CTagsOutput, "\r\n", "\n", -1)) + require.Equal(t, expectedOutput, strings.Replace(string(ctagsRunner.CtagsOutput), "\r\n", "\n", -1)) } func TestCTagsRunnerSketchWithTypename(t *testing.T) { @@ -92,6 +93,7 @@ func TestCTagsRunnerSketchWithTypename(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) ctx.Verbose = true + ctagsRunner := &builder.CTagsRunner{} commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, &builder.ContainerMergeCopySketchFiles{}, @@ -99,7 +101,7 @@ func TestCTagsRunnerSketchWithTypename(t *testing.T) { &builder.PrintUsedLibrariesIfVerbose{}, &builder.WarnAboutArchIncompatibleLibraries{}, &builder.CTagsTargetFileSaver{Source: &ctx.Source, TargetFileName: "ctags_target.cpp"}, - &builder.CTagsRunner{}, + ctagsRunner, } for _, command := range commands { err := command.Run(ctx) @@ -111,7 +113,7 @@ func TestCTagsRunnerSketchWithTypename(t *testing.T) { "setup\t" + quotedSketchLocation + "\t/^void setup() {$/;\"\tkind:function\tline:6\tsignature:()\treturntype:void\n" + "loop\t" + quotedSketchLocation + "\t/^void loop() {}$/;\"\tkind:function\tline:10\tsignature:()\treturntype:void\n" + "func\t" + quotedSketchLocation + "\t/^typename Foo::Bar func(){$/;\"\tkind:function\tline:12\tsignature:()\treturntype:Foo::Bar\n" - require.Equal(t, expectedOutput, strings.Replace(ctx.CTagsOutput, "\r\n", "\n", -1)) + require.Equal(t, expectedOutput, strings.Replace(string(ctagsRunner.CtagsOutput), "\r\n", "\n", -1)) } func TestCTagsRunnerSketchWithNamespace(t *testing.T) { @@ -120,6 +122,7 @@ func TestCTagsRunnerSketchWithNamespace(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) ctx.Verbose = true + ctagsRunner := &builder.CTagsRunner{} commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, &builder.ContainerMergeCopySketchFiles{}, @@ -127,7 +130,7 @@ func TestCTagsRunnerSketchWithNamespace(t *testing.T) { &builder.PrintUsedLibrariesIfVerbose{}, &builder.WarnAboutArchIncompatibleLibraries{}, &builder.CTagsTargetFileSaver{Source: &ctx.Source, TargetFileName: "ctags_target.cpp"}, - &builder.CTagsRunner{}, + ctagsRunner, } for _, command := range commands { err := command.Run(ctx) @@ -138,7 +141,7 @@ func TestCTagsRunnerSketchWithNamespace(t *testing.T) { expectedOutput := "value\t" + quotedSketchLocation + "\t/^\tint value() {$/;\"\tkind:function\tline:2\tnamespace:Test\tsignature:()\treturntype:int\n" + "setup\t" + quotedSketchLocation + "\t/^void setup() {}$/;\"\tkind:function\tline:7\tsignature:()\treturntype:void\n" + "loop\t" + quotedSketchLocation + "\t/^void loop() {}$/;\"\tkind:function\tline:8\tsignature:()\treturntype:void\n" - require.Equal(t, expectedOutput, strings.Replace(ctx.CTagsOutput, "\r\n", "\n", -1)) + require.Equal(t, expectedOutput, strings.Replace(string(ctagsRunner.CtagsOutput), "\r\n", "\n", -1)) } func TestCTagsRunnerSketchWithTemplates(t *testing.T) { @@ -147,6 +150,7 @@ func TestCTagsRunnerSketchWithTemplates(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) ctx.Verbose = true + ctagsRunner := &builder.CTagsRunner{} commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, &builder.ContainerMergeCopySketchFiles{}, @@ -154,7 +158,7 @@ func TestCTagsRunnerSketchWithTemplates(t *testing.T) { &builder.PrintUsedLibrariesIfVerbose{}, &builder.WarnAboutArchIncompatibleLibraries{}, &builder.CTagsTargetFileSaver{Source: &ctx.Source, TargetFileName: "ctags_target.cpp"}, - &builder.CTagsRunner{}, + ctagsRunner, } for _, command := range commands { err := command.Run(ctx) @@ -166,5 +170,5 @@ func TestCTagsRunnerSketchWithTemplates(t *testing.T) { "bVar\t" + quotedSketchLocation + "\t/^c< 8 > bVar;$/;\"\tkind:variable\tline:15\n" + "aVar\t" + quotedSketchLocation + "\t/^c< 1<<8 > aVar;$/;\"\tkind:variable\tline:16\n" + "func\t" + quotedSketchLocation + "\t/^template func( c< 1< & aParam) {$/;\"\tkind:function\tline:18\tsignature:( c< 1< & aParam)\treturntype:template\n" - require.Equal(t, expectedOutput, strings.Replace(ctx.CTagsOutput, "\r\n", "\n", -1)) + require.Equal(t, expectedOutput, strings.Replace(string(ctagsRunner.CtagsOutput), "\r\n", "\n", -1)) } diff --git a/legacy/builder/types/context.go b/legacy/builder/types/context.go index 0048d6d0c02..3c8473a8fef 100644 --- a/legacy/builder/types/context.go +++ b/legacy/builder/types/context.go @@ -115,7 +115,6 @@ type Context struct { UseCachedLibrariesResolution bool // C++ Parsing - CTagsOutput string CTagsTargetFile *paths.Path CTagsOfPreprocessedSource []*CTag LineOffset int From 4e7f2d8395bb541e0716e8f7a9caa044cf78e41d Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 30 Mar 2023 17:15:11 +0200 Subject: [PATCH 03/42] removing ctags-related data from builder ctx (part 2) --- legacy/builder/ctags/ctags_has_issues.go | 14 ++++++-------- legacy/builder/ctags_runner.go | 4 ++-- legacy/builder/types/context.go | 1 - 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/legacy/builder/ctags/ctags_has_issues.go b/legacy/builder/ctags/ctags_has_issues.go index 7cc693f2af6..a6cda249865 100644 --- a/legacy/builder/ctags/ctags_has_issues.go +++ b/legacy/builder/ctags/ctags_has_issues.go @@ -23,14 +23,12 @@ import ( "github.com/arduino/arduino-cli/legacy/builder/types" ) -func (p *CTagsParser) FixCLinkageTagsDeclarations(tags []*types.CTag) { - - linesMap := p.FindCLinkageLines(tags) - for i := range tags { - - if sliceContainsInt(linesMap[tags[i].Filename], tags[i].Line) && - !strings.Contains(tags[i].PrototypeModifiers, EXTERN) { - tags[i].PrototypeModifiers = tags[i].PrototypeModifiers + " " + EXTERN +func (p *CTagsParser) FixCLinkageTagsDeclarations() { + linesMap := p.FindCLinkageLines(p.tags) + for i := range p.tags { + if sliceContainsInt(linesMap[p.tags[i].Filename], p.tags[i].Line) && + !strings.Contains(p.tags[i].PrototypeModifiers, EXTERN) { + p.tags[i].PrototypeModifiers = p.tags[i].PrototypeModifiers + " " + EXTERN } } } diff --git a/legacy/builder/ctags_runner.go b/legacy/builder/ctags_runner.go index edf46a3d69a..dee108255e2 100644 --- a/legacy/builder/ctags_runner.go +++ b/legacy/builder/ctags_runner.go @@ -61,8 +61,8 @@ func (r *CTagsRunner) Run(ctx *types.Context) error { } parser := &ctags.CTagsParser{} - ctx.CTagsOfPreprocessedSource = parser.Parse(ctagsOutput, ctx.Sketch.MainFile) - parser.FixCLinkageTagsDeclarations(ctx.CTagsOfPreprocessedSource) + parser.Parse(ctagsOutput, ctx.Sketch.MainFile) + parser.FixCLinkageTagsDeclarations() protos, line := parser.GeneratePrototypes() if line != -1 { diff --git a/legacy/builder/types/context.go b/legacy/builder/types/context.go index 3c8473a8fef..6349e2fce4d 100644 --- a/legacy/builder/types/context.go +++ b/legacy/builder/types/context.go @@ -116,7 +116,6 @@ type Context struct { // C++ Parsing CTagsTargetFile *paths.Path - CTagsOfPreprocessedSource []*CTag LineOffset int PrototypesSection string PrototypesLineWhereToInsert int From d96ca5063ad6933226b7be3d8c6d0555aa2acbf8 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 30 Mar 2023 17:30:01 +0200 Subject: [PATCH 04/42] removing ctags-related data from builder ctx (part 3) --- legacy/builder/container_add_prototypes.go | 3 +-- legacy/builder/ctags_runner.go | 15 +++++++++++- legacy/builder/ctags_target_file_saver.go | 27 ---------------------- legacy/builder/test/ctags_runner_test.go | 15 ++++-------- legacy/builder/types/context.go | 1 - 5 files changed, 20 insertions(+), 41 deletions(-) diff --git a/legacy/builder/container_add_prototypes.go b/legacy/builder/container_add_prototypes.go index d710b82c979..5fe18da76d3 100644 --- a/legacy/builder/container_add_prototypes.go +++ b/legacy/builder/container_add_prototypes.go @@ -50,8 +50,7 @@ func PreprocessSketchWithCtags(ctx *types.Context) error { commands := []types.Command{ &ReadFileAndStoreInContext{FileToRead: targetFilePath, Target: &ctx.SourceGccMinusE}, &FilterSketchSource{Source: &ctx.SourceGccMinusE}, - &CTagsTargetFileSaver{Source: &ctx.SourceGccMinusE, TargetFileName: "ctags_target_for_gcc_minus_e.cpp"}, - &CTagsRunner{}, + &CTagsRunner{Source: &ctx.SourceGccMinusE, TargetFileName: "ctags_target_for_gcc_minus_e.cpp"}, &PrototypesAdder{}, } diff --git a/legacy/builder/ctags_runner.go b/legacy/builder/ctags_runner.go index dee108255e2..40ef36434d1 100644 --- a/legacy/builder/ctags_runner.go +++ b/legacy/builder/ctags_runner.go @@ -27,12 +27,25 @@ import ( ) type CTagsRunner struct { + Source *string + TargetFileName string + // Needed for unit-testing CtagsOutput []byte } func (r *CTagsRunner) Run(ctx *types.Context) error { - ctagsTargetFilePath := ctx.CTagsTargetFile + source := *r.Source + + preprocPath := ctx.PreprocPath + if err := preprocPath.MkdirAll(); err != nil { + return errors.WithStack(err) + } + + ctagsTargetFilePath := preprocPath.Join(r.TargetFileName) + if err := ctagsTargetFilePath.WriteFile([]byte(source)); err != nil { + return errors.WithStack(err) + } buildProperties := properties.NewMap() buildProperties.Set("tools.ctags.path", "{runtime.tools.ctags.path}") diff --git a/legacy/builder/ctags_target_file_saver.go b/legacy/builder/ctags_target_file_saver.go index a4014662c41..261bc7c759a 100644 --- a/legacy/builder/ctags_target_file_saver.go +++ b/legacy/builder/ctags_target_file_saver.go @@ -14,30 +14,3 @@ // To purchase a commercial license, send an email to license@arduino.cc. package builder - -import ( - "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/pkg/errors" -) - -type CTagsTargetFileSaver struct { - Source *string - TargetFileName string -} - -func (s *CTagsTargetFileSaver) Run(ctx *types.Context) error { - source := *s.Source - - preprocPath := ctx.PreprocPath - if err := preprocPath.MkdirAll(); err != nil { - return errors.WithStack(err) - } - - ctagsTargetFilePath := preprocPath.Join(s.TargetFileName) - if err := ctagsTargetFilePath.WriteFile([]byte(source)); err != nil { - return errors.WithStack(err) - } - - ctx.CTagsTargetFile = ctagsTargetFilePath - return nil -} diff --git a/legacy/builder/test/ctags_runner_test.go b/legacy/builder/test/ctags_runner_test.go index b5b9e823dc5..fe44735d66d 100644 --- a/legacy/builder/test/ctags_runner_test.go +++ b/legacy/builder/test/ctags_runner_test.go @@ -31,14 +31,13 @@ func TestCTagsRunner(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) ctx.Verbose = true - ctagsRunner := &builder.CTagsRunner{} + ctagsRunner := &builder.CTagsRunner{Source: &ctx.Source, TargetFileName: "ctags_target.cpp"} commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, &builder.ContainerMergeCopySketchFiles{}, &builder.ContainerFindIncludes{}, &builder.PrintUsedLibrariesIfVerbose{}, &builder.WarnAboutArchIncompatibleLibraries{}, - &builder.CTagsTargetFileSaver{Source: &ctx.Source, TargetFileName: "ctags_target.cpp"}, ctagsRunner, } for _, command := range commands { @@ -63,14 +62,13 @@ func TestCTagsRunnerSketchWithClass(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) ctx.Verbose = true - ctagsRunner := &builder.CTagsRunner{} + ctagsRunner := &builder.CTagsRunner{Source: &ctx.Source, TargetFileName: "ctags_target.cpp"} commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, &builder.ContainerMergeCopySketchFiles{}, &builder.ContainerFindIncludes{}, &builder.PrintUsedLibrariesIfVerbose{}, &builder.WarnAboutArchIncompatibleLibraries{}, - &builder.CTagsTargetFileSaver{Source: &ctx.Source, TargetFileName: "ctags_target.cpp"}, ctagsRunner, } for _, command := range commands { @@ -93,14 +91,13 @@ func TestCTagsRunnerSketchWithTypename(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) ctx.Verbose = true - ctagsRunner := &builder.CTagsRunner{} + ctagsRunner := &builder.CTagsRunner{Source: &ctx.Source, TargetFileName: "ctags_target.cpp"} commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, &builder.ContainerMergeCopySketchFiles{}, &builder.ContainerFindIncludes{}, &builder.PrintUsedLibrariesIfVerbose{}, &builder.WarnAboutArchIncompatibleLibraries{}, - &builder.CTagsTargetFileSaver{Source: &ctx.Source, TargetFileName: "ctags_target.cpp"}, ctagsRunner, } for _, command := range commands { @@ -122,14 +119,13 @@ func TestCTagsRunnerSketchWithNamespace(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) ctx.Verbose = true - ctagsRunner := &builder.CTagsRunner{} + ctagsRunner := &builder.CTagsRunner{Source: &ctx.Source, TargetFileName: "ctags_target.cpp"} commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, &builder.ContainerMergeCopySketchFiles{}, &builder.ContainerFindIncludes{}, &builder.PrintUsedLibrariesIfVerbose{}, &builder.WarnAboutArchIncompatibleLibraries{}, - &builder.CTagsTargetFileSaver{Source: &ctx.Source, TargetFileName: "ctags_target.cpp"}, ctagsRunner, } for _, command := range commands { @@ -150,14 +146,13 @@ func TestCTagsRunnerSketchWithTemplates(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) ctx.Verbose = true - ctagsRunner := &builder.CTagsRunner{} + ctagsRunner := &builder.CTagsRunner{Source: &ctx.Source, TargetFileName: "ctags_target.cpp"} commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, &builder.ContainerMergeCopySketchFiles{}, &builder.ContainerFindIncludes{}, &builder.PrintUsedLibrariesIfVerbose{}, &builder.WarnAboutArchIncompatibleLibraries{}, - &builder.CTagsTargetFileSaver{Source: &ctx.Source, TargetFileName: "ctags_target.cpp"}, ctagsRunner, } for _, command := range commands { diff --git a/legacy/builder/types/context.go b/legacy/builder/types/context.go index 6349e2fce4d..b37cbe5a467 100644 --- a/legacy/builder/types/context.go +++ b/legacy/builder/types/context.go @@ -115,7 +115,6 @@ type Context struct { UseCachedLibrariesResolution bool // C++ Parsing - CTagsTargetFile *paths.Path LineOffset int PrototypesSection string PrototypesLineWhereToInsert int From aebdfecbd521e1faed7f232cbcca745f2d60af46 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 31 Mar 2023 17:04:50 +0200 Subject: [PATCH 05/42] Clearly separate Source code processing phases --- legacy/builder/builder.go | 2 +- legacy/builder/container_add_prototypes.go | 10 ++-- .../container_merge_copy_sketch_files.go | 6 +-- legacy/builder/create_cmake_rule.go | 2 +- legacy/builder/preprocess_sketch.go | 11 ++--- legacy/builder/prototypes_adder.go | 9 ++-- legacy/builder/test/builder_test.go | 12 ++--- legacy/builder/test/ctags_runner_test.go | 10 ++-- legacy/builder/test/prototypes_adder_test.go | 46 +++++++++---------- .../read_file_and_store_in_context_test.go | 4 +- legacy/builder/types/context.go | 13 ++++-- 11 files changed, 65 insertions(+), 60 deletions(-) diff --git a/legacy/builder/builder.go b/legacy/builder/builder.go index d73ec778ca7..b55df582310 100644 --- a/legacy/builder/builder.go +++ b/legacy/builder/builder.go @@ -148,7 +148,7 @@ func (s *Preprocess) Run(ctx *types.Context) error { } // Output arduino-preprocessed source - ctx.WriteStdout([]byte(ctx.Source)) + ctx.WriteStdout([]byte(ctx.SketchSourceAfterArduinoPreprocessing)) return nil } diff --git a/legacy/builder/container_add_prototypes.go b/legacy/builder/container_add_prototypes.go index 5fe18da76d3..ddc6da270f5 100644 --- a/legacy/builder/container_add_prototypes.go +++ b/legacy/builder/container_add_prototypes.go @@ -28,7 +28,7 @@ func PreprocessSketchWithCtags(ctx *types.Context) error { if err := ctx.PreprocPath.MkdirAll(); err != nil { return errors.WithStack(err) } - targetFilePath := ctx.PreprocPath.Join("ctags_target_for_gcc_minus_e.cpp") + targetFilePath := ctx.PreprocPath.Join("sketch_merged.cpp") // Run preprocessor sourceFile := ctx.SketchBuildPath.Join(ctx.Sketch.MainFile.Base() + ".cpp") @@ -48,9 +48,9 @@ func PreprocessSketchWithCtags(ctx *types.Context) error { } commands := []types.Command{ - &ReadFileAndStoreInContext{FileToRead: targetFilePath, Target: &ctx.SourceGccMinusE}, - &FilterSketchSource{Source: &ctx.SourceGccMinusE}, - &CTagsRunner{Source: &ctx.SourceGccMinusE, TargetFileName: "ctags_target_for_gcc_minus_e.cpp"}, + &ReadFileAndStoreInContext{FileToRead: targetFilePath, Target: &ctx.SketchSourceAfterCppPreprocessing}, + &FilterSketchSource{Source: &ctx.SketchSourceAfterCppPreprocessing}, + &CTagsRunner{Source: &ctx.SketchSourceAfterCppPreprocessing, TargetFileName: "sketch_merged.cpp"}, &PrototypesAdder{}, } @@ -62,7 +62,7 @@ func PreprocessSketchWithCtags(ctx *types.Context) error { } } - if err := bldr.SketchSaveItemCpp(ctx.Sketch.MainFile, []byte(ctx.Source), ctx.SketchBuildPath); err != nil { + if err := bldr.SketchSaveItemCpp(ctx.Sketch.MainFile, []byte(ctx.SketchSourceAfterArduinoPreprocessing), ctx.SketchBuildPath); err != nil { return errors.WithStack(err) } diff --git a/legacy/builder/container_merge_copy_sketch_files.go b/legacy/builder/container_merge_copy_sketch_files.go index b98eaebb721..d0eeeaedd0a 100644 --- a/legacy/builder/container_merge_copy_sketch_files.go +++ b/legacy/builder/container_merge_copy_sketch_files.go @@ -24,14 +24,14 @@ import ( type ContainerMergeCopySketchFiles struct{} func (s *ContainerMergeCopySketchFiles) Run(ctx *types.Context) error { - offset, source, err := bldr.SketchMergeSources(ctx.Sketch, ctx.SourceOverride) + offset, mergedSource, err := bldr.SketchMergeSources(ctx.Sketch, ctx.SourceOverride) if err != nil { return err } ctx.LineOffset = offset - ctx.Source = source + ctx.SketchSourceMerged = mergedSource - if err := bldr.SketchSaveItemCpp(ctx.Sketch.MainFile, []byte(ctx.Source), ctx.SketchBuildPath); err != nil { + if err := bldr.SketchSaveItemCpp(ctx.Sketch.MainFile, []byte(mergedSource), ctx.SketchBuildPath); err != nil { return errors.WithStack(err) } diff --git a/legacy/builder/create_cmake_rule.go b/legacy/builder/create_cmake_rule.go index b7640ca58be..7eab7917638 100644 --- a/legacy/builder/create_cmake_rule.go +++ b/legacy/builder/create_cmake_rule.go @@ -120,7 +120,7 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error { // Use old ctags method to generate export file commands := []types.Command{ - &FilterSketchSource{Source: &ctx.Source, RemoveLineMarkers: true}, + &FilterSketchSource{Source: &ctx.SketchSourceMerged, RemoveLineMarkers: true}, } for _, command := range commands { command.Run(ctx) diff --git a/legacy/builder/preprocess_sketch.go b/legacy/builder/preprocess_sketch.go index 31c19ca3bc1..12c2d8f38ca 100644 --- a/legacy/builder/preprocess_sketch.go +++ b/legacy/builder/preprocess_sketch.go @@ -35,9 +35,9 @@ func PreprocessSketchWithArduinoPreprocessor(ctx *types.Context) error { } sourceFile := ctx.SketchBuildPath.Join(ctx.Sketch.MainFile.Base() + ".cpp") - GCCPreprocRunner(ctx, sourceFile, ctx.PreprocPath.Join("ctags_target_for_gcc_minus_e.cpp"), ctx.IncludeFolders) + targetFile := ctx.PreprocPath.Join("sketch_merged.cpp") + GCCPreprocRunner(ctx, sourceFile, targetFile, ctx.IncludeFolders) - targetFilePath := ctx.PreprocPath.Join("ctags_target_for_gcc_minus_e.cpp") buildProperties := properties.NewMap() buildProperties.Set("tools.arduino-preprocessor.path", "{runtime.tools.arduino-preprocessor.path}") buildProperties.Set("tools.arduino-preprocessor.cmd.path", "{path}/arduino-preprocessor") @@ -45,7 +45,7 @@ func PreprocessSketchWithArduinoPreprocessor(ctx *types.Context) error { buildProperties.Set("preproc.macros.flags", "-w -x c++ -E -CC") buildProperties.Merge(ctx.BuildProperties) buildProperties.Merge(buildProperties.SubTree("tools").SubTree("arduino-preprocessor")) - buildProperties.SetPath("source_file", targetFilePath) + buildProperties.SetPath("source_file", targetFile) pattern := buildProperties.Get("pattern") if pattern == "" { @@ -79,7 +79,6 @@ func PreprocessSketchWithArduinoPreprocessor(ctx *types.Context) error { result := utils.NormalizeUTF8(buf) //fmt.Printf("PREPROCESSOR OUTPUT:\n%s\n", output) - ctx.Source = string(result) - - return bldr.SketchSaveItemCpp(ctx.Sketch.MainFile, []byte(ctx.Source), ctx.SketchBuildPath) + ctx.SketchSourceAfterArduinoPreprocessing = string(result) + return bldr.SketchSaveItemCpp(ctx.Sketch.MainFile, []byte(ctx.SketchSourceAfterArduinoPreprocessing), ctx.SketchBuildPath) } diff --git a/legacy/builder/prototypes_adder.go b/legacy/builder/prototypes_adder.go index abc816b2763..03a2cffef3a 100644 --- a/legacy/builder/prototypes_adder.go +++ b/legacy/builder/prototypes_adder.go @@ -17,19 +17,20 @@ package builder import ( "fmt" + "strconv" + "strings" + "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" "github.com/arduino/arduino-cli/legacy/builder/utils" - "strconv" - "strings" ) type PrototypesAdder struct{} func (s *PrototypesAdder) Run(ctx *types.Context) error { debugOutput := ctx.DebugPreprocessor - source := ctx.Source + source := ctx.SketchSourceMerged source = strings.Replace(source, "\r\n", "\n", -1) source = strings.Replace(source, "\r", "\n", -1) @@ -61,7 +62,7 @@ func (s *PrototypesAdder) Run(ctx *types.Context) error { } fmt.Println("#END OF PREPROCESSED SOURCE") } - ctx.Source = source + ctx.SketchSourceAfterArduinoPreprocessing = source return nil } diff --git a/legacy/builder/test/builder_test.go b/legacy/builder/test/builder_test.go index 8e088eb50c1..e84e2d402dd 100644 --- a/legacy/builder/test/builder_test.go +++ b/legacy/builder/test/builder_test.go @@ -116,7 +116,7 @@ func TestBuilderEmptySketch(t *testing.T) { exist, err := buildPath.Join(constants.FOLDER_CORE, "HardwareSerial.cpp.o").ExistCheck() NoError(t, err) require.True(t, exist) - exist, err = buildPath.Join(constants.FOLDER_PREPROC, "ctags_target_for_gcc_minus_e.cpp").ExistCheck() + exist, err = buildPath.Join(constants.FOLDER_PREPROC, "sketch_merged.cpp").ExistCheck() NoError(t, err) require.True(t, exist) exist, err = buildPath.Join(constants.FOLDER_SKETCH, "sketch1.ino.cpp.o").ExistCheck() @@ -143,7 +143,7 @@ func TestBuilderBridge(t *testing.T) { exist, err := buildPath.Join(constants.FOLDER_CORE, "HardwareSerial.cpp.o").ExistCheck() NoError(t, err) require.True(t, exist) - exist, err = buildPath.Join(constants.FOLDER_PREPROC, "ctags_target_for_gcc_minus_e.cpp").ExistCheck() + exist, err = buildPath.Join(constants.FOLDER_PREPROC, "sketch_merged.cpp").ExistCheck() NoError(t, err) require.True(t, exist) exist, err = buildPath.Join(constants.FOLDER_SKETCH, "Bridge.ino.cpp.o").ExistCheck() @@ -173,7 +173,7 @@ func TestBuilderSketchWithConfig(t *testing.T) { exist, err := buildPath.Join(constants.FOLDER_CORE, "HardwareSerial.cpp.o").ExistCheck() NoError(t, err) require.True(t, exist) - exist, err = buildPath.Join(constants.FOLDER_PREPROC, "ctags_target_for_gcc_minus_e.cpp").ExistCheck() + exist, err = buildPath.Join(constants.FOLDER_PREPROC, "sketch_merged.cpp").ExistCheck() NoError(t, err) require.True(t, exist) exist, err = buildPath.Join(constants.FOLDER_SKETCH, "sketch_with_config.ino.cpp.o").ExistCheck() @@ -208,7 +208,7 @@ func TestBuilderBridgeTwice(t *testing.T) { exist, err := buildPath.Join(constants.FOLDER_CORE, "HardwareSerial.cpp.o").ExistCheck() NoError(t, err) require.True(t, exist) - exist, err = buildPath.Join(constants.FOLDER_PREPROC, "ctags_target_for_gcc_minus_e.cpp").ExistCheck() + exist, err = buildPath.Join(constants.FOLDER_PREPROC, "sketch_merged.cpp").ExistCheck() NoError(t, err) require.True(t, exist) exist, err = buildPath.Join(constants.FOLDER_SKETCH, "Bridge.ino.cpp.o").ExistCheck() @@ -245,7 +245,7 @@ func TestBuilderBridgeSAM(t *testing.T) { exist, err = buildPath.Join(constants.FOLDER_CORE, "avr", "dtostrf.c.d").ExistCheck() NoError(t, err) require.True(t, exist) - exist, err = buildPath.Join(constants.FOLDER_PREPROC, "ctags_target_for_gcc_minus_e.cpp").ExistCheck() + exist, err = buildPath.Join(constants.FOLDER_PREPROC, "sketch_merged.cpp").ExistCheck() NoError(t, err) require.True(t, exist) exist, err = buildPath.Join(constants.FOLDER_SKETCH, "Bridge.ino.cpp.o").ExistCheck() @@ -286,7 +286,7 @@ func TestBuilderBridgeRedBearLab(t *testing.T) { exist, err := buildPath.Join(constants.FOLDER_CORE, "HardwareSerial.cpp.o").ExistCheck() NoError(t, err) require.True(t, exist) - exist, err = buildPath.Join(constants.FOLDER_PREPROC, "ctags_target_for_gcc_minus_e.cpp").ExistCheck() + exist, err = buildPath.Join(constants.FOLDER_PREPROC, "sketch_merged.cpp").ExistCheck() NoError(t, err) require.True(t, exist) exist, err = buildPath.Join(constants.FOLDER_SKETCH, "Bridge.ino.cpp.o").ExistCheck() diff --git a/legacy/builder/test/ctags_runner_test.go b/legacy/builder/test/ctags_runner_test.go index fe44735d66d..d63a5314a49 100644 --- a/legacy/builder/test/ctags_runner_test.go +++ b/legacy/builder/test/ctags_runner_test.go @@ -31,7 +31,7 @@ func TestCTagsRunner(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) ctx.Verbose = true - ctagsRunner := &builder.CTagsRunner{Source: &ctx.Source, TargetFileName: "ctags_target.cpp"} + ctagsRunner := &builder.CTagsRunner{Source: &ctx.SketchSourceMerged, TargetFileName: "ctags_target.cpp"} commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, &builder.ContainerMergeCopySketchFiles{}, @@ -62,7 +62,7 @@ func TestCTagsRunnerSketchWithClass(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) ctx.Verbose = true - ctagsRunner := &builder.CTagsRunner{Source: &ctx.Source, TargetFileName: "ctags_target.cpp"} + ctagsRunner := &builder.CTagsRunner{Source: &ctx.SketchSourceMerged, TargetFileName: "ctags_target.cpp"} commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, &builder.ContainerMergeCopySketchFiles{}, @@ -91,7 +91,7 @@ func TestCTagsRunnerSketchWithTypename(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) ctx.Verbose = true - ctagsRunner := &builder.CTagsRunner{Source: &ctx.Source, TargetFileName: "ctags_target.cpp"} + ctagsRunner := &builder.CTagsRunner{Source: &ctx.SketchSourceMerged, TargetFileName: "ctags_target.cpp"} commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, &builder.ContainerMergeCopySketchFiles{}, @@ -119,7 +119,7 @@ func TestCTagsRunnerSketchWithNamespace(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) ctx.Verbose = true - ctagsRunner := &builder.CTagsRunner{Source: &ctx.Source, TargetFileName: "ctags_target.cpp"} + ctagsRunner := &builder.CTagsRunner{Source: &ctx.SketchSourceMerged, TargetFileName: "ctags_target.cpp"} commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, &builder.ContainerMergeCopySketchFiles{}, @@ -146,7 +146,7 @@ func TestCTagsRunnerSketchWithTemplates(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) ctx.Verbose = true - ctagsRunner := &builder.CTagsRunner{Source: &ctx.Source, TargetFileName: "ctags_target.cpp"} + ctagsRunner := &builder.CTagsRunner{Source: &ctx.SketchSourceMerged, TargetFileName: "ctags_target.cpp"} commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, &builder.ContainerMergeCopySketchFiles{}, diff --git a/legacy/builder/test/prototypes_adder_test.go b/legacy/builder/test/prototypes_adder_test.go index 67989677ea0..dbac210d086 100644 --- a/legacy/builder/test/prototypes_adder_test.go +++ b/legacy/builder/test/prototypes_adder_test.go @@ -48,7 +48,7 @@ func TestPrototypesAdderBridgeExample(t *testing.T) { } NoError(t, builder.PreprocessSketchWithCtags(ctx)) - require.Contains(t, ctx.Source, "#include \n#line 1 "+quotedSketchLocation+"\n") + require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") require.Equal(t, "#line 33 "+quotedSketchLocation+"\nvoid setup();\n#line 46 "+quotedSketchLocation+"\nvoid loop();\n#line 62 "+quotedSketchLocation+"\nvoid process(BridgeClient client);\n#line 82 "+quotedSketchLocation+"\nvoid digitalCommand(BridgeClient client);\n#line 109 "+quotedSketchLocation+"\nvoid analogCommand(BridgeClient client);\n#line 149 "+quotedSketchLocation+"\nvoid modeCommand(BridgeClient client);\n#line 33 "+quotedSketchLocation+"\n", ctx.PrototypesSection) } @@ -70,7 +70,7 @@ func TestPrototypesAdderSketchWithIfDef(t *testing.T) { NoError(t, builder.PreprocessSketchWithCtags(ctx)) preprocessed := LoadAndInterpolate(t, filepath.Join("SketchWithIfDef", "SketchWithIfDef.preprocessed.txt"), ctx) - require.Equal(t, preprocessed, strings.Replace(ctx.Source, "\r\n", "\n", -1)) + require.Equal(t, preprocessed, strings.Replace(ctx.SketchSourceAfterArduinoPreprocessing, "\r\n", "\n", -1)) } func TestPrototypesAdderBaladuino(t *testing.T) { @@ -91,7 +91,7 @@ func TestPrototypesAdderBaladuino(t *testing.T) { NoError(t, builder.PreprocessSketchWithCtags(ctx)) preprocessed := LoadAndInterpolate(t, filepath.Join("Baladuino", "Baladuino.preprocessed.txt"), ctx) - require.Equal(t, preprocessed, strings.Replace(ctx.Source, "\r\n", "\n", -1)) + require.Equal(t, preprocessed, strings.Replace(ctx.SketchSourceAfterArduinoPreprocessing, "\r\n", "\n", -1)) } func TestPrototypesAdderCharWithEscapedDoubleQuote(t *testing.T) { @@ -112,7 +112,7 @@ func TestPrototypesAdderCharWithEscapedDoubleQuote(t *testing.T) { NoError(t, builder.PreprocessSketchWithCtags(ctx)) preprocessed := LoadAndInterpolate(t, filepath.Join("CharWithEscapedDoubleQuote", "CharWithEscapedDoubleQuote.preprocessed.txt"), ctx) - require.Equal(t, preprocessed, strings.Replace(ctx.Source, "\r\n", "\n", -1)) + require.Equal(t, preprocessed, strings.Replace(ctx.SketchSourceAfterArduinoPreprocessing, "\r\n", "\n", -1)) } func TestPrototypesAdderIncludeBetweenMultilineComment(t *testing.T) { @@ -133,7 +133,7 @@ func TestPrototypesAdderIncludeBetweenMultilineComment(t *testing.T) { NoError(t, builder.PreprocessSketchWithCtags(ctx)) preprocessed := LoadAndInterpolate(t, filepath.Join("IncludeBetweenMultilineComment", "IncludeBetweenMultilineComment.preprocessed.txt"), ctx) - require.Equal(t, preprocessed, strings.Replace(ctx.Source, "\r\n", "\n", -1)) + require.Equal(t, preprocessed, strings.Replace(ctx.SketchSourceAfterArduinoPreprocessing, "\r\n", "\n", -1)) } func TestPrototypesAdderLineContinuations(t *testing.T) { @@ -154,7 +154,7 @@ func TestPrototypesAdderLineContinuations(t *testing.T) { NoError(t, builder.PreprocessSketchWithCtags(ctx)) preprocessed := LoadAndInterpolate(t, filepath.Join("LineContinuations", "LineContinuations.preprocessed.txt"), ctx) - require.Equal(t, preprocessed, strings.Replace(ctx.Source, "\r\n", "\n", -1)) + require.Equal(t, preprocessed, strings.Replace(ctx.SketchSourceAfterArduinoPreprocessing, "\r\n", "\n", -1)) } func TestPrototypesAdderStringWithComment(t *testing.T) { @@ -175,7 +175,7 @@ func TestPrototypesAdderStringWithComment(t *testing.T) { NoError(t, builder.PreprocessSketchWithCtags(ctx)) preprocessed := LoadAndInterpolate(t, filepath.Join("StringWithComment", "StringWithComment.preprocessed.txt"), ctx) - require.Equal(t, preprocessed, strings.Replace(ctx.Source, "\r\n", "\n", -1)) + require.Equal(t, preprocessed, strings.Replace(ctx.SketchSourceAfterArduinoPreprocessing, "\r\n", "\n", -1)) } func TestPrototypesAdderSketchWithStruct(t *testing.T) { @@ -196,7 +196,7 @@ func TestPrototypesAdderSketchWithStruct(t *testing.T) { NoError(t, builder.PreprocessSketchWithCtags(ctx)) preprocessed := LoadAndInterpolate(t, filepath.Join("SketchWithStruct", "SketchWithStruct.preprocessed.txt"), ctx) - obtained := strings.Replace(ctx.Source, "\r\n", "\n", -1) + obtained := strings.Replace(ctx.SketchSourceAfterArduinoPreprocessing, "\r\n", "\n", -1) // ctags based preprocessing removes the space after "dostuff", but this is still OK // TODO: remove this exception when moving to a more powerful parser preprocessed = strings.Replace(preprocessed, "void dostuff (A_NEW_TYPE * bar);", "void dostuff(A_NEW_TYPE * bar);", 1) @@ -224,11 +224,11 @@ func TestPrototypesAdderSketchWithConfig(t *testing.T) { } NoError(t, builder.PreprocessSketchWithCtags(ctx)) - require.Contains(t, ctx.Source, "#include \n#line 1 "+quotedSketchLocation+"\n") + require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") require.Equal(t, "#line 13 "+quotedSketchLocation+"\nvoid setup();\n#line 17 "+quotedSketchLocation+"\nvoid loop();\n#line 13 "+quotedSketchLocation+"\n", ctx.PrototypesSection) preprocessed := LoadAndInterpolate(t, filepath.Join("sketch_with_config", "sketch_with_config.preprocessed.txt"), ctx) - require.Equal(t, preprocessed, strings.Replace(ctx.Source, "\r\n", "\n", -1)) + require.Equal(t, preprocessed, strings.Replace(ctx.SketchSourceAfterArduinoPreprocessing, "\r\n", "\n", -1)) } func TestPrototypesAdderSketchNoFunctionsTwoFiles(t *testing.T) { @@ -251,7 +251,7 @@ func TestPrototypesAdderSketchNoFunctionsTwoFiles(t *testing.T) { } NoError(t, builder.PreprocessSketchWithCtags(ctx)) - require.Contains(t, ctx.Source, "#include \n#line 1 "+quotedSketchLocation+"\n") + require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") require.Equal(t, "", ctx.PrototypesSection) } @@ -275,7 +275,7 @@ func TestPrototypesAdderSketchNoFunctions(t *testing.T) { } NoError(t, builder.PreprocessSketchWithCtags(ctx)) - require.Contains(t, ctx.Source, "#include \n#line 1 "+quotedSketchLocation+"\n") + require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") require.Equal(t, "", ctx.PrototypesSection) } @@ -299,7 +299,7 @@ func TestPrototypesAdderSketchWithDefaultArgs(t *testing.T) { } NoError(t, builder.PreprocessSketchWithCtags(ctx)) - require.Contains(t, ctx.Source, "#include \n#line 1 "+quotedSketchLocation+"\n") + require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") require.Equal(t, "#line 4 "+quotedSketchLocation+"\nvoid setup();\n#line 7 "+quotedSketchLocation+"\nvoid loop();\n#line 1 "+quotedSketchLocation+"\n", ctx.PrototypesSection) } @@ -323,7 +323,7 @@ func TestPrototypesAdderSketchWithInlineFunction(t *testing.T) { } NoError(t, builder.PreprocessSketchWithCtags(ctx)) - require.Contains(t, ctx.Source, "#include \n#line 1 "+quotedSketchLocation+"\n") + require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") expected := "#line 1 " + quotedSketchLocation + "\nvoid setup();\n#line 2 " + quotedSketchLocation + "\nvoid loop();\n#line 4 " + quotedSketchLocation + "\nshort unsigned int testInt();\n#line 8 " + quotedSketchLocation + "\nstatic int8_t testInline();\n#line 12 " + quotedSketchLocation + "\n__attribute__((always_inline)) uint8_t testAttribute();\n#line 1 " + quotedSketchLocation + "\n" obtained := ctx.PrototypesSection @@ -358,7 +358,7 @@ func TestPrototypesAdderSketchWithFunctionSignatureInsideIFDEF(t *testing.T) { } NoError(t, builder.PreprocessSketchWithCtags(ctx)) - require.Contains(t, ctx.Source, "#include \n#line 1 "+quotedSketchLocation+"\n") + require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") require.Equal(t, "#line 1 "+quotedSketchLocation+"\nvoid setup();\n#line 3 "+quotedSketchLocation+"\nvoid loop();\n#line 15 "+quotedSketchLocation+"\nint8_t adalight();\n#line 1 "+quotedSketchLocation+"\n", ctx.PrototypesSection) } @@ -387,7 +387,7 @@ func TestPrototypesAdderSketchWithUSBCON(t *testing.T) { } NoError(t, builder.PreprocessSketchWithCtags(ctx)) - require.Contains(t, ctx.Source, "#include \n#line 1 "+quotedSketchLocation+"\n") + require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") require.Equal(t, "#line 5 "+quotedSketchLocation+"\nvoid ciao();\n#line 10 "+quotedSketchLocation+"\nvoid setup();\n#line 15 "+quotedSketchLocation+"\nvoid loop();\n#line 5 "+quotedSketchLocation+"\n", ctx.PrototypesSection) } @@ -415,7 +415,7 @@ func TestPrototypesAdderSketchWithTypename(t *testing.T) { } NoError(t, builder.PreprocessSketchWithCtags(ctx)) - require.Contains(t, ctx.Source, "#include \n#line 1 "+quotedSketchLocation+"\n") + require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") expected := "#line 6 " + quotedSketchLocation + "\nvoid setup();\n#line 10 " + quotedSketchLocation + "\nvoid loop();\n#line 12 " + quotedSketchLocation + "\ntypename Foo::Bar func();\n#line 6 " + quotedSketchLocation + "\n" obtained := ctx.PrototypesSection // ctags based preprocessing ignores line with typename @@ -445,11 +445,11 @@ func TestPrototypesAdderSketchWithIfDef2(t *testing.T) { } NoError(t, builder.PreprocessSketchWithCtags(ctx)) - require.Contains(t, ctx.Source, "#include \n#line 1 "+quotedSketchLocation+"\n") + require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") require.Equal(t, "#line 5 "+quotedSketchLocation+"\nvoid elseBranch();\n#line 9 "+quotedSketchLocation+"\nvoid f1();\n#line 10 "+quotedSketchLocation+"\nvoid f2();\n#line 12 "+quotedSketchLocation+"\nvoid setup();\n#line 14 "+quotedSketchLocation+"\nvoid loop();\n#line 5 "+quotedSketchLocation+"\n", ctx.PrototypesSection) expectedSource := LoadAndInterpolate(t, filepath.Join("sketch_with_ifdef", "sketch.preprocessed.txt"), ctx) - require.Equal(t, expectedSource, strings.Replace(ctx.Source, "\r\n", "\n", -1)) + require.Equal(t, expectedSource, strings.Replace(ctx.SketchSourceAfterArduinoPreprocessing, "\r\n", "\n", -1)) } func TestPrototypesAdderSketchWithIfDef2SAM(t *testing.T) { @@ -472,11 +472,11 @@ func TestPrototypesAdderSketchWithIfDef2SAM(t *testing.T) { } NoError(t, builder.PreprocessSketchWithCtags(ctx)) - require.Contains(t, ctx.Source, "#include \n#line 1 "+quotedSketchLocation+"\n") + require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") require.Equal(t, "#line 2 "+quotedSketchLocation+"\nvoid ifBranch();\n#line 9 "+quotedSketchLocation+"\nvoid f1();\n#line 10 "+quotedSketchLocation+"\nvoid f2();\n#line 12 "+quotedSketchLocation+"\nvoid setup();\n#line 14 "+quotedSketchLocation+"\nvoid loop();\n#line 2 "+quotedSketchLocation+"\n", ctx.PrototypesSection) expectedSource := LoadAndInterpolate(t, filepath.Join("sketch_with_ifdef", "sketch.preprocessed.SAM.txt"), ctx) - require.Equal(t, expectedSource, strings.Replace(ctx.Source, "\r\n", "\n", -1)) + require.Equal(t, expectedSource, strings.Replace(ctx.SketchSourceAfterArduinoPreprocessing, "\r\n", "\n", -1)) } func TestPrototypesAdderSketchWithConst(t *testing.T) { @@ -499,7 +499,7 @@ func TestPrototypesAdderSketchWithConst(t *testing.T) { } NoError(t, builder.PreprocessSketchWithCtags(ctx)) - require.Contains(t, ctx.Source, "#include \n#line 1 "+quotedSketchLocation+"\n") + require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") require.Equal(t, "#line 1 "+quotedSketchLocation+"\nvoid setup();\n#line 2 "+quotedSketchLocation+"\nvoid loop();\n#line 4 "+quotedSketchLocation+"\nconst __FlashStringHelper* test();\n#line 6 "+quotedSketchLocation+"\nconst int test3();\n#line 8 "+quotedSketchLocation+"\nvolatile __FlashStringHelper* test2();\n#line 10 "+quotedSketchLocation+"\nvolatile int test4();\n#line 1 "+quotedSketchLocation+"\n", ctx.PrototypesSection) } @@ -542,5 +542,5 @@ func TestPrototypesAdderSketchWithSubstringFunctionMember(t *testing.T) { } NoError(t, builder.PreprocessSketchWithCtags(ctx)) - require.Contains(t, ctx.Source, "class Foo {\nint blooper(int x) { return x+1; }\n};\n\nFoo foo;\n\n#line 7 "+quotedSketchLocation+"\nvoid setup();") + require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "class Foo {\nint blooper(int x) { return x+1; }\n};\n\nFoo foo;\n\n#line 7 "+quotedSketchLocation+"\nvoid setup();") } diff --git a/legacy/builder/test/read_file_and_store_in_context_test.go b/legacy/builder/test/read_file_and_store_in_context_test.go index c4e299e2c60..48f5a777dcb 100644 --- a/legacy/builder/test/read_file_and_store_in_context_test.go +++ b/legacy/builder/test/read_file_and_store_in_context_test.go @@ -36,9 +36,9 @@ func TestReadFileAndStoreInContext(t *testing.T) { ctx := &types.Context{} - command := &builder.ReadFileAndStoreInContext{FileToRead: file, Target: &ctx.SourceGccMinusE} + command := &builder.ReadFileAndStoreInContext{FileToRead: file, Target: &ctx.SketchSourceAfterCppPreprocessing} err = command.Run(ctx) NoError(t, err) - require.Equal(t, "test test\nciao", ctx.SourceGccMinusE) + require.Equal(t, "test test\nciao", ctx.SketchSourceAfterCppPreprocessing) } diff --git a/legacy/builder/types/context.go b/legacy/builder/types/context.go index b37cbe5a467..895c5ef043f 100644 --- a/legacy/builder/types/context.go +++ b/legacy/builder/types/context.go @@ -100,12 +100,17 @@ type Context struct { CollectedSourceFiles *UniqueSourceFileQueue - Sketch *sketch.Sketch - Source string - SourceGccMinusE string - + Sketch *sketch.Sketch WarningsLevel string + // Arduino sketch (.ino) to C++ (.cpp) conversion steps: + // 1. Concatenate *.ino files into a single merged source file -> SketchSourceMerged + SketchSourceMerged string + // 2. Run a pass of C++ preprocessor to remove macro definitions and ifdef-ed code -> SketchSourceAfterCppPreprocessing + SketchSourceAfterCppPreprocessing string + // 3. Do the Arduino preprocessing of the sketch (add missing prototypes) -> SketchSourceAfterArduinoPreprocessing + SketchSourceAfterArduinoPreprocessing string + // Libraries handling LibrariesManager *librariesmanager.LibrariesManager LibrariesResolver *librariesresolver.Cpp From e11e53e26384e5ed53d4df52d1ad3745cbdf372a Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 31 Mar 2023 19:57:24 +0200 Subject: [PATCH 06/42] Removed Command wrapper from ContainerMergeCopySketchFiles --- legacy/builder/builder.go | 12 +- .../container_merge_copy_sketch_files.go | 28 ++-- legacy/builder/test/ctags_runner_test.go | 30 ++++- .../test/includes_to_include_folders_test.go | 48 +++++-- legacy/builder/test/prototypes_adder_test.go | 127 +++++++++++++++--- legacy/builder/types/types.go | 6 + 6 files changed, 196 insertions(+), 55 deletions(-) diff --git a/legacy/builder/builder.go b/legacy/builder/builder.go index b55df582310..3e03e00f14a 100644 --- a/legacy/builder/builder.go +++ b/legacy/builder/builder.go @@ -39,6 +39,7 @@ func (s *Builder) Run(ctx *types.Context) error { return err } + var _err error commands := []types.Command{ &ContainerSetupHardwareToolsLibsSketchAndProps{}, @@ -46,7 +47,10 @@ func (s *Builder) Run(ctx *types.Context) error { &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.prebuild", Suffix: ".pattern"}, - &ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), utils.LogIfVerbose(false, tr("Detecting libraries used...")), &ContainerFindIncludes{}, @@ -127,6 +131,7 @@ func (s *Preprocess) Run(ctx *types.Context) error { return err } + var _err error commands := []types.Command{ &ContainerSetupHardwareToolsLibsSketchAndProps{}, @@ -134,7 +139,10 @@ func (s *Preprocess) Run(ctx *types.Context) error { &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.prebuild", Suffix: ".pattern"}, - &ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &ContainerFindIncludes{}, diff --git a/legacy/builder/container_merge_copy_sketch_files.go b/legacy/builder/container_merge_copy_sketch_files.go index d0eeeaedd0a..4e1ac6dd2a8 100644 --- a/legacy/builder/container_merge_copy_sketch_files.go +++ b/legacy/builder/container_merge_copy_sketch_files.go @@ -17,27 +17,19 @@ package builder import ( bldr "github.com/arduino/arduino-cli/arduino/builder" - "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/pkg/errors" + "github.com/arduino/arduino-cli/arduino/sketch" + "github.com/arduino/go-paths-helper" ) -type ContainerMergeCopySketchFiles struct{} - -func (s *ContainerMergeCopySketchFiles) Run(ctx *types.Context) error { - offset, mergedSource, err := bldr.SketchMergeSources(ctx.Sketch, ctx.SourceOverride) - if err != nil { - return err +func CopySketchFilesToBuildPath(sketch *sketch.Sketch, sourceOverrides map[string]string, buildPath *paths.Path) (offset int, mergedSource string, err error) { + if offset, mergedSource, err = bldr.SketchMergeSources(sketch, sourceOverrides); err != nil { + return } - ctx.LineOffset = offset - ctx.SketchSourceMerged = mergedSource - - if err := bldr.SketchSaveItemCpp(ctx.Sketch.MainFile, []byte(mergedSource), ctx.SketchBuildPath); err != nil { - return errors.WithStack(err) + if err = bldr.SketchSaveItemCpp(sketch.MainFile, []byte(mergedSource), buildPath); err != nil { + return } - - if err := bldr.SketchCopyAdditionalFiles(ctx.Sketch, ctx.SketchBuildPath, ctx.SourceOverride); err != nil { - return errors.WithStack(err) + if err = bldr.SketchCopyAdditionalFiles(sketch, buildPath, sourceOverrides); err != nil { + return } - - return nil + return } diff --git a/legacy/builder/test/ctags_runner_test.go b/legacy/builder/test/ctags_runner_test.go index d63a5314a49..25bcd7147d8 100644 --- a/legacy/builder/test/ctags_runner_test.go +++ b/legacy/builder/test/ctags_runner_test.go @@ -32,9 +32,13 @@ func TestCTagsRunner(t *testing.T) { ctx.Verbose = true ctagsRunner := &builder.CTagsRunner{Source: &ctx.SketchSourceMerged, TargetFileName: "ctags_target.cpp"} + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, &builder.PrintUsedLibrariesIfVerbose{}, &builder.WarnAboutArchIncompatibleLibraries{}, @@ -63,9 +67,13 @@ func TestCTagsRunnerSketchWithClass(t *testing.T) { ctx.Verbose = true ctagsRunner := &builder.CTagsRunner{Source: &ctx.SketchSourceMerged, TargetFileName: "ctags_target.cpp"} + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, &builder.PrintUsedLibrariesIfVerbose{}, &builder.WarnAboutArchIncompatibleLibraries{}, @@ -92,9 +100,13 @@ func TestCTagsRunnerSketchWithTypename(t *testing.T) { ctx.Verbose = true ctagsRunner := &builder.CTagsRunner{Source: &ctx.SketchSourceMerged, TargetFileName: "ctags_target.cpp"} + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, &builder.PrintUsedLibrariesIfVerbose{}, &builder.WarnAboutArchIncompatibleLibraries{}, @@ -120,9 +132,13 @@ func TestCTagsRunnerSketchWithNamespace(t *testing.T) { ctx.Verbose = true ctagsRunner := &builder.CTagsRunner{Source: &ctx.SketchSourceMerged, TargetFileName: "ctags_target.cpp"} + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, &builder.PrintUsedLibrariesIfVerbose{}, &builder.WarnAboutArchIncompatibleLibraries{}, @@ -147,9 +163,13 @@ func TestCTagsRunnerSketchWithTemplates(t *testing.T) { ctx.Verbose = true ctagsRunner := &builder.CTagsRunner{Source: &ctx.SketchSourceMerged, TargetFileName: "ctags_target.cpp"} + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, &builder.PrintUsedLibrariesIfVerbose{}, &builder.WarnAboutArchIncompatibleLibraries{}, diff --git a/legacy/builder/test/includes_to_include_folders_test.go b/legacy/builder/test/includes_to_include_folders_test.go index 537b58d327e..a33400eb69b 100644 --- a/legacy/builder/test/includes_to_include_folders_test.go +++ b/legacy/builder/test/includes_to_include_folders_test.go @@ -31,9 +31,13 @@ func TestIncludesToIncludeFolders(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -51,9 +55,13 @@ func TestIncludesToIncludeFoldersSketchWithIfDef(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -70,9 +78,13 @@ func TestIncludesToIncludeFoldersIRremoteLibrary(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -92,9 +104,13 @@ func TestIncludesToIncludeFoldersANewLibrary(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -119,9 +135,13 @@ func TestIncludesToIncludeFoldersDuplicateLibs(t *testing.T) { ctx = prepareBuilderTestContext(t, ctx, paths.New("user_hardware", "my_avr_platform", "avr", "libraries", "SPI", "examples", "BarometricPressureSensor", "BarometricPressureSensor.ino"), "my_avr_platform:avr:custom_yun") defer cleanUpBuilderTestContext(t, ctx) + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -147,9 +167,13 @@ func TestIncludesToIncludeFoldersDuplicateLibsWithConflictingLibsOutsideOfPlatfo ctx = prepareBuilderTestContext(t, ctx, paths.New("user_hardware", "my_avr_platform", "avr", "libraries", "SPI", "examples", "BarometricPressureSensor", "BarometricPressureSensor.ino"), "my_avr_platform:avr:custom_yun") defer cleanUpBuilderTestContext(t, ctx) + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -175,9 +199,13 @@ func TestIncludesToIncludeFoldersDuplicateLibs2(t *testing.T) { ctx = prepareBuilderTestContext(t, ctx, paths.New("sketch_usbhost", "sketch_usbhost.ino"), "arduino:samd:arduino_zero_native") defer cleanUpBuilderTestContext(t, ctx) + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -197,9 +225,13 @@ func TestIncludesToIncludeFoldersSubfolders(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { diff --git a/legacy/builder/test/prototypes_adder_test.go b/legacy/builder/test/prototypes_adder_test.go index dbac210d086..99d8c5ca7fc 100644 --- a/legacy/builder/test/prototypes_adder_test.go +++ b/legacy/builder/test/prototypes_adder_test.go @@ -37,9 +37,13 @@ func TestPrototypesAdderBridgeExample(t *testing.T) { ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -58,9 +62,13 @@ func TestPrototypesAdderSketchWithIfDef(t *testing.T) { ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -79,9 +87,13 @@ func TestPrototypesAdderBaladuino(t *testing.T) { ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -100,9 +112,13 @@ func TestPrototypesAdderCharWithEscapedDoubleQuote(t *testing.T) { ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -121,9 +137,13 @@ func TestPrototypesAdderIncludeBetweenMultilineComment(t *testing.T) { ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -142,9 +162,13 @@ func TestPrototypesAdderLineContinuations(t *testing.T) { ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -163,9 +187,13 @@ func TestPrototypesAdderStringWithComment(t *testing.T) { ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -184,9 +212,13 @@ func TestPrototypesAdderSketchWithStruct(t *testing.T) { ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -213,9 +245,13 @@ func TestPrototypesAdderSketchWithConfig(t *testing.T) { ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -240,9 +276,13 @@ func TestPrototypesAdderSketchNoFunctionsTwoFiles(t *testing.T) { ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -263,10 +303,13 @@ func TestPrototypesAdderSketchNoFunctions(t *testing.T) { sketchLocation := paths.New("sketch_no_functions", "sketch_no_functions.ino") quotedSketchLocation := utils.QuoteCppPath(Abs(t, sketchLocation)) - + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -288,9 +331,13 @@ func TestPrototypesAdderSketchWithDefaultArgs(t *testing.T) { ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -312,9 +359,13 @@ func TestPrototypesAdderSketchWithInlineFunction(t *testing.T) { ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -347,9 +398,13 @@ func TestPrototypesAdderSketchWithFunctionSignatureInsideIFDEF(t *testing.T) { ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -376,9 +431,13 @@ func TestPrototypesAdderSketchWithUSBCON(t *testing.T) { ctx = prepareBuilderTestContext(t, ctx, sketchLocation, "arduino:avr:leonardo") defer cleanUpBuilderTestContext(t, ctx) + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -404,9 +463,13 @@ func TestPrototypesAdderSketchWithTypename(t *testing.T) { ctx = prepareBuilderTestContext(t, ctx, sketchLocation, "arduino:avr:leonardo") defer cleanUpBuilderTestContext(t, ctx) + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -434,9 +497,13 @@ func TestPrototypesAdderSketchWithIfDef2(t *testing.T) { ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -461,9 +528,13 @@ func TestPrototypesAdderSketchWithIfDef2SAM(t *testing.T) { ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -488,9 +559,13 @@ func TestPrototypesAdderSketchWithConst(t *testing.T) { ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -509,9 +584,13 @@ func TestPrototypesAdderSketchWithDosEol(t *testing.T) { ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { @@ -531,9 +610,13 @@ func TestPrototypesAdderSketchWithSubstringFunctionMember(t *testing.T) { ctx.Verbose = true + var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - &builder.ContainerMergeCopySketchFiles{}, + types.BareCommand(func(ctx *types.Context) error { + ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + return _err + }), &builder.ContainerFindIncludes{}, } for _, command := range commands { diff --git a/legacy/builder/types/types.go b/legacy/builder/types/types.go index 855c0295ffb..d23fd306132 100644 --- a/legacy/builder/types/types.go +++ b/legacy/builder/types/types.go @@ -154,3 +154,9 @@ type CTag struct { type Command interface { Run(ctx *Context) error } + +type BareCommand func(ctx *Context) error + +func (cmd BareCommand) Run(ctx *Context) error { + return cmd(ctx) +} From 98cd03a87d8f19f49db4545416ace3c9bc1beadb Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Mon, 3 Apr 2023 10:28:08 +0200 Subject: [PATCH 07/42] Moved builder.CopySketchFilesToBuildPath into proper location --- arduino/builder/sketch.go | 35 +++++++++++---- arduino/builder/sketch_test.go | 13 +++--- legacy/builder/builder.go | 5 ++- .../container_merge_copy_sketch_files.go | 35 --------------- legacy/builder/test/ctags_runner_test.go | 11 ++--- .../test/includes_to_include_folders_test.go | 17 ++++---- legacy/builder/test/prototypes_adder_test.go | 43 ++++++++++--------- 7 files changed, 72 insertions(+), 87 deletions(-) delete mode 100644 legacy/builder/container_merge_copy_sketch_files.go diff --git a/arduino/builder/sketch.go b/arduino/builder/sketch.go index b5a6a97abb9..f8e8039e58a 100644 --- a/arduino/builder/sketch.go +++ b/arduino/builder/sketch.go @@ -34,15 +34,31 @@ var ( tr = i18n.Tr ) -// QuoteCppString returns the given string as a quoted string for use with the C +// quoteCppString returns the given string as a quoted string for use with the C // preprocessor. This adds double quotes around it and escapes any // double quotes and backslashes in the string. -func QuoteCppString(str string) string { +func quoteCppString(str string) string { str = strings.Replace(str, "\\", "\\\\", -1) str = strings.Replace(str, "\"", "\\\"", -1) return "\"" + str + "\"" } +// PrepareSketchBuildPath copies the sketch source files in the build path. +// The .ino files are merged together to create a .cpp file (by the way, the +// .cpp file still needs to be Arduino-preprocessed to compile). +func PrepareSketchBuildPath(sketch *sketch.Sketch, sourceOverrides map[string]string, buildPath *paths.Path) (offset int, mergedSource string, err error) { + if offset, mergedSource, err = sketchMergeSources(sketch, sourceOverrides); err != nil { + return + } + if err = SketchSaveItemCpp(sketch.MainFile, []byte(mergedSource), buildPath); err != nil { + return + } + if err = sketchCopyAdditionalFiles(sketch, buildPath, sourceOverrides); err != nil { + return + } + return +} + // SketchSaveItemCpp saves a preprocessed .cpp sketch file on disk func SketchSaveItemCpp(path *paths.Path, contents []byte, destPath *paths.Path) error { sketchName := path.Base() @@ -59,8 +75,9 @@ func SketchSaveItemCpp(path *paths.Path, contents []byte, destPath *paths.Path) return nil } -// SketchMergeSources merges all the source files included in a sketch -func SketchMergeSources(sk *sketch.Sketch, overrides map[string]string) (int, string, error) { +// sketchMergeSources merges all the .ino source files included in a sketch to produce +// a single .cpp file. +func sketchMergeSources(sk *sketch.Sketch, overrides map[string]string) (int, string, error) { lineOffset := 0 mergedSource := "" @@ -89,7 +106,7 @@ func SketchMergeSources(sk *sketch.Sketch, overrides map[string]string) (int, st lineOffset++ } - mergedSource += "#line 1 " + QuoteCppString(sk.MainFile.String()) + "\n" + mergedSource += "#line 1 " + quoteCppString(sk.MainFile.String()) + "\n" mergedSource += mainSrc + "\n" lineOffset++ @@ -98,16 +115,16 @@ func SketchMergeSources(sk *sketch.Sketch, overrides map[string]string) (int, st if err != nil { return 0, "", err } - mergedSource += "#line 1 " + QuoteCppString(file.String()) + "\n" + mergedSource += "#line 1 " + quoteCppString(file.String()) + "\n" mergedSource += src + "\n" } return lineOffset, mergedSource, nil } -// SketchCopyAdditionalFiles copies the additional files for a sketch to the +// sketchCopyAdditionalFiles copies the additional files for a sketch to the // specified destination directory. -func SketchCopyAdditionalFiles(sketch *sketch.Sketch, destPath *paths.Path, overrides map[string]string) error { +func sketchCopyAdditionalFiles(sketch *sketch.Sketch, destPath *paths.Path, overrides map[string]string) error { if err := destPath.MkdirAll(); err != nil { return errors.Wrap(err, tr("unable to create a folder to save the sketch files")) } @@ -138,7 +155,7 @@ func SketchCopyAdditionalFiles(sketch *sketch.Sketch, destPath *paths.Path, over } // tag each addtional file with the filename of the source it was copied from - sourceBytes = append([]byte("#line 1 "+QuoteCppString(file.String())+"\n"), sourceBytes...) + sourceBytes = append([]byte("#line 1 "+quoteCppString(file.String())+"\n"), sourceBytes...) err = writeIfDifferent(sourceBytes, targetPath) if err != nil { diff --git a/arduino/builder/sketch_test.go b/arduino/builder/sketch_test.go index a8f1cb6b1b0..5466cdff819 100644 --- a/arduino/builder/sketch_test.go +++ b/arduino/builder/sketch_test.go @@ -13,7 +13,7 @@ // Arduino software without disclosing the source code of your own applications. // To purchase a commercial license, send an email to license@arduino.cc. -package builder_test +package builder import ( "fmt" @@ -23,7 +23,6 @@ import ( "strings" "testing" - "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/arduino/sketch" "github.com/arduino/go-paths-helper" "github.com/stretchr/testify/require" @@ -48,7 +47,7 @@ func TestSaveSketch(t *testing.T) { t.Fatalf("unable to read golden file %s: %v", sketchFile, err) } - builder.SketchSaveItemCpp(paths.New(sketchName), source, tmp) + SketchSaveItemCpp(paths.New(sketchName), source, tmp) out, err := tmp.Join(outName).ReadFile() if err != nil { @@ -82,7 +81,7 @@ func TestMergeSketchSources(t *testing.T) { } mergedSources := strings.ReplaceAll(string(mergedBytes), "%s", pathToGoldenSource) - offset, source, err := builder.SketchMergeSources(s, nil) + offset, source, err := sketchMergeSources(s, nil) require.Nil(t, err) require.Equal(t, 2, offset) require.Equal(t, mergedSources, source) @@ -94,7 +93,7 @@ func TestMergeSketchSourcesArduinoIncluded(t *testing.T) { require.NotNil(t, s) // ensure not to include Arduino.h when it's already there - _, source, err := builder.SketchMergeSources(s, nil) + _, source, err := sketchMergeSources(s, nil) require.Nil(t, err) require.Equal(t, 1, strings.Count(source, "")) } @@ -110,7 +109,7 @@ func TestCopyAdditionalFiles(t *testing.T) { // copy the sketch over, create a fake main file we don't care about it // but we need it for `SketchLoad` to succeed later - err = builder.SketchCopyAdditionalFiles(s1, tmp, nil) + err = sketchCopyAdditionalFiles(s1, tmp, nil) require.Nil(t, err) fakeIno := tmp.Join(fmt.Sprintf("%s.ino", tmp.Base())) require.Nil(t, fakeIno.WriteFile([]byte{})) @@ -125,7 +124,7 @@ func TestCopyAdditionalFiles(t *testing.T) { require.Nil(t, err) // copy again - err = builder.SketchCopyAdditionalFiles(s1, tmp, nil) + err = sketchCopyAdditionalFiles(s1, tmp, nil) require.Nil(t, err) // verify file hasn't changed diff --git a/legacy/builder/builder.go b/legacy/builder/builder.go index 3e03e00f14a..b1b1f007c5d 100644 --- a/legacy/builder/builder.go +++ b/legacy/builder/builder.go @@ -19,6 +19,7 @@ import ( "reflect" "time" + "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/i18n" "github.com/arduino/arduino-cli/legacy/builder/phases" "github.com/arduino/arduino-cli/legacy/builder/types" @@ -48,7 +49,7 @@ func (s *Builder) Run(ctx *types.Context) error { &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.prebuild", Suffix: ".pattern"}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), @@ -140,7 +141,7 @@ func (s *Preprocess) Run(ctx *types.Context) error { &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.prebuild", Suffix: ".pattern"}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), diff --git a/legacy/builder/container_merge_copy_sketch_files.go b/legacy/builder/container_merge_copy_sketch_files.go deleted file mode 100644 index 4e1ac6dd2a8..00000000000 --- a/legacy/builder/container_merge_copy_sketch_files.go +++ /dev/null @@ -1,35 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package builder - -import ( - bldr "github.com/arduino/arduino-cli/arduino/builder" - "github.com/arduino/arduino-cli/arduino/sketch" - "github.com/arduino/go-paths-helper" -) - -func CopySketchFilesToBuildPath(sketch *sketch.Sketch, sourceOverrides map[string]string, buildPath *paths.Path) (offset int, mergedSource string, err error) { - if offset, mergedSource, err = bldr.SketchMergeSources(sketch, sourceOverrides); err != nil { - return - } - if err = bldr.SketchSaveItemCpp(sketch.MainFile, []byte(mergedSource), buildPath); err != nil { - return - } - if err = bldr.SketchCopyAdditionalFiles(sketch, buildPath, sourceOverrides); err != nil { - return - } - return -} diff --git a/legacy/builder/test/ctags_runner_test.go b/legacy/builder/test/ctags_runner_test.go index 25bcd7147d8..12194ccf744 100644 --- a/legacy/builder/test/ctags_runner_test.go +++ b/legacy/builder/test/ctags_runner_test.go @@ -19,6 +19,7 @@ import ( "strings" "testing" + bldr "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/legacy/builder" "github.com/arduino/arduino-cli/legacy/builder/types" paths "github.com/arduino/go-paths-helper" @@ -36,7 +37,7 @@ func TestCTagsRunner(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -71,7 +72,7 @@ func TestCTagsRunnerSketchWithClass(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -104,7 +105,7 @@ func TestCTagsRunnerSketchWithTypename(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -136,7 +137,7 @@ func TestCTagsRunnerSketchWithNamespace(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -167,7 +168,7 @@ func TestCTagsRunnerSketchWithTemplates(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, diff --git a/legacy/builder/test/includes_to_include_folders_test.go b/legacy/builder/test/includes_to_include_folders_test.go index a33400eb69b..a19d0ad57f2 100644 --- a/legacy/builder/test/includes_to_include_folders_test.go +++ b/legacy/builder/test/includes_to_include_folders_test.go @@ -20,6 +20,7 @@ import ( "sort" "testing" + bldr "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/legacy/builder" "github.com/arduino/arduino-cli/legacy/builder/types" paths "github.com/arduino/go-paths-helper" @@ -35,7 +36,7 @@ func TestIncludesToIncludeFolders(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -59,7 +60,7 @@ func TestIncludesToIncludeFoldersSketchWithIfDef(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -82,7 +83,7 @@ func TestIncludesToIncludeFoldersIRremoteLibrary(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -108,7 +109,7 @@ func TestIncludesToIncludeFoldersANewLibrary(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -139,7 +140,7 @@ func TestIncludesToIncludeFoldersDuplicateLibs(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -171,7 +172,7 @@ func TestIncludesToIncludeFoldersDuplicateLibsWithConflictingLibsOutsideOfPlatfo commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -203,7 +204,7 @@ func TestIncludesToIncludeFoldersDuplicateLibs2(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -229,7 +230,7 @@ func TestIncludesToIncludeFoldersSubfolders(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, diff --git a/legacy/builder/test/prototypes_adder_test.go b/legacy/builder/test/prototypes_adder_test.go index 99d8c5ca7fc..947f5bdbf02 100644 --- a/legacy/builder/test/prototypes_adder_test.go +++ b/legacy/builder/test/prototypes_adder_test.go @@ -21,6 +21,7 @@ import ( "strings" "testing" + bldr "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/legacy/builder" "github.com/arduino/arduino-cli/legacy/builder/types" "github.com/arduino/arduino-cli/legacy/builder/utils" @@ -41,7 +42,7 @@ func TestPrototypesAdderBridgeExample(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -66,7 +67,7 @@ func TestPrototypesAdderSketchWithIfDef(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -91,7 +92,7 @@ func TestPrototypesAdderBaladuino(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -116,7 +117,7 @@ func TestPrototypesAdderCharWithEscapedDoubleQuote(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -141,7 +142,7 @@ func TestPrototypesAdderIncludeBetweenMultilineComment(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -166,7 +167,7 @@ func TestPrototypesAdderLineContinuations(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -191,7 +192,7 @@ func TestPrototypesAdderStringWithComment(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -216,7 +217,7 @@ func TestPrototypesAdderSketchWithStruct(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -249,7 +250,7 @@ func TestPrototypesAdderSketchWithConfig(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -280,7 +281,7 @@ func TestPrototypesAdderSketchNoFunctionsTwoFiles(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -307,7 +308,7 @@ func TestPrototypesAdderSketchNoFunctions(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -335,7 +336,7 @@ func TestPrototypesAdderSketchWithDefaultArgs(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -363,7 +364,7 @@ func TestPrototypesAdderSketchWithInlineFunction(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -402,7 +403,7 @@ func TestPrototypesAdderSketchWithFunctionSignatureInsideIFDEF(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -435,7 +436,7 @@ func TestPrototypesAdderSketchWithUSBCON(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -467,7 +468,7 @@ func TestPrototypesAdderSketchWithTypename(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -501,7 +502,7 @@ func TestPrototypesAdderSketchWithIfDef2(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -532,7 +533,7 @@ func TestPrototypesAdderSketchWithIfDef2SAM(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -563,7 +564,7 @@ func TestPrototypesAdderSketchWithConst(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -588,7 +589,7 @@ func TestPrototypesAdderSketchWithDosEol(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -614,7 +615,7 @@ func TestPrototypesAdderSketchWithSubstringFunctionMember(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.CopySketchFilesToBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, From c39f992ee4b90b7a9e35778ee2deb306bc19c1b0 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Mon, 3 Apr 2023 11:03:23 +0200 Subject: [PATCH 08/42] legacy: Removed ReadFileAndStoreInContext command --- legacy/builder/container_add_prototypes.go | 7 ++- .../builder/read_file_and_store_in_context.go | 38 ---------------- .../read_file_and_store_in_context_test.go | 44 ------------------- 3 files changed, 6 insertions(+), 83 deletions(-) delete mode 100644 legacy/builder/read_file_and_store_in_context.go delete mode 100644 legacy/builder/test/read_file_and_store_in_context_test.go diff --git a/legacy/builder/container_add_prototypes.go b/legacy/builder/container_add_prototypes.go index ddc6da270f5..bf012ca1a6d 100644 --- a/legacy/builder/container_add_prototypes.go +++ b/legacy/builder/container_add_prototypes.go @@ -47,8 +47,13 @@ func PreprocessSketchWithCtags(ctx *types.Context) error { } } + if src, err := targetFilePath.ReadFile(); err != nil { + return err + } else { + ctx.SketchSourceAfterCppPreprocessing = string(src) + } + commands := []types.Command{ - &ReadFileAndStoreInContext{FileToRead: targetFilePath, Target: &ctx.SketchSourceAfterCppPreprocessing}, &FilterSketchSource{Source: &ctx.SketchSourceAfterCppPreprocessing}, &CTagsRunner{Source: &ctx.SketchSourceAfterCppPreprocessing, TargetFileName: "sketch_merged.cpp"}, &PrototypesAdder{}, diff --git a/legacy/builder/read_file_and_store_in_context.go b/legacy/builder/read_file_and_store_in_context.go deleted file mode 100644 index cdd75896387..00000000000 --- a/legacy/builder/read_file_and_store_in_context.go +++ /dev/null @@ -1,38 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package builder - -import ( - "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/arduino/go-paths-helper" - "github.com/pkg/errors" -) - -type ReadFileAndStoreInContext struct { - FileToRead *paths.Path - Target *string -} - -func (s *ReadFileAndStoreInContext) Run(ctx *types.Context) error { - bytes, err := s.FileToRead.ReadFile() - if err != nil { - return errors.WithStack(err) - } - - *s.Target = string(bytes) - - return nil -} diff --git a/legacy/builder/test/read_file_and_store_in_context_test.go b/legacy/builder/test/read_file_and_store_in_context_test.go deleted file mode 100644 index 48f5a777dcb..00000000000 --- a/legacy/builder/test/read_file_and_store_in_context_test.go +++ /dev/null @@ -1,44 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package test - -import ( - "os" - "testing" - - "github.com/arduino/arduino-cli/legacy/builder" - "github.com/arduino/arduino-cli/legacy/builder/types" - paths "github.com/arduino/go-paths-helper" - "github.com/stretchr/testify/require" -) - -func TestReadFileAndStoreInContext(t *testing.T) { - filePath, err := os.CreateTemp("", "test") - NoError(t, err) - - file := paths.New(filePath.Name()) - defer file.RemoveAll() - - file.WriteFile([]byte("test test\nciao")) - - ctx := &types.Context{} - - command := &builder.ReadFileAndStoreInContext{FileToRead: file, Target: &ctx.SketchSourceAfterCppPreprocessing} - err = command.Run(ctx) - NoError(t, err) - - require.Equal(t, "test test\nciao", ctx.SketchSourceAfterCppPreprocessing) -} From d2e6fe3a1ca200f3a8bdb7d2cc7d29552a38012e Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Mon, 3 Apr 2023 11:15:10 +0200 Subject: [PATCH 09/42] Converted FilterSketchSource into a function --- legacy/builder/container_add_prototypes.go | 4 ++-- legacy/builder/create_cmake_rule.go | 7 +------ legacy/builder/filter_sketch_source.go | 21 ++++++++------------- 3 files changed, 11 insertions(+), 21 deletions(-) diff --git a/legacy/builder/container_add_prototypes.go b/legacy/builder/container_add_prototypes.go index bf012ca1a6d..7fae47a86c8 100644 --- a/legacy/builder/container_add_prototypes.go +++ b/legacy/builder/container_add_prototypes.go @@ -16,6 +16,7 @@ package builder import ( + "bytes" "fmt" bldr "github.com/arduino/arduino-cli/arduino/builder" @@ -50,11 +51,10 @@ func PreprocessSketchWithCtags(ctx *types.Context) error { if src, err := targetFilePath.ReadFile(); err != nil { return err } else { - ctx.SketchSourceAfterCppPreprocessing = string(src) + ctx.SketchSourceAfterCppPreprocessing = FilterSketchSource(ctx.Sketch, bytes.NewReader(src), false) } commands := []types.Command{ - &FilterSketchSource{Source: &ctx.SketchSourceAfterCppPreprocessing}, &CTagsRunner{Source: &ctx.SketchSourceAfterCppPreprocessing, TargetFileName: "sketch_merged.cpp"}, &PrototypesAdder{}, } diff --git a/legacy/builder/create_cmake_rule.go b/legacy/builder/create_cmake_rule.go index 7eab7917638..47ed3199e85 100644 --- a/legacy/builder/create_cmake_rule.go +++ b/legacy/builder/create_cmake_rule.go @@ -119,12 +119,7 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error { } // Use old ctags method to generate export file - commands := []types.Command{ - &FilterSketchSource{Source: &ctx.SketchSourceMerged, RemoveLineMarkers: true}, - } - for _, command := range commands { - command.Run(ctx) - } + ctx.SketchSourceMerged = FilterSketchSource(ctx.Sketch, strings.NewReader(ctx.SketchSourceMerged), true) err = utils.CopyDir(ctx.SketchBuildPath.String(), cmakeFolder.Join("sketch").String(), validExportExtensions) if err != nil { diff --git a/legacy/builder/filter_sketch_source.go b/legacy/builder/filter_sketch_source.go index 507615a55b2..ef7a581b3da 100644 --- a/legacy/builder/filter_sketch_source.go +++ b/legacy/builder/filter_sketch_source.go @@ -17,34 +17,30 @@ package builder import ( "bufio" + "io" "strconv" "strings" "github.com/arduino/go-paths-helper" - "github.com/arduino/arduino-cli/legacy/builder/types" + "github.com/arduino/arduino-cli/arduino/sketch" "github.com/arduino/arduino-cli/legacy/builder/utils" ) -type FilterSketchSource struct { - Source *string - RemoveLineMarkers bool -} - -func (s *FilterSketchSource) Run(ctx *types.Context) error { +func FilterSketchSource(sketch *sketch.Sketch, source io.Reader, removeLineMarkers bool) string { fileNames := paths.NewPathList() - fileNames.Add(ctx.Sketch.MainFile) - fileNames.AddAll(ctx.Sketch.OtherSketchFiles) + fileNames.Add(sketch.MainFile) + fileNames.AddAll(sketch.OtherSketchFiles) inSketch := false filtered := "" - scanner := bufio.NewScanner(strings.NewReader(*s.Source)) + scanner := bufio.NewScanner(source) for scanner.Scan() { line := scanner.Text() if filename := parseLineMarker(line); filename != nil { inSketch = fileNames.Contains(filename) - if inSketch && s.RemoveLineMarkers { + if inSketch && removeLineMarkers { continue } } @@ -54,8 +50,7 @@ func (s *FilterSketchSource) Run(ctx *types.Context) error { } } - *s.Source = filtered - return nil + return filtered } // Parses the given line as a gcc line marker and returns the contained From 98a72e6071917d656dbe53814e70a70778d3d570 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Mon, 3 Apr 2023 11:26:06 +0200 Subject: [PATCH 10/42] Moved a couple of functions in the proper builder package --- arduino/builder/cpp.go | 74 +++++++++++++++++++ arduino/builder/cpp_test.go | 46 ++++++++++++ arduino/builder/sketch.go | 16 +--- legacy/builder/container_add_prototypes.go | 65 ++++++++++++++++- legacy/builder/create_cmake_rule.go | 2 +- legacy/builder/filter_sketch_source.go | 85 ---------------------- legacy/builder/test/utils_test.go | 38 ---------- legacy/builder/utils/utils.go | 45 ------------ 8 files changed, 188 insertions(+), 183 deletions(-) create mode 100644 arduino/builder/cpp.go create mode 100644 arduino/builder/cpp_test.go delete mode 100644 legacy/builder/filter_sketch_source.go diff --git a/arduino/builder/cpp.go b/arduino/builder/cpp.go new file mode 100644 index 00000000000..1054388f43f --- /dev/null +++ b/arduino/builder/cpp.go @@ -0,0 +1,74 @@ +// This file is part of arduino-cli. +// +// Copyright 2023 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +package builder + +import ( + "strings" + "unicode/utf8" +) + +// QuoteCppString returns the given string as a quoted string for use with the C +// preprocessor. This adds double quotes around it and escapes any +// double quotes and backslashes in the string. +func QuoteCppString(str string) string { + str = strings.Replace(str, "\\", "\\\\", -1) + str = strings.Replace(str, "\"", "\\\"", -1) + return "\"" + str + "\"" +} + +// Parse a C-preprocessor string as emitted by the preprocessor. This +// is a string contained in double quotes, with any backslashes or +// quotes escaped with a backslash. If a valid string was present at the +// start of the given line, returns the unquoted string contents, the +// remainder of the line (everything after the closing "), and true. +// Otherwise, returns the empty string, the entire line and false. +func ParseCppString(line string) (string, string, bool) { + // For details about how these strings are output by gcc, see: + // https://github.com/gcc-mirror/gcc/blob/a588355ab948cf551bc9d2b89f18e5ae5140f52c/libcpp/macro.c#L491-L511 + // Note that the documentation suggests all non-printable + // characters are also escaped, but the implementation does not + // actually do this. See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=51259 + if len(line) < 1 || line[0] != '"' { + return "", line, false + } + + i := 1 + res := "" + for { + if i >= len(line) { + return "", line, false + } + + c, width := utf8.DecodeRuneInString(line[i:]) + + switch c { + case '\\': + // Backslash, next character is used unmodified + i += width + if i >= len(line) { + return "", line, false + } + res += string(line[i]) + case '"': + // Quote, end of string + return res, line[i+width:], true + default: + res += string(c) + } + + i += width + } +} diff --git a/arduino/builder/cpp_test.go b/arduino/builder/cpp_test.go new file mode 100644 index 00000000000..84de5f0b254 --- /dev/null +++ b/arduino/builder/cpp_test.go @@ -0,0 +1,46 @@ +package builder_test + +import ( + "testing" + + "github.com/arduino/arduino-cli/arduino/builder" + "github.com/stretchr/testify/require" +) + +func TestParseCppString(t *testing.T) { + _, _, ok := builder.ParseCppString(`foo`) + require.Equal(t, false, ok) + + _, _, ok = builder.ParseCppString(`"foo`) + require.Equal(t, false, ok) + + str, rest, ok := builder.ParseCppString(`"foo"`) + require.Equal(t, true, ok) + require.Equal(t, `foo`, str) + require.Equal(t, ``, rest) + + str, rest, ok = builder.ParseCppString(`"foo\\bar"`) + require.Equal(t, true, ok) + require.Equal(t, `foo\bar`, str) + require.Equal(t, ``, rest) + + str, rest, ok = builder.ParseCppString(`"foo \"is\" quoted and \\\\bar\"\" escaped\\" and "then" some`) + require.Equal(t, true, ok) + require.Equal(t, `foo "is" quoted and \\bar"" escaped\`, str) + require.Equal(t, ` and "then" some`, rest) + + str, rest, ok = builder.ParseCppString(`" !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_abcdefghijklmnopqrstuvwxyz{|}~"`) + require.Equal(t, true, ok) + require.Equal(t, ` !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_abcdefghijklmnopqrstuvwxyz{|}~`, str) + require.Equal(t, ``, rest) + + str, rest, ok = builder.ParseCppString(`"/home/ççç/"`) + require.Equal(t, true, ok) + require.Equal(t, `/home/ççç/`, str) + require.Equal(t, ``, rest) + + str, rest, ok = builder.ParseCppString(`"/home/ççç/ /$sdsdd\\"`) + require.Equal(t, true, ok) + require.Equal(t, `/home/ççç/ /$sdsdd\`, str) + require.Equal(t, ``, rest) +} diff --git a/arduino/builder/sketch.go b/arduino/builder/sketch.go index f8e8039e58a..8560178a193 100644 --- a/arduino/builder/sketch.go +++ b/arduino/builder/sketch.go @@ -19,7 +19,6 @@ import ( "bytes" "fmt" "regexp" - "strings" "github.com/arduino/arduino-cli/arduino/sketch" "github.com/arduino/arduino-cli/i18n" @@ -34,15 +33,6 @@ var ( tr = i18n.Tr ) -// quoteCppString returns the given string as a quoted string for use with the C -// preprocessor. This adds double quotes around it and escapes any -// double quotes and backslashes in the string. -func quoteCppString(str string) string { - str = strings.Replace(str, "\\", "\\\\", -1) - str = strings.Replace(str, "\"", "\\\"", -1) - return "\"" + str + "\"" -} - // PrepareSketchBuildPath copies the sketch source files in the build path. // The .ino files are merged together to create a .cpp file (by the way, the // .cpp file still needs to be Arduino-preprocessed to compile). @@ -106,7 +96,7 @@ func sketchMergeSources(sk *sketch.Sketch, overrides map[string]string) (int, st lineOffset++ } - mergedSource += "#line 1 " + quoteCppString(sk.MainFile.String()) + "\n" + mergedSource += "#line 1 " + QuoteCppString(sk.MainFile.String()) + "\n" mergedSource += mainSrc + "\n" lineOffset++ @@ -115,7 +105,7 @@ func sketchMergeSources(sk *sketch.Sketch, overrides map[string]string) (int, st if err != nil { return 0, "", err } - mergedSource += "#line 1 " + quoteCppString(file.String()) + "\n" + mergedSource += "#line 1 " + QuoteCppString(file.String()) + "\n" mergedSource += src + "\n" } @@ -155,7 +145,7 @@ func sketchCopyAdditionalFiles(sketch *sketch.Sketch, destPath *paths.Path, over } // tag each addtional file with the filename of the source it was copied from - sourceBytes = append([]byte("#line 1 "+quoteCppString(file.String())+"\n"), sourceBytes...) + sourceBytes = append([]byte("#line 1 "+QuoteCppString(file.String())+"\n"), sourceBytes...) err = writeIfDifferent(sourceBytes, targetPath) if err != nil { diff --git a/legacy/builder/container_add_prototypes.go b/legacy/builder/container_add_prototypes.go index 7fae47a86c8..fe5e9064742 100644 --- a/legacy/builder/container_add_prototypes.go +++ b/legacy/builder/container_add_prototypes.go @@ -16,11 +16,17 @@ package builder import ( + "bufio" "bytes" "fmt" + "io" + "strconv" + "strings" bldr "github.com/arduino/arduino-cli/arduino/builder" + "github.com/arduino/arduino-cli/arduino/sketch" "github.com/arduino/arduino-cli/legacy/builder/types" + "github.com/arduino/go-paths-helper" "github.com/pkg/errors" ) @@ -51,7 +57,7 @@ func PreprocessSketchWithCtags(ctx *types.Context) error { if src, err := targetFilePath.ReadFile(); err != nil { return err } else { - ctx.SketchSourceAfterCppPreprocessing = FilterSketchSource(ctx.Sketch, bytes.NewReader(src), false) + ctx.SketchSourceAfterCppPreprocessing = filterSketchSource(ctx.Sketch, bytes.NewReader(src), false) } commands := []types.Command{ @@ -73,3 +79,60 @@ func PreprocessSketchWithCtags(ctx *types.Context) error { return nil } + +func filterSketchSource(sketch *sketch.Sketch, source io.Reader, removeLineMarkers bool) string { + fileNames := paths.NewPathList() + fileNames.Add(sketch.MainFile) + fileNames.AddAll(sketch.OtherSketchFiles) + + inSketch := false + filtered := "" + + scanner := bufio.NewScanner(source) + for scanner.Scan() { + line := scanner.Text() + if filename := parseLineMarker(line); filename != nil { + inSketch = fileNames.Contains(filename) + if inSketch && removeLineMarkers { + continue + } + } + + if inSketch { + filtered += line + "\n" + } + } + + return filtered +} + +// Parses the given line as a gcc line marker and returns the contained +// filename. +func parseLineMarker(line string) *paths.Path { + // A line marker contains the line number and filename and looks like: + // # 123 /path/to/file.cpp + // It can be followed by zero or more flag number that indicate the + // preprocessor state and can be ignored. + // For exact details on this format, see: + // https://github.com/gcc-mirror/gcc/blob/edd716b6b1caa1a5cb320a8cd7f626f30198e098/gcc/c-family/c-ppoutput.c#L413-L415 + + split := strings.SplitN(line, " ", 3) + if len(split) < 3 || len(split[0]) == 0 || split[0][0] != '#' { + return nil + } + + _, err := strconv.Atoi(split[1]) + if err != nil { + return nil + } + + // If we get here, we found a # followed by a line number, so + // assume this is a line marker and see if the rest of the line + // starts with a string containing the filename + str, rest, ok := bldr.ParseCppString(split[2]) + + if ok && (rest == "" || rest[0] == ' ') { + return paths.New(str) + } + return nil +} diff --git a/legacy/builder/create_cmake_rule.go b/legacy/builder/create_cmake_rule.go index 47ed3199e85..7c848ef8544 100644 --- a/legacy/builder/create_cmake_rule.go +++ b/legacy/builder/create_cmake_rule.go @@ -119,7 +119,7 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error { } // Use old ctags method to generate export file - ctx.SketchSourceMerged = FilterSketchSource(ctx.Sketch, strings.NewReader(ctx.SketchSourceMerged), true) + ctx.SketchSourceMerged = filterSketchSource(ctx.Sketch, strings.NewReader(ctx.SketchSourceMerged), true) err = utils.CopyDir(ctx.SketchBuildPath.String(), cmakeFolder.Join("sketch").String(), validExportExtensions) if err != nil { diff --git a/legacy/builder/filter_sketch_source.go b/legacy/builder/filter_sketch_source.go deleted file mode 100644 index ef7a581b3da..00000000000 --- a/legacy/builder/filter_sketch_source.go +++ /dev/null @@ -1,85 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package builder - -import ( - "bufio" - "io" - "strconv" - "strings" - - "github.com/arduino/go-paths-helper" - - "github.com/arduino/arduino-cli/arduino/sketch" - "github.com/arduino/arduino-cli/legacy/builder/utils" -) - -func FilterSketchSource(sketch *sketch.Sketch, source io.Reader, removeLineMarkers bool) string { - fileNames := paths.NewPathList() - fileNames.Add(sketch.MainFile) - fileNames.AddAll(sketch.OtherSketchFiles) - - inSketch := false - filtered := "" - - scanner := bufio.NewScanner(source) - for scanner.Scan() { - line := scanner.Text() - if filename := parseLineMarker(line); filename != nil { - inSketch = fileNames.Contains(filename) - if inSketch && removeLineMarkers { - continue - } - } - - if inSketch { - filtered += line + "\n" - } - } - - return filtered -} - -// Parses the given line as a gcc line marker and returns the contained -// filename. -func parseLineMarker(line string) *paths.Path { - // A line marker contains the line number and filename and looks like: - // # 123 /path/to/file.cpp - // It can be followed by zero or more flag number that indicate the - // preprocessor state and can be ignored. - // For exact details on this format, see: - // https://github.com/gcc-mirror/gcc/blob/edd716b6b1caa1a5cb320a8cd7f626f30198e098/gcc/c-family/c-ppoutput.c#L413-L415 - - split := strings.SplitN(line, " ", 3) - if len(split) < 3 || len(split[0]) == 0 || split[0][0] != '#' { - return nil - } - - _, err := strconv.Atoi(split[1]) - if err != nil { - return nil - } - - // If we get here, we found a # followed by a line number, so - // assume this is a line marker and see if the rest of the line - // starts with a string containing the filename - str, rest, ok := utils.ParseCppString(split[2]) - - if ok && (rest == "" || rest[0] == ' ') { - return paths.New(str) - } - return nil -} diff --git a/legacy/builder/test/utils_test.go b/legacy/builder/test/utils_test.go index b6f88555283..71a70be9dd6 100644 --- a/legacy/builder/test/utils_test.go +++ b/legacy/builder/test/utils_test.go @@ -65,41 +65,3 @@ func TestQuoteCppString(t *testing.T) { require.Equal(t, expected, utils.QuoteCppString(input)) } } - -func TestParseCppString(t *testing.T) { - _, _, ok := utils.ParseCppString(`foo`) - require.Equal(t, false, ok) - - _, _, ok = utils.ParseCppString(`"foo`) - require.Equal(t, false, ok) - - str, rest, ok := utils.ParseCppString(`"foo"`) - require.Equal(t, true, ok) - require.Equal(t, `foo`, str) - require.Equal(t, ``, rest) - - str, rest, ok = utils.ParseCppString(`"foo\\bar"`) - require.Equal(t, true, ok) - require.Equal(t, `foo\bar`, str) - require.Equal(t, ``, rest) - - str, rest, ok = utils.ParseCppString(`"foo \"is\" quoted and \\\\bar\"\" escaped\\" and "then" some`) - require.Equal(t, true, ok) - require.Equal(t, `foo "is" quoted and \\bar"" escaped\`, str) - require.Equal(t, ` and "then" some`, rest) - - str, rest, ok = utils.ParseCppString(`" !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_abcdefghijklmnopqrstuvwxyz{|}~"`) - require.Equal(t, true, ok) - require.Equal(t, ` !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_abcdefghijklmnopqrstuvwxyz{|}~`, str) - require.Equal(t, ``, rest) - - str, rest, ok = utils.ParseCppString(`"/home/ççç/"`) - require.Equal(t, true, ok) - require.Equal(t, `/home/ççç/`, str) - require.Equal(t, ``, rest) - - str, rest, ok = utils.ParseCppString(`"/home/ççç/ /$sdsdd\\"`) - require.Equal(t, true, ok) - require.Equal(t, `/home/ççç/ /$sdsdd\`, str) - require.Equal(t, ``, rest) -} diff --git a/legacy/builder/utils/utils.go b/legacy/builder/utils/utils.go index 87113cce633..0496ccff27a 100644 --- a/legacy/builder/utils/utils.go +++ b/legacy/builder/utils/utils.go @@ -26,7 +26,6 @@ import ( "path/filepath" "strings" "unicode" - "unicode/utf8" "github.com/arduino/arduino-cli/i18n" "github.com/arduino/arduino-cli/legacy/builder/gohasissues" @@ -309,50 +308,6 @@ func QuoteCppPath(path *paths.Path) string { return QuoteCppString(path.String()) } -// Parse a C-preprocessor string as emitted by the preprocessor. This -// is a string contained in double quotes, with any backslashes or -// quotes escaped with a backslash. If a valid string was present at the -// start of the given line, returns the unquoted string contents, the -// remainder of the line (everything after the closing "), and true. -// Otherwise, returns the empty string, the entire line and false. -func ParseCppString(line string) (string, string, bool) { - // For details about how these strings are output by gcc, see: - // https://github.com/gcc-mirror/gcc/blob/a588355ab948cf551bc9d2b89f18e5ae5140f52c/libcpp/macro.c#L491-L511 - // Note that the documentation suggests all non-printable - // characters are also escaped, but the implementation does not - // actually do this. See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=51259 - if len(line) < 1 || line[0] != '"' { - return "", line, false - } - - i := 1 - res := "" - for { - if i >= len(line) { - return "", line, false - } - - c, width := utf8.DecodeRuneInString(line[i:]) - - switch c { - case '\\': - // Backslash, next character is used unmodified - i += width - if i >= len(line) { - return "", line, false - } - res += string(line[i]) - case '"': - // Quote, end of string - return res, line[i+width:], true - default: - res += string(c) - } - - i += width - } -} - // Normalizes an UTF8 byte slice // TODO: use it more often troughout all the project (maybe on logger interface?) func NormalizeUTF8(buf []byte) []byte { From dcfdc5480850c1cf1bf234041d542d238a65b3d4 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Mon, 3 Apr 2023 11:32:40 +0200 Subject: [PATCH 11/42] Removed unused class --- legacy/builder/types/accessories.go | 34 ----------------------------- 1 file changed, 34 deletions(-) diff --git a/legacy/builder/types/accessories.go b/legacy/builder/types/accessories.go index b0c85d37e20..626380e89a0 100644 --- a/legacy/builder/types/accessories.go +++ b/legacy/builder/types/accessories.go @@ -15,11 +15,6 @@ package types -import ( - "bytes" - "sync" -) - type UniqueStringQueue []string func (queue UniqueStringQueue) Len() int { return len(queue) } @@ -65,32 +60,3 @@ func (queue *UniqueSourceFileQueue) Pop() SourceFile { func (queue *UniqueSourceFileQueue) Empty() bool { return queue.Len() == 0 } - -type BufferedUntilNewLineWriter struct { - PrintFunc PrintFunc - Buffer bytes.Buffer - lock sync.Mutex -} - -type PrintFunc func([]byte) - -func (w *BufferedUntilNewLineWriter) Write(p []byte) (n int, err error) { - w.lock.Lock() - defer w.lock.Unlock() - - writtenToBuffer, err := w.Buffer.Write(p) - return writtenToBuffer, err -} - -func (w *BufferedUntilNewLineWriter) Flush() { - w.lock.Lock() - defer w.lock.Unlock() - - remainingBytes := w.Buffer.Bytes() - if len(remainingBytes) > 0 { - if remainingBytes[len(remainingBytes)-1] != '\n' { - remainingBytes = append(remainingBytes, '\n') - } - w.PrintFunc(remainingBytes) - } -} From e8062531ce3663aae9c842b8d715355d8628eb88 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Mon, 3 Apr 2023 12:01:27 +0200 Subject: [PATCH 12/42] Replaced two accessory functions with stdlib functions --- .../go/golang.org/x/exp/constraints.dep.yml | 63 +++++++++++++++++++ .licenses/go/golang.org/x/exp/slices.dep.yml | 62 ++++++++++++++++++ legacy/builder/types/accessories.go | 9 ++- legacy/builder/types/utils.go | 35 ----------- 4 files changed, 132 insertions(+), 37 deletions(-) create mode 100644 .licenses/go/golang.org/x/exp/constraints.dep.yml create mode 100644 .licenses/go/golang.org/x/exp/slices.dep.yml delete mode 100644 legacy/builder/types/utils.go diff --git a/.licenses/go/golang.org/x/exp/constraints.dep.yml b/.licenses/go/golang.org/x/exp/constraints.dep.yml new file mode 100644 index 00000000000..14cd0ef4868 --- /dev/null +++ b/.licenses/go/golang.org/x/exp/constraints.dep.yml @@ -0,0 +1,63 @@ +--- +name: golang.org/x/exp/constraints +version: v0.0.0-20230321023759-10a507213a29 +type: go +summary: Package constraints defines a set of useful constraints to be used with type + parameters. +homepage: https://pkg.go.dev/golang.org/x/exp/constraints +license: bsd-3-clause +licenses: +- sources: exp@v0.0.0-20230321023759-10a507213a29/LICENSE + text: | + Copyright (c) 2009 The Go Authors. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +- sources: exp@v0.0.0-20230321023759-10a507213a29/PATENTS + text: | + Additional IP Rights Grant (Patents) + + "This implementation" means the copyrightable works distributed by + Google as part of the Go project. + + Google hereby grants to You a perpetual, worldwide, non-exclusive, + no-charge, royalty-free, irrevocable (except as stated in this section) + patent license to make, have made, use, offer to sell, sell, import, + transfer and otherwise run, modify and propagate the contents of this + implementation of Go, where such license applies only to those patent + claims, both currently owned or controlled by Google and acquired in + the future, licensable by Google that are necessarily infringed by this + implementation of Go. This grant does not include claims that would be + infringed only as a consequence of further modification of this + implementation. If you or your agent or exclusive licensee institute or + order or agree to the institution of patent litigation against any + entity (including a cross-claim or counterclaim in a lawsuit) alleging + that this implementation of Go or any code incorporated within this + implementation of Go constitutes direct or contributory patent + infringement, or inducement of patent infringement, then any patent + rights granted to you under this License for this implementation of Go + shall terminate as of the date such litigation is filed. +notices: [] diff --git a/.licenses/go/golang.org/x/exp/slices.dep.yml b/.licenses/go/golang.org/x/exp/slices.dep.yml new file mode 100644 index 00000000000..8fca8863335 --- /dev/null +++ b/.licenses/go/golang.org/x/exp/slices.dep.yml @@ -0,0 +1,62 @@ +--- +name: golang.org/x/exp/slices +version: v0.0.0-20230321023759-10a507213a29 +type: go +summary: Package slices defines various functions useful with slices of any type. +homepage: https://pkg.go.dev/golang.org/x/exp/slices +license: bsd-3-clause +licenses: +- sources: exp@v0.0.0-20230321023759-10a507213a29/LICENSE + text: | + Copyright (c) 2009 The Go Authors. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +- sources: exp@v0.0.0-20230321023759-10a507213a29/PATENTS + text: | + Additional IP Rights Grant (Patents) + + "This implementation" means the copyrightable works distributed by + Google as part of the Go project. + + Google hereby grants to You a perpetual, worldwide, non-exclusive, + no-charge, royalty-free, irrevocable (except as stated in this section) + patent license to make, have made, use, offer to sell, sell, import, + transfer and otherwise run, modify and propagate the contents of this + implementation of Go, where such license applies only to those patent + claims, both currently owned or controlled by Google and acquired in + the future, licensable by Google that are necessarily infringed by this + implementation of Go. This grant does not include claims that would be + infringed only as a consequence of further modification of this + implementation. If you or your agent or exclusive licensee institute or + order or agree to the institution of patent litigation against any + entity (including a cross-claim or counterclaim in a lawsuit) alleging + that this implementation of Go or any code incorporated within this + implementation of Go constitutes direct or contributory patent + infringement, or inducement of patent infringement, then any patent + rights granted to you under this License for this implementation of Go + shall terminate as of the date such litigation is filed. +notices: [] diff --git a/legacy/builder/types/accessories.go b/legacy/builder/types/accessories.go index 626380e89a0..a110977dde5 100644 --- a/legacy/builder/types/accessories.go +++ b/legacy/builder/types/accessories.go @@ -15,6 +15,8 @@ package types +import "golang.org/x/exp/slices" + type UniqueStringQueue []string func (queue UniqueStringQueue) Len() int { return len(queue) } @@ -22,7 +24,7 @@ func (queue UniqueStringQueue) Less(i, j int) bool { return false } func (queue UniqueStringQueue) Swap(i, j int) { panic("Who called me?!?") } func (queue *UniqueStringQueue) Push(value string) { - if !sliceContains(*queue, value) { + if !slices.Contains(*queue, value) { *queue = append(*queue, value) } } @@ -45,7 +47,10 @@ func (queue UniqueSourceFileQueue) Less(i, j int) bool { return false } func (queue UniqueSourceFileQueue) Swap(i, j int) { panic("Who called me?!?") } func (queue *UniqueSourceFileQueue) Push(value SourceFile) { - if !sliceContainsSourceFile(*queue, value) { + equals := func(elem SourceFile) bool { + return elem.Origin == value.Origin && elem.RelativePath.EqualsTo(value.RelativePath) + } + if !slices.ContainsFunc(*queue, equals) { *queue = append(*queue, value) } } diff --git a/legacy/builder/types/utils.go b/legacy/builder/types/utils.go deleted file mode 100644 index 7102cd555ca..00000000000 --- a/legacy/builder/types/utils.go +++ /dev/null @@ -1,35 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package types - -// duplication of utils.SliceContains! Thanks golang! Why? Because with golang you can't have import cycles -func sliceContains(slice []string, target string) bool { - for _, value := range slice { - if value == target { - return true - } - } - return false -} - -func sliceContainsSourceFile(slice []SourceFile, target SourceFile) bool { - for _, elem := range slice { - if elem.Origin == target.Origin && elem.RelativePath.EqualsTo(target.RelativePath) { - return true - } - } - return false -} From fbd1e50905f9517d8bf0962edec6d27a5d0ca476 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Mon, 3 Apr 2023 12:06:59 +0200 Subject: [PATCH 13/42] Movec ctags-related structs in ctags package --- legacy/builder/ctags/ctags_has_issues.go | 12 +++--- legacy/builder/ctags/ctags_parser.go | 37 +++++++++++++------ legacy/builder/ctags/ctags_parser_test.go | 4 +- legacy/builder/ctags/ctags_to_prototypes.go | 29 ++++++++++----- .../builder/ctags/ctags_to_prototypes_test.go | 3 +- legacy/builder/prototypes_adder.go | 7 ++-- legacy/builder/types/context.go | 3 +- legacy/builder/types/types.go | 30 --------------- 8 files changed, 59 insertions(+), 66 deletions(-) diff --git a/legacy/builder/ctags/ctags_has_issues.go b/legacy/builder/ctags/ctags_has_issues.go index a6cda249865..003aaa035da 100644 --- a/legacy/builder/ctags/ctags_has_issues.go +++ b/legacy/builder/ctags/ctags_has_issues.go @@ -19,8 +19,6 @@ import ( "bufio" "os" "strings" - - "github.com/arduino/arduino-cli/legacy/builder/types" ) func (p *CTagsParser) FixCLinkageTagsDeclarations() { @@ -42,7 +40,7 @@ func sliceContainsInt(s []int, e int) bool { return false } -func (p *CTagsParser) prototypeAndCodeDontMatch(tag *types.CTag) bool { +func (p *CTagsParser) prototypeAndCodeDontMatch(tag *CTag) bool { if tag.SkipMe { return true } @@ -107,7 +105,7 @@ func (p *CTagsParser) prototypeAndCodeDontMatch(tag *types.CTag) bool { return ret == -1 } -func findTemplateMultiline(tag *types.CTag) string { +func findTemplateMultiline(tag *CTag) string { code, _ := getFunctionProtoUntilTemplateToken(tag, tag.Code) return removeEverythingAfterClosingRoundBracket(code) } @@ -117,7 +115,7 @@ func removeEverythingAfterClosingRoundBracket(s string) string { return s[0 : n+1] } -func getFunctionProtoUntilTemplateToken(tag *types.CTag, code string) (string, int) { +func getFunctionProtoUntilTemplateToken(tag *CTag, code string) (string, int) { /* FIXME I'm ugly */ line := 0 @@ -150,7 +148,7 @@ func getFunctionProtoUntilTemplateToken(tag *types.CTag, code string) (string, i return code, line } -func getFunctionProtoWithNPreviousCharacters(tag *types.CTag, code string, n int) (string, int) { +func getFunctionProtoWithNPreviousCharacters(tag *CTag, code string, n int) (string, int) { /* FIXME I'm ugly */ expectedPrototypeLen := len(code) + n @@ -216,7 +214,7 @@ func removeComments(text string, multilinecomment bool) (string, bool) { /* This function scans the source files searching for "extern C" context * It save the line numbers in a map filename -> {lines...} */ -func (p *CTagsParser) FindCLinkageLines(tags []*types.CTag) map[string][]int { +func (p *CTagsParser) FindCLinkageLines(tags []*CTag) map[string][]int { lines := make(map[string][]int) diff --git a/legacy/builder/ctags/ctags_parser.go b/legacy/builder/ctags/ctags_parser.go index a2c49851338..61076101be5 100644 --- a/legacy/builder/ctags/ctags_parser.go +++ b/legacy/builder/ctags/ctags_parser.go @@ -20,8 +20,6 @@ import ( "strings" "github.com/arduino/go-paths-helper" - - "github.com/arduino/arduino-cli/legacy/builder/types" ) const KIND_PROTOTYPE = "prototype" @@ -39,11 +37,28 @@ var KNOWN_TAG_KINDS = map[string]bool{ } type CTagsParser struct { - tags []*types.CTag + tags []*CTag mainFile *paths.Path } -func (p *CTagsParser) Parse(ctagsOutput []byte, mainFile *paths.Path) []*types.CTag { +type CTag struct { + FunctionName string + Kind string + Line int + Code string + Class string + Struct string + Namespace string + Filename string + Typeref string + SkipMe bool + Signature string + + Prototype string + PrototypeModifiers string +} + +func (p *CTagsParser) Parse(ctagsOutput []byte, mainFile *paths.Path) []*CTag { rows := strings.Split(string(ctagsOutput), "\n") rows = removeEmpty(rows) @@ -71,7 +86,7 @@ func (p *CTagsParser) addPrototypes() { } } -func addPrototype(tag *types.CTag) { +func addPrototype(tag *CTag) { if strings.Index(tag.Prototype, TEMPLATE) == 0 { if strings.Index(tag.Code, TEMPLATE) == 0 { code := tag.Code @@ -129,7 +144,7 @@ func (p *CTagsParser) skipDuplicates() { } } -type skipFuncType func(tag *types.CTag) bool +type skipFuncType func(tag *CTag) bool func (p *CTagsParser) skipTagsWhere(skipFunc skipFuncType) { for _, tag := range p.tags { @@ -153,11 +168,11 @@ func removeSpacesAndTabs(s string) string { return s } -func tagIsUnhandled(tag *types.CTag) bool { +func tagIsUnhandled(tag *CTag) bool { return !isHandled(tag) } -func isHandled(tag *types.CTag) bool { +func isHandled(tag *CTag) bool { if tag.Class != "" { return false } @@ -170,12 +185,12 @@ func isHandled(tag *types.CTag) bool { return true } -func tagIsUnknown(tag *types.CTag) bool { +func tagIsUnknown(tag *CTag) bool { return !KNOWN_TAG_KINDS[tag.Kind] } -func parseTag(row string) *types.CTag { - tag := &types.CTag{} +func parseTag(row string) *CTag { + tag := &CTag{} parts := strings.Split(row, "\t") tag.FunctionName = parts[0] diff --git a/legacy/builder/ctags/ctags_parser_test.go b/legacy/builder/ctags/ctags_parser_test.go index 49711c27c18..d3c4abe6b54 100644 --- a/legacy/builder/ctags/ctags_parser_test.go +++ b/legacy/builder/ctags/ctags_parser_test.go @@ -20,12 +20,10 @@ import ( "path/filepath" "testing" - "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/stretchr/testify/require" ) -func produceTags(t *testing.T, filename string) []*types.CTag { +func produceTags(t *testing.T, filename string) []*CTag { bytes, err := os.ReadFile(filepath.Join("test_data", filename)) require.NoError(t, err) diff --git a/legacy/builder/ctags/ctags_to_prototypes.go b/legacy/builder/ctags/ctags_to_prototypes.go index dea077ff0aa..d5f750d2da7 100644 --- a/legacy/builder/ctags/ctags_to_prototypes.go +++ b/legacy/builder/ctags/ctags_to_prototypes.go @@ -16,12 +16,23 @@ package ctags import ( + "strconv" "strings" - - "github.com/arduino/arduino-cli/legacy/builder/types" ) -func (p *CTagsParser) GeneratePrototypes() ([]*types.Prototype, int) { +type Prototype struct { + FunctionName string + File string + Prototype string + Modifiers string + Line int +} + +func (proto *Prototype) String() string { + return proto.Modifiers + " " + proto.Prototype + " @ " + strconv.Itoa(proto.Line) +} + +func (p *CTagsParser) GeneratePrototypes() ([]*Prototype, int) { return p.toPrototypes(), p.findLineWhereToInsertPrototypes() } @@ -53,7 +64,7 @@ func (p *CTagsParser) firstFunctionPointerUsedAsArgument() int { return -1 } -func functionNameUsedAsFunctionPointerIn(tag *types.CTag, functionTags []*types.CTag) bool { +func functionNameUsedAsFunctionPointerIn(tag *CTag, functionTags []*CTag) bool { for _, functionTag := range functionTags { if tag.Line != functionTag.Line && strings.Contains(tag.Code, "&"+functionTag.FunctionName) { return true @@ -65,8 +76,8 @@ func functionNameUsedAsFunctionPointerIn(tag *types.CTag, functionTags []*types. return false } -func (p *CTagsParser) collectFunctions() []*types.CTag { - functionTags := []*types.CTag{} +func (p *CTagsParser) collectFunctions() []*CTag { + functionTags := []*CTag{} for _, tag := range p.tags { if tag.Kind == KIND_FUNCTION && !tag.SkipMe { functionTags = append(functionTags, tag) @@ -84,14 +95,14 @@ func (p *CTagsParser) firstFunctionAtLine() int { return -1 } -func (p *CTagsParser) toPrototypes() []*types.Prototype { - prototypes := []*types.Prototype{} +func (p *CTagsParser) toPrototypes() []*Prototype { + prototypes := []*Prototype{} for _, tag := range p.tags { if strings.TrimSpace(tag.Prototype) == "" { continue } if !tag.SkipMe { - prototype := &types.Prototype{ + prototype := &Prototype{ FunctionName: tag.FunctionName, File: tag.Filename, Prototype: tag.Prototype, diff --git a/legacy/builder/ctags/ctags_to_prototypes_test.go b/legacy/builder/ctags/ctags_to_prototypes_test.go index 7dcea2a0ab1..2e36c343b53 100644 --- a/legacy/builder/ctags/ctags_to_prototypes_test.go +++ b/legacy/builder/ctags/ctags_to_prototypes_test.go @@ -20,12 +20,11 @@ import ( "path/filepath" "testing" - "github.com/arduino/arduino-cli/legacy/builder/types" paths "github.com/arduino/go-paths-helper" "github.com/stretchr/testify/require" ) -func producePrototypes(t *testing.T, filename string, mainFile string) ([]*types.Prototype, int) { +func producePrototypes(t *testing.T, filename string, mainFile string) ([]*Prototype, int) { bytes, err := os.ReadFile(filepath.Join("test_data", filename)) require.NoError(t, err) diff --git a/legacy/builder/prototypes_adder.go b/legacy/builder/prototypes_adder.go index 03a2cffef3a..8a912f12993 100644 --- a/legacy/builder/prototypes_adder.go +++ b/legacy/builder/prototypes_adder.go @@ -21,6 +21,7 @@ import ( "strings" "github.com/arduino/arduino-cli/legacy/builder/constants" + "github.com/arduino/arduino-cli/legacy/builder/ctags" "github.com/arduino/arduino-cli/legacy/builder/types" "github.com/arduino/arduino-cli/legacy/builder/utils" ) @@ -67,7 +68,7 @@ func (s *PrototypesAdder) Run(ctx *types.Context) error { return nil } -func composePrototypeSection(line int, prototypes []*types.Prototype) string { +func composePrototypeSection(line int, prototypes []*ctags.Prototype) string { if len(prototypes) == 0 { return constants.EMPTY_STRING } @@ -81,7 +82,7 @@ func composePrototypeSection(line int, prototypes []*types.Prototype) string { return str } -func joinPrototypes(prototypes []*types.Prototype) string { +func joinPrototypes(prototypes []*ctags.Prototype) string { prototypesSlice := []string{} for _, proto := range prototypes { if signatureContainsaDefaultArg(proto) { @@ -98,7 +99,7 @@ func joinPrototypes(prototypes []*types.Prototype) string { return strings.Join(prototypesSlice, "\n") } -func signatureContainsaDefaultArg(proto *types.Prototype) bool { +func signatureContainsaDefaultArg(proto *ctags.Prototype) bool { return strings.Contains(proto.Prototype, "=") } diff --git a/legacy/builder/types/context.go b/legacy/builder/types/context.go index 895c5ef043f..f5b3bf44573 100644 --- a/legacy/builder/types/context.go +++ b/legacy/builder/types/context.go @@ -29,6 +29,7 @@ import ( "github.com/arduino/arduino-cli/arduino/libraries/librariesmanager" "github.com/arduino/arduino-cli/arduino/libraries/librariesresolver" "github.com/arduino/arduino-cli/arduino/sketch" + "github.com/arduino/arduino-cli/legacy/builder/ctags" rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" paths "github.com/arduino/go-paths-helper" properties "github.com/arduino/go-properties-orderedmap" @@ -123,7 +124,7 @@ type Context struct { LineOffset int PrototypesSection string PrototypesLineWhereToInsert int - Prototypes []*Prototype + Prototypes []*ctags.Prototype // Verbosity settings Verbose bool diff --git a/legacy/builder/types/types.go b/legacy/builder/types/types.go index d23fd306132..8c4ddbffaf9 100644 --- a/legacy/builder/types/types.go +++ b/legacy/builder/types/types.go @@ -17,7 +17,6 @@ package types import ( "fmt" - "strconv" "github.com/arduino/arduino-cli/arduino/libraries" "github.com/arduino/arduino-cli/arduino/sketch" @@ -117,40 +116,11 @@ type PlatforKeyRewrite struct { NewValue string } -type Prototype struct { - FunctionName string - File string - Prototype string - Modifiers string - Line int -} - -func (proto *Prototype) String() string { - return proto.Modifiers + " " + proto.Prototype + " @ " + strconv.Itoa(proto.Line) -} - type LibraryResolutionResult struct { Library *libraries.Library NotUsedLibraries []*libraries.Library } -type CTag struct { - FunctionName string - Kind string - Line int - Code string - Class string - Struct string - Namespace string - Filename string - Typeref string - SkipMe bool - Signature string - - Prototype string - PrototypeModifiers string -} - type Command interface { Run(ctx *Context) error } From 3e5f56f3fe6522f0f34c8b3ee0e9364baad750d4 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Mon, 3 Apr 2023 12:08:10 +0200 Subject: [PATCH 14/42] Removed unused structs --- .../builder/test/unique_string_queue_test.go | 34 ------------------- legacy/builder/types/accessories.go | 23 ------------- legacy/builder/types/types.go | 32 ----------------- 3 files changed, 89 deletions(-) delete mode 100644 legacy/builder/test/unique_string_queue_test.go diff --git a/legacy/builder/test/unique_string_queue_test.go b/legacy/builder/test/unique_string_queue_test.go deleted file mode 100644 index 20158792a53..00000000000 --- a/legacy/builder/test/unique_string_queue_test.go +++ /dev/null @@ -1,34 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package test - -import ( - "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/stretchr/testify/require" - "testing" -) - -func TestUniqueStringQueue(t *testing.T) { - queue := types.UniqueStringQueue{} - queue.Push("hello") - queue.Push("world") - queue.Push("hello") - queue.Push("world") - - require.Equal(t, "hello", queue.Pop()) - require.Equal(t, "world", queue.Pop()) - require.True(t, queue.Empty()) -} diff --git a/legacy/builder/types/accessories.go b/legacy/builder/types/accessories.go index a110977dde5..8decf206162 100644 --- a/legacy/builder/types/accessories.go +++ b/legacy/builder/types/accessories.go @@ -17,29 +17,6 @@ package types import "golang.org/x/exp/slices" -type UniqueStringQueue []string - -func (queue UniqueStringQueue) Len() int { return len(queue) } -func (queue UniqueStringQueue) Less(i, j int) bool { return false } -func (queue UniqueStringQueue) Swap(i, j int) { panic("Who called me?!?") } - -func (queue *UniqueStringQueue) Push(value string) { - if !slices.Contains(*queue, value) { - *queue = append(*queue, value) - } -} - -func (queue *UniqueStringQueue) Pop() interface{} { - old := *queue - x := old[0] - *queue = old[1:] - return x -} - -func (queue *UniqueStringQueue) Empty() bool { - return queue.Len() == 0 -} - type UniqueSourceFileQueue []SourceFile func (queue UniqueSourceFileQueue) Len() int { return len(queue) } diff --git a/legacy/builder/types/types.go b/legacy/builder/types/types.go index 8c4ddbffaf9..8d34056fff5 100644 --- a/legacy/builder/types/types.go +++ b/legacy/builder/types/types.go @@ -84,38 +84,6 @@ func (f *SourceFile) DepfilePath(ctx *Context) *paths.Path { return buildRoot(ctx, f.Origin).Join(f.RelativePath.String() + ".d") } -type SketchFile struct { - Name *paths.Path -} - -type SketchFileSortByName []SketchFile - -func (s SketchFileSortByName) Len() int { - return len(s) -} - -func (s SketchFileSortByName) Swap(i, j int) { - s[i], s[j] = s[j], s[i] -} - -func (s SketchFileSortByName) Less(i, j int) bool { - return s[i].Name.String() < s[j].Name.String() -} - -type PlatforKeysRewrite struct { - Rewrites []PlatforKeyRewrite -} - -func (p *PlatforKeysRewrite) Empty() bool { - return len(p.Rewrites) == 0 -} - -type PlatforKeyRewrite struct { - Key string - OldValue string - NewValue string -} - type LibraryResolutionResult struct { Library *libraries.Library NotUsedLibraries []*libraries.Library From d7475f76812fdf4d39236febff0a12f804352d85 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Mon, 3 Apr 2023 16:40:19 +0200 Subject: [PATCH 15/42] Unified GCCPreprocRunner* functions There is no need to duplicate a function that basically does the same thing. --- .../add_additional_entries_to_context.go | 2 -- legacy/builder/container_add_prototypes.go | 4 ++- legacy/builder/container_find_includes.go | 26 +++++++++--------- legacy/builder/gcc_preproc_runner.go | 27 +++---------------- legacy/builder/preprocess_sketch.go | 7 +++-- .../add_additional_entries_to_context_test.go | 4 --- legacy/builder/types/context.go | 2 -- 7 files changed, 26 insertions(+), 46 deletions(-) diff --git a/legacy/builder/add_additional_entries_to_context.go b/legacy/builder/add_additional_entries_to_context.go index e6ee138855f..fcb9639e37c 100644 --- a/legacy/builder/add_additional_entries_to_context.go +++ b/legacy/builder/add_additional_entries_to_context.go @@ -53,8 +53,6 @@ func (*AddAdditionalEntriesToContext) Run(ctx *types.Context) error { ctx.WarningsLevel = DEFAULT_WARNINGS_LEVEL } - ctx.CollectedSourceFiles = &types.UniqueSourceFileQueue{} - ctx.LibrariesResolutionResults = map[string]types.LibraryResolutionResult{} return nil diff --git a/legacy/builder/container_add_prototypes.go b/legacy/builder/container_add_prototypes.go index fe5e9064742..f7814e7d98c 100644 --- a/legacy/builder/container_add_prototypes.go +++ b/legacy/builder/container_add_prototypes.go @@ -39,7 +39,9 @@ func PreprocessSketchWithCtags(ctx *types.Context) error { // Run preprocessor sourceFile := ctx.SketchBuildPath.Join(ctx.Sketch.MainFile.Base() + ".cpp") - if err := GCCPreprocRunner(ctx, sourceFile, targetFilePath, ctx.IncludeFolders); err != nil { + stderr, err := GCCPreprocRunner(ctx, sourceFile, targetFilePath, ctx.IncludeFolders) + ctx.WriteStderr(stderr) + if err != nil { if !ctx.OnlyUpdateCompilationDatabase { return errors.WithStack(err) } diff --git a/legacy/builder/container_find_includes.go b/legacy/builder/container_find_includes.go index 189c3ff7b2c..824c5ddb73b 100644 --- a/legacy/builder/container_find_includes.go +++ b/legacy/builder/container_find_includes.go @@ -143,23 +143,24 @@ func (s *ContainerFindIncludes) findIncludes(ctx *types.Context) error { appendIncludeFolder(ctx, cache, nil, "", ctx.BuildProperties.GetPath("build.variant.path")) } + sourceFileQueue := &types.UniqueSourceFileQueue{} + if !ctx.UseCachedLibrariesResolution { sketch := ctx.Sketch mergedfile, err := types.MakeSourceFile(ctx, sketch, paths.New(sketch.MainFile.Base()+".cpp")) if err != nil { return errors.WithStack(err) } - ctx.CollectedSourceFiles.Push(mergedfile) + sourceFileQueue.Push(mergedfile) - sourceFilePaths := ctx.CollectedSourceFiles - queueSourceFilesFromFolder(ctx, sourceFilePaths, sketch, ctx.SketchBuildPath, false /* recurse */) + queueSourceFilesFromFolder(ctx, sourceFileQueue, sketch, ctx.SketchBuildPath, false /* recurse */) srcSubfolderPath := ctx.SketchBuildPath.Join("src") if srcSubfolderPath.IsDir() { - queueSourceFilesFromFolder(ctx, sourceFilePaths, sketch, srcSubfolderPath, true /* recurse */) + queueSourceFilesFromFolder(ctx, sourceFileQueue, sketch, srcSubfolderPath, true /* recurse */) } - for !sourceFilePaths.Empty() { - err := findIncludesUntilDone(ctx, cache, sourceFilePaths.Pop()) + for !sourceFileQueue.Empty() { + err := findIncludesUntilDone(ctx, cache, sourceFileQueue) if err != nil { cachePath.Remove() return errors.WithStack(err) @@ -314,7 +315,8 @@ func writeCache(cache *includeCache, path *paths.Path) error { return nil } -func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile types.SourceFile) error { +func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFileQueue *types.UniqueSourceFileQueue) error { + sourceFile := sourceFileQueue.Pop() sourcePath := sourceFile.SourcePath(ctx) targetFilePath := paths.NullPath() depPath := sourceFile.DepfilePath(ctx) @@ -367,7 +369,7 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile t ctx.Info(tr("Using cached library dependencies for file: %[1]s", sourcePath)) } } else { - preproc_stderr, preproc_err = GCCPreprocRunnerForDiscoveringIncludes(ctx, sourcePath, targetFilePath, includes) + preproc_stderr, preproc_err = GCCPreprocRunner(ctx, sourcePath, targetFilePath, includes) // Unwrap error and see if it is an ExitError. _, is_exit_error := errors.Cause(preproc_err).(*exec.ExitError) if preproc_err == nil { @@ -399,7 +401,7 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile t // return errors.WithStack(err) if preproc_err == nil || preproc_stderr == nil { // Filename came from cache, so run preprocessor to obtain error to show - preproc_stderr, preproc_err = GCCPreprocRunnerForDiscoveringIncludes(ctx, sourcePath, targetFilePath, includes) + preproc_stderr, preproc_err = GCCPreprocRunner(ctx, sourcePath, targetFilePath, includes) if preproc_err == nil { // If there is a missing #include in the cache, but running // gcc does not reproduce that, there is something wrong. @@ -419,13 +421,13 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile t appendIncludeFolder(ctx, cache, sourcePath, include, library.SourceDir) sourceDirs := library.SourceDirs() for _, sourceDir := range sourceDirs { - queueSourceFilesFromFolder(ctx, ctx.CollectedSourceFiles, library, sourceDir.Dir, sourceDir.Recurse) + queueSourceFilesFromFolder(ctx, sourceFileQueue, library, sourceDir.Dir, sourceDir.Recurse) } first = false } } -func queueSourceFilesFromFolder(ctx *types.Context, queue *types.UniqueSourceFileQueue, origin interface{}, folder *paths.Path, recurse bool) error { +func queueSourceFilesFromFolder(ctx *types.Context, sourceFileQueue *types.UniqueSourceFileQueue, origin interface{}, folder *paths.Path, recurse bool) error { sourceFileExtensions := []string{} for k := range globals.SourceFilesValidExtensions { sourceFileExtensions = append(sourceFileExtensions, k) @@ -440,7 +442,7 @@ func queueSourceFilesFromFolder(ctx *types.Context, queue *types.UniqueSourceFil if err != nil { return errors.WithStack(err) } - queue.Push(sourceFile) + sourceFileQueue.Push(sourceFile) } return nil diff --git a/legacy/builder/gcc_preproc_runner.go b/legacy/builder/gcc_preproc_runner.go index c1118d5889c..d8a0475fe82 100644 --- a/legacy/builder/gcc_preproc_runner.go +++ b/legacy/builder/gcc_preproc_runner.go @@ -27,32 +27,13 @@ import ( "github.com/pkg/errors" ) -func GCCPreprocRunner(ctx *types.Context, sourceFilePath *paths.Path, targetFilePath *paths.Path, includes paths.PathList) error { +func GCCPreprocRunner(ctx *types.Context, sourceFilePath *paths.Path, targetFilePath *paths.Path, includes paths.PathList) ([]byte, error) { cmd, err := prepareGCCPreprocRecipeProperties(ctx, sourceFilePath, targetFilePath, includes) if err != nil { - return errors.WithStack(err) + return nil, err } - - _, _, err = utils.ExecCommand(ctx, cmd /* stdout */, utils.ShowIfVerbose /* stderr */, utils.Show) - if err != nil { - return errors.WithStack(err) - } - - return nil -} - -func GCCPreprocRunnerForDiscoveringIncludes(ctx *types.Context, sourceFilePath *paths.Path, targetFilePath *paths.Path, includes paths.PathList) ([]byte, error) { - cmd, err := prepareGCCPreprocRecipeProperties(ctx, sourceFilePath, targetFilePath, includes) - if err != nil { - return nil, errors.WithStack(err) - } - - _, stderr, err := utils.ExecCommand(ctx, cmd /* stdout */, utils.ShowIfVerbose /* stderr */, utils.Capture) - if err != nil { - return stderr, errors.WithStack(err) - } - - return stderr, nil + _, stderr, err := utils.ExecCommand(ctx, cmd, utils.ShowIfVerbose /* stdout */, utils.Capture /* stderr */) + return stderr, err } func prepareGCCPreprocRecipeProperties(ctx *types.Context, sourceFilePath *paths.Path, targetFilePath *paths.Path, includes paths.PathList) (*exec.Cmd, error) { diff --git a/legacy/builder/preprocess_sketch.go b/legacy/builder/preprocess_sketch.go index 12c2d8f38ca..efd6721847d 100644 --- a/legacy/builder/preprocess_sketch.go +++ b/legacy/builder/preprocess_sketch.go @@ -36,8 +36,11 @@ func PreprocessSketchWithArduinoPreprocessor(ctx *types.Context) error { sourceFile := ctx.SketchBuildPath.Join(ctx.Sketch.MainFile.Base() + ".cpp") targetFile := ctx.PreprocPath.Join("sketch_merged.cpp") - GCCPreprocRunner(ctx, sourceFile, targetFile, ctx.IncludeFolders) - + stderr, err := GCCPreprocRunner(ctx, sourceFile, targetFile, ctx.IncludeFolders) + ctx.WriteStderr(stderr) + if err != nil { + return err + } buildProperties := properties.NewMap() buildProperties.Set("tools.arduino-preprocessor.path", "{runtime.tools.arduino-preprocessor.path}") buildProperties.Set("tools.arduino-preprocessor.cmd.path", "{path}/arduino-preprocessor") diff --git a/legacy/builder/test/add_additional_entries_to_context_test.go b/legacy/builder/test/add_additional_entries_to_context_test.go index ab64c26ec4c..51e5452daf3 100644 --- a/legacy/builder/test/add_additional_entries_to_context_test.go +++ b/legacy/builder/test/add_additional_entries_to_context_test.go @@ -38,8 +38,6 @@ func TestAddAdditionalEntriesToContextNoBuildPath(t *testing.T) { require.NotNil(t, ctx.WarningsLevel) - require.True(t, ctx.CollectedSourceFiles.Empty()) - require.Equal(t, 0, len(ctx.LibrariesResolutionResults)) } @@ -57,7 +55,5 @@ func TestAddAdditionalEntriesToContextWithBuildPath(t *testing.T) { require.NotNil(t, ctx.WarningsLevel) - require.True(t, ctx.CollectedSourceFiles.Empty()) - require.Equal(t, 0, len(ctx.LibrariesResolutionResults)) } diff --git a/legacy/builder/types/context.go b/legacy/builder/types/context.go index f5b3bf44573..ed21f25ebf1 100644 --- a/legacy/builder/types/context.go +++ b/legacy/builder/types/context.go @@ -99,8 +99,6 @@ type Context struct { SketchObjectFiles paths.PathList IgnoreSketchFolderNameErrors bool - CollectedSourceFiles *UniqueSourceFileQueue - Sketch *sketch.Sketch WarningsLevel string From c7cc15faf72f911110406811cb5127008352204b Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Mon, 3 Apr 2023 16:44:48 +0200 Subject: [PATCH 16/42] Replaced utils.SliceContains with stdlib function --- legacy/builder/create_cmake_rule.go | 5 +++-- legacy/builder/test/helper_tools_downloader.go | 6 +++--- .../builder/unused_compiled_libraries_remover.go | 4 ++-- legacy/builder/utils/utils.go | 14 +++----------- 4 files changed, 11 insertions(+), 18 deletions(-) diff --git a/legacy/builder/create_cmake_rule.go b/legacy/builder/create_cmake_rule.go index 7c848ef8544..03549563780 100644 --- a/legacy/builder/create_cmake_rule.go +++ b/legacy/builder/create_cmake_rule.go @@ -22,6 +22,7 @@ import ( "strings" properties "github.com/arduino/go-properties-orderedmap" + "golang.org/x/exp/slices" "github.com/arduino/arduino-cli/arduino/globals" "github.com/arduino/arduino-cli/legacy/builder/builder_utils" @@ -208,7 +209,7 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error { lib := staticLib.Base() lib = strings.TrimPrefix(lib, "lib") lib = strings.TrimSuffix(lib, ".a") - if !utils.SliceContains(dynamicLibsFromGccMinusL, lib) { + if !slices.Contains(dynamicLibsFromGccMinusL, lib) { linkGroup += " " + lib cmakelist += "add_library (" + lib + " STATIC IMPORTED)\n" location := strings.TrimPrefix(staticLib.String(), cmakeFolder.String()) @@ -263,7 +264,7 @@ func findUniqueFoldersRelative(slice []string, base string) string { for _, element := range slice { path := filepath.Dir(element) path = strings.TrimPrefix(path, base+"/") - if !utils.SliceContains(out, path) { + if !slices.Contains(out, path) { out = append(out, path) } } diff --git a/legacy/builder/test/helper_tools_downloader.go b/legacy/builder/test/helper_tools_downloader.go index 6e79e4d3aac..133122c4ab6 100644 --- a/legacy/builder/test/helper_tools_downloader.go +++ b/legacy/builder/test/helper_tools_downloader.go @@ -29,10 +29,10 @@ import ( "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/gohasissues" - "github.com/arduino/arduino-cli/legacy/builder/utils" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" + "golang.org/x/exp/slices" ) var hardwareFolder = paths.New("downloaded_hardware") @@ -692,7 +692,7 @@ func translateGOOSGOARCHToPackageIndexValue() []string { func findToolUrl(index map[string]interface{}, tool Tool, host []string) (string, error) { if len(tool.OsUrls) > 0 { for _, osUrl := range tool.OsUrls { - if utils.SliceContains(host, osUrl.Os) { + if slices.Contains(host, osUrl.Os) { return osUrl.Url, nil } } @@ -709,7 +709,7 @@ func findToolUrl(index map[string]interface{}, tool Tool, host []string) (string systems := packageTool["systems"].([]interface{}) for _, s := range systems { system := s.(map[string]interface{}) - if utils.SliceContains(host, system["host"].(string)) { + if slices.Contains(host, system["host"].(string)) { return system[constants.TOOL_URL].(string), nil } } diff --git a/legacy/builder/unused_compiled_libraries_remover.go b/legacy/builder/unused_compiled_libraries_remover.go index 595ce91302e..d1b36b147ac 100644 --- a/legacy/builder/unused_compiled_libraries_remover.go +++ b/legacy/builder/unused_compiled_libraries_remover.go @@ -18,8 +18,8 @@ package builder import ( "github.com/arduino/arduino-cli/arduino/libraries" "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/arduino/arduino-cli/legacy/builder/utils" "github.com/pkg/errors" + "golang.org/x/exp/slices" ) type UnusedCompiledLibrariesRemover struct{} @@ -39,7 +39,7 @@ func (s *UnusedCompiledLibrariesRemover) Run(ctx *types.Context) error { } for _, file := range files { if file.IsDir() { - if !utils.SliceContains(libraryNames, file.Base()) { + if !slices.Contains(libraryNames, file.Base()) { if err := file.RemoveAll(); err != nil { return errors.WithStack(err) } diff --git a/legacy/builder/utils/utils.go b/legacy/builder/utils/utils.go index 0496ccff27a..522f046a018 100644 --- a/legacy/builder/utils/utils.go +++ b/legacy/builder/utils/utils.go @@ -32,6 +32,7 @@ import ( "github.com/arduino/arduino-cli/legacy/builder/types" paths "github.com/arduino/go-paths-helper" "github.com/pkg/errors" + "golang.org/x/exp/slices" "golang.org/x/text/runes" "golang.org/x/text/transform" "golang.org/x/text/unicode/norm" @@ -63,7 +64,7 @@ func FilterFilesWithExtensions(extensions ...string) filterFiles { return func(files []os.FileInfo) []os.FileInfo { var filtered []os.FileInfo for _, file := range files { - if !file.IsDir() && SliceContains(extensions, filepath.Ext(file.Name())) { + if !file.IsDir() && slices.Contains(extensions, filepath.Ext(file.Name())) { filtered = append(filtered, file) } } @@ -118,15 +119,6 @@ func IsSCCSFile(file os.FileInfo) bool { return SOURCE_CONTROL_FOLDERS[name] } -func SliceContains(slice []string, target string) bool { - for _, value := range slice { - if value == target { - return true - } - } - return false -} - type mapFunc func(string) string func Map(slice []string, fn mapFunc) []string { @@ -262,7 +254,7 @@ func FindFilesInFolder(dir *paths.Path, recurse bool, extensions []string) (path func AppendIfNotPresent(target []string, elements ...string) []string { for _, element := range elements { - if !SliceContains(target, element) { + if !slices.Contains(target, element) { target = append(target, element) } } From 2adc6bd955c02dac70d6938908179a7972030d9e Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Mon, 3 Apr 2023 18:02:21 +0200 Subject: [PATCH 17/42] Implemented generic algorithms (at lease until they are available in go stdlib...) --- internal/algorithms/slices.go | 53 ++++++++++++++++++++++ internal/algorithms/slices_test.go | 53 ++++++++++++++++++++++ legacy/builder/builder_utils/utils.go | 13 ++---- legacy/builder/gcc_preproc_runner.go | 5 +- legacy/builder/phases/core_builder.go | 3 +- legacy/builder/phases/libraries_builder.go | 3 +- legacy/builder/phases/linker.go | 5 +- legacy/builder/phases/sketch_builder.go | 3 +- legacy/builder/test/utils_test.go | 12 ----- legacy/builder/utils/utils.go | 29 +----------- 10 files changed, 125 insertions(+), 54 deletions(-) create mode 100644 internal/algorithms/slices.go create mode 100644 internal/algorithms/slices_test.go diff --git a/internal/algorithms/slices.go b/internal/algorithms/slices.go new file mode 100644 index 00000000000..4cb2b9f8cea --- /dev/null +++ b/internal/algorithms/slices.go @@ -0,0 +1,53 @@ +// This file is part of arduino-cli. +// +// Copyright 2023 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +package f + +// Filter takes a slice of type []T and a function of type func(T) bool. It returns +// a newly allocated slice containing only those elements of the input slice that +// satisfy the test function. +func Filter[T any](values []T, test func(x T) bool) []T { + res := []T{} + for _, x := range values { + if test(x) { + res = append(res, x) + } + } + return res +} + +// Map applies the mapping function to each element of the slice and returns a new +// slice with the results in the same order. +func Map[T any](values []T, mapping func(x T) T) []T { + res := []T{} + for _, x := range values { + res = append(res, mapping(x)) + } + return res +} + +// Equals return a functor that matches the given value +func Equals[T comparable](value T) func(x T) bool { + return func(x T) bool { + return x == value + } +} + +// NotEquals return a functor that does not match the given value +func NotEquals[T comparable](value T) func(x T) bool { + return func(x T) bool { + return x != value + } +} diff --git a/internal/algorithms/slices_test.go b/internal/algorithms/slices_test.go new file mode 100644 index 00000000000..8ef58c660a9 --- /dev/null +++ b/internal/algorithms/slices_test.go @@ -0,0 +1,53 @@ +// This file is part of arduino-cli. +// +// Copyright 2023 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +package f_test + +import ( + "strings" + "testing" + + f "github.com/arduino/arduino-cli/internal/algorithms" + "github.com/stretchr/testify/require" +) + +func TestFilter(t *testing.T) { + a := []string{"aaa", "bbb", "ccc"} + require.Equal(t, []string{"bbb", "ccc"}, f.Filter(a, func(x string) bool { return x > "b" })) + b := []int{5, 9, 15, 2, 4, -2} + require.Equal(t, []int{5, 9, 15}, f.Filter(b, func(x int) bool { return x > 4 })) +} + +func TestIsEmpty(t *testing.T) { + require.True(t, f.Equals(int(0))(0)) + require.False(t, f.Equals(int(1))(0)) + require.True(t, f.Equals("")("")) + require.False(t, f.Equals("abc")("")) + require.False(t, f.NotEquals(int(0))(0)) + require.True(t, f.NotEquals(int(1))(0)) + require.False(t, f.NotEquals("")("")) + require.True(t, f.NotEquals("abc")("")) +} + +func TestMap(t *testing.T) { + value := "hello, world , how are,you? " + parts := f.Map(strings.Split(value, ","), strings.TrimSpace) + + require.Equal(t, 4, len(parts)) + require.Equal(t, "hello", parts[0]) + require.Equal(t, "world", parts[1]) + require.Equal(t, "how are", parts[2]) + require.Equal(t, "you?", parts[3]) +} diff --git a/legacy/builder/builder_utils/utils.go b/legacy/builder/builder_utils/utils.go index 44cc55c90af..5c25d2c86c0 100644 --- a/legacy/builder/builder_utils/utils.go +++ b/legacy/builder/builder_utils/utils.go @@ -26,6 +26,7 @@ import ( "github.com/arduino/arduino-cli/arduino/globals" "github.com/arduino/arduino-cli/i18n" + f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" "github.com/arduino/arduino-cli/legacy/builder/utils" @@ -262,10 +263,10 @@ func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile *paths.Path) (bool return false, errors.WithStack(err) } - rows = utils.Map(rows, removeEndingBackSlash) - rows = utils.Map(rows, strings.TrimSpace) - rows = utils.Map(rows, unescapeDep) - rows = utils.Filter(rows, nonEmptyString) + rows = f.Map(rows, removeEndingBackSlash) + rows = f.Map(rows, strings.TrimSpace) + rows = f.Map(rows, unescapeDep) + rows = f.Filter(rows, f.NotEquals("")) if len(rows) == 0 { return true, nil @@ -327,10 +328,6 @@ func removeEndingBackSlash(s string) string { return strings.TrimSuffix(s, "\\") } -func nonEmptyString(s string) bool { - return s != constants.EMPTY_STRING -} - func CoreOrReferencedCoreHasChanged(corePath, targetCorePath, targetFile *paths.Path) bool { targetFileStat, err := targetFile.Stat() diff --git a/legacy/builder/gcc_preproc_runner.go b/legacy/builder/gcc_preproc_runner.go index d8a0475fe82..10876f5d08b 100644 --- a/legacy/builder/gcc_preproc_runner.go +++ b/legacy/builder/gcc_preproc_runner.go @@ -19,6 +19,7 @@ import ( "os/exec" "strings" + f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/types" "github.com/arduino/arduino-cli/legacy/builder/utils" @@ -44,7 +45,7 @@ func prepareGCCPreprocRecipeProperties(ctx *types.Context, sourceFilePath *paths buildProperties.SetPath("source_file", sourceFilePath) buildProperties.SetPath("preprocessed_file_path", targetFilePath) - includesStrings := utils.Map(includes.AsStrings(), utils.WrapWithHyphenI) + includesStrings := f.Map(includes.AsStrings(), utils.WrapWithHyphenI) buildProperties.Set("includes", strings.Join(includesStrings, " ")) if buildProperties.Get("recipe.preproc.macros") == "" { @@ -65,7 +66,7 @@ func prepareGCCPreprocRecipeProperties(ctx *types.Context, sourceFilePath *paths // Remove -MMD argument if present. Leaving it will make gcc try // to create a /dev/null.d dependency file, which won't work. - cmd.Args = utils.Filter(cmd.Args, func(a string) bool { return a != "-MMD" }) + cmd.Args = f.Filter(cmd.Args, f.NotEquals("-MMD")) return cmd, nil } diff --git a/legacy/builder/phases/core_builder.go b/legacy/builder/phases/core_builder.go index a484c335f7b..e60af767b42 100644 --- a/legacy/builder/phases/core_builder.go +++ b/legacy/builder/phases/core_builder.go @@ -22,6 +22,7 @@ import ( "github.com/arduino/arduino-cli/buildcache" "github.com/arduino/arduino-cli/i18n" + f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" @@ -77,7 +78,7 @@ func compileCore(ctx *types.Context, buildPath *paths.Path, buildCachePath *path if variantFolder != nil && variantFolder.IsDir() { includes = append(includes, variantFolder.String()) } - includes = utils.Map(includes, utils.WrapWithHyphenI) + includes = f.Map(includes, utils.WrapWithHyphenI) var err error diff --git a/legacy/builder/phases/libraries_builder.go b/legacy/builder/phases/libraries_builder.go index 32bb0b2f01c..5ee81bb4385 100644 --- a/legacy/builder/phases/libraries_builder.go +++ b/legacy/builder/phases/libraries_builder.go @@ -19,6 +19,7 @@ import ( "strings" "github.com/arduino/arduino-cli/arduino/libraries" + f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" @@ -36,7 +37,7 @@ type LibrariesBuilder struct{} func (s *LibrariesBuilder) Run(ctx *types.Context) error { librariesBuildPath := ctx.LibrariesBuildPath buildProperties := ctx.BuildProperties - includes := utils.Map(ctx.IncludeFolders.AsStrings(), utils.WrapWithHyphenI) + includes := f.Map(ctx.IncludeFolders.AsStrings(), utils.WrapWithHyphenI) libs := ctx.ImportedLibraries if err := librariesBuildPath.MkdirAll(); err != nil { diff --git a/legacy/builder/phases/linker.go b/legacy/builder/phases/linker.go index 0f2f76d336e..cc7c293199d 100644 --- a/legacy/builder/phases/linker.go +++ b/legacy/builder/phases/linker.go @@ -18,6 +18,7 @@ package phases import ( "strings" + f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" @@ -64,7 +65,7 @@ func (s *Linker) Run(ctx *types.Context) error { } func link(ctx *types.Context, objectFiles paths.PathList, coreDotARelPath *paths.Path, coreArchiveFilePath *paths.Path, buildProperties *properties.Map) error { - objectFileList := strings.Join(utils.Map(objectFiles.AsStrings(), wrapWithDoubleQuotes), " ") + objectFileList := strings.Join(f.Map(objectFiles.AsStrings(), wrapWithDoubleQuotes), " ") // If command line length is too big (> 30000 chars), try to collect the object files into archives // and use that archives to complete the build. @@ -102,7 +103,7 @@ func link(ctx *types.Context, objectFiles paths.PathList, coreDotARelPath *paths } } - objectFileList = strings.Join(utils.Map(archives.AsStrings(), wrapWithDoubleQuotes), " ") + objectFileList = strings.Join(f.Map(archives.AsStrings(), wrapWithDoubleQuotes), " ") objectFileList = "-Wl,--whole-archive " + objectFileList + " -Wl,--no-whole-archive" } diff --git a/legacy/builder/phases/sketch_builder.go b/legacy/builder/phases/sketch_builder.go index d60f8ab9a0c..300cb6836af 100644 --- a/legacy/builder/phases/sketch_builder.go +++ b/legacy/builder/phases/sketch_builder.go @@ -16,6 +16,7 @@ package phases import ( + f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/types" "github.com/arduino/arduino-cli/legacy/builder/utils" @@ -27,7 +28,7 @@ type SketchBuilder struct{} func (s *SketchBuilder) Run(ctx *types.Context) error { sketchBuildPath := ctx.SketchBuildPath buildProperties := ctx.BuildProperties - includes := utils.Map(ctx.IncludeFolders.AsStrings(), utils.WrapWithHyphenI) + includes := f.Map(ctx.IncludeFolders.AsStrings(), utils.WrapWithHyphenI) if err := sketchBuildPath.MkdirAll(); err != nil { return errors.WithStack(err) diff --git a/legacy/builder/test/utils_test.go b/legacy/builder/test/utils_test.go index 71a70be9dd6..ec34a7bb670 100644 --- a/legacy/builder/test/utils_test.go +++ b/legacy/builder/test/utils_test.go @@ -16,7 +16,6 @@ package test import ( - "strings" "testing" "github.com/arduino/arduino-cli/legacy/builder/utils" @@ -42,17 +41,6 @@ func TestPrintableCommand(t *testing.T) { require.Equal(t, correct, result) } -func TestMapTrimSpace(t *testing.T) { - value := "hello, world , how are,you? " - parts := utils.Map(strings.Split(value, ","), utils.TrimSpace) - - require.Equal(t, 4, len(parts)) - require.Equal(t, "hello", parts[0]) - require.Equal(t, "world", parts[1]) - require.Equal(t, "how are", parts[2]) - require.Equal(t, "you?", parts[3]) -} - func TestQuoteCppString(t *testing.T) { cases := map[string]string{ `foo`: `"foo"`, diff --git a/legacy/builder/utils/utils.go b/legacy/builder/utils/utils.go index 522f046a018..056940308fe 100644 --- a/legacy/builder/utils/utils.go +++ b/legacy/builder/utils/utils.go @@ -28,6 +28,7 @@ import ( "unicode" "github.com/arduino/arduino-cli/i18n" + f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/arduino-cli/legacy/builder/gohasissues" "github.com/arduino/arduino-cli/legacy/builder/types" paths "github.com/arduino/go-paths-helper" @@ -119,36 +120,10 @@ func IsSCCSFile(file os.FileInfo) bool { return SOURCE_CONTROL_FOLDERS[name] } -type mapFunc func(string) string - -func Map(slice []string, fn mapFunc) []string { - newSlice := []string{} - for _, elem := range slice { - newSlice = append(newSlice, fn(elem)) - } - return newSlice -} - -type filterFunc func(string) bool - -func Filter(slice []string, fn filterFunc) []string { - newSlice := []string{} - for _, elem := range slice { - if fn(elem) { - newSlice = append(newSlice, elem) - } - } - return newSlice -} - func WrapWithHyphenI(value string) string { return "\"-I" + value + "\"" } -func TrimSpace(value string) string { - return strings.TrimSpace(value) -} - func printableArgument(arg string) string { if strings.ContainsAny(arg, "\"\\ \t") { arg = strings.Replace(arg, "\\", "\\\\", -1) @@ -164,7 +139,7 @@ func printableArgument(arg string) string { // probably not for shell interpretation. This essentially reverses // ParseCommandLine. func PrintableCommand(parts []string) string { - return strings.Join(Map(parts, printableArgument), " ") + return strings.Join(f.Map(parts, printableArgument), " ") } const ( From a58f3d566b46a7a60d3bc27da21bf02c46614625 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Tue, 4 Apr 2023 11:00:14 +0200 Subject: [PATCH 18/42] Removed useless call to FilterSketchSource --- legacy/builder/create_cmake_rule.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/legacy/builder/create_cmake_rule.go b/legacy/builder/create_cmake_rule.go index 03549563780..394b80ec49f 100644 --- a/legacy/builder/create_cmake_rule.go +++ b/legacy/builder/create_cmake_rule.go @@ -119,9 +119,6 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error { return err } - // Use old ctags method to generate export file - ctx.SketchSourceMerged = filterSketchSource(ctx.Sketch, strings.NewReader(ctx.SketchSourceMerged), true) - err = utils.CopyDir(ctx.SketchBuildPath.String(), cmakeFolder.Join("sketch").String(), validExportExtensions) if err != nil { fmt.Println(err) From c2328b5f3d12e245ff37b05b2e94134e051b58b8 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Tue, 4 Apr 2023 11:15:51 +0200 Subject: [PATCH 19/42] Converted AddPrototypes into a function --- legacy/builder/container_add_prototypes.go | 14 +++---------- legacy/builder/prototypes_adder.go | 24 ++++++---------------- 2 files changed, 9 insertions(+), 29 deletions(-) diff --git a/legacy/builder/container_add_prototypes.go b/legacy/builder/container_add_prototypes.go index f7814e7d98c..43d1fa36d35 100644 --- a/legacy/builder/container_add_prototypes.go +++ b/legacy/builder/container_add_prototypes.go @@ -62,18 +62,10 @@ func PreprocessSketchWithCtags(ctx *types.Context) error { ctx.SketchSourceAfterCppPreprocessing = filterSketchSource(ctx.Sketch, bytes.NewReader(src), false) } - commands := []types.Command{ - &CTagsRunner{Source: &ctx.SketchSourceAfterCppPreprocessing, TargetFileName: "sketch_merged.cpp"}, - &PrototypesAdder{}, - } - - for _, command := range commands { - PrintRingNameIfDebug(ctx, command) - err := command.Run(ctx) - if err != nil { - return errors.WithStack(err) - } + if err := (&CTagsRunner{Source: &ctx.SketchSourceAfterCppPreprocessing, TargetFileName: "sketch_merged.cpp"}).Run(ctx); err != nil { + return errors.WithStack(err) } + ctx.SketchSourceAfterArduinoPreprocessing, ctx.PrototypesSection = PrototypesAdder(ctx.SketchSourceMerged, ctx.PrototypesLineWhereToInsert, ctx.LineOffset, ctx.Prototypes, ctx.DebugPreprocessor) if err := bldr.SketchSaveItemCpp(ctx.Sketch.MainFile, []byte(ctx.SketchSourceAfterArduinoPreprocessing), ctx.SketchBuildPath); err != nil { return errors.WithStack(err) diff --git a/legacy/builder/prototypes_adder.go b/legacy/builder/prototypes_adder.go index 8a912f12993..a7676d42038 100644 --- a/legacy/builder/prototypes_adder.go +++ b/legacy/builder/prototypes_adder.go @@ -22,31 +22,21 @@ import ( "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/ctags" - "github.com/arduino/arduino-cli/legacy/builder/types" "github.com/arduino/arduino-cli/legacy/builder/utils" ) -type PrototypesAdder struct{} - -func (s *PrototypesAdder) Run(ctx *types.Context) error { - debugOutput := ctx.DebugPreprocessor - - source := ctx.SketchSourceMerged +func PrototypesAdder(source string, firstFunctionLine, lineOffset int, prototypes []*ctags.Prototype, debugOutput bool) (preprocessedSource, prototypeSection string) { source = strings.Replace(source, "\r\n", "\n", -1) source = strings.Replace(source, "\r", "\n", -1) - sourceRows := strings.Split(source, "\n") - - firstFunctionLine := ctx.PrototypesLineWhereToInsert if isFirstFunctionOutsideOfSource(firstFunctionLine, sourceRows) { - return nil + return } - insertionLine := firstFunctionLine + ctx.LineOffset - 1 + insertionLine := firstFunctionLine + lineOffset - 1 firstFunctionChar := len(strings.Join(sourceRows[:insertionLine], "\n")) + 1 - prototypeSection := composePrototypeSection(firstFunctionLine, ctx.Prototypes) - ctx.PrototypesSection = prototypeSection - source = source[:firstFunctionChar] + prototypeSection + source[firstFunctionChar:] + prototypeSection = composePrototypeSection(firstFunctionLine, prototypes) + preprocessedSource = source[:firstFunctionChar] + prototypeSection + source[firstFunctionChar:] if debugOutput { fmt.Println("#PREPROCESSED SOURCE") @@ -63,9 +53,7 @@ func (s *PrototypesAdder) Run(ctx *types.Context) error { } fmt.Println("#END OF PREPROCESSED SOURCE") } - ctx.SketchSourceAfterArduinoPreprocessing = source - - return nil + return } func composePrototypeSection(line int, prototypes []*ctags.Prototype) string { From f4dd2953e155b2c349b1ecb410656eaa5d97d728 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Wed, 5 Apr 2023 13:02:18 +0200 Subject: [PATCH 20/42] Converted CTagsRunner into a function --- legacy/builder/container_add_prototypes.go | 10 ++- legacy/builder/ctags_runner.go | 88 +++++++++++----------- legacy/builder/test/ctags_runner_test.go | 45 +++++++---- 3 files changed, 81 insertions(+), 62 deletions(-) diff --git a/legacy/builder/container_add_prototypes.go b/legacy/builder/container_add_prototypes.go index 43d1fa36d35..99059b0fb76 100644 --- a/legacy/builder/container_add_prototypes.go +++ b/legacy/builder/container_add_prototypes.go @@ -62,8 +62,14 @@ func PreprocessSketchWithCtags(ctx *types.Context) error { ctx.SketchSourceAfterCppPreprocessing = filterSketchSource(ctx.Sketch, bytes.NewReader(src), false) } - if err := (&CTagsRunner{Source: &ctx.SketchSourceAfterCppPreprocessing, TargetFileName: "sketch_merged.cpp"}).Run(ctx); err != nil { - return errors.WithStack(err) + var ctagsStderr []byte + _, ctagsStderr, ctx.PrototypesLineWhereToInsert, ctx.Prototypes, err = RunCTags( + ctx.Sketch, ctx.SketchSourceAfterCppPreprocessing, "sketch_merged.cpp", ctx.BuildProperties, ctx.PreprocPath) + if ctx.Verbose { + ctx.WriteStderr(ctagsStderr) + } + if err != nil { + return err } ctx.SketchSourceAfterArduinoPreprocessing, ctx.PrototypesSection = PrototypesAdder(ctx.SketchSourceMerged, ctx.PrototypesLineWhereToInsert, ctx.LineOffset, ctx.Prototypes, ctx.DebugPreprocessor) diff --git a/legacy/builder/ctags_runner.go b/legacy/builder/ctags_runner.go index 40ef36434d1..563444f0b7d 100644 --- a/legacy/builder/ctags_runner.go +++ b/legacy/builder/ctags_runner.go @@ -16,74 +16,72 @@ package builder import ( - "os" - "os/exec" + "bytes" + "strings" + "github.com/arduino/arduino-cli/arduino/sketch" + "github.com/arduino/arduino-cli/executils" "github.com/arduino/arduino-cli/legacy/builder/ctags" - "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/arduino/arduino-cli/legacy/builder/utils" + "github.com/arduino/go-paths-helper" properties "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" ) -type CTagsRunner struct { - Source *string - TargetFileName string - - // Needed for unit-testing - CtagsOutput []byte -} - -func (r *CTagsRunner) Run(ctx *types.Context) error { - source := *r.Source - - preprocPath := ctx.PreprocPath - if err := preprocPath.MkdirAll(); err != nil { - return errors.WithStack(err) +func RunCTags(sketch *sketch.Sketch, source string, targetFileName string, buildProperties *properties.Map, preprocPath *paths.Path, +) (ctagsStdout, ctagsStderr []byte, prototypesLineWhereToInsert int, prototypes []*ctags.Prototype, err error) { + if err = preprocPath.MkdirAll(); err != nil { + return } - ctagsTargetFilePath := preprocPath.Join(r.TargetFileName) - if err := ctagsTargetFilePath.WriteFile([]byte(source)); err != nil { - return errors.WithStack(err) + ctagsTargetFilePath := preprocPath.Join(targetFileName) + if err = ctagsTargetFilePath.WriteFile([]byte(source)); err != nil { + return } - buildProperties := properties.NewMap() - buildProperties.Set("tools.ctags.path", "{runtime.tools.ctags.path}") - buildProperties.Set("tools.ctags.cmd.path", "{path}/ctags") - buildProperties.Set("tools.ctags.pattern", `"{cmd.path}" -u --language-force=c++ -f - --c++-kinds=svpf --fields=KSTtzns --line-directives "{source_file}"`) - buildProperties.Merge(ctx.BuildProperties) - buildProperties.Merge(buildProperties.SubTree("tools").SubTree("ctags")) - buildProperties.SetPath("source_file", ctagsTargetFilePath) + ctagsBuildProperties := properties.NewMap() + ctagsBuildProperties.Set("tools.ctags.path", "{runtime.tools.ctags.path}") + ctagsBuildProperties.Set("tools.ctags.cmd.path", "{path}/ctags") + ctagsBuildProperties.Set("tools.ctags.pattern", `"{cmd.path}" -u --language-force=c++ -f - --c++-kinds=svpf --fields=KSTtzns --line-directives "{source_file}"`) + ctagsBuildProperties.Merge(buildProperties) + ctagsBuildProperties.Merge(ctagsBuildProperties.SubTree("tools").SubTree("ctags")) + ctagsBuildProperties.SetPath("source_file", ctagsTargetFilePath) - pattern := buildProperties.Get("pattern") + pattern := ctagsBuildProperties.Get("pattern") if pattern == "" { - return errors.Errorf(tr("%s pattern is missing"), "ctags") + err = errors.Errorf(tr("%s pattern is missing"), "ctags") + return } - commandLine := buildProperties.ExpandPropsInString(pattern) + commandLine := ctagsBuildProperties.ExpandPropsInString(pattern) parts, err := properties.SplitQuotedString(commandLine, `"'`, false) if err != nil { - return errors.WithStack(err) + return } - command := exec.Command(parts[0], parts[1:]...) - command.Env = append(os.Environ(), ctx.PackageManager.GetEnvVarsForSpawnedProcess()...) - - ctagsOutput, _, err := utils.ExecCommand(ctx, command, utils.Capture /* stdout */, utils.ShowIfVerbose /* stderr */) + proc, err := executils.NewProcess(nil, parts...) if err != nil { - return errors.WithStack(err) + return + } + stdout := &bytes.Buffer{} + stderr := &bytes.Buffer{} + proc.RedirectStdoutTo(stdout) + proc.RedirectStderrTo(stderr) + if err = proc.Run(); err != nil { + return + } + stderr.WriteString(strings.Join(parts, " ")) + ctagsStdout = stdout.Bytes() + ctagsStderr = stderr.Bytes() + if err != nil { + return } parser := &ctags.CTagsParser{} - parser.Parse(ctagsOutput, ctx.Sketch.MainFile) + parser.Parse(ctagsStdout, sketch.MainFile) parser.FixCLinkageTagsDeclarations() - protos, line := parser.GeneratePrototypes() + prototypes, line := parser.GeneratePrototypes() if line != -1 { - ctx.PrototypesLineWhereToInsert = line + prototypesLineWhereToInsert = line } - ctx.Prototypes = protos - - // Needed for unit-testing - r.CtagsOutput = ctagsOutput - return nil + return } diff --git a/legacy/builder/test/ctags_runner_test.go b/legacy/builder/test/ctags_runner_test.go index 12194ccf744..069339eef48 100644 --- a/legacy/builder/test/ctags_runner_test.go +++ b/legacy/builder/test/ctags_runner_test.go @@ -32,8 +32,8 @@ func TestCTagsRunner(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) ctx.Verbose = true - ctagsRunner := &builder.CTagsRunner{Source: &ctx.SketchSourceMerged, TargetFileName: "ctags_target.cpp"} var _err error + var ctagsOutput []byte commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { @@ -43,7 +43,10 @@ func TestCTagsRunner(t *testing.T) { &builder.ContainerFindIncludes{}, &builder.PrintUsedLibrariesIfVerbose{}, &builder.WarnAboutArchIncompatibleLibraries{}, - ctagsRunner, + types.BareCommand(func(ctx *types.Context) error { + ctagsOutput, _, _, _, _err = builder.RunCTags(ctx.Sketch, ctx.SketchSourceMerged, "ctags_target.cpp", ctx.BuildProperties, ctx.PreprocPath) + return _err + }), } for _, command := range commands { err := command.Run(ctx) @@ -58,7 +61,7 @@ func TestCTagsRunner(t *testing.T) { "digitalCommand " + quotedSketchLocation + " /^void digitalCommand(BridgeClient client) {$/;\" kind:function line:82 signature:(BridgeClient client) returntype:void\n" + "analogCommand " + quotedSketchLocation + " /^void analogCommand(BridgeClient client) {$/;\" kind:function line:109 signature:(BridgeClient client) returntype:void\n" + "modeCommand " + quotedSketchLocation + " /^void modeCommand(BridgeClient client) {$/;\" kind:function line:149 signature:(BridgeClient client) returntype:void\n" - require.Equal(t, expectedOutput, strings.Replace(string(ctagsRunner.CtagsOutput), "\r\n", "\n", -1)) + require.Equal(t, expectedOutput, strings.Replace(string(ctagsOutput), "\r\n", "\n", -1)) } func TestCTagsRunnerSketchWithClass(t *testing.T) { @@ -67,8 +70,8 @@ func TestCTagsRunnerSketchWithClass(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) ctx.Verbose = true - ctagsRunner := &builder.CTagsRunner{Source: &ctx.SketchSourceMerged, TargetFileName: "ctags_target.cpp"} var _err error + var ctagsOutput []byte commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { @@ -78,7 +81,10 @@ func TestCTagsRunnerSketchWithClass(t *testing.T) { &builder.ContainerFindIncludes{}, &builder.PrintUsedLibrariesIfVerbose{}, &builder.WarnAboutArchIncompatibleLibraries{}, - ctagsRunner, + types.BareCommand(func(ctx *types.Context) error { + ctagsOutput, _, _, _, _err = builder.RunCTags(ctx.Sketch, ctx.SketchSourceMerged, "ctags_target.cpp", ctx.BuildProperties, ctx.PreprocPath) + return _err + }), } for _, command := range commands { err := command.Run(ctx) @@ -91,7 +97,7 @@ func TestCTagsRunnerSketchWithClass(t *testing.T) { "set_values\t" + quotedSketchLocation + "\t/^void Rectangle::set_values (int x, int y) {$/;\"\tkind:function\tline:8\tclass:Rectangle\tsignature:(int x, int y)\treturntype:void\n" + "setup\t" + quotedSketchLocation + "\t/^void setup() {$/;\"\tkind:function\tline:13\tsignature:()\treturntype:void\n" + "loop\t" + quotedSketchLocation + "\t/^void loop() {$/;\"\tkind:function\tline:17\tsignature:()\treturntype:void\n" - require.Equal(t, expectedOutput, strings.Replace(string(ctagsRunner.CtagsOutput), "\r\n", "\n", -1)) + require.Equal(t, expectedOutput, strings.Replace(string(ctagsOutput), "\r\n", "\n", -1)) } func TestCTagsRunnerSketchWithTypename(t *testing.T) { @@ -100,8 +106,8 @@ func TestCTagsRunnerSketchWithTypename(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) ctx.Verbose = true - ctagsRunner := &builder.CTagsRunner{Source: &ctx.SketchSourceMerged, TargetFileName: "ctags_target.cpp"} var _err error + var ctagsOutput []byte commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { @@ -111,7 +117,10 @@ func TestCTagsRunnerSketchWithTypename(t *testing.T) { &builder.ContainerFindIncludes{}, &builder.PrintUsedLibrariesIfVerbose{}, &builder.WarnAboutArchIncompatibleLibraries{}, - ctagsRunner, + types.BareCommand(func(ctx *types.Context) error { + ctagsOutput, _, _, _, _err = builder.RunCTags(ctx.Sketch, ctx.SketchSourceMerged, "ctags_target.cpp", ctx.BuildProperties, ctx.PreprocPath) + return _err + }), } for _, command := range commands { err := command.Run(ctx) @@ -123,7 +132,7 @@ func TestCTagsRunnerSketchWithTypename(t *testing.T) { "setup\t" + quotedSketchLocation + "\t/^void setup() {$/;\"\tkind:function\tline:6\tsignature:()\treturntype:void\n" + "loop\t" + quotedSketchLocation + "\t/^void loop() {}$/;\"\tkind:function\tline:10\tsignature:()\treturntype:void\n" + "func\t" + quotedSketchLocation + "\t/^typename Foo::Bar func(){$/;\"\tkind:function\tline:12\tsignature:()\treturntype:Foo::Bar\n" - require.Equal(t, expectedOutput, strings.Replace(string(ctagsRunner.CtagsOutput), "\r\n", "\n", -1)) + require.Equal(t, expectedOutput, strings.Replace(string(ctagsOutput), "\r\n", "\n", -1)) } func TestCTagsRunnerSketchWithNamespace(t *testing.T) { @@ -132,8 +141,8 @@ func TestCTagsRunnerSketchWithNamespace(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) ctx.Verbose = true - ctagsRunner := &builder.CTagsRunner{Source: &ctx.SketchSourceMerged, TargetFileName: "ctags_target.cpp"} var _err error + var ctagsOutput []byte commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { @@ -143,7 +152,10 @@ func TestCTagsRunnerSketchWithNamespace(t *testing.T) { &builder.ContainerFindIncludes{}, &builder.PrintUsedLibrariesIfVerbose{}, &builder.WarnAboutArchIncompatibleLibraries{}, - ctagsRunner, + types.BareCommand(func(ctx *types.Context) error { + ctagsOutput, _, _, _, _err = builder.RunCTags(ctx.Sketch, ctx.SketchSourceMerged, "ctags_target.cpp", ctx.BuildProperties, ctx.PreprocPath) + return _err + }), } for _, command := range commands { err := command.Run(ctx) @@ -154,7 +166,7 @@ func TestCTagsRunnerSketchWithNamespace(t *testing.T) { expectedOutput := "value\t" + quotedSketchLocation + "\t/^\tint value() {$/;\"\tkind:function\tline:2\tnamespace:Test\tsignature:()\treturntype:int\n" + "setup\t" + quotedSketchLocation + "\t/^void setup() {}$/;\"\tkind:function\tline:7\tsignature:()\treturntype:void\n" + "loop\t" + quotedSketchLocation + "\t/^void loop() {}$/;\"\tkind:function\tline:8\tsignature:()\treturntype:void\n" - require.Equal(t, expectedOutput, strings.Replace(string(ctagsRunner.CtagsOutput), "\r\n", "\n", -1)) + require.Equal(t, expectedOutput, strings.Replace(string(ctagsOutput), "\r\n", "\n", -1)) } func TestCTagsRunnerSketchWithTemplates(t *testing.T) { @@ -163,8 +175,8 @@ func TestCTagsRunnerSketchWithTemplates(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) ctx.Verbose = true - ctagsRunner := &builder.CTagsRunner{Source: &ctx.SketchSourceMerged, TargetFileName: "ctags_target.cpp"} var _err error + var ctagsOutput []byte commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { @@ -174,7 +186,10 @@ func TestCTagsRunnerSketchWithTemplates(t *testing.T) { &builder.ContainerFindIncludes{}, &builder.PrintUsedLibrariesIfVerbose{}, &builder.WarnAboutArchIncompatibleLibraries{}, - ctagsRunner, + types.BareCommand(func(ctx *types.Context) error { + ctagsOutput, _, _, _, _err = builder.RunCTags(ctx.Sketch, ctx.SketchSourceMerged, "ctags_target.cpp", ctx.BuildProperties, ctx.PreprocPath) + return _err + }), } for _, command := range commands { err := command.Run(ctx) @@ -186,5 +201,5 @@ func TestCTagsRunnerSketchWithTemplates(t *testing.T) { "bVar\t" + quotedSketchLocation + "\t/^c< 8 > bVar;$/;\"\tkind:variable\tline:15\n" + "aVar\t" + quotedSketchLocation + "\t/^c< 1<<8 > aVar;$/;\"\tkind:variable\tline:16\n" + "func\t" + quotedSketchLocation + "\t/^template func( c< 1< & aParam) {$/;\"\tkind:function\tline:18\tsignature:( c< 1< & aParam)\treturntype:template\n" - require.Equal(t, expectedOutput, strings.Replace(string(ctagsRunner.CtagsOutput), "\r\n", "\n", -1)) + require.Equal(t, expectedOutput, strings.Replace(string(ctagsOutput), "\r\n", "\n", -1)) } From 53946e68ea6593bd1a3804a482830cce422835af Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Wed, 5 Apr 2023 13:03:58 +0200 Subject: [PATCH 21/42] Removed useless tasks from ctags_runner test --- legacy/builder/test/ctags_runner_test.go | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/legacy/builder/test/ctags_runner_test.go b/legacy/builder/test/ctags_runner_test.go index 069339eef48..7c3bdfaf75f 100644 --- a/legacy/builder/test/ctags_runner_test.go +++ b/legacy/builder/test/ctags_runner_test.go @@ -40,9 +40,6 @@ func TestCTagsRunner(t *testing.T) { ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), - &builder.ContainerFindIncludes{}, - &builder.PrintUsedLibrariesIfVerbose{}, - &builder.WarnAboutArchIncompatibleLibraries{}, types.BareCommand(func(ctx *types.Context) error { ctagsOutput, _, _, _, _err = builder.RunCTags(ctx.Sketch, ctx.SketchSourceMerged, "ctags_target.cpp", ctx.BuildProperties, ctx.PreprocPath) return _err @@ -78,9 +75,6 @@ func TestCTagsRunnerSketchWithClass(t *testing.T) { ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), - &builder.ContainerFindIncludes{}, - &builder.PrintUsedLibrariesIfVerbose{}, - &builder.WarnAboutArchIncompatibleLibraries{}, types.BareCommand(func(ctx *types.Context) error { ctagsOutput, _, _, _, _err = builder.RunCTags(ctx.Sketch, ctx.SketchSourceMerged, "ctags_target.cpp", ctx.BuildProperties, ctx.PreprocPath) return _err @@ -114,9 +108,6 @@ func TestCTagsRunnerSketchWithTypename(t *testing.T) { ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), - &builder.ContainerFindIncludes{}, - &builder.PrintUsedLibrariesIfVerbose{}, - &builder.WarnAboutArchIncompatibleLibraries{}, types.BareCommand(func(ctx *types.Context) error { ctagsOutput, _, _, _, _err = builder.RunCTags(ctx.Sketch, ctx.SketchSourceMerged, "ctags_target.cpp", ctx.BuildProperties, ctx.PreprocPath) return _err @@ -149,9 +140,6 @@ func TestCTagsRunnerSketchWithNamespace(t *testing.T) { ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), - &builder.ContainerFindIncludes{}, - &builder.PrintUsedLibrariesIfVerbose{}, - &builder.WarnAboutArchIncompatibleLibraries{}, types.BareCommand(func(ctx *types.Context) error { ctagsOutput, _, _, _, _err = builder.RunCTags(ctx.Sketch, ctx.SketchSourceMerged, "ctags_target.cpp", ctx.BuildProperties, ctx.PreprocPath) return _err @@ -183,9 +171,6 @@ func TestCTagsRunnerSketchWithTemplates(t *testing.T) { ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), - &builder.ContainerFindIncludes{}, - &builder.PrintUsedLibrariesIfVerbose{}, - &builder.WarnAboutArchIncompatibleLibraries{}, types.BareCommand(func(ctx *types.Context) error { ctagsOutput, _, _, _, _err = builder.RunCTags(ctx.Sketch, ctx.SketchSourceMerged, "ctags_target.cpp", ctx.BuildProperties, ctx.PreprocPath) return _err From 43515c22c6be0a141d12a649be9511043982acbb Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Wed, 5 Apr 2023 13:12:31 +0200 Subject: [PATCH 22/42] Simplified ctags_runner test --- legacy/builder/test/ctags_runner_test.go | 115 ++++++----------------- 1 file changed, 30 insertions(+), 85 deletions(-) diff --git a/legacy/builder/test/ctags_runner_test.go b/legacy/builder/test/ctags_runner_test.go index 7c3bdfaf75f..460a3cee3f4 100644 --- a/legacy/builder/test/ctags_runner_test.go +++ b/legacy/builder/test/ctags_runner_test.go @@ -32,23 +32,12 @@ func TestCTagsRunner(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) ctx.Verbose = true - var _err error - var ctagsOutput []byte - commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) - return _err - }), - types.BareCommand(func(ctx *types.Context) error { - ctagsOutput, _, _, _, _err = builder.RunCTags(ctx.Sketch, ctx.SketchSourceMerged, "ctags_target.cpp", ctx.BuildProperties, ctx.PreprocPath) - return _err - }), - } - for _, command := range commands { - err := command.Run(ctx) - NoError(t, err) - } + err := (&builder.ContainerSetupHardwareToolsLibsSketchAndProps{}).Run(ctx) + NoError(t, err) + _, source, err := bldr.PrepareSketchBuildPath(ctx.Sketch, nil, ctx.SketchBuildPath) + NoError(t, err) + ctagsOutput, _, _, _, err := builder.RunCTags(ctx.Sketch, source, "ctags_target.cpp", ctx.BuildProperties, ctx.PreprocPath) + NoError(t, err) quotedSketchLocation := strings.Replace(sketchLocation.String(), "\\", "\\\\", -1) expectedOutput := "server " + quotedSketchLocation + " /^BridgeServer server;$/;\" kind:variable line:31\n" + @@ -67,23 +56,12 @@ func TestCTagsRunnerSketchWithClass(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) ctx.Verbose = true - var _err error - var ctagsOutput []byte - commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) - return _err - }), - types.BareCommand(func(ctx *types.Context) error { - ctagsOutput, _, _, _, _err = builder.RunCTags(ctx.Sketch, ctx.SketchSourceMerged, "ctags_target.cpp", ctx.BuildProperties, ctx.PreprocPath) - return _err - }), - } - for _, command := range commands { - err := command.Run(ctx) - NoError(t, err) - } + err := (&builder.ContainerSetupHardwareToolsLibsSketchAndProps{}).Run(ctx) + NoError(t, err) + _, source, err := bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + NoError(t, err) + ctagsOutput, _, _, _, err := builder.RunCTags(ctx.Sketch, source, "ctags_target.cpp", ctx.BuildProperties, ctx.PreprocPath) + NoError(t, err) quotedSketchLocation := strings.Replace(sketchLocation.String(), "\\", "\\\\", -1) expectedOutput := "set_values\t" + quotedSketchLocation + "\t/^ void set_values (int,int);$/;\"\tkind:prototype\tline:4\tclass:Rectangle\tsignature:(int,int)\treturntype:void\n" + @@ -100,23 +78,12 @@ func TestCTagsRunnerSketchWithTypename(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) ctx.Verbose = true - var _err error - var ctagsOutput []byte - commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) - return _err - }), - types.BareCommand(func(ctx *types.Context) error { - ctagsOutput, _, _, _, _err = builder.RunCTags(ctx.Sketch, ctx.SketchSourceMerged, "ctags_target.cpp", ctx.BuildProperties, ctx.PreprocPath) - return _err - }), - } - for _, command := range commands { - err := command.Run(ctx) - NoError(t, err) - } + err := (&builder.ContainerSetupHardwareToolsLibsSketchAndProps{}).Run(ctx) + NoError(t, err) + _, source, err := bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + NoError(t, err) + ctagsOutput, _, _, _, err := builder.RunCTags(ctx.Sketch, source, "ctags_target.cpp", ctx.BuildProperties, ctx.PreprocPath) + NoError(t, err) quotedSketchLocation := strings.Replace(sketchLocation.String(), "\\", "\\\\", -1) expectedOutput := "Foo\t" + quotedSketchLocation + "\t/^ struct Foo{$/;\"\tkind:struct\tline:2\n" + @@ -132,23 +99,12 @@ func TestCTagsRunnerSketchWithNamespace(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) ctx.Verbose = true - var _err error - var ctagsOutput []byte - commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) - return _err - }), - types.BareCommand(func(ctx *types.Context) error { - ctagsOutput, _, _, _, _err = builder.RunCTags(ctx.Sketch, ctx.SketchSourceMerged, "ctags_target.cpp", ctx.BuildProperties, ctx.PreprocPath) - return _err - }), - } - for _, command := range commands { - err := command.Run(ctx) - NoError(t, err) - } + err := (&builder.ContainerSetupHardwareToolsLibsSketchAndProps{}).Run(ctx) + NoError(t, err) + _, source, err := bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + NoError(t, err) + ctagsOutput, _, _, _, err := builder.RunCTags(ctx.Sketch, source, "ctags_target.cpp", ctx.BuildProperties, ctx.PreprocPath) + NoError(t, err) quotedSketchLocation := strings.Replace(sketchLocation.String(), "\\", "\\\\", -1) expectedOutput := "value\t" + quotedSketchLocation + "\t/^\tint value() {$/;\"\tkind:function\tline:2\tnamespace:Test\tsignature:()\treturntype:int\n" + @@ -163,23 +119,12 @@ func TestCTagsRunnerSketchWithTemplates(t *testing.T) { defer cleanUpBuilderTestContext(t, ctx) ctx.Verbose = true - var _err error - var ctagsOutput []byte - commands := []types.Command{ - &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, - types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) - return _err - }), - types.BareCommand(func(ctx *types.Context) error { - ctagsOutput, _, _, _, _err = builder.RunCTags(ctx.Sketch, ctx.SketchSourceMerged, "ctags_target.cpp", ctx.BuildProperties, ctx.PreprocPath) - return _err - }), - } - for _, command := range commands { - err := command.Run(ctx) - NoError(t, err) - } + err := (&builder.ContainerSetupHardwareToolsLibsSketchAndProps{}).Run(ctx) + NoError(t, err) + _, source, err := bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + NoError(t, err) + ctagsOutput, _, _, _, err := builder.RunCTags(ctx.Sketch, source, "ctags_target.cpp", ctx.BuildProperties, ctx.PreprocPath) + NoError(t, err) quotedSketchLocation := strings.Replace(sketchLocation.String(), "\\", "\\\\", -1) expectedOutput := "printGyro\t" + quotedSketchLocation + "\t/^void printGyro()$/;\"\tkind:function\tline:10\tsignature:()\treturntype:void\n" + From efd739206e6bc30b146781ec658119ba4a472d8d Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Wed, 5 Apr 2023 16:57:12 +0200 Subject: [PATCH 23/42] Removed some ctags related fields from builder context --- legacy/builder/container_add_prototypes.go | 8 ++-- legacy/builder/ctags_runner.go | 41 ++++++-------------- legacy/builder/prototypes_adder.go | 20 +++++++--- legacy/builder/test/ctags_runner_test.go | 10 ++--- legacy/builder/test/prototypes_adder_test.go | 28 ++++++------- legacy/builder/types/context.go | 6 +-- 6 files changed, 51 insertions(+), 62 deletions(-) diff --git a/legacy/builder/container_add_prototypes.go b/legacy/builder/container_add_prototypes.go index 99059b0fb76..35b575b76a8 100644 --- a/legacy/builder/container_add_prototypes.go +++ b/legacy/builder/container_add_prototypes.go @@ -62,16 +62,16 @@ func PreprocessSketchWithCtags(ctx *types.Context) error { ctx.SketchSourceAfterCppPreprocessing = filterSketchSource(ctx.Sketch, bytes.NewReader(src), false) } - var ctagsStderr []byte - _, ctagsStderr, ctx.PrototypesLineWhereToInsert, ctx.Prototypes, err = RunCTags( - ctx.Sketch, ctx.SketchSourceAfterCppPreprocessing, "sketch_merged.cpp", ctx.BuildProperties, ctx.PreprocPath) + ctagsStdout, ctagsStderr, err := RunCTags( + ctx.SketchSourceAfterCppPreprocessing, "sketch_merged.cpp", ctx.BuildProperties, ctx.PreprocPath) if ctx.Verbose { ctx.WriteStderr(ctagsStderr) } if err != nil { return err } - ctx.SketchSourceAfterArduinoPreprocessing, ctx.PrototypesSection = PrototypesAdder(ctx.SketchSourceMerged, ctx.PrototypesLineWhereToInsert, ctx.LineOffset, ctx.Prototypes, ctx.DebugPreprocessor) + ctx.SketchSourceAfterArduinoPreprocessing = PrototypesAdder( + ctx.Sketch, ctx.SketchSourceMerged, ctagsStdout, ctx.LineOffset, ctx.DebugPreprocessor) if err := bldr.SketchSaveItemCpp(ctx.Sketch.MainFile, []byte(ctx.SketchSourceAfterArduinoPreprocessing), ctx.SketchBuildPath); err != nil { return errors.WithStack(err) diff --git a/legacy/builder/ctags_runner.go b/legacy/builder/ctags_runner.go index 563444f0b7d..38ddc4b4e0a 100644 --- a/legacy/builder/ctags_runner.go +++ b/legacy/builder/ctags_runner.go @@ -19,23 +19,21 @@ import ( "bytes" "strings" - "github.com/arduino/arduino-cli/arduino/sketch" "github.com/arduino/arduino-cli/executils" - "github.com/arduino/arduino-cli/legacy/builder/ctags" "github.com/arduino/go-paths-helper" properties "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" ) -func RunCTags(sketch *sketch.Sketch, source string, targetFileName string, buildProperties *properties.Map, preprocPath *paths.Path, -) (ctagsStdout, ctagsStderr []byte, prototypesLineWhereToInsert int, prototypes []*ctags.Prototype, err error) { - if err = preprocPath.MkdirAll(); err != nil { - return +func RunCTags(source string, targetFileName string, buildProperties *properties.Map, preprocPath *paths.Path, +) ([]byte, []byte, error) { + if err := preprocPath.MkdirAll(); err != nil { + return nil, nil, err } ctagsTargetFilePath := preprocPath.Join(targetFileName) - if err = ctagsTargetFilePath.WriteFile([]byte(source)); err != nil { - return + if err := ctagsTargetFilePath.WriteFile([]byte(source)); err != nil { + return nil, nil, err } ctagsBuildProperties := properties.NewMap() @@ -48,40 +46,25 @@ func RunCTags(sketch *sketch.Sketch, source string, targetFileName string, build pattern := ctagsBuildProperties.Get("pattern") if pattern == "" { - err = errors.Errorf(tr("%s pattern is missing"), "ctags") - return + return nil, nil, errors.Errorf(tr("%s pattern is missing"), "ctags") } commandLine := ctagsBuildProperties.ExpandPropsInString(pattern) parts, err := properties.SplitQuotedString(commandLine, `"'`, false) if err != nil { - return + return nil, nil, err } proc, err := executils.NewProcess(nil, parts...) if err != nil { - return + return nil, nil, err } stdout := &bytes.Buffer{} stderr := &bytes.Buffer{} proc.RedirectStdoutTo(stdout) proc.RedirectStderrTo(stderr) if err = proc.Run(); err != nil { - return + return nil, nil, err } - stderr.WriteString(strings.Join(parts, " ")) - ctagsStdout = stdout.Bytes() - ctagsStderr = stderr.Bytes() - if err != nil { - return - } - - parser := &ctags.CTagsParser{} - parser.Parse(ctagsStdout, sketch.MainFile) - parser.FixCLinkageTagsDeclarations() - - prototypes, line := parser.GeneratePrototypes() - if line != -1 { - prototypesLineWhereToInsert = line - } - return + _, _ = stderr.WriteString(strings.Join(parts, " ")) + return stdout.Bytes(), stderr.Bytes(), err } diff --git a/legacy/builder/prototypes_adder.go b/legacy/builder/prototypes_adder.go index a7676d42038..4d5f468ae83 100644 --- a/legacy/builder/prototypes_adder.go +++ b/legacy/builder/prototypes_adder.go @@ -20,23 +20,33 @@ import ( "strconv" "strings" + "github.com/arduino/arduino-cli/arduino/sketch" "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/ctags" "github.com/arduino/arduino-cli/legacy/builder/utils" ) -func PrototypesAdder(source string, firstFunctionLine, lineOffset int, prototypes []*ctags.Prototype, debugOutput bool) (preprocessedSource, prototypeSection string) { +func PrototypesAdder(sketch *sketch.Sketch, source string, ctagsStdout []byte, lineOffset int, debugOutput bool) string { + parser := &ctags.CTagsParser{} + parser.Parse(ctagsStdout, sketch.MainFile) + parser.FixCLinkageTagsDeclarations() + + prototypes, firstFunctionLine := parser.GeneratePrototypes() + if firstFunctionLine == -1 { + firstFunctionLine = 0 + } + source = strings.Replace(source, "\r\n", "\n", -1) source = strings.Replace(source, "\r", "\n", -1) sourceRows := strings.Split(source, "\n") if isFirstFunctionOutsideOfSource(firstFunctionLine, sourceRows) { - return + return "" } insertionLine := firstFunctionLine + lineOffset - 1 firstFunctionChar := len(strings.Join(sourceRows[:insertionLine], "\n")) + 1 - prototypeSection = composePrototypeSection(firstFunctionLine, prototypes) - preprocessedSource = source[:firstFunctionChar] + prototypeSection + source[firstFunctionChar:] + prototypeSection := composePrototypeSection(firstFunctionLine, prototypes) + preprocessedSource := source[:firstFunctionChar] + prototypeSection + source[firstFunctionChar:] if debugOutput { fmt.Println("#PREPROCESSED SOURCE") @@ -53,7 +63,7 @@ func PrototypesAdder(source string, firstFunctionLine, lineOffset int, prototype } fmt.Println("#END OF PREPROCESSED SOURCE") } - return + return preprocessedSource } func composePrototypeSection(line int, prototypes []*ctags.Prototype) string { diff --git a/legacy/builder/test/ctags_runner_test.go b/legacy/builder/test/ctags_runner_test.go index 460a3cee3f4..3b46a20751c 100644 --- a/legacy/builder/test/ctags_runner_test.go +++ b/legacy/builder/test/ctags_runner_test.go @@ -36,7 +36,7 @@ func TestCTagsRunner(t *testing.T) { NoError(t, err) _, source, err := bldr.PrepareSketchBuildPath(ctx.Sketch, nil, ctx.SketchBuildPath) NoError(t, err) - ctagsOutput, _, _, _, err := builder.RunCTags(ctx.Sketch, source, "ctags_target.cpp", ctx.BuildProperties, ctx.PreprocPath) + ctagsOutput, _, err := builder.RunCTags(source, "ctags_target.cpp", ctx.BuildProperties, ctx.PreprocPath) NoError(t, err) quotedSketchLocation := strings.Replace(sketchLocation.String(), "\\", "\\\\", -1) @@ -60,7 +60,7 @@ func TestCTagsRunnerSketchWithClass(t *testing.T) { NoError(t, err) _, source, err := bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) NoError(t, err) - ctagsOutput, _, _, _, err := builder.RunCTags(ctx.Sketch, source, "ctags_target.cpp", ctx.BuildProperties, ctx.PreprocPath) + ctagsOutput, _, err := builder.RunCTags(source, "ctags_target.cpp", ctx.BuildProperties, ctx.PreprocPath) NoError(t, err) quotedSketchLocation := strings.Replace(sketchLocation.String(), "\\", "\\\\", -1) @@ -82,7 +82,7 @@ func TestCTagsRunnerSketchWithTypename(t *testing.T) { NoError(t, err) _, source, err := bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) NoError(t, err) - ctagsOutput, _, _, _, err := builder.RunCTags(ctx.Sketch, source, "ctags_target.cpp", ctx.BuildProperties, ctx.PreprocPath) + ctagsOutput, _, err := builder.RunCTags(source, "ctags_target.cpp", ctx.BuildProperties, ctx.PreprocPath) NoError(t, err) quotedSketchLocation := strings.Replace(sketchLocation.String(), "\\", "\\\\", -1) @@ -103,7 +103,7 @@ func TestCTagsRunnerSketchWithNamespace(t *testing.T) { NoError(t, err) _, source, err := bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) NoError(t, err) - ctagsOutput, _, _, _, err := builder.RunCTags(ctx.Sketch, source, "ctags_target.cpp", ctx.BuildProperties, ctx.PreprocPath) + ctagsOutput, _, err := builder.RunCTags(source, "ctags_target.cpp", ctx.BuildProperties, ctx.PreprocPath) NoError(t, err) quotedSketchLocation := strings.Replace(sketchLocation.String(), "\\", "\\\\", -1) @@ -123,7 +123,7 @@ func TestCTagsRunnerSketchWithTemplates(t *testing.T) { NoError(t, err) _, source, err := bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) NoError(t, err) - ctagsOutput, _, _, _, err := builder.RunCTags(ctx.Sketch, source, "ctags_target.cpp", ctx.BuildProperties, ctx.PreprocPath) + ctagsOutput, _, err := builder.RunCTags(source, "ctags_target.cpp", ctx.BuildProperties, ctx.PreprocPath) NoError(t, err) quotedSketchLocation := strings.Replace(sketchLocation.String(), "\\", "\\\\", -1) diff --git a/legacy/builder/test/prototypes_adder_test.go b/legacy/builder/test/prototypes_adder_test.go index 947f5bdbf02..64630a7ba70 100644 --- a/legacy/builder/test/prototypes_adder_test.go +++ b/legacy/builder/test/prototypes_adder_test.go @@ -54,7 +54,7 @@ func TestPrototypesAdderBridgeExample(t *testing.T) { NoError(t, builder.PreprocessSketchWithCtags(ctx)) require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") - require.Equal(t, "#line 33 "+quotedSketchLocation+"\nvoid setup();\n#line 46 "+quotedSketchLocation+"\nvoid loop();\n#line 62 "+quotedSketchLocation+"\nvoid process(BridgeClient client);\n#line 82 "+quotedSketchLocation+"\nvoid digitalCommand(BridgeClient client);\n#line 109 "+quotedSketchLocation+"\nvoid analogCommand(BridgeClient client);\n#line 149 "+quotedSketchLocation+"\nvoid modeCommand(BridgeClient client);\n#line 33 "+quotedSketchLocation+"\n", ctx.PrototypesSection) + require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#line 33 "+quotedSketchLocation+"\nvoid setup();\n#line 46 "+quotedSketchLocation+"\nvoid loop();\n#line 62 "+quotedSketchLocation+"\nvoid process(BridgeClient client);\n#line 82 "+quotedSketchLocation+"\nvoid digitalCommand(BridgeClient client);\n#line 109 "+quotedSketchLocation+"\nvoid analogCommand(BridgeClient client);\n#line 149 "+quotedSketchLocation+"\nvoid modeCommand(BridgeClient client);\n#line 33 "+quotedSketchLocation+"\n") } func TestPrototypesAdderSketchWithIfDef(t *testing.T) { @@ -262,7 +262,7 @@ func TestPrototypesAdderSketchWithConfig(t *testing.T) { NoError(t, builder.PreprocessSketchWithCtags(ctx)) require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") - require.Equal(t, "#line 13 "+quotedSketchLocation+"\nvoid setup();\n#line 17 "+quotedSketchLocation+"\nvoid loop();\n#line 13 "+quotedSketchLocation+"\n", ctx.PrototypesSection) + require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#line 13 "+quotedSketchLocation+"\nvoid setup();\n#line 17 "+quotedSketchLocation+"\nvoid loop();\n#line 13 "+quotedSketchLocation+"\n") preprocessed := LoadAndInterpolate(t, filepath.Join("sketch_with_config", "sketch_with_config.preprocessed.txt"), ctx) require.Equal(t, preprocessed, strings.Replace(ctx.SketchSourceAfterArduinoPreprocessing, "\r\n", "\n", -1)) @@ -293,7 +293,7 @@ func TestPrototypesAdderSketchNoFunctionsTwoFiles(t *testing.T) { NoError(t, builder.PreprocessSketchWithCtags(ctx)) require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") - require.Equal(t, "", ctx.PrototypesSection) + require.Equal(t, ctx.SketchSourceMerged, ctx.SketchSourceAfterArduinoPreprocessing) // No prototypes added } func TestPrototypesAdderSketchNoFunctions(t *testing.T) { @@ -320,7 +320,7 @@ func TestPrototypesAdderSketchNoFunctions(t *testing.T) { NoError(t, builder.PreprocessSketchWithCtags(ctx)) require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") - require.Equal(t, "", ctx.PrototypesSection) + require.Equal(t, ctx.SketchSourceMerged, ctx.SketchSourceAfterArduinoPreprocessing) // No prototypes added } func TestPrototypesAdderSketchWithDefaultArgs(t *testing.T) { @@ -348,7 +348,7 @@ func TestPrototypesAdderSketchWithDefaultArgs(t *testing.T) { NoError(t, builder.PreprocessSketchWithCtags(ctx)) require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") - require.Equal(t, "#line 4 "+quotedSketchLocation+"\nvoid setup();\n#line 7 "+quotedSketchLocation+"\nvoid loop();\n#line 1 "+quotedSketchLocation+"\n", ctx.PrototypesSection) + require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#line 4 "+quotedSketchLocation+"\nvoid setup();\n#line 7 "+quotedSketchLocation+"\nvoid loop();\n#line 1 "+quotedSketchLocation+"\n") } func TestPrototypesAdderSketchWithInlineFunction(t *testing.T) { @@ -378,7 +378,7 @@ func TestPrototypesAdderSketchWithInlineFunction(t *testing.T) { require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") expected := "#line 1 " + quotedSketchLocation + "\nvoid setup();\n#line 2 " + quotedSketchLocation + "\nvoid loop();\n#line 4 " + quotedSketchLocation + "\nshort unsigned int testInt();\n#line 8 " + quotedSketchLocation + "\nstatic int8_t testInline();\n#line 12 " + quotedSketchLocation + "\n__attribute__((always_inline)) uint8_t testAttribute();\n#line 1 " + quotedSketchLocation + "\n" - obtained := ctx.PrototypesSection + obtained := ctx.SketchSourceAfterArduinoPreprocessing // ctags based preprocessing removes "inline" but this is still OK // TODO: remove this exception when moving to a more powerful parser expected = strings.Replace(expected, "static inline int8_t testInline();", "static int8_t testInline();", -1) @@ -387,7 +387,7 @@ func TestPrototypesAdderSketchWithInlineFunction(t *testing.T) { // TODO: remove this exception when moving to a more powerful parser expected = strings.Replace(expected, "__attribute__((always_inline)) uint8_t testAttribute();", "uint8_t testAttribute();", -1) obtained = strings.Replace(obtained, "__attribute__((always_inline)) uint8_t testAttribute();", "uint8_t testAttribute();", -1) - require.Equal(t, expected, obtained) + require.Contains(t, obtained, expected) } func TestPrototypesAdderSketchWithFunctionSignatureInsideIFDEF(t *testing.T) { @@ -415,7 +415,7 @@ func TestPrototypesAdderSketchWithFunctionSignatureInsideIFDEF(t *testing.T) { NoError(t, builder.PreprocessSketchWithCtags(ctx)) require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") - require.Equal(t, "#line 1 "+quotedSketchLocation+"\nvoid setup();\n#line 3 "+quotedSketchLocation+"\nvoid loop();\n#line 15 "+quotedSketchLocation+"\nint8_t adalight();\n#line 1 "+quotedSketchLocation+"\n", ctx.PrototypesSection) + require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#line 1 "+quotedSketchLocation+"\nvoid setup();\n#line 3 "+quotedSketchLocation+"\nvoid loop();\n#line 15 "+quotedSketchLocation+"\nint8_t adalight();\n#line 1 "+quotedSketchLocation+"\n") } func TestPrototypesAdderSketchWithUSBCON(t *testing.T) { @@ -448,7 +448,7 @@ func TestPrototypesAdderSketchWithUSBCON(t *testing.T) { NoError(t, builder.PreprocessSketchWithCtags(ctx)) require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") - require.Equal(t, "#line 5 "+quotedSketchLocation+"\nvoid ciao();\n#line 10 "+quotedSketchLocation+"\nvoid setup();\n#line 15 "+quotedSketchLocation+"\nvoid loop();\n#line 5 "+quotedSketchLocation+"\n", ctx.PrototypesSection) + require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#line 5 "+quotedSketchLocation+"\nvoid ciao();\n#line 10 "+quotedSketchLocation+"\nvoid setup();\n#line 15 "+quotedSketchLocation+"\nvoid loop();\n#line 5 "+quotedSketchLocation+"\n") } func TestPrototypesAdderSketchWithTypename(t *testing.T) { @@ -481,12 +481,12 @@ func TestPrototypesAdderSketchWithTypename(t *testing.T) { require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") expected := "#line 6 " + quotedSketchLocation + "\nvoid setup();\n#line 10 " + quotedSketchLocation + "\nvoid loop();\n#line 12 " + quotedSketchLocation + "\ntypename Foo::Bar func();\n#line 6 " + quotedSketchLocation + "\n" - obtained := ctx.PrototypesSection + obtained := ctx.SketchSourceAfterArduinoPreprocessing // ctags based preprocessing ignores line with typename // TODO: remove this exception when moving to a more powerful parser expected = strings.Replace(expected, "#line 12 "+quotedSketchLocation+"\ntypename Foo::Bar func();\n", "", -1) obtained = strings.Replace(obtained, "#line 12 "+quotedSketchLocation+"\ntypename Foo::Bar func();\n", "", -1) - require.Equal(t, expected, obtained) + require.Contains(t, obtained, expected) } func TestPrototypesAdderSketchWithIfDef2(t *testing.T) { @@ -514,7 +514,7 @@ func TestPrototypesAdderSketchWithIfDef2(t *testing.T) { NoError(t, builder.PreprocessSketchWithCtags(ctx)) require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") - require.Equal(t, "#line 5 "+quotedSketchLocation+"\nvoid elseBranch();\n#line 9 "+quotedSketchLocation+"\nvoid f1();\n#line 10 "+quotedSketchLocation+"\nvoid f2();\n#line 12 "+quotedSketchLocation+"\nvoid setup();\n#line 14 "+quotedSketchLocation+"\nvoid loop();\n#line 5 "+quotedSketchLocation+"\n", ctx.PrototypesSection) + require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#line 5 "+quotedSketchLocation+"\nvoid elseBranch();\n#line 9 "+quotedSketchLocation+"\nvoid f1();\n#line 10 "+quotedSketchLocation+"\nvoid f2();\n#line 12 "+quotedSketchLocation+"\nvoid setup();\n#line 14 "+quotedSketchLocation+"\nvoid loop();\n#line 5 "+quotedSketchLocation+"\n") expectedSource := LoadAndInterpolate(t, filepath.Join("sketch_with_ifdef", "sketch.preprocessed.txt"), ctx) require.Equal(t, expectedSource, strings.Replace(ctx.SketchSourceAfterArduinoPreprocessing, "\r\n", "\n", -1)) @@ -545,7 +545,7 @@ func TestPrototypesAdderSketchWithIfDef2SAM(t *testing.T) { NoError(t, builder.PreprocessSketchWithCtags(ctx)) require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") - require.Equal(t, "#line 2 "+quotedSketchLocation+"\nvoid ifBranch();\n#line 9 "+quotedSketchLocation+"\nvoid f1();\n#line 10 "+quotedSketchLocation+"\nvoid f2();\n#line 12 "+quotedSketchLocation+"\nvoid setup();\n#line 14 "+quotedSketchLocation+"\nvoid loop();\n#line 2 "+quotedSketchLocation+"\n", ctx.PrototypesSection) + require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#line 2 "+quotedSketchLocation+"\nvoid ifBranch();\n#line 9 "+quotedSketchLocation+"\nvoid f1();\n#line 10 "+quotedSketchLocation+"\nvoid f2();\n#line 12 "+quotedSketchLocation+"\nvoid setup();\n#line 14 "+quotedSketchLocation+"\nvoid loop();\n#line 2 "+quotedSketchLocation+"\n") expectedSource := LoadAndInterpolate(t, filepath.Join("sketch_with_ifdef", "sketch.preprocessed.SAM.txt"), ctx) require.Equal(t, expectedSource, strings.Replace(ctx.SketchSourceAfterArduinoPreprocessing, "\r\n", "\n", -1)) @@ -576,7 +576,7 @@ func TestPrototypesAdderSketchWithConst(t *testing.T) { NoError(t, builder.PreprocessSketchWithCtags(ctx)) require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") - require.Equal(t, "#line 1 "+quotedSketchLocation+"\nvoid setup();\n#line 2 "+quotedSketchLocation+"\nvoid loop();\n#line 4 "+quotedSketchLocation+"\nconst __FlashStringHelper* test();\n#line 6 "+quotedSketchLocation+"\nconst int test3();\n#line 8 "+quotedSketchLocation+"\nvolatile __FlashStringHelper* test2();\n#line 10 "+quotedSketchLocation+"\nvolatile int test4();\n#line 1 "+quotedSketchLocation+"\n", ctx.PrototypesSection) + require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#line 1 "+quotedSketchLocation+"\nvoid setup();\n#line 2 "+quotedSketchLocation+"\nvoid loop();\n#line 4 "+quotedSketchLocation+"\nconst __FlashStringHelper* test();\n#line 6 "+quotedSketchLocation+"\nconst int test3();\n#line 8 "+quotedSketchLocation+"\nvolatile __FlashStringHelper* test2();\n#line 10 "+quotedSketchLocation+"\nvolatile int test4();\n#line 1 "+quotedSketchLocation+"\n") } func TestPrototypesAdderSketchWithDosEol(t *testing.T) { diff --git a/legacy/builder/types/context.go b/legacy/builder/types/context.go index ed21f25ebf1..9f71f527198 100644 --- a/legacy/builder/types/context.go +++ b/legacy/builder/types/context.go @@ -29,7 +29,6 @@ import ( "github.com/arduino/arduino-cli/arduino/libraries/librariesmanager" "github.com/arduino/arduino-cli/arduino/libraries/librariesresolver" "github.com/arduino/arduino-cli/arduino/sketch" - "github.com/arduino/arduino-cli/legacy/builder/ctags" rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" paths "github.com/arduino/go-paths-helper" properties "github.com/arduino/go-properties-orderedmap" @@ -119,10 +118,7 @@ type Context struct { UseCachedLibrariesResolution bool // C++ Parsing - LineOffset int - PrototypesSection string - PrototypesLineWhereToInsert int - Prototypes []*ctags.Prototype + LineOffset int // Verbosity settings Verbose bool From 8c072163dad9aeea9d75e86a440a69d2c774f56d Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Wed, 5 Apr 2023 22:45:32 +0200 Subject: [PATCH 24/42] Simplified RunCTags / factored test subroutine --- legacy/builder/container_add_prototypes.go | 7 ++- legacy/builder/ctags_runner.go | 14 +---- legacy/builder/test/ctags_runner_test.go | 62 +++++++--------------- 3 files changed, 25 insertions(+), 58 deletions(-) diff --git a/legacy/builder/container_add_prototypes.go b/legacy/builder/container_add_prototypes.go index 35b575b76a8..49fecab0784 100644 --- a/legacy/builder/container_add_prototypes.go +++ b/legacy/builder/container_add_prototypes.go @@ -62,8 +62,11 @@ func PreprocessSketchWithCtags(ctx *types.Context) error { ctx.SketchSourceAfterCppPreprocessing = filterSketchSource(ctx.Sketch, bytes.NewReader(src), false) } - ctagsStdout, ctagsStderr, err := RunCTags( - ctx.SketchSourceAfterCppPreprocessing, "sketch_merged.cpp", ctx.BuildProperties, ctx.PreprocPath) + if err := targetFilePath.WriteFile([]byte(ctx.SketchSourceAfterCppPreprocessing)); err != nil { + return err + } + + ctagsStdout, ctagsStderr, err := RunCTags(targetFilePath, ctx.BuildProperties) if ctx.Verbose { ctx.WriteStderr(ctagsStderr) } diff --git a/legacy/builder/ctags_runner.go b/legacy/builder/ctags_runner.go index 38ddc4b4e0a..9463b635f7a 100644 --- a/legacy/builder/ctags_runner.go +++ b/legacy/builder/ctags_runner.go @@ -25,24 +25,14 @@ import ( "github.com/pkg/errors" ) -func RunCTags(source string, targetFileName string, buildProperties *properties.Map, preprocPath *paths.Path, -) ([]byte, []byte, error) { - if err := preprocPath.MkdirAll(); err != nil { - return nil, nil, err - } - - ctagsTargetFilePath := preprocPath.Join(targetFileName) - if err := ctagsTargetFilePath.WriteFile([]byte(source)); err != nil { - return nil, nil, err - } - +func RunCTags(sourceFile *paths.Path, buildProperties *properties.Map) ([]byte, []byte, error) { ctagsBuildProperties := properties.NewMap() ctagsBuildProperties.Set("tools.ctags.path", "{runtime.tools.ctags.path}") ctagsBuildProperties.Set("tools.ctags.cmd.path", "{path}/ctags") ctagsBuildProperties.Set("tools.ctags.pattern", `"{cmd.path}" -u --language-force=c++ -f - --c++-kinds=svpf --fields=KSTtzns --line-directives "{source_file}"`) ctagsBuildProperties.Merge(buildProperties) ctagsBuildProperties.Merge(ctagsBuildProperties.SubTree("tools").SubTree("ctags")) - ctagsBuildProperties.SetPath("source_file", ctagsTargetFilePath) + ctagsBuildProperties.SetPath("source_file", sourceFile) pattern := ctagsBuildProperties.Get("pattern") if pattern == "" { diff --git a/legacy/builder/test/ctags_runner_test.go b/legacy/builder/test/ctags_runner_test.go index 3b46a20751c..036547afae6 100644 --- a/legacy/builder/test/ctags_runner_test.go +++ b/legacy/builder/test/ctags_runner_test.go @@ -21,24 +21,34 @@ import ( bldr "github.com/arduino/arduino-cli/arduino/builder" "github.com/arduino/arduino-cli/legacy/builder" - "github.com/arduino/arduino-cli/legacy/builder/types" paths "github.com/arduino/go-paths-helper" "github.com/stretchr/testify/require" ) -func TestCTagsRunner(t *testing.T) { - sketchLocation := Abs(t, paths.New("downloaded_libraries", "Bridge", "examples", "Bridge", "Bridge.ino")) +func ctagsRunnerTestTemplate(t *testing.T, sketchLocation *paths.Path) []byte { ctx := prepareBuilderTestContext(t, nil, sketchLocation, "arduino:avr:leonardo") defer cleanUpBuilderTestContext(t, ctx) ctx.Verbose = true err := (&builder.ContainerSetupHardwareToolsLibsSketchAndProps{}).Run(ctx) NoError(t, err) + _, source, err := bldr.PrepareSketchBuildPath(ctx.Sketch, nil, ctx.SketchBuildPath) NoError(t, err) - ctagsOutput, _, err := builder.RunCTags(source, "ctags_target.cpp", ctx.BuildProperties, ctx.PreprocPath) + + target := ctx.BuildPath.Join("ctags_target.cpp") + NoError(t, target.WriteFile([]byte(source))) + + ctagsOutput, _, err := builder.RunCTags(target, ctx.BuildProperties) NoError(t, err) + return ctagsOutput +} + +func TestCTagsRunner(t *testing.T) { + sketchLocation := Abs(t, paths.New("downloaded_libraries", "Bridge", "examples", "Bridge", "Bridge.ino")) + ctagsOutput := ctagsRunnerTestTemplate(t, sketchLocation) + quotedSketchLocation := strings.Replace(sketchLocation.String(), "\\", "\\\\", -1) expectedOutput := "server " + quotedSketchLocation + " /^BridgeServer server;$/;\" kind:variable line:31\n" + "setup " + quotedSketchLocation + " /^void setup() {$/;\" kind:function line:33 signature:() returntype:void\n" + @@ -52,16 +62,7 @@ func TestCTagsRunner(t *testing.T) { func TestCTagsRunnerSketchWithClass(t *testing.T) { sketchLocation := Abs(t, paths.New("sketch_with_class", "sketch_with_class.ino")) - ctx := prepareBuilderTestContext(t, nil, sketchLocation, "arduino:avr:leonardo") - defer cleanUpBuilderTestContext(t, ctx) - ctx.Verbose = true - - err := (&builder.ContainerSetupHardwareToolsLibsSketchAndProps{}).Run(ctx) - NoError(t, err) - _, source, err := bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) - NoError(t, err) - ctagsOutput, _, err := builder.RunCTags(source, "ctags_target.cpp", ctx.BuildProperties, ctx.PreprocPath) - NoError(t, err) + ctagsOutput := ctagsRunnerTestTemplate(t, sketchLocation) quotedSketchLocation := strings.Replace(sketchLocation.String(), "\\", "\\\\", -1) expectedOutput := "set_values\t" + quotedSketchLocation + "\t/^ void set_values (int,int);$/;\"\tkind:prototype\tline:4\tclass:Rectangle\tsignature:(int,int)\treturntype:void\n" + @@ -74,16 +75,7 @@ func TestCTagsRunnerSketchWithClass(t *testing.T) { func TestCTagsRunnerSketchWithTypename(t *testing.T) { sketchLocation := Abs(t, paths.New("sketch_with_typename", "sketch_with_typename.ino")) - ctx := prepareBuilderTestContext(t, nil, sketchLocation, "arduino:avr:leonardo") - defer cleanUpBuilderTestContext(t, ctx) - ctx.Verbose = true - - err := (&builder.ContainerSetupHardwareToolsLibsSketchAndProps{}).Run(ctx) - NoError(t, err) - _, source, err := bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) - NoError(t, err) - ctagsOutput, _, err := builder.RunCTags(source, "ctags_target.cpp", ctx.BuildProperties, ctx.PreprocPath) - NoError(t, err) + ctagsOutput := ctagsRunnerTestTemplate(t, sketchLocation) quotedSketchLocation := strings.Replace(sketchLocation.String(), "\\", "\\\\", -1) expectedOutput := "Foo\t" + quotedSketchLocation + "\t/^ struct Foo{$/;\"\tkind:struct\tline:2\n" + @@ -95,16 +87,7 @@ func TestCTagsRunnerSketchWithTypename(t *testing.T) { func TestCTagsRunnerSketchWithNamespace(t *testing.T) { sketchLocation := Abs(t, paths.New("sketch_with_namespace", "sketch_with_namespace.ino")) - ctx := prepareBuilderTestContext(t, nil, sketchLocation, "arduino:avr:leonardo") - defer cleanUpBuilderTestContext(t, ctx) - ctx.Verbose = true - - err := (&builder.ContainerSetupHardwareToolsLibsSketchAndProps{}).Run(ctx) - NoError(t, err) - _, source, err := bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) - NoError(t, err) - ctagsOutput, _, err := builder.RunCTags(source, "ctags_target.cpp", ctx.BuildProperties, ctx.PreprocPath) - NoError(t, err) + ctagsOutput := ctagsRunnerTestTemplate(t, sketchLocation) quotedSketchLocation := strings.Replace(sketchLocation.String(), "\\", "\\\\", -1) expectedOutput := "value\t" + quotedSketchLocation + "\t/^\tint value() {$/;\"\tkind:function\tline:2\tnamespace:Test\tsignature:()\treturntype:int\n" + @@ -115,16 +98,7 @@ func TestCTagsRunnerSketchWithNamespace(t *testing.T) { func TestCTagsRunnerSketchWithTemplates(t *testing.T) { sketchLocation := Abs(t, paths.New("sketch_with_templates_and_shift", "sketch_with_templates_and_shift.ino")) - ctx := prepareBuilderTestContext(t, nil, sketchLocation, "arduino:avr:leonardo") - defer cleanUpBuilderTestContext(t, ctx) - ctx.Verbose = true - - err := (&builder.ContainerSetupHardwareToolsLibsSketchAndProps{}).Run(ctx) - NoError(t, err) - _, source, err := bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) - NoError(t, err) - ctagsOutput, _, err := builder.RunCTags(source, "ctags_target.cpp", ctx.BuildProperties, ctx.PreprocPath) - NoError(t, err) + ctagsOutput := ctagsRunnerTestTemplate(t, sketchLocation) quotedSketchLocation := strings.Replace(sketchLocation.String(), "\\", "\\\\", -1) expectedOutput := "printGyro\t" + quotedSketchLocation + "\t/^void printGyro()$/;\"\tkind:function\tline:10\tsignature:()\treturntype:void\n" + From 3c65cb837c6d91d516f5b2cdda00885059e27855 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Wed, 5 Apr 2023 22:55:53 +0200 Subject: [PATCH 25/42] Removed DebugPreprocessor from builder ctx --- legacy/builder/container_add_prototypes.go | 3 +-- legacy/builder/prototypes_adder.go | 6 ++++-- legacy/builder/test/try_build_of_problematic_sketch_test.go | 2 +- legacy/builder/types/context.go | 3 +-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/legacy/builder/container_add_prototypes.go b/legacy/builder/container_add_prototypes.go index 49fecab0784..e19549fd400 100644 --- a/legacy/builder/container_add_prototypes.go +++ b/legacy/builder/container_add_prototypes.go @@ -73,8 +73,7 @@ func PreprocessSketchWithCtags(ctx *types.Context) error { if err != nil { return err } - ctx.SketchSourceAfterArduinoPreprocessing = PrototypesAdder( - ctx.Sketch, ctx.SketchSourceMerged, ctagsStdout, ctx.LineOffset, ctx.DebugPreprocessor) + ctx.SketchSourceAfterArduinoPreprocessing = PrototypesAdder(ctx.Sketch, ctx.SketchSourceMerged, ctagsStdout, ctx.LineOffset) if err := bldr.SketchSaveItemCpp(ctx.Sketch.MainFile, []byte(ctx.SketchSourceAfterArduinoPreprocessing), ctx.SketchBuildPath); err != nil { return errors.WithStack(err) diff --git a/legacy/builder/prototypes_adder.go b/legacy/builder/prototypes_adder.go index 4d5f468ae83..fba8dd94e4c 100644 --- a/legacy/builder/prototypes_adder.go +++ b/legacy/builder/prototypes_adder.go @@ -26,7 +26,9 @@ import ( "github.com/arduino/arduino-cli/legacy/builder/utils" ) -func PrototypesAdder(sketch *sketch.Sketch, source string, ctagsStdout []byte, lineOffset int, debugOutput bool) string { +var DebugPreprocessor bool + +func PrototypesAdder(sketch *sketch.Sketch, source string, ctagsStdout []byte, lineOffset int) string { parser := &ctags.CTagsParser{} parser.Parse(ctagsStdout, sketch.MainFile) parser.FixCLinkageTagsDeclarations() @@ -48,7 +50,7 @@ func PrototypesAdder(sketch *sketch.Sketch, source string, ctagsStdout []byte, l prototypeSection := composePrototypeSection(firstFunctionLine, prototypes) preprocessedSource := source[:firstFunctionChar] + prototypeSection + source[firstFunctionChar:] - if debugOutput { + if DebugPreprocessor { fmt.Println("#PREPROCESSED SOURCE") prototypesRows := strings.Split(prototypeSection, "\n") prototypesRows = prototypesRows[:len(prototypesRows)-1] diff --git a/legacy/builder/test/try_build_of_problematic_sketch_test.go b/legacy/builder/test/try_build_of_problematic_sketch_test.go index c82c6e0b290..bb6457728ae 100644 --- a/legacy/builder/test/try_build_of_problematic_sketch_test.go +++ b/legacy/builder/test/try_build_of_problematic_sketch_test.go @@ -202,13 +202,13 @@ func TestTryBuild042(t *testing.T) { } func makeDefaultContext() *types.Context { + builder.DebugPreprocessor = true return &types.Context{ HardwareDirs: paths.NewPathList(filepath.Join("..", "hardware"), "downloaded_hardware", "downloaded_board_manager_stuff"), BuiltInToolsDirs: paths.NewPathList("downloaded_tools"), BuiltInLibrariesDirs: paths.New("downloaded_libraries"), OtherLibrariesDirs: paths.NewPathList("libraries"), Verbose: true, - DebugPreprocessor: true, } } diff --git a/legacy/builder/types/context.go b/legacy/builder/types/context.go index 9f71f527198..2b5fd27b679 100644 --- a/legacy/builder/types/context.go +++ b/legacy/builder/types/context.go @@ -121,8 +121,7 @@ type Context struct { LineOffset int // Verbosity settings - Verbose bool - DebugPreprocessor bool + Verbose bool // Dry run, only create progress map Progress ProgressStruct From 795c9ddc131af3fff39424017769f2ad525fa990 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 6 Apr 2023 00:05:24 +0200 Subject: [PATCH 26/42] Added executils.RunAndCaptureOutput --- executils/process.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/executils/process.go b/executils/process.go index 6044275af85..7e0ac135089 100644 --- a/executils/process.go +++ b/executils/process.go @@ -16,6 +16,7 @@ package executils import ( + "bytes" "context" "io" "os" @@ -174,3 +175,15 @@ func (p *Process) RunWithinContext(ctx context.Context) error { }() return p.Wait() } + +// RunAndCaptureOutput starts the specified command and waits for it to complete. If the given context +// is canceled before the normal process termination, the process is killed. The standard output and +// standard error of the process are captured and returned at process termination. +func (p *Process) RunAndCaptureOutput(ctx context.Context) ([]byte, []byte, error) { + stdout := &bytes.Buffer{} + stderr := &bytes.Buffer{} + p.RedirectStdoutTo(stdout) + p.RedirectStderrTo(stderr) + err := p.RunWithinContext(ctx) + return stdout.Bytes(), stderr.Bytes(), err +} From ed8764b68dc9823b92274c425ea26255d55d9082 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 6 Apr 2023 00:05:59 +0200 Subject: [PATCH 27/42] Moved RunCTags out of legacy package --- .../builder/preprocessor/ctags.go | 27 ++++++++++--------- legacy/builder/container_add_prototypes.go | 3 ++- legacy/builder/test/ctags_runner_test.go | 3 ++- 3 files changed, 18 insertions(+), 15 deletions(-) rename legacy/builder/ctags_runner.go => arduino/builder/preprocessor/ctags.go (81%) diff --git a/legacy/builder/ctags_runner.go b/arduino/builder/preprocessor/ctags.go similarity index 81% rename from legacy/builder/ctags_runner.go rename to arduino/builder/preprocessor/ctags.go index 9463b635f7a..336b8139a68 100644 --- a/legacy/builder/ctags_runner.go +++ b/arduino/builder/preprocessor/ctags.go @@ -1,6 +1,6 @@ // This file is part of arduino-cli. // -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) +// Copyright 2023 ARDUINO SA (http://www.arduino.cc/) // // This software is released under the GNU General Public License version 3, // which covers the main part of arduino-cli. @@ -13,18 +13,22 @@ // Arduino software without disclosing the source code of your own applications. // To purchase a commercial license, send an email to license@arduino.cc. -package builder +package preprocessor import ( - "bytes" + "context" + "fmt" "strings" "github.com/arduino/arduino-cli/executils" + "github.com/arduino/arduino-cli/i18n" "github.com/arduino/go-paths-helper" - properties "github.com/arduino/go-properties-orderedmap" + "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" ) +var tr = i18n.Tr + func RunCTags(sourceFile *paths.Path, buildProperties *properties.Map) ([]byte, []byte, error) { ctagsBuildProperties := properties.NewMap() ctagsBuildProperties.Set("tools.ctags.path", "{runtime.tools.ctags.path}") @@ -48,13 +52,10 @@ func RunCTags(sourceFile *paths.Path, buildProperties *properties.Map) ([]byte, if err != nil { return nil, nil, err } - stdout := &bytes.Buffer{} - stderr := &bytes.Buffer{} - proc.RedirectStdoutTo(stdout) - proc.RedirectStderrTo(stderr) - if err = proc.Run(); err != nil { - return nil, nil, err - } - _, _ = stderr.WriteString(strings.Join(parts, " ")) - return stdout.Bytes(), stderr.Bytes(), err + stdout, stderr, err := proc.RunAndCaptureOutput(context.Background()) + + // Append ctags arguments to stderr + args := fmt.Sprintln(strings.Join(parts, " ")) + stderr = append([]byte(args), stderr...) + return stdout, stderr, err } diff --git a/legacy/builder/container_add_prototypes.go b/legacy/builder/container_add_prototypes.go index e19549fd400..a372abb9250 100644 --- a/legacy/builder/container_add_prototypes.go +++ b/legacy/builder/container_add_prototypes.go @@ -24,6 +24,7 @@ import ( "strings" bldr "github.com/arduino/arduino-cli/arduino/builder" + "github.com/arduino/arduino-cli/arduino/builder/preprocessor" "github.com/arduino/arduino-cli/arduino/sketch" "github.com/arduino/arduino-cli/legacy/builder/types" "github.com/arduino/go-paths-helper" @@ -66,7 +67,7 @@ func PreprocessSketchWithCtags(ctx *types.Context) error { return err } - ctagsStdout, ctagsStderr, err := RunCTags(targetFilePath, ctx.BuildProperties) + ctagsStdout, ctagsStderr, err := preprocessor.RunCTags(targetFilePath, ctx.BuildProperties) if ctx.Verbose { ctx.WriteStderr(ctagsStderr) } diff --git a/legacy/builder/test/ctags_runner_test.go b/legacy/builder/test/ctags_runner_test.go index 036547afae6..d776a93c4bc 100644 --- a/legacy/builder/test/ctags_runner_test.go +++ b/legacy/builder/test/ctags_runner_test.go @@ -20,6 +20,7 @@ import ( "testing" bldr "github.com/arduino/arduino-cli/arduino/builder" + "github.com/arduino/arduino-cli/arduino/builder/preprocessor" "github.com/arduino/arduino-cli/legacy/builder" paths "github.com/arduino/go-paths-helper" "github.com/stretchr/testify/require" @@ -39,7 +40,7 @@ func ctagsRunnerTestTemplate(t *testing.T, sketchLocation *paths.Path) []byte { target := ctx.BuildPath.Join("ctags_target.cpp") NoError(t, target.WriteFile([]byte(source))) - ctagsOutput, _, err := builder.RunCTags(target, ctx.BuildProperties) + ctagsOutput, _, err := preprocessor.RunCTags(target, ctx.BuildProperties) NoError(t, err) return ctagsOutput From 91e7781e3359597861e5b3169d8b2ec474e74345 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 6 Apr 2023 00:14:58 +0200 Subject: [PATCH 28/42] Simplified GCCPreprocRunner function --- legacy/builder/gcc_preproc_runner.go | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/legacy/builder/gcc_preproc_runner.go b/legacy/builder/gcc_preproc_runner.go index 10876f5d08b..3fb2b8393a0 100644 --- a/legacy/builder/gcc_preproc_runner.go +++ b/legacy/builder/gcc_preproc_runner.go @@ -16,7 +16,6 @@ package builder import ( - "os/exec" "strings" f "github.com/arduino/arduino-cli/internal/algorithms" @@ -25,19 +24,9 @@ import ( "github.com/arduino/arduino-cli/legacy/builder/utils" "github.com/arduino/go-paths-helper" properties "github.com/arduino/go-properties-orderedmap" - "github.com/pkg/errors" ) func GCCPreprocRunner(ctx *types.Context, sourceFilePath *paths.Path, targetFilePath *paths.Path, includes paths.PathList) ([]byte, error) { - cmd, err := prepareGCCPreprocRecipeProperties(ctx, sourceFilePath, targetFilePath, includes) - if err != nil { - return nil, err - } - _, stderr, err := utils.ExecCommand(ctx, cmd, utils.ShowIfVerbose /* stdout */, utils.Capture /* stderr */) - return stderr, err -} - -func prepareGCCPreprocRecipeProperties(ctx *types.Context, sourceFilePath *paths.Path, targetFilePath *paths.Path, includes paths.PathList) (*exec.Cmd, error) { buildProperties := properties.NewMap() buildProperties.Set("preproc.macros.flags", "-w -x c++ -E -CC") buildProperties.Merge(ctx.BuildProperties) @@ -61,12 +50,13 @@ func prepareGCCPreprocRecipeProperties(ctx *types.Context, sourceFilePath *paths cmd, err := builder_utils.PrepareCommandForRecipe(buildProperties, "recipe.preproc.macros", true, ctx.PackageManager.GetEnvVarsForSpawnedProcess()) if err != nil { - return nil, errors.WithStack(err) + return nil, err } // Remove -MMD argument if present. Leaving it will make gcc try // to create a /dev/null.d dependency file, which won't work. cmd.Args = f.Filter(cmd.Args, f.NotEquals("-MMD")) - return cmd, nil + _, stderr, err := utils.ExecCommand(ctx, cmd, utils.ShowIfVerbose /* stdout */, utils.Capture /* stderr */) + return stderr, err } From de8a3b6a3d24368d419f2d23836fb375411a6b69 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 6 Apr 2023 01:43:13 +0200 Subject: [PATCH 29/42] Removed dependency on builder ctx in GCCPreprocRunner --- legacy/builder/container_add_prototypes.go | 7 ++- legacy/builder/container_find_includes.go | 15 ++++-- legacy/builder/gcc_preproc_runner.go | 53 ++++++++++++++-------- legacy/builder/preprocess_sketch.go | 8 +++- 4 files changed, 57 insertions(+), 26 deletions(-) diff --git a/legacy/builder/container_add_prototypes.go b/legacy/builder/container_add_prototypes.go index a372abb9250..0346bf87baf 100644 --- a/legacy/builder/container_add_prototypes.go +++ b/legacy/builder/container_add_prototypes.go @@ -40,8 +40,11 @@ func PreprocessSketchWithCtags(ctx *types.Context) error { // Run preprocessor sourceFile := ctx.SketchBuildPath.Join(ctx.Sketch.MainFile.Base() + ".cpp") - stderr, err := GCCPreprocRunner(ctx, sourceFile, targetFilePath, ctx.IncludeFolders) - ctx.WriteStderr(stderr) + gccStdout, gccStderr, err := GCCPreprocRunner(sourceFile, targetFilePath, ctx.IncludeFolders, ctx.BuildProperties) + if ctx.Verbose { + ctx.WriteStdout(gccStdout) + ctx.WriteStderr(gccStderr) + } if err != nil { if !ctx.OnlyUpdateCompilationDatabase { return errors.WithStack(err) diff --git a/legacy/builder/container_find_includes.go b/legacy/builder/container_find_includes.go index 824c5ddb73b..2aa5192d4d3 100644 --- a/legacy/builder/container_find_includes.go +++ b/legacy/builder/container_find_includes.go @@ -369,7 +369,12 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFileQu ctx.Info(tr("Using cached library dependencies for file: %[1]s", sourcePath)) } } else { - preproc_stderr, preproc_err = GCCPreprocRunner(ctx, sourcePath, targetFilePath, includes) + var preproc_stdout []byte + preproc_stdout, preproc_stderr, preproc_err = GCCPreprocRunner(sourcePath, targetFilePath, includes, ctx.BuildProperties) + if ctx.Verbose { + ctx.WriteStdout(preproc_stdout) + ctx.WriteStdout(preproc_stderr) + } // Unwrap error and see if it is an ExitError. _, is_exit_error := errors.Cause(preproc_err).(*exec.ExitError) if preproc_err == nil { @@ -397,11 +402,13 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFileQu library := ResolveLibrary(ctx, include) if library == nil { // Library could not be resolved, show error - // err := runCommand(ctx, &GCCPreprocRunner{SourceFilePath: sourcePath, TargetFileName: paths.New(constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E), Includes: includes}) - // return errors.WithStack(err) if preproc_err == nil || preproc_stderr == nil { // Filename came from cache, so run preprocessor to obtain error to show - preproc_stderr, preproc_err = GCCPreprocRunner(ctx, sourcePath, targetFilePath, includes) + var preproc_stdout []byte + preproc_stdout, preproc_stderr, preproc_err = GCCPreprocRunner(sourcePath, targetFilePath, includes, ctx.BuildProperties) + if ctx.Verbose { + ctx.WriteStdout(preproc_stdout) + } if preproc_err == nil { // If there is a missing #include in the cache, but running // gcc does not reproduce that, there is something wrong. diff --git a/legacy/builder/gcc_preproc_runner.go b/legacy/builder/gcc_preproc_runner.go index 3fb2b8393a0..f75c8c89260 100644 --- a/legacy/builder/gcc_preproc_runner.go +++ b/legacy/builder/gcc_preproc_runner.go @@ -16,47 +16,64 @@ package builder import ( + "context" + "fmt" "strings" + "github.com/arduino/arduino-cli/executils" f "github.com/arduino/arduino-cli/internal/algorithms" - "github.com/arduino/arduino-cli/legacy/builder/builder_utils" - "github.com/arduino/arduino-cli/legacy/builder/types" "github.com/arduino/arduino-cli/legacy/builder/utils" "github.com/arduino/go-paths-helper" properties "github.com/arduino/go-properties-orderedmap" + "github.com/pkg/errors" ) -func GCCPreprocRunner(ctx *types.Context, sourceFilePath *paths.Path, targetFilePath *paths.Path, includes paths.PathList) ([]byte, error) { - buildProperties := properties.NewMap() - buildProperties.Set("preproc.macros.flags", "-w -x c++ -E -CC") - buildProperties.Merge(ctx.BuildProperties) - buildProperties.Set("build.library_discovery_phase", "1") - buildProperties.SetPath("source_file", sourceFilePath) - buildProperties.SetPath("preprocessed_file_path", targetFilePath) +func GCCPreprocRunner(sourceFilePath *paths.Path, targetFilePath *paths.Path, includes paths.PathList, buildProperties *properties.Map) ([]byte, []byte, error) { + gccBuildProperties := properties.NewMap() + gccBuildProperties.Set("preproc.macros.flags", "-w -x c++ -E -CC") + gccBuildProperties.Merge(buildProperties) + gccBuildProperties.Set("build.library_discovery_phase", "1") + gccBuildProperties.SetPath("source_file", sourceFilePath) + gccBuildProperties.SetPath("preprocessed_file_path", targetFilePath) includesStrings := f.Map(includes.AsStrings(), utils.WrapWithHyphenI) - buildProperties.Set("includes", strings.Join(includesStrings, " ")) + gccBuildProperties.Set("includes", strings.Join(includesStrings, " ")) - if buildProperties.Get("recipe.preproc.macros") == "" { + const GCC_PATTERN_PROPERTY = "recipe.preproc.macros" + if gccBuildProperties.Get(GCC_PATTERN_PROPERTY) == "" { // autogenerate preprocess macros recipe from compile recipe - preprocPattern := buildProperties.Get("recipe.cpp.o.pattern") + preprocPattern := gccBuildProperties.Get("recipe.cpp.o.pattern") // add {preproc.macros.flags} to {compiler.cpp.flags} preprocPattern = strings.Replace(preprocPattern, "{compiler.cpp.flags}", "{compiler.cpp.flags} {preproc.macros.flags}", 1) // replace "{object_file}" with "{preprocessed_file_path}" preprocPattern = strings.Replace(preprocPattern, "{object_file}", "{preprocessed_file_path}", 1) - buildProperties.Set("recipe.preproc.macros", preprocPattern) + gccBuildProperties.Set(GCC_PATTERN_PROPERTY, preprocPattern) } - cmd, err := builder_utils.PrepareCommandForRecipe(buildProperties, "recipe.preproc.macros", true, ctx.PackageManager.GetEnvVarsForSpawnedProcess()) + pattern := gccBuildProperties.Get(GCC_PATTERN_PROPERTY) + if pattern == "" { + return nil, nil, errors.Errorf(tr("%s pattern is missing"), GCC_PATTERN_PROPERTY) + } + + commandLine := gccBuildProperties.ExpandPropsInString(pattern) + args, err := properties.SplitQuotedString(commandLine, `"'`, false) if err != nil { - return nil, err + return nil, nil, err } // Remove -MMD argument if present. Leaving it will make gcc try // to create a /dev/null.d dependency file, which won't work. - cmd.Args = f.Filter(cmd.Args, f.NotEquals("-MMD")) + args = f.Filter(args, f.NotEquals("-MMD")) + + proc, err := executils.NewProcess(nil, args...) + if err != nil { + return nil, nil, err + } + stdout, stderr, err := proc.RunAndCaptureOutput(context.Background()) + + // Append gcc arguments to stdout + stdout = append([]byte(fmt.Sprintln(strings.Join(args, " "))), stdout...) - _, stderr, err := utils.ExecCommand(ctx, cmd, utils.ShowIfVerbose /* stdout */, utils.Capture /* stderr */) - return stderr, err + return stdout, stderr, err } diff --git a/legacy/builder/preprocess_sketch.go b/legacy/builder/preprocess_sketch.go index efd6721847d..e2ff104fc76 100644 --- a/legacy/builder/preprocess_sketch.go +++ b/legacy/builder/preprocess_sketch.go @@ -36,11 +36,15 @@ func PreprocessSketchWithArduinoPreprocessor(ctx *types.Context) error { sourceFile := ctx.SketchBuildPath.Join(ctx.Sketch.MainFile.Base() + ".cpp") targetFile := ctx.PreprocPath.Join("sketch_merged.cpp") - stderr, err := GCCPreprocRunner(ctx, sourceFile, targetFile, ctx.IncludeFolders) - ctx.WriteStderr(stderr) + gccStdout, gccStderr, err := GCCPreprocRunner(sourceFile, targetFile, ctx.IncludeFolders, ctx.BuildProperties) + if ctx.Verbose { + ctx.WriteStdout(gccStdout) + ctx.WriteStderr(gccStderr) + } if err != nil { return err } + buildProperties := properties.NewMap() buildProperties.Set("tools.arduino-preprocessor.path", "{runtime.tools.arduino-preprocessor.path}") buildProperties.Set("tools.arduino-preprocessor.cmd.path", "{path}/arduino-preprocessor") From 493b6060460aaa3dbd3a2d7df5c97a13f5f2afa3 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 6 Apr 2023 01:48:35 +0200 Subject: [PATCH 30/42] Moved GCCPreprocRunner into proper place --- .../builder/preprocessor/gcc.go | 8 ++++---- legacy/builder/container_add_prototypes.go | 2 +- legacy/builder/container_find_includes.go | 5 +++-- legacy/builder/preprocess_sketch.go | 3 ++- 4 files changed, 10 insertions(+), 8 deletions(-) rename legacy/builder/gcc_preproc_runner.go => arduino/builder/preprocessor/gcc.go (90%) diff --git a/legacy/builder/gcc_preproc_runner.go b/arduino/builder/preprocessor/gcc.go similarity index 90% rename from legacy/builder/gcc_preproc_runner.go rename to arduino/builder/preprocessor/gcc.go index f75c8c89260..600e677834e 100644 --- a/legacy/builder/gcc_preproc_runner.go +++ b/arduino/builder/preprocessor/gcc.go @@ -1,6 +1,6 @@ // This file is part of arduino-cli. // -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) +// Copyright 2023 ARDUINO SA (http://www.arduino.cc/) // // This software is released under the GNU General Public License version 3, // which covers the main part of arduino-cli. @@ -13,7 +13,7 @@ // Arduino software without disclosing the source code of your own applications. // To purchase a commercial license, send an email to license@arduino.cc. -package builder +package preprocessor import ( "context" @@ -24,11 +24,11 @@ import ( f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/arduino-cli/legacy/builder/utils" "github.com/arduino/go-paths-helper" - properties "github.com/arduino/go-properties-orderedmap" + "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" ) -func GCCPreprocRunner(sourceFilePath *paths.Path, targetFilePath *paths.Path, includes paths.PathList, buildProperties *properties.Map) ([]byte, []byte, error) { +func GCC(sourceFilePath *paths.Path, targetFilePath *paths.Path, includes paths.PathList, buildProperties *properties.Map) ([]byte, []byte, error) { gccBuildProperties := properties.NewMap() gccBuildProperties.Set("preproc.macros.flags", "-w -x c++ -E -CC") gccBuildProperties.Merge(buildProperties) diff --git a/legacy/builder/container_add_prototypes.go b/legacy/builder/container_add_prototypes.go index 0346bf87baf..288bb03e1f4 100644 --- a/legacy/builder/container_add_prototypes.go +++ b/legacy/builder/container_add_prototypes.go @@ -40,7 +40,7 @@ func PreprocessSketchWithCtags(ctx *types.Context) error { // Run preprocessor sourceFile := ctx.SketchBuildPath.Join(ctx.Sketch.MainFile.Base() + ".cpp") - gccStdout, gccStderr, err := GCCPreprocRunner(sourceFile, targetFilePath, ctx.IncludeFolders, ctx.BuildProperties) + gccStdout, gccStderr, err := preprocessor.GCC(sourceFile, targetFilePath, ctx.IncludeFolders, ctx.BuildProperties) if ctx.Verbose { ctx.WriteStdout(gccStdout) ctx.WriteStderr(gccStderr) diff --git a/legacy/builder/container_find_includes.go b/legacy/builder/container_find_includes.go index 2aa5192d4d3..e7b8cbe0994 100644 --- a/legacy/builder/container_find_includes.go +++ b/legacy/builder/container_find_includes.go @@ -98,6 +98,7 @@ import ( "os/exec" "time" + "github.com/arduino/arduino-cli/arduino/builder/preprocessor" "github.com/arduino/arduino-cli/arduino/globals" "github.com/arduino/arduino-cli/arduino/libraries" "github.com/arduino/arduino-cli/legacy/builder/builder_utils" @@ -370,7 +371,7 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFileQu } } else { var preproc_stdout []byte - preproc_stdout, preproc_stderr, preproc_err = GCCPreprocRunner(sourcePath, targetFilePath, includes, ctx.BuildProperties) + preproc_stdout, preproc_stderr, preproc_err = preprocessor.GCC(sourcePath, targetFilePath, includes, ctx.BuildProperties) if ctx.Verbose { ctx.WriteStdout(preproc_stdout) ctx.WriteStdout(preproc_stderr) @@ -405,7 +406,7 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFileQu if preproc_err == nil || preproc_stderr == nil { // Filename came from cache, so run preprocessor to obtain error to show var preproc_stdout []byte - preproc_stdout, preproc_stderr, preproc_err = GCCPreprocRunner(sourcePath, targetFilePath, includes, ctx.BuildProperties) + preproc_stdout, preproc_stderr, preproc_err = preprocessor.GCC(sourcePath, targetFilePath, includes, ctx.BuildProperties) if ctx.Verbose { ctx.WriteStdout(preproc_stdout) } diff --git a/legacy/builder/preprocess_sketch.go b/legacy/builder/preprocess_sketch.go index e2ff104fc76..7f8aaccec38 100644 --- a/legacy/builder/preprocess_sketch.go +++ b/legacy/builder/preprocess_sketch.go @@ -23,6 +23,7 @@ import ( "runtime" bldr "github.com/arduino/arduino-cli/arduino/builder" + "github.com/arduino/arduino-cli/arduino/builder/preprocessor" "github.com/arduino/arduino-cli/legacy/builder/types" "github.com/arduino/arduino-cli/legacy/builder/utils" properties "github.com/arduino/go-properties-orderedmap" @@ -36,7 +37,7 @@ func PreprocessSketchWithArduinoPreprocessor(ctx *types.Context) error { sourceFile := ctx.SketchBuildPath.Join(ctx.Sketch.MainFile.Base() + ".cpp") targetFile := ctx.PreprocPath.Join("sketch_merged.cpp") - gccStdout, gccStderr, err := GCCPreprocRunner(sourceFile, targetFile, ctx.IncludeFolders, ctx.BuildProperties) + gccStdout, gccStderr, err := preprocessor.GCC(sourceFile, targetFile, ctx.IncludeFolders, ctx.BuildProperties) if ctx.Verbose { ctx.WriteStdout(gccStdout) ctx.WriteStderr(gccStderr) From fc92f8d50ca3e17426483b3fa4b9703fc588c932 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 6 Apr 2023 11:23:26 +0200 Subject: [PATCH 31/42] Replaced algorithm with stdlib equivalent --- legacy/builder/ctags/ctags_has_issues.go | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/legacy/builder/ctags/ctags_has_issues.go b/legacy/builder/ctags/ctags_has_issues.go index 003aaa035da..e7bef952bc3 100644 --- a/legacy/builder/ctags/ctags_has_issues.go +++ b/legacy/builder/ctags/ctags_has_issues.go @@ -19,27 +19,20 @@ import ( "bufio" "os" "strings" + + "golang.org/x/exp/slices" ) func (p *CTagsParser) FixCLinkageTagsDeclarations() { linesMap := p.FindCLinkageLines(p.tags) for i := range p.tags { - if sliceContainsInt(linesMap[p.tags[i].Filename], p.tags[i].Line) && + if slices.Contains(linesMap[p.tags[i].Filename], p.tags[i].Line) && !strings.Contains(p.tags[i].PrototypeModifiers, EXTERN) { p.tags[i].PrototypeModifiers = p.tags[i].PrototypeModifiers + " " + EXTERN } } } -func sliceContainsInt(s []int, e int) bool { - for _, a := range s { - if a == e { - return true - } - } - return false -} - func (p *CTagsParser) prototypeAndCodeDontMatch(tag *CTag) bool { if tag.SkipMe { return true From 7fd1a7f35b6e157aab3f54f92a6b8184923cb72d Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 6 Apr 2023 12:01:37 +0200 Subject: [PATCH 32/42] Reduced public API surface of CTags parser --- legacy/builder/ctags/ctags_has_issues.go | 2 +- legacy/builder/ctags/ctags_parser.go | 5 +++-- legacy/builder/ctags/ctags_parser_test.go | 4 +++- legacy/builder/ctags/ctags_to_prototypes.go | 4 ---- legacy/builder/ctags/ctags_to_prototypes_test.go | 3 +-- legacy/builder/prototypes_adder.go | 5 +---- 6 files changed, 9 insertions(+), 14 deletions(-) diff --git a/legacy/builder/ctags/ctags_has_issues.go b/legacy/builder/ctags/ctags_has_issues.go index e7bef952bc3..a61bf6c6c46 100644 --- a/legacy/builder/ctags/ctags_has_issues.go +++ b/legacy/builder/ctags/ctags_has_issues.go @@ -23,7 +23,7 @@ import ( "golang.org/x/exp/slices" ) -func (p *CTagsParser) FixCLinkageTagsDeclarations() { +func (p *CTagsParser) fixCLinkageTagsDeclarations() { linesMap := p.FindCLinkageLines(p.tags) for i := range p.tags { if slices.Contains(linesMap[p.tags[i].Filename], p.tags[i].Line) && diff --git a/legacy/builder/ctags/ctags_parser.go b/legacy/builder/ctags/ctags_parser.go index 61076101be5..5836f7bf337 100644 --- a/legacy/builder/ctags/ctags_parser.go +++ b/legacy/builder/ctags/ctags_parser.go @@ -58,7 +58,7 @@ type CTag struct { PrototypeModifiers string } -func (p *CTagsParser) Parse(ctagsOutput []byte, mainFile *paths.Path) []*CTag { +func (p *CTagsParser) Parse(ctagsOutput []byte, mainFile *paths.Path) ([]*Prototype, int) { rows := strings.Split(string(ctagsOutput), "\n") rows = removeEmpty(rows) @@ -74,8 +74,9 @@ func (p *CTagsParser) Parse(ctagsOutput []byte, mainFile *paths.Path) []*CTag { p.removeDefinedProtypes() p.skipDuplicates() p.skipTagsWhere(p.prototypeAndCodeDontMatch) + p.fixCLinkageTagsDeclarations() - return p.tags + return p.toPrototypes(), p.findLineWhereToInsertPrototypes() } func (p *CTagsParser) addPrototypes() { diff --git a/legacy/builder/ctags/ctags_parser_test.go b/legacy/builder/ctags/ctags_parser_test.go index d3c4abe6b54..97d5c85a4c9 100644 --- a/legacy/builder/ctags/ctags_parser_test.go +++ b/legacy/builder/ctags/ctags_parser_test.go @@ -20,6 +20,7 @@ import ( "path/filepath" "testing" + "github.com/arduino/go-paths-helper" "github.com/stretchr/testify/require" ) @@ -28,7 +29,8 @@ func produceTags(t *testing.T, filename string) []*CTag { require.NoError(t, err) parser := CTagsParser{} - return parser.Parse(bytes, nil) + parser.Parse(bytes, paths.New("sketch.ino")) + return parser.tags } func TestCTagsParserShouldListPrototypes(t *testing.T) { diff --git a/legacy/builder/ctags/ctags_to_prototypes.go b/legacy/builder/ctags/ctags_to_prototypes.go index d5f750d2da7..71cc072a6fa 100644 --- a/legacy/builder/ctags/ctags_to_prototypes.go +++ b/legacy/builder/ctags/ctags_to_prototypes.go @@ -32,10 +32,6 @@ func (proto *Prototype) String() string { return proto.Modifiers + " " + proto.Prototype + " @ " + strconv.Itoa(proto.Line) } -func (p *CTagsParser) GeneratePrototypes() ([]*Prototype, int) { - return p.toPrototypes(), p.findLineWhereToInsertPrototypes() -} - func (p *CTagsParser) findLineWhereToInsertPrototypes() int { firstFunctionLine := p.firstFunctionAtLine() firstFunctionPointerAsArgument := p.firstFunctionPointerUsedAsArgument() diff --git a/legacy/builder/ctags/ctags_to_prototypes_test.go b/legacy/builder/ctags/ctags_to_prototypes_test.go index 2e36c343b53..98455c22202 100644 --- a/legacy/builder/ctags/ctags_to_prototypes_test.go +++ b/legacy/builder/ctags/ctags_to_prototypes_test.go @@ -29,8 +29,7 @@ func producePrototypes(t *testing.T, filename string, mainFile string) ([]*Proto require.NoError(t, err) parser := &CTagsParser{} - parser.Parse(bytes, paths.New(mainFile)) - return parser.GeneratePrototypes() + return parser.Parse(bytes, paths.New(mainFile)) } func TestCTagsToPrototypesShouldListPrototypes(t *testing.T) { diff --git a/legacy/builder/prototypes_adder.go b/legacy/builder/prototypes_adder.go index fba8dd94e4c..e62a8486fa8 100644 --- a/legacy/builder/prototypes_adder.go +++ b/legacy/builder/prototypes_adder.go @@ -30,10 +30,7 @@ var DebugPreprocessor bool func PrototypesAdder(sketch *sketch.Sketch, source string, ctagsStdout []byte, lineOffset int) string { parser := &ctags.CTagsParser{} - parser.Parse(ctagsStdout, sketch.MainFile) - parser.FixCLinkageTagsDeclarations() - - prototypes, firstFunctionLine := parser.GeneratePrototypes() + prototypes, firstFunctionLine := parser.Parse(ctagsStdout, sketch.MainFile) if firstFunctionLine == -1 { firstFunctionLine = 0 } From bea1361746594c4dd7334d7a43aa238539d7a744 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 6 Apr 2023 12:08:06 +0200 Subject: [PATCH 33/42] Moved CTags parser out of legacy --- .../builder/preprocessor}/ctags/ctags_has_issues.go | 0 .../builder/preprocessor}/ctags/ctags_parser.go | 0 .../builder/preprocessor}/ctags/ctags_parser_test.go | 2 +- .../builder/preprocessor}/ctags/ctags_to_prototypes.go | 1 + .../builder/preprocessor}/ctags/ctags_to_prototypes_test.go | 2 +- .../testdata}/TestCTagsParserClassMembersAreFilteredOut.txt | 0 .../ctags/testdata}/TestCTagsParserDefaultArguments.txt | 0 .../ctags/testdata}/TestCTagsParserFunctionPointer.txt | 0 .../ctags/testdata}/TestCTagsParserFunctionPointers.txt | 0 .../testdata}/TestCTagsParserFunctionPointersNoIndirect.txt | 0 .../preprocessor/ctags/testdata}/TestCTagsParserNamespace.txt | 0 ...TestCTagsParserShouldDealFunctionWithDifferentSignatures.txt | 0 .../ctags/testdata}/TestCTagsParserShouldDealWithClasses.txt | 0 .../ctags/testdata}/TestCTagsParserShouldDealWithMacros.txt | 0 .../ctags/testdata}/TestCTagsParserShouldDealWithStructs.txt | 0 .../ctags/testdata}/TestCTagsParserShouldListPrototypes.txt | 0 .../ctags/testdata}/TestCTagsParserShouldListTemplates.txt | 0 .../ctags/testdata}/TestCTagsParserShouldListTemplates2.txt | 0 .../preprocessor/ctags/testdata}/TestCTagsParserStatic.txt | 0 .../ctags/testdata}/TestCTagsParserStructWithFunctions.txt | 0 .../ctags/testdata}/TestCTagsRunnerSketchWithClassFunction.txt | 0 .../ctags/testdata}/TestCTagsRunnerSketchWithMultifile.txt | 0 legacy/builder/prototypes_adder.go | 2 +- 23 files changed, 4 insertions(+), 3 deletions(-) rename {legacy/builder => arduino/builder/preprocessor}/ctags/ctags_has_issues.go (100%) rename {legacy/builder => arduino/builder/preprocessor}/ctags/ctags_parser.go (100%) rename {legacy/builder => arduino/builder/preprocessor}/ctags/ctags_parser_test.go (99%) rename {legacy/builder => arduino/builder/preprocessor}/ctags/ctags_to_prototypes.go (98%) rename {legacy/builder => arduino/builder/preprocessor}/ctags/ctags_to_prototypes_test.go (99%) rename {legacy/builder/ctags/test_data => arduino/builder/preprocessor/ctags/testdata}/TestCTagsParserClassMembersAreFilteredOut.txt (100%) rename {legacy/builder/ctags/test_data => arduino/builder/preprocessor/ctags/testdata}/TestCTagsParserDefaultArguments.txt (100%) rename {legacy/builder/ctags/test_data => arduino/builder/preprocessor/ctags/testdata}/TestCTagsParserFunctionPointer.txt (100%) rename {legacy/builder/ctags/test_data => arduino/builder/preprocessor/ctags/testdata}/TestCTagsParserFunctionPointers.txt (100%) rename {legacy/builder/ctags/test_data => arduino/builder/preprocessor/ctags/testdata}/TestCTagsParserFunctionPointersNoIndirect.txt (100%) rename {legacy/builder/ctags/test_data => arduino/builder/preprocessor/ctags/testdata}/TestCTagsParserNamespace.txt (100%) rename {legacy/builder/ctags/test_data => arduino/builder/preprocessor/ctags/testdata}/TestCTagsParserShouldDealFunctionWithDifferentSignatures.txt (100%) rename {legacy/builder/ctags/test_data => arduino/builder/preprocessor/ctags/testdata}/TestCTagsParserShouldDealWithClasses.txt (100%) rename {legacy/builder/ctags/test_data => arduino/builder/preprocessor/ctags/testdata}/TestCTagsParserShouldDealWithMacros.txt (100%) rename {legacy/builder/ctags/test_data => arduino/builder/preprocessor/ctags/testdata}/TestCTagsParserShouldDealWithStructs.txt (100%) rename {legacy/builder/ctags/test_data => arduino/builder/preprocessor/ctags/testdata}/TestCTagsParserShouldListPrototypes.txt (100%) rename {legacy/builder/ctags/test_data => arduino/builder/preprocessor/ctags/testdata}/TestCTagsParserShouldListTemplates.txt (100%) rename {legacy/builder/ctags/test_data => arduino/builder/preprocessor/ctags/testdata}/TestCTagsParserShouldListTemplates2.txt (100%) rename {legacy/builder/ctags/test_data => arduino/builder/preprocessor/ctags/testdata}/TestCTagsParserStatic.txt (100%) rename {legacy/builder/ctags/test_data => arduino/builder/preprocessor/ctags/testdata}/TestCTagsParserStructWithFunctions.txt (100%) rename {legacy/builder/ctags/test_data => arduino/builder/preprocessor/ctags/testdata}/TestCTagsRunnerSketchWithClassFunction.txt (100%) rename {legacy/builder/ctags/test_data => arduino/builder/preprocessor/ctags/testdata}/TestCTagsRunnerSketchWithMultifile.txt (100%) diff --git a/legacy/builder/ctags/ctags_has_issues.go b/arduino/builder/preprocessor/ctags/ctags_has_issues.go similarity index 100% rename from legacy/builder/ctags/ctags_has_issues.go rename to arduino/builder/preprocessor/ctags/ctags_has_issues.go diff --git a/legacy/builder/ctags/ctags_parser.go b/arduino/builder/preprocessor/ctags/ctags_parser.go similarity index 100% rename from legacy/builder/ctags/ctags_parser.go rename to arduino/builder/preprocessor/ctags/ctags_parser.go diff --git a/legacy/builder/ctags/ctags_parser_test.go b/arduino/builder/preprocessor/ctags/ctags_parser_test.go similarity index 99% rename from legacy/builder/ctags/ctags_parser_test.go rename to arduino/builder/preprocessor/ctags/ctags_parser_test.go index 97d5c85a4c9..dd6ef465440 100644 --- a/legacy/builder/ctags/ctags_parser_test.go +++ b/arduino/builder/preprocessor/ctags/ctags_parser_test.go @@ -25,7 +25,7 @@ import ( ) func produceTags(t *testing.T, filename string) []*CTag { - bytes, err := os.ReadFile(filepath.Join("test_data", filename)) + bytes, err := os.ReadFile(filepath.Join("testdata", filename)) require.NoError(t, err) parser := CTagsParser{} diff --git a/legacy/builder/ctags/ctags_to_prototypes.go b/arduino/builder/preprocessor/ctags/ctags_to_prototypes.go similarity index 98% rename from legacy/builder/ctags/ctags_to_prototypes.go rename to arduino/builder/preprocessor/ctags/ctags_to_prototypes.go index 71cc072a6fa..8f5e5720aa0 100644 --- a/legacy/builder/ctags/ctags_to_prototypes.go +++ b/arduino/builder/preprocessor/ctags/ctags_to_prototypes.go @@ -20,6 +20,7 @@ import ( "strings" ) +// Prototype is a C++ prototype generated from ctags output type Prototype struct { FunctionName string File string diff --git a/legacy/builder/ctags/ctags_to_prototypes_test.go b/arduino/builder/preprocessor/ctags/ctags_to_prototypes_test.go similarity index 99% rename from legacy/builder/ctags/ctags_to_prototypes_test.go rename to arduino/builder/preprocessor/ctags/ctags_to_prototypes_test.go index 98455c22202..74bd8e2756a 100644 --- a/legacy/builder/ctags/ctags_to_prototypes_test.go +++ b/arduino/builder/preprocessor/ctags/ctags_to_prototypes_test.go @@ -25,7 +25,7 @@ import ( ) func producePrototypes(t *testing.T, filename string, mainFile string) ([]*Prototype, int) { - bytes, err := os.ReadFile(filepath.Join("test_data", filename)) + bytes, err := os.ReadFile(filepath.Join("testdata", filename)) require.NoError(t, err) parser := &CTagsParser{} diff --git a/legacy/builder/ctags/test_data/TestCTagsParserClassMembersAreFilteredOut.txt b/arduino/builder/preprocessor/ctags/testdata/TestCTagsParserClassMembersAreFilteredOut.txt similarity index 100% rename from legacy/builder/ctags/test_data/TestCTagsParserClassMembersAreFilteredOut.txt rename to arduino/builder/preprocessor/ctags/testdata/TestCTagsParserClassMembersAreFilteredOut.txt diff --git a/legacy/builder/ctags/test_data/TestCTagsParserDefaultArguments.txt b/arduino/builder/preprocessor/ctags/testdata/TestCTagsParserDefaultArguments.txt similarity index 100% rename from legacy/builder/ctags/test_data/TestCTagsParserDefaultArguments.txt rename to arduino/builder/preprocessor/ctags/testdata/TestCTagsParserDefaultArguments.txt diff --git a/legacy/builder/ctags/test_data/TestCTagsParserFunctionPointer.txt b/arduino/builder/preprocessor/ctags/testdata/TestCTagsParserFunctionPointer.txt similarity index 100% rename from legacy/builder/ctags/test_data/TestCTagsParserFunctionPointer.txt rename to arduino/builder/preprocessor/ctags/testdata/TestCTagsParserFunctionPointer.txt diff --git a/legacy/builder/ctags/test_data/TestCTagsParserFunctionPointers.txt b/arduino/builder/preprocessor/ctags/testdata/TestCTagsParserFunctionPointers.txt similarity index 100% rename from legacy/builder/ctags/test_data/TestCTagsParserFunctionPointers.txt rename to arduino/builder/preprocessor/ctags/testdata/TestCTagsParserFunctionPointers.txt diff --git a/legacy/builder/ctags/test_data/TestCTagsParserFunctionPointersNoIndirect.txt b/arduino/builder/preprocessor/ctags/testdata/TestCTagsParserFunctionPointersNoIndirect.txt similarity index 100% rename from legacy/builder/ctags/test_data/TestCTagsParserFunctionPointersNoIndirect.txt rename to arduino/builder/preprocessor/ctags/testdata/TestCTagsParserFunctionPointersNoIndirect.txt diff --git a/legacy/builder/ctags/test_data/TestCTagsParserNamespace.txt b/arduino/builder/preprocessor/ctags/testdata/TestCTagsParserNamespace.txt similarity index 100% rename from legacy/builder/ctags/test_data/TestCTagsParserNamespace.txt rename to arduino/builder/preprocessor/ctags/testdata/TestCTagsParserNamespace.txt diff --git a/legacy/builder/ctags/test_data/TestCTagsParserShouldDealFunctionWithDifferentSignatures.txt b/arduino/builder/preprocessor/ctags/testdata/TestCTagsParserShouldDealFunctionWithDifferentSignatures.txt similarity index 100% rename from legacy/builder/ctags/test_data/TestCTagsParserShouldDealFunctionWithDifferentSignatures.txt rename to arduino/builder/preprocessor/ctags/testdata/TestCTagsParserShouldDealFunctionWithDifferentSignatures.txt diff --git a/legacy/builder/ctags/test_data/TestCTagsParserShouldDealWithClasses.txt b/arduino/builder/preprocessor/ctags/testdata/TestCTagsParserShouldDealWithClasses.txt similarity index 100% rename from legacy/builder/ctags/test_data/TestCTagsParserShouldDealWithClasses.txt rename to arduino/builder/preprocessor/ctags/testdata/TestCTagsParserShouldDealWithClasses.txt diff --git a/legacy/builder/ctags/test_data/TestCTagsParserShouldDealWithMacros.txt b/arduino/builder/preprocessor/ctags/testdata/TestCTagsParserShouldDealWithMacros.txt similarity index 100% rename from legacy/builder/ctags/test_data/TestCTagsParserShouldDealWithMacros.txt rename to arduino/builder/preprocessor/ctags/testdata/TestCTagsParserShouldDealWithMacros.txt diff --git a/legacy/builder/ctags/test_data/TestCTagsParserShouldDealWithStructs.txt b/arduino/builder/preprocessor/ctags/testdata/TestCTagsParserShouldDealWithStructs.txt similarity index 100% rename from legacy/builder/ctags/test_data/TestCTagsParserShouldDealWithStructs.txt rename to arduino/builder/preprocessor/ctags/testdata/TestCTagsParserShouldDealWithStructs.txt diff --git a/legacy/builder/ctags/test_data/TestCTagsParserShouldListPrototypes.txt b/arduino/builder/preprocessor/ctags/testdata/TestCTagsParserShouldListPrototypes.txt similarity index 100% rename from legacy/builder/ctags/test_data/TestCTagsParserShouldListPrototypes.txt rename to arduino/builder/preprocessor/ctags/testdata/TestCTagsParserShouldListPrototypes.txt diff --git a/legacy/builder/ctags/test_data/TestCTagsParserShouldListTemplates.txt b/arduino/builder/preprocessor/ctags/testdata/TestCTagsParserShouldListTemplates.txt similarity index 100% rename from legacy/builder/ctags/test_data/TestCTagsParserShouldListTemplates.txt rename to arduino/builder/preprocessor/ctags/testdata/TestCTagsParserShouldListTemplates.txt diff --git a/legacy/builder/ctags/test_data/TestCTagsParserShouldListTemplates2.txt b/arduino/builder/preprocessor/ctags/testdata/TestCTagsParserShouldListTemplates2.txt similarity index 100% rename from legacy/builder/ctags/test_data/TestCTagsParserShouldListTemplates2.txt rename to arduino/builder/preprocessor/ctags/testdata/TestCTagsParserShouldListTemplates2.txt diff --git a/legacy/builder/ctags/test_data/TestCTagsParserStatic.txt b/arduino/builder/preprocessor/ctags/testdata/TestCTagsParserStatic.txt similarity index 100% rename from legacy/builder/ctags/test_data/TestCTagsParserStatic.txt rename to arduino/builder/preprocessor/ctags/testdata/TestCTagsParserStatic.txt diff --git a/legacy/builder/ctags/test_data/TestCTagsParserStructWithFunctions.txt b/arduino/builder/preprocessor/ctags/testdata/TestCTagsParserStructWithFunctions.txt similarity index 100% rename from legacy/builder/ctags/test_data/TestCTagsParserStructWithFunctions.txt rename to arduino/builder/preprocessor/ctags/testdata/TestCTagsParserStructWithFunctions.txt diff --git a/legacy/builder/ctags/test_data/TestCTagsRunnerSketchWithClassFunction.txt b/arduino/builder/preprocessor/ctags/testdata/TestCTagsRunnerSketchWithClassFunction.txt similarity index 100% rename from legacy/builder/ctags/test_data/TestCTagsRunnerSketchWithClassFunction.txt rename to arduino/builder/preprocessor/ctags/testdata/TestCTagsRunnerSketchWithClassFunction.txt diff --git a/legacy/builder/ctags/test_data/TestCTagsRunnerSketchWithMultifile.txt b/arduino/builder/preprocessor/ctags/testdata/TestCTagsRunnerSketchWithMultifile.txt similarity index 100% rename from legacy/builder/ctags/test_data/TestCTagsRunnerSketchWithMultifile.txt rename to arduino/builder/preprocessor/ctags/testdata/TestCTagsRunnerSketchWithMultifile.txt diff --git a/legacy/builder/prototypes_adder.go b/legacy/builder/prototypes_adder.go index e62a8486fa8..7489aefa076 100644 --- a/legacy/builder/prototypes_adder.go +++ b/legacy/builder/prototypes_adder.go @@ -20,9 +20,9 @@ import ( "strconv" "strings" + "github.com/arduino/arduino-cli/arduino/builder/preprocessor/ctags" "github.com/arduino/arduino-cli/arduino/sketch" "github.com/arduino/arduino-cli/legacy/builder/constants" - "github.com/arduino/arduino-cli/legacy/builder/ctags" "github.com/arduino/arduino-cli/legacy/builder/utils" ) From 050e27999d4e3e534a745729f542c184e8b191d1 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Tue, 11 Apr 2023 10:45:00 +0200 Subject: [PATCH 34/42] Moved CTags preprocess subroutine in proper place --- arduino/builder/preprocessor/ctags.go | 98 +++++++++++++++++++ legacy/builder/container_add_prototypes.go | 12 ++- legacy/builder/preprocess_sketch.go | 2 +- legacy/builder/prototypes_adder.go | 89 ----------------- .../try_build_of_problematic_sketch_test.go | 3 +- 5 files changed, 108 insertions(+), 96 deletions(-) diff --git a/arduino/builder/preprocessor/ctags.go b/arduino/builder/preprocessor/ctags.go index 336b8139a68..e0d5f2a55f3 100644 --- a/arduino/builder/preprocessor/ctags.go +++ b/arduino/builder/preprocessor/ctags.go @@ -18,10 +18,14 @@ package preprocessor import ( "context" "fmt" + "strconv" "strings" + "github.com/arduino/arduino-cli/arduino/builder/preprocessor/ctags" + "github.com/arduino/arduino-cli/arduino/sketch" "github.com/arduino/arduino-cli/executils" "github.com/arduino/arduino-cli/i18n" + "github.com/arduino/arduino-cli/legacy/builder/utils" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" @@ -29,6 +33,100 @@ import ( var tr = i18n.Tr +// DebugPreprocessor when set to true the CTags preprocessor will output debugging info to stdout +// this is useful for unit-testing to provide more infos +var DebugPreprocessor bool + +func CTags(sourceFile *paths.Path, targetFile *paths.Path, sketch *sketch.Sketch, lineOffset int, buildProperties *properties.Map) ([]byte, error) { + ctagsOutput, ctagsStdErr, err := RunCTags(sourceFile, buildProperties) + if err != nil { + return ctagsStdErr, err + } + + // func PrototypesAdder(sketch *sketch.Sketch, source string, ctagsStdout []byte, lineOffset int) string { + parser := &ctags.CTagsParser{} + prototypes, firstFunctionLine := parser.Parse(ctagsOutput, sketch.MainFile) + if firstFunctionLine == -1 { + firstFunctionLine = 0 + } + + var source string + if sourceData, err := targetFile.ReadFile(); err != nil { + return nil, err + } else { + source = string(sourceData) + } + source = strings.Replace(source, "\r\n", "\n", -1) + source = strings.Replace(source, "\r", "\n", -1) + sourceRows := strings.Split(source, "\n") + if isFirstFunctionOutsideOfSource(firstFunctionLine, sourceRows) { + return nil, nil + } + + insertionLine := firstFunctionLine + lineOffset - 1 + firstFunctionChar := len(strings.Join(sourceRows[:insertionLine], "\n")) + 1 + prototypeSection := composePrototypeSection(firstFunctionLine, prototypes) + preprocessedSource := source[:firstFunctionChar] + prototypeSection + source[firstFunctionChar:] + + if DebugPreprocessor { + fmt.Println("#PREPROCESSED SOURCE") + prototypesRows := strings.Split(prototypeSection, "\n") + prototypesRows = prototypesRows[:len(prototypesRows)-1] + for i := 0; i < len(sourceRows)+len(prototypesRows); i++ { + if i < insertionLine { + fmt.Printf(" |%s\n", sourceRows[i]) + } else if i < insertionLine+len(prototypesRows) { + fmt.Printf("PRO|%s\n", prototypesRows[i-insertionLine]) + } else { + fmt.Printf(" |%s\n", sourceRows[i-len(prototypesRows)]) + } + } + fmt.Println("#END OF PREPROCESSED SOURCE") + } + + err = targetFile.WriteFile([]byte(preprocessedSource)) + return ctagsStdErr, err +} + +func composePrototypeSection(line int, prototypes []*ctags.Prototype) string { + if len(prototypes) == 0 { + return "" + } + + str := joinPrototypes(prototypes) + str += "\n#line " + str += strconv.Itoa(line) + str += " " + utils.QuoteCppString(prototypes[0].File) + str += "\n" + + return str +} + +func joinPrototypes(prototypes []*ctags.Prototype) string { + prototypesSlice := []string{} + for _, proto := range prototypes { + if signatureContainsaDefaultArg(proto) { + continue + } + prototypesSlice = append(prototypesSlice, "#line "+strconv.Itoa(proto.Line)+" "+utils.QuoteCppString(proto.File)) + prototypeParts := []string{} + if proto.Modifiers != "" { + prototypeParts = append(prototypeParts, proto.Modifiers) + } + prototypeParts = append(prototypeParts, proto.Prototype) + prototypesSlice = append(prototypesSlice, strings.Join(prototypeParts, " ")) + } + return strings.Join(prototypesSlice, "\n") +} + +func signatureContainsaDefaultArg(proto *ctags.Prototype) bool { + return strings.Contains(proto.Prototype, "=") +} + +func isFirstFunctionOutsideOfSource(firstFunctionLine int, sourceRows []string) bool { + return firstFunctionLine > len(sourceRows)-1 +} + func RunCTags(sourceFile *paths.Path, buildProperties *properties.Map) ([]byte, []byte, error) { ctagsBuildProperties := properties.NewMap() ctagsBuildProperties.Set("tools.ctags.path", "{runtime.tools.ctags.path}") diff --git a/legacy/builder/container_add_prototypes.go b/legacy/builder/container_add_prototypes.go index 288bb03e1f4..1ffaf04846d 100644 --- a/legacy/builder/container_add_prototypes.go +++ b/legacy/builder/container_add_prototypes.go @@ -70,19 +70,21 @@ func PreprocessSketchWithCtags(ctx *types.Context) error { return err } - ctagsStdout, ctagsStderr, err := preprocessor.RunCTags(targetFilePath, ctx.BuildProperties) + sketchCpp := ctx.SketchBuildPath.Join(fmt.Sprintf("%s.cpp", ctx.Sketch.MainFile.Base())) + ctagsStderr, err := preprocessor.CTags(targetFilePath, sketchCpp, ctx.Sketch, ctx.LineOffset, ctx.BuildProperties) if ctx.Verbose { ctx.WriteStderr(ctagsStderr) } if err != nil { return err } - ctx.SketchSourceAfterArduinoPreprocessing = PrototypesAdder(ctx.Sketch, ctx.SketchSourceMerged, ctagsStdout, ctx.LineOffset) - if err := bldr.SketchSaveItemCpp(ctx.Sketch.MainFile, []byte(ctx.SketchSourceAfterArduinoPreprocessing), ctx.SketchBuildPath); err != nil { - return errors.WithStack(err) + // Save preprocesssed source in context + if d, err := sketchCpp.ReadFile(); err != nil { + return err + } else { + ctx.SketchSourceAfterArduinoPreprocessing = string(d) } - return nil } diff --git a/legacy/builder/preprocess_sketch.go b/legacy/builder/preprocess_sketch.go index 7f8aaccec38..da6f5c9886a 100644 --- a/legacy/builder/preprocess_sketch.go +++ b/legacy/builder/preprocess_sketch.go @@ -88,5 +88,5 @@ func PreprocessSketchWithArduinoPreprocessor(ctx *types.Context) error { //fmt.Printf("PREPROCESSOR OUTPUT:\n%s\n", output) ctx.SketchSourceAfterArduinoPreprocessing = string(result) - return bldr.SketchSaveItemCpp(ctx.Sketch.MainFile, []byte(ctx.SketchSourceAfterArduinoPreprocessing), ctx.SketchBuildPath) + return bldr.SketchSaveItemCpp(ctx.Sketch.MainFile, result, ctx.SketchBuildPath) } diff --git a/legacy/builder/prototypes_adder.go b/legacy/builder/prototypes_adder.go index 7489aefa076..261bc7c759a 100644 --- a/legacy/builder/prototypes_adder.go +++ b/legacy/builder/prototypes_adder.go @@ -14,92 +14,3 @@ // To purchase a commercial license, send an email to license@arduino.cc. package builder - -import ( - "fmt" - "strconv" - "strings" - - "github.com/arduino/arduino-cli/arduino/builder/preprocessor/ctags" - "github.com/arduino/arduino-cli/arduino/sketch" - "github.com/arduino/arduino-cli/legacy/builder/constants" - "github.com/arduino/arduino-cli/legacy/builder/utils" -) - -var DebugPreprocessor bool - -func PrototypesAdder(sketch *sketch.Sketch, source string, ctagsStdout []byte, lineOffset int) string { - parser := &ctags.CTagsParser{} - prototypes, firstFunctionLine := parser.Parse(ctagsStdout, sketch.MainFile) - if firstFunctionLine == -1 { - firstFunctionLine = 0 - } - - source = strings.Replace(source, "\r\n", "\n", -1) - source = strings.Replace(source, "\r", "\n", -1) - sourceRows := strings.Split(source, "\n") - if isFirstFunctionOutsideOfSource(firstFunctionLine, sourceRows) { - return "" - } - - insertionLine := firstFunctionLine + lineOffset - 1 - firstFunctionChar := len(strings.Join(sourceRows[:insertionLine], "\n")) + 1 - prototypeSection := composePrototypeSection(firstFunctionLine, prototypes) - preprocessedSource := source[:firstFunctionChar] + prototypeSection + source[firstFunctionChar:] - - if DebugPreprocessor { - fmt.Println("#PREPROCESSED SOURCE") - prototypesRows := strings.Split(prototypeSection, "\n") - prototypesRows = prototypesRows[:len(prototypesRows)-1] - for i := 0; i < len(sourceRows)+len(prototypesRows); i++ { - if i < insertionLine { - fmt.Printf(" |%s\n", sourceRows[i]) - } else if i < insertionLine+len(prototypesRows) { - fmt.Printf("PRO|%s\n", prototypesRows[i-insertionLine]) - } else { - fmt.Printf(" |%s\n", sourceRows[i-len(prototypesRows)]) - } - } - fmt.Println("#END OF PREPROCESSED SOURCE") - } - return preprocessedSource -} - -func composePrototypeSection(line int, prototypes []*ctags.Prototype) string { - if len(prototypes) == 0 { - return constants.EMPTY_STRING - } - - str := joinPrototypes(prototypes) - str += "\n#line " - str += strconv.Itoa(line) - str += " " + utils.QuoteCppString(prototypes[0].File) - str += "\n" - - return str -} - -func joinPrototypes(prototypes []*ctags.Prototype) string { - prototypesSlice := []string{} - for _, proto := range prototypes { - if signatureContainsaDefaultArg(proto) { - continue - } - prototypesSlice = append(prototypesSlice, "#line "+strconv.Itoa(proto.Line)+" "+utils.QuoteCppString(proto.File)) - prototypeParts := []string{} - if proto.Modifiers != "" { - prototypeParts = append(prototypeParts, proto.Modifiers) - } - prototypeParts = append(prototypeParts, proto.Prototype) - prototypesSlice = append(prototypesSlice, strings.Join(prototypeParts, " ")) - } - return strings.Join(prototypesSlice, "\n") -} - -func signatureContainsaDefaultArg(proto *ctags.Prototype) bool { - return strings.Contains(proto.Prototype, "=") -} - -func isFirstFunctionOutsideOfSource(firstFunctionLine int, sourceRows []string) bool { - return firstFunctionLine > len(sourceRows)-1 -} diff --git a/legacy/builder/test/try_build_of_problematic_sketch_test.go b/legacy/builder/test/try_build_of_problematic_sketch_test.go index bb6457728ae..90f3633f2b6 100644 --- a/legacy/builder/test/try_build_of_problematic_sketch_test.go +++ b/legacy/builder/test/try_build_of_problematic_sketch_test.go @@ -20,6 +20,7 @@ import ( "path/filepath" "testing" + "github.com/arduino/arduino-cli/arduino/builder/preprocessor" "github.com/arduino/arduino-cli/legacy/builder" "github.com/arduino/arduino-cli/legacy/builder/types" paths "github.com/arduino/go-paths-helper" @@ -202,7 +203,7 @@ func TestTryBuild042(t *testing.T) { } func makeDefaultContext() *types.Context { - builder.DebugPreprocessor = true + preprocessor.DebugPreprocessor = true return &types.Context{ HardwareDirs: paths.NewPathList(filepath.Join("..", "hardware"), "downloaded_hardware", "downloaded_board_manager_stuff"), BuiltInToolsDirs: paths.NewPathList("downloaded_tools"), From 9794f6d153590750bd829883cfc0b8ae422bb9cc Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Tue, 11 Apr 2023 10:58:00 +0200 Subject: [PATCH 35/42] Factored all c++ source lines parsers --- arduino/builder/{ => cpp}/cpp.go | 44 +++++++++-- arduino/builder/cpp/cpp_test.go | 74 +++++++++++++++++++ arduino/builder/cpp_test.go | 46 ------------ arduino/builder/preprocessor/ctags.go | 6 +- arduino/builder/sketch.go | 7 +- .../integrationtest/compile_1/compile_test.go | 4 +- legacy/builder/container_add_prototypes.go | 4 +- legacy/builder/test/helper.go | 4 +- legacy/builder/test/prototypes_adder_test.go | 28 +++---- legacy/builder/test/utils_test.go | 13 ---- legacy/builder/utils/utils.go | 13 ---- 11 files changed, 140 insertions(+), 103 deletions(-) rename arduino/builder/{ => cpp}/cpp.go (62%) create mode 100644 arduino/builder/cpp/cpp_test.go delete mode 100644 arduino/builder/cpp_test.go diff --git a/arduino/builder/cpp.go b/arduino/builder/cpp/cpp.go similarity index 62% rename from arduino/builder/cpp.go rename to arduino/builder/cpp/cpp.go index 1054388f43f..71c2b696702 100644 --- a/arduino/builder/cpp.go +++ b/arduino/builder/cpp/cpp.go @@ -13,29 +13,63 @@ // Arduino software without disclosing the source code of your own applications. // To purchase a commercial license, send an email to license@arduino.cc. -package builder +package cpp import ( + "strconv" "strings" "unicode/utf8" + + "github.com/arduino/go-paths-helper" ) -// QuoteCppString returns the given string as a quoted string for use with the C +// QuoteString returns the given string as a quoted string for use with the C // preprocessor. This adds double quotes around it and escapes any // double quotes and backslashes in the string. -func QuoteCppString(str string) string { +func QuoteString(str string) string { str = strings.Replace(str, "\\", "\\\\", -1) str = strings.Replace(str, "\"", "\\\"", -1) return "\"" + str + "\"" } -// Parse a C-preprocessor string as emitted by the preprocessor. This +// ParseLineMarker parses the given line as a gcc line marker and returns the contained +// filename. +func ParseLineMarker(line string) *paths.Path { + // A line marker contains the line number and filename and looks like: + // # 123 /path/to/file.cpp + // It can be followed by zero or more flag number that indicate the + // preprocessor state and can be ignored. + // For exact details on this format, see: + // https://github.com/gcc-mirror/gcc/blob/edd716b6b1caa1a5cb320a8cd7f626f30198e098/gcc/c-family/c-ppoutput.c#L413-L415 + + split := strings.SplitN(line, " ", 3) + if len(split) < 3 || len(split[0]) == 0 || split[0][0] != '#' { + return nil + } + + _, err := strconv.Atoi(split[1]) + if err != nil { + return nil + } + + // If we get here, we found a # followed by a line number, so + // assume this is a line marker and see if the rest of the line + // starts with a string containing the filename + str, rest, ok := ParseString(split[2]) + + if ok && (rest == "" || rest[0] == ' ') { + return paths.New(str) + } + return nil +} + +// ParseString parse a string as emitted by the preprocessor. This // is a string contained in double quotes, with any backslashes or // quotes escaped with a backslash. If a valid string was present at the // start of the given line, returns the unquoted string contents, the // remainder of the line (everything after the closing "), and true. // Otherwise, returns the empty string, the entire line and false. -func ParseCppString(line string) (string, string, bool) { +func ParseString(line string) (string, string, bool) { // For details about how these strings are output by gcc, see: // https://github.com/gcc-mirror/gcc/blob/a588355ab948cf551bc9d2b89f18e5ae5140f52c/libcpp/macro.c#L491-L511 // Note that the documentation suggests all non-printable diff --git a/arduino/builder/cpp/cpp_test.go b/arduino/builder/cpp/cpp_test.go new file mode 100644 index 00000000000..b6ae543f558 --- /dev/null +++ b/arduino/builder/cpp/cpp_test.go @@ -0,0 +1,74 @@ +// This file is part of arduino-cli. +// +// Copyright 2023 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +package cpp_test + +import ( + "testing" + + "github.com/arduino/arduino-cli/arduino/builder/cpp" + "github.com/stretchr/testify/require" +) + +func TestParseString(t *testing.T) { + _, _, ok := cpp.ParseString(`foo`) + require.Equal(t, false, ok) + + _, _, ok = cpp.ParseString(`"foo`) + require.Equal(t, false, ok) + + str, rest, ok := cpp.ParseString(`"foo"`) + require.Equal(t, true, ok) + require.Equal(t, `foo`, str) + require.Equal(t, ``, rest) + + str, rest, ok = cpp.ParseString(`"foo\\bar"`) + require.Equal(t, true, ok) + require.Equal(t, `foo\bar`, str) + require.Equal(t, ``, rest) + + str, rest, ok = cpp.ParseString(`"foo \"is\" quoted and \\\\bar\"\" escaped\\" and "then" some`) + require.Equal(t, true, ok) + require.Equal(t, `foo "is" quoted and \\bar"" escaped\`, str) + require.Equal(t, ` and "then" some`, rest) + + str, rest, ok = cpp.ParseString(`" !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_abcdefghijklmnopqrstuvwxyz{|}~"`) + require.Equal(t, true, ok) + require.Equal(t, ` !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_abcdefghijklmnopqrstuvwxyz{|}~`, str) + require.Equal(t, ``, rest) + + str, rest, ok = cpp.ParseString(`"/home/ççç/"`) + require.Equal(t, true, ok) + require.Equal(t, `/home/ççç/`, str) + require.Equal(t, ``, rest) + + str, rest, ok = cpp.ParseString(`"/home/ççç/ /$sdsdd\\"`) + require.Equal(t, true, ok) + require.Equal(t, `/home/ççç/ /$sdsdd\`, str) + require.Equal(t, ``, rest) +} + +func TestQuoteString(t *testing.T) { + cases := map[string]string{ + `foo`: `"foo"`, + `foo\bar`: `"foo\\bar"`, + `foo "is" quoted and \\bar"" escaped\`: `"foo \"is\" quoted and \\\\bar\"\" escaped\\"`, + // ASCII 0x20 - 0x7e, excluding ` + ` !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_abcdefghijklmnopqrstuvwxyz{|}~`: `" !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_abcdefghijklmnopqrstuvwxyz{|}~"`, + } + for input, expected := range cases { + require.Equal(t, expected, cpp.QuoteString(input)) + } +} diff --git a/arduino/builder/cpp_test.go b/arduino/builder/cpp_test.go deleted file mode 100644 index 84de5f0b254..00000000000 --- a/arduino/builder/cpp_test.go +++ /dev/null @@ -1,46 +0,0 @@ -package builder_test - -import ( - "testing" - - "github.com/arduino/arduino-cli/arduino/builder" - "github.com/stretchr/testify/require" -) - -func TestParseCppString(t *testing.T) { - _, _, ok := builder.ParseCppString(`foo`) - require.Equal(t, false, ok) - - _, _, ok = builder.ParseCppString(`"foo`) - require.Equal(t, false, ok) - - str, rest, ok := builder.ParseCppString(`"foo"`) - require.Equal(t, true, ok) - require.Equal(t, `foo`, str) - require.Equal(t, ``, rest) - - str, rest, ok = builder.ParseCppString(`"foo\\bar"`) - require.Equal(t, true, ok) - require.Equal(t, `foo\bar`, str) - require.Equal(t, ``, rest) - - str, rest, ok = builder.ParseCppString(`"foo \"is\" quoted and \\\\bar\"\" escaped\\" and "then" some`) - require.Equal(t, true, ok) - require.Equal(t, `foo "is" quoted and \\bar"" escaped\`, str) - require.Equal(t, ` and "then" some`, rest) - - str, rest, ok = builder.ParseCppString(`" !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_abcdefghijklmnopqrstuvwxyz{|}~"`) - require.Equal(t, true, ok) - require.Equal(t, ` !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_abcdefghijklmnopqrstuvwxyz{|}~`, str) - require.Equal(t, ``, rest) - - str, rest, ok = builder.ParseCppString(`"/home/ççç/"`) - require.Equal(t, true, ok) - require.Equal(t, `/home/ççç/`, str) - require.Equal(t, ``, rest) - - str, rest, ok = builder.ParseCppString(`"/home/ççç/ /$sdsdd\\"`) - require.Equal(t, true, ok) - require.Equal(t, `/home/ççç/ /$sdsdd\`, str) - require.Equal(t, ``, rest) -} diff --git a/arduino/builder/preprocessor/ctags.go b/arduino/builder/preprocessor/ctags.go index e0d5f2a55f3..5cc9e5876c4 100644 --- a/arduino/builder/preprocessor/ctags.go +++ b/arduino/builder/preprocessor/ctags.go @@ -21,11 +21,11 @@ import ( "strconv" "strings" + "github.com/arduino/arduino-cli/arduino/builder/cpp" "github.com/arduino/arduino-cli/arduino/builder/preprocessor/ctags" "github.com/arduino/arduino-cli/arduino/sketch" "github.com/arduino/arduino-cli/executils" "github.com/arduino/arduino-cli/i18n" - "github.com/arduino/arduino-cli/legacy/builder/utils" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" @@ -96,7 +96,7 @@ func composePrototypeSection(line int, prototypes []*ctags.Prototype) string { str := joinPrototypes(prototypes) str += "\n#line " str += strconv.Itoa(line) - str += " " + utils.QuoteCppString(prototypes[0].File) + str += " " + cpp.QuoteString(prototypes[0].File) str += "\n" return str @@ -108,7 +108,7 @@ func joinPrototypes(prototypes []*ctags.Prototype) string { if signatureContainsaDefaultArg(proto) { continue } - prototypesSlice = append(prototypesSlice, "#line "+strconv.Itoa(proto.Line)+" "+utils.QuoteCppString(proto.File)) + prototypesSlice = append(prototypesSlice, "#line "+strconv.Itoa(proto.Line)+" "+cpp.QuoteString(proto.File)) prototypeParts := []string{} if proto.Modifiers != "" { prototypeParts = append(prototypeParts, proto.Modifiers) diff --git a/arduino/builder/sketch.go b/arduino/builder/sketch.go index 8560178a193..9ca21b6139e 100644 --- a/arduino/builder/sketch.go +++ b/arduino/builder/sketch.go @@ -20,6 +20,7 @@ import ( "fmt" "regexp" + "github.com/arduino/arduino-cli/arduino/builder/cpp" "github.com/arduino/arduino-cli/arduino/sketch" "github.com/arduino/arduino-cli/i18n" "github.com/arduino/go-paths-helper" @@ -96,7 +97,7 @@ func sketchMergeSources(sk *sketch.Sketch, overrides map[string]string) (int, st lineOffset++ } - mergedSource += "#line 1 " + QuoteCppString(sk.MainFile.String()) + "\n" + mergedSource += "#line 1 " + cpp.QuoteString(sk.MainFile.String()) + "\n" mergedSource += mainSrc + "\n" lineOffset++ @@ -105,7 +106,7 @@ func sketchMergeSources(sk *sketch.Sketch, overrides map[string]string) (int, st if err != nil { return 0, "", err } - mergedSource += "#line 1 " + QuoteCppString(file.String()) + "\n" + mergedSource += "#line 1 " + cpp.QuoteString(file.String()) + "\n" mergedSource += src + "\n" } @@ -145,7 +146,7 @@ func sketchCopyAdditionalFiles(sketch *sketch.Sketch, destPath *paths.Path, over } // tag each addtional file with the filename of the source it was copied from - sourceBytes = append([]byte("#line 1 "+QuoteCppString(file.String())+"\n"), sourceBytes...) + sourceBytes = append([]byte("#line 1 "+cpp.QuoteString(file.String())+"\n"), sourceBytes...) err = writeIfDifferent(sourceBytes, targetPath) if err != nil { diff --git a/internal/integrationtest/compile_1/compile_test.go b/internal/integrationtest/compile_1/compile_test.go index dcfb8736f19..58de62094fe 100644 --- a/internal/integrationtest/compile_1/compile_test.go +++ b/internal/integrationtest/compile_1/compile_test.go @@ -25,7 +25,7 @@ import ( "testing" "time" - "github.com/arduino/arduino-cli/arduino/builder" + "github.com/arduino/arduino-cli/arduino/builder/cpp" "github.com/arduino/arduino-cli/internal/integrationtest" "github.com/arduino/go-paths-helper" "github.com/stretchr/testify/require" @@ -1191,7 +1191,7 @@ void loop() { } ` - expected = strings.ReplaceAll(expected, "%SKETCH_PATH%", builder.QuoteCppString(sketchPath.Join("SketchSimple.ino").String())) + expected = strings.ReplaceAll(expected, "%SKETCH_PATH%", cpp.QuoteString(sketchPath.Join("SketchSimple.ino").String())) jsonOut, _, err := cli.Run("compile", "-b", fqbn, "--preprocess", sketchPath.String(), "--format", "json") require.NoError(t, err) diff --git a/legacy/builder/container_add_prototypes.go b/legacy/builder/container_add_prototypes.go index 1ffaf04846d..36a1b844368 100644 --- a/legacy/builder/container_add_prototypes.go +++ b/legacy/builder/container_add_prototypes.go @@ -23,7 +23,7 @@ import ( "strconv" "strings" - bldr "github.com/arduino/arduino-cli/arduino/builder" + "github.com/arduino/arduino-cli/arduino/builder/cpp" "github.com/arduino/arduino-cli/arduino/builder/preprocessor" "github.com/arduino/arduino-cli/arduino/sketch" "github.com/arduino/arduino-cli/legacy/builder/types" @@ -137,7 +137,7 @@ func parseLineMarker(line string) *paths.Path { // If we get here, we found a # followed by a line number, so // assume this is a line marker and see if the rest of the line // starts with a string containing the filename - str, rest, ok := bldr.ParseCppString(split[2]) + str, rest, ok := cpp.ParseString(split[2]) if ok && (rest == "" || rest[0] == ' ') { return paths.New(str) diff --git a/legacy/builder/test/helper.go b/legacy/builder/test/helper.go index 4581dd7c921..88432e5fc76 100644 --- a/legacy/builder/test/helper.go +++ b/legacy/builder/test/helper.go @@ -23,11 +23,11 @@ import ( "testing" "text/template" + "github.com/arduino/arduino-cli/arduino/builder/cpp" "github.com/arduino/arduino-cli/arduino/cores" "github.com/arduino/arduino-cli/arduino/libraries" "github.com/arduino/arduino-cli/legacy/builder/constants" "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/arduino/arduino-cli/legacy/builder/utils" paths "github.com/arduino/go-paths-helper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -35,7 +35,7 @@ import ( func LoadAndInterpolate(t *testing.T, filename string, ctx *types.Context) string { funcsMap := template.FuncMap{ - "QuoteCppString": utils.QuoteCppPath, + "QuoteCppString": func(p *paths.Path) string { return cpp.QuoteString(p.String()) }, } tpl, err := template.New(filepath.Base(filename)).Funcs(funcsMap).ParseFiles(filename) diff --git a/legacy/builder/test/prototypes_adder_test.go b/legacy/builder/test/prototypes_adder_test.go index 64630a7ba70..f175322fe64 100644 --- a/legacy/builder/test/prototypes_adder_test.go +++ b/legacy/builder/test/prototypes_adder_test.go @@ -22,16 +22,16 @@ import ( "testing" bldr "github.com/arduino/arduino-cli/arduino/builder" + "github.com/arduino/arduino-cli/arduino/builder/cpp" "github.com/arduino/arduino-cli/legacy/builder" "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/arduino/arduino-cli/legacy/builder/utils" paths "github.com/arduino/go-paths-helper" "github.com/stretchr/testify/require" ) func TestPrototypesAdderBridgeExample(t *testing.T) { sketchLocation := paths.New("downloaded_libraries", "Bridge", "examples", "Bridge", "Bridge.ino") - quotedSketchLocation := utils.QuoteCppPath(Abs(t, sketchLocation)) + quotedSketchLocation := cpp.QuoteString(Abs(t, sketchLocation).String()) ctx := prepareBuilderTestContext(t, nil, sketchLocation, "arduino:avr:leonardo") defer cleanUpBuilderTestContext(t, ctx) @@ -239,7 +239,7 @@ func TestPrototypesAdderSketchWithStruct(t *testing.T) { func TestPrototypesAdderSketchWithConfig(t *testing.T) { sketchLocation := paths.New("sketch_with_config", "sketch_with_config.ino") - quotedSketchLocation := utils.QuoteCppPath(Abs(t, sketchLocation)) + quotedSketchLocation := cpp.QuoteString(Abs(t, sketchLocation).String()) ctx := prepareBuilderTestContext(t, nil, sketchLocation, "arduino:avr:leonardo") defer cleanUpBuilderTestContext(t, ctx) @@ -270,7 +270,7 @@ func TestPrototypesAdderSketchWithConfig(t *testing.T) { func TestPrototypesAdderSketchNoFunctionsTwoFiles(t *testing.T) { sketchLocation := paths.New("sketch_no_functions_two_files", "sketch_no_functions_two_files.ino") - quotedSketchLocation := utils.QuoteCppPath(Abs(t, sketchLocation)) + quotedSketchLocation := cpp.QuoteString(Abs(t, sketchLocation).String()) ctx := prepareBuilderTestContext(t, nil, paths.New("sketch_no_functions_two_files", "sketch_no_functions_two_files.ino"), "arduino:avr:leonardo") defer cleanUpBuilderTestContext(t, ctx) @@ -303,7 +303,7 @@ func TestPrototypesAdderSketchNoFunctions(t *testing.T) { ctx.Verbose = true sketchLocation := paths.New("sketch_no_functions", "sketch_no_functions.ino") - quotedSketchLocation := utils.QuoteCppPath(Abs(t, sketchLocation)) + quotedSketchLocation := cpp.QuoteString(Abs(t, sketchLocation).String()) var _err error commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, @@ -325,7 +325,7 @@ func TestPrototypesAdderSketchNoFunctions(t *testing.T) { func TestPrototypesAdderSketchWithDefaultArgs(t *testing.T) { sketchLocation := paths.New("sketch_with_default_args", "sketch_with_default_args.ino") - quotedSketchLocation := utils.QuoteCppPath(Abs(t, sketchLocation)) + quotedSketchLocation := cpp.QuoteString(Abs(t, sketchLocation).String()) ctx := prepareBuilderTestContext(t, nil, sketchLocation, "arduino:avr:leonardo") defer cleanUpBuilderTestContext(t, ctx) @@ -353,7 +353,7 @@ func TestPrototypesAdderSketchWithDefaultArgs(t *testing.T) { func TestPrototypesAdderSketchWithInlineFunction(t *testing.T) { sketchLocation := paths.New("sketch_with_inline_function", "sketch_with_inline_function.ino") - quotedSketchLocation := utils.QuoteCppPath(Abs(t, sketchLocation)) + quotedSketchLocation := cpp.QuoteString(Abs(t, sketchLocation).String()) ctx := prepareBuilderTestContext(t, nil, sketchLocation, "arduino:avr:leonardo") defer cleanUpBuilderTestContext(t, ctx) @@ -392,7 +392,7 @@ func TestPrototypesAdderSketchWithInlineFunction(t *testing.T) { func TestPrototypesAdderSketchWithFunctionSignatureInsideIFDEF(t *testing.T) { sketchLocation := paths.New("sketch_with_function_signature_inside_ifdef", "sketch_with_function_signature_inside_ifdef.ino") - quotedSketchLocation := utils.QuoteCppPath(Abs(t, sketchLocation)) + quotedSketchLocation := cpp.QuoteString(Abs(t, sketchLocation).String()) ctx := prepareBuilderTestContext(t, nil, sketchLocation, "arduino:avr:leonardo") defer cleanUpBuilderTestContext(t, ctx) @@ -420,7 +420,7 @@ func TestPrototypesAdderSketchWithFunctionSignatureInsideIFDEF(t *testing.T) { func TestPrototypesAdderSketchWithUSBCON(t *testing.T) { sketchLocation := paths.New("sketch_with_usbcon", "sketch_with_usbcon.ino") - quotedSketchLocation := utils.QuoteCppPath(Abs(t, sketchLocation)) + quotedSketchLocation := cpp.QuoteString(Abs(t, sketchLocation).String()) ctx := &types.Context{ HardwareDirs: paths.NewPathList(filepath.Join("..", "hardware"), "downloaded_hardware"), @@ -453,7 +453,7 @@ func TestPrototypesAdderSketchWithUSBCON(t *testing.T) { func TestPrototypesAdderSketchWithTypename(t *testing.T) { sketchLocation := paths.New("sketch_with_typename", "sketch_with_typename.ino") - quotedSketchLocation := utils.QuoteCppPath(Abs(t, sketchLocation)) + quotedSketchLocation := cpp.QuoteString(Abs(t, sketchLocation).String()) ctx := &types.Context{ HardwareDirs: paths.NewPathList(filepath.Join("..", "hardware"), "downloaded_hardware"), @@ -491,7 +491,7 @@ func TestPrototypesAdderSketchWithTypename(t *testing.T) { func TestPrototypesAdderSketchWithIfDef2(t *testing.T) { sketchLocation := paths.New("sketch_with_ifdef", "sketch_with_ifdef.ino") - quotedSketchLocation := utils.QuoteCppPath(Abs(t, sketchLocation)) + quotedSketchLocation := cpp.QuoteString(Abs(t, sketchLocation).String()) ctx := prepareBuilderTestContext(t, nil, sketchLocation, "arduino:avr:yun") defer cleanUpBuilderTestContext(t, ctx) @@ -522,7 +522,7 @@ func TestPrototypesAdderSketchWithIfDef2(t *testing.T) { func TestPrototypesAdderSketchWithIfDef2SAM(t *testing.T) { sketchLocation := paths.New("sketch_with_ifdef", "sketch_with_ifdef.ino") - quotedSketchLocation := utils.QuoteCppPath(Abs(t, sketchLocation)) + quotedSketchLocation := cpp.QuoteString(Abs(t, sketchLocation).String()) ctx := prepareBuilderTestContext(t, nil, sketchLocation, "arduino:sam:arduino_due_x_dbg") defer cleanUpBuilderTestContext(t, ctx) @@ -553,7 +553,7 @@ func TestPrototypesAdderSketchWithIfDef2SAM(t *testing.T) { func TestPrototypesAdderSketchWithConst(t *testing.T) { sketchLocation := paths.New("sketch_with_const", "sketch_with_const.ino") - quotedSketchLocation := utils.QuoteCppPath(Abs(t, sketchLocation)) + quotedSketchLocation := cpp.QuoteString(Abs(t, sketchLocation).String()) ctx := prepareBuilderTestContext(t, nil, sketchLocation, "arduino:avr:uno") defer cleanUpBuilderTestContext(t, ctx) @@ -604,7 +604,7 @@ func TestPrototypesAdderSketchWithDosEol(t *testing.T) { func TestPrototypesAdderSketchWithSubstringFunctionMember(t *testing.T) { sketchLocation := paths.New("sketch_with_class_and_method_substring", "sketch_with_class_and_method_substring.ino") - quotedSketchLocation := utils.QuoteCppString(Abs(t, sketchLocation).String()) + quotedSketchLocation := cpp.QuoteString(Abs(t, sketchLocation).String()) ctx := prepareBuilderTestContext(t, nil, sketchLocation, "arduino:avr:uno") defer cleanUpBuilderTestContext(t, ctx) diff --git a/legacy/builder/test/utils_test.go b/legacy/builder/test/utils_test.go index ec34a7bb670..a6a4dea8fb4 100644 --- a/legacy/builder/test/utils_test.go +++ b/legacy/builder/test/utils_test.go @@ -40,16 +40,3 @@ func TestPrintableCommand(t *testing.T) { result := utils.PrintableCommand(parts) require.Equal(t, correct, result) } - -func TestQuoteCppString(t *testing.T) { - cases := map[string]string{ - `foo`: `"foo"`, - `foo\bar`: `"foo\\bar"`, - `foo "is" quoted and \\bar"" escaped\`: `"foo \"is\" quoted and \\\\bar\"\" escaped\\"`, - // ASCII 0x20 - 0x7e, excluding ` - ` !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_abcdefghijklmnopqrstuvwxyz{|}~`: `" !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_abcdefghijklmnopqrstuvwxyz{|}~"`, - } - for input, expected := range cases { - require.Equal(t, expected, utils.QuoteCppString(input)) - } -} diff --git a/legacy/builder/utils/utils.go b/legacy/builder/utils/utils.go index 056940308fe..3efc8b73d6d 100644 --- a/legacy/builder/utils/utils.go +++ b/legacy/builder/utils/utils.go @@ -262,19 +262,6 @@ func LogIfVerbose(warn bool, msg string) types.Command { return &loggerAction{onlyIfVerbose: true, warn: warn, msg: msg} } -// Returns the given string as a quoted string for use with the C -// preprocessor. This adds double quotes around it and escapes any -// double quotes and backslashes in the string. -func QuoteCppString(str string) string { - str = strings.Replace(str, "\\", "\\\\", -1) - str = strings.Replace(str, "\"", "\\\"", -1) - return "\"" + str + "\"" -} - -func QuoteCppPath(path *paths.Path) string { - return QuoteCppString(path.String()) -} - // Normalizes an UTF8 byte slice // TODO: use it more often troughout all the project (maybe on logger interface?) func NormalizeUTF8(buf []byte) []byte { From 0f77b623782bcba9e766a1cef8ab9a285f2e9896 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Tue, 11 Apr 2023 12:49:19 +0200 Subject: [PATCH 36/42] Removed useless builderCtx field SketchSourceAfterCppPreprocessing --- legacy/builder/container_add_prototypes.go | 9 ++++----- legacy/builder/types/context.go | 2 -- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/legacy/builder/container_add_prototypes.go b/legacy/builder/container_add_prototypes.go index 36a1b844368..ea1cb2bb1d2 100644 --- a/legacy/builder/container_add_prototypes.go +++ b/legacy/builder/container_add_prototypes.go @@ -63,11 +63,10 @@ func PreprocessSketchWithCtags(ctx *types.Context) error { if src, err := targetFilePath.ReadFile(); err != nil { return err } else { - ctx.SketchSourceAfterCppPreprocessing = filterSketchSource(ctx.Sketch, bytes.NewReader(src), false) - } - - if err := targetFilePath.WriteFile([]byte(ctx.SketchSourceAfterCppPreprocessing)); err != nil { - return err + filteredSource := filterSketchSource(ctx.Sketch, bytes.NewReader(src), false) + if err := targetFilePath.WriteFile([]byte(filteredSource)); err != nil { + return err + } } sketchCpp := ctx.SketchBuildPath.Join(fmt.Sprintf("%s.cpp", ctx.Sketch.MainFile.Base())) diff --git a/legacy/builder/types/context.go b/legacy/builder/types/context.go index 2b5fd27b679..47c499f47c3 100644 --- a/legacy/builder/types/context.go +++ b/legacy/builder/types/context.go @@ -104,8 +104,6 @@ type Context struct { // Arduino sketch (.ino) to C++ (.cpp) conversion steps: // 1. Concatenate *.ino files into a single merged source file -> SketchSourceMerged SketchSourceMerged string - // 2. Run a pass of C++ preprocessor to remove macro definitions and ifdef-ed code -> SketchSourceAfterCppPreprocessing - SketchSourceAfterCppPreprocessing string // 3. Do the Arduino preprocessing of the sketch (add missing prototypes) -> SketchSourceAfterArduinoPreprocessing SketchSourceAfterArduinoPreprocessing string From f5636a96e11d6ce9e624f1ffaac1afeac0f1e3b1 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Tue, 11 Apr 2023 12:59:39 +0200 Subject: [PATCH 37/42] Removed useless builderCtx field SketchSourceAfterArduinoPreprocessing --- legacy/builder/builder.go | 6 +- legacy/builder/container_add_prototypes.go | 12 +-- legacy/builder/preprocess_sketch.go | 3 - legacy/builder/test/prototypes_adder_test.go | 96 +++++++++++++------- legacy/builder/types/context.go | 2 - 5 files changed, 67 insertions(+), 52 deletions(-) diff --git a/legacy/builder/builder.go b/legacy/builder/builder.go index b1b1f007c5d..ba511a9a1b4 100644 --- a/legacy/builder/builder.go +++ b/legacy/builder/builder.go @@ -157,7 +157,11 @@ func (s *Preprocess) Run(ctx *types.Context) error { } // Output arduino-preprocessed source - ctx.WriteStdout([]byte(ctx.SketchSourceAfterArduinoPreprocessing)) + preprocessedSketch, err := ctx.SketchBuildPath.Join(ctx.Sketch.MainFile.Base() + ".cpp").ReadFile() + if err != nil { + return err + } + ctx.WriteStdout(preprocessedSketch) return nil } diff --git a/legacy/builder/container_add_prototypes.go b/legacy/builder/container_add_prototypes.go index ea1cb2bb1d2..14866f9497d 100644 --- a/legacy/builder/container_add_prototypes.go +++ b/legacy/builder/container_add_prototypes.go @@ -74,17 +74,7 @@ func PreprocessSketchWithCtags(ctx *types.Context) error { if ctx.Verbose { ctx.WriteStderr(ctagsStderr) } - if err != nil { - return err - } - - // Save preprocesssed source in context - if d, err := sketchCpp.ReadFile(); err != nil { - return err - } else { - ctx.SketchSourceAfterArduinoPreprocessing = string(d) - } - return nil + return err } func filterSketchSource(sketch *sketch.Sketch, source io.Reader, removeLineMarkers bool) string { diff --git a/legacy/builder/preprocess_sketch.go b/legacy/builder/preprocess_sketch.go index da6f5c9886a..8769b2da578 100644 --- a/legacy/builder/preprocess_sketch.go +++ b/legacy/builder/preprocess_sketch.go @@ -85,8 +85,5 @@ func PreprocessSketchWithArduinoPreprocessor(ctx *types.Context) error { } result := utils.NormalizeUTF8(buf) - - //fmt.Printf("PREPROCESSOR OUTPUT:\n%s\n", output) - ctx.SketchSourceAfterArduinoPreprocessing = string(result) return bldr.SketchSaveItemCpp(ctx.Sketch.MainFile, result, ctx.SketchBuildPath) } diff --git a/legacy/builder/test/prototypes_adder_test.go b/legacy/builder/test/prototypes_adder_test.go index f175322fe64..f3dd2d8804d 100644 --- a/legacy/builder/test/prototypes_adder_test.go +++ b/legacy/builder/test/prototypes_adder_test.go @@ -29,6 +29,12 @@ import ( "github.com/stretchr/testify/require" ) +func loadPreprocessedSketch(t *testing.T, ctx *types.Context) string { + res, err := ctx.SketchBuildPath.Join(ctx.Sketch.MainFile.Base() + ".cpp").ReadFile() + NoError(t, err) + return string(res) +} + func TestPrototypesAdderBridgeExample(t *testing.T) { sketchLocation := paths.New("downloaded_libraries", "Bridge", "examples", "Bridge", "Bridge.ino") quotedSketchLocation := cpp.QuoteString(Abs(t, sketchLocation).String()) @@ -53,8 +59,9 @@ func TestPrototypesAdderBridgeExample(t *testing.T) { } NoError(t, builder.PreprocessSketchWithCtags(ctx)) - require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") - require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#line 33 "+quotedSketchLocation+"\nvoid setup();\n#line 46 "+quotedSketchLocation+"\nvoid loop();\n#line 62 "+quotedSketchLocation+"\nvoid process(BridgeClient client);\n#line 82 "+quotedSketchLocation+"\nvoid digitalCommand(BridgeClient client);\n#line 109 "+quotedSketchLocation+"\nvoid analogCommand(BridgeClient client);\n#line 149 "+quotedSketchLocation+"\nvoid modeCommand(BridgeClient client);\n#line 33 "+quotedSketchLocation+"\n") + preprocessedSketch := loadPreprocessedSketch(t, ctx) + require.Contains(t, preprocessedSketch, "#include \n#line 1 "+quotedSketchLocation+"\n") + require.Contains(t, preprocessedSketch, "#line 33 "+quotedSketchLocation+"\nvoid setup();\n#line 46 "+quotedSketchLocation+"\nvoid loop();\n#line 62 "+quotedSketchLocation+"\nvoid process(BridgeClient client);\n#line 82 "+quotedSketchLocation+"\nvoid digitalCommand(BridgeClient client);\n#line 109 "+quotedSketchLocation+"\nvoid analogCommand(BridgeClient client);\n#line 149 "+quotedSketchLocation+"\nvoid modeCommand(BridgeClient client);\n#line 33 "+quotedSketchLocation+"\n") } func TestPrototypesAdderSketchWithIfDef(t *testing.T) { @@ -79,7 +86,8 @@ func TestPrototypesAdderSketchWithIfDef(t *testing.T) { NoError(t, builder.PreprocessSketchWithCtags(ctx)) preprocessed := LoadAndInterpolate(t, filepath.Join("SketchWithIfDef", "SketchWithIfDef.preprocessed.txt"), ctx) - require.Equal(t, preprocessed, strings.Replace(ctx.SketchSourceAfterArduinoPreprocessing, "\r\n", "\n", -1)) + preprocessedSketch := loadPreprocessedSketch(t, ctx) + require.Equal(t, preprocessed, strings.Replace(preprocessedSketch, "\r\n", "\n", -1)) } func TestPrototypesAdderBaladuino(t *testing.T) { @@ -104,7 +112,8 @@ func TestPrototypesAdderBaladuino(t *testing.T) { NoError(t, builder.PreprocessSketchWithCtags(ctx)) preprocessed := LoadAndInterpolate(t, filepath.Join("Baladuino", "Baladuino.preprocessed.txt"), ctx) - require.Equal(t, preprocessed, strings.Replace(ctx.SketchSourceAfterArduinoPreprocessing, "\r\n", "\n", -1)) + preprocessedSketch := loadPreprocessedSketch(t, ctx) + require.Equal(t, preprocessed, strings.Replace(preprocessedSketch, "\r\n", "\n", -1)) } func TestPrototypesAdderCharWithEscapedDoubleQuote(t *testing.T) { @@ -129,7 +138,8 @@ func TestPrototypesAdderCharWithEscapedDoubleQuote(t *testing.T) { NoError(t, builder.PreprocessSketchWithCtags(ctx)) preprocessed := LoadAndInterpolate(t, filepath.Join("CharWithEscapedDoubleQuote", "CharWithEscapedDoubleQuote.preprocessed.txt"), ctx) - require.Equal(t, preprocessed, strings.Replace(ctx.SketchSourceAfterArduinoPreprocessing, "\r\n", "\n", -1)) + preprocessedSketch := loadPreprocessedSketch(t, ctx) + require.Equal(t, preprocessed, strings.Replace(preprocessedSketch, "\r\n", "\n", -1)) } func TestPrototypesAdderIncludeBetweenMultilineComment(t *testing.T) { @@ -154,7 +164,8 @@ func TestPrototypesAdderIncludeBetweenMultilineComment(t *testing.T) { NoError(t, builder.PreprocessSketchWithCtags(ctx)) preprocessed := LoadAndInterpolate(t, filepath.Join("IncludeBetweenMultilineComment", "IncludeBetweenMultilineComment.preprocessed.txt"), ctx) - require.Equal(t, preprocessed, strings.Replace(ctx.SketchSourceAfterArduinoPreprocessing, "\r\n", "\n", -1)) + preprocessedSketch := loadPreprocessedSketch(t, ctx) + require.Equal(t, preprocessed, strings.Replace(preprocessedSketch, "\r\n", "\n", -1)) } func TestPrototypesAdderLineContinuations(t *testing.T) { @@ -179,7 +190,8 @@ func TestPrototypesAdderLineContinuations(t *testing.T) { NoError(t, builder.PreprocessSketchWithCtags(ctx)) preprocessed := LoadAndInterpolate(t, filepath.Join("LineContinuations", "LineContinuations.preprocessed.txt"), ctx) - require.Equal(t, preprocessed, strings.Replace(ctx.SketchSourceAfterArduinoPreprocessing, "\r\n", "\n", -1)) + preprocessedSketch := loadPreprocessedSketch(t, ctx) + require.Equal(t, preprocessed, strings.Replace(preprocessedSketch, "\r\n", "\n", -1)) } func TestPrototypesAdderStringWithComment(t *testing.T) { @@ -204,7 +216,8 @@ func TestPrototypesAdderStringWithComment(t *testing.T) { NoError(t, builder.PreprocessSketchWithCtags(ctx)) preprocessed := LoadAndInterpolate(t, filepath.Join("StringWithComment", "StringWithComment.preprocessed.txt"), ctx) - require.Equal(t, preprocessed, strings.Replace(ctx.SketchSourceAfterArduinoPreprocessing, "\r\n", "\n", -1)) + preprocessedSketch := loadPreprocessedSketch(t, ctx) + require.Equal(t, preprocessed, strings.Replace(preprocessedSketch, "\r\n", "\n", -1)) } func TestPrototypesAdderSketchWithStruct(t *testing.T) { @@ -229,7 +242,8 @@ func TestPrototypesAdderSketchWithStruct(t *testing.T) { NoError(t, builder.PreprocessSketchWithCtags(ctx)) preprocessed := LoadAndInterpolate(t, filepath.Join("SketchWithStruct", "SketchWithStruct.preprocessed.txt"), ctx) - obtained := strings.Replace(ctx.SketchSourceAfterArduinoPreprocessing, "\r\n", "\n", -1) + preprocessedSketch := loadPreprocessedSketch(t, ctx) + obtained := strings.Replace(preprocessedSketch, "\r\n", "\n", -1) // ctags based preprocessing removes the space after "dostuff", but this is still OK // TODO: remove this exception when moving to a more powerful parser preprocessed = strings.Replace(preprocessed, "void dostuff (A_NEW_TYPE * bar);", "void dostuff(A_NEW_TYPE * bar);", 1) @@ -261,11 +275,12 @@ func TestPrototypesAdderSketchWithConfig(t *testing.T) { } NoError(t, builder.PreprocessSketchWithCtags(ctx)) - require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") - require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#line 13 "+quotedSketchLocation+"\nvoid setup();\n#line 17 "+quotedSketchLocation+"\nvoid loop();\n#line 13 "+quotedSketchLocation+"\n") + preprocessedSketch := loadPreprocessedSketch(t, ctx) + require.Contains(t, preprocessedSketch, "#include \n#line 1 "+quotedSketchLocation+"\n") + require.Contains(t, preprocessedSketch, "#line 13 "+quotedSketchLocation+"\nvoid setup();\n#line 17 "+quotedSketchLocation+"\nvoid loop();\n#line 13 "+quotedSketchLocation+"\n") preprocessed := LoadAndInterpolate(t, filepath.Join("sketch_with_config", "sketch_with_config.preprocessed.txt"), ctx) - require.Equal(t, preprocessed, strings.Replace(ctx.SketchSourceAfterArduinoPreprocessing, "\r\n", "\n", -1)) + require.Equal(t, preprocessed, strings.Replace(preprocessedSketch, "\r\n", "\n", -1)) } func TestPrototypesAdderSketchNoFunctionsTwoFiles(t *testing.T) { @@ -292,8 +307,9 @@ func TestPrototypesAdderSketchNoFunctionsTwoFiles(t *testing.T) { } NoError(t, builder.PreprocessSketchWithCtags(ctx)) - require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") - require.Equal(t, ctx.SketchSourceMerged, ctx.SketchSourceAfterArduinoPreprocessing) // No prototypes added + preprocessedSketch := loadPreprocessedSketch(t, ctx) + require.Contains(t, preprocessedSketch, "#include \n#line 1 "+quotedSketchLocation+"\n") + require.Equal(t, ctx.SketchSourceMerged, preprocessedSketch) // No prototypes added } func TestPrototypesAdderSketchNoFunctions(t *testing.T) { @@ -319,8 +335,9 @@ func TestPrototypesAdderSketchNoFunctions(t *testing.T) { } NoError(t, builder.PreprocessSketchWithCtags(ctx)) - require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") - require.Equal(t, ctx.SketchSourceMerged, ctx.SketchSourceAfterArduinoPreprocessing) // No prototypes added + preprocessedSketch := loadPreprocessedSketch(t, ctx) + require.Contains(t, preprocessedSketch, "#include \n#line 1 "+quotedSketchLocation+"\n") + require.Equal(t, ctx.SketchSourceMerged, preprocessedSketch) // No prototypes added } func TestPrototypesAdderSketchWithDefaultArgs(t *testing.T) { @@ -347,8 +364,9 @@ func TestPrototypesAdderSketchWithDefaultArgs(t *testing.T) { } NoError(t, builder.PreprocessSketchWithCtags(ctx)) - require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") - require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#line 4 "+quotedSketchLocation+"\nvoid setup();\n#line 7 "+quotedSketchLocation+"\nvoid loop();\n#line 1 "+quotedSketchLocation+"\n") + preprocessedSketch := loadPreprocessedSketch(t, ctx) + require.Contains(t, preprocessedSketch, "#include \n#line 1 "+quotedSketchLocation+"\n") + require.Contains(t, preprocessedSketch, "#line 4 "+quotedSketchLocation+"\nvoid setup();\n#line 7 "+quotedSketchLocation+"\nvoid loop();\n#line 1 "+quotedSketchLocation+"\n") } func TestPrototypesAdderSketchWithInlineFunction(t *testing.T) { @@ -375,10 +393,11 @@ func TestPrototypesAdderSketchWithInlineFunction(t *testing.T) { } NoError(t, builder.PreprocessSketchWithCtags(ctx)) - require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") + preprocessedSketch := loadPreprocessedSketch(t, ctx) + require.Contains(t, preprocessedSketch, "#include \n#line 1 "+quotedSketchLocation+"\n") expected := "#line 1 " + quotedSketchLocation + "\nvoid setup();\n#line 2 " + quotedSketchLocation + "\nvoid loop();\n#line 4 " + quotedSketchLocation + "\nshort unsigned int testInt();\n#line 8 " + quotedSketchLocation + "\nstatic int8_t testInline();\n#line 12 " + quotedSketchLocation + "\n__attribute__((always_inline)) uint8_t testAttribute();\n#line 1 " + quotedSketchLocation + "\n" - obtained := ctx.SketchSourceAfterArduinoPreprocessing + obtained := preprocessedSketch // ctags based preprocessing removes "inline" but this is still OK // TODO: remove this exception when moving to a more powerful parser expected = strings.Replace(expected, "static inline int8_t testInline();", "static int8_t testInline();", -1) @@ -414,8 +433,9 @@ func TestPrototypesAdderSketchWithFunctionSignatureInsideIFDEF(t *testing.T) { } NoError(t, builder.PreprocessSketchWithCtags(ctx)) - require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") - require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#line 1 "+quotedSketchLocation+"\nvoid setup();\n#line 3 "+quotedSketchLocation+"\nvoid loop();\n#line 15 "+quotedSketchLocation+"\nint8_t adalight();\n#line 1 "+quotedSketchLocation+"\n") + preprocessedSketch := loadPreprocessedSketch(t, ctx) + require.Contains(t, preprocessedSketch, "#include \n#line 1 "+quotedSketchLocation+"\n") + require.Contains(t, preprocessedSketch, "#line 1 "+quotedSketchLocation+"\nvoid setup();\n#line 3 "+quotedSketchLocation+"\nvoid loop();\n#line 15 "+quotedSketchLocation+"\nint8_t adalight();\n#line 1 "+quotedSketchLocation+"\n") } func TestPrototypesAdderSketchWithUSBCON(t *testing.T) { @@ -447,8 +467,9 @@ func TestPrototypesAdderSketchWithUSBCON(t *testing.T) { } NoError(t, builder.PreprocessSketchWithCtags(ctx)) - require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") - require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#line 5 "+quotedSketchLocation+"\nvoid ciao();\n#line 10 "+quotedSketchLocation+"\nvoid setup();\n#line 15 "+quotedSketchLocation+"\nvoid loop();\n#line 5 "+quotedSketchLocation+"\n") + preprocessedSketch := loadPreprocessedSketch(t, ctx) + require.Contains(t, preprocessedSketch, "#include \n#line 1 "+quotedSketchLocation+"\n") + require.Contains(t, preprocessedSketch, "#line 5 "+quotedSketchLocation+"\nvoid ciao();\n#line 10 "+quotedSketchLocation+"\nvoid setup();\n#line 15 "+quotedSketchLocation+"\nvoid loop();\n#line 5 "+quotedSketchLocation+"\n") } func TestPrototypesAdderSketchWithTypename(t *testing.T) { @@ -479,9 +500,10 @@ func TestPrototypesAdderSketchWithTypename(t *testing.T) { } NoError(t, builder.PreprocessSketchWithCtags(ctx)) - require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") + preprocessedSketch := loadPreprocessedSketch(t, ctx) + require.Contains(t, preprocessedSketch, "#include \n#line 1 "+quotedSketchLocation+"\n") expected := "#line 6 " + quotedSketchLocation + "\nvoid setup();\n#line 10 " + quotedSketchLocation + "\nvoid loop();\n#line 12 " + quotedSketchLocation + "\ntypename Foo::Bar func();\n#line 6 " + quotedSketchLocation + "\n" - obtained := ctx.SketchSourceAfterArduinoPreprocessing + obtained := preprocessedSketch // ctags based preprocessing ignores line with typename // TODO: remove this exception when moving to a more powerful parser expected = strings.Replace(expected, "#line 12 "+quotedSketchLocation+"\ntypename Foo::Bar func();\n", "", -1) @@ -513,11 +535,12 @@ func TestPrototypesAdderSketchWithIfDef2(t *testing.T) { } NoError(t, builder.PreprocessSketchWithCtags(ctx)) - require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") - require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#line 5 "+quotedSketchLocation+"\nvoid elseBranch();\n#line 9 "+quotedSketchLocation+"\nvoid f1();\n#line 10 "+quotedSketchLocation+"\nvoid f2();\n#line 12 "+quotedSketchLocation+"\nvoid setup();\n#line 14 "+quotedSketchLocation+"\nvoid loop();\n#line 5 "+quotedSketchLocation+"\n") + preprocessedSketch := loadPreprocessedSketch(t, ctx) + require.Contains(t, preprocessedSketch, "#include \n#line 1 "+quotedSketchLocation+"\n") + require.Contains(t, preprocessedSketch, "#line 5 "+quotedSketchLocation+"\nvoid elseBranch();\n#line 9 "+quotedSketchLocation+"\nvoid f1();\n#line 10 "+quotedSketchLocation+"\nvoid f2();\n#line 12 "+quotedSketchLocation+"\nvoid setup();\n#line 14 "+quotedSketchLocation+"\nvoid loop();\n#line 5 "+quotedSketchLocation+"\n") expectedSource := LoadAndInterpolate(t, filepath.Join("sketch_with_ifdef", "sketch.preprocessed.txt"), ctx) - require.Equal(t, expectedSource, strings.Replace(ctx.SketchSourceAfterArduinoPreprocessing, "\r\n", "\n", -1)) + require.Equal(t, expectedSource, strings.Replace(preprocessedSketch, "\r\n", "\n", -1)) } func TestPrototypesAdderSketchWithIfDef2SAM(t *testing.T) { @@ -544,11 +567,12 @@ func TestPrototypesAdderSketchWithIfDef2SAM(t *testing.T) { } NoError(t, builder.PreprocessSketchWithCtags(ctx)) - require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") - require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#line 2 "+quotedSketchLocation+"\nvoid ifBranch();\n#line 9 "+quotedSketchLocation+"\nvoid f1();\n#line 10 "+quotedSketchLocation+"\nvoid f2();\n#line 12 "+quotedSketchLocation+"\nvoid setup();\n#line 14 "+quotedSketchLocation+"\nvoid loop();\n#line 2 "+quotedSketchLocation+"\n") + preprocessedSketch := loadPreprocessedSketch(t, ctx) + require.Contains(t, preprocessedSketch, "#include \n#line 1 "+quotedSketchLocation+"\n") + require.Contains(t, preprocessedSketch, "#line 2 "+quotedSketchLocation+"\nvoid ifBranch();\n#line 9 "+quotedSketchLocation+"\nvoid f1();\n#line 10 "+quotedSketchLocation+"\nvoid f2();\n#line 12 "+quotedSketchLocation+"\nvoid setup();\n#line 14 "+quotedSketchLocation+"\nvoid loop();\n#line 2 "+quotedSketchLocation+"\n") expectedSource := LoadAndInterpolate(t, filepath.Join("sketch_with_ifdef", "sketch.preprocessed.SAM.txt"), ctx) - require.Equal(t, expectedSource, strings.Replace(ctx.SketchSourceAfterArduinoPreprocessing, "\r\n", "\n", -1)) + require.Equal(t, expectedSource, strings.Replace(preprocessedSketch, "\r\n", "\n", -1)) } func TestPrototypesAdderSketchWithConst(t *testing.T) { @@ -575,8 +599,9 @@ func TestPrototypesAdderSketchWithConst(t *testing.T) { } NoError(t, builder.PreprocessSketchWithCtags(ctx)) - require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#include \n#line 1 "+quotedSketchLocation+"\n") - require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "#line 1 "+quotedSketchLocation+"\nvoid setup();\n#line 2 "+quotedSketchLocation+"\nvoid loop();\n#line 4 "+quotedSketchLocation+"\nconst __FlashStringHelper* test();\n#line 6 "+quotedSketchLocation+"\nconst int test3();\n#line 8 "+quotedSketchLocation+"\nvolatile __FlashStringHelper* test2();\n#line 10 "+quotedSketchLocation+"\nvolatile int test4();\n#line 1 "+quotedSketchLocation+"\n") + preprocessedSketch := loadPreprocessedSketch(t, ctx) + require.Contains(t, preprocessedSketch, "#include \n#line 1 "+quotedSketchLocation+"\n") + require.Contains(t, preprocessedSketch, "#line 1 "+quotedSketchLocation+"\nvoid setup();\n#line 2 "+quotedSketchLocation+"\nvoid loop();\n#line 4 "+quotedSketchLocation+"\nconst __FlashStringHelper* test();\n#line 6 "+quotedSketchLocation+"\nconst int test3();\n#line 8 "+quotedSketchLocation+"\nvolatile __FlashStringHelper* test2();\n#line 10 "+quotedSketchLocation+"\nvolatile int test4();\n#line 1 "+quotedSketchLocation+"\n") } func TestPrototypesAdderSketchWithDosEol(t *testing.T) { @@ -626,5 +651,6 @@ func TestPrototypesAdderSketchWithSubstringFunctionMember(t *testing.T) { } NoError(t, builder.PreprocessSketchWithCtags(ctx)) - require.Contains(t, ctx.SketchSourceAfterArduinoPreprocessing, "class Foo {\nint blooper(int x) { return x+1; }\n};\n\nFoo foo;\n\n#line 7 "+quotedSketchLocation+"\nvoid setup();") + preprocessedSketch := loadPreprocessedSketch(t, ctx) + require.Contains(t, preprocessedSketch, "class Foo {\nint blooper(int x) { return x+1; }\n};\n\nFoo foo;\n\n#line 7 "+quotedSketchLocation+"\nvoid setup();") } diff --git a/legacy/builder/types/context.go b/legacy/builder/types/context.go index 47c499f47c3..6a07e64ba64 100644 --- a/legacy/builder/types/context.go +++ b/legacy/builder/types/context.go @@ -104,8 +104,6 @@ type Context struct { // Arduino sketch (.ino) to C++ (.cpp) conversion steps: // 1. Concatenate *.ino files into a single merged source file -> SketchSourceMerged SketchSourceMerged string - // 3. Do the Arduino preprocessing of the sketch (add missing prototypes) -> SketchSourceAfterArduinoPreprocessing - SketchSourceAfterArduinoPreprocessing string // Libraries handling LibrariesManager *librariesmanager.LibrariesManager From 73c04fdf03930829b15825d1d798e8b1780078e5 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Tue, 11 Apr 2023 13:09:27 +0200 Subject: [PATCH 38/42] Removed useless builderCtx field SketchSourceMerged --- arduino/builder/sketch.go | 19 ++++---- legacy/builder/builder.go | 4 +- legacy/builder/test/ctags_runner_test.go | 3 +- .../test/includes_to_include_folders_test.go | 16 +++---- legacy/builder/test/prototypes_adder_test.go | 48 ++++++++++--------- legacy/builder/types/context.go | 4 -- 6 files changed, 46 insertions(+), 48 deletions(-) diff --git a/arduino/builder/sketch.go b/arduino/builder/sketch.go index 9ca21b6139e..a79bbac4bef 100644 --- a/arduino/builder/sketch.go +++ b/arduino/builder/sketch.go @@ -37,17 +37,16 @@ var ( // PrepareSketchBuildPath copies the sketch source files in the build path. // The .ino files are merged together to create a .cpp file (by the way, the // .cpp file still needs to be Arduino-preprocessed to compile). -func PrepareSketchBuildPath(sketch *sketch.Sketch, sourceOverrides map[string]string, buildPath *paths.Path) (offset int, mergedSource string, err error) { - if offset, mergedSource, err = sketchMergeSources(sketch, sourceOverrides); err != nil { - return - } - if err = SketchSaveItemCpp(sketch.MainFile, []byte(mergedSource), buildPath); err != nil { - return - } - if err = sketchCopyAdditionalFiles(sketch, buildPath, sourceOverrides); err != nil { - return +func PrepareSketchBuildPath(sketch *sketch.Sketch, sourceOverrides map[string]string, buildPath *paths.Path) (int, error) { + if offset, mergedSource, err := sketchMergeSources(sketch, sourceOverrides); err != nil { + return 0, err + } else if err := SketchSaveItemCpp(sketch.MainFile, []byte(mergedSource), buildPath); err != nil { + return 0, err + } else if err := sketchCopyAdditionalFiles(sketch, buildPath, sourceOverrides); err != nil { + return 0, err + } else { + return offset, nil } - return } // SketchSaveItemCpp saves a preprocessed .cpp sketch file on disk diff --git a/legacy/builder/builder.go b/legacy/builder/builder.go index ba511a9a1b4..36bed597616 100644 --- a/legacy/builder/builder.go +++ b/legacy/builder/builder.go @@ -49,7 +49,7 @@ func (s *Builder) Run(ctx *types.Context) error { &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.prebuild", Suffix: ".pattern"}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, _err = builder.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), @@ -141,7 +141,7 @@ func (s *Preprocess) Run(ctx *types.Context) error { &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.prebuild", Suffix: ".pattern"}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = builder.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, _err = builder.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), diff --git a/legacy/builder/test/ctags_runner_test.go b/legacy/builder/test/ctags_runner_test.go index d776a93c4bc..557aa34f57d 100644 --- a/legacy/builder/test/ctags_runner_test.go +++ b/legacy/builder/test/ctags_runner_test.go @@ -34,9 +34,10 @@ func ctagsRunnerTestTemplate(t *testing.T, sketchLocation *paths.Path) []byte { err := (&builder.ContainerSetupHardwareToolsLibsSketchAndProps{}).Run(ctx) NoError(t, err) - _, source, err := bldr.PrepareSketchBuildPath(ctx.Sketch, nil, ctx.SketchBuildPath) + _, err = bldr.PrepareSketchBuildPath(ctx.Sketch, nil, ctx.SketchBuildPath) NoError(t, err) + source := loadPreprocessedSketch(t, ctx) target := ctx.BuildPath.Join("ctags_target.cpp") NoError(t, target.WriteFile([]byte(source))) diff --git a/legacy/builder/test/includes_to_include_folders_test.go b/legacy/builder/test/includes_to_include_folders_test.go index a19d0ad57f2..f457e518185 100644 --- a/legacy/builder/test/includes_to_include_folders_test.go +++ b/legacy/builder/test/includes_to_include_folders_test.go @@ -36,7 +36,7 @@ func TestIncludesToIncludeFolders(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -60,7 +60,7 @@ func TestIncludesToIncludeFoldersSketchWithIfDef(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -83,7 +83,7 @@ func TestIncludesToIncludeFoldersIRremoteLibrary(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -109,7 +109,7 @@ func TestIncludesToIncludeFoldersANewLibrary(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -140,7 +140,7 @@ func TestIncludesToIncludeFoldersDuplicateLibs(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -172,7 +172,7 @@ func TestIncludesToIncludeFoldersDuplicateLibsWithConflictingLibsOutsideOfPlatfo commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -204,7 +204,7 @@ func TestIncludesToIncludeFoldersDuplicateLibs2(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -230,7 +230,7 @@ func TestIncludesToIncludeFoldersSubfolders(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, diff --git a/legacy/builder/test/prototypes_adder_test.go b/legacy/builder/test/prototypes_adder_test.go index f3dd2d8804d..f40e339f0f9 100644 --- a/legacy/builder/test/prototypes_adder_test.go +++ b/legacy/builder/test/prototypes_adder_test.go @@ -48,7 +48,7 @@ func TestPrototypesAdderBridgeExample(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -74,7 +74,7 @@ func TestPrototypesAdderSketchWithIfDef(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -100,7 +100,7 @@ func TestPrototypesAdderBaladuino(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -126,7 +126,7 @@ func TestPrototypesAdderCharWithEscapedDoubleQuote(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -152,7 +152,7 @@ func TestPrototypesAdderIncludeBetweenMultilineComment(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -178,7 +178,7 @@ func TestPrototypesAdderLineContinuations(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -204,7 +204,7 @@ func TestPrototypesAdderStringWithComment(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -230,7 +230,7 @@ func TestPrototypesAdderSketchWithStruct(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -264,7 +264,7 @@ func TestPrototypesAdderSketchWithConfig(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -296,7 +296,7 @@ func TestPrototypesAdderSketchNoFunctionsTwoFiles(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -305,11 +305,12 @@ func TestPrototypesAdderSketchNoFunctionsTwoFiles(t *testing.T) { err := command.Run(ctx) NoError(t, err) } + mergedSketch := loadPreprocessedSketch(t, ctx) NoError(t, builder.PreprocessSketchWithCtags(ctx)) preprocessedSketch := loadPreprocessedSketch(t, ctx) require.Contains(t, preprocessedSketch, "#include \n#line 1 "+quotedSketchLocation+"\n") - require.Equal(t, ctx.SketchSourceMerged, preprocessedSketch) // No prototypes added + require.Equal(t, mergedSketch, preprocessedSketch) // No prototypes added } func TestPrototypesAdderSketchNoFunctions(t *testing.T) { @@ -324,7 +325,7 @@ func TestPrototypesAdderSketchNoFunctions(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -333,11 +334,12 @@ func TestPrototypesAdderSketchNoFunctions(t *testing.T) { err := command.Run(ctx) NoError(t, err) } + mergedSketch := loadPreprocessedSketch(t, ctx) NoError(t, builder.PreprocessSketchWithCtags(ctx)) preprocessedSketch := loadPreprocessedSketch(t, ctx) require.Contains(t, preprocessedSketch, "#include \n#line 1 "+quotedSketchLocation+"\n") - require.Equal(t, ctx.SketchSourceMerged, preprocessedSketch) // No prototypes added + require.Equal(t, mergedSketch, preprocessedSketch) // No prototypes added } func TestPrototypesAdderSketchWithDefaultArgs(t *testing.T) { @@ -353,7 +355,7 @@ func TestPrototypesAdderSketchWithDefaultArgs(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -382,7 +384,7 @@ func TestPrototypesAdderSketchWithInlineFunction(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -422,7 +424,7 @@ func TestPrototypesAdderSketchWithFunctionSignatureInsideIFDEF(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -456,7 +458,7 @@ func TestPrototypesAdderSketchWithUSBCON(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -489,7 +491,7 @@ func TestPrototypesAdderSketchWithTypename(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -524,7 +526,7 @@ func TestPrototypesAdderSketchWithIfDef2(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -556,7 +558,7 @@ func TestPrototypesAdderSketchWithIfDef2SAM(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -588,7 +590,7 @@ func TestPrototypesAdderSketchWithConst(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -614,7 +616,7 @@ func TestPrototypesAdderSketchWithDosEol(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, @@ -640,7 +642,7 @@ func TestPrototypesAdderSketchWithSubstringFunctionMember(t *testing.T) { commands := []types.Command{ &builder.ContainerSetupHardwareToolsLibsSketchAndProps{}, types.BareCommand(func(ctx *types.Context) error { - ctx.LineOffset, ctx.SketchSourceMerged, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) + ctx.LineOffset, _err = bldr.PrepareSketchBuildPath(ctx.Sketch, ctx.SourceOverride, ctx.SketchBuildPath) return _err }), &builder.ContainerFindIncludes{}, diff --git a/legacy/builder/types/context.go b/legacy/builder/types/context.go index 6a07e64ba64..76533756923 100644 --- a/legacy/builder/types/context.go +++ b/legacy/builder/types/context.go @@ -101,10 +101,6 @@ type Context struct { Sketch *sketch.Sketch WarningsLevel string - // Arduino sketch (.ino) to C++ (.cpp) conversion steps: - // 1. Concatenate *.ino files into a single merged source file -> SketchSourceMerged - SketchSourceMerged string - // Libraries handling LibrariesManager *librariesmanager.LibrariesManager LibrariesResolver *librariesresolver.Cpp From 39548b5d6cebe476482455a4599666bb52e61478 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Tue, 11 Apr 2023 17:42:00 +0200 Subject: [PATCH 39/42] Moved ctag preprocessor into proper location --- arduino/builder/preprocessor/ctags.go | 94 ++++++++++-- .../integrationtest/compile_1/compile_test.go | 1 - .../add_additional_entries_to_context.go | 5 - legacy/builder/builder.go | 19 ++- legacy/builder/constants/constants.go | 1 - legacy/builder/container_add_prototypes.go | 135 ------------------ legacy/builder/create_cmake_rule.go | 2 +- legacy/builder/preprocess_sketch.go | 4 +- .../add_additional_entries_to_context_test.go | 2 - legacy/builder/test/builder_test.go | 18 --- legacy/builder/test/prototypes_adder_test.go | 42 +++--- legacy/builder/types/context.go | 1 - 12 files changed, 121 insertions(+), 203 deletions(-) delete mode 100644 legacy/builder/container_add_prototypes.go diff --git a/arduino/builder/preprocessor/ctags.go b/arduino/builder/preprocessor/ctags.go index 5cc9e5876c4..4d8a5a384b5 100644 --- a/arduino/builder/preprocessor/ctags.go +++ b/arduino/builder/preprocessor/ctags.go @@ -16,8 +16,11 @@ package preprocessor import ( + "bufio" + "bytes" "context" "fmt" + "io" "strconv" "strings" @@ -37,30 +40,74 @@ var tr = i18n.Tr // this is useful for unit-testing to provide more infos var DebugPreprocessor bool -func CTags(sourceFile *paths.Path, targetFile *paths.Path, sketch *sketch.Sketch, lineOffset int, buildProperties *properties.Map) ([]byte, error) { - ctagsOutput, ctagsStdErr, err := RunCTags(sourceFile, buildProperties) +// PreprocessSketchWithCtags performs preprocessing of the arduino sketch using CTags. +func PreprocessSketchWithCtags(sketch *sketch.Sketch, buildPath *paths.Path, includes paths.PathList, lineOffset int, buildProperties *properties.Map, onlyUpdateCompilationDatabase bool) ([]byte, []byte, error) { + // Create a temporary working directory + tmpDir, err := paths.MkTempDir("", "") if err != nil { - return ctagsStdErr, err + return nil, nil, err + } + defer tmpDir.RemoveAll() + ctagsTarget := tmpDir.Join("sketch_merged.cpp") + + normalOutput := &bytes.Buffer{} + verboseOutput := &bytes.Buffer{} + + // Run GCC preprocessor + sourceFile := buildPath.Join("sketch", sketch.MainFile.Base()+".cpp") + gccStdout, gccStderr, err := GCC(sourceFile, ctagsTarget, includes, buildProperties) + verboseOutput.Write(gccStdout) + verboseOutput.Write(gccStderr) + normalOutput.Write(gccStderr) + if err != nil { + if !onlyUpdateCompilationDatabase { + return normalOutput.Bytes(), verboseOutput.Bytes(), errors.WithStack(err) + } + + // Do not bail out if we are generating the compile commands database + normalOutput.WriteString(fmt.Sprintf("%s: %s", + tr("An error occurred adding prototypes"), + tr("the compilation database may be incomplete or inaccurate"))) + if err := sourceFile.CopyTo(ctagsTarget); err != nil { + return normalOutput.Bytes(), verboseOutput.Bytes(), errors.WithStack(err) + } } - // func PrototypesAdder(sketch *sketch.Sketch, source string, ctagsStdout []byte, lineOffset int) string { + if src, err := ctagsTarget.ReadFile(); err == nil { + filteredSource := filterSketchSource(sketch, bytes.NewReader(src), false) + if err := ctagsTarget.WriteFile([]byte(filteredSource)); err != nil { + return normalOutput.Bytes(), verboseOutput.Bytes(), err + } + } else { + return normalOutput.Bytes(), verboseOutput.Bytes(), err + } + + // Run CTags on gcc-preprocessed source + ctagsOutput, ctagsStdErr, err := RunCTags(ctagsTarget, buildProperties) + verboseOutput.Write(ctagsStdErr) + if err != nil { + return normalOutput.Bytes(), verboseOutput.Bytes(), err + } + + // Parse CTags output parser := &ctags.CTagsParser{} prototypes, firstFunctionLine := parser.Parse(ctagsOutput, sketch.MainFile) if firstFunctionLine == -1 { firstFunctionLine = 0 } + // Add prototypes to the original sketch source var source string - if sourceData, err := targetFile.ReadFile(); err != nil { - return nil, err - } else { + if sourceData, err := sourceFile.ReadFile(); err == nil { source = string(sourceData) + } else { + return normalOutput.Bytes(), verboseOutput.Bytes(), err } source = strings.Replace(source, "\r\n", "\n", -1) source = strings.Replace(source, "\r", "\n", -1) sourceRows := strings.Split(source, "\n") if isFirstFunctionOutsideOfSource(firstFunctionLine, sourceRows) { - return nil, nil + return normalOutput.Bytes(), verboseOutput.Bytes(), nil } insertionLine := firstFunctionLine + lineOffset - 1 @@ -84,8 +131,9 @@ func CTags(sourceFile *paths.Path, targetFile *paths.Path, sketch *sketch.Sketch fmt.Println("#END OF PREPROCESSED SOURCE") } - err = targetFile.WriteFile([]byte(preprocessedSource)) - return ctagsStdErr, err + // Write back arduino-preprocess output to the sourceFile + err = sourceFile.WriteFile([]byte(preprocessedSource)) + return normalOutput.Bytes(), verboseOutput.Bytes(), err } func composePrototypeSection(line int, prototypes []*ctags.Prototype) string { @@ -157,3 +205,29 @@ func RunCTags(sourceFile *paths.Path, buildProperties *properties.Map) ([]byte, stderr = append([]byte(args), stderr...) return stdout, stderr, err } + +func filterSketchSource(sketch *sketch.Sketch, source io.Reader, removeLineMarkers bool) string { + fileNames := paths.NewPathList() + fileNames.Add(sketch.MainFile) + fileNames.AddAll(sketch.OtherSketchFiles) + + inSketch := false + filtered := "" + + scanner := bufio.NewScanner(source) + for scanner.Scan() { + line := scanner.Text() + if filename := cpp.ParseLineMarker(line); filename != nil { + inSketch = fileNames.Contains(filename) + if inSketch && removeLineMarkers { + continue + } + } + + if inSketch { + filtered += line + "\n" + } + } + + return filtered +} diff --git a/internal/integrationtest/compile_1/compile_test.go b/internal/integrationtest/compile_1/compile_test.go index 58de62094fe..60c6552a678 100644 --- a/internal/integrationtest/compile_1/compile_test.go +++ b/internal/integrationtest/compile_1/compile_test.go @@ -1060,7 +1060,6 @@ func compileWithRelativeBuildPath(t *testing.T, env *integrationtest.Environment "core", "includes.cache", "libraries", - "preproc", "sketch", } diff --git a/legacy/builder/add_additional_entries_to_context.go b/legacy/builder/add_additional_entries_to_context.go index fcb9639e37c..9e9fdd80669 100644 --- a/legacy/builder/add_additional_entries_to_context.go +++ b/legacy/builder/add_additional_entries_to_context.go @@ -26,10 +26,6 @@ type AddAdditionalEntriesToContext struct{} func (*AddAdditionalEntriesToContext) Run(ctx *types.Context) error { if ctx.BuildPath != nil { buildPath := ctx.BuildPath - preprocPath, err := buildPath.Join(constants.FOLDER_PREPROC).Abs() - if err != nil { - return errors.WithStack(err) - } sketchBuildPath, err := buildPath.Join(constants.FOLDER_SKETCH).Abs() if err != nil { return errors.WithStack(err) @@ -43,7 +39,6 @@ func (*AddAdditionalEntriesToContext) Run(ctx *types.Context) error { return errors.WithStack(err) } - ctx.PreprocPath = preprocPath ctx.SketchBuildPath = sketchBuildPath ctx.LibrariesBuildPath = librariesBuildPath ctx.CoreBuildPath = coreBuildPath diff --git a/legacy/builder/builder.go b/legacy/builder/builder.go index 36bed597616..fe11d684faa 100644 --- a/legacy/builder/builder.go +++ b/legacy/builder/builder.go @@ -20,6 +20,7 @@ import ( "time" "github.com/arduino/arduino-cli/arduino/builder" + "github.com/arduino/arduino-cli/arduino/builder/preprocessor" "github.com/arduino/arduino-cli/i18n" "github.com/arduino/arduino-cli/legacy/builder/phases" "github.com/arduino/arduino-cli/legacy/builder/types" @@ -59,7 +60,7 @@ func (s *Builder) Run(ctx *types.Context) error { &WarnAboutArchIncompatibleLibraries{}, utils.LogIfVerbose(false, tr("Generating function prototypes...")), - &PreprocessSketch{}, + types.BareCommand(PreprocessSketch), utils.LogIfVerbose(false, tr("Compiling sketch...")), &RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.sketch.prebuild", Suffix: ".pattern"}, @@ -115,13 +116,19 @@ func (s *Builder) Run(ctx *types.Context) error { return otherErr } -type PreprocessSketch struct{} - -func (s *PreprocessSketch) Run(ctx *types.Context) error { +func PreprocessSketch(ctx *types.Context) error { if ctx.UseArduinoPreprocessor { return PreprocessSketchWithArduinoPreprocessor(ctx) } else { - return PreprocessSketchWithCtags(ctx) + normalOutput, verboseOutput, err := preprocessor.PreprocessSketchWithCtags( + ctx.Sketch, ctx.BuildPath, ctx.IncludeFolders, ctx.LineOffset, + ctx.BuildProperties, ctx.OnlyUpdateCompilationDatabase) + if ctx.Verbose { + ctx.WriteStdout(verboseOutput) + } else { + ctx.WriteStdout(normalOutput) + } + return err } } @@ -149,7 +156,7 @@ func (s *Preprocess) Run(ctx *types.Context) error { &WarnAboutArchIncompatibleLibraries{}, - &PreprocessSketch{}, + types.BareCommand(PreprocessSketch), } if err := runCommands(ctx, commands); err != nil { diff --git a/legacy/builder/constants/constants.go b/legacy/builder/constants/constants.go index 36d11764ebc..a91f454ffcf 100644 --- a/legacy/builder/constants/constants.go +++ b/legacy/builder/constants/constants.go @@ -34,7 +34,6 @@ const BUILD_PROPERTIES_RUNTIME_PLATFORM_PATH = "runtime.platform.path" const EMPTY_STRING = "" const FOLDER_BOOTLOADERS = "bootloaders" const FOLDER_CORE = "core" -const FOLDER_PREPROC = "preproc" const FOLDER_SKETCH = "sketch" const FOLDER_TOOLS = "tools" const FOLDER_LIBRARIES = "libraries" diff --git a/legacy/builder/container_add_prototypes.go b/legacy/builder/container_add_prototypes.go deleted file mode 100644 index 14866f9497d..00000000000 --- a/legacy/builder/container_add_prototypes.go +++ /dev/null @@ -1,135 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package builder - -import ( - "bufio" - "bytes" - "fmt" - "io" - "strconv" - "strings" - - "github.com/arduino/arduino-cli/arduino/builder/cpp" - "github.com/arduino/arduino-cli/arduino/builder/preprocessor" - "github.com/arduino/arduino-cli/arduino/sketch" - "github.com/arduino/arduino-cli/legacy/builder/types" - "github.com/arduino/go-paths-helper" - "github.com/pkg/errors" -) - -func PreprocessSketchWithCtags(ctx *types.Context) error { - // Generate the full pathname for the preproc output file - if err := ctx.PreprocPath.MkdirAll(); err != nil { - return errors.WithStack(err) - } - targetFilePath := ctx.PreprocPath.Join("sketch_merged.cpp") - - // Run preprocessor - sourceFile := ctx.SketchBuildPath.Join(ctx.Sketch.MainFile.Base() + ".cpp") - gccStdout, gccStderr, err := preprocessor.GCC(sourceFile, targetFilePath, ctx.IncludeFolders, ctx.BuildProperties) - if ctx.Verbose { - ctx.WriteStdout(gccStdout) - ctx.WriteStderr(gccStderr) - } - if err != nil { - if !ctx.OnlyUpdateCompilationDatabase { - return errors.WithStack(err) - } - - // Do not bail out if we are generating the compile commands database - ctx.Info( - fmt.Sprintf("%s: %s", - tr("An error occurred adding prototypes"), - tr("the compilation database may be incomplete or inaccurate"))) - if err := sourceFile.CopyTo(targetFilePath); err != nil { - return errors.WithStack(err) - } - } - - if src, err := targetFilePath.ReadFile(); err != nil { - return err - } else { - filteredSource := filterSketchSource(ctx.Sketch, bytes.NewReader(src), false) - if err := targetFilePath.WriteFile([]byte(filteredSource)); err != nil { - return err - } - } - - sketchCpp := ctx.SketchBuildPath.Join(fmt.Sprintf("%s.cpp", ctx.Sketch.MainFile.Base())) - ctagsStderr, err := preprocessor.CTags(targetFilePath, sketchCpp, ctx.Sketch, ctx.LineOffset, ctx.BuildProperties) - if ctx.Verbose { - ctx.WriteStderr(ctagsStderr) - } - return err -} - -func filterSketchSource(sketch *sketch.Sketch, source io.Reader, removeLineMarkers bool) string { - fileNames := paths.NewPathList() - fileNames.Add(sketch.MainFile) - fileNames.AddAll(sketch.OtherSketchFiles) - - inSketch := false - filtered := "" - - scanner := bufio.NewScanner(source) - for scanner.Scan() { - line := scanner.Text() - if filename := parseLineMarker(line); filename != nil { - inSketch = fileNames.Contains(filename) - if inSketch && removeLineMarkers { - continue - } - } - - if inSketch { - filtered += line + "\n" - } - } - - return filtered -} - -// Parses the given line as a gcc line marker and returns the contained -// filename. -func parseLineMarker(line string) *paths.Path { - // A line marker contains the line number and filename and looks like: - // # 123 /path/to/file.cpp - // It can be followed by zero or more flag number that indicate the - // preprocessor state and can be ignored. - // For exact details on this format, see: - // https://github.com/gcc-mirror/gcc/blob/edd716b6b1caa1a5cb320a8cd7f626f30198e098/gcc/c-family/c-ppoutput.c#L413-L415 - - split := strings.SplitN(line, " ", 3) - if len(split) < 3 || len(split[0]) == 0 || split[0][0] != '#' { - return nil - } - - _, err := strconv.Atoi(split[1]) - if err != nil { - return nil - } - - // If we get here, we found a # followed by a line number, so - // assume this is a line marker and see if the rest of the line - // starts with a string containing the filename - str, rest, ok := cpp.ParseString(split[2]) - - if ok && (rest == "" || rest[0] == ' ') { - return paths.New(str) - } - return nil -} diff --git a/legacy/builder/create_cmake_rule.go b/legacy/builder/create_cmake_rule.go index 394b80ec49f..9cc38434286 100644 --- a/legacy/builder/create_cmake_rule.go +++ b/legacy/builder/create_cmake_rule.go @@ -115,7 +115,7 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error { fmt.Println(err) } - if err := PreprocessSketchWithCtags(ctx); err != nil { + if err := PreprocessSketch(ctx); err != nil { return err } diff --git a/legacy/builder/preprocess_sketch.go b/legacy/builder/preprocess_sketch.go index 8769b2da578..7dff77c7dde 100644 --- a/legacy/builder/preprocess_sketch.go +++ b/legacy/builder/preprocess_sketch.go @@ -31,12 +31,12 @@ import ( ) func PreprocessSketchWithArduinoPreprocessor(ctx *types.Context) error { - if err := ctx.PreprocPath.MkdirAll(); err != nil { + if err := ctx.BuildPath.Join("preproc").MkdirAll(); err != nil { return errors.WithStack(err) } sourceFile := ctx.SketchBuildPath.Join(ctx.Sketch.MainFile.Base() + ".cpp") - targetFile := ctx.PreprocPath.Join("sketch_merged.cpp") + targetFile := ctx.BuildPath.Join("preproc", "sketch_merged.cpp") gccStdout, gccStderr, err := preprocessor.GCC(sourceFile, targetFile, ctx.IncludeFolders, ctx.BuildProperties) if ctx.Verbose { ctx.WriteStdout(gccStdout) diff --git a/legacy/builder/test/add_additional_entries_to_context_test.go b/legacy/builder/test/add_additional_entries_to_context_test.go index 51e5452daf3..fc1edb5acb2 100644 --- a/legacy/builder/test/add_additional_entries_to_context_test.go +++ b/legacy/builder/test/add_additional_entries_to_context_test.go @@ -31,7 +31,6 @@ func TestAddAdditionalEntriesToContextNoBuildPath(t *testing.T) { command := builder.AddAdditionalEntriesToContext{} NoError(t, command.Run(ctx)) - require.Empty(t, ctx.PreprocPath) require.Empty(t, ctx.SketchBuildPath) require.Empty(t, ctx.LibrariesBuildPath) require.Empty(t, ctx.CoreBuildPath) @@ -48,7 +47,6 @@ func TestAddAdditionalEntriesToContextWithBuildPath(t *testing.T) { command := builder.AddAdditionalEntriesToContext{} NoError(t, command.Run(ctx)) - require.Equal(t, Abs(t, paths.New("folder", constants.FOLDER_PREPROC)), ctx.PreprocPath) require.Equal(t, Abs(t, paths.New("folder", constants.FOLDER_SKETCH)), ctx.SketchBuildPath) require.Equal(t, Abs(t, paths.New("folder", "libraries")), ctx.LibrariesBuildPath) require.Equal(t, Abs(t, paths.New("folder", constants.FOLDER_CORE)), ctx.CoreBuildPath) diff --git a/legacy/builder/test/builder_test.go b/legacy/builder/test/builder_test.go index e84e2d402dd..3d5740d659d 100644 --- a/legacy/builder/test/builder_test.go +++ b/legacy/builder/test/builder_test.go @@ -116,9 +116,6 @@ func TestBuilderEmptySketch(t *testing.T) { exist, err := buildPath.Join(constants.FOLDER_CORE, "HardwareSerial.cpp.o").ExistCheck() NoError(t, err) require.True(t, exist) - exist, err = buildPath.Join(constants.FOLDER_PREPROC, "sketch_merged.cpp").ExistCheck() - NoError(t, err) - require.True(t, exist) exist, err = buildPath.Join(constants.FOLDER_SKETCH, "sketch1.ino.cpp.o").ExistCheck() NoError(t, err) require.True(t, exist) @@ -143,9 +140,6 @@ func TestBuilderBridge(t *testing.T) { exist, err := buildPath.Join(constants.FOLDER_CORE, "HardwareSerial.cpp.o").ExistCheck() NoError(t, err) require.True(t, exist) - exist, err = buildPath.Join(constants.FOLDER_PREPROC, "sketch_merged.cpp").ExistCheck() - NoError(t, err) - require.True(t, exist) exist, err = buildPath.Join(constants.FOLDER_SKETCH, "Bridge.ino.cpp.o").ExistCheck() NoError(t, err) require.True(t, exist) @@ -173,9 +167,6 @@ func TestBuilderSketchWithConfig(t *testing.T) { exist, err := buildPath.Join(constants.FOLDER_CORE, "HardwareSerial.cpp.o").ExistCheck() NoError(t, err) require.True(t, exist) - exist, err = buildPath.Join(constants.FOLDER_PREPROC, "sketch_merged.cpp").ExistCheck() - NoError(t, err) - require.True(t, exist) exist, err = buildPath.Join(constants.FOLDER_SKETCH, "sketch_with_config.ino.cpp.o").ExistCheck() NoError(t, err) require.True(t, exist) @@ -208,9 +199,6 @@ func TestBuilderBridgeTwice(t *testing.T) { exist, err := buildPath.Join(constants.FOLDER_CORE, "HardwareSerial.cpp.o").ExistCheck() NoError(t, err) require.True(t, exist) - exist, err = buildPath.Join(constants.FOLDER_PREPROC, "sketch_merged.cpp").ExistCheck() - NoError(t, err) - require.True(t, exist) exist, err = buildPath.Join(constants.FOLDER_SKETCH, "Bridge.ino.cpp.o").ExistCheck() NoError(t, err) require.True(t, exist) @@ -245,9 +233,6 @@ func TestBuilderBridgeSAM(t *testing.T) { exist, err = buildPath.Join(constants.FOLDER_CORE, "avr", "dtostrf.c.d").ExistCheck() NoError(t, err) require.True(t, exist) - exist, err = buildPath.Join(constants.FOLDER_PREPROC, "sketch_merged.cpp").ExistCheck() - NoError(t, err) - require.True(t, exist) exist, err = buildPath.Join(constants.FOLDER_SKETCH, "Bridge.ino.cpp.o").ExistCheck() NoError(t, err) require.True(t, exist) @@ -286,9 +271,6 @@ func TestBuilderBridgeRedBearLab(t *testing.T) { exist, err := buildPath.Join(constants.FOLDER_CORE, "HardwareSerial.cpp.o").ExistCheck() NoError(t, err) require.True(t, exist) - exist, err = buildPath.Join(constants.FOLDER_PREPROC, "sketch_merged.cpp").ExistCheck() - NoError(t, err) - require.True(t, exist) exist, err = buildPath.Join(constants.FOLDER_SKETCH, "Bridge.ino.cpp.o").ExistCheck() NoError(t, err) require.True(t, exist) diff --git a/legacy/builder/test/prototypes_adder_test.go b/legacy/builder/test/prototypes_adder_test.go index f40e339f0f9..024d86cf265 100644 --- a/legacy/builder/test/prototypes_adder_test.go +++ b/legacy/builder/test/prototypes_adder_test.go @@ -57,7 +57,7 @@ func TestPrototypesAdderBridgeExample(t *testing.T) { err := command.Run(ctx) NoError(t, err) } - NoError(t, builder.PreprocessSketchWithCtags(ctx)) + NoError(t, builder.PreprocessSketch(ctx)) preprocessedSketch := loadPreprocessedSketch(t, ctx) require.Contains(t, preprocessedSketch, "#include \n#line 1 "+quotedSketchLocation+"\n") @@ -83,7 +83,7 @@ func TestPrototypesAdderSketchWithIfDef(t *testing.T) { err := command.Run(ctx) NoError(t, err) } - NoError(t, builder.PreprocessSketchWithCtags(ctx)) + NoError(t, builder.PreprocessSketch(ctx)) preprocessed := LoadAndInterpolate(t, filepath.Join("SketchWithIfDef", "SketchWithIfDef.preprocessed.txt"), ctx) preprocessedSketch := loadPreprocessedSketch(t, ctx) @@ -109,7 +109,7 @@ func TestPrototypesAdderBaladuino(t *testing.T) { err := command.Run(ctx) NoError(t, err) } - NoError(t, builder.PreprocessSketchWithCtags(ctx)) + NoError(t, builder.PreprocessSketch(ctx)) preprocessed := LoadAndInterpolate(t, filepath.Join("Baladuino", "Baladuino.preprocessed.txt"), ctx) preprocessedSketch := loadPreprocessedSketch(t, ctx) @@ -135,7 +135,7 @@ func TestPrototypesAdderCharWithEscapedDoubleQuote(t *testing.T) { err := command.Run(ctx) NoError(t, err) } - NoError(t, builder.PreprocessSketchWithCtags(ctx)) + NoError(t, builder.PreprocessSketch(ctx)) preprocessed := LoadAndInterpolate(t, filepath.Join("CharWithEscapedDoubleQuote", "CharWithEscapedDoubleQuote.preprocessed.txt"), ctx) preprocessedSketch := loadPreprocessedSketch(t, ctx) @@ -161,7 +161,7 @@ func TestPrototypesAdderIncludeBetweenMultilineComment(t *testing.T) { err := command.Run(ctx) NoError(t, err) } - NoError(t, builder.PreprocessSketchWithCtags(ctx)) + NoError(t, builder.PreprocessSketch(ctx)) preprocessed := LoadAndInterpolate(t, filepath.Join("IncludeBetweenMultilineComment", "IncludeBetweenMultilineComment.preprocessed.txt"), ctx) preprocessedSketch := loadPreprocessedSketch(t, ctx) @@ -187,7 +187,7 @@ func TestPrototypesAdderLineContinuations(t *testing.T) { err := command.Run(ctx) NoError(t, err) } - NoError(t, builder.PreprocessSketchWithCtags(ctx)) + NoError(t, builder.PreprocessSketch(ctx)) preprocessed := LoadAndInterpolate(t, filepath.Join("LineContinuations", "LineContinuations.preprocessed.txt"), ctx) preprocessedSketch := loadPreprocessedSketch(t, ctx) @@ -213,7 +213,7 @@ func TestPrototypesAdderStringWithComment(t *testing.T) { err := command.Run(ctx) NoError(t, err) } - NoError(t, builder.PreprocessSketchWithCtags(ctx)) + NoError(t, builder.PreprocessSketch(ctx)) preprocessed := LoadAndInterpolate(t, filepath.Join("StringWithComment", "StringWithComment.preprocessed.txt"), ctx) preprocessedSketch := loadPreprocessedSketch(t, ctx) @@ -239,7 +239,7 @@ func TestPrototypesAdderSketchWithStruct(t *testing.T) { err := command.Run(ctx) NoError(t, err) } - NoError(t, builder.PreprocessSketchWithCtags(ctx)) + NoError(t, builder.PreprocessSketch(ctx)) preprocessed := LoadAndInterpolate(t, filepath.Join("SketchWithStruct", "SketchWithStruct.preprocessed.txt"), ctx) preprocessedSketch := loadPreprocessedSketch(t, ctx) @@ -273,7 +273,7 @@ func TestPrototypesAdderSketchWithConfig(t *testing.T) { err := command.Run(ctx) NoError(t, err) } - NoError(t, builder.PreprocessSketchWithCtags(ctx)) + NoError(t, builder.PreprocessSketch(ctx)) preprocessedSketch := loadPreprocessedSketch(t, ctx) require.Contains(t, preprocessedSketch, "#include \n#line 1 "+quotedSketchLocation+"\n") @@ -306,7 +306,7 @@ func TestPrototypesAdderSketchNoFunctionsTwoFiles(t *testing.T) { NoError(t, err) } mergedSketch := loadPreprocessedSketch(t, ctx) - NoError(t, builder.PreprocessSketchWithCtags(ctx)) + NoError(t, builder.PreprocessSketch(ctx)) preprocessedSketch := loadPreprocessedSketch(t, ctx) require.Contains(t, preprocessedSketch, "#include \n#line 1 "+quotedSketchLocation+"\n") @@ -335,7 +335,7 @@ func TestPrototypesAdderSketchNoFunctions(t *testing.T) { NoError(t, err) } mergedSketch := loadPreprocessedSketch(t, ctx) - NoError(t, builder.PreprocessSketchWithCtags(ctx)) + NoError(t, builder.PreprocessSketch(ctx)) preprocessedSketch := loadPreprocessedSketch(t, ctx) require.Contains(t, preprocessedSketch, "#include \n#line 1 "+quotedSketchLocation+"\n") @@ -364,7 +364,7 @@ func TestPrototypesAdderSketchWithDefaultArgs(t *testing.T) { err := command.Run(ctx) NoError(t, err) } - NoError(t, builder.PreprocessSketchWithCtags(ctx)) + NoError(t, builder.PreprocessSketch(ctx)) preprocessedSketch := loadPreprocessedSketch(t, ctx) require.Contains(t, preprocessedSketch, "#include \n#line 1 "+quotedSketchLocation+"\n") @@ -393,7 +393,7 @@ func TestPrototypesAdderSketchWithInlineFunction(t *testing.T) { err := command.Run(ctx) NoError(t, err) } - NoError(t, builder.PreprocessSketchWithCtags(ctx)) + NoError(t, builder.PreprocessSketch(ctx)) preprocessedSketch := loadPreprocessedSketch(t, ctx) require.Contains(t, preprocessedSketch, "#include \n#line 1 "+quotedSketchLocation+"\n") @@ -433,7 +433,7 @@ func TestPrototypesAdderSketchWithFunctionSignatureInsideIFDEF(t *testing.T) { err := command.Run(ctx) NoError(t, err) } - NoError(t, builder.PreprocessSketchWithCtags(ctx)) + NoError(t, builder.PreprocessSketch(ctx)) preprocessedSketch := loadPreprocessedSketch(t, ctx) require.Contains(t, preprocessedSketch, "#include \n#line 1 "+quotedSketchLocation+"\n") @@ -467,7 +467,7 @@ func TestPrototypesAdderSketchWithUSBCON(t *testing.T) { err := command.Run(ctx) NoError(t, err) } - NoError(t, builder.PreprocessSketchWithCtags(ctx)) + NoError(t, builder.PreprocessSketch(ctx)) preprocessedSketch := loadPreprocessedSketch(t, ctx) require.Contains(t, preprocessedSketch, "#include \n#line 1 "+quotedSketchLocation+"\n") @@ -500,7 +500,7 @@ func TestPrototypesAdderSketchWithTypename(t *testing.T) { err := command.Run(ctx) NoError(t, err) } - NoError(t, builder.PreprocessSketchWithCtags(ctx)) + NoError(t, builder.PreprocessSketch(ctx)) preprocessedSketch := loadPreprocessedSketch(t, ctx) require.Contains(t, preprocessedSketch, "#include \n#line 1 "+quotedSketchLocation+"\n") @@ -535,7 +535,7 @@ func TestPrototypesAdderSketchWithIfDef2(t *testing.T) { err := command.Run(ctx) NoError(t, err) } - NoError(t, builder.PreprocessSketchWithCtags(ctx)) + NoError(t, builder.PreprocessSketch(ctx)) preprocessedSketch := loadPreprocessedSketch(t, ctx) require.Contains(t, preprocessedSketch, "#include \n#line 1 "+quotedSketchLocation+"\n") @@ -567,7 +567,7 @@ func TestPrototypesAdderSketchWithIfDef2SAM(t *testing.T) { err := command.Run(ctx) NoError(t, err) } - NoError(t, builder.PreprocessSketchWithCtags(ctx)) + NoError(t, builder.PreprocessSketch(ctx)) preprocessedSketch := loadPreprocessedSketch(t, ctx) require.Contains(t, preprocessedSketch, "#include \n#line 1 "+quotedSketchLocation+"\n") @@ -599,7 +599,7 @@ func TestPrototypesAdderSketchWithConst(t *testing.T) { err := command.Run(ctx) NoError(t, err) } - NoError(t, builder.PreprocessSketchWithCtags(ctx)) + NoError(t, builder.PreprocessSketch(ctx)) preprocessedSketch := loadPreprocessedSketch(t, ctx) require.Contains(t, preprocessedSketch, "#include \n#line 1 "+quotedSketchLocation+"\n") @@ -625,7 +625,7 @@ func TestPrototypesAdderSketchWithDosEol(t *testing.T) { err := command.Run(ctx) NoError(t, err) } - NoError(t, builder.PreprocessSketchWithCtags(ctx)) + NoError(t, builder.PreprocessSketch(ctx)) // only requires no error as result } @@ -651,7 +651,7 @@ func TestPrototypesAdderSketchWithSubstringFunctionMember(t *testing.T) { err := command.Run(ctx) NoError(t, err) } - NoError(t, builder.PreprocessSketchWithCtags(ctx)) + NoError(t, builder.PreprocessSketch(ctx)) preprocessedSketch := loadPreprocessedSketch(t, ctx) require.Contains(t, preprocessedSketch, "class Foo {\nint blooper(int x) { return x+1; }\n};\n\nFoo foo;\n\n#line 7 "+quotedSketchLocation+"\nvoid setup();") diff --git a/legacy/builder/types/context.go b/legacy/builder/types/context.go index 76533756923..8bbd05f7308 100644 --- a/legacy/builder/types/context.go +++ b/legacy/builder/types/context.go @@ -94,7 +94,6 @@ type Context struct { CoreObjectsFiles paths.PathList LibrariesBuildPath *paths.Path LibrariesObjectFiles paths.PathList - PreprocPath *paths.Path SketchObjectFiles paths.PathList IgnoreSketchFolderNameErrors bool From f043154eafda056e95b22f122952693b04efd445 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Wed, 26 Apr 2023 18:37:23 +0200 Subject: [PATCH 40/42] Renamed variable for lint checks --- arduino/builder/preprocessor/gcc.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/arduino/builder/preprocessor/gcc.go b/arduino/builder/preprocessor/gcc.go index 600e677834e..bb34983a336 100644 --- a/arduino/builder/preprocessor/gcc.go +++ b/arduino/builder/preprocessor/gcc.go @@ -39,8 +39,8 @@ func GCC(sourceFilePath *paths.Path, targetFilePath *paths.Path, includes paths. includesStrings := f.Map(includes.AsStrings(), utils.WrapWithHyphenI) gccBuildProperties.Set("includes", strings.Join(includesStrings, " ")) - const GCC_PATTERN_PROPERTY = "recipe.preproc.macros" - if gccBuildProperties.Get(GCC_PATTERN_PROPERTY) == "" { + const gccPreprocRecipeProperty = "recipe.preproc.macros" + if gccBuildProperties.Get(gccPreprocRecipeProperty) == "" { // autogenerate preprocess macros recipe from compile recipe preprocPattern := gccBuildProperties.Get("recipe.cpp.o.pattern") // add {preproc.macros.flags} to {compiler.cpp.flags} @@ -48,12 +48,12 @@ func GCC(sourceFilePath *paths.Path, targetFilePath *paths.Path, includes paths. // replace "{object_file}" with "{preprocessed_file_path}" preprocPattern = strings.Replace(preprocPattern, "{object_file}", "{preprocessed_file_path}", 1) - gccBuildProperties.Set(GCC_PATTERN_PROPERTY, preprocPattern) + gccBuildProperties.Set(gccPreprocRecipeProperty, preprocPattern) } - pattern := gccBuildProperties.Get(GCC_PATTERN_PROPERTY) + pattern := gccBuildProperties.Get(gccPreprocRecipeProperty) if pattern == "" { - return nil, nil, errors.Errorf(tr("%s pattern is missing"), GCC_PATTERN_PROPERTY) + return nil, nil, errors.Errorf(tr("%s pattern is missing"), gccPreprocRecipeProperty) } commandLine := gccBuildProperties.ExpandPropsInString(pattern) From fef8d6784570fd9a7cfa5334991e72c2d259462d Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Wed, 26 Apr 2023 18:46:05 +0200 Subject: [PATCH 41/42] Moved ctags parser shenanings into internal package --- arduino/builder/preprocessor/ctags.go | 2 +- .../preprocessor/{ => internal}/ctags/ctags_has_issues.go | 0 .../builder/preprocessor/{ => internal}/ctags/ctags_parser.go | 0 .../preprocessor/{ => internal}/ctags/ctags_parser_test.go | 0 .../preprocessor/{ => internal}/ctags/ctags_to_prototypes.go | 0 .../{ => internal}/ctags/ctags_to_prototypes_test.go | 0 .../testdata/TestCTagsParserClassMembersAreFilteredOut.txt | 0 .../ctags/testdata/TestCTagsParserDefaultArguments.txt | 0 .../ctags/testdata/TestCTagsParserFunctionPointer.txt | 0 .../ctags/testdata/TestCTagsParserFunctionPointers.txt | 0 .../testdata/TestCTagsParserFunctionPointersNoIndirect.txt | 0 .../{ => internal}/ctags/testdata/TestCTagsParserNamespace.txt | 0 ...TestCTagsParserShouldDealFunctionWithDifferentSignatures.txt | 0 .../ctags/testdata/TestCTagsParserShouldDealWithClasses.txt | 0 .../ctags/testdata/TestCTagsParserShouldDealWithMacros.txt | 0 .../ctags/testdata/TestCTagsParserShouldDealWithStructs.txt | 0 .../ctags/testdata/TestCTagsParserShouldListPrototypes.txt | 0 .../ctags/testdata/TestCTagsParserShouldListTemplates.txt | 0 .../ctags/testdata/TestCTagsParserShouldListTemplates2.txt | 0 .../{ => internal}/ctags/testdata/TestCTagsParserStatic.txt | 0 .../ctags/testdata/TestCTagsParserStructWithFunctions.txt | 0 .../ctags/testdata/TestCTagsRunnerSketchWithClassFunction.txt | 0 .../ctags/testdata/TestCTagsRunnerSketchWithMultifile.txt | 0 23 files changed, 1 insertion(+), 1 deletion(-) rename arduino/builder/preprocessor/{ => internal}/ctags/ctags_has_issues.go (100%) rename arduino/builder/preprocessor/{ => internal}/ctags/ctags_parser.go (100%) rename arduino/builder/preprocessor/{ => internal}/ctags/ctags_parser_test.go (100%) rename arduino/builder/preprocessor/{ => internal}/ctags/ctags_to_prototypes.go (100%) rename arduino/builder/preprocessor/{ => internal}/ctags/ctags_to_prototypes_test.go (100%) rename arduino/builder/preprocessor/{ => internal}/ctags/testdata/TestCTagsParserClassMembersAreFilteredOut.txt (100%) rename arduino/builder/preprocessor/{ => internal}/ctags/testdata/TestCTagsParserDefaultArguments.txt (100%) rename arduino/builder/preprocessor/{ => internal}/ctags/testdata/TestCTagsParserFunctionPointer.txt (100%) rename arduino/builder/preprocessor/{ => internal}/ctags/testdata/TestCTagsParserFunctionPointers.txt (100%) rename arduino/builder/preprocessor/{ => internal}/ctags/testdata/TestCTagsParserFunctionPointersNoIndirect.txt (100%) rename arduino/builder/preprocessor/{ => internal}/ctags/testdata/TestCTagsParserNamespace.txt (100%) rename arduino/builder/preprocessor/{ => internal}/ctags/testdata/TestCTagsParserShouldDealFunctionWithDifferentSignatures.txt (100%) rename arduino/builder/preprocessor/{ => internal}/ctags/testdata/TestCTagsParserShouldDealWithClasses.txt (100%) rename arduino/builder/preprocessor/{ => internal}/ctags/testdata/TestCTagsParserShouldDealWithMacros.txt (100%) rename arduino/builder/preprocessor/{ => internal}/ctags/testdata/TestCTagsParserShouldDealWithStructs.txt (100%) rename arduino/builder/preprocessor/{ => internal}/ctags/testdata/TestCTagsParserShouldListPrototypes.txt (100%) rename arduino/builder/preprocessor/{ => internal}/ctags/testdata/TestCTagsParserShouldListTemplates.txt (100%) rename arduino/builder/preprocessor/{ => internal}/ctags/testdata/TestCTagsParserShouldListTemplates2.txt (100%) rename arduino/builder/preprocessor/{ => internal}/ctags/testdata/TestCTagsParserStatic.txt (100%) rename arduino/builder/preprocessor/{ => internal}/ctags/testdata/TestCTagsParserStructWithFunctions.txt (100%) rename arduino/builder/preprocessor/{ => internal}/ctags/testdata/TestCTagsRunnerSketchWithClassFunction.txt (100%) rename arduino/builder/preprocessor/{ => internal}/ctags/testdata/TestCTagsRunnerSketchWithMultifile.txt (100%) diff --git a/arduino/builder/preprocessor/ctags.go b/arduino/builder/preprocessor/ctags.go index 4d8a5a384b5..c7ed9e2ed92 100644 --- a/arduino/builder/preprocessor/ctags.go +++ b/arduino/builder/preprocessor/ctags.go @@ -25,7 +25,7 @@ import ( "strings" "github.com/arduino/arduino-cli/arduino/builder/cpp" - "github.com/arduino/arduino-cli/arduino/builder/preprocessor/ctags" + "github.com/arduino/arduino-cli/arduino/builder/preprocessor/internal/ctags" "github.com/arduino/arduino-cli/arduino/sketch" "github.com/arduino/arduino-cli/executils" "github.com/arduino/arduino-cli/i18n" diff --git a/arduino/builder/preprocessor/ctags/ctags_has_issues.go b/arduino/builder/preprocessor/internal/ctags/ctags_has_issues.go similarity index 100% rename from arduino/builder/preprocessor/ctags/ctags_has_issues.go rename to arduino/builder/preprocessor/internal/ctags/ctags_has_issues.go diff --git a/arduino/builder/preprocessor/ctags/ctags_parser.go b/arduino/builder/preprocessor/internal/ctags/ctags_parser.go similarity index 100% rename from arduino/builder/preprocessor/ctags/ctags_parser.go rename to arduino/builder/preprocessor/internal/ctags/ctags_parser.go diff --git a/arduino/builder/preprocessor/ctags/ctags_parser_test.go b/arduino/builder/preprocessor/internal/ctags/ctags_parser_test.go similarity index 100% rename from arduino/builder/preprocessor/ctags/ctags_parser_test.go rename to arduino/builder/preprocessor/internal/ctags/ctags_parser_test.go diff --git a/arduino/builder/preprocessor/ctags/ctags_to_prototypes.go b/arduino/builder/preprocessor/internal/ctags/ctags_to_prototypes.go similarity index 100% rename from arduino/builder/preprocessor/ctags/ctags_to_prototypes.go rename to arduino/builder/preprocessor/internal/ctags/ctags_to_prototypes.go diff --git a/arduino/builder/preprocessor/ctags/ctags_to_prototypes_test.go b/arduino/builder/preprocessor/internal/ctags/ctags_to_prototypes_test.go similarity index 100% rename from arduino/builder/preprocessor/ctags/ctags_to_prototypes_test.go rename to arduino/builder/preprocessor/internal/ctags/ctags_to_prototypes_test.go diff --git a/arduino/builder/preprocessor/ctags/testdata/TestCTagsParserClassMembersAreFilteredOut.txt b/arduino/builder/preprocessor/internal/ctags/testdata/TestCTagsParserClassMembersAreFilteredOut.txt similarity index 100% rename from arduino/builder/preprocessor/ctags/testdata/TestCTagsParserClassMembersAreFilteredOut.txt rename to arduino/builder/preprocessor/internal/ctags/testdata/TestCTagsParserClassMembersAreFilteredOut.txt diff --git a/arduino/builder/preprocessor/ctags/testdata/TestCTagsParserDefaultArguments.txt b/arduino/builder/preprocessor/internal/ctags/testdata/TestCTagsParserDefaultArguments.txt similarity index 100% rename from arduino/builder/preprocessor/ctags/testdata/TestCTagsParserDefaultArguments.txt rename to arduino/builder/preprocessor/internal/ctags/testdata/TestCTagsParserDefaultArguments.txt diff --git a/arduino/builder/preprocessor/ctags/testdata/TestCTagsParserFunctionPointer.txt b/arduino/builder/preprocessor/internal/ctags/testdata/TestCTagsParserFunctionPointer.txt similarity index 100% rename from arduino/builder/preprocessor/ctags/testdata/TestCTagsParserFunctionPointer.txt rename to arduino/builder/preprocessor/internal/ctags/testdata/TestCTagsParserFunctionPointer.txt diff --git a/arduino/builder/preprocessor/ctags/testdata/TestCTagsParserFunctionPointers.txt b/arduino/builder/preprocessor/internal/ctags/testdata/TestCTagsParserFunctionPointers.txt similarity index 100% rename from arduino/builder/preprocessor/ctags/testdata/TestCTagsParserFunctionPointers.txt rename to arduino/builder/preprocessor/internal/ctags/testdata/TestCTagsParserFunctionPointers.txt diff --git a/arduino/builder/preprocessor/ctags/testdata/TestCTagsParserFunctionPointersNoIndirect.txt b/arduino/builder/preprocessor/internal/ctags/testdata/TestCTagsParserFunctionPointersNoIndirect.txt similarity index 100% rename from arduino/builder/preprocessor/ctags/testdata/TestCTagsParserFunctionPointersNoIndirect.txt rename to arduino/builder/preprocessor/internal/ctags/testdata/TestCTagsParserFunctionPointersNoIndirect.txt diff --git a/arduino/builder/preprocessor/ctags/testdata/TestCTagsParserNamespace.txt b/arduino/builder/preprocessor/internal/ctags/testdata/TestCTagsParserNamespace.txt similarity index 100% rename from arduino/builder/preprocessor/ctags/testdata/TestCTagsParserNamespace.txt rename to arduino/builder/preprocessor/internal/ctags/testdata/TestCTagsParserNamespace.txt diff --git a/arduino/builder/preprocessor/ctags/testdata/TestCTagsParserShouldDealFunctionWithDifferentSignatures.txt b/arduino/builder/preprocessor/internal/ctags/testdata/TestCTagsParserShouldDealFunctionWithDifferentSignatures.txt similarity index 100% rename from arduino/builder/preprocessor/ctags/testdata/TestCTagsParserShouldDealFunctionWithDifferentSignatures.txt rename to arduino/builder/preprocessor/internal/ctags/testdata/TestCTagsParserShouldDealFunctionWithDifferentSignatures.txt diff --git a/arduino/builder/preprocessor/ctags/testdata/TestCTagsParserShouldDealWithClasses.txt b/arduino/builder/preprocessor/internal/ctags/testdata/TestCTagsParserShouldDealWithClasses.txt similarity index 100% rename from arduino/builder/preprocessor/ctags/testdata/TestCTagsParserShouldDealWithClasses.txt rename to arduino/builder/preprocessor/internal/ctags/testdata/TestCTagsParserShouldDealWithClasses.txt diff --git a/arduino/builder/preprocessor/ctags/testdata/TestCTagsParserShouldDealWithMacros.txt b/arduino/builder/preprocessor/internal/ctags/testdata/TestCTagsParserShouldDealWithMacros.txt similarity index 100% rename from arduino/builder/preprocessor/ctags/testdata/TestCTagsParserShouldDealWithMacros.txt rename to arduino/builder/preprocessor/internal/ctags/testdata/TestCTagsParserShouldDealWithMacros.txt diff --git a/arduino/builder/preprocessor/ctags/testdata/TestCTagsParserShouldDealWithStructs.txt b/arduino/builder/preprocessor/internal/ctags/testdata/TestCTagsParserShouldDealWithStructs.txt similarity index 100% rename from arduino/builder/preprocessor/ctags/testdata/TestCTagsParserShouldDealWithStructs.txt rename to arduino/builder/preprocessor/internal/ctags/testdata/TestCTagsParserShouldDealWithStructs.txt diff --git a/arduino/builder/preprocessor/ctags/testdata/TestCTagsParserShouldListPrototypes.txt b/arduino/builder/preprocessor/internal/ctags/testdata/TestCTagsParserShouldListPrototypes.txt similarity index 100% rename from arduino/builder/preprocessor/ctags/testdata/TestCTagsParserShouldListPrototypes.txt rename to arduino/builder/preprocessor/internal/ctags/testdata/TestCTagsParserShouldListPrototypes.txt diff --git a/arduino/builder/preprocessor/ctags/testdata/TestCTagsParserShouldListTemplates.txt b/arduino/builder/preprocessor/internal/ctags/testdata/TestCTagsParserShouldListTemplates.txt similarity index 100% rename from arduino/builder/preprocessor/ctags/testdata/TestCTagsParserShouldListTemplates.txt rename to arduino/builder/preprocessor/internal/ctags/testdata/TestCTagsParserShouldListTemplates.txt diff --git a/arduino/builder/preprocessor/ctags/testdata/TestCTagsParserShouldListTemplates2.txt b/arduino/builder/preprocessor/internal/ctags/testdata/TestCTagsParserShouldListTemplates2.txt similarity index 100% rename from arduino/builder/preprocessor/ctags/testdata/TestCTagsParserShouldListTemplates2.txt rename to arduino/builder/preprocessor/internal/ctags/testdata/TestCTagsParserShouldListTemplates2.txt diff --git a/arduino/builder/preprocessor/ctags/testdata/TestCTagsParserStatic.txt b/arduino/builder/preprocessor/internal/ctags/testdata/TestCTagsParserStatic.txt similarity index 100% rename from arduino/builder/preprocessor/ctags/testdata/TestCTagsParserStatic.txt rename to arduino/builder/preprocessor/internal/ctags/testdata/TestCTagsParserStatic.txt diff --git a/arduino/builder/preprocessor/ctags/testdata/TestCTagsParserStructWithFunctions.txt b/arduino/builder/preprocessor/internal/ctags/testdata/TestCTagsParserStructWithFunctions.txt similarity index 100% rename from arduino/builder/preprocessor/ctags/testdata/TestCTagsParserStructWithFunctions.txt rename to arduino/builder/preprocessor/internal/ctags/testdata/TestCTagsParserStructWithFunctions.txt diff --git a/arduino/builder/preprocessor/ctags/testdata/TestCTagsRunnerSketchWithClassFunction.txt b/arduino/builder/preprocessor/internal/ctags/testdata/TestCTagsRunnerSketchWithClassFunction.txt similarity index 100% rename from arduino/builder/preprocessor/ctags/testdata/TestCTagsRunnerSketchWithClassFunction.txt rename to arduino/builder/preprocessor/internal/ctags/testdata/TestCTagsRunnerSketchWithClassFunction.txt diff --git a/arduino/builder/preprocessor/ctags/testdata/TestCTagsRunnerSketchWithMultifile.txt b/arduino/builder/preprocessor/internal/ctags/testdata/TestCTagsRunnerSketchWithMultifile.txt similarity index 100% rename from arduino/builder/preprocessor/ctags/testdata/TestCTagsRunnerSketchWithMultifile.txt rename to arduino/builder/preprocessor/internal/ctags/testdata/TestCTagsRunnerSketchWithMultifile.txt From 651b3716f372c6a398f94feec91300be524e2fac Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Wed, 26 Apr 2023 19:03:05 +0200 Subject: [PATCH 42/42] Fixed linter warnings --- arduino/builder/preprocessor/ctags.go | 3 +- arduino/builder/preprocessor/gcc.go | 2 + .../internal/ctags/ctags_has_issues.go | 25 ++++---- .../internal/ctags/ctags_parser.go | 57 ++++++++++--------- .../internal/ctags/ctags_parser_test.go | 4 +- .../internal/ctags/ctags_to_prototypes.go | 21 ++++--- .../ctags/ctags_to_prototypes_test.go | 2 +- 7 files changed, 59 insertions(+), 55 deletions(-) diff --git a/arduino/builder/preprocessor/ctags.go b/arduino/builder/preprocessor/ctags.go index c7ed9e2ed92..bf5f25b4ca2 100644 --- a/arduino/builder/preprocessor/ctags.go +++ b/arduino/builder/preprocessor/ctags.go @@ -90,7 +90,7 @@ func PreprocessSketchWithCtags(sketch *sketch.Sketch, buildPath *paths.Path, inc } // Parse CTags output - parser := &ctags.CTagsParser{} + parser := &ctags.Parser{} prototypes, firstFunctionLine := parser.Parse(ctagsOutput, sketch.MainFile) if firstFunctionLine == -1 { firstFunctionLine = 0 @@ -175,6 +175,7 @@ func isFirstFunctionOutsideOfSource(firstFunctionLine int, sourceRows []string) return firstFunctionLine > len(sourceRows)-1 } +// RunCTags performs a run of ctags on the given source file. Returns the ctags output and the stderr contents. func RunCTags(sourceFile *paths.Path, buildProperties *properties.Map) ([]byte, []byte, error) { ctagsBuildProperties := properties.NewMap() ctagsBuildProperties.Set("tools.ctags.path", "{runtime.tools.ctags.path}") diff --git a/arduino/builder/preprocessor/gcc.go b/arduino/builder/preprocessor/gcc.go index bb34983a336..02fb62f47e1 100644 --- a/arduino/builder/preprocessor/gcc.go +++ b/arduino/builder/preprocessor/gcc.go @@ -28,6 +28,8 @@ import ( "github.com/pkg/errors" ) +// GCC performs a run of the gcc preprocess (macro/includes expansion). The function output the result +// to targetFilePath. Returns the stdout/stderr of gcc if any. func GCC(sourceFilePath *paths.Path, targetFilePath *paths.Path, includes paths.PathList, buildProperties *properties.Map) ([]byte, []byte, error) { gccBuildProperties := properties.NewMap() gccBuildProperties.Set("preproc.macros.flags", "-w -x c++ -E -CC") diff --git a/arduino/builder/preprocessor/internal/ctags/ctags_has_issues.go b/arduino/builder/preprocessor/internal/ctags/ctags_has_issues.go index a61bf6c6c46..4d4d7d48f65 100644 --- a/arduino/builder/preprocessor/internal/ctags/ctags_has_issues.go +++ b/arduino/builder/preprocessor/internal/ctags/ctags_has_issues.go @@ -23,17 +23,17 @@ import ( "golang.org/x/exp/slices" ) -func (p *CTagsParser) fixCLinkageTagsDeclarations() { +func (p *Parser) fixCLinkageTagsDeclarations() { linesMap := p.FindCLinkageLines(p.tags) for i := range p.tags { if slices.Contains(linesMap[p.tags[i].Filename], p.tags[i].Line) && - !strings.Contains(p.tags[i].PrototypeModifiers, EXTERN) { - p.tags[i].PrototypeModifiers = p.tags[i].PrototypeModifiers + " " + EXTERN + !strings.Contains(p.tags[i].PrototypeModifiers, keywordExternC) { + p.tags[i].PrototypeModifiers = p.tags[i].PrototypeModifiers + " " + keywordExternC } } } -func (p *CTagsParser) prototypeAndCodeDontMatch(tag *CTag) bool { +func (p *Parser) prototypeAndCodeDontMatch(tag *Tag) bool { if tag.SkipMe { return true } @@ -98,7 +98,7 @@ func (p *CTagsParser) prototypeAndCodeDontMatch(tag *CTag) bool { return ret == -1 } -func findTemplateMultiline(tag *CTag) string { +func findTemplateMultiline(tag *Tag) string { code, _ := getFunctionProtoUntilTemplateToken(tag, tag.Code) return removeEverythingAfterClosingRoundBracket(code) } @@ -108,7 +108,7 @@ func removeEverythingAfterClosingRoundBracket(s string) string { return s[0 : n+1] } -func getFunctionProtoUntilTemplateToken(tag *CTag, code string) (string, int) { +func getFunctionProtoUntilTemplateToken(tag *Tag, code string) (string, int) { /* FIXME I'm ugly */ line := 0 @@ -128,7 +128,7 @@ func getFunctionProtoUntilTemplateToken(tag *CTag, code string) (string, int) { textBuffer = append(textBuffer, text) } - for line > 0 && !strings.Contains(code, TEMPLATE) { + for line > 0 && !strings.Contains(code, keywordTemplate) { line = line - 1 text := textBuffer[line] @@ -141,7 +141,7 @@ func getFunctionProtoUntilTemplateToken(tag *CTag, code string) (string, int) { return code, line } -func getFunctionProtoWithNPreviousCharacters(tag *CTag, code string, n int) (string, int) { +func getFunctionProtoWithNPreviousCharacters(tag *Tag, code string, n int) (string, int) { /* FIXME I'm ugly */ expectedPrototypeLen := len(code) + n @@ -204,10 +204,9 @@ func removeComments(text string, multilinecomment bool) (string, bool) { return text, multilinecomment } -/* This function scans the source files searching for "extern C" context - * It save the line numbers in a map filename -> {lines...} - */ -func (p *CTagsParser) FindCLinkageLines(tags []*CTag) map[string][]int { +// FindCLinkageLines scans the source files searching for "extern C" context +// It save the line numbers in a map filename -> {lines...} +func (p *Parser) FindCLinkageLines(tags []*Tag) map[string][]int { lines := make(map[string][]int) @@ -245,7 +244,7 @@ func (p *CTagsParser) FindCLinkageLines(tags []*CTag) map[string][]int { indentLevels := 0 line := 0 - externCDecl := removeSpacesAndTabs(EXTERN) + externCDecl := removeSpacesAndTabs(keywordExternC) for scanner.Scan() { line++ diff --git a/arduino/builder/preprocessor/internal/ctags/ctags_parser.go b/arduino/builder/preprocessor/internal/ctags/ctags_parser.go index 5836f7bf337..c2e6e99585f 100644 --- a/arduino/builder/preprocessor/internal/ctags/ctags_parser.go +++ b/arduino/builder/preprocessor/internal/ctags/ctags_parser.go @@ -22,26 +22,28 @@ import ( "github.com/arduino/go-paths-helper" ) -const KIND_PROTOTYPE = "prototype" -const KIND_FUNCTION = "function" +const kindPrototype = "prototype" +const kindFunction = "function" //const KIND_PROTOTYPE_MODIFIERS = "prototype_modifiers" -const TEMPLATE = "template" -const STATIC = "static" -const EXTERN = "extern \"C\"" +const keywordTemplate = "template" +const keywordStatic = "static" +const keywordExternC = "extern \"C\"" -var KNOWN_TAG_KINDS = map[string]bool{ +var knownTagKinds = map[string]bool{ "prototype": true, "function": true, } -type CTagsParser struct { - tags []*CTag +// Parser is a parser for ctags output +type Parser struct { + tags []*Tag mainFile *paths.Path } -type CTag struct { +// Tag is a tag generated by ctags +type Tag struct { FunctionName string Kind string Line int @@ -58,7 +60,8 @@ type CTag struct { PrototypeModifiers string } -func (p *CTagsParser) Parse(ctagsOutput []byte, mainFile *paths.Path) ([]*Prototype, int) { +// Parse a ctags output and generates Prototypes +func (p *Parser) Parse(ctagsOutput []byte, mainFile *paths.Path) ([]*Prototype, int) { rows := strings.Split(string(ctagsOutput), "\n") rows = removeEmpty(rows) @@ -79,7 +82,7 @@ func (p *CTagsParser) Parse(ctagsOutput []byte, mainFile *paths.Path) ([]*Protot return p.toPrototypes(), p.findLineWhereToInsertPrototypes() } -func (p *CTagsParser) addPrototypes() { +func (p *Parser) addPrototypes() { for _, tag := range p.tags { if !tag.SkipMe { addPrototype(tag) @@ -87,9 +90,9 @@ func (p *CTagsParser) addPrototypes() { } } -func addPrototype(tag *CTag) { - if strings.Index(tag.Prototype, TEMPLATE) == 0 { - if strings.Index(tag.Code, TEMPLATE) == 0 { +func addPrototype(tag *Tag) { + if strings.Index(tag.Prototype, keywordTemplate) == 0 { + if strings.Index(tag.Code, keywordTemplate) == 0 { code := tag.Code if strings.Contains(code, "{") { code = code[:strings.Index(code, "{")] @@ -106,8 +109,8 @@ func addPrototype(tag *CTag) { } tag.PrototypeModifiers = "" - if strings.Contains(tag.Code, STATIC+" ") { - tag.PrototypeModifiers = tag.PrototypeModifiers + " " + STATIC + if strings.Contains(tag.Code, keywordStatic+" ") { + tag.PrototypeModifiers = tag.PrototypeModifiers + " " + keywordStatic } // Extern "C" modifier is now added in FixCLinkageTagsDeclarations @@ -115,10 +118,10 @@ func addPrototype(tag *CTag) { tag.PrototypeModifiers = strings.TrimSpace(tag.PrototypeModifiers) } -func (p *CTagsParser) removeDefinedProtypes() { +func (p *Parser) removeDefinedProtypes() { definedPrototypes := make(map[string]bool) for _, tag := range p.tags { - if tag.Kind == KIND_PROTOTYPE { + if tag.Kind == kindPrototype { definedPrototypes[tag.Prototype] = true } } @@ -133,7 +136,7 @@ func (p *CTagsParser) removeDefinedProtypes() { } } -func (p *CTagsParser) skipDuplicates() { +func (p *Parser) skipDuplicates() { definedPrototypes := make(map[string]bool) for _, tag := range p.tags { @@ -145,9 +148,9 @@ func (p *CTagsParser) skipDuplicates() { } } -type skipFuncType func(tag *CTag) bool +type skipFuncType func(tag *Tag) bool -func (p *CTagsParser) skipTagsWhere(skipFunc skipFuncType) { +func (p *Parser) skipTagsWhere(skipFunc skipFuncType) { for _, tag := range p.tags { if !tag.SkipMe { skip := skipFunc(tag) @@ -169,11 +172,11 @@ func removeSpacesAndTabs(s string) string { return s } -func tagIsUnhandled(tag *CTag) bool { +func tagIsUnhandled(tag *Tag) bool { return !isHandled(tag) } -func isHandled(tag *CTag) bool { +func isHandled(tag *Tag) bool { if tag.Class != "" { return false } @@ -186,12 +189,12 @@ func isHandled(tag *CTag) bool { return true } -func tagIsUnknown(tag *CTag) bool { - return !KNOWN_TAG_KINDS[tag.Kind] +func tagIsUnknown(tag *Tag) bool { + return !knownTagKinds[tag.Kind] } -func parseTag(row string) *CTag { - tag := &CTag{} +func parseTag(row string) *Tag { + tag := &Tag{} parts := strings.Split(row, "\t") tag.FunctionName = parts[0] diff --git a/arduino/builder/preprocessor/internal/ctags/ctags_parser_test.go b/arduino/builder/preprocessor/internal/ctags/ctags_parser_test.go index dd6ef465440..7fc390cec67 100644 --- a/arduino/builder/preprocessor/internal/ctags/ctags_parser_test.go +++ b/arduino/builder/preprocessor/internal/ctags/ctags_parser_test.go @@ -24,11 +24,11 @@ import ( "github.com/stretchr/testify/require" ) -func produceTags(t *testing.T, filename string) []*CTag { +func produceTags(t *testing.T, filename string) []*Tag { bytes, err := os.ReadFile(filepath.Join("testdata", filename)) require.NoError(t, err) - parser := CTagsParser{} + parser := Parser{} parser.Parse(bytes, paths.New("sketch.ino")) return parser.tags } diff --git a/arduino/builder/preprocessor/internal/ctags/ctags_to_prototypes.go b/arduino/builder/preprocessor/internal/ctags/ctags_to_prototypes.go index 8f5e5720aa0..95fc384f116 100644 --- a/arduino/builder/preprocessor/internal/ctags/ctags_to_prototypes.go +++ b/arduino/builder/preprocessor/internal/ctags/ctags_to_prototypes.go @@ -33,15 +33,14 @@ func (proto *Prototype) String() string { return proto.Modifiers + " " + proto.Prototype + " @ " + strconv.Itoa(proto.Line) } -func (p *CTagsParser) findLineWhereToInsertPrototypes() int { +func (p *Parser) findLineWhereToInsertPrototypes() int { firstFunctionLine := p.firstFunctionAtLine() firstFunctionPointerAsArgument := p.firstFunctionPointerUsedAsArgument() if firstFunctionLine != -1 && firstFunctionPointerAsArgument != -1 { if firstFunctionLine < firstFunctionPointerAsArgument { return firstFunctionLine - } else { - return firstFunctionPointerAsArgument } + return firstFunctionPointerAsArgument } else if firstFunctionLine != -1 { return firstFunctionLine } else if firstFunctionPointerAsArgument != -1 { @@ -51,7 +50,7 @@ func (p *CTagsParser) findLineWhereToInsertPrototypes() int { } } -func (p *CTagsParser) firstFunctionPointerUsedAsArgument() int { +func (p *Parser) firstFunctionPointerUsedAsArgument() int { functionTags := p.collectFunctions() for _, tag := range p.tags { if functionNameUsedAsFunctionPointerIn(tag, functionTags) { @@ -61,7 +60,7 @@ func (p *CTagsParser) firstFunctionPointerUsedAsArgument() int { return -1 } -func functionNameUsedAsFunctionPointerIn(tag *CTag, functionTags []*CTag) bool { +func functionNameUsedAsFunctionPointerIn(tag *Tag, functionTags []*Tag) bool { for _, functionTag := range functionTags { if tag.Line != functionTag.Line && strings.Contains(tag.Code, "&"+functionTag.FunctionName) { return true @@ -73,26 +72,26 @@ func functionNameUsedAsFunctionPointerIn(tag *CTag, functionTags []*CTag) bool { return false } -func (p *CTagsParser) collectFunctions() []*CTag { - functionTags := []*CTag{} +func (p *Parser) collectFunctions() []*Tag { + functionTags := []*Tag{} for _, tag := range p.tags { - if tag.Kind == KIND_FUNCTION && !tag.SkipMe { + if tag.Kind == kindFunction && !tag.SkipMe { functionTags = append(functionTags, tag) } } return functionTags } -func (p *CTagsParser) firstFunctionAtLine() int { +func (p *Parser) firstFunctionAtLine() int { for _, tag := range p.tags { - if !tagIsUnknown(tag) && isHandled(tag) && tag.Kind == KIND_FUNCTION && tag.Filename == p.mainFile.String() { + if !tagIsUnknown(tag) && isHandled(tag) && tag.Kind == kindFunction && tag.Filename == p.mainFile.String() { return tag.Line } } return -1 } -func (p *CTagsParser) toPrototypes() []*Prototype { +func (p *Parser) toPrototypes() []*Prototype { prototypes := []*Prototype{} for _, tag := range p.tags { if strings.TrimSpace(tag.Prototype) == "" { diff --git a/arduino/builder/preprocessor/internal/ctags/ctags_to_prototypes_test.go b/arduino/builder/preprocessor/internal/ctags/ctags_to_prototypes_test.go index 74bd8e2756a..bfec902d8d9 100644 --- a/arduino/builder/preprocessor/internal/ctags/ctags_to_prototypes_test.go +++ b/arduino/builder/preprocessor/internal/ctags/ctags_to_prototypes_test.go @@ -28,7 +28,7 @@ func producePrototypes(t *testing.T, filename string, mainFile string) ([]*Proto bytes, err := os.ReadFile(filepath.Join("testdata", filename)) require.NoError(t, err) - parser := &CTagsParser{} + parser := &Parser{} return parser.Parse(bytes, paths.New(mainFile)) }