[skip-changelog] legacy: Arduino preprocess subroutine refactorization (part 4) (#2194)

* Simplified GCCPreprocRunner function

* Renamed variable

* Removed dependency on builder ctx in GCCPreprocRunner

* Moved GCCPreprocRunner into proper place

* Replaced algorithm with stdlib equivalent

* Reduced public API surface of CTags parser

* Renamed variable for lint checks

Author: cmaglie

Diff for: arduino/builder/preprocessor/gcc.go

@@ -0,0 +1,81 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2023 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to [email protected].
+
+package preprocessor
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/arduino/arduino-cli/executils"
+	f "github.com/arduino/arduino-cli/internal/algorithms"
+	"github.com/arduino/arduino-cli/legacy/builder/utils"
+	"github.com/arduino/go-paths-helper"
+	"github.com/arduino/go-properties-orderedmap"
+	"github.com/pkg/errors"
+)
+
+// GCC performs a run of the gcc preprocess (macro/includes expansion). The function outputs the result
+// to targetFilePath. Returns the stdout/stderr of gcc if any.
+func GCC(sourceFilePath *paths.Path, targetFilePath *paths.Path, includes paths.PathList, buildProperties *properties.Map) ([]byte, []byte, error) {
+	gccBuildProperties := properties.NewMap()
+	gccBuildProperties.Set("preproc.macros.flags", "-w -x c++ -E -CC")
+	gccBuildProperties.Merge(buildProperties)
+	gccBuildProperties.Set("build.library_discovery_phase", "1")
+	gccBuildProperties.SetPath("source_file", sourceFilePath)
+	gccBuildProperties.SetPath("preprocessed_file_path", targetFilePath)
+
+	includesStrings := f.Map(includes.AsStrings(), utils.WrapWithHyphenI)
+	gccBuildProperties.Set("includes", strings.Join(includesStrings, " "))
+
+	const gccPreprocRecipeProperty = "recipe.preproc.macros"
+	if gccBuildProperties.Get(gccPreprocRecipeProperty) == "" {
+		// autogenerate preprocess macros recipe from compile recipe
+		preprocPattern := gccBuildProperties.Get("recipe.cpp.o.pattern")
+		// add {preproc.macros.flags} to {compiler.cpp.flags}
+		preprocPattern = strings.Replace(preprocPattern, "{compiler.cpp.flags}", "{compiler.cpp.flags} {preproc.macros.flags}", 1)
+		// replace "{object_file}" with "{preprocessed_file_path}"
+		preprocPattern = strings.Replace(preprocPattern, "{object_file}", "{preprocessed_file_path}", 1)
+
+		gccBuildProperties.Set(gccPreprocRecipeProperty, preprocPattern)
+	}
+
+	pattern := gccBuildProperties.Get(gccPreprocRecipeProperty)
+	if pattern == "" {
+		return nil, nil, errors.Errorf(tr("%s pattern is missing"), gccPreprocRecipeProperty)
+	}
+
+	commandLine := gccBuildProperties.ExpandPropsInString(pattern)
+	args, err := properties.SplitQuotedString(commandLine, `"'`, false)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// Remove -MMD argument if present. Leaving it will make gcc try
+	// to create a /dev/null.d dependency file, which won't work.
+	args = f.Filter(args, f.NotEquals("-MMD"))
+
+	proc, err := executils.NewProcess(nil, args...)
+	if err != nil {
+		return nil, nil, err
+	}
+	stdout, stderr, err := proc.RunAndCaptureOutput(context.Background())
+
+	// Append gcc arguments to stdout
+	stdout = append([]byte(fmt.Sprintln(strings.Join(args, " "))), stdout...)
+
+	return stdout, stderr, err
+}

Diff for: legacy/builder/container_add_prototypes.go

@@ -40,8 +40,11 @@ func PreprocessSketchWithCtags(ctx *types.Context) error {
 
 	// Run preprocessor
 	sourceFile := ctx.SketchBuildPath.Join(ctx.Sketch.MainFile.Base() + ".cpp")
-	stderr, err := GCCPreprocRunner(ctx, sourceFile, targetFilePath, ctx.IncludeFolders)
-	ctx.WriteStderr(stderr)
+	gccStdout, gccStderr, err := preprocessor.GCC(sourceFile, targetFilePath, ctx.IncludeFolders, ctx.BuildProperties)
+	if ctx.Verbose {
+		ctx.WriteStdout(gccStdout)
+		ctx.WriteStderr(gccStderr)
+	}
 	if err != nil {
 		if !ctx.OnlyUpdateCompilationDatabase {
 			return errors.WithStack(err)

Diff for: legacy/builder/container_find_includes.go

@@ -98,6 +98,7 @@ import (
 	"os/exec"
 	"time"
 
+	"github.com/arduino/arduino-cli/arduino/builder/preprocessor"
 	"github.com/arduino/arduino-cli/arduino/globals"
 	"github.com/arduino/arduino-cli/arduino/libraries"
 	"github.com/arduino/arduino-cli/legacy/builder/builder_utils"
@@ -369,7 +370,12 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFileQu
 				ctx.Info(tr("Using cached library dependencies for file: %[1]s", sourcePath))
 			}
 		} else {
-			preproc_stderr, preproc_err = GCCPreprocRunner(ctx, sourcePath, targetFilePath, includes)
+			var preproc_stdout []byte
+			preproc_stdout, preproc_stderr, preproc_err = preprocessor.GCC(sourcePath, targetFilePath, includes, ctx.BuildProperties)
+			if ctx.Verbose {
+				ctx.WriteStdout(preproc_stdout)
+				ctx.WriteStdout(preproc_stderr)
+			}
 			// Unwrap error and see if it is an ExitError.
 			_, is_exit_error := errors.Cause(preproc_err).(*exec.ExitError)
 			if preproc_err == nil {
@@ -397,11 +403,13 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFileQu
 		library := ResolveLibrary(ctx, include)
 		if library == nil {
 			// Library could not be resolved, show error
-			// err := runCommand(ctx, &GCCPreprocRunner{SourceFilePath: sourcePath, TargetFileName: paths.New(constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E), Includes: includes})
-			// return errors.WithStack(err)
 			if preproc_err == nil || preproc_stderr == nil {
 				// Filename came from cache, so run preprocessor to obtain error to show
-				preproc_stderr, preproc_err = GCCPreprocRunner(ctx, sourcePath, targetFilePath, includes)
+				var preproc_stdout []byte
+				preproc_stdout, preproc_stderr, preproc_err = preprocessor.GCC(sourcePath, targetFilePath, includes, ctx.BuildProperties)
+				if ctx.Verbose {
+					ctx.WriteStdout(preproc_stdout)
+				}
 				if preproc_err == nil {
 					// If there is a missing #include in the cache, but running
 					// gcc does not reproduce that, there is something wrong.

Diff for: legacy/builder/ctags/ctags_has_issues.go

@@ -19,27 +19,20 @@ import (
 	"bufio"
 	"os"
 	"strings"
+
+	"golang.org/x/exp/slices"
 )
 
-func (p *CTagsParser) FixCLinkageTagsDeclarations() {
+func (p *CTagsParser) fixCLinkageTagsDeclarations() {
 	linesMap := p.FindCLinkageLines(p.tags)
 	for i := range p.tags {
-		if sliceContainsInt(linesMap[p.tags[i].Filename], p.tags[i].Line) &&
+		if slices.Contains(linesMap[p.tags[i].Filename], p.tags[i].Line) &&
 			!strings.Contains(p.tags[i].PrototypeModifiers, EXTERN) {
 			p.tags[i].PrototypeModifiers = p.tags[i].PrototypeModifiers + " " + EXTERN
 		}
 	}
 }
 
-func sliceContainsInt(s []int, e int) bool {
-	for _, a := range s {
-		if a == e {
-			return true
-		}
-	}
-	return false
-}
-
 func (p *CTagsParser) prototypeAndCodeDontMatch(tag *CTag) bool {
 	if tag.SkipMe {
 		return true

Diff for: legacy/builder/ctags/ctags_parser.go

@@ -58,7 +58,7 @@ type CTag struct {
 	PrototypeModifiers string
 }
 
-func (p *CTagsParser) Parse(ctagsOutput []byte, mainFile *paths.Path) []*CTag {
+func (p *CTagsParser) Parse(ctagsOutput []byte, mainFile *paths.Path) ([]*Prototype, int) {
 	rows := strings.Split(string(ctagsOutput), "\n")
 	rows = removeEmpty(rows)
 
@@ -74,8 +74,9 @@ func (p *CTagsParser) Parse(ctagsOutput []byte, mainFile *paths.Path) []*CTag {
 	p.removeDefinedProtypes()
 	p.skipDuplicates()
 	p.skipTagsWhere(p.prototypeAndCodeDontMatch)
+	p.fixCLinkageTagsDeclarations()
 
-	return p.tags
+	return p.toPrototypes(), p.findLineWhereToInsertPrototypes()
 }
 
 func (p *CTagsParser) addPrototypes() {

Diff for: legacy/builder/ctags/ctags_parser_test.go

@@ -20,6 +20,7 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/arduino/go-paths-helper"
 	"github.com/stretchr/testify/require"
 )
 
@@ -28,7 +29,8 @@ func produceTags(t *testing.T, filename string) []*CTag {
 	require.NoError(t, err)
 
 	parser := CTagsParser{}
-	return parser.Parse(bytes, nil)
+	parser.Parse(bytes, paths.New("sketch.ino"))
+	return parser.tags
 }
 
 func TestCTagsParserShouldListPrototypes(t *testing.T) {

Diff for: legacy/builder/ctags/ctags_to_prototypes.go

@@ -32,10 +32,6 @@ func (proto *Prototype) String() string {
 	return proto.Modifiers + " " + proto.Prototype + " @ " + strconv.Itoa(proto.Line)
 }
 
-func (p *CTagsParser) GeneratePrototypes() ([]*Prototype, int) {
-	return p.toPrototypes(), p.findLineWhereToInsertPrototypes()
-}
-
 func (p *CTagsParser) findLineWhereToInsertPrototypes() int {
 	firstFunctionLine := p.firstFunctionAtLine()
 	firstFunctionPointerAsArgument := p.firstFunctionPointerUsedAsArgument()

Diff for: legacy/builder/ctags/ctags_to_prototypes_test.go

@@ -29,8 +29,7 @@ func producePrototypes(t *testing.T, filename string, mainFile string) ([]*Proto
 	require.NoError(t, err)
 
 	parser := &CTagsParser{}
-	parser.Parse(bytes, paths.New(mainFile))
-	return parser.GeneratePrototypes()
+	return parser.Parse(bytes, paths.New(mainFile))
 }
 
 func TestCTagsToPrototypesShouldListPrototypes(t *testing.T) {

Diff for: legacy/builder/gcc_preproc_runner.go

@@ -23,6 +23,7 @@ import (
 	"runtime"
 
 	bldr "github.com/arduino/arduino-cli/arduino/builder"
+	"github.com/arduino/arduino-cli/arduino/builder/preprocessor"
 	"github.com/arduino/arduino-cli/legacy/builder/types"
 	"github.com/arduino/arduino-cli/legacy/builder/utils"
 	properties "github.com/arduino/go-properties-orderedmap"
@@ -36,11 +37,15 @@ func PreprocessSketchWithArduinoPreprocessor(ctx *types.Context) error {
 
 	sourceFile := ctx.SketchBuildPath.Join(ctx.Sketch.MainFile.Base() + ".cpp")
 	targetFile := ctx.PreprocPath.Join("sketch_merged.cpp")
-	stderr, err := GCCPreprocRunner(ctx, sourceFile, targetFile, ctx.IncludeFolders)
-	ctx.WriteStderr(stderr)
+	gccStdout, gccStderr, err := preprocessor.GCC(sourceFile, targetFile, ctx.IncludeFolders, ctx.BuildProperties)
+	if ctx.Verbose {
+		ctx.WriteStdout(gccStdout)
+		ctx.WriteStderr(gccStderr)
+	}
 	if err != nil {
 		return err
 	}
+
 	buildProperties := properties.NewMap()
 	buildProperties.Set("tools.arduino-preprocessor.path", "{runtime.tools.arduino-preprocessor.path}")
 	buildProperties.Set("tools.arduino-preprocessor.cmd.path", "{path}/arduino-preprocessor")

Diff for: legacy/builder/preprocess_sketch.go

@@ -30,10 +30,7 @@ var DebugPreprocessor bool
 
 func PrototypesAdder(sketch *sketch.Sketch, source string, ctagsStdout []byte, lineOffset int) string {
 	parser := &ctags.CTagsParser{}
-	parser.Parse(ctagsStdout, sketch.MainFile)
-	parser.FixCLinkageTagsDeclarations()
-
-	prototypes, firstFunctionLine := parser.GeneratePrototypes()
+	prototypes, firstFunctionLine := parser.Parse(ctagsStdout, sketch.MainFile)
 	if firstFunctionLine == -1 {
 		firstFunctionLine = 0
 	}