[skip-changelog] legacy: Arduino preprocess subroutine refactorization (part 3) (arduino#2193)

* Converted CTagsRunner into a function

* Removed useless tasks from ctags_runner test

* Simplified ctags_runner test

* Removed some ctags related fields from builder context

The last piece in RunCTags:

 	parser := &ctags.CTagsParser{}
 	parser.Parse(ctagsStdout, sketch.MainFile)
 	parser.FixCLinkageTagsDeclarations()

 	prototypes, line := parser.GeneratePrototypes()
 	if line != -1 {
 		prototypesLineWhereToInsert = line
 	}

has been moved at the beginning of PrototypesAdder.
RunCTags now returns the output of ctags instead of `prototypes` and `line`.
This also allows to remove the context variables that keeps those
information.

* Simplified RunCTags / factored test subroutine

* Removed DebugPreprocessor from builder ctx

* Added executils.RunAndCaptureOutput

* Moved RunCTags out of legacy package

Author: cmaglie

arduino/builder/preprocessor/ctags.go

@@ -0,0 +1,62 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2023 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to [email protected].
+
+package preprocessor
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/arduino/arduino-cli/executils"
+	"github.com/arduino/arduino-cli/i18n"
+	"github.com/arduino/go-paths-helper"
+	"github.com/arduino/go-properties-orderedmap"
+	"github.com/pkg/errors"
+)
+
+var tr = i18n.Tr
+
+// RunCTags performs a run of ctags on the given source file. Returns the ctags output and the stderr contents.
+func RunCTags(sourceFile *paths.Path, buildProperties *properties.Map) ([]byte, []byte, error) {
+	ctagsBuildProperties := properties.NewMap()
+	ctagsBuildProperties.Set("tools.ctags.path", "{runtime.tools.ctags.path}")
+	ctagsBuildProperties.Set("tools.ctags.cmd.path", "{path}/ctags")
+	ctagsBuildProperties.Set("tools.ctags.pattern", `"{cmd.path}" -u --language-force=c++ -f - --c++-kinds=svpf --fields=KSTtzns --line-directives "{source_file}"`)
+	ctagsBuildProperties.Merge(buildProperties)
+	ctagsBuildProperties.Merge(ctagsBuildProperties.SubTree("tools").SubTree("ctags"))
+	ctagsBuildProperties.SetPath("source_file", sourceFile)
+
+	pattern := ctagsBuildProperties.Get("pattern")
+	if pattern == "" {
+		return nil, nil, errors.Errorf(tr("%s pattern is missing"), "ctags")
+	}
+
+	commandLine := ctagsBuildProperties.ExpandPropsInString(pattern)
+	parts, err := properties.SplitQuotedString(commandLine, `"'`, false)
+	if err != nil {
+		return nil, nil, err
+	}
+	proc, err := executils.NewProcess(nil, parts...)
+	if err != nil {
+		return nil, nil, err
+	}
+	stdout, stderr, err := proc.RunAndCaptureOutput(context.Background())
+
+	// Append ctags arguments to stderr
+	args := fmt.Sprintln(strings.Join(parts, " "))
+	stderr = append([]byte(args), stderr...)
+	return stdout, stderr, err
+}

executils/process.go

@@ -16,6 +16,7 @@
 package executils
 
 import (
+	"bytes"
 	"context"
 	"io"
 	"os"
@@ -174,3 +175,15 @@ func (p *Process) RunWithinContext(ctx context.Context) error {
 	}()
 	return p.Wait()
 }
+
+// RunAndCaptureOutput starts the specified command and waits for it to complete. If the given context
+// is canceled before the normal process termination, the process is killed. The standard output and
+// standard error of the process are captured and returned at process termination.
+func (p *Process) RunAndCaptureOutput(ctx context.Context) ([]byte, []byte, error) {
+	stdout := &bytes.Buffer{}
+	stderr := &bytes.Buffer{}
+	p.RedirectStdoutTo(stdout)
+	p.RedirectStderrTo(stderr)
+	err := p.RunWithinContext(ctx)
+	return stdout.Bytes(), stderr.Bytes(), err
+}

legacy/builder/container_add_prototypes.go

@@ -24,6 +24,7 @@ import (
 	"strings"
 
 	bldr "github.com/arduino/arduino-cli/arduino/builder"
+	"github.com/arduino/arduino-cli/arduino/builder/preprocessor"
 	"github.com/arduino/arduino-cli/arduino/sketch"
 	"github.com/arduino/arduino-cli/legacy/builder/types"
 	"github.com/arduino/go-paths-helper"
@@ -62,10 +63,18 @@ func PreprocessSketchWithCtags(ctx *types.Context) error {
 		ctx.SketchSourceAfterCppPreprocessing = filterSketchSource(ctx.Sketch, bytes.NewReader(src), false)
 	}
 
-	if err := (&CTagsRunner{Source: &ctx.SketchSourceAfterCppPreprocessing, TargetFileName: "sketch_merged.cpp"}).Run(ctx); err != nil {
-		return errors.WithStack(err)
+	if err := targetFilePath.WriteFile([]byte(ctx.SketchSourceAfterCppPreprocessing)); err != nil {
+		return err
+	}
+
+	ctagsStdout, ctagsStderr, err := preprocessor.RunCTags(targetFilePath, ctx.BuildProperties)
+	if ctx.Verbose {
+		ctx.WriteStderr(ctagsStderr)
+	}
+	if err != nil {
+		return err
 	}
-	ctx.SketchSourceAfterArduinoPreprocessing, ctx.PrototypesSection = PrototypesAdder(ctx.SketchSourceMerged, ctx.PrototypesLineWhereToInsert, ctx.LineOffset, ctx.Prototypes, ctx.DebugPreprocessor)
+	ctx.SketchSourceAfterArduinoPreprocessing = PrototypesAdder(ctx.Sketch, ctx.SketchSourceMerged, ctagsStdout, ctx.LineOffset)
 
 	if err := bldr.SketchSaveItemCpp(ctx.Sketch.MainFile, []byte(ctx.SketchSourceAfterArduinoPreprocessing), ctx.SketchBuildPath); err != nil {
 		return errors.WithStack(err)

legacy/builder/ctags_runner.go

@@ -20,25 +20,37 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/arduino/arduino-cli/arduino/sketch"
 	"github.com/arduino/arduino-cli/legacy/builder/constants"
 	"github.com/arduino/arduino-cli/legacy/builder/ctags"
 	"github.com/arduino/arduino-cli/legacy/builder/utils"
 )
 
-func PrototypesAdder(source string, firstFunctionLine, lineOffset int, prototypes []*ctags.Prototype, debugOutput bool) (preprocessedSource, prototypeSection string) {
+var DebugPreprocessor bool
+
+func PrototypesAdder(sketch *sketch.Sketch, source string, ctagsStdout []byte, lineOffset int) string {
+	parser := &ctags.CTagsParser{}
+	parser.Parse(ctagsStdout, sketch.MainFile)
+	parser.FixCLinkageTagsDeclarations()
+
+	prototypes, firstFunctionLine := parser.GeneratePrototypes()
+	if firstFunctionLine == -1 {
+		firstFunctionLine = 0
+	}
+
 	source = strings.Replace(source, "\r\n", "\n", -1)
 	source = strings.Replace(source, "\r", "\n", -1)
 	sourceRows := strings.Split(source, "\n")
 	if isFirstFunctionOutsideOfSource(firstFunctionLine, sourceRows) {
-		return
+		return ""
 	}
 
 	insertionLine := firstFunctionLine + lineOffset - 1
 	firstFunctionChar := len(strings.Join(sourceRows[:insertionLine], "\n")) + 1
-	prototypeSection = composePrototypeSection(firstFunctionLine, prototypes)
-	preprocessedSource = source[:firstFunctionChar] + prototypeSection + source[firstFunctionChar:]
+	prototypeSection := composePrototypeSection(firstFunctionLine, prototypes)
+	preprocessedSource := source[:firstFunctionChar] + prototypeSection + source[firstFunctionChar:]
 
-	if debugOutput {
+	if DebugPreprocessor {
 		fmt.Println("#PREPROCESSED SOURCE")
 		prototypesRows := strings.Split(prototypeSection, "\n")
 		prototypesRows = prototypesRows[:len(prototypesRows)-1]
@@ -53,7 +65,7 @@ func PrototypesAdder(source string, firstFunctionLine, lineOffset int, prototype
 		}
 		fmt.Println("#END OF PREPROCESSED SOURCE")
 	}
-	return
+	return preprocessedSource
 }
 
 func composePrototypeSection(line int, prototypes []*ctags.Prototype) string {