Moved ctag preprocessor into proper location

Author: cmaglie

Diff for: arduino/builder/preprocessor/ctags.go

@@ -16,11 +16,13 @@
 package preprocessor
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"strconv"
 	"strings"
 
+	"github.com/arduino/arduino-cli/arduino/builder"
 	"github.com/arduino/arduino-cli/arduino/builder/cpp"
 	"github.com/arduino/arduino-cli/arduino/builder/preprocessor/ctags"
 	"github.com/arduino/arduino-cli/arduino/sketch"
@@ -34,30 +36,73 @@ import (
 var tr = i18n.Tr
 var DebugPreprocessor bool
 
-func CTags(sourceFile *paths.Path, targetFile *paths.Path, sketch *sketch.Sketch, lineOffset int, buildProperties *properties.Map) ([]byte, error) {
-	ctagsOutput, ctagsStdErr, err := RunCTags(sourceFile, buildProperties)
+func PreprocessSketchWithCtags(sketch *sketch.Sketch, buildPath *paths.Path, includes paths.PathList, lineOffset int, buildProperties *properties.Map, onlyUpdateCompilationDatabase bool) ([]byte, []byte, error) {
+	// Create a temporary working directory
+	tmpDir, err := paths.MkTempDir("", "")
 	if err != nil {
-		return ctagsStdErr, err
+		return nil, nil, err
+	}
+	defer tmpDir.RemoveAll()
+	ctagsTarget := tmpDir.Join("sketch_merged.cpp")
+
+	normalOutput := &bytes.Buffer{}
+	verboseOutput := &bytes.Buffer{}
+
+	// Run GCC preprocessor
+	sourceFile := buildPath.Join("sketch", sketch.MainFile.Base()+".cpp")
+	gccStdout, gccStderr, err := GCC(sourceFile, ctagsTarget, includes, buildProperties)
+	verboseOutput.Write(gccStdout)
+	verboseOutput.Write(gccStderr)
+	normalOutput.Write(gccStderr)
+	if err != nil {
+		if !onlyUpdateCompilationDatabase {
+			return normalOutput.Bytes(), verboseOutput.Bytes(), errors.WithStack(err)
+		}
+
+		// Do not bail out if we are generating the compile commands database
+		normalOutput.WriteString(fmt.Sprintf("%s: %s",
+			tr("An error occurred adding prototypes"),
+			tr("the compilation database may be incomplete or inaccurate")))
+		if err := sourceFile.CopyTo(ctagsTarget); err != nil {
+			return normalOutput.Bytes(), verboseOutput.Bytes(), errors.WithStack(err)
+		}
+	}
+
+	if src, err := ctagsTarget.ReadFile(); err != nil {
+		return normalOutput.Bytes(), verboseOutput.Bytes(), err
+	} else {
+		filteredSource := builder.FilterSketchSource(sketch, bytes.NewReader(src), false)
+		if err := ctagsTarget.WriteFile([]byte(filteredSource)); err != nil {
+			return normalOutput.Bytes(), verboseOutput.Bytes(), err
+		}
+	}
+
+	// Run CTags on gcc-preprocessed source
+	ctagsOutput, ctagsStdErr, err := RunCTags(ctagsTarget, buildProperties)
+	verboseOutput.Write(ctagsStdErr)
+	if err != nil {
+		return normalOutput.Bytes(), verboseOutput.Bytes(), err
 	}
 
-	// func PrototypesAdder(sketch *sketch.Sketch, source string, ctagsStdout []byte, lineOffset int) string {
+	// Parse CTags output
 	parser := &ctags.CTagsParser{}
 	prototypes, firstFunctionLine := parser.Parse(ctagsOutput, sketch.MainFile)
 	if firstFunctionLine == -1 {
 		firstFunctionLine = 0
 	}
 
+	// Add prototypes to the original sketch source
 	var source string
-	if sourceData, err := targetFile.ReadFile(); err != nil {
-		return nil, err
+	if sourceData, err := sourceFile.ReadFile(); err != nil {
+		return normalOutput.Bytes(), verboseOutput.Bytes(), err
 	} else {
 		source = string(sourceData)
 	}
 	source = strings.Replace(source, "\r\n", "\n", -1)
 	source = strings.Replace(source, "\r", "\n", -1)
 	sourceRows := strings.Split(source, "\n")
 	if isFirstFunctionOutsideOfSource(firstFunctionLine, sourceRows) {
-		return nil, nil
+		return normalOutput.Bytes(), verboseOutput.Bytes(), nil
 	}
 
 	insertionLine := firstFunctionLine + lineOffset - 1
@@ -81,8 +126,9 @@ func CTags(sourceFile *paths.Path, targetFile *paths.Path, sketch *sketch.Sketch
 		fmt.Println("#END OF PREPROCESSED SOURCE")
 	}
 
-	err = targetFile.WriteFile([]byte(preprocessedSource))
-	return ctagsStdErr, err
+	// Write back arduino-preprocess output to the sourceFile
+	err = sourceFile.WriteFile([]byte(preprocessedSource))
+	return normalOutput.Bytes(), verboseOutput.Bytes(), err
 }
 
 func composePrototypeSection(line int, prototypes []*ctags.Prototype) string {

Diff for: legacy/builder/add_additional_entries_to_context.go

@@ -26,10 +26,6 @@ type AddAdditionalEntriesToContext struct{}
 func (*AddAdditionalEntriesToContext) Run(ctx *types.Context) error {
 	if ctx.BuildPath != nil {
 		buildPath := ctx.BuildPath
-		preprocPath, err := buildPath.Join(constants.FOLDER_PREPROC).Abs()
-		if err != nil {
-			return errors.WithStack(err)
-		}
 		sketchBuildPath, err := buildPath.Join(constants.FOLDER_SKETCH).Abs()
 		if err != nil {
 			return errors.WithStack(err)
@@ -43,7 +39,6 @@ func (*AddAdditionalEntriesToContext) Run(ctx *types.Context) error {
 			return errors.WithStack(err)
 		}
 
-		ctx.PreprocPath = preprocPath
 		ctx.SketchBuildPath = sketchBuildPath
 		ctx.LibrariesBuildPath = librariesBuildPath
 		ctx.CoreBuildPath = coreBuildPath

Diff for: legacy/builder/builder.go

@@ -20,6 +20,7 @@ import (
 	"time"
 
 	"github.com/arduino/arduino-cli/arduino/builder"
+	"github.com/arduino/arduino-cli/arduino/builder/preprocessor"
 	"github.com/arduino/arduino-cli/i18n"
 	"github.com/arduino/arduino-cli/legacy/builder/phases"
 	"github.com/arduino/arduino-cli/legacy/builder/types"
@@ -59,7 +60,7 @@ func (s *Builder) Run(ctx *types.Context) error {
 		&WarnAboutArchIncompatibleLibraries{},
 
 		utils.LogIfVerbose(false, tr("Generating function prototypes...")),
-		&PreprocessSketch{},
+		types.BareCommand(PreprocessSketch),
 
 		utils.LogIfVerbose(false, tr("Compiling sketch...")),
 		&RecipeByPrefixSuffixRunner{Prefix: "recipe.hooks.sketch.prebuild", Suffix: ".pattern"},
@@ -115,13 +116,19 @@ func (s *Builder) Run(ctx *types.Context) error {
 	return otherErr
 }
 
-type PreprocessSketch struct{}
-
-func (s *PreprocessSketch) Run(ctx *types.Context) error {
+func PreprocessSketch(ctx *types.Context) error {
 	if ctx.UseArduinoPreprocessor {
 		return PreprocessSketchWithArduinoPreprocessor(ctx)
 	} else {
-		return PreprocessSketchWithCtags(ctx)
+		normalOutput, verboseOutput, err := preprocessor.PreprocessSketchWithCtags(
+			ctx.Sketch, ctx.BuildPath, ctx.IncludeFolders, ctx.LineOffset,
+			ctx.BuildProperties, ctx.OnlyUpdateCompilationDatabase)
+		if ctx.Verbose {
+			ctx.WriteStdout(verboseOutput)
+		} else {
+			ctx.WriteStdout(normalOutput)
+		}
+		return err
 	}
 }
 
@@ -149,7 +156,7 @@ func (s *Preprocess) Run(ctx *types.Context) error {
 
 		&WarnAboutArchIncompatibleLibraries{},
 
-		&PreprocessSketch{},
+		types.BareCommand(PreprocessSketch),
 	}
 
 	if err := runCommands(ctx, commands); err != nil {

Diff for: legacy/builder/constants/constants.go

@@ -34,7 +34,6 @@ const BUILD_PROPERTIES_RUNTIME_PLATFORM_PATH = "runtime.platform.path"
 const EMPTY_STRING = ""
 const FOLDER_BOOTLOADERS = "bootloaders"
 const FOLDER_CORE = "core"
-const FOLDER_PREPROC = "preproc"
 const FOLDER_SKETCH = "sketch"
 const FOLDER_TOOLS = "tools"
 const FOLDER_LIBRARIES = "libraries"

Diff for: legacy/builder/container_add_prototypes.go

@@ -14,59 +14,3 @@
 // To purchase a commercial license, send an email to [email protected].
 
 package builder
-
-import (
-	"bytes"
-	"fmt"
-
-	bldr "github.com/arduino/arduino-cli/arduino/builder"
-	"github.com/arduino/arduino-cli/arduino/builder/preprocessor"
-	"github.com/arduino/arduino-cli/legacy/builder/types"
-	"github.com/pkg/errors"
-)
-
-func PreprocessSketchWithCtags(ctx *types.Context) error {
-	// Generate the full pathname for the preproc output file
-	if err := ctx.PreprocPath.MkdirAll(); err != nil {
-		return errors.WithStack(err)
-	}
-	targetFilePath := ctx.PreprocPath.Join("sketch_merged.cpp")
-
-	// Run preprocessor
-	sourceFile := ctx.SketchBuildPath.Join(ctx.Sketch.MainFile.Base() + ".cpp")
-	gccStdout, gccStderr, err := preprocessor.GCC(sourceFile, targetFilePath, ctx.IncludeFolders, ctx.BuildProperties)
-	if ctx.Verbose {
-		ctx.WriteStdout(gccStdout)
-		ctx.WriteStderr(gccStderr)
-	}
-	if err != nil {
-		if !ctx.OnlyUpdateCompilationDatabase {
-			return errors.WithStack(err)
-		}
-
-		// Do not bail out if we are generating the compile commands database
-		ctx.Info(
-			fmt.Sprintf("%s: %s",
-				tr("An error occurred adding prototypes"),
-				tr("the compilation database may be incomplete or inaccurate")))
-		if err := sourceFile.CopyTo(targetFilePath); err != nil {
-			return errors.WithStack(err)
-		}
-	}
-
-	if src, err := targetFilePath.ReadFile(); err != nil {
-		return err
-	} else {
-		filteredSource := bldr.FilterSketchSource(ctx.Sketch, bytes.NewReader(src), false)
-		if err := targetFilePath.WriteFile([]byte(filteredSource)); err != nil {
-			return err
-		}
-	}
-
-	sketchCpp := ctx.SketchBuildPath.Join(fmt.Sprintf("%s.cpp", ctx.Sketch.MainFile.Base()))
-	ctagsStderr, err := preprocessor.CTags(targetFilePath, sketchCpp, ctx.Sketch, ctx.LineOffset, ctx.BuildProperties)
-	if ctx.Verbose {
-		ctx.WriteStderr(ctagsStderr)
-	}
-	return err
-}

Diff for: legacy/builder/create_cmake_rule.go

@@ -115,7 +115,7 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error {
 		fmt.Println(err)
 	}
 
-	if err := PreprocessSketchWithCtags(ctx); err != nil {
+	if err := PreprocessSketch(ctx); err != nil {
 		return err
 	}
 

Diff for: legacy/builder/preprocess_sketch.go

@@ -31,12 +31,12 @@ import (
 )
 
 func PreprocessSketchWithArduinoPreprocessor(ctx *types.Context) error {
-	if err := ctx.PreprocPath.MkdirAll(); err != nil {
+	if err := ctx.BuildPath.Join("preproc").MkdirAll(); err != nil {
 		return errors.WithStack(err)
 	}
 
 	sourceFile := ctx.SketchBuildPath.Join(ctx.Sketch.MainFile.Base() + ".cpp")
-	targetFile := ctx.PreprocPath.Join("sketch_merged.cpp")
+	targetFile := ctx.BuildPath.Join("preproc", "sketch_merged.cpp")
 	gccStdout, gccStderr, err := preprocessor.GCC(sourceFile, targetFile, ctx.IncludeFolders, ctx.BuildProperties)
 	if ctx.Verbose {
 		ctx.WriteStdout(gccStdout)

Diff for: legacy/builder/test/add_additional_entries_to_context_test.go

@@ -31,7 +31,6 @@ func TestAddAdditionalEntriesToContextNoBuildPath(t *testing.T) {
 	command := builder.AddAdditionalEntriesToContext{}
 	NoError(t, command.Run(ctx))
 
-	require.Empty(t, ctx.PreprocPath)
 	require.Empty(t, ctx.SketchBuildPath)
 	require.Empty(t, ctx.LibrariesBuildPath)
 	require.Empty(t, ctx.CoreBuildPath)
@@ -48,7 +47,6 @@ func TestAddAdditionalEntriesToContextWithBuildPath(t *testing.T) {
 	command := builder.AddAdditionalEntriesToContext{}
 	NoError(t, command.Run(ctx))
 
-	require.Equal(t, Abs(t, paths.New("folder", constants.FOLDER_PREPROC)), ctx.PreprocPath)
 	require.Equal(t, Abs(t, paths.New("folder", constants.FOLDER_SKETCH)), ctx.SketchBuildPath)
 	require.Equal(t, Abs(t, paths.New("folder", "libraries")), ctx.LibrariesBuildPath)
 	require.Equal(t, Abs(t, paths.New("folder", constants.FOLDER_CORE)), ctx.CoreBuildPath)

Diff for: legacy/builder/test/builder_test.go

@@ -116,9 +116,6 @@ func TestBuilderEmptySketch(t *testing.T) {
 	exist, err := buildPath.Join(constants.FOLDER_CORE, "HardwareSerial.cpp.o").ExistCheck()
 	NoError(t, err)
 	require.True(t, exist)
-	exist, err = buildPath.Join(constants.FOLDER_PREPROC, "sketch_merged.cpp").ExistCheck()
-	NoError(t, err)
-	require.True(t, exist)
 	exist, err = buildPath.Join(constants.FOLDER_SKETCH, "sketch1.ino.cpp.o").ExistCheck()
 	NoError(t, err)
 	require.True(t, exist)
@@ -143,9 +140,6 @@ func TestBuilderBridge(t *testing.T) {
 	exist, err := buildPath.Join(constants.FOLDER_CORE, "HardwareSerial.cpp.o").ExistCheck()
 	NoError(t, err)
 	require.True(t, exist)
-	exist, err = buildPath.Join(constants.FOLDER_PREPROC, "sketch_merged.cpp").ExistCheck()
-	NoError(t, err)
-	require.True(t, exist)
 	exist, err = buildPath.Join(constants.FOLDER_SKETCH, "Bridge.ino.cpp.o").ExistCheck()
 	NoError(t, err)
 	require.True(t, exist)
@@ -173,9 +167,6 @@ func TestBuilderSketchWithConfig(t *testing.T) {
 	exist, err := buildPath.Join(constants.FOLDER_CORE, "HardwareSerial.cpp.o").ExistCheck()
 	NoError(t, err)
 	require.True(t, exist)
-	exist, err = buildPath.Join(constants.FOLDER_PREPROC, "sketch_merged.cpp").ExistCheck()
-	NoError(t, err)
-	require.True(t, exist)
 	exist, err = buildPath.Join(constants.FOLDER_SKETCH, "sketch_with_config.ino.cpp.o").ExistCheck()
 	NoError(t, err)
 	require.True(t, exist)
@@ -208,9 +199,6 @@ func TestBuilderBridgeTwice(t *testing.T) {
 	exist, err := buildPath.Join(constants.FOLDER_CORE, "HardwareSerial.cpp.o").ExistCheck()
 	NoError(t, err)
 	require.True(t, exist)
-	exist, err = buildPath.Join(constants.FOLDER_PREPROC, "sketch_merged.cpp").ExistCheck()
-	NoError(t, err)
-	require.True(t, exist)
 	exist, err = buildPath.Join(constants.FOLDER_SKETCH, "Bridge.ino.cpp.o").ExistCheck()
 	NoError(t, err)
 	require.True(t, exist)
@@ -245,9 +233,6 @@ func TestBuilderBridgeSAM(t *testing.T) {
 	exist, err = buildPath.Join(constants.FOLDER_CORE, "avr", "dtostrf.c.d").ExistCheck()
 	NoError(t, err)
 	require.True(t, exist)
-	exist, err = buildPath.Join(constants.FOLDER_PREPROC, "sketch_merged.cpp").ExistCheck()
-	NoError(t, err)
-	require.True(t, exist)
 	exist, err = buildPath.Join(constants.FOLDER_SKETCH, "Bridge.ino.cpp.o").ExistCheck()
 	NoError(t, err)
 	require.True(t, exist)
@@ -286,9 +271,6 @@ func TestBuilderBridgeRedBearLab(t *testing.T) {
 	exist, err := buildPath.Join(constants.FOLDER_CORE, "HardwareSerial.cpp.o").ExistCheck()
 	NoError(t, err)
 	require.True(t, exist)
-	exist, err = buildPath.Join(constants.FOLDER_PREPROC, "sketch_merged.cpp").ExistCheck()
-	NoError(t, err)
-	require.True(t, exist)
 	exist, err = buildPath.Join(constants.FOLDER_SKETCH, "Bridge.ino.cpp.o").ExistCheck()
 	NoError(t, err)
 	require.True(t, exist)