Add support for compile_command.json output

arduino/builder/compilation_database.go

@@ -0,0 +1,95 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to [email protected].
+
+package builder
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"os/exec"
+
+	"github.com/arduino/go-paths-helper"
+)
+
+// CompilationDatabase keeps track of all the compile commands run by the builder
+type CompilationDatabase struct {
+	Contents []CompilationCommand
+	File     *paths.Path
+}
+
+// CompilationCommand keeps track of a single run of a compile command
+type CompilationCommand struct {
+	Directory string   `json:"directory"`
+	Command   string   `json:"command,omitempty"`
+	Arguments []string `json:"arguments,omitempty"`
+	File      string   `json:"file"`
+}
+
+// NewCompilationDatabase creates an empty CompilationDatabase
+func NewCompilationDatabase(filename *paths.Path) *CompilationDatabase {
+	return &CompilationDatabase{
+		File: filename,
+	}
+}
+
+// LoadCompilationDatabase reads a compilation database from a file
+func LoadCompilationDatabase(file *paths.Path) (*CompilationDatabase, error) {
+	f, err := file.ReadFile()
+	if err != nil {
+		return nil, err
+	}
+	res := &CompilationDatabase{
+		File:     file,
+		Contents: []CompilationCommand{},
+	}
+	return res, json.Unmarshal(f, &res.Contents)
+}
+
+// SaveToFile save the CompilationDatabase to file as a clangd-compatible compile_commands.json,
+// see https://clang.llvm.org/docs/JSONCompilationDatabase.html
+func (db *CompilationDatabase) SaveToFile() {
+	if jsonContents, err := json.MarshalIndent(db.Contents, "", " "); err != nil {
+		fmt.Printf("Error serializing compilation database: %s", err)
+		return
+	} else if err := db.File.WriteFile(jsonContents); err != nil {
+		fmt.Printf("Error writing compilation database: %s", err)
+	}
+}
+
+func dirForCommand(command *exec.Cmd) string {
+	// This mimics what Cmd.Run also does: Use Dir if specified,
+	// current directory otherwise
+	if command.Dir != "" {
+		return command.Dir
+	}
+	dir, err := os.Getwd()
+	if err != nil {
+		fmt.Printf("Error getting current directory for compilation database: %s", err)
+		return ""
+	}
+	return dir
+}
+
+// Add adds a new CompilationDatabase entry
+func (db *CompilationDatabase) Add(target *paths.Path, command *exec.Cmd) {
+	entry := CompilationCommand{
+		Directory: dirForCommand(command),
+		Arguments: command.Args,
+		File:      target.String(),
+	}
+
+	db.Contents = append(db.Contents, entry)
+}

arduino/builder/compilation_database_test.go

@@ -0,0 +1,46 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to [email protected].
+
+package builder
+
+import (
+	"os/exec"
+	"testing"
+
+	"github.com/arduino/go-paths-helper"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCompilationDatabase(t *testing.T) {
+	tmpfile, err := paths.WriteToTempFile([]byte{}, nil, "")
+	require.NoError(t, err)
+	defer tmpfile.Remove()
+
+	cmd := exec.Command("gcc", "arg1", "arg2")
+	db := NewCompilationDatabase(tmpfile)
+	db.Add(paths.New("test"), cmd)
+	db.SaveToFile()
+
+	db2, err := LoadCompilationDatabase(tmpfile)
+	require.NoError(t, err)
+	require.Equal(t, db, db2)
+	require.Len(t, db2.Contents, 1)
+	require.Equal(t, db2.Contents[0].File, "test")
+	require.Equal(t, db2.Contents[0].Command, "")
+	require.Equal(t, db2.Contents[0].Arguments, []string{"gcc", "arg1", "arg2"})
+	cwd, err := paths.Getwd()
+	require.NoError(t, err)
+	require.Equal(t, db2.Contents[0].Directory, cwd.String())
+}

cli/compile/compile.go

@@ -35,25 +35,26 @@ import (
 )
 
 var (
-	fqbn               string   // Fully Qualified Board Name, e.g.: arduino:avr:uno.
-	showProperties     bool     // Show all build preferences used instead of compiling.
-	preprocess         bool     // Print preprocessed code to stdout.
-	buildCachePath     string   // Builds of 'core.a' are saved into this path to be cached and reused.
-	buildPath          string   // Path where to save compiled files.
-	buildProperties    []string // List of custom build properties separated by commas. Or can be used multiple times for multiple properties.
-	warnings           string   // Used to tell gcc which warning level to use.
-	verbose            bool     // Turns on verbose mode.
-	quiet              bool     // Suppresses almost every output.
-	vidPid             string   // VID/PID specific build properties.
-	uploadAfterCompile bool     // Upload the binary after the compilation.
-	port               string   // Upload port, e.g.: COM10 or /dev/ttyACM0.
-	verify             bool     // Upload, verify uploaded binary after the upload.
-	exportDir          string   // The compiled binary is written to this file
-	libraries          []string // List of custom libraries paths separated by commas. Or can be used multiple times for multiple libraries paths.
-	optimizeForDebug   bool     // Optimize compile output for debug, not for release
-	programmer         string   // Use the specified programmer to upload
-	clean              bool     // Cleanup the build folder and do not use any cached build
-	exportBinaries     bool     // Copies compiled binaries to sketch folder when true
+	fqbn                    string   // Fully Qualified Board Name, e.g.: arduino:avr:uno.
+	showProperties          bool     // Show all build preferences used instead of compiling.
+	preprocess              bool     // Print preprocessed code to stdout.
+	buildCachePath          string   // Builds of 'core.a' are saved into this path to be cached and reused.
+	buildPath               string   // Path where to save compiled files.
+	buildProperties         []string // List of custom build properties separated by commas. Or can be used multiple times for multiple properties.
+	warnings                string   // Used to tell gcc which warning level to use.
+	verbose                 bool     // Turns on verbose mode.
+	quiet                   bool     // Suppresses almost every output.
+	vidPid                  string   // VID/PID specific build properties.
+	uploadAfterCompile      bool     // Upload the binary after the compilation.
+	port                    string   // Upload port, e.g.: COM10 or /dev/ttyACM0.
+	verify                  bool     // Upload, verify uploaded binary after the upload.
+	exportDir               string   // The compiled binary is written to this file
+	libraries               []string // List of custom libraries paths separated by commas. Or can be used multiple times for multiple libraries paths.
+	optimizeForDebug        bool     // Optimize compile output for debug, not for release
+	programmer              string   // Use the specified programmer to upload
+	clean                   bool     // Cleanup the build folder and do not use any cached build
+	exportBinaries          bool     // Copies compiled binaries to sketch folder when true
+	compilationDatabaseOnly bool     // Only create compilation database without actually compiling
 )
 
 // NewCommand created a new `compile` command
@@ -94,6 +95,7 @@ func NewCommand() *cobra.Command {
 		"List of custom libraries paths separated by commas. Or can be used multiple times for multiple libraries paths.")
 	command.Flags().BoolVar(&optimizeForDebug, "optimize-for-debug", false, "Optional, optimize compile output for debugging, rather than for release.")
 	command.Flags().StringVarP(&programmer, "programmer", "P", "", "Optional, use the specified programmer to upload.")
+	command.Flags().BoolVar(&compilationDatabaseOnly, "only-compilation-database", false, "Just produce the compilation database, without actually compiling.")
 	command.Flags().BoolVar(&clean, "clean", false, "Optional, cleanup the build folder and do not use any cached build.")
 	// We must use the following syntax for this flag since it's also bound to settings, we could use the other one too
 	// but it wouldn't make sense since we still must explicitly set the exportBinaries variable by reading from settings.
@@ -127,23 +129,24 @@ func run(cmd *cobra.Command, args []string) {
 	exportBinaries = configuration.Settings.GetBool("sketch.always_export_binaries")
 
 	compileReq := &rpc.CompileReq{
-		Instance:         inst,
-		Fqbn:             fqbn,
-		SketchPath:       sketchPath.String(),
-		ShowProperties:   showProperties,
-		Preprocess:       preprocess,
-		BuildCachePath:   buildCachePath,
-		BuildPath:        buildPath,
-		BuildProperties:  buildProperties,
-		Warnings:         warnings,
-		Verbose:          verbose,
-		Quiet:            quiet,
-		VidPid:           vidPid,
-		ExportDir:        exportDir,
-		Libraries:        libraries,
-		OptimizeForDebug: optimizeForDebug,
-		Clean:            clean,
-		ExportBinaries:   exportBinaries,
+		Instance:                      inst,
+		Fqbn:                          fqbn,
+		SketchPath:                    sketchPath.String(),
+		ShowProperties:                showProperties,
+		Preprocess:                    preprocess,
+		BuildCachePath:                buildCachePath,
+		BuildPath:                     buildPath,
+		BuildProperties:               buildProperties,
+		Warnings:                      warnings,
+		Verbose:                       verbose,
+		Quiet:                         quiet,
+		VidPid:                        vidPid,
+		ExportDir:                     exportDir,
+		Libraries:                     libraries,
+		OptimizeForDebug:              optimizeForDebug,
+		Clean:                         clean,
+		ExportBinaries:                exportBinaries,
+		CreateCompilationDatabaseOnly: compilationDatabaseOnly,
 	}
 	compileOut := new(bytes.Buffer)
 	compileErr := new(bytes.Buffer)

commands/compile/compile.go

@@ -129,10 +129,12 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W
 	} else {
 		builderCtx.BuildPath = paths.New(req.GetBuildPath())
 	}
-
 	if err = builderCtx.BuildPath.MkdirAll(); err != nil {
 		return nil, fmt.Errorf("cannot create build directory: %s", err)
 	}
+	builderCtx.CompilationDatabase = bldr.NewCompilationDatabase(
+		builderCtx.BuildPath.Join("compile_commands.json"),
+	)
 
 	builderCtx.Verbose = req.GetVerbose()
 
@@ -188,6 +190,7 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W
 	builderCtx.ExecStderr = errStream
 	builderCtx.SetLogger(i18n.LoggerToCustomStreams{Stdout: outStream, Stderr: errStream})
 	builderCtx.Clean = req.GetClean()
+	builderCtx.OnlyUpdateCompilationDatabase = req.GetCreateCompilationDatabaseOnly()
 
 	// Use defer() to create an rpc.CompileResp with all the information available at the
 	// moment of return.

go.mod

@@ -4,7 +4,7 @@ go 1.14
 
 require (
 	github.com/arduino/board-discovery v0.0.0-20180823133458-1ba29327fb0c
-	github.com/arduino/go-paths-helper v1.3.2
+	github.com/arduino/go-paths-helper v1.4.0
 	github.com/arduino/go-properties-orderedmap v1.3.0
 	github.com/arduino/go-timeutils v0.0.0-20171220113728-d1dd9e313b1b
 	github.com/arduino/go-win32-utils v0.0.0-20180330194947-ed041402e83b

go.sum

@@ -17,6 +17,8 @@ github.com/arduino/go-paths-helper v1.0.1/go.mod h1:HpxtKph+g238EJHq4geEPv9p+gl3
 github.com/arduino/go-paths-helper v1.2.0/go.mod h1:HpxtKph+g238EJHq4geEPv9p+gl3v5YYu35Yb+w31Ck=
 github.com/arduino/go-paths-helper v1.3.2 h1:5U9TSKQODiwSVgTxskC0PNl0l0Vf40GUlp99Zy2SK8w=
 github.com/arduino/go-paths-helper v1.3.2/go.mod h1:HpxtKph+g238EJHq4geEPv9p+gl3v5YYu35Yb+w31Ck=
+github.com/arduino/go-paths-helper v1.4.0 h1:ilnseAdxmN1bFnLxxXHRtcdmt9jBf3O4jtYfWfqule4=
+github.com/arduino/go-paths-helper v1.4.0/go.mod h1:V82BWgAAp4IbmlybxQdk9Bpkz8M4Qyx+RAFKaG9NuvU=
 github.com/arduino/go-properties-orderedmap v1.3.0 h1:4No/vQopB36e7WUIk6H6TxiSEJPiMrVOCZylYmua39o=
 github.com/arduino/go-properties-orderedmap v1.3.0/go.mod h1:DKjD2VXY/NZmlingh4lSFMEYCVubfeArCsGPGDwb2yk=
 github.com/arduino/go-timeutils v0.0.0-20171220113728-d1dd9e313b1b h1:9hDi4F2st6dbLC3y4i02zFT5quS4X6iioWifGlVwfy4=

legacy/builder/builder.go

@@ -96,6 +96,10 @@ func (s *Builder) Run(ctx *types.Context) error {
 
 	mainErr := runCommands(ctx, commands)
 
+	if ctx.CompilationDatabase != nil {
+		ctx.CompilationDatabase.SaveToFile()
+	}
+
 	commands = []types.Command{
 		&PrintUsedAndNotUsedLibraries{SketchError: mainErr != nil},
 

legacy/builder/builder_utils/utils.go

@@ -247,18 +247,24 @@ func compileFileWithRecipe(ctx *types.Context, sourcePath *paths.Path, source *p
 	if err != nil {
 		return nil, errors.WithStack(err)
 	}
-	if !objIsUpToDate {
-		command, err := PrepareCommandForRecipe(properties, recipe, false)
-		if err != nil {
-			return nil, errors.WithStack(err)
-		}
-
+	command, err := PrepareCommandForRecipe(properties, recipe, false)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+	if ctx.CompilationDatabase != nil {
+		ctx.CompilationDatabase.Add(source, command)
+	}
+	if !objIsUpToDate && !ctx.OnlyUpdateCompilationDatabase {
 		_, _, err = utils.ExecCommand(ctx, command, utils.ShowIfVerbose /* stdout */, utils.Show /* stderr */)
 		if err != nil {
 			return nil, errors.WithStack(err)
 		}
 	} else if ctx.Verbose {
-		logger.Println(constants.LOG_LEVEL_INFO, constants.MSG_USING_PREVIOUS_COMPILED_FILE, objectFile)
+		if objIsUpToDate {
+			logger.Println(constants.LOG_LEVEL_INFO, constants.MSG_USING_PREVIOUS_COMPILED_FILE, objectFile)
+		} else {
+			logger.Println("info", "Skipping compile of: {0}", objectFile)
+		}
 	}
 
 	return objectFile, nil
@@ -452,10 +458,15 @@ func ArchiveCompiledFiles(ctx *types.Context, buildPath *paths.Path, archiveFile
 	logger := ctx.GetLogger()
 	archiveFilePath := buildPath.JoinPath(archiveFile)
 
-	rebuildArchive := false
+	if ctx.OnlyUpdateCompilationDatabase {
+		if ctx.Verbose {
+			logger.Println("info", "Skipping archive creation of: {0}", archiveFilePath)
+		}
+		return archiveFilePath, nil
+	}
 
 	if archiveFileStat, err := archiveFilePath.Stat(); err == nil {
-
+		rebuildArchive := false
 		for _, objectFile := range objectFilesToArchive {
 			objectFileStat, err := objectFile.Stat()
 			if err != nil || objectFileStat.ModTime().After(archiveFileStat.ModTime()) {

legacy/builder/merge_sketch_with_bootloader.go

@@ -31,6 +31,10 @@ import (
 type MergeSketchWithBootloader struct{}
 
 func (s *MergeSketchWithBootloader) Run(ctx *types.Context) error {
+	if ctx.OnlyUpdateCompilationDatabase {
+		return nil
+	}
+
 	buildProperties := ctx.BuildProperties
 	if !buildProperties.ContainsKey(constants.BUILD_PROPERTIES_BOOTLOADER_NOBLINK) && !buildProperties.ContainsKey(constants.BUILD_PROPERTIES_BOOTLOADER_FILE) {
 		return nil

legacy/builder/phases/core_builder.go

@@ -94,7 +94,9 @@ func compileCore(ctx *types.Context, buildPath *paths.Path, buildCachePath *path
 		archivedCoreName := GetCachedCoreArchiveFileName(buildProperties.Get(constants.BUILD_PROPERTIES_FQBN),
 			buildProperties.Get("compiler.optimization_flags"), realCoreFolder)
 		targetArchivedCore = buildCachePath.Join(archivedCoreName)
-		canUseArchivedCore := !ctx.Clean && !builder_utils.CoreOrReferencedCoreHasChanged(realCoreFolder, targetCoreFolder, targetArchivedCore)
+		canUseArchivedCore := !ctx.OnlyUpdateCompilationDatabase &&
+			!ctx.Clean &&
+			!builder_utils.CoreOrReferencedCoreHasChanged(realCoreFolder, targetCoreFolder, targetArchivedCore)
 
 		if canUseArchivedCore {
 			// use archived core
@@ -116,7 +118,7 @@ func compileCore(ctx *types.Context, buildPath *paths.Path, buildCachePath *path
 	}
 
 	// archive core.a
-	if targetArchivedCore != nil {
+	if targetArchivedCore != nil && !ctx.OnlyUpdateCompilationDatabase {
 		err := archiveFile.CopyTo(targetArchivedCore)
 		if ctx.Verbose {
 			if err == nil {

legacy/builder/phases/linker.go

@@ -30,6 +30,13 @@ import (
 type Linker struct{}
 
 func (s *Linker) Run(ctx *types.Context) error {
+	if ctx.OnlyUpdateCompilationDatabase {
+		if ctx.Verbose {
+			ctx.GetLogger().Println("info", "Skip linking of final executable.")
+		}
+		return nil
+	}
+
 	objectFilesSketch := ctx.SketchObjectFiles
 	objectFilesLibraries := ctx.LibrariesObjectFiles
 	objectFilesCore := ctx.CoreObjectsFiles

legacy/builder/phases/sizer.go

@@ -32,7 +32,9 @@ type Sizer struct {
 }
 
 func (s *Sizer) Run(ctx *types.Context) error {
-
+	if ctx.OnlyUpdateCompilationDatabase {
+		return nil
+	}
 	if s.SketchError {
 		return nil
 	}