Skip to content

Commit 441f8eb

Browse files
cmagliematthijskooijman
cmaglie
and
matthijskooijman
authored
Add support for compile_command.json output (#1081)
* Draft: Support generating a compile_commands.json file This is still very rough and unfinished. * Added compile flag to produce only the compilation database * Save compilation database inside build path * Removed 'Writing compilation database...' message Also slightly refactore code * Added missing (c) header * Renamed some functions, did some small cleanups * compilation database: made some fields public * compilation database: added LoadCompilationDatabase method * Added unit tests for compilation_database Co-authored-by: Matthijs Kooijman <[email protected]>
1 parent bf7a319 commit 441f8eb

File tree

16 files changed

+303
-97
lines changed

16 files changed

+303
-97
lines changed

Diff for: ‎arduino/builder/compilation_database.go

+95
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
// This file is part of arduino-cli.
2+
//
3+
// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
4+
//
5+
// This software is released under the GNU General Public License version 3,
6+
// which covers the main part of arduino-cli.
7+
// The terms of this license can be found at:
8+
// https://www.gnu.org/licenses/gpl-3.0.en.html
9+
//
10+
// You can be released from the requirements of the above licenses by purchasing
11+
// a commercial license. Buying such a license is mandatory if you want to
12+
// modify or otherwise use the software for commercial activities involving the
13+
// Arduino software without disclosing the source code of your own applications.
14+
// To purchase a commercial license, send an email to [email protected].
15+
16+
package builder
17+
18+
import (
19+
"encoding/json"
20+
"fmt"
21+
"os"
22+
"os/exec"
23+
24+
"github.com/arduino/go-paths-helper"
25+
)
26+
27+
// CompilationDatabase keeps track of all the compile commands run by the builder
28+
type CompilationDatabase struct {
29+
Contents []CompilationCommand
30+
File *paths.Path
31+
}
32+
33+
// CompilationCommand keeps track of a single run of a compile command
34+
type CompilationCommand struct {
35+
Directory string `json:"directory"`
36+
Command string `json:"command,omitempty"`
37+
Arguments []string `json:"arguments,omitempty"`
38+
File string `json:"file"`
39+
}
40+
41+
// NewCompilationDatabase creates an empty CompilationDatabase
42+
func NewCompilationDatabase(filename *paths.Path) *CompilationDatabase {
43+
return &CompilationDatabase{
44+
File: filename,
45+
}
46+
}
47+
48+
// LoadCompilationDatabase reads a compilation database from a file
49+
func LoadCompilationDatabase(file *paths.Path) (*CompilationDatabase, error) {
50+
f, err := file.ReadFile()
51+
if err != nil {
52+
return nil, err
53+
}
54+
res := &CompilationDatabase{
55+
File: file,
56+
Contents: []CompilationCommand{},
57+
}
58+
return res, json.Unmarshal(f, &res.Contents)
59+
}
60+
61+
// SaveToFile save the CompilationDatabase to file as a clangd-compatible compile_commands.json,
62+
// see https://clang.llvm.org/docs/JSONCompilationDatabase.html
63+
func (db *CompilationDatabase) SaveToFile() {
64+
if jsonContents, err := json.MarshalIndent(db.Contents, "", " "); err != nil {
65+
fmt.Printf("Error serializing compilation database: %s", err)
66+
return
67+
} else if err := db.File.WriteFile(jsonContents); err != nil {
68+
fmt.Printf("Error writing compilation database: %s", err)
69+
}
70+
}
71+
72+
func dirForCommand(command *exec.Cmd) string {
73+
// This mimics what Cmd.Run also does: Use Dir if specified,
74+
// current directory otherwise
75+
if command.Dir != "" {
76+
return command.Dir
77+
}
78+
dir, err := os.Getwd()
79+
if err != nil {
80+
fmt.Printf("Error getting current directory for compilation database: %s", err)
81+
return ""
82+
}
83+
return dir
84+
}
85+
86+
// Add adds a new CompilationDatabase entry
87+
func (db *CompilationDatabase) Add(target *paths.Path, command *exec.Cmd) {
88+
entry := CompilationCommand{
89+
Directory: dirForCommand(command),
90+
Arguments: command.Args,
91+
File: target.String(),
92+
}
93+
94+
db.Contents = append(db.Contents, entry)
95+
}

Diff for: ‎arduino/builder/compilation_database_test.go

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
// This file is part of arduino-cli.
2+
//
3+
// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
4+
//
5+
// This software is released under the GNU General Public License version 3,
6+
// which covers the main part of arduino-cli.
7+
// The terms of this license can be found at:
8+
// https://www.gnu.org/licenses/gpl-3.0.en.html
9+
//
10+
// You can be released from the requirements of the above licenses by purchasing
11+
// a commercial license. Buying such a license is mandatory if you want to
12+
// modify or otherwise use the software for commercial activities involving the
13+
// Arduino software without disclosing the source code of your own applications.
14+
// To purchase a commercial license, send an email to [email protected].
15+
16+
package builder
17+
18+
import (
19+
"os/exec"
20+
"testing"
21+
22+
"github.com/arduino/go-paths-helper"
23+
"github.com/stretchr/testify/require"
24+
)
25+
26+
func TestCompilationDatabase(t *testing.T) {
27+
tmpfile, err := paths.WriteToTempFile([]byte{}, nil, "")
28+
require.NoError(t, err)
29+
defer tmpfile.Remove()
30+
31+
cmd := exec.Command("gcc", "arg1", "arg2")
32+
db := NewCompilationDatabase(tmpfile)
33+
db.Add(paths.New("test"), cmd)
34+
db.SaveToFile()
35+
36+
db2, err := LoadCompilationDatabase(tmpfile)
37+
require.NoError(t, err)
38+
require.Equal(t, db, db2)
39+
require.Len(t, db2.Contents, 1)
40+
require.Equal(t, db2.Contents[0].File, "test")
41+
require.Equal(t, db2.Contents[0].Command, "")
42+
require.Equal(t, db2.Contents[0].Arguments, []string{"gcc", "arg1", "arg2"})
43+
cwd, err := paths.Getwd()
44+
require.NoError(t, err)
45+
require.Equal(t, db2.Contents[0].Directory, cwd.String())
46+
}

Diff for: ‎cli/compile/compile.go

+39-36
Original file line numberDiff line numberDiff line change
@@ -35,25 +35,26 @@ import (
3535
)
3636

3737
var (
38-
fqbn string // Fully Qualified Board Name, e.g.: arduino:avr:uno.
39-
showProperties bool // Show all build preferences used instead of compiling.
40-
preprocess bool // Print preprocessed code to stdout.
41-
buildCachePath string // Builds of 'core.a' are saved into this path to be cached and reused.
42-
buildPath string // Path where to save compiled files.
43-
buildProperties []string // List of custom build properties separated by commas. Or can be used multiple times for multiple properties.
44-
warnings string // Used to tell gcc which warning level to use.
45-
verbose bool // Turns on verbose mode.
46-
quiet bool // Suppresses almost every output.
47-
vidPid string // VID/PID specific build properties.
48-
uploadAfterCompile bool // Upload the binary after the compilation.
49-
port string // Upload port, e.g.: COM10 or /dev/ttyACM0.
50-
verify bool // Upload, verify uploaded binary after the upload.
51-
exportDir string // The compiled binary is written to this file
52-
libraries []string // List of custom libraries paths separated by commas. Or can be used multiple times for multiple libraries paths.
53-
optimizeForDebug bool // Optimize compile output for debug, not for release
54-
programmer string // Use the specified programmer to upload
55-
clean bool // Cleanup the build folder and do not use any cached build
56-
exportBinaries bool // Copies compiled binaries to sketch folder when true
38+
fqbn string // Fully Qualified Board Name, e.g.: arduino:avr:uno.
39+
showProperties bool // Show all build preferences used instead of compiling.
40+
preprocess bool // Print preprocessed code to stdout.
41+
buildCachePath string // Builds of 'core.a' are saved into this path to be cached and reused.
42+
buildPath string // Path where to save compiled files.
43+
buildProperties []string // List of custom build properties separated by commas. Or can be used multiple times for multiple properties.
44+
warnings string // Used to tell gcc which warning level to use.
45+
verbose bool // Turns on verbose mode.
46+
quiet bool // Suppresses almost every output.
47+
vidPid string // VID/PID specific build properties.
48+
uploadAfterCompile bool // Upload the binary after the compilation.
49+
port string // Upload port, e.g.: COM10 or /dev/ttyACM0.
50+
verify bool // Upload, verify uploaded binary after the upload.
51+
exportDir string // The compiled binary is written to this file
52+
libraries []string // List of custom libraries paths separated by commas. Or can be used multiple times for multiple libraries paths.
53+
optimizeForDebug bool // Optimize compile output for debug, not for release
54+
programmer string // Use the specified programmer to upload
55+
clean bool // Cleanup the build folder and do not use any cached build
56+
exportBinaries bool // Copies compiled binaries to sketch folder when true
57+
compilationDatabaseOnly bool // Only create compilation database without actually compiling
5758
)
5859

5960
// NewCommand created a new `compile` command
@@ -94,6 +95,7 @@ func NewCommand() *cobra.Command {
9495
"List of custom libraries paths separated by commas. Or can be used multiple times for multiple libraries paths.")
9596
command.Flags().BoolVar(&optimizeForDebug, "optimize-for-debug", false, "Optional, optimize compile output for debugging, rather than for release.")
9697
command.Flags().StringVarP(&programmer, "programmer", "P", "", "Optional, use the specified programmer to upload.")
98+
command.Flags().BoolVar(&compilationDatabaseOnly, "only-compilation-database", false, "Just produce the compilation database, without actually compiling.")
9799
command.Flags().BoolVar(&clean, "clean", false, "Optional, cleanup the build folder and do not use any cached build.")
98100
// We must use the following syntax for this flag since it's also bound to settings, we could use the other one too
99101
// but it wouldn't make sense since we still must explicitly set the exportBinaries variable by reading from settings.
@@ -127,23 +129,24 @@ func run(cmd *cobra.Command, args []string) {
127129
exportBinaries = configuration.Settings.GetBool("sketch.always_export_binaries")
128130

129131
compileReq := &rpc.CompileReq{
130-
Instance: inst,
131-
Fqbn: fqbn,
132-
SketchPath: sketchPath.String(),
133-
ShowProperties: showProperties,
134-
Preprocess: preprocess,
135-
BuildCachePath: buildCachePath,
136-
BuildPath: buildPath,
137-
BuildProperties: buildProperties,
138-
Warnings: warnings,
139-
Verbose: verbose,
140-
Quiet: quiet,
141-
VidPid: vidPid,
142-
ExportDir: exportDir,
143-
Libraries: libraries,
144-
OptimizeForDebug: optimizeForDebug,
145-
Clean: clean,
146-
ExportBinaries: exportBinaries,
132+
Instance: inst,
133+
Fqbn: fqbn,
134+
SketchPath: sketchPath.String(),
135+
ShowProperties: showProperties,
136+
Preprocess: preprocess,
137+
BuildCachePath: buildCachePath,
138+
BuildPath: buildPath,
139+
BuildProperties: buildProperties,
140+
Warnings: warnings,
141+
Verbose: verbose,
142+
Quiet: quiet,
143+
VidPid: vidPid,
144+
ExportDir: exportDir,
145+
Libraries: libraries,
146+
OptimizeForDebug: optimizeForDebug,
147+
Clean: clean,
148+
ExportBinaries: exportBinaries,
149+
CreateCompilationDatabaseOnly: compilationDatabaseOnly,
147150
}
148151
compileOut := new(bytes.Buffer)
149152
compileErr := new(bytes.Buffer)

Diff for: ‎commands/compile/compile.go

+4-1
Original file line numberDiff line numberDiff line change
@@ -129,10 +129,12 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W
129129
} else {
130130
builderCtx.BuildPath = paths.New(req.GetBuildPath())
131131
}
132-
133132
if err = builderCtx.BuildPath.MkdirAll(); err != nil {
134133
return nil, fmt.Errorf("cannot create build directory: %s", err)
135134
}
135+
builderCtx.CompilationDatabase = bldr.NewCompilationDatabase(
136+
builderCtx.BuildPath.Join("compile_commands.json"),
137+
)
136138

137139
builderCtx.Verbose = req.GetVerbose()
138140

@@ -188,6 +190,7 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W
188190
builderCtx.ExecStderr = errStream
189191
builderCtx.SetLogger(i18n.LoggerToCustomStreams{Stdout: outStream, Stderr: errStream})
190192
builderCtx.Clean = req.GetClean()
193+
builderCtx.OnlyUpdateCompilationDatabase = req.GetCreateCompilationDatabaseOnly()
191194

192195
// Use defer() to create an rpc.CompileResp with all the information available at the
193196
// moment of return.

Diff for: ‎go.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ go 1.14
44

55
require (
66
github.com/arduino/board-discovery v0.0.0-20180823133458-1ba29327fb0c
7-
github.com/arduino/go-paths-helper v1.3.2
7+
github.com/arduino/go-paths-helper v1.4.0
88
github.com/arduino/go-properties-orderedmap v1.3.0
99
github.com/arduino/go-timeutils v0.0.0-20171220113728-d1dd9e313b1b
1010
github.com/arduino/go-win32-utils v0.0.0-20180330194947-ed041402e83b

Diff for: ‎go.sum

+2
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ github.com/arduino/go-paths-helper v1.0.1/go.mod h1:HpxtKph+g238EJHq4geEPv9p+gl3
1717
github.com/arduino/go-paths-helper v1.2.0/go.mod h1:HpxtKph+g238EJHq4geEPv9p+gl3v5YYu35Yb+w31Ck=
1818
github.com/arduino/go-paths-helper v1.3.2 h1:5U9TSKQODiwSVgTxskC0PNl0l0Vf40GUlp99Zy2SK8w=
1919
github.com/arduino/go-paths-helper v1.3.2/go.mod h1:HpxtKph+g238EJHq4geEPv9p+gl3v5YYu35Yb+w31Ck=
20+
github.com/arduino/go-paths-helper v1.4.0 h1:ilnseAdxmN1bFnLxxXHRtcdmt9jBf3O4jtYfWfqule4=
21+
github.com/arduino/go-paths-helper v1.4.0/go.mod h1:V82BWgAAp4IbmlybxQdk9Bpkz8M4Qyx+RAFKaG9NuvU=
2022
github.com/arduino/go-properties-orderedmap v1.3.0 h1:4No/vQopB36e7WUIk6H6TxiSEJPiMrVOCZylYmua39o=
2123
github.com/arduino/go-properties-orderedmap v1.3.0/go.mod h1:DKjD2VXY/NZmlingh4lSFMEYCVubfeArCsGPGDwb2yk=
2224
github.com/arduino/go-timeutils v0.0.0-20171220113728-d1dd9e313b1b h1:9hDi4F2st6dbLC3y4i02zFT5quS4X6iioWifGlVwfy4=

Diff for: ‎legacy/builder/builder.go

+4
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,10 @@ func (s *Builder) Run(ctx *types.Context) error {
9696

9797
mainErr := runCommands(ctx, commands)
9898

99+
if ctx.CompilationDatabase != nil {
100+
ctx.CompilationDatabase.SaveToFile()
101+
}
102+
99103
commands = []types.Command{
100104
&PrintUsedAndNotUsedLibraries{SketchError: mainErr != nil},
101105

Diff for: ‎legacy/builder/builder_utils/utils.go

+20-9
Original file line numberDiff line numberDiff line change
@@ -247,18 +247,24 @@ func compileFileWithRecipe(ctx *types.Context, sourcePath *paths.Path, source *p
247247
if err != nil {
248248
return nil, errors.WithStack(err)
249249
}
250-
if !objIsUpToDate {
251-
command, err := PrepareCommandForRecipe(properties, recipe, false)
252-
if err != nil {
253-
return nil, errors.WithStack(err)
254-
}
255-
250+
command, err := PrepareCommandForRecipe(properties, recipe, false)
251+
if err != nil {
252+
return nil, errors.WithStack(err)
253+
}
254+
if ctx.CompilationDatabase != nil {
255+
ctx.CompilationDatabase.Add(source, command)
256+
}
257+
if !objIsUpToDate && !ctx.OnlyUpdateCompilationDatabase {
256258
_, _, err = utils.ExecCommand(ctx, command, utils.ShowIfVerbose /* stdout */, utils.Show /* stderr */)
257259
if err != nil {
258260
return nil, errors.WithStack(err)
259261
}
260262
} else if ctx.Verbose {
261-
logger.Println(constants.LOG_LEVEL_INFO, constants.MSG_USING_PREVIOUS_COMPILED_FILE, objectFile)
263+
if objIsUpToDate {
264+
logger.Println(constants.LOG_LEVEL_INFO, constants.MSG_USING_PREVIOUS_COMPILED_FILE, objectFile)
265+
} else {
266+
logger.Println("info", "Skipping compile of: {0}", objectFile)
267+
}
262268
}
263269

264270
return objectFile, nil
@@ -452,10 +458,15 @@ func ArchiveCompiledFiles(ctx *types.Context, buildPath *paths.Path, archiveFile
452458
logger := ctx.GetLogger()
453459
archiveFilePath := buildPath.JoinPath(archiveFile)
454460

455-
rebuildArchive := false
461+
if ctx.OnlyUpdateCompilationDatabase {
462+
if ctx.Verbose {
463+
logger.Println("info", "Skipping archive creation of: {0}", archiveFilePath)
464+
}
465+
return archiveFilePath, nil
466+
}
456467

457468
if archiveFileStat, err := archiveFilePath.Stat(); err == nil {
458-
469+
rebuildArchive := false
459470
for _, objectFile := range objectFilesToArchive {
460471
objectFileStat, err := objectFile.Stat()
461472
if err != nil || objectFileStat.ModTime().After(archiveFileStat.ModTime()) {

Diff for: ‎legacy/builder/merge_sketch_with_bootloader.go

+4
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,10 @@ import (
3131
type MergeSketchWithBootloader struct{}
3232

3333
func (s *MergeSketchWithBootloader) Run(ctx *types.Context) error {
34+
if ctx.OnlyUpdateCompilationDatabase {
35+
return nil
36+
}
37+
3438
buildProperties := ctx.BuildProperties
3539
if !buildProperties.ContainsKey(constants.BUILD_PROPERTIES_BOOTLOADER_NOBLINK) && !buildProperties.ContainsKey(constants.BUILD_PROPERTIES_BOOTLOADER_FILE) {
3640
return nil

Diff for: ‎legacy/builder/phases/core_builder.go

+4-2
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,9 @@ func compileCore(ctx *types.Context, buildPath *paths.Path, buildCachePath *path
9494
archivedCoreName := GetCachedCoreArchiveFileName(buildProperties.Get(constants.BUILD_PROPERTIES_FQBN),
9595
buildProperties.Get("compiler.optimization_flags"), realCoreFolder)
9696
targetArchivedCore = buildCachePath.Join(archivedCoreName)
97-
canUseArchivedCore := !ctx.Clean && !builder_utils.CoreOrReferencedCoreHasChanged(realCoreFolder, targetCoreFolder, targetArchivedCore)
97+
canUseArchivedCore := !ctx.OnlyUpdateCompilationDatabase &&
98+
!ctx.Clean &&
99+
!builder_utils.CoreOrReferencedCoreHasChanged(realCoreFolder, targetCoreFolder, targetArchivedCore)
98100

99101
if canUseArchivedCore {
100102
// use archived core
@@ -116,7 +118,7 @@ func compileCore(ctx *types.Context, buildPath *paths.Path, buildCachePath *path
116118
}
117119

118120
// archive core.a
119-
if targetArchivedCore != nil {
121+
if targetArchivedCore != nil && !ctx.OnlyUpdateCompilationDatabase {
120122
err := archiveFile.CopyTo(targetArchivedCore)
121123
if ctx.Verbose {
122124
if err == nil {

Diff for: ‎legacy/builder/phases/linker.go

+7
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,13 @@ import (
3030
type Linker struct{}
3131

3232
func (s *Linker) Run(ctx *types.Context) error {
33+
if ctx.OnlyUpdateCompilationDatabase {
34+
if ctx.Verbose {
35+
ctx.GetLogger().Println("info", "Skip linking of final executable.")
36+
}
37+
return nil
38+
}
39+
3340
objectFilesSketch := ctx.SketchObjectFiles
3441
objectFilesLibraries := ctx.LibrariesObjectFiles
3542
objectFilesCore := ctx.CoreObjectsFiles

Diff for: ‎legacy/builder/phases/sizer.go

+3-1
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,9 @@ type Sizer struct {
3232
}
3333

3434
func (s *Sizer) Run(ctx *types.Context) error {
35-
35+
if ctx.OnlyUpdateCompilationDatabase {
36+
return nil
37+
}
3638
if s.SketchError {
3739
return nil
3840
}

0 commit comments

Comments
 (0)