[skip-changelog] Added skip-libraries-discovery flag in Compile (#1777)

* Added skip-libraries-discovery flag in Compile

* Always send compile response even in case of error

Author: cmaglie

Diff for: cli/compile/compile.go

@@ -74,9 +74,10 @@ var (
 	// library and libraries sound similar but they're actually different.
 	// library expects a path to the root folder of one single library.
 	// libraries expects a path to a directory containing multiple libraries, similarly to the <directories.user>/libraries path.
-	library   []string // List of paths to libraries root folders. Can be used multiple times for different libraries
-	libraries []string // List of custom libraries dir paths separated by commas. Or can be used multiple times for multiple libraries paths.
-	tr        = i18n.Tr
+	library                []string // List of paths to libraries root folders. Can be used multiple times for different libraries
+	libraries              []string // List of custom libraries dir paths separated by commas. Or can be used multiple times for multiple libraries paths.
+	skipLibrariesDiscovery bool
+	tr                     = i18n.Tr
 )
 
 // NewCommand created a new `compile` command
@@ -135,7 +136,8 @@ func NewCommand() *cobra.Command {
 	compileCommand.Flags().BoolP("export-binaries", "e", false, tr("If set built binaries will be exported to the sketch folder."))
 	compileCommand.Flags().StringVar(&sourceOverrides, "source-override", "", tr("Optional. Path to a .json file that contains a set of replacements of the sketch source code."))
 	compileCommand.Flag("source-override").Hidden = true
-
+	compileCommand.Flags().BoolVar(&skipLibrariesDiscovery, "skip-libraries-discovery", false, "Skip libraries discovery. This flag is provided only for use in language server and other, very specific, use cases. Do not use for normal compiles")
+	compileCommand.Flag("skip-libraries-discovery").Hidden = true
 	configuration.Settings.BindPFlag("sketch.always_export_binaries", compileCommand.Flags().Lookup("export-binaries"))
 
 	compileCommand.Flags().MarkDeprecated("build-properties", tr("please use --build-property instead."))
@@ -220,6 +222,7 @@ func runCompileCommand(cmd *cobra.Command, args []string) {
 		KeysKeychain:                  keysKeychain,
 		SignKey:                       signKey,
 		EncryptKey:                    encryptKey,
+		SkipLibrariesDiscovery:        skipLibrariesDiscovery,
 	}
 	compileStdOut := new(bytes.Buffer)
 	compileStdErr := new(bytes.Buffer)

Diff for: commands/compile/compile.go

@@ -114,6 +114,7 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream
 	if pm.GetProfile() != nil {
 		builderCtx.LibrariesManager = lm
 	}
+	builderCtx.UseCachedLibrariesResolution = req.GetSkipLibrariesDiscovery()
 	builderCtx.FQBN = fqbn
 	builderCtx.SketchLocation = sk.FullPath
 	builderCtx.ProgressCB = progressCB

Diff for: commands/daemon/daemon.go

@@ -268,18 +268,19 @@ func (s *ArduinoCoreServerImpl) LoadSketch(ctx context.Context, req *rpc.LoadSke
 func (s *ArduinoCoreServerImpl) Compile(req *rpc.CompileRequest, stream rpc.ArduinoCoreService_CompileServer) error {
 	outStream, outCtx := utils.FeedStreamTo(func(data []byte) { stream.Send(&rpc.CompileResponse{OutStream: data}) })
 	errStream, errCtx := utils.FeedStreamTo(func(data []byte) { stream.Send(&rpc.CompileResponse{ErrStream: data}) })
-	resp, err := compile.Compile(
+	compileResp, compileErr := compile.Compile(
 		stream.Context(), req, outStream, errStream,
 		func(p *rpc.TaskProgress) { stream.Send(&rpc.CompileResponse{Progress: p}) },
 		false) // Set debug to false
 	outStream.Close()
 	errStream.Close()
-	if err != nil {
-		return convertErrorToRPCStatus(err)
-	}
 	<-outCtx.Done()
 	<-errCtx.Done()
-	return stream.Send(resp)
+	compileRespSendErr := stream.Send(compileResp)
+	if compileErr != nil {
+		return convertErrorToRPCStatus(compileErr)
+	}
+	return compileRespSendErr
 }
 
 // PlatformInstall FIXMEDOC

Diff for: legacy/builder/container_find_includes.go

@@ -122,6 +122,19 @@ func (s *ContainerFindIncludes) Run(ctx *types.Context) error {
 }
 
 func (s *ContainerFindIncludes) findIncludes(ctx *types.Context) error {
+	librariesResolutionCache := ctx.BuildPath.Join("libraries.cache")
+	if ctx.UseCachedLibrariesResolution && librariesResolutionCache.Exist() {
+		if d, err := librariesResolutionCache.ReadFile(); err != nil {
+			return err
+		} else if err := json.Unmarshal(d, &ctx.IncludeFolders); err != nil {
+			return err
+		}
+		if ctx.Verbose {
+			ctx.Info("Using cached library discovery: " + librariesResolutionCache.String())
+		}
+		return nil
+	}
+
 	cachePath := ctx.BuildPath.Join("includes.cache")
 	cache := readCache(cachePath)
 
@@ -130,38 +143,44 @@ func (s *ContainerFindIncludes) findIncludes(ctx *types.Context) error {
 		appendIncludeFolder(ctx, cache, nil, "", ctx.BuildProperties.GetPath("build.variant.path"))
 	}
 
-	sketch := ctx.Sketch
-	mergedfile, err := types.MakeSourceFile(ctx, sketch, paths.New(sketch.MainFile.Base()+".cpp"))
-	if err != nil {
-		return errors.WithStack(err)
-	}
-	ctx.CollectedSourceFiles.Push(mergedfile)
+	if !ctx.UseCachedLibrariesResolution {
+		sketch := ctx.Sketch
+		mergedfile, err := types.MakeSourceFile(ctx, sketch, paths.New(sketch.MainFile.Base()+".cpp"))
+		if err != nil {
+			return errors.WithStack(err)
+		}
+		ctx.CollectedSourceFiles.Push(mergedfile)
 
-	sourceFilePaths := ctx.CollectedSourceFiles
-	queueSourceFilesFromFolder(ctx, sourceFilePaths, sketch, ctx.SketchBuildPath, false /* recurse */)
-	srcSubfolderPath := ctx.SketchBuildPath.Join("src")
-	if srcSubfolderPath.IsDir() {
-		queueSourceFilesFromFolder(ctx, sourceFilePaths, sketch, srcSubfolderPath, true /* recurse */)
-	}
+		sourceFilePaths := ctx.CollectedSourceFiles
+		queueSourceFilesFromFolder(ctx, sourceFilePaths, sketch, ctx.SketchBuildPath, false /* recurse */)
+		srcSubfolderPath := ctx.SketchBuildPath.Join("src")
+		if srcSubfolderPath.IsDir() {
+			queueSourceFilesFromFolder(ctx, sourceFilePaths, sketch, srcSubfolderPath, true /* recurse */)
+		}
 
-	for !sourceFilePaths.Empty() {
-		err := findIncludesUntilDone(ctx, cache, sourceFilePaths.Pop())
-		if err != nil {
-			cachePath.Remove()
+		for !sourceFilePaths.Empty() {
+			err := findIncludesUntilDone(ctx, cache, sourceFilePaths.Pop())
+			if err != nil {
+				cachePath.Remove()
+				return errors.WithStack(err)
+			}
+		}
+
+		// Finalize the cache
+		cache.ExpectEnd()
+		if err := writeCache(cache, cachePath); err != nil {
 			return errors.WithStack(err)
 		}
 	}
 
-	// Finalize the cache
-	cache.ExpectEnd()
-	err = writeCache(cache, cachePath)
-	if err != nil {
+	if err := runCommand(ctx, &FailIfImportedLibraryIsWrong{}); err != nil {
 		return errors.WithStack(err)
 	}
 
-	err = runCommand(ctx, &FailIfImportedLibraryIsWrong{})
-	if err != nil {
-		return errors.WithStack(err)
+	if d, err := json.Marshal(ctx.IncludeFolders); err != nil {
+		return err
+	} else if err := librariesResolutionCache.WriteFile(d); err != nil {
+		return err
 	}
 
 	return nil

Diff for: legacy/builder/libraries_loader.go

@@ -26,7 +26,12 @@ import (
 type LibrariesLoader struct{}
 
 func (s *LibrariesLoader) Run(ctx *types.Context) error {
-	if ctx.LibrariesManager == nil {
+	if ctx.UseCachedLibrariesResolution {
+		// Since we are using the cached libraries resolution
+		// the library manager is not needed.
+		lm := librariesmanager.NewLibraryManager(nil, nil)
+		ctx.LibrariesManager = lm
+	} else if ctx.LibrariesManager == nil {
 		lm := librariesmanager.NewLibraryManager(nil, nil)
 		ctx.LibrariesManager = lm
 

Diff for: legacy/builder/types/context.go

@@ -120,12 +120,12 @@ type Context struct {
 	WarningsLevel string
 
 	// Libraries handling
-	LibrariesManager           *librariesmanager.LibrariesManager
-	LibrariesResolver          *librariesresolver.Cpp
-	ImportedLibraries          libraries.List
-	LibrariesResolutionResults map[string]LibraryResolutionResult
-	IncludeFolders             paths.PathList
-	//OutputGccMinusM            string
+	LibrariesManager             *librariesmanager.LibrariesManager
+	LibrariesResolver            *librariesresolver.Cpp
+	ImportedLibraries            libraries.List
+	LibrariesResolutionResults   map[string]LibraryResolutionResult
+	IncludeFolders               paths.PathList
+	UseCachedLibrariesResolution bool
 
 	// C++ Parsing
 	CTagsOutput                 string