From e0819ed3c59b23864f76202b5a3c771326e73d79 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Tue, 16 Oct 2018 10:32:25 +0200 Subject: [PATCH 1/3] Refactor ExecCommand to cache stdout/stderr This commits doesn't change the functionality of ExecCommand EXCEPT for programs that show progress bars or similar. For this kind of utilities, please use Stream "type" --- utils/utils.go | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/utils/utils.go b/utils/utils.go index 0f9ae060..9b0c2d46 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -281,9 +281,10 @@ func PrintableCommand(parts []string) string { const ( Ignore = 0 // Redirect to null - Show = 1 // Show on stdout/stderr as normal - ShowIfVerbose = 2 // Show if verbose is set, Ignore otherwise - Capture = 3 // Capture into buffer + Show = 1 // Capture and print the stream + ShowIfVerbose = 2 // Capture and print the stream only if verbose + Capture = 3 // Capture into buffer and don't print + Stream = 4 // Stream data as it comes in, no capture ) func ExecCommand(ctx *types.Context, command *exec.Cmd, stdout int, stderr int) ([]byte, []byte, error) { @@ -291,17 +292,17 @@ func ExecCommand(ctx *types.Context, command *exec.Cmd, stdout int, stderr int) ctx.GetLogger().UnformattedFprintln(os.Stdout, PrintableCommand(command.Args)) } - if stdout == Capture { + if stdout != Stream { buffer := &bytes.Buffer{} command.Stdout = buffer - } else if stdout == Show || stdout == ShowIfVerbose && ctx.Verbose { + } else { command.Stdout = os.Stdout } - if stderr == Capture { + if stderr != Stream { buffer := &bytes.Buffer{} command.Stderr = buffer - } else if stderr == Show || stderr == ShowIfVerbose && ctx.Verbose { + } else { command.Stderr = os.Stderr } @@ -313,6 +314,7 @@ func ExecCommand(ctx *types.Context, command *exec.Cmd, stdout int, stderr int) err = command.Wait() var outbytes, errbytes []byte + // this operation is a no-op in case of streaming if buf, ok := command.Stdout.(*bytes.Buffer); ok { outbytes = buf.Bytes() } @@ -320,6 +322,14 @@ func ExecCommand(ctx *types.Context, command *exec.Cmd, stdout int, stderr int) errbytes = buf.Bytes() } + if stdout == Show || (stdout == ShowIfVerbose && ctx.Verbose) { + ctx.GetLogger().UnformattedWrite(os.Stdout, outbytes) + } + + if stderr == Show || (stderr == ShowIfVerbose && ctx.Verbose) { + ctx.GetLogger().UnformattedWrite(os.Stderr, errbytes) + } + return outbytes, errbytes, i18n.WrapError(err) } From afbcc4d905543d2d7c8532984ff9f5fb59fc2658 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Tue, 16 Oct 2018 10:35:47 +0200 Subject: [PATCH 2/3] Introduce OutputCache file This file should cache warnings and other compiler output between compilations. Directly maps to ctx.OutputCache map --- builder.go | 2 ++ constants/constants.go | 1 + container_build_options.go | 1 + load_previous_build_options.go | 32 +++++++++++++++++++++++++++++--- store_build_options_map.go | 12 +++++++++++- types/context.go | 6 ++++++ 6 files changed, 50 insertions(+), 4 deletions(-) diff --git a/builder.go b/builder.go index 6c143996..2ec1ebac 100644 --- a/builder.go +++ b/builder.go @@ -129,6 +129,8 @@ func (s *Builder) Run(ctx *types.Context) error { &PrintUsedLibrariesIfVerbose{}, + &StoreOutputCacheMap{}, + &ExportProjectCMake{SketchError: mainErr != nil}, &phases.Sizer{SketchError: mainErr != nil}, diff --git a/constants/constants.go b/constants/constants.go index 366b22b0..6ad6df5a 100644 --- a/constants/constants.go +++ b/constants/constants.go @@ -90,6 +90,7 @@ const FILE_PLATFORM_LOCAL_TXT = "platform.local.txt" const FILE_PLATFORM_TXT = "platform.txt" const FILE_PROGRAMMERS_TXT = "programmers.txt" const FILE_INCLUDES_CACHE = "includes.cache" +const FILE_OUTPUT_CACHE = "output.cache" const FOLDER_BOOTLOADERS = "bootloaders" const FOLDER_CORE = "core" const FOLDER_CORES = "cores" diff --git a/container_build_options.go b/container_build_options.go index d2e22513..7658d2fa 100644 --- a/container_build_options.go +++ b/container_build_options.go @@ -41,6 +41,7 @@ func (s *ContainerBuildOptions) Run(ctx *types.Context) error { &CreateBuildOptionsMap{}, &LoadPreviousBuildOptionsMap{}, &WipeoutBuildPathIfBuildOptionsChanged{}, + &LoadPreviousOutputCacheMap{}, &StoreBuildOptionsMap{}, } diff --git a/load_previous_build_options.go b/load_previous_build_options.go index 3b9bd5d0..a7d797bb 100644 --- a/load_previous_build_options.go +++ b/load_previous_build_options.go @@ -30,12 +30,14 @@ package builder import ( - "github.com/arduino/arduino-builder/constants" - "github.com/arduino/arduino-builder/i18n" - "github.com/arduino/arduino-builder/types" + "encoding/json" "io/ioutil" "os" "path/filepath" + + "github.com/arduino/arduino-builder/constants" + "github.com/arduino/arduino-builder/i18n" + "github.com/arduino/arduino-builder/types" ) type LoadPreviousBuildOptionsMap struct{} @@ -60,3 +62,27 @@ func (s *LoadPreviousBuildOptionsMap) Run(ctx *types.Context) error { return nil } + +type LoadPreviousOutputCacheMap struct{} + +func (s *LoadPreviousOutputCacheMap) Run(ctx *types.Context) error { + cachedWarningsFile := filepath.Join(ctx.BuildPath, constants.FILE_OUTPUT_CACHE) + ctx.OutputCache = make(map[string]types.Streams) + + _, err := os.Stat(cachedWarningsFile) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return i18n.WrapError(err) + } + + bytes, err := ioutil.ReadFile(cachedWarningsFile) + if err != nil { + return i18n.WrapError(err) + } + + json.Unmarshal(bytes, &ctx.OutputCache) + + return nil +} diff --git a/store_build_options_map.go b/store_build_options_map.go index dc0c3220..405b2256 100644 --- a/store_build_options_map.go +++ b/store_build_options_map.go @@ -30,10 +30,12 @@ package builder import ( + "encoding/json" + "path/filepath" + "github.com/arduino/arduino-builder/constants" "github.com/arduino/arduino-builder/types" "github.com/arduino/arduino-builder/utils" - "path/filepath" ) type StoreBuildOptionsMap struct{} @@ -42,3 +44,11 @@ func (s *StoreBuildOptionsMap) Run(ctx *types.Context) error { utils.WriteFile(filepath.Join(ctx.BuildPath, constants.BUILD_OPTIONS_FILE), ctx.BuildOptionsJson) return nil } + +type StoreOutputCacheMap struct{} + +func (s *StoreOutputCacheMap) Run(ctx *types.Context) error { + data, _ := json.Marshal(ctx.OutputCache) + utils.WriteFile(filepath.Join(ctx.BuildPath, constants.FILE_OUTPUT_CACHE), string(data)) + return nil +} diff --git a/types/context.go b/types/context.go index e9368c08..9229db2b 100644 --- a/types/context.go +++ b/types/context.go @@ -14,6 +14,11 @@ type ProgressStruct struct { Progress float64 } +type Streams struct { + Stdout []byte + Stderr []byte +} + // Context structure type Context struct { // Build options @@ -65,6 +70,7 @@ type Context struct { CodeCompletions string WarningsLevel string + OutputCache map[string]Streams // Libraries handling Libraries []*Library From 672b0063394580e1a8bce640507164d51b35a393 Mon Sep 17 00:00:00 2001 From: Martino Facchin Date: Tue, 16 Oct 2018 10:39:38 +0200 Subject: [PATCH 3/3] Display cached warnings for previously compiled files The mutex is needed because the map could be concurrently accessed (when using parallel compilation) --- builder_utils/utils.go | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/builder_utils/utils.go b/builder_utils/utils.go index 9b4cb536..c6728d57 100644 --- a/builder_utils/utils.go +++ b/builder_utils/utils.go @@ -207,6 +207,8 @@ func compileFilesWithRecipe(ctx *types.Context, objectFiles []string, sourcePath } } +var outputCacheMutex sync.Mutex + func compileFileWithRecipe(ctx *types.Context, sourcePath string, source string, buildPath string, buildProperties properties.Map, includes []string, recipe string) (string, error) { logger := ctx.GetLogger() properties := buildProperties.Clone() @@ -230,12 +232,26 @@ func compileFileWithRecipe(ctx *types.Context, sourcePath string, source string, } if !objIsUpToDate { - _, _, err = ExecRecipe(ctx, properties, recipe, false /* stdout */, utils.ShowIfVerbose /* stderr */, utils.Show) + stdout, stderr, err := ExecRecipe(ctx, properties, recipe, false /* stdout */, utils.ShowIfVerbose /* stderr */, utils.Show) if err != nil { return "", i18n.WrapError(err) } - } else if ctx.Verbose { - logger.Println(constants.LOG_LEVEL_INFO, constants.MSG_USING_PREVIOUS_COMPILED_FILE, properties[constants.BUILD_PROPERTIES_OBJECT_FILE]) + outputCacheMutex.Lock() + ctx.OutputCache[source] = types.Streams{ + Stderr: stderr, + Stdout: stdout, + } + outputCacheMutex.Unlock() + } else { + if ctx.Verbose { + logger.Println(constants.LOG_LEVEL_INFO, constants.MSG_USING_PREVIOUS_COMPILED_FILE, properties[constants.BUILD_PROPERTIES_OBJECT_FILE]) + } + if len(ctx.OutputCache[source].Stderr) > 0 && ctx.WarningsLevel != "none" { + logger.UnformattedWrite(os.Stderr, ctx.OutputCache[source].Stderr) + } + if len(ctx.OutputCache[source].Stdout) > 0 && ctx.WarningsLevel != "none" { + logger.UnformattedWrite(os.Stdout, ctx.OutputCache[source].Stdout) + } } return properties[constants.BUILD_PROPERTIES_OBJECT_FILE], nil