From 4327e72f4da5e193e4bc86ee77c799c8a852091f Mon Sep 17 00:00:00 2001 From: Matthijs Kooijman Date: Thu, 2 Mar 2017 13:51:16 +0100 Subject: [PATCH 1/5] Pass debug level and logger to ObjFileIsUpToDate This does not change any behaviour, but only passed these variables along through the entire stack of functions leading up to ObjFileIsUpToDate. This prepares for the next commit. Signed-off-by: Matthijs Kooijman --- src/arduino.cc/builder/builder_utils/utils.go | 28 +++++++++---------- .../builder/container_find_includes.go | 2 +- src/arduino.cc/builder/phases/core_builder.go | 8 +++--- .../builder/phases/libraries_builder.go | 15 +++++----- .../builder/phases/sketch_builder.go | 5 ++-- 5 files changed, 30 insertions(+), 28 deletions(-) diff --git a/src/arduino.cc/builder/builder_utils/utils.go b/src/arduino.cc/builder/builder_utils/utils.go index d21420a3..9b5d8f1a 100644 --- a/src/arduino.cc/builder/builder_utils/utils.go +++ b/src/arduino.cc/builder/builder_utils/utils.go @@ -43,8 +43,8 @@ import ( "arduino.cc/properties" ) -func CompileFilesRecursive(objectFiles []string, sourcePath string, buildPath string, buildProperties properties.Map, includes []string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) { - objectFiles, err := CompileFiles(objectFiles, sourcePath, false, buildPath, buildProperties, includes, verbose, warningsLevel, logger) +func CompileFilesRecursive(objectFiles []string, sourcePath string, buildPath string, buildProperties properties.Map, includes []string, verbose bool, warningsLevel string, debugLevel int, logger i18n.Logger) ([]string, error) { + objectFiles, err := CompileFiles(objectFiles, sourcePath, false, buildPath, buildProperties, includes, verbose, warningsLevel, debugLevel, logger) if err != nil { return nil, i18n.WrapError(err) } @@ -55,7 +55,7 @@ func CompileFilesRecursive(objectFiles []string, sourcePath string, buildPath st } for _, folder := range folders { - objectFiles, err = CompileFilesRecursive(objectFiles, filepath.Join(sourcePath, folder.Name()), filepath.Join(buildPath, folder.Name()), buildProperties, includes, verbose, warningsLevel, logger) + objectFiles, err = CompileFilesRecursive(objectFiles, filepath.Join(sourcePath, folder.Name()), filepath.Join(buildPath, folder.Name()), buildProperties, includes, verbose, warningsLevel, debugLevel, logger) if err != nil { return nil, i18n.WrapError(err) } @@ -64,28 +64,28 @@ func CompileFilesRecursive(objectFiles []string, sourcePath string, buildPath st return objectFiles, nil } -func CompileFiles(objectFiles []string, sourcePath string, recurse bool, buildPath string, buildProperties properties.Map, includes []string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) { - objectFiles, err := compileFilesWithExtensionWithRecipe(objectFiles, sourcePath, recurse, buildPath, buildProperties, includes, ".S", constants.RECIPE_S_PATTERN, verbose, warningsLevel, logger) +func CompileFiles(objectFiles []string, sourcePath string, recurse bool, buildPath string, buildProperties properties.Map, includes []string, verbose bool, warningsLevel string, debugLevel int, logger i18n.Logger) ([]string, error) { + objectFiles, err := compileFilesWithExtensionWithRecipe(objectFiles, sourcePath, recurse, buildPath, buildProperties, includes, ".S", constants.RECIPE_S_PATTERN, verbose, warningsLevel, debugLevel, logger) if err != nil { return nil, i18n.WrapError(err) } - objectFiles, err = compileFilesWithExtensionWithRecipe(objectFiles, sourcePath, recurse, buildPath, buildProperties, includes, ".c", constants.RECIPE_C_PATTERN, verbose, warningsLevel, logger) + objectFiles, err = compileFilesWithExtensionWithRecipe(objectFiles, sourcePath, recurse, buildPath, buildProperties, includes, ".c", constants.RECIPE_C_PATTERN, verbose, warningsLevel, debugLevel, logger) if err != nil { return nil, i18n.WrapError(err) } - objectFiles, err = compileFilesWithExtensionWithRecipe(objectFiles, sourcePath, recurse, buildPath, buildProperties, includes, ".cpp", constants.RECIPE_CPP_PATTERN, verbose, warningsLevel, logger) + objectFiles, err = compileFilesWithExtensionWithRecipe(objectFiles, sourcePath, recurse, buildPath, buildProperties, includes, ".cpp", constants.RECIPE_CPP_PATTERN, verbose, warningsLevel, debugLevel, logger) if err != nil { return nil, i18n.WrapError(err) } return objectFiles, nil } -func compileFilesWithExtensionWithRecipe(objectFiles []string, sourcePath string, recurse bool, buildPath string, buildProperties properties.Map, includes []string, extension string, recipe string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) { +func compileFilesWithExtensionWithRecipe(objectFiles []string, sourcePath string, recurse bool, buildPath string, buildProperties properties.Map, includes []string, extension string, recipe string, verbose bool, warningsLevel string, debugLevel int, logger i18n.Logger) ([]string, error) { sources, err := findFilesInFolder(sourcePath, extension, recurse) if err != nil { return nil, i18n.WrapError(err) } - return compileFilesWithRecipe(objectFiles, sourcePath, sources, buildPath, buildProperties, includes, recipe, verbose, warningsLevel, logger) + return compileFilesWithRecipe(objectFiles, sourcePath, sources, buildPath, buildProperties, includes, recipe, verbose, warningsLevel, debugLevel, logger) } func findFilesInFolder(sourcePath string, extension string, recurse bool) ([]string, error) { @@ -116,9 +116,9 @@ func findFilesInFolder(sourcePath string, extension string, recurse bool) ([]str return sources, nil } -func compileFilesWithRecipe(objectFiles []string, sourcePath string, sources []string, buildPath string, buildProperties properties.Map, includes []string, recipe string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) { +func compileFilesWithRecipe(objectFiles []string, sourcePath string, sources []string, buildPath string, buildProperties properties.Map, includes []string, recipe string, verbose bool, warningsLevel string, debugLevel int, logger i18n.Logger) ([]string, error) { for _, source := range sources { - objectFile, err := compileFileWithRecipe(sourcePath, source, buildPath, buildProperties, includes, recipe, verbose, warningsLevel, logger) + objectFile, err := compileFileWithRecipe(sourcePath, source, buildPath, buildProperties, includes, recipe, verbose, warningsLevel, debugLevel, logger) if err != nil { return nil, i18n.WrapError(err) } @@ -128,7 +128,7 @@ func compileFilesWithRecipe(objectFiles []string, sourcePath string, sources []s return objectFiles, nil } -func compileFileWithRecipe(sourcePath string, source string, buildPath string, buildProperties properties.Map, includes []string, recipe string, verbose bool, warningsLevel string, logger i18n.Logger) (string, error) { +func compileFileWithRecipe(sourcePath string, source string, buildPath string, buildProperties properties.Map, includes []string, recipe string, verbose bool, warningsLevel string, debugLevel int, logger i18n.Logger) (string, error) { properties := buildProperties.Clone() properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS] = properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS+"."+warningsLevel] properties[constants.BUILD_PROPERTIES_INCLUDES] = strings.Join(includes, constants.SPACE) @@ -144,7 +144,7 @@ func compileFileWithRecipe(sourcePath string, source string, buildPath string, b return "", i18n.WrapError(err) } - objIsUpToDate, err := ObjFileIsUpToDate(properties[constants.BUILD_PROPERTIES_SOURCE_FILE], properties[constants.BUILD_PROPERTIES_OBJECT_FILE], filepath.Join(buildPath, relativeSource+".d")) + objIsUpToDate, err := ObjFileIsUpToDate(properties[constants.BUILD_PROPERTIES_SOURCE_FILE], properties[constants.BUILD_PROPERTIES_OBJECT_FILE], filepath.Join(buildPath, relativeSource+".d"), debugLevel, logger) if err != nil { return "", i18n.WrapError(err) } @@ -161,7 +161,7 @@ func compileFileWithRecipe(sourcePath string, source string, buildPath string, b return properties[constants.BUILD_PROPERTIES_OBJECT_FILE], nil } -func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile string) (bool, error) { +func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile string, debugLevel int, logger i18n.Logger) (bool, error) { sourceFile = filepath.Clean(sourceFile) objectFile = filepath.Clean(objectFile) dependencyFile = filepath.Clean(dependencyFile) diff --git a/src/arduino.cc/builder/container_find_includes.go b/src/arduino.cc/builder/container_find_includes.go index cc14a4f2..7ac9412a 100644 --- a/src/arduino.cc/builder/container_find_includes.go +++ b/src/arduino.cc/builder/container_find_includes.go @@ -303,7 +303,7 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile t // TODO: This reads the dependency file, but the actual building // does it again. Should the result be somehow cached? Perhaps // remove the object file if it is found to be stale? - unchanged, err := builder_utils.ObjFileIsUpToDate(sourcePath, sourceFile.ObjectPath(ctx), sourceFile.DepfilePath(ctx)) + unchanged, err := builder_utils.ObjFileIsUpToDate(sourcePath, sourceFile.ObjectPath(ctx), sourceFile.DepfilePath(ctx), ctx.DebugLevel, ctx.GetLogger()) if err != nil { return i18n.WrapError(err) } diff --git a/src/arduino.cc/builder/phases/core_builder.go b/src/arduino.cc/builder/phases/core_builder.go index 0a0c78c6..b43d3f0f 100644 --- a/src/arduino.cc/builder/phases/core_builder.go +++ b/src/arduino.cc/builder/phases/core_builder.go @@ -52,7 +52,7 @@ func (s *CoreBuilder) Run(ctx *types.Context) error { return i18n.WrapError(err) } - archiveFile, objectFiles, err := compileCore(coreBuildPath, buildProperties, verbose, warningsLevel, logger) + archiveFile, objectFiles, err := compileCore(coreBuildPath, buildProperties, verbose, warningsLevel, ctx.DebugLevel, logger) if err != nil { return i18n.WrapError(err) } @@ -63,7 +63,7 @@ func (s *CoreBuilder) Run(ctx *types.Context) error { return nil } -func compileCore(buildPath string, buildProperties properties.Map, verbose bool, warningsLevel string, logger i18n.Logger) (string, []string, error) { +func compileCore(buildPath string, buildProperties properties.Map, verbose bool, warningsLevel string, debugLevel int, logger i18n.Logger) (string, []string, error) { coreFolder := buildProperties[constants.BUILD_PROPERTIES_BUILD_CORE_PATH] variantFolder := buildProperties[constants.BUILD_PROPERTIES_BUILD_VARIANT_PATH] @@ -78,13 +78,13 @@ func compileCore(buildPath string, buildProperties properties.Map, verbose bool, variantObjectFiles := []string{} if variantFolder != constants.EMPTY_STRING { - variantObjectFiles, err = builder_utils.CompileFiles(variantObjectFiles, variantFolder, true, buildPath, buildProperties, includes, verbose, warningsLevel, logger) + variantObjectFiles, err = builder_utils.CompileFiles(variantObjectFiles, variantFolder, true, buildPath, buildProperties, includes, verbose, warningsLevel, debugLevel, logger) if err != nil { return "", nil, i18n.WrapError(err) } } - coreObjectFiles, err := builder_utils.CompileFiles([]string{}, coreFolder, true, buildPath, buildProperties, includes, verbose, warningsLevel, logger) + coreObjectFiles, err := builder_utils.CompileFiles([]string{}, coreFolder, true, buildPath, buildProperties, includes, verbose, warningsLevel, debugLevel, logger) if err != nil { return "", nil, i18n.WrapError(err) } diff --git a/src/arduino.cc/builder/phases/libraries_builder.go b/src/arduino.cc/builder/phases/libraries_builder.go index 816c512f..ec73c3ac 100644 --- a/src/arduino.cc/builder/phases/libraries_builder.go +++ b/src/arduino.cc/builder/phases/libraries_builder.go @@ -50,6 +50,7 @@ func (s *LibrariesBuilder) Run(ctx *types.Context) error { libraries := ctx.ImportedLibraries verbose := ctx.Verbose warningsLevel := ctx.WarningsLevel + debugLevel := ctx.DebugLevel logger := ctx.GetLogger() err := utils.EnsureFolderExists(librariesBuildPath) @@ -57,7 +58,7 @@ func (s *LibrariesBuilder) Run(ctx *types.Context) error { return i18n.WrapError(err) } - objectFiles, err := compileLibraries(libraries, librariesBuildPath, buildProperties, includes, verbose, warningsLevel, logger) + objectFiles, err := compileLibraries(libraries, librariesBuildPath, buildProperties, includes, verbose, warningsLevel, debugLevel, logger) if err != nil { return i18n.WrapError(err) } @@ -67,10 +68,10 @@ func (s *LibrariesBuilder) Run(ctx *types.Context) error { return nil } -func compileLibraries(libraries []*types.Library, buildPath string, buildProperties properties.Map, includes []string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) { +func compileLibraries(libraries []*types.Library, buildPath string, buildProperties properties.Map, includes []string, verbose bool, warningsLevel string, debugLevel int, logger i18n.Logger) ([]string, error) { objectFiles := []string{} for _, library := range libraries { - libraryObjectFiles, err := compileLibrary(library, buildPath, buildProperties, includes, verbose, warningsLevel, logger) + libraryObjectFiles, err := compileLibrary(library, buildPath, buildProperties, includes, verbose, warningsLevel, debugLevel, logger) if err != nil { return nil, i18n.WrapError(err) } @@ -81,7 +82,7 @@ func compileLibraries(libraries []*types.Library, buildPath string, buildPropert } -func compileLibrary(library *types.Library, buildPath string, buildProperties properties.Map, includes []string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) { +func compileLibrary(library *types.Library, buildPath string, buildProperties properties.Map, includes []string, verbose bool, warningsLevel string, debugLevel int, logger i18n.Logger) ([]string, error) { if verbose { logger.Println(constants.LOG_LEVEL_INFO, "Compiling library \"{0}\"", library.Name) } @@ -94,7 +95,7 @@ func compileLibrary(library *types.Library, buildPath string, buildProperties pr objectFiles := []string{} if library.Layout == types.LIBRARY_RECURSIVE { - objectFiles, err = builder_utils.CompileFilesRecursive(objectFiles, library.SrcFolder, libraryBuildPath, buildProperties, includes, verbose, warningsLevel, logger) + objectFiles, err = builder_utils.CompileFilesRecursive(objectFiles, library.SrcFolder, libraryBuildPath, buildProperties, includes, verbose, warningsLevel, debugLevel, logger) if err != nil { return nil, i18n.WrapError(err) } @@ -109,14 +110,14 @@ func compileLibrary(library *types.Library, buildPath string, buildProperties pr if library.UtilityFolder != "" { includes = append(includes, utils.WrapWithHyphenI(library.UtilityFolder)) } - objectFiles, err = builder_utils.CompileFiles(objectFiles, library.SrcFolder, false, libraryBuildPath, buildProperties, includes, verbose, warningsLevel, logger) + objectFiles, err = builder_utils.CompileFiles(objectFiles, library.SrcFolder, false, libraryBuildPath, buildProperties, includes, verbose, warningsLevel, debugLevel, logger) if err != nil { return nil, i18n.WrapError(err) } if library.UtilityFolder != "" { utilityBuildPath := filepath.Join(libraryBuildPath, constants.LIBRARY_FOLDER_UTILITY) - objectFiles, err = builder_utils.CompileFiles(objectFiles, library.UtilityFolder, false, utilityBuildPath, buildProperties, includes, verbose, warningsLevel, logger) + objectFiles, err = builder_utils.CompileFiles(objectFiles, library.UtilityFolder, false, utilityBuildPath, buildProperties, includes, verbose, warningsLevel, debugLevel, logger) if err != nil { return nil, i18n.WrapError(err) } diff --git a/src/arduino.cc/builder/phases/sketch_builder.go b/src/arduino.cc/builder/phases/sketch_builder.go index 68b5a859..c34dd337 100644 --- a/src/arduino.cc/builder/phases/sketch_builder.go +++ b/src/arduino.cc/builder/phases/sketch_builder.go @@ -48,6 +48,7 @@ func (s *SketchBuilder) Run(ctx *types.Context) error { includes = utils.Map(includes, utils.WrapWithHyphenI) verbose := ctx.Verbose warningsLevel := ctx.WarningsLevel + debugLevel := ctx.DebugLevel logger := ctx.GetLogger() err := utils.EnsureFolderExists(sketchBuildPath) @@ -56,7 +57,7 @@ func (s *SketchBuilder) Run(ctx *types.Context) error { } var objectFiles []string - objectFiles, err = builder_utils.CompileFiles(objectFiles, sketchBuildPath, false, sketchBuildPath, buildProperties, includes, verbose, warningsLevel, logger) + objectFiles, err = builder_utils.CompileFiles(objectFiles, sketchBuildPath, false, sketchBuildPath, buildProperties, includes, verbose, warningsLevel, debugLevel, logger) if err != nil { return i18n.WrapError(err) } @@ -64,7 +65,7 @@ func (s *SketchBuilder) Run(ctx *types.Context) error { // The "src/" subdirectory of a sketch is compiled recursively sketchSrcPath := filepath.Join(sketchBuildPath, constants.SKETCH_FOLDER_SRC) if info, err := os.Stat(sketchSrcPath); err == nil && info.IsDir() { - objectFiles, err = builder_utils.CompileFiles(objectFiles, sketchSrcPath, true, sketchSrcPath, buildProperties, includes, verbose, warningsLevel, logger) + objectFiles, err = builder_utils.CompileFiles(objectFiles, sketchSrcPath, true, sketchSrcPath, buildProperties, includes, verbose, warningsLevel, debugLevel, logger) if err != nil { return i18n.WrapError(err) } From 3a4083ca0e028543fa8cd7d5633b751df751d7d6 Mon Sep 17 00:00:00 2001 From: Matthijs Kooijman Date: Thu, 2 Mar 2017 13:53:54 +0100 Subject: [PATCH 2/5] Let ObjFileIsUpToDate output verbose debug output If -debug-level=20 is passed, whenever the cached file is not usable for whatever reason, a message is displayed. This should help debug caching problems. The messages are hardcoded in the source and not put into `constants`, since they are only debug messages. Signed-off-by: Matthijs Kooijman --- src/arduino.cc/builder/builder_utils/utils.go | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/arduino.cc/builder/builder_utils/utils.go b/src/arduino.cc/builder/builder_utils/utils.go index 9b5d8f1a..8e75966d 100644 --- a/src/arduino.cc/builder/builder_utils/utils.go +++ b/src/arduino.cc/builder/builder_utils/utils.go @@ -166,6 +166,10 @@ func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile string, debugLevel objectFile = filepath.Clean(objectFile) dependencyFile = filepath.Clean(dependencyFile) + if debugLevel >= 20 { + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "Checking previous results for {0} (result = {1}, dep = {2})", sourceFile, objectFile, dependencyFile) + } + sourceFileStat, err := os.Stat(sourceFile) if err != nil { return false, i18n.WrapError(err) @@ -174,6 +178,9 @@ func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile string, debugLevel objectFileStat, err := os.Stat(objectFile) if err != nil { if os.IsNotExist(err) { + if debugLevel >= 20 { + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "Not found: {0}", objectFile) + } return false, nil } else { return false, i18n.WrapError(err) @@ -183,6 +190,9 @@ func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile string, debugLevel dependencyFileStat, err := os.Stat(dependencyFile) if err != nil { if os.IsNotExist(err) { + if debugLevel >= 20 { + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "Not found: {0}", dependencyFile) + } return false, nil } else { return false, i18n.WrapError(err) @@ -190,9 +200,15 @@ func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile string, debugLevel } if sourceFileStat.ModTime().After(objectFileStat.ModTime()) { + if debugLevel >= 20 { + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "{0} newer than {1}", sourceFile, objectFile) + } return false, nil } if sourceFileStat.ModTime().After(dependencyFileStat.ModTime()) { + if debugLevel >= 20 { + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "{0} newer than {1}", sourceFile, dependencyFile) + } return false, nil } @@ -212,10 +228,16 @@ func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile string, debugLevel firstRow := rows[0] if !strings.HasSuffix(firstRow, ":") { + if debugLevel >= 20 { + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "No colon in first line of depfile") + } return false, nil } objFileInDepFile := firstRow[:len(firstRow)-1] if objFileInDepFile != objectFile { + if debugLevel >= 20 { + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "Depfile is about different file: {0}", objFileInDepFile) + } return false, nil } @@ -225,12 +247,22 @@ func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile string, debugLevel if err != nil && !os.IsNotExist(err) { // There is probably a parsing error of the dep file // Ignore the error and trigger a full rebuild anyway + if debugLevel >= 20 { + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "Failed to read: {0}", row) + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, i18n.WrapError(err).Error()) + } return false, nil } if os.IsNotExist(err) { + if debugLevel >= 20 { + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "Not found: {0}", row) + } return false, nil } if depStat.ModTime().After(objectFileStat.ModTime()) { + if debugLevel >= 20 { + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "{0} newer than {1}", row, objectFile) + } return false, nil } } From 824a5ff97d8e89aa8f6882cb598b55cc1fe9caec Mon Sep 17 00:00:00 2001 From: Matthijs Kooijman Date: Thu, 2 Mar 2017 16:24:02 +0100 Subject: [PATCH 3/5] ContainerFindIncludes: Add some temporary variables This slightly cleans up a function call. Signed-off-by: Matthijs Kooijman --- src/arduino.cc/builder/container_find_includes.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/arduino.cc/builder/container_find_includes.go b/src/arduino.cc/builder/container_find_includes.go index 7ac9412a..00dac379 100644 --- a/src/arduino.cc/builder/container_find_includes.go +++ b/src/arduino.cc/builder/container_find_includes.go @@ -289,6 +289,8 @@ func writeCache(cache *includeCache, path string) error { func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile types.SourceFile) error { sourcePath := sourceFile.SourcePath(ctx) + depPath := sourceFile.DepfilePath(ctx) + objPath := sourceFile.ObjectPath(ctx) targetFilePath := utils.NULLFile() // TODO: This should perhaps also compare against the @@ -303,7 +305,7 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile t // TODO: This reads the dependency file, but the actual building // does it again. Should the result be somehow cached? Perhaps // remove the object file if it is found to be stale? - unchanged, err := builder_utils.ObjFileIsUpToDate(sourcePath, sourceFile.ObjectPath(ctx), sourceFile.DepfilePath(ctx), ctx.DebugLevel, ctx.GetLogger()) + unchanged, err := builder_utils.ObjFileIsUpToDate(sourcePath, objPath, depPath, ctx.DebugLevel, ctx.GetLogger()) if err != nil { return i18n.WrapError(err) } From d4e907801c13b058cfbfdd18ce3b45f426519fe3 Mon Sep 17 00:00:00 2001 From: Matthijs Kooijman Date: Thu, 2 Mar 2017 16:27:26 +0100 Subject: [PATCH 4/5] Generalize ObjFileIsUpToDate to BuildResultIsUpToDate This renames this function and adds a resultFile parameter. This resultFile is used for all timestamp checks against other files (source file, dep file, header files from the dep file), while the objectFile passed is now only used to check against the contents of the dep file. Both calls to this function still pass the same filename as objectFile and resultFile, so this commit should not change any behavior. Signed-off-by: Matthijs Kooijman --- src/arduino.cc/builder/builder_utils/utils.go | 19 ++++++++++--------- .../builder/container_find_includes.go | 2 +- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/arduino.cc/builder/builder_utils/utils.go b/src/arduino.cc/builder/builder_utils/utils.go index 8e75966d..7c99a957 100644 --- a/src/arduino.cc/builder/builder_utils/utils.go +++ b/src/arduino.cc/builder/builder_utils/utils.go @@ -144,7 +144,7 @@ func compileFileWithRecipe(sourcePath string, source string, buildPath string, b return "", i18n.WrapError(err) } - objIsUpToDate, err := ObjFileIsUpToDate(properties[constants.BUILD_PROPERTIES_SOURCE_FILE], properties[constants.BUILD_PROPERTIES_OBJECT_FILE], filepath.Join(buildPath, relativeSource+".d"), debugLevel, logger) + objIsUpToDate, err := BuildResultIsUpToDate(properties[constants.BUILD_PROPERTIES_SOURCE_FILE], properties[constants.BUILD_PROPERTIES_OBJECT_FILE], properties[constants.BUILD_PROPERTIES_OBJECT_FILE], filepath.Join(buildPath, relativeSource+".d"), debugLevel, logger) if err != nil { return "", i18n.WrapError(err) } @@ -161,13 +161,14 @@ func compileFileWithRecipe(sourcePath string, source string, buildPath string, b return properties[constants.BUILD_PROPERTIES_OBJECT_FILE], nil } -func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile string, debugLevel int, logger i18n.Logger) (bool, error) { +func BuildResultIsUpToDate(sourceFile, resultFile, objectFile, dependencyFile string, debugLevel int, logger i18n.Logger) (bool, error) { sourceFile = filepath.Clean(sourceFile) + resultFile = filepath.Clean(resultFile) objectFile = filepath.Clean(objectFile) dependencyFile = filepath.Clean(dependencyFile) if debugLevel >= 20 { - logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "Checking previous results for {0} (result = {1}, dep = {2})", sourceFile, objectFile, dependencyFile) + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "Checking build results for {0} (result = {1}, dep = {2})", sourceFile, objectFile, dependencyFile) } sourceFileStat, err := os.Stat(sourceFile) @@ -175,11 +176,11 @@ func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile string, debugLevel return false, i18n.WrapError(err) } - objectFileStat, err := os.Stat(objectFile) + resultFileStat, err := os.Stat(resultFile) if err != nil { if os.IsNotExist(err) { if debugLevel >= 20 { - logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "Not found: {0}", objectFile) + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "Not found: {0}", resultFile) } return false, nil } else { @@ -199,9 +200,9 @@ func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile string, debugLevel } } - if sourceFileStat.ModTime().After(objectFileStat.ModTime()) { + if sourceFileStat.ModTime().After(resultFileStat.ModTime()) { if debugLevel >= 20 { - logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "{0} newer than {1}", sourceFile, objectFile) + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "{0} newer than {1}", sourceFile, resultFile) } return false, nil } @@ -259,9 +260,9 @@ func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile string, debugLevel } return false, nil } - if depStat.ModTime().After(objectFileStat.ModTime()) { + if depStat.ModTime().After(resultFileStat.ModTime()) { if debugLevel >= 20 { - logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "{0} newer than {1}", row, objectFile) + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "{0} newer than {1}", row, resultFile) } return false, nil } diff --git a/src/arduino.cc/builder/container_find_includes.go b/src/arduino.cc/builder/container_find_includes.go index 00dac379..980e33c2 100644 --- a/src/arduino.cc/builder/container_find_includes.go +++ b/src/arduino.cc/builder/container_find_includes.go @@ -305,7 +305,7 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile t // TODO: This reads the dependency file, but the actual building // does it again. Should the result be somehow cached? Perhaps // remove the object file if it is found to be stale? - unchanged, err := builder_utils.ObjFileIsUpToDate(sourcePath, objPath, depPath, ctx.DebugLevel, ctx.GetLogger()) + unchanged, err := builder_utils.BuildResultIsUpToDate(sourcePath, objPath, objPath, depPath, ctx.DebugLevel, ctx.GetLogger()) if err != nil { return i18n.WrapError(err) } From f13ade8f3296e11b00a2014345e8009f61d0e31a Mon Sep 17 00:00:00 2001 From: Matthijs Kooijman Date: Thu, 2 Mar 2017 13:57:04 +0100 Subject: [PATCH 5/5] Let include detection generate .d files too Previously, the including detection process caching relied on the .d files generated by compilation to know what .h files a given source files depends on. This works correctly, but if a project does not compile completely, not all .d files are generated, so not all cached include detection results can be used. In practice, this means that if a there is a compilation error in the first file that is compiled, include detection will run again for all files on the next run. If you have a few errors to solve and a big project, this gets annoying quickly. To fix this, the include detection process should generate .d files itself. At first glance it appears that there is a problematic case where the list of included header files changes, the include detection overwrites the .d file and then compilation only sees the new list (which was generated later than the .o file was generated). However, since this implies that changes are made to an #include directive in the source file itself or one of the files that are still included, this should be detected normally. There is still a corner case when a file is changed during the build, but that was already the case. Since include detections uses `-o /dev/null`, the compiler generates a slightly different .d file. During compilation, a file `foo.cpp.d` is generated in the output directory starting with: /path/to/foo.cpp.o: \ But when just passing `-MMD` to the preproc recipe, it generates a `foo.d` file in the source directory starting with: foo.o: \ To make these equal, `-MF` must be passed during include detection to set the .d filename, and `-MT` must be passed to set the .o filename inside the .d file. To enable this feature, platform.txt should be modified by adding ` -MMD -MF {dep_file} -MT {object_file}` to `preproc.macros.flags` (or `recipe.preproc.macros`). Without any changes to platform.txt, behaviour is unchanged. To allow this, this adds `{dep_file}` and `{object_file}` variables to the build properties for the preproc macros recipe. For consistency, `{dep_file}` is also added during normal compilation, though it is not currently used. --- src/arduino.cc/builder/builder_utils/utils.go | 1 + src/arduino.cc/builder/constants/constants.go | 1 + .../builder/container_add_prototypes.go | 4 +++- .../builder/container_find_includes.go | 10 +++++----- src/arduino.cc/builder/gcc_preproc_runner.go | 16 +++++++++++++--- 5 files changed, 23 insertions(+), 9 deletions(-) diff --git a/src/arduino.cc/builder/builder_utils/utils.go b/src/arduino.cc/builder/builder_utils/utils.go index 7c99a957..08a8ba24 100644 --- a/src/arduino.cc/builder/builder_utils/utils.go +++ b/src/arduino.cc/builder/builder_utils/utils.go @@ -138,6 +138,7 @@ func compileFileWithRecipe(sourcePath string, source string, buildPath string, b return "", i18n.WrapError(err) } properties[constants.BUILD_PROPERTIES_OBJECT_FILE] = filepath.Join(buildPath, relativeSource+".o") + properties[constants.BUILD_PROPERTIES_DEP_FILE] = filepath.Join(buildPath, relativeSource+".d") err = utils.EnsureFolderExists(filepath.Dir(properties[constants.BUILD_PROPERTIES_OBJECT_FILE])) if err != nil { diff --git a/src/arduino.cc/builder/constants/constants.go b/src/arduino.cc/builder/constants/constants.go index cdbdb72b..a0b79f1a 100644 --- a/src/arduino.cc/builder/constants/constants.go +++ b/src/arduino.cc/builder/constants/constants.go @@ -57,6 +57,7 @@ const BUILD_PROPERTIES_EXTRA_TIME_UTC = "extra.time.utc" const BUILD_PROPERTIES_EXTRA_TIME_ZONE = "extra.time.zone" const BUILD_PROPERTIES_INCLUDES = "includes" const BUILD_PROPERTIES_OBJECT_FILE = "object_file" +const BUILD_PROPERTIES_DEP_FILE = "dep_file" const BUILD_PROPERTIES_OBJECT_FILES = "object_files" const BUILD_PROPERTIES_PATTERN = "pattern" const BUILD_PROPERTIES_PID = "pid" diff --git a/src/arduino.cc/builder/container_add_prototypes.go b/src/arduino.cc/builder/container_add_prototypes.go index b69da297..af9ea8f6 100644 --- a/src/arduino.cc/builder/container_add_prototypes.go +++ b/src/arduino.cc/builder/container_add_prototypes.go @@ -41,8 +41,10 @@ type ContainerAddPrototypes struct{} func (s *ContainerAddPrototypes) Run(ctx *types.Context) error { sourceFile := filepath.Join(ctx.SketchBuildPath, filepath.Base(ctx.Sketch.MainFile.Name)+".cpp") + depFile := sourceFile + ".d" + objFile := sourceFile + ".o" commands := []types.Command{ - &GCCPreprocRunner{SourceFilePath: sourceFile, TargetFileName: constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E, Includes: ctx.IncludeFolders}, + &GCCPreprocRunner{SourceFilePath: sourceFile, ObjFilePath: objFile, DepFilePath: depFile, TargetFileName: constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E, Includes: ctx.IncludeFolders}, &ReadFileAndStoreInContext{Target: &ctx.SourceGccMinusE}, &FilterSketchSource{Source: &ctx.SourceGccMinusE}, &CTagsTargetFileSaver{Source: &ctx.SourceGccMinusE, TargetFileName: constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E}, diff --git a/src/arduino.cc/builder/container_find_includes.go b/src/arduino.cc/builder/container_find_includes.go index 980e33c2..196fb215 100644 --- a/src/arduino.cc/builder/container_find_includes.go +++ b/src/arduino.cc/builder/container_find_includes.go @@ -295,8 +295,8 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile t // TODO: This should perhaps also compare against the // include.cache file timestamp. Now, it only checks if the file - // changed after the object file was generated, but if it - // changed between generating the cache and the object file, + // changed after the dependency file was generated, but if it + // changed between generating the cache and the dependency file, // this could show the file as unchanged when it really is // changed. Changing files during a build isn't really // supported, but any problems from it should at least be @@ -305,7 +305,7 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile t // TODO: This reads the dependency file, but the actual building // does it again. Should the result be somehow cached? Perhaps // remove the object file if it is found to be stale? - unchanged, err := builder_utils.BuildResultIsUpToDate(sourcePath, objPath, objPath, depPath, ctx.DebugLevel, ctx.GetLogger()) + unchanged, err := builder_utils.BuildResultIsUpToDate(sourcePath, depPath, objPath, depPath, ctx.DebugLevel, ctx.GetLogger()) if err != nil { return i18n.WrapError(err) } @@ -326,7 +326,7 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile t } } else { commands := []types.Command{ - &GCCPreprocRunnerForDiscoveringIncludes{SourceFilePath: sourcePath, TargetFilePath: targetFilePath, Includes: includes}, + &GCCPreprocRunnerForDiscoveringIncludes{SourceFilePath: sourcePath, ObjFilePath: objPath, DepFilePath: depPath, TargetFilePath: targetFilePath, Includes: includes}, &IncludesFinderWithRegExp{Source: &ctx.SourceGccMinusE}, } for _, command := range commands { @@ -347,7 +347,7 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile t library := ResolveLibrary(ctx, include) if library == nil { // Library could not be resolved, show error - err := runCommand(ctx, &GCCPreprocRunner{SourceFilePath: sourcePath, TargetFileName: constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E, Includes: includes}) + err := runCommand(ctx, &GCCPreprocRunner{SourceFilePath: sourcePath, ObjFilePath: objPath, DepFilePath: depPath, TargetFileName: constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E, Includes: includes}) return i18n.WrapError(err) } diff --git a/src/arduino.cc/builder/gcc_preproc_runner.go b/src/arduino.cc/builder/gcc_preproc_runner.go index e735ac40..65d4706d 100644 --- a/src/arduino.cc/builder/gcc_preproc_runner.go +++ b/src/arduino.cc/builder/gcc_preproc_runner.go @@ -43,12 +43,14 @@ import ( type GCCPreprocRunner struct { SourceFilePath string + ObjFilePath string + DepFilePath string TargetFileName string Includes []string } func (s *GCCPreprocRunner) Run(ctx *types.Context) error { - properties, targetFilePath, err := prepareGCCPreprocRecipeProperties(ctx, s.SourceFilePath, s.TargetFileName, s.Includes) + properties, targetFilePath, err := prepareGCCPreprocRecipeProperties(ctx, s.SourceFilePath, s.ObjFilePath, s.DepFilePath, s.TargetFileName, s.Includes) if err != nil { return i18n.WrapError(err) } @@ -72,12 +74,14 @@ func (s *GCCPreprocRunner) Run(ctx *types.Context) error { type GCCPreprocRunnerForDiscoveringIncludes struct { SourceFilePath string + ObjFilePath string + DepFilePath string TargetFilePath string Includes []string } func (s *GCCPreprocRunnerForDiscoveringIncludes) Run(ctx *types.Context) error { - properties, _, err := prepareGCCPreprocRecipeProperties(ctx, s.SourceFilePath, s.TargetFilePath, s.Includes) + properties, _, err := prepareGCCPreprocRecipeProperties(ctx, s.SourceFilePath, s.ObjFilePath, s.DepFilePath, s.TargetFilePath, s.Includes) if err != nil { return i18n.WrapError(err) } @@ -100,7 +104,7 @@ func (s *GCCPreprocRunnerForDiscoveringIncludes) Run(ctx *types.Context) error { return nil } -func prepareGCCPreprocRecipeProperties(ctx *types.Context, sourceFilePath string, targetFilePath string, includes []string) (properties.Map, string, error) { +func prepareGCCPreprocRecipeProperties(ctx *types.Context, sourceFilePath string, objFilePath string, depFilePath string, targetFilePath string, includes []string) (properties.Map, string, error) { if targetFilePath != utils.NULLFile() { preprocPath := ctx.PreprocPath err := utils.EnsureFolderExists(preprocPath) @@ -113,6 +117,12 @@ func prepareGCCPreprocRecipeProperties(ctx *types.Context, sourceFilePath string properties := ctx.BuildProperties.Clone() properties[constants.BUILD_PROPERTIES_SOURCE_FILE] = sourceFilePath properties[constants.BUILD_PROPERTIES_PREPROCESSED_FILE_PATH] = targetFilePath + properties[constants.BUILD_PROPERTIES_DEP_FILE] = depFilePath + properties[constants.BUILD_PROPERTIES_OBJECT_FILE] = objFilePath + err := utils.EnsureFolderExists(filepath.Dir(depFilePath)) + if err != nil { + return nil, "", i18n.WrapError(err) + } includes = utils.Map(includes, utils.WrapWithHyphenI) properties[constants.BUILD_PROPERTIES_INCLUDES] = strings.Join(includes, constants.SPACE)