diff --git a/builder_utils/utils.go b/builder_utils/utils.go index 139ad770..5a8c1103 100644 --- a/builder_utils/utils.go +++ b/builder_utils/utils.go @@ -30,8 +30,6 @@ package builder_utils import ( - "bytes" - "fmt" "io" "os" "os/exec" @@ -40,12 +38,13 @@ import ( "github.com/arduino/arduino-builder/constants" "github.com/arduino/arduino-builder/i18n" + "github.com/arduino/arduino-builder/types" "github.com/arduino/arduino-builder/utils" "github.com/arduino/go-properties-map" ) -func CompileFilesRecursive(objectFiles []string, sourcePath string, buildPath string, buildProperties properties.Map, includes []string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) { - objectFiles, err := CompileFiles(objectFiles, sourcePath, false, buildPath, buildProperties, includes, verbose, warningsLevel, logger) +func CompileFilesRecursive(ctx *types.Context, objectFiles []string, sourcePath string, buildPath string, buildProperties properties.Map, includes []string) ([]string, error) { + objectFiles, err := CompileFiles(ctx, objectFiles, sourcePath, false, buildPath, buildProperties, includes) if err != nil { return nil, i18n.WrapError(err) } @@ -56,7 +55,7 @@ func CompileFilesRecursive(objectFiles []string, sourcePath string, buildPath st } for _, folder := range folders { - objectFiles, err = CompileFilesRecursive(objectFiles, filepath.Join(sourcePath, folder.Name()), filepath.Join(buildPath, folder.Name()), buildProperties, includes, verbose, warningsLevel, logger) + objectFiles, err = CompileFilesRecursive(ctx, objectFiles, filepath.Join(sourcePath, folder.Name()), filepath.Join(buildPath, folder.Name()), buildProperties, includes) if err != nil { return nil, i18n.WrapError(err) } @@ -65,28 +64,28 @@ func CompileFilesRecursive(objectFiles []string, sourcePath string, buildPath st return objectFiles, nil } -func CompileFiles(objectFiles []string, sourcePath string, recurse bool, buildPath string, buildProperties properties.Map, includes []string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) { - objectFiles, err := compileFilesWithExtensionWithRecipe(objectFiles, sourcePath, recurse, buildPath, buildProperties, includes, ".S", constants.RECIPE_S_PATTERN, verbose, warningsLevel, logger) +func CompileFiles(ctx *types.Context, objectFiles []string, sourcePath string, recurse bool, buildPath string, buildProperties properties.Map, includes []string) ([]string, error) { + objectFiles, err := compileFilesWithExtensionWithRecipe(ctx, objectFiles, sourcePath, recurse, buildPath, buildProperties, includes, ".S", constants.RECIPE_S_PATTERN) if err != nil { return nil, i18n.WrapError(err) } - objectFiles, err = compileFilesWithExtensionWithRecipe(objectFiles, sourcePath, recurse, buildPath, buildProperties, includes, ".c", constants.RECIPE_C_PATTERN, verbose, warningsLevel, logger) + objectFiles, err = compileFilesWithExtensionWithRecipe(ctx, objectFiles, sourcePath, recurse, buildPath, buildProperties, includes, ".c", constants.RECIPE_C_PATTERN) if err != nil { return nil, i18n.WrapError(err) } - objectFiles, err = compileFilesWithExtensionWithRecipe(objectFiles, sourcePath, recurse, buildPath, buildProperties, includes, ".cpp", constants.RECIPE_CPP_PATTERN, verbose, warningsLevel, logger) + objectFiles, err = compileFilesWithExtensionWithRecipe(ctx, objectFiles, sourcePath, recurse, buildPath, buildProperties, includes, ".cpp", constants.RECIPE_CPP_PATTERN) if err != nil { return nil, i18n.WrapError(err) } return objectFiles, nil } -func compileFilesWithExtensionWithRecipe(objectFiles []string, sourcePath string, recurse bool, buildPath string, buildProperties properties.Map, includes []string, extension string, recipe string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) { +func compileFilesWithExtensionWithRecipe(ctx *types.Context, objectFiles []string, sourcePath string, recurse bool, buildPath string, buildProperties properties.Map, includes []string, extension string, recipe string) ([]string, error) { sources, err := findFilesInFolder(sourcePath, extension, recurse) if err != nil { return nil, i18n.WrapError(err) } - return compileFilesWithRecipe(objectFiles, sourcePath, sources, buildPath, buildProperties, includes, recipe, verbose, warningsLevel, logger) + return compileFilesWithRecipe(ctx, objectFiles, sourcePath, sources, buildPath, buildProperties, includes, recipe) } func findFilesInFolder(sourcePath string, extension string, recurse bool) ([]string, error) { @@ -145,9 +144,9 @@ func findAllFilesInFolder(sourcePath string, recurse bool) ([]string, error) { return sources, nil } -func compileFilesWithRecipe(objectFiles []string, sourcePath string, sources []string, buildPath string, buildProperties properties.Map, includes []string, recipe string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) { +func compileFilesWithRecipe(ctx *types.Context, objectFiles []string, sourcePath string, sources []string, buildPath string, buildProperties properties.Map, includes []string, recipe string) ([]string, error) { for _, source := range sources { - objectFile, err := compileFileWithRecipe(sourcePath, source, buildPath, buildProperties, includes, recipe, verbose, warningsLevel, logger) + objectFile, err := compileFileWithRecipe(ctx, sourcePath, source, buildPath, buildProperties, includes, recipe) if err != nil { return nil, i18n.WrapError(err) } @@ -157,9 +156,10 @@ func compileFilesWithRecipe(objectFiles []string, sourcePath string, sources []s return objectFiles, nil } -func compileFileWithRecipe(sourcePath string, source string, buildPath string, buildProperties properties.Map, includes []string, recipe string, verbose bool, warningsLevel string, logger i18n.Logger) (string, error) { +func compileFileWithRecipe(ctx *types.Context, sourcePath string, source string, buildPath string, buildProperties properties.Map, includes []string, recipe string) (string, error) { + logger := ctx.GetLogger() properties := buildProperties.Clone() - properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS] = properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS+"."+warningsLevel] + properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS] = properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS+"."+ctx.WarningsLevel] properties[constants.BUILD_PROPERTIES_INCLUDES] = strings.Join(includes, constants.SPACE) properties[constants.BUILD_PROPERTIES_SOURCE_FILE] = source relativeSource, err := filepath.Rel(sourcePath, source) @@ -173,27 +173,33 @@ func compileFileWithRecipe(sourcePath string, source string, buildPath string, b return "", i18n.WrapError(err) } - objIsUpToDate, err := ObjFileIsUpToDate(properties[constants.BUILD_PROPERTIES_SOURCE_FILE], properties[constants.BUILD_PROPERTIES_OBJECT_FILE], filepath.Join(buildPath, relativeSource+".d")) + objIsUpToDate, err := ObjFileIsUpToDate(ctx, properties[constants.BUILD_PROPERTIES_SOURCE_FILE], properties[constants.BUILD_PROPERTIES_OBJECT_FILE], filepath.Join(buildPath, relativeSource+".d")) if err != nil { return "", i18n.WrapError(err) } if !objIsUpToDate { - _, err = ExecRecipe(properties, recipe, false, verbose, verbose, logger) + _, _, err = ExecRecipe(ctx, properties, recipe, false, /* stdout */ utils.ShowIfVerbose, /* stderr */ utils.Show) if err != nil { return "", i18n.WrapError(err) } - } else if verbose { + } else if ctx.Verbose { logger.Println(constants.LOG_LEVEL_INFO, constants.MSG_USING_PREVIOUS_COMPILED_FILE, properties[constants.BUILD_PROPERTIES_OBJECT_FILE]) } return properties[constants.BUILD_PROPERTIES_OBJECT_FILE], nil } -func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile string) (bool, error) { +func ObjFileIsUpToDate(ctx *types.Context, sourceFile, objectFile, dependencyFile string) (bool, error) { sourceFile = filepath.Clean(sourceFile) objectFile = filepath.Clean(objectFile) dependencyFile = filepath.Clean(dependencyFile) + logger := ctx.GetLogger() + debugLevel := ctx.DebugLevel + + if debugLevel >= 20 { + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "Checking previous results for {0} (result = {1}, dep = {2})", sourceFile, objectFile, dependencyFile) + } sourceFileStat, err := os.Stat(sourceFile) if err != nil { @@ -203,6 +209,9 @@ func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile string) (bool, err objectFileStat, err := os.Stat(objectFile) if err != nil { if os.IsNotExist(err) { + if debugLevel >= 20 { + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "Not found: {0}", objectFile) + } return false, nil } else { return false, i18n.WrapError(err) @@ -212,6 +221,9 @@ func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile string) (bool, err dependencyFileStat, err := os.Stat(dependencyFile) if err != nil { if os.IsNotExist(err) { + if debugLevel >= 20 { + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "Not found: {0}", dependencyFile) + } return false, nil } else { return false, i18n.WrapError(err) @@ -219,9 +231,15 @@ func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile string) (bool, err } if sourceFileStat.ModTime().After(objectFileStat.ModTime()) { + if debugLevel >= 20 { + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "{0} newer than {1}", sourceFile, objectFile) + } return false, nil } if sourceFileStat.ModTime().After(dependencyFileStat.ModTime()) { + if debugLevel >= 20 { + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "{0} newer than {1}", sourceFile, dependencyFile) + } return false, nil } @@ -241,10 +259,16 @@ func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile string) (bool, err firstRow := rows[0] if !strings.HasSuffix(firstRow, ":") { + if debugLevel >= 20 { + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "No colon in first line of depfile") + } return false, nil } objFileInDepFile := firstRow[:len(firstRow)-1] if objFileInDepFile != objectFile { + if debugLevel >= 20 { + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "Depfile is about different file: {0}", objFileInDepFile) + } return false, nil } @@ -254,12 +278,22 @@ func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile string) (bool, err if err != nil && !os.IsNotExist(err) { // There is probably a parsing error of the dep file // Ignore the error and trigger a full rebuild anyway + if debugLevel >= 20 { + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "Failed to read: {0}", row) + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, i18n.WrapError(err).Error()) + } return false, nil } if os.IsNotExist(err) { + if debugLevel >= 20 { + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "Not found: {0}", row) + } return false, nil } if depStat.ModTime().After(objectFileStat.ModTime()) { + if debugLevel >= 20 { + logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, "{0} newer than {1}", row, objectFile) + } return false, nil } } @@ -309,7 +343,8 @@ func CoreOrReferencedCoreHasChanged(corePath, targetCorePath, targetFile string) return true } -func ArchiveCompiledFiles(buildPath string, archiveFile string, objectFiles []string, buildProperties properties.Map, verbose bool, logger i18n.Logger) (string, error) { +func ArchiveCompiledFiles(ctx *types.Context, buildPath string, archiveFile string, objectFiles []string, buildProperties properties.Map) (string, error) { + logger := ctx.GetLogger() archiveFilePath := filepath.Join(buildPath, archiveFile) rebuildArchive := false @@ -332,7 +367,7 @@ func ArchiveCompiledFiles(buildPath string, archiveFile string, objectFiles []st return "", i18n.WrapError(err) } } else { - if verbose { + if ctx.Verbose { logger.Println(constants.LOG_LEVEL_INFO, constants.MSG_USING_PREVIOUS_COMPILED_FILE, archiveFilePath) } return archiveFilePath, nil @@ -345,7 +380,7 @@ func ArchiveCompiledFiles(buildPath string, archiveFile string, objectFiles []st properties[constants.BUILD_PROPERTIES_ARCHIVE_FILE_PATH] = archiveFilePath properties[constants.BUILD_PROPERTIES_OBJECT_FILE] = objectFile - _, err := ExecRecipe(properties, constants.RECIPE_AR_PATTERN, false, verbose, verbose, logger) + _, _, err := ExecRecipe(ctx, properties, constants.RECIPE_AR_PATTERN, false, /* stdout */ utils.ShowIfVerbose, /* stderr */ utils.Show) if err != nil { return "", i18n.WrapError(err) } @@ -354,28 +389,17 @@ func ArchiveCompiledFiles(buildPath string, archiveFile string, objectFiles []st return archiveFilePath, nil } -func ExecRecipe(properties properties.Map, recipe string, removeUnsetProperties bool, echoCommandLine bool, echoOutput bool, logger i18n.Logger) ([]byte, error) { - command, err := PrepareCommandForRecipe(properties, recipe, removeUnsetProperties, echoCommandLine, echoOutput, logger) +// See util.ExecCommand for stdout/stderr arguments +func ExecRecipe(ctx *types.Context, buildProperties properties.Map, recipe string, removeUnsetProperties bool, stdout int, stderr int) ([]byte, []byte, error) { + command, err := PrepareCommandForRecipe(ctx, buildProperties, recipe, removeUnsetProperties) if err != nil { - return nil, i18n.WrapError(err) + return nil, nil, i18n.WrapError(err) } - - if echoOutput { - command.Stdout = os.Stdout - } - - command.Stderr = os.Stderr - - if echoOutput { - err := command.Run() - return nil, i18n.WrapError(err) - } - - bytes, err := command.Output() - return bytes, i18n.WrapError(err) + return utils.ExecCommand(ctx, command, stdout, stderr) } -func PrepareCommandForRecipe(buildProperties properties.Map, recipe string, removeUnsetProperties bool, echoCommandLine bool, echoOutput bool, logger i18n.Logger) (*exec.Cmd, error) { +func PrepareCommandForRecipe(ctx *types.Context, buildProperties properties.Map, recipe string, removeUnsetProperties bool) (*exec.Cmd, error) { + logger := ctx.GetLogger() pattern := buildProperties[recipe] if pattern == constants.EMPTY_STRING { return nil, i18n.ErrorfWithLogger(logger, constants.MSG_PATTERN_MISSING, recipe) @@ -392,29 +416,9 @@ func PrepareCommandForRecipe(buildProperties properties.Map, recipe string, remo return nil, i18n.WrapError(err) } - if echoCommandLine { - fmt.Println(commandLine) - } - return command, nil } -func ExecRecipeCollectStdErr(buildProperties properties.Map, recipe string, removeUnsetProperties bool, echoCommandLine bool, echoOutput bool, logger i18n.Logger) (string, error) { - command, err := PrepareCommandForRecipe(buildProperties, recipe, removeUnsetProperties, echoCommandLine, echoOutput, logger) - if err != nil { - return "", i18n.WrapError(err) - } - - buffer := &bytes.Buffer{} - command.Stderr = buffer - command.Run() - return string(buffer.Bytes()), nil -} - -func RemoveHyphenMDDFlagFromGCCCommandLine(buildProperties properties.Map) { - buildProperties[constants.BUILD_PROPERTIES_COMPILER_CPP_FLAGS] = strings.Replace(buildProperties[constants.BUILD_PROPERTIES_COMPILER_CPP_FLAGS], "-MMD", "", -1) -} - // CopyFile copies the contents of the file named src to the file named // by dst. The file will be created if it does not already exist. If the // destination file exists, all it's contents will be replaced by the contents diff --git a/container_add_prototypes.go b/container_add_prototypes.go index cd0d56f7..13b777d3 100644 --- a/container_add_prototypes.go +++ b/container_add_prototypes.go @@ -35,15 +35,28 @@ import ( "github.com/arduino/arduino-builder/constants" "github.com/arduino/arduino-builder/i18n" "github.com/arduino/arduino-builder/types" + "github.com/arduino/arduino-builder/utils" ) type ContainerAddPrototypes struct{} func (s *ContainerAddPrototypes) Run(ctx *types.Context) error { sourceFile := filepath.Join(ctx.SketchBuildPath, filepath.Base(ctx.Sketch.MainFile.Name)+".cpp") + + // Generate the full pathname for the preproc output file + err := utils.EnsureFolderExists(ctx.PreprocPath) + if err != nil { + return i18n.WrapError(err) + } + targetFilePath := filepath.Join(ctx.PreprocPath, constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E) + + // Run preprocessor + err = GCCPreprocRunner(ctx, sourceFile, targetFilePath, ctx.IncludeFolders) + if err != nil { + return i18n.WrapError(err) + } commands := []types.Command{ - &GCCPreprocRunner{SourceFilePath: sourceFile, TargetFileName: constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E, Includes: ctx.IncludeFolders}, - &ReadFileAndStoreInContext{Target: &ctx.SourceGccMinusE}, + &ReadFileAndStoreInContext{FileToRead: targetFilePath, Target: &ctx.SourceGccMinusE}, &FilterSketchSource{Source: &ctx.SourceGccMinusE}, &CTagsTargetFileSaver{Source: &ctx.SourceGccMinusE, TargetFileName: constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E}, &CTagsRunner{}, diff --git a/container_find_includes.go b/container_find_includes.go index dbd31c71..df0fca81 100644 --- a/container_find_includes.go +++ b/container_find_includes.go @@ -110,6 +110,7 @@ import ( "encoding/json" "io/ioutil" "os" + "os/exec" "path/filepath" "time" @@ -118,6 +119,8 @@ import ( "github.com/arduino/arduino-builder/i18n" "github.com/arduino/arduino-builder/types" "github.com/arduino/arduino-builder/utils" + + "github.com/go-errors/errors" ) type ContainerFindIncludes struct{} @@ -213,7 +216,7 @@ func (cache *includeCache) Next() includeCacheEntry { // not, or no entry is available, the cache is invalidated. Does not // advance the cache. func (cache *includeCache) ExpectFile(sourcefile string) { - if cache.valid && cache.next < len(cache.entries) && cache.Next().Sourcefile != sourcefile { + if cache.valid && (cache.next >= len(cache.entries) || cache.Next().Sourcefile != sourcefile) { cache.valid = false cache.entries = cache.entries[:cache.next] } @@ -289,6 +292,8 @@ func writeCache(cache *includeCache, path string) error { func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile types.SourceFile) error { sourcePath := sourceFile.SourcePath(ctx) + depPath := sourceFile.DepfilePath(ctx) + objPath := sourceFile.ObjectPath(ctx) targetFilePath := utils.NULLFile() // TODO: This should perhaps also compare against the @@ -303,7 +308,7 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile t // TODO: This reads the dependency file, but the actual building // does it again. Should the result be somehow cached? Perhaps // remove the object file if it is found to be stale? - unchanged, err := builder_utils.ObjFileIsUpToDate(sourcePath, sourceFile.ObjectPath(ctx), sourceFile.DepfilePath(ctx)) + unchanged, err := builder_utils.ObjFileIsUpToDate(ctx, sourcePath, objPath, depPath) if err != nil { return i18n.WrapError(err) } @@ -317,23 +322,33 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile t if library, ok := sourceFile.Origin.(*types.Library); ok && library.UtilityFolder != "" { includes = append(includes, library.UtilityFolder) } + var preproc_err error + var preproc_stderr []byte if unchanged && cache.valid { include = cache.Next().Include if first && ctx.Verbose { ctx.GetLogger().Println(constants.LOG_LEVEL_INFO, constants.MSG_USING_CACHED_INCLUDES, sourcePath) } } else { - commands := []types.Command{ - &GCCPreprocRunnerForDiscoveringIncludes{SourceFilePath: sourcePath, TargetFilePath: targetFilePath, Includes: includes}, - &IncludesFinderWithRegExp{Source: &ctx.SourceGccMinusE}, - } - for _, command := range commands { - err := runCommand(ctx, command) - if err != nil { - return i18n.WrapError(err) + preproc_stderr, preproc_err = GCCPreprocRunnerForDiscoveringIncludes(ctx, sourcePath, targetFilePath, includes) + // Unwrap error and see if it is an ExitError. + _, is_exit_error := i18n.UnwrapError(preproc_err).(*exec.ExitError) + if preproc_err == nil { + // Preprocessor successful, done + include = "" + } else if !is_exit_error || preproc_stderr == nil { + // Ignore ExitErrors (e.g. gcc returning + // non-zero status), but bail out on + // other errors + return i18n.WrapError(preproc_err) + } else { + include = IncludesFinderWithRegExp(ctx, string(preproc_stderr)) + if include == "" { + // No include found? Bail out. + os.Stderr.Write(preproc_stderr) + return i18n.WrapError(preproc_err) } } - include = ctx.IncludeJustFound } if include == "" { @@ -345,8 +360,19 @@ func findIncludesUntilDone(ctx *types.Context, cache *includeCache, sourceFile t library := ResolveLibrary(ctx, include) if library == nil { // Library could not be resolved, show error - err := runCommand(ctx, &GCCPreprocRunner{SourceFilePath: sourcePath, TargetFileName: constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E, Includes: includes}) - return i18n.WrapError(err) + if preproc_err == nil || preproc_stderr == nil { + // Filename came from cache, so run preprocessor to obtain error to show + preproc_stderr, preproc_err = GCCPreprocRunnerForDiscoveringIncludes(ctx, sourcePath, targetFilePath, includes) + if preproc_err == nil { + // If there is a missing #include in the cache, but running + // gcc does not reproduce that, there is something wrong. + // Returning an error here will cause the cache to be + // deleted, so hopefully the next compilation will succeed. + return errors.New("Internal error in cache") + } + } + os.Stderr.Write(preproc_stderr) + return i18n.WrapError(preproc_err) } // Add this library to the list of libraries, the diff --git a/ctags_runner.go b/ctags_runner.go index 82e4165c..18850ca1 100644 --- a/ctags_runner.go +++ b/ctags_runner.go @@ -30,8 +30,6 @@ package builder import ( - "fmt" - "github.com/arduino/arduino-builder/constants" "github.com/arduino/arduino-builder/ctags" "github.com/arduino/arduino-builder/i18n" @@ -61,12 +59,7 @@ func (s *CTagsRunner) Run(ctx *types.Context) error { return i18n.WrapError(err) } - verbose := ctx.Verbose - if verbose { - fmt.Println(commandLine) - } - - sourceBytes, err := command.Output() + sourceBytes, _, err := utils.ExecCommand(ctx, command /* stdout */, utils.Capture /* stderr */, utils.Ignore) if err != nil { return i18n.WrapError(err) } diff --git a/gcc_preproc_runner.go b/gcc_preproc_runner.go index b571cacc..3c801f89 100644 --- a/gcc_preproc_runner.go +++ b/gcc_preproc_runner.go @@ -30,7 +30,7 @@ package builder import ( - "path/filepath" + "os/exec" "strings" "github.com/arduino/arduino-builder/builder_utils" @@ -38,87 +38,59 @@ import ( "github.com/arduino/arduino-builder/i18n" "github.com/arduino/arduino-builder/types" "github.com/arduino/arduino-builder/utils" - "github.com/arduino/go-properties-map" ) -type GCCPreprocRunner struct { - SourceFilePath string - TargetFileName string - Includes []string -} - -func (s *GCCPreprocRunner) Run(ctx *types.Context) error { - properties, targetFilePath, err := prepareGCCPreprocRecipeProperties(ctx, s.SourceFilePath, s.TargetFileName, s.Includes) +func GCCPreprocRunner(ctx *types.Context, sourceFilePath string, targetFilePath string, includes []string) error { + cmd, err := prepareGCCPreprocRecipeProperties(ctx, sourceFilePath, targetFilePath, includes) if err != nil { return i18n.WrapError(err) } - if properties[constants.RECIPE_PREPROC_MACROS] == constants.EMPTY_STRING { - //generate PREPROC_MACROS from RECIPE_CPP_PATTERN - properties[constants.RECIPE_PREPROC_MACROS] = GeneratePreprocPatternFromCompile(properties[constants.RECIPE_CPP_PATTERN]) - } - - verbose := ctx.Verbose - logger := ctx.GetLogger() - _, err = builder_utils.ExecRecipe(properties, constants.RECIPE_PREPROC_MACROS, true, verbose, false, logger) + _, _, err = utils.ExecCommand(ctx, cmd /* stdout */, utils.ShowIfVerbose /* stderr */, utils.Show) if err != nil { return i18n.WrapError(err) } - ctx.FileToRead = targetFilePath - return nil } -type GCCPreprocRunnerForDiscoveringIncludes struct { - SourceFilePath string - TargetFilePath string - Includes []string -} - -func (s *GCCPreprocRunnerForDiscoveringIncludes) Run(ctx *types.Context) error { - properties, _, err := prepareGCCPreprocRecipeProperties(ctx, s.SourceFilePath, s.TargetFilePath, s.Includes) +func GCCPreprocRunnerForDiscoveringIncludes(ctx *types.Context, sourceFilePath string, targetFilePath string, includes []string) ([]byte, error) { + cmd, err := prepareGCCPreprocRecipeProperties(ctx, sourceFilePath, targetFilePath, includes) if err != nil { - return i18n.WrapError(err) + return nil, i18n.WrapError(err) } - verbose := ctx.Verbose - logger := ctx.GetLogger() - - if properties[constants.RECIPE_PREPROC_MACROS] == constants.EMPTY_STRING { - //generate PREPROC_MACROS from RECIPE_CPP_PATTERN - properties[constants.RECIPE_PREPROC_MACROS] = GeneratePreprocPatternFromCompile(properties[constants.RECIPE_CPP_PATTERN]) - } - - stderr, err := builder_utils.ExecRecipeCollectStdErr(properties, constants.RECIPE_PREPROC_MACROS, true, verbose, false, logger) + _, stderr, err := utils.ExecCommand(ctx, cmd /* stdout */, utils.ShowIfVerbose /* stderr */, utils.Capture) if err != nil { - return i18n.WrapError(err) + return stderr, i18n.WrapError(err) } - ctx.SourceGccMinusE = string(stderr) - - return nil + return stderr, nil } -func prepareGCCPreprocRecipeProperties(ctx *types.Context, sourceFilePath string, targetFilePath string, includes []string) (properties.Map, string, error) { - if targetFilePath != utils.NULLFile() { - preprocPath := ctx.PreprocPath - err := utils.EnsureFolderExists(preprocPath) - if err != nil { - return nil, "", i18n.WrapError(err) - } - targetFilePath = filepath.Join(preprocPath, targetFilePath) - } - +func prepareGCCPreprocRecipeProperties(ctx *types.Context, sourceFilePath string, targetFilePath string, includes []string) (*exec.Cmd, error) { properties := ctx.BuildProperties.Clone() properties[constants.BUILD_PROPERTIES_SOURCE_FILE] = sourceFilePath properties[constants.BUILD_PROPERTIES_PREPROCESSED_FILE_PATH] = targetFilePath includes = utils.Map(includes, utils.WrapWithHyphenI) properties[constants.BUILD_PROPERTIES_INCLUDES] = strings.Join(includes, constants.SPACE) - builder_utils.RemoveHyphenMDDFlagFromGCCCommandLine(properties) - return properties, targetFilePath, nil + if properties[constants.RECIPE_PREPROC_MACROS] == constants.EMPTY_STRING { + //generate PREPROC_MACROS from RECIPE_CPP_PATTERN + properties[constants.RECIPE_PREPROC_MACROS] = GeneratePreprocPatternFromCompile(properties[constants.RECIPE_CPP_PATTERN]) + } + + cmd, err := builder_utils.PrepareCommandForRecipe(ctx, properties, constants.RECIPE_PREPROC_MACROS, true) + if err != nil { + return nil, i18n.WrapError(err) + } + + // Remove -MMD argument if present. Leaving it will make gcc try + // to create a /dev/null.d dependency file, which won't work. + cmd.Args = utils.Filter(cmd.Args, func(a string) bool { return a != "-MMD" }) + + return cmd, nil } func GeneratePreprocPatternFromCompile(compilePattern string) string { diff --git a/gcc_preproc_source_saver.go b/gcc_preproc_source_saver.go deleted file mode 100644 index 556cd276..00000000 --- a/gcc_preproc_source_saver.go +++ /dev/null @@ -1,53 +0,0 @@ -/* - * This file is part of Arduino Builder. - * - * Arduino Builder is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - * - * As a special exception, you may use this file as part of a free software - * library without restriction. Specifically, if other files instantiate - * templates or use macros or inline functions from this file, or you compile - * this file and link it with other files to produce an executable, this - * file does not by itself cause the resulting executable to be covered by - * the GNU General Public License. This exception does not however - * invalidate any other reasons why the executable file might be covered by - * the GNU General Public License. - * - * Copyright 2015 Arduino LLC (http://www.arduino.cc/) - */ - -package builder - -// XXX: OBSOLETE? - -import ( - "github.com/arduino/arduino-builder/constants" - "github.com/arduino/arduino-builder/i18n" - "github.com/arduino/arduino-builder/types" - "github.com/arduino/arduino-builder/utils" - "path/filepath" -) - -type GCCPreprocSourceSaver struct{} - -func (s *GCCPreprocSourceSaver) Run(ctx *types.Context) error { - preprocPath := ctx.PreprocPath - err := utils.EnsureFolderExists(preprocPath) - if err != nil { - return i18n.WrapError(err) - } - - err = utils.WriteFile(filepath.Join(preprocPath, constants.FILE_GCC_PREPROC_TARGET), ctx.Source) - return i18n.WrapError(err) -} diff --git a/i18n/errors.go b/i18n/errors.go index ba2a86ea..b3617800 100644 --- a/i18n/errors.go +++ b/i18n/errors.go @@ -18,3 +18,14 @@ func WrapError(err error) error { } return errors.Wrap(err, 0) } + +func UnwrapError(err error) error { + // Perhaps go-errors can do this already in later versions? + // See https://github.com/go-errors/errors/issues/14 + switch e := err.(type) { + case *errors.Error: + return e.Err + default: + return err + } +} diff --git a/includes_finder_with_regexp.go b/includes_finder_with_regexp.go index 56ac955e..abfe7635 100644 --- a/includes_finder_with_regexp.go +++ b/includes_finder_with_regexp.go @@ -37,21 +37,13 @@ import ( var INCLUDE_REGEXP = regexp.MustCompile("(?ms)^\\s*#[ \t]*include\\s*[<\"](\\S+)[\">]") -type IncludesFinderWithRegExp struct { - Source *string -} - -func (s *IncludesFinderWithRegExp) Run(ctx *types.Context) error { - source := *s.Source - +func IncludesFinderWithRegExp(ctx *types.Context, source string) string { match := INCLUDE_REGEXP.FindStringSubmatch(source) if match != nil { - ctx.IncludeJustFound = strings.TrimSpace(match[1]) + return strings.TrimSpace(match[1]) } else { - ctx.IncludeJustFound = findIncludeForOldCompilers(source) + return findIncludeForOldCompilers(source) } - - return nil } func findIncludeForOldCompilers(source string) string { diff --git a/phases/core_builder.go b/phases/core_builder.go index 75b6535d..cecea3de 100644 --- a/phases/core_builder.go +++ b/phases/core_builder.go @@ -46,9 +46,6 @@ func (s *CoreBuilder) Run(ctx *types.Context) error { coreBuildPath := ctx.CoreBuildPath coreBuildCachePath := ctx.CoreBuildCachePath buildProperties := ctx.BuildProperties - verbose := ctx.Verbose - warningsLevel := ctx.WarningsLevel - logger := ctx.GetLogger() err := utils.EnsureFolderExists(coreBuildPath) if err != nil { @@ -62,7 +59,7 @@ func (s *CoreBuilder) Run(ctx *types.Context) error { } } - archiveFile, objectFiles, err := compileCore(coreBuildPath, coreBuildCachePath, buildProperties, verbose, warningsLevel, logger) + archiveFile, objectFiles, err := compileCore(ctx, coreBuildPath, coreBuildCachePath, buildProperties) if err != nil { return i18n.WrapError(err) } @@ -73,7 +70,8 @@ func (s *CoreBuilder) Run(ctx *types.Context) error { return nil } -func compileCore(buildPath string, buildCachePath string, buildProperties properties.Map, verbose bool, warningsLevel string, logger i18n.Logger) (string, []string, error) { +func compileCore(ctx *types.Context, buildPath string, buildCachePath string, buildProperties properties.Map) (string, []string, error) { + logger := ctx.GetLogger() coreFolder := buildProperties[constants.BUILD_PROPERTIES_BUILD_CORE_PATH] variantFolder := buildProperties[constants.BUILD_PROPERTIES_BUILD_VARIANT_PATH] @@ -90,7 +88,7 @@ func compileCore(buildPath string, buildCachePath string, buildProperties proper variantObjectFiles := []string{} if variantFolder != constants.EMPTY_STRING { - variantObjectFiles, err = builder_utils.CompileFiles(variantObjectFiles, variantFolder, true, buildPath, buildProperties, includes, verbose, warningsLevel, logger) + variantObjectFiles, err = builder_utils.CompileFiles(ctx, variantObjectFiles, variantFolder, true, buildPath, buildProperties, includes) if err != nil { return "", nil, i18n.WrapError(err) } @@ -107,26 +105,26 @@ func compileCore(buildPath string, buildCachePath string, buildProperties proper if canUseArchivedCore { // use archived core - if verbose { + if ctx.Verbose { logger.Println(constants.LOG_LEVEL_INFO, "Using precompiled core: {0}", targetArchivedCore) } return targetArchivedCore, variantObjectFiles, nil } } - coreObjectFiles, err := builder_utils.CompileFiles([]string{}, coreFolder, true, buildPath, buildProperties, includes, verbose, warningsLevel, logger) + coreObjectFiles, err := builder_utils.CompileFiles(ctx, []string{}, coreFolder, true, buildPath, buildProperties, includes) if err != nil { return "", nil, i18n.WrapError(err) } - archiveFile, err := builder_utils.ArchiveCompiledFiles(buildPath, "core.a", coreObjectFiles, buildProperties, verbose, logger) + archiveFile, err := builder_utils.ArchiveCompiledFiles(ctx, buildPath, "core.a", coreObjectFiles, buildProperties) if err != nil { return "", nil, i18n.WrapError(err) } // archive core.a if targetArchivedCore != "" { - if verbose { + if ctx.Verbose { logger.Println(constants.LOG_LEVEL_INFO, constants.MSG_ARCHIVING_CORE_CACHE, targetArchivedCore) } builder_utils.CopyFile(archiveFile, targetArchivedCore) diff --git a/phases/libraries_builder.go b/phases/libraries_builder.go index fda56b80..dcbcaa4b 100644 --- a/phases/libraries_builder.go +++ b/phases/libraries_builder.go @@ -52,16 +52,13 @@ func (s *LibrariesBuilder) Run(ctx *types.Context) error { includes := ctx.IncludeFolders includes = utils.Map(includes, utils.WrapWithHyphenI) libraries := ctx.ImportedLibraries - verbose := ctx.Verbose - warningsLevel := ctx.WarningsLevel - logger := ctx.GetLogger() err := utils.EnsureFolderExists(librariesBuildPath) if err != nil { return i18n.WrapError(err) } - objectFiles, err := compileLibraries(libraries, librariesBuildPath, buildProperties, includes, verbose, warningsLevel, logger) + objectFiles, err := compileLibraries(ctx, libraries, librariesBuildPath, buildProperties, includes) if err != nil { return i18n.WrapError(err) } @@ -99,10 +96,10 @@ func fixLDFLAGforPrecompiledLibraries(ctx *types.Context, libraries []*types.Lib return nil } -func compileLibraries(libraries []*types.Library, buildPath string, buildProperties properties.Map, includes []string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) { +func compileLibraries(ctx *types.Context, libraries []*types.Library, buildPath string, buildProperties properties.Map, includes []string) ([]string, error) { objectFiles := []string{} for _, library := range libraries { - libraryObjectFiles, err := compileLibrary(library, buildPath, buildProperties, includes, verbose, warningsLevel, logger) + libraryObjectFiles, err := compileLibrary(ctx, library, buildPath, buildProperties, includes) if err != nil { return nil, i18n.WrapError(err) } @@ -113,8 +110,9 @@ func compileLibraries(libraries []*types.Library, buildPath string, buildPropert } -func compileLibrary(library *types.Library, buildPath string, buildProperties properties.Map, includes []string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) { - if verbose { +func compileLibrary(ctx *types.Context, library *types.Library, buildPath string, buildProperties properties.Map, includes []string) ([]string, error) { + logger := ctx.GetLogger() + if ctx.Verbose { logger.Println(constants.LOG_LEVEL_INFO, "Compiling library \"{0}\"", library.Name) } libraryBuildPath := filepath.Join(buildPath, library.Name) @@ -144,12 +142,12 @@ func compileLibrary(library *types.Library, buildPath string, buildProperties pr } if library.Layout == types.LIBRARY_RECURSIVE { - objectFiles, err = builder_utils.CompileFilesRecursive(objectFiles, library.SrcFolder, libraryBuildPath, buildProperties, includes, verbose, warningsLevel, logger) + objectFiles, err = builder_utils.CompileFilesRecursive(ctx, objectFiles, library.SrcFolder, libraryBuildPath, buildProperties, includes) if err != nil { return nil, i18n.WrapError(err) } if library.DotALinkage { - archiveFile, err := builder_utils.ArchiveCompiledFiles(libraryBuildPath, library.Name+".a", objectFiles, buildProperties, verbose, logger) + archiveFile, err := builder_utils.ArchiveCompiledFiles(ctx, libraryBuildPath, library.Name+".a", objectFiles, buildProperties) if err != nil { return nil, i18n.WrapError(err) } @@ -159,14 +157,14 @@ func compileLibrary(library *types.Library, buildPath string, buildProperties pr if library.UtilityFolder != "" { includes = append(includes, utils.WrapWithHyphenI(library.UtilityFolder)) } - objectFiles, err = builder_utils.CompileFiles(objectFiles, library.SrcFolder, false, libraryBuildPath, buildProperties, includes, verbose, warningsLevel, logger) + objectFiles, err = builder_utils.CompileFiles(ctx, objectFiles, library.SrcFolder, false, libraryBuildPath, buildProperties, includes) if err != nil { return nil, i18n.WrapError(err) } if library.UtilityFolder != "" { utilityBuildPath := filepath.Join(libraryBuildPath, constants.LIBRARY_FOLDER_UTILITY) - objectFiles, err = builder_utils.CompileFiles(objectFiles, library.UtilityFolder, false, utilityBuildPath, buildProperties, includes, verbose, warningsLevel, logger) + objectFiles, err = builder_utils.CompileFiles(ctx, objectFiles, library.UtilityFolder, false, utilityBuildPath, buildProperties, includes) if err != nil { return nil, i18n.WrapError(err) } diff --git a/phases/linker.go b/phases/linker.go index d35f4993..2514b406 100644 --- a/phases/linker.go +++ b/phases/linker.go @@ -61,11 +61,8 @@ func (s *Linker) Run(ctx *types.Context) error { } buildProperties := ctx.BuildProperties - verbose := ctx.Verbose - warningsLevel := ctx.WarningsLevel - logger := ctx.GetLogger() - err = link(objectFiles, coreDotARelPath, coreArchiveFilePath, buildProperties, verbose, warningsLevel, logger) + err = link(ctx, objectFiles, coreDotARelPath, coreArchiveFilePath, buildProperties) if err != nil { return i18n.WrapError(err) } @@ -73,7 +70,7 @@ func (s *Linker) Run(ctx *types.Context) error { return nil } -func link(objectFiles []string, coreDotARelPath string, coreArchiveFilePath string, buildProperties properties.Map, verbose bool, warningsLevel string, logger i18n.Logger) error { +func link(ctx *types.Context, objectFiles []string, coreDotARelPath string, coreArchiveFilePath string, buildProperties properties.Map) error { optRelax := addRelaxTrickIfATMEGA2560(buildProperties) objectFiles = utils.Map(objectFiles, wrapWithDoubleQuotes) @@ -81,12 +78,12 @@ func link(objectFiles []string, coreDotARelPath string, coreArchiveFilePath stri properties := buildProperties.Clone() properties[constants.BUILD_PROPERTIES_COMPILER_C_ELF_FLAGS] = properties[constants.BUILD_PROPERTIES_COMPILER_C_ELF_FLAGS] + optRelax - properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS] = properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS+"."+warningsLevel] + properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS] = properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS+"."+ctx.WarningsLevel] properties[constants.BUILD_PROPERTIES_ARCHIVE_FILE] = coreDotARelPath properties[constants.BUILD_PROPERTIES_ARCHIVE_FILE_PATH] = coreArchiveFilePath properties[constants.BUILD_PROPERTIES_OBJECT_FILES] = objectFileList - _, err := builder_utils.ExecRecipe(properties, constants.RECIPE_C_COMBINE_PATTERN, false, verbose, verbose, logger) + _, _, err := builder_utils.ExecRecipe(ctx, properties, constants.RECIPE_C_COMBINE_PATTERN, false, /* stdout */ utils.ShowIfVerbose, /* stderr */ utils.Show) return err } diff --git a/phases/sizer.go b/phases/sizer.go index 7ea86997..fa719e18 100644 --- a/phases/sizer.go +++ b/phases/sizer.go @@ -38,6 +38,7 @@ import ( "github.com/arduino/arduino-builder/constants" "github.com/arduino/arduino-builder/i18n" "github.com/arduino/arduino-builder/types" + "github.com/arduino/arduino-builder/utils" "github.com/arduino/go-properties-map" ) @@ -52,11 +53,8 @@ func (s *Sizer) Run(ctx *types.Context) error { } buildProperties := ctx.BuildProperties - verbose := ctx.Verbose - warningsLevel := ctx.WarningsLevel - logger := ctx.GetLogger() - err := checkSize(buildProperties, verbose, warningsLevel, logger) + err := checkSize(ctx, buildProperties) if err != nil { return i18n.WrapError(err) } @@ -64,10 +62,11 @@ func (s *Sizer) Run(ctx *types.Context) error { return nil } -func checkSize(buildProperties properties.Map, verbose bool, warningsLevel string, logger i18n.Logger) error { +func checkSize(ctx *types.Context, buildProperties properties.Map) error { + logger := ctx.GetLogger() properties := buildProperties.Clone() - properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS] = properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS+"."+warningsLevel] + properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS] = properties[constants.BUILD_PROPERTIES_COMPILER_WARNING_FLAGS+"."+ctx.WarningsLevel] maxTextSizeString := properties[constants.PROPERTY_UPLOAD_MAX_SIZE] maxDataSizeString := properties[constants.PROPERTY_UPLOAD_MAX_DATA_SIZE] @@ -89,7 +88,7 @@ func checkSize(buildProperties properties.Map, verbose bool, warningsLevel strin } } - textSize, dataSize, _, err := execSizeReceipe(properties, logger) + textSize, dataSize, _, err := execSizeRecipe(ctx, properties) if err != nil { logger.Println(constants.LOG_LEVEL_WARN, constants.MSG_SIZER_ERROR_NO_RULE) return nil @@ -127,8 +126,8 @@ func checkSize(buildProperties properties.Map, verbose bool, warningsLevel strin return nil } -func execSizeReceipe(properties properties.Map, logger i18n.Logger) (textSize int, dataSize int, eepromSize int, resErr error) { - out, err := builder_utils.ExecRecipe(properties, constants.RECIPE_SIZE_PATTERN, false, false, false, logger) +func execSizeRecipe(ctx *types.Context, properties properties.Map) (textSize int, dataSize int, eepromSize int, resErr error) { + out, _, err := builder_utils.ExecRecipe(ctx, properties, constants.RECIPE_SIZE_PATTERN, false /* stdout */, utils.Capture /* stderr */, utils.Show) if err != nil { resErr = errors.New("Error while determining sketch size: " + err.Error()) return diff --git a/phases/sketch_builder.go b/phases/sketch_builder.go index 04c7ded7..8a3158d9 100644 --- a/phases/sketch_builder.go +++ b/phases/sketch_builder.go @@ -46,9 +46,6 @@ func (s *SketchBuilder) Run(ctx *types.Context) error { buildProperties := ctx.BuildProperties includes := ctx.IncludeFolders includes = utils.Map(includes, utils.WrapWithHyphenI) - verbose := ctx.Verbose - warningsLevel := ctx.WarningsLevel - logger := ctx.GetLogger() err := utils.EnsureFolderExists(sketchBuildPath) if err != nil { @@ -56,7 +53,7 @@ func (s *SketchBuilder) Run(ctx *types.Context) error { } var objectFiles []string - objectFiles, err = builder_utils.CompileFiles(objectFiles, sketchBuildPath, false, sketchBuildPath, buildProperties, includes, verbose, warningsLevel, logger) + objectFiles, err = builder_utils.CompileFiles(ctx, objectFiles, sketchBuildPath, false, sketchBuildPath, buildProperties, includes) if err != nil { return i18n.WrapError(err) } @@ -64,7 +61,7 @@ func (s *SketchBuilder) Run(ctx *types.Context) error { // The "src/" subdirectory of a sketch is compiled recursively sketchSrcPath := filepath.Join(sketchBuildPath, constants.SKETCH_FOLDER_SRC) if info, err := os.Stat(sketchSrcPath); err == nil && info.IsDir() { - objectFiles, err = builder_utils.CompileFiles(objectFiles, sketchSrcPath, true, sketchSrcPath, buildProperties, includes, verbose, warningsLevel, logger) + objectFiles, err = builder_utils.CompileFiles(ctx, objectFiles, sketchSrcPath, true, sketchSrcPath, buildProperties, includes) if err != nil { return i18n.WrapError(err) } diff --git a/read_file_and_store_in_context.go b/read_file_and_store_in_context.go index a2533929..bd165617 100644 --- a/read_file_and_store_in_context.go +++ b/read_file_and_store_in_context.go @@ -36,11 +36,12 @@ import ( ) type ReadFileAndStoreInContext struct { + FileToRead string Target *string } func (s *ReadFileAndStoreInContext) Run(ctx *types.Context) error { - bytes, err := ioutil.ReadFile(ctx.FileToRead) + bytes, err := ioutil.ReadFile(s.FileToRead) if err != nil { return i18n.WrapError(err) } diff --git a/recipe_runner.go b/recipe_runner.go index bec23f90..ace82ade 100644 --- a/recipe_runner.go +++ b/recipe_runner.go @@ -34,6 +34,7 @@ import ( "github.com/arduino/arduino-builder/constants" "github.com/arduino/arduino-builder/i18n" "github.com/arduino/arduino-builder/types" + "github.com/arduino/arduino-builder/utils" "os" "sort" "strings" @@ -51,8 +52,6 @@ func (s *RecipeByPrefixSuffixRunner) Run(ctx *types.Context) error { } buildProperties := ctx.BuildProperties.Clone() - verbose := ctx.Verbose - recipes := findRecipes(buildProperties, s.Prefix, s.Suffix) properties := buildProperties.Clone() @@ -60,7 +59,7 @@ func (s *RecipeByPrefixSuffixRunner) Run(ctx *types.Context) error { if ctx.DebugLevel >= 10 { logger.Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, constants.MSG_RUNNING_RECIPE, recipe) } - _, err := builder_utils.ExecRecipe(properties, recipe, false, verbose, verbose, logger) + _, _, err := builder_utils.ExecRecipe(ctx, properties, recipe, false /* stdout */, utils.ShowIfVerbose /* stderr */, utils.Show) if err != nil { return i18n.WrapError(err) } diff --git a/test/builder_utils_test.go b/test/builder_utils_test.go index c6fc941c..32ff14dc 100644 --- a/test/builder_utils_test.go +++ b/test/builder_utils_test.go @@ -31,6 +31,7 @@ package test import ( "github.com/arduino/arduino-builder/builder_utils" + "github.com/arduino/arduino-builder/types" "github.com/arduino/arduino-builder/utils" "github.com/stretchr/testify/require" "io/ioutil" @@ -52,27 +53,33 @@ func tempFile(t *testing.T, prefix string) string { } func TestObjFileIsUpToDateObjMissing(t *testing.T) { + ctx := &types.Context{} + sourceFile := tempFile(t, "source") defer os.RemoveAll(sourceFile) - upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, "", "") + upToDate, err := builder_utils.ObjFileIsUpToDate(ctx, sourceFile, "", "") NoError(t, err) require.False(t, upToDate) } func TestObjFileIsUpToDateDepMissing(t *testing.T) { + ctx := &types.Context{} + sourceFile := tempFile(t, "source") defer os.RemoveAll(sourceFile) objFile := tempFile(t, "obj") defer os.RemoveAll(objFile) - upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, objFile, "") + upToDate, err := builder_utils.ObjFileIsUpToDate(ctx, sourceFile, objFile, "") NoError(t, err) require.False(t, upToDate) } func TestObjFileIsUpToDateObjOlder(t *testing.T) { + ctx := &types.Context{} + objFile := tempFile(t, "obj") defer os.RemoveAll(objFile) depFile := tempFile(t, "dep") @@ -83,12 +90,14 @@ func TestObjFileIsUpToDateObjOlder(t *testing.T) { sourceFile := tempFile(t, "source") defer os.RemoveAll(sourceFile) - upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) + upToDate, err := builder_utils.ObjFileIsUpToDate(ctx, sourceFile, objFile, depFile) NoError(t, err) require.False(t, upToDate) } func TestObjFileIsUpToDateObjNewer(t *testing.T) { + ctx := &types.Context{} + sourceFile := tempFile(t, "source") defer os.RemoveAll(sourceFile) @@ -99,12 +108,14 @@ func TestObjFileIsUpToDateObjNewer(t *testing.T) { depFile := tempFile(t, "dep") defer os.RemoveAll(depFile) - upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) + upToDate, err := builder_utils.ObjFileIsUpToDate(ctx, sourceFile, objFile, depFile) NoError(t, err) require.True(t, upToDate) } func TestObjFileIsUpToDateDepIsNewer(t *testing.T) { + ctx := &types.Context{} + sourceFile := tempFile(t, "source") defer os.RemoveAll(sourceFile) @@ -122,12 +133,14 @@ func TestObjFileIsUpToDateDepIsNewer(t *testing.T) { utils.WriteFile(depFile, objFile+": \\\n\t"+sourceFile+" \\\n\t"+headerFile) - upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) + upToDate, err := builder_utils.ObjFileIsUpToDate(ctx, sourceFile, objFile, depFile) NoError(t, err) require.False(t, upToDate) } func TestObjFileIsUpToDateDepIsOlder(t *testing.T) { + ctx := &types.Context{} + sourceFile := tempFile(t, "source") defer os.RemoveAll(sourceFile) @@ -143,12 +156,14 @@ func TestObjFileIsUpToDateDepIsOlder(t *testing.T) { utils.WriteFile(depFile, objFile+": \\\n\t"+sourceFile+" \\\n\t"+headerFile) - upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) + upToDate, err := builder_utils.ObjFileIsUpToDate(ctx, sourceFile, objFile, depFile) NoError(t, err) require.True(t, upToDate) } func TestObjFileIsUpToDateDepIsWrong(t *testing.T) { + ctx := &types.Context{} + sourceFile := tempFile(t, "source") defer os.RemoveAll(sourceFile) @@ -166,7 +181,7 @@ func TestObjFileIsUpToDateDepIsWrong(t *testing.T) { utils.WriteFile(depFile, sourceFile+": \\\n\t"+sourceFile+" \\\n\t"+headerFile) - upToDate, err := builder_utils.ObjFileIsUpToDate(sourceFile, objFile, depFile) + upToDate, err := builder_utils.ObjFileIsUpToDate(ctx, sourceFile, objFile, depFile) NoError(t, err) require.False(t, upToDate) } diff --git a/test/includes_finder_with_regexp_test.go b/test/includes_finder_with_regexp_test.go index d95153a5..c3c048cc 100644 --- a/test/includes_finder_with_regexp_test.go +++ b/test/includes_finder_with_regexp_test.go @@ -43,27 +43,17 @@ func TestIncludesFinderWithRegExp(t *testing.T) { "#include \n" + "^\n" + "compilation terminated." - ctx.Source = output + include := builder.IncludesFinderWithRegExp(ctx, output) - parser := builder.IncludesFinderWithRegExp{Source: &ctx.Source} - err := parser.Run(ctx) - NoError(t, err) - - require.Equal(t, "SPI.h", ctx.IncludeJustFound) + require.Equal(t, "SPI.h", include) } func TestIncludesFinderWithRegExpEmptyOutput(t *testing.T) { ctx := &types.Context{} - output := "" - - ctx.Source = output + include := builder.IncludesFinderWithRegExp(ctx, "") - parser := builder.IncludesFinderWithRegExp{Source: &ctx.Source} - err := parser.Run(ctx) - NoError(t, err) - - require.Equal(t, "", ctx.IncludeJustFound) + require.Equal(t, "", include) } func TestIncludesFinderWithRegExpPaddedIncludes(t *testing.T) { @@ -73,13 +63,9 @@ func TestIncludesFinderWithRegExpPaddedIncludes(t *testing.T) { " # include \n" + " ^\n" + "compilation terminated.\n" - ctx.Source = output - - parser := builder.IncludesFinderWithRegExp{Source: &ctx.Source} - err := parser.Run(ctx) - NoError(t, err) + include := builder.IncludesFinderWithRegExp(ctx, output) - require.Equal(t, "Wire.h", ctx.IncludeJustFound) + require.Equal(t, "Wire.h", include) } func TestIncludesFinderWithRegExpPaddedIncludes2(t *testing.T) { @@ -89,13 +75,9 @@ func TestIncludesFinderWithRegExpPaddedIncludes2(t *testing.T) { " #\t\t\tinclude \n" + " ^\n" + "compilation terminated.\n" - ctx.Source = output + include := builder.IncludesFinderWithRegExp(ctx, output) - parser := builder.IncludesFinderWithRegExp{Source: &ctx.Source} - err := parser.Run(ctx) - NoError(t, err) - - require.Equal(t, "Wire.h", ctx.IncludeJustFound) + require.Equal(t, "Wire.h", include) } func TestIncludesFinderWithRegExpPaddedIncludes3(t *testing.T) { @@ -104,13 +86,9 @@ func TestIncludesFinderWithRegExpPaddedIncludes3(t *testing.T) { output := "/some/path/sketch.ino:1:33: fatal error: SPI.h: No such file or directory\n" + "compilation terminated.\n" - ctx.Source = output - - parser := builder.IncludesFinderWithRegExp{Source: &ctx.Source} - err := parser.Run(ctx) - NoError(t, err) + include := builder.IncludesFinderWithRegExp(ctx, output) - require.Equal(t, "SPI.h", ctx.IncludeJustFound) + require.Equal(t, "SPI.h", include) } func TestIncludesFinderWithRegExpPaddedIncludes4(t *testing.T) { @@ -119,11 +97,7 @@ func TestIncludesFinderWithRegExpPaddedIncludes4(t *testing.T) { output := "In file included from /tmp/arduino_modified_sketch_815412/binouts.ino:52:0:\n" + "/tmp/arduino_build_static/sketch/regtable.h:31:22: fatal error: register.h: No such file or directory\n" - ctx.Source = output - - parser := builder.IncludesFinderWithRegExp{Source: &ctx.Source} - err := parser.Run(ctx) - NoError(t, err) + include := builder.IncludesFinderWithRegExp(ctx, output) - require.Equal(t, "register.h", ctx.IncludeJustFound) + require.Equal(t, "register.h", include) } diff --git a/test/read_file_and_store_in_context_test.go b/test/read_file_and_store_in_context_test.go index a83cfa0c..b33ff47c 100644 --- a/test/read_file_and_store_in_context_test.go +++ b/test/read_file_and_store_in_context_test.go @@ -47,9 +47,8 @@ func TestReadFileAndStoreInContext(t *testing.T) { utils.WriteFile(file.Name(), "test test\nciao") ctx := &types.Context{} - ctx.FileToRead = file.Name() - command := &builder.ReadFileAndStoreInContext{Target: &ctx.SourceGccMinusE} + command := &builder.ReadFileAndStoreInContext{FileToRead: file.Name(), Target: &ctx.SourceGccMinusE} err = command.Run(ctx) NoError(t, err) diff --git a/test/utils_test.go b/test/utils_test.go index da8395fa..6ada0c29 100644 --- a/test/utils_test.go +++ b/test/utils_test.go @@ -70,6 +70,25 @@ func TestCommandLineParser(t *testing.T) { require.Equal(t, "/tmp/sketch321469072.cpp", parts[22]) } +func TestPrintableCommand(t *testing.T) { + parts := []string{ + "/path/to/dir with spaces/cmd", + "arg1", + "arg-\"with\"-quotes", + "specialchar-`~!@#$%^&*()-_=+[{]}\\|;:'\",<.>/?-argument", + "arg with spaces", + "arg\twith\t\ttabs", + "lastarg", + } + correct := "\"/path/to/dir with spaces/cmd\"" + + " arg1 \"arg-\\\"with\\\"-quotes\"" + + " \"specialchar-`~!@#$%^&*()-_=+[{]}\\\\|;:'\\\",<.>/?-argument\"" + + " \"arg with spaces\" \"arg\twith\t\ttabs\"" + + " lastarg" + result := utils.PrintableCommand(parts) + require.Equal(t, correct, result) +} + func TestCommandLineParserError(t *testing.T) { command := "\"command missing quote" diff --git a/types/context.go b/types/context.go index bc9491dd..2e7ae92c 100644 --- a/types/context.go +++ b/types/context.go @@ -61,7 +61,6 @@ type Context struct { HeaderToLibraries map[string][]*Library ImportedLibraries []*Library LibrariesResolutionResults map[string]LibraryResolutionResult - IncludeJustFound string IncludeFolders []string OutputGccMinusM string @@ -85,9 +84,6 @@ type Context struct { // Logging logger i18n.Logger DebugLevel int - - // ReadFileAndStoreInContext command - FileToRead string } func (ctx *Context) ExtractBuildOptions() properties.Map { diff --git a/utils/utils.go b/utils/utils.go index 318b3984..21e6241e 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -30,6 +30,8 @@ package utils import ( + "fmt" + "bytes" "crypto/md5" "encoding/hex" "io/ioutil" @@ -254,6 +256,68 @@ func PrepareCommand(pattern string, logger i18n.Logger) (*exec.Cmd, error) { return PrepareCommandFilteredArgs(pattern, filterEmptyArg, logger) } +func printableArgument(arg string) string { + if strings.ContainsAny(arg, "\"\\ \t") { + arg = strings.Replace(arg, "\\", "\\\\", -1) + arg = strings.Replace(arg, "\"", "\\\"", -1) + return "\"" + arg + "\"" + } else { + return arg + } +} + +// Convert a command and argument slice back to a printable string. +// This adds basic escaping which is sufficient for debug output, but +// probably not for shell interpretation. This essentially reverses +// ParseCommandLine. +func PrintableCommand(parts []string) string { + return strings.Join(Map(parts, printableArgument), " ") +} + +const ( + Ignore = 0 // Redirect to null + Show = 1 // Show on stdout/stderr as normal + ShowIfVerbose = 2 // Show if verbose is set, Ignore otherwise + Capture = 3 // Capture into buffer +) + +func ExecCommand(ctx *types.Context, command *exec.Cmd, stdout int, stderr int) ([]byte, []byte, error) { + if ctx.Verbose { + fmt.Println(PrintableCommand(command.Args)) + } + + if stdout == Capture { + buffer := &bytes.Buffer{} + command.Stdout = buffer + } else if stdout == Show || stdout == ShowIfVerbose && ctx.Verbose { + command.Stdout = os.Stdout + } + + if stderr == Capture { + buffer := &bytes.Buffer{} + command.Stderr = buffer + } else if stderr == Show || stderr == ShowIfVerbose && ctx.Verbose { + command.Stderr = os.Stderr + } + + err := command.Start() + if err != nil { + return nil, nil, i18n.WrapError(err) + } + + err = command.Wait() + + var outbytes, errbytes []byte + if buf, ok := command.Stdout.(*bytes.Buffer); ok { + outbytes = buf.Bytes() + } + if buf, ok := command.Stderr.(*bytes.Buffer); ok { + errbytes = buf.Bytes() + } + + return outbytes, errbytes, i18n.WrapError(err) +} + func MapHas(aMap map[string]interface{}, key string) bool { _, ok := aMap[key] return ok