diff --git a/src/arduino.cc/arduino-builder/main.go b/src/arduino.cc/arduino-builder/main.go index 83ef32cc..ec5a3d51 100644 --- a/src/arduino.cc/arduino-builder/main.go +++ b/src/arduino.cc/arduino-builder/main.go @@ -64,6 +64,7 @@ const FLAG_FQBN = "fqbn" const FLAG_IDE_VERSION = "ide-version" const FLAG_CORE_API_VERSION = "core-api-version" const FLAG_BUILD_PATH = "build-path" +const FLAG_BUILD_CACHE = "build-cache" const FLAG_VERBOSE = "verbose" const FLAG_QUIET = "quiet" const FLAG_DEBUG_LEVEL = "debug-level" @@ -126,6 +127,7 @@ var fqbnFlag *string var coreAPIVersionFlag *string var ideVersionFlag *string var buildPathFlag *string +var buildCachePathFlag *string var verboseFlag *bool var quietFlag *bool var debugLevelFlag *int @@ -148,6 +150,7 @@ func init() { coreAPIVersionFlag = flag.String(FLAG_CORE_API_VERSION, "10600", "version of core APIs (used to populate ARDUINO #define)") ideVersionFlag = flag.String(FLAG_IDE_VERSION, "10600", "[deprecated] use '"+FLAG_CORE_API_VERSION+"' instead") buildPathFlag = flag.String(FLAG_BUILD_PATH, "", "build path") + buildCachePathFlag = flag.String(FLAG_BUILD_CACHE, "", "builds of 'core.a' are saved into this folder to be cached and reused") verboseFlag = flag.Bool(FLAG_VERBOSE, false, "if 'true' prints lots of stuff") quietFlag = flag.Bool(FLAG_QUIET, false, "if 'true' doesn't print any warnings or progress or whatever") debugLevelFlag = flag.Int(FLAG_DEBUG_LEVEL, builder.DEFAULT_DEBUG_LEVEL, "Turns on debugging messages. The higher, the chattier") @@ -256,6 +259,25 @@ func main() { } ctx.BuildPath = buildPath + // FLAG_BUILD_CACHE + buildCachePath, err := gohasissues.Unquote(*buildCachePathFlag) + if err != nil { + printCompleteError(err) + } + if buildCachePath != "" { + _, err := os.Stat(buildCachePath) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + err = utils.EnsureFolderExists(buildCachePath) + if err != nil { + printCompleteError(err) + } + } + ctx.BuildCachePath = buildCachePath + // FLAG_VID_PID if *vidPidFlag != "" { ctx.USBVidPid = *vidPidFlag diff --git a/src/arduino.cc/builder/add_additional_entries_to_context.go b/src/arduino.cc/builder/add_additional_entries_to_context.go index 91bc2a0e..795aea01 100644 --- a/src/arduino.cc/builder/add_additional_entries_to_context.go +++ b/src/arduino.cc/builder/add_additional_entries_to_context.go @@ -30,10 +30,11 @@ package builder import ( + "path/filepath" + "arduino.cc/builder/constants" "arduino.cc/builder/i18n" "arduino.cc/builder/types" - "path/filepath" ) type AddAdditionalEntriesToContext struct{} @@ -64,6 +65,15 @@ func (s *AddAdditionalEntriesToContext) Run(ctx *types.Context) error { ctx.CoreBuildPath = coreBuildPath } + if ctx.BuildCachePath != "" { + coreBuildCachePath, err := filepath.Abs(filepath.Join(ctx.BuildCachePath, constants.FOLDER_CORE)) + if err != nil { + return i18n.WrapError(err) + } + + ctx.CoreBuildCachePath = coreBuildCachePath + } + if ctx.WarningsLevel == "" { ctx.WarningsLevel = DEFAULT_WARNINGS_LEVEL } diff --git a/src/arduino.cc/builder/builder_utils/utils.go b/src/arduino.cc/builder/builder_utils/utils.go index d21420a3..97b6e355 100644 --- a/src/arduino.cc/builder/builder_utils/utils.go +++ b/src/arduino.cc/builder/builder_utils/utils.go @@ -32,6 +32,7 @@ package builder_utils import ( "bytes" "fmt" + "io" "os" "os/exec" "path/filepath" @@ -116,6 +117,34 @@ func findFilesInFolder(sourcePath string, extension string, recurse bool) ([]str return sources, nil } +func findAllFilesInFolder(sourcePath string, recurse bool) ([]string, error) { + files, err := utils.ReadDirFiltered(sourcePath, utils.FilterFiles()) + if err != nil { + return nil, i18n.WrapError(err) + } + var sources []string + for _, file := range files { + sources = append(sources, filepath.Join(sourcePath, file.Name())) + } + + if recurse { + folders, err := utils.ReadDirFiltered(sourcePath, utils.FilterDirs) + if err != nil { + return nil, i18n.WrapError(err) + } + + for _, folder := range folders { + otherSources, err := findAllFilesInFolder(filepath.Join(sourcePath, folder.Name()), recurse) + if err != nil { + return nil, i18n.WrapError(err) + } + sources = append(sources, otherSources...) + } + } + + return sources, nil +} + func compileFilesWithRecipe(objectFiles []string, sourcePath string, sources []string, buildPath string, buildProperties properties.Map, includes []string, recipe string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) { for _, source := range sources { objectFile, err := compileFileWithRecipe(sourcePath, source, buildPath, buildProperties, includes, recipe, verbose, warningsLevel, logger) @@ -258,6 +287,28 @@ func nonEmptyString(s string) bool { return s != constants.EMPTY_STRING } +func CoreOrReferencedCoreHasChanged(corePath, targetCorePath, targetFile string) bool { + + targetFileStat, err := os.Stat(targetFile) + if err == nil { + files, err := findAllFilesInFolder(corePath, true) + if err != nil { + return true + } + for _, file := range files { + fileStat, err := os.Stat(file) + if err != nil || fileStat.ModTime().After(targetFileStat.ModTime()) { + return true + } + } + if targetCorePath != constants.EMPTY_STRING && !strings.EqualFold(corePath, targetCorePath) { + return CoreOrReferencedCoreHasChanged(targetCorePath, constants.EMPTY_STRING, targetFile) + } + return false + } + return true +} + func ArchiveCompiledFiles(buildPath string, archiveFile string, objectFiles []string, buildProperties properties.Map, verbose bool, logger i18n.Logger) (string, error) { archiveFilePath := filepath.Join(buildPath, archiveFile) @@ -366,3 +417,59 @@ func ExecRecipeCollectStdErr(buildProperties properties.Map, recipe string, remo func RemoveHyphenMDDFlagFromGCCCommandLine(buildProperties properties.Map) { buildProperties[constants.BUILD_PROPERTIES_COMPILER_CPP_FLAGS] = strings.Replace(buildProperties[constants.BUILD_PROPERTIES_COMPILER_CPP_FLAGS], "-MMD", "", -1) } + +// CopyFile copies the contents of the file named src to the file named +// by dst. The file will be created if it does not already exist. If the +// destination file exists, all it's contents will be replaced by the contents +// of the source file. The file mode will be copied from the source and +// the copied data is synced/flushed to stable storage. +func CopyFile(src, dst string) (err error) { + in, err := os.Open(src) + if err != nil { + return + } + defer in.Close() + + out, err := os.Create(dst) + if err != nil { + return + } + defer func() { + if e := out.Close(); e != nil { + err = e + } + }() + + _, err = io.Copy(out, in) + if err != nil { + return + } + + err = out.Sync() + if err != nil { + return + } + + si, err := os.Stat(src) + if err != nil { + return + } + err = os.Chmod(dst, si.Mode()) + if err != nil { + return + } + + return +} + +// GetCachedCoreArchiveFileName returns the filename to be used to store +// the global cached core.a. +func GetCachedCoreArchiveFileName(fqbn, coreFolder string) string { + fqbnToUnderscore := strings.Replace(fqbn, ":", "_", -1) + fqbnToUnderscore = strings.Replace(fqbnToUnderscore, "=", "_", -1) + if absCoreFolder, err := filepath.Abs(coreFolder); err == nil { + coreFolder = absCoreFolder + } // silently continue if absolute path can't be detected + hash := utils.MD5Sum([]byte(coreFolder)) + return "core_" + fqbnToUnderscore + "_" + hash + ".a" +} diff --git a/src/arduino.cc/builder/constants/constants.go b/src/arduino.cc/builder/constants/constants.go index cdbdb72b..fdbf00cc 100644 --- a/src/arduino.cc/builder/constants/constants.go +++ b/src/arduino.cc/builder/constants/constants.go @@ -55,6 +55,7 @@ const BUILD_PROPERTIES_EXTRA_TIME_DST = "extra.time.dst" const BUILD_PROPERTIES_EXTRA_TIME_LOCAL = "extra.time.local" const BUILD_PROPERTIES_EXTRA_TIME_UTC = "extra.time.utc" const BUILD_PROPERTIES_EXTRA_TIME_ZONE = "extra.time.zone" +const BUILD_PROPERTIES_FQBN = "build.fqbn" const BUILD_PROPERTIES_INCLUDES = "includes" const BUILD_PROPERTIES_OBJECT_FILE = "object_file" const BUILD_PROPERTIES_OBJECT_FILES = "object_files" diff --git a/src/arduino.cc/builder/phases/core_builder.go b/src/arduino.cc/builder/phases/core_builder.go index 0a0c78c6..c906d042 100644 --- a/src/arduino.cc/builder/phases/core_builder.go +++ b/src/arduino.cc/builder/phases/core_builder.go @@ -30,6 +30,8 @@ package phases import ( + "path/filepath" + "arduino.cc/builder/builder_utils" "arduino.cc/builder/constants" "arduino.cc/builder/i18n" @@ -42,6 +44,7 @@ type CoreBuilder struct{} func (s *CoreBuilder) Run(ctx *types.Context) error { coreBuildPath := ctx.CoreBuildPath + coreBuildCachePath := ctx.CoreBuildCachePath buildProperties := ctx.BuildProperties verbose := ctx.Verbose warningsLevel := ctx.WarningsLevel @@ -52,7 +55,14 @@ func (s *CoreBuilder) Run(ctx *types.Context) error { return i18n.WrapError(err) } - archiveFile, objectFiles, err := compileCore(coreBuildPath, buildProperties, verbose, warningsLevel, logger) + if coreBuildCachePath != "" { + err := utils.EnsureFolderExists(coreBuildCachePath) + if err != nil { + return i18n.WrapError(err) + } + } + + archiveFile, objectFiles, err := compileCore(coreBuildPath, coreBuildCachePath, buildProperties, verbose, warningsLevel, logger) if err != nil { return i18n.WrapError(err) } @@ -63,10 +73,12 @@ func (s *CoreBuilder) Run(ctx *types.Context) error { return nil } -func compileCore(buildPath string, buildProperties properties.Map, verbose bool, warningsLevel string, logger i18n.Logger) (string, []string, error) { +func compileCore(buildPath string, buildCachePath string, buildProperties properties.Map, verbose bool, warningsLevel string, logger i18n.Logger) (string, []string, error) { coreFolder := buildProperties[constants.BUILD_PROPERTIES_BUILD_CORE_PATH] variantFolder := buildProperties[constants.BUILD_PROPERTIES_BUILD_VARIANT_PATH] + targetCoreFolder := buildProperties[constants.BUILD_PROPERTIES_RUNTIME_PLATFORM_PATH] + includes := []string{} includes = append(includes, coreFolder) if variantFolder != constants.EMPTY_STRING { @@ -84,6 +96,24 @@ func compileCore(buildPath string, buildProperties properties.Map, verbose bool, } } + // Recreate the archive if ANY of the core files (including platform.txt) has changed + realCoreFolder := utils.GetParentFolder(coreFolder, 2) + + var targetArchivedCore string + if buildCachePath != "" { + archivedCoreName := builder_utils.GetCachedCoreArchiveFileName(buildProperties[constants.BUILD_PROPERTIES_FQBN], realCoreFolder) + targetArchivedCore = filepath.Join(buildCachePath, archivedCoreName) + canUseArchivedCore := !builder_utils.CoreOrReferencedCoreHasChanged(realCoreFolder, targetCoreFolder, targetArchivedCore) + + if canUseArchivedCore { + // use archived core + if verbose { + logger.Println(constants.LOG_LEVEL_INFO, "Using precompiled core") + } + return targetArchivedCore, variantObjectFiles, nil + } + } + coreObjectFiles, err := builder_utils.CompileFiles([]string{}, coreFolder, true, buildPath, buildProperties, includes, verbose, warningsLevel, logger) if err != nil { return "", nil, i18n.WrapError(err) @@ -94,5 +124,11 @@ func compileCore(buildPath string, buildProperties properties.Map, verbose bool, return "", nil, i18n.WrapError(err) } + // archive core.a + if targetArchivedCore != "" { + logger.Println(constants.LOG_LEVEL_DEBUG, "Archiving built core (caching) in: "+targetArchivedCore) + builder_utils.CopyFile(archiveFile, targetArchivedCore) + } + return archiveFile, variantObjectFiles, nil } diff --git a/src/arduino.cc/builder/setup_build_properties.go b/src/arduino.cc/builder/setup_build_properties.go index e278a39d..02a272fc 100644 --- a/src/arduino.cc/builder/setup_build_properties.go +++ b/src/arduino.cc/builder/setup_build_properties.go @@ -70,6 +70,7 @@ func (s *SetupBuildProperties) Run(ctx *types.Context) error { buildProperties[constants.BUILD_PROPERTIES_RUNTIME_PLATFORM_PATH] = targetPlatform.Folder buildProperties[constants.BUILD_PROPERTIES_RUNTIME_HARDWARE_PATH] = filepath.Join(targetPlatform.Folder, "..") buildProperties[constants.BUILD_PROPERTIES_RUNTIME_IDE_VERSION] = ctx.ArduinoAPIVersion + buildProperties[constants.BUILD_PROPERTIES_FQBN] = ctx.FQBN buildProperties[constants.IDE_VERSION] = ctx.ArduinoAPIVersion buildProperties[constants.BUILD_PROPERTIES_RUNTIME_OS] = utils.PrettyOSName() diff --git a/src/arduino.cc/builder/test/builder_test.go b/src/arduino.cc/builder/test/builder_test.go index 3244c705..bfcc7edd 100644 --- a/src/arduino.cc/builder/test/builder_test.go +++ b/src/arduino.cc/builder/test/builder_test.go @@ -30,35 +30,42 @@ package test import ( - "arduino.cc/builder" - "arduino.cc/builder/constants" - "arduino.cc/builder/types" - "github.com/stretchr/testify/require" "os" "os/exec" "path/filepath" "testing" -) + "time" -func TestBuilderEmptySketch(t *testing.T) { - DownloadCoresAndToolsAndLibraries(t) + "arduino.cc/builder" + "arduino.cc/builder/builder_utils" + "arduino.cc/builder/constants" + "arduino.cc/builder/types" + "github.com/stretchr/testify/require" +) - ctx := &types.Context{ +func prepareBuilderTestContext(sketchPath, fqbn string) *types.Context { + return &types.Context{ + SketchLocation: sketchPath, + FQBN: fqbn, HardwareFolders: []string{filepath.Join("..", "hardware"), "hardware", "downloaded_hardware"}, ToolsFolders: []string{"downloaded_tools"}, BuiltInLibrariesFolders: []string{"downloaded_libraries"}, OtherLibrariesFolders: []string{"libraries"}, - SketchLocation: filepath.Join("sketch1", "sketch.ino"), - FQBN: "arduino:avr:uno", ArduinoAPIVersion: "10600", - Verbose: true, + Verbose: false, } +} - buildPath := SetupBuildPath(t, ctx) - defer os.RemoveAll(buildPath) +func TestBuilderEmptySketch(t *testing.T) { + DownloadCoresAndToolsAndLibraries(t) + ctx := prepareBuilderTestContext(filepath.Join("sketch1", "sketch.ino"), "arduino:avr:uno") ctx.DebugLevel = 10 + buildPath := SetupBuildPath(t, ctx) + defer os.RemoveAll(buildPath) + + // Run builder command := builder.Builder{} err := command.Run(ctx) NoError(t, err) @@ -78,20 +85,12 @@ func TestBuilderEmptySketch(t *testing.T) { func TestBuilderBridge(t *testing.T) { DownloadCoresAndToolsAndLibraries(t) - ctx := &types.Context{ - HardwareFolders: []string{filepath.Join("..", "hardware"), "hardware", "downloaded_hardware"}, - ToolsFolders: []string{"downloaded_tools"}, - BuiltInLibrariesFolders: []string{"downloaded_libraries"}, - OtherLibrariesFolders: []string{"libraries"}, - SketchLocation: filepath.Join("downloaded_libraries", "Bridge", "examples", "Bridge", "Bridge.ino"), - FQBN: "arduino:avr:leonardo", - ArduinoAPIVersion: "10600", - Verbose: true, - } + ctx := prepareBuilderTestContext(filepath.Join("downloaded_libraries", "Bridge", "examples", "Bridge", "Bridge.ino"), "arduino:avr:leonardo") buildPath := SetupBuildPath(t, ctx) defer os.RemoveAll(buildPath) + // Run builder command := builder.Builder{} err := command.Run(ctx) NoError(t, err) @@ -113,19 +112,12 @@ func TestBuilderBridge(t *testing.T) { func TestBuilderSketchWithConfig(t *testing.T) { DownloadCoresAndToolsAndLibraries(t) - ctx := &types.Context{ - HardwareFolders: []string{filepath.Join("..", "hardware"), "hardware", "downloaded_hardware"}, - ToolsFolders: []string{"downloaded_tools"}, - BuiltInLibrariesFolders: []string{"downloaded_libraries"}, - OtherLibrariesFolders: []string{"libraries"}, - SketchLocation: filepath.Join("sketch_with_config", "sketch_with_config.ino"), - FQBN: "arduino:avr:leonardo", - ArduinoAPIVersion: "10600", - } + ctx := prepareBuilderTestContext(filepath.Join("sketch_with_config", "sketch_with_config.ino"), "arduino:avr:leonardo") buildPath := SetupBuildPath(t, ctx) defer os.RemoveAll(buildPath) + // Run builder command := builder.Builder{} err := command.Run(ctx) NoError(t, err) @@ -147,23 +139,17 @@ func TestBuilderSketchWithConfig(t *testing.T) { func TestBuilderBridgeTwice(t *testing.T) { DownloadCoresAndToolsAndLibraries(t) - ctx := &types.Context{ - HardwareFolders: []string{filepath.Join("..", "hardware"), "hardware", "downloaded_hardware"}, - ToolsFolders: []string{"downloaded_tools"}, - BuiltInLibrariesFolders: []string{"downloaded_libraries"}, - OtherLibrariesFolders: []string{"libraries"}, - SketchLocation: filepath.Join("downloaded_libraries", "Bridge", "examples", "Bridge", "Bridge.ino"), - FQBN: "arduino:avr:leonardo", - ArduinoAPIVersion: "10600", - } + ctx := prepareBuilderTestContext(filepath.Join("downloaded_libraries", "Bridge", "examples", "Bridge", "Bridge.ino"), "arduino:avr:leonardo") buildPath := SetupBuildPath(t, ctx) defer os.RemoveAll(buildPath) + // Run builder command := builder.Builder{} err := command.Run(ctx) NoError(t, err) + // Run builder again command = builder.Builder{} err = command.Run(ctx) NoError(t, err) @@ -185,21 +171,13 @@ func TestBuilderBridgeTwice(t *testing.T) { func TestBuilderBridgeSAM(t *testing.T) { DownloadCoresAndToolsAndLibraries(t) - ctx := &types.Context{ - HardwareFolders: []string{filepath.Join("..", "hardware"), "hardware", "downloaded_hardware"}, - ToolsFolders: []string{"downloaded_tools"}, - BuiltInLibrariesFolders: []string{"downloaded_libraries"}, - OtherLibrariesFolders: []string{"libraries"}, - SketchLocation: filepath.Join("downloaded_libraries", "Bridge", "examples", "Bridge", "Bridge.ino"), - FQBN: "arduino:sam:arduino_due_x_dbg", - ArduinoAPIVersion: "10600", - } + ctx := prepareBuilderTestContext(filepath.Join("downloaded_libraries", "Bridge", "examples", "Bridge", "Bridge.ino"), "arduino:sam:arduino_due_x_dbg") + ctx.WarningsLevel = "all" buildPath := SetupBuildPath(t, ctx) defer os.RemoveAll(buildPath) - ctx.WarningsLevel = "all" - + // Run builder command := builder.Builder{} err := command.Run(ctx) NoError(t, err) @@ -230,19 +208,14 @@ func TestBuilderBridgeSAM(t *testing.T) { func TestBuilderBridgeRedBearLab(t *testing.T) { DownloadCoresAndToolsAndLibraries(t) - ctx := &types.Context{ - HardwareFolders: []string{filepath.Join("..", "hardware"), "hardware", "downloaded_hardware", "downloaded_board_manager_stuff"}, - ToolsFolders: []string{"downloaded_tools", "downloaded_board_manager_stuff"}, - BuiltInLibrariesFolders: []string{"downloaded_libraries"}, - OtherLibrariesFolders: []string{"libraries"}, - SketchLocation: filepath.Join("downloaded_libraries", "Bridge", "examples", "Bridge", "Bridge.ino"), - FQBN: "RedBearLab:avr:blend", - ArduinoAPIVersion: "10600", - } + ctx := prepareBuilderTestContext(filepath.Join("downloaded_libraries", "Bridge", "examples", "Bridge", "Bridge.ino"), "RedBearLab:avr:blend") + ctx.HardwareFolders = append(ctx.HardwareFolders, "downloaded_board_manager_stuff") + ctx.ToolsFolders = append(ctx.ToolsFolders, "downloaded_board_manager_stuff") buildPath := SetupBuildPath(t, ctx) defer os.RemoveAll(buildPath) + // Run builder command := builder.Builder{} err := command.Run(ctx) NoError(t, err) @@ -264,19 +237,14 @@ func TestBuilderBridgeRedBearLab(t *testing.T) { func TestBuilderSketchNoFunctions(t *testing.T) { DownloadCoresAndToolsAndLibraries(t) - ctx := &types.Context{ - HardwareFolders: []string{filepath.Join("..", "hardware"), "hardware", "downloaded_hardware", "downloaded_board_manager_stuff"}, - ToolsFolders: []string{"downloaded_tools", "downloaded_board_manager_stuff"}, - BuiltInLibrariesFolders: []string{"downloaded_libraries"}, - OtherLibrariesFolders: []string{"libraries"}, - SketchLocation: filepath.Join("sketch_no_functions", "main.ino"), - FQBN: "RedBearLab:avr:blend", - ArduinoAPIVersion: "10600", - } + ctx := prepareBuilderTestContext(filepath.Join("sketch_no_functions", "main.ino"), "RedBearLab:avr:blend") + ctx.HardwareFolders = append(ctx.HardwareFolders, "downloaded_board_manager_stuff") + ctx.ToolsFolders = append(ctx.ToolsFolders, "downloaded_board_manager_stuff") buildPath := SetupBuildPath(t, ctx) defer os.RemoveAll(buildPath) + // Run builder command := builder.Builder{} err := command.Run(ctx) require.Error(t, err) @@ -285,19 +253,14 @@ func TestBuilderSketchNoFunctions(t *testing.T) { func TestBuilderSketchWithBackup(t *testing.T) { DownloadCoresAndToolsAndLibraries(t) - ctx := &types.Context{ - HardwareFolders: []string{filepath.Join("..", "hardware"), "hardware", "downloaded_hardware", "downloaded_board_manager_stuff"}, - ToolsFolders: []string{"downloaded_tools", "downloaded_board_manager_stuff"}, - BuiltInLibrariesFolders: []string{"downloaded_libraries"}, - OtherLibrariesFolders: []string{"libraries"}, - SketchLocation: filepath.Join("sketch_with_backup_files", "sketch.ino"), - FQBN: "arduino:avr:uno", - ArduinoAPIVersion: "10600", - } + ctx := prepareBuilderTestContext(filepath.Join("sketch_with_backup_files", "sketch.ino"), "arduino:avr:uno") + ctx.HardwareFolders = append(ctx.HardwareFolders, "downloaded_board_manager_stuff") + ctx.ToolsFolders = append(ctx.ToolsFolders, "downloaded_board_manager_stuff") buildPath := SetupBuildPath(t, ctx) defer os.RemoveAll(buildPath) + // Run builder command := builder.Builder{} err := command.Run(ctx) NoError(t, err) @@ -306,19 +269,12 @@ func TestBuilderSketchWithBackup(t *testing.T) { func TestBuilderSketchWithOldLib(t *testing.T) { DownloadCoresAndToolsAndLibraries(t) - ctx := &types.Context{ - HardwareFolders: []string{filepath.Join("..", "hardware"), "hardware", "downloaded_hardware"}, - ToolsFolders: []string{"downloaded_tools"}, - BuiltInLibrariesFolders: []string{"downloaded_libraries"}, - OtherLibrariesFolders: []string{"libraries"}, - SketchLocation: filepath.Join("sketch_with_old_lib", "sketch.ino"), - FQBN: "arduino:avr:uno", - ArduinoAPIVersion: "10600", - } + ctx := prepareBuilderTestContext(filepath.Join("sketch_with_old_lib", "sketch.ino"), "arduino:avr:uno") buildPath := SetupBuildPath(t, ctx) defer os.RemoveAll(buildPath) + // Run builder command := builder.Builder{} err := command.Run(ctx) NoError(t, err) @@ -327,19 +283,12 @@ func TestBuilderSketchWithOldLib(t *testing.T) { func TestBuilderSketchWithSubfolders(t *testing.T) { DownloadCoresAndToolsAndLibraries(t) - ctx := &types.Context{ - HardwareFolders: []string{filepath.Join("..", "hardware"), "hardware", "downloaded_hardware"}, - ToolsFolders: []string{"downloaded_tools"}, - BuiltInLibrariesFolders: []string{"downloaded_libraries"}, - OtherLibrariesFolders: []string{"libraries"}, - SketchLocation: filepath.Join("sketch_with_subfolders", "sketch_with_subfolders.ino"), - FQBN: "arduino:avr:uno", - ArduinoAPIVersion: "10600", - } + ctx := prepareBuilderTestContext(filepath.Join("sketch_with_subfolders", "sketch_with_subfolders.ino"), "arduino:avr:uno") buildPath := SetupBuildPath(t, ctx) defer os.RemoveAll(buildPath) + // Run builder command := builder.Builder{} err := command.Run(ctx) NoError(t, err) @@ -348,21 +297,14 @@ func TestBuilderSketchWithSubfolders(t *testing.T) { func TestBuilderSketchBuildPathContainsUnusedPreviouslyCompiledLibrary(t *testing.T) { DownloadCoresAndToolsAndLibraries(t) - ctx := &types.Context{ - HardwareFolders: []string{filepath.Join("..", "hardware"), "hardware", "downloaded_hardware"}, - ToolsFolders: []string{"downloaded_tools"}, - BuiltInLibrariesFolders: []string{"downloaded_libraries"}, - OtherLibrariesFolders: []string{"libraries"}, - SketchLocation: filepath.Join("downloaded_libraries", "Bridge", "examples", "Bridge", "Bridge.ino"), - FQBN: "arduino:avr:leonardo", - ArduinoAPIVersion: "10600", - } + ctx := prepareBuilderTestContext(filepath.Join("downloaded_libraries", "Bridge", "examples", "Bridge", "Bridge.ino"), "arduino:avr:leonardo") buildPath := SetupBuildPath(t, ctx) defer os.RemoveAll(buildPath) NoError(t, os.MkdirAll(filepath.Join(buildPath, constants.FOLDER_LIBRARIES, "SPI"), os.FileMode(0755))) + // Run builder command := builder.Builder{} err := command.Run(ctx) NoError(t, err) @@ -377,22 +319,14 @@ func TestBuilderSketchBuildPathContainsUnusedPreviouslyCompiledLibrary(t *testin func TestBuilderWithBuildPathInSketchDir(t *testing.T) { DownloadCoresAndToolsAndLibraries(t) - ctx := &types.Context{ - HardwareFolders: []string{filepath.Join("..", "hardware"), "hardware", "downloaded_hardware"}, - ToolsFolders: []string{"downloaded_tools"}, - BuiltInLibrariesFolders: []string{"downloaded_libraries"}, - OtherLibrariesFolders: []string{"libraries"}, - SketchLocation: filepath.Join("sketch1", "sketch.ino"), - FQBN: "arduino:avr:uno", - ArduinoAPIVersion: "10600", - Verbose: true, - } + ctx := prepareBuilderTestContext(filepath.Join("sketch1", "sketch.ino"), "arduino:avr:uno") var err error ctx.BuildPath, err = filepath.Abs(filepath.Join("sketch1", "build")) NoError(t, err) defer os.RemoveAll(ctx.BuildPath) + // Run build command := builder.Builder{} err = command.Run(ctx) NoError(t, err) @@ -402,3 +336,48 @@ func TestBuilderWithBuildPathInSketchDir(t *testing.T) { err = command.Run(ctx) NoError(t, err) } + +func TestBuilderCacheCoreAFile(t *testing.T) { + DownloadCoresAndToolsAndLibraries(t) + + ctx := prepareBuilderTestContext(filepath.Join("sketch1", "sketch.ino"), "arduino:avr:uno") + + SetupBuildPath(t, ctx) + defer os.RemoveAll(ctx.BuildPath) + SetupBuildCachePath(t, ctx) + defer os.RemoveAll(ctx.BuildCachePath) + + // Run build + bldr := builder.Builder{} + err := bldr.Run(ctx) + NoError(t, err) + + // Pick timestamp of cached core + coreFolder := filepath.Join("downloaded_hardware", "arduino", "avr") + coreFileName := builder_utils.GetCachedCoreArchiveFileName(ctx.FQBN, coreFolder) + cachedCoreFile := filepath.Join(ctx.CoreBuildCachePath, coreFileName) + coreStatBefore, err := os.Stat(cachedCoreFile) + require.NoError(t, err) + + // Run build again, to verify that the builder skips rebuilding core.a + err = bldr.Run(ctx) + NoError(t, err) + + coreStatAfterRebuild, err := os.Stat(cachedCoreFile) + require.NoError(t, err) + require.Equal(t, coreStatBefore.ModTime(), coreStatAfterRebuild.ModTime()) + + // Touch a file of the core and check if the builder invalidate the cache + time.Sleep(time.Second) + now := time.Now().Local() + err = os.Chtimes(filepath.Join(coreFolder, "cores", "arduino", "Arduino.h"), now, now) + require.NoError(t, err) + + // Run build again, to verify that the builder rebuilds core.a + err = bldr.Run(ctx) + NoError(t, err) + + coreStatAfterTouch, err := os.Stat(cachedCoreFile) + require.NoError(t, err) + require.NotEqual(t, coreStatBefore.ModTime(), coreStatAfterTouch.ModTime()) +} diff --git a/src/arduino.cc/builder/test/helper.go b/src/arduino.cc/builder/test/helper.go index cd6e2b3d..dfb66338 100644 --- a/src/arduino.cc/builder/test/helper.go +++ b/src/arduino.cc/builder/test/helper.go @@ -31,17 +31,18 @@ package test import ( - "arduino.cc/builder/constants" - "arduino.cc/builder/types" - "arduino.cc/builder/utils" "bytes" "fmt" - "github.com/go-errors/errors" - "github.com/stretchr/testify/assert" "io/ioutil" "path/filepath" "testing" "text/template" + + "arduino.cc/builder/constants" + "arduino.cc/builder/types" + "arduino.cc/builder/utils" + "github.com/go-errors/errors" + "github.com/stretchr/testify/assert" ) func LoadAndInterpolate(t *testing.T, filename string, ctx *types.Context) string { @@ -78,12 +79,19 @@ func NoError(t *testing.T, err error, msgAndArgs ...interface{}) { } func SetupBuildPath(t *testing.T, ctx *types.Context) string { - buildPath, err := ioutil.TempDir(constants.EMPTY_STRING, "test") + buildPath, err := ioutil.TempDir(constants.EMPTY_STRING, "test_build_path") NoError(t, err) ctx.BuildPath = buildPath return buildPath } +func SetupBuildCachePath(t *testing.T, ctx *types.Context) string { + buildCachePath, err := ioutil.TempDir(constants.EMPTY_STRING, "test_build_cache") + NoError(t, err) + ctx.BuildCachePath = buildCachePath + return buildCachePath +} + type ByLibraryName []*types.Library func (s ByLibraryName) Len() int { diff --git a/src/arduino.cc/builder/test/wipeout_build_path_if_build_options_changed_test.go b/src/arduino.cc/builder/test/wipeout_build_path_if_build_options_changed_test.go index 9354e2d2..f8fc520d 100644 --- a/src/arduino.cc/builder/test/wipeout_build_path_if_build_options_changed_test.go +++ b/src/arduino.cc/builder/test/wipeout_build_path_if_build_options_changed_test.go @@ -30,14 +30,15 @@ package test import ( + "os" + "path/filepath" + "testing" + "arduino.cc/builder" "arduino.cc/builder/gohasissues" "arduino.cc/builder/types" "arduino.cc/builder/utils" "github.com/stretchr/testify/require" - "os" - "path/filepath" - "testing" ) func TestWipeoutBuildPathIfBuildOptionsChanged(t *testing.T) { @@ -100,34 +101,3 @@ func TestWipeoutBuildPathIfBuildOptionsChangedNoPreviousBuildOptions(t *testing. _, err = os.Stat(filepath.Join(buildPath, "should_not_be_deleted.txt")) NoError(t, err) } - -func TestWipeoutBuildPathIfBuildOptionsChangedBuildOptionsMatch(t *testing.T) { - ctx := &types.Context{} - - buildPath := SetupBuildPath(t, ctx) - defer os.RemoveAll(buildPath) - - ctx.BuildOptionsJsonPrevious = "{ \"old\":\"old\" }" - ctx.BuildOptionsJson = "{ \"old\":\"old\" }" - - utils.TouchFile(filepath.Join(buildPath, "should_not_be_deleted.txt")) - - commands := []types.Command{ - &builder.WipeoutBuildPathIfBuildOptionsChanged{}, - } - - for _, command := range commands { - err := command.Run(ctx) - NoError(t, err) - } - - _, err := os.Stat(buildPath) - NoError(t, err) - - files, err := gohasissues.ReadDir(buildPath) - NoError(t, err) - require.Equal(t, 1, len(files)) - - _, err = os.Stat(filepath.Join(buildPath, "should_not_be_deleted.txt")) - NoError(t, err) -} diff --git a/src/arduino.cc/builder/types/context.go b/src/arduino.cc/builder/types/context.go index 2a8a3db4..5e4fc478 100644 --- a/src/arduino.cc/builder/types/context.go +++ b/src/arduino.cc/builder/types/context.go @@ -37,8 +37,10 @@ type Context struct { BuildProperties properties.Map BuildCore string BuildPath string + BuildCachePath string SketchBuildPath string CoreBuildPath string + CoreBuildCachePath string CoreArchiveFilePath string CoreObjectsFiles []string LibrariesBuildPath string diff --git a/src/arduino.cc/builder/utils/utils.go b/src/arduino.cc/builder/utils/utils.go index a3bd44e9..0aa2e811 100644 --- a/src/arduino.cc/builder/utils/utils.go +++ b/src/arduino.cc/builder/utils/utils.go @@ -150,6 +150,18 @@ func FilterFilesWithExtensions(extensions ...string) filterFiles { } } +func FilterFiles() filterFiles { + return func(files []os.FileInfo) []os.FileInfo { + var filtered []os.FileInfo + for _, file := range files { + if !file.IsDir() { + filtered = append(filtered, file) + } + } + return filtered + } +} + var SOURCE_CONTROL_FOLDERS = map[string]bool{"CVS": true, "RCS": true, ".git": true, ".github": true, ".svn": true, ".hg": true, ".bzr": true, ".vscode": true} func IsSCCSOrHiddenFile(file os.FileInfo) bool { @@ -353,6 +365,16 @@ func FindFilesInFolder(files *[]string, folder string, extensions CheckExtension return gohasissues.Walk(folder, walkFunc) } +func GetParentFolder(basefolder string, n int) string { + tempFolder := basefolder + i := 0 + for i < n { + tempFolder = filepath.Dir(tempFolder) + i++ + } + return tempFolder +} + func AppendIfNotPresent(target []string, elements ...string) []string { for _, element := range elements { if !SliceContains(target, element) { diff --git a/src/arduino.cc/builder/wipeout_build_path_if_build_options_changed.go b/src/arduino.cc/builder/wipeout_build_path_if_build_options_changed.go index 5e4894c7..e08e7807 100644 --- a/src/arduino.cc/builder/wipeout_build_path_if_build_options_changed.go +++ b/src/arduino.cc/builder/wipeout_build_path_if_build_options_changed.go @@ -34,10 +34,12 @@ import ( "os" "path/filepath" + "arduino.cc/builder/builder_utils" "arduino.cc/builder/constants" "arduino.cc/builder/gohasissues" "arduino.cc/builder/i18n" "arduino.cc/builder/types" + "arduino.cc/builder/utils" "arduino.cc/properties" ) @@ -62,7 +64,17 @@ func (s *WipeoutBuildPathIfBuildOptionsChanged) Run(ctx *types.Context) error { delete(prevOpts, "sketchLocation") } - if opts.Equals(prevOpts) { + // check if any of the files contained in the core folders has changed + // since the json was generated - like platform.txt or similar + // if so, trigger a "safety" wipe + buildProperties := ctx.BuildProperties + targetCoreFolder := buildProperties[constants.BUILD_PROPERTIES_RUNTIME_PLATFORM_PATH] + coreFolder := buildProperties[constants.BUILD_PROPERTIES_BUILD_CORE_PATH] + realCoreFolder := utils.GetParentFolder(coreFolder, 2) + jsonPath := filepath.Join(ctx.BuildPath, constants.BUILD_OPTIONS_FILE) + coreHasChanged := builder_utils.CoreOrReferencedCoreHasChanged(realCoreFolder, targetCoreFolder, jsonPath) + + if opts.Equals(prevOpts) && !coreHasChanged { return nil }