From 165789cf556995e837d41b9b4151bbd1fcd61e04 Mon Sep 17 00:00:00 2001 From: Luca Bianconi Date: Mon, 9 Jan 2023 17:59:57 +0100 Subject: [PATCH 1/6] feat: purge cache after expiration time --- .gitignore | 3 + arduino/sketch/sketch.go | 7 +- arduino/sketch/sketch_test.go | 4 +- buildcache/build_cache.go | 72 +++++++++++++++ buildcache/build_cache_test.go | 80 +++++++++++++++++ commands/compile/compile.go | 31 ++++++- configuration/defaults.go | 3 + docs/configuration.md | 6 ++ internal/integrationtest/arduino-cli.go | 1 + .../integrationtest/compile_1/compile_test.go | 89 +++++++++++++++---- .../integrationtest/compile_2/compile_test.go | 2 +- internal/integrationtest/core/core_test.go | 2 +- .../upload_mock/upload_mock_test.go | 2 +- .../add_additional_entries_to_context.go | 2 +- legacy/builder/constants/constants.go | 1 + legacy/builder/phases/core_builder.go | 16 ++-- legacy/builder/test/builder_test.go | 7 +- 17 files changed, 293 insertions(+), 35 deletions(-) create mode 100644 buildcache/build_cache.go create mode 100644 buildcache/build_cache_test.go diff --git a/.gitignore b/.gitignore index bca7f6fdbfe..3d695480cd0 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,6 @@ venv /docsgen/arduino-cli.exe /docs/rpc/*.md /docs/commands/*.md + +# Delve debugger binary file +__debug_bin diff --git a/arduino/sketch/sketch.go b/arduino/sketch/sketch.go index 337aa904d57..6612c12b74e 100644 --- a/arduino/sketch/sketch.go +++ b/arduino/sketch/sketch.go @@ -302,5 +302,10 @@ func GenBuildPath(sketchPath *paths.Path) *paths.Path { } md5SumBytes := md5.Sum([]byte(path)) md5Sum := strings.ToUpper(hex.EncodeToString(md5SumBytes[:])) - return paths.TempDir().Join("arduino", "sketch-"+md5Sum) + + return getSketchesCacheDir().Join(md5Sum) +} + +func getSketchesCacheDir() *paths.Path { + return paths.TempDir().Join("arduino", "sketches").Canonical() } diff --git a/arduino/sketch/sketch_test.go b/arduino/sketch/sketch_test.go index cb7b624ab12..d1809f62717 100644 --- a/arduino/sketch/sketch_test.go +++ b/arduino/sketch/sketch_test.go @@ -286,10 +286,10 @@ func TestNewSketchFolderSymlink(t *testing.T) { } func TestGenBuildPath(t *testing.T) { - want := paths.TempDir().Join("arduino", "sketch-ACBD18DB4CC2F85CEDEF654FCCC4A4D8") + want := paths.TempDir().Join("arduino", "sketches", "ACBD18DB4CC2F85CEDEF654FCCC4A4D8") assert.True(t, GenBuildPath(paths.New("foo")).EquivalentTo(want)) - want = paths.TempDir().Join("arduino", "sketch-D41D8CD98F00B204E9800998ECF8427E") + want = paths.TempDir().Join("arduino", "sketches", "D41D8CD98F00B204E9800998ECF8427E") assert.True(t, GenBuildPath(nil).EquivalentTo(want)) } diff --git a/buildcache/build_cache.go b/buildcache/build_cache.go new file mode 100644 index 00000000000..5b7c8fb1d49 --- /dev/null +++ b/buildcache/build_cache.go @@ -0,0 +1,72 @@ +// This file is part of arduino-cli. +// +// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +package buildcache + +import ( + "time" + + "github.com/arduino/go-paths-helper" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +const lastUsedFileName = ".last-used" + +// GetOrCreate retrieves or creates the cache directory at the given path +// If the cache already exists the lifetime of the cache is extended. +func GetOrCreate(dir *paths.Path) (*paths.Path, error) { + if !dir.Exist() { + if err := dir.MkdirAll(); err != nil { + return nil, err + } + } + + if err := dir.Join(lastUsedFileName).WriteFile([]byte{}); err != nil { + return nil, err + } + return dir, nil +} + +// Purge removes all cache directories within baseDir that have expired +// To know how long ago a directory has been last used +// it checks into the .last-used file. +func Purge(baseDir *paths.Path, ttl time.Duration) { + files, err := baseDir.ReadDir() + if err != nil { + return + } + for _, file := range files { + if file.IsDir() { + removeIfExpired(file, ttl) + } + } +} + +func removeIfExpired(dir *paths.Path, ttl time.Duration) { + fileInfo, err := dir.Join().Stat() + if err != nil { + return + } + lifeExpectancy := ttl - time.Since(fileInfo.ModTime()) + if lifeExpectancy > 0 { + return + } + logrus.Tracef(`Purging cache directory "%s". Expired by %s`, dir, lifeExpectancy.Abs()) + err = dir.RemoveAll() + if err != nil { + logrus.Tracef(`Error while pruning cache directory "%s": %s`, dir, errors.WithStack(err)) + } +} diff --git a/buildcache/build_cache_test.go b/buildcache/build_cache_test.go new file mode 100644 index 00000000000..5aae8ca022c --- /dev/null +++ b/buildcache/build_cache_test.go @@ -0,0 +1,80 @@ +// This file is part of arduino-cli. +// +// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +package buildcache + +import ( + "testing" + "time" + + "github.com/arduino/go-paths-helper" + "github.com/stretchr/testify/require" +) + +func Test_UpdateLastUsedFileNotExisting(t *testing.T) { + testBuildDir := paths.New(t.TempDir(), "sketches", "xxx") + require.NoError(t, testBuildDir.MkdirAll()) + timeBeforeUpdating := time.Unix(0, 0) + requireCorrectUpdate(t, testBuildDir, timeBeforeUpdating) +} + +func Test_UpdateLastUsedFileExisting(t *testing.T) { + testBuildDir := paths.New(t.TempDir(), "sketches", "xxx") + require.NoError(t, testBuildDir.MkdirAll()) + + // create the file + preExistingFile := testBuildDir.Join(lastUsedFileName) + require.NoError(t, preExistingFile.WriteFile([]byte{})) + timeBeforeUpdating := time.Now().Add(-time.Second) + preExistingFile.Chtimes(time.Now(), timeBeforeUpdating) + requireCorrectUpdate(t, testBuildDir, timeBeforeUpdating) +} + +func requireCorrectUpdate(t *testing.T, dir *paths.Path, prevModTime time.Time) { + _, err := GetOrCreate(dir) + require.NoError(t, err) + expectedFile := dir.Join(lastUsedFileName) + fileInfo, err := expectedFile.Stat() + require.Nil(t, err) + require.Greater(t, fileInfo.ModTime(), prevModTime) +} + +func TestPurge(t *testing.T) { + ttl := time.Minute + + dirToPurge := paths.New(t.TempDir(), "root") + + lastUsedTimesByDirPath := map[*paths.Path]time.Time{ + (dirToPurge.Join("old")): time.Now().Add(-ttl - time.Hour), + (dirToPurge.Join("fresh")): time.Now().Add(-ttl + time.Minute), + } + + // create the metadata files + for dirPath, lastUsedTime := range lastUsedTimesByDirPath { + require.NoError(t, dirPath.MkdirAll()) + infoFilePath := dirPath.Join(lastUsedFileName).Canonical() + require.NoError(t, infoFilePath.WriteFile([]byte{})) + // make sure access time does not matter + accesstime := time.Now() + require.NoError(t, infoFilePath.Chtimes(accesstime, lastUsedTime)) + } + + Purge(dirToPurge, ttl) + + files, err := dirToPurge.Join("fresh").Stat() + require.Nil(t, err) + require.True(t, files.IsDir()) + require.True(t, dirToPurge.Exist()) +} diff --git a/commands/compile/compile.go b/commands/compile/compile.go index 2a881f23829..aeea6ec61bc 100644 --- a/commands/compile/compile.go +++ b/commands/compile/compile.go @@ -27,9 +27,11 @@ import ( "github.com/arduino/arduino-cli/arduino/cores" "github.com/arduino/arduino-cli/arduino/cores/packagemanager" "github.com/arduino/arduino-cli/arduino/sketch" + "github.com/arduino/arduino-cli/buildcache" "github.com/arduino/arduino-cli/commands" "github.com/arduino/arduino-cli/configuration" "github.com/arduino/arduino-cli/i18n" + "github.com/arduino/arduino-cli/inventory" "github.com/arduino/arduino-cli/legacy/builder" "github.com/arduino/arduino-cli/legacy/builder/types" rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" @@ -135,6 +137,11 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream if err = builderCtx.BuildPath.MkdirAll(); err != nil { return nil, &arduino.PermissionDeniedError{Message: tr("Cannot create build directory"), Cause: err} } + + buildcache.GetOrCreate(builderCtx.BuildPath) + // cache is purged after compilation to not remove entries that might be required + defer maybePurgeBuildCache() + builderCtx.CompilationDatabase = bldr.NewCompilationDatabase( builderCtx.BuildPath.Join("compile_commands.json"), ) @@ -143,8 +150,7 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream // Optimize for debug builderCtx.OptimizeForDebug = req.GetOptimizeForDebug() - - builderCtx.CoreBuildCachePath = paths.TempDir().Join("arduino", "core-cache") + builderCtx.CoreBuildCachePath = paths.TempDir().Join("arduino", "cores") builderCtx.Jobs = int(req.GetJobs()) @@ -284,3 +290,24 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream return r, nil } + +// maybePurgeBuildCache runs the build files cache purge if the policy conditions are met. +func maybePurgeBuildCache() { + + compilationsBeforePurge := configuration.Settings.GetUint("build_cache.compilations_before_purge") + // 0 means never purge + if compilationsBeforePurge == 0 { + return + } + compilationSinceLastPurge := inventory.Store.GetUint("build_cache.compilation_count_since_last_purge") + compilationSinceLastPurge++ + inventory.Store.Set("build_cache.compilation_count_since_last_purge", compilationSinceLastPurge) + defer inventory.WriteStore() + if compilationsBeforePurge == 0 || compilationSinceLastPurge < compilationsBeforePurge { + return + } + inventory.Store.Set("build_cache.compilation_count_since_last_purge", 0) + cacheTTL := configuration.Settings.GetDuration("build_cache.ttl").Abs() + buildcache.Purge(paths.TempDir().Join("arduino", "cores"), cacheTTL) + buildcache.Purge(paths.TempDir().Join("arduino", "sketches"), cacheTTL) +} diff --git a/configuration/defaults.go b/configuration/defaults.go index 323eb65203b..be1a0088a62 100644 --- a/configuration/defaults.go +++ b/configuration/defaults.go @@ -18,6 +18,7 @@ package configuration import ( "path/filepath" "strings" + "time" "github.com/spf13/viper" ) @@ -41,6 +42,8 @@ func SetDefaults(settings *viper.Viper) { // Sketch compilation settings.SetDefault("sketch.always_export_binaries", false) + settings.SetDefault("build_cache.ttl", time.Hour*24*30) + settings.SetDefault("build_cache.compilations_before_purge", 10) // daemon settings settings.SetDefault("daemon.port", "50051") diff --git a/docs/configuration.md b/docs/configuration.md index 16cacfcc5a6..f60402bf212 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -33,6 +33,12 @@ to the sketch folder. This is the equivalent of using the [`--export-binaries`][arduino-cli compile options] flag. - `updater` - configuration options related to Arduino CLI updates - `enable_notification` - set to `false` to disable notifications of new Arduino CLI releases, defaults to `true` +- `build_cache` configuration options related to the compilation cache + - `compilations_before_purge` - interval, in number of compilations, at which the cache is purged, defaults to `10`. + When `0` the cache is never purged. + - `ttl` - cache expiration time of build folders. If the cache is hit by a compilation the corresponding build files + lifetime is renewed. The value format must be a valid input for + [time.ParseDuration()](https://pkg.go.dev/time#ParseDuration), defaults to `720h` (30 days). ## Configuration methods diff --git a/internal/integrationtest/arduino-cli.go b/internal/integrationtest/arduino-cli.go index 5750189bead..92bce9e85bc 100644 --- a/internal/integrationtest/arduino-cli.go +++ b/internal/integrationtest/arduino-cli.go @@ -112,6 +112,7 @@ func NewArduinoCliWithinEnvironment(env *Environment, config *ArduinoCLIConfig) "ARDUINO_DATA_DIR": cli.dataDir.String(), "ARDUINO_DOWNLOADS_DIR": cli.stagingDir.String(), "ARDUINO_SKETCHBOOK_DIR": cli.sketchbookDir.String(), + "ARDUINO_BUILD_CACHE_COMPILATIONS_BEFORE_PURGE": "0", } env.RegisterCleanUpCallback(cli.CleanUp) return cli diff --git a/internal/integrationtest/compile_1/compile_test.go b/internal/integrationtest/compile_1/compile_test.go index 13cb87d8ecf..998fe7c0483 100644 --- a/internal/integrationtest/compile_1/compile_test.go +++ b/internal/integrationtest/compile_1/compile_test.go @@ -20,8 +20,10 @@ import ( "encoding/hex" "encoding/json" "os" + "sort" "strings" "testing" + "time" "github.com/arduino/arduino-cli/internal/integrationtest" "github.com/arduino/go-paths-helper" @@ -47,6 +49,7 @@ func TestCompile(t *testing.T) { {"WithoutFqbn", compileWithoutFqbn}, {"ErrorMessage", compileErrorMessage}, {"WithSimpleSketch", compileWithSimpleSketch}, + {"WithCachePurgeNeeded", compileWithCachePurgeNeeded}, {"OutputFlagDefaultPath", compileOutputFlagDefaultPath}, {"WithSketchWithSymlinkSelfloop", compileWithSketchWithSymlinkSelfloop}, {"BlacklistedSketchname", compileBlacklistedSketchname}, @@ -112,6 +115,35 @@ func compileErrorMessage(t *testing.T, env *integrationtest.Environment, cli *in } func compileWithSimpleSketch(t *testing.T, env *integrationtest.Environment, cli *integrationtest.ArduinoCLI) { + compileWithSimpleSketchCustomEnv(t, env, cli, cli.GetDefaultEnv()) +} + +func compileWithCachePurgeNeeded(t *testing.T, env *integrationtest.Environment, cli *integrationtest.ArduinoCLI) { + // create directories that must be purged + baseDir := paths.TempDir().Join("arduino", "sketches") + + // purge case: last used file too old + oldDir1 := baseDir.Join("test_old_sketch_1") + require.NoError(t, oldDir1.MkdirAll()) + require.NoError(t, oldDir1.Join(".last-used").WriteFile([]byte{})) + require.NoError(t, oldDir1.Join(".last-used").Chtimes(time.Now(), time.Unix(0, 0))) + // no purge case: last used file not existing + missingFileDir := baseDir.Join("test_sketch_2") + require.NoError(t, missingFileDir.MkdirAll()) + + defer oldDir1.RemoveAll() + defer missingFileDir.RemoveAll() + + customEnv := cli.GetDefaultEnv() + customEnv["ARDUINO_BUILD_CACHE_COMPILATIONS_BEFORE_PURGE"] = "1" + compileWithSimpleSketchCustomEnv(t, env, cli, customEnv) + + // check that purge has been run + require.NoFileExists(t, oldDir1.String()) + require.DirExists(t, missingFileDir.String()) +} + +func compileWithSimpleSketchCustomEnv(t *testing.T, env *integrationtest.Environment, cli *integrationtest.ArduinoCLI, customEnv map[string]string) { sketchName := "CompileIntegrationTest" sketchPath := cli.SketchbookDir().Join(sketchName) defer sketchPath.RemoveAll() @@ -127,7 +159,7 @@ func compileWithSimpleSketch(t *testing.T, env *integrationtest.Environment, cli require.NoError(t, err) // Build sketch for arduino:avr:uno with json output - stdout, _, err = cli.Run("compile", "-b", fqbn, sketchPath.String(), "--format", "json") + stdout, _, err = cli.RunWithCustomEnv(customEnv, "compile", "-b", fqbn, sketchPath.String(), "--format", "json") require.NoError(t, err) // check is a valid json and contains requested data var compileOutput map[string]interface{} @@ -140,7 +172,7 @@ func compileWithSimpleSketch(t *testing.T, env *integrationtest.Environment, cli md5 := md5.Sum(([]byte(sketchPath.String()))) sketchPathMd5 := strings.ToUpper(hex.EncodeToString(md5[:])) require.NotEmpty(t, sketchPathMd5) - buildDir := paths.TempDir().Join("arduino", "sketch-"+sketchPathMd5) + buildDir := paths.TempDir().Join("arduino", "sketches", sketchPathMd5) require.FileExists(t, buildDir.Join(sketchName+".ino.eep").String()) require.FileExists(t, buildDir.Join(sketchName+".ino.elf").String()) require.FileExists(t, buildDir.Join(sketchName+".ino.hex").String()) @@ -374,7 +406,7 @@ func compileWithOutputDirFlag(t *testing.T, env *integrationtest.Environment, cl md5 := md5.Sum(([]byte(sketchPath.String()))) sketchPathMd5 := strings.ToUpper(hex.EncodeToString(md5[:])) require.NotEmpty(t, sketchPathMd5) - buildDir := paths.TempDir().Join("arduino", "sketch-"+sketchPathMd5) + buildDir := paths.TempDir().Join("arduino", "sketches", sketchPathMd5) require.FileExists(t, buildDir.Join(sketchName+".ino.eep").String()) require.FileExists(t, buildDir.Join(sketchName+".ino.elf").String()) require.FileExists(t, buildDir.Join(sketchName+".ino.hex").String()) @@ -441,7 +473,7 @@ func compileWithCustomBuildPath(t *testing.T, env *integrationtest.Environment, md5 := md5.Sum(([]byte(sketchPath.String()))) sketchPathMd5 := strings.ToUpper(hex.EncodeToString(md5[:])) require.NotEmpty(t, sketchPathMd5) - buildDir := paths.TempDir().Join("arduino", "sketch-"+sketchPathMd5) + buildDir := paths.TempDir().Join("arduino", "sketches", sketchPathMd5) require.NoFileExists(t, buildDir.Join(sketchName+".ino.eep").String()) require.NoFileExists(t, buildDir.Join(sketchName+".ino.elf").String()) require.NoFileExists(t, buildDir.Join(sketchName+".ino.hex").String()) @@ -975,7 +1007,7 @@ func compileWithInvalidBuildOptionJson(t *testing.T, env *integrationtest.Enviro md5 := md5.Sum(([]byte(sketchPath.String()))) sketchPathMd5 := strings.ToUpper(hex.EncodeToString(md5[:])) require.NotEmpty(t, sketchPathMd5) - buildDir := paths.TempDir().Join("arduino", "sketch-"+sketchPathMd5) + buildDir := paths.TempDir().Join("arduino", "sketches", sketchPathMd5) _, _, err = cli.Run("compile", "-b", fqbn, sketchPath.String(), "--verbose") require.NoError(t, err) @@ -1008,18 +1040,41 @@ func compileWithRelativeBuildPath(t *testing.T, env *integrationtest.Environment absoluteBuildPath := cli.SketchbookDir().Join("build_path") builtFiles, err := absoluteBuildPath.ReadDir() require.NoError(t, err) - require.Contains(t, builtFiles[8].String(), sketchName+".ino.eep") - require.Contains(t, builtFiles[9].String(), sketchName+".ino.elf") - require.Contains(t, builtFiles[10].String(), sketchName+".ino.hex") - require.Contains(t, builtFiles[11].String(), sketchName+".ino.with_bootloader.bin") - require.Contains(t, builtFiles[12].String(), sketchName+".ino.with_bootloader.hex") - require.Contains(t, builtFiles[0].String(), "build.options.json") - require.Contains(t, builtFiles[1].String(), "compile_commands.json") - require.Contains(t, builtFiles[2].String(), "core") - require.Contains(t, builtFiles[3].String(), "includes.cache") - require.Contains(t, builtFiles[4].String(), "libraries") - require.Contains(t, builtFiles[6].String(), "preproc") - require.Contains(t, builtFiles[7].String(), "sketch") + + expectedFiles := []string{ + sketchName + ".ino.eep", + sketchName + ".ino.elf", + sketchName + ".ino.hex", + sketchName + ".ino.with_bootloader.bin", + sketchName + ".ino.with_bootloader.hex", + "build.options.json", + "compile_commands.json", + "core", + "includes.cache", + "libraries", + "preproc", + "sketch", + } + + foundFiles := []string{} + for _, builtFile := range builtFiles { + if sliceIncludes(expectedFiles, builtFile.Base()) { + foundFiles = append(foundFiles, builtFile.Base()) + } + } + sort.Strings(expectedFiles) + sort.Strings(foundFiles) + require.Equal(t, expectedFiles, foundFiles) +} + +// TODO: remove this when a generic library is introduced +func sliceIncludes[T comparable](slice []T, target T) bool { + for _, e := range slice { + if e == target { + return true + } + } + return false } func compileWithFakeSecureBootCore(t *testing.T, env *integrationtest.Environment, cli *integrationtest.ArduinoCLI) { diff --git a/internal/integrationtest/compile_2/compile_test.go b/internal/integrationtest/compile_2/compile_test.go index dc67aeaf2b0..f37f23eab19 100644 --- a/internal/integrationtest/compile_2/compile_test.go +++ b/internal/integrationtest/compile_2/compile_test.go @@ -145,7 +145,7 @@ func recompileWithDifferentLibrary(t *testing.T, env *integrationtest.Environmen md5 := md5.Sum(([]byte(sketchPath.String()))) sketchPathMd5 := strings.ToUpper(hex.EncodeToString(md5[:])) require.NotEmpty(t, sketchPathMd5) - buildDir := paths.TempDir().Join("arduino", "sketch-"+sketchPathMd5) + buildDir := paths.TempDir().Join("arduino", "sketches", sketchPathMd5) // Compile sketch using library not managed by CLI stdout, _, err := cli.Run("compile", "-b", fqbn, "--library", manuallyInstalledLibPath.String(), sketchPath.String(), "-v") diff --git a/internal/integrationtest/core/core_test.go b/internal/integrationtest/core/core_test.go index cea3c33195b..7e1259ee9c2 100644 --- a/internal/integrationtest/core/core_test.go +++ b/internal/integrationtest/core/core_test.go @@ -253,7 +253,7 @@ func TestCoreInstallEsp32(t *testing.T) { md5 := md5.Sum(([]byte(sketchPath.String()))) sketchPathMd5 := strings.ToUpper(hex.EncodeToString(md5[:])) require.NotEmpty(t, sketchPathMd5) - buildDir := paths.TempDir().Join("arduino", "sketch-"+sketchPathMd5) + buildDir := paths.TempDir().Join("arduino", "sketches", sketchPathMd5) require.FileExists(t, buildDir.Join(sketchName+".ino.partitions.bin").String()) } diff --git a/internal/integrationtest/upload_mock/upload_mock_test.go b/internal/integrationtest/upload_mock/upload_mock_test.go index 208e48fd772..ec0875d8706 100644 --- a/internal/integrationtest/upload_mock/upload_mock_test.go +++ b/internal/integrationtest/upload_mock/upload_mock_test.go @@ -697,7 +697,7 @@ func TestUploadSketch(t *testing.T) { func generateBuildDir(sketchPath *paths.Path, t *testing.T) *paths.Path { md5 := md5.Sum(([]byte(sketchPath.String()))) sketchPathMd5 := strings.ToUpper(hex.EncodeToString(md5[:])) - buildDir := paths.TempDir().Join("arduino", "sketch-"+sketchPathMd5) + buildDir := paths.TempDir().Join("arduino", "sketches", sketchPathMd5) require.NoError(t, buildDir.MkdirAll()) require.NoError(t, buildDir.ToAbs()) return buildDir diff --git a/legacy/builder/add_additional_entries_to_context.go b/legacy/builder/add_additional_entries_to_context.go index 775b6cdc63e..d7252674e56 100644 --- a/legacy/builder/add_additional_entries_to_context.go +++ b/legacy/builder/add_additional_entries_to_context.go @@ -35,7 +35,7 @@ func (*AddAdditionalEntriesToContext) Run(ctx *types.Context) error { if err != nil { return errors.WithStack(err) } - librariesBuildPath, err := buildPath.Join("libraries").Abs() + librariesBuildPath, err := buildPath.Join(constants.FOLDER_LIBRARIES).Abs() if err != nil { return errors.WithStack(err) } diff --git a/legacy/builder/constants/constants.go b/legacy/builder/constants/constants.go index e1266930fe3..5d7d3bb45a5 100644 --- a/legacy/builder/constants/constants.go +++ b/legacy/builder/constants/constants.go @@ -47,6 +47,7 @@ const FOLDER_CORE = "core" const FOLDER_PREPROC = "preproc" const FOLDER_SKETCH = "sketch" const FOLDER_TOOLS = "tools" +const FOLDER_LIBRARIES = "libraries" const LIBRARY_ALL_ARCHS = "*" const LIBRARY_EMAIL = "email" const LIBRARY_FOLDER_ARCH = "arch" diff --git a/legacy/builder/phases/core_builder.go b/legacy/builder/phases/core_builder.go index 76cb731062c..ecd2a327861 100644 --- a/legacy/builder/phases/core_builder.go +++ b/legacy/builder/phases/core_builder.go @@ -19,6 +19,7 @@ import ( "os" "strings" + "github.com/arduino/arduino-cli/buildcache" "github.com/arduino/arduino-cli/i18n" "github.com/arduino/arduino-cli/legacy/builder/builder_utils" "github.com/arduino/arduino-cli/legacy/builder/constants" @@ -92,9 +93,10 @@ func compileCore(ctx *types.Context, buildPath *paths.Path, buildCachePath *path var targetArchivedCore *paths.Path if buildCachePath != nil { - archivedCoreName := GetCachedCoreArchiveFileName(buildProperties.Get(constants.BUILD_PROPERTIES_FQBN), + archivedCoreName := GetCachedCoreArchiveDirName(buildProperties.Get(constants.BUILD_PROPERTIES_FQBN), buildProperties.Get("compiler.optimization_flags"), realCoreFolder) - targetArchivedCore = buildCachePath.Join(archivedCoreName) + buildcache.GetOrCreate(buildCachePath.Join(archivedCoreName)) + targetArchivedCore = buildCachePath.Join(archivedCoreName, "core.a") canUseArchivedCore := !ctx.OnlyUpdateCompilationDatabase && !ctx.Clean && !builder_utils.CoreOrReferencedCoreHasChanged(realCoreFolder, targetCoreFolder, targetArchivedCore) @@ -137,19 +139,19 @@ func compileCore(ctx *types.Context, buildPath *paths.Path, buildCachePath *path return archiveFile, variantObjectFiles, nil } -// GetCachedCoreArchiveFileName returns the filename to be used to store +// GetCachedCoreArchiveDirName returns the directory name to be used to store // the global cached core.a. -func GetCachedCoreArchiveFileName(fqbn string, optimizationFlags string, coreFolder *paths.Path) string { +func GetCachedCoreArchiveDirName(fqbn string, optimizationFlags string, coreFolder *paths.Path) string { fqbnToUnderscore := strings.Replace(fqbn, ":", "_", -1) fqbnToUnderscore = strings.Replace(fqbnToUnderscore, "=", "_", -1) if absCoreFolder, err := coreFolder.Abs(); err == nil { coreFolder = absCoreFolder } // silently continue if absolute path can't be detected hash := utils.MD5Sum([]byte(coreFolder.String() + optimizationFlags)) - realName := "core_" + fqbnToUnderscore + "_" + hash + ".a" + realName := fqbnToUnderscore + "_" + hash if len(realName) > 100 { - // avoid really long names, simply hash the final part - realName = "core_" + utils.MD5Sum([]byte(fqbnToUnderscore+"_"+hash)) + ".a" + // avoid really long names, simply hash the name + realName = utils.MD5Sum([]byte(fqbnToUnderscore + "_" + hash)) } return realName } diff --git a/legacy/builder/test/builder_test.go b/legacy/builder/test/builder_test.go index 21107425fd5..913dd547add 100644 --- a/legacy/builder/test/builder_test.go +++ b/legacy/builder/test/builder_test.go @@ -380,10 +380,13 @@ func TestBuilderCacheCoreAFile(t *testing.T) { // Pick timestamp of cached core coreFolder := paths.New("downloaded_hardware", "arduino", "avr") - coreFileName := phases.GetCachedCoreArchiveFileName(ctx.FQBN.String(), ctx.OptimizationFlags, coreFolder) - cachedCoreFile := ctx.CoreBuildCachePath.Join(coreFileName) + coreFileName := phases.GetCachedCoreArchiveDirName(ctx.FQBN.String(), ctx.OptimizationFlags, coreFolder) + cachedCoreFile := ctx.CoreBuildCachePath.Join(coreFileName, "core.a") coreStatBefore, err := cachedCoreFile.Stat() require.NoError(t, err) + lastUsedFile := ctx.CoreBuildCachePath.Join(coreFileName, ".last-used") + _, err = lastUsedFile.Stat() + require.NoError(t, err) // Run build again, to verify that the builder skips rebuilding core.a err = bldr.Run(ctx) From b65c37b5aa8ffc4398acfc4f96fc221044d1e0ee Mon Sep 17 00:00:00 2001 From: Luca Bianconi Date: Wed, 18 Jan 2023 10:42:17 +0100 Subject: [PATCH 2/6] fix: handle cache dir not created --- legacy/builder/phases/core_builder.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/legacy/builder/phases/core_builder.go b/legacy/builder/phases/core_builder.go index ecd2a327861..cf5596a6792 100644 --- a/legacy/builder/phases/core_builder.go +++ b/legacy/builder/phases/core_builder.go @@ -16,6 +16,7 @@ package phases import ( + "fmt" "os" "strings" @@ -92,11 +93,13 @@ func compileCore(ctx *types.Context, buildPath *paths.Path, buildCachePath *path realCoreFolder := coreFolder.Parent().Parent() var targetArchivedCore *paths.Path + var buildCacheErr error if buildCachePath != nil { archivedCoreName := GetCachedCoreArchiveDirName(buildProperties.Get(constants.BUILD_PROPERTIES_FQBN), buildProperties.Get("compiler.optimization_flags"), realCoreFolder) - buildcache.GetOrCreate(buildCachePath.Join(archivedCoreName)) targetArchivedCore = buildCachePath.Join(archivedCoreName, "core.a") + _, buildCacheErr = buildcache.GetOrCreate(targetArchivedCore.Parent()) + canUseArchivedCore := !ctx.OnlyUpdateCompilationDatabase && !ctx.Clean && !builder_utils.CoreOrReferencedCoreHasChanged(realCoreFolder, targetCoreFolder, targetArchivedCore) @@ -122,6 +125,11 @@ func compileCore(ctx *types.Context, buildPath *paths.Path, buildCachePath *path // archive core.a if targetArchivedCore != nil && !ctx.OnlyUpdateCompilationDatabase { + if buildCacheErr != nil { + if err := targetArchivedCore.Parent().Mkdir(); err != nil { + return nil, nil, fmt.Errorf(tr("creating core cache folder: %s", err)) + } + } err := archiveFile.CopyTo(targetArchivedCore) if ctx.Verbose { if err == nil { From 776d66c158cb5208943497556654097c6b473945 Mon Sep 17 00:00:00 2001 From: Luca Bianconi Date: Fri, 27 Jan 2023 13:35:40 +0100 Subject: [PATCH 3/6] refactor: new interface --- buildcache/build_cache.go | 26 +++++++++++++++++--------- buildcache/build_cache_test.go | 4 ++-- commands/compile/compile.go | 6 +++--- legacy/builder/phases/core_builder.go | 2 +- 4 files changed, 23 insertions(+), 15 deletions(-) diff --git a/buildcache/build_cache.go b/buildcache/build_cache.go index 5b7c8fb1d49..d3d12b6f61d 100644 --- a/buildcache/build_cache.go +++ b/buildcache/build_cache.go @@ -25,26 +25,29 @@ import ( const lastUsedFileName = ".last-used" +type buildCache struct { + baseDir *paths.Path +} + // GetOrCreate retrieves or creates the cache directory at the given path // If the cache already exists the lifetime of the cache is extended. -func GetOrCreate(dir *paths.Path) (*paths.Path, error) { - if !dir.Exist() { - if err := dir.MkdirAll(); err != nil { - return nil, err - } +func (bc *buildCache) GetOrCreate(key string) (*paths.Path, error) { + keyDir := bc.baseDir.Join(key) + if err := keyDir.MkdirAll(); err != nil { + return nil, err } - if err := dir.Join(lastUsedFileName).WriteFile([]byte{}); err != nil { + if err := keyDir.Join(lastUsedFileName).WriteFile([]byte{}); err != nil { return nil, err } - return dir, nil + return keyDir, nil } // Purge removes all cache directories within baseDir that have expired // To know how long ago a directory has been last used // it checks into the .last-used file. -func Purge(baseDir *paths.Path, ttl time.Duration) { - files, err := baseDir.ReadDir() +func (bc *buildCache) Purge(ttl time.Duration) { + files, err := bc.baseDir.ReadDir() if err != nil { return } @@ -55,6 +58,11 @@ func Purge(baseDir *paths.Path, ttl time.Duration) { } } +// New instantiates a build cache +func New(baseDir *paths.Path) *buildCache { + return &buildCache{baseDir} +} + func removeIfExpired(dir *paths.Path, ttl time.Duration) { fileInfo, err := dir.Join().Stat() if err != nil { diff --git a/buildcache/build_cache_test.go b/buildcache/build_cache_test.go index 5aae8ca022c..e08b10b2962 100644 --- a/buildcache/build_cache_test.go +++ b/buildcache/build_cache_test.go @@ -43,7 +43,7 @@ func Test_UpdateLastUsedFileExisting(t *testing.T) { } func requireCorrectUpdate(t *testing.T, dir *paths.Path, prevModTime time.Time) { - _, err := GetOrCreate(dir) + _, err := New(dir.Parent()).GetOrCreate(dir.Base()) require.NoError(t, err) expectedFile := dir.Join(lastUsedFileName) fileInfo, err := expectedFile.Stat() @@ -71,7 +71,7 @@ func TestPurge(t *testing.T) { require.NoError(t, infoFilePath.Chtimes(accesstime, lastUsedTime)) } - Purge(dirToPurge, ttl) + New(dirToPurge).Purge(ttl) files, err := dirToPurge.Join("fresh").Stat() require.Nil(t, err) diff --git a/commands/compile/compile.go b/commands/compile/compile.go index ced440bebf5..664368e067c 100644 --- a/commands/compile/compile.go +++ b/commands/compile/compile.go @@ -138,7 +138,7 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream return nil, &arduino.PermissionDeniedError{Message: tr("Cannot create build directory"), Cause: err} } - buildcache.GetOrCreate(builderCtx.BuildPath) + buildcache.New(builderCtx.BuildPath.Parent()).GetOrCreate(builderCtx.BuildPath.Base()) // cache is purged after compilation to not remove entries that might be required defer maybePurgeBuildCache() @@ -312,6 +312,6 @@ func maybePurgeBuildCache() { } inventory.Store.Set("build_cache.compilation_count_since_last_purge", 0) cacheTTL := configuration.Settings.GetDuration("build_cache.ttl").Abs() - buildcache.Purge(paths.TempDir().Join("arduino", "cores"), cacheTTL) - buildcache.Purge(paths.TempDir().Join("arduino", "sketches"), cacheTTL) + buildcache.New(paths.TempDir().Join("arduino", "cores")).Purge(cacheTTL) + buildcache.New(paths.TempDir().Join("arduino", "sketches")).Purge(cacheTTL) } diff --git a/legacy/builder/phases/core_builder.go b/legacy/builder/phases/core_builder.go index cf5596a6792..bd2c07cc784 100644 --- a/legacy/builder/phases/core_builder.go +++ b/legacy/builder/phases/core_builder.go @@ -98,7 +98,7 @@ func compileCore(ctx *types.Context, buildPath *paths.Path, buildCachePath *path archivedCoreName := GetCachedCoreArchiveDirName(buildProperties.Get(constants.BUILD_PROPERTIES_FQBN), buildProperties.Get("compiler.optimization_flags"), realCoreFolder) targetArchivedCore = buildCachePath.Join(archivedCoreName, "core.a") - _, buildCacheErr = buildcache.GetOrCreate(targetArchivedCore.Parent()) + _, buildCacheErr = buildcache.New(buildCachePath).GetOrCreate(archivedCoreName) canUseArchivedCore := !ctx.OnlyUpdateCompilationDatabase && !ctx.Clean && From 446e2af0087e1c8e4898db0d371531cc2757dd58 Mon Sep 17 00:00:00 2001 From: Luca Bianconi Date: Fri, 27 Jan 2023 13:49:21 +0100 Subject: [PATCH 4/6] style: lint --- buildcache/build_cache.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/buildcache/build_cache.go b/buildcache/build_cache.go index d3d12b6f61d..18a799f780d 100644 --- a/buildcache/build_cache.go +++ b/buildcache/build_cache.go @@ -25,13 +25,16 @@ import ( const lastUsedFileName = ".last-used" -type buildCache struct { +// BuildCache represents a cache of built files (sketches and cores), it's designed +// to work on directories. Given a directory as "base" it handles direct subdirectories as +// keys +type BuildCache struct { baseDir *paths.Path } // GetOrCreate retrieves or creates the cache directory at the given path // If the cache already exists the lifetime of the cache is extended. -func (bc *buildCache) GetOrCreate(key string) (*paths.Path, error) { +func (bc *BuildCache) GetOrCreate(key string) (*paths.Path, error) { keyDir := bc.baseDir.Join(key) if err := keyDir.MkdirAll(); err != nil { return nil, err @@ -46,7 +49,7 @@ func (bc *buildCache) GetOrCreate(key string) (*paths.Path, error) { // Purge removes all cache directories within baseDir that have expired // To know how long ago a directory has been last used // it checks into the .last-used file. -func (bc *buildCache) Purge(ttl time.Duration) { +func (bc *BuildCache) Purge(ttl time.Duration) { files, err := bc.baseDir.ReadDir() if err != nil { return @@ -59,8 +62,8 @@ func (bc *buildCache) Purge(ttl time.Duration) { } // New instantiates a build cache -func New(baseDir *paths.Path) *buildCache { - return &buildCache{baseDir} +func New(baseDir *paths.Path) *BuildCache { + return &BuildCache{baseDir} } func removeIfExpired(dir *paths.Path, ttl time.Duration) { From 6ac7fce2f1671aca8b9da5f5b74d002c5edfcd0d Mon Sep 17 00:00:00 2001 From: Luca Bianconi Date: Fri, 3 Feb 2023 18:31:13 +0100 Subject: [PATCH 5/6] fix: cache age check --- buildcache/build_cache.go | 2 +- buildcache/build_cache_test.go | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/buildcache/build_cache.go b/buildcache/build_cache.go index 18a799f780d..300d6ef5dbc 100644 --- a/buildcache/build_cache.go +++ b/buildcache/build_cache.go @@ -67,7 +67,7 @@ func New(baseDir *paths.Path) *BuildCache { } func removeIfExpired(dir *paths.Path, ttl time.Duration) { - fileInfo, err := dir.Join().Stat() + fileInfo, err := dir.Join(lastUsedFileName).Stat() if err != nil { return } diff --git a/buildcache/build_cache_test.go b/buildcache/build_cache_test.go index e08b10b2962..9e79f927a22 100644 --- a/buildcache/build_cache_test.go +++ b/buildcache/build_cache_test.go @@ -73,8 +73,6 @@ func TestPurge(t *testing.T) { New(dirToPurge).Purge(ttl) - files, err := dirToPurge.Join("fresh").Stat() - require.Nil(t, err) - require.True(t, files.IsDir()) - require.True(t, dirToPurge.Exist()) + require.False(t, dirToPurge.Join("old").Exist()) + require.True(t, dirToPurge.Join("fresh").Exist()) } From bf58720ed7d088aeb767ab3574d595e9d9d935a0 Mon Sep 17 00:00:00 2001 From: Luca Bianconi Date: Mon, 6 Feb 2023 20:11:20 +0100 Subject: [PATCH 6/6] refactor: cleaner flow --- buildcache/build_cache.go | 23 +++++++++++++++++++++-- legacy/builder/phases/core_builder.go | 9 ++++----- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/buildcache/build_cache.go b/buildcache/build_cache.go index 300d6ef5dbc..d77586caf7a 100644 --- a/buildcache/build_cache.go +++ b/buildcache/build_cache.go @@ -23,6 +23,25 @@ import ( "github.com/sirupsen/logrus" ) +type wrapError struct { + wrapped error +} + +func (e wrapError) Error() string { + return e.wrapped.Error() +} + +func (e wrapError) Unwrap() error { + return e.wrapped +} + +type ErrCreateBaseDir struct { + wrapError +} +type ErrWriteLastUsedFile struct { + wrapError +} + const lastUsedFileName = ".last-used" // BuildCache represents a cache of built files (sketches and cores), it's designed @@ -37,11 +56,11 @@ type BuildCache struct { func (bc *BuildCache) GetOrCreate(key string) (*paths.Path, error) { keyDir := bc.baseDir.Join(key) if err := keyDir.MkdirAll(); err != nil { - return nil, err + return nil, &ErrCreateBaseDir{wrapError{err}} } if err := keyDir.Join(lastUsedFileName).WriteFile([]byte{}); err != nil { - return nil, err + return nil, &ErrWriteLastUsedFile{wrapError{err}} } return keyDir, nil } diff --git a/legacy/builder/phases/core_builder.go b/legacy/builder/phases/core_builder.go index bd2c07cc784..022a563de71 100644 --- a/legacy/builder/phases/core_builder.go +++ b/legacy/builder/phases/core_builder.go @@ -100,6 +100,10 @@ func compileCore(ctx *types.Context, buildPath *paths.Path, buildCachePath *path targetArchivedCore = buildCachePath.Join(archivedCoreName, "core.a") _, buildCacheErr = buildcache.New(buildCachePath).GetOrCreate(archivedCoreName) + if errors.As(buildCacheErr, &buildcache.ErrCreateBaseDir{}) { + return nil, nil, fmt.Errorf(tr("creating core cache folder: %s", err)) + } + canUseArchivedCore := !ctx.OnlyUpdateCompilationDatabase && !ctx.Clean && !builder_utils.CoreOrReferencedCoreHasChanged(realCoreFolder, targetCoreFolder, targetArchivedCore) @@ -125,11 +129,6 @@ func compileCore(ctx *types.Context, buildPath *paths.Path, buildCachePath *path // archive core.a if targetArchivedCore != nil && !ctx.OnlyUpdateCompilationDatabase { - if buildCacheErr != nil { - if err := targetArchivedCore.Parent().Mkdir(); err != nil { - return nil, nil, fmt.Errorf(tr("creating core cache folder: %s", err)) - } - } err := archiveFile.CopyTo(targetArchivedCore) if ctx.Verbose { if err == nil {