feat(wip): purge cache after expiration time

Author: Luca Bianconi

.gitignore

@@ -28,3 +28,6 @@ venv
 /docsgen/arduino-cli.exe
 /docs/rpc/*.md
 /docs/commands/*.md
+
+# Delve debugger binary file
+__debug_bin

arduino/sketch/sketch.go

@@ -302,5 +302,10 @@ func GenBuildPath(sketchPath *paths.Path) *paths.Path {
 	}
 	md5SumBytes := md5.Sum([]byte(path))
 	md5Sum := strings.ToUpper(hex.EncodeToString(md5SumBytes[:]))
-	return paths.TempDir().Join("arduino", "sketch-"+md5Sum)
+
+	return getSketchesCacheDir().Join("sketch-" + md5Sum)
+}
+
+func getSketchesCacheDir() *paths.Path {
+	return paths.TempDir().Join("arduino", "sketches").Canonical()
 }

buildcache/build_cache.go

@@ -0,0 +1,85 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to [email protected].
+
+package buildcache
+
+import (
+	"os"
+	"time"
+
+	"github.com/arduino/go-paths-helper"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	holdFileName = ".cache-metadata.yaml"
+)
+
+func getHoldFileModTime(p string) time.Time {
+	fileInfo, err := paths.New(p).Stat()
+	if err != nil {
+		return time.Unix(0, 0)
+	}
+	return fileInfo.ModTime()
+}
+
+func updateHoldFile(p string) error {
+	return paths.New(p).WriteFile([]byte{})
+}
+
+// UpdateLastUsedTime registers the last used time in the cache metadata file
+func UpdateLastUsedTime(path *paths.Path) error {
+	dir := requireDirectory(path)
+	return updateHoldFile(dir.Join(holdFileName).String())
+}
+
+// Purge removes all cache directories within baseDir that have expired
+// To know how long ago a directory has been last used
+// it checks into the hold file. If the file does not exist
+// then the directory is purged.
+func Purge(baseDir *paths.Path, ttl time.Duration) {
+	files, err := os.ReadDir(baseDir.String())
+	if err != nil {
+		return
+	}
+	for _, file := range files {
+		if file.IsDir() {
+			deleteIfExpired(baseDir.Join(file.Name()), ttl)
+		}
+	}
+}
+
+func deleteIfExpired(dir *paths.Path, ttl time.Duration) {
+	modTime := getHoldFileModTime(dir.Join(holdFileName).String())
+	if time.Since(modTime) < ttl {
+		return
+	}
+	logrus.Tracef(`Pruning cache directory "%s". Expired by %s`, dir, (time.Since(modTime) - ttl))
+	err := os.RemoveAll(dir.String())
+
+	if err != nil {
+		logrus.Tracef(`Error while pruning cache directory "%s".\n%s\n`, dir, errors.WithStack(err))
+	}
+}
+
+// requireDirectory returns p if it's a directory, its parent directory otherwise
+func requireDirectory(p *paths.Path) *paths.Path {
+	directoryPath := p
+	if !directoryPath.IsDir() {
+		directoryPath = p.Parent()
+	}
+	return directoryPath
+}

buildcache/build_cache_test.go

@@ -0,0 +1,78 @@
+package buildcache
+
+import (
+	"os"
+	"testing"
+	"time"
+
+	"github.com/arduino/go-paths-helper"
+	"github.com/stretchr/testify/require"
+)
+
+func Test_UpdateLastUsedFileNotExisting(t *testing.T) {
+
+	testBuildDir := paths.New(t.TempDir(), "sketches", "sketch-xxx")
+	err := os.MkdirAll(testBuildDir.String(), 0770)
+	require.Nil(t, err)
+	requireCorrectUpdate(t, testBuildDir)
+}
+
+func Test_UpdateLastUsedFileExisting(t *testing.T) {
+
+	testBuildDir := paths.New(t.TempDir(), "sketches", "sketch-xxx")
+	err := os.MkdirAll(testBuildDir.String(), 0770)
+	require.Nil(t, err)
+
+	// create the file
+	err = paths.New(testBuildDir.Join(holdFileName).String()).WriteFile([]byte{})
+	require.Nil(t, err)
+
+	requireCorrectUpdate(t, testBuildDir)
+}
+
+func requireCorrectUpdate(t *testing.T, dir *paths.Path) {
+	timeBeforeUpdating := time.Now()
+	err := UpdateLastUsedTime(dir)
+	require.Nil(t, err)
+	expectedMetadataFile := dir.Join(holdFileName)
+	_, err = os.Stat(expectedMetadataFile.String())
+	require.Nil(t, err)
+
+	// content can be decoded
+	err = updateHoldFile(expectedMetadataFile.String())
+	require.Nil(t, err)
+	lastUsedTime := getHoldFileModTime(expectedMetadataFile.String())
+	require.GreaterOrEqual(t, lastUsedTime, timeBeforeUpdating)
+}
+
+func TestPurge(t *testing.T) {
+	ttl := time.Minute
+
+	dirToPurge := paths.New(t.TempDir(), "root")
+
+	lastUsedTimesByDirPath := map[*paths.Path]time.Time{
+		(dirToPurge.Join("old")):   time.Now().Add(-ttl - time.Hour),
+		(dirToPurge.Join("fresh")): time.Now().Add(-ttl + time.Minute),
+	}
+
+	// create the metadata files
+	for dirPath, lastUsedTime := range lastUsedTimesByDirPath {
+		err := os.MkdirAll(dirPath.Canonical().String(), 0770)
+		require.Nil(t, err)
+		holdFilePath := dirPath.Join(holdFileName).Canonical().String()
+		err = updateHoldFile(holdFilePath)
+		require.Nil(t, err)
+		// make sure access time does not matter
+		accesstime := time.Now()
+		err = os.Chtimes(holdFilePath, accesstime, lastUsedTime)
+		require.Nil(t, err)
+	}
+
+	Purge(dirToPurge, ttl)
+
+	fileinfo, err := os.Stat(dirToPurge.Join("fresh").String())
+	require.Nil(t, err)
+	require.True(t, fileinfo.IsDir())
+	_, err = os.Stat(dirToPurge.Join("old").String())
+	require.ErrorIs(t, err, os.ErrNotExist)
+}

commands/compile/compile.go

@@ -26,9 +26,11 @@ import (
 	"github.com/arduino/arduino-cli/arduino/cores"
 	"github.com/arduino/arduino-cli/arduino/cores/packagemanager"
 	"github.com/arduino/arduino-cli/arduino/sketch"
+	"github.com/arduino/arduino-cli/buildcache"
 	"github.com/arduino/arduino-cli/commands"
 	"github.com/arduino/arduino-cli/configuration"
 	"github.com/arduino/arduino-cli/i18n"
+	"github.com/arduino/arduino-cli/inventory"
 	"github.com/arduino/arduino-cli/legacy/builder"
 	"github.com/arduino/arduino-cli/legacy/builder/types"
 	rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
@@ -64,6 +66,8 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream
 	if lm == nil {
 		return nil, &arduino.InvalidInstanceError{}
 	}
+	// cache is purged after compilation to not remove entries that might be required
+	defer maybePurgeBuildCache()
 
 	logrus.Tracef("Compile %s for %s started", req.GetSketchPath(), req.GetFqbn())
 	if req.GetSketchPath() == "" {
@@ -142,8 +146,7 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream
 
 	// Optimize for debug
 	builderCtx.OptimizeForDebug = req.GetOptimizeForDebug()
-
-	builderCtx.CoreBuildCachePath = paths.TempDir().Join("arduino", "core-cache")
+	builderCtx.CoreBuildCachePath = paths.TempDir().Join("arduino", "cores")
 
 	builderCtx.Jobs = int(req.GetJobs())
 
@@ -213,6 +216,8 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream
 		r.UsedLibraries = importedLibs
 	}()
 
+	defer buildcache.UpdateLastUsedTime(builderCtx.BuildPath)
+
 	// if it's a regular build, go on...
 	if err := builder.RunBuilder(builderCtx); err != nil {
 		return r, &arduino.CompileFailedError{Message: err.Error()}
@@ -268,3 +273,24 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream
 
 	return r, nil
 }
+
+// maybePurgeBuildCache runs the build files cache purge if the policy conditions are met.
+func maybePurgeBuildCache() {
+
+	compilationSinceLastPurge := inventory.Store.GetInt("build_cache.compilation_count_since_last_purge")
+	compilationSinceLastPurge++
+	inventory.Store.Set("build_cache.compilation_count_since_last_purge", compilationSinceLastPurge)
+	defer inventory.WriteStore()
+
+	// 0 means never purge
+	purgeAfterCompilationCount := configuration.Settings.GetInt("build_cache.purge_at_compilation_count")
+
+	if purgeAfterCompilationCount == 0 || compilationSinceLastPurge < purgeAfterCompilationCount {
+		return
+	}
+	inventory.Store.Set("build_cache.compilation_count_since_last_purge", 0)
+
+	cacheTTL := configuration.Settings.GetDuration("build_cache.ttl")
+	buildcache.Purge(paths.TempDir().Join("arduino", "cores"), cacheTTL)
+	buildcache.Purge(paths.TempDir().Join("arduino", "sketches"), cacheTTL)
+}

configuration/defaults.go

@@ -18,6 +18,7 @@ package configuration
 import (
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/spf13/viper"
 )
@@ -41,6 +42,8 @@ func SetDefaults(settings *viper.Viper) {
 
 	// Sketch compilation
 	settings.SetDefault("sketch.always_export_binaries", false)
+	settings.SetDefault("build_cache.ttl", time.Hour*24*7)
+	settings.SetDefault("build_cache.purge_at_compilation_count", 10)
 
 	// daemon settings
 	settings.SetDefault("daemon.port", "50051")

legacy/builder/add_additional_entries_to_context.go

@@ -35,7 +35,7 @@ func (*AddAdditionalEntriesToContext) Run(ctx *types.Context) error {
 		if err != nil {
 			return errors.WithStack(err)
 		}
-		librariesBuildPath, err := buildPath.Join("libraries").Abs()
+		librariesBuildPath, err := buildPath.Join(constants.FOLDER_LIBRARIES).Abs()
 		if err != nil {
 			return errors.WithStack(err)
 		}

legacy/builder/constants/constants.go

@@ -47,6 +47,7 @@ const FOLDER_CORE = "core"
 const FOLDER_PREPROC = "preproc"
 const FOLDER_SKETCH = "sketch"
 const FOLDER_TOOLS = "tools"
+const FOLDER_LIBRARIES = "libraries"
 const LIBRARY_ALL_ARCHS = "*"
 const LIBRARY_EMAIL = "email"
 const LIBRARY_FOLDER_ARCH = "arch"

legacy/builder/phases/core_builder.go

@@ -19,6 +19,7 @@ import (
 	"os"
 	"strings"
 
+	"github.com/arduino/arduino-cli/buildcache"
 	"github.com/arduino/arduino-cli/i18n"
 	"github.com/arduino/arduino-cli/legacy/builder/builder_utils"
 	"github.com/arduino/arduino-cli/legacy/builder/constants"
@@ -91,10 +92,12 @@ func compileCore(ctx *types.Context, buildPath *paths.Path, buildCachePath *path
 	realCoreFolder := coreFolder.Parent().Parent()
 
 	var targetArchivedCore *paths.Path
+	var targetArchivedCoreDir *paths.Path
 	if buildCachePath != nil {
-		archivedCoreName := GetCachedCoreArchiveFileName(buildProperties.Get(constants.BUILD_PROPERTIES_FQBN),
+		archivedCoreName := GetCachedCoreArchiveDirName(buildProperties.Get(constants.BUILD_PROPERTIES_FQBN),
 			buildProperties.Get("compiler.optimization_flags"), realCoreFolder)
-		targetArchivedCore = buildCachePath.Join(archivedCoreName)
+		targetArchivedCoreDir = buildCachePath.Join(archivedCoreName)
+		targetArchivedCore = targetArchivedCoreDir.Join("core.a")
 		canUseArchivedCore := !ctx.OnlyUpdateCompilationDatabase &&
 			!ctx.Clean &&
 			!builder_utils.CoreOrReferencedCoreHasChanged(realCoreFolder, targetCoreFolder, targetArchivedCore)
@@ -120,6 +123,8 @@ func compileCore(ctx *types.Context, buildPath *paths.Path, buildCachePath *path
 
 	// archive core.a
 	if targetArchivedCore != nil && !ctx.OnlyUpdateCompilationDatabase {
+		defer buildcache.UpdateLastUsedTime(targetArchivedCore)
+		targetArchivedCoreDir.MkdirAll()
 		err := archiveFile.CopyTo(targetArchivedCore)
 		if ctx.Verbose {
 			if err == nil {
@@ -137,19 +142,19 @@ func compileCore(ctx *types.Context, buildPath *paths.Path, buildCachePath *path
 	return archiveFile, variantObjectFiles, nil
 }
 
-// GetCachedCoreArchiveFileName returns the filename to be used to store
+// GetCachedCoreArchiveDirName returns the directory name to be used to store
 // the global cached core.a.
-func GetCachedCoreArchiveFileName(fqbn string, optimizationFlags string, coreFolder *paths.Path) string {
+func GetCachedCoreArchiveDirName(fqbn string, optimizationFlags string, coreFolder *paths.Path) string {
 	fqbnToUnderscore := strings.Replace(fqbn, ":", "_", -1)
 	fqbnToUnderscore = strings.Replace(fqbnToUnderscore, "=", "_", -1)
 	if absCoreFolder, err := coreFolder.Abs(); err == nil {
 		coreFolder = absCoreFolder
 	} // silently continue if absolute path can't be detected
 	hash := utils.MD5Sum([]byte(coreFolder.String() + optimizationFlags))
-	realName := "core_" + fqbnToUnderscore + "_" + hash + ".a"
+	realName := "core_" + fqbnToUnderscore + "_" + hash
 	if len(realName) > 100 {
 		// avoid really long names, simply hash the final part
-		realName = "core_" + utils.MD5Sum([]byte(fqbnToUnderscore+"_"+hash)) + ".a"
+		realName = "core_" + utils.MD5Sum([]byte(fqbnToUnderscore+"_"+hash))
 	}
 	return realName
 }

legacy/builder/test/builder_test.go

@@ -380,7 +380,7 @@ func TestBuilderCacheCoreAFile(t *testing.T) {
 
 	// Pick timestamp of cached core
 	coreFolder := paths.New("downloaded_hardware", "arduino", "avr")
-	coreFileName := phases.GetCachedCoreArchiveFileName(ctx.FQBN.String(), ctx.OptimizationFlags, coreFolder)
+	coreFileName := phases.GetCachedCoreArchiveDirName(ctx.FQBN.String(), ctx.OptimizationFlags, coreFolder)
 	cachedCoreFile := ctx.CoreBuildCachePath.Join(coreFileName)
 	coreStatBefore, err := cachedCoreFile.Stat()
 	require.NoError(t, err)