Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit d9fc74b

Browse files
author
Luca Bianconi
committedJan 11, 2023
refactoring(wip): cache interface
1 parent 7e7ea35 commit d9fc74b

File tree

12 files changed

+152
-82
lines changed

12 files changed

+152
-82
lines changed
 

‎arduino/sketch/sketch_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -286,10 +286,10 @@ func TestNewSketchFolderSymlink(t *testing.T) {
286286
}
287287

288288
func TestGenBuildPath(t *testing.T) {
289-
want := paths.TempDir().Join("arduino", "sketch-ACBD18DB4CC2F85CEDEF654FCCC4A4D8")
289+
want := paths.TempDir().Join("arduino", "sketches", "sketch-ACBD18DB4CC2F85CEDEF654FCCC4A4D8")
290290
assert.True(t, GenBuildPath(paths.New("foo")).EquivalentTo(want))
291291

292-
want = paths.TempDir().Join("arduino", "sketch-D41D8CD98F00B204E9800998ECF8427E")
292+
want = paths.TempDir().Join("arduino", "sketches", "sketch-D41D8CD98F00B204E9800998ECF8427E")
293293
assert.True(t, GenBuildPath(nil).EquivalentTo(want))
294294
}
295295

‎buildcache/build_cache.go

Lines changed: 7 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -16,70 +16,22 @@
1616
package buildcache
1717

1818
import (
19-
"os"
2019
"time"
2120

2221
"github.com/arduino/go-paths-helper"
23-
"github.com/pkg/errors"
24-
"github.com/sirupsen/logrus"
2522
)
2623

27-
const (
28-
holdFileName = ".cache-metadata.yaml"
29-
)
30-
31-
func getHoldFileModTime(p string) time.Time {
32-
fileInfo, err := paths.New(p).Stat()
33-
if err != nil {
34-
return time.Unix(0, 0)
35-
}
36-
return fileInfo.ModTime()
37-
}
38-
39-
func updateHoldFile(p string) error {
40-
return paths.New(p).WriteFile([]byte{})
41-
}
42-
43-
// UpdateLastUsedTime registers the last used time in the cache metadata file
44-
func UpdateLastUsedTime(path *paths.Path) error {
45-
dir := requireDirectory(path)
46-
return updateHoldFile(dir.Join(holdFileName).String())
24+
// UpdateLastUsedTime registers the last used time in the cache hold file
25+
func UpdateLastUsedTime(dir *paths.Path) error {
26+
_, err := newLastUsedDirCache(dir.Parent().String(), time.Hour).
27+
GetOrCreate(dir.Base(), time.Now())
28+
return err
4729
}
4830

4931
// Purge removes all cache directories within baseDir that have expired
5032
// To know how long ago a directory has been last used
51-
// it checks into the hold file. If the file does not exist
33+
// it checks into the last used file. If the file does not exist
5234
// then the directory is purged.
5335
func Purge(baseDir *paths.Path, ttl time.Duration) {
54-
files, err := os.ReadDir(baseDir.String())
55-
if err != nil {
56-
return
57-
}
58-
for _, file := range files {
59-
if file.IsDir() {
60-
deleteIfExpired(baseDir.Join(file.Name()), ttl)
61-
}
62-
}
63-
}
64-
65-
func deleteIfExpired(dir *paths.Path, ttl time.Duration) {
66-
modTime := getHoldFileModTime(dir.Join(holdFileName).String())
67-
if time.Since(modTime) < ttl {
68-
return
69-
}
70-
logrus.Tracef(`Pruning cache directory "%s". Expired by %s`, dir, (time.Since(modTime) - ttl))
71-
err := os.RemoveAll(dir.String())
72-
73-
if err != nil {
74-
logrus.Tracef(`Error while pruning cache directory "%s".\n%s\n`, dir, errors.WithStack(err))
75-
}
76-
}
77-
78-
// requireDirectory returns p if it's a directory, its parent directory otherwise
79-
func requireDirectory(p *paths.Path) *paths.Path {
80-
directoryPath := p
81-
if !directoryPath.IsDir() {
82-
directoryPath = p.Parent()
83-
}
84-
return directoryPath
36+
newLastUsedDirCache(baseDir.String(), ttl).Purge()
8537
}

‎buildcache/build_cache_test.go

Lines changed: 7 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ func Test_UpdateLastUsedFileExisting(t *testing.T) {
2424
require.Nil(t, err)
2525

2626
// create the file
27-
err = paths.New(testBuildDir.Join(holdFileName).String()).WriteFile([]byte{})
27+
err = paths.New(testBuildDir.Join(lastUsedFileName).String()).WriteFile([]byte{})
2828
require.Nil(t, err)
2929

3030
requireCorrectUpdate(t, testBuildDir)
@@ -34,15 +34,11 @@ func requireCorrectUpdate(t *testing.T, dir *paths.Path) {
3434
timeBeforeUpdating := time.Now()
3535
err := UpdateLastUsedTime(dir)
3636
require.Nil(t, err)
37-
expectedMetadataFile := dir.Join(holdFileName)
38-
_, err = os.Stat(expectedMetadataFile.String())
37+
expectedFile := dir.Join(lastUsedFileName)
38+
fileInfo, err := os.Stat(expectedFile.String())
3939
require.Nil(t, err)
4040

41-
// content can be decoded
42-
err = updateHoldFile(expectedMetadataFile.String())
43-
require.Nil(t, err)
44-
lastUsedTime := getHoldFileModTime(expectedMetadataFile.String())
45-
require.GreaterOrEqual(t, lastUsedTime, timeBeforeUpdating)
41+
require.GreaterOrEqual(t, fileInfo.ModTime(), timeBeforeUpdating)
4642
}
4743

4844
func TestPurge(t *testing.T) {
@@ -59,12 +55,12 @@ func TestPurge(t *testing.T) {
5955
for dirPath, lastUsedTime := range lastUsedTimesByDirPath {
6056
err := os.MkdirAll(dirPath.Canonical().String(), 0770)
6157
require.Nil(t, err)
62-
holdFilePath := dirPath.Join(holdFileName).Canonical().String()
63-
err = updateHoldFile(holdFilePath)
58+
infoFilePath := dirPath.Join(lastUsedFileName).Canonical().String()
59+
err = paths.New(infoFilePath).WriteFile([]byte{})
6460
require.Nil(t, err)
6561
// make sure access time does not matter
6662
accesstime := time.Now()
67-
err = os.Chtimes(holdFilePath, accesstime, lastUsedTime)
63+
err = os.Chtimes(infoFilePath, accesstime, lastUsedTime)
6864
require.Nil(t, err)
6965
}
7066

‎buildcache/directory_cache.go

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
package buildcache
2+
3+
import (
4+
"os"
5+
"time"
6+
7+
"github.com/arduino/go-paths-helper"
8+
"github.com/pkg/errors"
9+
"github.com/sirupsen/logrus"
10+
)
11+
12+
const (
13+
lastUsedFileName = ".last-used"
14+
)
15+
16+
type directoryCache struct {
17+
baseDir string
18+
ttl time.Duration
19+
}
20+
21+
func (dc *directoryCache) basePath() *paths.Path {
22+
return paths.New(dc.baseDir)
23+
}
24+
25+
func (dc *directoryCache) isExpired(key string, ttl time.Duration) bool {
26+
modTime, _ := dc.modTime(key)
27+
return time.Since(modTime) >= ttl
28+
}
29+
30+
func (dc *directoryCache) modTime(key string) (time.Time, error) {
31+
fileInfo, err := os.Stat(dc.basePath().Join(key, lastUsedFileName).String())
32+
if err != nil {
33+
return time.Unix(0, 0), err
34+
}
35+
36+
return fileInfo.ModTime(), nil
37+
}
38+
39+
func (dc *directoryCache) GetOrCreate(key string, value time.Time) (time.Time, error) {
40+
existing := true
41+
modTime, err := dc.modTime(key)
42+
if err != nil {
43+
existing = false
44+
}
45+
46+
subDir := dc.basePath().Join(key)
47+
os.MkdirAll(subDir.String(), 0770)
48+
err = subDir.Join(lastUsedFileName).WriteFile([]byte{})
49+
if err != nil || existing {
50+
return modTime, err
51+
}
52+
return time.Now(), nil
53+
}
54+
55+
func (dc *directoryCache) Purge() error {
56+
files, err := os.ReadDir(dc.basePath().String())
57+
if err != nil {
58+
return err
59+
}
60+
for _, file := range files {
61+
if file.IsDir() && dc.isExpired(file.Name(), dc.ttl) {
62+
subDir := dc.basePath().Join(file.Name())
63+
modTime, _ := dc.modTime(file.Name())
64+
logrus.Tracef(`Purging cache directory "%s". Expired by %s\n`, subDir, (time.Since(modTime) - dc.ttl))
65+
err := os.RemoveAll(subDir.String())
66+
67+
if err != nil {
68+
logrus.Tracef(`Error while pruning cache directory "%s".\n%s\n`, subDir, errors.WithStack(err))
69+
}
70+
}
71+
}
72+
return nil
73+
}
74+
75+
func newLastUsedDirCache(baseDir string, ttl time.Duration) cache[string, time.Time] {
76+
return &directoryCache{
77+
baseDir: baseDir,
78+
ttl: ttl,
79+
}
80+
}

‎buildcache/interface.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
package buildcache
2+
3+
type cache[K comparable, V any] interface {
4+
GetOrCreate(key K, createvalue V) (V, error)
5+
Purge() error
6+
}

‎commands/compile/compile.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -283,7 +283,7 @@ func maybePurgeBuildCache() {
283283
defer inventory.WriteStore()
284284

285285
// 0 means never purge
286-
purgeAfterCompilationCount := configuration.Settings.GetInt("build_cache.purge_at_compilation_count")
286+
purgeAfterCompilationCount := configuration.Settings.GetInt("build_cache.compilations_before_purge")
287287

288288
if purgeAfterCompilationCount == 0 || compilationSinceLastPurge < purgeAfterCompilationCount {
289289
return

‎configuration/defaults.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ func SetDefaults(settings *viper.Viper) {
4343
// Sketch compilation
4444
settings.SetDefault("sketch.always_export_binaries", false)
4545
settings.SetDefault("build_cache.ttl", time.Hour*24*7)
46-
settings.SetDefault("build_cache.purge_at_compilation_count", 10)
46+
settings.SetDefault("build_cache.compilations_before_purge", 10)
4747

4848
// daemon settings
4949
settings.SetDefault("daemon.port", "50051")

‎internal/integrationtest/arduino-cli.go

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -108,10 +108,11 @@ func NewArduinoCliWithinEnvironment(env *Environment, config *ArduinoCLIConfig)
108108
}
109109

110110
cli.cliEnvVars = map[string]string{
111-
"LANG": "en",
112-
"ARDUINO_DATA_DIR": cli.dataDir.String(),
113-
"ARDUINO_DOWNLOADS_DIR": cli.stagingDir.String(),
114-
"ARDUINO_SKETCHBOOK_DIR": cli.sketchbookDir.String(),
111+
"LANG": "en",
112+
"ARDUINO_DATA_DIR": cli.dataDir.String(),
113+
"ARDUINO_DOWNLOADS_DIR": cli.stagingDir.String(),
114+
"ARDUINO_SKETCHBOOK_DIR": cli.sketchbookDir.String(),
115+
"ARDUINO_BUILD_CACHE_COMPILATIONS_BEFORE_PURGE": "0",
115116
}
116117
env.RegisterCleanUpCallback(cli.CleanUp)
117118
return cli

‎internal/integrationtest/compile_1/compile_test.go

Lines changed: 40 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
"os"
2323
"strings"
2424
"testing"
25+
"time"
2526

2627
"github.com/arduino/arduino-cli/internal/integrationtest"
2728
"github.com/arduino/go-paths-helper"
@@ -47,6 +48,7 @@ func TestCompile(t *testing.T) {
4748
{"WithoutFqbn", compileWithoutFqbn},
4849
{"ErrorMessage", compileErrorMessage},
4950
{"WithSimpleSketch", compileWithSimpleSketch},
51+
{"WithCachePurgeNeeded", compileWithCachePurgeNeeded},
5052
{"OutputFlagDefaultPath", compileOutputFlagDefaultPath},
5153
{"WithSketchWithSymlinkSelfloop", compileWithSketchWithSymlinkSelfloop},
5254
{"BlacklistedSketchname", compileBlacklistedSketchname},
@@ -112,6 +114,39 @@ func compileErrorMessage(t *testing.T, env *integrationtest.Environment, cli *in
112114
}
113115

114116
func compileWithSimpleSketch(t *testing.T, env *integrationtest.Environment, cli *integrationtest.ArduinoCLI) {
117+
compileWithSimpleSketchCustomEnv(t, env, cli, cli.GetDefaultEnv())
118+
}
119+
120+
func compileWithCachePurgeNeeded(t *testing.T, env *integrationtest.Environment, cli *integrationtest.ArduinoCLI) {
121+
// create directories that must be purged
122+
baseDir := paths.TempDir().Join("arduino", "sketches")
123+
124+
// purge case: hold file too old
125+
oldDir1 := baseDir.Join("test_old_sketch_1")
126+
err := os.MkdirAll(oldDir1.String(), 0770)
127+
require.Nil(t, err)
128+
err = oldDir1.Join(".last-used").WriteFile([]byte{})
129+
require.Nil(t, err)
130+
err = os.Chtimes(oldDir1.Join(".last-used").String(), time.Now(), time.Unix(0, 0))
131+
require.Nil(t, err)
132+
// purge case: hold file not existing
133+
oldDir2 := baseDir.Join("test_old_sketch_2")
134+
err = os.MkdirAll(oldDir2.String(), 0770)
135+
require.Nil(t, err)
136+
137+
defer os.RemoveAll(oldDir1.String())
138+
defer os.RemoveAll(oldDir2.String())
139+
140+
customEnv := cli.GetDefaultEnv()
141+
customEnv["ARDUINO_BUILD_CACHE_COMPILATIONS_BEFORE_PURGE"] = "1"
142+
compileWithSimpleSketchCustomEnv(t, env, cli, customEnv)
143+
144+
// check that purge has been run
145+
require.NoFileExists(t, oldDir1.String())
146+
require.NoFileExists(t, oldDir2.String())
147+
}
148+
149+
func compileWithSimpleSketchCustomEnv(t *testing.T, env *integrationtest.Environment, cli *integrationtest.ArduinoCLI, customEnv map[string]string) {
115150
sketchName := "CompileIntegrationTest"
116151
sketchPath := cli.SketchbookDir().Join(sketchName)
117152
defer sketchPath.RemoveAll()
@@ -127,7 +162,7 @@ func compileWithSimpleSketch(t *testing.T, env *integrationtest.Environment, cli
127162
require.NoError(t, err)
128163

129164
// Build sketch for arduino:avr:uno with json output
130-
stdout, _, err = cli.Run("compile", "-b", fqbn, sketchPath.String(), "--format", "json")
165+
stdout, _, err = cli.RunWithCustomEnv(customEnv, "compile", "-b", fqbn, sketchPath.String(), "--format", "json")
131166
require.NoError(t, err)
132167
// check is a valid json and contains requested data
133168
var compileOutput map[string]interface{}
@@ -140,7 +175,7 @@ func compileWithSimpleSketch(t *testing.T, env *integrationtest.Environment, cli
140175
md5 := md5.Sum(([]byte(sketchPath.String())))
141176
sketchPathMd5 := strings.ToUpper(hex.EncodeToString(md5[:]))
142177
require.NotEmpty(t, sketchPathMd5)
143-
buildDir := paths.TempDir().Join("arduino", "sketch-"+sketchPathMd5)
178+
buildDir := paths.TempDir().Join("arduino", "sketches", "sketch-"+sketchPathMd5)
144179
require.FileExists(t, buildDir.Join(sketchName+".ino.eep").String())
145180
require.FileExists(t, buildDir.Join(sketchName+".ino.elf").String())
146181
require.FileExists(t, buildDir.Join(sketchName+".ino.hex").String())
@@ -374,7 +409,7 @@ func compileWithOutputDirFlag(t *testing.T, env *integrationtest.Environment, cl
374409
md5 := md5.Sum(([]byte(sketchPath.String())))
375410
sketchPathMd5 := strings.ToUpper(hex.EncodeToString(md5[:]))
376411
require.NotEmpty(t, sketchPathMd5)
377-
buildDir := paths.TempDir().Join("arduino", "sketch-"+sketchPathMd5)
412+
buildDir := paths.TempDir().Join("arduino", "sketches", "sketch-"+sketchPathMd5)
378413
require.FileExists(t, buildDir.Join(sketchName+".ino.eep").String())
379414
require.FileExists(t, buildDir.Join(sketchName+".ino.elf").String())
380415
require.FileExists(t, buildDir.Join(sketchName+".ino.hex").String())
@@ -441,7 +476,7 @@ func compileWithCustomBuildPath(t *testing.T, env *integrationtest.Environment,
441476
md5 := md5.Sum(([]byte(sketchPath.String())))
442477
sketchPathMd5 := strings.ToUpper(hex.EncodeToString(md5[:]))
443478
require.NotEmpty(t, sketchPathMd5)
444-
buildDir := paths.TempDir().Join("arduino", "sketch-"+sketchPathMd5)
479+
buildDir := paths.TempDir().Join("arduino", "sketches", "sketch-"+sketchPathMd5)
445480
require.NoFileExists(t, buildDir.Join(sketchName+".ino.eep").String())
446481
require.NoFileExists(t, buildDir.Join(sketchName+".ino.elf").String())
447482
require.NoFileExists(t, buildDir.Join(sketchName+".ino.hex").String())
@@ -975,7 +1010,7 @@ func compileWithInvalidBuildOptionJson(t *testing.T, env *integrationtest.Enviro
9751010
md5 := md5.Sum(([]byte(sketchPath.String())))
9761011
sketchPathMd5 := strings.ToUpper(hex.EncodeToString(md5[:]))
9771012
require.NotEmpty(t, sketchPathMd5)
978-
buildDir := paths.TempDir().Join("arduino", "sketch-"+sketchPathMd5)
1013+
buildDir := paths.TempDir().Join("arduino", "sketches", "sketch-"+sketchPathMd5)
9791014

9801015
_, _, err = cli.Run("compile", "-b", fqbn, sketchPath.String(), "--verbose")
9811016
require.NoError(t, err)

‎internal/integrationtest/compile_2/compile_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,7 @@ func recompileWithDifferentLibrary(t *testing.T, env *integrationtest.Environmen
145145
md5 := md5.Sum(([]byte(sketchPath.String())))
146146
sketchPathMd5 := strings.ToUpper(hex.EncodeToString(md5[:]))
147147
require.NotEmpty(t, sketchPathMd5)
148-
buildDir := paths.TempDir().Join("arduino", "sketch-"+sketchPathMd5)
148+
buildDir := paths.TempDir().Join("arduino", "sketches", "sketch-"+sketchPathMd5)
149149

150150
// Compile sketch using library not managed by CLI
151151
stdout, _, err := cli.Run("compile", "-b", fqbn, "--library", manuallyInstalledLibPath.String(), sketchPath.String(), "-v")

‎internal/integrationtest/core/core_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -253,7 +253,7 @@ func TestCoreInstallEsp32(t *testing.T) {
253253
md5 := md5.Sum(([]byte(sketchPath.String())))
254254
sketchPathMd5 := strings.ToUpper(hex.EncodeToString(md5[:]))
255255
require.NotEmpty(t, sketchPathMd5)
256-
buildDir := paths.TempDir().Join("arduino", "sketch-"+sketchPathMd5)
256+
buildDir := paths.TempDir().Join("arduino", "sketches", "sketch-"+sketchPathMd5)
257257
require.FileExists(t, buildDir.Join(sketchName+".ino.partitions.bin").String())
258258
}
259259

‎internal/integrationtest/upload_mock/upload_mock_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -697,7 +697,7 @@ func TestUploadSketch(t *testing.T) {
697697
func generateBuildDir(sketchPath *paths.Path, t *testing.T) *paths.Path {
698698
md5 := md5.Sum(([]byte(sketchPath.String())))
699699
sketchPathMd5 := strings.ToUpper(hex.EncodeToString(md5[:]))
700-
buildDir := paths.TempDir().Join("arduino", "sketch-"+sketchPathMd5)
700+
buildDir := paths.TempDir().Join("arduino", "sketches", "sketch-"+sketchPathMd5)
701701
require.NoError(t, buildDir.MkdirAll())
702702
require.NoError(t, buildDir.ToAbs())
703703
return buildDir

0 commit comments

Comments
 (0)
Please sign in to comment.