Skip to content

Commit 303ce81

Browse files
johnstcnjohnstcn
committed
feat: support starting from an already-built image (#296)
- Extracts 'magic directives' to constants.go. - Adds 'magic image' file to signify envbuilder should skip the destructive 'build image' stage. - Modifies logic for copying binary into built image: we now copy to build context and remove it after build finishes to avoid leaving around files owned by root:root. Also ensures files are created with consistent permissions. (cherry picked from commit 6afe89e)
1 parent 069a0aa commit 303ce81

File tree

5 files changed

+140
-62
lines changed

5 files changed

+140
-62
lines changed

‎cmd/envbuilder/main.go

+1
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ func envbuilderCmd() serpent.Command {
6868
img, err := envbuilder.RunCacheProbe(inv.Context(), o)
6969
if err != nil {
7070
o.Logger(log.LevelError, "error: %s", err)
71+
return err
7172
}
7273
digest, err := img.Digest()
7374
if err != nil {

‎constants/constants.go

+29
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package constants
22

33
import (
44
"errors"
5+
"fmt"
56
"path/filepath"
67
)
78

@@ -32,4 +33,32 @@ var (
3233
// MagicFile is the location of the build context when
3334
// using remote build mode.
3435
MagicRemoteRepoDir = filepath.Join(MagicDir, "repo")
36+
37+
// MagicBinaryLocation is the expected location of the envbuilder binary
38+
// inside a builder image.
39+
MagicBinaryLocation = filepath.Join(MagicDir, "bin", "envbuilder")
40+
41+
// MagicImage is a file that is created in the image when
42+
// envbuilder has already been run. This is used to skip
43+
// the destructive initial build step when 'resuming' envbuilder
44+
// from a previously built image.
45+
MagicImage = filepath.Join(MagicDir, "image")
46+
47+
// MagicTempDir is a directory inside the build context inside which
48+
// we place files referenced by MagicDirectives.
49+
MagicTempDir = ".envbuilder.tmp"
50+
51+
// MagicDirectives are directives automatically appended to Dockerfiles
52+
// when pushing the image. These directives allow the built image to be
53+
// 're-used'.
54+
MagicDirectives = fmt.Sprintf(`
55+
COPY --chmod=0755 %[1]s %[2]s
56+
COPY --chmod=0644 %[3]s %[4]s
57+
USER root
58+
WORKDIR /
59+
ENTRYPOINT [%[2]q]
60+
`,
61+
".envbuilder.tmp/envbuilder", MagicBinaryLocation,
62+
".envbuilder.tmp/image", MagicImage,
63+
)
3564
)

‎envbuilder.go

+107-59
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import (
2727
"github.com/coder/envbuilder/constants"
2828
"github.com/coder/envbuilder/git"
2929
"github.com/coder/envbuilder/options"
30+
"github.com/go-git/go-billy/v5"
3031

3132
"github.com/GoogleContainerTools/kaniko/pkg/config"
3233
"github.com/GoogleContainerTools/kaniko/pkg/creds"
@@ -311,26 +312,58 @@ func Run(ctx context.Context, opts options.Options) error {
311312
}
312313

313314
// In order to allow 'resuming' envbuilder, embed the binary into the image
314-
// if it is being pushed
315+
// if it is being pushed.
316+
// As these files will be owned by root, it is considerate to clean up
317+
// after we're done!
318+
cleanupBuildContext := func() {}
315319
if opts.PushImage {
316-
exePath, err := os.Executable()
317-
if err != nil {
318-
return xerrors.Errorf("get exe path: %w", err)
320+
// Add exceptions in Kaniko's ignorelist for these magic files we add.
321+
if err := util.AddAllowedPathToDefaultIgnoreList(opts.BinaryPath); err != nil {
322+
return fmt.Errorf("add envbuilder binary to ignore list: %w", err)
323+
}
324+
if err := util.AddAllowedPathToDefaultIgnoreList(constants.MagicImage); err != nil {
325+
return fmt.Errorf("add magic image file to ignore list: %w", err)
319326
}
320-
// Add an exception for the current running binary in kaniko ignore list
321-
if err := util.AddAllowedPathToDefaultIgnoreList(exePath); err != nil {
322-
return xerrors.Errorf("add exe path to ignore list: %w", err)
327+
magicTempDir := filepath.Join(buildParams.BuildContext, constants.MagicTempDir)
328+
if err := opts.Filesystem.MkdirAll(magicTempDir, 0o755); err != nil {
329+
return fmt.Errorf("create magic temp dir in build context: %w", err)
323330
}
331+
// Add the magic directives that embed the binary into the built image.
332+
buildParams.DockerfileContent += constants.MagicDirectives
324333
// Copy the envbuilder binary into the build context.
325-
buildParams.DockerfileContent += fmt.Sprintf(`
326-
COPY --chmod=0755 %s %s
327-
USER root
328-
WORKDIR /
329-
ENTRYPOINT [%q]`, exePath, exePath, exePath)
330-
dst := filepath.Join(buildParams.BuildContext, exePath)
331-
if err := copyFile(exePath, dst); err != nil {
332-
return xerrors.Errorf("copy running binary to build context: %w", err)
334+
// External callers will need to specify the path to the desired envbuilder binary.
335+
envbuilderBinDest := filepath.Join(
336+
magicTempDir,
337+
filepath.Base(constants.MagicBinaryLocation),
338+
)
339+
// Also touch the magic file that signifies the image has been built!
340+
magicImageDest := filepath.Join(
341+
magicTempDir,
342+
filepath.Base(constants.MagicImage),
343+
)
344+
// Clean up after build!
345+
var cleanupOnce sync.Once
346+
cleanupBuildContext = func() {
347+
cleanupOnce.Do(func() {
348+
for _, path := range []string{magicImageDest, envbuilderBinDest, magicTempDir} {
349+
if err := opts.Filesystem.Remove(path); err != nil {
350+
opts.Logger(log.LevelWarn, "failed to clean up magic temp dir from build context: %w", err)
351+
}
352+
}
353+
})
354+
}
355+
defer cleanupBuildContext()
356+
357+
opts.Logger(log.LevelDebug, "copying envbuilder binary at %q to build context %q", opts.BinaryPath, envbuilderBinDest)
358+
if err := copyFile(opts.Filesystem, opts.BinaryPath, envbuilderBinDest, 0o755); err != nil {
359+
return fmt.Errorf("copy envbuilder binary to build context: %w", err)
360+
}
361+
362+
opts.Logger(log.LevelDebug, "touching magic image file at %q in build context %q", magicImageDest, buildParams.BuildContext)
363+
if err := touchFile(opts.Filesystem, magicImageDest, 0o755); err != nil {
364+
return fmt.Errorf("touch magic image file in build context: %w", err)
333365
}
366+
334367
}
335368

336369
// temp move of all ro mounts
@@ -354,8 +387,10 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath)
354387
stderrWriter, closeStderr := log.Writer(opts.Logger)
355388
defer closeStderr()
356389
build := func() (v1.Image, error) {
357-
_, err := opts.Filesystem.Stat(constants.MagicFile)
358-
if err == nil && opts.SkipRebuild {
390+
defer cleanupBuildContext()
391+
_, alreadyBuiltErr := opts.Filesystem.Stat(constants.MagicFile)
392+
_, isImageErr := opts.Filesystem.Stat(constants.MagicImage)
393+
if (alreadyBuiltErr == nil && opts.SkipRebuild) || isImageErr == nil {
359394
endStage := startStage("🏗️ Skipping build because of cache...")
360395
imageRef, err := devcontainer.ImageFromDockerfile(buildParams.DockerfileContent)
361396
if err != nil {
@@ -381,26 +416,7 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath)
381416
if err := maybeDeleteFilesystem(opts.Logger, opts.ForceSafe); err != nil {
382417
return nil, fmt.Errorf("delete filesystem: %w", err)
383418
}
384-
/*
385-
stdoutReader, stdoutWriter := io.Pipe()
386-
stderrReader, stderrWriter := io.Pipe()
387-
defer stdoutReader.Close()
388-
defer stdoutWriter.Close()
389-
defer stderrReader.Close()
390-
defer stderrWriter.Close()
391-
go func() {
392-
scanner := bufio.NewScanner(stdoutReader)
393-
for scanner.Scan() {
394-
opts.Logger(log.LevelInfo, "%s", scanner.Text())
395-
}
396-
}()
397-
go func() {
398-
scanner := bufio.NewScanner(stderrReader)
399-
for scanner.Scan() {
400-
opts.Logger(log.LevelInfo, "%s", scanner.Text())
401-
}
402-
}()
403-
*/
419+
404420
cacheTTL := time.Hour * 24 * 7
405421
if opts.CacheTTLDays != 0 {
406422
cacheTTL = time.Hour * 24 * time.Duration(opts.CacheTTLDays)
@@ -1064,23 +1080,41 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error)
10641080
// We expect an image built and pushed by envbuilder to have the envbuilder
10651081
// binary present at a predefined path. In order to correctly replicate the
10661082
// build via executor.RunCacheProbe we need to have the *exact* copy of the
1067-
// envbuilder binary available used to build the image.
1068-
exePath := opts.BinaryPath
1069-
// Add an exception for the current running binary in kaniko ignore list
1070-
if err := util.AddAllowedPathToDefaultIgnoreList(exePath); err != nil {
1071-
return nil, xerrors.Errorf("add exe path to ignore list: %w", err)
1072-
}
1083+
// envbuilder binary available used to build the image and we also need to
1084+
// add the magic directives to the Dockerfile content.
1085+
buildParams.DockerfileContent += constants.MagicDirectives
1086+
magicTempDir := filepath.Join(buildParams.BuildContext, constants.MagicTempDir)
1087+
if err := opts.Filesystem.MkdirAll(magicTempDir, 0o755); err != nil {
1088+
return nil, fmt.Errorf("create magic temp dir in build context: %w", err)
1089+
}
1090+
envbuilderBinDest := filepath.Join(
1091+
magicTempDir,
1092+
filepath.Base(constants.MagicBinaryLocation),
1093+
)
1094+
10731095
// Copy the envbuilder binary into the build context.
1074-
buildParams.DockerfileContent += fmt.Sprintf(`
1075-
COPY --chmod=0755 %s %s
1076-
USER root
1077-
WORKDIR /
1078-
ENTRYPOINT [%q]`, exePath, exePath, exePath)
1079-
dst := filepath.Join(buildParams.BuildContext, exePath)
1080-
if err := copyFile(exePath, dst); err != nil {
1096+
opts.Logger(log.LevelDebug, "copying envbuilder binary at %q to build context %q", opts.BinaryPath, buildParams.BuildContext)
1097+
if err := copyFile(opts.Filesystem, opts.BinaryPath, envbuilderBinDest, 0o755); err != nil {
10811098
return nil, xerrors.Errorf("copy envbuilder binary to build context: %w", err)
10821099
}
10831100

1101+
// Also touch the magic file that signifies the image has been built!
1102+
magicImageDest := filepath.Join(
1103+
magicTempDir,
1104+
filepath.Base(constants.MagicImage),
1105+
)
1106+
if err := touchFile(opts.Filesystem, magicImageDest, 0o755); err != nil {
1107+
return nil, fmt.Errorf("touch magic image file in build context: %w", err)
1108+
}
1109+
defer func() {
1110+
// Clean up after we're done!
1111+
for _, path := range []string{magicImageDest, envbuilderBinDest, magicTempDir} {
1112+
if err := opts.Filesystem.Remove(path); err != nil {
1113+
opts.Logger(log.LevelWarn, "failed to clean up magic temp dir from build context: %w", err)
1114+
}
1115+
}
1116+
}()
1117+
10841118
stdoutWriter, closeStdout := log.Writer(opts.Logger)
10851119
defer closeStdout()
10861120
stderrWriter, closeStderr := log.Writer(opts.Logger)
@@ -1138,8 +1172,8 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath)
11381172
},
11391173
SrcContext: buildParams.BuildContext,
11401174

1141-
// For cached image utilization, produce reproducible builds.
1142-
Reproducible: opts.PushImage,
1175+
// When performing a cache probe, always perform reproducible snapshots.
1176+
Reproducible: true,
11431177
}
11441178

11451179
endStage := startStage("🏗️ Checking for cached image...")
@@ -1382,24 +1416,38 @@ func maybeDeleteFilesystem(logger log.Func, force bool) error {
13821416
return util.DeleteFilesystem()
13831417
}
13841418

1385-
func copyFile(src, dst string) error {
1386-
content, err := os.ReadFile(src)
1419+
func copyFile(fs billy.Filesystem, src, dst string, mode fs.FileMode) error {
1420+
srcF, err := fs.Open(src)
13871421
if err != nil {
1388-
return fmt.Errorf("read file failed: %w", err)
1422+
return fmt.Errorf("open src file: %w", err)
13891423
}
1424+
defer srcF.Close()
13901425

1391-
err = os.MkdirAll(filepath.Dir(dst), 0o755)
1426+
err = fs.MkdirAll(filepath.Dir(dst), mode)
13921427
if err != nil {
1393-
return fmt.Errorf("mkdir all failed: %w", err)
1428+
return fmt.Errorf("create destination dir failed: %w", err)
13941429
}
13951430

1396-
err = os.WriteFile(dst, content, 0o644)
1431+
dstF, err := fs.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode)
13971432
if err != nil {
1398-
return fmt.Errorf("write file failed: %w", err)
1433+
return fmt.Errorf("open dest file for writing: %w", err)
1434+
}
1435+
defer dstF.Close()
1436+
1437+
if _, err := io.Copy(dstF, srcF); err != nil {
1438+
return fmt.Errorf("copy failed: %w", err)
13991439
}
14001440
return nil
14011441
}
14021442

1443+
func touchFile(fs billy.Filesystem, dst string, mode fs.FileMode) error {
1444+
f, err := fs.OpenFile(dst, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode)
1445+
if err != nil {
1446+
return xerrors.Errorf("failed to touch file: %w", err)
1447+
}
1448+
return f.Close()
1449+
}
1450+
14031451
func initDockerConfigJSON(dockerConfigBase64 string) (func() error, error) {
14041452
var cleanupOnce sync.Once
14051453
noop := func() error { return nil }

‎go.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ go 1.22.4
44

55
// There are a few options we need added to Kaniko!
66
// See: https://github.com/GoogleContainerTools/kaniko/compare/main...coder:kaniko:main
7-
replace github.com/GoogleContainerTools/kaniko => github.com/coder/kaniko v0.0.0-20240717115058-0ba2908ca4d3
7+
replace github.com/GoogleContainerTools/kaniko => github.com/coder/kaniko v0.0.0-20240803153527-10d1800455b9
88

99
// Required to import codersdk due to gvisor dependency.
1010
replace tailscale.com => github.com/coder/tailscale v1.1.1-0.20240702054557-aa558fbe5374

‎go.sum

+2-2
Original file line numberDiff line numberDiff line change
@@ -171,8 +171,8 @@ github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoC
171171
github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
172172
github.com/coder/coder/v2 v2.10.1-0.20240704130443-c2d44d16a352 h1:L/EjCuZxs5tOcqqCaASj/nu65TRYEFcTt8qRQfHZXX0=
173173
github.com/coder/coder/v2 v2.10.1-0.20240704130443-c2d44d16a352/go.mod h1:P1KoQSgnKEAG6Mnd3YlGzAophty+yKA9VV48LpfNRvo=
174-
github.com/coder/kaniko v0.0.0-20240717115058-0ba2908ca4d3 h1:Q7L6cjKfw3DIyhKIcgCJEmgxnUTBajmMDrHxXvxgBZs=
175-
github.com/coder/kaniko v0.0.0-20240717115058-0ba2908ca4d3/go.mod h1:YMK7BlxerzLlMwihGxNWUaFoN9LXCij4P+w/8/fNlcM=
174+
github.com/coder/kaniko v0.0.0-20240803153527-10d1800455b9 h1:d01T5YbPN1yc1mXjIXG59YcQQoT/9idvqFErjWHfsZ4=
175+
github.com/coder/kaniko v0.0.0-20240803153527-10d1800455b9/go.mod h1:YMK7BlxerzLlMwihGxNWUaFoN9LXCij4P+w/8/fNlcM=
176176
github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0 h1:3A0ES21Ke+FxEM8CXx9n47SZOKOpgSE1bbJzlE4qPVs=
177177
github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0/go.mod h1:5UuS2Ts+nTToAMeOjNlnHFkPahrtDkmpydBen/3wgZc=
178178
github.com/coder/quartz v0.1.0 h1:cLL+0g5l7xTf6ordRnUMMiZtRE8Sq5LxpghS63vEXrQ=

0 commit comments

Comments
 (0)