From 0497bed2fdda6de012c4da7716c3278eae5e913a Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 2 Aug 2024 23:53:53 +0100 Subject: [PATCH 1/6] feat: support starting from an already-built image --- constants/constants.go | 22 +++++++ envbuilder.go | 135 +++++++++++++++++++++++------------------ 2 files changed, 99 insertions(+), 58 deletions(-) diff --git a/constants/constants.go b/constants/constants.go index 042660dd..0fd98c45 100644 --- a/constants/constants.go +++ b/constants/constants.go @@ -2,6 +2,7 @@ package constants import ( "errors" + "fmt" "path/filepath" ) @@ -32,4 +33,25 @@ var ( // MagicFile is the location of the build context when // using remote build mode. MagicRemoteRepoDir = filepath.Join(MagicDir, "repo") + + // MagicBinaryLocation is the expected location of the envbuilder binary + // inside a builder image. + MagicBinaryLocation = filepath.Join(MagicDir, "bin", "envbuilder") + + // MagicImage is a file that is created in the image when + // envbuilder has already been run. This is used to skip + // the destructive initial build step when 'resuming' envbuilder + // from a previously built image. + MagicImage = filepath.Join(MagicDir, "image") + + // MagicDirectives are directives automatically appended to Dockerfiles + // when pushing the image. These directives allow the built image to be + // 're-used'. + MagicDirectives = fmt.Sprintf(` +COPY --chmod=0755 %[1]s %[2]s +COPY --chmod=0644 %[3]s %[4]s +USER root +WORKDIR / +ENTRYPOINT [%[2]q] +`, filepath.Base(MagicBinaryLocation), MagicBinaryLocation, filepath.Base(MagicImage), MagicImage) ) diff --git a/envbuilder.go b/envbuilder.go index f24dd14a..c2059bac 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -311,26 +311,45 @@ func Run(ctx context.Context, opts options.Options) error { } // In order to allow 'resuming' envbuilder, embed the binary into the image - // if it is being pushed + // if it is being pushed. + // As these files will be owned by root, it is considerate to clean up + // after we're done! + cleanupBuildContext := func() {} if opts.PushImage { - exePath, err := os.Executable() - if err != nil { - return xerrors.Errorf("get exe path: %w", err) + // Add exceptions in Kaniko's ignorelist for these magic files we add. + if err := util.AddAllowedPathToDefaultIgnoreList(opts.BinaryPath); err != nil { + return fmt.Errorf("add envbuilder binary to ignore list: %w", err) } - // Add an exception for the current running binary in kaniko ignore list - if err := util.AddAllowedPathToDefaultIgnoreList(exePath); err != nil { - return xerrors.Errorf("add exe path to ignore list: %w", err) + if err := util.AddAllowedPathToDefaultIgnoreList(constants.MagicImage); err != nil { + return fmt.Errorf("add magic image file to ignore list: %w", err) } + // Add the magic directives that embed the binary into the built image. + buildParams.DockerfileContent += constants.MagicDirectives // Copy the envbuilder binary into the build context. - buildParams.DockerfileContent += fmt.Sprintf(` -COPY --chmod=0755 %s %s -USER root -WORKDIR / -ENTRYPOINT [%q]`, exePath, exePath, exePath) - dst := filepath.Join(buildParams.BuildContext, exePath) - if err := copyFile(exePath, dst); err != nil { - return xerrors.Errorf("copy running binary to build context: %w", err) + // External callers will need to specify the path to the desired envbuilder binary. + envbuilderBinDest := filepath.Join(buildParams.BuildContext, filepath.Base(constants.MagicBinaryLocation)) + opts.Logger(log.LevelDebug, "copying envbuilder binary at %q to build context %q", opts.BinaryPath, buildParams.BuildContext) + if err := copyFile(opts.BinaryPath, envbuilderBinDest, 0o755); err != nil { + return fmt.Errorf("copy envbuilder binary to build context: %w", err) + } + + // Also touch the magic file that signifies the image has been built! + magicImageDest := filepath.Join(buildParams.BuildContext, filepath.Base(constants.MagicImage)) + opts.Logger(log.LevelDebug, "copying envbuilder binary at %q to build context %q", opts.BinaryPath, buildParams.BuildContext) + if err := touchFile(magicImageDest, 0o755); err != nil { + return fmt.Errorf("copy envbuilder binary to build context: %w", err) + } + + // Clean up after build! + cleanupBuildContext = func() { + if err := os.Remove(envbuilderBinDest); err != nil { + opts.Logger(log.LevelWarn, "failed to clean up envbuilder binary from build context: %w", err) + } + if err := os.Remove(magicImageDest); err != nil { + opts.Logger(log.LevelWarn, "failed to clean up magic image file from build context: %w", err) + } } + defer cleanupBuildContext() } // temp move of all ro mounts @@ -354,8 +373,10 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) stderrWriter, closeStderr := log.Writer(opts.Logger) defer closeStderr() build := func() (v1.Image, error) { - _, err := opts.Filesystem.Stat(constants.MagicFile) - if err == nil && opts.SkipRebuild { + defer cleanupBuildContext() + _, alreadyBuiltErr := opts.Filesystem.Stat(constants.MagicFile) + _, isImageErr := opts.Filesystem.Stat(constants.MagicImage) + if (alreadyBuiltErr == nil && opts.SkipRebuild) || isImageErr == nil { endStage := startStage("🏗️ Skipping build because of cache...") imageRef, err := devcontainer.ImageFromDockerfile(buildParams.DockerfileContent) if err != nil { @@ -381,26 +402,7 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) if err := maybeDeleteFilesystem(opts.Logger, opts.ForceSafe); err != nil { return nil, fmt.Errorf("delete filesystem: %w", err) } - /* - stdoutReader, stdoutWriter := io.Pipe() - stderrReader, stderrWriter := io.Pipe() - defer stdoutReader.Close() - defer stdoutWriter.Close() - defer stderrReader.Close() - defer stderrWriter.Close() - go func() { - scanner := bufio.NewScanner(stdoutReader) - for scanner.Scan() { - opts.Logger(log.LevelInfo, "%s", scanner.Text()) - } - }() - go func() { - scanner := bufio.NewScanner(stderrReader) - for scanner.Scan() { - opts.Logger(log.LevelInfo, "%s", scanner.Text()) - } - }() - */ + cacheTTL := time.Hour * 24 * 7 if opts.CacheTTLDays != 0 { cacheTTL = time.Hour * 24 * time.Duration(opts.CacheTTLDays) @@ -1064,23 +1066,32 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) // We expect an image built and pushed by envbuilder to have the envbuilder // binary present at a predefined path. In order to correctly replicate the // build via executor.RunCacheProbe we need to have the *exact* copy of the - // envbuilder binary available used to build the image. - exePath := opts.BinaryPath - // Add an exception for the current running binary in kaniko ignore list - if err := util.AddAllowedPathToDefaultIgnoreList(exePath); err != nil { - return nil, xerrors.Errorf("add exe path to ignore list: %w", err) - } + // envbuilder binary available used to build the image and we also need to + // add the magic directives to the Dockerfile content. + buildParams.DockerfileContent += constants.MagicDirectives + envbuilderBinDest := filepath.Join(buildParams.BuildContext, "envbuilder") + // Copy the envbuilder binary into the build context. - buildParams.DockerfileContent += fmt.Sprintf(` -COPY --chmod=0755 %s %s -USER root -WORKDIR / -ENTRYPOINT [%q]`, exePath, exePath, exePath) - dst := filepath.Join(buildParams.BuildContext, exePath) - if err := copyFile(exePath, dst); err != nil { + opts.Logger(log.LevelDebug, "copying envbuilder binary at %q to build context %q", opts.BinaryPath, buildParams.BuildContext) + if err := copyFile(opts.BinaryPath, envbuilderBinDest, 0o755); err != nil { return nil, xerrors.Errorf("copy envbuilder binary to build context: %w", err) } + // Also touch the magic file that signifies the image has been built! + magicImageDest := filepath.Join(buildParams.BuildContext, filepath.Base(constants.MagicImage)) + if err := touchFile(magicImageDest, 0o755); err != nil { + return nil, fmt.Errorf("touch magic image file in build context: %w", err) + } + defer func() { + // Clean up after we're done! + if err := os.Remove(envbuilderBinDest); err != nil { + opts.Logger(log.LevelWarn, "failed to clean up envbuilder binary from build context: %w", err) + } + if err := os.Remove(magicImageDest); err != nil { + opts.Logger(log.LevelWarn, "failed to clean up magic image file from build context: %w", err) + } + }() + stdoutWriter, closeStdout := log.Writer(opts.Logger) defer closeStdout() stderrWriter, closeStderr := log.Writer(opts.Logger) @@ -1138,8 +1149,8 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) }, SrcContext: buildParams.BuildContext, - // For cached image utilization, produce reproducible builds. - Reproducible: opts.PushImage, + // When performing a cache probe, always perform reproducible snapshots. + Reproducible: true, } endStage := startStage("🏗️ Checking for cached image...") @@ -1382,24 +1393,32 @@ func maybeDeleteFilesystem(logger log.Func, force bool) error { return util.DeleteFilesystem() } -func copyFile(src, dst string) error { +func copyFile(src, dst string, mode fs.FileMode) error { content, err := os.ReadFile(src) if err != nil { - return fmt.Errorf("read file failed: %w", err) + return fmt.Errorf("read src file failed: %w", err) } - err = os.MkdirAll(filepath.Dir(dst), 0o755) + err = os.MkdirAll(filepath.Dir(dst), mode) if err != nil { - return fmt.Errorf("mkdir all failed: %w", err) + return fmt.Errorf("create destination dir failed: %w", err) } - err = os.WriteFile(dst, content, 0o644) + err = os.WriteFile(dst, content, mode) if err != nil { - return fmt.Errorf("write file failed: %w", err) + return fmt.Errorf("write dest file failed: %w", err) } return nil } +func touchFile(dst string, mode fs.FileMode) error { + f, err := os.OpenFile(dst, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode) + if err != nil { + return xerrors.Errorf("failed to touch file: %w", err) + } + return f.Close() +} + func initDockerConfigJSON(dockerConfigBase64 string) (func() error, error) { var cleanupOnce sync.Once noop := func() error { return nil } From 372bf32e075322951287b3d0ae51ffb01be96bbf Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Sat, 3 Aug 2024 16:18:14 +0100 Subject: [PATCH 2/6] address PR comments --- constants/constants.go | 9 +++++- envbuilder.go | 65 +++++++++++++++++++++++++++--------------- 2 files changed, 50 insertions(+), 24 deletions(-) diff --git a/constants/constants.go b/constants/constants.go index 0fd98c45..fefa1394 100644 --- a/constants/constants.go +++ b/constants/constants.go @@ -44,6 +44,10 @@ var ( // from a previously built image. MagicImage = filepath.Join(MagicDir, "image") + // MagicTempDir is a directory inside the build context inside which + // we place files referenced by MagicDirectives. + MagicTempDir = ".envbuilder.tmp" + // MagicDirectives are directives automatically appended to Dockerfiles // when pushing the image. These directives allow the built image to be // 're-used'. @@ -53,5 +57,8 @@ COPY --chmod=0644 %[3]s %[4]s USER root WORKDIR / ENTRYPOINT [%[2]q] -`, filepath.Base(MagicBinaryLocation), MagicBinaryLocation, filepath.Base(MagicImage), MagicImage) +`, + ".envbuilder.tmp/envbuilder", MagicBinaryLocation, + ".envbuilder.tmp/image", MagicImage, + ) ) diff --git a/envbuilder.go b/envbuilder.go index c2059bac..9777e394 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -27,6 +27,7 @@ import ( "github.com/coder/envbuilder/constants" "github.com/coder/envbuilder/git" "github.com/coder/envbuilder/options" + "github.com/go-git/go-billy/v5" "github.com/GoogleContainerTools/kaniko/pkg/config" "github.com/GoogleContainerTools/kaniko/pkg/creds" @@ -323,30 +324,39 @@ func Run(ctx context.Context, opts options.Options) error { if err := util.AddAllowedPathToDefaultIgnoreList(constants.MagicImage); err != nil { return fmt.Errorf("add magic image file to ignore list: %w", err) } + magicTempDir := filepath.Join(buildParams.BuildContext, constants.MagicTempDir) + if err := opts.Filesystem.MkdirAll(magicTempDir, 0o755); err != nil { + return fmt.Errorf("create magic temp dir in build context: %w", err) + } // Add the magic directives that embed the binary into the built image. buildParams.DockerfileContent += constants.MagicDirectives // Copy the envbuilder binary into the build context. // External callers will need to specify the path to the desired envbuilder binary. - envbuilderBinDest := filepath.Join(buildParams.BuildContext, filepath.Base(constants.MagicBinaryLocation)) - opts.Logger(log.LevelDebug, "copying envbuilder binary at %q to build context %q", opts.BinaryPath, buildParams.BuildContext) - if err := copyFile(opts.BinaryPath, envbuilderBinDest, 0o755); err != nil { + envbuilderBinDest := filepath.Join( + magicTempDir, + filepath.Base(constants.MagicBinaryLocation), + ) + opts.Logger(log.LevelDebug, "copying envbuilder binary at %q to build context %q", opts.BinaryPath, envbuilderBinDest) + if err := copyFile(opts.Filesystem, opts.BinaryPath, envbuilderBinDest, 0o755); err != nil { return fmt.Errorf("copy envbuilder binary to build context: %w", err) } // Also touch the magic file that signifies the image has been built! - magicImageDest := filepath.Join(buildParams.BuildContext, filepath.Base(constants.MagicImage)) - opts.Logger(log.LevelDebug, "copying envbuilder binary at %q to build context %q", opts.BinaryPath, buildParams.BuildContext) - if err := touchFile(magicImageDest, 0o755); err != nil { - return fmt.Errorf("copy envbuilder binary to build context: %w", err) + magicImageDest := filepath.Join( + magicTempDir, + filepath.Base(constants.MagicImage), + ) + opts.Logger(log.LevelDebug, "touching magic image file at %q in build context %q", magicImageDest, buildParams.BuildContext) + if err := touchFile(opts.Filesystem, magicImageDest, 0o755); err != nil { + return fmt.Errorf("touch magic image file in build context: %w", err) } // Clean up after build! cleanupBuildContext = func() { - if err := os.Remove(envbuilderBinDest); err != nil { - opts.Logger(log.LevelWarn, "failed to clean up envbuilder binary from build context: %w", err) - } - if err := os.Remove(magicImageDest); err != nil { - opts.Logger(log.LevelWarn, "failed to clean up magic image file from build context: %w", err) + for _, path := range []string{magicImageDest, envbuilderBinDest, magicTempDir} { + if err := opts.Filesystem.Remove(path); err != nil { + opts.Logger(log.LevelWarn, "failed to clean up magic temp dir from build context: %w", err) + } } } defer cleanupBuildContext() @@ -1069,26 +1079,35 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) // envbuilder binary available used to build the image and we also need to // add the magic directives to the Dockerfile content. buildParams.DockerfileContent += constants.MagicDirectives - envbuilderBinDest := filepath.Join(buildParams.BuildContext, "envbuilder") + magicTempDir := filepath.Join(buildParams.BuildContext, constants.MagicTempDir) + if err := opts.Filesystem.MkdirAll(magicTempDir, 0o755); err != nil { + return nil, fmt.Errorf("create magic temp dir in build context: %w", err) + } + envbuilderBinDest := filepath.Join( + magicTempDir, + filepath.Base(constants.MagicBinaryLocation), + ) // Copy the envbuilder binary into the build context. opts.Logger(log.LevelDebug, "copying envbuilder binary at %q to build context %q", opts.BinaryPath, buildParams.BuildContext) - if err := copyFile(opts.BinaryPath, envbuilderBinDest, 0o755); err != nil { + if err := copyFile(opts.Filesystem, opts.BinaryPath, envbuilderBinDest, 0o755); err != nil { return nil, xerrors.Errorf("copy envbuilder binary to build context: %w", err) } // Also touch the magic file that signifies the image has been built! - magicImageDest := filepath.Join(buildParams.BuildContext, filepath.Base(constants.MagicImage)) - if err := touchFile(magicImageDest, 0o755); err != nil { + magicImageDest := filepath.Join( + magicTempDir, + filepath.Base(constants.MagicImage), + ) + if err := touchFile(opts.Filesystem, magicImageDest, 0o755); err != nil { return nil, fmt.Errorf("touch magic image file in build context: %w", err) } defer func() { // Clean up after we're done! - if err := os.Remove(envbuilderBinDest); err != nil { - opts.Logger(log.LevelWarn, "failed to clean up envbuilder binary from build context: %w", err) - } - if err := os.Remove(magicImageDest); err != nil { - opts.Logger(log.LevelWarn, "failed to clean up magic image file from build context: %w", err) + for _, path := range []string{magicImageDest, envbuilderBinDest, magicTempDir} { + if err := opts.Filesystem.Remove(path); err != nil { + opts.Logger(log.LevelWarn, "failed to clean up magic temp dir from build context: %w", err) + } } }() @@ -1393,7 +1412,7 @@ func maybeDeleteFilesystem(logger log.Func, force bool) error { return util.DeleteFilesystem() } -func copyFile(src, dst string, mode fs.FileMode) error { +func copyFile(fs billy.Filesystem, src, dst string, mode fs.FileMode) error { content, err := os.ReadFile(src) if err != nil { return fmt.Errorf("read src file failed: %w", err) @@ -1411,7 +1430,7 @@ func copyFile(src, dst string, mode fs.FileMode) error { return nil } -func touchFile(dst string, mode fs.FileMode) error { +func touchFile(fs billy.Filesystem, dst string, mode fs.FileMode) error { f, err := os.OpenFile(dst, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode) if err != nil { return xerrors.Errorf("failed to touch file: %w", err) From 238540db64b37f3bd412af34c2ae5fb234e40937 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Sat, 3 Aug 2024 16:37:45 +0100 Subject: [PATCH 3/6] update kaniko fork --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 270cb077..e06edc53 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.22.4 // There are a few options we need added to Kaniko! // See: https://github.com/GoogleContainerTools/kaniko/compare/main...coder:kaniko:main -replace github.com/GoogleContainerTools/kaniko => github.com/coder/kaniko v0.0.0-20240717115058-0ba2908ca4d3 +replace github.com/GoogleContainerTools/kaniko => github.com/coder/kaniko v0.0.0-20240803153527-10d1800455b9 // Required to import codersdk due to gvisor dependency. replace tailscale.com => github.com/coder/tailscale v1.1.1-0.20240702054557-aa558fbe5374 diff --git a/go.sum b/go.sum index 86887005..25bdf7fc 100644 --- a/go.sum +++ b/go.sum @@ -171,8 +171,8 @@ github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoC github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/coder/coder/v2 v2.10.1-0.20240704130443-c2d44d16a352 h1:L/EjCuZxs5tOcqqCaASj/nu65TRYEFcTt8qRQfHZXX0= github.com/coder/coder/v2 v2.10.1-0.20240704130443-c2d44d16a352/go.mod h1:P1KoQSgnKEAG6Mnd3YlGzAophty+yKA9VV48LpfNRvo= -github.com/coder/kaniko v0.0.0-20240717115058-0ba2908ca4d3 h1:Q7L6cjKfw3DIyhKIcgCJEmgxnUTBajmMDrHxXvxgBZs= -github.com/coder/kaniko v0.0.0-20240717115058-0ba2908ca4d3/go.mod h1:YMK7BlxerzLlMwihGxNWUaFoN9LXCij4P+w/8/fNlcM= +github.com/coder/kaniko v0.0.0-20240803153527-10d1800455b9 h1:d01T5YbPN1yc1mXjIXG59YcQQoT/9idvqFErjWHfsZ4= +github.com/coder/kaniko v0.0.0-20240803153527-10d1800455b9/go.mod h1:YMK7BlxerzLlMwihGxNWUaFoN9LXCij4P+w/8/fNlcM= github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0 h1:3A0ES21Ke+FxEM8CXx9n47SZOKOpgSE1bbJzlE4qPVs= github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0/go.mod h1:5UuS2Ts+nTToAMeOjNlnHFkPahrtDkmpydBen/3wgZc= github.com/coder/quartz v0.1.0 h1:cLL+0g5l7xTf6ordRnUMMiZtRE8Sq5LxpghS63vEXrQ= From 592e9ad35a789b7d711008cbe1bee1923eb48ad2 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Sat, 3 Aug 2024 19:27:05 +0100 Subject: [PATCH 4/6] fix nil ptr defer --- cmd/envbuilder/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/envbuilder/main.go b/cmd/envbuilder/main.go index 1910568e..410e0897 100644 --- a/cmd/envbuilder/main.go +++ b/cmd/envbuilder/main.go @@ -68,6 +68,7 @@ func envbuilderCmd() serpent.Command { img, err := envbuilder.RunCacheProbe(inv.Context(), o) if err != nil { o.Logger(log.LevelError, "error: %s", err) + return err } digest, err := img.Digest() if err != nil { From 4e5c0acf93075c5112498a7657b3059e0f9c4aa5 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Sat, 3 Aug 2024 19:27:49 +0100 Subject: [PATCH 5/6] cleanup in sync.Once --- envbuilder.go | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/envbuilder.go b/envbuilder.go index 9777e394..9e40d0bc 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -336,30 +336,34 @@ func Run(ctx context.Context, opts options.Options) error { magicTempDir, filepath.Base(constants.MagicBinaryLocation), ) - opts.Logger(log.LevelDebug, "copying envbuilder binary at %q to build context %q", opts.BinaryPath, envbuilderBinDest) - if err := copyFile(opts.Filesystem, opts.BinaryPath, envbuilderBinDest, 0o755); err != nil { - return fmt.Errorf("copy envbuilder binary to build context: %w", err) - } - // Also touch the magic file that signifies the image has been built! magicImageDest := filepath.Join( magicTempDir, filepath.Base(constants.MagicImage), ) - opts.Logger(log.LevelDebug, "touching magic image file at %q in build context %q", magicImageDest, buildParams.BuildContext) - if err := touchFile(opts.Filesystem, magicImageDest, 0o755); err != nil { - return fmt.Errorf("touch magic image file in build context: %w", err) - } - // Clean up after build! + var cleanupOnce sync.Once cleanupBuildContext = func() { - for _, path := range []string{magicImageDest, envbuilderBinDest, magicTempDir} { - if err := opts.Filesystem.Remove(path); err != nil { - opts.Logger(log.LevelWarn, "failed to clean up magic temp dir from build context: %w", err) + cleanupOnce.Do(func() { + for _, path := range []string{magicImageDest, envbuilderBinDest, magicTempDir} { + if err := opts.Filesystem.Remove(path); err != nil { + opts.Logger(log.LevelWarn, "failed to clean up magic temp dir from build context: %w", err) + } } - } + }) } defer cleanupBuildContext() + + opts.Logger(log.LevelDebug, "copying envbuilder binary at %q to build context %q", opts.BinaryPath, envbuilderBinDest) + if err := copyFile(opts.Filesystem, opts.BinaryPath, envbuilderBinDest, 0o755); err != nil { + return fmt.Errorf("copy envbuilder binary to build context: %w", err) + } + + opts.Logger(log.LevelDebug, "touching magic image file at %q in build context %q", magicImageDest, buildParams.BuildContext) + if err := touchFile(opts.Filesystem, magicImageDest, 0o755); err != nil { + return fmt.Errorf("touch magic image file in build context: %w", err) + } + } // temp move of all ro mounts From a07577f68c3ff9c5d921611695f67b5a91a2d745 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Sat, 3 Aug 2024 19:28:15 +0100 Subject: [PATCH 6/6] actually use fs --- envbuilder.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/envbuilder.go b/envbuilder.go index 9e40d0bc..215ccc48 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -1417,25 +1417,31 @@ func maybeDeleteFilesystem(logger log.Func, force bool) error { } func copyFile(fs billy.Filesystem, src, dst string, mode fs.FileMode) error { - content, err := os.ReadFile(src) + srcF, err := fs.Open(src) if err != nil { - return fmt.Errorf("read src file failed: %w", err) + return fmt.Errorf("open src file: %w", err) } + defer srcF.Close() - err = os.MkdirAll(filepath.Dir(dst), mode) + err = fs.MkdirAll(filepath.Dir(dst), mode) if err != nil { return fmt.Errorf("create destination dir failed: %w", err) } - err = os.WriteFile(dst, content, mode) + dstF, err := fs.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode) if err != nil { - return fmt.Errorf("write dest file failed: %w", err) + return fmt.Errorf("open dest file for writing: %w", err) + } + defer dstF.Close() + + if _, err := io.Copy(dstF, srcF); err != nil { + return fmt.Errorf("copy failed: %w", err) } return nil } func touchFile(fs billy.Filesystem, dst string, mode fs.FileMode) error { - f, err := os.OpenFile(dst, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode) + f, err := fs.OpenFile(dst, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode) if err != nil { return xerrors.Errorf("failed to touch file: %w", err) }