From 3c8066f642c9b3983a564059ef223770f8a64c4b Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 26 Jul 2024 17:31:28 +0100 Subject: [PATCH 01/10] extract RunCacheProbe function --- cmd/envbuilder/main.go | 14 ++ envbuilder.go | 416 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 429 insertions(+), 1 deletion(-) diff --git a/cmd/envbuilder/main.go b/cmd/envbuilder/main.go index fcbf26d0..1910568e 100644 --- a/cmd/envbuilder/main.go +++ b/cmd/envbuilder/main.go @@ -36,6 +36,7 @@ func envbuilderCmd() serpent.Command { Use: "envbuilder", Options: o.CLI(), Handler: func(inv *serpent.Invocation) error { + o.SetDefaults() o.Logger = log.New(os.Stderr, o.Verbose) if o.CoderAgentURL != "" { if o.CoderAgentToken == "" { @@ -63,6 +64,19 @@ func envbuilderCmd() serpent.Command { } } + if o.GetCachedImage { + img, err := envbuilder.RunCacheProbe(inv.Context(), o) + if err != nil { + o.Logger(log.LevelError, "error: %s", err) + } + digest, err := img.Digest() + if err != nil { + return fmt.Errorf("get cached image digest: %w", err) + } + _, _ = fmt.Fprintf(inv.Stdout, "ENVBUILDER_CACHED_IMAGE=%s@%s\n", o.CacheRepo, digest.String()) + return nil + } + err := envbuilder.Run(inv.Context(), o) if err != nil { o.Logger(log.LevelError, "error: %s", err) diff --git a/envbuilder.go b/envbuilder.go index 98adab03..7e57124e 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -59,7 +59,9 @@ type DockerConfig configfile.ConfigFile // Filesystem is the filesystem to use for all operations. // Defaults to the host filesystem. func Run(ctx context.Context, opts options.Options) error { - opts.SetDefaults() + if opts.GetCachedImage { + return fmt.Errorf("developer error: use RunCacheProbe instead") + } if opts.CacheRepo == "" && opts.PushImage { return fmt.Errorf("--cache-repo must be set when using --push-image") @@ -902,6 +904,418 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) return nil } +// RunCacheProbe performs a 'dry-run' build of the image and checks that +// all of the resulting layers are present in options.CacheRepo. +// Logger is the logf to use for all operations. +// Filesystem is the filesystem to use for all operations. +// Defaults to the host filesystem. +func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) { + if !opts.GetCachedImage { + return nil, fmt.Errorf("developer error: RunCacheProbe must be run with --get-cached-image") + } + if opts.CacheRepo == "" { + return nil, fmt.Errorf("--cache-repo must be set when using --get-cached-image") + } + + // Default to the shell! + // initArgs := []string{"-c", opts.InitScript} + // if opts.InitArgs != "" { + // var err error + // initArgs, err = shellquote.Split(opts.InitArgs) + // if err != nil { + // return fmt.Errorf("parse init args: %w", err) + // } + // } + + stageNumber := 0 + startStage := func(format string, args ...any) func(format string, args ...any) { + now := time.Now() + stageNumber++ + stageNum := stageNumber + opts.Logger(log.LevelInfo, "#%d: %s", stageNum, fmt.Sprintf(format, args...)) + + return func(format string, args ...any) { + opts.Logger(log.LevelInfo, "#%d: %s [%s]", stageNum, fmt.Sprintf(format, args...), time.Since(now)) + } + } + + opts.Logger(log.LevelInfo, "%s - Build development environments from repositories in a container", newColor(color.Bold).Sprintf("envbuilder")) + + var caBundle []byte + if opts.SSLCertBase64 != "" { + certPool, err := x509.SystemCertPool() + if err != nil { + return nil, xerrors.Errorf("get global system cert pool: %w", err) + } + data, err := base64.StdEncoding.DecodeString(opts.SSLCertBase64) + if err != nil { + return nil, xerrors.Errorf("base64 decode ssl cert: %w", err) + } + ok := certPool.AppendCertsFromPEM(data) + if !ok { + return nil, xerrors.Errorf("failed to append the ssl cert to the global pool: %s", data) + } + caBundle = data + } + + if opts.DockerConfigBase64 != "" { + decoded, err := base64.StdEncoding.DecodeString(opts.DockerConfigBase64) + if err != nil { + return nil, fmt.Errorf("decode docker config: %w", err) + } + var configFile DockerConfig + decoded, err = hujson.Standardize(decoded) + if err != nil { + return nil, fmt.Errorf("humanize json for docker config: %w", err) + } + err = json.Unmarshal(decoded, &configFile) + if err != nil { + return nil, fmt.Errorf("parse docker config: %w", err) + } + err = os.WriteFile(filepath.Join(constants.MagicDir, "config.json"), decoded, 0o644) + if err != nil { + return nil, fmt.Errorf("write docker config: %w", err) + } + } + + var fallbackErr error + var cloned bool + if opts.GitURL != "" { + endStage := startStage("📦 Cloning %s to %s...", + newColor(color.FgCyan).Sprintf(opts.GitURL), + newColor(color.FgCyan).Sprintf(opts.WorkspaceFolder), + ) + + reader, writer := io.Pipe() + defer reader.Close() + defer writer.Close() + go func() { + data := make([]byte, 4096) + for { + read, err := reader.Read(data) + if err != nil { + return + } + content := data[:read] + for _, line := range strings.Split(string(content), "\r") { + if line == "" { + continue + } + opts.Logger(log.LevelInfo, "#1: %s", strings.TrimSpace(line)) + } + } + }() + + cloneOpts := git.CloneRepoOptions{ + Path: opts.WorkspaceFolder, + Storage: opts.Filesystem, + Insecure: opts.Insecure, + Progress: writer, + SingleBranch: opts.GitCloneSingleBranch, + Depth: int(opts.GitCloneDepth), + CABundle: caBundle, + } + + cloneOpts.RepoAuth = git.SetupRepoAuth(&opts) + if opts.GitHTTPProxyURL != "" { + cloneOpts.ProxyOptions = transport.ProxyOptions{ + URL: opts.GitHTTPProxyURL, + } + } + cloneOpts.RepoURL = opts.GitURL + + cloned, fallbackErr = git.CloneRepo(ctx, cloneOpts) + if fallbackErr == nil { + if cloned { + endStage("📦 Cloned repository!") + } else { + endStage("📦 The repository already exists!") + } + } else { + opts.Logger(log.LevelError, "Failed to clone repository: %s", fallbackErr.Error()) + opts.Logger(log.LevelError, "Falling back to the default image...") + } + } + + defaultBuildParams := func() (*devcontainer.Compiled, error) { + dockerfile := filepath.Join(constants.MagicDir, "Dockerfile") + file, err := opts.Filesystem.OpenFile(dockerfile, os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + return nil, err + } + defer file.Close() + if opts.FallbackImage == "" { + if fallbackErr != nil { + return nil, xerrors.Errorf("%s: %w", fallbackErr.Error(), constants.ErrNoFallbackImage) + } + // We can't use errors.Join here because our tests + // don't support parsing a multiline error. + return nil, constants.ErrNoFallbackImage + } + content := "FROM " + opts.FallbackImage + _, err = file.Write([]byte(content)) + if err != nil { + return nil, err + } + return &devcontainer.Compiled{ + DockerfilePath: dockerfile, + DockerfileContent: content, + BuildContext: constants.MagicDir, + }, nil + } + + var ( + buildParams *devcontainer.Compiled + devcontainerPath string + ) + if opts.DockerfilePath == "" { + // Only look for a devcontainer if a Dockerfile wasn't specified. + // devcontainer is a standard, so it's reasonable to be the default. + var devcontainerDir string + var err error + devcontainerPath, devcontainerDir, err = findDevcontainerJSON(opts) + if err != nil { + opts.Logger(log.LevelError, "Failed to locate devcontainer.json: %s", err.Error()) + opts.Logger(log.LevelError, "Falling back to the default image...") + } else { + // We know a devcontainer exists. + // Let's parse it and use it! + file, err := opts.Filesystem.Open(devcontainerPath) + if err != nil { + return nil, fmt.Errorf("open devcontainer.json: %w", err) + } + defer file.Close() + content, err := io.ReadAll(file) + if err != nil { + return nil, fmt.Errorf("read devcontainer.json: %w", err) + } + devContainer, err := devcontainer.Parse(content) + if err == nil { + var fallbackDockerfile string + if !devContainer.HasImage() && !devContainer.HasDockerfile() { + defaultParams, err := defaultBuildParams() + if err != nil { + return nil, fmt.Errorf("no Dockerfile or image found: %w", err) + } + opts.Logger(log.LevelInfo, "No Dockerfile or image specified; falling back to the default image...") + fallbackDockerfile = defaultParams.DockerfilePath + } + buildParams, err = devContainer.Compile(opts.Filesystem, devcontainerDir, constants.MagicDir, fallbackDockerfile, opts.WorkspaceFolder, false, os.LookupEnv) + if err != nil { + return nil, fmt.Errorf("compile devcontainer.json: %w", err) + } + } else { + opts.Logger(log.LevelError, "Failed to parse devcontainer.json: %s", err.Error()) + opts.Logger(log.LevelError, "Falling back to the default image...") + } + } + } else { + // If a Dockerfile was specified, we use that. + dockerfilePath := filepath.Join(opts.WorkspaceFolder, opts.DockerfilePath) + + // If the dockerfilePath is specified and deeper than the base of WorkspaceFolder AND the BuildContextPath is + // not defined, show a warning + dockerfileDir := filepath.Dir(dockerfilePath) + if dockerfileDir != filepath.Clean(opts.WorkspaceFolder) && opts.BuildContextPath == "" { + opts.Logger(log.LevelWarn, "given dockerfile %q is below %q and no custom build context has been defined", dockerfilePath, opts.WorkspaceFolder) + opts.Logger(log.LevelWarn, "\t-> set BUILD_CONTEXT_PATH to %q to fix", dockerfileDir) + } + + dockerfile, err := opts.Filesystem.Open(dockerfilePath) + if err == nil { + content, err := io.ReadAll(dockerfile) + if err != nil { + return nil, fmt.Errorf("read Dockerfile: %w", err) + } + buildParams = &devcontainer.Compiled{ + DockerfilePath: dockerfilePath, + DockerfileContent: string(content), + BuildContext: filepath.Join(opts.WorkspaceFolder, opts.BuildContextPath), + } + } + } + + // When probing the build cache, there is no fallback! + if buildParams == nil { + return nil, fmt.Errorf("no Dockerfile or devcontainer.json found") + } + + HijackLogrus(func(entry *logrus.Entry) { + for _, line := range strings.Split(entry.Message, "\r") { + opts.Logger(log.LevelInfo, "#%d: %s", stageNumber, color.HiBlackString(line)) + } + }) + + var closeAfterBuild func() + // Allows quick testing of layer caching using a local directory! + if opts.LayerCacheDir != "" { + cfg := &configuration.Configuration{ + Storage: configuration.Storage{ + "filesystem": configuration.Parameters{ + "rootdirectory": opts.LayerCacheDir, + }, + }, + } + cfg.Log.Level = "error" + + // Spawn an in-memory registry to cache built layers... + registry := handlers.NewApp(ctx, cfg) + + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return nil, fmt.Errorf("start listener for in-memory registry: %w", err) + } + tcpAddr, ok := listener.Addr().(*net.TCPAddr) + if !ok { + return nil, fmt.Errorf("listener addr was of wrong type: %T", listener.Addr()) + } + srv := &http.Server{ + Handler: registry, + } + go func() { + err := srv.Serve(listener) + if err != nil && !errors.Is(err, http.ErrServerClosed) { + opts.Logger(log.LevelError, "Failed to serve registry: %s", err.Error()) + } + }() + closeAfterBuild = func() { + _ = srv.Close() + _ = listener.Close() + } + if opts.CacheRepo != "" { + opts.Logger(log.LevelWarn, "Overriding cache repo with local registry...") + } + opts.CacheRepo = fmt.Sprintf("localhost:%d/local/cache", tcpAddr.Port) + } + + // IgnorePaths in the Kaniko opts doesn't properly ignore paths. + // So we add them to the default ignore list. See: + // https://github.com/GoogleContainerTools/kaniko/blob/63be4990ca5a60bdf06ddc4d10aa4eca0c0bc714/cmd/executor/cmd/root.go#L136 + ignorePaths := append([]string{ + constants.MagicDir, + opts.WorkspaceFolder, + // See: https://github.com/coder/envbuilder/issues/37 + "/etc/resolv.conf", + }, opts.IgnorePaths...) + + if opts.LayerCacheDir != "" { + ignorePaths = append(ignorePaths, opts.LayerCacheDir) + } + + for _, ignorePath := range ignorePaths { + util.AddToDefaultIgnoreList(util.IgnoreListEntry{ + Path: ignorePath, + PrefixMatchOnly: false, + AllowedPaths: nil, + }) + } + + stdoutReader, stdoutWriter := io.Pipe() + stderrReader, stderrWriter := io.Pipe() + defer stdoutReader.Close() + defer stdoutWriter.Close() + defer stderrReader.Close() + defer stderrWriter.Close() + go func() { + scanner := bufio.NewScanner(stdoutReader) + for scanner.Scan() { + opts.Logger(log.LevelInfo, "%s", scanner.Text()) + } + }() + go func() { + scanner := bufio.NewScanner(stderrReader) + for scanner.Scan() { + opts.Logger(log.LevelInfo, "%s", scanner.Text()) + } + }() + cacheTTL := time.Hour * 24 * 7 + if opts.CacheTTLDays != 0 { + cacheTTL = time.Hour * 24 * time.Duration(opts.CacheTTLDays) + } + + // At this point we have all the context, we can now build! + registryMirror := []string{} + if val, ok := os.LookupEnv("KANIKO_REGISTRY_MIRROR"); ok { + registryMirror = strings.Split(val, ";") + } + var destinations []string + if opts.CacheRepo != "" { + destinations = append(destinations, opts.CacheRepo) + } + kOpts := &config.KanikoOptions{ + // Boilerplate! + CustomPlatform: platforms.Format(platforms.Normalize(platforms.DefaultSpec())), + SnapshotMode: "redo", + RunV2: true, + RunStdout: stdoutWriter, + RunStderr: stderrWriter, + Destinations: destinations, + NoPush: !opts.PushImage || len(destinations) == 0, + CacheRunLayers: true, + CacheCopyLayers: true, + CompressedCaching: true, + Compression: config.ZStd, + // Maps to "default" level, ~100-300 MB/sec according to + // benchmarks in klauspost/compress README + // https://github.com/klauspost/compress/blob/67a538e2b4df11f8ec7139388838a13bce84b5d5/zstd/encoder_options.go#L188 + CompressionLevel: 3, + CacheOptions: config.CacheOptions{ + // Cache for a week by default! + CacheTTL: cacheTTL, + CacheDir: opts.BaseImageCacheDir, + }, + ForceUnpack: true, + BuildArgs: buildParams.BuildArgs, + CacheRepo: opts.CacheRepo, + Cache: opts.CacheRepo != "" || opts.BaseImageCacheDir != "", + DockerfilePath: buildParams.DockerfilePath, + DockerfileContent: buildParams.DockerfileContent, + RegistryOptions: config.RegistryOptions{ + Insecure: opts.Insecure, + InsecurePull: opts.Insecure, + SkipTLSVerify: opts.Insecure, + // Enables registry mirror features in Kaniko, see more in link below + // https://github.com/GoogleContainerTools/kaniko?tab=readme-ov-file#flag---registry-mirror + // Related to PR #114 + // https://github.com/coder/envbuilder/pull/114 + RegistryMirrors: registryMirror, + }, + SrcContext: buildParams.BuildContext, + + // For cached image utilization, produce reproducible builds. + Reproducible: opts.PushImage, + } + + endStage := startStage("🏗️ Checking for cached image...") + image, err := executor.DoCacheProbe(kOpts) + if err != nil { + return nil, xerrors.Errorf("get cached image: %w", err) + } + endStage("🏗️ Found cached image!") + + if closeAfterBuild != nil { + closeAfterBuild() + } + + // Sanitize the environment of any opts! + options.UnsetEnv() + + // Remove the Docker config secret file! + if opts.DockerConfigBase64 != "" { + c := filepath.Join(constants.MagicDir, "config.json") + err = os.Remove(c) + if err != nil { + if !errors.Is(err, fs.ErrNotExist) { + return nil, fmt.Errorf("remove docker config: %w", err) + } else { + fmt.Fprintln(os.Stderr, "failed to remove the Docker config secret file: %w", c) + } + } + } + + return image, nil +} + type userInfo struct { uid int gid int From eeb25c17ac40467b6575515e9e4a53d85b22a079 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 30 Jul 2024 16:58:01 +0100 Subject: [PATCH 02/10] extract CABundle and docker config JSON init functions --- envbuilder.go | 173 +++++++++++++++++++++++--------------------------- 1 file changed, 78 insertions(+), 95 deletions(-) diff --git a/envbuilder.go b/envbuilder.go index 7e57124e..4894a285 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -21,6 +21,7 @@ import ( "sort" "strconv" "strings" + "sync" "syscall" "time" @@ -91,41 +92,16 @@ func Run(ctx context.Context, opts options.Options) error { opts.Logger(log.LevelInfo, "%s - Build development environments from repositories in a container", newColor(color.Bold).Sprintf("envbuilder")) var caBundle []byte - if opts.SSLCertBase64 != "" { - certPool, err := x509.SystemCertPool() - if err != nil { - return xerrors.Errorf("get global system cert pool: %w", err) - } - data, err := base64.StdEncoding.DecodeString(opts.SSLCertBase64) - if err != nil { - return xerrors.Errorf("base64 decode ssl cert: %w", err) - } - ok := certPool.AppendCertsFromPEM(data) - if !ok { - return xerrors.Errorf("failed to append the ssl cert to the global pool: %s", data) - } - caBundle = data + caBundle, err := initCABundle(opts.SSLCertBase64) + if err != nil { + return err } - if opts.DockerConfigBase64 != "" { - decoded, err := base64.StdEncoding.DecodeString(opts.DockerConfigBase64) - if err != nil { - return fmt.Errorf("decode docker config: %w", err) - } - var configFile DockerConfig - decoded, err = hujson.Standardize(decoded) - if err != nil { - return fmt.Errorf("humanize json for docker config: %w", err) - } - err = json.Unmarshal(decoded, &configFile) - if err != nil { - return fmt.Errorf("parse docker config: %w", err) - } - err = os.WriteFile(filepath.Join(constants.MagicDir, "config.json"), decoded, 0o644) - if err != nil { - return fmt.Errorf("write docker config: %w", err) - } + cleanupDockerConfigJSON, err := initDockerConfigJSON(opts.DockerConfigBase64) + if err != nil { + return err } + defer func() { _ = cleanupDockerConfigJSON() }() // best effort var fallbackErr error var cloned bool @@ -644,16 +620,8 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) options.UnsetEnv() // Remove the Docker config secret file! - if opts.DockerConfigBase64 != "" { - c := filepath.Join(constants.MagicDir, "config.json") - err = os.Remove(c) - if err != nil { - if !errors.Is(err, fs.ErrNotExist) { - return fmt.Errorf("remove docker config: %w", err) - } else { - fmt.Fprintln(os.Stderr, "failed to remove the Docker config secret file: %w", c) - } - } + if err := cleanupDockerConfigJSON(); err != nil { + return err } environ, err := os.ReadFile("/etc/environment") @@ -917,16 +885,6 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) return nil, fmt.Errorf("--cache-repo must be set when using --get-cached-image") } - // Default to the shell! - // initArgs := []string{"-c", opts.InitScript} - // if opts.InitArgs != "" { - // var err error - // initArgs, err = shellquote.Split(opts.InitArgs) - // if err != nil { - // return fmt.Errorf("parse init args: %w", err) - // } - // } - stageNumber := 0 startStage := func(format string, args ...any) func(format string, args ...any) { now := time.Now() @@ -941,42 +899,16 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) opts.Logger(log.LevelInfo, "%s - Build development environments from repositories in a container", newColor(color.Bold).Sprintf("envbuilder")) - var caBundle []byte - if opts.SSLCertBase64 != "" { - certPool, err := x509.SystemCertPool() - if err != nil { - return nil, xerrors.Errorf("get global system cert pool: %w", err) - } - data, err := base64.StdEncoding.DecodeString(opts.SSLCertBase64) - if err != nil { - return nil, xerrors.Errorf("base64 decode ssl cert: %w", err) - } - ok := certPool.AppendCertsFromPEM(data) - if !ok { - return nil, xerrors.Errorf("failed to append the ssl cert to the global pool: %s", data) - } - caBundle = data + caBundle, err := initCABundle(opts.SSLCertBase64) + if err != nil { + return nil, err } - if opts.DockerConfigBase64 != "" { - decoded, err := base64.StdEncoding.DecodeString(opts.DockerConfigBase64) - if err != nil { - return nil, fmt.Errorf("decode docker config: %w", err) - } - var configFile DockerConfig - decoded, err = hujson.Standardize(decoded) - if err != nil { - return nil, fmt.Errorf("humanize json for docker config: %w", err) - } - err = json.Unmarshal(decoded, &configFile) - if err != nil { - return nil, fmt.Errorf("parse docker config: %w", err) - } - err = os.WriteFile(filepath.Join(constants.MagicDir, "config.json"), decoded, 0o644) - if err != nil { - return nil, fmt.Errorf("write docker config: %w", err) - } + cleanupDockerConfigJSON, err := initDockerConfigJSON(opts.DockerConfigBase64) + if err != nil { + return nil, err } + defer func() { _ = cleanupDockerConfigJSON() }() // best effort var fallbackErr error var cloned bool @@ -1301,16 +1233,8 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) options.UnsetEnv() // Remove the Docker config secret file! - if opts.DockerConfigBase64 != "" { - c := filepath.Join(constants.MagicDir, "config.json") - err = os.Remove(c) - if err != nil { - if !errors.Is(err, fs.ErrNotExist) { - return nil, fmt.Errorf("remove docker config: %w", err) - } else { - fmt.Fprintln(os.Stderr, "failed to remove the Docker config secret file: %w", c) - } - } + if err := cleanupDockerConfigJSON(); err != nil { + return nil, err } return image, nil @@ -1551,3 +1475,62 @@ func copyFile(src, dst string) error { } return nil } + +func initCABundle(sslCertBase64 string) ([]byte, error) { + if sslCertBase64 == "" { + return []byte{}, nil + } + certPool, err := x509.SystemCertPool() + if err != nil { + return nil, xerrors.Errorf("get global system cert pool: %w", err) + } + data, err := base64.StdEncoding.DecodeString(sslCertBase64) + if err != nil { + return nil, xerrors.Errorf("base64 decode ssl cert: %w", err) + } + ok := certPool.AppendCertsFromPEM(data) + if !ok { + return nil, xerrors.Errorf("failed to append the ssl cert to the global pool: %s", data) + } + return data, nil +} + +func initDockerConfigJSON(dockerConfigBase64 string) (func() error, error) { + var cleanupOnce sync.Once + noop := func() error { return nil } + if dockerConfigBase64 == "" { + return noop, nil + } + cfgPath := filepath.Join(constants.MagicDir, "config.json") + decoded, err := base64.StdEncoding.DecodeString(dockerConfigBase64) + if err != nil { + return noop, fmt.Errorf("decode docker config: %w", err) + } + var configFile DockerConfig + decoded, err = hujson.Standardize(decoded) + if err != nil { + return noop, fmt.Errorf("humanize json for docker config: %w", err) + } + err = json.Unmarshal(decoded, &configFile) + if err != nil { + return noop, fmt.Errorf("parse docker config: %w", err) + } + err = os.WriteFile(cfgPath, decoded, 0o644) + if err != nil { + return noop, fmt.Errorf("write docker config: %w", err) + } + cleanup := func() error { + var cleanupErr error + cleanupOnce.Do(func() { + // Remove the Docker config secret file! + if cleanupErr = os.Remove(cfgPath); err != nil { + if !errors.Is(err, fs.ErrNotExist) { + cleanupErr = fmt.Errorf("remove docker config: %w", cleanupErr) + } + _, _ = fmt.Fprintln(os.Stderr, "failed to remove the Docker config secret file: %w", cfgPath) + } + }) + return cleanupErr + } + return cleanup, err +} From 83496026a8607716292701e0224205def18a92ce Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 30 Jul 2024 17:54:52 +0100 Subject: [PATCH 03/10] extract local cache registry func --- envbuilder.go | 139 ++++++++++++++++++++++---------------------------- 1 file changed, 60 insertions(+), 79 deletions(-) diff --git a/envbuilder.go b/envbuilder.go index 4894a285..30941959 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -279,46 +279,16 @@ func Run(ctx context.Context, opts options.Options) error { } }) - var closeAfterBuild func() - // Allows quick testing of layer caching using a local directory! if opts.LayerCacheDir != "" { - cfg := &configuration.Configuration{ - Storage: configuration.Storage{ - "filesystem": configuration.Parameters{ - "rootdirectory": opts.LayerCacheDir, - }, - }, + if opts.CacheRepo != "" { + opts.Logger(log.LevelWarn, "Overriding cache repo with local registry...") } - cfg.Log.Level = "error" - - // Spawn an in-memory registry to cache built layers... - registry := handlers.NewApp(ctx, cfg) - - listener, err := net.Listen("tcp", "127.0.0.1:0") + localRegistry, closeLocalRegistry, err := serveLocalRegistry(ctx, opts.Logger, opts.LayerCacheDir) if err != nil { return err } - tcpAddr, ok := listener.Addr().(*net.TCPAddr) - if !ok { - return fmt.Errorf("listener addr was of wrong type: %T", listener.Addr()) - } - srv := &http.Server{ - Handler: registry, - } - go func() { - err := srv.Serve(listener) - if err != nil && !errors.Is(err, http.ErrServerClosed) { - opts.Logger(log.LevelError, "Failed to serve registry: %s", err.Error()) - } - }() - closeAfterBuild = func() { - _ = srv.Close() - _ = listener.Close() - } - if opts.CacheRepo != "" { - opts.Logger(log.LevelWarn, "Overriding cache repo with local registry...") - } - opts.CacheRepo = fmt.Sprintf("localhost:%d/local/cache", tcpAddr.Port) + defer closeLocalRegistry() + opts.CacheRepo = localRegistry } // IgnorePaths in the Kaniko opts doesn't properly ignore paths. @@ -556,10 +526,6 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) return fmt.Errorf("build with kaniko: %w", err) } - if closeAfterBuild != nil { - closeAfterBuild() - } - if err := restoreMounts(); err != nil { return fmt.Errorf("restore mounts: %w", err) } @@ -1078,46 +1044,16 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) } }) - var closeAfterBuild func() - // Allows quick testing of layer caching using a local directory! if opts.LayerCacheDir != "" { - cfg := &configuration.Configuration{ - Storage: configuration.Storage{ - "filesystem": configuration.Parameters{ - "rootdirectory": opts.LayerCacheDir, - }, - }, - } - cfg.Log.Level = "error" - - // Spawn an in-memory registry to cache built layers... - registry := handlers.NewApp(ctx, cfg) - - listener, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - return nil, fmt.Errorf("start listener for in-memory registry: %w", err) - } - tcpAddr, ok := listener.Addr().(*net.TCPAddr) - if !ok { - return nil, fmt.Errorf("listener addr was of wrong type: %T", listener.Addr()) - } - srv := &http.Server{ - Handler: registry, - } - go func() { - err := srv.Serve(listener) - if err != nil && !errors.Is(err, http.ErrServerClosed) { - opts.Logger(log.LevelError, "Failed to serve registry: %s", err.Error()) - } - }() - closeAfterBuild = func() { - _ = srv.Close() - _ = listener.Close() - } if opts.CacheRepo != "" { opts.Logger(log.LevelWarn, "Overriding cache repo with local registry...") } - opts.CacheRepo = fmt.Sprintf("localhost:%d/local/cache", tcpAddr.Port) + localRegistry, closeLocalRegistry, err := serveLocalRegistry(ctx, opts.Logger, opts.LayerCacheDir) + if err != nil { + return nil, err + } + defer closeLocalRegistry() + opts.CacheRepo = localRegistry } // IgnorePaths in the Kaniko opts doesn't properly ignore paths. @@ -1225,10 +1161,6 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) } endStage("🏗️ Found cached image!") - if closeAfterBuild != nil { - closeAfterBuild() - } - // Sanitize the environment of any opts! options.UnsetEnv() @@ -1534,3 +1466,52 @@ func initDockerConfigJSON(dockerConfigBase64 string) (func() error, error) { } return cleanup, err } + +// Allows quick testing of layer caching using a local directory! +func serveLocalRegistry(ctx context.Context, logf log.Func, layerCacheDir string) (string, func(), error) { + noop := func() {} + if layerCacheDir == "" { + return "", noop, nil + } + cfg := &configuration.Configuration{ + Storage: configuration.Storage{ + "filesystem": configuration.Parameters{ + "rootdirectory": layerCacheDir, + }, + }, + } + cfg.Log.Level = "error" + + // Spawn an in-memory registry to cache built layers... + registry := handlers.NewApp(ctx, cfg) + + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return "", nil, fmt.Errorf("start listener for in-memory registry: %w", err) + } + tcpAddr, ok := listener.Addr().(*net.TCPAddr) + if !ok { + return "", noop, fmt.Errorf("listener addr was of wrong type: %T", listener.Addr()) + } + srv := &http.Server{ + Handler: registry, + } + done := make(chan struct{}) + go func() { + defer close(done) + err := srv.Serve(listener) + if err != nil && !errors.Is(err, http.ErrServerClosed) { + logf(log.LevelError, "Failed to serve registry: %s", err.Error()) + } + }() + var closeOnce sync.Once + closer := func() { + closeOnce.Do(func() { + _ = srv.Close() + _ = listener.Close() + <-done + }) + } + addr := fmt.Sprintf("localhost:%d/local/cache", tcpAddr.Port) + return addr, closer, nil +} From 28aaec44a16203af7097960927282de8f515782e Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 31 Jul 2024 09:56:31 +0100 Subject: [PATCH 04/10] rm outdated comments --- envbuilder.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/envbuilder.go b/envbuilder.go index 30941959..47af8e52 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -431,7 +431,6 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) // https://github.com/klauspost/compress/blob/67a538e2b4df11f8ec7139388838a13bce84b5d5/zstd/encoder_options.go#L188 CompressionLevel: 3, CacheOptions: config.CacheOptions{ - // Cache for a week by default! CacheTTL: cacheTTL, CacheDir: opts.BaseImageCacheDir, }, @@ -840,9 +839,6 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) // RunCacheProbe performs a 'dry-run' build of the image and checks that // all of the resulting layers are present in options.CacheRepo. -// Logger is the logf to use for all operations. -// Filesystem is the filesystem to use for all operations. -// Defaults to the host filesystem. func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) { if !opts.GetCachedImage { return nil, fmt.Errorf("developer error: RunCacheProbe must be run with --get-cached-image") @@ -1128,7 +1124,6 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) // https://github.com/klauspost/compress/blob/67a538e2b4df11f8ec7139388838a13bce84b5d5/zstd/encoder_options.go#L188 CompressionLevel: 3, CacheOptions: config.CacheOptions{ - // Cache for a week by default! CacheTTL: cacheTTL, CacheDir: opts.BaseImageCacheDir, }, From 2d4b8fd4aa18dbc708d8428fbe2a26d26e815b9f Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 31 Jul 2024 09:56:50 +0100 Subject: [PATCH 05/10] xerrors -> fmt --- envbuilder.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/envbuilder.go b/envbuilder.go index 47af8e52..94f3418c 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -940,7 +940,7 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) defer file.Close() if opts.FallbackImage == "" { if fallbackErr != nil { - return nil, xerrors.Errorf("%s: %w", fallbackErr.Error(), constants.ErrNoFallbackImage) + return nil, fmt.Errorf("%s: %w", fallbackErr.Error(), constants.ErrNoFallbackImage) } // We can't use errors.Join here because our tests // don't support parsing a multiline error. @@ -1152,7 +1152,7 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) endStage := startStage("🏗️ Checking for cached image...") image, err := executor.DoCacheProbe(kOpts) if err != nil { - return nil, xerrors.Errorf("get cached image: %w", err) + return nil, fmt.Errorf("get cached image: %w", err) } endStage("🏗️ Found cached image!") @@ -1388,17 +1388,17 @@ func maybeDeleteFilesystem(logger log.Func, force bool) error { func copyFile(src, dst string) error { content, err := os.ReadFile(src) if err != nil { - return xerrors.Errorf("read file failed: %w", err) + return fmt.Errorf("read file failed: %w", err) } err = os.MkdirAll(filepath.Dir(dst), 0o755) if err != nil { - return xerrors.Errorf("mkdir all failed: %w", err) + return fmt.Errorf("mkdir all failed: %w", err) } err = os.WriteFile(dst, content, 0o644) if err != nil { - return xerrors.Errorf("write file failed: %w", err) + return fmt.Errorf("write file failed: %w", err) } return nil } @@ -1409,15 +1409,15 @@ func initCABundle(sslCertBase64 string) ([]byte, error) { } certPool, err := x509.SystemCertPool() if err != nil { - return nil, xerrors.Errorf("get global system cert pool: %w", err) + return nil, fmt.Errorf("get global system cert pool: %w", err) } data, err := base64.StdEncoding.DecodeString(sslCertBase64) if err != nil { - return nil, xerrors.Errorf("base64 decode ssl cert: %w", err) + return nil, fmt.Errorf("base64 decode ssl cert: %w", err) } ok := certPool.AppendCertsFromPEM(data) if !ok { - return nil, xerrors.Errorf("failed to append the ssl cert to the global pool: %s", data) + return nil, fmt.Errorf("failed to append the ssl cert to the global pool: %s", data) } return data, nil } From 8d39505c4a04f21f7e5abe30e0e70c21e47be875 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 31 Jul 2024 09:57:19 +0100 Subject: [PATCH 06/10] rm DoCacheProbe invocation in Run --- envbuilder.go | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/envbuilder.go b/envbuilder.go index 94f3418c..055760b0 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -456,21 +456,6 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) Reproducible: opts.PushImage, } - if opts.GetCachedImage { - endStage := startStage("🏗️ Checking for cached image...") - image, err := executor.DoCacheProbe(kOpts) - if err != nil { - return nil, xerrors.Errorf("get cached image: %w", err) - } - digest, err := image.Digest() - if err != nil { - return nil, xerrors.Errorf("get cached image digest: %w", err) - } - endStage("🏗️ Found cached image!") - _, _ = fmt.Fprintf(os.Stdout, "ENVBUILDER_CACHED_IMAGE=%s@%s\n", kOpts.CacheRepo, digest.String()) - os.Exit(0) - } - endStage := startStage("🏗️ Building image...") image, err := executor.DoBuild(kOpts) if err != nil { From a7b4d39309309d698318d893a895a9b513388e96 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 31 Jul 2024 10:15:23 +0100 Subject: [PATCH 07/10] rm empty file --- envbuilder_test.go | 1 - 1 file changed, 1 deletion(-) delete mode 100644 envbuilder_test.go diff --git a/envbuilder_test.go b/envbuilder_test.go deleted file mode 100644 index aa4205c7..00000000 --- a/envbuilder_test.go +++ /dev/null @@ -1 +0,0 @@ -package envbuilder_test From ddfa66f6186a47d40cb535c63bcb46c797109c65 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 31 Jul 2024 12:22:59 +0100 Subject: [PATCH 08/10] extract log.Writer func that allows stopping --- envbuilder.go | 65 ++++++++++++++++++++++----------------------------- log/log.go | 29 +++++++++++++++++++++++ 2 files changed, 57 insertions(+), 37 deletions(-) diff --git a/envbuilder.go b/envbuilder.go index 055760b0..62b9bda7 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -352,6 +352,10 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) } skippedRebuild := false + stdoutWriter, closeStdout := log.Writer(opts.Logger) + defer closeStdout() + stderrWriter, closeStderr := log.Writer(opts.Logger) + defer closeStderr() build := func() (v1.Image, error) { _, err := opts.Filesystem.Stat(constants.MagicFile) if err == nil && opts.SkipRebuild { @@ -380,25 +384,26 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) if err := maybeDeleteFilesystem(opts.Logger, opts.ForceSafe); err != nil { return nil, fmt.Errorf("delete filesystem: %w", err) } - - stdoutReader, stdoutWriter := io.Pipe() - stderrReader, stderrWriter := io.Pipe() - defer stdoutReader.Close() - defer stdoutWriter.Close() - defer stderrReader.Close() - defer stderrWriter.Close() - go func() { - scanner := bufio.NewScanner(stdoutReader) - for scanner.Scan() { - opts.Logger(log.LevelInfo, "%s", scanner.Text()) - } - }() - go func() { - scanner := bufio.NewScanner(stderrReader) - for scanner.Scan() { - opts.Logger(log.LevelInfo, "%s", scanner.Text()) - } - }() + /* + stdoutReader, stdoutWriter := io.Pipe() + stderrReader, stderrWriter := io.Pipe() + defer stdoutReader.Close() + defer stdoutWriter.Close() + defer stderrReader.Close() + defer stderrWriter.Close() + go func() { + scanner := bufio.NewScanner(stdoutReader) + for scanner.Scan() { + opts.Logger(log.LevelInfo, "%s", scanner.Text()) + } + }() + go func() { + scanner := bufio.NewScanner(stderrReader) + for scanner.Scan() { + opts.Logger(log.LevelInfo, "%s", scanner.Text()) + } + }() + */ cacheTTL := time.Hour * 24 * 7 if opts.CacheTTLDays != 0 { cacheTTL = time.Hour * 24 * time.Duration(opts.CacheTTLDays) @@ -1059,24 +1064,10 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) }) } - stdoutReader, stdoutWriter := io.Pipe() - stderrReader, stderrWriter := io.Pipe() - defer stdoutReader.Close() - defer stdoutWriter.Close() - defer stderrReader.Close() - defer stderrWriter.Close() - go func() { - scanner := bufio.NewScanner(stdoutReader) - for scanner.Scan() { - opts.Logger(log.LevelInfo, "%s", scanner.Text()) - } - }() - go func() { - scanner := bufio.NewScanner(stderrReader) - for scanner.Scan() { - opts.Logger(log.LevelInfo, "%s", scanner.Text()) - } - }() + stdoutWriter, closeStdout := log.Writer(opts.Logger) + defer closeStdout() + stderrWriter, closeStderr := log.Writer(opts.Logger) + defer closeStderr() cacheTTL := time.Hour * 24 * 7 if opts.CacheTTLDays != 0 { cacheTTL = time.Hour * 24 * time.Duration(opts.CacheTTLDays) diff --git a/log/log.go b/log/log.go index da308266..8519d6b0 100644 --- a/log/log.go +++ b/log/log.go @@ -1,6 +1,7 @@ package log import ( + "bufio" "fmt" "io" "strings" @@ -45,3 +46,31 @@ func Wrap(fs ...Func) Func { } } } + +// Writer returns an io.Writer that logs all writes in a separate goroutine. +// It is the responsibility of the caller to call the returned +// function to stop the goroutine. +func Writer(logf Func) (io.Writer, func()) { + pipeReader, pipeWriter := io.Pipe() + doneCh := make(chan struct{}) + go func() { + defer pipeWriter.Close() + defer pipeReader.Close() + scanner := bufio.NewScanner(pipeReader) + for { + select { + case <-doneCh: + return + default: + if !scanner.Scan() { + return + } + logf(LevelInfo, "%s", scanner.Text()) + } + } + }() + closer := func() { + close(doneCh) + } + return pipeWriter, closer +} From 499bf1700526386754549511229512662396d44b Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 31 Jul 2024 12:32:40 +0100 Subject: [PATCH 09/10] log errors cleaning up docker config json in defer --- envbuilder.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/envbuilder.go b/envbuilder.go index 62b9bda7..767d0ff6 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -101,7 +101,11 @@ func Run(ctx context.Context, opts options.Options) error { if err != nil { return err } - defer func() { _ = cleanupDockerConfigJSON() }() // best effort + defer func() { + if err := cleanupDockerConfigJSON(); err != nil { + opts.Logger(log.LevelError, "failed to cleanup docker config JSON: %w", err) + } + }() // best effort var fallbackErr error var cloned bool @@ -860,7 +864,11 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) if err != nil { return nil, err } - defer func() { _ = cleanupDockerConfigJSON() }() // best effort + defer func() { + if err := cleanupDockerConfigJSON(); err != nil { + opts.Logger(log.LevelError, "failed to cleanup docker config JSON: %w", err) + } + }() // best effort var fallbackErr error var cloned bool @@ -1430,7 +1438,7 @@ func initDockerConfigJSON(dockerConfigBase64 string) (func() error, error) { if !errors.Is(err, fs.ErrNotExist) { cleanupErr = fmt.Errorf("remove docker config: %w", cleanupErr) } - _, _ = fmt.Fprintln(os.Stderr, "failed to remove the Docker config secret file: %w", cfgPath) + _, _ = fmt.Fprintf(os.Stderr, "failed to remove the Docker config secret file: %s\n", cleanupErr) } }) return cleanupErr From e15ecbcd0da53358932f22ada0b5de0e33688bbf Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 31 Jul 2024 12:38:12 +0100 Subject: [PATCH 10/10] defer options.UnsetEnv() --- envbuilder.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/envbuilder.go b/envbuilder.go index 767d0ff6..dc2ead8e 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -60,6 +60,7 @@ type DockerConfig configfile.ConfigFile // Filesystem is the filesystem to use for all operations. // Defaults to the host filesystem. func Run(ctx context.Context, opts options.Options) error { + defer options.UnsetEnv() if opts.GetCachedImage { return fmt.Errorf("developer error: use RunCacheProbe instead") } @@ -834,6 +835,7 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) // RunCacheProbe performs a 'dry-run' build of the image and checks that // all of the resulting layers are present in options.CacheRepo. func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) { + defer options.UnsetEnv() if !opts.GetCachedImage { return nil, fmt.Errorf("developer error: RunCacheProbe must be run with --get-cached-image") }