From 4843bdc618ede5d127662cd489959cbd1c41d630 Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Mon, 23 Sep 2024 11:45:12 +0000
Subject: [PATCH 1/4] fix: cached image start and caching of features

Fixes #334
---
 envbuilder.go                   | 749 +++++++++++++++++---------------
 integration/integration_test.go | 449 +++++++++----------
 internal/magicdir/magicdir.go   |   5 +
 3 files changed, 616 insertions(+), 587 deletions(-)

diff --git a/envbuilder.go b/envbuilder.go
index 3df35622..5ee564aa 100644
--- a/envbuilder.go
+++ b/envbuilder.go
@@ -276,407 +276,421 @@ func Run(ctx context.Context, opts options.Options) error {
 		}
 	}
 
-	if buildParams == nil {
-		// If there isn't a devcontainer.json file in the repository,
-		// we fallback to whatever the `DefaultImage` is.
-		var err error
-		buildParams, err = defaultBuildParams()
-		if err != nil {
-			return fmt.Errorf("no Dockerfile or devcontainer.json found: %w", err)
+	var (
+		username       string
+		skippedRebuild bool
+	)
+	if _, err := os.Stat(magicDir.Image()); errors.Is(err, fs.ErrNotExist) {
+		if buildParams == nil {
+			// If there isn't a devcontainer.json file in the repository,
+			// we fallback to whatever the `DefaultImage` is.
+			var err error
+			buildParams, err = defaultBuildParams()
+			if err != nil {
+				return fmt.Errorf("no Dockerfile or devcontainer.json found: %w", err)
+			}
 		}
-	}
 
-	lvl := log.LevelInfo
-	if opts.Verbose {
-		lvl = log.LevelDebug
-	}
-	log.HijackLogrus(lvl, func(entry *logrus.Entry) {
-		for _, line := range strings.Split(entry.Message, "\r") {
-			opts.Logger(log.FromLogrus(entry.Level), "#%d: %s", stageNumber, color.HiBlackString(line))
+		lvl := log.LevelInfo
+		if opts.Verbose {
+			lvl = log.LevelDebug
 		}
-	})
+		log.HijackLogrus(lvl, func(entry *logrus.Entry) {
+			for _, line := range strings.Split(entry.Message, "\r") {
+				opts.Logger(log.FromLogrus(entry.Level), "#%d: %s", stageNumber, color.HiBlackString(line))
+			}
+		})
 
-	if opts.LayerCacheDir != "" {
-		if opts.CacheRepo != "" {
-			opts.Logger(log.LevelWarn, "Overriding cache repo with local registry...")
-		}
-		localRegistry, closeLocalRegistry, err := serveLocalRegistry(ctx, opts.Logger, opts.LayerCacheDir)
-		if err != nil {
-			return err
+		if opts.LayerCacheDir != "" {
+			if opts.CacheRepo != "" {
+				opts.Logger(log.LevelWarn, "Overriding cache repo with local registry...")
+			}
+			localRegistry, closeLocalRegistry, err := serveLocalRegistry(ctx, opts.Logger, opts.LayerCacheDir)
+			if err != nil {
+				return err
+			}
+			defer closeLocalRegistry()
+			opts.CacheRepo = localRegistry
 		}
-		defer closeLocalRegistry()
-		opts.CacheRepo = localRegistry
-	}
 
-	// IgnorePaths in the Kaniko opts doesn't properly ignore paths.
-	// So we add them to the default ignore list. See:
-	// https://github.com/GoogleContainerTools/kaniko/blob/63be4990ca5a60bdf06ddc4d10aa4eca0c0bc714/cmd/executor/cmd/root.go#L136
-	ignorePaths := append([]string{
-		magicDir.Path(),
-		opts.WorkspaceFolder,
-		// See: https://github.com/coder/envbuilder/issues/37
-		"/etc/resolv.conf",
-	}, opts.IgnorePaths...)
-
-	if opts.LayerCacheDir != "" {
-		ignorePaths = append(ignorePaths, opts.LayerCacheDir)
-	}
+		// IgnorePaths in the Kaniko opts doesn't properly ignore paths.
+		// So we add them to the default ignore list. See:
+		// https://github.com/GoogleContainerTools/kaniko/blob/63be4990ca5a60bdf06ddc4d10aa4eca0c0bc714/cmd/executor/cmd/root.go#L136
+		ignorePaths := append([]string{
+			magicDir.Path(),
+			opts.WorkspaceFolder,
+			// See: https://github.com/coder/envbuilder/issues/37
+			"/etc/resolv.conf",
+		}, opts.IgnorePaths...)
 
-	for _, ignorePath := range ignorePaths {
-		util.AddToDefaultIgnoreList(util.IgnoreListEntry{
-			Path:            ignorePath,
-			PrefixMatchOnly: false,
-			AllowedPaths:    nil,
-		})
-	}
+		if opts.LayerCacheDir != "" {
+			ignorePaths = append(ignorePaths, opts.LayerCacheDir)
+		}
 
-	// In order to allow 'resuming' envbuilder, embed the binary into the image
-	// if it is being pushed.
-	// As these files will be owned by root, it is considerate to clean up
-	// after we're done!
-	cleanupBuildContext := func() {}
-	if opts.PushImage {
-		// Add exceptions in Kaniko's ignorelist for these magic files we add.
-		if err := util.AddAllowedPathToDefaultIgnoreList(opts.BinaryPath); err != nil {
-			return fmt.Errorf("add envbuilder binary to ignore list: %w", err)
-		}
-		if err := util.AddAllowedPathToDefaultIgnoreList(magicDir.Image()); err != nil {
-			return fmt.Errorf("add magic image file to ignore list: %w", err)
-		}
-		magicTempDir := magicdir.At(buildParams.BuildContext, magicdir.TempDir)
-		if err := opts.Filesystem.MkdirAll(magicTempDir.Path(), 0o755); err != nil {
-			return fmt.Errorf("create magic temp dir in build context: %w", err)
-		}
-		// Add the magic directives that embed the binary into the built image.
-		buildParams.DockerfileContent += magicdir.Directives
-		// Copy the envbuilder binary into the build context.
-		// External callers will need to specify the path to the desired envbuilder binary.
-		envbuilderBinDest := filepath.Join(magicTempDir.Path(), "envbuilder")
-		// Also touch the magic file that signifies the image has been built!
-		magicImageDest := magicTempDir.Image()
-		// Clean up after build!
-		var cleanupOnce sync.Once
-		cleanupBuildContext = func() {
-			cleanupOnce.Do(func() {
-				for _, path := range []string{magicImageDest, envbuilderBinDest, magicTempDir.Path()} {
-					if err := opts.Filesystem.Remove(path); err != nil {
-						opts.Logger(log.LevelWarn, "failed to clean up magic temp dir from build context: %w", err)
-					}
-				}
+		for _, ignorePath := range ignorePaths {
+			util.AddToDefaultIgnoreList(util.IgnoreListEntry{
+				Path:            ignorePath,
+				PrefixMatchOnly: false,
+				AllowedPaths:    nil,
 			})
 		}
-		defer cleanupBuildContext()
 
-		opts.Logger(log.LevelDebug, "copying envbuilder binary at %q to build context %q", opts.BinaryPath, envbuilderBinDest)
-		if err := copyFile(opts.Filesystem, opts.BinaryPath, envbuilderBinDest, 0o755); err != nil {
-			return fmt.Errorf("copy envbuilder binary to build context: %w", err)
-		}
+		// In order to allow 'resuming' envbuilder, embed the binary into the image
+		// if it is being pushed.
+		// As these files will be owned by root, it is considerate to clean up
+		// after we're done!
+		cleanupBuildContext := func() {}
+		if opts.PushImage {
+			// Add exceptions in Kaniko's ignorelist for these magic files we add.
+			if err := util.AddAllowedPathToDefaultIgnoreList(opts.BinaryPath); err != nil {
+				return fmt.Errorf("add envbuilder binary to ignore list: %w", err)
+			}
+			if err := util.AddAllowedPathToDefaultIgnoreList(magicDir.Image()); err != nil {
+				return fmt.Errorf("add magic image file to ignore list: %w", err)
+			}
+			if err := util.AddAllowedPathToDefaultIgnoreList(magicDir.Features()); err != nil {
+				return fmt.Errorf("add features to ignore list: %w", err)
+			}
+			magicTempDir := magicdir.At(buildParams.BuildContext, magicdir.TempDir)
+			if err := opts.Filesystem.MkdirAll(magicTempDir.Path(), 0o755); err != nil {
+				return fmt.Errorf("create magic temp dir in build context: %w", err)
+			}
+			// Add the magic directives that embed the binary into the built image.
+			buildParams.DockerfileContent += magicdir.Directives
+			// Copy the envbuilder binary into the build context.
+			// External callers will need to specify the path to the desired envbuilder binary.
+			envbuilderBinDest := filepath.Join(magicTempDir.Path(), "envbuilder")
+			// Also touch the magic file that signifies the image has been built!
+			magicImageDest := magicTempDir.Image()
+			// Clean up after build!
+			var cleanupOnce sync.Once
+			cleanupBuildContext = func() {
+				cleanupOnce.Do(func() {
+					for _, path := range []string{magicImageDest, envbuilderBinDest, magicTempDir.Path()} {
+						if err := opts.Filesystem.Remove(path); err != nil {
+							opts.Logger(log.LevelWarn, "failed to clean up magic temp dir from build context: %w", err)
+						}
+					}
+				})
+			}
+			defer cleanupBuildContext()
 
-		opts.Logger(log.LevelDebug, "touching magic image file at %q in build context %q", magicImageDest, magicTempDir)
-		if err := touchFile(opts.Filesystem, magicImageDest, 0o755); err != nil {
-			return fmt.Errorf("touch magic image file in build context: %w", err)
-		}
-	}
+			opts.Logger(log.LevelDebug, "copying envbuilder binary at %q to build context %q", opts.BinaryPath, envbuilderBinDest)
+			if err := copyFile(opts.Filesystem, opts.BinaryPath, envbuilderBinDest, 0o755); err != nil {
+				return fmt.Errorf("copy envbuilder binary to build context: %w", err)
+			}
 
-	// temp move of all ro mounts
-	tempRemountDest := magicDir.Join("mnt")
-	// ignorePrefixes is a superset of ignorePaths that we pass to kaniko's
-	// IgnoreList.
-	ignorePrefixes := append([]string{"/dev", "/proc", "/sys"}, ignorePaths...)
-	restoreMounts, err := ebutil.TempRemount(opts.Logger, tempRemountDest, ignorePrefixes...)
-	defer func() { // restoreMounts should never be nil
-		if err := restoreMounts(); err != nil {
-			opts.Logger(log.LevelError, "restore mounts: %s", err.Error())
+			opts.Logger(log.LevelDebug, "writing magic image file at %q in build context %q", magicImageDest, magicTempDir)
+			if err := writeFile(opts.Filesystem, magicImageDest, 0o755, fmt.Sprintf("USER=%s\n", buildParams.User)); err != nil {
+				return fmt.Errorf("write magic image file in build context: %w", err)
+			}
 		}
-	}()
-	if err != nil {
-		return fmt.Errorf("temp remount: %w", err)
-	}
 
-	skippedRebuild := false
-	stdoutWriter, closeStdout := log.Writer(opts.Logger)
-	defer closeStdout()
-	stderrWriter, closeStderr := log.Writer(opts.Logger)
-	defer closeStderr()
-	build := func() (v1.Image, error) {
-		defer cleanupBuildContext()
-		_, alreadyBuiltErr := opts.Filesystem.Stat(magicDir.Built())
-		_, isImageErr := opts.Filesystem.Stat(magicDir.Image())
-		if (alreadyBuiltErr == nil && opts.SkipRebuild) || isImageErr == nil {
-			endStage := startStage("🏗️ Skipping build because of cache...")
-			imageRef, err := devcontainer.ImageFromDockerfile(buildParams.DockerfileContent)
-			if err != nil {
-				return nil, fmt.Errorf("image from dockerfile: %w", err)
+		// temp move of all ro mounts
+		tempRemountDest := magicDir.Join("mnt")
+		// ignorePrefixes is a superset of ignorePaths that we pass to kaniko's
+		// IgnoreList.
+		ignorePrefixes := append([]string{"/dev", "/proc", "/sys"}, ignorePaths...)
+		restoreMounts, err := ebutil.TempRemount(opts.Logger, tempRemountDest, ignorePrefixes...)
+		defer func() { // restoreMounts should never be nil
+			if err := restoreMounts(); err != nil {
+				opts.Logger(log.LevelError, "restore mounts: %s", err.Error())
 			}
-			image, err := remote.Image(imageRef, remote.WithAuthFromKeychain(creds.GetKeychain()))
+		}()
+		if err != nil {
+			return fmt.Errorf("temp remount: %w", err)
+		}
+
+		stdoutWriter, closeStdout := log.Writer(opts.Logger)
+		defer closeStdout()
+		stderrWriter, closeStderr := log.Writer(opts.Logger)
+		defer closeStderr()
+		build := func() (v1.Image, error) {
+			defer cleanupBuildContext()
+			_, alreadyBuiltErr := opts.Filesystem.Stat(magicDir.Built())
+			_, isImageErr := opts.Filesystem.Stat(magicDir.Image())
+			if (alreadyBuiltErr == nil && opts.SkipRebuild) || isImageErr == nil {
+				endStage := startStage("🏗️ Skipping build because of cache...")
+				imageRef, err := devcontainer.ImageFromDockerfile(buildParams.DockerfileContent)
+				if err != nil {
+					return nil, fmt.Errorf("image from dockerfile: %w", err)
+				}
+				image, err := remote.Image(imageRef, remote.WithAuthFromKeychain(creds.GetKeychain()))
+				if err != nil {
+					return nil, fmt.Errorf("image from remote: %w", err)
+				}
+				endStage("🏗️ Found image from remote!")
+				skippedRebuild = true
+				return image, nil
+			}
+
+			// This is required for deleting the filesystem prior to build!
+			err = util.InitIgnoreList()
 			if err != nil {
-				return nil, fmt.Errorf("image from remote: %w", err)
+				return nil, fmt.Errorf("init ignore list: %w", err)
 			}
-			endStage("🏗️ Found image from remote!")
-			skippedRebuild = true
-			return image, nil
-		}
 
-		// This is required for deleting the filesystem prior to build!
-		err = util.InitIgnoreList()
-		if err != nil {
-			return nil, fmt.Errorf("init ignore list: %w", err)
-		}
+			// It's possible that the container will already have files in it, and
+			// we don't want to merge a new container with the old one.
+			if err := maybeDeleteFilesystem(opts.Logger, opts.ForceSafe); err != nil {
+				return nil, fmt.Errorf("delete filesystem: %w", err)
+			}
 
-		// It's possible that the container will already have files in it, and
-		// we don't want to merge a new container with the old one.
-		if err := maybeDeleteFilesystem(opts.Logger, opts.ForceSafe); err != nil {
-			return nil, fmt.Errorf("delete filesystem: %w", err)
-		}
+			cacheTTL := time.Hour * 24 * 7
+			if opts.CacheTTLDays != 0 {
+				cacheTTL = time.Hour * 24 * time.Duration(opts.CacheTTLDays)
+			}
 
-		cacheTTL := time.Hour * 24 * 7
-		if opts.CacheTTLDays != 0 {
-			cacheTTL = time.Hour * 24 * time.Duration(opts.CacheTTLDays)
-		}
+			// At this point we have all the context, we can now build!
+			registryMirror := []string{}
+			if val, ok := os.LookupEnv("KANIKO_REGISTRY_MIRROR"); ok {
+				registryMirror = strings.Split(val, ";")
+			}
+			var destinations []string
+			if opts.CacheRepo != "" {
+				destinations = append(destinations, opts.CacheRepo)
+			}
+			kOpts := &config.KanikoOptions{
+				// Boilerplate!
+				CustomPlatform:    platforms.Format(platforms.Normalize(platforms.DefaultSpec())),
+				SnapshotMode:      "redo",
+				RunV2:             true,
+				RunStdout:         stdoutWriter,
+				RunStderr:         stderrWriter,
+				Destinations:      destinations,
+				NoPush:            !opts.PushImage || len(destinations) == 0,
+				CacheRunLayers:    true,
+				CacheCopyLayers:   true,
+				CompressedCaching: true,
+				Compression:       config.ZStd,
+				// Maps to "default" level, ~100-300 MB/sec according to
+				// benchmarks in klauspost/compress README
+				// https://github.com/klauspost/compress/blob/67a538e2b4df11f8ec7139388838a13bce84b5d5/zstd/encoder_options.go#L188
+				CompressionLevel: 3,
+				CacheOptions: config.CacheOptions{
+					CacheTTL: cacheTTL,
+					CacheDir: opts.BaseImageCacheDir,
+				},
+				ForceUnpack:       true,
+				BuildArgs:         buildParams.BuildArgs,
+				CacheRepo:         opts.CacheRepo,
+				Cache:             opts.CacheRepo != "" || opts.BaseImageCacheDir != "",
+				DockerfilePath:    buildParams.DockerfilePath,
+				DockerfileContent: buildParams.DockerfileContent,
+				RegistryOptions: config.RegistryOptions{
+					Insecure:      opts.Insecure,
+					InsecurePull:  opts.Insecure,
+					SkipTLSVerify: opts.Insecure,
+					// Enables registry mirror features in Kaniko, see more in link below
+					// https://github.com/GoogleContainerTools/kaniko?tab=readme-ov-file#flag---registry-mirror
+					// Related to PR #114
+					// https://github.com/coder/envbuilder/pull/114
+					RegistryMirrors: registryMirror,
+				},
+				SrcContext: buildParams.BuildContext,
+
+				// For cached image utilization, produce reproducible builds.
+				Reproducible: opts.PushImage,
+			}
 
-		// At this point we have all the context, we can now build!
-		registryMirror := []string{}
-		if val, ok := os.LookupEnv("KANIKO_REGISTRY_MIRROR"); ok {
-			registryMirror = strings.Split(val, ";")
-		}
-		var destinations []string
-		if opts.CacheRepo != "" {
-			destinations = append(destinations, opts.CacheRepo)
-		}
-		kOpts := &config.KanikoOptions{
-			// Boilerplate!
-			CustomPlatform:    platforms.Format(platforms.Normalize(platforms.DefaultSpec())),
-			SnapshotMode:      "redo",
-			RunV2:             true,
-			RunStdout:         stdoutWriter,
-			RunStderr:         stderrWriter,
-			Destinations:      destinations,
-			NoPush:            !opts.PushImage || len(destinations) == 0,
-			CacheRunLayers:    true,
-			CacheCopyLayers:   true,
-			CompressedCaching: true,
-			Compression:       config.ZStd,
-			// Maps to "default" level, ~100-300 MB/sec according to
-			// benchmarks in klauspost/compress README
-			// https://github.com/klauspost/compress/blob/67a538e2b4df11f8ec7139388838a13bce84b5d5/zstd/encoder_options.go#L188
-			CompressionLevel: 3,
-			CacheOptions: config.CacheOptions{
-				CacheTTL: cacheTTL,
-				CacheDir: opts.BaseImageCacheDir,
-			},
-			ForceUnpack:       true,
-			BuildArgs:         buildParams.BuildArgs,
-			CacheRepo:         opts.CacheRepo,
-			Cache:             opts.CacheRepo != "" || opts.BaseImageCacheDir != "",
-			DockerfilePath:    buildParams.DockerfilePath,
-			DockerfileContent: buildParams.DockerfileContent,
-			RegistryOptions: config.RegistryOptions{
-				Insecure:      opts.Insecure,
-				InsecurePull:  opts.Insecure,
-				SkipTLSVerify: opts.Insecure,
-				// Enables registry mirror features in Kaniko, see more in link below
-				// https://github.com/GoogleContainerTools/kaniko?tab=readme-ov-file#flag---registry-mirror
-				// Related to PR #114
-				// https://github.com/coder/envbuilder/pull/114
-				RegistryMirrors: registryMirror,
-			},
-			SrcContext: buildParams.BuildContext,
+			endStage := startStage("🏗️ Building image...")
+			image, err := executor.DoBuild(kOpts)
+			if err != nil {
+				return nil, xerrors.Errorf("do build: %w", err)
+			}
+			endStage("🏗️ Built image!")
+			if opts.PushImage {
+				endStage = startStage("🏗️ Pushing image...")
+				if err := executor.DoPush(image, kOpts); err != nil {
+					return nil, xerrors.Errorf("do push: %w", err)
+				}
+				endStage("🏗️ Pushed image!")
+			}
 
-			// For cached image utilization, produce reproducible builds.
-			Reproducible: opts.PushImage,
+			return image, err
 		}
 
-		endStage := startStage("🏗️ Building image...")
-		image, err := executor.DoBuild(kOpts)
+		// At this point we have all the context, we can now build!
+		image, err := build()
 		if err != nil {
-			return nil, xerrors.Errorf("do build: %w", err)
-		}
-		endStage("🏗️ Built image!")
-		if opts.PushImage {
-			endStage = startStage("🏗️ Pushing image...")
-			if err := executor.DoPush(image, kOpts); err != nil {
-				return nil, xerrors.Errorf("do push: %w", err)
+			fallback := false
+			switch {
+			case strings.Contains(err.Error(), "parsing dockerfile"):
+				fallback = true
+				fallbackErr = err
+			case strings.Contains(err.Error(), "error building stage"):
+				fallback = true
+				fallbackErr = err
+			// This occurs when the image cannot be found!
+			case strings.Contains(err.Error(), "authentication required"):
+				fallback = true
+				fallbackErr = err
+			// This occurs from Docker Hub when the image cannot be found!
+			case strings.Contains(err.Error(), "manifest unknown"):
+				fallback = true
+				fallbackErr = err
+			case strings.Contains(err.Error(), "unexpected status code 401 Unauthorized"):
+				opts.Logger(log.LevelError, "Unable to pull the provided image. Ensure your registry credentials are correct!")
 			}
-			endStage("🏗️ Pushed image!")
-		}
-
-		return image, err
-	}
-
-	// At this point we have all the context, we can now build!
-	image, err := build()
-	if err != nil {
-		fallback := false
-		switch {
-		case strings.Contains(err.Error(), "parsing dockerfile"):
-			fallback = true
-			fallbackErr = err
-		case strings.Contains(err.Error(), "error building stage"):
-			fallback = true
-			fallbackErr = err
-		// This occurs when the image cannot be found!
-		case strings.Contains(err.Error(), "authentication required"):
-			fallback = true
-			fallbackErr = err
-		// This occurs from Docker Hub when the image cannot be found!
-		case strings.Contains(err.Error(), "manifest unknown"):
-			fallback = true
-			fallbackErr = err
-		case strings.Contains(err.Error(), "unexpected status code 401 Unauthorized"):
-			opts.Logger(log.LevelError, "Unable to pull the provided image. Ensure your registry credentials are correct!")
-		}
-		if !fallback || opts.ExitOnBuildFailure {
-			return err
+			if !fallback || opts.ExitOnBuildFailure {
+				return err
+			}
+			opts.Logger(log.LevelError, "Failed to build: %s", err)
+			opts.Logger(log.LevelError, "Falling back to the default image...")
+			buildParams, err = defaultBuildParams()
+			if err != nil {
+				return err
+			}
+			image, err = build()
 		}
-		opts.Logger(log.LevelError, "Failed to build: %s", err)
-		opts.Logger(log.LevelError, "Falling back to the default image...")
-		buildParams, err = defaultBuildParams()
 		if err != nil {
-			return err
+			return fmt.Errorf("build with kaniko: %w", err)
 		}
-		image, err = build()
-	}
-	if err != nil {
-		return fmt.Errorf("build with kaniko: %w", err)
-	}
 
-	if err := restoreMounts(); err != nil {
-		return fmt.Errorf("restore mounts: %w", err)
-	}
-
-	// Create the magic file to indicate that this build
-	// has already been ran before!
-	file, err := opts.Filesystem.Create(magicDir.Built())
-	if err != nil {
-		return fmt.Errorf("create magic file: %w", err)
-	}
-	_ = file.Close()
-
-	configFile, err := image.ConfigFile()
-	if err != nil {
-		return fmt.Errorf("get image config: %w", err)
-	}
-
-	containerEnv := make(map[string]string)
-	remoteEnv := make(map[string]string)
+		if err := restoreMounts(); err != nil {
+			return fmt.Errorf("restore mounts: %w", err)
+		}
 
-	// devcontainer metadata can be persisted through a standard label
-	devContainerMetadata, exists := configFile.Config.Labels["devcontainer.metadata"]
-	if exists {
-		var devContainer []*devcontainer.Spec
-		devContainerMetadataBytes, err := hujson.Standardize([]byte(devContainerMetadata))
+		// Create the magic file to indicate that this build
+		// has already been ran before!
+		file, err := opts.Filesystem.Create(magicDir.Built())
 		if err != nil {
-			return fmt.Errorf("humanize json for dev container metadata: %w", err)
+			return fmt.Errorf("create magic file: %w", err)
 		}
-		err = json.Unmarshal(devContainerMetadataBytes, &devContainer)
+		_ = file.Close()
+
+		configFile, err := image.ConfigFile()
 		if err != nil {
-			return fmt.Errorf("unmarshal metadata: %w", err)
+			return fmt.Errorf("get image config: %w", err)
 		}
-		opts.Logger(log.LevelInfo, "#%d: 👀 Found devcontainer.json label metadata in image...", stageNumber)
-		for _, container := range devContainer {
-			if container.RemoteUser != "" {
-				opts.Logger(log.LevelInfo, "#%d: 🧑 Updating the user to %q!", stageNumber, container.RemoteUser)
 
-				configFile.Config.User = container.RemoteUser
-			}
-			maps.Copy(containerEnv, container.ContainerEnv)
-			maps.Copy(remoteEnv, container.RemoteEnv)
-			if !container.OnCreateCommand.IsEmpty() {
-				scripts.OnCreateCommand = container.OnCreateCommand
-			}
-			if !container.UpdateContentCommand.IsEmpty() {
-				scripts.UpdateContentCommand = container.UpdateContentCommand
+		containerEnv := make(map[string]string)
+		remoteEnv := make(map[string]string)
+
+		// devcontainer metadata can be persisted through a standard label
+		devContainerMetadata, exists := configFile.Config.Labels["devcontainer.metadata"]
+		if exists {
+			var devContainer []*devcontainer.Spec
+			devContainerMetadataBytes, err := hujson.Standardize([]byte(devContainerMetadata))
+			if err != nil {
+				return fmt.Errorf("humanize json for dev container metadata: %w", err)
 			}
-			if !container.PostCreateCommand.IsEmpty() {
-				scripts.PostCreateCommand = container.PostCreateCommand
+			err = json.Unmarshal(devContainerMetadataBytes, &devContainer)
+			if err != nil {
+				return fmt.Errorf("unmarshal metadata: %w", err)
 			}
-			if !container.PostStartCommand.IsEmpty() {
-				scripts.PostStartCommand = container.PostStartCommand
+			opts.Logger(log.LevelInfo, "#%d: 👀 Found devcontainer.json label metadata in image...", stageNumber)
+			for _, container := range devContainer {
+				if container.RemoteUser != "" {
+					opts.Logger(log.LevelInfo, "#%d: 🧑 Updating the user to %q!", stageNumber, container.RemoteUser)
+
+					configFile.Config.User = container.RemoteUser
+				}
+				maps.Copy(containerEnv, container.ContainerEnv)
+				maps.Copy(remoteEnv, container.RemoteEnv)
+				if !container.OnCreateCommand.IsEmpty() {
+					scripts.OnCreateCommand = container.OnCreateCommand
+				}
+				if !container.UpdateContentCommand.IsEmpty() {
+					scripts.UpdateContentCommand = container.UpdateContentCommand
+				}
+				if !container.PostCreateCommand.IsEmpty() {
+					scripts.PostCreateCommand = container.PostCreateCommand
+				}
+				if !container.PostStartCommand.IsEmpty() {
+					scripts.PostStartCommand = container.PostStartCommand
+				}
 			}
 		}
-	}
 
-	// Sanitize the environment of any opts!
-	options.UnsetEnv()
+		// Sanitize the environment of any opts!
+		options.UnsetEnv()
 
-	// Remove the Docker config secret file!
-	if err := cleanupDockerConfigJSON(); err != nil {
-		return err
-	}
+		// Remove the Docker config secret file!
+		if err := cleanupDockerConfigJSON(); err != nil {
+			return err
+		}
 
-	environ, err := os.ReadFile("/etc/environment")
-	if err == nil {
-		for _, env := range strings.Split(string(environ), "\n") {
-			pair := strings.SplitN(env, "=", 2)
-			if len(pair) != 2 {
-				continue
+		environ, err := os.ReadFile("/etc/environment")
+		if err == nil {
+			for _, env := range strings.Split(string(environ), "\n") {
+				pair := strings.SplitN(env, "=", 2)
+				if len(pair) != 2 {
+					continue
+				}
+				os.Setenv(pair[0], pair[1])
 			}
-			os.Setenv(pair[0], pair[1])
 		}
-	}
-
-	allEnvKeys := make(map[string]struct{})
 
-	// It must be set in this parent process otherwise nothing will be found!
-	for _, env := range configFile.Config.Env {
-		pair := strings.SplitN(env, "=", 2)
-		os.Setenv(pair[0], pair[1])
-		allEnvKeys[pair[0]] = struct{}{}
-	}
-	maps.Copy(containerEnv, buildParams.ContainerEnv)
-	maps.Copy(remoteEnv, buildParams.RemoteEnv)
+		allEnvKeys := make(map[string]struct{})
 
-	// Set Envbuilder runtime markers
-	containerEnv["ENVBUILDER"] = "true"
-	if devcontainerPath != "" {
-		containerEnv["DEVCONTAINER"] = "true"
-		containerEnv["DEVCONTAINER_CONFIG"] = devcontainerPath
-	}
+		// It must be set in this parent process otherwise nothing will be found!
+		for _, env := range configFile.Config.Env {
+			pair := strings.SplitN(env, "=", 2)
+			os.Setenv(pair[0], pair[1])
+			allEnvKeys[pair[0]] = struct{}{}
+		}
+		maps.Copy(containerEnv, buildParams.ContainerEnv)
+		maps.Copy(remoteEnv, buildParams.RemoteEnv)
 
-	for _, env := range []map[string]string{containerEnv, remoteEnv} {
-		envKeys := make([]string, 0, len(env))
-		for key := range env {
-			envKeys = append(envKeys, key)
-			allEnvKeys[key] = struct{}{}
+		// Set Envbuilder runtime markers
+		containerEnv["ENVBUILDER"] = "true"
+		if devcontainerPath != "" {
+			containerEnv["DEVCONTAINER"] = "true"
+			containerEnv["DEVCONTAINER_CONFIG"] = devcontainerPath
 		}
-		sort.Strings(envKeys)
-		for _, envVar := range envKeys {
-			value := devcontainer.SubstituteVars(env[envVar], opts.WorkspaceFolder, os.LookupEnv)
-			os.Setenv(envVar, value)
+
+		for _, env := range []map[string]string{containerEnv, remoteEnv} {
+			envKeys := make([]string, 0, len(env))
+			for key := range env {
+				envKeys = append(envKeys, key)
+				allEnvKeys[key] = struct{}{}
+			}
+			sort.Strings(envKeys)
+			for _, envVar := range envKeys {
+				value := devcontainer.SubstituteVars(env[envVar], opts.WorkspaceFolder, os.LookupEnv)
+				os.Setenv(envVar, value)
+			}
 		}
-	}
 
-	// Do not export env if we skipped a rebuild, because ENV directives
-	// from the Dockerfile would not have been processed and we'd miss these
-	// in the export. We should have generated a complete set of environment
-	// on the intial build, so exporting environment variables a second time
-	// isn't useful anyway.
-	if opts.ExportEnvFile != "" && !skippedRebuild {
-		exportEnvFile, err := os.Create(opts.ExportEnvFile)
-		if err != nil {
-			return fmt.Errorf("failed to open EXPORT_ENV_FILE %q: %w", opts.ExportEnvFile, err)
+		// Do not export env if we skipped a rebuild, because ENV directives
+		// from the Dockerfile would not have been processed and we'd miss these
+		// in the export. We should have generated a complete set of environment
+		// on the intial build, so exporting environment variables a second time
+		// isn't useful anyway.
+		if opts.ExportEnvFile != "" && !skippedRebuild {
+			exportEnvFile, err := os.Create(opts.ExportEnvFile)
+			if err != nil {
+				return fmt.Errorf("failed to open EXPORT_ENV_FILE %q: %w", opts.ExportEnvFile, err)
+			}
+
+			envKeys := make([]string, 0, len(allEnvKeys))
+			for key := range allEnvKeys {
+				envKeys = append(envKeys, key)
+			}
+			sort.Strings(envKeys)
+			for _, key := range envKeys {
+				fmt.Fprintf(exportEnvFile, "%s=%s\n", key, os.Getenv(key))
+			}
+
+			exportEnvFile.Close()
 		}
 
-		envKeys := make([]string, 0, len(allEnvKeys))
-		for key := range allEnvKeys {
-			envKeys = append(envKeys, key)
+		username = configFile.Config.User
+		if buildParams.User != "" {
+			username = buildParams.User
 		}
-		sort.Strings(envKeys)
-		for _, key := range envKeys {
-			fmt.Fprintf(exportEnvFile, "%s=%s\n", key, os.Getenv(key))
+	} else {
+		skippedRebuild = true
+		magicEnv, err := parseMagicImageFile(opts.Filesystem, magicDir.Image())
+		if err != nil {
+			return fmt.Errorf("parse magic env: %w", err)
 		}
-
-		exportEnvFile.Close()
-	}
-
-	username := configFile.Config.User
-	if buildParams.User != "" {
-		username = buildParams.User
+		username = magicEnv["USER"]
 	}
 	if username == "" {
 		opts.Logger(log.LevelWarn, "#%d: no user specified, using root", stageNumber)
 	}
-
 	userInfo, err := getUser(username)
 	if err != nil {
 		return fmt.Errorf("update user: %w", err)
@@ -957,7 +971,7 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error)
 	}
 
 	defaultBuildParams := func() (*devcontainer.Compiled, error) {
-		dockerfile := filepath.Join(buildTimeWorkspaceFolder, "Dockerfile")
+		dockerfile := magicDir.Join("Dockerfile")
 		file, err := opts.Filesystem.OpenFile(dockerfile, os.O_CREATE|os.O_WRONLY, 0o644)
 		if err != nil {
 			return nil, err
@@ -979,7 +993,7 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error)
 		return &devcontainer.Compiled{
 			DockerfilePath:    dockerfile,
 			DockerfileContent: content,
-			BuildContext:      buildTimeWorkspaceFolder,
+			BuildContext:      magicDir.Path(),
 		}, nil
 	}
 
@@ -1019,7 +1033,7 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error)
 					opts.Logger(log.LevelInfo, "No Dockerfile or image specified; falling back to the default image...")
 					fallbackDockerfile = defaultParams.DockerfilePath
 				}
-				buildParams, err = devContainer.Compile(opts.Filesystem, devcontainerDir, buildTimeWorkspaceFolder, fallbackDockerfile, opts.WorkspaceFolder, false, os.LookupEnv)
+				buildParams, err = devContainer.Compile(opts.Filesystem, devcontainerDir, magicDir.Path(), fallbackDockerfile, opts.WorkspaceFolder, false, os.LookupEnv)
 				if err != nil {
 					return nil, fmt.Errorf("compile devcontainer.json: %w", err)
 				}
@@ -1122,11 +1136,11 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error)
 		return nil, xerrors.Errorf("copy envbuilder binary to build context: %w", err)
 	}
 
-	// Also touch the magic file that signifies the image has been built!A
+	// Also write the magic file that signifies the image has been built!
 	magicImageDest := filepath.Join(magicTempDir, "image")
-	opts.Logger(log.LevelDebug, "touching magic image file at %q in build context %q", magicImageDest, magicTempDir)
-	if err := touchFile(opts.Filesystem, magicImageDest, 0o755); err != nil {
-		return nil, fmt.Errorf("touch magic image file at %q: %w", magicImageDest, err)
+	opts.Logger(log.LevelDebug, "writing magic image file at %q in build context %q", magicImageDest, magicTempDir)
+	if err := writeFile(opts.Filesystem, magicImageDest, 0o755, fmt.Sprintf("USER=%s\n", buildParams.User)); err != nil {
+		return nil, fmt.Errorf("write magic image file in build context: %w", err)
 	}
 	defer func() {
 		// Clean up after we're done!
@@ -1465,12 +1479,43 @@ func copyFile(fs billy.Filesystem, src, dst string, mode fs.FileMode) error {
 	return nil
 }
 
-func touchFile(fs billy.Filesystem, dst string, mode fs.FileMode) error {
-	f, err := fs.OpenFile(dst, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode)
+func writeFile(fs billy.Filesystem, dst string, mode fs.FileMode, content string) error {
+	f, err := fs.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode)
 	if err != nil {
-		return xerrors.Errorf("failed to touch file: %w", err)
+		return fmt.Errorf("open file: %w", err)
+	}
+	defer f.Close()
+	_, err = f.Write([]byte(content))
+	if err != nil {
+		return fmt.Errorf("write file: %w", err)
+	}
+	return nil
+}
+
+func parseMagicImageFile(fs billy.Filesystem, path string) (map[string]string, error) {
+	file, err := fs.Open(path)
+	if err != nil {
+		return nil, fmt.Errorf("open magic image file: %w", err)
+	}
+	defer file.Close()
+
+	env := make(map[string]string)
+	s := bufio.NewScanner(file)
+	for s.Scan() {
+		line := strings.TrimSpace(s.Text())
+		if line == "" {
+			continue
+		}
+		parts := strings.SplitN(line, "=", 2)
+		if len(parts) != 2 {
+			return nil, fmt.Errorf("invalid magic image file format: %q", line)
+		}
+		env[parts[0]] = parts[1]
+	}
+	if err := s.Err(); err != nil {
+		return nil, fmt.Errorf("scan magic image file: %w", err)
 	}
-	return f.Close()
+	return env, nil
 }
 
 func initDockerConfigJSON(logf log.Func, magicDir magicdir.MagicDir, dockerConfigBase64 string) (func() error, error) {
diff --git a/integration/integration_test.go b/integration/integration_test.go
index 29b5e8a7..f94b8fe9 100644
--- a/integration/integration_test.go
+++ b/integration/integration_test.go
@@ -17,6 +17,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"testing"
 	"time"
@@ -41,6 +42,7 @@ import (
 	"github.com/google/go-containerregistry/pkg/authn"
 	"github.com/google/go-containerregistry/pkg/name"
 	"github.com/google/go-containerregistry/pkg/registry"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
 	"github.com/google/go-containerregistry/pkg/v1/remote/transport"
 	"github.com/google/uuid"
@@ -1160,7 +1162,7 @@ RUN date --utc > /root/date.txt`, testImageAlpine),
 		_, err = remote.Image(ref)
 		require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push")
 
-		// When: we run envbuilder with PUSH_IMAGE set
+		// When: we run envbuilder with no PUSH_IMAGE set
 		_, err = runEnvbuilder(t, runOpts{env: []string{
 			envbuilderEnv("GIT_URL", srv.URL),
 			envbuilderEnv("CACHE_REPO", testRepo),
@@ -1184,6 +1186,9 @@ RUN date --utc > /root/date.txt`, testImageAlpine),
 	t.Run("CacheAndPush", func(t *testing.T) {
 		t.Parallel()
 
+		ctx, cancel := context.WithCancel(context.Background())
+		t.Cleanup(cancel)
+
 		srv := gittest.CreateGitServer(t, gittest.Options{
 			Files: map[string]string{
 				".devcontainer/Dockerfile": fmt.Sprintf(`FROM %s
@@ -1210,90 +1215,33 @@ RUN date --utc > /root/date.txt`, testImageAlpine),
 		_, err = remote.Image(ref)
 		require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push")
 
-		// When: we run envbuilder with GET_CACHED_IMAGE
-		_, err = runEnvbuilder(t, runOpts{env: []string{
+		opts := []string{
 			envbuilderEnv("GIT_URL", srv.URL),
 			envbuilderEnv("CACHE_REPO", testRepo),
-			envbuilderEnv("GET_CACHED_IMAGE", "1"),
 			envbuilderEnv("VERBOSE", "1"),
-		}})
+		}
+
+		// When: we run envbuilder with GET_CACHED_IMAGE
+		_, err = runEnvbuilder(t, runOpts{env: append(opts,
+			envbuilderEnv("GET_CACHED_IMAGE", "1"),
+		)})
 		require.ErrorContains(t, err, "error probing build cache: uncached RUN command")
 		// Then: it should fail to build the image and nothing should be pushed
 		_, err = remote.Image(ref)
 		require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push")
 
 		// When: we run envbuilder with PUSH_IMAGE set
-		_, err = runEnvbuilder(t, runOpts{env: []string{
-			envbuilderEnv("GIT_URL", srv.URL),
-			envbuilderEnv("CACHE_REPO", testRepo),
-			envbuilderEnv("PUSH_IMAGE", "1"),
-			envbuilderEnv("VERBOSE", "1"),
-		}})
-		require.NoError(t, err)
-
-		// Then: the image should be pushed
-		img, err := remote.Image(ref)
-		require.NoError(t, err, "expected image to be present after build + push")
-
-		// Then: the image should have its directives replaced with those required
-		// to run envbuilder automatically
-		configFile, err := img.ConfigFile()
-		require.NoError(t, err, "expected image to return a config file")
-
-		assert.Equal(t, "root", configFile.Config.User, "user must be root")
-		assert.Equal(t, "/", configFile.Config.WorkingDir, "workdir must be /")
-		if assert.Len(t, configFile.Config.Entrypoint, 1) {
-			assert.Equal(t, "/.envbuilder/bin/envbuilder", configFile.Config.Entrypoint[0], "incorrect entrypoint")
-		}
-
-		// Then: re-running envbuilder with GET_CACHED_IMAGE should succeed
-		ctrID, err := runEnvbuilder(t, runOpts{env: []string{
-			envbuilderEnv("GIT_URL", srv.URL),
-			envbuilderEnv("CACHE_REPO", testRepo),
-			envbuilderEnv("GET_CACHED_IMAGE", "1"),
-		}})
-		require.NoError(t, err)
+		_ = pushImage(t, ref, nil, opts...)
 
-		// Then: the cached image ref should be emitted in the container logs
-		ctx, cancel := context.WithCancel(context.Background())
-		t.Cleanup(cancel)
 		cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
 		require.NoError(t, err)
 		defer cli.Close()
-		logs, err := cli.ContainerLogs(ctx, ctrID, container.LogsOptions{
-			ShowStdout: true,
-			ShowStderr: true,
-		})
-		require.NoError(t, err)
-		defer logs.Close()
-		logBytes, err := io.ReadAll(logs)
-		require.NoError(t, err)
-		require.Regexp(t, `ENVBUILDER_CACHED_IMAGE=(\S+)`, string(logBytes))
 
-		// When: we pull the image we just built
-		rc, err := cli.ImagePull(ctx, ref.String(), image.PullOptions{})
-		require.NoError(t, err)
-		t.Cleanup(func() { _ = rc.Close() })
-		_, err = io.ReadAll(rc)
-		require.NoError(t, err)
+		// Then: re-running envbuilder with GET_CACHED_IMAGE should succeed
+		cachedRef := getCachedImage(ctx, t, cli, opts...)
 
 		// When: we run the image we just built
-		ctr, err := cli.ContainerCreate(ctx, &container.Config{
-			Image:      ref.String(),
-			Entrypoint: []string{"sleep", "infinity"},
-			Labels: map[string]string{
-				testContainerLabel: "true",
-			},
-		}, nil, nil, nil, "")
-		require.NoError(t, err)
-		t.Cleanup(func() {
-			_ = cli.ContainerRemove(ctx, ctr.ID, container.RemoveOptions{
-				RemoveVolumes: true,
-				Force:         true,
-			})
-		})
-		err = cli.ContainerStart(ctx, ctr.ID, container.StartOptions{})
-		require.NoError(t, err)
+		ctr := startContainerFromRef(ctx, t, cli, cachedRef)
 
 		// Then: the envbuilder binary exists in the image!
 		out := execContainer(t, ctr.ID, "/.envbuilder/bin/envbuilder --help")
@@ -1305,6 +1253,9 @@ RUN date --utc > /root/date.txt`, testImageAlpine),
 	t.Run("CacheAndPushDevcontainerOnly", func(t *testing.T) {
 		t.Parallel()
 
+		ctx, cancel := context.WithCancel(context.Background())
+		t.Cleanup(cancel)
+
 		srv := gittest.CreateGitServer(t, gittest.Options{
 			Files: map[string]string{
 				".devcontainer/devcontainer.json": fmt.Sprintf(`{"image": %q}`, testImageAlpine),
@@ -1319,88 +1270,32 @@ RUN date --utc > /root/date.txt`, testImageAlpine),
 		_, err = remote.Image(ref)
 		require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push")
 
-		// When: we run envbuilder with GET_CACHED_IMAGE
-		_, err = runEnvbuilder(t, runOpts{env: []string{
+		opts := []string{
 			envbuilderEnv("GIT_URL", srv.URL),
 			envbuilderEnv("CACHE_REPO", testRepo),
+		}
+
+		// When: we run envbuilder with GET_CACHED_IMAGE
+		_, err = runEnvbuilder(t, runOpts{env: append(opts,
 			envbuilderEnv("GET_CACHED_IMAGE", "1"),
-		}})
+		)})
 		require.ErrorContains(t, err, "error probing build cache: uncached COPY command")
 		// Then: it should fail to build the image and nothing should be pushed
 		_, err = remote.Image(ref)
 		require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push")
 
 		// When: we run envbuilder with PUSH_IMAGE set
-		_, err = runEnvbuilder(t, runOpts{env: []string{
-			envbuilderEnv("GIT_URL", srv.URL),
-			envbuilderEnv("CACHE_REPO", testRepo),
-			envbuilderEnv("PUSH_IMAGE", "1"),
-		}})
-		require.NoError(t, err)
-
-		// Then: the image should be pushed
-		img, err := remote.Image(ref)
-		require.NoError(t, err, "expected image to be present after build + push")
+		_ = pushImage(t, ref, nil, opts...)
 
-		// Then: the image should have its directives replaced with those required
-		// to run envbuilder automatically
-		configFile, err := img.ConfigFile()
-		require.NoError(t, err, "expected image to return a config file")
-
-		assert.Equal(t, "root", configFile.Config.User, "user must be root")
-		assert.Equal(t, "/", configFile.Config.WorkingDir, "workdir must be /")
-		if assert.Len(t, configFile.Config.Entrypoint, 1) {
-			assert.Equal(t, "/.envbuilder/bin/envbuilder", configFile.Config.Entrypoint[0], "incorrect entrypoint")
-		}
-
-		// Then: re-running envbuilder with GET_CACHED_IMAGE should succeed
-		ctrID, err := runEnvbuilder(t, runOpts{env: []string{
-			envbuilderEnv("GIT_URL", srv.URL),
-			envbuilderEnv("CACHE_REPO", testRepo),
-			envbuilderEnv("GET_CACHED_IMAGE", "1"),
-		}})
-		require.NoError(t, err)
-
-		// Then: the cached image ref should be emitted in the container logs
-		ctx, cancel := context.WithCancel(context.Background())
-		t.Cleanup(cancel)
 		cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
 		require.NoError(t, err)
 		defer cli.Close()
-		logs, err := cli.ContainerLogs(ctx, ctrID, container.LogsOptions{
-			ShowStdout: true,
-			ShowStderr: true,
-		})
-		require.NoError(t, err)
-		defer logs.Close()
-		logBytes, err := io.ReadAll(logs)
-		require.NoError(t, err)
-		require.Regexp(t, `ENVBUILDER_CACHED_IMAGE=(\S+)`, string(logBytes))
 
-		// When: we pull the image we just built
-		rc, err := cli.ImagePull(ctx, ref.String(), image.PullOptions{})
-		require.NoError(t, err)
-		t.Cleanup(func() { _ = rc.Close() })
-		_, err = io.ReadAll(rc)
-		require.NoError(t, err)
+		// Then: re-running envbuilder with GET_CACHED_IMAGE should succeed
+		cachedRef := getCachedImage(ctx, t, cli, opts...)
 
 		// When: we run the image we just built
-		ctr, err := cli.ContainerCreate(ctx, &container.Config{
-			Image:      ref.String(),
-			Entrypoint: []string{"sleep", "infinity"},
-			Labels: map[string]string{
-				testContainerLabel: "true",
-			},
-		}, nil, nil, nil, "")
-		require.NoError(t, err)
-		t.Cleanup(func() {
-			_ = cli.ContainerRemove(ctx, ctr.ID, container.RemoveOptions{
-				RemoveVolumes: true,
-				Force:         true,
-			})
-		})
-		err = cli.ContainerStart(ctx, ctr.ID, container.StartOptions{})
-		require.NoError(t, err)
+		ctr := startContainerFromRef(ctx, t, cli, cachedRef)
 
 		// Then: the envbuilder binary exists in the image!
 		out := execContainer(t, ctr.ID, "/.envbuilder/bin/envbuilder --help")
@@ -1430,18 +1325,18 @@ RUN date --utc > /root/date.txt`, testImageAlpine),
 		})
 
 		// Given: an empty registry
-		opts := setupInMemoryRegistryOpts{
+		authOpts := setupInMemoryRegistryOpts{
 			Username: "testing",
 			Password: "testing",
 		}
-		remoteAuthOpt := remote.WithAuth(&authn.Basic{Username: opts.Username, Password: opts.Password})
-		testReg := setupInMemoryRegistry(t, opts)
+		remoteAuthOpt := remote.WithAuth(&authn.Basic{Username: authOpts.Username, Password: authOpts.Password})
+		testReg := setupInMemoryRegistry(t, authOpts)
 		testRepo := testReg + "/test"
 		regAuthJSON, err := json.Marshal(envbuilder.DockerConfig{
 			AuthConfigs: map[string]clitypes.AuthConfig{
 				testRepo: {
-					Username: opts.Username,
-					Password: opts.Password,
+					Username: authOpts.Username,
+					Password: authOpts.Password,
 				},
 			},
 		})
@@ -1451,37 +1346,32 @@ RUN date --utc > /root/date.txt`, testImageAlpine),
 		_, err = remote.Image(ref, remoteAuthOpt)
 		require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push")
 
-		// When: we run envbuilder with GET_CACHED_IMAGE
-		_, err = runEnvbuilder(t, runOpts{env: []string{
+		opts := []string{
 			envbuilderEnv("GIT_URL", srv.URL),
 			envbuilderEnv("CACHE_REPO", testRepo),
+			envbuilderEnv("DOCKER_CONFIG_BASE64", base64.StdEncoding.EncodeToString(regAuthJSON)),
+		}
+
+		// When: we run envbuilder with GET_CACHED_IMAGE
+		_, err = runEnvbuilder(t, runOpts{env: append(opts,
 			envbuilderEnv("GET_CACHED_IMAGE", "1"),
-		}})
+		)})
 		require.ErrorContains(t, err, "error probing build cache: uncached RUN command")
 		// Then: it should fail to build the image and nothing should be pushed
 		_, err = remote.Image(ref, remoteAuthOpt)
 		require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push")
 
 		// When: we run envbuilder with PUSH_IMAGE set
-		_, err = runEnvbuilder(t, runOpts{env: []string{
-			envbuilderEnv("GIT_URL", srv.URL),
-			envbuilderEnv("CACHE_REPO", testRepo),
-			envbuilderEnv("PUSH_IMAGE", "1"),
-			envbuilderEnv("DOCKER_CONFIG_BASE64", base64.StdEncoding.EncodeToString(regAuthJSON)),
-		}})
-		require.NoError(t, err)
+		_ = pushImage(t, ref, remoteAuthOpt, opts...)
 
 		// Then: the image should be pushed
 		_, err = remote.Image(ref, remoteAuthOpt)
 		require.NoError(t, err, "expected image to be present after build + push")
 
 		// Then: re-running envbuilder with GET_CACHED_IMAGE should succeed
-		_, err = runEnvbuilder(t, runOpts{env: []string{
-			envbuilderEnv("GIT_URL", srv.URL),
-			envbuilderEnv("CACHE_REPO", testRepo),
+		_, err = runEnvbuilder(t, runOpts{env: append(opts,
 			envbuilderEnv("GET_CACHED_IMAGE", "1"),
-			envbuilderEnv("DOCKER_CONFIG_BASE64", base64.StdEncoding.EncodeToString(regAuthJSON)),
-		}})
+		)})
 		require.NoError(t, err)
 	})
 
@@ -1507,35 +1397,36 @@ RUN date --utc > /root/date.txt`, testImageAlpine),
 		})
 
 		// Given: an empty registry
-		opts := setupInMemoryRegistryOpts{
+		authOpts := setupInMemoryRegistryOpts{
 			Username: "testing",
 			Password: "testing",
 		}
-		remoteAuthOpt := remote.WithAuth(&authn.Basic{Username: opts.Username, Password: opts.Password})
-		testReg := setupInMemoryRegistry(t, opts)
+		remoteAuthOpt := remote.WithAuth(&authn.Basic{Username: authOpts.Username, Password: authOpts.Password})
+		testReg := setupInMemoryRegistry(t, authOpts)
 		testRepo := testReg + "/test"
 		ref, err := name.ParseReference(testRepo + ":latest")
 		require.NoError(t, err)
 		_, err = remote.Image(ref, remoteAuthOpt)
 		require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push")
 
-		// When: we run envbuilder with GET_CACHED_IMAGE
-		_, err = runEnvbuilder(t, runOpts{env: []string{
+		opts := []string{
 			envbuilderEnv("GIT_URL", srv.URL),
 			envbuilderEnv("CACHE_REPO", testRepo),
+		}
+
+		// When: we run envbuilder with GET_CACHED_IMAGE
+		_, err = runEnvbuilder(t, runOpts{env: append(opts,
 			envbuilderEnv("GET_CACHED_IMAGE", "1"),
-		}})
+		)})
 		require.ErrorContains(t, err, "error probing build cache: uncached RUN command")
 		// Then: it should fail to build the image and nothing should be pushed
 		_, err = remote.Image(ref, remoteAuthOpt)
 		require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push")
 
 		// When: we run envbuilder with PUSH_IMAGE set
-		_, err = runEnvbuilder(t, runOpts{env: []string{
-			envbuilderEnv("GIT_URL", srv.URL),
-			envbuilderEnv("CACHE_REPO", testRepo),
+		_, err = runEnvbuilder(t, runOpts{env: append(opts,
 			envbuilderEnv("PUSH_IMAGE", "1"),
-		}})
+		)})
 		// Then: it should fail with an Unauthorized error
 		require.ErrorContains(t, err, "401 Unauthorized", "expected unauthorized error using no auth when cache repo requires it")
 
@@ -1547,6 +1438,9 @@ RUN date --utc > /root/date.txt`, testImageAlpine),
 	t.Run("CacheAndPushMultistage", func(t *testing.T) {
 		t.Parallel()
 
+		ctx, cancel := context.WithCancel(context.Background())
+		t.Cleanup(cancel)
+
 		srv := gittest.CreateGitServer(t, gittest.Options{
 			Files: map[string]string{
 				"Dockerfile": fmt.Sprintf(`
@@ -1576,80 +1470,33 @@ COPY --from=prebuild /the-future/hello.txt /the-future/hello.txt
 		_, err = remote.Image(ref)
 		require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push")
 
-		// When: we run envbuilder with GET_CACHED_IMAGE
-		_, err = runEnvbuilder(t, runOpts{env: []string{
+		opts := []string{
 			envbuilderEnv("GIT_URL", srv.URL),
 			envbuilderEnv("CACHE_REPO", testRepo),
-			envbuilderEnv("GET_CACHED_IMAGE", "1"),
 			envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"),
-		}})
+		}
+
+		// When: we run envbuilder with GET_CACHED_IMAGE
+		_, err = runEnvbuilder(t, runOpts{env: append(opts,
+			envbuilderEnv("GET_CACHED_IMAGE", "1"),
+		)})
 		require.ErrorContains(t, err, "error probing build cache: uncached RUN command")
 		// Then: it should fail to build the image and nothing should be pushed
 		_, err = remote.Image(ref)
 		require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push")
 
 		// When: we run envbuilder with PUSH_IMAGE set
-		_, err = runEnvbuilder(t, runOpts{env: []string{
-			envbuilderEnv("GIT_URL", srv.URL),
-			envbuilderEnv("CACHE_REPO", testRepo),
-			envbuilderEnv("PUSH_IMAGE", "1"),
-			envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"),
-		}})
-		require.NoError(t, err)
-
-		// Then: the image should be pushed
-		_, err = remote.Image(ref)
-		require.NoError(t, err, "expected image to be present after build + push")
-
-		// Then: re-running envbuilder with GET_CACHED_IMAGE should succeed
-		ctrID, err := runEnvbuilder(t, runOpts{env: []string{
-			envbuilderEnv("GIT_URL", srv.URL),
-			envbuilderEnv("CACHE_REPO", testRepo),
-			envbuilderEnv("GET_CACHED_IMAGE", "1"),
-			envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"),
-		}})
-		require.NoError(t, err)
+		_ = pushImage(t, ref, nil, opts...)
 
-		// Then: the cached image ref should be emitted in the container logs
-		ctx, cancel := context.WithCancel(context.Background())
-		t.Cleanup(cancel)
 		cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
 		require.NoError(t, err)
 		defer cli.Close()
-		logs, err := cli.ContainerLogs(ctx, ctrID, container.LogsOptions{
-			ShowStdout: true,
-			ShowStderr: true,
-		})
-		require.NoError(t, err)
-		defer logs.Close()
-		logBytes, err := io.ReadAll(logs)
-		require.NoError(t, err)
-		require.Regexp(t, `ENVBUILDER_CACHED_IMAGE=(\S+)`, string(logBytes))
 
-		// When: we pull the image we just built
-		rc, err := cli.ImagePull(ctx, ref.String(), image.PullOptions{})
-		require.NoError(t, err)
-		t.Cleanup(func() { _ = rc.Close() })
-		_, err = io.ReadAll(rc)
-		require.NoError(t, err)
+		// Then: re-running envbuilder with GET_CACHED_IMAGE should succeed
+		cachedRef := getCachedImage(ctx, t, cli, opts...)
 
 		// When: we run the image we just built
-		ctr, err := cli.ContainerCreate(ctx, &container.Config{
-			Image:      ref.String(),
-			Entrypoint: []string{"sleep", "infinity"},
-			Labels: map[string]string{
-				testContainerLabel: "true",
-			},
-		}, nil, nil, nil, "")
-		require.NoError(t, err)
-		t.Cleanup(func() {
-			_ = cli.ContainerRemove(ctx, ctr.ID, container.RemoveOptions{
-				RemoveVolumes: true,
-				Force:         true,
-			})
-		})
-		err = cli.ContainerStart(ctx, ctr.ID, container.StartOptions{})
-		require.NoError(t, err)
+		ctr := startContainerFromRef(ctx, t, cli, cachedRef)
 
 		// Then: The files from the prebuild stage are present.
 		out := execContainer(t, ctr.ID, "/bin/sh -c 'cat /the-past/hello.txt /the-future/hello.txt; readlink -f /the-past/hello.link'")
@@ -1698,17 +1545,11 @@ RUN date --utc > /root/date.txt
 		require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push")
 
 		// When: we run envbuilder with PUSH_IMAGE set
-		_, err = runEnvbuilder(t, runOpts{env: []string{
+		_ = pushImage(t, ref, nil,
 			envbuilderEnv("GIT_URL", srv.URL),
 			envbuilderEnv("CACHE_REPO", testRepo),
-			envbuilderEnv("PUSH_IMAGE", "1"),
 			envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"),
-		}})
-		require.NoError(t, err)
-
-		// Then: the image should be pushed
-		_, err = remote.Image(ref)
-		require.NoError(t, err, "expected image to be present after build + push")
+		)
 
 		// Then: re-running envbuilder with GET_CACHED_IMAGE should succeed
 		_, err = runEnvbuilder(t, runOpts{env: []string{
@@ -1725,13 +1566,11 @@ RUN date --utc > /root/date.txt
 		srv = newServer(dockerfilePrebuildContents)
 
 		// When: we rebuild the prebuild stage so that the cache is created
-		_, err = runEnvbuilder(t, runOpts{env: []string{
+		_ = pushImage(t, ref, nil,
 			envbuilderEnv("GIT_URL", srv.URL),
 			envbuilderEnv("CACHE_REPO", testRepo),
-			envbuilderEnv("PUSH_IMAGE", "1"),
 			envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"),
-		}})
-		require.NoError(t, err)
+		)
 
 		// Then: re-running envbuilder with GET_CACHED_IMAGE should still fail
 		// on the second stage because the first stage file has changed.
@@ -1814,6 +1653,60 @@ RUN date --utc > /root/date.txt`, testImageAlpine),
 		// Then: envbuilder should fail with a descriptive error
 		require.ErrorContains(t, err, "failed to push to destination")
 	})
+
+	t.Run("CacheAndPushDevcontainerFeatures", func(t *testing.T) {
+		t.Parallel()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		t.Cleanup(cancel)
+
+		srv := gittest.CreateGitServer(t, gittest.Options{
+			Files: map[string]string{
+				// NOTE(mafredri): We can't cache the feature in our local
+				// registry because the image media type is incompatible.
+				".devcontainer/devcontainer.json": fmt.Sprintf(`
+{
+	"image": %q,
+	"features": {
+		"ghcr.io/devcontainers/feature-starter/color:1": {
+				"favorite": "green"
+		}
+	}
+}
+`, testImageUbuntu),
+			},
+		})
+
+		// Given: an empty registry
+		testReg := setupInMemoryRegistry(t, setupInMemoryRegistryOpts{})
+		testRepo := testReg + "/test"
+		ref, err := name.ParseReference(testRepo + ":latest")
+		require.NoError(t, err)
+		_, err = remote.Image(ref)
+		require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push")
+
+		opts := []string{
+			envbuilderEnv("GIT_URL", srv.URL),
+			envbuilderEnv("CACHE_REPO", testRepo),
+		}
+
+		// When: we run envbuilder with PUSH_IMAGE set
+		_ = pushImage(t, ref, nil, opts...)
+
+		cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
+		require.NoError(t, err)
+		defer cli.Close()
+
+		// Then: re-running envbuilder with GET_CACHED_IMAGE should succeed
+		cachedRef := getCachedImage(ctx, t, cli, opts...)
+
+		// When: we run the image we just built
+		ctr := startContainerFromRef(ctx, t, cli, cachedRef)
+
+		// Check that the feature is present in the image.
+		out := execContainer(t, ctr.ID, "/usr/local/bin/color")
+		require.Contains(t, strings.TrimSpace(out), "my favorite color is green")
+	})
 }
 
 func TestChownHomedir(t *testing.T) {
@@ -1935,6 +1828,92 @@ func cleanOldEnvbuilders() {
 	}
 }
 
+func pushImage(t *testing.T, ref name.Reference, remoteOpt remote.Option, env ...string) v1.Image {
+	t.Helper()
+
+	var remoteOpts []remote.Option
+	if remoteOpt != nil {
+		remoteOpts = append(remoteOpts, remoteOpt)
+	}
+
+	_, err := runEnvbuilder(t, runOpts{env: append(env, envbuilderEnv("PUSH_IMAGE", "1"))})
+	require.NoError(t, err, "envbuilder push image failed")
+
+	img, err := remote.Image(ref, remoteOpts...)
+	require.NoError(t, err, "expected image to be present after build + push")
+
+	// The image should have its directives replaced with those required
+	// to run envbuilder automatically
+	configFile, err := img.ConfigFile()
+	require.NoError(t, err, "expected image to return a config file")
+
+	assert.Equal(t, "root", configFile.Config.User, "user must be root")
+	assert.Equal(t, "/", configFile.Config.WorkingDir, "workdir must be /")
+	if assert.Len(t, configFile.Config.Entrypoint, 1) {
+		assert.Equal(t, "/.envbuilder/bin/envbuilder", configFile.Config.Entrypoint[0], "incorrect entrypoint")
+	}
+
+	require.False(t, t.Failed(), "pushImage failed")
+
+	return img
+}
+
+func getCachedImage(ctx context.Context, t *testing.T, cli *client.Client, env ...string) name.Reference {
+	ctrID, err := runEnvbuilder(t, runOpts{env: append(env, envbuilderEnv("GET_CACHED_IMAGE", "1"))})
+	require.NoError(t, err)
+
+	logs, err := cli.ContainerLogs(ctx, ctrID, container.LogsOptions{
+		ShowStdout: true,
+		ShowStderr: true,
+	})
+	require.NoError(t, err)
+	defer logs.Close()
+	logBytes, err := io.ReadAll(logs)
+	require.NoError(t, err)
+
+	re := regexp.MustCompile(`ENVBUILDER_CACHED_IMAGE=(\S+)`)
+	matches := re.FindStringSubmatch(string(logBytes))
+	require.Len(t, matches, 2, "envbuilder cached image not found")
+	ref, err := name.ParseReference(matches[1])
+	require.NoError(t, err, "failed to parse cached image reference")
+	return ref
+}
+
+func startContainerFromRef(ctx context.Context, t *testing.T, cli *client.Client, ref name.Reference) container.CreateResponse {
+	// Ensure that we can pull the image.
+	rc, err := cli.ImagePull(ctx, ref.String(), image.PullOptions{})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = rc.Close() })
+	_, err = io.ReadAll(rc)
+	require.NoError(t, err)
+
+	// Start the container.
+	ctr, err := cli.ContainerCreate(ctx, &container.Config{
+		Image:      ref.String(),
+		Entrypoint: []string{"sleep", "infinity"},
+		Labels: map[string]string{
+			testContainerLabel: "true",
+		},
+	}, nil, nil, nil, "")
+	require.NoError(t, err)
+
+	t.Cleanup(func() {
+		// Start a new context to ensure that the container is removed.
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+
+		_ = cli.ContainerRemove(ctx, ctr.ID, container.RemoveOptions{
+			RemoveVolumes: true,
+			Force:         true,
+		})
+	})
+
+	err = cli.ContainerStart(ctx, ctr.ID, container.StartOptions{})
+	require.NoError(t, err)
+
+	return ctr
+}
+
 type runOpts struct {
 	binds   []string
 	env     []string
diff --git a/internal/magicdir/magicdir.go b/internal/magicdir/magicdir.go
index 31bcd7c9..5e062514 100644
--- a/internal/magicdir/magicdir.go
+++ b/internal/magicdir/magicdir.go
@@ -76,3 +76,8 @@ func (m MagicDir) Built() string {
 func (m MagicDir) Image() string {
 	return m.Join("image")
 }
+
+// Features is a directory that contains feature files.
+func (m MagicDir) Features() string {
+	return m.Join("features")
+}

From 4788b25d0a031ec38004e2f485921572dfeb6624 Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Mon, 23 Sep 2024 12:07:59 +0000
Subject: [PATCH 2/4] add user test

---
 integration/integration_test.go | 55 +++++++++++++++++++++++++++++++--
 1 file changed, 53 insertions(+), 2 deletions(-)

diff --git a/integration/integration_test.go b/integration/integration_test.go
index f94b8fe9..78fe2854 100644
--- a/integration/integration_test.go
+++ b/integration/integration_test.go
@@ -1707,6 +1707,58 @@ RUN date --utc > /root/date.txt`, testImageAlpine),
 		out := execContainer(t, ctr.ID, "/usr/local/bin/color")
 		require.Contains(t, strings.TrimSpace(out), "my favorite color is green")
 	})
+
+	t.Run("CacheAndPushUser", func(t *testing.T) {
+		t.Parallel()
+
+		ctx, cancel := context.WithCancel(context.Background())
+		t.Cleanup(cancel)
+
+		srv := gittest.CreateGitServer(t, gittest.Options{
+			Files: map[string]string{
+				".devcontainer/devcontainer.json": `{
+					"name": "Test",
+					"build": {
+						"dockerfile": "Dockerfile"
+					},
+				}`,
+				".devcontainer/Dockerfile": fmt.Sprintf(`FROM %s
+RUN useradd -m -s /bin/bash devalot
+USER devalot
+`, testImageUbuntu),
+			},
+		})
+
+		// Given: an empty registry
+		testReg := setupInMemoryRegistry(t, setupInMemoryRegistryOpts{})
+		testRepo := testReg + "/test"
+		ref, err := name.ParseReference(testRepo + ":latest")
+		require.NoError(t, err)
+		_, err = remote.Image(ref)
+		require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push")
+
+		opts := []string{
+			envbuilderEnv("GIT_URL", srv.URL),
+			envbuilderEnv("CACHE_REPO", testRepo),
+		}
+
+		// When: we run envbuilder with PUSH_IMAGE set
+		_ = pushImage(t, ref, nil, opts...)
+
+		cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
+		require.NoError(t, err)
+		defer cli.Close()
+
+		// Then: re-running envbuilder with GET_CACHED_IMAGE should succeed
+		cachedRef := getCachedImage(ctx, t, cli, opts...)
+
+		// When: we run the image we just built
+		ctr := startContainerFromRef(ctx, t, cli, cachedRef)
+
+		// Check that the user is present in the image.
+		out := execContainer(t, ctr.ID, "ps aux | awk '/^devalot / {print $1}' | sort -u")
+		require.Contains(t, strings.TrimSpace(out), "devalot")
+	})
 }
 
 func TestChownHomedir(t *testing.T) {
@@ -1889,8 +1941,7 @@ func startContainerFromRef(ctx context.Context, t *testing.T, cli *client.Client
 
 	// Start the container.
 	ctr, err := cli.ContainerCreate(ctx, &container.Config{
-		Image:      ref.String(),
-		Entrypoint: []string{"sleep", "infinity"},
+		Image: ref.String(),
 		Labels: map[string]string{
 			testContainerLabel: "true",
 		},

From fda5091118e094267c3b743a9244e735908a0b5c Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Mon, 23 Sep 2024 12:19:06 +0000
Subject: [PATCH 3/4] fix user tests race

---
 integration/integration_test.go | 27 ++++++++++++++++++++++-----
 1 file changed, 22 insertions(+), 5 deletions(-)

diff --git a/integration/integration_test.go b/integration/integration_test.go
index 78fe2854..f88829aa 100644
--- a/integration/integration_test.go
+++ b/integration/integration_test.go
@@ -157,8 +157,17 @@ FROM a AS b`, testImageUbuntu),
 	}})
 	require.NoError(t, err)
 
-	output := execContainer(t, ctr, "ps aux | awk '/^pickme / {print $1}' | sort -u")
-	require.Equal(t, "pickme", strings.TrimSpace(output))
+	// Check that envbuilder started command as user.
+	// Since envbuilder starts as root, probe for up to 10 seconds.
+	for i := 0; i < 10; i++ {
+		out := execContainer(t, ctr, "ps aux | awk '/^pickme * 1 / {print $1}' | sort -u")
+		got := strings.TrimSpace(out)
+		if got == "pickme" {
+			return
+		}
+		time.Sleep(time.Second)
+	}
+	require.Fail(t, "expected pid 1 to be running as pickme")
 }
 
 func TestUidGid(t *testing.T) {
@@ -1755,9 +1764,17 @@ USER devalot
 		// When: we run the image we just built
 		ctr := startContainerFromRef(ctx, t, cli, cachedRef)
 
-		// Check that the user is present in the image.
-		out := execContainer(t, ctr.ID, "ps aux | awk '/^devalot / {print $1}' | sort -u")
-		require.Contains(t, strings.TrimSpace(out), "devalot")
+		// Check that envbuilder started command as user.
+		// Since envbuilder starts as root, probe for up to 10 seconds.
+		for i := 0; i < 10; i++ {
+			out := execContainer(t, ctr.ID, "ps aux | awk '/^devalot * 1 / {print $1}' | sort -u")
+			got := strings.TrimSpace(out)
+			if got == "devalot" {
+				return
+			}
+			time.Sleep(time.Second)
+		}
+		require.Fail(t, "expected pid 1 to be running as devalot")
 	})
 }
 

From 8226cd67d4a50472fd0624f787a7f4fcf0244920 Mon Sep 17 00:00:00 2001
From: Mathias Fredriksson <mafredri@gmail.com>
Date: Tue, 24 Sep 2024 11:01:31 +0000
Subject: [PATCH 4/4] add comment explaining content of magic image file

---
 envbuilder.go | 37 +++++++++++++++++++++++--------------
 1 file changed, 23 insertions(+), 14 deletions(-)

diff --git a/envbuilder.go b/envbuilder.go
index 5ee564aa..bc94d89c 100644
--- a/envbuilder.go
+++ b/envbuilder.go
@@ -357,11 +357,10 @@ func Run(ctx context.Context, opts options.Options) error {
 			}
 			// Add the magic directives that embed the binary into the built image.
 			buildParams.DockerfileContent += magicdir.Directives
-			// Copy the envbuilder binary into the build context.
-			// External callers will need to specify the path to the desired envbuilder binary.
+
 			envbuilderBinDest := filepath.Join(magicTempDir.Path(), "envbuilder")
-			// Also touch the magic file that signifies the image has been built!
 			magicImageDest := magicTempDir.Image()
+
 			// Clean up after build!
 			var cleanupOnce sync.Once
 			cleanupBuildContext = func() {
@@ -375,11 +374,16 @@ func Run(ctx context.Context, opts options.Options) error {
 			}
 			defer cleanupBuildContext()
 
+			// Copy the envbuilder binary into the build context. External callers
+			// will need to specify the path to the desired envbuilder binary.
 			opts.Logger(log.LevelDebug, "copying envbuilder binary at %q to build context %q", opts.BinaryPath, envbuilderBinDest)
 			if err := copyFile(opts.Filesystem, opts.BinaryPath, envbuilderBinDest, 0o755); err != nil {
 				return fmt.Errorf("copy envbuilder binary to build context: %w", err)
 			}
 
+			// Also write the magic file that signifies the image has been built.
+			// Since the user in the image is set to root, we also store the user
+			// in the magic file to be used by envbuilder when the image is run.
 			opts.Logger(log.LevelDebug, "writing magic image file at %q in build context %q", magicImageDest, magicTempDir)
 			if err := writeFile(opts.Filesystem, magicImageDest, 0o755, fmt.Sprintf("USER=%s\n", buildParams.User)); err != nil {
 				return fmt.Errorf("write magic image file in build context: %w", err)
@@ -1124,32 +1128,37 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error)
 	// add the magic directives to the Dockerfile content.
 	// MAGICDIR
 	buildParams.DockerfileContent += magicdir.Directives
+
 	magicTempDir := filepath.Join(buildParams.BuildContext, magicdir.TempDir)
 	if err := opts.Filesystem.MkdirAll(magicTempDir, 0o755); err != nil {
 		return nil, fmt.Errorf("create magic temp dir in build context: %w", err)
 	}
 	envbuilderBinDest := filepath.Join(magicTempDir, "envbuilder")
+	magicImageDest := filepath.Join(magicTempDir, "image")
+
+	// Clean up after probe!
+	defer func() {
+		for _, path := range []string{magicImageDest, envbuilderBinDest, magicTempDir} {
+			if err := opts.Filesystem.Remove(path); err != nil {
+				opts.Logger(log.LevelWarn, "failed to clean up magic temp dir from build context: %w", err)
+			}
+		}
+	}()
 
-	// Copy the envbuilder binary into the build context.
+	// Copy the envbuilder binary into the build context. External callers
+	// will need to specify the path to the desired envbuilder binary.
 	opts.Logger(log.LevelDebug, "copying envbuilder binary at %q to build context %q", opts.BinaryPath, envbuilderBinDest)
 	if err := copyFile(opts.Filesystem, opts.BinaryPath, envbuilderBinDest, 0o755); err != nil {
 		return nil, xerrors.Errorf("copy envbuilder binary to build context: %w", err)
 	}
 
-	// Also write the magic file that signifies the image has been built!
-	magicImageDest := filepath.Join(magicTempDir, "image")
+	// Also write the magic file that signifies the image has been built.
+	// Since the user in the image is set to root, we also store the user
+	// in the magic file to be used by envbuilder when the image is run.
 	opts.Logger(log.LevelDebug, "writing magic image file at %q in build context %q", magicImageDest, magicTempDir)
 	if err := writeFile(opts.Filesystem, magicImageDest, 0o755, fmt.Sprintf("USER=%s\n", buildParams.User)); err != nil {
 		return nil, fmt.Errorf("write magic image file in build context: %w", err)
 	}
-	defer func() {
-		// Clean up after we're done!
-		for _, path := range []string{magicImageDest, envbuilderBinDest, magicTempDir} {
-			if err := opts.Filesystem.Remove(path); err != nil {
-				opts.Logger(log.LevelWarn, "failed to clean up magic temp dir from build context: %w", err)
-			}
-		}
-	}()
 
 	stdoutWriter, closeStdout := log.Writer(opts.Logger)
 	defer closeStdout()