feat: implement reproducible build and get cached image

README.md

@@ -362,4 +362,6 @@ On MacOS or Windows systems, we recommend either using a VM or the provided `.de
 | `--coder-agent-url` | `CODER_AGENT_URL` |  | URL of the Coder deployment. If CODER_AGENT_TOKEN is also set, logs from envbuilder will be forwarded here and will be visible in the workspace build logs. |
 | `--coder-agent-token` | `CODER_AGENT_TOKEN` |  | Authentication token for a Coder agent. If this is set, then CODER_AGENT_URL must also be set. |
 | `--coder-agent-subsystem` | `CODER_AGENT_SUBSYSTEM` |  | Coder agent subsystems to report when forwarding logs. The envbuilder subsystem is always included. |
+| `--push-image` | `ENVBUILDER_PUSH_IMAGE` |  | Push the built image to a remote registry.This option forces a reproducible build. |
+| `--get-cached-image` | `ENVBUILDER_GET_CACHED_IMAGE` |  | Print the digest of the cached image, if available.Exits with an error if not found. |
 <!--- END docsgen --->

envbuilder.go

@@ -98,6 +98,9 @@ func Run(ctx context.Context, options Options) error {
 	if options.InitCommand == "" {
 		options.InitCommand = "/bin/sh"
 	}
+	if options.CacheRepo == "" && options.PushImage {
+		return fmt.Errorf("--cache-repo must be set when using --push-image")
+	}
 	// Default to the shell!
 	initArgs := []string{"-c", options.InitScript}
 	if options.InitArgs != "" {
@@ -118,11 +121,11 @@ func Run(ctx context.Context, options Options) error {
 		options.WorkspaceFolder = f
 	}
 
-	stageNumber := 1
+	stageNumber := 0
 	startStage := func(format string, args ...any) func(format string, args ...any) {
 		now := time.Now()
-		stageNum := stageNumber
 		stageNumber++
+		stageNum := stageNumber
 		options.Logger(notcodersdk.LogLevelInfo, "#%d: %s", stageNum, fmt.Sprintf(format, args...))
 
 		return func(format string, args ...any) {
@@ -341,7 +344,7 @@ func Run(ctx context.Context, options Options) error {
 
 	HijackLogrus(func(entry *logrus.Entry) {
 		for _, line := range strings.Split(entry.Message, "\r") {
-			options.Logger(notcodersdk.LogLevelInfo, "#2: %s", color.HiBlackString(line))
+			options.Logger(notcodersdk.LogLevelInfo, "#%d: %s", stageNumber, color.HiBlackString(line))
 		}
 	})
 
@@ -471,20 +474,24 @@ func Run(ctx context.Context, options Options) error {
 			cacheTTL = time.Hour * 24 * time.Duration(options.CacheTTLDays)
 		}
 
-		endStage := startStage("🏗️ Building image...")
 		// At this point we have all the context, we can now build!
 		registryMirror := []string{}
 		if val, ok := os.LookupEnv("KANIKO_REGISTRY_MIRROR"); ok {
 			registryMirror = strings.Split(val, ";")
 		}
-		image, err := executor.DoBuild(&config.KanikoOptions{
+		var destinations []string
+		if options.CacheRepo != "" {
+			destinations = append(destinations, options.CacheRepo)
+		}
+		opts := &config.KanikoOptions{
 			// Boilerplate!
 			CustomPlatform:    platforms.Format(platforms.Normalize(platforms.DefaultSpec())),
 			SnapshotMode:      "redo",
 			RunV2:             true,
 			RunStdout:         stdoutWriter,
 			RunStderr:         stderrWriter,
-			Destinations:      []string{"local"},
+			Destinations:      destinations,
+			NoPush:            !options.PushImage || len(destinations) == 0,
 			CacheRunLayers:    true,
 			CacheCopyLayers:   true,
 			CompressedCaching: true,
@@ -515,11 +522,40 @@ func Run(ctx context.Context, options Options) error {
 				RegistryMirrors: registryMirror,
 			},
 			SrcContext: buildParams.BuildContext,
-		})
+
+			// For cached image utilization, produce reproducible builds.
+			Reproducible: options.PushImage,
+		}
+
+		if options.GetCachedImage {
+			endStage := startStage("🏗️ Checking for cached image...")
+			image, err := executor.DoCacheProbe(opts)
+			if err != nil {
+				return nil, xerrors.Errorf("get cached image: %w", err)
+			}
+			digest, err := image.Digest()
+			if err != nil {
+				return nil, xerrors.Errorf("get cached image digest: %w", err)
+			}
+			endStage("🏗️ Found cached image!")
+			_, _ = fmt.Fprintf(os.Stdout, "%s@%s\n", options.CacheRepo, digest.String())
+			os.Exit(0)
+		}
+
+		endStage := startStage("🏗️ Building image...")
+		image, err := executor.DoBuild(opts)
 		if err != nil {
-			return nil, err
+			return nil, xerrors.Errorf("do build: %w", err)
 		}
 		endStage("🏗️ Built image!")
+		if options.PushImage {
+			endStage = startStage("🏗️ Pushing image...")
+			if err := executor.DoPush(image, opts); err != nil {
+				return nil, xerrors.Errorf("do push: %w", err)
+			}
+			endStage("🏗️ Pushed image!")
+		}
+
 		return image, err
 	}
 

go.mod

@@ -6,7 +6,7 @@ toolchain go1.22.3
 
 // There are a few options we need added to Kaniko!
 // See: https://github.com/GoogleContainerTools/kaniko/compare/main...coder:kaniko:main
-replace github.com/GoogleContainerTools/kaniko => github.com/coder/kaniko v0.0.0-20240524082248-9d0d55902c34
+replace github.com/GoogleContainerTools/kaniko => github.com/coder/kaniko v0.0.0-20240610145227-0a73fcd89e3f
 
 require (
 	cdr.dev/slog v1.6.2-0.20240126064726-20367d4aede6

go.sum

@@ -123,8 +123,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX
 github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo=
 github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA=
 github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
-github.com/coder/kaniko v0.0.0-20240524082248-9d0d55902c34 h1:Wm7sMNc1aTN5l0NerYHb3LZdQJVQp4QrW4v83N21sfc=
-github.com/coder/kaniko v0.0.0-20240524082248-9d0d55902c34/go.mod h1:YMK7BlxerzLlMwihGxNWUaFoN9LXCij4P+w/8/fNlcM=
+github.com/coder/kaniko v0.0.0-20240610145227-0a73fcd89e3f h1:o5p2u5sF1KVMgH+pcoMNfStpfXe76O3Enwg0h1KlzIU=
+github.com/coder/kaniko v0.0.0-20240610145227-0a73fcd89e3f/go.mod h1:YMK7BlxerzLlMwihGxNWUaFoN9LXCij4P+w/8/fNlcM=
 github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0 h1:3A0ES21Ke+FxEM8CXx9n47SZOKOpgSE1bbJzlE4qPVs=
 github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0/go.mod h1:5UuS2Ts+nTToAMeOjNlnHFkPahrtDkmpydBen/3wgZc=
 github.com/coder/retry v1.5.1 h1:iWu8YnD8YqHs3XwqrqsjoBTAVqT9ml6z9ViJ2wlMiqc=

integration/integration_test.go

@@ -34,6 +34,8 @@ import (
 	"github.com/go-git/go-billy/v5/memfs"
 	"github.com/google/go-containerregistry/pkg/authn"
 	"github.com/google/go-containerregistry/pkg/name"
+	"github.com/google/go-containerregistry/pkg/registry"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
 	"github.com/google/go-containerregistry/pkg/v1/remote/transport"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -987,6 +989,252 @@ COPY %s .`, testImageAlpine, inclFile)
 	}
 }
 
+func TestPushImage(t *testing.T) {
+	t.Parallel()
+
+	t.Run("CacheWithoutPush", func(t *testing.T) {
+		t.Parallel()
+
+		srv := createGitServer(t, gitServerOptions{
+			files: map[string]string{
+				".devcontainer/Dockerfile": fmt.Sprintf("FROM %s\nRUN date --utc > /root/date.txt", testImageAlpine),
+				".devcontainer/devcontainer.json": `{
+				"name": "Test",
+				"build": {
+					"dockerfile": "Dockerfile"
+				},
+			}`,
+			},
+		})
+
+		// Given: an empty registry
+		testReg := setupInMemoryRegistry(t)
+		testRepo := testReg + "/test"
+		ref, err := name.ParseReference(testRepo + ":latest")
+		require.NoError(t, err)
+		_, err = remote.Image(ref)
+		require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push")
+
+		// When: we run envbuilder with GET_CACHED_IMAGE
+		_, err = runEnvbuilder(t, options{env: []string{
+			envbuilderEnv("GIT_URL", srv.URL),
+			envbuilderEnv("CACHE_REPO", testRepo),
+			envbuilderEnv("GET_CACHED_IMAGE", "1"),
+		}})
+		require.ErrorContains(t, err, "not supported in fake build")
+		// Then: it should fail to build the image and nothing should be pushed
+		_, err = remote.Image(ref)
+		require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push")
+
+		// When: we run envbuilder with PUSH_IMAGE set
+		_, err = runEnvbuilder(t, options{env: []string{
+			envbuilderEnv("GIT_URL", srv.URL),
+			envbuilderEnv("CACHE_REPO", testRepo),
+		}})
+		require.NoError(t, err)
+
+		// Then: the image tag should not be present, only the layers
+		_, err = remote.Image(ref)
+		require.ErrorContains(t, err, "MANIFEST_UNKNOWN", "expected image to not be present before build + push")
+
+		// Then: re-running envbuilder with GET_CACHED_IMAGE should succeed
+		_, err = runEnvbuilder(t, options{env: []string{
+			envbuilderEnv("GIT_URL", srv.URL),
+			envbuilderEnv("CACHE_REPO", testRepo),
+			envbuilderEnv("GET_CACHED_IMAGE", "1"),
+		}})
+		require.NoError(t, err)
+	})
+
+	t.Run("CacheAndPush", func(t *testing.T) {
+		t.Parallel()
+
+		srv := createGitServer(t, gitServerOptions{
+			files: map[string]string{
+				".devcontainer/Dockerfile": fmt.Sprintf("FROM %s\nRUN date --utc > /root/date.txt", testImageAlpine),
+				".devcontainer/devcontainer.json": `{
+				"name": "Test",
+				"build": {
+					"dockerfile": "Dockerfile"
+				},
+			}`,
+			},
+		})
+
+		// Given: an empty registry
+		testReg := setupInMemoryRegistry(t)
+		testRepo := testReg + "/test"
+		ref, err := name.ParseReference(testRepo + ":latest")
+		require.NoError(t, err)
+		_, err = remote.Image(ref)
+		require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push")
+
+		// When: we run envbuilder with GET_CACHED_IMAGE
+		_, err = runEnvbuilder(t, options{env: []string{
+			envbuilderEnv("GIT_URL", srv.URL),
+			envbuilderEnv("CACHE_REPO", testRepo),
+			envbuilderEnv("GET_CACHED_IMAGE", "1"),
+		}})
+		require.ErrorContains(t, err, "not supported in fake build")
+		// Then: it should fail to build the image and nothing should be pushed
+		_, err = remote.Image(ref)
+		require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push")
+
+		// When: we run envbuilder with PUSH_IMAGE set
+		_, err = runEnvbuilder(t, options{env: []string{
+			envbuilderEnv("GIT_URL", srv.URL),
+			envbuilderEnv("CACHE_REPO", testRepo),
+			envbuilderEnv("PUSH_IMAGE", "1"),
+		}})
+		require.NoError(t, err)
+
+		// Then: the image should be pushed
+		_, err = remote.Image(ref)
+		require.NoError(t, err, "expected image to be present after build + push")
+
+		// Then: re-running envbuilder with GET_CACHED_IMAGE should succeed
+		_, err = runEnvbuilder(t, options{env: []string{
+			envbuilderEnv("GIT_URL", srv.URL),
+			envbuilderEnv("CACHE_REPO", testRepo),
+			envbuilderEnv("GET_CACHED_IMAGE", "1"),
+		}})
+		require.NoError(t, err)
+	})
+
+	t.Run("CacheAndPushMultistage", func(t *testing.T) {
+		// Currently fails with:
+		//         /home/coder/src/coder/envbuilder/integration/integration_test.go:1417: "error: unable to get cached image: error fake building stage: failed to optimize instructions: failed to get files used from context: failed to get fileinfo for /.envbuilder/0/root/date.txt: lstat /.envbuilder/0/root/date.txt: no such file or directory"
+		// /home/coder/src/coder/envbuilder/integration/integration_test.go:1156:
+		// Error Trace:	/home/coder/src/coder/envbuilder/integration/integration_test.go:1156
+		// Error:      	Received unexpected error:
+		// error: unable to get cached image: error fake building stage: failed to optimize instructions: failed to get files used from context: failed to get fileinfo for /.envbuilder/0/root/date.txt: lstat /.envbuilder/0/root/date.txt: no such file or directory
+		// Test:       	TestPushImage/CacheAndPushMultistage
+		t.Skip("TODO: https://github.com/coder/envbuilder/issues/230")
+		t.Parallel()
+
+		srv := createGitServer(t, gitServerOptions{
+			files: map[string]string{
+				"Dockerfile": fmt.Sprintf(`FROM %s AS a
+RUN date --utc > /root/date.txt
+FROM %s as b
+COPY --from=a /root/date.txt /date.txt`, testImageAlpine, testImageAlpine),
+			},
+		})
+
+		// Given: an empty registry
+		testReg := setupInMemoryRegistry(t)
+		testRepo := testReg + "/test"
+		ref, err := name.ParseReference(testRepo + ":latest")
+		require.NoError(t, err)
+		_, err = remote.Image(ref)
+		require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push")
+
+		// When: we run envbuilder with GET_CACHED_IMAGE
+		_, err = runEnvbuilder(t, options{env: []string{
+			envbuilderEnv("GIT_URL", srv.URL),
+			envbuilderEnv("CACHE_REPO", testRepo),
+			envbuilderEnv("GET_CACHED_IMAGE", "1"),
+			envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"),
+		}})
+		require.ErrorContains(t, err, "not supported in fake build")
+		// Then: it should fail to build the image and nothing should be pushed
+		_, err = remote.Image(ref)
+		require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push")
+
+		// When: we run envbuilder with PUSH_IMAGE set
+		ctrID, err := runEnvbuilder(t, options{env: []string{
+			envbuilderEnv("GIT_URL", srv.URL),
+			envbuilderEnv("CACHE_REPO", testRepo),
+			envbuilderEnv("PUSH_IMAGE", "1"),
+			envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"),
+		}})
+		require.NoError(t, err)
+		// Then: The file copied from stage a should be present
+		out := execContainer(t, ctrID, "cat /date.txt")
+		require.NotEmpty(t, out)
+
+		// Then: the image should be pushed
+		_, err = remote.Image(ref)
+		require.NoError(t, err, "expected image to be present after build + push")
+
+		// Then: re-running envbuilder with GET_CACHED_IMAGE should succeed
+		_, err = runEnvbuilder(t, options{env: []string{
+			envbuilderEnv("GIT_URL", srv.URL),
+			envbuilderEnv("CACHE_REPO", testRepo),
+			envbuilderEnv("GET_CACHED_IMAGE", "1"),
+			envbuilderEnv("DOCKERFILE_PATH", "Dockerfile"),
+		}})
+		require.NoError(t, err)
+	})
+
+	t.Run("PushImageRequiresCache", func(t *testing.T) {
+		t.Parallel()
+
+		srv := createGitServer(t, gitServerOptions{
+			files: map[string]string{
+				".devcontainer/Dockerfile": fmt.Sprintf("FROM %s\nRUN date --utc > /root/date.txt", testImageAlpine),
+				".devcontainer/devcontainer.json": `{
+				"name": "Test",
+				"build": {
+					"dockerfile": "Dockerfile"
+				},
+			}`,
+			},
+		})
+
+		// When: we run envbuilder with PUSH_IMAGE set but no cache repo set
+		_, err := runEnvbuilder(t, options{env: []string{
+			envbuilderEnv("GIT_URL", srv.URL),
+			envbuilderEnv("PUSH_IMAGE", "1"),
+		}})
+
+		// Then: Envbuilder should fail explicitly, as it does not make sense to
+		// specify PUSH_IMAGE
+		require.ErrorContains(t, err, "--cache-repo must be set when using --push-image")
+	})
+
+	t.Run("PushErr", func(t *testing.T) {
+		t.Parallel()
+
+		srv := createGitServer(t, gitServerOptions{
+			files: map[string]string{
+				".devcontainer/Dockerfile": fmt.Sprintf("FROM %s\nRUN date --utc > /root/date.txt", testImageAlpine),
+				".devcontainer/devcontainer.json": `{
+				"name": "Test",
+				"build": {
+					"dockerfile": "Dockerfile"
+				},
+			}`,
+			},
+		})
+
+		// Given: registry is not set up (in this case, not a registry)
+		notRegSrv := httptest.NewServer(http.NotFoundHandler())
+		notRegURL := strings.TrimPrefix(notRegSrv.URL, "http://") + "/test"
+
+		// When: we run envbuilder with PUSH_IMAGE set
+		_, err := runEnvbuilder(t, options{env: []string{
+			envbuilderEnv("GIT_URL", srv.URL),
+			envbuilderEnv("CACHE_REPO", notRegURL),
+			envbuilderEnv("PUSH_IMAGE", "1"),
+		}})
+
+		// Then: envbuilder should fail with a descriptive error
+		require.ErrorContains(t, err, "failed to push to destination")
+	})
+}
+
+func setupInMemoryRegistry(t *testing.T) string {
+	t.Helper()
+	tempDir := t.TempDir()
+	testReg := registry.New(registry.WithBlobHandler(registry.NewDiskBlobHandler(tempDir)))
+	regSrv := httptest.NewServer(testReg)
+	t.Cleanup(func() { regSrv.Close() })
+	regSrvURL, err := url.Parse(regSrv.URL)
+	require.NoError(t, err)
+	return fmt.Sprintf("localhost:%s", regSrvURL.Port())
+}
+
 // TestMain runs before all tests to build the envbuilder image.
 func TestMain(m *testing.M) {
 	checkTestRegistry()