feat: add support for copying certs to dockerd's registry directory (#101)

Author: sreya

cli/clitest/cli.go

@@ -49,7 +49,7 @@ func New(t *testing.T, cmd string, args ...string) (context.Context, *cobra.Comm
 
 	var (
 		execer = NewFakeExecer()
-		fs     = NewMemFS()
+		fs     = xunixfake.NewMemFS()
 		mnt    = &mount.FakeMounter{}
 		client = NewFakeDockerClient()
 		iface  = GetNetLink(t)

cli/clitest/fake.go

@@ -8,21 +8,13 @@ import (
 	"strings"
 
 	dockertypes "github.com/docker/docker/api/types"
-	"github.com/spf13/afero"
 	testingexec "k8s.io/utils/exec/testing"
 
 	"github.com/coder/envbox/dockerutil"
 	"github.com/coder/envbox/dockerutil/dockerfake"
 	"github.com/coder/envbox/xunix/xunixfake"
 )
 
-func NewMemFS() *xunixfake.MemFS {
-	return &xunixfake.MemFS{
-		MemMapFs: &afero.MemMapFs{},
-		Owner:    map[string]xunixfake.FileOwner{},
-	}
-}
-
 func NewFakeExecer() *xunixfake.FakeExec {
 	return &xunixfake.FakeExec{
 		Commands: map[string]*xunixfake.FakeCmd{},

cli/docker.go

@@ -157,20 +157,20 @@ func dockerCmd() *cobra.Command {
 		RunE: func(cmd *cobra.Command, args []string) (err error) {
 			var (
 				ctx                  = cmd.Context()
-				log                  = slog.Make(slogjson.Sink(io.Discard))
-				blog buildlog.Logger = buildlog.NopLogger{}
+				log                  = slog.Make(slogjson.Sink(cmd.ErrOrStderr()), slogkubeterminate.Make()).Leveled(slog.LevelDebug)
+				blog buildlog.Logger = buildlog.JSONLogger{Encoder: json.NewEncoder(os.Stderr)}
 			)
 
+			if flags.noStartupLogs {
+				log = slog.Make(slogjson.Sink(io.Discard))
+				blog = buildlog.NopLogger{}
+			}
+
 			httpClient, err := xhttp.Client(log, flags.extraCertsPath)
 			if err != nil {
 				return xerrors.Errorf("http client: %w", err)
 			}
 
-			if !flags.noStartupLogs {
-				log = slog.Make(slogjson.Sink(cmd.ErrOrStderr()), slogkubeterminate.Make()).Leveled(slog.LevelDebug)
-				blog = buildlog.JSONLogger{Encoder: json.NewEncoder(os.Stderr)}
-			}
-
 			if !flags.noStartupLogs && flags.agentToken != "" && flags.coderURL != "" {
 				coderURL, err := url.Parse(flags.coderURL)
 				if err != nil {
@@ -197,7 +197,6 @@ func dockerCmd() *cobra.Command {
 				}
 			}(&err)
 
-			blog.Info("Waiting for dockerd to startup...")
 			sysboxArgs := []string{}
 			if flags.disableIDMappedMount {
 				sysboxArgs = append(sysboxArgs, "--disable-idmapped-mount")
@@ -231,6 +230,7 @@ func dockerCmd() *cobra.Command {
 
 			log.Debug(ctx, "starting dockerd", slog.F("args", args))
 
+			blog.Info("Waiting for sysbox processes to startup...")
 			dockerd := background.New(ctx, log, "dockerd", dargs...)
 			err = dockerd.Start()
 			if err != nil {
@@ -289,11 +289,32 @@ func dockerCmd() *cobra.Command {
 			// We wait for the daemon after spawning the goroutine in case
 			// startup causes the daemon to encounter encounter a 'no space left
 			// on device' error.
+			blog.Info("Waiting for dockerd to startup...")
 			err = dockerutil.WaitForDaemon(ctx, client)
 			if err != nil {
 				return xerrors.Errorf("wait for dockerd: %w", err)
 			}
 
+			if flags.extraCertsPath != "" {
+				// Parse the registry from the inner image
+				registry, err := name.ParseReference(flags.innerImage)
+				if err != nil {
+					return xerrors.Errorf("invalid image: %w", err)
+				}
+				registryName := registry.Context().RegistryStr()
+
+				// Write certificates for the registry
+				err = dockerutil.WriteCertsForRegistry(ctx, registryName, flags.extraCertsPath)
+				if err != nil {
+					return xerrors.Errorf("write certs for registry: %w", err)
+				}
+
+				blog.Infof("Successfully copied certificates from %q to %q", flags.extraCertsPath, filepath.Join("/etc/docker/certs.d", registryName))
+				log.Debug(ctx, "wrote certificates for registry", slog.F("registry", registryName),
+					slog.F("extra_certs_path", flags.extraCertsPath),
+				)
+			}
+
 			err = runDockerCVM(ctx, log, client, blog, flags)
 			if err != nil {
 				// It's possible we failed because we ran out of disk while

dockerutil/image.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"strings"
 	"time"
 
 	dockertypes "github.com/docker/docker/api/types"
@@ -65,6 +66,14 @@ func PullImage(ctx context.Context, config *PullImageConfig) error {
 	}
 
 	err = pullImageFn()
+	// We should bail early if we're going to fail due to a
+	// certificate error. We can't xerrors.As here since this is
+	// returned from the daemon so the client is reporting
+	// essentially an unwrapped error.
+	if isTLSVerificationErr(err) {
+		return err
+	}
+
 	if err == nil {
 		return nil
 	}
@@ -253,3 +262,7 @@ func DefaultLogImagePullFn(log buildlog.Logger) func(ImagePullEvent) error {
 		return nil
 	}
 }
+
+func isTLSVerificationErr(err error) bool {
+	return err != nil && strings.Contains(err.Error(), "tls: failed to verify certificate: x509: certificate signed by unknown authority")
+}

dockerutil/registry.go

@@ -0,0 +1,92 @@
+package dockerutil
+
+import (
+	"context"
+	"io"
+	"path/filepath"
+
+	"github.com/spf13/afero"
+	"golang.org/x/xerrors"
+
+	"github.com/coder/envbox/xunix"
+)
+
+// WriteCertsForRegistry writes the certificates found in the provided directory
+// to the correct subdirectory that the Docker daemon uses when pulling images
+// from the specified private registry.
+func WriteCertsForRegistry(ctx context.Context, registryName, certsDir string) error {
+	fs := xunix.GetFS(ctx)
+
+	// Docker certs directory.
+	registryCertsDir := filepath.Join("/etc/docker/certs.d", registryName)
+
+	// If the directory already exists it means someone
+	// has either wrapped the image or has mounted in certs
+	// manually. We should assume the user knows what they're
+	// doing and avoid mucking with their solution.
+	if _, err := fs.Stat(registryCertsDir); err == nil {
+		return nil
+	}
+
+	// Ensure the registry certs directory exists.
+	err := fs.MkdirAll(registryCertsDir, 0o755)
+	if err != nil {
+		return xerrors.Errorf("create registry certs directory: %w", err)
+	}
+
+	// Check if certsDir is a file.
+	fileInfo, err := fs.Stat(certsDir)
+	if err != nil {
+		return xerrors.Errorf("stat certs directory/file: %w", err)
+	}
+
+	if !fileInfo.IsDir() {
+		// If it's a file, copy it directly
+		err = copyCertFile(fs, certsDir, filepath.Join(registryCertsDir, "ca.crt"))
+		if err != nil {
+			return xerrors.Errorf("copy cert file: %w", err)
+		}
+		return nil
+	}
+
+	// If it's a directory, copy all cert files in the root of the directory
+	entries, err := afero.ReadDir(fs, certsDir)
+	if err != nil {
+		return xerrors.Errorf("read certs directory: %w", err)
+	}
+
+	for _, entry := range entries {
+		if entry.IsDir() {
+			continue
+		}
+		srcPath := filepath.Join(certsDir, entry.Name())
+		dstPath := filepath.Join(registryCertsDir, entry.Name())
+		err = copyCertFile(fs, srcPath, dstPath)
+		if err != nil {
+			return xerrors.Errorf("copy cert file %s: %w", entry.Name(), err)
+		}
+	}
+
+	return nil
+}
+
+func copyCertFile(fs xunix.FS, src, dst string) error {
+	srcFile, err := fs.Open(src)
+	if err != nil {
+		return xerrors.Errorf("open source file: %w", err)
+	}
+	defer srcFile.Close()
+
+	dstFile, err := fs.Create(dst)
+	if err != nil {
+		return xerrors.Errorf("create destination file: %w", err)
+	}
+	defer dstFile.Close()
+
+	_, err = io.Copy(dstFile, srcFile)
+	if err != nil {
+		return xerrors.Errorf("copy file contents: %w", err)
+	}
+
+	return nil
+}

dockerutil/registry_test.go

@@ -0,0 +1,100 @@
+package dockerutil_test
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/coder/envbox/dockerutil"
+	"github.com/coder/envbox/xunix"
+	"github.com/coder/envbox/xunix/xunixfake"
+)
+
+func TestWriteCertsForRegistry(t *testing.T) {
+	t.Parallel()
+
+	t.Run("SingleCertFile", func(t *testing.T) {
+		t.Parallel()
+		// Test setup
+		fs := xunixfake.NewMemFS()
+		ctx := xunix.WithFS(context.Background(), fs)
+
+		// Create a test certificate file
+		certContent := []byte("test certificate content")
+		err := afero.WriteFile(fs, "/certs/ca.crt", certContent, 0o644)
+		require.NoError(t, err)
+
+		// Run the function
+		err = dockerutil.WriteCertsForRegistry(ctx, "test.registry.com", "/certs/ca.crt")
+		require.NoError(t, err)
+
+		// Check the result
+		copiedContent, err := afero.ReadFile(fs, "/etc/docker/certs.d/test.registry.com/ca.crt")
+		require.NoError(t, err)
+		assert.Equal(t, certContent, copiedContent)
+	})
+
+	t.Run("MultipleCertFiles", func(t *testing.T) {
+		t.Parallel()
+		// Test setup
+		fs := xunixfake.NewMemFS()
+		ctx := xunix.WithFS(context.Background(), fs)
+
+		// Create test certificate files
+		certFiles := []string{"ca.crt", "client.cert", "client.key"}
+		for _, file := range certFiles {
+			err := afero.WriteFile(fs, filepath.Join("/certs", file), []byte("content of "+file), 0o644)
+			require.NoError(t, err)
+		}
+
+		// Run the function
+		err := dockerutil.WriteCertsForRegistry(ctx, "test.registry.com", "/certs")
+		require.NoError(t, err)
+
+		// Check the results
+		for _, file := range certFiles {
+			copiedContent, err := afero.ReadFile(fs, filepath.Join("/etc/docker/certs.d/test.registry.com", file))
+			require.NoError(t, err)
+			assert.Equal(t, []byte("content of "+file), copiedContent)
+		}
+	})
+	t.Run("ExistingRegistryCertsDir", func(t *testing.T) {
+		t.Parallel()
+		// Test setup
+		fs := xunixfake.NewMemFS()
+		ctx := xunix.WithFS(context.Background(), fs)
+
+		// Create an existing registry certs directory
+		registryCertsDir := "/etc/docker/certs.d/test.registry.com"
+		err := fs.MkdirAll(registryCertsDir, 0o755)
+		require.NoError(t, err)
+
+		// Create a file in the existing directory
+		existingContent := []byte("existing certificate content")
+		err = afero.WriteFile(fs, filepath.Join(registryCertsDir, "existing.crt"), existingContent, 0o644)
+		require.NoError(t, err)
+
+		// Create a test certificate file in the source directory
+		certContent := []byte("new certificate content")
+		err = afero.WriteFile(fs, "/certs/ca.crt", certContent, 0o644)
+		require.NoError(t, err)
+
+		// Run the function
+		err = dockerutil.WriteCertsForRegistry(ctx, "test.registry.com", "/certs")
+		require.NoError(t, err)
+
+		// Check that the existing file was not modified
+		existingFileContent, err := afero.ReadFile(fs, filepath.Join(registryCertsDir, "existing.crt"))
+		require.NoError(t, err)
+		assert.Equal(t, existingContent, existingFileContent)
+
+		// Check that the new file was not copied
+		_, err = fs.Stat(filepath.Join(registryCertsDir, "ca.crt"))
+		assert.True(t, os.IsNotExist(err), "New certificate file should not have been copied")
+	})
+}