feat: add option to specify additional certs (#98)

Author: sreya

README.md

@@ -27,6 +27,7 @@ The environment variables can be used to configure various aspects of the inner
 | `CODER_CPUS`                   | Dictates the number of CPUs to allocate the inner container. It is recommended to set this using the Kubernetes [Downward API](https://kubernetes.io/docs/tasks/inject-data-application/environment-variable-expose-pod-information/#use-container-fields-as-values-for-environment-variables).                                                                                                                                                                                                                                | false    |
 | `CODER_MEMORY`                 | Dictates the max memory (in bytes) to allocate the inner container. It is recommended to set this using the Kubernetes [Downward API](https://kubernetes.io/docs/tasks/inject-data-application/environment-variable-expose-pod-information/#use-container-fields-as-values-for-environment-variables).                                                                                                                                                                                                                         | false    |
 | `CODER_DISABLE_IDMAPPED_MOUNT` | Disables idmapped mounts in sysbox. For more information, see the [Sysbox Documentation](https://github.com/nestybox/sysbox/blob/master/docs/user-guide/configuration.md#disabling-id-mapped-mounts-on-sysbox).                                                                                                                                                                                                                                                                                                                | false    |
+| `CODER_EXTRA_CERTS_PATH`       | A path to a file or directory containing CA certificates that should be made when communicating to external services (e.g. the Coder control plane or a Docker registry)                                                                                                                                                                                                                                                                                                                                                       | false    |
 
 ## Coder Template
 

buildlog/coder.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"net/http"
 	"net/url"
 	"time"
 
@@ -143,9 +144,10 @@ func newAgentClientV2(ctx context.Context, logger slog.Logger, client *agentsdk.
 	}, nil
 }
 
-func OpenCoderClient(ctx context.Context, accessURL *url.URL, logger slog.Logger, token string) (CoderClient, error) {
+func OpenCoderClient(ctx context.Context, logger slog.Logger, accessURL *url.URL, hc *http.Client, token string) (CoderClient, error) {
 	client := agentsdk.New(accessURL)
 	client.SetSessionToken(token)
+	client.SDK.HTTPClient = hc
 
 	resp, err := client.SDK.BuildInfo(ctx)
 	if err != nil {

cli/docker.go

@@ -28,6 +28,7 @@ import (
 	"github.com/coder/envbox/dockerutil"
 	"github.com/coder/envbox/slogkubeterminate"
 	"github.com/coder/envbox/sysboxutil"
+	"github.com/coder/envbox/xhttp"
 	"github.com/coder/envbox/xunix"
 )
 
@@ -101,6 +102,7 @@ var (
 	EnvDockerConfig         = "CODER_DOCKER_CONFIG"
 	EnvDebug                = "CODER_DEBUG"
 	EnvDisableIDMappedMount = "CODER_DISABLE_IDMAPPED_MOUNT"
+	EnvExtraCertsPath       = "CODER_EXTRA_CERTS_PATH"
 )
 
 var envboxPrivateMounts = map[string]struct{}{
@@ -138,6 +140,7 @@ type flags struct {
 	cpus                 int
 	memory               int
 	disableIDMappedMount bool
+	extraCertsPath       string
 
 	// Test flags.
 	noStartupLogs bool
@@ -158,6 +161,11 @@ func dockerCmd() *cobra.Command {
 				blog buildlog.Logger = buildlog.NopLogger{}
 			)
 
+			httpClient, err := xhttp.Client(log, flags.extraCertsPath)
+			if err != nil {
+				return xerrors.Errorf("http client: %w", err)
+			}
+
 			if !flags.noStartupLogs {
 				log = slog.Make(slogjson.Sink(cmd.ErrOrStderr()), slogkubeterminate.Make()).Leveled(slog.LevelDebug)
 				blog = buildlog.JSONLogger{Encoder: json.NewEncoder(os.Stderr)}
@@ -169,7 +177,7 @@ func dockerCmd() *cobra.Command {
 					return xerrors.Errorf("parse coder URL %q: %w", flags.coderURL, err)
 				}
 
-				agent, err := buildlog.OpenCoderClient(ctx, coderURL, log, flags.agentToken)
+				agent, err := buildlog.OpenCoderClient(ctx, log, coderURL, httpClient, flags.agentToken)
 				if err != nil {
 					// Don't fail workspace startup on
 					// an inability to push build logs.
@@ -349,6 +357,7 @@ func dockerCmd() *cobra.Command {
 	cliflag.IntVarP(cmd.Flags(), &flags.cpus, "cpus", "", EnvCPUs, 0, "Number of CPUs to allocate inner container. e.g. 2")
 	cliflag.IntVarP(cmd.Flags(), &flags.memory, "memory", "", EnvMemory, 0, "Max memory to allocate to the inner container in bytes.")
 	cliflag.BoolVarP(cmd.Flags(), &flags.disableIDMappedMount, "disable-idmapped-mount", "", EnvDisableIDMappedMount, false, "Disable idmapped mounts in sysbox. Note that you may need an alternative (e.g. shiftfs).")
+	cliflag.StringVarP(cmd.Flags(), &flags.extraCertsPath, "extra-certs-path", "", EnvExtraCertsPath, "", "The path to a directory or file containing extra CA certificates.")
 
 	// Test flags.
 	cliflag.BoolVarP(cmd.Flags(), &flags.noStartupLogs, "no-startup-log", "", "", false, "Do not log startup logs. Useful for testing.")

go.mod

@@ -6,6 +6,8 @@ go 1.22.4
 // https://github.com/tailscale/tailscale/compare/main...coder:tailscale:main
 replace tailscale.com => github.com/coder/tailscale v1.1.1-0.20240530071520-1ac63d3a4ee3
 
+replace github.com/gliderlabs/ssh => github.com/coder/ssh v0.0.0-20231128192721-70855dedb788
+
 require (
 	cdr.dev/slog v1.6.2-0.20240126064726-20367d4aede6
 	github.com/coder/coder/v2 v2.12.0

integration/docker_test.go

@@ -5,6 +5,7 @@ package integration_test
 
 import (
 	"fmt"
+	"net"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -37,9 +38,9 @@ func TestDocker(t *testing.T) {
 		runEnvbox := func() *dockertest.Resource {
 			// Run the envbox container.
 			resource := integrationtest.RunEnvbox(t, pool, &integrationtest.CreateDockerCVMConfig{
-				Image:    integrationtest.DockerdImage,
-				Username: "root",
-				Binds:    binds,
+				Image:       integrationtest.DockerdImage,
+				Username:    "root",
+				OuterMounts: binds,
 			})
 
 			// Wait for the inner container's docker daemon.
@@ -98,8 +99,8 @@ func TestDocker(t *testing.T) {
 		require.NoError(t, err)
 
 		binds = append(binds,
-			bindMount(homeDir, "/home/coder", false),
-			bindMount(secretDir, "/var/secrets", true),
+			integrationtest.BindMount(homeDir, "/home/coder", false),
+			integrationtest.BindMount(secretDir, "/var/secrets", true),
 		)
 
 		var (
@@ -144,7 +145,7 @@ func TestDocker(t *testing.T) {
 			Username:        "coder",
 			InnerEnvFilter:  envFilter,
 			Envs:            envs,
-			Binds:           binds,
+			OuterMounts:     binds,
 			AddFUSE:         true,
 			AddTUN:          true,
 			BootstrapScript: bootstrapScript,
@@ -272,6 +273,83 @@ func TestDocker(t *testing.T) {
 		require.NoError(t, err)
 		require.Equal(t, expectedHostname, strings.TrimSpace(string(hostname)))
 	})
+
+	t.Run("SelfSignedCerts", func(t *testing.T) {
+		t.Parallel()
+
+		var (
+			dir   = integrationtest.TmpDir(t)
+			binds = integrationtest.DefaultBinds(t, dir)
+		)
+
+		pool, err := dockertest.NewPool("")
+		require.NoError(t, err)
+
+		// Create some listeners for the Docker and Coder
+		// services we'll be running with self signed certs.
+		bridgeIP := integrationtest.DockerBridgeIP(t)
+		coderListener, err := net.Listen("tcp", fmt.Sprintf("%s:0", bridgeIP))
+		require.NoError(t, err)
+		defer coderListener.Close()
+		coderAddr := tcpAddr(t, coderListener)
+
+		registryListener, err := net.Listen("tcp", fmt.Sprintf("%s:0", bridgeIP))
+		require.NoError(t, err)
+		err = registryListener.Close()
+		require.NoError(t, err)
+		registryAddr := tcpAddr(t, registryListener)
+
+		coderCert := integrationtest.GenerateTLSCertificate(t, "host.docker.internal", coderAddr.IP.String())
+		dockerCert := integrationtest.GenerateTLSCertificate(t, "host.docker.internal", registryAddr.IP.String())
+
+		// Startup our fake Coder "control-plane".
+		recorder := integrationtest.FakeBuildLogRecorder(t, coderListener, coderCert)
+
+		certDir := integrationtest.MkdirAll(t, dir, "certs")
+
+		// Write the Coder cert disk.
+		coderCertPath := filepath.Join(certDir, "coder_cert.pem")
+		coderKeyPath := filepath.Join(certDir, "coder_key.pem")
+		integrationtest.WriteCertificate(t, coderCert, coderCertPath, coderKeyPath)
+		coderCertMount := integrationtest.BindMount(certDir, "/tmp/certs", false)
+
+		// Write the Registry cert to disk.
+		regCertPath := filepath.Join(certDir, "registry_cert.crt")
+		regKeyPath := filepath.Join(certDir, "registry_key.pem")
+		integrationtest.WriteCertificate(t, dockerCert, regCertPath, regKeyPath)
+
+		// Start up the docker registry and push an image
+		// to it that we can reference.
+		image := integrationtest.RunLocalDockerRegistry(t, pool, integrationtest.RegistryConfig{
+			HostCertPath: regCertPath,
+			HostKeyPath:  regKeyPath,
+			Image:        integrationtest.UbuntuImage,
+			TLSPort:      strconv.Itoa(registryAddr.Port),
+		})
+
+		// Mount the cert into the expected location
+		// for the Envbox Docker daemon.
+		regCAPath := filepath.Join("/etc/docker/certs.d", image.Registry(), "ca.crt")
+		registryCAMount := integrationtest.BindMount(regCertPath, regCAPath, false)
+
+		envs := []string{
+			integrationtest.EnvVar(cli.EnvAgentToken, "faketoken"),
+			integrationtest.EnvVar(cli.EnvAgentURL, fmt.Sprintf("https://%s:%d", "host.docker.internal", coderAddr.Port)),
+			integrationtest.EnvVar(cli.EnvExtraCertsPath, "/tmp/certs"),
+		}
+
+		// Run the envbox container.
+		_ = integrationtest.RunEnvbox(t, pool, &integrationtest.CreateDockerCVMConfig{
+			Image:       image.String(),
+			Username:    "coder",
+			Envs:        envs,
+			OuterMounts: append(binds, coderCertMount, registryCAMount),
+		})
+
+		// This indicates we've made it all the way to end
+		// of the logs we attempt to push.
+		require.True(t, recorder.ContainsLog("Bootstrapping workspace..."))
+	})
 }
 
 func requireSliceNoContains(t *testing.T, ss []string, els ...string) {
@@ -297,9 +375,10 @@ func requireSliceContains(t *testing.T, ss []string, els ...string) {
 	}
 }
 
-func bindMount(src, dest string, ro bool) string {
-	if ro {
-		return fmt.Sprintf("%s:%s:%s", src, dest, "ro")
-	}
-	return fmt.Sprintf("%s:%s", src, dest)
+func tcpAddr(t testing.TB, l net.Listener) *net.TCPAddr {
+	t.Helper()
+
+	tcpAddr, ok := l.Addr().(*net.TCPAddr)
+	require.True(t, ok)
+	return tcpAddr
 }

integration/integrationtest/certs.go

@@ -0,0 +1,82 @@
+package integrationtest
+
+import (
+	"bytes"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"math/big"
+	"net"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func GenerateTLSCertificate(t testing.TB, commonName string, ipAddr string) tls.Certificate {
+	t.Helper()
+
+	privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	require.NoError(t, err)
+	template := x509.Certificate{
+		SerialNumber: big.NewInt(1),
+		Subject: pkix.Name{
+			Organization: []string{"Acme Co"},
+			CommonName:   commonName,
+		},
+		DNSNames:  []string{commonName},
+		NotBefore: time.Now(),
+		NotAfter:  time.Now().Add(time.Hour * 24 * 180),
+
+		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+		IPAddresses:           []net.IP{net.ParseIP(ipAddr)},
+		IsCA:                  true,
+	}
+
+	derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &privateKey.PublicKey, privateKey)
+	require.NoError(t, err)
+	var certFile bytes.Buffer
+	require.NoError(t, err)
+	_, err = certFile.Write(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derBytes}))
+	require.NoError(t, err)
+	privateKeyBytes, err := x509.MarshalPKCS8PrivateKey(privateKey)
+	require.NoError(t, err)
+	var keyFile bytes.Buffer
+	err = pem.Encode(&keyFile, &pem.Block{Type: "PRIVATE KEY", Bytes: privateKeyBytes})
+	require.NoError(t, err)
+	cert, err := tls.X509KeyPair(certFile.Bytes(), keyFile.Bytes())
+	require.NoError(t, err)
+	return cert
+}
+
+func writePEM(t testing.TB, path string, typ string, contents []byte) {
+	t.Helper()
+
+	f, err := os.Create(path)
+	require.NoError(t, err)
+	defer f.Close()
+
+	err = pem.Encode(f, &pem.Block{
+		Type:  typ,
+		Bytes: contents,
+	})
+	require.NoError(t, err)
+}
+
+func WriteCertificate(t testing.TB, c tls.Certificate, certPath, keyPath string) {
+	require.Len(t, c.Certificate, 1, "expecting 1 certificate")
+	key, err := x509.MarshalPKCS8PrivateKey(c.PrivateKey)
+	require.NoError(t, err)
+
+	cert := c.Certificate[0]
+
+	writePEM(t, keyPath, "PRIVATE KEY", key)
+	writePEM(t, certPath, "CERTIFICATE", cert)
+}