feat: support cloning over SSH via private key auth

README.md

@@ -288,6 +288,8 @@ On MacOS or Windows systems, we recommend either using a VM or the provided `.de
 | `--git-clone-single-branch` | `GIT_CLONE_SINGLE_BRANCH` |  | Clone only a single branch of the Git repository. |
 | `--git-username` | `GIT_USERNAME` |  | The username to use for Git authentication. This is optional. |
 | `--git-password` | `GIT_PASSWORD` |  | The password to use for Git authentication. This is optional. |
+| `--git-ssh-private-key-path` | `GIT_SSH_PRIVATE_KEY_PATH` |  | Path to a SSH private key to be used for Git authentication. |
+| `--git-ssh-known-hosts-base64` | `GIT_SSH_KNOWN_HOSTS_BASE64` |  | Base64-encoded content of a known hosts file. If not specified, host keys will be scanned and logged, but not checked. |
 | `--git-http-proxy-url` | `GIT_HTTP_PROXY_URL` |  | The URL for the HTTP proxy. This is optional. |
 | `--workspace-folder` | `WORKSPACE_FOLDER` |  | The path to the workspace folder that will be built. This is optional. |
 | `--ssl-cert-base64` | `SSL_CERT_BASE64` |  | The content of an SSL cert file. This is useful for self-signed certificates. |

envbuilder.go

@@ -45,6 +45,7 @@ import (
 	"github.com/go-git/go-billy/v5/osfs"
 	"github.com/go-git/go-git/v5/plumbing/transport"
 	githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
+	gitssh "github.com/go-git/go-git/v5/plumbing/transport/ssh"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
 	"github.com/sirupsen/logrus"
@@ -156,6 +157,36 @@ func Run(ctx context.Context, options Options) error {
 		}
 	}
 
+	if options.GitURL != "" {
+		gitURLParsed, err := ParseGitURL(options.GitURL)
+		if err != nil {
+			return fmt.Errorf("invalid git URL: %w", err)
+		}
+		// If we're cloning over SSH, we need a known_hosts file.
+		if gitURLParsed.Scheme == "ssh" {
+			var knownHostsContent []byte
+			if options.GitSSHKnownHostsBase64 != "" {
+				if kh, err := base64.StdEncoding.DecodeString(options.GitSSHKnownHostsBase64); err != nil {
+					return fmt.Errorf("invalid known_hosts content: %w", err)
+				} else {
+					knownHostsContent = kh
+				}
+			} else {
+				// This is a best-effort.
+				kh, err := KeyScan(options.Logger, gitURLParsed)
+				if err == nil {
+					knownHostsContent = kh
+				}
+			}
+			knownHostsPath := filepath.Join(MagicDir, "known_hosts")
+			if err := os.WriteFile(knownHostsPath, knownHostsContent, 0644); err != nil {
+				return fmt.Errorf("write known_hosts file: %w", err)
+			}
+			// go-git will read this file to validate the server host keys.
+			_ = os.Setenv("SSH_KNOWN_HOSTS", knownHostsPath)
+		}
+	}
+
 	var fallbackErr error
 	var cloned bool
 	if options.GitURL != "" {
@@ -201,6 +232,15 @@ func Run(ctx context.Context, options Options) error {
 				Username: options.GitUsername,
 				Password: options.GitPassword,
 			}
+		} else if options.GitSSHPrivateKeyPath != "" {
+			signer, err := ReadPrivateKey(options.GitSSHPrivateKeyPath)
+			if err != nil {
+				return xerrors.Errorf("read private key: %w", err)
+			}
+			cloneOpts.RepoAuth = &gitssh.PublicKeys{
+				User:   "git",
+				Signer: signer,
+			}
 		}
 		if options.GitHTTPProxyURL != "" {
 			cloneOpts.ProxyOptions = transport.ProxyOptions{

git.go

@@ -1,11 +1,20 @@
 package envbuilder
 
 import (
+	"bytes"
 	"context"
+	"encoding/base64"
 	"errors"
 	"fmt"
+	"io"
+	"net"
 	"net/url"
+	"os"
+	"regexp"
+	"strconv"
+	"strings"
 
+	"github.com/coder/coder/v2/codersdk"
 	"github.com/go-git/go-billy/v5"
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/plumbing"
@@ -14,6 +23,7 @@ import (
 	"github.com/go-git/go-git/v5/plumbing/protocol/packp/sideband"
 	"github.com/go-git/go-git/v5/plumbing/transport"
 	"github.com/go-git/go-git/v5/storage/filesystem"
+	gossh "golang.org/x/crypto/ssh"
 )
 
 type CloneRepoOptions struct {
@@ -113,3 +123,92 @@ func CloneRepo(ctx context.Context, opts CloneRepoOptions) (bool, error) {
 	}
 	return true, nil
 }
+
+// ReadPrivateKey attempts to read an SSH private key from path
+// and returns an ssh.Signer.
+func ReadPrivateKey(path string) (gossh.Signer, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, fmt.Errorf("open private key file: %w", err)
+	}
+	bs, err := io.ReadAll(f)
+	if err != nil {
+		return nil, fmt.Errorf("read private key file: %w", err)
+	}
+	k, err := gossh.ParsePrivateKey(bs)
+	if err != nil {
+		return nil, fmt.Errorf("parse private key file: %w", err)
+	}
+	return k, nil
+}
+
+// KeyScan dials the server located at gitURL and fetches the SSH
+// public keys returned in a format accepted by known_hosts.
+// If no host keys found, returns an error.
+func KeyScan(log LoggerFunc, gitURL *url.URL) ([]byte, error) {
+	var buf bytes.Buffer
+	conf := &gossh.ClientConfig{
+		// Accept and record all host keys
+		HostKeyCallback: func(dialAddr string, _ net.Addr, key gossh.PublicKey) error {
+			kh, err := KnownHostsLine(dialAddr, key)
+			if err != nil {
+				return fmt.Errorf("ssh keyscan: generate known hosts line: %w", err)
+			}
+			log(codersdk.LogLevelInfo, "ssh keyscan: %s", kh)
+			buf.WriteString(kh)
+			buf.WriteString("\n")
+			return nil
+		},
+	}
+	dialAddr := hostPort(gitURL)
+	client, err := gossh.Dial("tcp", dialAddr, conf)
+	if err != nil {
+		// The dial may fail due to no authentication methods, but this is fine.
+		if netErr, ok := err.(net.Error); ok {
+			return nil, fmt.Errorf("ssh keyscan: dial %s: %w", dialAddr, netErr)
+		}
+		// Otherwise, assume success.
+	}
+	defer func() {
+		if client != nil {
+			_ = client.Close()
+		}
+	}()
+
+	bs := buf.Bytes()
+	if len(bs) == 0 {
+		return nil, fmt.Errorf("ssh keyscan: found no host keys")
+	}
+	return buf.Bytes(), nil
+}
+
+// KnownHostsLine generates a corresponding line for known_hosts
+// given a dial address in the format host:port and a public key.
+func KnownHostsLine(dialAddr string, key gossh.PublicKey) (string, error) {
+	if !strings.Contains(dialAddr, ":") {
+		return "", fmt.Errorf("invalid dialAddr, expected host:port")
+	}
+	h := strings.Split(dialAddr, ":")[0]
+	k64 := base64.StdEncoding.EncodeToString(key.Marshal())
+	return fmt.Sprintf("%s %s %s", h, key.Type(), k64), nil
+}
+
+func hostPort(u *url.URL) string {
+	p := 22 // assume default SSH port
+	if tmp, err := strconv.Atoi(u.Port()); err == nil {
+		p = tmp
+	}
+	return fmt.Sprintf("%s:%d", u.Host, p)
+}
+
+var schemaRe = regexp.MustCompile(`^[a-zA-Z]+://`)
+
+// ParseGitURL will normalize a git URL without a leading schema.
+// If no schema is provided, we will default to ssh://.
+func ParseGitURL(gitURL string) (*url.URL, error) {
+	if !schemaRe.MatchString(gitURL) {
+		gitURL = "ssh://" + gitURL
+
+	}
+	return url.Parse(gitURL)
+}

git_test.go

@@ -2,6 +2,7 @@ package envbuilder_test
 
 import (
 	"context"
+	"crypto/ed25519"
 	"fmt"
 	"io"
 	"net/http/httptest"
@@ -14,8 +15,12 @@ import (
 	"github.com/coder/envbuilder/testutil/gittest"
 	"github.com/go-git/go-billy/v5"
 	"github.com/go-git/go-billy/v5/memfs"
+	"github.com/go-git/go-billy/v5/osfs"
 	githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
+	gitssh "github.com/go-git/go-git/v5/plumbing/transport/ssh"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	gossh "golang.org/x/crypto/ssh"
 )
 
 func TestCloneRepo(t *testing.T) {
@@ -159,6 +164,136 @@ func TestCloneRepo(t *testing.T) {
 	}
 }
 
+func TestCloneRepoSSH(t *testing.T) {
+	t.Parallel()
+
+	t.Run("AuthSuccess", func(t *testing.T) {
+		t.Parallel()
+
+		// TODO: test the rest of the cloning flow. This just tests successful auth.
+		tmpDir := t.TempDir()
+		srvFS := osfs.New(tmpDir, osfs.WithChrootOS())
+
+		_ = gittest.NewRepo(t, srvFS, gittest.Commit(t, "README.md", "Hello, world!", "Wow!"))
+		key := randKeygen(t)
+		tr := gittest.NewServerSSH(t, srvFS, key.PublicKey())
+		gitURL := tr.String()
+		clientFS := memfs.New()
+
+		cloned, err := envbuilder.CloneRepo(context.Background(), envbuilder.CloneRepoOptions{
+			Path:    "/workspace",
+			RepoURL: gitURL,
+			Storage: clientFS,
+			RepoAuth: &gitssh.PublicKeys{
+				User:   "",
+				Signer: key,
+				HostKeyCallbackHelper: gitssh.HostKeyCallbackHelper{
+					// Not testing host keys here.
+					HostKeyCallback: gossh.InsecureIgnoreHostKey(),
+				},
+			},
+		})
+		// TODO: ideally, we want to test the entire cloning flow.
+		// For now, this indicates successful ssh key auth.
+		require.ErrorContains(t, err, "repository not found")
+		require.False(t, cloned)
+	})
+
+	t.Run("AuthFailure", func(t *testing.T) {
+		t.Parallel()
+
+		tmpDir := t.TempDir()
+		srvFS := osfs.New(tmpDir, osfs.WithChrootOS())
+
+		_ = gittest.NewRepo(t, srvFS, gittest.Commit(t, "README.md", "Hello, world!", "Wow!"))
+		key := randKeygen(t)
+		tr := gittest.NewServerSSH(t, srvFS, key.PublicKey())
+		gitURL := tr.String()
+		clientFS := memfs.New()
+
+		anotherKey := randKeygen(t)
+		cloned, err := envbuilder.CloneRepo(context.Background(), envbuilder.CloneRepoOptions{
+			Path:    "/workspace",
+			RepoURL: gitURL,
+			Storage: clientFS,
+			RepoAuth: &gitssh.PublicKeys{
+				User:   "",
+				Signer: anotherKey,
+				HostKeyCallbackHelper: gitssh.HostKeyCallbackHelper{
+					// Not testing host keys here.
+					HostKeyCallback: gossh.InsecureIgnoreHostKey(),
+				},
+			},
+		})
+		require.ErrorContains(t, err, "handshake failed")
+		require.False(t, cloned)
+	})
+
+	// nolint: paralleltest // t.Setenv
+	t.Run("PrivateKeyHostKeyMismatch", func(t *testing.T) {
+		t.Parallel()
+
+		tmpDir := t.TempDir()
+		srvFS := osfs.New(tmpDir, osfs.WithChrootOS())
+
+		_ = gittest.NewRepo(t, srvFS, gittest.Commit(t, "README.md", "Hello, world!", "Wow!"))
+		key := randKeygen(t)
+		tr := gittest.NewServerSSH(t, srvFS, key.PublicKey())
+		gitURL := tr.String()
+		clientFS := memfs.New()
+
+		cloned, err := envbuilder.CloneRepo(context.Background(), envbuilder.CloneRepoOptions{
+			Path:    "/workspace",
+			RepoURL: gitURL,
+			Storage: clientFS,
+			RepoAuth: &gitssh.PublicKeys{
+				User:   "",
+				Signer: key,
+				HostKeyCallbackHelper: gitssh.HostKeyCallbackHelper{
+					HostKeyCallback: gossh.FixedHostKey(randKeygen(t).PublicKey()),
+				},
+			},
+		})
+		require.ErrorContains(t, err, "ssh: host key mismatch")
+		require.False(t, cloned)
+	})
+}
+
+func TestParseGitURL(t *testing.T) {
+	t.Parallel()
+
+	for _, tc := range []struct {
+		url           string
+		expected      string
+		expectedError string
+	}{
+		{
+			url:      "https://user:[email protected]/repo",
+			expected: "https://user:[email protected]/repo",
+		},
+		{
+			url:      "http://user:[email protected]/repo",
+			expected: "http://user:[email protected]/repo",
+		},
+		{
+			url:      "ssh://[email protected]/repo",
+			expected: "ssh://[email protected]/repo",
+		},
+		{
+			url:      "[email protected]/repo",
+			expected: "ssh://[email protected]/repo",
+		},
+	} {
+		actual, err := envbuilder.ParseGitURL(tc.url)
+		if tc.expectedError == "" {
+			assert.NoError(t, err)
+			assert.Equal(t, tc.expected, actual.String())
+			continue
+		}
+		assert.ErrorContains(t, err, tc.expectedError)
+	}
+}
+
 func mustRead(t *testing.T, fs billy.Filesystem, path string) string {
 	t.Helper()
 	f, err := fs.OpenFile(path, os.O_RDONLY, 0644)
@@ -167,3 +302,13 @@ func mustRead(t *testing.T, fs billy.Filesystem, path string) string {
 	require.NoError(t, err)
 	return string(content)
 }
+
+// generates a random ed25519 private key
+func randKeygen(t *testing.T) gossh.Signer {
+	t.Helper()
+	_, key, err := ed25519.GenerateKey(nil)
+	require.NoError(t, err)
+	signer, err := gossh.NewSignerFromKey(key)
+	require.NoError(t, err)
+	return signer
+}

go.mod

@@ -26,6 +26,7 @@ require (
 	github.com/docker/cli v26.1.0+incompatible
 	github.com/docker/docker v23.0.8+incompatible
 	github.com/fatih/color v1.16.0
+	github.com/gliderlabs/ssh v0.3.7
 	github.com/go-git/go-billy/v5 v5.5.0
 	github.com/go-git/go-git/v5 v5.12.0
 	github.com/google/go-containerregistry v0.15.2
@@ -36,6 +37,7 @@ require (
 	github.com/sirupsen/logrus v1.9.3
 	github.com/stretchr/testify v1.9.0
 	github.com/tailscale/hujson v0.0.0-20221223112325-20486734a56a
+	golang.org/x/crypto v0.21.0
 	golang.org/x/sync v0.7.0
 	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028
 )
@@ -70,6 +72,7 @@ require (
 	github.com/agext/levenshtein v1.2.3 // indirect
 	github.com/akutz/memconn v0.1.0 // indirect
 	github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 // indirect
+	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
 	github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
 	github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
 	github.com/aws/aws-sdk-go-v2 v1.20.3 // indirect
@@ -261,7 +264,6 @@ require (
 	go4.org/mem v0.0.0-20220726221520-4f986261bf13 // indirect
 	go4.org/netipx v0.0.0-20230728180743-ad4cb58a6516 // indirect
 	go4.org/unsafe/assume-no-moving-gc v0.0.0-20230525183740-e7c30c78aeb2 // indirect
-	golang.org/x/crypto v0.21.0 // indirect
 	golang.org/x/exp v0.0.0-20240213143201-ec583247a57a // indirect
 	golang.org/x/mod v0.15.0 // indirect
 	golang.org/x/net v0.23.0 // indirect