feat: support cloning over SSH via private key auth

README.md

@@ -288,8 +288,7 @@ On MacOS or Windows systems, we recommend either using a VM or the provided `.de
 | `--git-clone-single-branch` | `GIT_CLONE_SINGLE_BRANCH` |  | Clone only a single branch of the Git repository. |
 | `--git-username` | `GIT_USERNAME` |  | The username to use for Git authentication. This is optional. |
 | `--git-password` | `GIT_PASSWORD` |  | The password to use for Git authentication. This is optional. |
-| `--git-ssh-private-key-path` | `GIT_SSH_PRIVATE_KEY_PATH` |  | Path to a SSH private key to be used for Git authentication. |
-| `--git-ssh-known-hosts-base64` | `GIT_SSH_KNOWN_HOSTS_BASE64` |  | Base64-encoded content of a known hosts file. If not specified, host keys will be scanned and logged, but not checked. |
+| `--git-ssh-private-key-path` | `GIT_SSH_PRIVATE_KEY_PATH` |  | Path to an SSH private key to be used for Git authentication. |
 | `--git-http-proxy-url` | `GIT_HTTP_PROXY_URL` |  | The URL for the HTTP proxy. This is optional. |
 | `--workspace-folder` | `WORKSPACE_FOLDER` |  | The path to the workspace folder that will be built. This is optional. |
 | `--ssl-cert-base64` | `SSL_CERT_BASE64` |  | The content of an SSL cert file. This is useful for self-signed certificates. |

envbuilder.go

@@ -44,8 +44,6 @@ import (
 	"github.com/go-git/go-billy/v5"
 	"github.com/go-git/go-billy/v5/osfs"
 	"github.com/go-git/go-git/v5/plumbing/transport"
-	githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
-	gitssh "github.com/go-git/go-git/v5/plumbing/transport/ssh"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
 	"github.com/sirupsen/logrus"
@@ -157,36 +155,6 @@ func Run(ctx context.Context, options Options) error {
 		}
 	}
 
-	if options.GitURL != "" {
-		gitURLParsed, err := ParseGitURL(options.GitURL)
-		if err != nil {
-			return fmt.Errorf("invalid git URL: %w", err)
-		}
-		// If we're cloning over SSH, we need a known_hosts file.
-		if gitURLParsed.Scheme == "ssh" {
-			var knownHostsContent []byte
-			if options.GitSSHKnownHostsBase64 != "" {
-				if kh, err := base64.StdEncoding.DecodeString(options.GitSSHKnownHostsBase64); err != nil {
-					return fmt.Errorf("invalid known_hosts content: %w", err)
-				} else {
-					knownHostsContent = kh
-				}
-			} else {
-				// This is a best-effort.
-				kh, err := KeyScan(options.Logger, gitURLParsed)
-				if err == nil {
-					knownHostsContent = kh
-				}
-			}
-			knownHostsPath := filepath.Join(MagicDir, "known_hosts")
-			if err := os.WriteFile(knownHostsPath, knownHostsContent, 0644); err != nil {
-				return fmt.Errorf("write known_hosts file: %w", err)
-			}
-			// go-git will read this file to validate the server host keys.
-			_ = os.Setenv("SSH_KNOWN_HOSTS", knownHostsPath)
-		}
-	}
-
 	var fallbackErr error
 	var cloned bool
 	if options.GitURL != "" {
@@ -225,23 +193,7 @@ func Run(ctx context.Context, options Options) error {
 			CABundle:     caBundle,
 		}
 
-		if options.GitUsername != "" || options.GitPassword != "" {
-			// NOTE: we previously inserted the credentials into the repo URL.
-			// This was removed in https://github.com/coder/envbuilder/pull/141
-			cloneOpts.RepoAuth = &githttp.BasicAuth{
-				Username: options.GitUsername,
-				Password: options.GitPassword,
-			}
-		} else if options.GitSSHPrivateKeyPath != "" {
-			signer, err := ReadPrivateKey(options.GitSSHPrivateKeyPath)
-			if err != nil {
-				return xerrors.Errorf("read private key: %w", err)
-			}
-			cloneOpts.RepoAuth = &gitssh.PublicKeys{
-				User:   "git",
-				Signer: signer,
-			}
-		}
+		cloneOpts.RepoAuth = SetupRepoAuth(&options)
 		if options.GitHTTPProxyURL != "" {
 			cloneOpts.ProxyOptions = transport.ProxyOptions{
 				URL: options.GitHTTPProxyURL,

git.go

@@ -1,17 +1,13 @@
 package envbuilder
 
 import (
-	"bytes"
 	"context"
-	"encoding/base64"
 	"errors"
 	"fmt"
 	"io"
 	"net"
 	"net/url"
 	"os"
-	"regexp"
-	"strconv"
 	"strings"
 
 	"github.com/coder/coder/v2/codersdk"
@@ -22,7 +18,11 @@ import (
 	"github.com/go-git/go-git/v5/plumbing/protocol/packp/capability"
 	"github.com/go-git/go-git/v5/plumbing/protocol/packp/sideband"
 	"github.com/go-git/go-git/v5/plumbing/transport"
+	githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
+	gitssh "github.com/go-git/go-git/v5/plumbing/transport/ssh"
 	"github.com/go-git/go-git/v5/storage/filesystem"
+	"github.com/skeema/knownhosts"
+	"golang.org/x/crypto/ssh"
 	gossh "golang.org/x/crypto/ssh"
 )
 
@@ -142,73 +142,112 @@ func ReadPrivateKey(path string) (gossh.Signer, error) {
 	return k, nil
 }
 
-// KeyScan dials the server located at gitURL and fetches the SSH
-// public keys returned in a format accepted by known_hosts.
-// If no host keys found, returns an error.
-func KeyScan(log LoggerFunc, gitURL *url.URL) ([]byte, error) {
-	var buf bytes.Buffer
-	conf := &gossh.ClientConfig{
-		// Accept and record all host keys
-		HostKeyCallback: func(dialAddr string, _ net.Addr, key gossh.PublicKey) error {
-			kh, err := KnownHostsLine(dialAddr, key)
-			if err != nil {
-				return fmt.Errorf("ssh keyscan: generate known hosts line: %w", err)
-			}
-			log(codersdk.LogLevelInfo, "ssh keyscan: %s", kh)
-			buf.WriteString(kh)
-			buf.WriteString("\n")
-			return nil
-		},
-	}
-	dialAddr := hostPort(gitURL)
-	client, err := gossh.Dial("tcp", dialAddr, conf)
-	if err != nil {
-		// The dial may fail due to no authentication methods, but this is fine.
-		if netErr, ok := err.(net.Error); ok {
-			return nil, fmt.Errorf("ssh keyscan: dial %s: %w", dialAddr, netErr)
+// LogHostKeyCallback is a HostKeyCallback that just logs host keys
+// and does nothing else.
+func LogHostKeyCallback(log LoggerFunc) gossh.HostKeyCallback {
+	return func(hostname string, remote net.Addr, key gossh.PublicKey) error {
+		var sb strings.Builder
+		_ = knownhosts.WriteKnownHost(&sb, hostname, remote, key)
+		// skeema/knownhosts uses a fake public key to determine the host key
+		// algorithms. Ignore this one.
+		if s := sb.String(); !strings.Contains(s, "fake-public-key ZmFrZSBwdWJsaWMga2V5") {
+			log(codersdk.LogLevelInfo, "#1: 🔑 Got host key: %s", strings.TrimSpace(s))
 		}
-		// Otherwise, assume success.
+		return nil
 	}
-	defer func() {
-		if client != nil {
-			_ = client.Close()
+}
+
+// SetupRepoAuth determines the desired AuthMethod based on options.GitURL:
+//
+// | Git URL format          | GIT_USERNAME | GIT_PASSWORD | Auth Method |
+// | ------------------------|--------------|--------------|-------------|
+// | https?://host.tld/repo  | Not Set      | Not Set      | None        |
+// | https?://host.tld/repo  | Not Set      | Set          | HTTP Basic  |
+// | https?://host.tld/repo  | Set          | Not Set      | HTTP Basic  |
+// | https?://host.tld/repo  | Set          | Set          | HTTP Basic  |
+// | All other formats       | -            | -            | SSH         |
+//
+// For SSH authentication, the default username is "git" but will honour
+// GIT_USERNAME if set.
+//
+// If SSH_PRIVATE_KEY_PATH is set, an SSH private key will be read from
+// that path and the SSH auth method will be configured with that key.
+//
+// If SSH_KNOWN_HOSTS is not set, the SSH auth method will be configured
+// to accept and log all host keys. Otherwise, host key checking will be
+// performed as usual.
+func SetupRepoAuth(options *Options) transport.AuthMethod {
+	if options.GitURL == "" {
+		options.Logger(codersdk.LogLevelInfo, "#1: ❔ No Git URL supplied!")
+		return nil
+	}
+	if strings.HasPrefix(options.GitURL, "http://") || strings.HasPrefix(options.GitURL, "https://") {
+		// Special case: no auth
+		if options.GitUsername == "" && options.GitPassword == "" {
+			options.Logger(codersdk.LogLevelInfo, "#1: 👤 Using no authentication!")
+			return nil
+		}
+		// Basic Auth
+		// NOTE: we previously inserted the credentials into the repo URL.
+		// This was removed in https://github.com/coder/envbuilder/pull/141
+		options.Logger(codersdk.LogLevelInfo, "#1: 🔒 Using HTTP basic authentication!")
+		return &githttp.BasicAuth{
+			Username: options.GitUsername,
+			Password: options.GitPassword,
 		}
-	}()
+	}
 
-	bs := buf.Bytes()
-	if len(bs) == 0 {
-		return nil, fmt.Errorf("ssh keyscan: found no host keys")
+	// Generally git clones over SSH use the 'git' user, but respect
+	// GIT_USERNAME if set.
+	authUser := options.GitUsername
+	if authUser == "" {
+		authUser = "git"
 	}
-	return buf.Bytes(), nil
-}
 
-// KnownHostsLine generates a corresponding line for known_hosts
-// given a dial address in the format host:port and a public key.
-func KnownHostsLine(dialAddr string, key gossh.PublicKey) (string, error) {
-	if !strings.Contains(dialAddr, ":") {
-		return "", fmt.Errorf("invalid dialAddr, expected host:port")
+	// Assume SSH auth for all other formats.
+	options.Logger(codersdk.LogLevelInfo, "#1: 🔑 Using SSH authentication!")
+
+	var signer ssh.Signer
+	if options.GitSSHPrivateKeyPath != "" {
+		s, err := ReadPrivateKey(options.GitSSHPrivateKeyPath)
+		if err != nil {
+			options.Logger(codersdk.LogLevelError, "#1: ❌ Failed to read private key from %s: %s", options.GitSSHPrivateKeyPath, err.Error())
+		} else {
+			options.Logger(codersdk.LogLevelInfo, "#1: 🔑 Using %s key!", s.PublicKey().Type())
+			signer = s
+		}
 	}
-	h := strings.Split(dialAddr, ":")[0]
-	k64 := base64.StdEncoding.EncodeToString(key.Marshal())
-	return fmt.Sprintf("%s %s %s", h, key.Type(), k64), nil
-}
 
-func hostPort(u *url.URL) string {
-	p := 22 // assume default SSH port
-	if tmp, err := strconv.Atoi(u.Port()); err == nil {
-		p = tmp
+	// If no SSH key set, fall back to agent auth.
+	if signer == nil {
+		options.Logger(codersdk.LogLevelError, "#1: 🔑 No SSH key found, falling back to agent!")
+		auth, err := gitssh.NewSSHAgentAuth(authUser)
+		if err != nil {
+			options.Logger(codersdk.LogLevelError, "#1: ❌ Failed to connect to SSH agent: %s", err.Error())
+			return nil // nothing else we can do
+		}
+		if os.Getenv("SSH_KNOWN_HOSTS") == "" {
+			options.Logger(codersdk.LogLevelWarn, "#1: 🔓 SSH_KNOWN_HOSTS not set, accepting all host keys!")
+			auth.HostKeyCallback = LogHostKeyCallback(options.Logger)
+		}
+		return auth
 	}
-	return fmt.Sprintf("%s:%d", u.Host, p)
-}
 
-var schemaRe = regexp.MustCompile(`^[a-zA-Z]+://`)
+	auth := &gitssh.PublicKeys{
+		User:   options.GitUsername,
+		Signer: signer,
+	}
 
-// ParseGitURL will normalize a git URL without a leading schema.
-// If no schema is provided, we will default to ssh://.
-func ParseGitURL(gitURL string) (*url.URL, error) {
-	if !schemaRe.MatchString(gitURL) {
-		gitURL = "ssh://" + gitURL
+	// Generally git clones over SSH use the 'git' user, but respect
+	// GIT_USERNAME if set.
+	if auth.User == "" {
+		auth.User = "git"
+	}
 
+	// Duplicated code due to Go's type system.
+	if os.Getenv("SSH_KNOWN_HOSTS") == "" {
+		options.Logger(codersdk.LogLevelWarn, "#1: 🔓 SSH_KNOWN_HOSTS not set, accepting all host keys!")
+		auth.HostKeyCallback = LogHostKeyCallback(options.Logger)
 	}
-	return url.Parse(gitURL)
+	return auth
 }