feat: implement repo-mode

Fixes #218

Author: mafredri

constants/constants.go

@@ -28,4 +28,8 @@ var (
 	// to skip building when a container is restarting.
 	// e.g. docker stop -> docker start
 	MagicFile = filepath.Join(MagicDir, "built")
+
+	// MagicFile is the location of the build context when
+	// using remote build mode.
+	MagicRemoteRepoDir = filepath.Join(MagicDir, "repo")
 )

envbuilder.go

@@ -4,7 +4,6 @@ import (
 	"bufio"
 	"bytes"
 	"context"
-	"crypto/x509"
 	"encoding/base64"
 	"encoding/json"
 	"errors"
@@ -41,7 +40,6 @@ import (
 	_ "github.com/distribution/distribution/v3/registry/storage/driver/filesystem"
 	"github.com/docker/cli/cli/config/configfile"
 	"github.com/fatih/color"
-	"github.com/go-git/go-git/v5/plumbing/transport"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
 	"github.com/kballard/go-shellquote"
@@ -88,23 +86,6 @@ func Run(ctx context.Context, opts options.Options) error {
 
 	opts.Logger(log.LevelInfo, "%s - Build development environments from repositories in a container", newColor(color.Bold).Sprintf("envbuilder"))
 
-	var caBundle []byte
-	if opts.SSLCertBase64 != "" {
-		certPool, err := x509.SystemCertPool()
-		if err != nil {
-			return xerrors.Errorf("get global system cert pool: %w", err)
-		}
-		data, err := base64.StdEncoding.DecodeString(opts.SSLCertBase64)
-		if err != nil {
-			return xerrors.Errorf("base64 decode ssl cert: %w", err)
-		}
-		ok := certPool.AppendCertsFromPEM(data)
-		if !ok {
-			return xerrors.Errorf("failed to append the ssl cert to the global pool: %s", data)
-		}
-		caBundle = data
-	}
-
 	if opts.DockerConfigBase64 != "" {
 		decoded, err := base64.StdEncoding.DecodeString(opts.DockerConfigBase64)
 		if err != nil {
@@ -125,51 +106,23 @@ func Run(ctx context.Context, opts options.Options) error {
 		}
 	}
 
+	buildTimeWorkspaceFolder := opts.WorkspaceFolder
 	var fallbackErr error
 	var cloned bool
 	if opts.GitURL != "" {
+		cloneOpts, err := git.CloneOptionsFromOptions(opts)
+		if err != nil {
+			return fmt.Errorf("git clone options: %w", err)
+		}
+
 		endStage := startStage("📦 Cloning %s to %s...",
 			newColor(color.FgCyan).Sprintf(opts.GitURL),
-			newColor(color.FgCyan).Sprintf(opts.WorkspaceFolder),
+			newColor(color.FgCyan).Sprintf(cloneOpts.Path),
 		)
 
-		reader, writer := io.Pipe()
-		defer reader.Close()
-		defer writer.Close()
-		go func() {
-			data := make([]byte, 4096)
-			for {
-				read, err := reader.Read(data)
-				if err != nil {
-					return
-				}
-				content := data[:read]
-				for _, line := range strings.Split(string(content), "\r") {
-					if line == "" {
-						continue
-					}
-					opts.Logger(log.LevelInfo, "#1: %s", strings.TrimSpace(line))
-				}
-			}
-		}()
-
-		cloneOpts := git.CloneRepoOptions{
-			Path:         opts.WorkspaceFolder,
-			Storage:      opts.Filesystem,
-			Insecure:     opts.Insecure,
-			Progress:     writer,
-			SingleBranch: opts.GitCloneSingleBranch,
-			Depth:        int(opts.GitCloneDepth),
-			CABundle:     caBundle,
-		}
-
-		cloneOpts.RepoAuth = git.SetupRepoAuth(&opts)
-		if opts.GitHTTPProxyURL != "" {
-			cloneOpts.ProxyOptions = transport.ProxyOptions{
-				URL: opts.GitHTTPProxyURL,
-			}
-		}
-		cloneOpts.RepoURL = opts.GitURL
+		w := git.ProgressWriter(func(line string) { opts.Logger(log.LevelInfo, "#%d: %s", stageNumber, line) })
+		defer w.Close()
+		cloneOpts.Progress = w
 
 		cloned, fallbackErr = git.CloneRepo(ctx, cloneOpts)
 		if fallbackErr == nil {
@@ -182,6 +135,40 @@ func Run(ctx context.Context, opts options.Options) error {
 			opts.Logger(log.LevelError, "Failed to clone repository: %s", fallbackErr.Error())
 			opts.Logger(log.LevelError, "Falling back to the default image...")
 		}
+
+		// The repo wasn't cloned (i.e. may not be up-to-date with remote), so
+		// we need to clone it to get the right build context. This is only
+		// necessary when the repo is in remote build mode. If the repo is in
+		// local build mode, the build context is already available on the host
+		// filesystem.
+		//
+		// Skipping clone here if we believe workspace repo matches remote repo
+		// is a performance optimization.
+		if fallbackErr == nil && !cloned && opts.RepoBuildMode == options.RepoBuildModeRemote {
+			cloneOpts, err := git.CloneOptionsFromOptions(opts)
+			if err != nil {
+				return fmt.Errorf("git clone options: %w", err)
+			}
+			cloneOpts.Path = constants.MagicRemoteRepoDir
+
+			endStage := startStage("📦 Remote build mode enabled, cloning %s to %s for build context...",
+				newColor(color.FgCyan).Sprintf(opts.GitURL),
+				newColor(color.FgCyan).Sprintf(cloneOpts.Path),
+			)
+
+			w := git.ProgressWriter(func(line string) { opts.Logger(log.LevelInfo, "#%d: %s", stageNumber, line) })
+			defer w.Close()
+			cloneOpts.Progress = w
+
+			fallbackErr = git.ShallowCloneRepo(ctx, cloneOpts)
+			if fallbackErr == nil {
+				endStage("📦 Cloned repository!")
+				buildTimeWorkspaceFolder = cloneOpts.Path
+			} else {
+				opts.Logger(log.LevelError, "Failed to clone repository for remote repo mode: %s", err.Error())
+				opts.Logger(log.LevelError, "Falling back to the default image...")
+			}
+		}
 	}
 
 	defaultBuildParams := func() (*devcontainer.Compiled, error) {
@@ -222,7 +209,7 @@ func Run(ctx context.Context, opts options.Options) error {
 		// devcontainer is a standard, so it's reasonable to be the default.
 		var devcontainerDir string
 		var err error
-		devcontainerPath, devcontainerDir, err = findDevcontainerJSON(opts)
+		devcontainerPath, devcontainerDir, err = findDevcontainerJSON(buildTimeWorkspaceFolder, opts)
 		if err != nil {
 			opts.Logger(log.LevelError, "Failed to locate devcontainer.json: %s", err.Error())
 			opts.Logger(log.LevelError, "Falling back to the default image...")
@@ -614,10 +601,10 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath)
 		if err != nil {
 			return fmt.Errorf("unmarshal metadata: %w", err)
 		}
-		opts.Logger(log.LevelInfo, "#3: 👀 Found devcontainer.json label metadata in image...")
+		opts.Logger(log.LevelInfo, "#%d: 👀 Found devcontainer.json label metadata in image...", stageNumber)
 		for _, container := range devContainer {
 			if container.RemoteUser != "" {
-				opts.Logger(log.LevelInfo, "#3: 🧑 Updating the user to %q!", container.RemoteUser)
+				opts.Logger(log.LevelInfo, "#%d: 🧑 Updating the user to %q!", stageNumber, container.RemoteUser)
 
 				configFile.Config.User = container.RemoteUser
 			}
@@ -724,7 +711,7 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath)
 		username = buildParams.User
 	}
 	if username == "" {
-		opts.Logger(log.LevelWarn, "#3: no user specified, using root")
+		opts.Logger(log.LevelWarn, "#%d: no user specified, using root", stageNumber)
 	}
 
 	userInfo, err := getUser(username)
@@ -1030,7 +1017,11 @@ func newColor(value ...color.Attribute) *color.Color {
 	return c
 }
 
-func findDevcontainerJSON(options options.Options) (string, string, error) {
+func findDevcontainerJSON(workspaceFolder string, options options.Options) (string, string, error) {
+	if workspaceFolder == "" {
+		workspaceFolder = options.WorkspaceFolder
+	}
+
 	// 0. Check if custom devcontainer directory or path is provided.
 	if options.DevcontainerDir != "" || options.DevcontainerJSONPath != "" {
 		devcontainerDir := options.DevcontainerDir
@@ -1040,7 +1031,7 @@ func findDevcontainerJSON(options options.Options) (string, string, error) {
 
 		// If `devcontainerDir` is not an absolute path, assume it is relative to the workspace folder.
 		if !filepath.IsAbs(devcontainerDir) {
-			devcontainerDir = filepath.Join(options.WorkspaceFolder, devcontainerDir)
+			devcontainerDir = filepath.Join(workspaceFolder, devcontainerDir)
 		}
 
 		// An absolute location always takes a precedence.
@@ -1059,20 +1050,20 @@ func findDevcontainerJSON(options options.Options) (string, string, error) {
 		return devcontainerPath, devcontainerDir, nil
 	}
 
-	// 1. Check `options.WorkspaceFolder`/.devcontainer/devcontainer.json.
-	location := filepath.Join(options.WorkspaceFolder, ".devcontainer", "devcontainer.json")
+	// 1. Check `workspaceFolder`/.devcontainer/devcontainer.json.
+	location := filepath.Join(workspaceFolder, ".devcontainer", "devcontainer.json")
 	if _, err := options.Filesystem.Stat(location); err == nil {
 		return location, filepath.Dir(location), nil
 	}
 
-	// 2. Check `options.WorkspaceFolder`/devcontainer.json.
-	location = filepath.Join(options.WorkspaceFolder, "devcontainer.json")
+	// 2. Check `workspaceFolder`/devcontainer.json.
+	location = filepath.Join(workspaceFolder, "devcontainer.json")
 	if _, err := options.Filesystem.Stat(location); err == nil {
 		return location, filepath.Dir(location), nil
 	}
 
-	// 3. Check every folder: `options.WorkspaceFolder`/.devcontainer/<folder>/devcontainer.json.
-	devcontainerDir := filepath.Join(options.WorkspaceFolder, ".devcontainer")
+	// 3. Check every folder: `workspaceFolder`/.devcontainer/<folder>/devcontainer.json.
+	devcontainerDir := filepath.Join(workspaceFolder, ".devcontainer")
 
 	fileInfos, err := options.Filesystem.ReadDir(devcontainerDir)
 	if err != nil {