Merge branch 'main' of github.com:coder/envbuilder into bq/docs

Author: BrunoQuaresma

.github/workflows/ci.yaml

@@ -48,4 +48,4 @@ jobs:
           go-version: "~1.21"
 
       - name: Test
-        run: go test ./...
+        run: make test

.gitignore

@@ -1 +1,2 @@
 scripts/envbuilder-*
+.registry-cache

Makefile

@@ -0,0 +1,45 @@
+GOARCH := $(shell go env GOARCH)
+PWD=$(shell pwd)
+
+develop:
+	./scripts/develop.sh
+
+build: scripts/envbuilder-$(GOARCH)
+	./scripts/build.sh
+
+.PHONY: test
+test: test-registry test-images
+	go test -count=1 ./...
+
+test-race:
+	go test -race -count=3 ./...
+
+# Starts a local Docker registry on port 5000 with a local disk cache.
+.PHONY: test-registry
+test-registry: .registry-cache
+	if ! curl -fsSL http://localhost:5000/v2/_catalog > /dev/null 2>&1; then \
+		docker rm -f envbuilder-registry && \
+		docker run -d -p 5000:5000 --name envbuilder-registry --volume $(PWD)/.registry-cache:/var/lib/registry registry:2; \
+	fi
+
+# Pulls images referenced in integration tests and pushes them to the local cache.
+.PHONY: test-images
+test-images: .registry-cache .registry-cache/docker/registry/v2/repositories/envbuilder-test-alpine .registry-cache/docker/registry/v2/repositories/envbuilder-test-ubuntu .registry-cache/docker/registry/v2/repositories/envbuilder-test-codercom-code-server
+
+.registry-cache:
+	mkdir -p .registry-cache && chmod -R ag+w .registry-cache
+
+.registry-cache/docker/registry/v2/repositories/envbuilder-test-alpine:
+	docker pull alpine:latest
+	docker tag alpine:latest localhost:5000/envbuilder-test-alpine:latest
+	docker push localhost:5000/envbuilder-test-alpine:latest
+
+.registry-cache/docker/registry/v2/repositories/envbuilder-test-ubuntu:
+	docker pull ubuntu:latest
+	docker tag ubuntu:latest localhost:5000/envbuilder-test-ubuntu:latest
+	docker push localhost:5000/envbuilder-test-ubuntu:latest
+
+.registry-cache/docker/registry/v2/repositories/envbuilder-test-codercom-code-server:
+	docker pull codercom/code-server:latest
+	docker tag codercom/code-server:latest localhost:5000/envbuilder-test-codercom-code-server:latest
+	docker push localhost:5000/envbuilder-test-codercom-code-server:latest

README.md

@@ -224,3 +224,23 @@ docker run -it --rm \
 - [`SSL_CERT_FILE`](https://go.dev/src/crypto/x509/root_unix.go#L19): Specifies the path to an SSL certificate.
 - [`SSL_CERT_DIR`](https://go.dev/src/crypto/x509/root_unix.go#L25): Identifies which directory to check for SSL certificate files.
 - `SSL_CERT_BASE64`: Specifies a base64-encoded SSL certificate that will be added to the global certificate pool on start.
+
+
+# Local Development
+
+Building `envbuilder` currently **requires** a Linux system.
+
+On MacOS or Windows systems, we recommend either using a VM or the provided `.devcontainer` for development.
+
+**Additional Requirements:**
+
+- `go 1.21`
+- `make`
+- Docker daemon (for running tests)
+
+**Makefile targets:**
+
+- `build`: builds and tags `envbuilder:latest` for your current architecture.
+- `develop`: runs `envbuilder:latest` against a sample Git repository.
+- `test`: run tests.
+- `test-registry`: stands up a local registry for caching images used in tests.

devcontainer/devcontainer_test.go

@@ -78,7 +78,7 @@ func TestCompileWithFeatures(t *testing.T) {
     "context": ".",
   },
   // Comments here!
-  "image": "codercom/code-server:latest",
+  "image": "localhost:5000/envbuilder-test-codercom-code-server:latest",
   "features": {
 	"` + featureOne + `": {},
 	"` + featureTwo + `": "potato"
@@ -96,7 +96,7 @@ func TestCompileWithFeatures(t *testing.T) {
 	featureTwoMD5 := md5.Sum([]byte(featureTwo))
 	featureTwoDir := fmt.Sprintf("/.envbuilder/features/two-%x", featureTwoMD5[:4])
 
-	require.Equal(t, `FROM codercom/code-server:latest
+	require.Equal(t, `FROM localhost:5000/envbuilder-test-codercom-code-server:latest
 
 USER root
 # Rust tomato - Example description!
@@ -116,7 +116,7 @@ func TestCompileDevContainer(t *testing.T) {
 		t.Parallel()
 		fs := memfs.New()
 		dc := &devcontainer.Spec{
-			Image: "codercom/code-server:latest",
+			Image: "localhost:5000/envbuilder-test-ubuntu:latest",
 		}
 		params, err := dc.Compile(fs, "", magicDir, "", "", false)
 		require.NoError(t, err)
@@ -141,7 +141,7 @@ func TestCompileDevContainer(t *testing.T) {
 		require.NoError(t, err)
 		file, err := fs.OpenFile(filepath.Join(dcDir, "Dockerfile"), os.O_CREATE|os.O_WRONLY, 0644)
 		require.NoError(t, err)
-		_, err = io.WriteString(file, "FROM ubuntu")
+		_, err = io.WriteString(file, "FROM localhost:5000/envbuilder-test-ubuntu:latest")
 		require.NoError(t, err)
 		_ = file.Close()
 		params, err := dc.Compile(fs, dcDir, magicDir, "", "/var/workspace", false)

envbuilder.go

@@ -267,22 +267,11 @@ func Run(ctx context.Context, options OptionsMap, deps Dependencies) error {
 	if options.GetString("DockerFilepath") == "" {
 		// Only look for a devcontainer if a Dockerfile wasn't specified.
 		// devcontainer is a standard, so it's reasonable to be the default.
-		devcontainerDir := options.GetString("DevcontainerDir")
-		if devcontainerDir == "" {
-			devcontainerDir = ".devcontainer"
-		}
-		if !filepath.IsAbs(devcontainerDir) {
-			devcontainerDir = filepath.Join(options.GetString("WorkspaceFolder"), devcontainerDir)
-		}
-		devcontainerPath := options.GetString("DevcontainerJSONPath")
-		if devcontainerPath == "" {
-			devcontainerPath = "devcontainer.json"
-		}
-		if !filepath.IsAbs(devcontainerPath) {
-			devcontainerPath = filepath.Join(devcontainerDir, devcontainerPath)
-		}
-		_, err := deps.Filesystem.Stat(devcontainerPath)
-		if err == nil {
+		devcontainerPath, devcontainerDir, err := findDevcontainerJSON(&options, &deps)
+		if err != nil {
+			logf(codersdk.LogLevelError, "Failed to locate devcontainer.json: %s", err.Error())
+			logf(codersdk.LogLevelError, "Falling back to the default image...")
+		} else {
 			// We know a devcontainer exists.
 			// Let's parse it and use it!
 			file, err := deps.Filesystem.Open(devcontainerPath)
@@ -317,7 +306,16 @@ func Run(ctx context.Context, options OptionsMap, deps Dependencies) error {
 		}
 	} else {
 		// If a Dockerfile was specified, we use that.
-		dockerfilePath := filepath.Join(options.GetString("WorkspaceFolder"), options.GetString("DockerFilepath"))
+		dockerfilePath := filepath.Join(options.GetString("WorkspaceFolder"), options.GetString("DockerfilePath"))
+
+		// If the dockerfilePath is specified and deeper than the base of WorkspaceFolder AND the BuildContextPath is
+		// not defined, show a warning
+		dockerfileDir := filepath.Dir(dockerfilePath)
+		if dockerfileDir != filepath.Clean(options.GetString("WorkspaceFolder")) && options.GetString("BuildContextPath") == "" {
+			logf(codersdk.LogLevelWarn, "given dockerfile %q is below %q and no custom build context has been defined", dockerfilePath, options.GetString("WorkspaceFolder"))
+			logf(codersdk.LogLevelWarn, "\t-> set BUILD_CONTEXT_PATH to %q to fix", dockerfileDir)
+		}
+
 		dockerfile, err := deps.Filesystem.Open(dockerfilePath)
 		if err == nil {
 			content, err := io.ReadAll(dockerfile)
@@ -327,7 +325,7 @@ func Run(ctx context.Context, options OptionsMap, deps Dependencies) error {
 			buildParams = &devcontainer.Compiled{
 				DockerfilePath:    dockerfilePath,
 				DockerfileContent: string(content),
-				BuildContext:      options.GetString("WorkspaceFolder"),
+				BuildContext:      filepath.Join(options.GetString("WorkspaceFolder"), options.GetString("BuildContextPath")),
 			}
 		}
 	}
@@ -1002,3 +1000,71 @@ type osfsWithChmod struct {
 func (fs *osfsWithChmod) Chmod(name string, mode os.FileMode) error {
 	return os.Chmod(name, mode)
 }
+
+func findDevcontainerJSON(options *OptionsMap, deps *Dependencies) (string, string, error) {
+	// 0. Check if custom devcontainer directory or path is provided.
+	if options.GetString("DevcontainerDir") != "" || options.GetString("DevcontainerJSONPath") != "" {
+		devcontainerDir := options.GetString("DevcontainerDir")
+		if devcontainerDir == "" {
+			devcontainerDir = ".devcontainer"
+		}
+
+		// If `devcontainerDir` is not an absolute path, assume it is relative to the workspace folder.
+		if !filepath.IsAbs(devcontainerDir) {
+			devcontainerDir = filepath.Join(options.GetString("WorkspaceFolder"), devcontainerDir)
+		}
+
+		// An absolute location always takes a precedence.
+		devcontainerPath := options.GetString("DevcontainerJSONPath")
+		if filepath.IsAbs(devcontainerPath) {
+			return options.GetString("DevcontainerJSONPath"), devcontainerDir, nil
+		}
+		// If an override is not provided, assume it is just `devcontainer.json`.
+		if devcontainerPath == "" {
+			devcontainerPath = "devcontainer.json"
+		}
+
+		if !filepath.IsAbs(devcontainerPath) {
+			devcontainerPath = filepath.Join(devcontainerDir, devcontainerPath)
+		}
+		return devcontainerPath, devcontainerDir, nil
+	}
+
+	// 1. Check `options.WorkspaceFolder`/.devcontainer/devcontainer.json.
+	location := filepath.Join(options.GetString("WorkspaceFolder"), ".devcontainer", "devcontainer.json")
+	if _, err := deps.Filesystem.Stat(location); err == nil {
+		return location, filepath.Dir(location), nil
+	}
+
+	// 2. Check `options.WorkspaceFolder`/devcontainer.json.
+	location = filepath.Join(options.GetString("WorkspaceFolder"), "devcontainer.json")
+	if _, err := deps.Filesystem.Stat(location); err == nil {
+		return location, filepath.Dir(location), nil
+	}
+
+	// 3. Check every folder: `options.WorkspaceFolder`/.devcontainer/<folder>/devcontainer.json.
+	devcontainerDir := filepath.Join(options.GetString("WorkspaceFolder"), ".devcontainer")
+
+	fileInfos, err := deps.Filesystem.ReadDir(devcontainerDir)
+	if err != nil {
+		return "", "", err
+	}
+
+	logf := deps.Logger
+	for _, fileInfo := range fileInfos {
+		if !fileInfo.IsDir() {
+			logf(codersdk.LogLevelDebug, `%s is a file`, fileInfo.Name())
+			continue
+		}
+
+		location := filepath.Join(devcontainerDir, fileInfo.Name(), "devcontainer.json")
+		if _, err := deps.Filesystem.Stat(location); err != nil {
+			logf(codersdk.LogLevelDebug, `stat %s failed: %s`, location, err.Error())
+			continue
+		}
+
+		return location, filepath.Dir(location), nil
+	}
+
+	return "", "", errors.New("can't find devcontainer.json, is it a correct spec?")
+}

envbuilder_internal_test.go

@@ -0,0 +1,147 @@
+package envbuilder
+
+import (
+	"testing"
+
+	"github.com/go-git/go-billy/v5/memfs"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFindDevcontainerJSON(t *testing.T) {
+	t.Parallel()
+
+	t.Run("empty filesystem", func(t *testing.T) {
+		t.Parallel()
+
+		// given
+		fs := memfs.New()
+
+		// when
+		options := DefaultOptions()
+		options.SetString("WorkspaceFolder", "/workspace")
+		_, _, err := findDevcontainerJSON(&options, &Dependencies{Filesystem: fs})
+
+		// then
+		require.Error(t, err)
+	})
+
+	t.Run("devcontainers.json is missing", func(t *testing.T) {
+		t.Parallel()
+
+		// given
+		fs := memfs.New()
+		err := fs.MkdirAll("/workspace/.devcontainer", 0600)
+		require.NoError(t, err)
+
+		// when
+		options := DefaultOptions()
+		options.SetString("WorkspaceFolder", "/workspace")
+		_, _, err = findDevcontainerJSON(&options, &Dependencies{Filesystem: fs})
+
+		// then
+		require.Error(t, err)
+	})
+
+	t.Run("default configuration", func(t *testing.T) {
+		t.Parallel()
+
+		// given
+		fs := memfs.New()
+		err := fs.MkdirAll("/workspace/.devcontainer", 0600)
+		require.NoError(t, err)
+		fs.Create("/workspace/.devcontainer/devcontainer.json")
+
+		// when
+		options := DefaultOptions()
+		options.SetString("WorkspaceFolder", "/workspace")
+		devcontainerPath, devcontainerDir, err := findDevcontainerJSON(&options, &Dependencies{Filesystem: fs})
+
+		// then
+		require.NoError(t, err)
+		assert.Equal(t, "/workspace/.devcontainer/devcontainer.json", devcontainerPath)
+		assert.Equal(t, "/workspace/.devcontainer", devcontainerDir)
+	})
+
+	t.Run("overridden .devcontainer directory", func(t *testing.T) {
+		t.Parallel()
+
+		// given
+		fs := memfs.New()
+		err := fs.MkdirAll("/workspace/experimental-devcontainer", 0600)
+		require.NoError(t, err)
+		fs.Create("/workspace/experimental-devcontainer/devcontainer.json")
+
+		// when
+		options := DefaultOptions()
+		options.SetString("WorkspaceFolder", "/workspace")
+		options.SetString("DevcontainerDir", "/experimental-devcontainer")
+		devcontainerPath, devcontainerDir, err := findDevcontainerJSON(&options, &Dependencies{Filesystem: fs})
+
+		// then
+		require.NoError(t, err)
+		assert.Equal(t, "/workspace/experimental-devcontainer/devcontainer.json", devcontainerPath)
+		assert.Equal(t, "/workspace/experimental-devcontainer", devcontainerDir)
+	})
+
+	t.Run("overridden devcontainer.json path", func(t *testing.T) {
+		t.Parallel()
+
+		// given
+		fs := memfs.New()
+		err := fs.MkdirAll("/workspace/.devcontainer", 0600)
+		require.NoError(t, err)
+		fs.Create("/workspace/.devcontainer/experimental.json")
+
+		// when
+		options := DefaultOptions()
+		options.SetString("WorkspaceFolder", "/workspace")
+		options.SetString("DevcontainerJSONPath", "experimental.json")
+		devcontainerPath, devcontainerDir, err := findDevcontainerJSON(&options, &Dependencies{Filesystem: fs})
+
+		// then
+		require.NoError(t, err)
+		assert.Equal(t, "/workspace/.devcontainer/experimental.json", devcontainerPath)
+		assert.Equal(t, "/workspace/.devcontainer", devcontainerDir)
+	})
+
+	t.Run("devcontainer.json in workspace root", func(t *testing.T) {
+		t.Parallel()
+
+		// given
+		fs := memfs.New()
+		err := fs.MkdirAll("/workspace", 0600)
+		require.NoError(t, err)
+		fs.Create("/workspace/devcontainer.json")
+
+		// when
+		options := DefaultOptions()
+		options.SetString("WorkspaceFolder", "/workspace")
+		devcontainerPath, devcontainerDir, err := findDevcontainerJSON(&options, &Dependencies{Filesystem: fs})
+
+		// then
+		require.NoError(t, err)
+		assert.Equal(t, "/workspace/devcontainer.json", devcontainerPath)
+		assert.Equal(t, "/workspace", devcontainerDir)
+	})
+
+	t.Run("devcontainer.json in subfolder of .devcontainer", func(t *testing.T) {
+		t.Parallel()
+
+		// given
+		fs := memfs.New()
+		err := fs.MkdirAll("/workspace/.devcontainer/sample", 0600)
+		require.NoError(t, err)
+		fs.Create("/workspace/.devcontainer/sample/devcontainer.json")
+
+		// when
+		options := DefaultOptions()
+		options.SetString("WorkspaceFolder", "/workspace")
+		devcontainerPath, devcontainerDir, err := findDevcontainerJSON(&options, &Dependencies{Filesystem: fs})
+
+		// then
+		require.NoError(t, err)
+		assert.Equal(t, "/workspace/.devcontainer/sample/devcontainer.json", devcontainerPath)
+		assert.Equal(t, "/workspace/.devcontainer/sample", devcontainerDir)
+	})
+}