diff --git a/go.mod b/go.mod index f52ab41..0358cdb 100644 --- a/go.mod +++ b/go.mod @@ -10,9 +10,11 @@ replace tailscale.com => github.com/coder/tailscale v1.1.1-0.20240702054557-aa55 require ( github.com/GoogleContainerTools/kaniko v1.9.2 - github.com/coder/envbuilder v1.0.0-rc.0.0.20240803183847-6afe89e6950e + github.com/coder/envbuilder v1.0.0-rc.0.0.20240805094524-c1f9917dfb61 github.com/docker/docker v26.1.4+incompatible + github.com/gliderlabs/ssh v0.3.7 github.com/go-git/go-billy/v5 v5.5.0 + github.com/go-git/go-git/v5 v5.12.0 github.com/google/go-containerregistry v0.19.1 github.com/hashicorp/terraform-plugin-docs v0.19.4 github.com/hashicorp/terraform-plugin-framework v1.10.0 @@ -58,6 +60,7 @@ require ( github.com/agext/levenshtein v1.2.3 // indirect github.com/akutz/memconn v0.1.0 // indirect github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 // indirect + github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect github.com/armon/go-radix v1.0.1-0.20221118154546-54df44f2176c // indirect github.com/aws/aws-sdk-go-v2 v1.30.0 // indirect @@ -128,7 +131,6 @@ require ( github.com/fxamacker/cbor/v2 v2.4.0 // indirect github.com/go-chi/chi/v5 v5.0.10 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect - github.com/go-git/go-git/v5 v5.12.0 // indirect github.com/go-jose/go-jose/v4 v4.0.1 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect diff --git a/go.sum b/go.sum index 97ffc84..7044326 100644 --- a/go.sum +++ b/go.sum @@ -186,8 +186,8 @@ github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoC github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/coder/coder/v2 v2.10.1-0.20240704130443-c2d44d16a352 h1:L/EjCuZxs5tOcqqCaASj/nu65TRYEFcTt8qRQfHZXX0= github.com/coder/coder/v2 v2.10.1-0.20240704130443-c2d44d16a352/go.mod h1:P1KoQSgnKEAG6Mnd3YlGzAophty+yKA9VV48LpfNRvo= -github.com/coder/envbuilder v1.0.0-rc.0.0.20240803183847-6afe89e6950e h1:gchZb6E2C5giRJwS2wPjbwHfxle4rJX7NqHCpN1XaT0= -github.com/coder/envbuilder v1.0.0-rc.0.0.20240803183847-6afe89e6950e/go.mod h1:SCpGkbd04qsTIHUYRWEJMgt4R+uK+q4lGnOhEyTorjU= +github.com/coder/envbuilder v1.0.0-rc.0.0.20240805094524-c1f9917dfb61 h1:SPOT1R13rgJie9l+VUsqd4TiqzSeGD2AmEv8wzmAcDE= +github.com/coder/envbuilder v1.0.0-rc.0.0.20240805094524-c1f9917dfb61/go.mod h1:SCpGkbd04qsTIHUYRWEJMgt4R+uK+q4lGnOhEyTorjU= github.com/coder/kaniko v0.0.0-20240803153527-10d1800455b9 h1:d01T5YbPN1yc1mXjIXG59YcQQoT/9idvqFErjWHfsZ4= github.com/coder/kaniko v0.0.0-20240803153527-10d1800455b9/go.mod h1:YMK7BlxerzLlMwihGxNWUaFoN9LXCij4P+w/8/fNlcM= github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0 h1:3A0ES21Ke+FxEM8CXx9n47SZOKOpgSE1bbJzlE4qPVs= diff --git a/internal/provider/cached_image_data_source.go b/internal/provider/cached_image_data_source.go index 325fb24..2e876d2 100644 --- a/internal/provider/cached_image_data_source.go +++ b/internal/provider/cached_image_data_source.go @@ -295,13 +295,15 @@ func (d *CachedImageDataSource) Read(ctx context.Context, req datasource.ReadReq // This may require changing this to be a resource instead of a data source. opts := eboptions.Options{ // These options are always required - CacheRepo: data.CacheRepo.ValueString(), - Filesystem: osfs.New("/"), - ForceSafe: false, // This should never be set to true, as this may be running outside of a container! - GetCachedImage: true, // always! - Logger: tfLogFunc(ctx), - Verbose: data.Verbose.ValueBool(), - WorkspaceFolder: workspaceFolder, + CacheRepo: data.CacheRepo.ValueString(), + Filesystem: osfs.New("/"), + ForceSafe: false, // This should never be set to true, as this may be running outside of a container! + GetCachedImage: true, // always! + Logger: tfLogFunc(ctx), + Verbose: data.Verbose.ValueBool(), + WorkspaceFolder: workspaceFolder, + RemoteRepoBuildMode: true, + RemoteRepoDir: filepath.Join(tmpDir, "repo"), // Hidden option used by this provider. // Options related to compiling the devcontainer BuildContextPath: data.BuildContextPath.ValueString(), diff --git a/internal/provider/cached_image_data_source_test.go b/internal/provider/cached_image_data_source_test.go index ea45864..8dccdbb 100644 --- a/internal/provider/cached_image_data_source_test.go +++ b/internal/provider/cached_image_data_source_test.go @@ -18,24 +18,26 @@ import ( func TestAccCachedImageDataSource(t *testing.T) { t.Run("Found", func(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) - t.Cleanup(cancel) + defer cancel() files := map[string]string{ ".devcontainer/devcontainer.json": `{"build": { "dockerfile": "Dockerfile" }}`, ".devcontainer/Dockerfile": `FROM localhost:5000/test-ubuntu:latest RUN apt-get update && apt-get install -y cowsay`, } - deps := setup(t, files) + + deps := setup(ctx, t, files) seedCache(ctx, t, deps) tfCfg := fmt.Sprintf(`data "envbuilder_cached_image" "test" { builder_image = %q workspace_folder = %q git_url = %q + git_ssh_private_key_path = %q extra_env = { "FOO" : "bar" } cache_repo = %q verbose = true -}`, deps.BuilderImage, deps.RepoDir, deps.RepoDir, deps.CacheRepo) +}`, deps.BuilderImage, "/workspace", deps.Repo.URL, deps.Repo.Key, deps.CacheRepo) resource.Test(t, resource.TestCase{ PreCheck: func() { testAccPreCheck(t) }, ProtoV6ProviderFactories: testAccProtoV6ProviderFactories, @@ -46,7 +48,7 @@ func TestAccCachedImageDataSource(t *testing.T) { // Inputs should still be present. resource.TestCheckResourceAttr("data.envbuilder_cached_image.test", "cache_repo", deps.CacheRepo), resource.TestCheckResourceAttr("data.envbuilder_cached_image.test", "extra_env.FOO", "bar"), - resource.TestCheckResourceAttr("data.envbuilder_cached_image.test", "git_url", deps.RepoDir), + resource.TestCheckResourceAttr("data.envbuilder_cached_image.test", "git_url", deps.Repo.URL), // Should be empty resource.TestCheckNoResourceAttr("data.envbuilder_cached_image.test", "git_username"), resource.TestCheckNoResourceAttr("data.envbuilder_cached_image.test", "git_password"), @@ -78,23 +80,26 @@ func TestAccCachedImageDataSource(t *testing.T) { }) t.Run("NotFound", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() files := map[string]string{ ".devcontainer/devcontainer.json": `{"build": { "dockerfile": "Dockerfile" }}`, ".devcontainer/Dockerfile": `FROM localhost:5000/test-ubuntu:latest RUN apt-get update && apt-get install -y cowsay`, } - deps := setup(t, files) + deps := setup(ctx, t, files) // We do not seed the cache. tfCfg := fmt.Sprintf(`data "envbuilder_cached_image" "test" { builder_image = %q workspace_folder = %q git_url = %q + git_ssh_private_key_path = %q extra_env = { "FOO" : "bar" } cache_repo = %q verbose = true -}`, deps.BuilderImage, deps.RepoDir, deps.RepoDir, deps.CacheRepo) +}`, deps.BuilderImage, "/workspace", deps.Repo.URL, deps.Repo.Key, deps.CacheRepo) resource.Test(t, resource.TestCase{ PreCheck: func() { testAccPreCheck(t) }, ProtoV6ProviderFactories: testAccProtoV6ProviderFactories, @@ -105,7 +110,7 @@ func TestAccCachedImageDataSource(t *testing.T) { // Inputs should still be present. resource.TestCheckResourceAttr("data.envbuilder_cached_image.test", "cache_repo", deps.CacheRepo), resource.TestCheckResourceAttr("data.envbuilder_cached_image.test", "extra_env.FOO", "bar"), - resource.TestCheckResourceAttr("data.envbuilder_cached_image.test", "git_url", deps.RepoDir), + resource.TestCheckResourceAttr("data.envbuilder_cached_image.test", "git_url", deps.Repo.URL), resource.TestCheckResourceAttr("data.envbuilder_cached_image.test", "exists", "false"), resource.TestCheckResourceAttr("data.envbuilder_cached_image.test", "image", deps.BuilderImage), // Should be empty diff --git a/internal/provider/git_test.go b/internal/provider/git_test.go new file mode 100644 index 0000000..a512a85 --- /dev/null +++ b/internal/provider/git_test.go @@ -0,0 +1,172 @@ +package provider + +import ( + "context" + "errors" + "io" + "net" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/gliderlabs/ssh" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// nolint:gosec // Throw-away key for testing. DO NOT REUSE. +const ( + testSSHKey = `-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW +QyNTUxOQAAACCtxz9h0yXzi/HqZBpSkA2xFo28v5W8O4HimI0ZzNpQkwAAAKhv/+X2b//l +9gAAAAtzc2gtZWQyNTUxOQAAACCtxz9h0yXzi/HqZBpSkA2xFo28v5W8O4HimI0ZzNpQkw +AAAED/G0HuohvSa8q6NzkZ+wRPW0PhPpo9Th8fvcBQDaxCia3HP2HTJfOL8epkGlKQDbEW +jby/lbw7geKYjRnM2lCTAAAAInRlcnJhZm9ybS1wcm92aWRlci1lbnZidWlsZGVyLXRlc3 +QBAgM= +-----END OPENSSH PRIVATE KEY-----` + testSSHPubKey = `ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK3HP2HTJfOL8epkGlKQDbEWjby/lbw7geKYjRnM2lCT terraform-provider-envbuilder-test` +) + +func setupGitRepo(t testing.TB, files map[string]string) string { + t.Helper() + + dir := filepath.Join(t.TempDir(), "repo") + + writeFiles(t, dir, files) + + repo, err := git.PlainInitWithOptions(dir, &git.PlainInitOptions{ + InitOptions: git.InitOptions{ + DefaultBranch: plumbing.ReferenceName("refs/heads/main"), + }, + }) + require.NoError(t, err, "init git repo") + wt, err := repo.Worktree() + require.NoError(t, err, "get worktree") + _, err = wt.Add(".") + require.NoError(t, err, "add files") + _, err = wt.Commit("initial commit", &git.CommitOptions{ + Author: &object.Signature{ + Name: "test", + Email: "test@coder.com", + }, + }) + require.NoError(t, err, "commit files") + t.Logf("initialized git repo at %s", dir) + + return dir +} + +func writeFiles(t testing.TB, destPath string, files map[string]string) { + t.Helper() + + err := os.MkdirAll(destPath, 0o755) + require.NoError(t, err, "create dest path") + + for relPath, content := range files { + absPath := filepath.Join(destPath, relPath) + d := filepath.Dir(absPath) + bs := []byte(content) + require.NoError(t, os.MkdirAll(d, 0o755)) + require.NoError(t, os.WriteFile(absPath, bs, 0o644)) + t.Logf("wrote %d bytes to %s", len(bs), absPath) + } +} + +type testGitRepoSSH struct { + Dir string + URL string + Key string +} + +func serveGitRepoSSH(ctx context.Context, t testing.TB, dir string) testGitRepoSSH { + t.Helper() + + sshDir := filepath.Join(t.TempDir(), "ssh") + require.NoError(t, os.Mkdir(sshDir, 0o700)) + + keyPath := filepath.Join(sshDir, "id_ed25519") + require.NoError(t, os.WriteFile(keyPath, []byte(testSSHKey), 0o600)) + + // Start SSH server + addr := startSSHServer(ctx, t) + + // Serve git repo + repoURL := "ssh://" + addr + dir + return testGitRepoSSH{ + Dir: dir, + URL: repoURL, + Key: keyPath, + } +} + +func startSSHServer(ctx context.Context, t testing.TB) string { + t.Helper() + + s := &ssh.Server{ + PublicKeyHandler: func(ctx ssh.Context, key ssh.PublicKey) bool { + return true // Allow all keys. + }, + Handler: func(s ssh.Session) { + t.Logf("session started: %s", s.RawCommand()) + + args := s.Command() + cmd := exec.CommandContext(ctx, args[0], args[1:]...) + + in, err := cmd.StdinPipe() + assert.NoError(t, err, "stdin pipe") + out, err := cmd.StdoutPipe() + assert.NoError(t, err, "stdout pipe") + err = cmd.Start() + if err != nil { + t.Logf("command failed: %s", err) + return + } + t.Cleanup(func() { + _ = in.Close() + _ = out.Close() + _ = cmd.Process.Kill() + }) + + go func() { + _, _ = io.Copy(in, s) + _ = in.Close() + }() + go func() { + _, _ = io.Copy(s, out) + _ = out.Close() + _ = s.CloseWrite() + }() + err = cmd.Wait() + if err != nil { + t.Logf("command failed: %s", err) + } + + t.Logf("session ended: %s", s.RawCommand()) + + err = s.Exit(cmd.ProcessState.ExitCode()) + if err != nil { + t.Logf("session exit failed: %s", err) + } + }, + } + + ln, err := (&net.ListenConfig{}).Listen(ctx, "tcp", "localhost:0") + require.NoError(t, err, "listen") + + go func() { + err := s.Serve(ln) + if !errors.Is(err, ssh.ErrServerClosed) { + require.NoError(t, err) + } + }() + t.Cleanup(func() { + _ = s.Close() + _ = ln.Close() + }) + + return ln.Addr().String() +} diff --git a/internal/provider/provider_test.go b/internal/provider/provider_test.go index 1a88710..0a7edc3 100644 --- a/internal/provider/provider_test.go +++ b/internal/provider/provider_test.go @@ -8,7 +8,6 @@ import ( "context" "io" "os" - "path/filepath" "slices" "strings" "testing" @@ -43,11 +42,11 @@ func testAccPreCheck(t *testing.T) { type testDependencies struct { BuilderImage string - RepoDir string CacheRepo string + Repo testGitRepoSSH } -func setup(t testing.TB, files map[string]string) testDependencies { +func setup(ctx context.Context, t testing.TB, files map[string]string) testDependencies { t.Helper() envbuilderImage := getEnvOrDefault("ENVBUILDER_IMAGE", "ghcr.io/coder/envbuilder-preview") @@ -56,22 +55,31 @@ func setup(t testing.TB, files map[string]string) testDependencies { // TODO: envbuilder creates /.envbuilder/bin/envbuilder owned by root:root which we are unable to clean up. // This causes tests to fail. - repoDir := t.TempDir() regDir := t.TempDir() reg := registrytest.New(t, regDir) - writeFiles(t, files, repoDir) + + repoDir := setupGitRepo(t, files) + gitRepo := serveGitRepoSSH(ctx, t, repoDir) + return testDependencies{ BuilderImage: envbuilderImageRef, CacheRepo: reg + "/test", - RepoDir: repoDir, + Repo: gitRepo, } } func seedCache(ctx context.Context, t testing.TB, deps testDependencies) { + t.Helper() + + t.Logf("seeding cache with %s", deps.CacheRepo) + defer t.Logf("finished seeding cache with %s", deps.CacheRepo) + cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) require.NoError(t, err, "init docker client") t.Cleanup(func() { _ = cli.Close() }) + ensureImage(ctx, t, cli, deps.BuilderImage) + // Run envbuilder using this dir as a local layer cache ctr, err := cli.ContainerCreate(ctx, &container.Config{ Image: deps.BuilderImage, @@ -81,14 +89,19 @@ func seedCache(ctx context.Context, t testing.TB, deps testDependencies) { "ENVBUILDER_INIT_SCRIPT=exit", "ENVBUILDER_PUSH_IMAGE=true", "ENVBUILDER_VERBOSE=true", + "ENVBUILDER_GIT_URL=" + deps.Repo.URL, + "ENVBUILDER_GIT_SSH_PRIVATE_KEY_PATH=/id_ed25519", }, Labels: map[string]string{ testContainerLabel: "true", }, }, &container.HostConfig{ NetworkMode: container.NetworkMode("host"), - Binds: []string{deps.RepoDir + ":" + "/workspaces/empty"}, + Binds: []string{ + deps.Repo.Key + ":/id_ed25519", + }, }, nil, nil, "") + require.NoError(t, err, "failed to run envbuilder to seed cache") t.Cleanup(func() { _ = cli.ContainerRemove(ctx, ctr.ID, container.RemoveOptions{ @@ -133,17 +146,6 @@ func getEnvOrDefault(env, defVal string) string { return defVal } -func writeFiles(t testing.TB, files map[string]string, destPath string) { - for relPath, content := range files { - absPath := filepath.Join(destPath, relPath) - d := filepath.Dir(absPath) - bs := []byte(content) - require.NoError(t, os.MkdirAll(d, 0o755)) - require.NoError(t, os.WriteFile(absPath, bs, 0o644)) - t.Logf("wrote %d bytes to %s", len(bs), absPath) - } -} - func ensureImage(ctx context.Context, t testing.TB, cli *client.Client, ref string) { t.Helper()