Skip to content

feat(envbuilder.go): add support for build secrets #391

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 16 commits into from
Oct 29, 2024
Merged

feat(envbuilder.go): add support for build secrets #391

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
c9a1bf1
chore(envbuilder.go): prove the concept of build secret environment v…
SasSwart Oct 16, 2024
56924ed
feat(devcontainer): add support for build-secrets
SasSwart Oct 18, 2024
1416077
feat(devcontainer): add support for build-secrets
SasSwart Oct 24, 2024
0b4ea02
fix(envbuilder.go): fix self review notes and linting
SasSwart Oct 24, 2024
87e395a
Merge remote-tracking branch 'origin/main' into jjs/buildsecrets
SasSwart Oct 24, 2024
9227192
fix(envbuilder.go): fix linting
SasSwart Oct 24, 2024
7fff300
fix(envbuilder.go): add unit tests for build secret filtering
SasSwart Oct 25, 2024
c112f25
fix(integration): add integration tests for build secrets
SasSwart Oct 25, 2024
dbf7879
fix(integration): fix a test case for duplicate secret mounts
SasSwart Oct 28, 2024
58ae81c
fix(integration): fix a test case for duplicate secret mounts
SasSwart Oct 28, 2024
f2e8acd
Merge remote-tracking branch 'origin/main' into jjs/buildsecrets
SasSwart Oct 28, 2024
e1a0a59
fix(options): further prevent flakes in options testing
SasSwart Oct 28, 2024
1127121
Merge remote-tracking branch 'origin' into jjs/buildsecrets
SasSwart Oct 29, 2024
3d9aac2
fix(envbuilder.go): properly declare an empty slice
SasSwart Oct 29, 2024
39fe438
fix(envbuilder.go): document an exported function
SasSwart Oct 29, 2024
b0ee401
fix(options): add a note to some tests that cannot be run in parallel
SasSwart Oct 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions devcontainer/devcontainer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -400,11 +400,11 @@ func ImageFromDockerfile(dockerfileContent string) (name.Reference, error) {
arg = strings.TrimSpace(arg)
if strings.Contains(arg, "=") {
parts := strings.SplitN(arg, "=", 2)
key, err := lexer.ProcessWord(parts[0], args)
key, _, err := lexer.ProcessWord(parts[0], shell.EnvsFromSlice(args))
if err != nil {
return nil, fmt.Errorf("processing %q: %w", line, err)
}
val, err := lexer.ProcessWord(parts[1], args)
val, _, err := lexer.ProcessWord(parts[1], shell.EnvsFromSlice(args))
if err != nil {
return nil, fmt.Errorf("processing %q: %w", line, err)
}
Expand All @@ -421,7 +421,7 @@ func ImageFromDockerfile(dockerfileContent string) (name.Reference, error) {
if imageRef == "" {
return nil, fmt.Errorf("no FROM directive found")
}
imageRef, err := lexer.ProcessWord(imageRef, args)
imageRef, _, err := lexer.ProcessWord(imageRef, shell.EnvsFromSlice(args))
if err != nil {
return nil, fmt.Errorf("processing %q: %w", imageRef, err)
}
Expand Down
19 changes: 13 additions & 6 deletions envbuilder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -411,6 +411,10 @@ func run(ctx context.Context, opts options.Options, execArgs *execArgsInfo) erro
})
}

magicTempDir := workingdir.At(buildParams.BuildContext, workingdir.TempDir)
if err := opts.Filesystem.MkdirAll(magicTempDir.Path(), 0o755); err != nil {
return fmt.Errorf("create magic temp dir in build context: %w", err)
}
// In order to allow 'resuming' envbuilder, embed the binary into the image
// if it is being pushed.
// As these files will be owned by root, it is considerate to clean up
Expand All @@ -427,10 +431,6 @@ func run(ctx context.Context, opts options.Options, execArgs *execArgsInfo) erro
if err := util.AddAllowedPathToDefaultIgnoreList(workingDir.Features()); err != nil {
return fmt.Errorf("add features to ignore list: %w", err)
}
magicTempDir := workingdir.At(buildParams.BuildContext, workingdir.TempDir)
if err := opts.Filesystem.MkdirAll(magicTempDir.Path(), 0o755); err != nil {
return fmt.Errorf("create magic temp dir in build context: %w", err)
}
// Add the magic directives that embed the binary into the built image.
buildParams.DockerfileContent += workingdir.Directives

Expand Down Expand Up @@ -525,10 +525,15 @@ func run(ctx context.Context, opts options.Options, execArgs *execArgsInfo) erro
if val, ok := os.LookupEnv("KANIKO_REGISTRY_MIRROR"); ok {
registryMirror = strings.Split(val, ";")
}
var destinations []string
var destinations = []string{"image"}
if opts.CacheRepo != "" {
destinations = append(destinations, opts.CacheRepo)
}

buildSecrets := options.GetBuildSecrets(os.Environ())
// Ensure that build secrets do not make it into the runtime environment or the setup script:
options.ClearBuildSecrets()

kOpts := &config.KanikoOptions{
// Boilerplate!
CustomPlatform: platforms.Format(platforms.Normalize(platforms.DefaultSpec())),
Expand All @@ -538,6 +543,7 @@ func run(ctx context.Context, opts options.Options, execArgs *execArgsInfo) erro
RunStderr: stderrWriter,
Destinations: destinations,
NoPush: !opts.PushImage || len(destinations) == 0,
TarPath: filepath.Join(magicTempDir.Path(), "image.tar"),
CacheRunLayers: true,
CacheCopyLayers: true,
ForceBuildMetadata: opts.PushImage, // Force layers with no changes to be cached, required for cache probing.
Expand All @@ -553,6 +559,7 @@ func run(ctx context.Context, opts options.Options, execArgs *execArgsInfo) erro
},
ForceUnpack: true,
BuildArgs: buildParams.BuildArgs,
BuildSecrets: buildSecrets,
CacheRepo: opts.CacheRepo,
Cache: opts.CacheRepo != "" || opts.BaseImageCacheDir != "",
DockerfilePath: buildParams.DockerfilePath,
Expand All @@ -579,7 +586,7 @@ func run(ctx context.Context, opts options.Options, execArgs *execArgsInfo) erro
return nil, xerrors.Errorf("do build: %w", err)
}
endStage("🏗️ Built image!")
if opts.PushImage {
if opts.PushImage || true {
endStage = startStage("🏗️ Pushing image...")
if err := executor.DoPush(image, kOpts); err != nil {
return nil, xerrors.Errorf("do push: %w", err)
Expand Down
27 changes: 13 additions & 14 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ go 1.22.4

// There are a few options we need added to Kaniko!
// See: https://github.com/GoogleContainerTools/kaniko/compare/main...coder:kaniko:main
replace github.com/GoogleContainerTools/kaniko => github.com/coder/kaniko v0.0.0-20240925122543-caa18967f374
replace github.com/GoogleContainerTools/kaniko => github.com/coder/kaniko v0.0.0-20241024065816-8f144a699d23

// Required to import codersdk due to gvisor dependency.
replace tailscale.com => github.com/coder/tailscale v1.1.1-0.20240702054557-aa558fbe5374
Expand All @@ -19,8 +19,8 @@ require (
github.com/coder/serpent v0.7.0
github.com/containerd/platforms v0.2.1
github.com/distribution/distribution/v3 v3.0.0-alpha.1
github.com/docker/cli v27.2.0+incompatible
github.com/docker/docker v26.1.5+incompatible
github.com/docker/cli v27.2.1+incompatible
github.com/docker/docker v27.3.1+incompatible
github.com/fatih/color v1.17.0
github.com/gliderlabs/ssh v0.3.7
github.com/go-git/go-billy/v5 v5.5.0
Expand All @@ -31,7 +31,7 @@ require (
github.com/hashicorp/go-multierror v1.1.1
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
github.com/mattn/go-isatty v0.0.20
github.com/moby/buildkit v0.13.1
github.com/moby/buildkit v0.16.0
github.com/otiai10/copy v1.14.0
github.com/prometheus/procfs v0.15.1
github.com/sirupsen/logrus v1.9.3
Expand Down Expand Up @@ -100,25 +100,22 @@ require (
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/charmbracelet/lipgloss v0.8.0 // indirect
github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589 // indirect
github.com/cilium/ebpf v0.12.3 // indirect
github.com/cloudflare/circl v1.3.7 // indirect
github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0 // indirect
github.com/coder/quartz v0.1.0 // indirect
github.com/coder/terraform-provider-coder v0.23.0 // indirect
github.com/containerd/cgroups v1.1.0 // indirect
github.com/containerd/cgroups/v3 v3.0.2 // indirect
github.com/containerd/containerd v1.7.19 // indirect
github.com/containerd/containerd v1.7.21 // indirect
github.com/containerd/containerd/api v1.7.19 // indirect
github.com/containerd/continuity v0.4.3 // indirect
github.com/containerd/errdefs v0.1.0 // indirect
github.com/containerd/fifo v1.1.0 // indirect
github.com/containerd/log v0.1.0 // indirect
github.com/containerd/stargz-snapshotter/estargz v0.15.1 // indirect
github.com/containerd/ttrpc v1.2.5 // indirect
github.com/containerd/typeurl/v2 v2.1.1 // indirect
github.com/containerd/typeurl/v2 v2.2.0 // indirect
github.com/coreos/go-iptables v0.6.0 // indirect
github.com/coreos/go-oidc/v3 v3.10.0 // indirect
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
github.com/cyphar/filepath-securejoin v0.2.4 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
Expand Down Expand Up @@ -151,7 +148,7 @@ require (
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/btree v1.1.2 // indirect
github.com/google/nftables v0.2.0 // indirect
github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b // indirect
github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect
github.com/gorilla/handlers v1.5.1 // indirect
github.com/gorilla/mux v1.8.1 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0 // indirect
Expand All @@ -164,7 +161,7 @@ require (
github.com/hashicorp/go-version v1.7.0 // indirect
github.com/hashicorp/golang-lru v1.0.2 // indirect
github.com/hashicorp/golang-lru/arc/v2 v2.0.5 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.5 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
github.com/hashicorp/hcl/v2 v2.21.0 // indirect
github.com/hashicorp/logutils v1.0.0 // indirect
github.com/hashicorp/terraform-plugin-go v0.12.0 // indirect
Expand Down Expand Up @@ -202,11 +199,12 @@ require (
github.com/moby/patternmatcher v0.6.0 // indirect
github.com/moby/swarmkit/v2 v2.0.0-20230315203717-e28e8ba9bc83 // indirect
github.com/moby/sys/mount v0.3.3 // indirect
github.com/moby/sys/mountinfo v0.7.1 // indirect
github.com/moby/sys/mountinfo v0.7.2 // indirect
github.com/moby/sys/sequential v0.5.0 // indirect
github.com/moby/sys/signal v0.7.0 // indirect
github.com/moby/sys/signal v0.7.1 // indirect
github.com/moby/sys/symlink v0.2.0 // indirect
github.com/moby/sys/user v0.1.0 // indirect
github.com/moby/sys/user v0.3.0 // indirect
github.com/moby/sys/userns v0.1.0 // indirect
github.com/moby/term v0.5.0 // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/muesli/reflow v0.3.0 // indirect
Expand Down Expand Up @@ -245,6 +243,7 @@ require (
github.com/tailscale/wireguard-go v0.0.0-20231121184858-cc193a0b3272 // indirect
github.com/tcnksm/go-httpstat v0.2.0 // indirect
github.com/tinylib/msgp v1.1.8 // indirect
github.com/tonistiigi/go-csvvalue v0.0.0-20240710180619-ddb21b71c0b4 // indirect
github.com/twpayne/go-vfs/v5 v5.0.4 // indirect
github.com/u-root/uio v0.0.0-20240209044354-b3d14b93376a // indirect
github.com/valyala/fasthttp v1.55.0 // indirect
Expand Down
Loading
Loading