Skip to content

Commit 350cbb8

Browse files
authored
Merge pull request #30 from coder/jjs/build-secrets
feat(pkg/commands): add support for build secrets
2 parents 51b3a16 + a8c3296 commit 350cbb8

File tree

11 files changed

+379
-20
lines changed

11 files changed

+379
-20
lines changed

cmd/executor/cmd/root.go

+14
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ func init() {
6262

6363
addKanikoOptionsFlags()
6464
addHiddenFlags(RootCmd)
65+
opts.BuildSecrets = readBuildSecrets(os.Environ())
6566
RootCmd.PersistentFlags().BoolVarP(&opts.IgnoreVarRun, "whitelist-var-run", "", true, "Ignore /var/run directory when taking image snapshot. Set it to false to preserve /var/run/ in destination image.")
6667
RootCmd.PersistentFlags().MarkDeprecated("whitelist-var-run", "Please use ignore-var-run instead.")
6768
}
@@ -296,6 +297,19 @@ func addHiddenFlags(cmd *cobra.Command) {
296297
cmd.PersistentFlags().MarkHidden("bucket")
297298
}
298299

300+
const buildSecretPrefix = "KANIKO_BUILD_SECRET_"
301+
302+
func readBuildSecrets(environment []string) []string {
303+
var buildSecrets []string
304+
for _, secret := range environment {
305+
if strings.HasPrefix(secret, buildSecretPrefix) {
306+
buildSecrets = append(buildSecrets, strings.TrimPrefix(secret, buildSecretPrefix))
307+
}
308+
}
309+
310+
return buildSecrets
311+
}
312+
299313
// checkKanikoDir will check whether the executor is operating in the default '/kaniko' directory,
300314
// conducting the relevant operations if it is not
301315
func checkKanikoDir(dir string) error {
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# Copyright 2020 Google, Inc. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
FROM debian:10.13
16+
# Make sure we set secrets to env the same way docker does
17+
RUN --mount=type=secret,id=FOO,env=FOO,required=true echo "$FOO" > /etc/foo
18+
# Make sure we set secrets to default disk location the same way docker does
19+
RUN --mount=type=secret,id=BAR cat /run/secrets/BAR > /etc/bar
20+
# Make sure we set secrets to custom disk location the same way docker does
21+
RUN --mount=type=secret,id=BAZ,target=/baz.secret cat /baz.secret > /etc/baz
22+
# Make sure relative targets for secret mounts work:
23+
WORKDIR /etc
24+
RUN --mount=type=secret,id=QUX,target=qux.secret cat qux.secret > /etc/qux
25+
26+
# Test with ARG so that we know our implementation of secrets don't override args
27+
ARG file
28+
RUN echo "run" > $file

integration/images.go

+31-1
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ var argsMap = map[string][]string{
5555
"Dockerfile_test_run": {"file=/file"},
5656
"Dockerfile_test_run_new": {"file=/file"},
5757
"Dockerfile_test_run_redo": {"file=/file"},
58+
"Dockerfile_test_run_mount": {"file=/file"},
5859
"Dockerfile_test_workdir": {"workdir=/arg/workdir"},
5960
"Dockerfile_test_add": {"file=context/foo"},
6061
"Dockerfile_test_arg_secret": {"SSH_PRIVATE_KEY", "SSH_PUBLIC_KEY=Pµbl1cK€Y"},
@@ -68,6 +69,20 @@ var argsMap = map[string][]string{
6869
"Dockerfile_test_multistage": {"file=/foo2"},
6970
}
7071

72+
type buildSecret struct {
73+
name string
74+
value string
75+
}
76+
77+
var secretsMap = map[string][]buildSecret{
78+
"Dockerfile_test_run_mount": {
79+
{name: "FOO", value: "foo"},
80+
{name: "BAR", value: "bar"},
81+
{name: "BAZ", value: "baz"},
82+
{name: "QUX", value: "qux"},
83+
},
84+
}
85+
7186
// Environment to build Dockerfiles with, used for both docker and kaniko builds
7287
var envsMap = map[string][]string{
7388
"Dockerfile_test_arg_secret": {"SSH_PRIVATE_KEY=ThEPriv4t3Key"},
@@ -257,8 +272,15 @@ func (d *DockerFileBuilder) BuildDockerImage(t *testing.T, imageRepo, dockerfile
257272
buildArgs = append(buildArgs, buildArgFlag, arg)
258273
}
259274

275+
var buildSecrets []string
276+
secretFlag := "--secret"
277+
for _, secret := range secretsMap[dockerfile] {
278+
buildSecrets = append(buildSecrets, secretFlag, "id="+secret.name)
279+
}
280+
260281
// build docker image
261-
additionalFlags := append(buildArgs, additionalDockerFlagsMap[dockerfile]...)
282+
additionalFlags := append(buildArgs, buildSecrets...)
283+
additionalFlags = append(additionalFlags, additionalDockerFlagsMap[dockerfile]...)
262284
dockerImage := strings.ToLower(imageRepo + dockerPrefix + dockerfile)
263285

264286
dockerArgs := []string{
@@ -278,6 +300,9 @@ func (d *DockerFileBuilder) BuildDockerImage(t *testing.T, imageRepo, dockerfile
278300
if env, ok := envsMap[dockerfile]; ok {
279301
dockerCmd.Env = append(dockerCmd.Env, env...)
280302
}
303+
for _, secret := range secretsMap[dockerfile] {
304+
dockerCmd.Env = append(dockerCmd.Env, fmt.Sprintf("%s=%s", secret.name, secret.value))
305+
}
281306

282307
out, err := RunCommandWithoutTest(dockerCmd)
283308
if err != nil {
@@ -506,6 +531,11 @@ func buildKanikoImage(
506531
}
507532
}
508533

534+
if secrets, ok := secretsMap[dockerfile]; ok {
535+
for _, secret := range secrets {
536+
dockerRunFlags = append(dockerRunFlags, "-e", fmt.Sprintf("KANIKO_BUILD_SECRET_%s=%s", secret.name, secret.value))
537+
}
538+
}
509539
dockerRunFlags = addServiceAccountFlags(dockerRunFlags, serviceAccount)
510540

511541
kanikoDockerfilePath := path.Join(buildContextPath, dockerfilesPath, dockerfile)

integration/integration_test.go

+3-2
Original file line numberDiff line numberDiff line change
@@ -591,8 +591,9 @@ func TestBuildWithHTTPError(t *testing.T) {
591591

592592
func TestLayers(t *testing.T) {
593593
offset := map[string]int{
594-
"Dockerfile_test_add": 12,
595-
"Dockerfile_test_scratch": 3,
594+
"Dockerfile_test_add": 12,
595+
"Dockerfile_test_scratch": 3,
596+
"Dockerfile_test_run_mount": 1, // The WORKDIR layer is not present in the kaniko image
596597
}
597598

598599
if os.Getenv("CI") == "true" {

pkg/commands/commands.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -64,13 +64,13 @@ type DockerCommand interface {
6464
IsArgsEnvsRequiredInCache() bool
6565
}
6666

67-
func GetCommand(cmd instructions.Command, fileContext util.FileContext, useNewRun bool, cacheCopy bool, cacheRun bool, output *RunOutput) (DockerCommand, error) {
67+
func GetCommand(cmd instructions.Command, fileContext util.FileContext, useNewRun bool, cacheCopy bool, cacheRun bool, output *RunOutput, buildSecrets []string) (DockerCommand, error) {
6868
switch c := cmd.(type) {
6969
case *instructions.RunCommand:
7070
if useNewRun {
71-
return &RunMarkerCommand{cmd: c, shdCache: cacheRun, output: output}, nil
71+
return &RunMarkerCommand{cmd: c, shdCache: cacheRun, output: output, buildSecrets: buildSecrets}, nil
7272
}
73-
return &RunCommand{cmd: c, shdCache: cacheRun, output: output}, nil
73+
return &RunCommand{cmd: c, shdCache: cacheRun, output: output, buildSecrets: buildSecrets}, nil
7474
case *instructions.CopyCommand:
7575
return &CopyCommand{cmd: c, fileContext: fileContext, shdCache: cacheCopy}, nil
7676
case *instructions.ExposeCommand:

pkg/commands/run.go

+153-5
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,10 @@ package commands
1919
import (
2020
"fmt"
2121
"io"
22+
"io/fs"
2223
"os"
2324
"os/exec"
25+
"path/filepath"
2426
"strings"
2527
"syscall"
2628

@@ -42,11 +44,14 @@ type RunOutput struct {
4244

4345
type RunCommand struct {
4446
BaseCommand
45-
cmd *instructions.RunCommand
46-
output *RunOutput
47-
shdCache bool
47+
cmd *instructions.RunCommand
48+
output *RunOutput
49+
buildSecrets []string
50+
shdCache bool
4851
}
4952

53+
const secretsDir = "/run/secrets"
54+
5055
// for testing
5156
var (
5257
userLookup = util.LookupUser
@@ -57,10 +62,10 @@ func (r *RunCommand) IsArgsEnvsRequiredInCache() bool {
5762
}
5863

5964
func (r *RunCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.BuildArgs) error {
60-
return runCommandInExec(config, buildArgs, r.cmd, r.output)
65+
return runCommandInExec(config, buildArgs, r.cmd, r.output, r.buildSecrets)
6166
}
6267

63-
func runCommandInExec(config *v1.Config, buildArgs *dockerfile.BuildArgs, cmdRun *instructions.RunCommand, output *RunOutput) error {
68+
func runCommandInExec(config *v1.Config, buildArgs *dockerfile.BuildArgs, cmdRun *instructions.RunCommand, output *RunOutput, buildSecrets []string) (err error) {
6469
if output == nil {
6570
output = &RunOutput{}
6671
}
@@ -131,6 +136,82 @@ func runCommandInExec(config *v1.Config, buildArgs *dockerfile.BuildArgs, cmdRun
131136
return errors.Wrap(err, "adding default HOME variable")
132137
}
133138

139+
cmdRun.Expand(func(word string) (string, error) {
140+
// NOTE(SasSwart): This is a noop function. It's here to satisfy the buildkit parser.
141+
// Without this, the buildkit parser won't parse --mount flags for RUN directives.
142+
// Support for expansion in RUN directives deferred until its needed.
143+
// https://docs.docker.com/build/building/variables/
144+
return word, nil
145+
})
146+
147+
buildSecretsMap := make(map[string]string)
148+
for _, s := range buildSecrets {
149+
secretName, secretValue, found := strings.Cut(s, "=")
150+
if !found {
151+
return fmt.Errorf("invalid secret %s", s)
152+
}
153+
buildSecretsMap[secretName] = secretValue
154+
}
155+
156+
secretFileManager := fileCreatorCleaner{}
157+
defer func() {
158+
cleanupErr := secretFileManager.Clean()
159+
if err == nil {
160+
err = cleanupErr
161+
}
162+
}()
163+
164+
mounts := instructions.GetMounts(cmdRun)
165+
for _, mount := range mounts {
166+
switch mount.Type {
167+
case instructions.MountTypeSecret:
168+
// Implemented as per:
169+
// https://docs.docker.com/reference/dockerfile/#run---mounttypesecret
170+
171+
envName := mount.CacheID
172+
secret, secretSet := buildSecretsMap[envName]
173+
if !secretSet && mount.Required {
174+
return fmt.Errorf("required secret %s not found", mount.CacheID)
175+
}
176+
177+
// If a target is specified, we write to the file specified by the target:
178+
// If no target is specified and no env is specified, we write to /run/secrets/<id>
179+
// If no target is specified and an env is specified, we set the env and don't write to file
180+
if mount.Env == nil || mount.Target != "" {
181+
targetFile := mount.Target
182+
if targetFile == "" {
183+
targetFile = filepath.Join(secretsDir, mount.CacheID)
184+
}
185+
if !filepath.IsAbs(targetFile) {
186+
targetFile = filepath.Join(config.WorkingDir, targetFile)
187+
}
188+
secretFileManager.MkdirAndWriteFile(targetFile, []byte(secret), 0700, 0600)
189+
}
190+
191+
// We don't return in the block above, because its possible to have both a target and an env.
192+
// As such we need this guard clause or we risk getting a nil pointer below.
193+
if mount.Env == nil {
194+
continue
195+
}
196+
197+
targetEnv := *mount.Env
198+
if targetEnv == "" {
199+
targetEnv = mount.CacheID
200+
}
201+
202+
env = append(env, fmt.Sprintf("%s=%s", targetEnv, secret))
203+
// NOTE(SasSwart):
204+
// Buildkit v0.16.0 brought support for `RUN --mount` flags. Kaniko support for the mount
205+
// types below is deferred until its needed.
206+
// case instructions.MountTypeBind:
207+
// case instructions.MountTypeTmpfs:
208+
// case instructions.MountTypeCache:
209+
// case instructions.MountTypeSSH
210+
default:
211+
logrus.Warnf("Mount type %s is not supported", mount.Type)
212+
}
213+
}
214+
134215
cmd.Env = env
135216

136217
logrus.Infof("Running: %s", cmd.Args)
@@ -153,6 +234,73 @@ func runCommandInExec(config *v1.Config, buildArgs *dockerfile.BuildArgs, cmdRun
153234
return nil
154235
}
155236

237+
// fileCreatorCleaner keeps tracks of all files and directories that it created in the order that they were created.
238+
// Once asked to clean up, it will remove all files and directories in the reverse order that they were created.
239+
type fileCreatorCleaner struct {
240+
filesToClean []string
241+
dirsToClean []string
242+
}
243+
244+
func (s *fileCreatorCleaner) MkdirAndWriteFile(path string, data []byte, dirPerm, filePerm os.FileMode) error {
245+
dirPath := filepath.Dir(path)
246+
parentDirs := strings.Split(dirPath, string(os.PathSeparator))
247+
248+
// Start at the root directory
249+
currentPath := string(os.PathSeparator)
250+
251+
for _, nextDirDown := range parentDirs {
252+
if nextDirDown == "" {
253+
continue
254+
}
255+
// Traverse one level down
256+
currentPath = filepath.Join(currentPath, nextDirDown)
257+
258+
if _, err := filesystem.FS.Stat(currentPath); errors.Is(err, os.ErrNotExist) {
259+
if err := filesystem.FS.Mkdir(currentPath, dirPerm); err != nil {
260+
return err
261+
}
262+
s.dirsToClean = append(s.dirsToClean, currentPath)
263+
}
264+
}
265+
266+
// With all parent directories created, we can now create the actual secret file
267+
if err := filesystem.FS.WriteFile(path, []byte(data), 0600); err != nil {
268+
return errors.Wrap(err, "writing secret to file")
269+
}
270+
s.filesToClean = append(s.filesToClean, path)
271+
272+
return nil
273+
}
274+
275+
func (s *fileCreatorCleaner) Clean() error {
276+
for i := len(s.filesToClean) - 1; i >= 0; i-- {
277+
if err := filesystem.FS.Remove(s.filesToClean[i]); err != nil {
278+
if errors.Is(err, os.ErrNotExist) {
279+
continue
280+
}
281+
return err
282+
}
283+
}
284+
285+
for i := len(s.dirsToClean) - 1; i >= 0; i-- {
286+
if err := filesystem.FS.Remove(s.dirsToClean[i]); err != nil {
287+
pathErr := new(fs.PathError)
288+
// If a path that we need to clean up is not empty, then that means
289+
// that a third party has placed something in there since we created it.
290+
// In that case, we should not remove it, because it no longer belongs exclusively to us.
291+
if errors.As(err, &pathErr) && pathErr.Err == syscall.ENOTEMPTY {
292+
continue
293+
}
294+
if errors.Is(err, os.ErrNotExist) {
295+
continue
296+
}
297+
return err
298+
}
299+
}
300+
301+
return nil
302+
}
303+
156304
// addDefaultHOME adds the default value for HOME if it isn't already set
157305
func addDefaultHOME(u string, envs []string) ([]string, error) {
158306
for _, env := range envs {

pkg/commands/run_marker.go

+6-5
Original file line numberDiff line numberDiff line change
@@ -28,17 +28,18 @@ import (
2828

2929
type RunMarkerCommand struct {
3030
BaseCommand
31-
cmd *instructions.RunCommand
32-
output *RunOutput
33-
Files []string
34-
shdCache bool
31+
cmd *instructions.RunCommand
32+
output *RunOutput
33+
Files []string
34+
buildSecrets []string
35+
shdCache bool
3536
}
3637

3738
func (r *RunMarkerCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.BuildArgs) error {
3839
// run command `touch filemarker`
3940
logrus.Debugf("Using new RunMarker command")
4041
prevFilesMap, _ := util.GetFSInfoMap("/", map[string]os.FileInfo{})
41-
if err := runCommandInExec(config, buildArgs, r.cmd, r.output); err != nil {
42+
if err := runCommandInExec(config, buildArgs, r.cmd, r.output, r.buildSecrets); err != nil {
4243
return err
4344
}
4445
_, r.Files = util.GetFSInfoMap("/", prevFilesMap)

0 commit comments

Comments
 (0)