Skip to content

Commit d205864

Browse files
dlipovetskydlipovetsky
authored
refactor: Write configuration under /etc/caren (#656)
**What problem does this PR solve?**: Previously, we wrote some configuration to /etc/cre. Now that the project name is CAREN, we should use /etc/caren. We also wrote containerd configuration to /etc/containerd/cre.d, but this configuration is not read by containerd directly. Instead, it is read by a script that merges it to the primary containerd configuration. For that reason, this configuration belongs under /etc/caren. **Which issue(s) this PR fixes**: Fixes # **How Has This Been Tested?**: <!-- Please describe the tests that you ran to verify your changes. Provide output from the tests and any manual steps needed to replicate the tests. --> **Special notes for your reviewer**: <!-- Use this to provide any additional information to the reviewers. This may include: - Best way to review the PR. - Where the author wants the most review attention on. - etc. -->
1 parent 2deeee9 commit d205864

File tree

10 files changed

+122
-83
lines changed

10 files changed

+122
-83
lines changed

‎pkg/common/containerd.go

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
// Copyright 2024 Nutanix. All rights reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
package common
5+
6+
import "path/filepath"
7+
8+
const (
9+
// ConfigDirOnRemote is the directory on the machine where we write CAREN configuration (e.g. scripts, patches
10+
// etc) as files.
11+
// These files are later applied by one or more commands that run on the machine.
12+
ConfigDirOnRemote = "/etc/caren"
13+
14+
// ContainerdScriptsDirOnRemote is the directory where we write scripts that relate to containerd as files.
15+
// It is a subdirectory of the root config directory.
16+
ContainerdScriptsDirOnRemote = ConfigDirOnRemote + "/containerd"
17+
18+
// ContainerdPatchDirOnRemote is the directory where we write containerd configuration patches as files.
19+
// It is a subdirectory of the containerd config directory.
20+
ContainerdPatchDirOnRemote = ConfigDirOnRemote + "/containerd/patches"
21+
)
22+
23+
// ConfigFilePathOnRemote returns the absolute path of a file that CAREN deploys onto the machine.
24+
func ConfigFilePathOnRemote(relativePath string) string {
25+
return filepath.Join(ConfigDirOnRemote, relativePath)
26+
}
27+
28+
// ContainerPathOnRemote returns the absolute path of a script that relates to containerd on the machine.
29+
func ContainerdScriptPathOnRemote(relativePath string) string {
30+
return filepath.Join(ContainerdScriptsDirOnRemote, relativePath)
31+
}
32+
33+
// ContainerdPatchPathOnRemote returns the absolute path of a containerd configuration patch on the machine.
34+
func ContainerdPatchPathOnRemote(relativePath string) string {
35+
return filepath.Join(ContainerdPatchDirOnRemote, relativePath)
36+
}

‎pkg/handlers/generic/mutation/containerdapplypatchesandrestart/apply_patches.go

Lines changed: 39 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -3,25 +3,53 @@
33
package containerdapplypatchesandrestart
44

55
import (
6+
"bytes"
67
_ "embed"
8+
"fmt"
9+
"text/template"
710

811
bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
12+
13+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/common"
914
)
1015

1116
const (
12-
ContainerdRestartScriptOnRemote = "/etc/containerd/restart.sh"
13-
ContainerdRestartScriptOnRemoteCommand = "/bin/bash " + ContainerdRestartScriptOnRemote
17+
tomlMergeImage = "ghcr.io/mesosphere/toml-merge:v0.2.0"
18+
)
19+
20+
var (
21+
containerdApplyPatchesScriptOnRemote = common.ContainerdScriptPathOnRemote(
22+
"apply-patches.sh",
23+
)
24+
containerdApplyPatchesScriptOnRemoteCommand = "/bin/bash " + containerdApplyPatchesScriptOnRemote
1425
)
1526

16-
//go:embed templates/containerd-restart.sh
17-
var containerdRestartScript []byte
27+
//go:embed templates/containerd-apply-patches.sh.gotmpl
28+
var containerdApplyConfigPatchesScript []byte
29+
30+
func generateContainerdApplyPatchesScript() (bootstrapv1.File, string, error) {
31+
t, err := template.New("").Parse(string(containerdApplyConfigPatchesScript))
32+
if err != nil {
33+
return bootstrapv1.File{}, "", fmt.Errorf("failed to parse go template: %w", err)
34+
}
35+
36+
templateInput := struct {
37+
TOMLMergeImage string
38+
PatchDir string
39+
}{
40+
TOMLMergeImage: tomlMergeImage,
41+
PatchDir: common.ContainerdPatchDirOnRemote,
42+
}
43+
44+
var b bytes.Buffer
45+
err = t.Execute(&b, templateInput)
46+
if err != nil {
47+
return bootstrapv1.File{}, "", fmt.Errorf("failed executing template: %w", err)
48+
}
1849

19-
//nolint:gocritic // no need for named return values
20-
func generateContainerdRestartScript() (bootstrapv1.File, string) {
2150
return bootstrapv1.File{
22-
Path: ContainerdRestartScriptOnRemote,
23-
Content: string(containerdRestartScript),
24-
Permissions: "0700",
25-
},
26-
ContainerdRestartScriptOnRemoteCommand
51+
Path: containerdApplyPatchesScriptOnRemote,
52+
Content: b.String(),
53+
Permissions: "0700",
54+
}, containerdApplyPatchesScriptOnRemoteCommand, nil
2755
}

‎pkg/handlers/generic/mutation/containerdapplypatchesandrestart/inject_test.go

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ var _ = Describe("Generate Containerd apply patches and restart patches", func()
4242
"path", containerdApplyPatchesScriptOnRemote,
4343
),
4444
gomega.HaveKeyWithValue(
45-
"path", ContainerdRestartScriptOnRemote,
45+
"path", containerdRestartScriptOnRemote,
4646
),
4747
),
4848
},
@@ -51,7 +51,7 @@ var _ = Describe("Generate Containerd apply patches and restart patches", func()
5151
Path: "/spec/template/spec/kubeadmConfigSpec/preKubeadmCommands",
5252
ValueMatcher: gomega.HaveExactElements(
5353
containerdApplyPatchesScriptOnRemoteCommand,
54-
ContainerdRestartScriptOnRemoteCommand,
54+
containerdRestartScriptOnRemoteCommand,
5555
),
5656
},
5757
},
@@ -78,7 +78,7 @@ var _ = Describe("Generate Containerd apply patches and restart patches", func()
7878
"path", containerdApplyPatchesScriptOnRemote,
7979
),
8080
gomega.HaveKeyWithValue(
81-
"path", ContainerdRestartScriptOnRemote,
81+
"path", containerdRestartScriptOnRemote,
8282
),
8383
),
8484
},
@@ -87,7 +87,7 @@ var _ = Describe("Generate Containerd apply patches and restart patches", func()
8787
Path: "/spec/template/spec/preKubeadmCommands",
8888
ValueMatcher: gomega.HaveExactElements(
8989
containerdApplyPatchesScriptOnRemoteCommand,
90-
ContainerdRestartScriptOnRemoteCommand,
90+
containerdRestartScriptOnRemoteCommand,
9191
),
9292
},
9393
},
@@ -109,7 +109,7 @@ var _ = Describe("Generate Containerd apply patches and restart patches", func()
109109

110110
func Test_generateContainerdApplyPatchesScript(t *testing.T) {
111111
wantFile := bootstrapv1.File{
112-
Path: "/etc/containerd/apply-patches.sh",
112+
Path: "/etc/caren/containerd/apply-patches.sh",
113113
Owner: "",
114114
Permissions: "0700",
115115
Encoding: "",
@@ -126,7 +126,7 @@ IFS=$'\n\t'
126126
# using -e does not work with globs.
127127
# See https://github.com/koalaman/shellcheck/wiki/SC2144 for an explanation of the following loop.
128128
patches_exist=false
129-
for file in "/etc/containerd/cre.d"/*.toml; do
129+
for file in "/etc/caren/containerd/patches"/*.toml; do
130130
if [ -e "${file}" ]; then
131131
patches_exist=true
132132
fi
@@ -135,7 +135,7 @@ for file in "/etc/containerd/cre.d"/*.toml; do
135135
done
136136
137137
if [ "${patches_exist}" = false ]; then
138-
echo "No TOML files found in the patch directory: /etc/containerd/cre.d - nothing to do"
138+
echo "No TOML files found in the patch directory: /etc/caren/containerd/patches - nothing to do"
139139
exit 0
140140
fi
141141
@@ -158,10 +158,10 @@ readonly tmp_ctr_mount_dir="$(mktemp -d)"
158158
159159
# Mount the toml-merge image filesystem and run the toml-merge binary to merge the TOML files.
160160
ctr --namespace k8s.io images mount "${TOML_MERGE_IMAGE}" "${tmp_ctr_mount_dir}"
161-
"${tmp_ctr_mount_dir}/usr/local/bin/toml-merge" -i --patch-file "/etc/containerd/cre.d/*.toml" /etc/containerd/config.toml
161+
"${tmp_ctr_mount_dir}/usr/local/bin/toml-merge" -i --patch-file "/etc/caren/containerd/patches/*.toml" /etc/containerd/config.toml
162162
`,
163163
}
164-
wantCmd := "/bin/bash /etc/containerd/apply-patches.sh"
164+
wantCmd := "/bin/bash /etc/caren/containerd/apply-patches.sh"
165165
file, cmd, _ := generateContainerdApplyPatchesScript()
166166
assert.Equal(t, wantFile, file)
167167
assert.Equal(t, wantCmd, cmd)

‎pkg/handlers/generic/mutation/containerdapplypatchesandrestart/restart.go

Lines changed: 14 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -3,47 +3,27 @@
33
package containerdapplypatchesandrestart
44

55
import (
6-
"bytes"
76
_ "embed"
8-
"fmt"
9-
"text/template"
107

118
bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
12-
)
139

14-
const (
15-
tomlMergeImage = "ghcr.io/mesosphere/toml-merge:v0.2.0"
16-
containerdPatchesDirOnRemote = "/etc/containerd/cre.d"
17-
containerdApplyPatchesScriptOnRemote = "/etc/containerd/apply-patches.sh"
18-
containerdApplyPatchesScriptOnRemoteCommand = "/bin/bash " + containerdApplyPatchesScriptOnRemote
10+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/common"
1911
)
2012

21-
//go:embed templates/containerd-apply-patches.sh.gotmpl
22-
var containerdApplyConfigPatchesScript []byte
23-
24-
func generateContainerdApplyPatchesScript() (bootstrapv1.File, string, error) {
25-
t, err := template.New("").Parse(string(containerdApplyConfigPatchesScript))
26-
if err != nil {
27-
return bootstrapv1.File{}, "", fmt.Errorf("failed to parse go template: %w", err)
28-
}
29-
30-
templateInput := struct {
31-
TOMLMergeImage string
32-
PatchDir string
33-
}{
34-
TOMLMergeImage: tomlMergeImage,
35-
PatchDir: containerdPatchesDirOnRemote,
36-
}
13+
var (
14+
containerdRestartScriptOnRemote = common.ContainerdScriptPathOnRemote("restart.sh")
15+
containerdRestartScriptOnRemoteCommand = "/bin/bash " + containerdRestartScriptOnRemote
16+
)
3717

38-
var b bytes.Buffer
39-
err = t.Execute(&b, templateInput)
40-
if err != nil {
41-
return bootstrapv1.File{}, "", fmt.Errorf("failed executing template: %w", err)
42-
}
18+
//go:embed templates/containerd-restart.sh
19+
var containerdRestartScript []byte
4320

21+
//nolint:gocritic // no need for named return values
22+
func generateContainerdRestartScript() (bootstrapv1.File, string) {
4423
return bootstrapv1.File{
45-
Path: containerdApplyPatchesScriptOnRemote,
46-
Content: b.String(),
47-
Permissions: "0700",
48-
}, containerdApplyPatchesScriptOnRemoteCommand, nil
24+
Path: containerdRestartScriptOnRemote,
25+
Content: string(containerdRestartScript),
26+
Permissions: "0700",
27+
},
28+
containerdRestartScriptOnRemoteCommand
4929
}

‎pkg/handlers/generic/mutation/containerdmetrics/metrics.go

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,23 +4,16 @@ package containerdmetrics
44

55
import (
66
_ "embed"
7-
"path"
87

98
cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
10-
)
119

12-
const (
13-
// TODO Factor out this constant to a common package.
14-
containerdPatchesDirOnRemote = "/etc/containerd/cre.d"
10+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/common"
1511
)
1612

1713
var (
1814
//go:embed files/metrics-config.toml
1915
metricsConfigDropIn []byte
20-
metricsConfigDropInFileOnRemote = path.Join(
21-
containerdPatchesDirOnRemote,
22-
"metrics-config.toml",
23-
)
16+
metricsConfigDropInFileOnRemote = common.ContainerdPatchPathOnRemote("metrics-config.toml")
2417
)
2518

2619
func generateMetricsConfigDropIn() cabpkv1.File {

‎pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_install_files.go

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,14 +10,18 @@ import (
1010
"text/template"
1111

1212
cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
13+
14+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/common"
1315
)
1416

15-
const (
16-
//nolint:gosec // Does not contain hard coded credentials.
17-
installKubeletCredentialProvidersScriptOnRemote = "/etc/cre/install-kubelet-credential-providers.sh"
17+
var (
18+
installKubeletCredentialProvidersScriptOnRemote = common.ConfigFilePathOnRemote(
19+
"install-kubelet-credential-providers.sh")
1820

1921
installKubeletCredentialProvidersScriptOnRemoteCommand = "/bin/bash " + installKubeletCredentialProvidersScriptOnRemote
22+
)
2023

24+
const (
2125
//nolint:gosec // Does not contain hard coded credentials.
2226
dynamicCredentialProviderImage = "ghcr.io/mesosphere/dynamic-credential-provider:v0.5.0"
2327

‎pkg/handlers/generic/mutation/imageregistries/credentials/inject_test.go

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -161,7 +161,7 @@ var _ = Describe("Generate Image registry patches", func() {
161161
Path: "/spec/template/spec/kubeadmConfigSpec/files",
162162
ValueMatcher: gomega.ContainElements(
163163
gomega.HaveKeyWithValue(
164-
"path", "/etc/cre/install-kubelet-credential-providers.sh",
164+
"path", "/etc/caren/install-kubelet-credential-providers.sh",
165165
),
166166
gomega.HaveKeyWithValue(
167167
"path", "/etc/kubernetes/image-credential-provider-config.yaml",
@@ -175,7 +175,7 @@ var _ = Describe("Generate Image registry patches", func() {
175175
Operation: "add",
176176
Path: "/spec/template/spec/kubeadmConfigSpec/preKubeadmCommands",
177177
ValueMatcher: gomega.ContainElement(
178-
"/bin/bash /etc/cre/install-kubelet-credential-providers.sh",
178+
"/bin/bash /etc/caren/install-kubelet-credential-providers.sh",
179179
),
180180
},
181181
{
@@ -222,7 +222,7 @@ var _ = Describe("Generate Image registry patches", func() {
222222
Path: "/spec/template/spec/kubeadmConfigSpec/files",
223223
ValueMatcher: gomega.ContainElements(
224224
gomega.HaveKeyWithValue(
225-
"path", "/etc/cre/install-kubelet-credential-providers.sh",
225+
"path", "/etc/caren/install-kubelet-credential-providers.sh",
226226
),
227227
gomega.HaveKeyWithValue(
228228
"path", "/etc/kubernetes/image-credential-provider-config.yaml",
@@ -239,7 +239,7 @@ var _ = Describe("Generate Image registry patches", func() {
239239
Operation: "add",
240240
Path: "/spec/template/spec/kubeadmConfigSpec/preKubeadmCommands",
241241
ValueMatcher: gomega.ContainElement(
242-
"/bin/bash /etc/cre/install-kubelet-credential-providers.sh",
242+
"/bin/bash /etc/caren/install-kubelet-credential-providers.sh",
243243
),
244244
},
245245
{
@@ -286,7 +286,7 @@ var _ = Describe("Generate Image registry patches", func() {
286286
Path: "/spec/template/spec/files",
287287
ValueMatcher: gomega.ContainElements(
288288
gomega.HaveKeyWithValue(
289-
"path", "/etc/cre/install-kubelet-credential-providers.sh",
289+
"path", "/etc/caren/install-kubelet-credential-providers.sh",
290290
),
291291
gomega.HaveKeyWithValue(
292292
"path", "/etc/kubernetes/image-credential-provider-config.yaml",
@@ -300,7 +300,7 @@ var _ = Describe("Generate Image registry patches", func() {
300300
Operation: "add",
301301
Path: "/spec/template/spec/preKubeadmCommands",
302302
ValueMatcher: gomega.ContainElement(
303-
"/bin/bash /etc/cre/install-kubelet-credential-providers.sh",
303+
"/bin/bash /etc/caren/install-kubelet-credential-providers.sh",
304304
),
305305
},
306306
{
@@ -344,7 +344,7 @@ var _ = Describe("Generate Image registry patches", func() {
344344
Path: "/spec/template/spec/files",
345345
ValueMatcher: gomega.ContainElements(
346346
gomega.HaveKeyWithValue(
347-
"path", "/etc/cre/install-kubelet-credential-providers.sh",
347+
"path", "/etc/caren/install-kubelet-credential-providers.sh",
348348
),
349349
gomega.HaveKeyWithValue(
350350
"path", "/etc/kubernetes/image-credential-provider-config.yaml",
@@ -361,7 +361,7 @@ var _ = Describe("Generate Image registry patches", func() {
361361
Operation: "add",
362362
Path: "/spec/template/spec/preKubeadmCommands",
363363
ValueMatcher: gomega.ContainElement(
364-
"/bin/bash /etc/cre/install-kubelet-credential-providers.sh",
364+
"/bin/bash /etc/caren/install-kubelet-credential-providers.sh",
365365
),
366366
},
367367
{

‎pkg/handlers/generic/mutation/mirrors/inject_test.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ var _ = Describe("Generate Global mirror patches", func() {
7070
"path", "/etc/containerd/certs.d/_default/hosts.toml",
7171
),
7272
gomega.HaveKeyWithValue(
73-
"path", "/etc/containerd/cre.d/registry-config.toml",
73+
"path", "/etc/caren/containerd/patches/registry-config.toml",
7474
),
7575
),
7676
},
@@ -105,7 +105,7 @@ var _ = Describe("Generate Global mirror patches", func() {
105105
"path", "/etc/certs/mirror.pem",
106106
),
107107
gomega.HaveKeyWithValue(
108-
"path", "/etc/containerd/cre.d/registry-config.toml",
108+
"path", "/etc/caren/containerd/patches/registry-config.toml",
109109
),
110110
),
111111
},
@@ -140,7 +140,7 @@ var _ = Describe("Generate Global mirror patches", func() {
140140
"path", "/etc/containerd/certs.d/_default/hosts.toml",
141141
),
142142
gomega.HaveKeyWithValue(
143-
"path", "/etc/containerd/cre.d/registry-config.toml",
143+
"path", "/etc/caren/containerd/patches/registry-config.toml",
144144
),
145145
),
146146
},
@@ -183,7 +183,7 @@ var _ = Describe("Generate Global mirror patches", func() {
183183
"path", "/etc/certs/mirror.pem",
184184
),
185185
gomega.HaveKeyWithValue(
186-
"path", "/etc/containerd/cre.d/registry-config.toml",
186+
"path", "/etc/caren/containerd/patches/registry-config.toml",
187187
),
188188
),
189189
},

0 commit comments

Comments
 (0)