Skip to content

Commit 3f763d1

Browse files
dkoshkindkoshkin
committed
feat: use registryMirror addon as Containerd mirror
1 parent 91c70c3 commit 3f763d1

File tree

2 files changed

+144
-7
lines changed

2 files changed

+144
-7
lines changed

‎pkg/handlers/generic/mutation/mirrors/inject.go

Lines changed: 55 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
1212
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
1313
"k8s.io/apimachinery/pkg/runtime"
14+
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
1415
bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
1516
controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
1617
runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
@@ -22,6 +23,7 @@ import (
2223
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches"
2324
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches/selectors"
2425
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/variables"
26+
registrymirrorutils "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/registrymirror/utils"
2527
handlersutils "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/utils"
2628
)
2729

@@ -62,8 +64,8 @@ func (h *globalMirrorPatchHandler) Mutate(
6264
obj *unstructured.Unstructured,
6365
vars map[string]apiextensionsv1.JSON,
6466
holderRef runtimehooksv1.HolderReference,
65-
clusterKey ctrlclient.ObjectKey,
66-
_ mutation.ClusterGetter,
67+
_ ctrlclient.ObjectKey,
68+
clusterGetter mutation.ClusterGetter,
6769
) error {
6870
log := ctrl.LoggerFrom(ctx).WithValues(
6971
"holderRef", holderRef,
@@ -82,14 +84,24 @@ func (h *globalMirrorPatchHandler) Mutate(
8284
v1alpha1.ImageRegistriesVariableName,
8385
)
8486

87+
_, registryMirrorErr := variables.Get[v1alpha1.RegistryMirror](
88+
vars,
89+
v1alpha1.ClusterConfigVariableName,
90+
[]string{"addons", v1alpha1.RegistryMirrorVariableName}...)
91+
8592
switch {
86-
case variables.IsNotFoundError(imageRegistriesErr) && variables.IsNotFoundError(globalMirrorErr):
87-
log.V(5).Info("Image Registry Credentials and Global Registry Mirror variable not defined")
93+
case variables.IsNotFoundError(imageRegistriesErr) &&
94+
variables.IsNotFoundError(globalMirrorErr) &&
95+
variables.IsNotFoundError(registryMirrorErr):
96+
log.V(5).
97+
Info("Image Registry Credentials and Global Registry Mirror and Registry Mirror Addon variable not defined")
8898
return nil
8999
case imageRegistriesErr != nil && !variables.IsNotFoundError(imageRegistriesErr):
90100
return imageRegistriesErr
91101
case globalMirrorErr != nil && !variables.IsNotFoundError(globalMirrorErr):
92102
return globalMirrorErr
103+
case registryMirrorErr != nil && !variables.IsNotFoundError(registryMirrorErr):
104+
return registryMirrorErr
93105
}
94106

95107
var registriesWithOptionalCA []containerdConfig //nolint:prealloc // We don't know the size of the slice yet.
@@ -121,6 +133,26 @@ func (h *globalMirrorPatchHandler) Mutate(
121133
registryWithOptionalCredentials,
122134
)
123135
}
136+
if registryMirrorErr == nil {
137+
cluster, err := clusterGetter(ctx)
138+
if err != nil {
139+
log.Error(
140+
err,
141+
"failed to get cluster from Global Mirror mutation handler",
142+
)
143+
return err
144+
}
145+
146+
registryConfig, err := containerdConfigFromRegistryMirrorAddon(
147+
ctx,
148+
h.client,
149+
cluster,
150+
)
151+
if err != nil {
152+
return err
153+
}
154+
registriesWithOptionalCA = append(registriesWithOptionalCA, registryConfig)
155+
}
124156

125157
needConfiguration := needContainerdConfiguration(
126158
registriesWithOptionalCA,
@@ -234,6 +266,25 @@ func containerdConfigFromImageRegistry(
234266
return configWithOptionalCACert, nil
235267
}
236268

269+
func containerdConfigFromRegistryMirrorAddon(
270+
_ context.Context,
271+
_ ctrlclient.Client,
272+
cluster *clusterv1.Cluster,
273+
) (containerdConfig, error) {
274+
serviceIP, err := registrymirrorutils.ServiceIPForCluster(cluster)
275+
if err != nil {
276+
return containerdConfig{}, fmt.Errorf("error getting service IP for the registry mirror addon: %w", err)
277+
}
278+
279+
config := containerdConfig{
280+
// FIXME: Generate a self-signed CA.
281+
URL: fmt.Sprintf("http://%s", serviceIP),
282+
Mirror: true,
283+
}
284+
285+
return config, nil
286+
}
287+
237288
func generateFiles(
238289
registriesWithOptionalCA []containerdConfig,
239290
) ([]bootstrapv1.File, error) {

‎pkg/handlers/generic/mutation/mirrors/inject_test.go

Lines changed: 89 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,11 @@ import (
1111
"github.com/stretchr/testify/assert"
1212
corev1 "k8s.io/api/core/v1"
1313
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
14+
"k8s.io/apimachinery/pkg/runtime"
15+
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
1416
"k8s.io/apiserver/pkg/storage/names"
17+
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
18+
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
1519
runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
1620

1721
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1"
@@ -32,6 +36,10 @@ func TestMirrorsPatch(t *testing.T) {
3236
}
3337

3438
var _ = Describe("Generate Global mirror patches", func() {
39+
clientScheme := runtime.NewScheme()
40+
utilruntime.Must(clientgoscheme.AddToScheme(clientScheme))
41+
utilruntime.Must(clusterv1.AddToScheme(clientScheme))
42+
3543
patchGenerator := func() mutation.GeneratePatches {
3644
// Always initialize the testEnv variable in the closure.
3745
// This will allow ginkgo to initialize testEnv variable during test execution time.
@@ -40,7 +48,7 @@ var _ = Describe("Generate Global mirror patches", func() {
4048
// that are written by the tests.
4149
// Test cases writes credentials secret that the mutator handler reads.
4250
// Using direct client will enable reading it immediately.
43-
client, err := testEnv.GetK8sClient()
51+
client, err := testEnv.GetK8sClientWithScheme(clientScheme)
4452
gomega.Expect(err).To(gomega.BeNil())
4553
return mutation.NewMetaGeneratePatchesHandler("", client, NewPatch(client)).(mutation.GeneratePatches)
4654
}
@@ -330,11 +338,69 @@ var _ = Describe("Generate Global mirror patches", func() {
330338
},
331339
},
332340
},
341+
{
342+
Name: "files added in KubeadmControlPlaneTemplate for registry mirror addon",
343+
Vars: []runtimehooksv1.Variable{
344+
capitest.VariableWithValue(
345+
v1alpha1.ClusterConfigVariableName,
346+
v1alpha1.RegistryMirror{},
347+
[]string{"addons", v1alpha1.RegistryMirrorVariableName}...,
348+
),
349+
},
350+
RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""),
351+
ExpectedPatchMatchers: []capitest.JSONPatchMatcher{
352+
{
353+
Operation: "add",
354+
Path: "/spec/template/spec/kubeadmConfigSpec/files",
355+
ValueMatcher: gomega.HaveExactElements(
356+
gomega.HaveKeyWithValue(
357+
"path", "/etc/containerd/certs.d/_default/hosts.toml",
358+
),
359+
gomega.HaveKeyWithValue(
360+
"path", "/etc/caren/containerd/patches/registry-config.toml",
361+
),
362+
),
363+
},
364+
},
365+
},
366+
{
367+
Name: "files added in KubeadmConfigTemplate for registry mirror addon",
368+
Vars: []runtimehooksv1.Variable{
369+
capitest.VariableWithValue(
370+
v1alpha1.ClusterConfigVariableName,
371+
v1alpha1.RegistryMirror{},
372+
[]string{"addons", v1alpha1.RegistryMirrorVariableName}...,
373+
),
374+
capitest.VariableWithValue(
375+
"builtin",
376+
map[string]any{
377+
"machineDeployment": map[string]any{
378+
"class": names.SimpleNameGenerator.GenerateName("worker-"),
379+
},
380+
},
381+
),
382+
},
383+
RequestItem: request.NewKubeadmConfigTemplateRequestItem(""),
384+
ExpectedPatchMatchers: []capitest.JSONPatchMatcher{
385+
{
386+
Operation: "add",
387+
Path: "/spec/template/spec/files",
388+
ValueMatcher: gomega.HaveExactElements(
389+
gomega.HaveKeyWithValue(
390+
"path", "/etc/containerd/certs.d/_default/hosts.toml",
391+
),
392+
gomega.HaveKeyWithValue(
393+
"path", "/etc/caren/containerd/patches/registry-config.toml",
394+
),
395+
),
396+
},
397+
},
398+
},
333399
}
334400

335401
// Create credentials secret before each test
336402
BeforeEach(func(ctx SpecContext) {
337-
client, err := helpers.TestEnv.GetK8sClient()
403+
client, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme)
338404
gomega.Expect(err).To(gomega.BeNil())
339405
gomega.Expect(client.Create(
340406
ctx,
@@ -344,11 +410,21 @@ var _ = Describe("Generate Global mirror patches", func() {
344410
ctx,
345411
newMirrorSecretWithoutCA(validMirrorNoCASecretName, request.Namespace),
346412
)).To(gomega.BeNil())
413+
414+
gomega.Expect(client.Create(
415+
ctx,
416+
&clusterv1.Cluster{
417+
ObjectMeta: metav1.ObjectMeta{
418+
Name: request.ClusterName,
419+
Namespace: request.Namespace,
420+
},
421+
},
422+
)).To(gomega.BeNil())
347423
})
348424

349425
// Delete credentials secret after each test
350426
AfterEach(func(ctx SpecContext) {
351-
client, err := helpers.TestEnv.GetK8sClient()
427+
client, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme)
352428
gomega.Expect(err).To(gomega.BeNil())
353429
gomega.Expect(client.Delete(
354430
ctx,
@@ -358,6 +434,16 @@ var _ = Describe("Generate Global mirror patches", func() {
358434
ctx,
359435
newMirrorSecretWithoutCA(validMirrorNoCASecretName, request.Namespace),
360436
)).To(gomega.BeNil())
437+
438+
gomega.Expect(client.Delete(
439+
ctx,
440+
&clusterv1.Cluster{
441+
ObjectMeta: metav1.ObjectMeta{
442+
Name: request.ClusterName,
443+
Namespace: request.Namespace,
444+
},
445+
},
446+
)).To(gomega.BeNil())
361447
})
362448

363449
// create test node for each case

0 commit comments

Comments
 (0)