Skip to content

Commit f2818d7

Browse files
dkoshkindkoshkin
authored
feat: use registryMirror addon as Containerd mirror (#1117)
**What problem does this PR solve?**: Depends on #1116 Automatically sets the registryMirror as a Containerd mirror. We're not updating the Cluster object because this should not be a user controller configuration and the IP used as the mirror is determined based on the addon handler and the Service CIDRs. Tested in a Docker cluster: ``` $ kubectl port-forward \ --address=0.0.0.0 \ --namespace registry-mirror-system \ pod/registry-mirror-docker-registry-0 5000:5000 # Push an image tag that doesn't exist in dockerhub $ crane copy nginx:latest 0.0.0.0:5000/library/nginx:dkoshkin --insecure $ kubectl run nginx-working --image=docker.io/library/nginx:dkoshkin $ kubectl run nginx-should-be-broken --image=docker.io/library/nginx:dne $ kubectl get pods NAME READY STATUS RESTARTS AGE cluster-autoscaler-0196931c-cb53-7abf-aa89-49c82c42ced5-86w5j8c 0/1 ContainerCreating 0 19m nginx-should-be-broken 0/1 ErrImagePull 0 11m nginx-working 1/1 Running 0 11m ``` **Which issue(s) this PR fixes**: Fixes # **How Has This Been Tested?**: <!-- Please describe the tests that you ran to verify your changes. Provide output from the tests and any manual steps needed to replicate the tests. --> **Special notes for your reviewer**: <!-- Use this to provide any additional information to the reviewers. This may include: - Best way to review the PR. - Where the author wants the most review attention on. - etc. -->
1 parent 6fcafb7 commit f2818d7

File tree

2 files changed

+206
-7
lines changed

2 files changed

+206
-7
lines changed

‎pkg/handlers/generic/mutation/mirrors/inject.go

Lines changed: 47 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
1212
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
1313
"k8s.io/apimachinery/pkg/runtime"
14+
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
1415
bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
1516
controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
1617
runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
@@ -22,6 +23,7 @@ import (
2223
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches"
2324
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches/selectors"
2425
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/variables"
26+
registryutils "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/registry/utils"
2527
handlersutils "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/utils"
2628
)
2729

@@ -62,8 +64,8 @@ func (h *globalMirrorPatchHandler) Mutate(
6264
obj *unstructured.Unstructured,
6365
vars map[string]apiextensionsv1.JSON,
6466
holderRef runtimehooksv1.HolderReference,
65-
clusterKey ctrlclient.ObjectKey,
66-
_ mutation.ClusterGetter,
67+
_ ctrlclient.ObjectKey,
68+
clusterGetter mutation.ClusterGetter,
6769
) error {
6870
log := ctrl.LoggerFrom(ctx).WithValues(
6971
"holderRef", holderRef,
@@ -82,14 +84,24 @@ func (h *globalMirrorPatchHandler) Mutate(
8284
v1alpha1.ImageRegistriesVariableName,
8385
)
8486

87+
_, registryAddonErr := variables.Get[v1alpha1.RegistryAddon](
88+
vars,
89+
v1alpha1.ClusterConfigVariableName,
90+
[]string{"addons", v1alpha1.RegistryAddonVariableName}...)
91+
8592
switch {
86-
case variables.IsNotFoundError(imageRegistriesErr) && variables.IsNotFoundError(globalMirrorErr):
87-
log.V(5).Info("Image Registry Credentials and Global Registry Mirror variable not defined")
93+
case variables.IsNotFoundError(imageRegistriesErr) &&
94+
variables.IsNotFoundError(globalMirrorErr) &&
95+
variables.IsNotFoundError(registryAddonErr):
96+
log.V(5).
97+
Info("Image Registry Credentials and Global Registry Mirror and Registry Addon variable not defined")
8898
return nil
8999
case imageRegistriesErr != nil && !variables.IsNotFoundError(imageRegistriesErr):
90100
return imageRegistriesErr
91101
case globalMirrorErr != nil && !variables.IsNotFoundError(globalMirrorErr):
92102
return globalMirrorErr
103+
case registryAddonErr != nil && !variables.IsNotFoundError(registryAddonErr):
104+
return registryAddonErr
93105
}
94106

95107
var registriesWithOptionalCA []containerdConfig //nolint:prealloc // We don't know the size of the slice yet.
@@ -121,6 +133,22 @@ func (h *globalMirrorPatchHandler) Mutate(
121133
registryWithOptionalCredentials,
122134
)
123135
}
136+
if registryAddonErr == nil {
137+
cluster, err := clusterGetter(ctx)
138+
if err != nil {
139+
log.Error(
140+
err,
141+
"failed to get cluster from Global Mirror mutation handler",
142+
)
143+
return err
144+
}
145+
146+
registryConfig, err := containerdConfigFromRegistryAddon(cluster)
147+
if err != nil {
148+
return err
149+
}
150+
registriesWithOptionalCA = append(registriesWithOptionalCA, registryConfig)
151+
}
124152

125153
needConfiguration := needContainerdConfiguration(
126154
registriesWithOptionalCA,
@@ -234,6 +262,21 @@ func containerdConfigFromImageRegistry(
234262
return configWithOptionalCACert, nil
235263
}
236264

265+
func containerdConfigFromRegistryAddon(cluster *clusterv1.Cluster) (containerdConfig, error) {
266+
serviceIP, err := registryutils.ServiceIPForCluster(cluster)
267+
if err != nil {
268+
return containerdConfig{}, fmt.Errorf("error getting service IP for the registry addon: %w", err)
269+
}
270+
271+
config := containerdConfig{
272+
// FIXME: Generate a self-signed CA.
273+
URL: fmt.Sprintf("http://%s", serviceIP),
274+
Mirror: true,
275+
}
276+
277+
return config, nil
278+
}
279+
237280
func generateFiles(
238281
registriesWithOptionalCA []containerdConfig,
239282
) ([]bootstrapv1.File, error) {

‎pkg/handlers/generic/mutation/mirrors/inject_test.go

Lines changed: 159 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,20 @@
44
package mirrors
55

66
import (
7+
"fmt"
78
"testing"
89

910
. "github.com/onsi/ginkgo/v2"
1011
"github.com/onsi/gomega"
1112
"github.com/stretchr/testify/assert"
13+
"github.com/stretchr/testify/require"
1214
corev1 "k8s.io/api/core/v1"
1315
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
16+
"k8s.io/apimachinery/pkg/runtime"
17+
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
1418
"k8s.io/apiserver/pkg/storage/names"
19+
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
20+
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
1521
runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
1622

1723
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1"
@@ -32,6 +38,10 @@ func TestMirrorsPatch(t *testing.T) {
3238
}
3339

3440
var _ = Describe("Generate Global mirror patches", func() {
41+
clientScheme := runtime.NewScheme()
42+
utilruntime.Must(clientgoscheme.AddToScheme(clientScheme))
43+
utilruntime.Must(clusterv1.AddToScheme(clientScheme))
44+
3545
patchGenerator := func() mutation.GeneratePatches {
3646
// Always initialize the testEnv variable in the closure.
3747
// This will allow ginkgo to initialize testEnv variable during test execution time.
@@ -40,7 +50,7 @@ var _ = Describe("Generate Global mirror patches", func() {
4050
// that are written by the tests.
4151
// Test cases writes credentials secret that the mutator handler reads.
4252
// Using direct client will enable reading it immediately.
43-
client, err := testEnv.GetK8sClient()
53+
client, err := testEnv.GetK8sClientWithScheme(clientScheme)
4454
gomega.Expect(err).To(gomega.BeNil())
4555
return mutation.NewMetaGeneratePatchesHandler("", client, NewPatch(client)).(mutation.GeneratePatches)
4656
}
@@ -330,11 +340,69 @@ var _ = Describe("Generate Global mirror patches", func() {
330340
},
331341
},
332342
},
343+
{
344+
Name: "files added in KubeadmControlPlaneTemplate for registry addon",
345+
Vars: []runtimehooksv1.Variable{
346+
capitest.VariableWithValue(
347+
v1alpha1.ClusterConfigVariableName,
348+
v1alpha1.RegistryAddon{},
349+
[]string{"addons", v1alpha1.RegistryAddonVariableName}...,
350+
),
351+
},
352+
RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""),
353+
ExpectedPatchMatchers: []capitest.JSONPatchMatcher{
354+
{
355+
Operation: "add",
356+
Path: "/spec/template/spec/kubeadmConfigSpec/files",
357+
ValueMatcher: gomega.HaveExactElements(
358+
gomega.HaveKeyWithValue(
359+
"path", "/etc/containerd/certs.d/_default/hosts.toml",
360+
),
361+
gomega.HaveKeyWithValue(
362+
"path", "/etc/caren/containerd/patches/registry-config.toml",
363+
),
364+
),
365+
},
366+
},
367+
},
368+
{
369+
Name: "files added in KubeadmConfigTemplate for registry addon",
370+
Vars: []runtimehooksv1.Variable{
371+
capitest.VariableWithValue(
372+
v1alpha1.ClusterConfigVariableName,
373+
v1alpha1.RegistryAddon{},
374+
[]string{"addons", v1alpha1.RegistryAddonVariableName}...,
375+
),
376+
capitest.VariableWithValue(
377+
"builtin",
378+
map[string]any{
379+
"machineDeployment": map[string]any{
380+
"class": names.SimpleNameGenerator.GenerateName("worker-"),
381+
},
382+
},
383+
),
384+
},
385+
RequestItem: request.NewKubeadmConfigTemplateRequestItem(""),
386+
ExpectedPatchMatchers: []capitest.JSONPatchMatcher{
387+
{
388+
Operation: "add",
389+
Path: "/spec/template/spec/files",
390+
ValueMatcher: gomega.HaveExactElements(
391+
gomega.HaveKeyWithValue(
392+
"path", "/etc/containerd/certs.d/_default/hosts.toml",
393+
),
394+
gomega.HaveKeyWithValue(
395+
"path", "/etc/caren/containerd/patches/registry-config.toml",
396+
),
397+
),
398+
},
399+
},
400+
},
333401
}
334402

335403
// Create credentials secret before each test
336404
BeforeEach(func(ctx SpecContext) {
337-
client, err := helpers.TestEnv.GetK8sClient()
405+
client, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme)
338406
gomega.Expect(err).To(gomega.BeNil())
339407
gomega.Expect(client.Create(
340408
ctx,
@@ -344,11 +412,28 @@ var _ = Describe("Generate Global mirror patches", func() {
344412
ctx,
345413
newMirrorSecretWithoutCA(validMirrorNoCASecretName, request.Namespace),
346414
)).To(gomega.BeNil())
415+
416+
gomega.Expect(client.Create(
417+
ctx,
418+
&clusterv1.Cluster{
419+
ObjectMeta: metav1.ObjectMeta{
420+
Name: request.ClusterName,
421+
Namespace: request.Namespace,
422+
},
423+
Spec: clusterv1.ClusterSpec{
424+
ClusterNetwork: &clusterv1.ClusterNetwork{
425+
Services: &clusterv1.NetworkRanges{
426+
CIDRBlocks: []string{"192.168.0.1/16"},
427+
},
428+
},
429+
},
430+
},
431+
)).To(gomega.BeNil())
347432
})
348433

349434
// Delete credentials secret after each test
350435
AfterEach(func(ctx SpecContext) {
351-
client, err := helpers.TestEnv.GetK8sClient()
436+
client, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme)
352437
gomega.Expect(err).To(gomega.BeNil())
353438
gomega.Expect(client.Delete(
354439
ctx,
@@ -358,6 +443,16 @@ var _ = Describe("Generate Global mirror patches", func() {
358443
ctx,
359444
newMirrorSecretWithoutCA(validMirrorNoCASecretName, request.Namespace),
360445
)).To(gomega.BeNil())
446+
447+
gomega.Expect(client.Delete(
448+
ctx,
449+
&clusterv1.Cluster{
450+
ObjectMeta: metav1.ObjectMeta{
451+
Name: request.ClusterName,
452+
Namespace: request.Namespace,
453+
},
454+
},
455+
)).To(gomega.BeNil())
361456
})
362457

363458
// create test node for each case
@@ -406,6 +501,67 @@ func newMirrorSecretWithoutCA(name, namespace string) *corev1.Secret {
406501
}
407502
}
408503

504+
func Test_containerdConfigFromRegistryAddon(t *testing.T) {
505+
t.Parallel()
506+
tests := []struct {
507+
name string
508+
cluster *clusterv1.Cluster
509+
want containerdConfig
510+
wantErr error
511+
}{
512+
{
513+
name: "valid input",
514+
cluster: &clusterv1.Cluster{
515+
ObjectMeta: metav1.ObjectMeta{
516+
Name: request.ClusterName,
517+
Namespace: request.Namespace,
518+
},
519+
Spec: clusterv1.ClusterSpec{
520+
ClusterNetwork: &clusterv1.ClusterNetwork{
521+
Services: &clusterv1.NetworkRanges{
522+
CIDRBlocks: []string{"192.168.0.1/16"},
523+
},
524+
},
525+
},
526+
},
527+
want: containerdConfig{
528+
URL: "http://192.168.0.20",
529+
Mirror: true,
530+
},
531+
},
532+
{
533+
name: "missing Services CIDR",
534+
cluster: &clusterv1.Cluster{
535+
ObjectMeta: metav1.ObjectMeta{
536+
Name: request.ClusterName,
537+
Namespace: request.Namespace,
538+
},
539+
Spec: clusterv1.ClusterSpec{
540+
ClusterNetwork: &clusterv1.ClusterNetwork{},
541+
},
542+
},
543+
wantErr: fmt.Errorf(
544+
"error getting service IP for the registry addon: " +
545+
"error getting a service IP for a cluster: " +
546+
"unexpected empty service Subnets",
547+
),
548+
},
549+
}
550+
for idx := range tests {
551+
tt := tests[idx]
552+
t.Run(tt.name, func(t *testing.T) {
553+
t.Parallel()
554+
got, err := containerdConfigFromRegistryAddon(tt.cluster)
555+
if tt.wantErr != nil {
556+
require.EqualError(t, err, tt.wantErr.Error())
557+
} else {
558+
require.NoError(t, err)
559+
}
560+
assert.Equal(t, tt.want, got)
561+
})
562+
}
563+
}
564+
409565
func Test_needContainerdConfiguration(t *testing.T) {
410566
t.Parallel()
411567
tests := []struct {

0 commit comments

Comments
 (0)