Skip to content

Commit 176d403

Browse files
supershalsupershal
committed
test: fix race condition between registry credentials and mirror tests
1 parent d54a86e commit 176d403

File tree

5 files changed

+314
-193
lines changed

5 files changed

+314
-193
lines changed

‎common/pkg/testutils/capitest/request/items.go

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,8 @@ import (
2121

2222
const (
2323
ClusterName = "test-cluster"
24-
KubeadmConfigTemplateRequestObjectName = "test-kubeadmconfigtemplate"
25-
KubeadmControlPlaneTemplateRequestObjectName = "test-kubeadmcontrolplanetemplate"
24+
kubeadmConfigTemplateRequestObjectName = "test-kubeadmconfigtemplate"
25+
kubeadmControlPlaneTemplateRequestObjectName = "test-kubeadmcontrolplanetemplate"
2626
Namespace = corev1.NamespaceDefault
2727
)
2828

@@ -45,15 +45,24 @@ func NewRequestItem(
4545
}
4646
}
4747

48-
func NewKubeadmConfigTemplateRequestItem(uid types.UID) runtimehooksv1.GeneratePatchesRequestItem {
48+
func NewKubeadmConfigTemplateRequestItem(
49+
uid types.UID,
50+
) runtimehooksv1.GeneratePatchesRequestItem {
51+
return NewKubeadmConfigTemplateRequest(uid, kubeadmConfigTemplateRequestObjectName)
52+
}
53+
54+
func NewKubeadmConfigTemplateRequest(
55+
uid types.UID,
56+
name string,
57+
) runtimehooksv1.GeneratePatchesRequestItem {
4958
return NewRequestItem(
5059
&bootstrapv1.KubeadmConfigTemplate{
5160
TypeMeta: metav1.TypeMeta{
5261
APIVersion: bootstrapv1.GroupVersion.String(),
5362
Kind: "KubeadmConfigTemplate",
5463
},
5564
ObjectMeta: metav1.ObjectMeta{
56-
Name: KubeadmConfigTemplateRequestObjectName,
65+
Name: name,
5766
Namespace: Namespace,
5867
},
5968
Spec: bootstrapv1.KubeadmConfigTemplateSpec{
@@ -75,8 +84,9 @@ func NewKubeadmConfigTemplateRequestItem(uid types.UID) runtimehooksv1.GenerateP
7584
)
7685
}
7786

78-
func NewKubeadmControlPlaneTemplateRequestItem(
87+
func NewKubeadmControlPlaneTemplateRequest(
7988
uid types.UID,
89+
name string,
8090
) runtimehooksv1.GeneratePatchesRequestItem {
8191
return NewRequestItem(
8292
&controlplanev1.KubeadmControlPlaneTemplate{
@@ -85,7 +95,7 @@ func NewKubeadmControlPlaneTemplateRequestItem(
8595
Kind: "KubeadmControlPlaneTemplate",
8696
},
8797
ObjectMeta: metav1.ObjectMeta{
88-
Name: KubeadmControlPlaneTemplateRequestObjectName,
98+
Name: name,
8999
Namespace: Namespace,
90100
},
91101
Spec: controlplanev1.KubeadmControlPlaneTemplateSpec{
@@ -113,6 +123,12 @@ func NewKubeadmControlPlaneTemplateRequestItem(
113123
)
114124
}
115125

126+
func NewKubeadmControlPlaneTemplateRequestItem(
127+
uid types.UID,
128+
) runtimehooksv1.GeneratePatchesRequestItem {
129+
return NewKubeadmControlPlaneTemplateRequest(uid, kubeadmControlPlaneTemplateRequestObjectName)
130+
}
131+
116132
func NewAWSClusterTemplateRequestItem(
117133
uid types.UID,
118134
existingSpec ...capav1.AWSClusterTemplateSpec,

‎pkg/handlers/aws/mutation/metapatch_handler_test.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,14 @@ func TestGeneratePatches(t *testing.T) {
156156
imageregistries.VariableName,
157157
)
158158

159+
imageregistrycredentialstests.TestGenerateMirrorPatches(
160+
t,
161+
metaPatchGeneratorFunc(mgr),
162+
mgr.GetClient(),
163+
clusterconfig.MetaVariableName,
164+
imageregistries.VariableName,
165+
)
166+
159167
amitests.TestControlPlaneGeneratePatches(
160168
t,
161169
metaPatchGeneratorFunc(mgr),

‎pkg/handlers/docker/mutation/metapatch_handler_test.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,4 +112,12 @@ func TestGeneratePatches(t *testing.T) {
112112
clusterconfig.MetaVariableName,
113113
imageregistries.VariableName,
114114
)
115+
116+
imageregistrycredentialstests.TestGenerateMirrorPatches(
117+
t,
118+
metaPatchGeneratorFunc(mgr),
119+
mgr.GetClient(),
120+
clusterconfig.MetaVariableName,
121+
imageregistries.VariableName,
122+
)
115123
}

‎pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_mirror_patches.go

Lines changed: 262 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,262 @@
1+
// Copyright 2023 D2iQ, Inc. All rights reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
package tests
5+
6+
import (
7+
"context"
8+
"fmt"
9+
"testing"
10+
11+
"github.com/onsi/gomega"
12+
"github.com/stretchr/testify/require"
13+
corev1 "k8s.io/api/core/v1"
14+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
15+
"k8s.io/apiserver/pkg/storage/names"
16+
runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
17+
"sigs.k8s.io/controller-runtime/pkg/client"
18+
19+
"github.com/d2iq-labs/capi-runtime-extensions/api/v1alpha1"
20+
"github.com/d2iq-labs/capi-runtime-extensions/common/pkg/capi/clustertopology/handlers/mutation"
21+
"github.com/d2iq-labs/capi-runtime-extensions/common/pkg/testutils/capitest"
22+
"github.com/d2iq-labs/capi-runtime-extensions/common/pkg/testutils/capitest/request"
23+
)
24+
25+
const (
26+
validMirrorCredentialsSecretName = "my-mirror-registry-credentials"
27+
validMirrorCASecretName = "myregistry-mirror-cacert"
28+
//nolint:gosec // Does not contain hard coded credentials.
29+
cpRegistryAsMirrorCreds = "kubeadmControlPlaneRegistryAsMirrorCreds"
30+
//nolint:gosec // Does not contain hard coded credentials.
31+
workerRegistryAsMirrorCreds = "kubeadmConfigTemplateRegistryAsMirrorCreds"
32+
registryStaticCredentialsSecretSuffix = "registry-config"
33+
)
34+
35+
func TestGenerateMirrorPatches(
36+
t *testing.T,
37+
generatorFunc func() mutation.GeneratePatches,
38+
fakeClient client.Client,
39+
variableName string,
40+
variablePath ...string,
41+
) {
42+
t.Helper()
43+
44+
require.NoError(
45+
t,
46+
fakeClient.Create(
47+
context.Background(),
48+
newRegistryCredentialsSecret(validMirrorCredentialsSecretName, request.Namespace),
49+
),
50+
)
51+
52+
require.NoError(
53+
t,
54+
fakeClient.Create(
55+
context.Background(),
56+
newMirrorSecret(validMirrorCASecretName, request.Namespace),
57+
),
58+
)
59+
60+
// Server side apply does not work with the fake client, hack around it by pre-creating empty Secrets
61+
// https://github.com/kubernetes-sigs/controller-runtime/issues/2341
62+
require.NoError(
63+
t,
64+
fakeClient.Create(
65+
context.Background(),
66+
newEmptySecret(
67+
fmt.Sprintf(
68+
"%s-%s",
69+
cpRegistryAsMirrorCreds,
70+
registryStaticCredentialsSecretSuffix,
71+
),
72+
request.Namespace,
73+
),
74+
),
75+
)
76+
77+
require.NoError(
78+
t,
79+
fakeClient.Create(
80+
context.Background(),
81+
newEmptySecret(
82+
fmt.Sprintf(
83+
"%s-%s",
84+
workerRegistryAsMirrorCreds,
85+
registryStaticCredentialsSecretSuffix,
86+
),
87+
request.Namespace,
88+
),
89+
),
90+
)
91+
92+
capitest.ValidateGeneratePatches(
93+
t,
94+
generatorFunc,
95+
capitest.PatchTestDef{
96+
Name: "files added in KubeadmControlPlaneTemplate for registry with mirror without CA Certificate",
97+
Vars: []runtimehooksv1.Variable{
98+
capitest.VariableWithValue(
99+
variableName,
100+
v1alpha1.ImageRegistries{
101+
v1alpha1.ImageRegistry{
102+
URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com",
103+
Mirror: &v1alpha1.RegistryMirror{},
104+
},
105+
},
106+
variablePath...,
107+
),
108+
},
109+
RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""),
110+
ExpectedPatchMatchers: []capitest.JSONPatchMatcher{
111+
{
112+
Operation: "add",
113+
Path: "/spec/template/spec/kubeadmConfigSpec/files",
114+
ValueMatcher: gomega.ContainElements(
115+
gomega.HaveKeyWithValue(
116+
"path", "/etc/containerd/certs.d/_default/hosts.toml",
117+
),
118+
),
119+
},
120+
},
121+
},
122+
capitest.PatchTestDef{
123+
Name: "files added in KubeadmControlPlaneTemplate for registry with mirror with CA Certificate",
124+
Vars: []runtimehooksv1.Variable{
125+
capitest.VariableWithValue(
126+
variableName,
127+
v1alpha1.ImageRegistries{
128+
v1alpha1.ImageRegistry{
129+
URL: "https://mirror-registry.com",
130+
Credentials: &v1alpha1.ImageCredentials{
131+
SecretRef: &corev1.ObjectReference{
132+
Name: validSecretName,
133+
},
134+
},
135+
Mirror: &v1alpha1.RegistryMirror{
136+
SecretRef: &corev1.ObjectReference{
137+
Name: validMirrorCASecretName,
138+
},
139+
},
140+
},
141+
},
142+
variablePath...,
143+
),
144+
},
145+
RequestItem: request.NewKubeadmControlPlaneTemplateRequest("", cpRegistryAsMirrorCreds),
146+
ExpectedPatchMatchers: []capitest.JSONPatchMatcher{
147+
{
148+
Operation: "add",
149+
Path: "/spec/template/spec/kubeadmConfigSpec/files",
150+
ValueMatcher: gomega.ContainElements(
151+
gomega.HaveKeyWithValue(
152+
"path", "/etc/containerd/certs.d/_default/hosts.toml",
153+
),
154+
gomega.HaveKeyWithValue(
155+
"path", "/etc/certs/mirror.pem",
156+
),
157+
),
158+
},
159+
},
160+
},
161+
capitest.PatchTestDef{
162+
Name: "files added in KubeadmConfigTemplate for registry mirror wihthout CA certificate",
163+
Vars: []runtimehooksv1.Variable{
164+
capitest.VariableWithValue(
165+
variableName,
166+
v1alpha1.ImageRegistries{
167+
v1alpha1.ImageRegistry{
168+
URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com",
169+
Mirror: &v1alpha1.RegistryMirror{},
170+
},
171+
},
172+
variablePath...,
173+
),
174+
capitest.VariableWithValue(
175+
"builtin",
176+
map[string]any{
177+
"machineDeployment": map[string]any{
178+
"class": names.SimpleNameGenerator.GenerateName("worker-"),
179+
},
180+
},
181+
),
182+
},
183+
RequestItem: request.NewKubeadmConfigTemplateRequestItem(""),
184+
ExpectedPatchMatchers: []capitest.JSONPatchMatcher{
185+
{
186+
Operation: "add",
187+
Path: "/spec/template/spec/files",
188+
ValueMatcher: gomega.ContainElements(
189+
gomega.HaveKeyWithValue(
190+
"path", "/etc/containerd/certs.d/_default/hosts.toml",
191+
),
192+
),
193+
},
194+
},
195+
},
196+
capitest.PatchTestDef{
197+
Name: "files added in KubeadmConfigTemplate for registry mirror with secret for CA certificate",
198+
Vars: []runtimehooksv1.Variable{
199+
capitest.VariableWithValue(
200+
variableName,
201+
v1alpha1.ImageRegistries{
202+
v1alpha1.ImageRegistry{
203+
URL: "https://mirror-registry.io",
204+
Credentials: &v1alpha1.ImageCredentials{
205+
SecretRef: &corev1.ObjectReference{
206+
Name: validSecretName,
207+
},
208+
},
209+
Mirror: &v1alpha1.RegistryMirror{
210+
SecretRef: &corev1.ObjectReference{
211+
Name: validMirrorCASecretName,
212+
},
213+
},
214+
},
215+
},
216+
variablePath...,
217+
),
218+
capitest.VariableWithValue(
219+
"builtin",
220+
map[string]any{
221+
"machineDeployment": map[string]any{
222+
"class": names.SimpleNameGenerator.GenerateName("worker-"),
223+
},
224+
},
225+
),
226+
},
227+
RequestItem: request.NewKubeadmConfigTemplateRequest("", workerRegistryAsMirrorCreds),
228+
ExpectedPatchMatchers: []capitest.JSONPatchMatcher{
229+
{
230+
Operation: "add",
231+
Path: "/spec/template/spec/files",
232+
ValueMatcher: gomega.ContainElements(
233+
gomega.HaveKeyWithValue(
234+
"path", "/etc/containerd/certs.d/_default/hosts.toml",
235+
),
236+
gomega.HaveKeyWithValue(
237+
"path", "/etc/certs/mirror.pem",
238+
),
239+
),
240+
},
241+
},
242+
},
243+
)
244+
}
245+
246+
func newMirrorSecret(name, namespace string) *corev1.Secret {
247+
secretData := map[string][]byte{
248+
"ca.crt": []byte("myCACert"),
249+
}
250+
return &corev1.Secret{
251+
TypeMeta: metav1.TypeMeta{
252+
APIVersion: "v1",
253+
Kind: "Secret",
254+
},
255+
ObjectMeta: metav1.ObjectMeta{
256+
Name: name,
257+
Namespace: namespace,
258+
},
259+
Data: secretData,
260+
Type: corev1.SecretTypeOpaque,
261+
}
262+
}

0 commit comments

Comments
 (0)