Skip to content

Commit 4dd5b1f

Browse files
supershalsupershal
committed
feat: global image registry mirror variable
1 parent 176d403 commit 4dd5b1f

19 files changed

+389
-307
lines changed

‎api/v1alpha1/clusterconfig_types.go

Lines changed: 21 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,9 @@ type GenericClusterConfig struct {
9090
// +optional
9191
ImageRegistries ImageRegistries `json:"imageRegistries,omitempty"`
9292

93+
// +optional
94+
GlobalImageRegistryMirror *GlobalImageRegistryMirror `json:"globalImageRegistryMirror,omitempty"`
95+
9396
// +optional
9497
Addons *Addons `json:"addons,omitempty"`
9598
}
@@ -108,7 +111,8 @@ func (s GenericClusterConfig) VariableSchema() clusterv1.VariableSchema { //noli
108111
"",
109112
).VariableSchema().
110113
OpenAPIV3Schema,
111-
"imageRegistries": ImageRegistries{}.VariableSchema().OpenAPIV3Schema,
114+
"imageRegistries": ImageRegistries{}.VariableSchema().OpenAPIV3Schema,
115+
"globalImageRegistryMirror": GlobalImageRegistryMirror{}.VariableSchema().OpenAPIV3Schema,
112116
},
113117
},
114118
}
@@ -240,7 +244,7 @@ func (ExtraAPIServerCertSANs) VariableSchema() clusterv1.VariableSchema {
240244

241245
type ImageCredentials struct {
242246
// The Secret containing the registry credentials and CA certificate
243-
// The Secret should have keys 'username', 'password' and 'caCert'
247+
// The Secret should have keys 'username', 'password' and 'ca.crt'
244248
// This credentials Secret is not required for some registries, e.g. ECR.
245249
// +optional
246250
SecretRef *corev1.ObjectReference `json:"secretRef,omitempty"`
@@ -253,7 +257,7 @@ func (ImageCredentials) VariableSchema() clusterv1.VariableSchema {
253257
Properties: map[string]clusterv1.JSONSchemaProps{
254258
"secretRef": {
255259
Description: "The Secret containing the registry credentials. " +
256-
"The Secret should have keys 'username', 'password'. " +
260+
"The Secret should have keys 'username', 'password' and 'ca.crt' " +
257261
"This credentials Secret is not required for some registries, e.g. ECR.",
258262
Type: "object",
259263
Properties: map[string]clusterv1.JSONSchemaProps{
@@ -274,37 +278,28 @@ func (ImageCredentials) VariableSchema() clusterv1.VariableSchema {
274278
}
275279
}
276280

277-
type RegistryMirror struct {
278-
// The secret containing CA certificate for the registry mirror.
279-
// The secret should have 'ca.crt' key
281+
// GlobalImageRegistryMirror sets default mirror configuration for all the image registries.
282+
type GlobalImageRegistryMirror struct {
283+
// Registry URL.
284+
URL string `json:"url"`
285+
286+
// Credentials and CA certificate for the image registry mirror
280287
// +optional
281-
SecretRef *corev1.ObjectReference `json:"secretRef,omitempty"`
288+
Credentials *ImageCredentials `json:"credentials,omitempty"`
282289
}
283290

284-
func (RegistryMirror) VariableSchema() clusterv1.VariableSchema {
291+
func (GlobalImageRegistryMirror) VariableSchema() clusterv1.VariableSchema {
285292
return clusterv1.VariableSchema{
286293
OpenAPIV3Schema: clusterv1.JSONSchemaProps{
287294
Type: "object",
288295
Properties: map[string]clusterv1.JSONSchemaProps{
289-
"secretRef": {
290-
Description: "The Secret containing the registry CA certificate. " +
291-
"The Secret should have keys 'ca.crt'. " +
292-
"This credentials Secret is not required for public registries.",
293-
Type: "object",
294-
Properties: map[string]clusterv1.JSONSchemaProps{
295-
"name": {
296-
Description: "The name of the Secret containing the registry CA certificate.",
297-
Type: "string",
298-
},
299-
"namespace": {
300-
Description: "The namespace of the Secret containing the registry CA certificate. " +
301-
"Defaults to the namespace of the KubeadmControlPlaneTemplate and KubeadmConfigTemplate" +
302-
" that reference this variable.",
303-
Type: "string",
304-
},
305-
},
296+
"url": {
297+
Description: "Registry mirror URL.",
298+
Type: "string",
306299
},
300+
"credentials": ImageCredentials{}.VariableSchema().OpenAPIV3Schema,
307301
},
302+
Required: []string{"url"},
308303
},
309304
}
310305
}
@@ -313,13 +308,9 @@ type ImageRegistry struct {
313308
// Registry URL.
314309
URL string `json:"url"`
315310

316-
// Credentials for the image registry
311+
// Credentials and CA certificate for the image registry
317312
// +optional
318313
Credentials *ImageCredentials `json:"credentials,omitempty"`
319-
320-
// Use this registry as a mirror
321-
// +optional
322-
Mirror *RegistryMirror `json:"mirror,omitempty"`
323314
}
324315

325316
func (ImageRegistry) VariableSchema() clusterv1.VariableSchema {
@@ -332,7 +323,6 @@ func (ImageRegistry) VariableSchema() clusterv1.VariableSchema {
332323
Type: "string",
333324
},
334325
"credentials": ImageCredentials{}.VariableSchema().OpenAPIV3Schema,
335-
"mirror": RegistryMirror{}.VariableSchema().OpenAPIV3Schema,
336326
},
337327
Required: []string{"url"},
338328
},

‎api/v1alpha1/zz_generated.deepcopy.go

Lines changed: 25 additions & 25 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎docs/content/customization/generic/global-mirror.md

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
+++
2+
title = "Global Image Registry Mirror"
3+
+++
4+
5+
Add containerd image registry mirror configuration to all Nodes in the cluster.
6+
7+
When the `globalImageRegistryMirror` variable is set, `files` with configurations for
8+
[Containerd default mirror](https://github.com/containerd/containerd/blob/main/docs/hosts.md#setup-default-mirror-for-all-registries) will be added.
9+
10+
This customization will be available when the
11+
[provider-specific cluster configuration patch]({{< ref "..">}}) is included in the `ClusterClass`.
12+
13+
## Example
14+
15+
To provide image registry mirror with CA certificate, specify the following configuration:
16+
17+
If your registry mirror requires self signed CA certifate, create a Kubernetes Secret with keys for `ca.crt`:
18+
19+
```shell
20+
kubectl create secret generic my-mirror-ca-cert-secret \
21+
--from-file=ca.crt=registry-ca.crt
22+
```
23+
24+
```yaml
25+
apiVersion: cluster.x-k8s.io/v1beta1
26+
kind: Cluster
27+
metadata:
28+
name: <NAME>
29+
spec:
30+
topology:
31+
variables:
32+
- name: clusterConfig
33+
value:
34+
globalImageRegistryMirror:
35+
url: https://my-mirror.io
36+
credentials:
37+
secretRef:
38+
name: my-mirror-ca-cert-secret
39+
```
40+
41+
Applying this configuration will result in following new files on the
42+
`KubeadmControlPlaneTemplate` and `KubeadmConfigTemplate`
43+
44+
- `/etc/containerd/certs.d/_default/hosts.toml`
45+
- `/etc/certs/mirror.pem`
46+
47+
To use a public hosted image registry (ex. ECR) as mirror, specify the following configuration:
48+
49+
```yaml
50+
apiVersion: cluster.x-k8s.io/v1beta1
51+
kind: Cluster
52+
metadata:
53+
name: <NAME>
54+
spec:
55+
topology:
56+
variables:
57+
- name: clusterConfig
58+
value:
59+
globalImageRegistryMirror:
60+
url: https://123456789.dkr.ecr.us-east-1.amazonaws.com
61+
```
62+
63+
Applying this configuration will result in following new files on the
64+
`KubeadmControlPlaneTemplate` and `KubeadmConfigTemplate`
65+
66+
- `/etc/containerd/certs.d/_default/hosts.toml`

‎pkg/handlers/aws/mutation/metapatch_handler_test.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@ import (
3838
imageregistrycredentialstests "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/imageregistries/credentials/tests"
3939
"github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/kubernetesimagerepository"
4040
kubernetesimagerepositorytests "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/kubernetesimagerepository/tests"
41+
"github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/mirrors"
42+
globalimageregistrymirrortests "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/mirrors/tests"
4143
"github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/workerconfig"
4244
)
4345

@@ -156,12 +158,12 @@ func TestGeneratePatches(t *testing.T) {
156158
imageregistries.VariableName,
157159
)
158160

159-
imageregistrycredentialstests.TestGenerateMirrorPatches(
161+
globalimageregistrymirrortests.TestGeneratePatches(
160162
t,
161163
metaPatchGeneratorFunc(mgr),
162164
mgr.GetClient(),
163165
clusterconfig.MetaVariableName,
164-
imageregistries.VariableName,
166+
mirrors.GlobalMirrorVariableName,
165167
)
166168

167169
amitests.TestControlPlaneGeneratePatches(

‎pkg/handlers/docker/mutation/metapatch_handler_test.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@ import (
2828
imageregistrycredentialstests "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/imageregistries/credentials/tests"
2929
"github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/kubernetesimagerepository"
3030
kubernetesimagerepositorytests "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/kubernetesimagerepository/tests"
31+
"github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/mirrors"
32+
globalimageregistrymirrortests "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/mirrors/tests"
3133
"github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/workerconfig"
3234
)
3335

@@ -113,11 +115,11 @@ func TestGeneratePatches(t *testing.T) {
113115
imageregistries.VariableName,
114116
)
115117

116-
imageregistrycredentialstests.TestGenerateMirrorPatches(
118+
globalimageregistrymirrortests.TestGeneratePatches(
117119
t,
118120
metaPatchGeneratorFunc(mgr),
119121
mgr.GetClient(),
120122
clusterconfig.MetaVariableName,
121-
imageregistries.VariableName,
123+
mirrors.GlobalMirrorVariableName,
122124
)
123125
}

‎pkg/handlers/generic/mutation/handlers.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
"github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/httpproxy"
1515
"github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/imageregistries/credentials"
1616
"github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/kubernetesimagerepository"
17+
"github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/mirrors"
1718
)
1819

1920
// MetaMutators returns all generic patch handlers.
@@ -25,6 +26,7 @@ func MetaMutators(mgr manager.Manager) []mutation.MetaMutator {
2526
httpproxy.NewPatch(mgr.GetClient()),
2627
kubernetesimagerepository.NewPatch(),
2728
credentials.NewPatch(mgr.GetClient()),
29+
mirrors.NewPatch(mgr.GetClient()),
2830
calico.NewPatch(),
2931
}
3032
}

‎pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -52,10 +52,7 @@ func (c providerConfig) isCredentialsEmpty() bool {
5252
c.Password == ""
5353
}
5454

55-
func templateFilesForImageCredentialProviderConfigs(
56-
config providerConfig,
57-
mirror *mirrorConfig,
58-
) ([]cabpkv1.File, error) {
55+
func templateFilesForImageCredentialProviderConfigs(config providerConfig) ([]cabpkv1.File, error) {
5956
var files []cabpkv1.File
6057

6158
kubeletCredentialProviderConfigFile, err := templateKubeletCredentialProviderConfig()
@@ -68,7 +65,6 @@ func templateFilesForImageCredentialProviderConfigs(
6865

6966
kubeletDynamicCredentialProviderConfigFile, err := templateDynamicCredentialProviderConfig(
7067
config,
71-
mirror,
7268
)
7369
if err != nil {
7470
return nil, err
@@ -104,7 +100,6 @@ func templateKubeletCredentialProviderConfig() (*cabpkv1.File, error) {
104100

105101
func templateDynamicCredentialProviderConfig(
106102
config providerConfig,
107-
mirror *mirrorConfig,
108103
) (*cabpkv1.File, error) {
109104
registryURL, err := url.ParseRequestURI(config.URL)
110105
if err != nil {
@@ -142,13 +137,11 @@ func templateDynamicCredentialProviderConfig(
142137
ProviderBinary string
143138
ProviderArgs []string
144139
ProviderAPIVersion string
145-
Mirror *mirrorConfig
146140
}{
147141
RegistryHost: registryHostWithPath,
148142
ProviderBinary: providerBinary,
149143
ProviderArgs: providerArgs,
150144
ProviderAPIVersion: providerAPIVersion,
151-
Mirror: mirror,
152145
}
153146

154147
return fileFromTemplate(t, templateInput, kubeletDynamicCredentialProviderConfigOnRemote)

0 commit comments

Comments
 (0)