diff --git a/api/v1alpha1/clusterconfig_types.go b/api/v1alpha1/clusterconfig_types.go index c04ba04e1..0eed0f94c 100644 --- a/api/v1alpha1/clusterconfig_types.go +++ b/api/v1alpha1/clusterconfig_types.go @@ -8,6 +8,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/openapi/patterns" @@ -89,6 +90,9 @@ type GenericClusterConfig struct { // +optional ImageRegistries ImageRegistries `json:"imageRegistries,omitempty"` + // +optional + GlobalImageRegistryMirror *GlobalImageRegistryMirror `json:"globalImageRegistryMirror,omitempty"` + // +optional Addons *Addons `json:"addons,omitempty"` } @@ -107,7 +111,8 @@ func (s GenericClusterConfig) VariableSchema() clusterv1.VariableSchema { //noli "", ).VariableSchema(). OpenAPIV3Schema, - "imageRegistries": ImageRegistries{}.VariableSchema().OpenAPIV3Schema, + "imageRegistries": ImageRegistries{}.VariableSchema().OpenAPIV3Schema, + "globalImageRegistryMirror": GlobalImageRegistryMirror{}.VariableSchema().OpenAPIV3Schema, }, }, } @@ -237,52 +242,81 @@ func (ExtraAPIServerCertSANs) VariableSchema() clusterv1.VariableSchema { } } -type ImageRegistries struct { +type RegistryCredentials struct { + // The Secret containing the registry credentials and optional CA certificate + // using the keys `username`, `password` and `ca.crt`. + // This credentials Secret is not required for some registries, e.g. ECR. // +optional - ImageRegistryCredentials ImageRegistryCredentials `json:"credentials,omitempty"` + SecretRef *corev1.ObjectReference `json:"secretRef,omitempty"` } -func (ImageRegistries) VariableSchema() clusterv1.VariableSchema { +func (RegistryCredentials) VariableSchema() clusterv1.VariableSchema { return clusterv1.VariableSchema{ OpenAPIV3Schema: clusterv1.JSONSchemaProps{ - Description: "Configuration for image registries.", - Type: "object", + Type: "object", Properties: map[string]clusterv1.JSONSchemaProps{ - "credentials": ImageRegistryCredentials{}.VariableSchema().OpenAPIV3Schema, + "secretRef": { + Description: "A reference to the Secret containing the registry credentials. " + + "The Secret should have keys 'username', 'password' and optional 'ca.crt'. " + + "This credentials Secret is not required for some registries, e.g. ECR.", + Type: "object", + Properties: map[string]clusterv1.JSONSchemaProps{ + "name": { + Description: "The name of the Secret containing the registry credentials.", + Type: "string", + }, + "namespace": { + Description: "The namespace of the Secret containing the registry credentials. " + + "Defaults to the namespace of the Cluster. " + + "that reference this variable.", + Type: "string", + }, + }, + Required: []string{"name"}, + }, }, }, } } -type ImageRegistryCredentials []ImageRegistryCredentialsResource +// GlobalImageRegistryMirror sets default mirror configuration for all the image registries. +type GlobalImageRegistryMirror struct { + // Registry URL. + URL string `json:"url"` -func (ImageRegistryCredentials) VariableSchema() clusterv1.VariableSchema { - resourceSchema := ImageRegistryCredentialsResource{}.VariableSchema().OpenAPIV3Schema + // Credentials and CA certificate for the image registry mirror + // +optional + Credentials *RegistryCredentials `json:"credentials,omitempty"` +} +func (GlobalImageRegistryMirror) VariableSchema() clusterv1.VariableSchema { return clusterv1.VariableSchema{ OpenAPIV3Schema: clusterv1.JSONSchemaProps{ - Description: "Image registry credentials to set up on all Nodes in the cluster. " + - "Enabling this will configure the Kubelets with " + - "https://kubernetes.io/docs/tasks/administer-cluster/kubelet-credential-provider/.", - Type: "array", - Items: &resourceSchema, + Type: "object", + Properties: map[string]clusterv1.JSONSchemaProps{ + "url": { + Description: "Registry mirror URL.", + Type: "string", + Format: "uri", + Pattern: "^https?://", + }, + "credentials": RegistryCredentials{}.VariableSchema().OpenAPIV3Schema, + }, + Required: []string{"url"}, }, } } -// ImageRegistryCredentialsResource required for providing credentials for an image registry URL. -type ImageRegistryCredentialsResource struct { +type ImageRegistry struct { // Registry URL. URL string `json:"url"` - // The Secret containing the registry credentials. - // The Secret should have keys 'username' and 'password'. - // This credentials Secret is not required for some registries, e.g. ECR. + // Credentials and CA certificate for the image registry // +optional - Secret *corev1.ObjectReference `json:"secretRef,omitempty"` + Credentials *RegistryCredentials `json:"credentials,omitempty"` } -func (ImageRegistryCredentialsResource) VariableSchema() clusterv1.VariableSchema { +func (ImageRegistry) VariableSchema() clusterv1.VariableSchema { return clusterv1.VariableSchema{ OpenAPIV3Schema: clusterv1.JSONSchemaProps{ Type: "object", @@ -290,31 +324,29 @@ func (ImageRegistryCredentialsResource) VariableSchema() clusterv1.VariableSchem "url": { Description: "Registry URL.", Type: "string", + Format: "uri", + Pattern: "^https?://", }, - "secretRef": { - Description: "The Secret containing the registry credentials. " + - "The Secret should have keys 'username' and 'password'. " + - "This credentials Secret is not required for some registries, e.g. ECR.", - Type: "object", - Properties: map[string]clusterv1.JSONSchemaProps{ - "name": { - Description: "The name of the Secret containing the registry credentials.", - Type: "string", - }, - "namespace": { - Description: "The namespace of the Secret containing the registry credentials. " + - "Defaults to the namespace of the KubeadmControlPlaneTemplate and KubeadmConfigTemplate" + - " that reference this variable.", - Type: "string", - }, - }, - }, + "credentials": RegistryCredentials{}.VariableSchema().OpenAPIV3Schema, }, Required: []string{"url"}, }, } } +type ImageRegistries []ImageRegistry + +func (ImageRegistries) VariableSchema() clusterv1.VariableSchema { + return clusterv1.VariableSchema{ + OpenAPIV3Schema: clusterv1.JSONSchemaProps{ + Description: "Configuration for image registries.", + Type: "array", + Items: ptr.To(ImageRegistry{}.VariableSchema().OpenAPIV3Schema), + MaxItems: ptr.To[int64](1), + }, + } +} + func init() { SchemeBuilder.Register(&ClusterConfig{}) } diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 27897b8dd..76d67d4bc 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -435,7 +435,18 @@ func (in *GenericClusterConfig) DeepCopyInto(out *GenericClusterConfig) { *out = make(ExtraAPIServerCertSANs, len(*in)) copy(*out, *in) } - in.ImageRegistries.DeepCopyInto(&out.ImageRegistries) + if in.ImageRegistries != nil { + in, out := &in.ImageRegistries, &out.ImageRegistries + *out = make(ImageRegistries, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.GlobalImageRegistryMirror != nil { + in, out := &in.GlobalImageRegistryMirror, &out.GlobalImageRegistryMirror + *out = new(GlobalImageRegistryMirror) + (*in).DeepCopyInto(*out) + } if in.Addons != nil { in, out := &in.Addons, &out.Addons *out = new(Addons) @@ -468,6 +479,26 @@ func (in *GenericNodeConfig) DeepCopy() *GenericNodeConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GlobalImageRegistryMirror) DeepCopyInto(out *GlobalImageRegistryMirror) { + *out = *in + if in.Credentials != nil { + in, out := &in.Credentials, &out.Credentials + *out = new(RegistryCredentials) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GlobalImageRegistryMirror. +func (in *GlobalImageRegistryMirror) DeepCopy() *GlobalImageRegistryMirror { + if in == nil { + return nil + } + out := new(GlobalImageRegistryMirror) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HTTPProxy) DeepCopyInto(out *HTTPProxy) { *out = *in @@ -504,64 +535,42 @@ func (in *Image) DeepCopy() *Image { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ImageRegistries) DeepCopyInto(out *ImageRegistries) { - *out = *in - if in.ImageRegistryCredentials != nil { - in, out := &in.ImageRegistryCredentials, &out.ImageRegistryCredentials - *out = make(ImageRegistryCredentials, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImageRegistries. -func (in *ImageRegistries) DeepCopy() *ImageRegistries { - if in == nil { - return nil - } - out := new(ImageRegistries) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in ImageRegistryCredentials) DeepCopyInto(out *ImageRegistryCredentials) { +func (in ImageRegistries) DeepCopyInto(out *ImageRegistries) { { in := &in - *out = make(ImageRegistryCredentials, len(*in)) + *out = make(ImageRegistries, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImageRegistryCredentials. -func (in ImageRegistryCredentials) DeepCopy() ImageRegistryCredentials { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImageRegistries. +func (in ImageRegistries) DeepCopy() ImageRegistries { if in == nil { return nil } - out := new(ImageRegistryCredentials) + out := new(ImageRegistries) in.DeepCopyInto(out) return *out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ImageRegistryCredentialsResource) DeepCopyInto(out *ImageRegistryCredentialsResource) { +func (in *ImageRegistry) DeepCopyInto(out *ImageRegistry) { *out = *in - if in.Secret != nil { - in, out := &in.Secret, &out.Secret - *out = new(v1.ObjectReference) - **out = **in + if in.Credentials != nil { + in, out := &in.Credentials, &out.Credentials + *out = new(RegistryCredentials) + (*in).DeepCopyInto(*out) } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImageRegistryCredentialsResource. -func (in *ImageRegistryCredentialsResource) DeepCopy() *ImageRegistryCredentialsResource { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImageRegistry. +func (in *ImageRegistry) DeepCopy() *ImageRegistry { if in == nil { return nil } - out := new(ImageRegistryCredentialsResource) + out := new(ImageRegistry) in.DeepCopyInto(out) return out } @@ -661,6 +670,26 @@ func (in *ObjectMeta) DeepCopy() *ObjectMeta { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RegistryCredentials) DeepCopyInto(out *RegistryCredentials) { + *out = *in + if in.SecretRef != nil { + in, out := &in.SecretRef, &out.SecretRef + *out = new(v1.ObjectReference) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RegistryCredentials. +func (in *RegistryCredentials) DeepCopy() *RegistryCredentials { + if in == nil { + return nil + } + out := new(RegistryCredentials) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SecurityGroup) DeepCopyInto(out *SecurityGroup) { *out = *in diff --git a/common/pkg/testutils/capitest/request/items.go b/common/pkg/testutils/capitest/request/items.go index a0313cb70..5444bd4b8 100644 --- a/common/pkg/testutils/capitest/request/items.go +++ b/common/pkg/testutils/capitest/request/items.go @@ -21,8 +21,8 @@ import ( const ( ClusterName = "test-cluster" - KubeadmConfigTemplateRequestObjectName = "test-kubeadmconfigtemplate" - KubeadmControlPlaneTemplateRequestObjectName = "test-kubeadmcontrolplanetemplate" + kubeadmConfigTemplateRequestObjectName = "test-kubeadmconfigtemplate" + kubeadmControlPlaneTemplateRequestObjectName = "test-kubeadmcontrolplanetemplate" Namespace = corev1.NamespaceDefault ) @@ -45,7 +45,16 @@ func NewRequestItem( } } -func NewKubeadmConfigTemplateRequestItem(uid types.UID) runtimehooksv1.GeneratePatchesRequestItem { +func NewKubeadmConfigTemplateRequestItem( + uid types.UID, +) runtimehooksv1.GeneratePatchesRequestItem { + return NewKubeadmConfigTemplateRequest(uid, kubeadmConfigTemplateRequestObjectName) +} + +func NewKubeadmConfigTemplateRequest( + uid types.UID, + name string, +) runtimehooksv1.GeneratePatchesRequestItem { return NewRequestItem( &bootstrapv1.KubeadmConfigTemplate{ TypeMeta: metav1.TypeMeta{ @@ -53,7 +62,7 @@ func NewKubeadmConfigTemplateRequestItem(uid types.UID) runtimehooksv1.GenerateP Kind: "KubeadmConfigTemplate", }, ObjectMeta: metav1.ObjectMeta{ - Name: KubeadmConfigTemplateRequestObjectName, + Name: name, Namespace: Namespace, }, Spec: bootstrapv1.KubeadmConfigTemplateSpec{ @@ -75,8 +84,9 @@ func NewKubeadmConfigTemplateRequestItem(uid types.UID) runtimehooksv1.GenerateP ) } -func NewKubeadmControlPlaneTemplateRequestItem( +func NewKubeadmControlPlaneTemplateRequest( uid types.UID, + name string, ) runtimehooksv1.GeneratePatchesRequestItem { return NewRequestItem( &controlplanev1.KubeadmControlPlaneTemplate{ @@ -85,7 +95,7 @@ func NewKubeadmControlPlaneTemplateRequestItem( Kind: "KubeadmControlPlaneTemplate", }, ObjectMeta: metav1.ObjectMeta{ - Name: KubeadmControlPlaneTemplateRequestObjectName, + Name: name, Namespace: Namespace, }, Spec: controlplanev1.KubeadmControlPlaneTemplateSpec{ @@ -113,6 +123,12 @@ func NewKubeadmControlPlaneTemplateRequestItem( ) } +func NewKubeadmControlPlaneTemplateRequestItem( + uid types.UID, +) runtimehooksv1.GeneratePatchesRequestItem { + return NewKubeadmControlPlaneTemplateRequest(uid, kubeadmControlPlaneTemplateRequestObjectName) +} + func NewAWSClusterTemplateRequestItem( uid types.UID, existingSpec ...capav1.AWSClusterTemplateSpec, diff --git a/docs/content/customization/generic/global-mirror.md b/docs/content/customization/generic/global-mirror.md new file mode 100644 index 000000000..a653b53e2 --- /dev/null +++ b/docs/content/customization/generic/global-mirror.md @@ -0,0 +1,67 @@ ++++ +title = "Global Image Registry Mirror" ++++ + +Add containerd image registry mirror configuration to all Nodes in the cluster. + +When the `globalImageRegistryMirror` variable is set, `files` with configurations for +[Containerd default mirror](https://github.com/containerd/containerd/blob/main/docs/hosts.md). + +This customization will be available when the +[provider-specific cluster configuration patch]({{< ref "..">}}) is included in the `ClusterClass`. + +## Example + +To provide an image registry mirror with a CA certificate, specify the following configuration: + +If the registry mirror requires a private or self-signed CA certificate, +create a Kubernetes Secret with the `ca.crt` key populated with the CA certificate in PEM format: + +```shell +kubectl create secret generic my-mirror-ca-cert \ + --from-file=ca.crt=registry-ca.crt +``` + +```yaml +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: +spec: + topology: + variables: + - name: clusterConfig + value: + globalImageRegistryMirror: + url: https://example.com + credentials: + secretRef: + name: my-mirror-ca-cert +``` + +Applying this configuration will result in following new files on the +`KubeadmControlPlaneTemplate` and `KubeadmConfigTemplate` resources: + +- `/etc/containerd/certs.d/_default/hosts.toml` +- `/etc/certs/mirror.pem` + +To use a public hosted image registry (e.g. ECR) as a registry mirror, specify the following configuration: + +```yaml +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: +spec: + topology: + variables: + - name: clusterConfig + value: + globalImageRegistryMirror: + url: https://123456789.dkr.ecr.us-east-1.amazonaws.com +``` + +Applying this configuration will result in following new files on the +`KubeadmControlPlaneTemplate` and `KubeadmConfigTemplate` resources: + +- `/etc/containerd/certs.d/_default/hosts.toml` diff --git a/docs/content/customization/generic/image-registries.md b/docs/content/customization/generic/image-registries.md index ed7cecd70..2c1e8b336 100644 --- a/docs/content/customization/generic/image-registries.md +++ b/docs/content/customization/generic/image-registries.md @@ -33,8 +33,8 @@ spec: - name: clusterConfig value: imageRegistries: - credentials: - - url: https://my-registry.io + - url: https://my-registry.io + credentials: secretRef: name: my-registry-credentials ``` diff --git a/pkg/handlers/aws/mutation/metapatch_handler_test.go b/pkg/handlers/aws/mutation/metapatch_handler_test.go index 87a288229..2e8e63b5a 100644 --- a/pkg/handlers/aws/mutation/metapatch_handler_test.go +++ b/pkg/handlers/aws/mutation/metapatch_handler_test.go @@ -35,10 +35,11 @@ import ( "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/httpproxy" httpproxytests "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/httpproxy/tests" "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/imageregistries" - imageregistrycredentials "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/imageregistries/credentials" imageregistrycredentialstests "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/imageregistries/credentials/tests" "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/kubernetesimagerepository" kubernetesimagerepositorytests "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/kubernetesimagerepository/tests" + "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/mirrors" + globalimageregistrymirrortests "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/mirrors/tests" "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/workerconfig" ) @@ -155,7 +156,14 @@ func TestGeneratePatches(t *testing.T) { mgr.GetClient(), clusterconfig.MetaVariableName, imageregistries.VariableName, - imageregistrycredentials.VariableName, + ) + + globalimageregistrymirrortests.TestGeneratePatches( + t, + metaPatchGeneratorFunc(mgr), + mgr.GetClient(), + clusterconfig.MetaVariableName, + mirrors.GlobalMirrorVariableName, ) amitests.TestControlPlaneGeneratePatches( diff --git a/pkg/handlers/docker/mutation/metapatch_handler_test.go b/pkg/handlers/docker/mutation/metapatch_handler_test.go index a74a7cc94..6fd017258 100644 --- a/pkg/handlers/docker/mutation/metapatch_handler_test.go +++ b/pkg/handlers/docker/mutation/metapatch_handler_test.go @@ -25,10 +25,11 @@ import ( "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/httpproxy" httpproxytests "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/httpproxy/tests" "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/imageregistries" - imageregistrycredentials "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/imageregistries/credentials" imageregistrycredentialstests "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/imageregistries/credentials/tests" "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/kubernetesimagerepository" kubernetesimagerepositorytests "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/kubernetesimagerepository/tests" + "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/mirrors" + globalimageregistrymirrortests "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/mirrors/tests" "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/workerconfig" ) @@ -112,6 +113,13 @@ func TestGeneratePatches(t *testing.T) { mgr.GetClient(), clusterconfig.MetaVariableName, imageregistries.VariableName, - imageregistrycredentials.VariableName, + ) + + globalimageregistrymirrortests.TestGeneratePatches( + t, + metaPatchGeneratorFunc(mgr), + mgr.GetClient(), + clusterconfig.MetaVariableName, + mirrors.GlobalMirrorVariableName, ) } diff --git a/pkg/handlers/generic/mutation/handlers.go b/pkg/handlers/generic/mutation/handlers.go index aa2160737..7ac6f66f6 100644 --- a/pkg/handlers/generic/mutation/handlers.go +++ b/pkg/handlers/generic/mutation/handlers.go @@ -14,6 +14,7 @@ import ( "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/httpproxy" "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/imageregistries/credentials" "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/kubernetesimagerepository" + "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/mirrors" ) // MetaMutators returns all generic patch handlers. @@ -25,6 +26,7 @@ func MetaMutators(mgr manager.Manager) []mutation.MetaMutator { httpproxy.NewPatch(mgr.GetClient()), kubernetesimagerepository.NewPatch(), credentials.NewPatch(mgr.GetClient()), + mirrors.NewPatch(mgr.GetClient()), calico.NewPatch(), } } diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go index e8ce8dd87..09dc598ec 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files_test.go @@ -192,7 +192,7 @@ credentialProviders: { name: "error for a registry with no credentials", credentials: providerConfig{ - URL: "https://myregistry.com", + URL: "https://registry.example.com", }, wantErr: ErrCredentialsNotFound, }, diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go b/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go index a1ac10c1e..b7a8a9f1e 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go @@ -26,11 +26,6 @@ import ( "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/imageregistries" ) -const ( - // VariableName is the external patch variable name. - VariableName = "credentials" -) - type imageRegistriesPatchHandler struct { client ctrlclient.Client @@ -45,7 +40,6 @@ func NewPatch( cl, clusterconfig.MetaVariableName, imageregistries.VariableName, - VariableName, ) } @@ -75,7 +69,7 @@ func (h *imageRegistriesPatchHandler) Mutate( "holderRef", holderRef, ) - imageRegistryCredentials, found, err := variables.Get[v1alpha1.ImageRegistryCredentials]( + imageRegistries, found, err := variables.Get[v1alpha1.ImageRegistries]( vars, h.variableName, h.variableFieldPath..., @@ -88,127 +82,129 @@ func (h *imageRegistriesPatchHandler) Mutate( return nil } - // TODO: Add support for multiple registries. - if len(imageRegistryCredentials) > 1 { - return fmt.Errorf("multiple Image Registry Credentials are not supported at this time") - } - - credentials := imageRegistryCredentials[0] - - log = log.WithValues( - "variableName", - h.variableName, - "variableFieldPath", - h.variableFieldPath, - "variableValue", - credentials, - ) + // TODO: Support for multiple registries is constrained with variable schema of ImageRegistries. + // currently only one registry is supported. Implement support for multiple registries in + // DynamicCredentialProviderConfig + for _, imageRegistry := range imageRegistries { + log = log.WithValues( + "variableName", + h.variableName, + "variableFieldPath", + h.variableFieldPath, + "variableValue", + imageRegistry, + ) - if err := patches.MutateIfApplicable( - obj, vars, &holderRef, selectors.ControlPlane(), log, - func(obj *controlplanev1.KubeadmControlPlaneTemplate) error { - registryWithOptionalCredentials, generateErr := registryWithOptionalCredentialsFromImageRegistryCredentials( - ctx, h.client, credentials, obj, - ) - if generateErr != nil { - return generateErr - } - files, commands, generateErr := generateFilesAndCommands(registryWithOptionalCredentials, obj.GetName()) - if generateErr != nil { - return generateErr - } - - log.WithValues( - "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), - "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), - ).Info("adding files to control plane kubeadm config spec") - obj.Spec.Template.Spec.KubeadmConfigSpec.Files = append( - obj.Spec.Template.Spec.KubeadmConfigSpec.Files, - files..., - ) - - log.WithValues( - "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), - "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), - ).Info("adding PreKubeadmCommands to control plane kubeadm config spec") - obj.Spec.Template.Spec.KubeadmConfigSpec.PreKubeadmCommands = append( - obj.Spec.Template.Spec.KubeadmConfigSpec.PreKubeadmCommands, - commands..., - ) - - generateErr = createSecretIfNeeded(ctx, h.client, registryWithOptionalCredentials, obj, clusterKey) - if generateErr != nil { - return generateErr - } - - initConfiguration := obj.Spec.Template.Spec.KubeadmConfigSpec.InitConfiguration - if initConfiguration == nil { - initConfiguration = &bootstrapv1.InitConfiguration{} - } - obj.Spec.Template.Spec.KubeadmConfigSpec.InitConfiguration = initConfiguration - if initConfiguration.NodeRegistration.KubeletExtraArgs == nil { - initConfiguration.NodeRegistration.KubeletExtraArgs = map[string]string{} - } - addImageCredentialProviderArgs(initConfiguration.NodeRegistration.KubeletExtraArgs) - - joinConfiguration := obj.Spec.Template.Spec.KubeadmConfigSpec.JoinConfiguration - if joinConfiguration == nil { - joinConfiguration = &bootstrapv1.JoinConfiguration{} - } - obj.Spec.Template.Spec.KubeadmConfigSpec.JoinConfiguration = joinConfiguration - if joinConfiguration.NodeRegistration.KubeletExtraArgs == nil { - joinConfiguration.NodeRegistration.KubeletExtraArgs = map[string]string{} - } - addImageCredentialProviderArgs(joinConfiguration.NodeRegistration.KubeletExtraArgs) - return nil - }); err != nil { - return err - } + if err := patches.MutateIfApplicable( + obj, vars, &holderRef, selectors.ControlPlane(), log, + func(obj *controlplanev1.KubeadmControlPlaneTemplate) error { + registryWithOptionalCredentials, generateErr := registryWithOptionalCredentialsFromImageRegistryCredentials( + ctx, h.client, imageRegistry, obj, + ) + if generateErr != nil { + return generateErr + } + files, commands, generateErr := generateFilesAndCommands( + registryWithOptionalCredentials, + obj.GetName()) + if generateErr != nil { + return generateErr + } + + log.WithValues( + "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), + "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), + ).Info("adding files to control plane kubeadm config spec") + obj.Spec.Template.Spec.KubeadmConfigSpec.Files = append( + obj.Spec.Template.Spec.KubeadmConfigSpec.Files, + files..., + ) + + log.WithValues( + "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), + "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), + ).Info("adding PreKubeadmCommands to control plane kubeadm config spec") + obj.Spec.Template.Spec.KubeadmConfigSpec.PreKubeadmCommands = append( + obj.Spec.Template.Spec.KubeadmConfigSpec.PreKubeadmCommands, + commands..., + ) + + generateErr = createSecretIfNeeded(ctx, h.client, registryWithOptionalCredentials, obj, clusterKey) + if generateErr != nil { + return generateErr + } + + initConfiguration := obj.Spec.Template.Spec.KubeadmConfigSpec.InitConfiguration + if initConfiguration == nil { + initConfiguration = &bootstrapv1.InitConfiguration{} + } + obj.Spec.Template.Spec.KubeadmConfigSpec.InitConfiguration = initConfiguration + if initConfiguration.NodeRegistration.KubeletExtraArgs == nil { + initConfiguration.NodeRegistration.KubeletExtraArgs = map[string]string{} + } + addImageCredentialProviderArgs(initConfiguration.NodeRegistration.KubeletExtraArgs) + + joinConfiguration := obj.Spec.Template.Spec.KubeadmConfigSpec.JoinConfiguration + if joinConfiguration == nil { + joinConfiguration = &bootstrapv1.JoinConfiguration{} + } + obj.Spec.Template.Spec.KubeadmConfigSpec.JoinConfiguration = joinConfiguration + if joinConfiguration.NodeRegistration.KubeletExtraArgs == nil { + joinConfiguration.NodeRegistration.KubeletExtraArgs = map[string]string{} + } + addImageCredentialProviderArgs(joinConfiguration.NodeRegistration.KubeletExtraArgs) + return nil + }); err != nil { + return err + } - if err := patches.MutateIfApplicable( - obj, vars, &holderRef, selectors.WorkersKubeadmConfigTemplateSelector(), log, - func(obj *bootstrapv1.KubeadmConfigTemplate) error { - registryWithOptionalCredentials, generateErr := registryWithOptionalCredentialsFromImageRegistryCredentials( - ctx, h.client, credentials, obj, - ) - if generateErr != nil { - return generateErr - } - files, commands, generateErr := generateFilesAndCommands(registryWithOptionalCredentials, obj.GetName()) - if generateErr != nil { - return generateErr - } - - log.WithValues( - "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), - "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), - ).Info("adding files to worker node kubeadm config template") - obj.Spec.Template.Spec.Files = append(obj.Spec.Template.Spec.Files, files...) - - log.WithValues( - "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), - "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), - ).Info("adding PreKubeadmCommands to worker node kubeadm config template") - obj.Spec.Template.Spec.PreKubeadmCommands = append(obj.Spec.Template.Spec.PreKubeadmCommands, commands...) - - generateErr = createSecretIfNeeded(ctx, h.client, registryWithOptionalCredentials, obj, clusterKey) - if generateErr != nil { - return generateErr - } - - joinConfiguration := obj.Spec.Template.Spec.JoinConfiguration - if joinConfiguration == nil { - joinConfiguration = &bootstrapv1.JoinConfiguration{} - } - obj.Spec.Template.Spec.JoinConfiguration = joinConfiguration - if joinConfiguration.NodeRegistration.KubeletExtraArgs == nil { - joinConfiguration.NodeRegistration.KubeletExtraArgs = map[string]string{} - } - addImageCredentialProviderArgs(joinConfiguration.NodeRegistration.KubeletExtraArgs) - - return nil - }); err != nil { - return err + if err := patches.MutateIfApplicable( + obj, vars, &holderRef, selectors.WorkersKubeadmConfigTemplateSelector(), log, + func(obj *bootstrapv1.KubeadmConfigTemplate) error { + registryWithOptionalCredentials, generateErr := registryWithOptionalCredentialsFromImageRegistryCredentials( + ctx, h.client, imageRegistry, obj, + ) + if generateErr != nil { + return generateErr + } + files, commands, generateErr := generateFilesAndCommands( + registryWithOptionalCredentials, + obj.GetName()) + if generateErr != nil { + return generateErr + } + + log.WithValues( + "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), + "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), + ).Info("adding files to worker node kubeadm config template") + obj.Spec.Template.Spec.Files = append(obj.Spec.Template.Spec.Files, files...) + + log.WithValues( + "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), + "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), + ).Info("adding PreKubeadmCommands to worker node kubeadm config template") + obj.Spec.Template.Spec.PreKubeadmCommands = append(obj.Spec.Template.Spec.PreKubeadmCommands, commands...) + + generateErr = createSecretIfNeeded(ctx, h.client, registryWithOptionalCredentials, obj, clusterKey) + if generateErr != nil { + return generateErr + } + + joinConfiguration := obj.Spec.Template.Spec.JoinConfiguration + if joinConfiguration == nil { + joinConfiguration = &bootstrapv1.JoinConfiguration{} + } + obj.Spec.Template.Spec.JoinConfiguration = joinConfiguration + if joinConfiguration.NodeRegistration.KubeletExtraArgs == nil { + joinConfiguration.NodeRegistration.KubeletExtraArgs = map[string]string{} + } + addImageCredentialProviderArgs(joinConfiguration.NodeRegistration.KubeletExtraArgs) + + return nil + }); err != nil { + return err + } } return nil @@ -217,23 +213,23 @@ func (h *imageRegistriesPatchHandler) Mutate( func registryWithOptionalCredentialsFromImageRegistryCredentials( ctx context.Context, c ctrlclient.Client, - credentials v1alpha1.ImageRegistryCredentialsResource, + imageRegistry v1alpha1.ImageRegistry, obj ctrlclient.Object, ) (providerConfig, error) { registryWithOptionalCredentials := providerConfig{ - URL: credentials.URL, + URL: imageRegistry.URL, } secret, err := secretForImageRegistryCredentials( ctx, c, - credentials, + imageRegistry, obj.GetNamespace(), ) if err != nil { return providerConfig{}, fmt.Errorf( - "error getting secret %s/%s from Image Registry Credentials variable: %w", + "error getting secret %s/%s from Image Registry variable: %w", obj.GetNamespace(), - credentials.Secret, + imageRegistry.Credentials.SecretRef.Name, err, ) } @@ -253,7 +249,7 @@ func generateFilesAndCommands( files, commands, err := templateFilesAndCommandsForInstallKubeletCredentialProviders() if err != nil { return nil, nil, fmt.Errorf( - "error generating insall files and commands for Image Registry Credentials variable: %w", + "error generating install files and commands for Image Registry Credentials variable: %w", err, ) } @@ -270,7 +266,6 @@ func generateFilesAndCommands( files = append( files, generateCredentialsSecretFile(registryWithOptionalCredentials, objName)...) - return files, commands, err } @@ -307,20 +302,20 @@ func createSecretIfNeeded( func secretForImageRegistryCredentials( ctx context.Context, c ctrlclient.Reader, - credentials v1alpha1.ImageRegistryCredentialsResource, + registry v1alpha1.ImageRegistry, objectNamespace string, ) (*corev1.Secret, error) { - if credentials.Secret == nil { + if registry.Credentials == nil || registry.Credentials.SecretRef == nil { return nil, nil } namespace := objectNamespace - if credentials.Secret.Namespace != "" { - namespace = credentials.Secret.Namespace + if registry.Credentials.SecretRef.Namespace != "" { + namespace = registry.Credentials.SecretRef.Namespace } key := ctrlclient.ObjectKey{ - Name: credentials.Secret.Name, + Name: registry.Credentials.SecretRef.Name, Namespace: namespace, } secret := &corev1.Secret{} diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_patches.go b/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_patches.go index 5fc1704d8..13b51e7bf 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_patches.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/tests/generate_patches.go @@ -5,6 +5,7 @@ package tests import ( "context" + "fmt" "testing" "github.com/onsi/gomega" @@ -23,6 +24,11 @@ import ( const ( validSecretName = "myregistry-credentials" + //nolint:gosec // Does not contain hard coded credentials. + cpRegistryCreds = "kubeadmControlPlaneRegistryWithCredentials" + //nolint:gosec // Does not contain hard coded credentials. + workerRegistryCreds = "kubeadmConfigTemplateRegistryWithCreds" + registryStaticCredentialsSecretSuffix = "registry-config" ) func TestGeneratePatches( @@ -34,31 +40,33 @@ func TestGeneratePatches( ) { t.Helper() - // Server side apply does not work with the fake client, hack around it by pre-creating empty Secrets - // https://github.com/kubernetes-sigs/controller-runtime/issues/2341 require.NoError( t, fakeClient.Create( context.Background(), - newTestSecret(validSecretName, request.Namespace), + newRegistryCredentialsSecret(validSecretName, request.Namespace), ), ) + + // Server side apply does not work with the fake client, hack around it by pre-creating empty Secrets + // https://github.com/kubernetes-sigs/controller-runtime/issues/2341 require.NoError( t, fakeClient.Create( context.Background(), newEmptySecret( - request.KubeadmControlPlaneTemplateRequestObjectName+"-registry-config", + fmt.Sprintf("%s-%s", cpRegistryCreds, registryStaticCredentialsSecretSuffix), request.Namespace, ), ), ) + require.NoError( t, fakeClient.Create( context.Background(), newEmptySecret( - request.KubeadmConfigTemplateRequestObjectName+"-registry-config", + fmt.Sprintf("%s-%s", workerRegistryCreds, registryStaticCredentialsSecretSuffix), request.Namespace, ), ), @@ -75,8 +83,8 @@ func TestGeneratePatches( Vars: []runtimehooksv1.Variable{ capitest.VariableWithValue( variableName, - v1alpha1.ImageRegistryCredentials{ - v1alpha1.ImageRegistryCredentialsResource{ + v1alpha1.ImageRegistries{ + v1alpha1.ImageRegistry{ URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", }, }, @@ -130,18 +138,20 @@ func TestGeneratePatches( Vars: []runtimehooksv1.Variable{ capitest.VariableWithValue( variableName, - v1alpha1.ImageRegistryCredentials{ - v1alpha1.ImageRegistryCredentialsResource{ - URL: "https://my-registry.io", - Secret: &corev1.ObjectReference{ - Name: validSecretName, + v1alpha1.ImageRegistries{ + v1alpha1.ImageRegistry{ + URL: "https://registry.example.com", + Credentials: &v1alpha1.RegistryCredentials{ + SecretRef: &corev1.ObjectReference{ + Name: validSecretName, + }, }, }, }, variablePath..., ), }, - RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""), + RequestItem: request.NewKubeadmControlPlaneTemplateRequest("", cpRegistryCreds), ExpectedPatchMatchers: []capitest.JSONPatchMatcher{ { Operation: "add", @@ -191,8 +201,8 @@ func TestGeneratePatches( Vars: []runtimehooksv1.Variable{ capitest.VariableWithValue( variableName, - v1alpha1.ImageRegistryCredentials{ - v1alpha1.ImageRegistryCredentialsResource{ + v1alpha1.ImageRegistries{ + v1alpha1.ImageRegistry{ URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", }, }, @@ -246,11 +256,13 @@ func TestGeneratePatches( Vars: []runtimehooksv1.Variable{ capitest.VariableWithValue( variableName, - v1alpha1.ImageRegistryCredentials{ - v1alpha1.ImageRegistryCredentialsResource{ - URL: "https://my-registry.io", - Secret: &corev1.ObjectReference{ - Name: validSecretName, + v1alpha1.ImageRegistries{ + v1alpha1.ImageRegistry{ + URL: "https://registry.example.com", + Credentials: &v1alpha1.RegistryCredentials{ + SecretRef: &corev1.ObjectReference{ + Name: validSecretName, + }, }, }, }, @@ -265,7 +277,7 @@ func TestGeneratePatches( }, ), }, - RequestItem: request.NewKubeadmConfigTemplateRequestItem(""), + RequestItem: request.NewKubeadmConfigTemplateRequest("", workerRegistryCreds), ExpectedPatchMatchers: []capitest.JSONPatchMatcher{ { Operation: "add", @@ -307,9 +319,9 @@ func TestGeneratePatches( Vars: []runtimehooksv1.Variable{ capitest.VariableWithValue( variableName, - v1alpha1.ImageRegistryCredentials{ - v1alpha1.ImageRegistryCredentialsResource{ - URL: "https://my-registry.io", + v1alpha1.ImageRegistries{ + v1alpha1.ImageRegistry{ + URL: "https://registry.example.com", }, }, variablePath..., @@ -321,7 +333,7 @@ func TestGeneratePatches( ) } -func newTestSecret(name, namespace string) *corev1.Secret { +func newRegistryCredentialsSecret(name, namespace string) *corev1.Secret { secretData := map[string][]byte{ "username": []byte("myuser"), "password": []byte("mypassword"), diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/variables_test.go b/pkg/handlers/generic/mutation/imageregistries/credentials/variables_test.go index ef7bff7b5..2c416bcf7 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/variables_test.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/variables_test.go @@ -22,25 +22,23 @@ func TestVariableValidation(t *testing.T) { false, clusterconfig.NewVariable, capitest.VariableTestDef{ - Name: "without a Secret", + Name: "without a credentials secret", Vals: v1alpha1.GenericClusterConfig{ - ImageRegistries: v1alpha1.ImageRegistries{ - ImageRegistryCredentials: []v1alpha1.ImageRegistryCredentialsResource{ - { - URL: "http://a.b.c.example.com", - }, + ImageRegistries: []v1alpha1.ImageRegistry{ + { + URL: "http://a.b.c.example.com", }, }, }, }, capitest.VariableTestDef{ - Name: "with a Secret", + Name: "with a credentials secret", Vals: v1alpha1.GenericClusterConfig{ - ImageRegistries: v1alpha1.ImageRegistries{ - ImageRegistryCredentials: []v1alpha1.ImageRegistryCredentialsResource{ - { - URL: "http://a.b.c.example.com", - Secret: &corev1.ObjectReference{ + ImageRegistries: []v1alpha1.ImageRegistry{ + { + URL: "https://a.b.c.example.com/a/b/c", + Credentials: &v1alpha1.RegistryCredentials{ + SecretRef: &corev1.ObjectReference{ Name: "a.b.c.example.com-creds", }, }, @@ -48,5 +46,41 @@ func TestVariableValidation(t *testing.T) { }, }, }, + capitest.VariableTestDef{ + Name: "support for only single image registry", + Vals: v1alpha1.GenericClusterConfig{ + ImageRegistries: []v1alpha1.ImageRegistry{ + { + URL: "http://first-image-registry.example.com", + }, + { + URL: "http://second-image-registry.example.com", + }, + }, + }, + ExpectError: true, + }, + capitest.VariableTestDef{ + Name: "invalid registry URL", + Vals: v1alpha1.GenericClusterConfig{ + ImageRegistries: []v1alpha1.ImageRegistry{ + { + URL: "unsupportedformat://a.b.c.example.com", + }, + }, + }, + ExpectError: true, + }, + capitest.VariableTestDef{ + Name: "registry URL without format", + Vals: v1alpha1.GenericClusterConfig{ + ImageRegistries: []v1alpha1.ImageRegistry{ + { + URL: "a.b.c.example.com/a/b/c", + }, + }, + }, + ExpectError: true, + }, ) } diff --git a/pkg/handlers/generic/mutation/mirrors/constants.go b/pkg/handlers/generic/mutation/mirrors/constants.go new file mode 100644 index 000000000..13639af8c --- /dev/null +++ b/pkg/handlers/generic/mutation/mirrors/constants.go @@ -0,0 +1,8 @@ +// Copyright 2023 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package mirrors + +const ( + GlobalMirrorVariableName = "globalImageRegistryMirror" +) diff --git a/pkg/handlers/generic/mutation/mirrors/inject.go b/pkg/handlers/generic/mutation/mirrors/inject.go new file mode 100644 index 000000000..e9c42320f --- /dev/null +++ b/pkg/handlers/generic/mutation/mirrors/inject.go @@ -0,0 +1,169 @@ +// Copyright 2023 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package mirrors + +import ( + "context" + + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" + controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" + runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" + ctrl "sigs.k8s.io/controller-runtime" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/d2iq-labs/capi-runtime-extensions/api/v1alpha1" + "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/capi/clustertopology/patches" + "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/capi/clustertopology/patches/selectors" + "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/capi/clustertopology/variables" + "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/clusterconfig" +) + +type globalMirrorPatchHandler struct { + client ctrlclient.Client + + variableName string + variableFieldPath []string +} + +func NewPatch( + cl ctrlclient.Client, +) *globalMirrorPatchHandler { + return newGlobalMirrorPatchHandler( + cl, + clusterconfig.MetaVariableName, + GlobalMirrorVariableName, + ) +} + +func newGlobalMirrorPatchHandler( + cl ctrlclient.Client, + variableName string, + variableFieldPath ...string, +) *globalMirrorPatchHandler { + scheme := runtime.NewScheme() + _ = bootstrapv1.AddToScheme(scheme) + _ = controlplanev1.AddToScheme(scheme) + return &globalMirrorPatchHandler{ + client: cl, + variableName: variableName, + variableFieldPath: variableFieldPath, + } +} + +func (h *globalMirrorPatchHandler) Mutate( + ctx context.Context, + obj *unstructured.Unstructured, + vars map[string]apiextensionsv1.JSON, + holderRef runtimehooksv1.HolderReference, + clusterKey ctrlclient.ObjectKey, +) error { + log := ctrl.LoggerFrom(ctx).WithValues( + "holderRef", holderRef, + ) + + globalMirror, found, err := variables.Get[v1alpha1.GlobalImageRegistryMirror]( + vars, + h.variableName, + h.variableFieldPath..., + ) + if err != nil { + return err + } + if !found { + log.V(5).Info("Global registry mirror variable not defined") + return nil + } + + log = log.WithValues( + "variableName", + h.variableName, + "variableFieldPath", + h.variableFieldPath, + "variableValue", + globalMirror, + ) + + if err := patches.MutateIfApplicable( + obj, vars, &holderRef, selectors.ControlPlane(), log, + func(obj *controlplanev1.KubeadmControlPlaneTemplate) error { + mirrorConfig, err := mirrorConfigForGlobalMirror( + ctx, + h.client, + globalMirror, + obj, + ) + if err != nil { + return err + } + files, generateErr := generateFiles( + mirrorConfig, + globalMirror) + if generateErr != nil { + return generateErr + } + + log.WithValues( + "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), + "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), + ).Info("adding global registry mirror files to control plane kubeadm config spec") + obj.Spec.Template.Spec.KubeadmConfigSpec.Files = append( + obj.Spec.Template.Spec.KubeadmConfigSpec.Files, + files..., + ) + + return nil + }); err != nil { + return err + } + + if err := patches.MutateIfApplicable( + obj, vars, &holderRef, selectors.WorkersKubeadmConfigTemplateSelector(), log, + func(obj *bootstrapv1.KubeadmConfigTemplate) error { + mirrorConfig, err := mirrorConfigForGlobalMirror( + ctx, + h.client, + globalMirror, + obj, + ) + if err != nil { + return err + } + files, generateErr := generateFiles( + mirrorConfig, + globalMirror) + if generateErr != nil { + return generateErr + } + + log.WithValues( + "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), + "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), + ).Info("adding global registry mirror files to worker node kubeadm config template") + obj.Spec.Template.Spec.Files = append(obj.Spec.Template.Spec.Files, files...) + return nil + }); err != nil { + return err + } + + return nil +} + +func generateFiles( + mirrorConfig *mirrorConfig, + globalMirror v1alpha1.GlobalImageRegistryMirror, +) ([]bootstrapv1.File, error) { + // Generate default registry mirror file + files, err := generateGlobalRegistryMirrorFile(mirrorConfig) + if err != nil { + return nil, err + } + // generate CA certificate file for registry mirror + mirrorCAFile := generateMirrorCACertFile(mirrorConfig, globalMirror) + files = append(files, mirrorCAFile...) + + return files, err +} diff --git a/pkg/handlers/generic/mutation/mirrors/mirror.go b/pkg/handlers/generic/mutation/mirrors/mirror.go new file mode 100644 index 000000000..409d0fea9 --- /dev/null +++ b/pkg/handlers/generic/mutation/mirrors/mirror.go @@ -0,0 +1,148 @@ +// Copyright 2023 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package mirrors + +import ( + "bytes" + "context" + _ "embed" + "fmt" + "text/template" + + corev1 "k8s.io/api/core/v1" + cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/d2iq-labs/capi-runtime-extensions/api/v1alpha1" +) + +const ( + mirrorCACertPathOnRemote = "/etc/certs/mirror.pem" + defaultRegistryMirrorConfigPathOnRemote = "/etc/containerd/certs.d/_default/hosts.toml" + secretKeyForMirrorCACert = "ca.crt" +) + +//go:embed templates/hosts.toml.gotmpl +var defaultRegistryMirrorPatch []byte + +type mirrorConfig struct { + URL string + CACert string +} + +func mirrorConfigForGlobalMirror( + ctx context.Context, + c ctrlclient.Client, + globalMirror v1alpha1.GlobalImageRegistryMirror, + obj ctrlclient.Object, +) (*mirrorConfig, error) { + mirrorWithOptionalCACert := &mirrorConfig{ + URL: globalMirror.URL, + } + secret, err := secretForMirrorCACert( + ctx, + c, + globalMirror, + obj.GetNamespace(), + ) + if err != nil { + return &mirrorConfig{}, fmt.Errorf( + "error getting secret %s/%s from Image Registry variable: %w", + obj.GetNamespace(), + globalMirror.Credentials.SecretRef.Name, + err, + ) + } + + if secret != nil { + mirrorWithOptionalCACert.CACert = string(secret.Data[secretKeyForMirrorCACert]) + } + + return mirrorWithOptionalCACert, nil +} + +// secretForMirrorCACert returns the Secret for the given mirror's CA certificate. +// Returns nil if the secret field is empty. +func secretForMirrorCACert( + ctx context.Context, + c ctrlclient.Reader, + globalMirror v1alpha1.GlobalImageRegistryMirror, + objectNamespace string, +) (*corev1.Secret, error) { + if globalMirror.Credentials == nil || globalMirror.Credentials.SecretRef == nil { + return nil, nil + } + + namespace := objectNamespace + if globalMirror.Credentials.SecretRef.Namespace != "" { + namespace = globalMirror.Credentials.SecretRef.Namespace + } + + key := ctrlclient.ObjectKey{ + Name: globalMirror.Credentials.SecretRef.Name, + Namespace: namespace, + } + secret := &corev1.Secret{} + err := c.Get(ctx, key, secret) + return secret, err +} + +// Default Mirror for all registries. +// Containerd configuration for global mirror will be created at /etc/containerd/certs.d/_default/hosts.toml +// The upstream registry will be automatically used after all defined mirrors have been tried. +// reference: https://github.com/containerd/containerd/blob/main/docs/hosts.md#setup-default-mirror-for-all-registries +func generateGlobalRegistryMirrorFile(mirror *mirrorConfig) ([]cabpkv1.File, error) { + if mirror == nil { + return nil, nil + } + t, err := template.New("").Parse(string(defaultRegistryMirrorPatch)) + if err != nil { + return nil, fmt.Errorf("fail to parse go template for registry mirror: %w", err) + } + templateInput := struct { + URL string + CACertPath string + }{ + URL: mirror.URL, + } + // CA cert is optional for mirror registry. + // i.e. registry is using signed certificates. Insecure registry will not be allowed. + if mirror.CACert != "" { + templateInput.CACertPath = mirrorCACertPathOnRemote + } + + var b bytes.Buffer + err = t.Execute(&b, templateInput) + if err != nil { + return nil, fmt.Errorf("failed executing template for registry mirror: %w", err) + } + return []cabpkv1.File{ + { + Path: defaultRegistryMirrorConfigPathOnRemote, + Content: b.String(), + Permissions: "0600", + }, + }, nil +} + +func generateMirrorCACertFile( + config *mirrorConfig, + globalMirror v1alpha1.GlobalImageRegistryMirror, +) []cabpkv1.File { + if config == nil || config.CACert == "" { + return nil + } + return []cabpkv1.File{ + { + Path: mirrorCACertPathOnRemote, + Permissions: "0600", + ContentFrom: &cabpkv1.FileSource{ + Secret: cabpkv1.SecretFileSource{ + Name: globalMirror.Credentials.SecretRef.Name, + Key: secretKeyForMirrorCACert, + }, + }, + }, + } +} diff --git a/pkg/handlers/generic/mutation/mirrors/mirror_test.go b/pkg/handlers/generic/mutation/mirrors/mirror_test.go new file mode 100644 index 000000000..f9693579c --- /dev/null +++ b/pkg/handlers/generic/mutation/mirrors/mirror_test.go @@ -0,0 +1,132 @@ +// Copyright 2023 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package mirrors + +import ( + "testing" + + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" + + "github.com/d2iq-labs/capi-runtime-extensions/api/v1alpha1" +) + +func Test_generateDefaultRegistryMirrorFile(t *testing.T) { + t.Parallel() + tests := []struct { + name string + config *mirrorConfig + want []cabpkv1.File + wantErr error + }{ + { + name: "ECR image registry and no CA certificate", + config: &mirrorConfig{URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com"}, + want: []cabpkv1.File{ + { + Path: "/etc/containerd/certs.d/_default/hosts.toml", + Owner: "", + Permissions: "0600", + Encoding: "", + Append: false, + Content: `[host."https://123456789.dkr.ecr.us-east-1.amazonaws.com"] + capabilities = ["pull", "resolve"] +`, + }, + }, + wantErr: nil, + }, + { + name: "image registry with CA certificates", + config: &mirrorConfig{ + URL: "https://myregistry.com", + CACert: "mycacert", + }, + want: []cabpkv1.File{ + { + Path: "/etc/containerd/certs.d/_default/hosts.toml", + Owner: "", + Permissions: "0600", + Encoding: "", + Append: false, + Content: `[host."https://myregistry.com"] + capabilities = ["pull", "resolve"] + ca = "/etc/certs/mirror.pem" +`, + }, + }, + wantErr: nil, + }, + } + for idx := range tests { + tt := tests[idx] + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + file, err := generateGlobalRegistryMirrorFile(tt.config) + assert.ErrorIs(t, err, tt.wantErr) + assert.Equal(t, tt.want, file) + }) + } +} + +func Test_generateMirrorCACertFile(t *testing.T) { + t.Parallel() + tests := []struct { + name string + config *mirrorConfig + registry v1alpha1.GlobalImageRegistryMirror + want []cabpkv1.File + }{ + { + name: "Mirror registry with no CA certificate", + config: &mirrorConfig{ + URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", + }, + registry: v1alpha1.GlobalImageRegistryMirror{ + URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", + }, + want: nil, + }, + { + name: "Mirror registry with CA certificate", + config: &mirrorConfig{ + URL: "https://myregistry.com", + CACert: "mycacert", + }, + registry: v1alpha1.GlobalImageRegistryMirror{ + URL: "https://registry.example.com", + + Credentials: &v1alpha1.RegistryCredentials{ + SecretRef: &v1.ObjectReference{ + Name: "my-registry-credentials-secret", + }, + }, + }, + want: []cabpkv1.File{ + { + Path: "/etc/certs/mirror.pem", + Owner: "", + Permissions: "0600", + Encoding: "", + Append: false, + ContentFrom: &cabpkv1.FileSource{ + Secret: cabpkv1.SecretFileSource{ + Name: "my-registry-credentials-secret", + Key: "ca.crt", + }, + }, + }, + }, + }, + } + for idx := range tests { + tt := tests[idx] + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + file := generateMirrorCACertFile(tt.config, tt.registry) + assert.Equal(t, tt.want, file) + }) + } +} diff --git a/pkg/handlers/generic/mutation/mirrors/templates/hosts.toml.gotmpl b/pkg/handlers/generic/mutation/mirrors/templates/hosts.toml.gotmpl new file mode 100644 index 000000000..46237f6e3 --- /dev/null +++ b/pkg/handlers/generic/mutation/mirrors/templates/hosts.toml.gotmpl @@ -0,0 +1,5 @@ +[host."{{ .URL }}"] + capabilities = ["pull", "resolve"] + {{- if .CACertPath }} + ca = "{{ .CACertPath }}" + {{- end }} diff --git a/pkg/handlers/generic/mutation/mirrors/tests/generate_patches.go b/pkg/handlers/generic/mutation/mirrors/tests/generate_patches.go new file mode 100644 index 000000000..86c749d11 --- /dev/null +++ b/pkg/handlers/generic/mutation/mirrors/tests/generate_patches.go @@ -0,0 +1,199 @@ +// Copyright 2023 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package tests + +import ( + "context" + "testing" + + "github.com/onsi/gomega" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apiserver/pkg/storage/names" + runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/d2iq-labs/capi-runtime-extensions/api/v1alpha1" + "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/capi/clustertopology/handlers/mutation" + "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/testutils/capitest" + "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/testutils/capitest/request" +) + +const ( + validMirrorCASecretName = "myregistry-mirror-cacert" + //nolint:gosec // Does not contain hard coded credentials. + cpRegistryAsMirrorCreds = "kubeadmControlPlaneRegistryAsMirrorCreds" + //nolint:gosec // Does not contain hard coded credentials. + workerRegistryAsMirrorCreds = "kubeadmConfigTemplateRegistryAsMirrorCreds" +) + +func TestGeneratePatches( + t *testing.T, + generatorFunc func() mutation.GeneratePatches, + fakeClient client.Client, + variableName string, + variablePath ...string, +) { + t.Helper() + + require.NoError( + t, + fakeClient.Create( + context.Background(), + newMirrorSecret(validMirrorCASecretName, request.Namespace), + ), + ) + + capitest.ValidateGeneratePatches( + t, + generatorFunc, + capitest.PatchTestDef{ + Name: "files added in KubeadmControlPlaneTemplate for registry with mirror without CA Certificate", + Vars: []runtimehooksv1.Variable{ + capitest.VariableWithValue( + variableName, + v1alpha1.GlobalImageRegistryMirror{ + URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", + }, + variablePath..., + ), + }, + RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""), + ExpectedPatchMatchers: []capitest.JSONPatchMatcher{ + { + Operation: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/files", + ValueMatcher: gomega.ContainElements( + gomega.HaveKeyWithValue( + "path", "/etc/containerd/certs.d/_default/hosts.toml", + ), + ), + }, + }, + }, + capitest.PatchTestDef{ + Name: "files added in KubeadmControlPlaneTemplate for registry with mirror with CA Certificate", + Vars: []runtimehooksv1.Variable{ + capitest.VariableWithValue( + variableName, + v1alpha1.GlobalImageRegistryMirror{ + URL: "https://registry.example.com", + Credentials: &v1alpha1.RegistryCredentials{ + SecretRef: &corev1.ObjectReference{ + Name: validMirrorCASecretName, + }, + }, + }, + variablePath..., + ), + }, + RequestItem: request.NewKubeadmControlPlaneTemplateRequest("", cpRegistryAsMirrorCreds), + ExpectedPatchMatchers: []capitest.JSONPatchMatcher{ + { + Operation: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/files", + ValueMatcher: gomega.ContainElements( + gomega.HaveKeyWithValue( + "path", "/etc/containerd/certs.d/_default/hosts.toml", + ), + gomega.HaveKeyWithValue( + "path", "/etc/certs/mirror.pem", + ), + ), + }, + }, + }, + capitest.PatchTestDef{ + Name: "files added in KubeadmConfigTemplate for registry mirror wihthout CA certificate", + Vars: []runtimehooksv1.Variable{ + capitest.VariableWithValue( + variableName, + v1alpha1.GlobalImageRegistryMirror{ + URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", + }, + variablePath..., + ), + capitest.VariableWithValue( + "builtin", + map[string]any{ + "machineDeployment": map[string]any{ + "class": names.SimpleNameGenerator.GenerateName("worker-"), + }, + }, + ), + }, + RequestItem: request.NewKubeadmConfigTemplateRequestItem(""), + ExpectedPatchMatchers: []capitest.JSONPatchMatcher{ + { + Operation: "add", + Path: "/spec/template/spec/files", + ValueMatcher: gomega.ContainElements( + gomega.HaveKeyWithValue( + "path", "/etc/containerd/certs.d/_default/hosts.toml", + ), + ), + }, + }, + }, + capitest.PatchTestDef{ + Name: "files added in KubeadmConfigTemplate for registry mirror with secret for CA certificate", + Vars: []runtimehooksv1.Variable{ + capitest.VariableWithValue( + variableName, + v1alpha1.GlobalImageRegistryMirror{ + URL: "https://registry.example.com", + Credentials: &v1alpha1.RegistryCredentials{ + SecretRef: &corev1.ObjectReference{ + Name: validMirrorCASecretName, + }, + }, + }, + variablePath..., + ), + capitest.VariableWithValue( + "builtin", + map[string]any{ + "machineDeployment": map[string]any{ + "class": names.SimpleNameGenerator.GenerateName("worker-"), + }, + }, + ), + }, + RequestItem: request.NewKubeadmConfigTemplateRequest("", workerRegistryAsMirrorCreds), + ExpectedPatchMatchers: []capitest.JSONPatchMatcher{ + { + Operation: "add", + Path: "/spec/template/spec/files", + ValueMatcher: gomega.ContainElements( + gomega.HaveKeyWithValue( + "path", "/etc/containerd/certs.d/_default/hosts.toml", + ), + gomega.HaveKeyWithValue( + "path", "/etc/certs/mirror.pem", + ), + ), + }, + }, + }, + ) +} + +func newMirrorSecret(name, namespace string) *corev1.Secret { + secretData := map[string][]byte{ + "ca.crt": []byte("myCACert"), + } + return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Data: secretData, + Type: corev1.SecretTypeOpaque, + } +} diff --git a/pkg/handlers/generic/mutation/mirrors/variables_test.go b/pkg/handlers/generic/mutation/mirrors/variables_test.go new file mode 100644 index 000000000..332ddc815 --- /dev/null +++ b/pkg/handlers/generic/mutation/mirrors/variables_test.go @@ -0,0 +1,64 @@ +// Copyright 2023 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package mirrors + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/ptr" + + "github.com/d2iq-labs/capi-runtime-extensions/api/v1alpha1" + "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/testutils/capitest" + "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/clusterconfig" +) + +func TestVariableValidation(t *testing.T) { + capitest.ValidateDiscoverVariables( + t, + clusterconfig.MetaVariableName, + ptr.To(v1alpha1.GenericClusterConfig{}.VariableSchema()), + false, + clusterconfig.NewVariable, + capitest.VariableTestDef{ + Name: "without a credentials secret", + Vals: v1alpha1.GenericClusterConfig{ + GlobalImageRegistryMirror: &v1alpha1.GlobalImageRegistryMirror{ + URL: "http://a.b.c.example.com", + }, + }, + }, + capitest.VariableTestDef{ + Name: "with a credentials CA secret", + Vals: v1alpha1.GenericClusterConfig{ + GlobalImageRegistryMirror: &v1alpha1.GlobalImageRegistryMirror{ + URL: "http://a.b.c.example.com", + Credentials: &v1alpha1.RegistryCredentials{ + SecretRef: &corev1.ObjectReference{ + Name: "a.b.c.example.com-ca-cert-creds", + }, + }, + }, + }, + }, + capitest.VariableTestDef{ + Name: "invalid mirror registry URL", + Vals: v1alpha1.GenericClusterConfig{ + GlobalImageRegistryMirror: &v1alpha1.GlobalImageRegistryMirror{ + URL: "unsupportedformat://a.b.c.example.com", + }, + }, + ExpectError: true, + }, + capitest.VariableTestDef{ + Name: "mirror URL without format", + Vals: v1alpha1.GenericClusterConfig{ + GlobalImageRegistryMirror: &v1alpha1.GlobalImageRegistryMirror{ + URL: "a.b.c.example.com/a/b/c", + }, + }, + ExpectError: true, + }, + ) +}