diff --git a/charts/cluster-api-runtime-extensions-nutanix/defaultclusterclasses/nutanix-cluster-class.yaml b/charts/cluster-api-runtime-extensions-nutanix/defaultclusterclasses/nutanix-cluster-class.yaml index eeae7dca8..cf67af219 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/defaultclusterclasses/nutanix-cluster-class.yaml +++ b/charts/cluster-api-runtime-extensions-nutanix/defaultclusterclasses/nutanix-cluster-class.yaml @@ -67,7 +67,7 @@ spec: patches: - external: discoverVariablesExtension: nutanixclusterconfigvars-dv.cluster-api-runtime-extensions-nutanix - generateExtension: nutanixclusterconfigpatch-gp.cluster-api-runtime-extensions-nutanix + generateExtension: nutanixclusterv2configpatch-gp.cluster-api-runtime-extensions-nutanix name: cluster-config - external: discoverVariablesExtension: nutanixworkerconfigvars-dv.cluster-api-runtime-extensions-nutanix diff --git a/hack/examples/overlays/clusterclasses/nutanix/kustomization.yaml.tmpl b/hack/examples/overlays/clusterclasses/nutanix/kustomization.yaml.tmpl index e14d9af5c..2ad0f0c28 100644 --- a/hack/examples/overlays/clusterclasses/nutanix/kustomization.yaml.tmpl +++ b/hack/examples/overlays/clusterclasses/nutanix/kustomization.yaml.tmpl @@ -19,7 +19,7 @@ patches: value: - name: "cluster-config" external: - generateExtension: "nutanixclusterconfigpatch-gp.cluster-api-runtime-extensions-nutanix" + generateExtension: "nutanixclusterv2configpatch-gp.cluster-api-runtime-extensions-nutanix" discoverVariablesExtension: "nutanixclusterconfigvars-dv.cluster-api-runtime-extensions-nutanix" - name: "worker-config" external: diff --git a/pkg/handlers/deleteinv0280/generic/mutation/handlers.go b/pkg/handlers/deleteinv0280/generic/mutation/handlers.go new file mode 100644 index 000000000..9ccafedfe --- /dev/null +++ b/pkg/handlers/deleteinv0280/generic/mutation/handlers.go @@ -0,0 +1,56 @@ +// Copyright 2023 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package mutation + +import ( + "sigs.k8s.io/controller-runtime/pkg/manager" + + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/aws/mutation/cni/calico" + deleteinv0280mirrors "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/deleteinv0280/generic/mutation/mirrors" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/auditpolicy" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/autorenewcerts" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/containerdapplypatchesandrestart" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/containerdmetrics" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/containerdunprivilegedports" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/coredns" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/encryptionatrest" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/etcd" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/extraapiservercertsans" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/httpproxy" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/imageregistries/credentials" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/kubernetesimagerepository" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/users" +) + +// MetaMutators returns all generic patch handlers. +func MetaMutators(mgr manager.Manager) []mutation.MetaMutator { + return []mutation.MetaMutator{ + auditpolicy.NewPatch(), + etcd.NewPatch(), + coredns.NewPatch(), + extraapiservercertsans.NewPatch(), + httpproxy.NewPatch(mgr.GetClient()), + kubernetesimagerepository.NewPatch(), + credentials.NewPatch(mgr.GetClient()), + deleteinv0280mirrors.NewPatch(mgr.GetClient()), + calico.NewPatch(), + users.NewPatch(), + containerdmetrics.NewPatch(), + containerdunprivilegedports.NewPatch(), + encryptionatrest.NewPatch(mgr.GetClient(), encryptionatrest.RandomTokenGenerator), + autorenewcerts.NewPatch(), + + // Some patches may have changed containerd configuration. + // We write the configuration changes to disk, and must run a command + // to apply the changes to the actual containerd configuration. + // We then must restart containerd for the configuration to take effect. + // Therefore, we must apply this patch last. + // + // Containerd restart and readiness altogether could take ~5s. + // We want to keep patch independent of each other and not share any state. + // Therefore, We must always apply this patch regardless any other patch modified containerd configuration. + containerdapplypatchesandrestart.NewPatch(), + } +} diff --git a/pkg/handlers/deleteinv0280/generic/mutation/mirrors/containerd_files.go b/pkg/handlers/deleteinv0280/generic/mutation/mirrors/containerd_files.go new file mode 100644 index 000000000..16f3bf1bf --- /dev/null +++ b/pkg/handlers/deleteinv0280/generic/mutation/mirrors/containerd_files.go @@ -0,0 +1,196 @@ +// Copyright 2023 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package mirrors + +import ( + "bytes" + _ "embed" + "fmt" + "net/url" + "path" + "strings" + "text/template" + + cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" + + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/common" +) + +const ( + containerdHostsConfigurationOnRemote = "/etc/containerd/certs.d/_default/hosts.toml" + secretKeyForCACert = "ca.crt" +) + +var ( + //go:embed templates/hosts.toml.gotmpl + containerdHostsConfiguration []byte + + containerdHostsConfigurationTemplate = template.Must( + template.New("").Parse(string(containerdHostsConfiguration)), + ) + + //go:embed templates/containerd-registry-config-drop-in.toml + containerdRegistryConfigDropIn []byte + containerdRegistryConfigDropInFileOnRemote = common.ContainerdPatchPathOnRemote( + "registry-config.toml", + ) + + mirrorCACertPathOnRemoteFmt = "/etc/certs/%s.pem" +) + +type containerdConfig struct { + URL string + CASecretName string + CACert string + Mirror bool +} + +// fileNameFromURL returns a file name for a registry URL. +// Follows a convention of replacing all non-alphanumeric characters with "-". +func (c containerdConfig) filePathFromURL() (string, error) { + registryURL, err := url.ParseRequestURI(c.URL) + if err != nil { + return "", fmt.Errorf("failed parsing registry URL: %w", err) + } + + registryHostWithPath := registryURL.Host + if registryURL.Path != "" { + registryHostWithPath = path.Join(registryURL.Host, registryURL.Path) + } + + replaced := strings.ReplaceAll(registryHostWithPath, "/", "-") + + return fmt.Sprintf(mirrorCACertPathOnRemoteFmt, replaced), nil +} + +// Return true if configuration is a mirror or has a CA certificate. +func (c containerdConfig) needContainerdConfiguration() bool { + return c.CACert != "" || c.Mirror +} + +// Containerd registry configuration created at /etc/containerd/certs.d/_default/hosts.toml for: +// +// 1. Set the default mirror for all registries. +// The upstream registry will be automatically used after all defined mirrors have been tried. +// https://github.com/containerd/containerd/blob/main/docs/hosts.md#setup-default-mirror-for-all-registries +// +// 2. Setting CA certificate for global image registry mirror and image registries. +func generateContainerdHostsFile( + configs []containerdConfig, +) (*cabpkv1.File, error) { + if len(configs) == 0 { + return nil, nil + } + + type templateInput struct { + URL string + CACertPath string + Mirror bool + } + + inputs := make([]templateInput, 0, len(configs)) + + for _, config := range configs { + if !config.needContainerdConfiguration() { + continue + } + + formattedURL, err := formatURLForContainerd(config.URL) + if err != nil { + return nil, fmt.Errorf("failed formatting image registry URL for Containerd: %w", err) + } + + input := templateInput{ + URL: formattedURL, + Mirror: config.Mirror, + } + // CA cert is optional for mirror registry. + // i.e. registry is using signed certificates. Insecure registry will not be allowed. + if config.CACert != "" { + var registryCACertPathOnRemote string + registryCACertPathOnRemote, err = config.filePathFromURL() + if err != nil { + return nil, fmt.Errorf( + "failed generating CA certificate file path from URL: %w", + err, + ) + } + input.CACertPath = registryCACertPathOnRemote + } + + inputs = append(inputs, input) + } + + var b bytes.Buffer + err := containerdHostsConfigurationTemplate.Execute(&b, inputs) + if err != nil { + return nil, fmt.Errorf("failed executing template for Containerd hosts.toml file: %w", err) + } + return &cabpkv1.File{ + Path: containerdHostsConfigurationOnRemote, + // Trimming the leading and trailing whitespaces in the template did not work as expected with multiple configs. + Content: fmt.Sprintf("%s\n", strings.TrimSpace(b.String())), + Permissions: "0600", + }, nil +} + +func generateRegistryCACertFiles( + configs []containerdConfig, +) ([]cabpkv1.File, error) { + if len(configs) == 0 { + return nil, nil + } + + var files []cabpkv1.File //nolint:prealloc // We don't know the size of the slice yet. + + for _, config := range configs { + if config.CASecretName == "" { + continue + } + + registryCACertPathOnRemote, err := config.filePathFromURL() + if err != nil { + return nil, fmt.Errorf("failed generating CA certificate file path from URL: %w", err) + } + files = append(files, cabpkv1.File{ + Path: registryCACertPathOnRemote, + Permissions: "0600", + ContentFrom: &cabpkv1.FileSource{ + Secret: cabpkv1.SecretFileSource{ + Name: config.CASecretName, + Key: secretKeyForCACert, + }, + }, + }) + } + + return files, nil +} + +func generateContainerdRegistryConfigDropInFile() []cabpkv1.File { + return []cabpkv1.File{ + { + Path: containerdRegistryConfigDropInFileOnRemote, + Content: string(containerdRegistryConfigDropIn), + Permissions: "0600", + }, + } +} + +func formatURLForContainerd(uri string) (string, error) { + mirrorURL, err := url.ParseRequestURI(uri) + if err != nil { + return "", fmt.Errorf("failed parsing mirror: %w", err) + } + + mirror := fmt.Sprintf("%s://%s", mirrorURL.Scheme, mirrorURL.Host) + // assume Containerd expects the following pattern: + // scheme://host/v2/path + mirrorPath := "v2" + if mirrorURL.Path != "" { + mirrorPath = path.Join(mirrorPath, mirrorURL.Path) + } + // using path.Join on all elements incorrectly drops a "/" from "https://" + return fmt.Sprintf("%s/%s", mirror, mirrorPath), nil +} diff --git a/pkg/handlers/deleteinv0280/generic/mutation/mirrors/containerd_files_test.go b/pkg/handlers/deleteinv0280/generic/mutation/mirrors/containerd_files_test.go new file mode 100644 index 000000000..4bfe71996 --- /dev/null +++ b/pkg/handlers/deleteinv0280/generic/mutation/mirrors/containerd_files_test.go @@ -0,0 +1,234 @@ +// Copyright 2023 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package mirrors + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" +) + +func Test_generateContainerdHostsFile(t *testing.T) { + t.Parallel() + tests := []struct { + name string + configs []containerdConfig + want *cabpkv1.File + wantErr error + }{ + { + name: "ECR mirror image registry and no CA certificate", + configs: []containerdConfig{ + { + URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", + Mirror: true, + }, + }, + want: &cabpkv1.File{ + Path: "/etc/containerd/certs.d/_default/hosts.toml", + Owner: "", + Permissions: "0600", + Encoding: "", + Append: false, + Content: `[host."https://123456789.dkr.ecr.us-east-1.amazonaws.com/v2"] + capabilities = ["pull", "resolve"] + # don't rely on Containerd to add the v2/ suffix + # there is a bug where it is added incorrectly for mirrors with a path + override_path = true +`, + }, + wantErr: nil, + }, + { + name: "ECR mirror image registry with a path and no CA certificate", + configs: []containerdConfig{ + { + URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com/myproject", + Mirror: true, + }, + }, + want: &cabpkv1.File{ + Path: "/etc/containerd/certs.d/_default/hosts.toml", + Owner: "", + Permissions: "0600", + Encoding: "", + Append: false, + Content: `[host."https://123456789.dkr.ecr.us-east-1.amazonaws.com/v2/myproject"] + capabilities = ["pull", "resolve"] + # don't rely on Containerd to add the v2/ suffix + # there is a bug where it is added incorrectly for mirrors with a path + override_path = true +`, + }, + wantErr: nil, + }, + { + name: "Mirror image registry with a CA and an image registry with no CA certificate", + configs: []containerdConfig{ + { + URL: "https://mymirror.com", + CACert: "mymirrorcert", + Mirror: true, + }, + { + URL: "https://myregistry.com", + }, + }, + want: &cabpkv1.File{ + Path: "/etc/containerd/certs.d/_default/hosts.toml", + Owner: "", + Permissions: "0600", + Encoding: "", + Append: false, + Content: `[host."https://mymirror.com/v2"] + capabilities = ["pull", "resolve"] + ca = "/etc/certs/mymirror.com.pem" + # don't rely on Containerd to add the v2/ suffix + # there is a bug where it is added incorrectly for mirrors with a path + override_path = true +`, + }, + wantErr: nil, + }, + { + name: "Mirror image registry with a CA and an image registry with a CA", + configs: []containerdConfig{ + { + URL: "https://mymirror.com", + CACert: "mymirrorcert", + Mirror: true, + }, + { + URL: "https://myregistry.com", + CACert: "myregistrycert", + }, + { + URL: "https://172.100.0.10:5000/myproject", + CACert: "myregistrycert", + }, + }, + want: &cabpkv1.File{ + Path: "/etc/containerd/certs.d/_default/hosts.toml", + Owner: "", + Permissions: "0600", + Encoding: "", + Append: false, + Content: `[host."https://mymirror.com/v2"] + capabilities = ["pull", "resolve"] + ca = "/etc/certs/mymirror.com.pem" + # don't rely on Containerd to add the v2/ suffix + # there is a bug where it is added incorrectly for mirrors with a path + override_path = true +[host."https://myregistry.com/v2"] + ca = "/etc/certs/myregistry.com.pem" +[host."https://172.100.0.10:5000/v2/myproject"] + ca = "/etc/certs/172.100.0.10:5000-myproject.pem" +`, + }, + wantErr: nil, + }, + { + name: "Image registry with a CA", + configs: []containerdConfig{ + { + URL: "https://myregistry.com", + CACert: "myregistrycert", + }, + }, + want: &cabpkv1.File{ + Path: "/etc/containerd/certs.d/_default/hosts.toml", + Owner: "", + Permissions: "0600", + Encoding: "", + Append: false, + Content: `[host."https://myregistry.com/v2"] + ca = "/etc/certs/myregistry.com.pem" +`, + }, + wantErr: nil, + }, + } + for idx := range tests { + tt := tests[idx] + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + file, err := generateContainerdHostsFile(tt.configs) + require.ErrorIs(t, err, tt.wantErr) + assert.Equal(t, tt.want, file) + }) + } +} + +func Test_generateRegistryCACertFiles(t *testing.T) { + t.Parallel() + tests := []struct { + name string + configs []containerdConfig + want []cabpkv1.File + }{ + { + name: "ECR mirror image registry with no CA certificate", + configs: []containerdConfig{ + { + URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", + Mirror: true, + }, + }, + want: nil, + }, + { + name: "Mirror image registry with CA certificate", + configs: []containerdConfig{ + { + URL: "https://registry.example.com", + CASecretName: "my-registry-credentials-secret", + Mirror: true, + }, + }, + want: []cabpkv1.File{ + { + Path: "/etc/certs/registry.example.com.pem", + Owner: "", + Permissions: "0600", + Encoding: "", + Append: false, + ContentFrom: &cabpkv1.FileSource{ + Secret: cabpkv1.SecretFileSource{ + Name: "my-registry-credentials-secret", + Key: "ca.crt", + }, + }, + }, + }, + }, + } + for idx := range tests { + tt := tests[idx] + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + file, err := generateRegistryCACertFiles(tt.configs) + require.NoError(t, err) + assert.Equal(t, tt.want, file) + }) + } +} + +func Test_generateContainerdRegistryConfigDropInFile(t *testing.T) { + want := []cabpkv1.File{ + { + Path: "/etc/caren/containerd/patches/registry-config.toml", + Owner: "", + Permissions: "0600", + Encoding: "", + Append: false, + Content: `[plugins."io.containerd.grpc.v1.cri".registry] + config_path = "/etc/containerd/certs.d" +`, + }, + } + file := generateContainerdRegistryConfigDropInFile() + assert.Equal(t, want, file) +} diff --git a/pkg/handlers/deleteinv0280/generic/mutation/mirrors/inject.go b/pkg/handlers/deleteinv0280/generic/mutation/mirrors/inject.go new file mode 100644 index 000000000..8eead175b --- /dev/null +++ b/pkg/handlers/deleteinv0280/generic/mutation/mirrors/inject.go @@ -0,0 +1,283 @@ +// Copyright 2023 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package mirrors + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" + controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" + runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" + ctrl "sigs.k8s.io/controller-runtime" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches/selectors" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/variables" + handlersutils "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/utils" +) + +type globalMirrorPatchHandler struct { + client ctrlclient.Client + + variableName string + variableFieldPath []string +} + +func NewPatch( + cl ctrlclient.Client, +) *globalMirrorPatchHandler { + return newGlobalMirrorPatchHandler( + cl, + v1alpha1.ClusterConfigVariableName, + v1alpha1.GlobalMirrorVariableName, + ) +} + +func newGlobalMirrorPatchHandler( + cl ctrlclient.Client, + variableName string, + variableFieldPath ...string, +) *globalMirrorPatchHandler { + scheme := runtime.NewScheme() + _ = bootstrapv1.AddToScheme(scheme) + _ = controlplanev1.AddToScheme(scheme) + return &globalMirrorPatchHandler{ + client: cl, + variableName: variableName, + variableFieldPath: variableFieldPath, + } +} + +func (h *globalMirrorPatchHandler) Mutate( + ctx context.Context, + obj *unstructured.Unstructured, + vars map[string]apiextensionsv1.JSON, + holderRef runtimehooksv1.HolderReference, + clusterKey ctrlclient.ObjectKey, + _ mutation.ClusterGetter, +) error { + log := ctrl.LoggerFrom(ctx).WithValues( + "holderRef", holderRef, + ) + + globalMirror, globalMirrorErr := variables.Get[v1alpha1.GlobalImageRegistryMirror]( + vars, + h.variableName, + h.variableFieldPath..., + ) + + // add CA certificate for image registries + imageRegistries, imageRegistriesErr := variables.Get[[]v1alpha1.ImageRegistry]( + vars, + h.variableName, + v1alpha1.ImageRegistriesVariableName, + ) + + switch { + case variables.IsNotFoundError(imageRegistriesErr) && variables.IsNotFoundError(globalMirrorErr): + log.V(5).Info("Image Registry Credentials and Global Registry Mirror variable not defined") + return nil + case imageRegistriesErr != nil && !variables.IsNotFoundError(imageRegistriesErr): + return imageRegistriesErr + case globalMirrorErr != nil && !variables.IsNotFoundError(globalMirrorErr): + return globalMirrorErr + } + + var registriesWithOptionalCA []containerdConfig //nolint:prealloc // We don't know the size of the slice yet. + if globalMirrorErr == nil { + registryConfig, err := containerdConfigFromGlobalMirror( + ctx, + h.client, + globalMirror, + obj, + ) + if err != nil { + return err + } + registriesWithOptionalCA = append(registriesWithOptionalCA, registryConfig) + } + for _, imageRegistry := range imageRegistries { + registryWithOptionalCredentials, generateErr := containerdConfigFromImageRegistry( + ctx, + h.client, + imageRegistry, + obj, + ) + if generateErr != nil { + return generateErr + } + + registriesWithOptionalCA = append( + registriesWithOptionalCA, + registryWithOptionalCredentials, + ) + } + + needConfiguration := needContainerdConfiguration( + registriesWithOptionalCA, + ) + if !needConfiguration { + log.V(5).Info("Only Image Registry Configuration is defined but without CA certificates") + return nil + } + + files, err := generateFiles(registriesWithOptionalCA) + if err != nil { + return err + } + + if err := patches.MutateIfApplicable( + obj, vars, &holderRef, selectors.ControlPlane(), log, + func(obj *controlplanev1.KubeadmControlPlaneTemplate) error { + log.WithValues( + "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), + "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), + ).Info("adding global registry mirror files to control plane kubeadm config spec") + obj.Spec.Template.Spec.KubeadmConfigSpec.Files = append( + obj.Spec.Template.Spec.KubeadmConfigSpec.Files, + files..., + ) + + return nil + }); err != nil { + return err + } + + if err := patches.MutateIfApplicable( + obj, vars, &holderRef, selectors.WorkersKubeadmConfigTemplateSelector(), log, + func(obj *bootstrapv1.KubeadmConfigTemplate) error { + log.WithValues( + "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), + "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), + ).Info("adding global registry mirror files to worker node kubeadm config template") + obj.Spec.Template.Spec.Files = append(obj.Spec.Template.Spec.Files, files...) + + return nil + }); err != nil { + return err + } + + return nil +} + +func containerdConfigFromGlobalMirror( + ctx context.Context, + c ctrlclient.Client, + globalMirror v1alpha1.GlobalImageRegistryMirror, + obj ctrlclient.Object, +) (containerdConfig, error) { + configWithOptionalCACert := containerdConfig{ + URL: globalMirror.URL, + Mirror: true, + } + secret, err := handlersutils.SecretForImageRegistryCredentials( + ctx, + c, + globalMirror.Credentials, + obj.GetNamespace(), + ) + if err != nil { + return containerdConfig{}, fmt.Errorf( + "error getting secret %s/%s from Global Image Registry Mirror variable: %w", + obj.GetNamespace(), + globalMirror.Credentials.SecretRef.Name, + err, + ) + } + + if secretHasCACert(secret) { + configWithOptionalCACert.CASecretName = secret.Name + configWithOptionalCACert.CACert = string(secret.Data[secretKeyForCACert]) + } + + return configWithOptionalCACert, nil +} + +func containerdConfigFromImageRegistry( + ctx context.Context, + c ctrlclient.Client, + imageRegistry v1alpha1.ImageRegistry, + obj ctrlclient.Object, +) (containerdConfig, error) { + configWithOptionalCACert := containerdConfig{ + URL: imageRegistry.URL, + } + secret, err := handlersutils.SecretForImageRegistryCredentials( + ctx, + c, + imageRegistry.Credentials, + obj.GetNamespace(), + ) + if err != nil { + return containerdConfig{}, fmt.Errorf( + "error getting secret %s/%s from Image Registry variable: %w", + obj.GetNamespace(), + imageRegistry.Credentials.SecretRef.Name, + err, + ) + } + + if secretHasCACert(secret) { + configWithOptionalCACert.CASecretName = secret.Name + configWithOptionalCACert.CACert = string(secret.Data[secretKeyForCACert]) + } + + return configWithOptionalCACert, nil +} + +func generateFiles( + registriesWithOptionalCA []containerdConfig, +) ([]bootstrapv1.File, error) { + var files []bootstrapv1.File + // generate default registry mirror file + containerdHostsFile, err := generateContainerdHostsFile(registriesWithOptionalCA) + if err != nil { + return nil, err + } + if containerdHostsFile != nil { + files = append(files, *containerdHostsFile) + } + + // generate CA certificate file for registry mirror + mirrorCAFiles, err := generateRegistryCACertFiles(registriesWithOptionalCA) + if err != nil { + return nil, err + } + files = append(files, mirrorCAFiles...) + + // generate Containerd registry config drop-in file + registryConfigDropIn := generateContainerdRegistryConfigDropInFile() + files = append(files, registryConfigDropIn...) + + return files, err +} + +// This handler reads input from two user provided variables: globalImageRegistryMirror and imageRegistries. +// The handler will be used to either add configuration for a global mirror or CA certificates for image registries. +func needContainerdConfiguration(configs []containerdConfig) bool { + for _, config := range configs { + if config.needContainerdConfiguration() { + return true + } + } + + return false +} + +func secretHasCACert(secret *corev1.Secret) bool { + if secret == nil { + return false + } + + _, ok := secret.Data[secretKeyForCACert] + return ok +} diff --git a/pkg/handlers/deleteinv0280/generic/mutation/mirrors/inject_test.go b/pkg/handlers/deleteinv0280/generic/mutation/mirrors/inject_test.go new file mode 100644 index 000000000..a6243352c --- /dev/null +++ b/pkg/handlers/deleteinv0280/generic/mutation/mirrors/inject_test.go @@ -0,0 +1,489 @@ +// Copyright 2023 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package mirrors + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apiserver/pkg/storage/names" + runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" + + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest/request" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/test/helpers" +) + +const ( + validMirrorCASecretName = "myregistry-mirror-cacert" + validMirrorNoCASecretName = "myregistry-mirror-no-cacert" +) + +func TestMirrorsPatch(t *testing.T) { + gomega.RegisterFailHandler(Fail) + RunSpecs(t, "Global mirror mutator suite") +} + +var _ = Describe("Generate Global mirror patches", func() { + patchGenerator := func() mutation.GeneratePatches { + // Always initialize the testEnv variable in the closure. + // This will allow ginkgo to initialize testEnv variable during test execution time. + testEnv := helpers.TestEnv + // use direct client instead of controller client. This will allow the patch handler to read k8s object + // that are written by the tests. + // Test cases writes credentials secret that the mutator handler reads. + // Using direct client will enable reading it immediately. + client, err := testEnv.GetK8sClient() + gomega.Expect(err).To(gomega.BeNil()) + return mutation.NewMetaGeneratePatchesHandler("", client, NewPatch(client)).(mutation.GeneratePatches) + } + + testDefs := []capitest.PatchTestDef{ + { + Name: "files added in KubeadmControlPlaneTemplate for registry with mirror without CA Certificate secret", + Vars: []runtimehooksv1.Variable{ + capitest.VariableWithValue( + v1alpha1.ClusterConfigVariableName, + v1alpha1.GlobalImageRegistryMirror{ + URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", + }, + v1alpha1.GlobalMirrorVariableName, + ), + }, + RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""), + ExpectedPatchMatchers: []capitest.JSONPatchMatcher{ + { + Operation: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/files", + ValueMatcher: gomega.HaveExactElements( + gomega.HaveKeyWithValue( + "path", "/etc/containerd/certs.d/_default/hosts.toml", + ), + gomega.HaveKeyWithValue( + "path", "/etc/caren/containerd/patches/registry-config.toml", + ), + ), + }, + }, + }, + { + Name: "files added in KubeadmControlPlaneTemplate for registry with mirror with CA Certificate", + Vars: []runtimehooksv1.Variable{ + capitest.VariableWithValue( + v1alpha1.ClusterConfigVariableName, + v1alpha1.GlobalImageRegistryMirror{ + URL: "https://registry.example.com", + Credentials: &v1alpha1.RegistryCredentials{ + SecretRef: &v1alpha1.LocalObjectReference{ + Name: validMirrorCASecretName, + }, + }, + }, + v1alpha1.GlobalMirrorVariableName, + ), + }, + RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""), + ExpectedPatchMatchers: []capitest.JSONPatchMatcher{ + { + Operation: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/files", + ValueMatcher: gomega.HaveExactElements( + gomega.HaveKeyWithValue( + "path", "/etc/containerd/certs.d/_default/hosts.toml", + ), + gomega.HaveKeyWithValue( + "path", "/etc/certs/registry.example.com.pem", + ), + gomega.HaveKeyWithValue( + "path", "/etc/caren/containerd/patches/registry-config.toml", + ), + ), + }, + }, + }, + { + Name: "files added in KubeadmControlPlaneTemplate for registry mirror with secret but missing CA certificate key", + Vars: []runtimehooksv1.Variable{ + capitest.VariableWithValue( + v1alpha1.ClusterConfigVariableName, + v1alpha1.GlobalImageRegistryMirror{ + URL: "https://registry.example.com", + Credentials: &v1alpha1.RegistryCredentials{ + SecretRef: &v1alpha1.LocalObjectReference{ + Name: validMirrorNoCASecretName, + }, + }, + }, + v1alpha1.GlobalMirrorVariableName, + ), + }, + RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""), + ExpectedPatchMatchers: []capitest.JSONPatchMatcher{ + { + Operation: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/files", + ValueMatcher: gomega.HaveExactElements( + gomega.HaveKeyWithValue( + "path", "/etc/containerd/certs.d/_default/hosts.toml", + ), + gomega.HaveKeyWithValue( + "path", "/etc/caren/containerd/patches/registry-config.toml", + ), + ), + }, + }, + }, + { + Name: "files added in KubeadmControlPlaneTemplate for image registry with CA Certificate secret", + Vars: []runtimehooksv1.Variable{ + capitest.VariableWithValue( + v1alpha1.ClusterConfigVariableName, + []v1alpha1.ImageRegistry{{ + URL: "https://registry.example.com", + Credentials: &v1alpha1.RegistryCredentials{ + SecretRef: &v1alpha1.LocalObjectReference{ + Name: validMirrorCASecretName, + }, + }, + }}, + v1alpha1.ImageRegistriesVariableName, + ), + }, + RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""), + ExpectedPatchMatchers: []capitest.JSONPatchMatcher{ + { + Operation: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/files", + ValueMatcher: gomega.HaveExactElements( + gomega.HaveKeyWithValue( + "path", "/etc/containerd/certs.d/_default/hosts.toml", + ), + gomega.HaveKeyWithValue( + "path", "/etc/certs/registry.example.com.pem", + ), + gomega.HaveKeyWithValue( + "path", "/etc/caren/containerd/patches/registry-config.toml", + ), + ), + }, + }, + }, + { + Name: "files added in KubeadmConfigTemplate for registry mirror without CA certificate secret", + Vars: []runtimehooksv1.Variable{ + capitest.VariableWithValue( + v1alpha1.ClusterConfigVariableName, + v1alpha1.GlobalImageRegistryMirror{ + URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", + }, + v1alpha1.GlobalMirrorVariableName, + ), + capitest.VariableWithValue( + "builtin", + map[string]any{ + "machineDeployment": map[string]any{ + "class": names.SimpleNameGenerator.GenerateName("worker-"), + }, + }, + ), + }, + RequestItem: request.NewKubeadmConfigTemplateRequestItem(""), + ExpectedPatchMatchers: []capitest.JSONPatchMatcher{ + { + Operation: "add", + Path: "/spec/template/spec/files", + ValueMatcher: gomega.HaveExactElements( + gomega.HaveKeyWithValue( + "path", "/etc/containerd/certs.d/_default/hosts.toml", + ), + gomega.HaveKeyWithValue( + "path", "/etc/caren/containerd/patches/registry-config.toml", + ), + ), + }, + }, + }, + { + Name: "files added in KubeadmConfigTemplate for registry mirror with secret for CA certificate", + Vars: []runtimehooksv1.Variable{ + capitest.VariableWithValue( + v1alpha1.ClusterConfigVariableName, + v1alpha1.GlobalImageRegistryMirror{ + URL: "https://registry.example.com", + Credentials: &v1alpha1.RegistryCredentials{ + SecretRef: &v1alpha1.LocalObjectReference{ + Name: validMirrorCASecretName, + }, + }, + }, + v1alpha1.GlobalMirrorVariableName, + ), + capitest.VariableWithValue( + "builtin", + map[string]any{ + "machineDeployment": map[string]any{ + "class": names.SimpleNameGenerator.GenerateName("worker-"), + }, + }, + ), + }, + RequestItem: request.NewKubeadmConfigTemplateRequestItem(""), + ExpectedPatchMatchers: []capitest.JSONPatchMatcher{ + { + Operation: "add", + Path: "/spec/template/spec/files", + ValueMatcher: gomega.HaveExactElements( + gomega.HaveKeyWithValue( + "path", "/etc/containerd/certs.d/_default/hosts.toml", + ), + gomega.HaveKeyWithValue( + "path", "/etc/certs/registry.example.com.pem", + ), + gomega.HaveKeyWithValue( + "path", "/etc/caren/containerd/patches/registry-config.toml", + ), + ), + }, + }, + }, + { + Name: "files added in KubeadmConfigTemplate for registry mirror with secret but missing CA certificate key", + Vars: []runtimehooksv1.Variable{ + capitest.VariableWithValue( + v1alpha1.ClusterConfigVariableName, + v1alpha1.GlobalImageRegistryMirror{ + URL: "https://registry.example.com", + Credentials: &v1alpha1.RegistryCredentials{ + SecretRef: &v1alpha1.LocalObjectReference{ + Name: validMirrorNoCASecretName, + }, + }, + }, + v1alpha1.GlobalMirrorVariableName, + ), + capitest.VariableWithValue( + "builtin", + map[string]any{ + "machineDeployment": map[string]any{ + "class": names.SimpleNameGenerator.GenerateName("worker-"), + }, + }, + ), + }, + RequestItem: request.NewKubeadmConfigTemplateRequestItem(""), + ExpectedPatchMatchers: []capitest.JSONPatchMatcher{ + { + Operation: "add", + Path: "/spec/template/spec/files", + ValueMatcher: gomega.HaveExactElements( + gomega.HaveKeyWithValue( + "path", "/etc/containerd/certs.d/_default/hosts.toml", + ), + gomega.HaveKeyWithValue( + "path", "/etc/caren/containerd/patches/registry-config.toml", + ), + ), + }, + }, + }, + { + Name: "files added in KubeadmConfigTemplate for image registry with secret for CA certificate", + Vars: []runtimehooksv1.Variable{ + capitest.VariableWithValue( + v1alpha1.ClusterConfigVariableName, + []v1alpha1.ImageRegistry{{ + URL: "https://registry.example.com", + Credentials: &v1alpha1.RegistryCredentials{ + SecretRef: &v1alpha1.LocalObjectReference{ + Name: validMirrorCASecretName, + }, + }, + }}, + v1alpha1.ImageRegistriesVariableName, + ), + capitest.VariableWithValue( + "builtin", + map[string]any{ + "machineDeployment": map[string]any{ + "class": names.SimpleNameGenerator.GenerateName("worker-"), + }, + }, + ), + }, + RequestItem: request.NewKubeadmConfigTemplateRequestItem(""), + ExpectedPatchMatchers: []capitest.JSONPatchMatcher{ + { + Operation: "add", + Path: "/spec/template/spec/files", + ValueMatcher: gomega.HaveExactElements( + gomega.HaveKeyWithValue( + "path", "/etc/containerd/certs.d/_default/hosts.toml", + ), + gomega.HaveKeyWithValue( + "path", "/etc/certs/registry.example.com.pem", + ), + gomega.HaveKeyWithValue( + "path", "/etc/caren/containerd/patches/registry-config.toml", + ), + ), + }, + }, + }, + } + + // Create credentials secret before each test + BeforeEach(func(ctx SpecContext) { + client, err := helpers.TestEnv.GetK8sClient() + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(client.Create( + ctx, + newMirrorSecretWithCA(validMirrorCASecretName, request.Namespace), + )).To(gomega.BeNil()) + gomega.Expect(client.Create( + ctx, + newMirrorSecretWithoutCA(validMirrorNoCASecretName, request.Namespace), + )).To(gomega.BeNil()) + }) + + // Delete credentials secret after each test + AfterEach(func(ctx SpecContext) { + client, err := helpers.TestEnv.GetK8sClient() + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(client.Delete( + ctx, + newMirrorSecretWithCA(validMirrorCASecretName, request.Namespace), + )).To(gomega.BeNil()) + gomega.Expect(client.Delete( + ctx, + newMirrorSecretWithoutCA(validMirrorNoCASecretName, request.Namespace), + )).To(gomega.BeNil()) + }) + + // create test node for each case + for testIdx := range testDefs { + tt := testDefs[testIdx] + It(tt.Name, func() { + capitest.AssertGeneratePatches(GinkgoT(), patchGenerator, &tt) + }) + } +}) + +func newMirrorSecretWithCA(name, namespace string) *corev1.Secret { + secretData := map[string][]byte{ + "ca.crt": []byte("myCACert"), + } + return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Data: secretData, + Type: corev1.SecretTypeOpaque, + } +} + +func newMirrorSecretWithoutCA(name, namespace string) *corev1.Secret { + secretData := map[string][]byte{ + "username": []byte("user"), + "password": []byte("pass"), + } + return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Data: secretData, + Type: corev1.SecretTypeOpaque, + } +} + +func Test_needContainerdConfiguration(t *testing.T) { + t.Parallel() + tests := []struct { + name string + configs []containerdConfig + want bool + }{ + { + name: "ECR mirror image registry with no CA certificate", + configs: []containerdConfig{ + { + URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", + Mirror: true, + }, + }, + want: true, + }, + { + name: "ECR mirror image registry with a path and no CA certificate", + configs: []containerdConfig{ + { + URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com/myproject", + Mirror: true, + }, + }, + want: true, + }, + { + name: "Mirror image registry with a CA and an image registry with no CA certificate", + configs: []containerdConfig{ + { + URL: "https://mymirror.com", + CACert: "mymirrorcert", + Mirror: true, + }, + { + URL: "https://myregistry.com", + }, + }, + want: true, + }, + { + name: "Mirror image registry with a CA and an image registry with a CA", + configs: []containerdConfig{ + { + URL: "https://mymirror.com", + CACert: "mymirrorcert", + Mirror: true, + }, + { + URL: "https://myregistry.com", + CACert: "myregistrycert", + }, + }, + want: true, + }, + { + name: "Image registry with no CA certificate", + configs: []containerdConfig{ + { + URL: "https://myregistry.com", + }, + }, + want: false, + }, + } + for idx := range tests { + tt := tests[idx] + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := needContainerdConfiguration(tt.configs) + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/pkg/handlers/deleteinv0280/generic/mutation/mirrors/templates/containerd-registry-config-drop-in.toml b/pkg/handlers/deleteinv0280/generic/mutation/mirrors/templates/containerd-registry-config-drop-in.toml new file mode 100644 index 000000000..e6adbd676 --- /dev/null +++ b/pkg/handlers/deleteinv0280/generic/mutation/mirrors/templates/containerd-registry-config-drop-in.toml @@ -0,0 +1,2 @@ +[plugins."io.containerd.grpc.v1.cri".registry] + config_path = "/etc/containerd/certs.d" diff --git a/pkg/handlers/deleteinv0280/generic/mutation/mirrors/templates/hosts.toml.gotmpl b/pkg/handlers/deleteinv0280/generic/mutation/mirrors/templates/hosts.toml.gotmpl new file mode 100644 index 000000000..c8118c14c --- /dev/null +++ b/pkg/handlers/deleteinv0280/generic/mutation/mirrors/templates/hosts.toml.gotmpl @@ -0,0 +1,15 @@ +{{- range . }} +{{- if .Mirror }} +[host."{{ .URL }}"] + capabilities = ["pull", "resolve"] + {{- if .CACertPath }} + ca = "{{ .CACertPath }}" + {{- end }} + # don't rely on Containerd to add the v2/ suffix + # there is a bug where it is added incorrectly for mirrors with a path + override_path = true +{{- else }} +[host."{{ .URL }}"] + ca = "{{ .CACertPath }}" +{{- end }} +{{- end }} diff --git a/pkg/handlers/deleteinv0280/generic/mutation/mirrors/variables_test.go b/pkg/handlers/deleteinv0280/generic/mutation/mirrors/variables_test.go new file mode 100644 index 000000000..d25235fd8 --- /dev/null +++ b/pkg/handlers/deleteinv0280/generic/mutation/mirrors/variables_test.go @@ -0,0 +1,86 @@ +// Copyright 2023 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package mirrors + +import ( + "testing" + + "k8s.io/utils/ptr" + + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest" + awsclusterconfig "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/aws/clusterconfig" + dockerclusterconfig "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/docker/clusterconfig" + nutanixclusterconfig "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/nutanix/clusterconfig" +) + +var testDefs = []capitest.VariableTestDef{{ + Name: "without a credentials secret", + Vals: v1alpha1.GenericClusterConfigSpec{ + GlobalImageRegistryMirror: &v1alpha1.GlobalImageRegistryMirror{ + URL: "http://a.b.c.example.com", + }, + }, +}, { + Name: "with a credentials CA secret", + Vals: v1alpha1.GenericClusterConfigSpec{ + GlobalImageRegistryMirror: &v1alpha1.GlobalImageRegistryMirror{ + URL: "http://a.b.c.example.com", + Credentials: &v1alpha1.RegistryCredentials{ + SecretRef: &v1alpha1.LocalObjectReference{ + Name: "a.b.c.example.com-ca-cert-creds", + }, + }, + }, + }, +}, { + Name: "invalid mirror registry URL", + Vals: v1alpha1.GenericClusterConfigSpec{ + GlobalImageRegistryMirror: &v1alpha1.GlobalImageRegistryMirror{ + URL: "unsupportedformat://a.b.c.example.com", + }, + }, + ExpectError: true, +}, { + Name: "mirror URL without format", + Vals: v1alpha1.GenericClusterConfigSpec{ + GlobalImageRegistryMirror: &v1alpha1.GlobalImageRegistryMirror{ + URL: "a.b.c.example.com/a/b/c", + }, + }, + ExpectError: true, +}} + +func TestVariableValidation_AWS(t *testing.T) { + capitest.ValidateDiscoverVariables( + t, + v1alpha1.ClusterConfigVariableName, + ptr.To(v1alpha1.AWSClusterConfig{}.VariableSchema()), + true, + awsclusterconfig.NewVariable, + testDefs..., + ) +} + +func TestVariableValidation_Docker(t *testing.T) { + capitest.ValidateDiscoverVariables( + t, + v1alpha1.ClusterConfigVariableName, + ptr.To(v1alpha1.DockerClusterConfig{}.VariableSchema()), + true, + dockerclusterconfig.NewVariable, + testDefs..., + ) +} + +func TestVariableValidation_Nutanix(t *testing.T) { + capitest.ValidateDiscoverVariables( + t, + v1alpha1.ClusterConfigVariableName, + ptr.To(v1alpha1.NutanixClusterConfig{}.VariableSchema()), + true, + nutanixclusterconfig.NewVariable, + testDefs..., + ) +} diff --git a/pkg/handlers/deleteinv0280/nutanix/mutation/metapatch_handler.go b/pkg/handlers/deleteinv0280/nutanix/mutation/metapatch_handler.go new file mode 100644 index 000000000..aeaadc5bd --- /dev/null +++ b/pkg/handlers/deleteinv0280/nutanix/mutation/metapatch_handler.go @@ -0,0 +1,36 @@ +// Copyright 2024 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package mutation + +import ( + "sigs.k8s.io/controller-runtime/pkg/manager" + + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation" + deleteinv0280genericmutation "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/deleteinv0280/generic/mutation" + genericmutation "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/controlplanevirtualip" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/nutanix/mutation/controlplaneendpoint" + nutanixcontrolplanevirtualip "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/nutanix/mutation/controlplanevirtualip" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/nutanix/mutation/machinedetails" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/nutanix/mutation/prismcentralendpoint" +) + +// MetaPatchHandler returns a meta patch handler for mutating CAPX clusters. +func MetaPatchHandler(mgr manager.Manager, cfg *controlplanevirtualip.Config) handlers.Named { + patchHandlers := []mutation.MetaMutator{ + controlplaneendpoint.NewPatch(), + nutanixcontrolplanevirtualip.NewPatch(mgr.GetClient(), cfg), + prismcentralendpoint.NewPatch(), + machinedetails.NewControlPlanePatch(), + } + patchHandlers = append(patchHandlers, deleteinv0280genericmutation.MetaMutators(mgr)...) + patchHandlers = append(patchHandlers, genericmutation.ControlPlaneMetaMutators()...) + + return mutation.NewMetaGeneratePatchesHandler( + "nutanixClusterConfigPatch", + mgr.GetClient(), + patchHandlers..., + ) +} diff --git a/pkg/handlers/generic/mutation/mirrors/containerd_files.go b/pkg/handlers/generic/mutation/mirrors/containerd_files.go index 16f3bf1bf..8faff58e8 100644 --- a/pkg/handlers/generic/mutation/mirrors/containerd_files.go +++ b/pkg/handlers/generic/mutation/mirrors/containerd_files.go @@ -26,7 +26,7 @@ var ( //go:embed templates/hosts.toml.gotmpl containerdHostsConfiguration []byte - containerdHostsConfigurationTemplate = template.Must( + containerdDefaultHostsConfigurationTemplate = template.Must( template.New("").Parse(string(containerdHostsConfiguration)), ) @@ -36,7 +36,7 @@ var ( "registry-config.toml", ) - mirrorCACertPathOnRemoteFmt = "/etc/certs/%s.pem" + caCertPathOnRemoteFmt = "/etc/containerd/certs.d/%s/ca.crt" ) type containerdConfig struct { @@ -54,14 +54,7 @@ func (c containerdConfig) filePathFromURL() (string, error) { return "", fmt.Errorf("failed parsing registry URL: %w", err) } - registryHostWithPath := registryURL.Host - if registryURL.Path != "" { - registryHostWithPath = path.Join(registryURL.Host, registryURL.Path) - } - - replaced := strings.ReplaceAll(registryHostWithPath, "/", "-") - - return fmt.Sprintf(mirrorCACertPathOnRemoteFmt, replaced), nil + return fmt.Sprintf(caCertPathOnRemoteFmt, registryURL.Host), nil } // Return true if configuration is a mirror or has a CA certificate. @@ -71,12 +64,12 @@ func (c containerdConfig) needContainerdConfiguration() bool { // Containerd registry configuration created at /etc/containerd/certs.d/_default/hosts.toml for: // -// 1. Set the default mirror for all registries. +// 1. Setting the default mirror for all registries. // The upstream registry will be automatically used after all defined mirrors have been tried. // https://github.com/containerd/containerd/blob/main/docs/hosts.md#setup-default-mirror-for-all-registries // -// 2. Setting CA certificate for global image registry mirror and image registries. -func generateContainerdHostsFile( +// 2. Setting CA certificate for global image registry mirror. +func generateContainerdDefaultHostsFile( configs []containerdConfig, ) (*cabpkv1.File, error) { if len(configs) == 0 { @@ -86,13 +79,12 @@ func generateContainerdHostsFile( type templateInput struct { URL string CACertPath string - Mirror bool } inputs := make([]templateInput, 0, len(configs)) for _, config := range configs { - if !config.needContainerdConfiguration() { + if !config.Mirror { continue } @@ -102,14 +94,12 @@ func generateContainerdHostsFile( } input := templateInput{ - URL: formattedURL, - Mirror: config.Mirror, + URL: formattedURL, } // CA cert is optional for mirror registry. // i.e. registry is using signed certificates. Insecure registry will not be allowed. if config.CACert != "" { - var registryCACertPathOnRemote string - registryCACertPathOnRemote, err = config.filePathFromURL() + registryCACertPathOnRemote, err := config.filePathFromURL() if err != nil { return nil, fmt.Errorf( "failed generating CA certificate file path from URL: %w", @@ -123,7 +113,7 @@ func generateContainerdHostsFile( } var b bytes.Buffer - err := containerdHostsConfigurationTemplate.Execute(&b, inputs) + err := containerdDefaultHostsConfigurationTemplate.Execute(&b, inputs) if err != nil { return nil, fmt.Errorf("failed executing template for Containerd hosts.toml file: %w", err) } diff --git a/pkg/handlers/generic/mutation/mirrors/containerd_files_test.go b/pkg/handlers/generic/mutation/mirrors/containerd_files_test.go index 4bfe71996..5748c93e1 100644 --- a/pkg/handlers/generic/mutation/mirrors/containerd_files_test.go +++ b/pkg/handlers/generic/mutation/mirrors/containerd_files_test.go @@ -11,7 +11,7 @@ import ( cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" ) -func Test_generateContainerdHostsFile(t *testing.T) { +func Test_generateContainerdDefaultHostsFile(t *testing.T) { t.Parallel() tests := []struct { name string @@ -85,7 +85,7 @@ func Test_generateContainerdHostsFile(t *testing.T) { Append: false, Content: `[host."https://mymirror.com/v2"] capabilities = ["pull", "resolve"] - ca = "/etc/certs/mymirror.com.pem" + ca = "/etc/containerd/certs.d/mymirror.com/ca.crt" # don't rely on Containerd to add the v2/ suffix # there is a bug where it is added incorrectly for mirrors with a path override_path = true @@ -118,14 +118,10 @@ func Test_generateContainerdHostsFile(t *testing.T) { Append: false, Content: `[host."https://mymirror.com/v2"] capabilities = ["pull", "resolve"] - ca = "/etc/certs/mymirror.com.pem" + ca = "/etc/containerd/certs.d/mymirror.com/ca.crt" # don't rely on Containerd to add the v2/ suffix # there is a bug where it is added incorrectly for mirrors with a path override_path = true -[host."https://myregistry.com/v2"] - ca = "/etc/certs/myregistry.com.pem" -[host."https://172.100.0.10:5000/v2/myproject"] - ca = "/etc/certs/172.100.0.10:5000-myproject.pem" `, }, wantErr: nil, @@ -144,8 +140,7 @@ func Test_generateContainerdHostsFile(t *testing.T) { Permissions: "0600", Encoding: "", Append: false, - Content: `[host."https://myregistry.com/v2"] - ca = "/etc/certs/myregistry.com.pem" + Content: ` `, }, wantErr: nil, @@ -155,7 +150,7 @@ func Test_generateContainerdHostsFile(t *testing.T) { tt := tests[idx] t.Run(tt.name, func(t *testing.T) { t.Parallel() - file, err := generateContainerdHostsFile(tt.configs) + file, err := generateContainerdDefaultHostsFile(tt.configs) require.ErrorIs(t, err, tt.wantErr) assert.Equal(t, tt.want, file) }) @@ -190,7 +185,7 @@ func Test_generateRegistryCACertFiles(t *testing.T) { }, want: []cabpkv1.File{ { - Path: "/etc/certs/registry.example.com.pem", + Path: "/etc/containerd/certs.d/registry.example.com/ca.crt", Owner: "", Permissions: "0600", Encoding: "", diff --git a/pkg/handlers/generic/mutation/mirrors/inject.go b/pkg/handlers/generic/mutation/mirrors/inject.go index 8eead175b..03f6eec7d 100644 --- a/pkg/handlers/generic/mutation/mirrors/inject.go +++ b/pkg/handlers/generic/mutation/mirrors/inject.go @@ -239,7 +239,7 @@ func generateFiles( ) ([]bootstrapv1.File, error) { var files []bootstrapv1.File // generate default registry mirror file - containerdHostsFile, err := generateContainerdHostsFile(registriesWithOptionalCA) + containerdHostsFile, err := generateContainerdDefaultHostsFile(registriesWithOptionalCA) if err != nil { return nil, err } diff --git a/pkg/handlers/generic/mutation/mirrors/inject_test.go b/pkg/handlers/generic/mutation/mirrors/inject_test.go index a6243352c..b10a69ed2 100644 --- a/pkg/handlers/generic/mutation/mirrors/inject_test.go +++ b/pkg/handlers/generic/mutation/mirrors/inject_test.go @@ -99,7 +99,7 @@ var _ = Describe("Generate Global mirror patches", func() { "path", "/etc/containerd/certs.d/_default/hosts.toml", ), gomega.HaveKeyWithValue( - "path", "/etc/certs/registry.example.com.pem", + "path", "/etc/containerd/certs.d/registry.example.com/ca.crt", ), gomega.HaveKeyWithValue( "path", "/etc/caren/containerd/patches/registry-config.toml", @@ -166,7 +166,7 @@ var _ = Describe("Generate Global mirror patches", func() { "path", "/etc/containerd/certs.d/_default/hosts.toml", ), gomega.HaveKeyWithValue( - "path", "/etc/certs/registry.example.com.pem", + "path", "/etc/containerd/certs.d/registry.example.com/ca.crt", ), gomega.HaveKeyWithValue( "path", "/etc/caren/containerd/patches/registry-config.toml", @@ -244,7 +244,7 @@ var _ = Describe("Generate Global mirror patches", func() { "path", "/etc/containerd/certs.d/_default/hosts.toml", ), gomega.HaveKeyWithValue( - "path", "/etc/certs/registry.example.com.pem", + "path", "/etc/containerd/certs.d/registry.example.com/ca.crt", ), gomega.HaveKeyWithValue( "path", "/etc/caren/containerd/patches/registry-config.toml", @@ -299,7 +299,7 @@ var _ = Describe("Generate Global mirror patches", func() { capitest.VariableWithValue( v1alpha1.ClusterConfigVariableName, []v1alpha1.ImageRegistry{{ - URL: "https://registry.example.com", + URL: "https://registry.example.com:5050", Credentials: &v1alpha1.RegistryCredentials{ SecretRef: &v1alpha1.LocalObjectReference{ Name: validMirrorCASecretName, @@ -327,7 +327,7 @@ var _ = Describe("Generate Global mirror patches", func() { "path", "/etc/containerd/certs.d/_default/hosts.toml", ), gomega.HaveKeyWithValue( - "path", "/etc/certs/registry.example.com.pem", + "path", "/etc/containerd/certs.d/registry.example.com:5050/ca.crt", ), gomega.HaveKeyWithValue( "path", "/etc/caren/containerd/patches/registry-config.toml", diff --git a/pkg/handlers/generic/mutation/mirrors/templates/hosts.toml.gotmpl b/pkg/handlers/generic/mutation/mirrors/templates/hosts.toml.gotmpl index c8118c14c..b80742dcc 100644 --- a/pkg/handlers/generic/mutation/mirrors/templates/hosts.toml.gotmpl +++ b/pkg/handlers/generic/mutation/mirrors/templates/hosts.toml.gotmpl @@ -1,5 +1,4 @@ {{- range . }} -{{- if .Mirror }} [host."{{ .URL }}"] capabilities = ["pull", "resolve"] {{- if .CACertPath }} @@ -8,8 +7,4 @@ # don't rely on Containerd to add the v2/ suffix # there is a bug where it is added incorrectly for mirrors with a path override_path = true -{{- else }} -[host."{{ .URL }}"] - ca = "{{ .CACertPath }}" -{{- end }} {{- end }} diff --git a/pkg/handlers/nutanix/handlers.go b/pkg/handlers/nutanix/handlers.go index 6315853dd..c1b28f656 100644 --- a/pkg/handlers/nutanix/handlers.go +++ b/pkg/handlers/nutanix/handlers.go @@ -8,6 +8,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers" + deleteinv0280nutanixmutation "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/deleteinv0280/nutanix/mutation" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/controlplanevirtualip" nutanixclusterconfig "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/nutanix/clusterconfig" nutanixmutation "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/nutanix/mutation" @@ -33,6 +34,7 @@ func (h *Handlers) AllHandlers(mgr manager.Manager) []handlers.Named { nutanixclusterconfig.NewVariable(), nutanixworkerconfig.NewVariable(), nutanixmutation.MetaPatchHandler(mgr, h.controlPlaneVirtualIPConfig), + deleteinv0280nutanixmutation.MetaPatchHandler(mgr, h.controlPlaneVirtualIPConfig), nutanixmutation.MetaWorkerPatchHandler(mgr), } } diff --git a/pkg/handlers/nutanix/mutation/metapatch_handler.go b/pkg/handlers/nutanix/mutation/metapatch_handler.go index 7e975ad43..d63ee0567 100644 --- a/pkg/handlers/nutanix/mutation/metapatch_handler.go +++ b/pkg/handlers/nutanix/mutation/metapatch_handler.go @@ -28,7 +28,7 @@ func MetaPatchHandler(mgr manager.Manager, cfg *controlplanevirtualip.Config) ha patchHandlers = append(patchHandlers, genericmutation.ControlPlaneMetaMutators()...) return mutation.NewMetaGeneratePatchesHandler( - "nutanixClusterConfigPatch", + "nutanixClusterV2ConfigPatch", mgr.GetClient(), patchHandlers..., )