diff --git a/api/v1alpha1/constants.go b/api/v1alpha1/constants.go index 6493ef621..1efbcecd9 100644 --- a/api/v1alpha1/constants.go +++ b/api/v1alpha1/constants.go @@ -28,6 +28,11 @@ const ( // ServiceLoadBalancerVariableName is the Service LoadBalancer config patch variable name. ServiceLoadBalancerVariableName = "serviceLoadBalancer" + // GlobalMirrorVariableName is the global image registry mirror patch variable name. + GlobalMirrorVariableName = "globalImageRegistryMirror" + // ImageRegistriesVariableName is the image registries patch variable name. + ImageRegistriesVariableName = "imageRegistries" + // NamespaceSyncLabelKey is a label that can be applied to a namespace. // // When a namespace has a label with this key, ClusterClasses and their Templates are diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 94f848d4f..d821469a1 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -10,7 +10,7 @@ package v1alpha1 import ( "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/external/github.com/nutanix-cloud-native/cluster-api-provider-nutanix/api/v1beta1" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/external/sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" - v1 "k8s.io/api/core/v1" + "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -1289,9 +1289,7 @@ func (in *NutanixNodeSpec) DeepCopy() *NutanixNodeSpec { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *NutanixPrismCentralEndpointCredentials) DeepCopyInto( - out *NutanixPrismCentralEndpointCredentials, -) { +func (in *NutanixPrismCentralEndpointCredentials) DeepCopyInto(out *NutanixPrismCentralEndpointCredentials) { *out = *in out.SecretRef = in.SecretRef } diff --git a/docs/content/customization/generic/image-registries.md b/docs/content/customization/generic/image-registries.md index 2c1e8b336..86e4ce9b4 100644 --- a/docs/content/customization/generic/image-registries.md +++ b/docs/content/customization/generic/image-registries.md @@ -20,7 +20,24 @@ kubectl create secret generic my-registry-credentials \ --from-literal username=${REGISTRY_USERNAME} --from-literal password=${REGISTRY_PASSWORD} ``` -To add image registry credentials, specify the following configuration: +If your registry requires a private or self-signed CA certificate, +create a Kubernetes Secret with the `ca.crt` key populated with the CA certificate in PEM format: + +```shell +kubectl create secret generic my-mirror-ca-cert \ + --from-file=ca.crt=registry-ca.crt +``` + +To set both image registry credentials and CA certificate, +create a Kubernetes Secret with keys for `username`, `password`, and `ca.crt`: + +```shell +kubectl create secret generic my-registry-credentials \ + --from-literal username=${REGISTRY_USERNAME} --from-literal password=${REGISTRY_PASSWORD} \ + --from-file=ca.crt=registry-ca.crt +``` + +To add image registry credentials and/or CA certificate, specify the following configuration: ```yaml apiVersion: cluster.x-k8s.io/v1beta1 diff --git a/pkg/handlers/generic/mutation/imageregistries/constants.go b/pkg/handlers/generic/mutation/imageregistries/constants.go deleted file mode 100644 index 336d0ab12..000000000 --- a/pkg/handlers/generic/mutation/imageregistries/constants.go +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright 2023 Nutanix. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package imageregistries - -const ( - VariableName = "imageRegistries" -) diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go index d7bbc6e7f..5a82c06d3 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go @@ -108,7 +108,9 @@ func templateFilesForImageCredentialProviderConfigs( files = append(files, *kubeletCredentialProviderConfigFile) } - kubeletDynamicCredentialProviderConfigFile, err := templateDynamicCredentialProviderConfig(configs) + kubeletDynamicCredentialProviderConfigFile, err := templateDynamicCredentialProviderConfig( + configs, + ) if err != nil { return nil, err } diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go b/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go index 3a2a45e67..9a6a04adc 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/inject.go @@ -8,7 +8,6 @@ import ( "errors" "fmt" - corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" @@ -26,8 +25,6 @@ import ( "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches/selectors" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/variables" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/k8s/client" - "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/imageregistries" - "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/mirrors" handlersutils "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/utils" ) @@ -46,7 +43,7 @@ func NewPatch( return newImageRegistriesPatchHandler( cl, v1alpha1.ClusterConfigVariableName, - imageregistries.VariableName, + v1alpha1.ImageRegistriesVariableName, ) } @@ -87,12 +84,12 @@ func (h *imageRegistriesPatchHandler) Mutate( globalMirror, globalMirrorErr := variables.Get[v1alpha1.GlobalImageRegistryMirror]( vars, h.variableName, - mirrors.GlobalMirrorVariableName, + v1alpha1.GlobalMirrorVariableName, ) switch { case variables.IsNotFoundError(imageRegistriesErr) && variables.IsNotFoundError(globalMirrorErr): - log.V(5).Info("Image Registry Credentials variable not defined") + log.V(5).Info("Image Registry Credentials and Global Registry Mirror variable not defined") return nil case imageRegistriesErr != nil && !variables.IsNotFoundError(imageRegistriesErr): return imageRegistriesErr @@ -287,7 +284,7 @@ func ensureOwnerReferenceOnCredentialsSecrets( } for _, credential := range credentials { - if secretName := secretNameForImageRegistryCredentials(credential); secretName != "" { + if secretName := handlersutils.SecretNameForImageRegistryCredentials(credential); secretName != "" { // Ensure the Secret is owned by the Cluster so it is correctly moved and deleted with the Cluster. // This code assumes that Secret exists and that was validated before calling this function. err := handlersutils.EnsureOwnerReferenceForSecret( @@ -317,7 +314,7 @@ func registryWithOptionalCredentialsFromImageRegistryCredentials( registryWithOptionalCredentials := providerConfig{ URL: imageRegistry.URL, } - secret, err := secretForImageRegistryCredentials( + secret, err := handlersutils.SecretForImageRegistryCredentials( ctx, c, imageRegistry.Credentials, @@ -350,7 +347,7 @@ func mirrorConfigFromGlobalImageRegistryMirror( URL: mirror.URL, Mirror: true, } - secret, err := secretForImageRegistryCredentials( + secret, err := handlersutils.SecretForImageRegistryCredentials( ctx, c, mirror.Credentials, @@ -438,28 +435,6 @@ func createSecretIfNeeded( return nil } -// secretForImageRegistryCredentials returns the Secret for the given ImageRegistryCredentials. -// Returns nil if the secret field is empty. -func secretForImageRegistryCredentials( - ctx context.Context, - c ctrlclient.Reader, - credentials *v1alpha1.RegistryCredentials, - objectNamespace string, -) (*corev1.Secret, error) { - name := secretNameForImageRegistryCredentials(credentials) - if name == "" { - return nil, nil - } - - key := ctrlclient.ObjectKey{ - Name: name, - Namespace: objectNamespace, - } - secret := &corev1.Secret{} - err := c.Get(ctx, key, secret) - return secret, err -} - // This handler reads input from two user provided variables: globalImageRegistryMirror and imageRegistries. // We expect if imageRegistries is set it will either have static credentials // or be for a registry where the credential plugin returns the credentials, ie ECR, GCR, ACR, etc, @@ -491,12 +466,3 @@ func needImageRegistryCredentialsConfiguration(configs []providerConfig) (bool, return true, nil } - -// secretForImageRegistryCredentials returns the name of the Secret for the given RegistryCredentials. -// Returns an empty string if the credentials or secret field is empty. -func secretNameForImageRegistryCredentials(credentials *v1alpha1.RegistryCredentials) string { - if credentials == nil || credentials.SecretRef == nil || credentials.SecretRef.Name == "" { - return "" - } - return credentials.SecretRef.Name -} diff --git a/pkg/handlers/generic/mutation/imageregistries/credentials/inject_test.go b/pkg/handlers/generic/mutation/imageregistries/credentials/inject_test.go index 1ff863973..e32e8f855 100644 --- a/pkg/handlers/generic/mutation/imageregistries/credentials/inject_test.go +++ b/pkg/handlers/generic/mutation/imageregistries/credentials/inject_test.go @@ -26,7 +26,6 @@ import ( "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest/request" - "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/imageregistries" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/test/helpers" ) @@ -161,7 +160,7 @@ var _ = Describe("Generate Image registry patches", func() { []v1alpha1.ImageRegistry{{ URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", }}, - imageregistries.VariableName, + v1alpha1.ImageRegistriesVariableName, ), }, RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""), @@ -221,7 +220,7 @@ var _ = Describe("Generate Image registry patches", func() { }, }, }}, - imageregistries.VariableName, + v1alpha1.ImageRegistriesVariableName, ), }, RequestItem: request.NewKubeadmControlPlaneTemplateRequest( @@ -283,7 +282,7 @@ var _ = Describe("Generate Image registry patches", func() { []v1alpha1.ImageRegistry{{ URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", }}, - imageregistries.VariableName, + v1alpha1.ImageRegistriesVariableName, ), capitest.VariableWithValue( "builtin", @@ -343,7 +342,7 @@ var _ = Describe("Generate Image registry patches", func() { }, }, }}, - imageregistries.VariableName, + v1alpha1.ImageRegistriesVariableName, ), capitest.VariableWithValue( "builtin", @@ -405,7 +404,7 @@ var _ = Describe("Generate Image registry patches", func() { []v1alpha1.ImageRegistry{{ URL: "https://registry.example.com", }}, - imageregistries.VariableName, + v1alpha1.ImageRegistriesVariableName, ), }, RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""), diff --git a/pkg/handlers/generic/mutation/mirrors/constants.go b/pkg/handlers/generic/mutation/mirrors/constants.go deleted file mode 100644 index 1a187452c..000000000 --- a/pkg/handlers/generic/mutation/mirrors/constants.go +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright 2023 Nutanix. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package mirrors - -const ( - GlobalMirrorVariableName = "globalImageRegistryMirror" -) diff --git a/pkg/handlers/generic/mutation/mirrors/containerd_files.go b/pkg/handlers/generic/mutation/mirrors/containerd_files.go new file mode 100644 index 000000000..9e6aece96 --- /dev/null +++ b/pkg/handlers/generic/mutation/mirrors/containerd_files.go @@ -0,0 +1,196 @@ +// Copyright 2023 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package mirrors + +import ( + "bytes" + _ "embed" + "fmt" + "net/url" + "path" + "strings" + "text/template" + + cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" + + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/common" +) + +const ( + containerdHostsConfigurationOnRemote = "/etc/containerd/certs.d/_default/hosts.toml" + secretKeyForMirrorCACert = "ca.crt" +) + +var ( + //go:embed templates/hosts.toml.gotmpl + containerdHostsConfiguration []byte + + containerdHostsConfigurationTemplate = template.Must( + template.New("").Parse(string(containerdHostsConfiguration)), + ) + + //go:embed templates/containerd-registry-config-drop-in.toml + containerdRegistryConfigDropIn []byte + containerdRegistryConfigDropInFileOnRemote = common.ContainerdPatchPathOnRemote( + "registry-config.toml", + ) + + mirrorCACertPathOnRemoteFmt = "/etc/certs/%s.pem" +) + +type containerdConfig struct { + URL string + CASecretName string + CACert string + Mirror bool +} + +// fileNameFromURL returns a file name for a registry URL. +// Follows a convention of replacing all non-alphanumeric characters with "-". +func (c containerdConfig) filePathFromURL() (string, error) { + registryURL, err := url.ParseRequestURI(c.URL) + if err != nil { + return "", fmt.Errorf("failed parsing registry URL: %w", err) + } + + registryHostWithPath := registryURL.Host + if registryURL.Path != "" { + registryHostWithPath = path.Join(registryURL.Host, registryURL.Path) + } + + replaced := strings.ReplaceAll(registryHostWithPath, "/", "-") + + return fmt.Sprintf(mirrorCACertPathOnRemoteFmt, replaced), nil +} + +// Return true if configuration is a mirror or has a CA certificate. +func (c containerdConfig) needContainerdConfiguration() bool { + return c.CACert != "" || c.Mirror +} + +// Containerd registry configuration created at /etc/containerd/certs.d/_default/hosts.toml for: +// +// 1. Set the default mirror for all registries. +// The upstream registry will be automatically used after all defined mirrors have been tried. +// https://github.com/containerd/containerd/blob/main/docs/hosts.md#setup-default-mirror-for-all-registries +// +// 2. Setting CA certificate for global image registry mirror and image registries. +func generateContainerdHostsFile( + configs []containerdConfig, +) (*cabpkv1.File, error) { + if len(configs) == 0 { + return nil, nil + } + + type templateInput struct { + URL string + CACertPath string + Mirror bool + } + + inputs := make([]templateInput, 0, len(configs)) + + for _, config := range configs { + if !config.needContainerdConfiguration() { + continue + } + + formattedURL, err := formatURLForContainerd(config.URL) + if err != nil { + return nil, fmt.Errorf("failed formatting image registry URL for Containerd: %w", err) + } + + input := templateInput{ + URL: formattedURL, + Mirror: config.Mirror, + } + // CA cert is optional for mirror registry. + // i.e. registry is using signed certificates. Insecure registry will not be allowed. + if config.CACert != "" { + var registryCACertPathOnRemote string + registryCACertPathOnRemote, err = config.filePathFromURL() + if err != nil { + return nil, fmt.Errorf( + "failed generating CA certificate file path from URL: %w", + err, + ) + } + input.CACertPath = registryCACertPathOnRemote + } + + inputs = append(inputs, input) + } + + var b bytes.Buffer + err := containerdHostsConfigurationTemplate.Execute(&b, inputs) + if err != nil { + return nil, fmt.Errorf("failed executing template for Containerd hosts.toml file: %w", err) + } + return &cabpkv1.File{ + Path: containerdHostsConfigurationOnRemote, + // Trimming the leading and trailing whitespaces in the template did not work as expected with multiple configs. + Content: fmt.Sprintf("%s\n", strings.TrimSpace(b.String())), + Permissions: "0600", + }, nil +} + +func generateRegistryCACertFiles( + configs []containerdConfig, +) ([]cabpkv1.File, error) { + if len(configs) == 0 { + return nil, nil + } + + var files []cabpkv1.File //nolint:prealloc // We don't know the size of the slice yet. + + for _, config := range configs { + if config.CASecretName == "" { + continue + } + + registryCACertPathOnRemote, err := config.filePathFromURL() + if err != nil { + return nil, fmt.Errorf("failed generating CA certificate file path from URL: %w", err) + } + files = append(files, cabpkv1.File{ + Path: registryCACertPathOnRemote, + Permissions: "0600", + ContentFrom: &cabpkv1.FileSource{ + Secret: cabpkv1.SecretFileSource{ + Name: config.CASecretName, + Key: secretKeyForMirrorCACert, + }, + }, + }) + } + + return files, nil +} + +func generateContainerdRegistryConfigDropInFile() []cabpkv1.File { + return []cabpkv1.File{ + { + Path: containerdRegistryConfigDropInFileOnRemote, + Content: string(containerdRegistryConfigDropIn), + Permissions: "0600", + }, + } +} + +func formatURLForContainerd(uri string) (string, error) { + mirrorURL, err := url.ParseRequestURI(uri) + if err != nil { + return "", fmt.Errorf("failed parsing mirror: %w", err) + } + + mirror := fmt.Sprintf("%s://%s", mirrorURL.Scheme, mirrorURL.Host) + // assume Containerd expects the following pattern: + // scheme://host/v2/path + mirrorPath := "v2" + if mirrorURL.Path != "" { + mirrorPath = path.Join(mirrorPath, mirrorURL.Path) + } + // using path.Join on all elements incorrectly drops a "/" from "https://" + return fmt.Sprintf("%s/%s", mirror, mirrorPath), nil +} diff --git a/pkg/handlers/generic/mutation/mirrors/containerd_files_test.go b/pkg/handlers/generic/mutation/mirrors/containerd_files_test.go new file mode 100644 index 000000000..4bfe71996 --- /dev/null +++ b/pkg/handlers/generic/mutation/mirrors/containerd_files_test.go @@ -0,0 +1,234 @@ +// Copyright 2023 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package mirrors + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" +) + +func Test_generateContainerdHostsFile(t *testing.T) { + t.Parallel() + tests := []struct { + name string + configs []containerdConfig + want *cabpkv1.File + wantErr error + }{ + { + name: "ECR mirror image registry and no CA certificate", + configs: []containerdConfig{ + { + URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", + Mirror: true, + }, + }, + want: &cabpkv1.File{ + Path: "/etc/containerd/certs.d/_default/hosts.toml", + Owner: "", + Permissions: "0600", + Encoding: "", + Append: false, + Content: `[host."https://123456789.dkr.ecr.us-east-1.amazonaws.com/v2"] + capabilities = ["pull", "resolve"] + # don't rely on Containerd to add the v2/ suffix + # there is a bug where it is added incorrectly for mirrors with a path + override_path = true +`, + }, + wantErr: nil, + }, + { + name: "ECR mirror image registry with a path and no CA certificate", + configs: []containerdConfig{ + { + URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com/myproject", + Mirror: true, + }, + }, + want: &cabpkv1.File{ + Path: "/etc/containerd/certs.d/_default/hosts.toml", + Owner: "", + Permissions: "0600", + Encoding: "", + Append: false, + Content: `[host."https://123456789.dkr.ecr.us-east-1.amazonaws.com/v2/myproject"] + capabilities = ["pull", "resolve"] + # don't rely on Containerd to add the v2/ suffix + # there is a bug where it is added incorrectly for mirrors with a path + override_path = true +`, + }, + wantErr: nil, + }, + { + name: "Mirror image registry with a CA and an image registry with no CA certificate", + configs: []containerdConfig{ + { + URL: "https://mymirror.com", + CACert: "mymirrorcert", + Mirror: true, + }, + { + URL: "https://myregistry.com", + }, + }, + want: &cabpkv1.File{ + Path: "/etc/containerd/certs.d/_default/hosts.toml", + Owner: "", + Permissions: "0600", + Encoding: "", + Append: false, + Content: `[host."https://mymirror.com/v2"] + capabilities = ["pull", "resolve"] + ca = "/etc/certs/mymirror.com.pem" + # don't rely on Containerd to add the v2/ suffix + # there is a bug where it is added incorrectly for mirrors with a path + override_path = true +`, + }, + wantErr: nil, + }, + { + name: "Mirror image registry with a CA and an image registry with a CA", + configs: []containerdConfig{ + { + URL: "https://mymirror.com", + CACert: "mymirrorcert", + Mirror: true, + }, + { + URL: "https://myregistry.com", + CACert: "myregistrycert", + }, + { + URL: "https://172.100.0.10:5000/myproject", + CACert: "myregistrycert", + }, + }, + want: &cabpkv1.File{ + Path: "/etc/containerd/certs.d/_default/hosts.toml", + Owner: "", + Permissions: "0600", + Encoding: "", + Append: false, + Content: `[host."https://mymirror.com/v2"] + capabilities = ["pull", "resolve"] + ca = "/etc/certs/mymirror.com.pem" + # don't rely on Containerd to add the v2/ suffix + # there is a bug where it is added incorrectly for mirrors with a path + override_path = true +[host."https://myregistry.com/v2"] + ca = "/etc/certs/myregistry.com.pem" +[host."https://172.100.0.10:5000/v2/myproject"] + ca = "/etc/certs/172.100.0.10:5000-myproject.pem" +`, + }, + wantErr: nil, + }, + { + name: "Image registry with a CA", + configs: []containerdConfig{ + { + URL: "https://myregistry.com", + CACert: "myregistrycert", + }, + }, + want: &cabpkv1.File{ + Path: "/etc/containerd/certs.d/_default/hosts.toml", + Owner: "", + Permissions: "0600", + Encoding: "", + Append: false, + Content: `[host."https://myregistry.com/v2"] + ca = "/etc/certs/myregistry.com.pem" +`, + }, + wantErr: nil, + }, + } + for idx := range tests { + tt := tests[idx] + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + file, err := generateContainerdHostsFile(tt.configs) + require.ErrorIs(t, err, tt.wantErr) + assert.Equal(t, tt.want, file) + }) + } +} + +func Test_generateRegistryCACertFiles(t *testing.T) { + t.Parallel() + tests := []struct { + name string + configs []containerdConfig + want []cabpkv1.File + }{ + { + name: "ECR mirror image registry with no CA certificate", + configs: []containerdConfig{ + { + URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", + Mirror: true, + }, + }, + want: nil, + }, + { + name: "Mirror image registry with CA certificate", + configs: []containerdConfig{ + { + URL: "https://registry.example.com", + CASecretName: "my-registry-credentials-secret", + Mirror: true, + }, + }, + want: []cabpkv1.File{ + { + Path: "/etc/certs/registry.example.com.pem", + Owner: "", + Permissions: "0600", + Encoding: "", + Append: false, + ContentFrom: &cabpkv1.FileSource{ + Secret: cabpkv1.SecretFileSource{ + Name: "my-registry-credentials-secret", + Key: "ca.crt", + }, + }, + }, + }, + }, + } + for idx := range tests { + tt := tests[idx] + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + file, err := generateRegistryCACertFiles(tt.configs) + require.NoError(t, err) + assert.Equal(t, tt.want, file) + }) + } +} + +func Test_generateContainerdRegistryConfigDropInFile(t *testing.T) { + want := []cabpkv1.File{ + { + Path: "/etc/caren/containerd/patches/registry-config.toml", + Owner: "", + Permissions: "0600", + Encoding: "", + Append: false, + Content: `[plugins."io.containerd.grpc.v1.cri".registry] + config_path = "/etc/containerd/certs.d" +`, + }, + } + file := generateContainerdRegistryConfigDropInFile() + assert.Equal(t, want, file) +} diff --git a/pkg/handlers/generic/mutation/mirrors/inject.go b/pkg/handlers/generic/mutation/mirrors/inject.go index 517e66817..095246071 100644 --- a/pkg/handlers/generic/mutation/mirrors/inject.go +++ b/pkg/handlers/generic/mutation/mirrors/inject.go @@ -5,6 +5,7 @@ package mirrors import ( "context" + "fmt" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -20,6 +21,7 @@ import ( "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches/selectors" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/variables" + handlersutils "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/utils" ) type globalMirrorPatchHandler struct { @@ -35,7 +37,7 @@ func NewPatch( return newGlobalMirrorPatchHandler( cl, v1alpha1.ClusterConfigVariableName, - GlobalMirrorVariableName, + v1alpha1.GlobalMirrorVariableName, ) } @@ -66,47 +68,75 @@ func (h *globalMirrorPatchHandler) Mutate( "holderRef", holderRef, ) - globalMirror, err := variables.Get[v1alpha1.GlobalImageRegistryMirror]( + globalMirror, globalMirrorErr := variables.Get[v1alpha1.GlobalImageRegistryMirror]( vars, h.variableName, h.variableFieldPath..., ) - if err != nil { - if variables.IsNotFoundError(err) { - log.V(5).Info("Global registry mirror variable not defined") - return nil + + // add CA certificate for image registries + imageRegistries, imageRegistriesErr := variables.Get[[]v1alpha1.ImageRegistry]( + vars, + h.variableName, + v1alpha1.ImageRegistriesVariableName, + ) + + switch { + case variables.IsNotFoundError(imageRegistriesErr) && variables.IsNotFoundError(globalMirrorErr): + log.V(5).Info("Image Registry Credentials and Global Registry Mirror variable not defined") + return nil + case imageRegistriesErr != nil && !variables.IsNotFoundError(imageRegistriesErr): + return imageRegistriesErr + case globalMirrorErr != nil && !variables.IsNotFoundError(globalMirrorErr): + return globalMirrorErr + } + + var registriesWithOptionalCA []containerdConfig //nolint:prealloc // We don't know the size of the slice yet. + if globalMirrorErr == nil { + registryConfig, err := containerdConfigFromGlobalMirror( + ctx, + h.client, + globalMirror, + obj, + ) + if err != nil { + return err } - return err + registriesWithOptionalCA = append(registriesWithOptionalCA, registryConfig) } + for _, imageRegistry := range imageRegistries { + registryWithOptionalCredentials, generateErr := containerdConfigFromImageRegistry( + ctx, + h.client, + imageRegistry, + obj, + ) + if generateErr != nil { + return generateErr + } - log = log.WithValues( - "variableName", - h.variableName, - "variableFieldPath", - h.variableFieldPath, - "variableValue", - globalMirror, + registriesWithOptionalCA = append( + registriesWithOptionalCA, + registryWithOptionalCredentials, + ) + } + + needConfiguration := needContainerdConfiguration( + registriesWithOptionalCA, ) + if !needConfiguration { + log.V(5).Info("Only Image Registry Configuration is defined but without CA certificates") + return nil + } + + files, err := generateFiles(registriesWithOptionalCA) + if err != nil { + return err + } if err := patches.MutateIfApplicable( obj, vars, &holderRef, selectors.ControlPlane(), log, func(obj *controlplanev1.KubeadmControlPlaneTemplate) error { - mirrorConfig, err := mirrorConfigForGlobalMirror( - ctx, - h.client, - globalMirror, - obj, - ) - if err != nil { - return err - } - files, generateErr := generateFilesAndCommands( - mirrorConfig, - globalMirror) - if generateErr != nil { - return generateErr - } - log.WithValues( "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), @@ -124,22 +154,6 @@ func (h *globalMirrorPatchHandler) Mutate( if err := patches.MutateIfApplicable( obj, vars, &holderRef, selectors.WorkersKubeadmConfigTemplateSelector(), log, func(obj *bootstrapv1.KubeadmConfigTemplate) error { - mirrorConfig, err := mirrorConfigForGlobalMirror( - ctx, - h.client, - globalMirror, - obj, - ) - if err != nil { - return err - } - files, generateErr := generateFilesAndCommands( - mirrorConfig, - globalMirror) - if generateErr != nil { - return generateErr - } - log.WithValues( "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), @@ -154,21 +168,106 @@ func (h *globalMirrorPatchHandler) Mutate( return nil } -func generateFilesAndCommands( - mirrorConfig *mirrorConfig, +func containerdConfigFromGlobalMirror( + ctx context.Context, + c ctrlclient.Client, globalMirror v1alpha1.GlobalImageRegistryMirror, + obj ctrlclient.Object, +) (containerdConfig, error) { + configWithOptionalCACert := containerdConfig{ + URL: globalMirror.URL, + Mirror: true, + } + secret, err := handlersutils.SecretForImageRegistryCredentials( + ctx, + c, + globalMirror.Credentials, + obj.GetNamespace(), + ) + if err != nil { + return containerdConfig{}, fmt.Errorf( + "error getting secret %s/%s from Global Image Registry Mirror variable: %w", + obj.GetNamespace(), + globalMirror.Credentials.SecretRef.Name, + err, + ) + } + + if secret != nil { + configWithOptionalCACert.CASecretName = secret.Name + configWithOptionalCACert.CACert = string(secret.Data[secretKeyForMirrorCACert]) + } + + return configWithOptionalCACert, nil +} + +func containerdConfigFromImageRegistry( + ctx context.Context, + c ctrlclient.Client, + imageRegistry v1alpha1.ImageRegistry, + obj ctrlclient.Object, +) (containerdConfig, error) { + configWithOptionalCACert := containerdConfig{ + URL: imageRegistry.URL, + } + secret, err := handlersutils.SecretForImageRegistryCredentials( + ctx, + c, + imageRegistry.Credentials, + obj.GetNamespace(), + ) + if err != nil { + return containerdConfig{}, fmt.Errorf( + "error getting secret %s/%s from Image Registry variable: %w", + obj.GetNamespace(), + imageRegistry.Credentials.SecretRef.Name, + err, + ) + } + + if secret != nil { + configWithOptionalCACert.CASecretName = secret.Name + configWithOptionalCACert.CACert = string(secret.Data[secretKeyForMirrorCACert]) + } + + return configWithOptionalCACert, nil +} + +func generateFiles( + registriesWithOptionalCA []containerdConfig, ) ([]bootstrapv1.File, error) { + var files []bootstrapv1.File // generate default registry mirror file - files, err := generateGlobalRegistryMirrorFile(mirrorConfig) + containerdHostsFile, err := generateContainerdHostsFile(registriesWithOptionalCA) if err != nil { return nil, err } + if containerdHostsFile != nil { + files = append(files, *containerdHostsFile) + } + // generate CA certificate file for registry mirror - mirrorCAFile := generateMirrorCACertFile(mirrorConfig, globalMirror) - files = append(files, mirrorCAFile...) + mirrorCAFiles, err := generateRegistryCACertFiles(registriesWithOptionalCA) + if err != nil { + return nil, err + } + files = append(files, mirrorCAFiles...) + // generate Containerd registry config drop-in file registryConfigDropIn := generateContainerdRegistryConfigDropInFile() files = append(files, registryConfigDropIn...) return files, err } + +// This handler reads input from two user provided variables: globalImageRegistryMirror and imageRegistries. +// The handler will be used to either add configuration for a global mirror or CA certificates for image registries. +func needContainerdConfiguration(configs []containerdConfig) bool { + for _, config := range configs { + if config.needContainerdConfiguration() { + return true + } + } + + return false +} diff --git a/pkg/handlers/generic/mutation/mirrors/inject_test.go b/pkg/handlers/generic/mutation/mirrors/inject_test.go index e4decf69c..44b526fd1 100644 --- a/pkg/handlers/generic/mutation/mirrors/inject_test.go +++ b/pkg/handlers/generic/mutation/mirrors/inject_test.go @@ -8,6 +8,7 @@ import ( . "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" + "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apiserver/pkg/storage/names" @@ -56,7 +57,7 @@ var _ = Describe("Generate Global mirror patches", func() { v1alpha1.GlobalImageRegistryMirror{ URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", }, - GlobalMirrorVariableName, + v1alpha1.GlobalMirrorVariableName, ), }, RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""), @@ -88,7 +89,7 @@ var _ = Describe("Generate Global mirror patches", func() { }, }, }, - GlobalMirrorVariableName, + v1alpha1.GlobalMirrorVariableName, ), }, RequestItem: request.NewKubeadmControlPlaneTemplateRequest("", cpRegistryAsMirrorCreds), @@ -101,7 +102,7 @@ var _ = Describe("Generate Global mirror patches", func() { "path", "/etc/containerd/certs.d/_default/hosts.toml", ), gomega.HaveKeyWithValue( - "path", "/etc/certs/mirror.pem", + "path", "/etc/certs/registry.example.com.pem", ), gomega.HaveKeyWithValue( "path", "/etc/caren/containerd/patches/registry-config.toml", @@ -118,7 +119,7 @@ var _ = Describe("Generate Global mirror patches", func() { v1alpha1.GlobalImageRegistryMirror{ URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", }, - GlobalMirrorVariableName, + v1alpha1.GlobalMirrorVariableName, ), capitest.VariableWithValue( "builtin", @@ -158,7 +159,7 @@ var _ = Describe("Generate Global mirror patches", func() { }, }, }, - GlobalMirrorVariableName, + v1alpha1.GlobalMirrorVariableName, ), capitest.VariableWithValue( "builtin", @@ -179,7 +180,7 @@ var _ = Describe("Generate Global mirror patches", func() { "path", "/etc/containerd/certs.d/_default/hosts.toml", ), gomega.HaveKeyWithValue( - "path", "/etc/certs/mirror.pem", + "path", "/etc/certs/registry.example.com.pem", ), gomega.HaveKeyWithValue( "path", "/etc/caren/containerd/patches/registry-config.toml", @@ -236,3 +237,79 @@ func newMirrorSecret(name, namespace string) *corev1.Secret { Type: corev1.SecretTypeOpaque, } } + +func Test_needContainerdConfiguration(t *testing.T) { + t.Parallel() + tests := []struct { + name string + configs []containerdConfig + want bool + }{ + { + name: "ECR mirror image registry with no CA certificate", + configs: []containerdConfig{ + { + URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", + Mirror: true, + }, + }, + want: true, + }, + { + name: "ECR mirror image registry with a path and no CA certificate", + configs: []containerdConfig{ + { + URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com/myproject", + Mirror: true, + }, + }, + want: true, + }, + { + name: "Mirror image registry with a CA and an image registry with no CA certificate", + configs: []containerdConfig{ + { + URL: "https://mymirror.com", + CACert: "mymirrorcert", + Mirror: true, + }, + { + URL: "https://myregistry.com", + }, + }, + want: true, + }, + { + name: "Mirror image registry with a CA and an image registry with a CA", + configs: []containerdConfig{ + { + URL: "https://mymirror.com", + CACert: "mymirrorcert", + Mirror: true, + }, + { + URL: "https://myregistry.com", + CACert: "myregistrycert", + }, + }, + want: true, + }, + { + name: "Image registry with no CA certificate", + configs: []containerdConfig{ + { + URL: "https://myregistry.com", + }, + }, + want: false, + }, + } + for idx := range tests { + tt := tests[idx] + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := needContainerdConfiguration(tt.configs) + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/pkg/handlers/generic/mutation/mirrors/mirror.go b/pkg/handlers/generic/mutation/mirrors/mirror.go deleted file mode 100644 index e536773c3..000000000 --- a/pkg/handlers/generic/mutation/mirrors/mirror.go +++ /dev/null @@ -1,185 +0,0 @@ -// Copyright 2023 Nutanix. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package mirrors - -import ( - "bytes" - "context" - _ "embed" - "fmt" - "net/url" - "path" - "text/template" - - corev1 "k8s.io/api/core/v1" - cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" - ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" - "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/common" -) - -const ( - mirrorCACertPathOnRemote = "/etc/certs/mirror.pem" - defaultRegistryMirrorConfigPathOnRemote = "/etc/containerd/certs.d/_default/hosts.toml" - secretKeyForMirrorCACert = "ca.crt" -) - -var ( - //go:embed templates/hosts.toml.gotmpl - defaultRegistryMirrorPatch []byte - - defaultRegistryMirrorPatchTemplate = template.Must( - template.New("").Parse(string(defaultRegistryMirrorPatch)), - ) - - //go:embed templates/containerd-registry-config-drop-in.toml - containerdRegistryConfigDropIn []byte - containerdRegistryConfigDropInFileOnRemote = common.ContainerdPatchPathOnRemote( - "registry-config.toml", - ) -) - -type mirrorConfig struct { - URL string - CACert string -} - -func mirrorConfigForGlobalMirror( - ctx context.Context, - c ctrlclient.Client, - globalMirror v1alpha1.GlobalImageRegistryMirror, - obj ctrlclient.Object, -) (*mirrorConfig, error) { - mirrorWithOptionalCACert := &mirrorConfig{ - URL: globalMirror.URL, - } - secret, err := secretForMirrorCACert( - ctx, - c, - globalMirror, - obj.GetNamespace(), - ) - if err != nil { - return &mirrorConfig{}, fmt.Errorf( - "error getting secret %s/%s from Global Image Registry Mirror variable: %w", - obj.GetNamespace(), - globalMirror.Credentials.SecretRef.Name, - err, - ) - } - - if secret != nil { - mirrorWithOptionalCACert.CACert = string(secret.Data[secretKeyForMirrorCACert]) - } - - return mirrorWithOptionalCACert, nil -} - -// secretForMirrorCACert returns the Secret for the given mirror's CA certificate. -// Returns nil if the secret field is empty. -func secretForMirrorCACert( - ctx context.Context, - c ctrlclient.Reader, - globalMirror v1alpha1.GlobalImageRegistryMirror, - objectNamespace string, -) (*corev1.Secret, error) { - if globalMirror.Credentials == nil || globalMirror.Credentials.SecretRef == nil { - return nil, nil - } - - key := ctrlclient.ObjectKey{ - Name: globalMirror.Credentials.SecretRef.Name, - Namespace: objectNamespace, - } - secret := &corev1.Secret{} - err := c.Get(ctx, key, secret) - return secret, err -} - -// Default Mirror for all registries. -// Containerd configuration for global mirror will be created at /etc/containerd/certs.d/_default/hosts.toml -// The upstream registry will be automatically used after all defined mirrors have been tried. -// reference: https://github.com/containerd/containerd/blob/main/docs/hosts.md#setup-default-mirror-for-all-registries -func generateGlobalRegistryMirrorFile(mirror *mirrorConfig) ([]cabpkv1.File, error) { - if mirror == nil { - return nil, nil - } - formattedURL, err := formatURLForContainerd(mirror.URL) - if err != nil { - return nil, fmt.Errorf("failed formatting registry mirror URL for Containerd: %w", err) - } - templateInput := struct { - URL string - CACertPath string - }{ - URL: formattedURL, - } - // CA cert is optional for mirror registry. - // i.e. registry is using signed certificates. Insecure registry will not be allowed. - if mirror.CACert != "" { - templateInput.CACertPath = mirrorCACertPathOnRemote - } - - var b bytes.Buffer - err = defaultRegistryMirrorPatchTemplate.Execute(&b, templateInput) - if err != nil { - return nil, fmt.Errorf("failed executing template for registry mirror: %w", err) - } - return []cabpkv1.File{ - { - Path: defaultRegistryMirrorConfigPathOnRemote, - Content: b.String(), - Permissions: "0600", - }, - }, nil -} - -func generateMirrorCACertFile( - mirror *mirrorConfig, - globalMirror v1alpha1.GlobalImageRegistryMirror, -) []cabpkv1.File { - if mirror == nil || mirror.CACert == "" { - return nil - } - return []cabpkv1.File{ - { - Path: mirrorCACertPathOnRemote, - Permissions: "0600", - ContentFrom: &cabpkv1.FileSource{ - Secret: cabpkv1.SecretFileSource{ - Name: globalMirror.Credentials.SecretRef.Name, - Key: secretKeyForMirrorCACert, - }, - }, - }, - } -} - -func formatURLForContainerd(uri string) (string, error) { - mirrorURL, err := url.ParseRequestURI(uri) - if err != nil { - return "", fmt.Errorf("failed parsing mirror: %w", err) - } - - mirror := fmt.Sprintf("%s://%s", mirrorURL.Scheme, mirrorURL.Host) - // assume Containerd expects the following pattern: - // scheme://host/v2/path - mirrorPath := "v2" - if mirrorURL.Path != "" { - mirrorPath = path.Join(mirrorPath, mirrorURL.Path) - } - // using path.Join on all elements incorrectly drops a "/" from "https://" - return fmt.Sprintf("%s/%s", mirror, mirrorPath), nil -} - -func generateContainerdRegistryConfigDropInFile() []cabpkv1.File { - return []cabpkv1.File{ - { - Path: containerdRegistryConfigDropInFileOnRemote, - Content: string(containerdRegistryConfigDropIn), - Permissions: "0600", - }, - } -} diff --git a/pkg/handlers/generic/mutation/mirrors/mirror_test.go b/pkg/handlers/generic/mutation/mirrors/mirror_test.go deleted file mode 100644 index 574bda27b..000000000 --- a/pkg/handlers/generic/mutation/mirrors/mirror_test.go +++ /dev/null @@ -1,177 +0,0 @@ -// Copyright 2023 Nutanix. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package mirrors - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" - - "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" -) - -func Test_generateDefaultRegistryMirrorFile(t *testing.T) { - t.Parallel() - tests := []struct { - name string - config *mirrorConfig - want []cabpkv1.File - wantErr error - }{ - { - name: "ECR image registry and no CA certificate", - config: &mirrorConfig{URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com"}, - want: []cabpkv1.File{ - { - Path: "/etc/containerd/certs.d/_default/hosts.toml", - Owner: "", - Permissions: "0600", - Encoding: "", - Append: false, - Content: `[host."https://123456789.dkr.ecr.us-east-1.amazonaws.com/v2"] - capabilities = ["pull", "resolve"] - # don't rely on Containerd to add the v2/ suffix - # there is a bug where it is added incorrectly for mirrors with a path - override_path = true -`, - }, - }, - wantErr: nil, - }, - { - name: "ECR image registry with a path and no CA certificate", - config: &mirrorConfig{ - URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com/myproject", - }, - want: []cabpkv1.File{ - { - Path: "/etc/containerd/certs.d/_default/hosts.toml", - Owner: "", - Permissions: "0600", - Encoding: "", - Append: false, - Content: `[host."https://123456789.dkr.ecr.us-east-1.amazonaws.com/v2/myproject"] - capabilities = ["pull", "resolve"] - # don't rely on Containerd to add the v2/ suffix - # there is a bug where it is added incorrectly for mirrors with a path - override_path = true -`, - }, - }, - wantErr: nil, - }, - { - name: "image registry with CA certificates", - config: &mirrorConfig{ - URL: "https://myregistry.com", - CACert: "mycacert", - }, - want: []cabpkv1.File{ - { - Path: "/etc/containerd/certs.d/_default/hosts.toml", - Owner: "", - Permissions: "0600", - Encoding: "", - Append: false, - Content: `[host."https://myregistry.com/v2"] - capabilities = ["pull", "resolve"] - ca = "/etc/certs/mirror.pem" - # don't rely on Containerd to add the v2/ suffix - # there is a bug where it is added incorrectly for mirrors with a path - override_path = true -`, - }, - }, - wantErr: nil, - }, - } - for idx := range tests { - tt := tests[idx] - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - file, err := generateGlobalRegistryMirrorFile(tt.config) - require.ErrorIs(t, err, tt.wantErr) - assert.Equal(t, tt.want, file) - }) - } -} - -func Test_generateMirrorCACertFile(t *testing.T) { - t.Parallel() - tests := []struct { - name string - config *mirrorConfig - registry v1alpha1.GlobalImageRegistryMirror - want []cabpkv1.File - }{ - { - name: "Mirror registry with no CA certificate", - config: &mirrorConfig{ - URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", - }, - registry: v1alpha1.GlobalImageRegistryMirror{ - URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com", - }, - want: nil, - }, - { - name: "Mirror registry with CA certificate", - config: &mirrorConfig{ - URL: "https://myregistry.com", - CACert: "mycacert", - }, - registry: v1alpha1.GlobalImageRegistryMirror{ - URL: "https://registry.example.com", - - Credentials: &v1alpha1.RegistryCredentials{ - SecretRef: &v1alpha1.LocalObjectReference{ - Name: "my-registry-credentials-secret", - }, - }, - }, - want: []cabpkv1.File{ - { - Path: "/etc/certs/mirror.pem", - Owner: "", - Permissions: "0600", - Encoding: "", - Append: false, - ContentFrom: &cabpkv1.FileSource{ - Secret: cabpkv1.SecretFileSource{ - Name: "my-registry-credentials-secret", - Key: "ca.crt", - }, - }, - }, - }, - }, - } - for idx := range tests { - tt := tests[idx] - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - file := generateMirrorCACertFile(tt.config, tt.registry) - assert.Equal(t, tt.want, file) - }) - } -} - -func Test_generateContainerdRegistryConfigDropInFile(t *testing.T) { - want := []cabpkv1.File{ - { - Path: "/etc/caren/containerd/patches/registry-config.toml", - Owner: "", - Permissions: "0600", - Encoding: "", - Append: false, - Content: `[plugins."io.containerd.grpc.v1.cri".registry] - config_path = "/etc/containerd/certs.d" -`, - }, - } - file := generateContainerdRegistryConfigDropInFile() - assert.Equal(t, want, file) -} diff --git a/pkg/handlers/generic/mutation/mirrors/templates/hosts.toml.gotmpl b/pkg/handlers/generic/mutation/mirrors/templates/hosts.toml.gotmpl index 41335b8dd..c8118c14c 100644 --- a/pkg/handlers/generic/mutation/mirrors/templates/hosts.toml.gotmpl +++ b/pkg/handlers/generic/mutation/mirrors/templates/hosts.toml.gotmpl @@ -1,3 +1,5 @@ +{{- range . }} +{{- if .Mirror }} [host."{{ .URL }}"] capabilities = ["pull", "resolve"] {{- if .CACertPath }} @@ -6,3 +8,8 @@ # don't rely on Containerd to add the v2/ suffix # there is a bug where it is added incorrectly for mirrors with a path override_path = true +{{- else }} +[host."{{ .URL }}"] + ca = "{{ .CACertPath }}" +{{- end }} +{{- end }} diff --git a/pkg/handlers/utils/secrets.go b/pkg/handlers/utils/secrets.go index 3358b5276..de5200199 100644 --- a/pkg/handlers/utils/secrets.go +++ b/pkg/handlers/utils/secrets.go @@ -14,6 +14,7 @@ import ( ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/k8s/client" ) @@ -105,3 +106,34 @@ func getSecretForCluster( } return secret, c.Get(ctx, ctrlclient.ObjectKeyFromObject(secret), secret) } + +// SecretForImageRegistryCredentials returns the Secret for the given ImageRegistryCredentials. +// Returns nil if the secret field is empty. +func SecretForImageRegistryCredentials( + ctx context.Context, + c ctrlclient.Reader, + credentials *v1alpha1.RegistryCredentials, + objectNamespace string, +) (*corev1.Secret, error) { + name := SecretNameForImageRegistryCredentials(credentials) + if name == "" { + return nil, nil + } + + key := ctrlclient.ObjectKey{ + Name: name, + Namespace: objectNamespace, + } + secret := &corev1.Secret{} + err := c.Get(ctx, key, secret) + return secret, err +} + +// SecretNameForImageRegistryCredentials returns the name of the Secret for the given RegistryCredentials. +// Returns an empty string if the credentials or secret field is empty. +func SecretNameForImageRegistryCredentials(credentials *v1alpha1.RegistryCredentials) string { + if credentials == nil || credentials.SecretRef == nil || credentials.SecretRef.Name == "" { + return "" + } + return credentials.SecretRef.Name +}