fix: image registries not handling CA certificates (#729)

**What problem does this PR solve?**:
While working on
#724
I noticed that setting imageRegistry CA certificate had no effect on the
generated configuration.

I structured `/mirrors/inject.go` similar to
`/imageregistries/credentials/inject.go` where it reads both the
`globalImageRegistryMirror` and `imageRegistries` variables and handles
them together when generating the necessary files.

**Which issue(s) this PR fixes**:
Fixes #

**How Has This Been Tested?**:
<!--
Please describe the tests that you ran to verify your changes.
Provide output from the tests and any manual steps needed to replicate
the tests.
-->
```
# Tested by setting an imageRegistries. The mirror Pod failure is expected because the cluster is not configured as a mirror
$ kubectl get pods
NAME                                                 READY   STATUS              RESTARTS   AGE
cluster-autoscaler-docker-creds-1-77c5df58c4-gs6m2   0/1     ContainerCreating   0          94s
nginx-latest                                         1/1     Running             0          10s
nginx-mirror                                         0/1     ErrImagePull        0          10s
nginx-stable                                         1/1     Running             0          10s

# Tested by setting an globalImageRegistryMirror
$ kubectl get pods
NAME                                                 READY   STATUS              RESTARTS   AGE
cluster-autoscaler-docker-creds-1-77c5df58c4-nncdc   0/1     ContainerCreating   0          22m
nginx-latest                                         1/1     Running             0          9s
nginx-mirror                                         1/1     Running             0          9s
nginx-stable                                         1/1     Running             0          9s
```

**Special notes for your reviewer**:
<!--
Use this to provide any additional information to the reviewers.
This may include:
- Best way to review the PR.
- Where the author wants the most review attention on.
- etc.
-->

Author: dkoshkin

api/v1alpha1/constants.go

@@ -28,6 +28,11 @@ const (
 	// ServiceLoadBalancerVariableName is the Service LoadBalancer config patch variable name.
 	ServiceLoadBalancerVariableName = "serviceLoadBalancer"
 
+	// GlobalMirrorVariableName is the global image registry mirror patch variable name.
+	GlobalMirrorVariableName = "globalImageRegistryMirror"
+	// ImageRegistriesVariableName is the image registries patch variable name.
+	ImageRegistriesVariableName = "imageRegistries"
+
 	// NamespaceSyncLabelKey is a label that can be applied to a namespace.
 	//
 	// When a namespace has a label with this key, ClusterClasses and their Templates are

api/v1alpha1/zz_generated.deepcopy.go

@@ -20,7 +20,24 @@ kubectl create secret generic my-registry-credentials \
   --from-literal username=${REGISTRY_USERNAME} --from-literal password=${REGISTRY_PASSWORD}
 ```
 
-To add image registry credentials, specify the following configuration:
+If your registry requires a private or self-signed CA certificate,
+create a Kubernetes Secret with the `ca.crt` key populated with the CA certificate in PEM format:
+
+```shell
+kubectl create secret generic my-mirror-ca-cert \
+  --from-file=ca.crt=registry-ca.crt
+```
+
+To set both image registry credentials and CA certificate,
+create a Kubernetes Secret with keys for `username`, `password`, and `ca.crt`:
+
+```shell
+kubectl create secret generic my-registry-credentials \
+  --from-literal username=${REGISTRY_USERNAME} --from-literal password=${REGISTRY_PASSWORD} \
+  --from-file=ca.crt=registry-ca.crt
+```
+
+To add image registry credentials and/or CA certificate, specify the following configuration:
 
 ```yaml
 apiVersion: cluster.x-k8s.io/v1beta1

docs/content/customization/generic/image-registries.md

@@ -108,7 +108,9 @@ func templateFilesForImageCredentialProviderConfigs(
 		files = append(files, *kubeletCredentialProviderConfigFile)
 	}
 
-	kubeletDynamicCredentialProviderConfigFile, err := templateDynamicCredentialProviderConfig(configs)
+	kubeletDynamicCredentialProviderConfigFile, err := templateDynamicCredentialProviderConfig(
+		configs,
+	)
 	if err != nil {
 		return nil, err
 	}

pkg/handlers/generic/mutation/imageregistries/constants.go

@@ -8,7 +8,6 @@ import (
 	"errors"
 	"fmt"
 
-	corev1 "k8s.io/api/core/v1"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -26,8 +25,6 @@ import (
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches/selectors"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/variables"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/k8s/client"
-	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/imageregistries"
-	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/mirrors"
 	handlersutils "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/utils"
 )
 
@@ -46,7 +43,7 @@ func NewPatch(
 	return newImageRegistriesPatchHandler(
 		cl,
 		v1alpha1.ClusterConfigVariableName,
-		imageregistries.VariableName,
+		v1alpha1.ImageRegistriesVariableName,
 	)
 }
 
@@ -87,12 +84,12 @@ func (h *imageRegistriesPatchHandler) Mutate(
 	globalMirror, globalMirrorErr := variables.Get[v1alpha1.GlobalImageRegistryMirror](
 		vars,
 		h.variableName,
-		mirrors.GlobalMirrorVariableName,
+		v1alpha1.GlobalMirrorVariableName,
 	)
 
 	switch {
 	case variables.IsNotFoundError(imageRegistriesErr) && variables.IsNotFoundError(globalMirrorErr):
-		log.V(5).Info("Image Registry Credentials variable not defined")
+		log.V(5).Info("Image Registry Credentials and Global Registry Mirror variable not defined")
 		return nil
 	case imageRegistriesErr != nil && !variables.IsNotFoundError(imageRegistriesErr):
 		return imageRegistriesErr
@@ -287,7 +284,7 @@ func ensureOwnerReferenceOnCredentialsSecrets(
 	}
 
 	for _, credential := range credentials {
-		if secretName := secretNameForImageRegistryCredentials(credential); secretName != "" {
+		if secretName := handlersutils.SecretNameForImageRegistryCredentials(credential); secretName != "" {
 			// Ensure the Secret is owned by the Cluster so it is correctly moved and deleted with the Cluster.
 			// This code assumes that Secret exists and that was validated before calling this function.
 			err := handlersutils.EnsureOwnerReferenceForSecret(
@@ -317,7 +314,7 @@ func registryWithOptionalCredentialsFromImageRegistryCredentials(
 	registryWithOptionalCredentials := providerConfig{
 		URL: imageRegistry.URL,
 	}
-	secret, err := secretForImageRegistryCredentials(
+	secret, err := handlersutils.SecretForImageRegistryCredentials(
 		ctx,
 		c,
 		imageRegistry.Credentials,
@@ -350,7 +347,7 @@ func mirrorConfigFromGlobalImageRegistryMirror(
 		URL:    mirror.URL,
 		Mirror: true,
 	}
-	secret, err := secretForImageRegistryCredentials(
+	secret, err := handlersutils.SecretForImageRegistryCredentials(
 		ctx,
 		c,
 		mirror.Credentials,
@@ -438,28 +435,6 @@ func createSecretIfNeeded(
 	return nil
 }
 
-// secretForImageRegistryCredentials returns the Secret for the given ImageRegistryCredentials.
-// Returns nil if the secret field is empty.
-func secretForImageRegistryCredentials(
-	ctx context.Context,
-	c ctrlclient.Reader,
-	credentials *v1alpha1.RegistryCredentials,
-	objectNamespace string,
-) (*corev1.Secret, error) {
-	name := secretNameForImageRegistryCredentials(credentials)
-	if name == "" {
-		return nil, nil
-	}
-
-	key := ctrlclient.ObjectKey{
-		Name:      name,
-		Namespace: objectNamespace,
-	}
-	secret := &corev1.Secret{}
-	err := c.Get(ctx, key, secret)
-	return secret, err
-}
-
 // This handler reads input from two user provided variables: globalImageRegistryMirror and imageRegistries.
 // We expect if imageRegistries is set it will either have static credentials
 // or be for a registry where the credential plugin returns the credentials, ie ECR, GCR, ACR, etc,
@@ -491,12 +466,3 @@ func needImageRegistryCredentialsConfiguration(configs []providerConfig) (bool,
 
 	return true, nil
 }
-
-// secretForImageRegistryCredentials returns the name of the Secret for the given RegistryCredentials.
-// Returns an empty string if the credentials or secret field is empty.
-func secretNameForImageRegistryCredentials(credentials *v1alpha1.RegistryCredentials) string {
-	if credentials == nil || credentials.SecretRef == nil || credentials.SecretRef.Name == "" {
-		return ""
-	}
-	return credentials.SecretRef.Name
-}

pkg/handlers/generic/mutation/imageregistries/credentials/credential_provider_config_files.go

@@ -26,7 +26,6 @@ import (
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest/request"
-	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/imageregistries"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/test/helpers"
 )
 
@@ -161,7 +160,7 @@ var _ = Describe("Generate Image registry patches", func() {
 						[]v1alpha1.ImageRegistry{{
 							URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com",
 						}},
-						imageregistries.VariableName,
+						v1alpha1.ImageRegistriesVariableName,
 					),
 				},
 				RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""),
@@ -221,7 +220,7 @@ var _ = Describe("Generate Image registry patches", func() {
 								},
 							},
 						}},
-						imageregistries.VariableName,
+						v1alpha1.ImageRegistriesVariableName,
 					),
 				},
 				RequestItem: request.NewKubeadmControlPlaneTemplateRequest(
@@ -283,7 +282,7 @@ var _ = Describe("Generate Image registry patches", func() {
 						[]v1alpha1.ImageRegistry{{
 							URL: "https://123456789.dkr.ecr.us-east-1.amazonaws.com",
 						}},
-						imageregistries.VariableName,
+						v1alpha1.ImageRegistriesVariableName,
 					),
 					capitest.VariableWithValue(
 						"builtin",
@@ -343,7 +342,7 @@ var _ = Describe("Generate Image registry patches", func() {
 								},
 							},
 						}},
-						imageregistries.VariableName,
+						v1alpha1.ImageRegistriesVariableName,
 					),
 					capitest.VariableWithValue(
 						"builtin",
@@ -405,7 +404,7 @@ var _ = Describe("Generate Image registry patches", func() {
 						[]v1alpha1.ImageRegistry{{
 							URL: "https://registry.example.com",
 						}},
-						imageregistries.VariableName,
+						v1alpha1.ImageRegistriesVariableName,
 					),
 				},
 				RequestItem:     request.NewKubeadmControlPlaneTemplateRequestItem(""),