feat: add imageRegistryCredentials handler

Author: dkoshkin

api/v1alpha1/clusterconfig_types.go

@@ -28,6 +28,10 @@ type GenericClusterConfig struct {
 	// +optional
 	ExtraAPIServerCertSANs ExtraAPIServerCertSANs `json:"extraAPIServerCertSANs,omitempty"`
 
+	// TODO: Add support for multiple registries.
+	// +optional
+	ImageRegistryCredentials ImageRegistryCredentials `json:"imageRegistryCredentials,omitempty"`
+
 	// +optional
 	Addons *Addons `json:"addons,omitempty"`
 }
@@ -46,6 +50,7 @@ func (GenericClusterConfig) VariableSchema() clusterv1.VariableSchema {
 					"",
 				).VariableSchema().
 					OpenAPIV3Schema,
+				"imageRegistryCredentials": ImageRegistryCredentials{}.VariableSchema().OpenAPIV3Schema,
 			},
 		},
 	}
@@ -175,6 +180,40 @@ func (ExtraAPIServerCertSANs) VariableSchema() clusterv1.VariableSchema {
 	}
 }
 
+// ImageRegistryCredentials required for providing credentials for an image registry URL.
+type ImageRegistryCredentials struct {
+	// Registry URL.
+	URL string `json:"url"`
+
+	// Name of the Secret containing the registry credentials.
+	// The Secret should have keys 'username' and 'password'.
+	// This credentials Secret is not required for some registries, e.g. ECR.
+	// +optional
+	Secret string `json:"secret,omitempty"`
+}
+
+func (ImageRegistryCredentials) VariableSchema() clusterv1.VariableSchema {
+	return clusterv1.VariableSchema{
+		OpenAPIV3Schema: clusterv1.JSONSchemaProps{
+			Description: "Extra Subject Alternative Names for the API Server signing cert",
+			Type:        "object",
+			Properties: map[string]clusterv1.JSONSchemaProps{
+				"url": {
+					Description: "Registry URL.",
+					Type:        "string",
+				},
+				"secret": {
+					Description: "Name of the Secret containing the registry credentials. " +
+						"The Secret should have keys 'username' and 'password'. " +
+						"This credentials Secret is not required for some registries, e.g. ECR.",
+					Type: "string",
+				},
+			},
+			Required: []string{"url"},
+		},
+	}
+}
+
 type Addons struct {
 	// +optional
 	CNI *CNI `json:"cni,omitempty"`

api/v1alpha1/zz_generated.deepcopy.go

@@ -23,8 +23,10 @@ rules:
   resources:
   - secrets
   verbs:
+  - create
   - get
   - list
+  - patch
   - watch
 - apiGroups:
   - addons.cluster.x-k8s.io

charts/capi-runtime-extensions/templates/role.yaml

@@ -37,6 +37,7 @@ import (
 	"github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/etcd"
 	"github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/extraapiservercertsans"
 	"github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/httpproxy"
+	"github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/imageregistrycredentials"
 	"github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/kubernetesimagerepository"
 )
 
@@ -133,6 +134,7 @@ func main() {
 		extraapiservercertsans.NewMetaPatch(),
 		httpproxy.NewMetaPatch(mgr.GetClient()),
 		kubernetesimagerepository.NewMetaPatch(),
+		imageregistrycredentials.NewMetaPatch(mgr.GetClient()),
 	}
 
 	// awsMetaPatchHandlers combines all AWS patch and variable handlers under a single handler.

cmd/main.go

@@ -11,6 +11,7 @@ require (
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.8.4
 	gomodules.xyz/jsonpatch/v2 v2.4.0
+	k8s.io/api v0.28.2
 	k8s.io/apiextensions-apiserver v0.28.2
 	k8s.io/apimachinery v0.28.2
 	k8s.io/utils v0.0.0-20230726121419-3b25d923346b
@@ -74,7 +75,6 @@ require (
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/api v0.28.2 // indirect
 	k8s.io/apiserver v0.28.2 // indirect
 	k8s.io/client-go v0.28.2 // indirect
 	k8s.io/cluster-bootstrap v0.27.2 // indirect

common/go.mod

@@ -6,6 +6,7 @@ package capitest
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"testing"
 
 	"github.com/onsi/gomega"
@@ -55,7 +56,7 @@ func ValidateGeneratePatches[T mutation.GeneratePatches](
 			}
 			resp := &runtimehooksv1.GeneratePatchesResponse{}
 			h.GeneratePatches(context.Background(), req, resp)
-			g.Expect(resp.Status).To(gomega.Equal(runtimehooksv1.ResponseStatusSuccess))
+			g.Expect(resp.Status).To(gomega.Equal(runtimehooksv1.ResponseStatusSuccess), fmt.Sprintf("Message: %s", resp.Message))
 
 			if len(tt.ExpectedPatchMatchers) == 0 {
 				g.Expect(resp.Items).To(gomega.BeEmpty())

common/pkg/testutils/capitest/patches.go

@@ -4,6 +4,7 @@
 package request
 
 import (
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
@@ -15,6 +16,13 @@ import (
 	"github.com/d2iq-labs/capi-runtime-extensions/common/pkg/testutils/capitest/serializer"
 )
 
+const (
+	ClusterName                                  = "test-cluster"
+	KubeadmConfigTemplateRequestObjectName       = "test-kubeadmconfigtemplate"
+	KubeadmControlPlaneTemplateRequestObjectName = "test-kubeadmcontrolplanetemplate"
+	Namespace                                    = corev1.NamespaceDefault
+)
+
 // NewRequestItem returns a GeneratePatchesRequestItem with the given variables and object.
 func NewRequestItem(
 	object runtime.Object,
@@ -41,6 +49,10 @@ func NewKubeadmConfigTemplateRequestItem(uid types.UID) runtimehooksv1.GenerateP
 				Kind:       "KubeadmConfigTemplate",
 				APIVersion: bootstrapv1.GroupVersion.String(),
 			},
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      KubeadmConfigTemplateRequestObjectName,
+				Namespace: Namespace,
+			},
 			Spec: bootstrapv1.KubeadmConfigTemplateSpec{
 				Template: bootstrapv1.KubeadmConfigTemplateResource{
 					Spec: bootstrapv1.KubeadmConfigSpec{
@@ -66,10 +78,16 @@ func NewKubeadmControlPlaneTemplateRequestItem(
 				Kind:       "KubeadmControlPlaneTemplate",
 				APIVersion: controlplanev1.GroupVersion.String(),
 			},
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      KubeadmControlPlaneTemplateRequestObjectName,
+				Namespace: Namespace,
+			},
 		},
 		&runtimehooksv1.HolderReference{
 			Kind:      "Cluster",
 			FieldPath: "spec.controlPlaneRef",
+			Name:      ClusterName,
+			Namespace: Namespace,
 		},
 		uid,
 	)

common/pkg/testutils/capitest/request/items.go

@@ -38,6 +38,6 @@ configure defaults specific for their environment rather than compiling the defa
 The Helm chart comes with default configurations for the Calico Installation CRS per supported provider, but overriding
 is possible. To do so, specify:
 
-```bash
+```shell
 --set-file handlers.CalicoCNI.defaultInstallationConfigMaps.DockerCluster.configMap.content=<file>
 ```

docs/content/addons/calico-cni.md

@@ -39,6 +39,9 @@ spec:
             additionalNo:
               - no-proxy-1.example.com
               - no-proxy-2.example.com
+          imageRegistryCredentials:
+            url: https://my-registry.io
+            secret: my-registry-credentials
           cni:
             provider: calico
 ```

docs/content/customization/_index.md

@@ -0,0 +1,44 @@
++++
+title = "Image registry credentials"
++++
+
+In some network environments it is necessary to use HTTP proxy to successfuly execute HTTP requests.
+To configure Kubernetes components (`containerd`, `kubelet`) to use HTTP proxy use the `httpproxypatch`
+external patch that will generate appropriate configuration for control plane and worker nodes.
+
+Add image registry credentials to all Nodes in the cluster.
+When this handle is enabled, the handler will add `files` and `preKubeadmnCommands` with configurations for
+[Kubelet image credential provider](https://kubernetes.io/docs/tasks/administer-cluster/kubelet-credential-provider/)
+and [dynamic credential provider](https://github.com/mesosphere/dynamic-credential-provider).
+
+This customization will be available when the
+[provider-specific cluster configuration patch]({{< ref "..">}}) is included in the `ClusterClass`.
+
+## Example
+
+If your registry requires static credentials, create a Kubernetes Secret with keys for `username` and `password`:
+
+```shell
+kubectl create secret generic my-registry-credentials \
+  --from-literal username=${REGISTRY_USERNAME} password=${REGISTRY_PASSWORD}
+```
+
+On the cluster resource then specify desired image registry credentials:
+
+```yaml
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  name: <NAME>
+spec:
+  topology:
+    variables:
+      - name: clusterConfig
+        value:
+          imageRegistryCredentials:
+            url: https://my-registry.io
+            secret: my-registry-credentials
+```
+
+Applying this configuration will result in new files and preKubeadmCommands
+on the `KubeadmControlPlaneTemplate` and `KubeadmConfigTemplate`.

docs/content/customization/generic/image-registry-credentials.md

@@ -20,6 +20,7 @@ require (
 	k8s.io/client-go v0.28.2
 	k8s.io/component-base v0.28.2
 	k8s.io/klog/v2 v2.100.1
+	k8s.io/kubelet v0.28.2
 	k8s.io/utils v0.0.0-20230726121419-3b25d923346b
 	sigs.k8s.io/cluster-api v1.5.1
 	sigs.k8s.io/controller-runtime v0.16.2

go.mod

@@ -512,6 +512,8 @@ k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg=
 k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
 k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ=
 k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM=
+k8s.io/kubelet v0.28.2 h1:wqe5zKtVhNWwtdABU0mpcWVe8hc6VdVvs2kqQridZRw=
+k8s.io/kubelet v0.28.2/go.mod h1:rvd0e7T5TjPcfZvy62P90XhFzp0IhPIOy+Pqy3Rtipo=
 k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI=
 k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=