refactor: add VirtualIPSpec API and a handler

Author: dkoshkin

api/v1alpha1/common_types.go

@@ -37,4 +37,10 @@ type ControlPlaneEndpointSpec struct {
 	// +kubebuilder:validation:Minimum=1
 	// +kubebuilder:validation:Maximum=65535
 	Port int32 `json:"port"`
+
+	// Configuration for the virtual IP provider.
+	// +optional
+	VirtualIPSpec *ControlPlaneVirtualIPSpec `json:"virtualIP,omitempty"`
 }
+
+type ControlPlaneVirtualIPSpec struct{}

api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml

@@ -476,6 +476,9 @@ spec:
                         maximum: 65535
                         minimum: 1
                         type: integer
+                      virtualIP:
+                        description: Configuration for the virtual IP provider.
+                        type: object
                     required:
                     - host
                     - port

api/v1alpha1/zz_generated.deepcopy.go

@@ -22,6 +22,7 @@ spec:
             controlPlaneEndpoint:
               host: x.x.x.x
               port: 6443
+              virtualIP: {}
 ```
 
 Applying this configuration will result in the following value being set:
@@ -36,3 +37,22 @@ spec:
         host: x.x.x.x
         port: 6443
 ```
+
+- `KubeadmControlPlaneTemplate`
+
+```yaml
+  spec:
+    kubeadmConfigSpec:
+      files:
+      - content: |
+          apiVersion: v1
+          kind: Pod
+          metadata:
+            name: kube-vip
+            namespace: kube-system
+          spec:
+          ...
+        owner: root:root
+        path: /etc/kubernetes/manifests/kube-vip.yaml
+        permissions: "0600"
+```

docs/content/customization/nutanix/control-plane-endpoint.md

@@ -112,6 +112,7 @@ spec:
           controlPlaneEndpoint:
             host: ${CONTROL_PLANE_ENDPOINT_IP}
             port: 6443
+            virtualIP: {}
           prismCentralEndpoint:
             credentials:
               name: ${CLUSTER_NAME}-pc-creds

examples/capi-quick-start/nutanix-cluster-calico-crs.yaml

@@ -112,6 +112,7 @@ spec:
           controlPlaneEndpoint:
             host: ${CONTROL_PLANE_ENDPOINT_IP}
             port: 6443
+            virtualIP: {}
           prismCentralEndpoint:
             credentials:
               name: ${CLUSTER_NAME}-pc-creds

examples/capi-quick-start/nutanix-cluster-calico-helm-addon.yaml

@@ -112,6 +112,7 @@ spec:
           controlPlaneEndpoint:
             host: ${CONTROL_PLANE_ENDPOINT_IP}
             port: 6443
+            virtualIP: {}
           prismCentralEndpoint:
             credentials:
               name: ${CLUSTER_NAME}-pc-creds

examples/capi-quick-start/nutanix-cluster-cilium-crs.yaml

@@ -112,6 +112,7 @@ spec:
           controlPlaneEndpoint:
             host: ${CONTROL_PLANE_ENDPOINT_IP}
             port: 6443
+            virtualIP: {}
           prismCentralEndpoint:
             credentials:
               name: ${CLUSTER_NAME}-pc-creds

examples/capi-quick-start/nutanix-cluster-cilium-helm-addon.yaml

@@ -39,7 +39,7 @@ patches:
       path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/certSANs"
 
 # Delete the kube-vip file.
-# Will be templated and added back in the handler.
+# Will be templated and added back in the handler if enabled.
 - target:
     kind: KubeadmControlPlaneTemplate
   patch: |-

hack/examples/bases/nutanix/clusterclass/kustomization.yaml.tmpl

@@ -10,6 +10,7 @@
     controlPlaneEndpoint:
       host: ${CONTROL_PLANE_ENDPOINT_IP}
       port: 6443
+      virtualIP: {}
     prismCentralEndpoint:
       url: https://${NUTANIX_ENDPOINT}:9440
       insecure: ${NUTANIX_INSECURE}

hack/examples/patches/nutanix/initialize-variables.yaml

@@ -0,0 +1,144 @@
+// Copyright 2023 D2iQ, Inc. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package controlplanevirtualip
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/spf13/pflag"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
+	runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches/selectors"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/variables"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/controlplanevirtualip/providers"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/options"
+)
+
+const (
+	// VariableName is the external patch variable name.
+	VariableName = "controlPlaneEndpoint"
+)
+
+type Config struct {
+	*options.GlobalOptions
+
+	defaultKubeVipConfigMapName string
+}
+
+func (c *Config) AddFlags(prefix string, flags *pflag.FlagSet) {
+	flags.StringVar(
+		&c.defaultKubeVipConfigMapName,
+		prefix+".default-kube-vip-template-configmap-name",
+		"default-kube-vip-template",
+		"default ConfigMap name that holds the kube-vip template used for the control-plane virtual IP",
+	)
+}
+
+type ControlPlaneVirtualIP struct {
+	client client.Reader
+	config *Config
+
+	variableName      string
+	variableFieldPath []string
+}
+
+// NewControlPlaneVirtualIP is different from other generic handlers.
+// It requires variableName and variableFieldPath to be passed from another provider specific handler.
+// The code is here to be shared across different providers.
+func NewControlPlaneVirtualIP(
+	cl client.Reader,
+	config *Config,
+	variableName string,
+	variableFieldPath ...string,
+) *ControlPlaneVirtualIP {
+	return &ControlPlaneVirtualIP{
+		client:            cl,
+		config:            config,
+		variableName:      variableName,
+		variableFieldPath: variableFieldPath,
+	}
+}
+
+func (h *ControlPlaneVirtualIP) Mutate(
+	ctx context.Context,
+	obj *unstructured.Unstructured,
+	vars map[string]apiextensionsv1.JSON,
+	holderRef runtimehooksv1.HolderReference,
+	_ client.ObjectKey,
+	_ mutation.ClusterGetter,
+) error {
+	log := ctrl.LoggerFrom(ctx).WithValues(
+		"holderRef", holderRef,
+	)
+
+	controlPlaneEndpointVar, err := variables.Get[v1alpha1.ControlPlaneEndpointSpec](
+		vars,
+		h.variableName,
+		h.variableFieldPath...,
+	)
+	if err != nil {
+		if variables.IsNotFoundError(err) {
+			log.V(5).Info("ControlPlaneEndpoint variable not defined")
+			return nil
+		}
+		return err
+	}
+
+	log = log.WithValues(
+		"variableName",
+		h.variableName,
+		"variableFieldPath",
+		h.variableFieldPath,
+		"variableValue",
+		controlPlaneEndpointVar,
+	)
+
+	if controlPlaneEndpointVar.VirtualIPSpec == nil {
+		log.V(5).Info("ControlPlane VirtualIP not set")
+		return nil
+	}
+
+	// only kube-vip is supported, but more providers can be added in the future
+	virtualIPProvider := providers.NewKubeVIPFromConfigMapProvider(
+		h.client,
+		h.config.defaultKubeVipConfigMapName,
+		h.config.DefaultsNamespace(),
+	)
+
+	return patches.MutateIfApplicable(
+		obj,
+		vars,
+		&holderRef,
+		selectors.ControlPlane(),
+		log,
+		func(obj *controlplanev1.KubeadmControlPlaneTemplate) error {
+			virtualIPProviderFile, getFileErr := virtualIPProvider.GetFile(ctx, controlPlaneEndpointVar)
+			if getFileErr != nil {
+				return getFileErr
+			}
+
+			log.WithValues(
+				"patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(),
+				"patchedObjectName", client.ObjectKeyFromObject(obj),
+			).Info(fmt.Sprintf(
+				"adding %s static Pod file to control plane kubeadm config spec",
+				virtualIPProvider.Name(),
+			))
+			obj.Spec.Template.Spec.KubeadmConfigSpec.Files = append(
+				obj.Spec.Template.Spec.KubeadmConfigSpec.Files,
+				*virtualIPProviderFile,
+			)
+			return nil
+		},
+	)
+}