feat: Virtual IP configuration to set different address/port (#986)

**What problem does this PR solve?**:
New configuration to support setting different virutalIP address from
the external controlPlaneEndpoint. This is useful for Clusters like a
Nutanix Cluster in a VPC where the virtual IP needs to be some IP from
same Subnet as the VMs, but the conrtol-plane endpoint is a floating IP
mapped to the virtual IP accessible from outside the VPC.
```
spec:
  topology:
    variables:
      - name: clusterConfig
        value:
          nutanix:
            controlPlaneEndpoint:
              host: x.x.x.x
              port: 6443
              virtualIP:
                configuration:
                  address: y.y.y.y
```
The "configuration" keyword comes from an existing API here
https://github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/blob/8e2b30770c64d6976a7990274988d53f9626f169/api/v1alpha1/addon_types.go#L260

**Which issue(s) this PR fixes**:
Fixes #

**How Has This Been Tested?**:
<!--
Please describe the tests that you ran to verify your changes.
Provide output from the tests and any manual steps needed to replicate
the tests.
-->
* New unit tests
* Created a Nutanix Cluster inside a VPC

Assigned a floating IP `10.22.198.34` to an internal IP `172.16.0.101`
<img width="1123" alt="image"
src="https://github.com/user-attachments/assets/1b099de8-1b88-4820-93fc-e4c2ea3e7586">
One of the control-plane VMs got the internal VIP assigned.
<img width="547" alt="image"
src="https://github.com/user-attachments/assets/bcc17ce8-f2c8-45ab-9a87-a1261a2c0bf0">

And the Cluster is accessible outside of the VPC.
```
kubectl get nodes
NAME                                          STATUS   ROLES           AGE    VERSION
dkoshkin-floating-ip-k9d74-d8jxp              Ready    control-plane   122m   v1.30.5
dkoshkin-floating-ip-k9d74-fb5pb              Ready    control-plane   125m   v1.30.5
dkoshkin-floating-ip-k9d74-wtl5w              Ready    control-plane   123m   v1.30.5
dkoshkin-floating-ip-md-0-qkg7q-pm5z9-6zf8f   Ready    <none>          124m   v1.30.5
```


**Special notes for your reviewer**:
<!--
Use this to provide any additional information to the reviewers.
This may include:
- Best way to review the PR.
- Where the author wants the most review attention on.
- etc.
-->

Author: dkoshkin

api/v1alpha1/common_types.go

@@ -47,6 +47,25 @@ type ControlPlaneVirtualIPSpec struct {
 	// +kubebuilder:validation:Enum=KubeVIP
 	// +kubebuilder:default=KubeVIP
 	Provider string `json:"provider,omitempty"`
+
+	// Configuration for the chosen control-plane virtual IP provider.
+	// +kubebuilder:validation:Optional
+	Configuration *ControlPlaneVirtualIPConfiguration `json:"configuration,omitempty"`
+}
+
+type ControlPlaneVirtualIPConfiguration struct {
+	// The virtual IP on which the API server is serving.
+	// If left empty, the value from controlPlaneEndpoint.host will be used.
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:validation:Format=ipv4
+	Address string `json:"address,omitempty"`
+
+	// The port on which the API server is serving.
+	// If left empty, the value from controlPlaneEndpoint.port will be used.
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:validation:Minimum=1
+	// +kubebuilder:validation:Maximum=65535
+	Port int32 `json:"port,omitempty"`
 }
 
 // LocalObjectReference contains enough information to let you locate the

api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml

@@ -613,6 +613,24 @@ spec:
                         virtualIP:
                           description: Configuration for the virtual IP provider.
                           properties:
+                            configuration:
+                              description: Configuration for the chosen control-plane virtual IP provider.
+                              properties:
+                                address:
+                                  description: |-
+                                    The virtual IP on which the API server is serving.
+                                    If left empty, the value from controlPlaneEndpoint.host will be used.
+                                  format: ipv4
+                                  type: string
+                                port:
+                                  description: |-
+                                    The port on which the API server is serving.
+                                    If left empty, the value from controlPlaneEndpoint.port will be used.
+                                  format: int32
+                                  maximum: 65535
+                                  minimum: 1
+                                  type: integer
+                              type: object
                             provider:
                               default: KubeVIP
                               description: Virtual IP provider to deploy.

api/v1alpha1/zz_generated.deepcopy.go

@@ -4,9 +4,9 @@ prismCentralInsecure: {{ .PrismCentralInsecure }}
 {{- with .PrismCentralAdditionalTrustBundle }}
 prismCentralAdditionalTrustBundle: "{{ . }}"
 {{- end }}
-{{- with .ControlPlaneEndpointHost }}
-ignoredNodeIPs: [ {{ printf "%q" . }} ]
- {{- end }}
+{{- with .IPsToIgnore }}
+ignoredNodeIPs: [ {{ joinQuoted . }} ]
+{{- end }}
 
 # The Secret containing the credentials will be created by the handler.
 createSecret: false

charts/cluster-api-runtime-extensions-nutanix/addons/ccm/nutanix/values-template.yaml

@@ -22,7 +22,7 @@ data:
             - name: vip_arp
               value: "true"
             - name: port
-              value: '{{ `{{ .ControlPlaneEndpoint.Port }}` }}'
+              value: '{{ `{{ .Port }}` }}'
             - name: vip_nodename
               valueFrom:
                 fieldRef:
@@ -46,7 +46,7 @@ data:
             - name: vip_retryperiod
               value: "2"
             - name: address
-              value: '{{ `{{ .ControlPlaneEndpoint.Host }}` }}'
+              value: '{{ `{{ .Address }}` }}'
             - name: prometheus_server
           image: ghcr.io/kube-vip/kube-vip:v0.8.3
           imagePullPolicy: IfNotPresent

charts/cluster-api-runtime-extensions-nutanix/templates/virtual-ip/kube-vip/manifests/kube-vip-configmap.yaml

@@ -2,7 +2,7 @@
 title = "Control Plane Endpoint"
 +++
 
-Configure Control Plane Endpoint. Defines the host IP and port of the CAPX Kubernetes cluster.
+Configure Control Plane Endpoint. Defines the host IP and port of the Nutanix Kubernetes cluster.
 
 ## Examples
 
@@ -51,6 +51,93 @@ spec:
             name: kube-vip
             namespace: kube-system
           spec:
+            containers:
+            - name: kube-vip
+              args:
+              - manager
+              env:
+              - name: port
+                value: '6443'
+              - name: address
+                value: 'x.x.x.x'
+          ...
+        owner: root:root
+        path: /etc/kubernetes/manifests/kube-vip.yaml
+        permissions: "0600"
+      postKubeadmCommands:
+        # Only added for clusters version >=v1.29.0
+        - |-
+          if [ -f /run/kubeadm/kubeadm.yaml ]; then
+            sed -i 's#path: /etc/kubernetes/super-admin.conf#path: ...
+          fi
+      preKubeadmCommands:
+        # Only added for clusters version >=v1.29.0
+        - |-
+          if [ -f /run/kubeadm/kubeadm.yaml ]; then
+            sed -i 's#path: /etc/kubernetes/admin.conf#path: ...
+          fi
+```
+
+### Set Control Plane Endpoint and a Different Virtual IP
+
+It is also possible to set a separate virtual IP to be used by kube-vip from the control plane endpoint.
+This is useful in VPC setups or other instances
+when you have an external floating IP already associated with the virtual IP.
+
+```yaml
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  name: <NAME>
+spec:
+  topology:
+    variables:
+      - name: clusterConfig
+        value:
+          nutanix:
+            controlPlaneEndpoint:
+              host: x.x.x.x
+              port: 6443
+              virtualIP:
+                configuration:
+                  address: y.y.y.y
+```
+
+Applying this configuration will result in the following value being set:
+
+- `NutanixCluster`:
+
+```yaml
+spec:
+  template:
+    spec:
+      controlPlaneEndpoint:
+        host: x.x.x.x
+        port: 6443
+```
+
+- `KubeadmControlPlaneTemplate`
+
+```yaml
+  spec:
+    kubeadmConfigSpec:
+      files:
+      - content: |
+          apiVersion: v1
+          kind: Pod
+          metadata:
+            name: kube-vip
+            namespace: kube-system
+          spec:
+            containers:
+            - name: kube-vip
+              args:
+              - manager
+              env:
+              - name: port
+                value: '6443'
+              - name: address
+                value: 'y.y.y.y'
           ...
         owner: root:root
         path: /etc/kubernetes/manifests/kube-vip.yaml

docs/content/customization/nutanix/control-plane-endpoint.md

@@ -33,8 +33,8 @@ docker container run --rm ghcr.io/kube-vip/kube-vip:"${KUBE_VIP_VERSION}" \
   gojq --yaml-input --yaml-output \
     'del(.metadata.creationTimestamp, .status) |
      .spec.containers[].imagePullPolicy |= "IfNotPresent" |
-     (.spec.containers[0].env[] | select(.name == "port").value) |= "{{ `{{ .ControlPlaneEndpoint.Port }}` }}" |
-     (.spec.containers[0].env[] | select(.name == "address").value) |= "{{ `{{ .ControlPlaneEndpoint.Host }}` }}"
+     (.spec.containers[0].env[] | select(.name == "port").value) |= "{{ `{{ .Port }}` }}" |
+     (.spec.containers[0].env[] | select(.name == "address").value) |= "{{ `{{ .Address }}` }}"
     ' >"${ASSETS_DIR}/${FILE_NAME}"
 
 kubectl create configmap "{{ .Values.hooks.virtualIP.kubeVip.defaultTemplateConfigMap.name }}" --dry-run=client --output yaml \

hack/addons/update-kube-vip-manifests.sh

@@ -8,6 +8,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"strings"
 	"text/template"
 
 	"github.com/go-logr/logr"
@@ -140,7 +141,15 @@ func templateValuesFunc(
 	nutanixConfig *v1alpha1.NutanixSpec,
 ) func(*clusterv1.Cluster, string) (string, error) {
 	return func(_ *clusterv1.Cluster, valuesTemplate string) (string, error) {
-		helmValuesTemplate, err := template.New("").Parse(valuesTemplate)
+		joinQuoted := template.FuncMap{
+			"joinQuoted": func(items []string) string {
+				for i, item := range items {
+					items[i] = fmt.Sprintf("%q", item)
+				}
+				return strings.Join(items, ", ")
+			},
+		}
+		helmValuesTemplate, err := template.New("").Funcs(joinQuoted).Parse(valuesTemplate)
 		if err != nil {
 			return "", fmt.Errorf("failed to parse Helm values template: %w", err)
 		}
@@ -150,7 +159,7 @@ func templateValuesFunc(
 			PrismCentralPort                  uint16
 			PrismCentralInsecure              bool
 			PrismCentralAdditionalTrustBundle string
-			ControlPlaneEndpointHost          string
+			IPsToIgnore                       []string
 		}
 
 		address, port, err := nutanixConfig.PrismCentralEndpoint.ParseURL()
@@ -162,7 +171,7 @@ func templateValuesFunc(
 			PrismCentralPort:                  port,
 			PrismCentralInsecure:              nutanixConfig.PrismCentralEndpoint.Insecure,
 			PrismCentralAdditionalTrustBundle: nutanixConfig.PrismCentralEndpoint.AdditionalTrustBundle,
-			ControlPlaneEndpointHost:          nutanixConfig.ControlPlaneEndpoint.Host,
+			IPsToIgnore:                       ipsToIgnore(nutanixConfig),
 		}
 
 		var b bytes.Buffer
@@ -174,3 +183,17 @@ func templateValuesFunc(
 		return b.String(), nil
 	}
 }
+
+func ipsToIgnore(nutanixConfig *v1alpha1.NutanixSpec) []string {
+	toIgnore := []string{nutanixConfig.ControlPlaneEndpoint.Host}
+	// Also ignore the virtual IP if it is set.
+	if nutanixConfig.ControlPlaneEndpoint.VirtualIPSpec != nil &&
+		nutanixConfig.ControlPlaneEndpoint.VirtualIPSpec.Configuration != nil &&
+		nutanixConfig.ControlPlaneEndpoint.VirtualIPSpec.Configuration.Address != "" {
+		toIgnore = append(
+			toIgnore,
+			nutanixConfig.ControlPlaneEndpoint.VirtualIPSpec.Configuration.Address,
+		)
+	}
+	return toIgnore
+}

pkg/handlers/generic/lifecycle/ccm/nutanix/handler.go

@@ -39,6 +39,16 @@ prismCentralPort: 9440
 prismCentralInsecure: true
 ignoredNodeIPs: [ "1.2.3.4" ]
 
+# The Secret containing the credentials will be created by the handler.
+createSecret: false
+secretName: nutanix-ccm-credentials
+`
+
+	expectedWithVirtualIPSet = `prismCentralEndPoint: prism-central.nutanix.com
+prismCentralPort: 9440
+prismCentralInsecure: true
+ignoredNodeIPs: [ "1.2.3.4", "5.6.7.8" ]
+
 # The Secret containing the credentials will be created by the handler.
 createSecret: false
 secretName: nutanix-ccm-credentials
@@ -127,6 +137,41 @@ func Test_templateValues(t *testing.T) {
 			in:       valuesTemplate,
 			expected: expectedWithoutAdditionalTrustBundle,
 		},
+		{
+			name: "With VirtualIP Set",
+			clusterConfig: &apivariables.ClusterConfigSpec{
+				Addons: &apivariables.Addons{
+					GenericAddons: v1alpha1.GenericAddons{
+						CCM: &v1alpha1.CCM{
+							Credentials: &v1alpha1.CCMCredentials{
+								SecretRef: v1alpha1.LocalObjectReference{
+									Name: "creds",
+								},
+							},
+						},
+					},
+				},
+				Nutanix: &v1alpha1.NutanixSpec{
+					PrismCentralEndpoint: v1alpha1.NutanixPrismCentralEndpointSpec{
+						URL: fmt.Sprintf(
+							"https://prism-central.nutanix.com:%d",
+							v1alpha1.DefaultPrismCentralPort,
+						),
+						Insecure: true,
+					},
+					ControlPlaneEndpoint: v1alpha1.ControlPlaneEndpointSpec{
+						Host: "1.2.3.4",
+						VirtualIPSpec: &v1alpha1.ControlPlaneVirtualIPSpec{
+							Configuration: &v1alpha1.ControlPlaneVirtualIPConfiguration{
+								Address: "5.6.7.8",
+							},
+						},
+					},
+				},
+			},
+			in:       valuesTemplate,
+			expected: expectedWithVirtualIPSet,
+		},
 	}
 	for idx := range tests {
 		tt := tests[idx]

pkg/handlers/generic/lifecycle/ccm/nutanix/handler_test.go

@@ -264,7 +264,7 @@ spec:
         - name: vip_arp
           value: "true"
         - name: address
-          value: "{{ .ControlPlaneEndpoint.Host }}"
+          value: "{{ .Address }}"
         - name: port
-          value: "{{ .ControlPlaneEndpoint.Port }}"
+          value: "{{ .Port }}"
 `