feat: add kube-vip static Pod in a Nutanix handler (#558)

**What problem does this PR solve?**:
Stacked on
#554

This PR moves the responsibility of adding the control-plan kube-vip to
CAREN away from infra providers.
This has a big benefit of being able to keep this implementation and the
image version consistent across the different providers.

~Because CAPX is always requires kube-vip to be installed there are no
API changes, but in the future we can add support to disable or change
the provider.~
Added an empty `virtualIP{}` struct to Nutanix `controlPlaneEndpoint`.
```
spec:
  topology:
    variables:
      - name: clusterConfig
        value:
          nutanix:
            controlPlaneEndpoint:
              host: x.x.x.x
              port: 6443
              virtualIP: {}
```


**Which issue(s) this PR fixes**:
Fixes https://jira.nutanix.com/browse/D2IQ-100364

**How Has This Been Tested?**:
<!--
Please describe the tests that you ran to verify your changes.
Provide output from the tests and any manual steps needed to replicate
the tests.
-->
New unit tests, and tested manually too:
```
clusterctl describe cluster $CLUSTER_NAME    
NAME                                                                                              READY  SEVERITY  REASON  SINCE  MESSAGE 
Cluster/dkkonvoy-042324-nutanix-03                                                                True                     76m             
├─ClusterInfrastructure - NutanixCluster/dkkonvoy-042324-nutanix-03-wvv6d                                                                  
├─ControlPlane - KubeadmControlPlane/dkkonvoy-042324-nutanix-03-sv2zb                             True                     76m             
│ └─Machine/dkkonvoy-042324-nutanix-03-sv2zb-ds5mv                                                True                     76m             
│   └─MachineInfrastructure - NutanixMachine/dkkonvoy-042324-nutanix-03-sv2zb-ds5mv                                                        
└─Workers                                                                                                                                  
  └─MachineDeployment/dkkonvoy-042324-nutanix-03-md-0-w6mls                                       True                     74m             
    └─Machine/dkkonvoy-042324-nutanix-03-md-0-w6mls-979mw-c582s                                   True                     74m             
      └─MachineInfrastructure - NutanixMachine/dkkonvoy-042324-nutanix-03-md-0-w6mls-979mw-c582s                                      
```

**Special notes for your reviewer**:
<!--
Use this to provide any additional information to the reviewers.
This may include:
- Best way to review the PR.
- Where the author wants the most review attention on.
- etc.
-->

---------

Co-authored-by: Jimmi Dyson <[email protected]>

Author: dkoshkin

api/v1alpha1/common_types.go

@@ -5,6 +5,8 @@ package v1alpha1
 
 const (
 	APIServerPort = 6443
+
+	VirtualIPProviderKubeVIP = "KubeVIP"
 )
 
 // ObjectMeta is metadata that all persisted resources must have, which includes all objects
@@ -37,4 +39,16 @@ type ControlPlaneEndpointSpec struct {
 	// +kubebuilder:validation:Minimum=1
 	// +kubebuilder:validation:Maximum=65535
 	Port int32 `json:"port"`
+
+	// Configuration for the virtual IP provider.
+	// +optional
+	VirtualIPSpec *ControlPlaneVirtualIPSpec `json:"virtualIP,omitempty"`
+}
+
+type ControlPlaneVirtualIPSpec struct {
+	// Virtual IP provider to deploy.
+	// +kubebuilder:validation:Enum=KubeVIP
+	// +kubebuilder:default=KubeVIP
+	// +optional
+	Provider string `json:"provider,omitempty"`
 }

api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml

@@ -476,6 +476,16 @@ spec:
                         maximum: 65535
                         minimum: 1
                         type: integer
+                      virtualIP:
+                        description: Configuration for the virtual IP provider.
+                        properties:
+                          provider:
+                            default: KubeVIP
+                            description: Virtual IP provider to deploy.
+                            enum:
+                            - KubeVIP
+                            type: string
+                        type: object
                     required:
                     - host
                     - port

api/v1alpha1/zz_generated.deepcopy.go

@@ -62,6 +62,8 @@ A Helm chart for cluster-api-runtime-extensions-nutanix
 | hooks.nfd.crsStrategy.defaultInstallationConfigMap.name | string | `"node-feature-discovery"` |  |
 | hooks.nfd.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` |  |
 | hooks.nfd.helmAddonStrategy.defaultValueTemplateConfigMap.name | string | `"default-nfd-helm-values-template"` |  |
+| hooks.virtualIP.kubeVip.defaultTemplateConfigMap.create | bool | `true` |  |
+| hooks.virtualIP.kubeVip.defaultTemplateConfigMap.name | string | `"default-kube-vip-template"` |  |
 | image.pullPolicy | string | `"IfNotPresent"` |  |
 | image.repository | string | `"ghcr.io/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix"` |  |
 | image.tag | string | `""` |  |

charts/cluster-api-runtime-extensions-nutanix/README.md

@@ -132,72 +132,7 @@ spec:
           scheduler:
             extraArgs:
               tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256
-        files:
-        - content: |
-            apiVersion: v1
-            kind: Pod
-            metadata:
-              name: kube-vip
-              namespace: kube-system
-            spec:
-              containers:
-                - name: kube-vip
-                  image: ghcr.io/kube-vip/kube-vip:v0.6.4
-                  imagePullPolicy: IfNotPresent
-                  args:
-                    - manager
-                  env:
-                    - name: vip_arp
-                      value: "true"
-                    - name: address
-                      value: "control_plane_endpoint_ip"
-                    - name: port
-                      value: "control_plane_endpoint_port"
-                    - name: vip_cidr
-                      value: "32"
-                    - name: cp_enable
-                      value: "true"
-                    - name: cp_namespace
-                      value: kube-system
-                    - name: vip_ddns
-                      value: "false"
-                    - name: vip_leaderelection
-                      value: "true"
-                    - name: vip_leaseduration
-                      value: "15"
-                    - name: vip_renewdeadline
-                      value: "10"
-                    - name: vip_retryperiod
-                      value: "2"
-                    - name: svc_enable
-                      value: "false"
-                    - name: lb_enable
-                      value: "false"
-                    - name: enableServicesElection
-                      value: "false"
-                  securityContext:
-                    capabilities:
-                      add:
-                        - NET_ADMIN
-                        - SYS_TIME
-                        - NET_RAW
-                  volumeMounts:
-                    - mountPath: /etc/kubernetes/admin.conf
-                      name: kubeconfig
-                  resources: {}
-              hostNetwork: true
-              hostAliases:
-                - hostnames:
-                    - kubernetes
-                  ip: 127.0.0.1
-              volumes:
-                - name: kubeconfig
-                  hostPath:
-                    type: FileOrCreate
-                    path: /etc/kubernetes/admin.conf
-            status: {}
-          owner: root:root
-          path: /etc/kubernetes/manifests/kube-vip.yaml
+        files: []
         initConfiguration:
           nodeRegistration:
             kubeletExtraArgs:
@@ -212,14 +147,6 @@ spec:
               tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256
         postKubeadmCommands:
         - echo export KUBECONFIG=/etc/kubernetes/admin.conf >> /root/.bashrc
-        - |
-          KUBERNETES_VERSION_NO_V=${KUBERNETES_VERSION#v}
-          VERSION_TO_COMPARE=1.29.0
-          if [ "$(printf '%s\n' "$KUBERNETES_VERSION_NO_V" "$VERSION_TO_COMPARE" | sort -V | head -n1)" != "$KUBERNETES_VERSION_NO_V" ]; then
-            if [ -f /run/kubeadm/kubeadm.yaml ]; then
-              sed -i 's#path: /etc/kubernetes/super-admin.conf#path: /etc/kubernetes/admin.conf#' /etc/kubernetes/manifests/kube-vip.yaml;
-            fi
-          fi
         - echo "after kubeadm call" > /var/log/postkubeadm.log
         preKubeadmCommands:
         - echo "before kubeadm call" > /var/log/prekubeadm.log
@@ -228,14 +155,6 @@ spec:
         - echo "127.0.0.1   localhost" >>/etc/hosts
         - echo "127.0.0.1   kubernetes" >>/etc/hosts
         - echo "127.0.0.1   {{ ds.meta_data.hostname }}" >> /etc/hosts
-        - |
-          KUBERNETES_VERSION_NO_V=${KUBERNETES_VERSION#v}
-          VERSION_TO_COMPARE=1.29.0
-          if [ "$(printf '%s\n' "$KUBERNETES_VERSION_NO_V" "$VERSION_TO_COMPARE" | sort -V | head -n1)" != "$KUBERNETES_VERSION_NO_V" ]; then
-            if [ -f /run/kubeadm/kubeadm.yaml ]; then
-              sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' /etc/kubernetes/manifests/kube-vip.yaml;
-            fi
-          fi
         useExperimentalRetryJoin: true
         verbosity: 10
 ---

charts/cluster-api-runtime-extensions-nutanix/defaultclusterclasses/nutanix-cluster-class.yaml

@@ -0,0 +1,75 @@
+# Copyright 2023 D2iQ, Inc. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+#=================================================================
+#                 DO NOT EDIT THIS FILE
+#  IT HAS BEEN GENERATED BY /hack/addons/update-kube-vip-manifests.sh
+#=================================================================
+{{- if .Values.hooks.virtualIP.kubeVip.defaultTemplateConfigMap.create }}
+apiVersion: v1
+data:
+  kube-vip.yaml: |
+    apiVersion: v1
+    kind: Pod
+    metadata:
+      name: kube-vip
+      namespace: kube-system
+    spec:
+      containers:
+        - args:
+            - manager
+          env:
+            - name: vip_arp
+              value: "true"
+            - name: port
+              value: '{{ `{{ .ControlPlaneEndpoint.Port }}` }}'
+            - name: vip_nodename
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: vip_cidr
+              value: "32"
+            - name: dns_mode
+              value: first
+            - name: cp_enable
+              value: "true"
+            - name: cp_namespace
+              value: kube-system
+            - name: vip_leaderelection
+              value: "true"
+            - name: vip_leasename
+              value: plndr-cp-lock
+            - name: vip_leaseduration
+              value: "15"
+            - name: vip_renewdeadline
+              value: "10"
+            - name: vip_retryperiod
+              value: "2"
+            - name: address
+              value: '{{ `{{ .ControlPlaneEndpoint.Host }}` }}'
+          image: ghcr.io/kube-vip/kube-vip:v0.8.0
+          imagePullPolicy: IfNotPresent
+          name: kube-vip
+          resources: {}
+          securityContext:
+            capabilities:
+              add:
+                - NET_ADMIN
+                - NET_RAW
+          volumeMounts:
+            - mountPath: /etc/kubernetes/admin.conf
+              name: kubeconfig
+      hostAliases:
+        - hostnames:
+            - kubernetes
+          ip: 127.0.0.1
+      hostNetwork: true
+      volumes:
+        - hostPath:
+            path: /etc/kubernetes/admin.conf
+          name: kubeconfig
+kind: ConfigMap
+metadata:
+  creationTimestamp: null
+  name: '{{ .Values.hooks.virtualIP.kubeVip.defaultTemplateConfigMap.name }}'
+{{- end -}}

charts/cluster-api-runtime-extensions-nutanix/templates/virtual-ip/kube-vip/manifests/kube-vip-configmap.yaml

@@ -72,6 +72,12 @@ hooks:
         create: true
         name: default-cluster-autoscaler-helm-values-template
 
+  virtualIP:
+    kubeVip:
+      defaultTemplateConfigMap:
+        create: true
+        name: default-kube-vip-template
+
 helmAddonsConfigMap: default-helm-addons-config
 
 deployDefaultClusterClasses: true

charts/cluster-api-runtime-extensions-nutanix/values.yaml

@@ -22,6 +22,7 @@ spec:
             controlPlaneEndpoint:
               host: x.x.x.x
               port: 6443
+              virtualIP: {}
 ```
 
 Applying this configuration will result in the following value being set:
@@ -36,3 +37,34 @@ spec:
         host: x.x.x.x
         port: 6443
 ```
+
+- `KubeadmControlPlaneTemplate`
+
+```yaml
+  spec:
+    kubeadmConfigSpec:
+      files:
+      - content: |
+          apiVersion: v1
+          kind: Pod
+          metadata:
+            name: kube-vip
+            namespace: kube-system
+          spec:
+          ...
+        owner: root:root
+        path: /etc/kubernetes/manifests/kube-vip.yaml
+        permissions: "0600"
+      postKubeadmCommands:
+        # Only added for clusters version >=v1.29.0
+        - |-
+          if [ -f /run/kubeadm/kubeadm.yaml ]; then
+            sed -i 's#path: /etc/kubernetes/super-admin.conf#path: ...
+          fi
+      preKubeadmCommands:
+        # Only added for clusters version >=v1.29.0
+        - |-
+          if [ -f /run/kubeadm/kubeadm.yaml ]; then
+            sed -i 's#path: /etc/kubernetes/admin.conf#path: ...
+          fi
+```

docs/content/customization/nutanix/control-plane-endpoint.md

@@ -112,6 +112,8 @@ spec:
           controlPlaneEndpoint:
             host: ${CONTROL_PLANE_ENDPOINT_IP}
             port: 6443
+            virtualIP:
+              provider: KubeVIP
           prismCentralEndpoint:
             credentials:
               name: ${CLUSTER_NAME}-pc-creds

examples/capi-quick-start/nutanix-cluster-calico-crs.yaml

@@ -112,6 +112,8 @@ spec:
           controlPlaneEndpoint:
             host: ${CONTROL_PLANE_ENDPOINT_IP}
             port: 6443
+            virtualIP:
+              provider: KubeVIP
           prismCentralEndpoint:
             credentials:
               name: ${CLUSTER_NAME}-pc-creds

examples/capi-quick-start/nutanix-cluster-calico-helm-addon.yaml

@@ -112,6 +112,8 @@ spec:
           controlPlaneEndpoint:
             host: ${CONTROL_PLANE_ENDPOINT_IP}
             port: 6443
+            virtualIP:
+              provider: KubeVIP
           prismCentralEndpoint:
             credentials:
               name: ${CLUSTER_NAME}-pc-creds

examples/capi-quick-start/nutanix-cluster-cilium-crs.yaml

@@ -112,6 +112,8 @@ spec:
           controlPlaneEndpoint:
             host: ${CONTROL_PLANE_ENDPOINT_IP}
             port: 6443
+            virtualIP:
+              provider: KubeVIP
           prismCentralEndpoint:
             credentials:
               name: ${CLUSTER_NAME}-pc-creds