nutanix-cloud-native · jimmidyson · Apr 26, 2024 · Apr 22, 2024 · Apr 23, 2024 · Apr 23, 2024
diff --git a/api/v1alpha1/common_types.go b/api/v1alpha1/common_types.go
@@ -37,4 +37,10 @@ type ControlPlaneEndpointSpec struct {
 	// +kubebuilder:validation:Minimum=1
 	// +kubebuilder:validation:Maximum=65535
 	Port int32 `json:"port"`
+
+	// Configuration for the virtual IP provider.
+	// +optional
+	VirtualIPSpec *ControlPlaneVirtualIPSpec `json:"virtualIP,omitempty"`
 }
+
+type ControlPlaneVirtualIPSpec struct{}
diff --git a/api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml b/api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml
@@ -476,6 +476,9 @@ spec:
                         maximum: 65535
                         minimum: 1
                         type: integer
+                      virtualIP:
+                        description: Configuration for the virtual IP provider.
+                        type: object
                     required:
                     - host
                     - port

diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
diff --git a/charts/cluster-api-runtime-extensions-nutanix/README.md b/charts/cluster-api-runtime-extensions-nutanix/README.md
@@ -59,6 +59,8 @@ A Helm chart for cluster-api-runtime-extensions-nutanix
 | hooks.cni.cilium.helmAddonStrategy.defaultValueTemplateConfigMap.name | string | `"default-cilium-cni-helm-values-template"` |  |
 | hooks.csi.nutanix.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` |  |
 | hooks.csi.nutanix.helmAddonStrategy.defaultValueTemplateConfigMap.name | string | `"default-nutanix-csi-helm-values-template"` |  |
+| hooks.kubeVip.defaultTemplateConfigMap.create | bool | `true` |  |
+| hooks.kubeVip.defaultTemplateConfigMap.name | string | `"default-kube-vip-template"` |  |
 | hooks.nfd.crsStrategy.defaultInstallationConfigMap.name | string | `"node-feature-discovery"` |  |
 | hooks.nfd.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` |  |
 | hooks.nfd.helmAddonStrategy.defaultValueTemplateConfigMap.name | string | `"default-nfd-helm-values-template"` |  |

diff --git a/...s/cluster-api-runtime-extensions-nutanix/defaultclusterclasses/nutanix-cluster-class.yaml b/...s/cluster-api-runtime-extensions-nutanix/defaultclusterclasses/nutanix-cluster-class.yaml
@@ -132,72 +132,7 @@ spec:
           scheduler:
             extraArgs:
               tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256
-        files:
-        - content: |
-            apiVersion: v1
-            kind: Pod
-            metadata:
-              name: kube-vip
-              namespace: kube-system
-            spec:
-              containers:
-                - name: kube-vip
-                  image: ghcr.io/kube-vip/kube-vip:v0.6.4
-                  imagePullPolicy: IfNotPresent
-                  args:
-                    - manager
-                  env:
-                    - name: vip_arp
-                      value: "true"
-                    - name: address
-                      value: "control_plane_endpoint_ip"
-                    - name: port
-                      value: "control_plane_endpoint_port"
-                    - name: vip_cidr
-                      value: "32"
-                    - name: cp_enable
-                      value: "true"
-                    - name: cp_namespace
-                      value: kube-system
-                    - name: vip_ddns
-                      value: "false"
-                    - name: vip_leaderelection
-                      value: "true"
-                    - name: vip_leaseduration
-                      value: "15"
-                    - name: vip_renewdeadline
-                      value: "10"
-                    - name: vip_retryperiod
-                      value: "2"
-                    - name: svc_enable
-                      value: "false"
-                    - name: lb_enable
-                      value: "false"
-                    - name: enableServicesElection
-                      value: "false"
-                  securityContext:
-                    capabilities:
-                      add:
-                        - NET_ADMIN
-                        - SYS_TIME
-                        - NET_RAW
-                  volumeMounts:
-                    - mountPath: /etc/kubernetes/admin.conf
-                      name: kubeconfig
-                  resources: {}
-              hostNetwork: true
-              hostAliases:
-                - hostnames:
-                    - kubernetes
-                  ip: 127.0.0.1
-              volumes:
-                - name: kubeconfig
-                  hostPath:
-                    type: FileOrCreate
-                    path: /etc/kubernetes/admin.conf
-            status: {}
-          owner: root:root
-          path: /etc/kubernetes/manifests/kube-vip.yaml
+        files: []
         initConfiguration:
           nodeRegistration:
             kubeletExtraArgs:
@@ -212,14 +147,6 @@ spec:
               tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256
         postKubeadmCommands:
         - echo export KUBECONFIG=/etc/kubernetes/admin.conf >> /root/.bashrc
-        - |
-          KUBERNETES_VERSION_NO_V=${KUBERNETES_VERSION#v}
-          VERSION_TO_COMPARE=1.29.0
-          if [ "$(printf '%s\n' "$KUBERNETES_VERSION_NO_V" "$VERSION_TO_COMPARE" | sort -V | head -n1)" != "$KUBERNETES_VERSION_NO_V" ]; then
-            if [ -f /run/kubeadm/kubeadm.yaml ]; then
-              sed -i 's#path: /etc/kubernetes/super-admin.conf#path: /etc/kubernetes/admin.conf#' /etc/kubernetes/manifests/kube-vip.yaml;
-            fi
-          fi
         - echo "after kubeadm call" > /var/log/postkubeadm.log
         preKubeadmCommands:
         - echo "before kubeadm call" > /var/log/prekubeadm.log
@@ -228,14 +155,6 @@ spec:
         - echo "127.0.0.1   localhost" >>/etc/hosts
         - echo "127.0.0.1   kubernetes" >>/etc/hosts
         - echo "127.0.0.1   {{ ds.meta_data.hostname }}" >> /etc/hosts
-        - |
-          KUBERNETES_VERSION_NO_V=${KUBERNETES_VERSION#v}
-          VERSION_TO_COMPARE=1.29.0
-          if [ "$(printf '%s\n' "$KUBERNETES_VERSION_NO_V" "$VERSION_TO_COMPARE" | sort -V | head -n1)" != "$KUBERNETES_VERSION_NO_V" ]; then
-            if [ -f /run/kubeadm/kubeadm.yaml ]; then
-              sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' /etc/kubernetes/manifests/kube-vip.yaml;
-            fi
-          fi
         useExperimentalRetryJoin: true
         verbosity: 10
 ---

diff --git a/...uster-api-runtime-extensions-nutanix/templates/kube-vip/manifests/kube-vip-configmap.yaml b/...uster-api-runtime-extensions-nutanix/templates/kube-vip/manifests/kube-vip-configmap.yaml
@@ -0,0 +1,75 @@
+# Copyright 2023 D2iQ, Inc. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+#=================================================================
+#                 DO NOT EDIT THIS FILE
+#  IT HAS BEEN GENERATED BY /hack/addons/update-kube-vip-manifests.sh
+#=================================================================
+{{- if .Values.hooks.kubeVip.defaultTemplateConfigMap.create }}
+apiVersion: v1
+data:
+  kube-vip.yaml: |
+    apiVersion: v1
+    kind: Pod
+    metadata:
+      name: kube-vip
+      namespace: kube-system
+    spec:
+      containers:
+        - args:
+            - manager
+          env:
+            - name: vip_arp
+              value: "true"
+            - name: port
+              value: '{{ `{{ .ControlPlaneEndpoint.Port }}` }}'
+            - name: vip_nodename
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: vip_cidr
+              value: "32"
+            - name: dns_mode
+              value: first
+            - name: cp_enable
+              value: "true"
+            - name: cp_namespace
+              value: kube-system
+            - name: vip_leaderelection
+              value: "true"
+            - name: vip_leasename
+              value: plndr-cp-lock
+            - name: vip_leaseduration
+              value: "15"
+            - name: vip_renewdeadline
+              value: "10"
+            - name: vip_retryperiod
+              value: "2"
+            - name: address
+              value: '{{ `{{ .ControlPlaneEndpoint.Host }}` }}'
+          image: ghcr.io/kube-vip/kube-vip:v0.8.0
+          imagePullPolicy: IfNotPresent
+          name: kube-vip
+          resources: {}
+          securityContext:
+            capabilities:
+              add:
+                - NET_ADMIN
+                - NET_RAW
+          volumeMounts:
+            - mountPath: /etc/kubernetes/admin.conf
+              name: kubeconfig
+      hostAliases:
+        - hostnames:
+            - kubernetes
+          ip: 127.0.0.1
+      hostNetwork: true
+      volumes:
+        - hostPath:
+            path: /etc/kubernetes/admin.conf
+          name: kubeconfig
+kind: ConfigMap
+metadata:
+  creationTimestamp: null
+  name: '{{ .Values.hooks.kubeVip.defaultTemplateConfigMap.name }}'
+{{- end -}}
diff --git a/charts/cluster-api-runtime-extensions-nutanix/values.yaml b/charts/cluster-api-runtime-extensions-nutanix/values.yaml
@@ -71,6 +71,10 @@ hooks:
       defaultValueTemplateConfigMap:
         create: true
         name: default-cluster-autoscaler-helm-values-template
+  kubeVip:
+    defaultTemplateConfigMap:
+      create: true
+      name: default-kube-vip-template
 
 helmAddonsConfigMap: default-helm-addons-config
 

diff --git a/docs/content/customization/nutanix/control-plane-endpoint.md b/docs/content/customization/nutanix/control-plane-endpoint.md
@@ -22,6 +22,7 @@ spec:
             controlPlaneEndpoint:
               host: x.x.x.x
               port: 6443
+              virtualIP: {}
 ```
 
 Applying this configuration will result in the following value being set:
@@ -36,3 +37,34 @@ spec:
         host: x.x.x.x
         port: 6443
 ```
+
+- `KubeadmControlPlaneTemplate`
+
+```yaml
+  spec:
+    kubeadmConfigSpec:
+      files:
+      - content: |
+          apiVersion: v1
+          kind: Pod
+          metadata:
+            name: kube-vip
+            namespace: kube-system
+          spec:
+          ...
+        owner: root:root
+        path: /etc/kubernetes/manifests/kube-vip.yaml
+        permissions: "0600"
+      postKubeadmCommands:
+        # Only added for clusters version >=v1.29.0
+        - |-
+          if [ -f /run/kubeadm/kubeadm.yaml ]; then
+            sed -i 's#path: /etc/kubernetes/super-admin.conf#path: ...
+          fi
+      preKubeadmCommands:
+        # Only added for clusters version >=v1.29.0
+        - |-
+          if [ -f /run/kubeadm/kubeadm.yaml ]; then
+            sed -i 's#path: /etc/kubernetes/admin.conf#path: ...
+          fi
+```
diff --git a/examples/capi-quick-start/nutanix-cluster-calico-crs.yaml b/examples/capi-quick-start/nutanix-cluster-calico-crs.yaml
@@ -112,6 +112,7 @@ spec:
           controlPlaneEndpoint:
             host: ${CONTROL_PLANE_ENDPOINT_IP}
             port: 6443
+            virtualIP: {}
           prismCentralEndpoint:
             credentials:
               name: ${CLUSTER_NAME}-pc-creds

diff --git a/examples/capi-quick-start/nutanix-cluster-calico-helm-addon.yaml b/examples/capi-quick-start/nutanix-cluster-calico-helm-addon.yaml
@@ -112,6 +112,7 @@ spec:
           controlPlaneEndpoint:
             host: ${CONTROL_PLANE_ENDPOINT_IP}
             port: 6443
+            virtualIP: {}
           prismCentralEndpoint:
             credentials:
               name: ${CLUSTER_NAME}-pc-creds

diff --git a/examples/capi-quick-start/nutanix-cluster-cilium-crs.yaml b/examples/capi-quick-start/nutanix-cluster-cilium-crs.yaml
@@ -112,6 +112,7 @@ spec:
           controlPlaneEndpoint:
             host: ${CONTROL_PLANE_ENDPOINT_IP}
             port: 6443
+            virtualIP: {}
           prismCentralEndpoint:
             credentials:
               name: ${CLUSTER_NAME}-pc-creds

diff --git a/examples/capi-quick-start/nutanix-cluster-cilium-helm-addon.yaml b/examples/capi-quick-start/nutanix-cluster-cilium-helm-addon.yaml
@@ -112,6 +112,7 @@ spec:
           controlPlaneEndpoint:
             host: ${CONTROL_PLANE_ENDPOINT_IP}
             port: 6443
+            virtualIP: {}
           prismCentralEndpoint:
             credentials:
               name: ${CLUSTER_NAME}-pc-creds

diff --git a/hack/addons/update-kube-vip-manifests.sh b/hack/addons/update-kube-vip-manifests.sh
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+readonly SCRIPT_DIR
+
+# shellcheck source=hack/common.sh
+source "${SCRIPT_DIR}/../common.sh"
+
+if [ -z "${KUBE_VIP_VERSION:-}" ]; then
+  echo "Missing argument: KUBE_VIP_VERSION"
+  exit 1
+fi
+
+ASSETS_DIR="$(mktemp -d -p "${TMPDIR:-/tmp}")"
+readonly ASSETS_DIR
+trap_add "rm -rf ${ASSETS_DIR}" EXIT
+
+readonly FILE_NAME="kube-vip.yaml"
+
+docker container run --rm ghcr.io/kube-vip/kube-vip:"${KUBE_VIP_VERSION}" \
+  manifest pod \
+  --arp \
+  --address='{{ `{{ .ControlPlaneEndpoint.Host }}` }}' \
+  --port=-99999 \
+  --controlplane \
+  --leaderElection \
+  --leaseDuration=15 \
+  --leaseRenewDuration=10 \
+  --leaseRetry=2 \
+  --prometheusHTTPServer='' |
+  gojq --yaml-input --yaml-output 'del(.metadata.creationTimestamp, .status) | .spec.containers[].imagePullPolicy |= "IfNotPresent"' |
+  sed "s/\"-99999\"/'{{ \`{{ .ControlPlaneEndpoint.Port }}\` }}'/" >"${ASSETS_DIR}/${FILE_NAME}"
+
+kubectl create configmap "{{ .Values.hooks.kubeVip.defaultTemplateConfigMap.name }}" --dry-run=client --output yaml \
+  --from-file "${ASSETS_DIR}/${FILE_NAME}" \
+  >"${ASSETS_DIR}/kube-vip-configmap.yaml"
+
+# add warning not to edit file directly
+cat <<EOF >"${GIT_REPO_ROOT}/charts/cluster-api-runtime-extensions-nutanix/templates/kube-vip/manifests/kube-vip-configmap.yaml"
+$(cat "${GIT_REPO_ROOT}/hack/license-header.yaml.txt")
+
+#=================================================================
+#                 DO NOT EDIT THIS FILE
+#  IT HAS BEEN GENERATED BY /hack/addons/update-kube-vip-manifests.sh
+#=================================================================
+{{- if .Values.hooks.kubeVip.defaultTemplateConfigMap.create }}
+$(cat "${ASSETS_DIR}/kube-vip-configmap.yaml")
+{{- end -}}
+EOF