feat: add Cluster Autoscaler Addon with HelmAddon (nutanix-cloud-native#427)

Co-authored-by: Jimmi Dyson <[email protected]>

Author: dkoshkin

api/v1alpha1/addon_types.go

@@ -125,6 +125,7 @@ func (ClusterAutoscaler) VariableSchema() clusterv1.VariableSchema {
 					Type: "string",
 					Enum: variables.MustMarshalValuesToEnumJSON(
 						AddonStrategyClusterResourceSet,
+						AddonStrategyHelmAddon,
 					),
 				},
 			},

charts/capi-runtime-extensions/README.md

@@ -32,6 +32,8 @@ A Helm chart for capi-runtime-extensions
 | deployment.replicas | int | `1` |  |
 | env | object | `{}` |  |
 | hooks.clusterAutoscaler.crsStrategy.defaultInstallationConfigMap.name | string | `"cluster-autoscaler"` |  |
+| hooks.clusterAutoscaler.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` |  |
+| hooks.clusterAutoscaler.helmAddonStrategy.defaultValueTemplateConfigMap.name | string | `"default-cluster-autoscaler-helm-values-template"` |  |
 | hooks.cni.calico.crsStrategy.defaultInstallationConfigMaps.AWSCluster.configMap.content | string | `""` |  |
 | hooks.cni.calico.crsStrategy.defaultInstallationConfigMaps.AWSCluster.configMap.name | string | `"calico-cni-crs-installation-awscluster"` |  |
 | hooks.cni.calico.crsStrategy.defaultInstallationConfigMaps.AWSCluster.create | bool | `true` |  |

charts/capi-runtime-extensions/templates/cluster-autoscaler/manifests/cluster-autoscaler-configmap.yaml

@@ -221,4 +221,5 @@ data:
 kind: ConfigMap
 metadata:
   creationTimestamp: null
-  name: cluster-autoscaler
+  name: '{{ .Values.hooks.clusterAutoscaler.crsStrategy.defaultInstallationConfigMap.name
+    }}'

charts/capi-runtime-extensions/templates/cluster-autoscaler/manifests/helm-addon-installation.yaml

@@ -0,0 +1,48 @@
+# Copyright 2024 D2iQ, Inc. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+{{- if .Values.hooks.clusterAutoscaler.helmAddonStrategy.defaultValueTemplateConfigMap.create }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: '{{ .Values.hooks.clusterAutoscaler.helmAddonStrategy.defaultValueTemplateConfigMap.name }}'
+data:
+  values.yaml: |-
+    ---
+    fullnameOverride: "cluster-autoscaler-{{ `{{ .Cluster.metadata.name }}` }}"
+
+    cloudProvider: clusterapi
+
+    # Always trigger a scale-out if replicas are less than the min.
+    extraArgs:
+      enforce-node-group-min-size: true
+
+    # Enable it to run in a 1 Node cluster.
+    tolerations:
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/control-plane
+
+    # Limit a single cluster-autoscaler Deployment to a single Cluster.
+    autoDiscovery:
+      clusterName: "{{ `{{ .Cluster.metadata.name }}` }}"
+      # The controller failed with an RBAC error trying to watch CAPI objects at the cluster scope without this.
+      labels:
+        - namespace: "{{ `{{ .Cluster.metadata.namespace }}` }}"
+
+    clusterAPIConfigMapsNamespace: "{{ `{{ .Cluster.metadata.namespace }}` }}"
+    # For workload clusters it is not possible to use the in-cluster client.
+    # To simplify the configuration, use the admin kubeconfig generated by CAPI for all clusters.
+    clusterAPIMode: kubeconfig-incluster
+    clusterAPIWorkloadKubeconfigPath: /cluster/kubeconfig
+    extraVolumeSecrets:
+      kubeconfig:
+        name: "{{ `{{ .Cluster.metadata.name }}` }}-kubeconfig"
+        mountPath: /cluster
+        readOnly: true
+        items:
+          - key: value
+            path: kubeconfig
+    rbac:
+      # Create a Role instead of a ClusterRoles to update cluster-api objects
+      clusterScoped: false
+{{- end -}}

charts/capi-runtime-extensions/values.yaml

@@ -47,6 +47,10 @@ hooks:
     crsStrategy:
       defaultInstallationConfigMap:
         name: cluster-autoscaler
+    helmAddonStrategy:
+      defaultValueTemplateConfigMap:
+        create: true
+        name: default-cluster-autoscaler-helm-values-template
 
 deployDefaultClusterClasses: true
 

examples/capi-quick-start/aws-cluster-calico-helm-addon.yaml

@@ -21,6 +21,8 @@ spec:
       - name: clusterConfig
         value:
           addons:
+            clusterAutoscaler:
+              strategy: HelmAddon
             cni:
               provider: Calico
               strategy: HelmAddon
@@ -51,5 +53,8 @@ spec:
     workers:
       machineDeployments:
         - class: default-worker
+          metadata:
+            annotations:
+              cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "${WORKER_MACHINE_COUNT}"
+              cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "${WORKER_MACHINE_COUNT}"
           name: md-0
-          replicas: ${WORKER_MACHINE_COUNT}

examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml

@@ -21,6 +21,8 @@ spec:
       - name: clusterConfig
         value:
           addons:
+            clusterAutoscaler:
+              strategy: HelmAddon
             cni:
               provider: Cilium
               strategy: HelmAddon
@@ -51,5 +53,8 @@ spec:
     workers:
       machineDeployments:
         - class: default-worker
+          metadata:
+            annotations:
+              cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "${WORKER_MACHINE_COUNT}"
+              cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "${WORKER_MACHINE_COUNT}"
           name: md-0
-          replicas: ${WORKER_MACHINE_COUNT}

examples/capi-quick-start/docker-cluster-calico-helm-addon.yaml

@@ -22,6 +22,8 @@ spec:
       - name: clusterConfig
         value:
           addons:
+            clusterAutoscaler:
+              strategy: HelmAddon
             cni:
               provider: Calico
               strategy: HelmAddon
@@ -32,5 +34,8 @@ spec:
     workers:
       machineDeployments:
         - class: default-worker
+          metadata:
+            annotations:
+              cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "${WORKER_MACHINE_COUNT}"
+              cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "${WORKER_MACHINE_COUNT}"
           name: md-0
-          replicas: ${WORKER_MACHINE_COUNT}

examples/capi-quick-start/docker-cluster-cilium-helm-addon.yaml

@@ -22,6 +22,8 @@ spec:
       - name: clusterConfig
         value:
           addons:
+            clusterAutoscaler:
+              strategy: HelmAddon
             cni:
               provider: Cilium
               strategy: HelmAddon
@@ -32,5 +34,8 @@ spec:
     workers:
       machineDeployments:
         - class: default-worker
+          metadata:
+            annotations:
+              cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "${WORKER_MACHINE_COUNT}"
+              cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "${WORKER_MACHINE_COUNT}"
           name: md-0
-          replicas: ${WORKER_MACHINE_COUNT}

hack/addons/update-cluster-autoscaler.sh

@@ -25,7 +25,7 @@ cp "${KUSTOMIZE_BASE_DIR}"/*.yaml "${ASSETS_DIR}"
 
 kustomize build --enable-helm "${ASSETS_DIR}" >"${ASSETS_DIR}/${FILE_NAME}"
 
-kubectl create configmap cluster-autoscaler --dry-run=client --output yaml \
+kubectl create configmap "{{ .Values.hooks.clusterAutoscaler.crsStrategy.defaultInstallationConfigMap.name }}" --dry-run=client --output yaml \
   --from-file "${ASSETS_DIR}/${FILE_NAME}" \
   >"${ASSETS_DIR}/cluster-autoscaler-configmap.yaml"
 

hack/examples/bases/aws/calico/crs/kustomization.yaml.tmpl

@@ -32,15 +32,6 @@ patches:
       path: "/spec/topology/variables/0/value/addons/clusterAutoscaler"
       value:
         strategy: ClusterResourceSet
-    - op: "remove"
-      path: "/spec/topology/workers/machineDeployments/0/replicas"
-    - op: "add"
-      path: "/spec/topology/workers/machineDeployments/0/metadata"
-      value:
-        annotations:
-          cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "$${WORKER_MACHINE_COUNT}"
-          cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "$${WORKER_MACHINE_COUNT}"
-
 
 # Delete all ClusterClass related resources to prevent duplicate resources because we use the same resources definition
 # above- we only actually want the Cluster resource to mutate here.

hack/examples/bases/aws/calico/helm-addon/kustomization.yaml.tmpl

@@ -28,3 +28,7 @@ patches:
       path: "/spec/topology/variables/0/value/addons/nfd"
       value:
         strategy: HelmAddon
+    - op: "add"
+      path: "/spec/topology/variables/0/value/addons/clusterAutoscaler"
+      value:
+        strategy: HelmAddon

hack/examples/bases/aws/cilium/crs/kustomization.yaml.tmpl

@@ -32,14 +32,6 @@ patches:
       path: "/spec/topology/variables/0/value/addons/clusterAutoscaler"
       value:
         strategy: ClusterResourceSet
-    - op: "remove"
-      path: "/spec/topology/workers/machineDeployments/0/replicas"
-    - op: "add"
-      path: "/spec/topology/workers/machineDeployments/0/metadata"
-      value:
-        annotations:
-          cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "$${WORKER_MACHINE_COUNT}"
-          cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "$${WORKER_MACHINE_COUNT}"
 
 
 # Delete all ClusterClass related resources to prevent duplicate resources because we use the same resources definition

hack/examples/bases/aws/cilium/helm-addon/kustomization.yaml.tmpl

@@ -28,6 +28,10 @@ patches:
       path: "/spec/topology/variables/0/value/addons/nfd"
       value:
         strategy: HelmAddon
+    - op: "add"
+      path: "/spec/topology/variables/0/value/addons/clusterAutoscaler"
+      value:
+        strategy: HelmAddon
 
 # Delete all ClusterClass related resources to prevent duplicate resources because we use the same resources definition
 # above- we only actually want the Cluster resource to mutate here.

hack/examples/bases/docker/calico/crs/kustomization.yaml.tmpl

@@ -32,11 +32,3 @@ patches:
       path: "/spec/topology/variables/0/value/addons/clusterAutoscaler"
       value:
         strategy: ClusterResourceSet
-    - op: "remove"
-      path: "/spec/topology/workers/machineDeployments/0/replicas"
-    - op: "add"
-      path: "/spec/topology/workers/machineDeployments/0/metadata"
-      value:
-        annotations:
-          cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "$${WORKER_MACHINE_COUNT}"
-          cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "$${WORKER_MACHINE_COUNT}"

hack/examples/bases/docker/calico/helm-addon/kustomization.yaml.tmpl

@@ -28,3 +28,7 @@ patches:
       path: "/spec/topology/variables/0/value/addons/nfd"
       value:
         strategy: HelmAddon
+    - op: "add"
+      path: "/spec/topology/variables/0/value/addons/clusterAutoscaler"
+      value:
+        strategy: HelmAddon

hack/examples/bases/docker/cilium/crs/kustomization.yaml.tmpl

@@ -32,11 +32,3 @@ patches:
       path: "/spec/topology/variables/0/value/addons/clusterAutoscaler"
       value:
         strategy: ClusterResourceSet
-    - op: "remove"
-      path: "/spec/topology/workers/machineDeployments/0/replicas"
-    - op: "add"
-      path: "/spec/topology/workers/machineDeployments/0/metadata"
-      value:
-        annotations:
-          cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "$${WORKER_MACHINE_COUNT}"
-          cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "$${WORKER_MACHINE_COUNT}"

hack/examples/bases/docker/cilium/helm-addon/kustomization.yaml.tmpl

@@ -28,3 +28,7 @@ patches:
       path: "/spec/topology/variables/0/value/addons/nfd"
       value:
         strategy: HelmAddon
+    - op: "add"
+      path: "/spec/topology/variables/0/value/addons/clusterAutoscaler"
+      value:
+        strategy: HelmAddon

hack/examples/kustomization.yaml.tmpl

@@ -32,3 +32,11 @@ patches:
     - op: "add"
       path: "/spec/clusterNetwork/serviceDomain"
       value: "cluster.local"
+    - op: "remove"
+      path: "/spec/topology/workers/machineDeployments/0/replicas"
+    - op: "add"
+      path: "/spec/topology/workers/machineDeployments/0/metadata"
+      value:
+        annotations:
+          cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "$${WORKER_MACHINE_COUNT}"
+          cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "$${WORKER_MACHINE_COUNT}"

pkg/handlers/generic/lifecycle/clusterautoscaler/handler.go

@@ -18,18 +18,28 @@ import (
 	"github.com/d2iq-labs/capi-runtime-extensions/common/pkg/capi/clustertopology/handlers/lifecycle"
 	"github.com/d2iq-labs/capi-runtime-extensions/common/pkg/capi/clustertopology/variables"
 	"github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/clusterconfig"
+	"github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/options"
 )
 
 type addonStrategy interface {
-	apply(context.Context, *runtimehooksv1.AfterControlPlaneInitializedRequest, logr.Logger) error
+	apply(
+		context.Context,
+		*runtimehooksv1.AfterControlPlaneInitializedRequest,
+		string,
+		logr.Logger,
+	) error
 }
 
 type Config struct {
-	crsConfig crsConfig
+	*options.GlobalOptions
+
+	crsConfig       crsConfig
+	helmAddonConfig helmAddonConfig
 }
 
 func (c *Config) AddFlags(prefix string, flags *pflag.FlagSet) {
 	c.crsConfig.AddFlags(prefix+".crs", flags)
+	c.helmAddonConfig.AddFlags(prefix+".helm-addon", flags)
 }
 
 type DefaultClusterAutoscaler struct {
@@ -106,6 +116,11 @@ func (n *DefaultClusterAutoscaler) AfterControlPlaneInitialized(
 			config: n.config.crsConfig,
 			client: n.client,
 		}
+	case v1alpha1.AddonStrategyHelmAddon:
+		strategy = helmAddonStrategy{
+			config: n.config.helmAddonConfig,
+			client: n.client,
+		}
 	default:
 		resp.SetStatus(runtimehooksv1.ResponseStatusFailure)
 		resp.SetMessage(
@@ -114,7 +129,7 @@ func (n *DefaultClusterAutoscaler) AfterControlPlaneInitialized(
 		return
 	}
 
-	if err := strategy.apply(ctx, req, log); err != nil {
+	if err = strategy.apply(ctx, req, n.config.DefaultsNamespace(), log); err != nil {
 		resp.SetStatus(runtimehooksv1.ResponseStatusFailure)
 		resp.SetMessage(err.Error())
 		return

pkg/handlers/generic/lifecycle/clusterautoscaler/strategy_crs.go

@@ -15,7 +15,6 @@ import (
 	runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
 	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
 
-	capiutils "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/capi/utils"
 	"github.com/d2iq-labs/capi-runtime-extensions/common/pkg/k8s/client"
 	"github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/lifecycle/utils"
 )
@@ -51,6 +50,7 @@ type crsStrategy struct {
 func (s crsStrategy) apply(
 	ctx context.Context,
 	req *runtimehooksv1.AfterControlPlaneInitializedRequest,
+	defaultsNamespace string,
 	log logr.Logger,
 ) error {
 	defaultCM := &corev1.ConfigMap{
@@ -59,7 +59,7 @@ func (s crsStrategy) apply(
 			Kind:       "ConfigMap",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Namespace: s.config.defaultsNamespace,
+			Namespace: defaultsNamespace,
 			Name:      s.config.defaultClusterAutoscalerConfigMap,
 		},
 	}
@@ -97,22 +97,12 @@ func (s crsStrategy) apply(
 		)
 	}
 
-	existingManagementCluster, err := capiutils.ManagementCluster(ctx, s.client)
-	if err != nil {
-		return fmt.Errorf(
-			"failed to get management Cluster: %w",
-			err,
-		)
-	}
 	// The cluster-autoscaler is different from other addons.
 	// It requires all resources to be created in the management cluster,
 	// which means creating the ClusterResourceSet always targeting the management cluster.
-	// In most cases, target the management cluster.
-	// But if existingManagementCluster is nil, i.e. when s.client points to a bootstrap cluster,
-	// target the cluster and assume that will become the management cluster.
-	targetCluster := existingManagementCluster
-	if targetCluster == nil {
-		targetCluster = cluster
+	targetCluster, err := findTargetCluster(ctx, s.client, cluster)
+	if err != nil {
+		return err
 	}
 
 	// In the case when existingManagementCluster is nil, i.e. when s.client points to a bootstrap cluster,