feat: patch the cluster-autoscaler annotations

Author: dkoshkin

README.md

@@ -73,7 +73,7 @@ To create an example cluster:
 
 ```shell
 clusterctl generate cluster docker-quick-start-helm-addon-cilium \
-  --from examples/capi-quick-start/docker-cluster-cilium-helm-addon.yaml \
+  --from examples/capi-quick-start/docker-cluster-calico-crs.yaml \
   --kubernetes-version v1.29.1 \
   --worker-machine-count 1 | \
   kubectl apply --server-side -f -

api/v1alpha1/node_types.go

@@ -10,14 +10,14 @@ import (
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 )
 
-//+kubebuilder:object:root=true
+// +kubebuilder:object:root=true
 
 // NodeConfig is the Schema for the workerconfigs API.
 type NodeConfig struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`
 
-	//+optional
+	// +optional
 	Spec NodeConfigSpec `json:"spec,omitempty"`
 }
 
@@ -29,6 +29,8 @@ type NodeConfigSpec struct {
 	AWS *AWSNodeSpec `json:"aws,omitempty"`
 	// +optional
 	Docker *DockerNodeSpec `json:"docker,omitempty"`
+
+	GenericNodeConfig `json:",inline"`
 }
 
 func (s NodeConfigSpec) VariableSchema() clusterv1.VariableSchema {
@@ -54,14 +56,42 @@ func (s NodeConfigSpec) VariableSchema() clusterv1.VariableSchema {
 	return nodeConfigProps
 }
 
-type GenericNodeConfig struct{}
+type GenericNodeConfig struct {
+	// +optional
+	ClusterAutoscaler *ClusterAutoscalerNodeConfig `json:"clusterAutoscaler,omitempty"`
+}
+
+type ClusterAutoscalerNodeConfig struct {
+	MinimumSize int32 `json:"minimumSize"`
+	MaximumSize int32 `json:"maximumSize"`
+}
+
+func (ClusterAutoscalerNodeConfig) VariableSchema() clusterv1.VariableSchema {
+	return clusterv1.VariableSchema{
+		OpenAPIV3Schema: clusterv1.JSONSchemaProps{
+			Type: "object",
+			Properties: map[string]clusterv1.JSONSchemaProps{
+				"minimumSize": {
+					Description: "Minimum number of Machines the cluster-autoscaler can scale down to.",
+					Type:        "integer",
+				},
+				"maximumSize": {
+					Description: "Maximum number of Machines the cluster-autoscaler can scale up to.",
+					Type:        "integer",
+				},
+			},
+		},
+	}
+}
 
 func (GenericNodeConfig) VariableSchema() clusterv1.VariableSchema {
 	return clusterv1.VariableSchema{
 		OpenAPIV3Schema: clusterv1.JSONSchemaProps{
 			Description: "Node configuration",
 			Type:        "object",
-			Properties:  map[string]clusterv1.JSONSchemaProps{},
+			Properties: map[string]clusterv1.JSONSchemaProps{
+				"clusterAutoscaler": ClusterAutoscalerNodeConfig{}.VariableSchema().OpenAPIV3Schema,
+			},
 		},
 	}
 }

api/v1alpha1/zz_generated.deepcopy.go

@@ -34,6 +34,13 @@ func WorkersKubeadmConfigTemplateSelector() clusterv1.PatchSelector {
 	}
 }
 
+func MachineDeploymentSelector() clusterv1.PatchSelector {
+	return clusterv1.PatchSelector{
+		APIVersion: clusterv1.GroupVersion.String(),
+		Kind:       "MachineDeployment",
+	}
+}
+
 // InfrastructureCluster selector matches against infrastructure clusters.
 // Passing in the API version (not the API group) is required because different providers could support different API
 // versions. This also allows for a patch to select multiple infrastructure versions for the same provider.

common/pkg/capi/clustertopology/patches/selectors/selectors.go

@@ -29,13 +29,13 @@ spec:
     workers:
       machineDeployments:
         - class: default-worker
-          metadata:
-            annotations:
-              # Set the following annotations to configure the Cluster Autoscaler
-              # The initial MachineDeployment will have 1 Machine
-              cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "3"
-              cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "1"
           name: md-0
+          variables:
+            - name: workerConfig
+              value:
+                clusterAutoscaler:
+                  minimumSize: 1
+                  maximumSize: 3
           # Remove the replicas field, otherwise the topology controller will revert back the aytoscaler's changes
 ```
 

docs/content/addons/cluster-autoscaler.md

@@ -53,8 +53,5 @@ spec:
     workers:
       machineDeployments:
         - class: default-worker
-          metadata:
-            annotations:
-              cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "${WORKER_MACHINE_COUNT}"
-              cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "${WORKER_MACHINE_COUNT}"
           name: md-0
+          replicas: ${WORKER_MACHINE_COUNT}

examples/capi-quick-start/aws-cluster-calico-crs.yaml

@@ -53,8 +53,5 @@ spec:
     workers:
       machineDeployments:
         - class: default-worker
-          metadata:
-            annotations:
-              cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "${WORKER_MACHINE_COUNT}"
-              cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "${WORKER_MACHINE_COUNT}"
           name: md-0
+          replicas: ${WORKER_MACHINE_COUNT}

examples/capi-quick-start/aws-cluster-cilium-crs.yaml

@@ -34,8 +34,5 @@ spec:
     workers:
       machineDeployments:
         - class: default-worker
-          metadata:
-            annotations:
-              cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "${WORKER_MACHINE_COUNT}"
-              cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "${WORKER_MACHINE_COUNT}"
           name: md-0
+          replicas: ${WORKER_MACHINE_COUNT}

examples/capi-quick-start/docker-cluster-calico-crs.yaml

@@ -34,8 +34,5 @@ spec:
     workers:
       machineDeployments:
         - class: default-worker
-          metadata:
-            annotations:
-              cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "${WORKER_MACHINE_COUNT}"
-              cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "${WORKER_MACHINE_COUNT}"
           name: md-0
+          replicas: ${WORKER_MACHINE_COUNT}

examples/capi-quick-start/docker-cluster-cilium-crs.yaml

@@ -32,14 +32,6 @@ patches:
       path: "/spec/topology/variables/0/value/addons/clusterAutoscaler"
       value:
         strategy: ClusterResourceSet
-    - op: "remove"
-      path: "/spec/topology/workers/machineDeployments/0/replicas"
-    - op: "add"
-      path: "/spec/topology/workers/machineDeployments/0/metadata"
-      value:
-        annotations:
-          cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "$${WORKER_MACHINE_COUNT}"
-          cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "$${WORKER_MACHINE_COUNT}"
 
 
 # Delete all ClusterClass related resources to prevent duplicate resources because we use the same resources definition

hack/examples/bases/aws/calico/crs/kustomization.yaml.tmpl

@@ -32,14 +32,6 @@ patches:
       path: "/spec/topology/variables/0/value/addons/clusterAutoscaler"
       value:
         strategy: ClusterResourceSet
-    - op: "remove"
-      path: "/spec/topology/workers/machineDeployments/0/replicas"
-    - op: "add"
-      path: "/spec/topology/workers/machineDeployments/0/metadata"
-      value:
-        annotations:
-          cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "$${WORKER_MACHINE_COUNT}"
-          cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "$${WORKER_MACHINE_COUNT}"
 
 
 # Delete all ClusterClass related resources to prevent duplicate resources because we use the same resources definition

hack/examples/bases/aws/cilium/crs/kustomization.yaml.tmpl

@@ -32,11 +32,3 @@ patches:
       path: "/spec/topology/variables/0/value/addons/clusterAutoscaler"
       value:
         strategy: ClusterResourceSet
-    - op: "remove"
-      path: "/spec/topology/workers/machineDeployments/0/replicas"
-    - op: "add"
-      path: "/spec/topology/workers/machineDeployments/0/metadata"
-      value:
-        annotations:
-          cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "$${WORKER_MACHINE_COUNT}"
-          cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "$${WORKER_MACHINE_COUNT}"

hack/examples/bases/docker/calico/crs/kustomization.yaml.tmpl

@@ -32,11 +32,3 @@ patches:
       path: "/spec/topology/variables/0/value/addons/clusterAutoscaler"
       value:
         strategy: ClusterResourceSet
-    - op: "remove"
-      path: "/spec/topology/workers/machineDeployments/0/replicas"
-    - op: "add"
-      path: "/spec/topology/workers/machineDeployments/0/metadata"
-      value:
-        annotations:
-          cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "$${WORKER_MACHINE_COUNT}"
-          cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "$${WORKER_MACHINE_COUNT}"

hack/examples/bases/docker/cilium/crs/kustomization.yaml.tmpl

@@ -111,7 +111,3 @@ sed -i'' 's/${AMI_LOOKUP_ORG}/"${AMI_LOOKUP_ORG}"/' "${EXAMPLE_CLUSTERS_DIR}"/*.
 
 # TODO Remove once CAPA templates default to using external cloud provider.
 sed -i'' 's/cloud-provider:\ aws/cloud-provider:\ external/g' "${EXAMPLE_CLUSTERCLASSES_DIR}/aws-cluster-class.yaml"
-
-# TODO Remove once kustomize supports retaining quotes in what will be numeric values.
-#shellcheck disable=SC2016
-sed -i'' 's/size\: ${WORKER_MACHINE_COUNT}/size\: "${WORKER_MACHINE_COUNT}"/' "${EXAMPLE_CLUSTERS_DIR}"/*.yaml

hack/examples/sync.sh

@@ -17,6 +17,7 @@ import (
 	"github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/aws/mutation/region"
 	"github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/aws/mutation/securitygroups"
 	genericmutation "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation"
+	"github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/clusterautoscaler"
 )
 
 // MetaPatchHandler returns a meta patch handler for mutating CAPA clusters.
@@ -48,6 +49,7 @@ func MetaWorkerPatchHandler() handlers.Named {
 		instancetype.NewWorkerPatch(),
 		ami.NewWorkerPatch(),
 		securitygroups.NewWorkerPatch(),
+		clusterautoscaler.NewWorkerPatch(),
 	}
 
 	return mutation.NewMetaGeneratePatchesHandler(

pkg/handlers/aws/mutation/metapatch_handler.go

@@ -10,6 +10,7 @@ import (
 	"github.com/d2iq-labs/capi-runtime-extensions/common/pkg/capi/clustertopology/handlers/mutation"
 	"github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/docker/mutation/customimage"
 	genericmutation "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation"
+	"github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/mutation/clusterautoscaler"
 )
 
 // MetaPatchHandler returns a meta patch handler for mutating CAPD clusters.
@@ -31,6 +32,7 @@ func MetaPatchHandler(mgr manager.Manager) handlers.Named {
 func MetaWorkerPatchHandler() handlers.Named {
 	patchHandlers := []mutation.MetaMutator{
 		customimage.NewWorkerPatch(),
+		clusterautoscaler.NewWorkerPatch(),
 	}
 
 	return mutation.NewMetaGeneratePatchesHandler(

pkg/handlers/docker/mutation/metapatch_handler.go

@@ -0,0 +1,100 @@
+// Copyright 2023 D2iQ, Inc. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package clusterautoscaler
+
+import (
+	"context"
+	"fmt"
+
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/d2iq-labs/capi-runtime-extensions/api/v1alpha1"
+	"github.com/d2iq-labs/capi-runtime-extensions/common/pkg/capi/clustertopology/patches"
+	"github.com/d2iq-labs/capi-runtime-extensions/common/pkg/capi/clustertopology/patches/selectors"
+	"github.com/d2iq-labs/capi-runtime-extensions/common/pkg/capi/clustertopology/variables"
+	"github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/workerconfig"
+)
+
+const (
+	// VariableName is the external patch variable name.
+	VariableName = "clusterAutoscaler"
+)
+
+type clusterAuotscalerPatchHandler struct {
+	variableName      string
+	variableFieldPath []string
+}
+
+func NewWorkerPatch() *clusterAuotscalerPatchHandler {
+	return newClusterAutoscalerPatchHandler(workerconfig.MetaVariableName, VariableName)
+}
+
+func newClusterAutoscalerPatchHandler(
+	variableName string,
+	variableFieldPath ...string,
+) *clusterAuotscalerPatchHandler {
+	return &clusterAuotscalerPatchHandler{
+		variableName:      variableName,
+		variableFieldPath: variableFieldPath,
+	}
+}
+
+func (h *clusterAuotscalerPatchHandler) Mutate(
+	ctx context.Context,
+	obj *unstructured.Unstructured,
+	vars map[string]apiextensionsv1.JSON,
+	holderRef runtimehooksv1.HolderReference,
+	_ client.ObjectKey,
+) error {
+	log := ctrl.LoggerFrom(ctx).WithValues(
+		"holderRef", holderRef,
+	)
+
+	clusterAutoscaler, found, err := variables.Get[v1alpha1.ClusterAutoscalerNodeConfig](
+		vars,
+		h.variableName,
+		h.variableFieldPath...,
+	)
+	if err != nil {
+		return err
+	}
+	if !found {
+		log.V(5).Info("clusterAutoscaler variable not defined")
+		return nil
+	}
+	log.V(5).Info("clusterAutoscaler variable defined")
+
+	log.V(5).Info("clusterAutoscaler variable defined")
+	log = log.WithValues(
+		"variableName",
+		h.variableName,
+		"variableFieldPath",
+		h.variableFieldPath,
+		"variableValue",
+		clusterAutoscaler,
+	)
+
+	return patches.MutateIfApplicable(
+		obj, vars, &holderRef, selectors.MachineDeploymentSelector(), log,
+		func(obj *clusterv1.MachineDeployment) error {
+			log.WithValues(
+				"patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(),
+				"patchedObjectName", client.ObjectKeyFromObject(obj),
+			).Info("setting annotations for cluster-autoscaler in MachineDeployment")
+
+			if obj.Annotations == nil {
+				obj.Annotations = map[string]string{}
+			}
+			obj.Annotations[clusterv1.AutoscalerMinSizeAnnotation] = fmt.Sprint(clusterAutoscaler.MinimumSize)
+			obj.Annotations[clusterv1.AutoscalerMaxSizeAnnotation] = fmt.Sprint(clusterAutoscaler.MaximumSize)
+
+			return nil
+		},
+	)
+}