nutanix-cloud-native · jimmidyson · Jun 26, 2024 · Jun 19, 2024
diff --git a/charts/cluster-api-runtime-extensions-nutanix/README.md b/charts/cluster-api-runtime-extensions-nutanix/README.md
@@ -60,6 +60,8 @@ A Helm chart for cluster-api-runtime-extensions-nutanix
 | hooks.cni.cilium.crsStrategy.defaultCiliumConfigMap.name | string | `"cilium"` |  |
 | hooks.cni.cilium.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` |  |
 | hooks.cni.cilium.helmAddonStrategy.defaultValueTemplateConfigMap.name | string | `"default-cilium-cni-helm-values-template"` |  |
+| hooks.csi.aws-ebs.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` |  |
+| hooks.csi.aws-ebs.helmAddonStrategy.defaultValueTemplateConfigMap.name | string | `"default-aws-ebs-csi-helm-values-template"` |  |
 | hooks.csi.local-path.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` |  |
 | hooks.csi.local-path.helmAddonStrategy.defaultValueTemplateConfigMap.name | string | `"default-local-path-csi-helm-values-template"` |  |
 | hooks.csi.nutanix.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` |  |

diff --git a/...i-runtime-extensions-nutanix/templates/csi/aws-ebs/manifests/helm-addon-installation.yaml b/...i-runtime-extensions-nutanix/templates/csi/aws-ebs/manifests/helm-addon-installation.yaml
@@ -0,0 +1,36 @@
+# Copyright 2024 Nutanix. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+{{- if (index .Values.hooks.csi "aws-ebs").helmAddonStrategy.defaultValueTemplateConfigMap.create }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: '{{ (index .Values.hooks.csi "aws-ebs").helmAddonStrategy.defaultValueTemplateConfigMap.name }}'
+data:
+  values.yaml: |-
+    controller:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: node-role.kubernetes.io/control-plane
+                    operator: Exists
+      tolerations:
+        - key: CriticalAddonsOnly
+          operator: Exists
+        - effect: NoExecute
+          operator: Exists
+          tolerationSeconds: 300
+        - effect: NoSchedule
+          key: node-role.kubernetes.io/master
+          operator: Exists
+        - effect: NoSchedule
+          key: node-role.kubernetes.io/control-plane
+          operator: Exists
+    node:
+      priorityClassName: system-node-critical
+    sidecars:
+      snapshotter:
+        forceEnable: true
+{{- end -}}
diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/deployment.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/deployment.yaml
@@ -37,6 +37,7 @@ spec:
         - --cni.cilium.helm-addon.default-values-template-configmap-name={{ .Values.hooks.cni.cilium.helmAddonStrategy.defaultValueTemplateConfigMap.name }}
         - --nfd.helm-addon.default-values-template-configmap-name={{ .Values.hooks.nfd.helmAddonStrategy.defaultValueTemplateConfigMap.name }}
         - --csi.local-path.helm-addon.default-values-template-configmap-name={{ (index .Values.hooks.csi "local-path").helmAddonStrategy.defaultValueTemplateConfigMap.name }}
+        - --csi.aws-ebs.helm-addon.default-values-template-configmap-name={{ (index .Values.hooks.csi "aws-ebs").helmAddonStrategy.defaultValueTemplateConfigMap.name }}
         {{- range $key, $value := .Values.extraArgs }}
         - --{{ $key }}={{ $value }}
         {{- end }}

diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml
@@ -7,6 +7,10 @@
 #=================================================================
 apiVersion: v1
 data:
+  aws-ebs-csi: |
+    ChartName: aws-ebs-csi-driver
+    ChartVersion: 2.28.1
+    RepositoryURL: {{ if .Values.selfHostedRegistry }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://kubernetes-sigs.github.io/aws-ebs-csi-driver{{ end }}
   cilium: |
     ChartName: cilium
     ChartVersion: 1.15.5

diff --git a/charts/cluster-api-runtime-extensions-nutanix/values.schema.json b/charts/cluster-api-runtime-extensions-nutanix/values.schema.json
@@ -281,6 +281,27 @@
                 },
                 "csi": {
                     "properties": {
+                        "aws-ebs": {
+                            "properties": {
+                                "helmAddonStrategy": {
+                                    "properties": {
+                                        "defaultValueTemplateConfigMap": {
+                                            "properties": {
+                                                "create": {
+                                                    "type": "boolean"
+                                                },
+                                                "name": {
+                                                    "type": "string"
+                                                }
+                                            },
+                                            "type": "object"
+                                        }
+                                    },
+                                    "type": "object"
+                                }
+                            },
+                            "type": "object"
+                        },
                         "local-path": {
                             "properties": {
                                 "helmAddonStrategy": {

diff --git a/charts/cluster-api-runtime-extensions-nutanix/values.yaml b/charts/cluster-api-runtime-extensions-nutanix/values.yaml
@@ -49,6 +49,11 @@ hooks:
         defaultValueTemplateConfigMap:
           create: true
           name: default-nutanix-csi-helm-values-template
+    aws-ebs:
+      helmAddonStrategy:
+        defaultValueTemplateConfigMap:
+          create: true
+          name: default-aws-ebs-csi-helm-values-template
     local-path:
       helmAddonStrategy:
         defaultValueTemplateConfigMap:

diff --git a/examples/capi-quick-start/aws-cluster-calico-helm-addon.yaml b/examples/capi-quick-start/aws-cluster-calico-helm-addon.yaml
@@ -35,7 +35,7 @@ spec:
               aws-ebs:
                 storageClassConfigs:
                   default: {}
-                strategy: ClusterResourceSet
+                strategy: HelmAddon
           nfd:
             strategy: HelmAddon
         aws:

diff --git a/examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml b/examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml
@@ -35,7 +35,7 @@ spec:
               aws-ebs:
                 storageClassConfigs:
                   default: {}
-                strategy: ClusterResourceSet
+                strategy: HelmAddon
           nfd:
             strategy: HelmAddon
         aws:

diff --git a/hack/addons/kustomize/aws-ebs-csi/storage-class.yaml b/hack/addons/kustomize/aws-ebs-csi/storage-class.yaml
diff --git a/hack/addons/mindthegap-helm-registry/repos.yaml b/hack/addons/mindthegap-helm-registry/repos.yaml
@@ -6,6 +6,11 @@
 #  IT HAS BEEN GENERATED BY /hack/addons/generate-mindthegap-repofile.sh
 #=================================================================
 repositories:
+  aws-ebs-csi-driver:
+    repoURL: https://kubernetes-sigs.github.io/aws-ebs-csi-driver
+    charts:
+      aws-ebs-csi-driver:
+      - 2.28.1
   cilium:
     repoURL: https://helm.cilium.io/
     charts:

diff --git a/hack/examples/overlays/clusters/aws/calico/crs/kustomization.yaml.tmpl b/hack/examples/overlays/clusters/aws/calico/crs/kustomization.yaml.tmpl
@@ -17,3 +17,6 @@ patches:
   - target:
       kind: Cluster
     path: ../../../../../patches/crs-strategy.yaml
+  - target:
+      kind: Cluster
+    path: ../../../../../patches/aws/csi-crs-strategy.yaml
diff --git a/hack/examples/overlays/clusters/aws/calico/helm-addon/kustomization.yaml.tmpl b/hack/examples/overlays/clusters/aws/calico/helm-addon/kustomization.yaml.tmpl
@@ -17,3 +17,6 @@ patches:
   - target:
       kind: Cluster
     path: ../../../../../patches/helm-addon-strategy.yaml
+  - target:
+      kind: Cluster
+    path: ../../../../../patches/aws/csi-helm-addon-strategy.yaml
diff --git a/hack/examples/overlays/clusters/aws/cilium/crs/kustomization.yaml.tmpl b/hack/examples/overlays/clusters/aws/cilium/crs/kustomization.yaml.tmpl
@@ -17,3 +17,6 @@ patches:
   - target:
       kind: Cluster
     path: ../../../../../patches/crs-strategy.yaml
+  - target:
+      kind: Cluster
+    path: ../../../../../patches/aws/csi-crs-strategy.yaml
diff --git a/hack/examples/overlays/clusters/aws/cilium/helm-addon/kustomization.yaml.tmpl b/hack/examples/overlays/clusters/aws/cilium/helm-addon/kustomization.yaml.tmpl
@@ -17,3 +17,6 @@ patches:
   - target:
       kind: Cluster
     path: ../../../../../patches/helm-addon-strategy.yaml
+  - target:
+      kind: Cluster
+    path: ../../../../../patches/aws/csi-helm-addon-strategy.yaml
diff --git a/hack/examples/patches/aws/csi-crs-strategy.yaml b/hack/examples/patches/aws/csi-crs-strategy.yaml
@@ -0,0 +1,6 @@
+# Copyright 2024 Nutanix. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+- op: "add"
+  path: "/spec/topology/variables/0/value/addons/csi/providers/aws-ebs/strategy"
+  value: ClusterResourceSet
diff --git a/hack/examples/patches/aws/csi-helm-addon-strategy.yaml b/hack/examples/patches/aws/csi-helm-addon-strategy.yaml
@@ -0,0 +1,6 @@
+# Copyright 2024 Nutanix. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+- op: "add"
+  path: "/spec/topology/variables/0/value/addons/csi/providers/aws-ebs/strategy"
+  value: HelmAddon
diff --git a/hack/examples/patches/aws/csi.yaml b/hack/examples/patches/aws/csi.yaml
@@ -9,6 +9,5 @@
       storageClassConfig: default
     providers:
       aws-ebs:
-        strategy: ClusterResourceSet
         storageClassConfigs:
           default: {}
diff --git a/hack/tools/helm-cm/main.go b/hack/tools/helm-cm/main.go
@@ -154,7 +154,6 @@ func createConfigMapFromDir(kustomizeDir string) (*corev1.ConfigMap, error) {
 
 var ignored = []string{
 	"aws-ccm",
-	"aws-ebs-csi",
 }
 
 func isIgnored(filepath string) bool {

diff --git a/make/addons.mk b/make/addons.mk
@@ -6,7 +6,7 @@ export CILIUM_VERSION := 1.15.5
 export NODE_FEATURE_DISCOVERY_VERSION := 0.15.2
 export CLUSTER_AUTOSCALER_VERSION := 9.37.0
 export AWS_CSI_SNAPSHOT_CONTROLLER_VERSION := v6.3.3
-export AWS_EBS_CSI_CHART_VERSION := v2.28.1
+export AWS_EBS_CSI_CHART_VERSION := 2.28.1
 export NUTANIX_STORAGE_CSI_CHART_VERSION := 3.0.0-beta.1912
 export NUTANIX_SNAPSHOT_CSI_CHART_VERSION := 6.3.2
 export LOCAL_PATH_CSI_CHART_VERSION := 0.0.29

diff --git a/pkg/handlers/generic/lifecycle/config/cm.go b/pkg/handlers/generic/lifecycle/config/cm.go
@@ -26,6 +26,7 @@ const (
 	NutanixCCM              Component = "nutanix-ccm"
 	MetalLB                 Component = "metallb"
 	LocalPathProvisionerCSI Component = "local-path-provisioner-csi"
+	AWSEBSCSI               Component = "aws-ebs-csi"
 )
 
 type HelmChartGetter struct {

diff --git a/pkg/handlers/generic/lifecycle/csi/awsebs/handler.go b/pkg/handlers/generic/lifecycle/csi/awsebs/handler.go
@@ -0,0 +1,116 @@
+// Copyright 2023 Nutanix. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package awsebs
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/go-logr/logr"
+	"github.com/spf13/pflag"
+	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/config"
+	csiutils "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/csi/utils"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/options"
+)
+
+const (
+	defaultHelmReleaseName      = "aws-ebs-csi-driver"
+	defaultHelmReleaseNamespace = "kube-system"
+)
+
+var DefaultStorageClassParameters = map[string]string{
+	"csi.storage.k8s.io/fstype": "ext4",
+	"type":                      "gp3",
+}
+
+type addonStrategy interface {
+	apply(
+		context.Context,
+		*clusterv1.Cluster,
+		string,
+		logr.Logger,
+	) error
+}
+
+type AWSEBSConfig struct {
+	*options.GlobalOptions
+
+	crsConfig       crsConfig
+	helmAddonConfig helmAddonConfig
+}
+
+func (a *AWSEBSConfig) AddFlags(prefix string, flags *pflag.FlagSet) {
+	a.crsConfig.AddFlags(prefix+".crs", flags)
+	a.helmAddonConfig.AddFlags(prefix+".helm-addon", flags)
+}
+
+type AWSEBS struct {
+	client              ctrlclient.Client
+	helmChartInfoGetter *config.HelmChartGetter
+	config              *AWSEBSConfig
+}
+
+func New(
+	c ctrlclient.Client,
+	cfg *AWSEBSConfig,
+	helmChartInfoGetter *config.HelmChartGetter,
+) *AWSEBS {
+	return &AWSEBS{
+		client:              c,
+		config:              cfg,
+		helmChartInfoGetter: helmChartInfoGetter,
+	}
+}
+
+func (a *AWSEBS) Apply(
+	ctx context.Context,
+	provider v1alpha1.CSIProvider,
+	defaultStorage v1alpha1.DefaultStorage,
+	cluster *clusterv1.Cluster,
+	log logr.Logger,
+) error {
+	var strategy addonStrategy
+	switch provider.Strategy {
+	case v1alpha1.AddonStrategyHelmAddon:
+		helmChart, err := a.helmChartInfoGetter.For(ctx, log, config.AWSEBSCSI)
+		if err != nil {
+			return fmt.Errorf("failed to get configuration to create helm addon: %w", err)
+		}
+		strategy = helmAddonStrategy{
+			config:    a.config.helmAddonConfig,
+			client:    a.client,
+			helmChart: helmChart,
+		}
+	case v1alpha1.AddonStrategyClusterResourceSet:
+		strategy = crsStrategy{
+			config: a.config.crsConfig,
+			client: a.client,
+		}
+	default:
+		return fmt.Errorf("strategy %s not implemented", strategy)
+	}
+
+	if err := strategy.apply(ctx, cluster, a.config.DefaultsNamespace(), log); err != nil {
+		return fmt.Errorf("failed to apply aws-ebs CSI addon: %w", err)
+	}
+
+	err := csiutils.CreateStorageClassesOnRemote(
+		ctx,
+		a.client,
+		provider.StorageClassConfigs,
+		cluster,
+		defaultStorage,
+		v1alpha1.CSIProviderAWSEBS,
+		v1alpha1.AWSEBSProvisioner,
+		DefaultStorageClassParameters,
+	)
+	if err != nil {
+		return fmt.Errorf("error creating StorageClasses for the AWS EBS CSI driver: %w", err)
+	}
+	return nil
+}