From 68020def306c69f7f216667eab2af0f185f2ae30 Mon Sep 17 00:00:00 2001 From: Jimmi Dyson Date: Wed, 19 Jun 2024 14:06:46 +0100 Subject: [PATCH] feat: Support HelmAddon strategy for AWS EBS Also add e2e tests for AWS EBS deployment. --- .../README.md | 2 + .../manifests/helm-addon-installation.yaml | 36 +++++ .../templates/deployment.yaml | 1 + .../templates/helm-config.yaml | 4 + .../values.schema.json | 21 +++ .../values.yaml | 5 + .../aws-cluster-calico-helm-addon.yaml | 2 +- .../aws-cluster-cilium-helm-addon.yaml | 2 +- .../kustomize/aws-ebs-csi/storage-class.yaml | 12 -- .../mindthegap-helm-registry/repos.yaml | 5 + .../aws/calico/crs/kustomization.yaml.tmpl | 3 + .../calico/helm-addon/kustomization.yaml.tmpl | 3 + .../aws/cilium/crs/kustomization.yaml.tmpl | 3 + .../cilium/helm-addon/kustomization.yaml.tmpl | 3 + .../patches/aws/csi-crs-strategy.yaml | 6 + .../patches/aws/csi-helm-addon-strategy.yaml | 6 + hack/examples/patches/aws/csi.yaml | 1 - hack/tools/helm-cm/main.go | 1 - make/addons.mk | 2 +- pkg/handlers/generic/lifecycle/config/cm.go | 1 + .../generic/lifecycle/csi/awsebs/handler.go | 116 +++++++++++++++ .../handler.go => awsebs/strategy_crs.go} | 83 +++-------- .../csi/awsebs/strategy_helmaddon.go | 97 +++++++++++++ pkg/handlers/generic/lifecycle/handlers.go | 10 +- test/e2e/csi_helpers.go | 135 ++++++++++++++++-- 25 files changed, 462 insertions(+), 98 deletions(-) create mode 100644 charts/cluster-api-runtime-extensions-nutanix/templates/csi/aws-ebs/manifests/helm-addon-installation.yaml delete mode 100644 hack/addons/kustomize/aws-ebs-csi/storage-class.yaml create mode 100644 hack/examples/patches/aws/csi-crs-strategy.yaml create mode 100644 hack/examples/patches/aws/csi-helm-addon-strategy.yaml create mode 100644 pkg/handlers/generic/lifecycle/csi/awsebs/handler.go rename pkg/handlers/generic/lifecycle/csi/{aws-ebs/handler.go => awsebs/strategy_crs.go} (50%) create mode 100644 pkg/handlers/generic/lifecycle/csi/awsebs/strategy_helmaddon.go diff --git a/charts/cluster-api-runtime-extensions-nutanix/README.md b/charts/cluster-api-runtime-extensions-nutanix/README.md index 82c43de62..fcfae8a12 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/README.md +++ b/charts/cluster-api-runtime-extensions-nutanix/README.md @@ -60,6 +60,8 @@ A Helm chart for cluster-api-runtime-extensions-nutanix | hooks.cni.cilium.crsStrategy.defaultCiliumConfigMap.name | string | `"cilium"` | | | hooks.cni.cilium.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` | | | hooks.cni.cilium.helmAddonStrategy.defaultValueTemplateConfigMap.name | string | `"default-cilium-cni-helm-values-template"` | | +| hooks.csi.aws-ebs.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` | | +| hooks.csi.aws-ebs.helmAddonStrategy.defaultValueTemplateConfigMap.name | string | `"default-aws-ebs-csi-helm-values-template"` | | | hooks.csi.local-path.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` | | | hooks.csi.local-path.helmAddonStrategy.defaultValueTemplateConfigMap.name | string | `"default-local-path-csi-helm-values-template"` | | | hooks.csi.nutanix.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` | | diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/csi/aws-ebs/manifests/helm-addon-installation.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/csi/aws-ebs/manifests/helm-addon-installation.yaml new file mode 100644 index 000000000..54c2b4781 --- /dev/null +++ b/charts/cluster-api-runtime-extensions-nutanix/templates/csi/aws-ebs/manifests/helm-addon-installation.yaml @@ -0,0 +1,36 @@ +# Copyright 2024 Nutanix. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{{- if (index .Values.hooks.csi "aws-ebs").helmAddonStrategy.defaultValueTemplateConfigMap.create }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: '{{ (index .Values.hooks.csi "aws-ebs").helmAddonStrategy.defaultValueTemplateConfigMap.name }}' +data: + values.yaml: |- + controller: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + tolerationSeconds: 300 + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + node: + priorityClassName: system-node-critical + sidecars: + snapshotter: + forceEnable: true +{{- end -}} diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/deployment.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/deployment.yaml index e55921d38..6e421f3fd 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/templates/deployment.yaml +++ b/charts/cluster-api-runtime-extensions-nutanix/templates/deployment.yaml @@ -37,6 +37,7 @@ spec: - --cni.cilium.helm-addon.default-values-template-configmap-name={{ .Values.hooks.cni.cilium.helmAddonStrategy.defaultValueTemplateConfigMap.name }} - --nfd.helm-addon.default-values-template-configmap-name={{ .Values.hooks.nfd.helmAddonStrategy.defaultValueTemplateConfigMap.name }} - --csi.local-path.helm-addon.default-values-template-configmap-name={{ (index .Values.hooks.csi "local-path").helmAddonStrategy.defaultValueTemplateConfigMap.name }} + - --csi.aws-ebs.helm-addon.default-values-template-configmap-name={{ (index .Values.hooks.csi "aws-ebs").helmAddonStrategy.defaultValueTemplateConfigMap.name }} {{- range $key, $value := .Values.extraArgs }} - --{{ $key }}={{ $value }} {{- end }} diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml index 497b3ce6b..d0aab0f22 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml +++ b/charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml @@ -7,6 +7,10 @@ #================================================================= apiVersion: v1 data: + aws-ebs-csi: | + ChartName: aws-ebs-csi-driver + ChartVersion: 2.28.1 + RepositoryURL: {{ if .Values.selfHostedRegistry }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://kubernetes-sigs.github.io/aws-ebs-csi-driver{{ end }} cilium: | ChartName: cilium ChartVersion: 1.15.5 diff --git a/charts/cluster-api-runtime-extensions-nutanix/values.schema.json b/charts/cluster-api-runtime-extensions-nutanix/values.schema.json index 3c60a0d68..1e713fcd8 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/values.schema.json +++ b/charts/cluster-api-runtime-extensions-nutanix/values.schema.json @@ -281,6 +281,27 @@ }, "csi": { "properties": { + "aws-ebs": { + "properties": { + "helmAddonStrategy": { + "properties": { + "defaultValueTemplateConfigMap": { + "properties": { + "create": { + "type": "boolean" + }, + "name": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + } + }, + "type": "object" + }, "local-path": { "properties": { "helmAddonStrategy": { diff --git a/charts/cluster-api-runtime-extensions-nutanix/values.yaml b/charts/cluster-api-runtime-extensions-nutanix/values.yaml index 9f8428a1c..dbbda8ce0 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/values.yaml +++ b/charts/cluster-api-runtime-extensions-nutanix/values.yaml @@ -49,6 +49,11 @@ hooks: defaultValueTemplateConfigMap: create: true name: default-nutanix-csi-helm-values-template + aws-ebs: + helmAddonStrategy: + defaultValueTemplateConfigMap: + create: true + name: default-aws-ebs-csi-helm-values-template local-path: helmAddonStrategy: defaultValueTemplateConfigMap: diff --git a/examples/capi-quick-start/aws-cluster-calico-helm-addon.yaml b/examples/capi-quick-start/aws-cluster-calico-helm-addon.yaml index fcc2564fa..52f1fd8f1 100644 --- a/examples/capi-quick-start/aws-cluster-calico-helm-addon.yaml +++ b/examples/capi-quick-start/aws-cluster-calico-helm-addon.yaml @@ -35,7 +35,7 @@ spec: aws-ebs: storageClassConfigs: default: {} - strategy: ClusterResourceSet + strategy: HelmAddon nfd: strategy: HelmAddon aws: diff --git a/examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml b/examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml index 7dba7c586..1221850a8 100644 --- a/examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml +++ b/examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml @@ -35,7 +35,7 @@ spec: aws-ebs: storageClassConfigs: default: {} - strategy: ClusterResourceSet + strategy: HelmAddon nfd: strategy: HelmAddon aws: diff --git a/hack/addons/kustomize/aws-ebs-csi/storage-class.yaml b/hack/addons/kustomize/aws-ebs-csi/storage-class.yaml deleted file mode 100644 index 71115f01c..000000000 --- a/hack/addons/kustomize/aws-ebs-csi/storage-class.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright 2023 Nutanix. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: ebs-sc -provisioner: ebs.csi.aws.com -volumeBindingMode: WaitForFirstConsumer -parameters: - csi.storage.k8s.io/fstype: ext4 - type: gp3 diff --git a/hack/addons/mindthegap-helm-registry/repos.yaml b/hack/addons/mindthegap-helm-registry/repos.yaml index db390c266..7bfb0431f 100644 --- a/hack/addons/mindthegap-helm-registry/repos.yaml +++ b/hack/addons/mindthegap-helm-registry/repos.yaml @@ -6,6 +6,11 @@ # IT HAS BEEN GENERATED BY /hack/addons/generate-mindthegap-repofile.sh #================================================================= repositories: + aws-ebs-csi-driver: + repoURL: https://kubernetes-sigs.github.io/aws-ebs-csi-driver + charts: + aws-ebs-csi-driver: + - 2.28.1 cilium: repoURL: https://helm.cilium.io/ charts: diff --git a/hack/examples/overlays/clusters/aws/calico/crs/kustomization.yaml.tmpl b/hack/examples/overlays/clusters/aws/calico/crs/kustomization.yaml.tmpl index 0fd0e8532..8835e241b 100644 --- a/hack/examples/overlays/clusters/aws/calico/crs/kustomization.yaml.tmpl +++ b/hack/examples/overlays/clusters/aws/calico/crs/kustomization.yaml.tmpl @@ -17,3 +17,6 @@ patches: - target: kind: Cluster path: ../../../../../patches/crs-strategy.yaml + - target: + kind: Cluster + path: ../../../../../patches/aws/csi-crs-strategy.yaml diff --git a/hack/examples/overlays/clusters/aws/calico/helm-addon/kustomization.yaml.tmpl b/hack/examples/overlays/clusters/aws/calico/helm-addon/kustomization.yaml.tmpl index 5cc5781a9..615237bfb 100644 --- a/hack/examples/overlays/clusters/aws/calico/helm-addon/kustomization.yaml.tmpl +++ b/hack/examples/overlays/clusters/aws/calico/helm-addon/kustomization.yaml.tmpl @@ -17,3 +17,6 @@ patches: - target: kind: Cluster path: ../../../../../patches/helm-addon-strategy.yaml + - target: + kind: Cluster + path: ../../../../../patches/aws/csi-helm-addon-strategy.yaml diff --git a/hack/examples/overlays/clusters/aws/cilium/crs/kustomization.yaml.tmpl b/hack/examples/overlays/clusters/aws/cilium/crs/kustomization.yaml.tmpl index 3b4b6e91f..ec7896af9 100644 --- a/hack/examples/overlays/clusters/aws/cilium/crs/kustomization.yaml.tmpl +++ b/hack/examples/overlays/clusters/aws/cilium/crs/kustomization.yaml.tmpl @@ -17,3 +17,6 @@ patches: - target: kind: Cluster path: ../../../../../patches/crs-strategy.yaml + - target: + kind: Cluster + path: ../../../../../patches/aws/csi-crs-strategy.yaml diff --git a/hack/examples/overlays/clusters/aws/cilium/helm-addon/kustomization.yaml.tmpl b/hack/examples/overlays/clusters/aws/cilium/helm-addon/kustomization.yaml.tmpl index cdbf58e23..a394efcd8 100644 --- a/hack/examples/overlays/clusters/aws/cilium/helm-addon/kustomization.yaml.tmpl +++ b/hack/examples/overlays/clusters/aws/cilium/helm-addon/kustomization.yaml.tmpl @@ -17,3 +17,6 @@ patches: - target: kind: Cluster path: ../../../../../patches/helm-addon-strategy.yaml + - target: + kind: Cluster + path: ../../../../../patches/aws/csi-helm-addon-strategy.yaml diff --git a/hack/examples/patches/aws/csi-crs-strategy.yaml b/hack/examples/patches/aws/csi-crs-strategy.yaml new file mode 100644 index 000000000..152a3bbcd --- /dev/null +++ b/hack/examples/patches/aws/csi-crs-strategy.yaml @@ -0,0 +1,6 @@ +# Copyright 2024 Nutanix. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +- op: "add" + path: "/spec/topology/variables/0/value/addons/csi/providers/aws-ebs/strategy" + value: ClusterResourceSet diff --git a/hack/examples/patches/aws/csi-helm-addon-strategy.yaml b/hack/examples/patches/aws/csi-helm-addon-strategy.yaml new file mode 100644 index 000000000..ca323fe8b --- /dev/null +++ b/hack/examples/patches/aws/csi-helm-addon-strategy.yaml @@ -0,0 +1,6 @@ +# Copyright 2024 Nutanix. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +- op: "add" + path: "/spec/topology/variables/0/value/addons/csi/providers/aws-ebs/strategy" + value: HelmAddon diff --git a/hack/examples/patches/aws/csi.yaml b/hack/examples/patches/aws/csi.yaml index 2d7646b1d..5d76e8604 100644 --- a/hack/examples/patches/aws/csi.yaml +++ b/hack/examples/patches/aws/csi.yaml @@ -9,6 +9,5 @@ storageClassConfig: default providers: aws-ebs: - strategy: ClusterResourceSet storageClassConfigs: default: {} diff --git a/hack/tools/helm-cm/main.go b/hack/tools/helm-cm/main.go index 2762d30b4..d75f7fd83 100644 --- a/hack/tools/helm-cm/main.go +++ b/hack/tools/helm-cm/main.go @@ -154,7 +154,6 @@ func createConfigMapFromDir(kustomizeDir string) (*corev1.ConfigMap, error) { var ignored = []string{ "aws-ccm", - "aws-ebs-csi", } func isIgnored(filepath string) bool { diff --git a/make/addons.mk b/make/addons.mk index 080c9e660..0e459f85f 100644 --- a/make/addons.mk +++ b/make/addons.mk @@ -6,7 +6,7 @@ export CILIUM_VERSION := 1.15.5 export NODE_FEATURE_DISCOVERY_VERSION := 0.15.2 export CLUSTER_AUTOSCALER_VERSION := 9.37.0 export AWS_CSI_SNAPSHOT_CONTROLLER_VERSION := v6.3.3 -export AWS_EBS_CSI_CHART_VERSION := v2.28.1 +export AWS_EBS_CSI_CHART_VERSION := 2.28.1 export NUTANIX_STORAGE_CSI_CHART_VERSION := 3.0.0-beta.1912 export NUTANIX_SNAPSHOT_CSI_CHART_VERSION := 6.3.2 export LOCAL_PATH_CSI_CHART_VERSION := 0.0.29 diff --git a/pkg/handlers/generic/lifecycle/config/cm.go b/pkg/handlers/generic/lifecycle/config/cm.go index 08eb8af4a..d7947201f 100644 --- a/pkg/handlers/generic/lifecycle/config/cm.go +++ b/pkg/handlers/generic/lifecycle/config/cm.go @@ -26,6 +26,7 @@ const ( NutanixCCM Component = "nutanix-ccm" MetalLB Component = "metallb" LocalPathProvisionerCSI Component = "local-path-provisioner-csi" + AWSEBSCSI Component = "aws-ebs-csi" ) type HelmChartGetter struct { diff --git a/pkg/handlers/generic/lifecycle/csi/awsebs/handler.go b/pkg/handlers/generic/lifecycle/csi/awsebs/handler.go new file mode 100644 index 000000000..b5f4d5600 --- /dev/null +++ b/pkg/handlers/generic/lifecycle/csi/awsebs/handler.go @@ -0,0 +1,116 @@ +// Copyright 2023 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package awsebs + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + "github.com/spf13/pflag" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/config" + csiutils "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/csi/utils" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/options" +) + +const ( + defaultHelmReleaseName = "aws-ebs-csi-driver" + defaultHelmReleaseNamespace = "kube-system" +) + +var DefaultStorageClassParameters = map[string]string{ + "csi.storage.k8s.io/fstype": "ext4", + "type": "gp3", +} + +type addonStrategy interface { + apply( + context.Context, + *clusterv1.Cluster, + string, + logr.Logger, + ) error +} + +type AWSEBSConfig struct { + *options.GlobalOptions + + crsConfig crsConfig + helmAddonConfig helmAddonConfig +} + +func (a *AWSEBSConfig) AddFlags(prefix string, flags *pflag.FlagSet) { + a.crsConfig.AddFlags(prefix+".crs", flags) + a.helmAddonConfig.AddFlags(prefix+".helm-addon", flags) +} + +type AWSEBS struct { + client ctrlclient.Client + helmChartInfoGetter *config.HelmChartGetter + config *AWSEBSConfig +} + +func New( + c ctrlclient.Client, + cfg *AWSEBSConfig, + helmChartInfoGetter *config.HelmChartGetter, +) *AWSEBS { + return &AWSEBS{ + client: c, + config: cfg, + helmChartInfoGetter: helmChartInfoGetter, + } +} + +func (a *AWSEBS) Apply( + ctx context.Context, + provider v1alpha1.CSIProvider, + defaultStorage v1alpha1.DefaultStorage, + cluster *clusterv1.Cluster, + log logr.Logger, +) error { + var strategy addonStrategy + switch provider.Strategy { + case v1alpha1.AddonStrategyHelmAddon: + helmChart, err := a.helmChartInfoGetter.For(ctx, log, config.AWSEBSCSI) + if err != nil { + return fmt.Errorf("failed to get configuration to create helm addon: %w", err) + } + strategy = helmAddonStrategy{ + config: a.config.helmAddonConfig, + client: a.client, + helmChart: helmChart, + } + case v1alpha1.AddonStrategyClusterResourceSet: + strategy = crsStrategy{ + config: a.config.crsConfig, + client: a.client, + } + default: + return fmt.Errorf("strategy %s not implemented", strategy) + } + + if err := strategy.apply(ctx, cluster, a.config.DefaultsNamespace(), log); err != nil { + return fmt.Errorf("failed to apply aws-ebs CSI addon: %w", err) + } + + err := csiutils.CreateStorageClassesOnRemote( + ctx, + a.client, + provider.StorageClassConfigs, + cluster, + defaultStorage, + v1alpha1.CSIProviderAWSEBS, + v1alpha1.AWSEBSProvisioner, + DefaultStorageClassParameters, + ) + if err != nil { + return fmt.Errorf("error creating StorageClasses for the AWS EBS CSI driver: %w", err) + } + return nil +} diff --git a/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go b/pkg/handlers/generic/lifecycle/csi/awsebs/strategy_crs.go similarity index 50% rename from pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go rename to pkg/handlers/generic/lifecycle/csi/awsebs/strategy_crs.go index be1318a97..2d33aaa34 100644 --- a/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/awsebs/strategy_crs.go @@ -1,7 +1,7 @@ -// Copyright 2023 Nutanix. All rights reserved. +// Copyright 2024 Nutanix. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -package aws +package awsebs import ( "context" @@ -14,94 +14,45 @@ import ( clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/k8s/client" - csiutils "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/csi/utils" - "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/options" handlersutils "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/utils" ) -var defaultStorageClassParameters = map[string]string{ - "csi.storage.k8s.io/fstype": "ext4", - "type": "gp3", -} - -type AWSEBSConfig struct { - *options.GlobalOptions +type crsConfig struct { defaultAWSEBSConfigMapName string } -func (a *AWSEBSConfig) AddFlags(prefix string, flags *pflag.FlagSet) { +func (c *crsConfig) AddFlags(prefix string, flags *pflag.FlagSet) { flags.StringVar( - &a.defaultAWSEBSConfigMapName, - prefix+".aws-ebs-provider-configmap-name", + &c.defaultAWSEBSConfigMapName, + prefix+".default-aws-ebs-csi-configmap-name", "aws-ebs-csi", "name of the ConfigMap used to deploy AWS EBS CSI driver", ) } -type AWSEBS struct { - client ctrlclient.Client - config *AWSEBSConfig -} +type crsStrategy struct { + config crsConfig -func New( - c ctrlclient.Client, - cfg *AWSEBSConfig, -) *AWSEBS { - return &AWSEBS{ - client: c, - config: cfg, - } + client ctrlclient.Client } -func (a *AWSEBS) Apply( +func (s crsStrategy) apply( ctx context.Context, - provider v1alpha1.CSIProvider, - defaultStorage v1alpha1.DefaultStorage, - cluster *clusterv1.Cluster, - _ logr.Logger, -) error { - strategy := provider.Strategy - switch strategy { - case v1alpha1.AddonStrategyClusterResourceSet: - err := a.handleCRSApply(ctx, cluster) - if err != nil { - return err - } - case v1alpha1.AddonStrategyHelmAddon: - default: - return fmt.Errorf("stategy %s not implemented", strategy) - } - err := csiutils.CreateStorageClassesOnRemote( - ctx, - a.client, - provider.StorageClassConfigs, - cluster, - defaultStorage, - v1alpha1.CSIProviderAWSEBS, - v1alpha1.AWSEBSProvisioner, - defaultStorageClassParameters, - ) - if err != nil { - return fmt.Errorf("error creating StorageClasses for the AWS EBS CSI driver: %w", err) - } - return nil -} - -func (a *AWSEBS) handleCRSApply(ctx context.Context, cluster *clusterv1.Cluster, + defaultsNamespace string, + log logr.Logger, ) error { awsEBSCSIConfigMap := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Namespace: a.config.DefaultsNamespace(), - Name: a.config.defaultAWSEBSConfigMapName, + Namespace: defaultsNamespace, + Name: s.config.defaultAWSEBSConfigMapName, }, } defaultAWSEBSCSIConfigMapObjName := ctrlclient.ObjectKeyFromObject( awsEBSCSIConfigMap, ) - err := a.client.Get(ctx, defaultAWSEBSCSIConfigMapObjName, awsEBSCSIConfigMap) + err := s.client.Get(ctx, defaultAWSEBSCSIConfigMapObjName, awsEBSCSIConfigMap) if err != nil { return fmt.Errorf( "failed to retrieve default AWS EBS CSI manifests ConfigMap %q: %w", @@ -110,7 +61,7 @@ func (a *AWSEBS) handleCRSApply(ctx context.Context, ) } cm := generateAWSEBSCSIConfigMap(awsEBSCSIConfigMap, cluster) - if err := client.ServerSideApply(ctx, a.client, cm, client.ForceOwnership); err != nil { + if err := client.ServerSideApply(ctx, s.client, cm, client.ForceOwnership); err != nil { return fmt.Errorf( "failed to apply AWS EBS CSI manifests ConfigMap: %w", err, @@ -119,7 +70,7 @@ func (a *AWSEBS) handleCRSApply(ctx context.Context, err = handlersutils.EnsureCRSForClusterFromObjects( ctx, cm.Name, - a.client, + s.client, cluster, cm, ) diff --git a/pkg/handlers/generic/lifecycle/csi/awsebs/strategy_helmaddon.go b/pkg/handlers/generic/lifecycle/csi/awsebs/strategy_helmaddon.go new file mode 100644 index 000000000..1c8ba814f --- /dev/null +++ b/pkg/handlers/generic/lifecycle/csi/awsebs/strategy_helmaddon.go @@ -0,0 +1,97 @@ +// Copyright 2023 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package awsebs + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + "github.com/spf13/pflag" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + caaphv1 "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/external/sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/k8s/client" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/config" + handlersutils "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/utils" +) + +type helmAddonConfig struct { + defaultValuesTemplateConfigMapName string +} + +func (c *helmAddonConfig) AddFlags(prefix string, flags *pflag.FlagSet) { + flags.StringVar( + &c.defaultValuesTemplateConfigMapName, + prefix+".default-values-template-configmap-name", + "default-aws-ebs-csi-helm-values-template", + "default values ConfigMap name", + ) +} + +type helmAddonStrategy struct { + config helmAddonConfig + client ctrlclient.Client + helmChart *config.HelmChart +} + +func (s helmAddonStrategy) apply( + ctx context.Context, + cluster *clusterv1.Cluster, + defaultsNamespace string, + log logr.Logger, +) error { + log.Info("Retrieving aws-ebs CSI installation values template for cluster") + values, err := handlersutils.RetrieveValuesTemplate( + ctx, + s.client, + s.config.defaultValuesTemplateConfigMapName, + defaultsNamespace, + ) + if err != nil { + return fmt.Errorf( + "failed to retrieve aws-ebs CSI installation values template for cluster: %w", + err, + ) + } + + chartProxy := &caaphv1.HelmChartProxy{ + TypeMeta: metav1.TypeMeta{ + APIVersion: caaphv1.GroupVersion.String(), + Kind: "HelmChartProxy", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: cluster.Namespace, + Name: "aws-ebs-csi-" + cluster.Name, + }, + Spec: caaphv1.HelmChartProxySpec{ + RepoURL: s.helmChart.Repository, + ChartName: s.helmChart.Name, + ClusterSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{clusterv1.ClusterNameLabel: cluster.Name}, + }, + ReleaseNamespace: defaultHelmReleaseNamespace, + ReleaseName: defaultHelmReleaseName, + Version: s.helmChart.Version, + ValuesTemplate: values, + }, + } + handlersutils.SetTLSConfigForHelmChartProxyIfNeeded(chartProxy) + if err = controllerutil.SetOwnerReference(cluster, chartProxy, s.client.Scheme()); err != nil { + return fmt.Errorf( + "failed to set owner reference on HelmChartProxy %q: %w", + chartProxy.Name, + err, + ) + } + + if err = client.ServerSideApply(ctx, s.client, chartProxy, client.ForceOwnership); err != nil { + return fmt.Errorf("failed to apply HelmChartProxy %q: %w", chartProxy.Name, err) + } + + return nil +} diff --git a/pkg/handlers/generic/lifecycle/handlers.go b/pkg/handlers/generic/lifecycle/handlers.go index be54a480d..4bf5f119c 100644 --- a/pkg/handlers/generic/lifecycle/handlers.go +++ b/pkg/handlers/generic/lifecycle/handlers.go @@ -17,7 +17,7 @@ import ( "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/cni/cilium" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/config" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/csi" - awsebs "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/csi/aws-ebs" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/csi/awsebs" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/csi/localpath" nutanixcsi "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/csi/nutanix-csi" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/nfd" @@ -68,7 +68,11 @@ func (h *Handlers) AllHandlers(mgr manager.Manager) []handlers.Named { mgr.GetClient(), ) csiHandlers := map[string]csi.CSIProvider{ - v1alpha1.CSIProviderAWSEBS: awsebs.New(mgr.GetClient(), h.ebsConfig), + v1alpha1.CSIProviderAWSEBS: awsebs.New( + mgr.GetClient(), + h.ebsConfig, + helmChartInfoGetter, + ), v1alpha1.CSIProviderNutanix: nutanixcsi.New( mgr.GetClient(), h.nutanixCSIConfig, @@ -112,7 +116,7 @@ func (h *Handlers) AddFlags(flagSet *pflag.FlagSet) { h.clusterAutoscalerConfig.AddFlags("cluster-autoscaler", flagSet) h.calicoCNIConfig.AddFlags("cni.calico", flagSet) h.ciliumCNIConfig.AddFlags("cni.cilium", flagSet) - h.ebsConfig.AddFlags("awsebs", pflag.CommandLine) + h.ebsConfig.AddFlags("csi.aws-ebs", pflag.CommandLine) h.awsccmConfig.AddFlags("awsccm", pflag.CommandLine) h.nutanixCSIConfig.AddFlags("nutanixcsi", flagSet) h.nutanixCCMConfig.AddFlags("nutanixccm", flagSet) diff --git a/test/e2e/csi_helpers.go b/test/e2e/csi_helpers.go index b468484cd..279a0219f 100644 --- a/test/e2e/csi_helpers.go +++ b/test/e2e/csi_helpers.go @@ -8,6 +8,7 @@ package e2e import ( "context" "fmt" + "maps" "time" . "github.com/onsi/ginkgo/v2" @@ -25,6 +26,7 @@ import ( "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" apivariables "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/variables" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/csi/awsebs" ) type WaitForCSIToBeReadyInWorkloadClusterInput struct { @@ -45,6 +47,8 @@ func WaitForCSIToBeReadyInWorkloadCluster( return } + defaultStorageClassParameters := make(map[string]map[string]string, len(input.CSI.Providers)) + for providerName, providerConfig := range input.CSI.Providers { switch providerName { case v1alpha1.CSIProviderLocalPath: @@ -59,6 +63,21 @@ func WaitForCSIToBeReadyInWorkloadCluster( clusterResourceSetIntervals: input.ClusterResourceSetIntervals, }, ) + defaultStorageClassParameters[providerName] = map[string]string{} + case v1alpha1.CSIProviderAWSEBS: + waitForAWSEBSCSIToBeReadyInWorkloadCluster( + ctx, + waitForAWSEBSCSIToBeReadyInWorkloadClusterInput{ + strategy: providerConfig.Strategy, + workloadCluster: input.WorkloadCluster, + clusterProxy: input.ClusterProxy, + deploymentIntervals: input.DeploymentIntervals, + daemonSetIntervals: input.DaemonSetIntervals, + helmReleaseIntervals: input.HelmReleaseIntervals, + clusterResourceSetIntervals: input.ClusterResourceSetIntervals, + }, + ) + defaultStorageClassParameters[providerName] = awsebs.DefaultStorageClassParameters default: Fail( fmt.Sprintf( @@ -71,11 +90,12 @@ func WaitForCSIToBeReadyInWorkloadCluster( waitForStorageClassesToExistInWorkloadCluster( ctx, waitForStorageClassesToExistInWorkloadClusterInput{ - storageClasses: providerConfig.StorageClassConfigs, - workloadCluster: input.WorkloadCluster, - clusterProxy: input.ClusterProxy, - providerName: providerName, - defaultStorage: input.CSI.DefaultStorage, + storageClasses: providerConfig.StorageClassConfigs, + workloadCluster: input.WorkloadCluster, + clusterProxy: input.ClusterProxy, + providerName: providerName, + defaultStorage: input.CSI.DefaultStorage, + defaultStorageClassParameters: defaultStorageClassParameters[providerName], }, ) } @@ -149,12 +169,92 @@ func waitForLocalPathCSIToBeReadyInWorkloadCluster( }, input.deploymentIntervals...) } +type waitForAWSEBSCSIToBeReadyInWorkloadClusterInput struct { + strategy v1alpha1.AddonStrategy + workloadCluster *clusterv1.Cluster + clusterProxy framework.ClusterProxy + deploymentIntervals []interface{} + daemonSetIntervals []interface{} + helmReleaseIntervals []interface{} + clusterResourceSetIntervals []interface{} +} + +func waitForAWSEBSCSIToBeReadyInWorkloadCluster( + ctx context.Context, + input waitForAWSEBSCSIToBeReadyInWorkloadClusterInput, //nolint:gocritic // This hugeParam is OK in tests. +) { + switch input.strategy { + case v1alpha1.AddonStrategyClusterResourceSet: + crs := &addonsv1.ClusterResourceSet{} + Expect(input.clusterProxy.GetClient().Get( + ctx, + types.NamespacedName{ + Name: "aws-ebs-csi-" + input.workloadCluster.Name, + Namespace: input.workloadCluster.Namespace, + }, + crs, + )).To(Succeed()) + + framework.WaitForClusterResourceSetToApplyResources( + ctx, + framework.WaitForClusterResourceSetToApplyResourcesInput{ + ClusterResourceSet: crs, + ClusterProxy: input.clusterProxy, + Cluster: input.workloadCluster, + }, + input.clusterResourceSetIntervals..., + ) + case v1alpha1.AddonStrategyHelmAddon: + WaitForHelmReleaseProxyReadyForCluster( + ctx, + WaitForHelmReleaseProxyReadyForClusterInput{ + GetLister: input.clusterProxy.GetClient(), + Cluster: input.workloadCluster, + HelmChartProxyName: "aws-ebs-csi-" + input.workloadCluster.Name, + }, + input.helmReleaseIntervals..., + ) + default: + Fail( + fmt.Sprintf( + "Do not know how to wait for local-path-provisioner CSI using strategy %s to be ready", + input.strategy, + ), + ) + } + + workloadClusterClient := input.clusterProxy.GetWorkloadCluster( + ctx, input.workloadCluster.Namespace, input.workloadCluster.Name, + ).GetClient() + + WaitForDeploymentsAvailable(ctx, framework.WaitForDeploymentsAvailableInput{ + Getter: workloadClusterClient, + Deployment: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ebs-csi-controller", + Namespace: metav1.NamespaceSystem, + }, + }, + }, input.deploymentIntervals...) + + WaitForDaemonSetsAvailable(ctx, WaitForDaemonSetsAvailableInput{ + Getter: workloadClusterClient, + DaemonSet: &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ebs-csi-node", + Namespace: metav1.NamespaceSystem, + }, + }, + }, input.deploymentIntervals...) +} + type waitForStorageClassesToExistInWorkloadClusterInput struct { - providerName string - storageClasses map[string]v1alpha1.StorageClassConfig - defaultStorage v1alpha1.DefaultStorage - workloadCluster *clusterv1.Cluster - clusterProxy framework.ClusterProxy + providerName string + storageClasses map[string]v1alpha1.StorageClassConfig + defaultStorage v1alpha1.DefaultStorage + workloadCluster *clusterv1.Cluster + clusterProxy framework.ClusterProxy + defaultStorageClassParameters map[string]string } func waitForStorageClassesToExistInWorkloadCluster( @@ -169,6 +269,8 @@ func waitForStorageClassesToExistInWorkloadCluster( switch input.providerName { case v1alpha1.CSIProviderLocalPath: provisioner = string(v1alpha1.LocalPathProvisioner) + case v1alpha1.CSIProviderAWSEBS: + provisioner = string(v1alpha1.AWSEBSProvisioner) default: Fail( fmt.Sprintf( @@ -181,6 +283,9 @@ func waitForStorageClassesToExistInWorkloadCluster( for storageClassName, storageClassConfig := range input.storageClasses { isDefault := input.providerName == input.defaultStorage.Provider && storageClassName == input.defaultStorage.StorageClassConfig + storageClassParametersWithDefaults := maps.Clone(input.defaultStorageClassParameters) + maps.Copy(storageClassParametersWithDefaults, storageClassConfig.Parameters) + storageClassConfig.Parameters = storageClassParametersWithDefaults waitForStorageClassToExistInWorkloadCluster( ctx, @@ -217,7 +322,11 @@ func waitForStorageClassToExistInWorkloadCluster( }, intervals...).Should(BeTrue()) Expect(gotSC.Provisioner).To(Equal(provisioner)) - Expect(gotSC.Parameters).To(Equal(scConfig.Parameters)) + if len(scConfig.Parameters) == 0 { + Expect(gotSC.Parameters).To(SatisfyAny(BeNil(), BeEmpty())) + } else { + Expect(gotSC.Parameters).To(Equal(scConfig.Parameters)) + } if scConfig.ReclaimPolicy != nil { Expect(gotSC.ReclaimPolicy).To(Equal(scConfig.ReclaimPolicy)) } else { @@ -230,7 +339,9 @@ func waitForStorageClassToExistInWorkloadCluster( } Expect(gotSC.AllowVolumeExpansion).To(HaveValue(Equal(scConfig.AllowExpansion))) if isDefault { - Expect(gotSC.Annotations).To(HaveKeyWithValue("storageclass.kubernetes.io/is-default-class", "true")) + Expect( + gotSC.Annotations, + ).To(HaveKeyWithValue("storageclass.kubernetes.io/is-default-class", "true")) } else { Expect(gotSC.Annotations).ToNot(HaveKeyWithValue("storageclass.kubernetes.io/is-default-class", "true")) }