Skip to content

Commit 5869295

Browse files
jimmidysonjimmidyson
committed
feat: Support HelmAddon strategy for AWS EBS
Also add e2e tests for AWS EBS deployment.
1 parent 7aa8b34 commit 5869295

File tree

24 files changed

+441
-98
lines changed

24 files changed

+441
-98
lines changed

‎charts/cluster-api-runtime-extensions-nutanix/README.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,8 @@ A Helm chart for cluster-api-runtime-extensions-nutanix
6060
| hooks.cni.cilium.crsStrategy.defaultCiliumConfigMap.name | string | `"cilium"` | |
6161
| hooks.cni.cilium.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` | |
6262
| hooks.cni.cilium.helmAddonStrategy.defaultValueTemplateConfigMap.name | string | `"default-cilium-cni-helm-values-template"` | |
63+
| hooks.csi.aws-ebs.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` | |
64+
| hooks.csi.aws-ebs.helmAddonStrategy.defaultValueTemplateConfigMap.name | string | `"default-aws-ebs-csi-helm-values-template"` | |
6365
| hooks.csi.local-path.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` | |
6466
| hooks.csi.local-path.helmAddonStrategy.defaultValueTemplateConfigMap.name | string | `"default-local-path-csi-helm-values-template"` | |
6567
| hooks.csi.nutanix.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` | |

‎charts/cluster-api-runtime-extensions-nutanix/templates/csi/aws-ebs/manifests/helm-addon-installation.yaml

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
# Copyright 2024 Nutanix. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
{{- if (index .Values.hooks.csi "aws-ebs").helmAddonStrategy.defaultValueTemplateConfigMap.create }}
5+
apiVersion: v1
6+
kind: ConfigMap
7+
metadata:
8+
name: '{{ (index .Values.hooks.csi "aws-ebs").helmAddonStrategy.defaultValueTemplateConfigMap.name }}'
9+
data:
10+
values.yaml: |-
11+
controller:
12+
affinity:
13+
nodeAffinity:
14+
requiredDuringSchedulingIgnoredDuringExecution:
15+
nodeSelectorTerms:
16+
- matchExpressions:
17+
- key: node-role.kubernetes.io/control-plane
18+
operator: Exists
19+
tolerations:
20+
- key: CriticalAddonsOnly
21+
operator: Exists
22+
- effect: NoExecute
23+
operator: Exists
24+
tolerationSeconds: 300
25+
- effect: NoSchedule
26+
key: node-role.kubernetes.io/master
27+
operator: Exists
28+
- effect: NoSchedule
29+
key: node-role.kubernetes.io/control-plane
30+
operator: Exists
31+
node:
32+
priorityClassName: system-node-critical
33+
sidecars:
34+
snapshotter:
35+
forceEnable: true
36+
{{- end -}}

‎charts/cluster-api-runtime-extensions-nutanix/templates/deployment.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ spec:
3434
- --cni.cilium.helm-addon.default-values-template-configmap-name={{ .Values.hooks.cni.cilium.helmAddonStrategy.defaultValueTemplateConfigMap.name }}
3535
- --nfd.helm-addon.default-values-template-configmap-name={{ .Values.hooks.nfd.helmAddonStrategy.defaultValueTemplateConfigMap.name }}
3636
- --csi.local-path.helm-addon.default-values-template-configmap-name={{ (index .Values.hooks.csi "local-path").helmAddonStrategy.defaultValueTemplateConfigMap.name }}
37+
- --csi.aws-ebs.helm-addon.default-values-template-configmap-name={{ (index .Values.hooks.csi "aws-ebs").helmAddonStrategy.defaultValueTemplateConfigMap.name }}
3738
{{- range $key, $value := .Values.extraArgs }}
3839
- --{{ $key }}={{ $value }}
3940
{{- end }}

‎charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,10 @@
77
#=================================================================
88
apiVersion: v1
99
data:
10+
aws-ebs-csi: |
11+
ChartName: aws-ebs-csi-driver
12+
ChartVersion: 2.28.1
13+
RepositoryURL: {{ if .Values.selfHostedRegistry }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://kubernetes-sigs.github.io/aws-ebs-csi-driver{{ end }}
1014
cilium: |
1115
ChartName: cilium
1216
ChartVersion: 1.15.5

‎charts/cluster-api-runtime-extensions-nutanix/values.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,11 @@ hooks:
4949
defaultValueTemplateConfigMap:
5050
create: true
5151
name: default-nutanix-csi-helm-values-template
52+
aws-ebs:
53+
helmAddonStrategy:
54+
defaultValueTemplateConfigMap:
55+
create: true
56+
name: default-aws-ebs-csi-helm-values-template
5257
local-path:
5358
helmAddonStrategy:
5459
defaultValueTemplateConfigMap:

‎examples/capi-quick-start/aws-cluster-calico-helm-addon.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ spec:
3535
aws-ebs:
3636
storageClassConfigs:
3737
default: {}
38-
strategy: ClusterResourceSet
38+
strategy: HelmAddon
3939
nfd:
4040
strategy: HelmAddon
4141
aws:

‎examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ spec:
3535
aws-ebs:
3636
storageClassConfigs:
3737
default: {}
38-
strategy: ClusterResourceSet
38+
strategy: HelmAddon
3939
nfd:
4040
strategy: HelmAddon
4141
aws:

‎hack/addons/kustomize/aws-ebs-csi/storage-class.yaml

Lines changed: 0 additions & 12 deletions
This file was deleted.

‎hack/addons/mindthegap-helm-registry/repos.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,11 @@
66
# IT HAS BEEN GENERATED BY /hack/addons/generate-mindthegap-repofile.sh
77
#=================================================================
88
repositories:
9+
aws-ebs-csi-driver:
10+
repoURL: https://kubernetes-sigs.github.io/aws-ebs-csi-driver
11+
charts:
12+
aws-ebs-csi-driver:
13+
- 2.28.1
914
cilium:
1015
repoURL: https://helm.cilium.io/
1116
charts:

‎hack/examples/overlays/clusters/aws/calico/crs/kustomization.yaml.tmpl

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,3 +17,6 @@ patches:
1717
- target:
1818
kind: Cluster
1919
path: ../../../../../patches/crs-strategy.yaml
20+
- target:
21+
kind: Cluster
22+
path: ../../../../../patches/aws/csi-crs-strategy.yaml

‎hack/examples/overlays/clusters/aws/calico/helm-addon/kustomization.yaml.tmpl

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,3 +17,6 @@ patches:
1717
- target:
1818
kind: Cluster
1919
path: ../../../../../patches/helm-addon-strategy.yaml
20+
- target:
21+
kind: Cluster
22+
path: ../../../../../patches/aws/csi-helm-addon-strategy.yaml

‎hack/examples/overlays/clusters/aws/cilium/crs/kustomization.yaml.tmpl

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,3 +17,6 @@ patches:
1717
- target:
1818
kind: Cluster
1919
path: ../../../../../patches/crs-strategy.yaml
20+
- target:
21+
kind: Cluster
22+
path: ../../../../../patches/aws/csi-crs-strategy.yaml

‎hack/examples/overlays/clusters/aws/cilium/helm-addon/kustomization.yaml.tmpl

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,3 +17,6 @@ patches:
1717
- target:
1818
kind: Cluster
1919
path: ../../../../../patches/helm-addon-strategy.yaml
20+
- target:
21+
kind: Cluster
22+
path: ../../../../../patches/aws/csi-helm-addon-strategy.yaml

‎hack/examples/patches/aws/csi-crs-strategy.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# Copyright 2024 Nutanix. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
- op: "add"
5+
path: "/spec/topology/variables/0/value/addons/csi/providers/aws-ebs/strategy"
6+
value: ClusterResourceSet

‎hack/examples/patches/aws/csi-helm-addon-strategy.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# Copyright 2024 Nutanix. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
- op: "add"
5+
path: "/spec/topology/variables/0/value/addons/csi/providers/aws-ebs/strategy"
6+
value: HelmAddon

‎hack/examples/patches/aws/csi.yaml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,5 @@
99
storageClassConfig: default
1010
providers:
1111
aws-ebs:
12-
strategy: ClusterResourceSet
1312
storageClassConfigs:
1413
default: {}

‎hack/tools/helm-cm/main.go

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -154,7 +154,6 @@ func createConfigMapFromDir(kustomizeDir string) (*corev1.ConfigMap, error) {
154154

155155
var ignored = []string{
156156
"aws-ccm",
157-
"aws-ebs-csi",
158157
}
159158

160159
func isIgnored(filepath string) bool {

‎make/addons.mk

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ export CILIUM_VERSION := 1.15.5
66
export NODE_FEATURE_DISCOVERY_VERSION := 0.15.2
77
export CLUSTER_AUTOSCALER_VERSION := 9.37.0
88
export AWS_CSI_SNAPSHOT_CONTROLLER_VERSION := v6.3.3
9-
export AWS_EBS_CSI_CHART_VERSION := v2.28.1
9+
export AWS_EBS_CSI_CHART_VERSION := 2.28.1
1010
export NUTANIX_STORAGE_CSI_CHART_VERSION := 3.0.0-beta.1912
1111
export NUTANIX_SNAPSHOT_CSI_CHART_VERSION := 6.3.2
1212
export LOCAL_PATH_CSI_CHART_VERSION := 0.0.29

‎pkg/handlers/generic/lifecycle/config/cm.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ const (
2626
NutanixCCM Component = "nutanix-ccm"
2727
MetalLB Component = "metallb"
2828
LocalPathProvisionerCSI Component = "local-path-provisioner-csi"
29+
AWSEBSCSI Component = "aws-ebs-csi"
2930
)
3031

3132
type HelmChartGetter struct {

‎pkg/handlers/generic/lifecycle/csi/awsebs/handler.go

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
// Copyright 2023 Nutanix. All rights reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
package awsebs
5+
6+
import (
7+
"context"
8+
"fmt"
9+
10+
"github.com/go-logr/logr"
11+
"github.com/spf13/pflag"
12+
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
13+
ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
14+
15+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1"
16+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/config"
17+
csiutils "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/csi/utils"
18+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/options"
19+
)
20+
21+
const (
22+
defaultHelmReleaseName = "aws-ebs-csi-driver"
23+
defaultHelmReleaseNamespace = "kube-system"
24+
)
25+
26+
var DefaultStorageClassParameters = map[string]string{
27+
"csi.storage.k8s.io/fstype": "ext4",
28+
"type": "gp3",
29+
}
30+
31+
type addonStrategy interface {
32+
apply(
33+
context.Context,
34+
*clusterv1.Cluster,
35+
string,
36+
logr.Logger,
37+
) error
38+
}
39+
40+
type AWSEBSConfig struct {
41+
*options.GlobalOptions
42+
43+
crsConfig crsConfig
44+
helmAddonConfig helmAddonConfig
45+
}
46+
47+
func (a *AWSEBSConfig) AddFlags(prefix string, flags *pflag.FlagSet) {
48+
a.crsConfig.AddFlags(prefix+".crs", flags)
49+
a.helmAddonConfig.AddFlags(prefix+".helm-addon", flags)
50+
}
51+
52+
type AWSEBS struct {
53+
client ctrlclient.Client
54+
helmChartInfoGetter *config.HelmChartGetter
55+
config *AWSEBSConfig
56+
}
57+
58+
func New(
59+
c ctrlclient.Client,
60+
cfg *AWSEBSConfig,
61+
helmChartInfoGetter *config.HelmChartGetter,
62+
) *AWSEBS {
63+
return &AWSEBS{
64+
client: c,
65+
config: cfg,
66+
helmChartInfoGetter: helmChartInfoGetter,
67+
}
68+
}
69+
70+
func (a *AWSEBS) Apply(
71+
ctx context.Context,
72+
provider v1alpha1.CSIProvider,
73+
defaultStorage v1alpha1.DefaultStorage,
74+
cluster *clusterv1.Cluster,
75+
log logr.Logger,
76+
) error {
77+
var strategy addonStrategy
78+
switch provider.Strategy {
79+
case v1alpha1.AddonStrategyHelmAddon:
80+
helmChart, err := a.helmChartInfoGetter.For(ctx, log, config.AWSEBSCSI)
81+
if err != nil {
82+
return fmt.Errorf("failed to get configuration to create helm addon: %w", err)
83+
}
84+
strategy = helmAddonStrategy{
85+
config: a.config.helmAddonConfig,
86+
client: a.client,
87+
helmChart: helmChart,
88+
}
89+
case v1alpha1.AddonStrategyClusterResourceSet:
90+
strategy = crsStrategy{
91+
config: a.config.crsConfig,
92+
client: a.client,
93+
}
94+
default:
95+
return fmt.Errorf("strategy %s not implemented", strategy)
96+
}
97+
98+
if err := strategy.apply(ctx, cluster, a.config.DefaultsNamespace(), log); err != nil {
99+
return fmt.Errorf("failed to apply aws-ebs CSI addon: %w", err)
100+
}
101+
102+
err := csiutils.CreateStorageClassesOnRemote(
103+
ctx,
104+
a.client,
105+
provider.StorageClassConfigs,
106+
cluster,
107+
defaultStorage,
108+
v1alpha1.CSIProviderAWSEBS,
109+
v1alpha1.AWSEBSProvisioner,
110+
DefaultStorageClassParameters,
111+
)
112+
if err != nil {
113+
return fmt.Errorf("error creating StorageClasses for the AWS EBS CSI driver: %w", err)
114+
}
115+
return nil
116+
}

0 commit comments

Comments
 (0)