feat: Upgrade AWS ESB CSI and switch to using Helm chart (nutanix-cloud-native#393)

This will make it easier to adopt the HelmAddon strategy for deploying
the AWS ESB CSI addon.

Depends on nutanix-cloud-native#392

Author: jimmidyson

charts/capi-runtime-extensions/templates/csi/aws-ebs/manifests/aws-ebs-csi-configmap.yaml

@@ -9,10 +9,11 @@ metadata:
 
 helmCharts:
 - name: aws-cloud-controller-manager
-  includeCRDs: true
+  repo: https://kubernetes.github.io/cloud-provider-aws
   releaseName: aws-cloud-controller-manager
   version: ${AWS_CPI_CHART_VERSION}
-  repo: https://kubernetes.github.io/cloud-provider-aws
+  includeCRDs: true
+  skipTests: true
 
 patches:
 - patch: |-

hack/addons/kustomize/aws-cpi/kustomization.yaml.tmpl

@@ -0,0 +1,36 @@
+# Copyright 2024 D2iQ, Inc. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+---
+controller:
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+          - matchExpressions:
+              - key: node-role.kubernetes.io/control-plane
+                operator: Exists
+  tolerations:
+    - key: CriticalAddonsOnly
+      operator: Exists
+    - effect: NoExecute
+      operator: Exists
+      tolerationSeconds: 300
+    - effect: NoSchedule
+      key: node-role.kubernetes.io/master
+      operator: Exists
+    - effect: NoSchedule
+      key: node-role.kubernetes.io/control-plane
+      operator: Exists
+node:
+  priorityClassName: system-node-critical
+sidecars:
+  snapshotter:
+    forceEnable: true
+storageClasses:
+- metadata:
+  name: ebs-sc
+  volumeBindingMode: WaitForFirstConsumer
+  parameters:
+    csi.storage.k8s.io/fstype: ext4
+    type: gp3

hack/addons/kustomize/aws-ebs-csi/helm-values.yaml

@@ -7,37 +7,16 @@ kind: Kustomization
 metadata:
   name: aws-ebs-csi-kustomize
 
-resources:
-- github.com/kubernetes-sigs/aws-ebs-csi-driver/deploy/kubernetes/overlays/stable/?ref=${AWS_EBS_CSI_VERSION}
-- storage-class.yaml
-- github.com/kubernetes-csi/external-snapshotter/client/config/crd/?ref=${AWS_CSI_SNAPSHOT_CONTROLLER_VERSION}
-- github.com/kubernetes-csi/external-snapshotter/deploy/kubernetes/snapshot-controller/?ref=${AWS_CSI_SNAPSHOT_CONTROLLER_VERSION}
+namespace: kube-system
 
-patches:
-- path: ./overlays/controller-nodeAffinity.yaml
-  target:
-    kind: Deployment
-    name: ebs-csi-controller
-    namespace: kube-system
-- path: ./overlays/controller-nodeAffinity.yaml
-  target:
-    kind: Deployment
-    name: snapshot-controller
-    namespace: kube-system
-- path: ./overlays/deployment-priorityClassName.yaml
-  target:
-    kind: Deployment
-    name: snapshot-controller
-    namespace: kube-system
-- path: ./overlays/ds-tolerations.yaml
-  target:
-    kind: DaemonSet
-    name: ebs-csi-node
-    namespace: kube-system
+helmCharts:
+- name: aws-ebs-csi-driver
+  repo: https://kubernetes-sigs.github.io/aws-ebs-csi-driver
+  releaseName: aws-ebs-csi-driver
+  version: ${AWS_EBS_CSI_CHART_VERSION}
+  valuesFile: helm-values.yaml
+  includeCRDs: true
+  skipTests: true
 
-# always override the images to match the version from the set envs
-images:
-- name: public.ecr.aws/ebs-csi-driver/aws-ebs-csi-driver
-  newTag: ${AWS_EBS_CSI_VERSION}
-- name: registry.k8s.io/sig-storage/snapshot-controller
-  newTag: ${AWS_CSI_SNAPSHOT_CONTROLLER_VERSION}
+resources:
+- ../external-snapshotter

hack/addons/kustomize/aws-ebs-csi/kustomization.yaml.tmpl

@@ -12,10 +12,11 @@ sortOptions:
 
 helmCharts:
 - name: cilium
-  includeCRDs: true
-  valuesFile: helm-values.yaml
+  repo: https://helm.cilium.io/
   releaseName: cilium
   version: ${CILIUM_VERSION}
-  repo: https://helm.cilium.io/
+  valuesFile: helm-values.yaml
+  includeCRDs: true
+  skipTests: true
 
 namespace: kube-system

hack/addons/kustomize/aws-ebs-csi/overlays/controller-priorityClassName.yaml

@@ -0,0 +1,31 @@
+# Copyright 2023 D2iQ, Inc. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+metadata:
+  name: external-snapshotter-kustomize
+
+namespace: kube-system
+
+resources:
+- github.com/kubernetes-csi/external-snapshotter/client/config/crd/?ref=${AWS_CSI_SNAPSHOT_CONTROLLER_VERSION}
+- github.com/kubernetes-csi/external-snapshotter/deploy/kubernetes/snapshot-controller/?ref=${AWS_CSI_SNAPSHOT_CONTROLLER_VERSION}
+
+patches:
+- path: ./overlays/deployment-nodeAffinity.yaml
+  target:
+    kind: Deployment
+    name: snapshot-controller
+    namespace: kube-system
+- path: ./overlays/deployment-priorityClassName.yaml
+  target:
+    kind: Deployment
+    name: snapshot-controller
+    namespace: kube-system
+
+# always override the images to match the version from the set envs
+images:
+- name: registry.k8s.io/sig-storage/snapshot-controller
+  newTag: ${AWS_CSI_SNAPSHOT_CONTROLLER_VERSION}

hack/addons/kustomize/aws-ebs-csi/overlays/ds-priorityClassName.yaml

@@ -15,9 +15,6 @@ spec:
               - matchExpressions:
                   - key: node-role.kubernetes.io/control-plane
                     operator: Exists
-      nodeSelector:
-        node-role.kubernetes.io/control-plane: null
-        node-role.kubernetes.io/master: null
       tolerations:
         - key: CriticalAddonsOnly
           operator: Exists

hack/addons/kustomize/aws-ebs-csi/overlays/ds-tolerations.yaml

@@ -12,13 +12,14 @@ resources:
 
 helmCharts:
 - name: node-feature-discovery
-  includeCRDs: true
+  repo: https://kubernetes-sigs.github.io/node-feature-discovery/charts
+  releaseName: node-feature-discovery
+  version: ${NODE_FEATURE_DISCOVERY_VERSION}
   valuesFile: helm-values.yaml
   valuesInline:
     image:
       tag: "v${NODE_FEATURE_DISCOVERY_VERSION}-minimal"
-  releaseName: node-feature-discovery
-  version: ${NODE_FEATURE_DISCOVERY_VERSION}
-  repo: https://kubernetes-sigs.github.io/node-feature-discovery/charts
+  includeCRDs: true
+  skipTests: true
 
 namespace: node-feature-discovery

hack/addons/kustomize/cilium/kustomization.yaml.tmpl

@@ -15,10 +15,11 @@ resources:
 
 helmCharts:
 - name: tigera-operator
-  includeCRDs: true
-  valuesFile: helm-values.yaml
+  repo: https://docs.tigera.io/calico/charts
   releaseName: tigera-operator
   version: ${CALICO_VERSION}
-  repo: https://docs.tigera.io/calico/charts
+  valuesFile: helm-values.yaml
+  includeCRDs: true
+  skipTests: true
 
 namespace: tigera-operator

hack/addons/kustomize/external-snapshotter/kustomization.yaml.tmpl

@@ -8,8 +8,12 @@ readonly SCRIPT_DIR
 # shellcheck source=hack/common.sh
 source "${SCRIPT_DIR}/../common.sh"
 
-if [ -z "${AWS_EBS_CSI_VERSION:-}" ]; then
-  echo "Missing environment variable: AWS_EBS_CSI_VERSION"
+if [ -z "${AWS_EBS_CSI_CHART_VERSION:-}" ]; then
+  echo "Missing environment variable: AWS_EBS_CSI_CHART_VERSION"
+  exit 1
+fi
+if [ -z "${AWS_CSI_SNAPSHOT_CONTROLLER_VERSION:-}" ]; then
+  echo "Missing environment variable: AWS_CSI_SNAPSHOT_CONTROLLER_VERSION"
   exit 1
 fi
 
@@ -19,11 +23,17 @@ trap_add "rm -rf ${ASSETS_DIR}" EXIT
 
 readonly FILE_NAME="aws-ebs-csi.yaml"
 
-readonly KUSTOMIZE_BASE_DIR="${SCRIPT_DIR}/kustomize/aws-ebs-csi/"
-envsubst -no-unset <"${KUSTOMIZE_BASE_DIR}/kustomization.yaml.tmpl" >"${ASSETS_DIR}/kustomization.yaml"
-cp "${KUSTOMIZE_BASE_DIR}"/*.yaml "${ASSETS_DIR}"
-cp -r "${KUSTOMIZE_BASE_DIR}"/overlays "${ASSETS_DIR}"
-kustomize build --enable-helm "${ASSETS_DIR}" >"${ASSETS_DIR}/${FILE_NAME}"
+readonly KUSTOMIZE_BASE_DIR="${SCRIPT_DIR}/kustomize/aws-ebs-csi"
+mkdir -p "${ASSETS_DIR}/aws-ebs-csi"
+envsubst -no-unset <"${KUSTOMIZE_BASE_DIR}/kustomization.yaml.tmpl" >"${ASSETS_DIR}/aws-ebs-csi/kustomization.yaml"
+cp -r "${KUSTOMIZE_BASE_DIR}"/*.yaml "${ASSETS_DIR}/aws-ebs-csi/"
+
+readonly EXTERNAL_SNAPSHOTTER_BASE_DIR="${SCRIPT_DIR}/kustomize/external-snapshotter"
+mkdir -p "${ASSETS_DIR}/external-snapshotter"
+envsubst -no-unset <"${EXTERNAL_SNAPSHOTTER_BASE_DIR}/kustomization.yaml.tmpl" >"${ASSETS_DIR}/external-snapshotter/kustomization.yaml"
+cp -r "${EXTERNAL_SNAPSHOTTER_BASE_DIR}/overlays" "${ASSETS_DIR}/external-snapshotter/"
+
+kustomize build --enable-helm "${ASSETS_DIR}/aws-ebs-csi/" >"${ASSETS_DIR}/${FILE_NAME}"
 
 kubectl create configmap aws-ebs-csi --dry-run=client --output yaml \
   --from-file "${ASSETS_DIR}/${FILE_NAME}" \

hack/addons/kustomize/aws-ebs-csi/overlays/controller-nodeAffinity.yaml renamed to hack/addons/kustomize/external-snapshotter/overlays/deployment-nodeAffinity.yaml

@@ -4,8 +4,8 @@
 export CALICO_VERSION := $(shell goprintconst -file pkg/handlers/generic/lifecycle/cni/calico/strategy_helmaddon.go -name defaultCalicoHelmChartVersion)
 export CILIUM_VERSION := $(shell goprintconst -file pkg/handlers/generic/lifecycle/cni/cilium/strategy_helmaddon.go -name defaultCiliumHelmChartVersion)
 export NODE_FEATURE_DISCOVERY_VERSION := $(shell goprintconst -file pkg/handlers/generic/lifecycle/nfd/strategy_helmaddon.go -name defaultHelmChartVersion)
-export AWS_CSI_SNAPSHOT_CONTROLLER_VERSION := v6.3.0
-export AWS_EBS_CSI_VERSION := v1.25.0
+export AWS_CSI_SNAPSHOT_CONTROLLER_VERSION := v6.3.3
+export AWS_EBS_CSI_CHART_VERSION := v2.28.1
 # a map of AWS CPI versions
 export AWS_CPI_VERSION_127 := v1.27.1
 export AWS_CPI_CHART_VERSION_127 := 0.0.8