From ef05b880c0510afeda88c9d6bf1ab0ef011853f0 Mon Sep 17 00:00:00 2001 From: Jimmi Dyson Date: Wed, 19 Jun 2024 09:27:56 +0100 Subject: [PATCH] feat: Support CRS for local-path provisioner and add CSI e2e Further strategy support for other providers coming in follow up PR, plus more e2e tests for those other providers. --- .github/workflows/checks.yml | 2 +- .../README.md | 2 + .../manifests/helm-addon-installation.yaml | 28 ++ .../local-path-provisioner-csi-configmap.yaml | 264 ++++++++++++++++++ .../templates/deployment.yaml | 1 + .../values.schema.json | 21 ++ .../values.yaml | 5 + .../docker-cluster-calico-crs.yaml | 2 +- .../docker-cluster-cilium-crs.yaml | 2 +- .../helm-values.yaml | 21 ++ .../kustomization.yaml.tmpl | 1 + .../update-local-path-provisioner-csi.sh | 42 +++ .../docker/calico/crs/kustomization.yaml.tmpl | 3 + .../calico/helm-addon/kustomization.yaml.tmpl | 3 + .../docker/cilium/crs/kustomization.yaml.tmpl | 3 + .../cilium/helm-addon/kustomization.yaml.tmpl | 3 + .../patches/docker/csi-crs-strategy.yaml | 6 + .../docker/csi-helm-addon-strategy.yaml | 6 + hack/examples/patches/docker/csi.yaml | 1 - make/addons.mk | 4 + .../generic/lifecycle/csi/aws-ebs/handler.go | 2 +- .../lifecycle/csi/localpath/handler.go | 111 +++----- .../lifecycle/csi/localpath/strategy_crs.go | 96 +++++++ .../csi/localpath/strategy_helmaddon.go | 97 +++++++ .../lifecycle/csi/nutanix-csi/handler.go | 2 +- .../generic/lifecycle/csi/utils/scs.go | 2 +- pkg/handlers/generic/lifecycle/handlers.go | 9 +- test/e2e/addon_helpers.go | 13 + test/e2e/csi_helpers.go | 239 ++++++++++++++++ test/e2e/e2e_suite_test.go | 2 + 30 files changed, 919 insertions(+), 74 deletions(-) create mode 100644 charts/cluster-api-runtime-extensions-nutanix/templates/csi/local-path/manifests/helm-addon-installation.yaml create mode 100644 charts/cluster-api-runtime-extensions-nutanix/templates/csi/local-path/manifests/local-path-provisioner-csi-configmap.yaml create mode 100644 hack/addons/kustomize/local-path-provisioner-csi/helm-values.yaml create mode 100755 hack/addons/update-local-path-provisioner-csi.sh create mode 100644 hack/examples/patches/docker/csi-crs-strategy.yaml create mode 100644 hack/examples/patches/docker/csi-helm-addon-strategy.yaml create mode 100644 pkg/handlers/generic/lifecycle/csi/localpath/strategy_crs.go create mode 100644 pkg/handlers/generic/lifecycle/csi/localpath/strategy_helmaddon.go create mode 100644 test/e2e/csi_helpers.go diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 62949d3c6..40844fd4b 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -239,7 +239,7 @@ jobs: devbox run -- \ ct install \ --config charts/ct-config.yaml \ - --helm-extra-set-args "--set-string image.repository=ko.local/cluster-api-runtime-extensions-nutanix --set-string image.tag=$(devbox run -- gojq -r .version dist/metadata.json) --set-string helmRepositoryImage.tag=$(devbox run -- gojq -r .version dist/metadata.json)" + --helm-extra-set-args "--set-string image.repository=ko.local/cluster-api-runtime-extensions-nutanix --set-string image.tag=$(devbox run -- gojq -r .version dist/metadata.json) --set-string helmRepositoryImage.tag=$(devbox run -- gojq -r .version dist/metadata.json)-$(devbox run -- go env GOARCH)" env: KUBECONFIG: ct-kind-kubeconfig diff --git a/charts/cluster-api-runtime-extensions-nutanix/README.md b/charts/cluster-api-runtime-extensions-nutanix/README.md index 510f0e5e1..112fb78ba 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/README.md +++ b/charts/cluster-api-runtime-extensions-nutanix/README.md @@ -60,6 +60,8 @@ A Helm chart for cluster-api-runtime-extensions-nutanix | hooks.cni.cilium.crsStrategy.defaultCiliumConfigMap.name | string | `"cilium"` | | | hooks.cni.cilium.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` | | | hooks.cni.cilium.helmAddonStrategy.defaultValueTemplateConfigMap.name | string | `"default-cilium-cni-helm-values-template"` | | +| hooks.csi.local-path.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` | | +| hooks.csi.local-path.helmAddonStrategy.defaultValueTemplateConfigMap.name | string | `"default-local-path-csi-helm-values-template"` | | | hooks.csi.nutanix.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` | | | hooks.csi.nutanix.helmAddonStrategy.defaultValueTemplateConfigMap.name | string | `"default-nutanix-csi-helm-values-template"` | | | hooks.nfd.crsStrategy.defaultInstallationConfigMap.name | string | `"node-feature-discovery"` | | diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/csi/local-path/manifests/helm-addon-installation.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/csi/local-path/manifests/helm-addon-installation.yaml new file mode 100644 index 000000000..69441057f --- /dev/null +++ b/charts/cluster-api-runtime-extensions-nutanix/templates/csi/local-path/manifests/helm-addon-installation.yaml @@ -0,0 +1,28 @@ +# Copyright 2024 Nutanix. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{{- if (index .Values.hooks.csi "local-path").helmAddonStrategy.defaultValueTemplateConfigMap.create }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: '{{ (index .Values.hooks.csi "local-path").helmAddonStrategy.defaultValueTemplateConfigMap.name }}' +data: + values.yaml: |- + storageClass: + create: false + provisionerName: rancher.io/local-path + helperImage: + tag: 1.36.1 + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + tolerationSeconds: 300 + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists +{{- end -}} diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/csi/local-path/manifests/local-path-provisioner-csi-configmap.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/csi/local-path/manifests/local-path-provisioner-csi-configmap.yaml new file mode 100644 index 000000000..a2a5b9708 --- /dev/null +++ b/charts/cluster-api-runtime-extensions-nutanix/templates/csi/local-path/manifests/local-path-provisioner-csi-configmap.yaml @@ -0,0 +1,264 @@ +# Copyright 2024 Nutanix. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +#================================================================= +# DO NOT EDIT THIS FILE +# IT HAS BEEN GENERATED BY /hack/addons/update-local-path-provisioner-csi.sh +#================================================================= +apiVersion: v1 +data: + local-path-provisioner-csi.yaml: | + apiVersion: v1 + imagePullSecrets: null + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/instance: local-path-provisioner + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: local-path-provisioner + app.kubernetes.io/version: v0.0.27 + helm.sh/chart: local-path-provisioner-0.0.29 + name: local-path-provisioner + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: + labels: + app.kubernetes.io/instance: local-path-provisioner + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: local-path-provisioner + app.kubernetes.io/version: v0.0.27 + helm.sh/chart: local-path-provisioner-0.0.29 + name: local-path-provisioner + namespace: kube-system + rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + - create + - patch + - update + - delete + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/instance: local-path-provisioner + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: local-path-provisioner + app.kubernetes.io/version: v0.0.27 + helm.sh/chart: local-path-provisioner-0.0.29 + name: local-path-provisioner + rules: + - apiGroups: + - "" + resources: + - nodes + - persistentvolumeclaims + - configmaps + - pods + - pods/log + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - patch + - update + - delete + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + labels: + app.kubernetes.io/instance: local-path-provisioner + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: local-path-provisioner + app.kubernetes.io/version: v0.0.27 + helm.sh/chart: local-path-provisioner-0.0.29 + name: local-path-provisioner + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: local-path-provisioner + subjects: + - kind: ServiceAccount + name: local-path-provisioner + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/instance: local-path-provisioner + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: local-path-provisioner + app.kubernetes.io/version: v0.0.27 + helm.sh/chart: local-path-provisioner-0.0.29 + name: local-path-provisioner + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: local-path-provisioner + subjects: + - kind: ServiceAccount + name: local-path-provisioner + namespace: kube-system + --- + apiVersion: v1 + data: + config.json: |- + { + "nodePathMap": [ + { + "node": "DEFAULT_PATH_FOR_NON_LISTED_NODES", + "paths": [ + "/opt/local-path-provisioner" + ] + } + ] + } + helperPod.yaml: |- + apiVersion: v1 + kind: Pod + metadata: + name: helper-pod + namespace: kube-system + spec: + priorityClassName: system-node-critical + tolerations: + - key: node.kubernetes.io/disk-pressure + operator: Exists + effect: NoSchedule + containers: + - name: helper-pod + image: busybox:1.36.1 + imagePullPolicy: IfNotPresent + resources: + {} + setup: |- + #!/bin/sh + set -eu + mkdir -m 0777 -p "$VOL_DIR" + teardown: |- + #!/bin/sh + set -eu + rm -rf "$VOL_DIR" + kind: ConfigMap + metadata: + labels: + app.kubernetes.io/instance: local-path-provisioner + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: local-path-provisioner + app.kubernetes.io/version: v0.0.27 + helm.sh/chart: local-path-provisioner-0.0.29 + name: local-path-config + namespace: kube-system + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app.kubernetes.io/instance: local-path-provisioner + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: local-path-provisioner + app.kubernetes.io/version: v0.0.27 + helm.sh/chart: local-path-provisioner-0.0.29 + name: local-path-provisioner + namespace: kube-system + spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/instance: local-path-provisioner + app.kubernetes.io/name: local-path-provisioner + template: + metadata: + labels: + app.kubernetes.io/instance: local-path-provisioner + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: local-path-provisioner + app.kubernetes.io/version: v0.0.27 + helm.sh/chart: local-path-provisioner-0.0.29 + spec: + containers: + - command: + - local-path-provisioner + - --debug + - start + - --config + - /etc/config/config.json + - --service-account-name + - local-path-provisioner + - --provisioner-name + - rancher.io/local-path + - --helper-image + - busybox:1.36.1 + - --configmap-name + - local-path-config + env: + - name: POD_NAMESPACE + value: kube-system + - name: CONFIG_MOUNT_PATH + value: /etc/config/ + image: rancher/local-path-provisioner:v0.0.27 + imagePullPolicy: IfNotPresent + name: local-path-provisioner + resources: {} + securityContext: {} + volumeMounts: + - mountPath: /etc/config/ + name: config-volume + securityContext: {} + serviceAccountName: local-path-provisioner + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + tolerationSeconds: 300 + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + volumes: + - configMap: + name: local-path-config + name: config-volume +kind: ConfigMap +metadata: + creationTimestamp: null + name: local-path-provisioner-csi diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/deployment.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/deployment.yaml index bee2322df..c195b5c69 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/templates/deployment.yaml +++ b/charts/cluster-api-runtime-extensions-nutanix/templates/deployment.yaml @@ -33,6 +33,7 @@ spec: - --helm-addons-configmap={{ .Values.helmAddonsConfigMap }} - --cni.cilium.helm-addon.default-values-template-configmap-name={{ .Values.hooks.cni.cilium.helmAddonStrategy.defaultValueTemplateConfigMap.name }} - --nfd.helm-addon.default-values-template-configmap-name={{ .Values.hooks.nfd.helmAddonStrategy.defaultValueTemplateConfigMap.name }} + - --csi.local-path.helm-addon.default-values-template-configmap-name={{ (index .Values.hooks.csi "local-path").helmAddonStrategy.defaultValueTemplateConfigMap.name }} {{- range $key, $value := .Values.extraArgs }} - --{{ $key }}={{ $value }} {{- end }} diff --git a/charts/cluster-api-runtime-extensions-nutanix/values.schema.json b/charts/cluster-api-runtime-extensions-nutanix/values.schema.json index 5cc9b2610..9161937c1 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/values.schema.json +++ b/charts/cluster-api-runtime-extensions-nutanix/values.schema.json @@ -281,6 +281,27 @@ }, "csi": { "properties": { + "local-path": { + "properties": { + "helmAddonStrategy": { + "properties": { + "defaultValueTemplateConfigMap": { + "properties": { + "create": { + "type": "boolean" + }, + "name": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + } + }, + "type": "object" + }, "nutanix": { "properties": { "helmAddonStrategy": { diff --git a/charts/cluster-api-runtime-extensions-nutanix/values.yaml b/charts/cluster-api-runtime-extensions-nutanix/values.yaml index 33062ba13..52cf7cd70 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/values.yaml +++ b/charts/cluster-api-runtime-extensions-nutanix/values.yaml @@ -49,6 +49,11 @@ hooks: defaultValueTemplateConfigMap: create: true name: default-nutanix-csi-helm-values-template + local-path: + helmAddonStrategy: + defaultValueTemplateConfigMap: + create: true + name: default-local-path-csi-helm-values-template ccm: nutanix: helmAddonStrategy: diff --git a/examples/capi-quick-start/docker-cluster-calico-crs.yaml b/examples/capi-quick-start/docker-cluster-calico-crs.yaml index 37621d8e7..dc368b28e 100644 --- a/examples/capi-quick-start/docker-cluster-calico-crs.yaml +++ b/examples/capi-quick-start/docker-cluster-calico-crs.yaml @@ -35,7 +35,7 @@ spec: local-path: storageClassConfigs: default: {} - strategy: HelmAddon + strategy: ClusterResourceSet nfd: strategy: ClusterResourceSet encryptionAtRest: diff --git a/examples/capi-quick-start/docker-cluster-cilium-crs.yaml b/examples/capi-quick-start/docker-cluster-cilium-crs.yaml index dc931bd9c..c4963acb0 100644 --- a/examples/capi-quick-start/docker-cluster-cilium-crs.yaml +++ b/examples/capi-quick-start/docker-cluster-cilium-crs.yaml @@ -35,7 +35,7 @@ spec: local-path: storageClassConfigs: default: {} - strategy: HelmAddon + strategy: ClusterResourceSet nfd: strategy: ClusterResourceSet encryptionAtRest: diff --git a/hack/addons/kustomize/local-path-provisioner-csi/helm-values.yaml b/hack/addons/kustomize/local-path-provisioner-csi/helm-values.yaml new file mode 100644 index 000000000..069fd2879 --- /dev/null +++ b/hack/addons/kustomize/local-path-provisioner-csi/helm-values.yaml @@ -0,0 +1,21 @@ +# Copyright 2024 Nutanix. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +storageClass: + create: false + provisionerName: rancher.io/local-path +helperImage: + tag: 1.36.1 +tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + tolerationSeconds: 300 + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists diff --git a/hack/addons/kustomize/local-path-provisioner-csi/kustomization.yaml.tmpl b/hack/addons/kustomize/local-path-provisioner-csi/kustomization.yaml.tmpl index c662c41b0..338ae300e 100644 --- a/hack/addons/kustomize/local-path-provisioner-csi/kustomization.yaml.tmpl +++ b/hack/addons/kustomize/local-path-provisioner-csi/kustomization.yaml.tmpl @@ -14,6 +14,7 @@ helmCharts: repo: https://charts.containeroo.ch releaseName: local-path-provisioner version: ${LOCAL_PATH_CSI_CHART_VERSION} + valuesFile: helm-values.yaml includeCRDs: true skipTests: true namespace: kube-system diff --git a/hack/addons/update-local-path-provisioner-csi.sh b/hack/addons/update-local-path-provisioner-csi.sh new file mode 100755 index 000000000..9a27e821c --- /dev/null +++ b/hack/addons/update-local-path-provisioner-csi.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -euo pipefail +IFS=$'\n\t' + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly SCRIPT_DIR + +# shellcheck source=hack/common.sh +source "${SCRIPT_DIR}/../common.sh" + +if [ -z "${LOCAL_PATH_CSI_CHART_VERSION:-}" ]; then + echo "Missing environment variable: LOCAL_PATH_CSI_CHART_VERSION" + exit 1 +fi + +ASSETS_DIR="$(mktemp -d -p "${TMPDIR:-/tmp}")" +readonly ASSETS_DIR +trap_add "rm -rf ${ASSETS_DIR}" EXIT + +readonly FILE_NAME="local-path-provisioner-csi.yaml" + +readonly KUSTOMIZE_BASE_DIR="${SCRIPT_DIR}/kustomize/local-path-provisioner-csi" +mkdir -p "${ASSETS_DIR}/local-path-provisioner-csi" +envsubst -no-unset <"${KUSTOMIZE_BASE_DIR}/kustomization.yaml.tmpl" >"${ASSETS_DIR}/local-path-provisioner-csi/kustomization.yaml" +cp -r "${KUSTOMIZE_BASE_DIR}"/*.yaml "${ASSETS_DIR}/local-path-provisioner-csi/" + +kustomize build --enable-helm "${ASSETS_DIR}/local-path-provisioner-csi/" >"${ASSETS_DIR}/${FILE_NAME}" + +kubectl create configmap local-path-provisioner-csi --dry-run=client --output yaml \ + --from-file "${ASSETS_DIR}/${FILE_NAME}" \ + >"${ASSETS_DIR}/local-path-provisioner-csi-configmap.yaml" + +# add warning not to edit file directly +cat <"${GIT_REPO_ROOT}/charts/cluster-api-runtime-extensions-nutanix/templates/csi/local-path/manifests/local-path-provisioner-csi-configmap.yaml" +$(cat "${GIT_REPO_ROOT}/hack/license-header.yaml.txt") + +#================================================================= +# DO NOT EDIT THIS FILE +# IT HAS BEEN GENERATED BY /hack/addons/update-local-path-provisioner-csi.sh +#================================================================= +$(cat "${ASSETS_DIR}/local-path-provisioner-csi-configmap.yaml") +EOF diff --git a/hack/examples/overlays/clusters/docker/calico/crs/kustomization.yaml.tmpl b/hack/examples/overlays/clusters/docker/calico/crs/kustomization.yaml.tmpl index 523858a21..a2a628f0c 100644 --- a/hack/examples/overlays/clusters/docker/calico/crs/kustomization.yaml.tmpl +++ b/hack/examples/overlays/clusters/docker/calico/crs/kustomization.yaml.tmpl @@ -17,3 +17,6 @@ patches: - target: kind: Cluster path: ../../../../../patches/crs-strategy.yaml + - target: + kind: Cluster + path: ../../../../../patches/docker/csi-crs-strategy.yaml diff --git a/hack/examples/overlays/clusters/docker/calico/helm-addon/kustomization.yaml.tmpl b/hack/examples/overlays/clusters/docker/calico/helm-addon/kustomization.yaml.tmpl index d402ae85b..7e44f63f5 100644 --- a/hack/examples/overlays/clusters/docker/calico/helm-addon/kustomization.yaml.tmpl +++ b/hack/examples/overlays/clusters/docker/calico/helm-addon/kustomization.yaml.tmpl @@ -17,3 +17,6 @@ patches: - target: kind: Cluster path: ../../../../../patches/helm-addon-strategy.yaml + - target: + kind: Cluster + path: ../../../../../patches/docker/csi-helm-addon-strategy.yaml diff --git a/hack/examples/overlays/clusters/docker/cilium/crs/kustomization.yaml.tmpl b/hack/examples/overlays/clusters/docker/cilium/crs/kustomization.yaml.tmpl index c5dec8ea7..6f784918d 100644 --- a/hack/examples/overlays/clusters/docker/cilium/crs/kustomization.yaml.tmpl +++ b/hack/examples/overlays/clusters/docker/cilium/crs/kustomization.yaml.tmpl @@ -17,3 +17,6 @@ patches: - target: kind: Cluster path: ../../../../../patches/crs-strategy.yaml + - target: + kind: Cluster + path: ../../../../../patches/docker/csi-crs-strategy.yaml diff --git a/hack/examples/overlays/clusters/docker/cilium/helm-addon/kustomization.yaml.tmpl b/hack/examples/overlays/clusters/docker/cilium/helm-addon/kustomization.yaml.tmpl index e0db60b9d..51af2f960 100644 --- a/hack/examples/overlays/clusters/docker/cilium/helm-addon/kustomization.yaml.tmpl +++ b/hack/examples/overlays/clusters/docker/cilium/helm-addon/kustomization.yaml.tmpl @@ -17,3 +17,6 @@ patches: - target: kind: Cluster path: ../../../../../patches/helm-addon-strategy.yaml + - target: + kind: Cluster + path: ../../../../../patches/docker/csi-helm-addon-strategy.yaml diff --git a/hack/examples/patches/docker/csi-crs-strategy.yaml b/hack/examples/patches/docker/csi-crs-strategy.yaml new file mode 100644 index 000000000..91e8b3bb8 --- /dev/null +++ b/hack/examples/patches/docker/csi-crs-strategy.yaml @@ -0,0 +1,6 @@ +# Copyright 2024 Nutanix. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +- op: "add" + path: "/spec/topology/variables/0/value/addons/csi/providers/local-path/strategy" + value: ClusterResourceSet diff --git a/hack/examples/patches/docker/csi-helm-addon-strategy.yaml b/hack/examples/patches/docker/csi-helm-addon-strategy.yaml new file mode 100644 index 000000000..b2f66976e --- /dev/null +++ b/hack/examples/patches/docker/csi-helm-addon-strategy.yaml @@ -0,0 +1,6 @@ +# Copyright 2024 Nutanix. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +- op: "add" + path: "/spec/topology/variables/0/value/addons/csi/providers/local-path/strategy" + value: HelmAddon diff --git a/hack/examples/patches/docker/csi.yaml b/hack/examples/patches/docker/csi.yaml index 3491ac94f..89f833103 100644 --- a/hack/examples/patches/docker/csi.yaml +++ b/hack/examples/patches/docker/csi.yaml @@ -9,6 +9,5 @@ storageClassConfig: default providers: local-path: - strategy: HelmAddon storageClassConfigs: default: {} diff --git a/make/addons.mk b/make/addons.mk index 58e096dbf..080c9e660 100644 --- a/make/addons.mk +++ b/make/addons.mk @@ -47,6 +47,10 @@ update-addon.cluster-autoscaler: ; $(info $(M) updating cluster-autoscaler manif update-addon.aws-ebs-csi: ; $(info $(M) updating aws ebs csi manifests) ./hack/addons/update-aws-ebs-csi.sh +.PHONY: update-addon.local-path-provisioner-csi +update-addon.local-path-provisioner-csi: ; $(info $(M) updating local-path-provisioner csi manifests) + ./hack/addons/update-local-path-provisioner-csi.sh + .PHONY: update-addon.aws-ccm.% update-addon.aws-ccm.%: ; $(info $(M) updating aws ccm $* manifests) ./hack/addons/update-aws-ccm.sh $(AWS_CCM_VERSION_$*) $(AWS_CCM_CHART_VERSION_$*) diff --git a/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go b/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go index c7ac315b3..be1318a97 100644 --- a/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go @@ -73,7 +73,7 @@ func (a *AWSEBS) Apply( default: return fmt.Errorf("stategy %s not implemented", strategy) } - err := csiutils.CreateStorageClassOnRemote( + err := csiutils.CreateStorageClassesOnRemote( ctx, a.client, provider.StorageClassConfigs, diff --git a/pkg/handlers/generic/lifecycle/csi/localpath/handler.go b/pkg/handlers/generic/lifecycle/csi/localpath/handler.go index f80f3a8b9..d1f41bf01 100644 --- a/pkg/handlers/generic/lifecycle/csi/localpath/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/localpath/handler.go @@ -8,17 +8,14 @@ import ( "fmt" "github.com/go-logr/logr" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/spf13/pflag" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - caaphv1 "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/external/sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" - "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/k8s/client" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/config" csiutils "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/csi/utils" - handlersutils "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/utils" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/options" ) const ( @@ -26,17 +23,41 @@ const ( defaultHelmReleaseNamespace = "kube-system" ) +type addonStrategy interface { + apply( + context.Context, + *clusterv1.Cluster, + string, + logr.Logger, + ) error +} + +type Config struct { + *options.GlobalOptions + + crsConfig crsConfig + helmAddonConfig helmAddonConfig +} + type LocalPathProvisionerCSI struct { client ctrlclient.Client helmChartInfoGetter *config.HelmChartGetter + config *Config +} + +func (c *Config) AddFlags(prefix string, flags *pflag.FlagSet) { + c.crsConfig.AddFlags(prefix+".crs", flags) + c.helmAddonConfig.AddFlags(prefix+".helm-addon", flags) } func New( c ctrlclient.Client, + cfg *Config, helmChartInfoGetter *config.HelmChartGetter, ) *LocalPathProvisionerCSI { return &LocalPathProvisionerCSI{ client: c, + config: cfg, helmChartInfoGetter: helmChartInfoGetter, } } @@ -48,19 +69,32 @@ func (l *LocalPathProvisionerCSI) Apply( cluster *clusterv1.Cluster, log logr.Logger, ) error { - strategy := provider.Strategy - switch strategy { + var strategy addonStrategy + switch provider.Strategy { case v1alpha1.AddonStrategyHelmAddon: - err := l.handleHelmAddonApply(ctx, cluster, log) + helmChart, err := l.helmChartInfoGetter.For(ctx, log, config.LocalPathProvisionerCSI) if err != nil { - return err + return fmt.Errorf("failed to get configuration to create helm addon: %w", err) + } + strategy = helmAddonStrategy{ + config: l.config.helmAddonConfig, + client: l.client, + helmChart: helmChart, } case v1alpha1.AddonStrategyClusterResourceSet: + strategy = crsStrategy{ + config: l.config.crsConfig, + client: l.client, + } default: return fmt.Errorf("strategy %s not implemented", strategy) } - err := csiutils.CreateStorageClassOnRemote( + if err := strategy.apply(ctx, cluster, l.config.DefaultsNamespace(), log); err != nil { + return fmt.Errorf("failed to apply local-path CSI addon: %w", err) + } + + err := csiutils.CreateStorageClassesOnRemote( ctx, l.client, provider.StorageClassConfigs, @@ -72,64 +106,9 @@ func (l *LocalPathProvisionerCSI) Apply( ) if err != nil { return fmt.Errorf( - "error creating StorageClasses for the local-path-provisioner CSI driver: %w", - err, - ) - } - return nil -} - -func (l *LocalPathProvisionerCSI) handleHelmAddonApply( - ctx context.Context, - cluster *clusterv1.Cluster, - log logr.Logger, -) error { - chart, err := l.helmChartInfoGetter.For(ctx, log, config.LocalPathProvisionerCSI) - if err != nil { - return fmt.Errorf("failed to get helm chart %q: %w", config.LocalPathProvisionerCSI, err) - } - - valuesTemplate := ` -storageClass: - create: false - provisionerName: rancher.io/local-path -helperImage: - tag: 1.36.1 -` - - chartProxy := &caaphv1.HelmChartProxy{ - TypeMeta: metav1.TypeMeta{ - APIVersion: caaphv1.GroupVersion.String(), - Kind: "HelmChartProxy", - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: cluster.Namespace, - Name: "local-path-provisioner-csi-" + cluster.Name, - }, - Spec: caaphv1.HelmChartProxySpec{ - RepoURL: chart.Repository, - ChartName: chart.Name, - ClusterSelector: metav1.LabelSelector{ - MatchLabels: map[string]string{clusterv1.ClusterNameLabel: cluster.Name}, - }, - ReleaseNamespace: defaultHelmReleaseNamespace, - ReleaseName: defaultHelmReleaseName, - Version: chart.Version, - ValuesTemplate: valuesTemplate, - }, - } - handlersutils.SetTLSConfigForHelmChartProxyIfNeeded(chartProxy) - if err = controllerutil.SetOwnerReference(cluster, chartProxy, l.client.Scheme()); err != nil { - return fmt.Errorf( - "failed to set owner reference on HelmChartProxy %q: %w", - chartProxy.Name, + "error creating StorageClasses for the local-path CSI driver: %w", err, ) } - - if err = client.ServerSideApply(ctx, l.client, chartProxy, client.ForceOwnership); err != nil { - return fmt.Errorf("failed to apply HelmChartProxy %q: %w", chartProxy.Name, err) - } - return nil } diff --git a/pkg/handlers/generic/lifecycle/csi/localpath/strategy_crs.go b/pkg/handlers/generic/lifecycle/csi/localpath/strategy_crs.go new file mode 100644 index 000000000..cbea01342 --- /dev/null +++ b/pkg/handlers/generic/lifecycle/csi/localpath/strategy_crs.go @@ -0,0 +1,96 @@ +// Copyright 2024 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package localpath + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + "github.com/spf13/pflag" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/k8s/client" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/utils" +) + +type crsConfig struct { + defaultLocalPathCSIConfigMapName string +} + +func (c *crsConfig) AddFlags(prefix string, flags *pflag.FlagSet) { + flags.StringVar( + &c.defaultLocalPathCSIConfigMapName, + prefix+".default-local-path-csi-configmap-name", + "local-path-provisioner-csi", + "name of the ConfigMap used to deploy local-path CSI", + ) +} + +type crsStrategy struct { + config crsConfig + + client ctrlclient.Client +} + +func (s crsStrategy) apply( + ctx context.Context, + cluster *clusterv1.Cluster, + defaultsNamespace string, + log logr.Logger, +) error { + defaultLocalPathCSIConfigMap := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: defaultsNamespace, + Name: s.config.defaultLocalPathCSIConfigMapName, + }, + } + + err := s.client.Get( + ctx, + ctrlclient.ObjectKeyFromObject(defaultLocalPathCSIConfigMap), + defaultLocalPathCSIConfigMap, + ) + if err != nil { + return fmt.Errorf("failed to get default local-path CSI ConfigMap: %w", err) + } + + log.Info("Ensuring local-path CSI installation CRS and ConfigMap exist for cluster") + + cm := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: cluster.Namespace, + Name: "local-path-provisioner-csi-" + cluster.Name, + }, + Data: defaultLocalPathCSIConfigMap.Data, + BinaryData: defaultLocalPathCSIConfigMap.BinaryData, + } + + if err := client.ServerSideApply(ctx, s.client, cm, client.ForceOwnership); err != nil { + return fmt.Errorf( + "failed to apply local-path CSI installation ConfigMap: %w", + err, + ) + } + + if err := utils.EnsureCRSForClusterFromObjects(ctx, cm.Name, s.client, cluster, cm); err != nil { + return fmt.Errorf( + "failed to apply local-path CSI installation ClusterResourceSet: %w", + err, + ) + } + + return nil +} diff --git a/pkg/handlers/generic/lifecycle/csi/localpath/strategy_helmaddon.go b/pkg/handlers/generic/lifecycle/csi/localpath/strategy_helmaddon.go new file mode 100644 index 000000000..a953d1aec --- /dev/null +++ b/pkg/handlers/generic/lifecycle/csi/localpath/strategy_helmaddon.go @@ -0,0 +1,97 @@ +// Copyright 2023 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package localpath + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + "github.com/spf13/pflag" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + caaphv1 "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/external/sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/k8s/client" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/config" + handlersutils "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/utils" +) + +type helmAddonConfig struct { + defaultValuesTemplateConfigMapName string +} + +func (c *helmAddonConfig) AddFlags(prefix string, flags *pflag.FlagSet) { + flags.StringVar( + &c.defaultValuesTemplateConfigMapName, + prefix+".default-values-template-configmap-name", + "default-local-path-csi-helm-values-template", + "default values ConfigMap name", + ) +} + +type helmAddonStrategy struct { + config helmAddonConfig + client ctrlclient.Client + helmChart *config.HelmChart +} + +func (s helmAddonStrategy) apply( + ctx context.Context, + cluster *clusterv1.Cluster, + defaultsNamespace string, + log logr.Logger, +) error { + log.Info("Retrieving local-path CSI installation values template for cluster") + values, err := handlersutils.RetrieveValuesTemplate( + ctx, + s.client, + s.config.defaultValuesTemplateConfigMapName, + defaultsNamespace, + ) + if err != nil { + return fmt.Errorf( + "failed to retrieve local-path CSI installation values template for cluster: %w", + err, + ) + } + + chartProxy := &caaphv1.HelmChartProxy{ + TypeMeta: metav1.TypeMeta{ + APIVersion: caaphv1.GroupVersion.String(), + Kind: "HelmChartProxy", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: cluster.Namespace, + Name: "local-path-provisioner-csi-" + cluster.Name, + }, + Spec: caaphv1.HelmChartProxySpec{ + RepoURL: s.helmChart.Repository, + ChartName: s.helmChart.Name, + ClusterSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{clusterv1.ClusterNameLabel: cluster.Name}, + }, + ReleaseNamespace: defaultHelmReleaseNamespace, + ReleaseName: defaultHelmReleaseName, + Version: s.helmChart.Version, + ValuesTemplate: values, + }, + } + handlersutils.SetTLSConfigForHelmChartProxyIfNeeded(chartProxy) + if err = controllerutil.SetOwnerReference(cluster, chartProxy, s.client.Scheme()); err != nil { + return fmt.Errorf( + "failed to set owner reference on HelmChartProxy %q: %w", + chartProxy.Name, + err, + ) + } + + if err = client.ServerSideApply(ctx, s.client, chartProxy, client.ForceOwnership); err != nil { + return fmt.Errorf("failed to apply HelmChartProxy %q: %w", chartProxy.Name, err) + } + + return nil +} diff --git a/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go b/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go index f52980503..f070e0c78 100644 --- a/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go @@ -128,7 +128,7 @@ func (n *NutanixCSI) Apply( } } - err := csiutils.CreateStorageClassOnRemote( + err := csiutils.CreateStorageClassesOnRemote( ctx, n.client, provider.StorageClassConfigs, diff --git a/pkg/handlers/generic/lifecycle/csi/utils/scs.go b/pkg/handlers/generic/lifecycle/csi/utils/scs.go index 9307544e8..37c9a9be8 100644 --- a/pkg/handlers/generic/lifecycle/csi/utils/scs.go +++ b/pkg/handlers/generic/lifecycle/csi/utils/scs.go @@ -65,7 +65,7 @@ func CreateStorageClass( return &sc } -func CreateStorageClassOnRemote( +func CreateStorageClassesOnRemote( ctx context.Context, cl ctrlclient.Client, configs map[string]v1alpha1.StorageClassConfig, diff --git a/pkg/handlers/generic/lifecycle/handlers.go b/pkg/handlers/generic/lifecycle/handlers.go index 09bcd1dff..be54a480d 100644 --- a/pkg/handlers/generic/lifecycle/handlers.go +++ b/pkg/handlers/generic/lifecycle/handlers.go @@ -38,6 +38,7 @@ type Handlers struct { awsccmConfig *awsccm.AWSCCMConfig nutanixCCMConfig *nutanixccm.Config metalLBConfig *metallb.Config + localPathCSIConfig *localpath.Config } func New( @@ -56,6 +57,7 @@ func New( nutanixCSIConfig: &nutanixcsi.NutanixCSIConfig{GlobalOptions: globalOptions}, nutanixCCMConfig: &nutanixccm.Config{GlobalOptions: globalOptions}, metalLBConfig: &metallb.Config{GlobalOptions: globalOptions}, + localPathCSIConfig: &localpath.Config{GlobalOptions: globalOptions}, } } @@ -72,7 +74,11 @@ func (h *Handlers) AllHandlers(mgr manager.Manager) []handlers.Named { h.nutanixCSIConfig, helmChartInfoGetter, ), - v1alpha1.CSIProviderLocalPath: localpath.New(mgr.GetClient(), helmChartInfoGetter), + v1alpha1.CSIProviderLocalPath: localpath.New( + mgr.GetClient(), + h.localPathCSIConfig, + helmChartInfoGetter, + ), } ccmHandlers := map[string]ccm.CCMProvider{ v1alpha1.CCMProviderAWS: awsccm.New(mgr.GetClient(), h.awsccmConfig), @@ -111,4 +117,5 @@ func (h *Handlers) AddFlags(flagSet *pflag.FlagSet) { h.nutanixCSIConfig.AddFlags("nutanixcsi", flagSet) h.nutanixCCMConfig.AddFlags("nutanixccm", flagSet) h.metalLBConfig.AddFlags("metallb", flagSet) + h.localPathCSIConfig.AddFlags("csi.local-path", flagSet) } diff --git a/test/e2e/addon_helpers.go b/test/e2e/addon_helpers.go index 3e747d358..331ee04d0 100644 --- a/test/e2e/addon_helpers.go +++ b/test/e2e/addon_helpers.go @@ -66,4 +66,17 @@ func WaitForAddonsToBeReadyInWorkloadCluster( ClusterResourceSetIntervals: input.ClusterResourceSetIntervals, }, ) + + WaitForCSIToBeReadyInWorkloadCluster( + ctx, + WaitForCSIToBeReadyInWorkloadClusterInput{ + CSI: input.AddonsConfig.CSI, + WorkloadCluster: input.WorkloadCluster, + ClusterProxy: input.ClusterProxy, + DeploymentIntervals: input.DeploymentIntervals, + DaemonSetIntervals: input.DaemonSetIntervals, + HelmReleaseIntervals: input.HelmReleaseIntervals, + ClusterResourceSetIntervals: input.ClusterResourceSetIntervals, + }, + ) } diff --git a/test/e2e/csi_helpers.go b/test/e2e/csi_helpers.go new file mode 100644 index 000000000..b468484cd --- /dev/null +++ b/test/e2e/csi_helpers.go @@ -0,0 +1,239 @@ +//go:build e2e + +// Copyright 2024 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "context" + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + storagev1 "k8s.io/api/storage/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + addonsv1 "sigs.k8s.io/cluster-api/exp/addons/api/v1beta1" + capie2e "sigs.k8s.io/cluster-api/test/e2e" + "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" + apivariables "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/variables" +) + +type WaitForCSIToBeReadyInWorkloadClusterInput struct { + CSI *apivariables.CSI + WorkloadCluster *clusterv1.Cluster + ClusterProxy framework.ClusterProxy + DeploymentIntervals []interface{} + DaemonSetIntervals []interface{} + HelmReleaseIntervals []interface{} + ClusterResourceSetIntervals []interface{} +} + +func WaitForCSIToBeReadyInWorkloadCluster( + ctx context.Context, + input WaitForCSIToBeReadyInWorkloadClusterInput, //nolint:gocritic // This hugeParam is OK in tests. +) { + if input.CSI == nil { + return + } + + for providerName, providerConfig := range input.CSI.Providers { + switch providerName { + case v1alpha1.CSIProviderLocalPath: + waitForLocalPathCSIToBeReadyInWorkloadCluster( + ctx, + waitForLocalPathCSIToBeReadyInWorkloadClusterInput{ + strategy: providerConfig.Strategy, + workloadCluster: input.WorkloadCluster, + clusterProxy: input.ClusterProxy, + deploymentIntervals: input.DeploymentIntervals, + helmReleaseIntervals: input.HelmReleaseIntervals, + clusterResourceSetIntervals: input.ClusterResourceSetIntervals, + }, + ) + default: + Fail( + fmt.Sprintf( + "Do not know how to wait for CSI provider %s to be ready", + providerName, + ), + ) + } + + waitForStorageClassesToExistInWorkloadCluster( + ctx, + waitForStorageClassesToExistInWorkloadClusterInput{ + storageClasses: providerConfig.StorageClassConfigs, + workloadCluster: input.WorkloadCluster, + clusterProxy: input.ClusterProxy, + providerName: providerName, + defaultStorage: input.CSI.DefaultStorage, + }, + ) + } +} + +type waitForLocalPathCSIToBeReadyInWorkloadClusterInput struct { + strategy v1alpha1.AddonStrategy + workloadCluster *clusterv1.Cluster + clusterProxy framework.ClusterProxy + deploymentIntervals []interface{} + helmReleaseIntervals []interface{} + clusterResourceSetIntervals []interface{} +} + +func waitForLocalPathCSIToBeReadyInWorkloadCluster( + ctx context.Context, + input waitForLocalPathCSIToBeReadyInWorkloadClusterInput, //nolint:gocritic // This hugeParam is OK in tests. +) { + switch input.strategy { + case v1alpha1.AddonStrategyClusterResourceSet: + crs := &addonsv1.ClusterResourceSet{} + Expect(input.clusterProxy.GetClient().Get( + ctx, + types.NamespacedName{ + Name: "local-path-provisioner-csi-" + input.workloadCluster.Name, + Namespace: input.workloadCluster.Namespace, + }, + crs, + )).To(Succeed()) + + framework.WaitForClusterResourceSetToApplyResources( + ctx, + framework.WaitForClusterResourceSetToApplyResourcesInput{ + ClusterResourceSet: crs, + ClusterProxy: input.clusterProxy, + Cluster: input.workloadCluster, + }, + input.clusterResourceSetIntervals..., + ) + case v1alpha1.AddonStrategyHelmAddon: + WaitForHelmReleaseProxyReadyForCluster( + ctx, + WaitForHelmReleaseProxyReadyForClusterInput{ + GetLister: input.clusterProxy.GetClient(), + Cluster: input.workloadCluster, + HelmChartProxyName: "local-path-provisioner-csi-" + input.workloadCluster.Name, + }, + input.helmReleaseIntervals..., + ) + default: + Fail( + fmt.Sprintf( + "Do not know how to wait for local-path-provisioner CSI using strategy %s to be ready", + input.strategy, + ), + ) + } + + workloadClusterClient := input.clusterProxy.GetWorkloadCluster( + ctx, input.workloadCluster.Namespace, input.workloadCluster.Name, + ).GetClient() + + WaitForDeploymentsAvailable(ctx, framework.WaitForDeploymentsAvailableInput{ + Getter: workloadClusterClient, + Deployment: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "local-path-provisioner", + Namespace: metav1.NamespaceSystem, + }, + }, + }, input.deploymentIntervals...) +} + +type waitForStorageClassesToExistInWorkloadClusterInput struct { + providerName string + storageClasses map[string]v1alpha1.StorageClassConfig + defaultStorage v1alpha1.DefaultStorage + workloadCluster *clusterv1.Cluster + clusterProxy framework.ClusterProxy +} + +func waitForStorageClassesToExistInWorkloadCluster( + ctx context.Context, + input waitForStorageClassesToExistInWorkloadClusterInput, //nolint:gocritic // This hugeParam is OK in tests. +) { + workloadClusterClient := input.clusterProxy.GetWorkloadCluster( + ctx, input.workloadCluster.Namespace, input.workloadCluster.Name, + ).GetClient() + + var provisioner string + switch input.providerName { + case v1alpha1.CSIProviderLocalPath: + provisioner = string(v1alpha1.LocalPathProvisioner) + default: + Fail( + fmt.Sprintf( + "Do not know how to wait for storage classes for CSI provider %s", + input.providerName, + ), + ) + } + + for storageClassName, storageClassConfig := range input.storageClasses { + isDefault := input.providerName == input.defaultStorage.Provider && + storageClassName == input.defaultStorage.StorageClassConfig + + waitForStorageClassToExistInWorkloadCluster( + ctx, + workloadClusterClient, + client.ObjectKey{Name: input.providerName + "-" + storageClassName}, + provisioner, + storageClassConfig, + isDefault, + ) + } +} + +func waitForStorageClassToExistInWorkloadCluster( + ctx context.Context, + workloadClusterClient client.Client, + scKey client.ObjectKey, + provisioner string, + scConfig v1alpha1.StorageClassConfig, + isDefault bool, + intervals ...interface{}, +) { + start := time.Now() + capie2e.Byf("waiting for storageclass %v to exist", scKey) + Log("starting to wait for storageclass to exist") + var gotSC storagev1.StorageClass + Eventually(func() bool { + if err := workloadClusterClient.Get(ctx, scKey, &gotSC); err != nil { + if apierrors.IsNotFound(err) { + return false + } + Expect(err).NotTo(HaveOccurred()) + } + return true + }, intervals...).Should(BeTrue()) + + Expect(gotSC.Provisioner).To(Equal(provisioner)) + Expect(gotSC.Parameters).To(Equal(scConfig.Parameters)) + if scConfig.ReclaimPolicy != nil { + Expect(gotSC.ReclaimPolicy).To(Equal(scConfig.ReclaimPolicy)) + } else { + Expect(gotSC.ReclaimPolicy).To(BeNil()) + } + if scConfig.VolumeBindingMode != nil { + Expect(gotSC.VolumeBindingMode).To(Equal(scConfig.VolumeBindingMode)) + } else { + Expect(gotSC.VolumeBindingMode).To(BeNil()) + } + Expect(gotSC.AllowVolumeExpansion).To(HaveValue(Equal(scConfig.AllowExpansion))) + if isDefault { + Expect(gotSC.Annotations).To(HaveKeyWithValue("storageclass.kubernetes.io/is-default-class", "true")) + } else { + Expect(gotSC.Annotations).ToNot(HaveKeyWithValue("storageclass.kubernetes.io/is-default-class", "true")) + } + + Logf("StorageClass %v now exists, took %v", scKey, time.Since(start)) +} diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index f73bdc062..a4e6518a7 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -19,6 +19,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/klog/v2" capie2e "sigs.k8s.io/cluster-api/test/e2e" @@ -209,6 +210,7 @@ func initScheme() *runtime.Scheme { scheme := runtime.NewScheme() capie2eframework.TryAddDefaultSchemes(scheme) Expect(helmaddonsv1.AddToScheme(scheme)).To(Succeed()) + Expect(storagev1.AddToScheme(scheme)).To(Succeed()) return scheme }