diff --git a/api/v1alpha1/addon_types.go b/api/v1alpha1/addon_types.go index 975c94291..b302911a4 100644 --- a/api/v1alpha1/addon_types.go +++ b/api/v1alpha1/addon_types.go @@ -19,8 +19,9 @@ const ( CNIProviderCalico = "Calico" CNIProviderCilium = "Cilium" - CSIProviderAWSEBS = "aws-ebs" - CSIProviderNutanix = "nutanix" + CSIProviderAWSEBS = "aws-ebs" + CSIProviderNutanix = "nutanix" + CSIProviderLocalPath = "local-path" VirtualIPProviderKubeVIP = "KubeVIP" @@ -45,8 +46,9 @@ const ( type StorageProvisioner string const ( - AWSEBSProvisioner StorageProvisioner = "ebs.csi.aws.com" - NutanixProvisioner StorageProvisioner = "csi.nutanix.com" + AWSEBSProvisioner StorageProvisioner = "ebs.csi.aws.com" + NutanixProvisioner StorageProvisioner = "csi.nutanix.com" + LocalPathProvisioner StorageProvisioner = "rancher.io/local-path" ) // FIXME: Remove the CCM providers from the API. Users do not provider this @@ -110,7 +112,7 @@ type ClusterAutoscaler struct { type DefaultStorage struct { // Name of the CSI Provider for the default storage class. // +kubebuilder:validation:Required - // +kubebuilder:validation:Enum=aws-ebs;nutanix + // +kubebuilder:validation:Enum=aws-ebs;nutanix;local-path ProviderName string `json:"providerName"` // Name of storage class config in any of the provider objects. @@ -131,7 +133,7 @@ type CSI struct { type CSIProvider struct { // Name of the CSI Provider. // +kubebuilder:validation:Required - // +kubebuilder:validation:Enum=aws-ebs;nutanix + // +kubebuilder:validation:Enum=aws-ebs;nutanix;local-path Name string `json:"name"` // StorageClassConfig is a list of storage class configurations for this CSI provider. diff --git a/api/v1alpha1/crds/caren.nutanix.com_awsclusterconfigs.yaml b/api/v1alpha1/crds/caren.nutanix.com_awsclusterconfigs.yaml index 1fe58146a..60ad28d02 100644 --- a/api/v1alpha1/crds/caren.nutanix.com_awsclusterconfigs.yaml +++ b/api/v1alpha1/crds/caren.nutanix.com_awsclusterconfigs.yaml @@ -113,6 +113,7 @@ spec: enum: - aws-ebs - nutanix + - local-path type: string storageClassConfigName: description: Name of storage class config in any of the @@ -151,6 +152,7 @@ spec: enum: - aws-ebs - nutanix + - local-path type: string storageClassConfig: description: StorageClassConfig is a list of storage diff --git a/api/v1alpha1/crds/caren.nutanix.com_dockerclusterconfigs.yaml b/api/v1alpha1/crds/caren.nutanix.com_dockerclusterconfigs.yaml index 5aa474f29..7ef37615c 100644 --- a/api/v1alpha1/crds/caren.nutanix.com_dockerclusterconfigs.yaml +++ b/api/v1alpha1/crds/caren.nutanix.com_dockerclusterconfigs.yaml @@ -114,6 +114,7 @@ spec: enum: - aws-ebs - nutanix + - local-path type: string storageClassConfigName: description: Name of storage class config in any of the @@ -152,6 +153,7 @@ spec: enum: - aws-ebs - nutanix + - local-path type: string storageClassConfig: description: StorageClassConfig is a list of storage diff --git a/api/v1alpha1/crds/caren.nutanix.com_genericclusterconfigs.yaml b/api/v1alpha1/crds/caren.nutanix.com_genericclusterconfigs.yaml index 771258d1b..2afd5e87f 100644 --- a/api/v1alpha1/crds/caren.nutanix.com_genericclusterconfigs.yaml +++ b/api/v1alpha1/crds/caren.nutanix.com_genericclusterconfigs.yaml @@ -122,6 +122,7 @@ spec: enum: - aws-ebs - nutanix + - local-path type: string storageClassConfigName: description: Name of storage class config in any of the @@ -160,6 +161,7 @@ spec: enum: - aws-ebs - nutanix + - local-path type: string storageClassConfig: description: StorageClassConfig is a list of storage diff --git a/api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml b/api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml index 0b3446067..4c2f64ce1 100644 --- a/api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml +++ b/api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml @@ -114,6 +114,7 @@ spec: enum: - aws-ebs - nutanix + - local-path type: string storageClassConfigName: description: Name of storage class config in any of the @@ -152,6 +153,7 @@ spec: enum: - aws-ebs - nutanix + - local-path type: string storageClassConfig: description: StorageClassConfig is a list of storage diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml index d5dbd994b..cf5de6dde 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml +++ b/charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml @@ -15,6 +15,10 @@ data: ChartName: cluster-autoscaler ChartVersion: 9.37.0 RepositoryURL: https://kubernetes.github.io/autoscaler + local-path-provisioner-csi: | + ChartName: local-path-provisioner + ChartVersion: v0.0.29 + RepositoryURL: https://charts.containeroo.ch metallb: | ChartName: metallb ChartVersion: v0.14.5 diff --git a/examples/capi-quick-start/docker-cluster-calico-crs.yaml b/examples/capi-quick-start/docker-cluster-calico-crs.yaml index 551aab200..34244f074 100644 --- a/examples/capi-quick-start/docker-cluster-calico-crs.yaml +++ b/examples/capi-quick-start/docker-cluster-calico-crs.yaml @@ -27,6 +27,15 @@ spec: cni: provider: Calico strategy: ClusterResourceSet + csi: + defaultStorage: + providerName: local-path + storageClassConfigName: local-path + providers: + - name: local-path + storageClassConfig: + - name: local-path + strategy: HelmAddon nfd: strategy: ClusterResourceSet encryptionAtRest: diff --git a/examples/capi-quick-start/docker-cluster-calico-helm-addon.yaml b/examples/capi-quick-start/docker-cluster-calico-helm-addon.yaml index 53c5cc4e7..f6b278094 100644 --- a/examples/capi-quick-start/docker-cluster-calico-helm-addon.yaml +++ b/examples/capi-quick-start/docker-cluster-calico-helm-addon.yaml @@ -27,6 +27,15 @@ spec: cni: provider: Calico strategy: HelmAddon + csi: + defaultStorage: + providerName: local-path + storageClassConfigName: local-path + providers: + - name: local-path + storageClassConfig: + - name: local-path + strategy: HelmAddon nfd: strategy: HelmAddon encryptionAtRest: diff --git a/examples/capi-quick-start/docker-cluster-cilium-crs.yaml b/examples/capi-quick-start/docker-cluster-cilium-crs.yaml index 0688f562a..062a1eb2e 100644 --- a/examples/capi-quick-start/docker-cluster-cilium-crs.yaml +++ b/examples/capi-quick-start/docker-cluster-cilium-crs.yaml @@ -27,6 +27,15 @@ spec: cni: provider: Cilium strategy: ClusterResourceSet + csi: + defaultStorage: + providerName: local-path + storageClassConfigName: local-path + providers: + - name: local-path + storageClassConfig: + - name: local-path + strategy: HelmAddon nfd: strategy: ClusterResourceSet encryptionAtRest: diff --git a/examples/capi-quick-start/docker-cluster-cilium-helm-addon.yaml b/examples/capi-quick-start/docker-cluster-cilium-helm-addon.yaml index 1eda93a6e..8412a058f 100644 --- a/examples/capi-quick-start/docker-cluster-cilium-helm-addon.yaml +++ b/examples/capi-quick-start/docker-cluster-cilium-helm-addon.yaml @@ -27,6 +27,15 @@ spec: cni: provider: Cilium strategy: HelmAddon + csi: + defaultStorage: + providerName: local-path + storageClassConfigName: local-path + providers: + - name: local-path + storageClassConfig: + - name: local-path + strategy: HelmAddon nfd: strategy: HelmAddon encryptionAtRest: diff --git a/hack/addons/kustomize/local-path-provisioner-csi/kustomization.yaml.tmpl b/hack/addons/kustomize/local-path-provisioner-csi/kustomization.yaml.tmpl new file mode 100644 index 000000000..c662c41b0 --- /dev/null +++ b/hack/addons/kustomize/local-path-provisioner-csi/kustomization.yaml.tmpl @@ -0,0 +1,19 @@ +# Copyright 2023 Nutanix. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +metadata: + name: local-path-provisioner + +namespace: kube-system + +helmCharts: +- name: local-path-provisioner + repo: https://charts.containeroo.ch + releaseName: local-path-provisioner + version: ${LOCAL_PATH_CSI_CHART_VERSION} + includeCRDs: true + skipTests: true + namespace: kube-system diff --git a/hack/examples/bases/docker/cluster/kustomization.yaml.tmpl b/hack/examples/bases/docker/cluster/kustomization.yaml.tmpl index 7a4e271eb..e498b447e 100644 --- a/hack/examples/bases/docker/cluster/kustomization.yaml.tmpl +++ b/hack/examples/bases/docker/cluster/kustomization.yaml.tmpl @@ -35,6 +35,9 @@ patches: - target: kind: Cluster path: ../../../patches/cluster-autoscaler.yaml +- target: + kind: Cluster + path: ../../../patches/docker/csi.yaml - target: kind: Cluster path: ../../../patches/encryption.yaml diff --git a/hack/examples/patches/docker/csi.yaml b/hack/examples/patches/docker/csi.yaml new file mode 100644 index 000000000..5463358c1 --- /dev/null +++ b/hack/examples/patches/docker/csi.yaml @@ -0,0 +1,14 @@ +# Copyright 2024 Nutanix. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +- op: "add" + path: "/spec/topology/variables/0/value/addons/csi" + value: + defaultStorage: + providerName: local-path + storageClassConfigName: local-path + providers: + - name: local-path + storageClassConfig: + - name: local-path + strategy: HelmAddon diff --git a/make/addons.mk b/make/addons.mk index 19d5b9c49..1055b8518 100644 --- a/make/addons.mk +++ b/make/addons.mk @@ -9,6 +9,7 @@ export AWS_CSI_SNAPSHOT_CONTROLLER_VERSION := v6.3.3 export AWS_EBS_CSI_CHART_VERSION := v2.28.1 export NUTANIX_STORAGE_CSI_CHART_VERSION := v3.0.0-beta.1912 export NUTANIX_SNAPSHOT_CSI_CHART_VERSION := v6.3.2 +export LOCAL_PATH_CSI_CHART_VERSION := v0.0.29 # a map of AWS CCM versions export AWS_CCM_VERSION_127 := v1.27.1 export AWS_CCM_CHART_VERSION_127 := 0.0.8 diff --git a/pkg/handlers/generic/lifecycle/config/cm.go b/pkg/handlers/generic/lifecycle/config/cm.go index 35bfb25e5..08eb8af4a 100644 --- a/pkg/handlers/generic/lifecycle/config/cm.go +++ b/pkg/handlers/generic/lifecycle/config/cm.go @@ -17,14 +17,15 @@ import ( type Component string const ( - Autoscaler Component = "cluster-autoscaler" - Tigera Component = "tigera-operator" - Cilium Component = "cilium" - NFD Component = "nfd" - NutanixStorageCSI Component = "nutanix-storage-csi" - NutanixSnapshotCSI Component = "nutanix-snapshot-csi" - NutanixCCM Component = "nutanix-ccm" - MetalLB Component = "metallb" + Autoscaler Component = "cluster-autoscaler" + Tigera Component = "tigera-operator" + Cilium Component = "cilium" + NFD Component = "nfd" + NutanixStorageCSI Component = "nutanix-storage-csi" + NutanixSnapshotCSI Component = "nutanix-snapshot-csi" + NutanixCCM Component = "nutanix-ccm" + MetalLB Component = "metallb" + LocalPathProvisionerCSI Component = "local-path-provisioner-csi" ) type HelmChartGetter struct { diff --git a/pkg/handlers/generic/lifecycle/csi/handler.go b/pkg/handlers/generic/lifecycle/csi/handler.go index 8670e78c3..ae20c424f 100644 --- a/pkg/handlers/generic/lifecycle/csi/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/handler.go @@ -79,7 +79,7 @@ func (c *CSIHandler) AfterControlPlaneInitialized( c.variablePath...) if err != nil { if variables.IsNotFoundError(err) { - log.Info("Skipping CSI handler, the cluster does not define the CSI variable") + log.V(5).Info("Skipping CSI handler, the cluster does not define the CSI variable") return } msg := "failed to read the CSI variable from the cluster" diff --git a/pkg/handlers/generic/lifecycle/csi/localpath/handler.go b/pkg/handlers/generic/lifecycle/csi/localpath/handler.go new file mode 100644 index 000000000..bd1cb1758 --- /dev/null +++ b/pkg/handlers/generic/lifecycle/csi/localpath/handler.go @@ -0,0 +1,135 @@ +// Copyright 2023 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package localpath + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + caaphv1 "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/external/sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/k8s/client" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/config" + lifecycleutils "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/utils" +) + +const ( + defaultHelmReleaseName = "local-path-provisioner" + defaultHelmReleaseNamespace = "kube-system" +) + +type LocalPathProvisionerCSI struct { + client ctrlclient.Client + helmChartInfoGetter *config.HelmChartGetter +} + +func New( + c ctrlclient.Client, + helmChartInfoGetter *config.HelmChartGetter, +) *LocalPathProvisionerCSI { + return &LocalPathProvisionerCSI{ + client: c, + helmChartInfoGetter: helmChartInfoGetter, + } +} + +func (l *LocalPathProvisionerCSI) Apply( + ctx context.Context, + provider v1alpha1.CSIProvider, + defaultStorageConfig v1alpha1.DefaultStorage, + req *runtimehooksv1.AfterControlPlaneInitializedRequest, + log logr.Logger, +) error { + strategy := provider.Strategy + switch strategy { + case v1alpha1.AddonStrategyHelmAddon: + err := l.handleHelmAddonApply(ctx, req, log) + if err != nil { + return err + } + case v1alpha1.AddonStrategyClusterResourceSet: + default: + return fmt.Errorf("strategy %s not implemented", strategy) + } + + err := lifecycleutils.CreateStorageClassOnRemote( + ctx, + l.client, + provider.StorageClassConfig, + &req.Cluster, + defaultStorageConfig, + v1alpha1.CSIProviderLocalPath, + v1alpha1.LocalPathProvisioner, + nil, + ) + if err != nil { + return fmt.Errorf( + "error creating StorageClasses for the local-path-provisioner CSI driver: %w", + err, + ) + } + return nil +} + +func (l *LocalPathProvisionerCSI) handleHelmAddonApply( + ctx context.Context, + req *runtimehooksv1.AfterControlPlaneInitializedRequest, + log logr.Logger, +) error { + chart, err := l.helmChartInfoGetter.For(ctx, log, config.LocalPathProvisionerCSI) + if err != nil { + return fmt.Errorf("failed to get helm chart %q: %w", config.LocalPathProvisionerCSI, err) + } + + valuesTemplate := ` +storageClass: + create: false + provisionerName: rancher.io/local-path +helperImage: + tag: 1.36.1 +` + + chartProxy := &caaphv1.HelmChartProxy{ + TypeMeta: metav1.TypeMeta{ + APIVersion: caaphv1.GroupVersion.String(), + Kind: "HelmChartProxy", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: req.Cluster.Namespace, + Name: "local-path-provisioner-csi-" + req.Cluster.Name, + }, + Spec: caaphv1.HelmChartProxySpec{ + RepoURL: chart.Repository, + ChartName: chart.Name, + ClusterSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{clusterv1.ClusterNameLabel: req.Cluster.Name}, + }, + ReleaseNamespace: defaultHelmReleaseNamespace, + ReleaseName: defaultHelmReleaseName, + Version: chart.Version, + ValuesTemplate: valuesTemplate, + }, + } + + if err = controllerutil.SetOwnerReference(&req.Cluster, chartProxy, l.client.Scheme()); err != nil { + return fmt.Errorf( + "failed to set owner reference on HelmChartProxy %q: %w", + chartProxy.Name, + err, + ) + } + + if err = client.ServerSideApply(ctx, l.client, chartProxy, client.ForceOwnership); err != nil { + return fmt.Errorf("failed to apply HelmChartProxy %q: %w", chartProxy.Name, err) + } + + return nil +} diff --git a/pkg/handlers/generic/lifecycle/handlers.go b/pkg/handlers/generic/lifecycle/handlers.go index 8b482b93c..09bcd1dff 100644 --- a/pkg/handlers/generic/lifecycle/handlers.go +++ b/pkg/handlers/generic/lifecycle/handlers.go @@ -18,6 +18,7 @@ import ( "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/config" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/csi" awsebs "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/csi/aws-ebs" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/csi/localpath" nutanixcsi "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/csi/nutanix-csi" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/nfd" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/servicelbgc" @@ -71,6 +72,7 @@ func (h *Handlers) AllHandlers(mgr manager.Manager) []handlers.Named { h.nutanixCSIConfig, helmChartInfoGetter, ), + v1alpha1.CSIProviderLocalPath: localpath.New(mgr.GetClient(), helmChartInfoGetter), } ccmHandlers := map[string]ccm.CCMProvider{ v1alpha1.CCMProviderAWS: awsccm.New(mgr.GetClient(), h.awsccmConfig),