refactor: deploy storage class as a cluster resource set

Author: faiq

api/v1alpha1/clusterconfig_types.go

@@ -14,9 +14,13 @@ import (
 	"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/openapi/patterns"
 )
 
+type StorageDriver string
+
 const (
-	CNIProviderCalico = "Calico"
-	CNIProviderCilium = "Cilium"
+	CNIProviderCalico               = "Calico"
+	CNIProviderCilium               = "Cilium"
+	AWSEBSDriver      StorageDriver = "ebs.csi.aws.com"
+	NutanixDriver     StorageDriver = "csi.nutanix.com"
 
 	CSIProviderAWSEBS  = "aws-ebs"
 	CSIProviderNutanix = "nutanix"

pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go

@@ -10,6 +10,7 @@ import (
 	"github.com/spf13/pflag"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
 	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
@@ -20,10 +21,6 @@ import (
 	"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/options"
 )
 
-const (
-	awsEBSProvisionerName = "ebs.csi.aws.com"
-)
-
 type AWSEBSConfig struct {
 	*options.GlobalOptions
 	defaultAWSEBSConfigMapName string
@@ -83,23 +80,33 @@ func (a *AWSEBS) createStorageClasses(ctx context.Context,
 	cluster *clusterv1.Cluster,
 	defaultStorageConfig *v1alpha1.DefaultStorage,
 ) error {
+	allStorageClasses := make([]runtime.Object, 0, len(configs))
 	for _, c := range configs {
 		setAsDefault := c.Name == defaultStorageConfig.StorageClassConfigName &&
 			v1alpha1.CSIProviderAWSEBS == defaultStorageConfig.ProviderName
-		err := lifecycleutils.CreateStorageClass(
-			ctx,
-			a.client,
+		allStorageClasses = append(allStorageClasses, lifecycleutils.CreateStorageClass(
 			c,
 			cluster,
 			a.config.GlobalOptions.DefaultsNamespace(),
-			awsEBSProvisionerName,
+			v1alpha1.CSIProviderAWSEBS,
 			setAsDefault,
-		)
-		if err != nil {
-			return fmt.Errorf("failed to create storageclass %w", err)
-		}
+		))
 	}
-	return nil
+	cm, err := lifecycleutils.CreateConfigMapForCRS(
+		"aws-storageclass-cm",
+		a.config.DefaultsNamespace(),
+		allStorageClasses...,
+	)
+	if err != nil {
+		return err
+	}
+	return lifecycleutils.EnsureCRSForClusterFromConfigMaps(
+		ctx,
+		"aws-storageclass-crs",
+		a.client,
+		cluster,
+		cm,
+	)
 }
 
 func (a *AWSEBS) handleCRSApply(ctx context.Context,

pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/spf13/pflag"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
 	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
@@ -17,7 +18,7 @@ import (
 	caaphv1 "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/external/sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1"
 	"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/v1alpha1"
 	"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/k8s/client"
-	"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/utils"
+	lifecycleutils "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/utils"
 	"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/options"
 )
 
@@ -26,7 +27,6 @@ const (
 	defaultHelmChartVersion        = "v2.6.6"
 	defaultHelmChartName           = "nutanix-csi-storage"
 	defaultHelmReleaseNameTemplate = "nutanix-csi-storage-%s"
-	nutanixCSIProvisionerName      = "csi.nutanix.com"
 )
 
 type NutanixCSIConfig struct {
@@ -87,7 +87,7 @@ func (n *NutanixCSI) handleHelmAddonApply(
 	ctx context.Context,
 	req *runtimehooksv1.AfterControlPlaneInitializedRequest,
 ) error {
-	valuesTemplateConfigMap, err := utils.RetrieveValuesTemplateConfigMap(ctx,
+	valuesTemplateConfigMap, err := lifecycleutils.RetrieveValuesTemplateConfigMap(ctx,
 		n.client,
 		n.config.defaultValuesTemplateConfigMapName,
 		n.config.DefaultsNamespace())
@@ -140,21 +140,31 @@ func (n *NutanixCSI) createStorageClasses(ctx context.Context,
 	cluster *clusterv1.Cluster,
 	defaultStorageConfig *v1alpha1.DefaultStorage,
 ) error {
+	allStorageClasses := make([]runtime.Object, 0, len(configs))
 	for _, c := range configs {
 		setAsDefault := c.Name == defaultStorageConfig.StorageClassConfigName &&
 			v1alpha1.CSIProviderNutanix == defaultStorageConfig.ProviderName
-		err := utils.CreateStorageClass(
-			ctx,
-			n.client,
+		allStorageClasses = append(allStorageClasses, lifecycleutils.CreateStorageClass(
 			c,
 			cluster,
-			nutanixCSIProvisionerName,
 			n.config.GlobalOptions.DefaultsNamespace(),
+			v1alpha1.NutanixDriver,
 			setAsDefault,
-		)
-		if err != nil {
-			return fmt.Errorf("failed to create storageclass %w", err)
-		}
+		))
 	}
-	return nil
+	cm, err := lifecycleutils.CreateConfigMapForCRS(
+		"nutanix-storageclass-cm",
+		n.config.DefaultsNamespace(),
+		allStorageClasses...,
+	)
+	if err != nil {
+		return err
+	}
+	return lifecycleutils.EnsureCRSForClusterFromConfigMaps(
+		ctx,
+		"nutanix-storageclass-crs",
+		n.client,
+		cluster,
+		cm,
+	)
 }

pkg/handlers/generic/lifecycle/utils/utils.go

@@ -11,11 +11,12 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	storagev1 "k8s.io/api/storage/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/utils/ptr"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
-	"sigs.k8s.io/cluster-api/controllers/remote"
 	crsv1 "sigs.k8s.io/cluster-api/exp/addons/api/v1beta1"
-	capiutil "sigs.k8s.io/cluster-api/util"
+	utilyaml "sigs.k8s.io/cluster-api/util/yaml"
 	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
@@ -24,7 +25,8 @@ import (
 )
 
 const (
-	kindStorageClass = "StorageClass"
+	kindStorageClass       = "StorageClass"
+	defaultCRSConfigMapKey = "custom-resources.yaml"
 )
 
 var (
@@ -135,20 +137,17 @@ func RetrieveValuesTemplateConfigMap(
 }
 
 func CreateStorageClass(
-	ctx context.Context,
-	cl ctrlclient.Client,
 	storageConfig v1alpha1.StorageClassConfig,
 	cluster *clusterv1.Cluster,
-	defaultsNamespace,
-	provisionerName string,
+	defaultsNamespace string,
+	provisionerName v1alpha1.StorageDriver,
 	isDefault bool,
-) error {
+) *storagev1.StorageClass {
 	var volumeBindingMode *storagev1.VolumeBindingMode
 	switch storageConfig.VolumeBindingMode {
 	case v1alpha1.VolumeBindingImmediate:
 		volumeBindingMode = ptr.To(storagev1.VolumeBindingImmediate)
 	case v1alpha1.VolumeBindingWaitForFirstConsumer:
-	default:
 		volumeBindingMode = ptr.To(storagev1.VolumeBindingWaitForFirstConsumer)
 	}
 	var reclaimPolicy *corev1.PersistentVolumeReclaimPolicy
@@ -173,29 +172,47 @@ func CreateStorageClass(
 			Name:      storageConfig.Name,
 			Namespace: defaultsNamespace,
 		},
-		Provisioner:       provisionerName,
+		Provisioner:       string(provisionerName),
 		Parameters:        params,
 		VolumeBindingMode: volumeBindingMode,
 		ReclaimPolicy:     reclaimPolicy,
 	}
 	if isDefault {
 		sc.ObjectMeta.Annotations = defaultStorageClassMap
 	}
-	workloadClient, err := remote.NewClusterClient(
-		ctx,
-		"",
-		cl,
-		capiutil.ObjectKey(cluster),
-	)
-	if err != nil {
-		return err
-	}
+	return &sc
+}
 
-	if err := client.ServerSideApply(ctx, workloadClient, &sc); err != nil {
-		return fmt.Errorf(
-			"failed to create storage class %w",
-			err,
-		)
+func CreateConfigMapForCRS(configMapName, configMapNamespace string,
+	objs ...runtime.Object,
+) (*corev1.ConfigMap, error) {
+	cm := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      configMapName,
+			Namespace: configMapNamespace,
+		},
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: corev1.SchemeGroupVersion.String(),
+			Kind:       "ConfigMap",
+		},
+		Data: make(map[string]string),
+	}
+	l := make([][]byte, 0, len(objs))
+	for _, v := range objs {
+		obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(v)
+		if err != nil {
+			return nil, err
+		}
+		objYaml, err := utilyaml.FromUnstructured([]unstructured.Unstructured{
+			{
+				Object: obj,
+			},
+		})
+		if err != nil {
+			return nil, err
+		}
+		l = append(l, objYaml)
 	}
-	return nil
+	cm.Data[defaultCRSConfigMapKey] = fmt.Sprintf("|\n%s", string(utilyaml.JoinYaml(l...)))
+	return cm, nil
 }

pkg/handlers/generic/lifecycle/utils/utils_test.go

@@ -0,0 +1,95 @@
+// Copyright 2023 D2iQ, Inc. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package utils
+
+import (
+	"testing"
+
+	corev1 "k8s.io/api/core/v1"
+	storagev1 "k8s.io/api/storage/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+)
+
+func TestCreateConfigMapForCRS(t *testing.T) {
+	tests := []struct {
+		name          string
+		testCMName    string
+		testNamespace string
+		objs          []runtime.Object
+		expectedCM    corev1.ConfigMap
+	}{
+		{
+			name:          "multiple storage class objects",
+			testCMName:    "test",
+			testNamespace: "default",
+			objs: []runtime.Object{
+				&storagev1.StorageClass{
+					TypeMeta: metav1.TypeMeta{
+						Kind:       kindStorageClass,
+						APIVersion: storagev1.SchemeGroupVersion.String(),
+					},
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "test",
+						Namespace: "default",
+					},
+				},
+				&storagev1.StorageClass{
+					TypeMeta: metav1.TypeMeta{
+						Kind:       kindStorageClass,
+						APIVersion: storagev1.SchemeGroupVersion.String(),
+					},
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "test-2",
+						Namespace: "default",
+					},
+				},
+			},
+			expectedCM: corev1.ConfigMap{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test",
+					Namespace: "default",
+				},
+				TypeMeta: metav1.TypeMeta{
+					APIVersion: corev1.SchemeGroupVersion.String(),
+					Kind:       "ConfigMap",
+				},
+				Data: map[string]string{
+					defaultCRSConfigMapKey: `|
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  creationTimestamp: null
+  name: test
+  namespace: default
+provisioner: ""
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  creationTimestamp: null
+  name: test-2
+  namespace: default
+provisioner: ""`,
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cm, err := CreateConfigMapForCRS(tt.testCMName, tt.testNamespace, tt.objs...)
+			if err != nil {
+				t.Errorf("failed to create cm with error %v", err)
+			}
+			data, ok := cm.Data[defaultCRSConfigMapKey]
+			if !ok {
+				t.Errorf("expected %s to exist in cm.Data. got %v", defaultCRSConfigMapKey, cm.Data)
+			}
+			expected := tt.expectedCM.Data[defaultCRSConfigMapKey]
+			if data != expected {
+				t.Errorf("expected %s \n got %s", expected, data)
+			}
+		})
+	}
+}