update test setup to avoid running Datacache setup on machines not supporting LSSDs

Author: hungnguyen243

cmd/gce-pd-csi-driver/main.go

@@ -75,7 +75,7 @@ var (
 	fallbackRequisiteZonesFlag  = flag.String("fallback-requisite-zones", "", "Comma separated list of requisite zones that will be used if there are not sufficient zones present in requisite topologies when provisioning a disk")
 	enableStoragePoolsFlag      = flag.Bool("enable-storage-pools", false, "If set to true, the CSI Driver will allow volumes to be provisioned in Storage Pools")
 	enableHdHAFlag              = flag.Bool("allow-hdha-provisioning", false, "If set to true, will allow the driver to provision Hyperdisk-balanced High Availability disks")
-	enableDataCacheFlag         = flag.Bool("enable-data-cache", false, "If set to true, the CSI Driver will allow volumes to be provisioned with data cache configuration")
+	enableDataCacheFlag         = flag.Bool("enable-data-cache", false, "If set to true, the CSI Driver will allow volumes to be provisioned with Data Cache configuration")
 	nodeName                    = flag.String("node-name", "", "The node this driver is running on")
 
 	multiZoneVolumeHandleDiskTypesFlag = flag.String("multi-zone-volume-handle-disk-types", "", "Comma separated list of allowed disk types that can use the multi-zone volumeHandle. Used only if --multi-zone-volume-handle-enable")
@@ -258,10 +258,10 @@ func handle() {
 
 	if *enableDataCacheFlag {
 		if nodeName == nil || *nodeName == "" {
-			klog.Errorf("Data cache enabled, but --node-name not passed")
+			klog.Errorf("Data Cache enabled, but --node-name not passed")
 		}
 		if err := setupDataCache(ctx, *nodeName); err != nil {
-			klog.Errorf("DataCache setup failed: %v", err)
+			klog.Errorf("Data Cache setup failed: %v", err)
 		}
 	}
 
@@ -370,7 +370,7 @@ func fetchLssdsForRaiding(lssdCount int) ([]string, error) {
 		return nil, fmt.Errorf("Error listing LSSDs with empty mountpoint: %v", err)
 	}
 
-	// We need to ensure the disks to be used for Datacache are both unRAIDed & not containing mountpoints for ephemeral storage already
+	// We need to ensure the disks to be used for Data Cache are both unRAIDed & not containing mountpoints for ephemeral storage already
 	availableLssds := slices.Filter(nil, unRaidedLssds, func(e string) bool {
 		return slices.Contains(LSSDsWithEmptyMountPoint, e)
 	})
@@ -390,31 +390,31 @@ func setupDataCache(ctx context.Context, nodeName string) error {
 	if err != nil {
 		klog.V(2).Infof("Errored while scanning for available LocalSSDs err:%v; continuing Raiding", err)
 	} else if isAlreadyRaided {
-		klog.V(2).Infof("Local SSDs are already RAIDed. Skipping Datacache setup.")
+		klog.V(2).Infof("Local SSDs are already RAIDed. Skipping Data Cache setup.")
 		return nil
 	}
 
 	lssdCount := common.LocalSSDCountForDataCache
 	if nodeName != common.TestNode {
 		var err error
 		lssdCount, err = driver.GetDataCacheCountFromNodeLabel(ctx, nodeName)
-		if lssdCount == 0 {
-			klog.Infof("Datacache is not enabled on node %v", nodeName)
-			return nil
-		}
 		if err != nil {
 			return err
 		}
+		if lssdCount == 0 {
+			klog.Infof("Data Cache is not enabled on node %v", nodeName)
+			return nil
+		}
 	}
 	lssdNames, err := fetchLssdsForRaiding(lssdCount)
 	if err != nil {
-		klog.Fatalf("Failed to get sufficient SSDs for Datacache's caching setup: %v", err)
+		klog.Fatalf("Failed to get sufficient SSDs for Data Cache's caching setup: %v", err)
 	}
-	klog.V(2).Infof("Raiding local ssds to setup data cache: %v", lssdNames)
+	klog.V(2).Infof("Raiding local ssds to setup Data Cache: %v", lssdNames)
 	if err := driver.RaidLocalSsds(lssdNames); err != nil {
-		return fmt.Errorf("Failed to Raid local SSDs, unable to setup data caching, got error %v", err)
+		return fmt.Errorf("Failed to Raid local SSDs, unable to setup Data Cache, got error %v", err)
 	}
 
-	klog.V(2).Infof("Datacache enabled for node %s", nodeName)
+	klog.V(2).Infof("Data Cache enabled for node %s", nodeName)
 	return nil
 }

pkg/common/constants.go

@@ -48,7 +48,7 @@ const (
 	// Default LSSD count for datacache E2E tests
 	LocalSSDCountForDataCache = 2
 
-	// Node label for datacache
+	// Node label for Data Cache (only applicable to GKE nodes)
 	NodeLabelPrefix         = "cloud.google.com/%s"
 	DataCacheLssdCountLabel = "gke-data-cache-disk"
 )

pkg/gce-pd-csi-driver/cache.go

@@ -16,11 +16,10 @@ import (
 )
 
 const (
-	cacheSuffix               = "csi-fast"
-	mainLvSuffix              = "csi-main"
-	raidedLocalSsdName        = "csi-driver-data-cache"
-	raidMode                  = "0"
-	initialRaidedLocalSsdPath = "/dev/md0"
+	cacheSuffix        = "csi-fast"
+	mainLvSuffix       = "csi-main"
+	raidedLocalSsdName = "csi-driver-data-cache"
+	raidMode           = "0"
 )
 
 func fetchRAIDedLocalSsdPath() (string, error) {
@@ -30,7 +29,7 @@ func fetchRAIDedLocalSsdPath() (string, error) {
 	}
 	info, err := common.RunCommand("grep", []string{raidedLocalSsdName}, "mdadm", args...)
 	if err != nil || len(info) == 0 {
-		return "", fmt.Errorf("Error getting RAIDed device path for Datacache %v, output:%v ===============", err, string(info))
+		return "", fmt.Errorf("Error getting RAIDed device path for Data Cache %v, output:%v", err, string(info))
 	}
 	infoString := strings.TrimSpace(string(info))
 	infoSlice := strings.Split(infoString, " ")
@@ -82,9 +81,9 @@ func setupCaching(devicePath string, req *csi.NodeStageVolumeRequest, nodeId str
 	infoString = strings.ReplaceAll(infoString, "\"", "")
 	infoSlice := strings.Split(strings.TrimSpace(infoString), " ")
 	vgNameForPv := strings.TrimSpace(infoSlice[(len(infoSlice) - 1)])
-	klog.V(2).Infof("============================== Physical volume is part of Volume group: %v ==============================", vgNameForPv)
+	klog.V(2).Infof("Physical volume is part of Volume group: %v", vgNameForPv)
 	if vgNameForPv == volumeGroupName {
-		klog.V(2).Infof("============================== Physical Volume(PV) already exists in the Volume Group ==============================")
+		klog.V(2).Infof("Physical Volume(PV) already exists in the Volume Group")
 	} else if vgNameForPv != "VG" && vgNameForPv != "" {
 
 		info, err = common.RunCommand("" /* pipedCmd */, nil /* pipedCmdArg */, "vgchange", []string{"-an", vgNameForPv}...)
@@ -218,48 +217,50 @@ func ValidateDataCacheConfig(dataCacheMode string, datacacheSize string, ctx con
 		}
 		return nil
 	}
-	klog.Infof("Data cache is not enabled for PVC")
+	klog.Infof("Data Cache is not enabled for PVC")
 	return nil
 }
 
 func GetDataCacheCountFromNodeLabel(ctx context.Context, nodeName string) (int, error) {
-	if nodeName == common.TestNode {
-		return common.LocalSSDCountForDataCache, nil
-	}
 	cfg, err := rest.InClusterConfig()
 	// We want to capture API errors with node label fetching, so return -1
 	// in those cases instead of 0.
 	if err != nil {
-		return -1, err
+		return 0, err
 	}
 	kubeClient, err := kubernetes.NewForConfig(cfg)
 	if err != nil {
-		return -1, err
+		return 0, err
 	}
 	node, err := kubeClient.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{})
 	if err != nil {
 		// We could retry, but this error will also crashloop the driver which may be as good a way to retry as any.
-		return -1, err
+		return 0, err
 	}
 	if val, found := node.GetLabels()[fmt.Sprintf(common.NodeLabelPrefix, common.DataCacheLssdCountLabel)]; found {
 		dataCacheCount, err := strconv.Atoi(val)
 		if err != nil {
-			return -1, fmt.Errorf("Error getting Datacache's LSSD count from node label: %v", err)
+			return 0, fmt.Errorf("Error getting Data Cache's LSSD count from node label: %v", err)
 		}
-		klog.Infof("Number of local SSDs requested for Datacache: %v", dataCacheCount)
+		klog.Infof("Number of local SSDs requested for Data Cache: %v", dataCacheCount)
 		return dataCacheCount, nil
 	}
-	return 0, fmt.Errorf("Cannot get Datacache's LSSD count from node label")
+	// This will be returned for a non-Data-Cache node pool
+	return 0, nil
 }
 
 func FetchRaidedLssdCountForDatacache() (int, error) {
+	raidedPath, err := fetchRAIDedLocalSsdPath()
+	if err != nil {
+		return 0, err
+	}
 	args := []string{
 		"--detail",
-		initialRaidedLocalSsdPath,
+		raidedPath,
 	}
 	info, err := common.RunCommand("grep", []string{"Raid Devices"}, "mdadm", args...)
 	if err != nil {
-		return 0, fmt.Errorf("Error getting RAIDed devices for Datacache")
+		return 0, fmt.Errorf("Error getting RAIDed devices for Data Cache")
 	}
 	if len(info) != 0 {
 		raidedDeviceInfo := strings.Split(strings.TrimSpace(string(info)), ":")
@@ -358,6 +359,7 @@ func cleanupCache(volumeId string, nodeId string) error {
 		// If volume group doesn't exist then there's nothing to uncache
 		return nil
 	}
+	reduceVolumeGroup(volumeGroupName, true)
 	mainLvName := getLvName(mainLvSuffix, volumeId)
 	args := []string{
 		"-an",
@@ -431,7 +433,7 @@ func reduceVolumeGroup(volumeGroupName string, force bool) {
 func RaidLocalSsds(availableLssds []string) error {
 	args := []string{
 		"--create",
-		initialRaidedLocalSsdPath,
+		raidedLocalSsdName,
 		"--name",
 		raidedLocalSsdName,
 		"-l" + raidMode,

pkg/gce-pd-csi-driver/node.go

@@ -347,7 +347,7 @@ func (ns *GCENodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStage
 		}
 		configError := ValidateDataCacheConfig(req.GetPublishContext()[common.ContextDataCacheMode], req.GetPublishContext()[common.ContextDataCacheSize], ctx, nodeId)
 		if configError != nil {
-			return nil, status.Error(codes.Internal, fmt.Sprintf("Error validate configuration for Datacache: %v", err.Error()))
+			return nil, status.Error(codes.Internal, fmt.Sprintf("Error validate configuration for Data Cache: %v", err.Error()))
 		}
 		devicePath, err = setupCaching(devFsPath, req, nodeId)
 		if err != nil {

test/e2e/utils/utils.go

@@ -71,8 +71,11 @@ func GCEClientAndDriverSetup(instance *remote.InstanceInfo, driverConfig DriverC
 		"--allow-hdha-provisioning",
 		"--device-in-use-timeout=10s", // Set lower than the usual value to expedite tests
 		fmt.Sprintf("--fallback-requisite-zones=%s", strings.Join(driverConfig.Zones, ",")),
-		"--enable-data-cache",
-		fmt.Sprintf("--node-name=%s", utilcommon.TestNode),
+	}
+
+	if instance.GetLocalSSD() > 0 {
+		extra_flags = append(extra_flags, "--enable-data-cache")
+		extra_flags = append(extra_flags, fmt.Sprintf("--node-name=%s", utilcommon.TestNode))
 	}
 	extra_flags = append(extra_flags, fmt.Sprintf("--compute-endpoint=%s", driverConfig.ComputeEndpoint))
 	extra_flags = append(extra_flags, driverConfig.ExtraFlags...)

test/remote/instance.go

@@ -80,6 +80,10 @@ func (i *InstanceInfo) GetNodeID() string {
 	return common.CreateNodeID(i.cfg.Project, i.cfg.Zone, i.cfg.Name)
 }
 
+func (i *InstanceInfo) GetLocalSSD() int64 {
+	return i.cfg.LocalSSDCount
+}
+
 func machineTypeMismatch(curInst *compute.Instance, newInst *compute.Instance) bool {
 	if !strings.Contains(curInst.MachineType, newInst.MachineType) {
 		klog.Infof("Machine type mismatch")