From 9d1ce534e5be4e018c932c9ea69a684d94d57913 Mon Sep 17 00:00:00 2001 From: Matthew Cary Date: Fri, 2 Oct 2020 12:05:35 -0700 Subject: [PATCH] Add test support for regional PD --- test/k8s-integration/cluster.go | 4 ++ test/k8s-integration/config/sc-regional.yaml | 9 +++++ .../config/test-config-template.in | 3 +- test/k8s-integration/driver-config.go | 30 ++++++++++----- test/k8s-integration/main.go | 37 ++++++++++++++++--- test/run-k8s-integration-ci.sh | 9 ++++- 6 files changed, 75 insertions(+), 17 deletions(-) create mode 100644 test/k8s-integration/config/sc-regional.yaml diff --git a/test/k8s-integration/cluster.go b/test/k8s-integration/cluster.go index f8a207832..30e4c26ee 100644 --- a/test/k8s-integration/cluster.go +++ b/test/k8s-integration/cluster.go @@ -30,6 +30,10 @@ func gkeLocationArgs(gceZone, gceRegion string) (locationArg, locationVal string return } +func isRegionalGKECluster(gceZone, gceRegion string) bool { + return len(gceRegion) > 0 +} + func clusterDownGCE(k8sDir string) error { cmd := exec.Command(filepath.Join(k8sDir, "hack", "e2e-internal", "e2e-down.sh")) err := runCommand("Bringing Down E2E Cluster on GCE", cmd) diff --git a/test/k8s-integration/config/sc-regional.yaml b/test/k8s-integration/config/sc-regional.yaml new file mode 100644 index 000000000..4ad0b7792 --- /dev/null +++ b/test/k8s-integration/config/sc-regional.yaml @@ -0,0 +1,9 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: csi-gcepd-regional +provisioner: pd.csi.storage.gke.io +parameters: + type: pd-standard + replication-type: regional-pd +volumeBindingMode: WaitForFirstConsumer diff --git a/test/k8s-integration/config/test-config-template.in b/test/k8s-integration/config/test-config-template.in index 3693a14b4..7f08565a1 100644 --- a/test/k8s-integration/config/test-config-template.in +++ b/test/k8s-integration/config/test-config-template.in @@ -19,7 +19,8 @@ DriverInfo: debug: nouid32: SupportedSizeRange: - Min: 5Gi + Min: {{.MinimumVolumeSize}} Max: 64Ti TopologyKeys: - topology.gke.io/zone + NumAllowedTopologies: {{.NumAllowedTopologies}} \ No newline at end of file diff --git a/test/k8s-integration/driver-config.go b/test/k8s-integration/driver-config.go index 73a0d1ec3..eaba8b664 100644 --- a/test/k8s-integration/driver-config.go +++ b/test/k8s-integration/driver-config.go @@ -10,11 +10,13 @@ import ( ) type driverConfig struct { - StorageClassFile string - StorageClass string - SnapshotClassFile string - Capabilities []string - SupportedFsType []string + StorageClassFile string + StorageClass string + SnapshotClassFile string + Capabilities []string + SupportedFsType []string + MinimumVolumeSize string + NumAllowedTopologies int } const ( @@ -98,12 +100,20 @@ func generateDriverConfigFile(platform, pkgDir, storageClassFile, snapshotClassF absSnapshotClassFilePath = filepath.Join(pkgDir, testConfigDir, snapshotClassFile) } + minimumVolumeSize := "5Gi" + numAllowedTopologies := 1 + if storageClassFile == regionalPDStorageClass { + minimumVolumeSize = "200Gi" + numAllowedTopologies = 2 + } params := driverConfig{ - StorageClassFile: filepath.Join(pkgDir, testConfigDir, storageClassFile), - StorageClass: storageClassFile[:strings.LastIndex(storageClassFile, ".")], - SnapshotClassFile: absSnapshotClassFilePath, - SupportedFsType: fsTypes, - Capabilities: caps, + StorageClassFile: filepath.Join(pkgDir, testConfigDir, storageClassFile), + StorageClass: storageClassFile[:strings.LastIndex(storageClassFile, ".")], + SnapshotClassFile: absSnapshotClassFilePath, + SupportedFsType: fsTypes, + Capabilities: caps, + MinimumVolumeSize: minimumVolumeSize, + NumAllowedTopologies: numAllowedTopologies, } // Write config file diff --git a/test/k8s-integration/main.go b/test/k8s-integration/main.go index 75a0f6c97..1c3d88514 100644 --- a/test/k8s-integration/main.go +++ b/test/k8s-integration/main.go @@ -49,6 +49,7 @@ var ( gkeReleaseChannel = flag.String("gke-release-channel", "", "GKE release channel to be used for cluster deploy. One of 'rapid', 'stable' or 'regular'") gkeTestClusterName = flag.String("gke-cluster-name", "gcp-pd-csi-driver-test-cluster", "GKE cluster name") gkeNodeVersion = flag.String("gke-node-version", "", "GKE cluster worker node version") + isRegionalCluster = flag.Bool("is-regional-cluster", false, "tell the test that a regional cluster is being used. Should be used for running on an existing regional cluster (ie, --bringup-cluster=false). The test will fail if a zonal GKE cluster is created when this flag is true") // Test infrastructure flags boskosResourceType = flag.String("boskos-resource-type", "gce-project", "name of the boskos resource type to reserve") @@ -74,6 +75,7 @@ const ( k8sOutOfDockerBuildBinDir = "_output/bin" externalDriverNamespace = "gce-pd-csi-driver" managedDriverNamespace = "kube-system" + regionalPDStorageClass = "sc-regional.yaml" ) func init() { @@ -97,11 +99,11 @@ func main() { ensureVariable(deployOverlayName, false, "'deploy-overlay-name' must not be set when using GKE managed driver") } - if *deployOverlayName != "noauth" { - ensureVariable(saFile, true, "service-account-file is a required flag") - } if !*useGKEManagedDriver { ensureVariable(deployOverlayName, true, "deploy-overlay-name is a required flag") + if *deployOverlayName != "noauth" { + ensureVariable(saFile, true, "service-account-file is a required flag") + } } ensureVariable(testFocus, true, "test-focus is a required flag") @@ -122,6 +124,9 @@ func main() { ensureVariable(kubeFeatureGates, false, "kube-feature-gates set but not bringing up new cluster") } else { ensureVariable(imageType, true, "image type is a required flag. Available options include 'cos' and 'ubuntu'") + if *isRegionalCluster { + klog.Error("is-regional-cluster can only be set when using an existing cluster") + } } if *platform == "windows" { @@ -269,6 +274,14 @@ func handle() error { testDir = k8sDir } + if *deploymentStrat == "gke" { + gkeRegional := isRegionalGKECluster(*gceZone, *gceRegion) + if *isRegionalCluster && !gkeRegional { + return fmt.Errorf("--is-regional-cluster set but deployed GKE cluster would be zonal") + } + *isRegionalCluster = gkeRegional + } + // Create a cluster either through GKE or GCE if *bringupCluster { var err error = nil @@ -418,10 +431,24 @@ func handle() error { // Run the tests using the testDir kubernetes if len(*storageClassFiles) != 0 { - storageClasses := strings.Split(*storageClassFiles, ",") + applicableStorageClassFiles := []string{} + for _, rawScFile := range strings.Split(*storageClassFiles, ",") { + scFile := strings.TrimSpace(rawScFile) + if len(scFile) == 0 { + continue + } + if scFile == regionalPDStorageClass && !*isRegionalCluster { + klog.Warningf("Skipping regional StorageClass in zonal cluster") + continue + } + applicableStorageClassFiles = append(applicableStorageClassFiles, scFile) + } + if len(applicableStorageClassFiles) == 0 { + return fmt.Errorf("No applicable storage classes found") + } var ginkgoErrors []string var testOutputDirs []string - for _, scFile := range storageClasses { + for _, scFile := range applicableStorageClassFiles { outputDir := strings.TrimSuffix(scFile, ".yaml") testOutputDirs = append(testOutputDirs, outputDir) if err = runCSITests(*platform, pkgDir, testDir, *testFocus, testSkip, scFile, diff --git a/test/run-k8s-integration-ci.sh b/test/run-k8s-integration-ci.sh index cd245588f..1a1483b08 100755 --- a/test/run-k8s-integration-ci.sh +++ b/test/run-k8s-integration-ci.sh @@ -25,6 +25,13 @@ readonly gke_release_channel=${GKE_RELEASE_CHANNEL:-""} readonly teardown_driver=${GCE_PD_TEARDOWN_DRIVER:-true} readonly gke_node_version=${GKE_NODE_VERSION:-} readonly run_intree_plugin_tests=${RUN_INTREE_PLUGIN_TESTS:-false} + +storage_classes=sc-standard.yaml,sc-balanced.yaml,sc-ssd.yaml + +if [[ -n $gce_region ]] ; then + storage_classes="${storage_classes}",sc-regional +fi + export GCE_PD_VERBOSITY=9 make -C "${PKGDIR}" test-k8s-integration @@ -32,7 +39,7 @@ make -C "${PKGDIR}" test-k8s-integration base_cmd="${PKGDIR}/bin/k8s-integration-test \ --run-in-prow=true --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \ --do-driver-build=${do_driver_build} --teardown-driver=${teardown_driver} --boskos-resource-type=${boskos_resource_type} \ - --storageclass-files=sc-standard.yaml,sc-balanced.yaml,sc-ssd.yaml --snapshotclass-file=pd-volumesnapshotclass.yaml \ + --storageclass-files="${storage_classes}" --snapshotclass-file=pd-volumesnapshotclass.yaml \ --deployment-strategy=${deployment_strategy} --test-version=${test_version} \ --num-nodes=3 --image-type=${image_type}"