wip: run images and snapshots in integration tests

Author: luohao

test/k8s-integration/main.go

@@ -57,7 +57,7 @@ var (
 	// Test infrastructure flags
 	boskosResourceType = flag.String("boskos-resource-type", "gce-project", "name of the boskos resource type to reserve")
 	storageClassFiles  = flag.String("storageclass-files", "", "name of storageclass yaml file to use for test relative to test/k8s-integration/config. This may be a comma-separated list to test multiple storage classes")
-	snapshotClassFile  = flag.String("snapshotclass-file", "", "name of snapshotclass yaml file to use for test relative to test/k8s-integration/config")
+	snapshotClassFiles = flag.String("snapshotclass-files", "", "name of snapshotclass yaml file to use for test relative to test/k8s-integration/config. This may be a comma-separated list to test multiple storage classes")
 	inProw             = flag.Bool("run-in-prow", false, "is the test running in PROW")
 
 	// Driver flags
@@ -203,7 +203,6 @@ func handle() error {
 	testParams := &testParameters{
 		platform:            *platform,
 		testFocus:           *testFocus,
-		snapshotClassFile:   *snapshotClassFile,
 		stagingVersion:      string(uuid.NewUUID()),
 		deploymentStrategy:  *deploymentStrat,
 		useGKEManagedDriver: *useGKEManagedDriver,
@@ -481,6 +480,7 @@ func handle() error {
 	// Run the tests using the k8sSourceDir kubernetes
 	if len(*storageClassFiles) != 0 {
 		applicableStorageClassFiles := []string{}
+		applicableSnapshotClassFiles := []string{}
 		for _, rawScFile := range strings.Split(*storageClassFiles, ",") {
 			scFile := strings.TrimSpace(rawScFile)
 			if len(scFile) == 0 {
@@ -495,13 +495,28 @@ func handle() error {
 		if len(applicableStorageClassFiles) == 0 {
 			return fmt.Errorf("No applicable storage classes found")
 		}
+		for _, rawSnapshotClassFile := range strings.Split(*snapshotClassFiles, ",") {
+			snapshotClassFile := strings.TrimSpace(rawSnapshotClassFile)
+			if len(snapshotClassFile) != 0 {
+				applicableSnapshotClassFiles = append(applicableStorageClassFiles, snapshotClassFile)
+			}
+		}
+		if len(applicableSnapshotClassFiles) == 0 {
+			applicableSnapshotClassFiles = append(applicableStorageClassFiles, "")
+		}
 		var ginkgoErrors []string
 		var testOutputDirs []string
 		for _, scFile := range applicableStorageClassFiles {
 			outputDir := strings.TrimSuffix(scFile, ".yaml")
-			testOutputDirs = append(testOutputDirs, outputDir)
-			if err = runCSITests(testParams, scFile, outputDir); err != nil {
-				ginkgoErrors = append(ginkgoErrors, err.Error())
+			for _, snapshotClassFile := range applicableSnapshotClassFiles {
+				if len(snapshotClassFile) != 0 {
+					outputDir = fmt.Sprintf("%s--%s", outputDir, strings.TrimSuffix(snapshotClassFile, ".yaml"))
+				}
+				testOutputDirs = append(testOutputDirs, outputDir)
+				testParams.snapshotClassFile = snapshotClassFile
+				if err = runCSITests(testParams, scFile, outputDir); err != nil {
+					ginkgoErrors = append(ginkgoErrors, err.Error())
+				}
 			}
 		}
 		if err = mergeArtifacts(testOutputDirs); err != nil {

test/run-k8s-integration-ci.sh

@@ -102,9 +102,9 @@ if [ -n "$gke_node_version" ]; then
 fi
 
 if [ "$test_disk_image_snapshot" = true ]; then
-  base_cmd="${base_cmd} --snapshotclass-file=image-volumesnapshotclass.yaml"
+  base_cmd="${base_cmd} --snapshotclass-files=image-volumesnapshotclass.yaml,pd-volumesnapshotclass.yaml"
 else
-  base_cmd="${base_cmd} --snapshotclass-file=pd-volumesnapshotclass.yaml"
+  base_cmd="${base_cmd} --snapshotclass-files=pd-volumesnapshotclass.yaml"
 fi
 
 eval "$base_cmd"

test/run-k8s-integration-local.sh

@@ -50,7 +50,7 @@ make -C "${PKGDIR}" test-k8s-integration
 #${PKGDIR}/bin/k8s-integration-test --run-in-prow=false \
 #--staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \
 #--deploy-overlay-name=prow-canary-sidecar --bringup-cluster=false --teardown-cluster=false --test-focus="External.*Storage.*snapshot" --local-k8s-dir=$KTOP \
-#--storageclass-files=sc-standard.yaml,sc-balanced.yaml,sc-ssd.yaml --snapshotclass-file=pd-volumesnapshotclass.yaml --do-driver-build=true \
+#--storageclass-files=sc-standard.yaml,sc-balanced.yaml,sc-ssd.yaml --snapshotclass-files=pd-volumesnapshotclass.yaml --do-driver-build=true \
 #--gce-zone="us-central1-b" --num-nodes=${NUM_NODES:-3}
 
 # This version of the command builds and deploys the GCE PD CSI driver.
@@ -60,15 +60,15 @@ make -C "${PKGDIR}" test-k8s-integration
 #${PKGDIR}/bin/k8s-integration-test --run-in-prow=false \
 #--staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \
 #--deploy-overlay-name=prow-canary-sidecar --bringup-cluster=false --teardown-cluster=false --test-focus="External.*Storage.*snapshot" --local-k8s-dir=$KTOP \
-#--storageclass-files=sc-standard.yaml,sc-balanced.yaml,sc-ssd.yaml --snapshotclass-file=image-volumesnapshotclass.yaml --do-driver-build=true \
+#--storageclass-files=sc-standard.yaml,sc-balanced.yaml,sc-ssd.yaml --snapshotclass-files=image-volumesnapshotclass.yaml --do-driver-build=true \
 #--gce-zone="us-central1-b" --num-nodes=${NUM_NODES:-3}
 
 # This version of the command brings up (and subsequently tears down) a GKE
 # cluster with managed GCE PersistentDisk CSI driver add-on enabled, and points to
 # the local K8s repository to get the e2e test binary.
 # ${PKGDIR}/bin/k8s-integration-test --run-in-prow=false --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \
 # --test-focus="External.Storage" --local-k8s-dir=$KTOP --storageclass-files=sc-standard.yaml,sc-balanced.yaml,sc-ssd.yaml \
-# --snapshotclass-file=pd-volumesnapshotclass.yaml --do-driver-build=false --teardown-driver=false \
+# --snapshotclass-files=pd-volumesnapshotclass.yaml --do-driver-build=false --teardown-driver=false \
 # --gce-zone="us-central1-c" --num-nodes=${NUM_NODES:-3} --gke-cluster-version="latest" --deployment-strategy="gke" \
 # --use-gke-managed-driver=true --teardown-cluster=true
 
@@ -77,7 +77,7 @@ make -C "${PKGDIR}" test-k8s-integration
 # the local K8s repository to get the e2e test binary.
 # ${PKGDIR}/bin/k8s-integration-test --run-in-prow=false --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \
 # --test-focus="External.Storage" --local-k8s-dir=$KTOP --storageclass-files=sc-standard.yaml,sc-balanced.yaml,sc-ssd.yaml \
-# --snapshotclass-file=pd-volumesnapshotclass.yaml --do-driver-build=false --teardown-driver=false \
+# --snapshotclass-files=pd-volumesnapshotclass.yaml --do-driver-build=false --teardown-driver=false \
 # --gce-zone="us-central1-c" --num-nodes=${NUM_NODES:-3} --gke-release-channel="rapid" --deployment-strategy="gke" \
 # --use-gke-managed-driver=true --teardown-cluster=true
 

test/run-k8s-integration.sh

@@ -44,7 +44,7 @@ fi
 base_cmd="${PKGDIR}/bin/k8s-integration-test \
             --run-in-prow=true --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \
             --do-driver-build=${do_driver_build} --teardown-driver=${teardown_driver} --boskos-resource-type=${boskos_resource_type} \
-            --storageclass-files=sc-standard.yaml --snapshotclass-file=pd-volumesnapshotclass.yaml \
+            --storageclass-files=sc-standard.yaml --snapshotclass-files=pd-volumesnapshotclass.yaml \
             --deployment-strategy=${deployment_strategy} --test-version=${test_version} \
             --num-nodes=3 --image-type=${image_type} --use-kubetest2=${use_kubetest2}"
 

test/run-windows-k8s-integration.sh

@@ -51,6 +51,6 @@ ${PKGDIR}/bin/k8s-integration-test \
     --test-version="${test_version}" \
     --kube-version="${kube_version}" \
     --storageclass-files=sc-windows.yaml \
-    --snapshotclass-file=pd-volumesnapshotclass.yaml \
+    --snapshotclass-files=pd-volumesnapshotclass.yaml \
     --test-focus='External.Storage' \
     --use-kubetest2="${use_kubetest2}"

test/run-windows-k8s-migration.sh

@@ -60,6 +60,6 @@ ${PKGDIR}/bin/k8s-integration-test \
     --kube-version="${kube_version}" \
     --kube-feature-gates="${feature_gates}" \
     --storageclass-files=sc-windows.yaml \
-    --snapshotclass-file=pd-volumesnapshotclass.yaml \
+    --snapshotclass-files=pd-volumesnapshotclass.yaml \
     --test-focus="${GCE_PD_TEST_FOCUS}" \
     --use-kubetest2="${use_kubetest2}"