Merge pull request #929 from luohao/hluo/image-integration-tests

Add e2e/integration tests for image snapshot

Author: k8s-ci-robot

pkg/gce-cloud-provider/compute/cloud-disk.go

@@ -188,6 +188,17 @@ func (d *CloudDisk) GetSourceDiskId() string {
 	}
 }
 
+func (d *CloudDisk) GetImageId() string {
+	switch {
+	case d.disk != nil:
+		return d.disk.SourceImageId
+	case d.betaDisk != nil:
+		return d.betaDisk.SourceImageId
+	default:
+		return ""
+	}
+}
+
 func (d *CloudDisk) GetKMSKeyName() string {
 	switch {
 	case d.disk != nil:

pkg/gce-cloud-provider/compute/fake-gce.go

@@ -196,16 +196,30 @@ func (cloud *FakeCloudProvider) InsertDisk(ctx context.Context, project string,
 	}
 
 	computeDisk := &computev1.Disk{
-		Name:             volKey.Name,
-		SizeGb:           common.BytesToGbRoundUp(capBytes),
-		Description:      "Disk created by GCE-PD CSI Driver",
-		Type:             cloud.GetDiskTypeURI(project, volKey, params.DiskType),
-		SourceSnapshotId: snapshotID,
-		SourceDiskId:     volumeContentSourceVolumeID,
-		SourceImageId:    snapshotID,
-		Status:           cloud.mockDiskStatus,
-		Labels:           params.Labels,
+		Name:         volKey.Name,
+		SizeGb:       common.BytesToGbRoundUp(capBytes),
+		Description:  "Disk created by GCE-PD CSI Driver",
+		Type:         cloud.GetDiskTypeURI(project, volKey, params.DiskType),
+		SourceDiskId: volumeContentSourceVolumeID,
+		Status:       cloud.mockDiskStatus,
+		Labels:       params.Labels,
 	}
+
+	if snapshotID != "" {
+		_, snapshotType, _, err := common.SnapshotIDToProjectKey(snapshotID)
+		if err != nil {
+			return err
+		}
+		switch snapshotType {
+		case common.DiskSnapshotType:
+			computeDisk.SourceSnapshotId = snapshotID
+		case common.DiskImageType:
+			computeDisk.SourceImageId = snapshotID
+		default:
+			return fmt.Errorf("invalid snapshot type in snapshot ID: %s", snapshotType)
+		}
+	}
+
 	if params.DiskEncryptionKMSKey != "" {
 		computeDisk.DiskEncryptionKey = &computev1.CustomerEncryptionKey{
 			KmsKeyName: params.DiskEncryptionKMSKey,

pkg/gce-cloud-provider/compute/gce-compute.go

@@ -894,7 +894,7 @@ func (cloud *CloudProvider) CreateImage(ctx context.Context, project string, vol
 		Description:      description,
 	}
 
-	_, err = cloud.service.Images.Insert(project, image).Context(ctx).Do()
+	_, err = cloud.service.Images.Insert(project, image).Context(ctx).ForceCreate(true).Do()
 	if err != nil {
 		return nil, err
 	}

pkg/gce-pd-csi-driver/controller.go

@@ -1480,8 +1480,9 @@ func generateCreateVolumeResponse(disk *gce.CloudDisk, zones []string) *csi.Crea
 		},
 	}
 	snapshotID := disk.GetSnapshotId()
+	imageID := disk.GetImageId()
 	diskID := disk.GetSourceDiskId()
-	if diskID != "" || snapshotID != "" {
+	if diskID != "" || snapshotID != "" || imageID != "" {
 		contentSource := &csi.VolumeContentSource{}
 		if snapshotID != "" {
 			contentSource = &csi.VolumeContentSource{
@@ -1501,6 +1502,15 @@ func generateCreateVolumeResponse(disk *gce.CloudDisk, zones []string) *csi.Crea
 				},
 			}
 		}
+		if imageID != "" {
+			contentSource = &csi.VolumeContentSource{
+				Type: &csi.VolumeContentSource_Snapshot{
+					Snapshot: &csi.VolumeContentSource_SnapshotSource{
+						SnapshotId: imageID,
+					},
+				},
+			}
+		}
 		createResp.Volume.ContentSource = contentSource
 	}
 	return createResp

test/e2e/tests/single_zone_e2e_test.go

@@ -952,6 +952,65 @@ var _ = Describe("GCE PD CSI Driver", func() {
 			Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected snapshot to not be found")
 		}()
 	})
+
+	// Use the region of the test location.
+	It("Should successfully create snapshot backed by disk image", func() {
+		testContext := getRandomTestContext()
+
+		p, z, _ := testContext.Instance.GetIdentity()
+		client := testContext.Client
+
+		// Create Disk
+		volName, volID := createAndValidateUniqueZonalDisk(client, p, z)
+
+		// Create Snapshot
+		snapshotName := testNamePrefix + string(uuid.NewUUID())
+		testImageFamily := "test-family"
+
+		snapshotParams := map[string]string{common.ParameterKeySnapshotType: common.DiskImageType, common.ParameterKeyImageFamily: testImageFamily}
+		snapshotID, err := client.CreateSnapshot(snapshotName, volID, snapshotParams)
+		Expect(err).To(BeNil(), "CreateSnapshot failed with error: %v", err)
+
+		// Validate Snapshot Created
+		snapshot, err := computeService.Images.Get(p, snapshotName).Do()
+		Expect(err).To(BeNil(), "Could not get snapshot from cloud directly")
+		Expect(snapshot.Name).To(Equal(snapshotName))
+
+		err = wait.Poll(10*time.Second, 5*time.Minute, func() (bool, error) {
+			snapshot, err := computeService.Images.Get(p, snapshotName).Do()
+			Expect(err).To(BeNil(), "Could not get snapshot from cloud directly")
+			if snapshot.Status == "READY" {
+				return true, nil
+			}
+			return false, nil
+		})
+		Expect(err).To(BeNil(), "Could not wait for snapshot be ready")
+
+		// Check Snapshot Type
+		snapshot, err = computeService.Images.Get(p, snapshotName).Do()
+		Expect(err).To(BeNil(), "Could not get snapshot from cloud directly")
+		_, snapshotType, _, err := common.SnapshotIDToProjectKey(cleanSelfLink(snapshot.SelfLink))
+		Expect(err).To(BeNil(), "Failed to parse snapshot ID")
+		Expect(snapshotType).To(Equal(common.DiskImageType), "Expected images type in snapshot ID")
+
+		defer func() {
+			// Delete Disk
+			err := client.DeleteVolume(volID)
+			Expect(err).To(BeNil(), "DeleteVolume failed")
+
+			// Validate Disk Deleted
+			_, err = computeService.Disks.Get(p, z, volName).Do()
+			Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found")
+
+			// Delete Snapshot
+			err = client.DeleteSnapshot(snapshotID)
+			Expect(err).To(BeNil(), "DeleteSnapshot failed")
+
+			// Validate Snapshot Deleted
+			_, err = computeService.Images.Get(p, snapshotName).Do()
+			Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected snapshot to not be found")
+		}()
+	})
 })
 
 func equalWithinEpsilon(a, b, epsiolon int64) bool {
@@ -1031,3 +1090,9 @@ func createAndValidateUniqueZonalMultiWriterDisk(client *remote.CsiClient, proje
 
 	return volName, volID
 }
+
+func cleanSelfLink(selfLink string) string {
+	temp := strings.TrimPrefix(selfLink, gce.GCEComputeAPIEndpoint)
+	temp = strings.TrimPrefix(temp, gce.GCEComputeBetaAPIEndpoint)
+	return strings.TrimPrefix(temp, gce.GCEComputeAlphaAPIEndpoint)
+}

test/k8s-integration/config/image-volumesnapshotclass.yaml

@@ -0,0 +1,9 @@
+apiVersion: snapshot.storage.k8s.io/v1beta1
+kind: VolumeSnapshotClass
+metadata:
+  name: csi-gce-image-snapshot-class
+driver: pd.csi.storage.gke.io
+deletionPolicy: Delete
+parameters:
+  snapshot-type: images
+  image-family: integration-test

test/k8s-integration/config/test-config-template.in

@@ -10,7 +10,7 @@ Timeouts:
   {{ end }}
 {{end}}
 DriverInfo:
-  Name: csi-gcepd-{{.StorageClass}}
+  Name: csi-gcepd-{{.StorageClass}}--{{.SnapshotClass}}
   SupportedFsType:
   {{range .SupportedFsType}}  {{ . }}:
   {{end}}
@@ -28,4 +28,4 @@ DriverInfo:
     Max: 64Ti
   TopologyKeys:
     - topology.gke.io/zone
-  NumAllowedTopologies: {{.NumAllowedTopologies}}
+  NumAllowedTopologies: {{.NumAllowedTopologies}}

test/k8s-integration/driver-config.go

@@ -13,6 +13,7 @@ type driverConfig struct {
 	StorageClassFile     string
 	StorageClass         string
 	SnapshotClassFile    string
+	SnapshotClass        string
 	Capabilities         []string
 	SupportedFsType      []string
 	MinimumVolumeSize    string
@@ -115,11 +116,15 @@ func generateDriverConfigFile(testParams *testParameters, storageClassFile strin
 	}
 
 	var absSnapshotClassFilePath string
+	var snapshotClassName string
 	// If snapshot class is passed in as argument, include snapshot specific driver capabiltiites.
 	if testParams.snapshotClassFile != "" {
 		caps = append(caps, "snapshotDataSource")
 		// Update the absolute file path pointing to the snapshot class file, if it is provided as an argument.
 		absSnapshotClassFilePath = filepath.Join(testParams.pkgDir, testConfigDir, testParams.snapshotClassFile)
+		snapshotClassName = testParams.snapshotClassFile[:strings.LastIndex(testParams.snapshotClassFile, ".")]
+	} else {
+		snapshotClassName = "no-volumesnapshotclass"
 	}
 
 	caps = append(caps, "pvcDataSource")
@@ -136,6 +141,7 @@ func generateDriverConfigFile(testParams *testParameters, storageClassFile strin
 		StorageClassFile:     filepath.Join(testParams.pkgDir, testConfigDir, storageClassFile),
 		StorageClass:         storageClassFile[:strings.LastIndex(storageClassFile, ".")],
 		SnapshotClassFile:    absSnapshotClassFilePath,
+		SnapshotClass:        snapshotClassName,
 		SupportedFsType:      fsTypes,
 		Capabilities:         caps,
 		MinimumVolumeSize:    minimumVolumeSize,

test/k8s-integration/main.go

@@ -57,7 +57,7 @@ var (
 	// Test infrastructure flags
 	boskosResourceType = flag.String("boskos-resource-type", "gce-project", "name of the boskos resource type to reserve")
 	storageClassFiles  = flag.String("storageclass-files", "", "name of storageclass yaml file to use for test relative to test/k8s-integration/config. This may be a comma-separated list to test multiple storage classes")
-	snapshotClassFile  = flag.String("snapshotclass-file", "", "name of snapshotclass yaml file to use for test relative to test/k8s-integration/config")
+	snapshotClassFiles = flag.String("snapshotclass-files", "", "name of snapshotclass yaml file to use for test relative to test/k8s-integration/config. This may be a comma-separated list to test multiple storage classes")
 	inProw             = flag.Bool("run-in-prow", false, "is the test running in PROW")
 
 	// Driver flags
@@ -203,7 +203,6 @@ func handle() error {
 	testParams := &testParameters{
 		platform:            *platform,
 		testFocus:           *testFocus,
-		snapshotClassFile:   *snapshotClassFile,
 		stagingVersion:      string(uuid.NewUUID()),
 		deploymentStrategy:  *deploymentStrat,
 		useGKEManagedDriver: *useGKEManagedDriver,
@@ -481,6 +480,7 @@ func handle() error {
 	// Run the tests using the k8sSourceDir kubernetes
 	if len(*storageClassFiles) != 0 {
 		applicableStorageClassFiles := []string{}
+		applicableSnapshotClassFiles := []string{}
 		for _, rawScFile := range strings.Split(*storageClassFiles, ",") {
 			scFile := strings.TrimSpace(rawScFile)
 			if len(scFile) == 0 {
@@ -495,13 +495,29 @@ func handle() error {
 		if len(applicableStorageClassFiles) == 0 {
 			return fmt.Errorf("No applicable storage classes found")
 		}
+		for _, rawSnapshotClassFile := range strings.Split(*snapshotClassFiles, ",") {
+			snapshotClassFile := strings.TrimSpace(rawSnapshotClassFile)
+			if len(snapshotClassFile) != 0 {
+				applicableSnapshotClassFiles = append(applicableSnapshotClassFiles, snapshotClassFile)
+			}
+		}
+		if len(applicableSnapshotClassFiles) == 0 {
+			// when no snapshot class specified, we run the tests without snapshot capability
+			applicableSnapshotClassFiles = append(applicableSnapshotClassFiles, "")
+		}
 		var ginkgoErrors []string
 		var testOutputDirs []string
 		for _, scFile := range applicableStorageClassFiles {
 			outputDir := strings.TrimSuffix(scFile, ".yaml")
-			testOutputDirs = append(testOutputDirs, outputDir)
-			if err = runCSITests(testParams, scFile, outputDir); err != nil {
-				ginkgoErrors = append(ginkgoErrors, err.Error())
+			for _, snapshotClassFile := range applicableSnapshotClassFiles {
+				if len(snapshotClassFile) != 0 {
+					outputDir = fmt.Sprintf("%s--%s", outputDir, strings.TrimSuffix(snapshotClassFile, ".yaml"))
+				}
+				testOutputDirs = append(testOutputDirs, outputDir)
+				testParams.snapshotClassFile = snapshotClassFile
+				if err = runCSITests(testParams, scFile, outputDir); err != nil {
+					ginkgoErrors = append(ginkgoErrors, err.Error())
+				}
 			}
 		}
 		if err = mergeArtifacts(testOutputDirs); err != nil {

test/run-k8s-integration-ci.sh

@@ -29,6 +29,7 @@ readonly run_intree_plugin_tests=${RUN_INTREE_PLUGIN_TESTS:-false}
 readonly use_kubetest2=${USE_KUBETEST2:-true}
 readonly test_pd_labels=${TEST_PD_LABELS:-true}
 readonly migration_test=${MIGRATION_TEST:-false}
+readonly test_disk_image_snapshot=${TEST_DISK_IMAGE_SNAPSHOT:-true}
 
 readonly GCE_PD_TEST_FOCUS="PersistentVolumes\sGCEPD|[V|v]olume\sexpand|\[sig-storage\]\sIn-tree\sVolumes\s\[Driver:\sgcepd\]|allowedTopologies|Pod\sDisks|PersistentVolumes\sDefault"
 
@@ -62,7 +63,7 @@ fi
 base_cmd="${PKGDIR}/bin/k8s-integration-test \
             --run-in-prow=true --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \
             --do-driver-build=${do_driver_build} --teardown-driver=${teardown_driver} --boskos-resource-type=${boskos_resource_type} \
-            --storageclass-files="${storage_classes}" --snapshotclass-file=pd-volumesnapshotclass.yaml \
+            --storageclass-files="${storage_classes}" \
             --deployment-strategy=${deployment_strategy} --test-version=${test_version} \
             --num-nodes=3 --image-type=${image_type} --use-kubetest2=${use_kubetest2}"
 
@@ -100,4 +101,10 @@ if [ -n "$gke_node_version" ]; then
   base_cmd="${base_cmd} --gke-node-version=${gke_node_version}"
 fi
 
+if [ "$test_disk_image_snapshot" = true ]; then
+  base_cmd="${base_cmd} --snapshotclass-files=image-volumesnapshotclass.yaml,pd-volumesnapshotclass.yaml"
+else
+  base_cmd="${base_cmd} --snapshotclass-files=pd-volumesnapshotclass.yaml"
+fi
+
 eval "$base_cmd"

test/run-k8s-integration-local.sh

@@ -50,15 +50,25 @@ make -C "${PKGDIR}" test-k8s-integration
 #${PKGDIR}/bin/k8s-integration-test --run-in-prow=false \
 #--staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \
 #--deploy-overlay-name=prow-canary-sidecar --bringup-cluster=false --teardown-cluster=false --test-focus="External.*Storage.*snapshot" --local-k8s-dir=$KTOP \
-#--storageclass-files=sc-standard.yaml,sc-balanced.yaml,sc-ssd.yaml --snapshotclass-file=pd-volumesnapshotclass.yaml --do-driver-build=true \
+#--storageclass-files=sc-standard.yaml,sc-balanced.yaml,sc-ssd.yaml --snapshotclass-files=pd-volumesnapshotclass.yaml --do-driver-build=true \
+#--gce-zone="us-central1-b" --num-nodes=${NUM_NODES:-3}
+
+# This version of the command builds and deploys the GCE PD CSI driver.
+# Points to a local K8s repository to get the e2e test binary, does not bring up
+# or tear down the kubernetes cluster. In addition, it runs External Storage
+# snapshot tests for the PD CSI driver using disk image snapshots.
+#${PKGDIR}/bin/k8s-integration-test --run-in-prow=false \
+#--staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \
+#--deploy-overlay-name=prow-canary-sidecar --bringup-cluster=false --teardown-cluster=false --test-focus="External.*Storage.*snapshot" --local-k8s-dir=$KTOP \
+#--storageclass-files=sc-standard.yaml,sc-balanced.yaml,sc-ssd.yaml --snapshotclass-files=image-volumesnapshotclass.yaml --do-driver-build=true \
 #--gce-zone="us-central1-b" --num-nodes=${NUM_NODES:-3}
 
 # This version of the command brings up (and subsequently tears down) a GKE
 # cluster with managed GCE PersistentDisk CSI driver add-on enabled, and points to
 # the local K8s repository to get the e2e test binary.
 # ${PKGDIR}/bin/k8s-integration-test --run-in-prow=false --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \
 # --test-focus="External.Storage" --local-k8s-dir=$KTOP --storageclass-files=sc-standard.yaml,sc-balanced.yaml,sc-ssd.yaml \
-# --snapshotclass-file=pd-volumesnapshotclass.yaml --do-driver-build=false --teardown-driver=false \
+# --snapshotclass-files=pd-volumesnapshotclass.yaml --do-driver-build=false --teardown-driver=false \
 # --gce-zone="us-central1-c" --num-nodes=${NUM_NODES:-3} --gke-cluster-version="latest" --deployment-strategy="gke" \
 # --use-gke-managed-driver=true --teardown-cluster=true
 
@@ -67,7 +77,7 @@ make -C "${PKGDIR}" test-k8s-integration
 # the local K8s repository to get the e2e test binary.
 # ${PKGDIR}/bin/k8s-integration-test --run-in-prow=false --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \
 # --test-focus="External.Storage" --local-k8s-dir=$KTOP --storageclass-files=sc-standard.yaml,sc-balanced.yaml,sc-ssd.yaml \
-# --snapshotclass-file=pd-volumesnapshotclass.yaml --do-driver-build=false --teardown-driver=false \
+# --snapshotclass-files=pd-volumesnapshotclass.yaml --do-driver-build=false --teardown-driver=false \
 # --gce-zone="us-central1-c" --num-nodes=${NUM_NODES:-3} --gke-release-channel="rapid" --deployment-strategy="gke" \
 # --use-gke-managed-driver=true --teardown-cluster=true
 
@@ -96,4 +106,4 @@ make -C "${PKGDIR}" test-k8s-integration
 # --staging-image="${GCE_PD_CSI_STAGING_IMAGE}" --service-account-file="${GCE_PD_SA_DIR}/cloud-sa.json" \
 # --deploy-overlay-name=dev --bringup-cluster=false --teardown-cluster=false --local-k8s-dir="$KTOP" \
 # --storageclass-files=sc-standard.yaml --do-driver-build=false --test-focus='External.Storage' \
-# --gce-zone="us-central1-b" --num-nodes="${NUM_NODES:-3}"
+# --gce-zone="us-central1-b" --num-nodes="${NUM_NODES:-3}"

test/run-k8s-integration.sh

@@ -44,7 +44,7 @@ fi
 base_cmd="${PKGDIR}/bin/k8s-integration-test \
             --run-in-prow=true --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \
             --do-driver-build=${do_driver_build} --teardown-driver=${teardown_driver} --boskos-resource-type=${boskos_resource_type} \
-            --storageclass-files=sc-standard.yaml --snapshotclass-file=pd-volumesnapshotclass.yaml \
+            --storageclass-files=sc-standard.yaml --snapshotclass-files=pd-volumesnapshotclass.yaml \
             --deployment-strategy=${deployment_strategy} --test-version=${test_version} \
             --num-nodes=3 --image-type=${image_type} --use-kubetest2=${use_kubetest2}"
 

test/run-windows-k8s-integration.sh

@@ -51,6 +51,6 @@ ${PKGDIR}/bin/k8s-integration-test \
     --test-version="${test_version}" \
     --kube-version="${kube_version}" \
     --storageclass-files=sc-windows.yaml \
-    --snapshotclass-file=pd-volumesnapshotclass.yaml \
+    --snapshotclass-files=pd-volumesnapshotclass.yaml \
     --test-focus='External.Storage' \
     --use-kubetest2="${use_kubetest2}"

test/run-windows-k8s-migration.sh

@@ -60,6 +60,6 @@ ${PKGDIR}/bin/k8s-integration-test \
     --kube-version="${kube_version}" \
     --kube-feature-gates="${feature_gates}" \
     --storageclass-files=sc-windows.yaml \
-    --snapshotclass-file=pd-volumesnapshotclass.yaml \
+    --snapshotclass-files=pd-volumesnapshotclass.yaml \
     --test-focus="${GCE_PD_TEST_FOCUS}" \
     --use-kubetest2="${use_kubetest2}"