diff --git a/pkg/gce-cloud-provider/compute/cloud-disk.go b/pkg/gce-cloud-provider/compute/cloud-disk.go index e6a4cd6b7..99968d00a 100644 --- a/pkg/gce-cloud-provider/compute/cloud-disk.go +++ b/pkg/gce-cloud-provider/compute/cloud-disk.go @@ -188,6 +188,17 @@ func (d *CloudDisk) GetSourceDiskId() string { } } +func (d *CloudDisk) GetImageId() string { + switch { + case d.disk != nil: + return d.disk.SourceImageId + case d.betaDisk != nil: + return d.betaDisk.SourceImageId + default: + return "" + } +} + func (d *CloudDisk) GetKMSKeyName() string { switch { case d.disk != nil: diff --git a/pkg/gce-cloud-provider/compute/fake-gce.go b/pkg/gce-cloud-provider/compute/fake-gce.go index dd9953167..962079f4a 100644 --- a/pkg/gce-cloud-provider/compute/fake-gce.go +++ b/pkg/gce-cloud-provider/compute/fake-gce.go @@ -196,16 +196,30 @@ func (cloud *FakeCloudProvider) InsertDisk(ctx context.Context, project string, } computeDisk := &computev1.Disk{ - Name: volKey.Name, - SizeGb: common.BytesToGbRoundUp(capBytes), - Description: "Disk created by GCE-PD CSI Driver", - Type: cloud.GetDiskTypeURI(project, volKey, params.DiskType), - SourceSnapshotId: snapshotID, - SourceDiskId: volumeContentSourceVolumeID, - SourceImageId: snapshotID, - Status: cloud.mockDiskStatus, - Labels: params.Labels, + Name: volKey.Name, + SizeGb: common.BytesToGbRoundUp(capBytes), + Description: "Disk created by GCE-PD CSI Driver", + Type: cloud.GetDiskTypeURI(project, volKey, params.DiskType), + SourceDiskId: volumeContentSourceVolumeID, + Status: cloud.mockDiskStatus, + Labels: params.Labels, } + + if snapshotID != "" { + _, snapshotType, _, err := common.SnapshotIDToProjectKey(snapshotID) + if err != nil { + return err + } + switch snapshotType { + case common.DiskSnapshotType: + computeDisk.SourceSnapshotId = snapshotID + case common.DiskImageType: + computeDisk.SourceImageId = snapshotID + default: + return fmt.Errorf("invalid snapshot type in snapshot ID: %s", snapshotType) + } + } + if params.DiskEncryptionKMSKey != "" { computeDisk.DiskEncryptionKey = &computev1.CustomerEncryptionKey{ KmsKeyName: params.DiskEncryptionKMSKey, diff --git a/pkg/gce-cloud-provider/compute/gce-compute.go b/pkg/gce-cloud-provider/compute/gce-compute.go index 8d1818c50..7b0b8b211 100644 --- a/pkg/gce-cloud-provider/compute/gce-compute.go +++ b/pkg/gce-cloud-provider/compute/gce-compute.go @@ -871,7 +871,7 @@ func (cloud *CloudProvider) CreateImage(ctx context.Context, project string, vol StorageLocations: snapshotParams.StorageLocations, } - _, err = cloud.service.Images.Insert(project, image).Context(ctx).Do() + _, err = cloud.service.Images.Insert(project, image).Context(ctx).ForceCreate(true).Do() if err != nil { return nil, err } diff --git a/pkg/gce-pd-csi-driver/controller.go b/pkg/gce-pd-csi-driver/controller.go index 0d44458be..0dbd5269b 100644 --- a/pkg/gce-pd-csi-driver/controller.go +++ b/pkg/gce-pd-csi-driver/controller.go @@ -1480,8 +1480,9 @@ func generateCreateVolumeResponse(disk *gce.CloudDisk, zones []string) *csi.Crea }, } snapshotID := disk.GetSnapshotId() + imageID := disk.GetImageId() diskID := disk.GetSourceDiskId() - if diskID != "" || snapshotID != "" { + if diskID != "" || snapshotID != "" || imageID != "" { contentSource := &csi.VolumeContentSource{} if snapshotID != "" { contentSource = &csi.VolumeContentSource{ @@ -1501,6 +1502,15 @@ func generateCreateVolumeResponse(disk *gce.CloudDisk, zones []string) *csi.Crea }, } } + if imageID != "" { + contentSource = &csi.VolumeContentSource{ + Type: &csi.VolumeContentSource_Snapshot{ + Snapshot: &csi.VolumeContentSource_SnapshotSource{ + SnapshotId: imageID, + }, + }, + } + } createResp.Volume.ContentSource = contentSource } return createResp diff --git a/test/e2e/tests/single_zone_e2e_test.go b/test/e2e/tests/single_zone_e2e_test.go index 89c560981..959b5ac8a 100644 --- a/test/e2e/tests/single_zone_e2e_test.go +++ b/test/e2e/tests/single_zone_e2e_test.go @@ -946,6 +946,65 @@ var _ = Describe("GCE PD CSI Driver", func() { Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected snapshot to not be found") }() }) + + // Use the region of the test location. + It("Should successfully create snapshot backed by disk image", func() { + testContext := getRandomTestContext() + + p, z, _ := testContext.Instance.GetIdentity() + client := testContext.Client + + // Create Disk + volName, volID := createAndValidateUniqueZonalDisk(client, p, z) + + // Create Snapshot + snapshotName := testNamePrefix + string(uuid.NewUUID()) + testImageFamily := "test-family" + + snapshotParams := map[string]string{common.ParameterKeySnapshotType: common.DiskImageType, common.ParameterKeyImageFamily: testImageFamily} + snapshotID, err := client.CreateSnapshot(snapshotName, volID, snapshotParams) + Expect(err).To(BeNil(), "CreateSnapshot failed with error: %v", err) + + // Validate Snapshot Created + snapshot, err := computeService.Images.Get(p, snapshotName).Do() + Expect(err).To(BeNil(), "Could not get snapshot from cloud directly") + Expect(snapshot.Name).To(Equal(snapshotName)) + + err = wait.Poll(10*time.Second, 5*time.Minute, func() (bool, error) { + snapshot, err := computeService.Images.Get(p, snapshotName).Do() + Expect(err).To(BeNil(), "Could not get snapshot from cloud directly") + if snapshot.Status == "READY" { + return true, nil + } + return false, nil + }) + Expect(err).To(BeNil(), "Could not wait for snapshot be ready") + + // Check Snapshot Type + snapshot, err = computeService.Images.Get(p, snapshotName).Do() + Expect(err).To(BeNil(), "Could not get snapshot from cloud directly") + _, snapshotType, _, err := common.SnapshotIDToProjectKey(cleanSelfLink(snapshot.SelfLink)) + Expect(err).To(BeNil(), "Failed to parse snapshot ID") + Expect(snapshotType).To(Equal(common.DiskImageType), "Expected images type in snapshot ID") + + defer func() { + // Delete Disk + err := client.DeleteVolume(volID) + Expect(err).To(BeNil(), "DeleteVolume failed") + + // Validate Disk Deleted + _, err = computeService.Disks.Get(p, z, volName).Do() + Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") + + // Delete Snapshot + err = client.DeleteSnapshot(snapshotID) + Expect(err).To(BeNil(), "DeleteSnapshot failed") + + // Validate Snapshot Deleted + _, err = computeService.Images.Get(p, snapshotName).Do() + Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected snapshot to not be found") + }() + }) }) func equalWithinEpsilon(a, b, epsiolon int64) bool { @@ -1025,3 +1084,9 @@ func createAndValidateUniqueZonalMultiWriterDisk(client *remote.CsiClient, proje return volName, volID } + +func cleanSelfLink(selfLink string) string { + temp := strings.TrimPrefix(selfLink, gce.GCEComputeAPIEndpoint) + temp = strings.TrimPrefix(temp, gce.GCEComputeBetaAPIEndpoint) + return strings.TrimPrefix(temp, gce.GCEComputeAlphaAPIEndpoint) +} diff --git a/test/k8s-integration/config/image-volumesnapshotclass.yaml b/test/k8s-integration/config/image-volumesnapshotclass.yaml new file mode 100644 index 000000000..ee62f2150 --- /dev/null +++ b/test/k8s-integration/config/image-volumesnapshotclass.yaml @@ -0,0 +1,9 @@ +apiVersion: snapshot.storage.k8s.io/v1beta1 +kind: VolumeSnapshotClass +metadata: + name: csi-gce-image-snapshot-class +driver: pd.csi.storage.gke.io +deletionPolicy: Delete +parameters: + snapshot-type: images + image-family: integration-test diff --git a/test/k8s-integration/config/test-config-template.in b/test/k8s-integration/config/test-config-template.in index b88ea90a0..6571ae9e5 100644 --- a/test/k8s-integration/config/test-config-template.in +++ b/test/k8s-integration/config/test-config-template.in @@ -10,7 +10,7 @@ Timeouts: {{ end }} {{end}} DriverInfo: - Name: csi-gcepd-{{.StorageClass}} + Name: csi-gcepd-{{.StorageClass}}--{{.SnapshotClass}} SupportedFsType: {{range .SupportedFsType}} {{ . }}: {{end}} @@ -28,4 +28,4 @@ DriverInfo: Max: 64Ti TopologyKeys: - topology.gke.io/zone - NumAllowedTopologies: {{.NumAllowedTopologies}} \ No newline at end of file + NumAllowedTopologies: {{.NumAllowedTopologies}} diff --git a/test/k8s-integration/driver-config.go b/test/k8s-integration/driver-config.go index 958a3417a..e2e83f25a 100644 --- a/test/k8s-integration/driver-config.go +++ b/test/k8s-integration/driver-config.go @@ -13,6 +13,7 @@ type driverConfig struct { StorageClassFile string StorageClass string SnapshotClassFile string + SnapshotClass string Capabilities []string SupportedFsType []string MinimumVolumeSize string @@ -115,11 +116,15 @@ func generateDriverConfigFile(testParams *testParameters, storageClassFile strin } var absSnapshotClassFilePath string + var snapshotClassName string // If snapshot class is passed in as argument, include snapshot specific driver capabiltiites. if testParams.snapshotClassFile != "" { caps = append(caps, "snapshotDataSource") // Update the absolute file path pointing to the snapshot class file, if it is provided as an argument. absSnapshotClassFilePath = filepath.Join(testParams.pkgDir, testConfigDir, testParams.snapshotClassFile) + snapshotClassName = testParams.snapshotClassFile[:strings.LastIndex(testParams.snapshotClassFile, ".")] + } else { + snapshotClassName = "no-volumesnapshotclass" } caps = append(caps, "pvcDataSource") @@ -136,6 +141,7 @@ func generateDriverConfigFile(testParams *testParameters, storageClassFile strin StorageClassFile: filepath.Join(testParams.pkgDir, testConfigDir, storageClassFile), StorageClass: storageClassFile[:strings.LastIndex(storageClassFile, ".")], SnapshotClassFile: absSnapshotClassFilePath, + SnapshotClass: snapshotClassName, SupportedFsType: fsTypes, Capabilities: caps, MinimumVolumeSize: minimumVolumeSize, diff --git a/test/k8s-integration/main.go b/test/k8s-integration/main.go index 7a83c5858..701815675 100644 --- a/test/k8s-integration/main.go +++ b/test/k8s-integration/main.go @@ -57,7 +57,7 @@ var ( // Test infrastructure flags boskosResourceType = flag.String("boskos-resource-type", "gce-project", "name of the boskos resource type to reserve") storageClassFiles = flag.String("storageclass-files", "", "name of storageclass yaml file to use for test relative to test/k8s-integration/config. This may be a comma-separated list to test multiple storage classes") - snapshotClassFile = flag.String("snapshotclass-file", "", "name of snapshotclass yaml file to use for test relative to test/k8s-integration/config") + snapshotClassFiles = flag.String("snapshotclass-files", "", "name of snapshotclass yaml file to use for test relative to test/k8s-integration/config. This may be a comma-separated list to test multiple storage classes") inProw = flag.Bool("run-in-prow", false, "is the test running in PROW") // Driver flags @@ -203,7 +203,6 @@ func handle() error { testParams := &testParameters{ platform: *platform, testFocus: *testFocus, - snapshotClassFile: *snapshotClassFile, stagingVersion: string(uuid.NewUUID()), deploymentStrategy: *deploymentStrat, useGKEManagedDriver: *useGKEManagedDriver, @@ -481,6 +480,7 @@ func handle() error { // Run the tests using the k8sSourceDir kubernetes if len(*storageClassFiles) != 0 { applicableStorageClassFiles := []string{} + applicableSnapshotClassFiles := []string{} for _, rawScFile := range strings.Split(*storageClassFiles, ",") { scFile := strings.TrimSpace(rawScFile) if len(scFile) == 0 { @@ -495,13 +495,29 @@ func handle() error { if len(applicableStorageClassFiles) == 0 { return fmt.Errorf("No applicable storage classes found") } + for _, rawSnapshotClassFile := range strings.Split(*snapshotClassFiles, ",") { + snapshotClassFile := strings.TrimSpace(rawSnapshotClassFile) + if len(snapshotClassFile) != 0 { + applicableSnapshotClassFiles = append(applicableSnapshotClassFiles, snapshotClassFile) + } + } + if len(applicableSnapshotClassFiles) == 0 { + // when no snapshot class specified, we run the tests without snapshot capability + applicableSnapshotClassFiles = append(applicableSnapshotClassFiles, "") + } var ginkgoErrors []string var testOutputDirs []string for _, scFile := range applicableStorageClassFiles { outputDir := strings.TrimSuffix(scFile, ".yaml") - testOutputDirs = append(testOutputDirs, outputDir) - if err = runCSITests(testParams, scFile, outputDir); err != nil { - ginkgoErrors = append(ginkgoErrors, err.Error()) + for _, snapshotClassFile := range applicableSnapshotClassFiles { + if len(snapshotClassFile) != 0 { + outputDir = fmt.Sprintf("%s--%s", outputDir, strings.TrimSuffix(snapshotClassFile, ".yaml")) + } + testOutputDirs = append(testOutputDirs, outputDir) + testParams.snapshotClassFile = snapshotClassFile + if err = runCSITests(testParams, scFile, outputDir); err != nil { + ginkgoErrors = append(ginkgoErrors, err.Error()) + } } } if err = mergeArtifacts(testOutputDirs); err != nil { diff --git a/test/run-k8s-integration-ci.sh b/test/run-k8s-integration-ci.sh index e2fbeffbd..b1d578029 100755 --- a/test/run-k8s-integration-ci.sh +++ b/test/run-k8s-integration-ci.sh @@ -29,6 +29,7 @@ readonly run_intree_plugin_tests=${RUN_INTREE_PLUGIN_TESTS:-false} readonly use_kubetest2=${USE_KUBETEST2:-true} readonly test_pd_labels=${TEST_PD_LABELS:-true} readonly migration_test=${MIGRATION_TEST:-false} +readonly test_disk_image_snapshot=${TEST_DISK_IMAGE_SNAPSHOT:-true} readonly GCE_PD_TEST_FOCUS="PersistentVolumes\sGCEPD|[V|v]olume\sexpand|\[sig-storage\]\sIn-tree\sVolumes\s\[Driver:\sgcepd\]|allowedTopologies|Pod\sDisks|PersistentVolumes\sDefault" @@ -62,7 +63,7 @@ fi base_cmd="${PKGDIR}/bin/k8s-integration-test \ --run-in-prow=true --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \ --do-driver-build=${do_driver_build} --teardown-driver=${teardown_driver} --boskos-resource-type=${boskos_resource_type} \ - --storageclass-files="${storage_classes}" --snapshotclass-file=pd-volumesnapshotclass.yaml \ + --storageclass-files="${storage_classes}" \ --deployment-strategy=${deployment_strategy} --test-version=${test_version} \ --num-nodes=3 --image-type=${image_type} --use-kubetest2=${use_kubetest2}" @@ -100,4 +101,10 @@ if [ -n "$gke_node_version" ]; then base_cmd="${base_cmd} --gke-node-version=${gke_node_version}" fi +if [ "$test_disk_image_snapshot" = true ]; then + base_cmd="${base_cmd} --snapshotclass-files=image-volumesnapshotclass.yaml,pd-volumesnapshotclass.yaml" +else + base_cmd="${base_cmd} --snapshotclass-files=pd-volumesnapshotclass.yaml" +fi + eval "$base_cmd" diff --git a/test/run-k8s-integration-local.sh b/test/run-k8s-integration-local.sh index fe620cef9..c551b0212 100755 --- a/test/run-k8s-integration-local.sh +++ b/test/run-k8s-integration-local.sh @@ -50,7 +50,17 @@ make -C "${PKGDIR}" test-k8s-integration #${PKGDIR}/bin/k8s-integration-test --run-in-prow=false \ #--staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \ #--deploy-overlay-name=prow-canary-sidecar --bringup-cluster=false --teardown-cluster=false --test-focus="External.*Storage.*snapshot" --local-k8s-dir=$KTOP \ -#--storageclass-files=sc-standard.yaml,sc-balanced.yaml,sc-ssd.yaml --snapshotclass-file=pd-volumesnapshotclass.yaml --do-driver-build=true \ +#--storageclass-files=sc-standard.yaml,sc-balanced.yaml,sc-ssd.yaml --snapshotclass-files=pd-volumesnapshotclass.yaml --do-driver-build=true \ +#--gce-zone="us-central1-b" --num-nodes=${NUM_NODES:-3} + +# This version of the command builds and deploys the GCE PD CSI driver. +# Points to a local K8s repository to get the e2e test binary, does not bring up +# or tear down the kubernetes cluster. In addition, it runs External Storage +# snapshot tests for the PD CSI driver using disk image snapshots. +#${PKGDIR}/bin/k8s-integration-test --run-in-prow=false \ +#--staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \ +#--deploy-overlay-name=prow-canary-sidecar --bringup-cluster=false --teardown-cluster=false --test-focus="External.*Storage.*snapshot" --local-k8s-dir=$KTOP \ +#--storageclass-files=sc-standard.yaml,sc-balanced.yaml,sc-ssd.yaml --snapshotclass-files=image-volumesnapshotclass.yaml --do-driver-build=true \ #--gce-zone="us-central1-b" --num-nodes=${NUM_NODES:-3} # This version of the command brings up (and subsequently tears down) a GKE @@ -58,7 +68,7 @@ make -C "${PKGDIR}" test-k8s-integration # the local K8s repository to get the e2e test binary. # ${PKGDIR}/bin/k8s-integration-test --run-in-prow=false --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \ # --test-focus="External.Storage" --local-k8s-dir=$KTOP --storageclass-files=sc-standard.yaml,sc-balanced.yaml,sc-ssd.yaml \ -# --snapshotclass-file=pd-volumesnapshotclass.yaml --do-driver-build=false --teardown-driver=false \ +# --snapshotclass-files=pd-volumesnapshotclass.yaml --do-driver-build=false --teardown-driver=false \ # --gce-zone="us-central1-c" --num-nodes=${NUM_NODES:-3} --gke-cluster-version="latest" --deployment-strategy="gke" \ # --use-gke-managed-driver=true --teardown-cluster=true @@ -67,7 +77,7 @@ make -C "${PKGDIR}" test-k8s-integration # the local K8s repository to get the e2e test binary. # ${PKGDIR}/bin/k8s-integration-test --run-in-prow=false --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \ # --test-focus="External.Storage" --local-k8s-dir=$KTOP --storageclass-files=sc-standard.yaml,sc-balanced.yaml,sc-ssd.yaml \ -# --snapshotclass-file=pd-volumesnapshotclass.yaml --do-driver-build=false --teardown-driver=false \ +# --snapshotclass-files=pd-volumesnapshotclass.yaml --do-driver-build=false --teardown-driver=false \ # --gce-zone="us-central1-c" --num-nodes=${NUM_NODES:-3} --gke-release-channel="rapid" --deployment-strategy="gke" \ # --use-gke-managed-driver=true --teardown-cluster=true @@ -96,4 +106,4 @@ make -C "${PKGDIR}" test-k8s-integration # --staging-image="${GCE_PD_CSI_STAGING_IMAGE}" --service-account-file="${GCE_PD_SA_DIR}/cloud-sa.json" \ # --deploy-overlay-name=dev --bringup-cluster=false --teardown-cluster=false --local-k8s-dir="$KTOP" \ # --storageclass-files=sc-standard.yaml --do-driver-build=false --test-focus='External.Storage' \ -# --gce-zone="us-central1-b" --num-nodes="${NUM_NODES:-3}" \ No newline at end of file +# --gce-zone="us-central1-b" --num-nodes="${NUM_NODES:-3}" diff --git a/test/run-k8s-integration.sh b/test/run-k8s-integration.sh index cf360f04b..7590f6dc9 100755 --- a/test/run-k8s-integration.sh +++ b/test/run-k8s-integration.sh @@ -44,7 +44,7 @@ fi base_cmd="${PKGDIR}/bin/k8s-integration-test \ --run-in-prow=true --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \ --do-driver-build=${do_driver_build} --teardown-driver=${teardown_driver} --boskos-resource-type=${boskos_resource_type} \ - --storageclass-files=sc-standard.yaml --snapshotclass-file=pd-volumesnapshotclass.yaml \ + --storageclass-files=sc-standard.yaml --snapshotclass-files=pd-volumesnapshotclass.yaml \ --deployment-strategy=${deployment_strategy} --test-version=${test_version} \ --num-nodes=3 --image-type=${image_type} --use-kubetest2=${use_kubetest2}" diff --git a/test/run-windows-k8s-integration.sh b/test/run-windows-k8s-integration.sh index cc4c8a77f..dd90ee9e0 100755 --- a/test/run-windows-k8s-integration.sh +++ b/test/run-windows-k8s-integration.sh @@ -51,6 +51,6 @@ ${PKGDIR}/bin/k8s-integration-test \ --test-version="${test_version}" \ --kube-version="${kube_version}" \ --storageclass-files=sc-windows.yaml \ - --snapshotclass-file=pd-volumesnapshotclass.yaml \ + --snapshotclass-files=pd-volumesnapshotclass.yaml \ --test-focus='External.Storage' \ --use-kubetest2="${use_kubetest2}" diff --git a/test/run-windows-k8s-migration.sh b/test/run-windows-k8s-migration.sh index 2e07004ce..995c5366c 100755 --- a/test/run-windows-k8s-migration.sh +++ b/test/run-windows-k8s-migration.sh @@ -60,6 +60,6 @@ ${PKGDIR}/bin/k8s-integration-test \ --kube-version="${kube_version}" \ --kube-feature-gates="${feature_gates}" \ --storageclass-files=sc-windows.yaml \ - --snapshotclass-file=pd-volumesnapshotclass.yaml \ + --snapshotclass-files=pd-volumesnapshotclass.yaml \ --test-focus="${GCE_PD_TEST_FOCUS}" \ --use-kubetest2="${use_kubetest2}"