Skip to content

Commit 9681ab1

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request kubernetes-sigs#929 from luohao/hluo/image-integration-tests
Add e2e/integration tests for image snapshot
2 parents e008326 + 31db7bb commit 9681ab1

14 files changed

+174
-26
lines changed

‎pkg/gce-cloud-provider/compute/cloud-disk.go

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,17 @@ func (d *CloudDisk) GetSourceDiskId() string {
188188
}
189189
}
190190

191+
func (d *CloudDisk) GetImageId() string {
192+
switch {
193+
case d.disk != nil:
194+
return d.disk.SourceImageId
195+
case d.betaDisk != nil:
196+
return d.betaDisk.SourceImageId
197+
default:
198+
return ""
199+
}
200+
}
201+
191202
func (d *CloudDisk) GetKMSKeyName() string {
192203
switch {
193204
case d.disk != nil:

‎pkg/gce-cloud-provider/compute/fake-gce.go

Lines changed: 23 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -196,16 +196,30 @@ func (cloud *FakeCloudProvider) InsertDisk(ctx context.Context, project string,
196196
}
197197

198198
computeDisk := &computev1.Disk{
199-
Name: volKey.Name,
200-
SizeGb: common.BytesToGbRoundUp(capBytes),
201-
Description: "Disk created by GCE-PD CSI Driver",
202-
Type: cloud.GetDiskTypeURI(project, volKey, params.DiskType),
203-
SourceSnapshotId: snapshotID,
204-
SourceDiskId: volumeContentSourceVolumeID,
205-
SourceImageId: snapshotID,
206-
Status: cloud.mockDiskStatus,
207-
Labels: params.Labels,
199+
Name: volKey.Name,
200+
SizeGb: common.BytesToGbRoundUp(capBytes),
201+
Description: "Disk created by GCE-PD CSI Driver",
202+
Type: cloud.GetDiskTypeURI(project, volKey, params.DiskType),
203+
SourceDiskId: volumeContentSourceVolumeID,
204+
Status: cloud.mockDiskStatus,
205+
Labels: params.Labels,
208206
}
207+
208+
if snapshotID != "" {
209+
_, snapshotType, _, err := common.SnapshotIDToProjectKey(snapshotID)
210+
if err != nil {
211+
return err
212+
}
213+
switch snapshotType {
214+
case common.DiskSnapshotType:
215+
computeDisk.SourceSnapshotId = snapshotID
216+
case common.DiskImageType:
217+
computeDisk.SourceImageId = snapshotID
218+
default:
219+
return fmt.Errorf("invalid snapshot type in snapshot ID: %s", snapshotType)
220+
}
221+
}
222+
209223
if params.DiskEncryptionKMSKey != "" {
210224
computeDisk.DiskEncryptionKey = &computev1.CustomerEncryptionKey{
211225
KmsKeyName: params.DiskEncryptionKMSKey,

‎pkg/gce-cloud-provider/compute/gce-compute.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -894,7 +894,7 @@ func (cloud *CloudProvider) CreateImage(ctx context.Context, project string, vol
894894
Description: description,
895895
}
896896

897-
_, err = cloud.service.Images.Insert(project, image).Context(ctx).Do()
897+
_, err = cloud.service.Images.Insert(project, image).Context(ctx).ForceCreate(true).Do()
898898
if err != nil {
899899
return nil, err
900900
}

‎pkg/gce-pd-csi-driver/controller.go

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1480,8 +1480,9 @@ func generateCreateVolumeResponse(disk *gce.CloudDisk, zones []string) *csi.Crea
14801480
},
14811481
}
14821482
snapshotID := disk.GetSnapshotId()
1483+
imageID := disk.GetImageId()
14831484
diskID := disk.GetSourceDiskId()
1484-
if diskID != "" || snapshotID != "" {
1485+
if diskID != "" || snapshotID != "" || imageID != "" {
14851486
contentSource := &csi.VolumeContentSource{}
14861487
if snapshotID != "" {
14871488
contentSource = &csi.VolumeContentSource{
@@ -1501,6 +1502,15 @@ func generateCreateVolumeResponse(disk *gce.CloudDisk, zones []string) *csi.Crea
15011502
},
15021503
}
15031504
}
1505+
if imageID != "" {
1506+
contentSource = &csi.VolumeContentSource{
1507+
Type: &csi.VolumeContentSource_Snapshot{
1508+
Snapshot: &csi.VolumeContentSource_SnapshotSource{
1509+
SnapshotId: imageID,
1510+
},
1511+
},
1512+
}
1513+
}
15041514
createResp.Volume.ContentSource = contentSource
15051515
}
15061516
return createResp

‎test/e2e/tests/single_zone_e2e_test.go

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -952,6 +952,65 @@ var _ = Describe("GCE PD CSI Driver", func() {
952952
Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected snapshot to not be found")
953953
}()
954954
})
955+
956+
// Use the region of the test location.
957+
It("Should successfully create snapshot backed by disk image", func() {
958+
testContext := getRandomTestContext()
959+
960+
p, z, _ := testContext.Instance.GetIdentity()
961+
client := testContext.Client
962+
963+
// Create Disk
964+
volName, volID := createAndValidateUniqueZonalDisk(client, p, z)
965+
966+
// Create Snapshot
967+
snapshotName := testNamePrefix + string(uuid.NewUUID())
968+
testImageFamily := "test-family"
969+
970+
snapshotParams := map[string]string{common.ParameterKeySnapshotType: common.DiskImageType, common.ParameterKeyImageFamily: testImageFamily}
971+
snapshotID, err := client.CreateSnapshot(snapshotName, volID, snapshotParams)
972+
Expect(err).To(BeNil(), "CreateSnapshot failed with error: %v", err)
973+
974+
// Validate Snapshot Created
975+
snapshot, err := computeService.Images.Get(p, snapshotName).Do()
976+
Expect(err).To(BeNil(), "Could not get snapshot from cloud directly")
977+
Expect(snapshot.Name).To(Equal(snapshotName))
978+
979+
err = wait.Poll(10*time.Second, 5*time.Minute, func() (bool, error) {
980+
snapshot, err := computeService.Images.Get(p, snapshotName).Do()
981+
Expect(err).To(BeNil(), "Could not get snapshot from cloud directly")
982+
if snapshot.Status == "READY" {
983+
return true, nil
984+
}
985+
return false, nil
986+
})
987+
Expect(err).To(BeNil(), "Could not wait for snapshot be ready")
988+
989+
// Check Snapshot Type
990+
snapshot, err = computeService.Images.Get(p, snapshotName).Do()
991+
Expect(err).To(BeNil(), "Could not get snapshot from cloud directly")
992+
_, snapshotType, _, err := common.SnapshotIDToProjectKey(cleanSelfLink(snapshot.SelfLink))
993+
Expect(err).To(BeNil(), "Failed to parse snapshot ID")
994+
Expect(snapshotType).To(Equal(common.DiskImageType), "Expected images type in snapshot ID")
995+
996+
defer func() {
997+
// Delete Disk
998+
err := client.DeleteVolume(volID)
999+
Expect(err).To(BeNil(), "DeleteVolume failed")
1000+
1001+
// Validate Disk Deleted
1002+
_, err = computeService.Disks.Get(p, z, volName).Do()
1003+
Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found")
1004+
1005+
// Delete Snapshot
1006+
err = client.DeleteSnapshot(snapshotID)
1007+
Expect(err).To(BeNil(), "DeleteSnapshot failed")
1008+
1009+
// Validate Snapshot Deleted
1010+
_, err = computeService.Images.Get(p, snapshotName).Do()
1011+
Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected snapshot to not be found")
1012+
}()
1013+
})
9551014
})
9561015

9571016
func equalWithinEpsilon(a, b, epsiolon int64) bool {
@@ -1031,3 +1090,9 @@ func createAndValidateUniqueZonalMultiWriterDisk(client *remote.CsiClient, proje
10311090

10321091
return volName, volID
10331092
}
1093+
1094+
func cleanSelfLink(selfLink string) string {
1095+
temp := strings.TrimPrefix(selfLink, gce.GCEComputeAPIEndpoint)
1096+
temp = strings.TrimPrefix(temp, gce.GCEComputeBetaAPIEndpoint)
1097+
return strings.TrimPrefix(temp, gce.GCEComputeAlphaAPIEndpoint)
1098+
}

‎test/k8s-integration/config/image-volumesnapshotclass.yaml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
apiVersion: snapshot.storage.k8s.io/v1beta1
2+
kind: VolumeSnapshotClass
3+
metadata:
4+
name: csi-gce-image-snapshot-class
5+
driver: pd.csi.storage.gke.io
6+
deletionPolicy: Delete
7+
parameters:
8+
snapshot-type: images
9+
image-family: integration-test

‎test/k8s-integration/config/test-config-template.in

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ Timeouts:
1010
{{ end }}
1111
{{end}}
1212
DriverInfo:
13-
Name: csi-gcepd-{{.StorageClass}}
13+
Name: csi-gcepd-{{.StorageClass}}--{{.SnapshotClass}}
1414
SupportedFsType:
1515
{{range .SupportedFsType}} {{ . }}:
1616
{{end}}
@@ -28,4 +28,4 @@ DriverInfo:
2828
Max: 64Ti
2929
TopologyKeys:
3030
- topology.gke.io/zone
31-
NumAllowedTopologies: {{.NumAllowedTopologies}}
31+
NumAllowedTopologies: {{.NumAllowedTopologies}}

‎test/k8s-integration/driver-config.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ type driverConfig struct {
1313
StorageClassFile string
1414
StorageClass string
1515
SnapshotClassFile string
16+
SnapshotClass string
1617
Capabilities []string
1718
SupportedFsType []string
1819
MinimumVolumeSize string
@@ -115,11 +116,15 @@ func generateDriverConfigFile(testParams *testParameters, storageClassFile strin
115116
}
116117

117118
var absSnapshotClassFilePath string
119+
var snapshotClassName string
118120
// If snapshot class is passed in as argument, include snapshot specific driver capabiltiites.
119121
if testParams.snapshotClassFile != "" {
120122
caps = append(caps, "snapshotDataSource")
121123
// Update the absolute file path pointing to the snapshot class file, if it is provided as an argument.
122124
absSnapshotClassFilePath = filepath.Join(testParams.pkgDir, testConfigDir, testParams.snapshotClassFile)
125+
snapshotClassName = testParams.snapshotClassFile[:strings.LastIndex(testParams.snapshotClassFile, ".")]
126+
} else {
127+
snapshotClassName = "no-volumesnapshotclass"
123128
}
124129

125130
caps = append(caps, "pvcDataSource")
@@ -136,6 +141,7 @@ func generateDriverConfigFile(testParams *testParameters, storageClassFile strin
136141
StorageClassFile: filepath.Join(testParams.pkgDir, testConfigDir, storageClassFile),
137142
StorageClass: storageClassFile[:strings.LastIndex(storageClassFile, ".")],
138143
SnapshotClassFile: absSnapshotClassFilePath,
144+
SnapshotClass: snapshotClassName,
139145
SupportedFsType: fsTypes,
140146
Capabilities: caps,
141147
MinimumVolumeSize: minimumVolumeSize,

‎test/k8s-integration/main.go

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ var (
5757
// Test infrastructure flags
5858
boskosResourceType = flag.String("boskos-resource-type", "gce-project", "name of the boskos resource type to reserve")
5959
storageClassFiles = flag.String("storageclass-files", "", "name of storageclass yaml file to use for test relative to test/k8s-integration/config. This may be a comma-separated list to test multiple storage classes")
60-
snapshotClassFile = flag.String("snapshotclass-file", "", "name of snapshotclass yaml file to use for test relative to test/k8s-integration/config")
60+
snapshotClassFiles = flag.String("snapshotclass-files", "", "name of snapshotclass yaml file to use for test relative to test/k8s-integration/config. This may be a comma-separated list to test multiple storage classes")
6161
inProw = flag.Bool("run-in-prow", false, "is the test running in PROW")
6262

6363
// Driver flags
@@ -203,7 +203,6 @@ func handle() error {
203203
testParams := &testParameters{
204204
platform: *platform,
205205
testFocus: *testFocus,
206-
snapshotClassFile: *snapshotClassFile,
207206
stagingVersion: string(uuid.NewUUID()),
208207
deploymentStrategy: *deploymentStrat,
209208
useGKEManagedDriver: *useGKEManagedDriver,
@@ -481,6 +480,7 @@ func handle() error {
481480
// Run the tests using the k8sSourceDir kubernetes
482481
if len(*storageClassFiles) != 0 {
483482
applicableStorageClassFiles := []string{}
483+
applicableSnapshotClassFiles := []string{}
484484
for _, rawScFile := range strings.Split(*storageClassFiles, ",") {
485485
scFile := strings.TrimSpace(rawScFile)
486486
if len(scFile) == 0 {
@@ -495,13 +495,29 @@ func handle() error {
495495
if len(applicableStorageClassFiles) == 0 {
496496
return fmt.Errorf("No applicable storage classes found")
497497
}
498+
for _, rawSnapshotClassFile := range strings.Split(*snapshotClassFiles, ",") {
499+
snapshotClassFile := strings.TrimSpace(rawSnapshotClassFile)
500+
if len(snapshotClassFile) != 0 {
501+
applicableSnapshotClassFiles = append(applicableSnapshotClassFiles, snapshotClassFile)
502+
}
503+
}
504+
if len(applicableSnapshotClassFiles) == 0 {
505+
// when no snapshot class specified, we run the tests without snapshot capability
506+
applicableSnapshotClassFiles = append(applicableSnapshotClassFiles, "")
507+
}
498508
var ginkgoErrors []string
499509
var testOutputDirs []string
500510
for _, scFile := range applicableStorageClassFiles {
501511
outputDir := strings.TrimSuffix(scFile, ".yaml")
502-
testOutputDirs = append(testOutputDirs, outputDir)
503-
if err = runCSITests(testParams, scFile, outputDir); err != nil {
504-
ginkgoErrors = append(ginkgoErrors, err.Error())
512+
for _, snapshotClassFile := range applicableSnapshotClassFiles {
513+
if len(snapshotClassFile) != 0 {
514+
outputDir = fmt.Sprintf("%s--%s", outputDir, strings.TrimSuffix(snapshotClassFile, ".yaml"))
515+
}
516+
testOutputDirs = append(testOutputDirs, outputDir)
517+
testParams.snapshotClassFile = snapshotClassFile
518+
if err = runCSITests(testParams, scFile, outputDir); err != nil {
519+
ginkgoErrors = append(ginkgoErrors, err.Error())
520+
}
505521
}
506522
}
507523
if err = mergeArtifacts(testOutputDirs); err != nil {

‎test/run-k8s-integration-ci.sh

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ readonly run_intree_plugin_tests=${RUN_INTREE_PLUGIN_TESTS:-false}
2929
readonly use_kubetest2=${USE_KUBETEST2:-true}
3030
readonly test_pd_labels=${TEST_PD_LABELS:-true}
3131
readonly migration_test=${MIGRATION_TEST:-false}
32+
readonly test_disk_image_snapshot=${TEST_DISK_IMAGE_SNAPSHOT:-true}
3233

3334
readonly GCE_PD_TEST_FOCUS="PersistentVolumes\sGCEPD|[V|v]olume\sexpand|\[sig-storage\]\sIn-tree\sVolumes\s\[Driver:\sgcepd\]|allowedTopologies|Pod\sDisks|PersistentVolumes\sDefault"
3435

@@ -62,7 +63,7 @@ fi
6263
base_cmd="${PKGDIR}/bin/k8s-integration-test \
6364
--run-in-prow=true --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \
6465
--do-driver-build=${do_driver_build} --teardown-driver=${teardown_driver} --boskos-resource-type=${boskos_resource_type} \
65-
--storageclass-files="${storage_classes}" --snapshotclass-file=pd-volumesnapshotclass.yaml \
66+
--storageclass-files="${storage_classes}" \
6667
--deployment-strategy=${deployment_strategy} --test-version=${test_version} \
6768
--num-nodes=3 --image-type=${image_type} --use-kubetest2=${use_kubetest2}"
6869

@@ -100,4 +101,10 @@ if [ -n "$gke_node_version" ]; then
100101
base_cmd="${base_cmd} --gke-node-version=${gke_node_version}"
101102
fi
102103

104+
if [ "$test_disk_image_snapshot" = true ]; then
105+
base_cmd="${base_cmd} --snapshotclass-files=image-volumesnapshotclass.yaml,pd-volumesnapshotclass.yaml"
106+
else
107+
base_cmd="${base_cmd} --snapshotclass-files=pd-volumesnapshotclass.yaml"
108+
fi
109+
103110
eval "$base_cmd"

‎test/run-k8s-integration-local.sh

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -50,15 +50,25 @@ make -C "${PKGDIR}" test-k8s-integration
5050
#${PKGDIR}/bin/k8s-integration-test --run-in-prow=false \
5151
#--staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \
5252
#--deploy-overlay-name=prow-canary-sidecar --bringup-cluster=false --teardown-cluster=false --test-focus="External.*Storage.*snapshot" --local-k8s-dir=$KTOP \
53-
#--storageclass-files=sc-standard.yaml,sc-balanced.yaml,sc-ssd.yaml --snapshotclass-file=pd-volumesnapshotclass.yaml --do-driver-build=true \
53+
#--storageclass-files=sc-standard.yaml,sc-balanced.yaml,sc-ssd.yaml --snapshotclass-files=pd-volumesnapshotclass.yaml --do-driver-build=true \
54+
#--gce-zone="us-central1-b" --num-nodes=${NUM_NODES:-3}
55+
56+
# This version of the command builds and deploys the GCE PD CSI driver.
57+
# Points to a local K8s repository to get the e2e test binary, does not bring up
58+
# or tear down the kubernetes cluster. In addition, it runs External Storage
59+
# snapshot tests for the PD CSI driver using disk image snapshots.
60+
#${PKGDIR}/bin/k8s-integration-test --run-in-prow=false \
61+
#--staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \
62+
#--deploy-overlay-name=prow-canary-sidecar --bringup-cluster=false --teardown-cluster=false --test-focus="External.*Storage.*snapshot" --local-k8s-dir=$KTOP \
63+
#--storageclass-files=sc-standard.yaml,sc-balanced.yaml,sc-ssd.yaml --snapshotclass-files=image-volumesnapshotclass.yaml --do-driver-build=true \
5464
#--gce-zone="us-central1-b" --num-nodes=${NUM_NODES:-3}
5565

5666
# This version of the command brings up (and subsequently tears down) a GKE
5767
# cluster with managed GCE PersistentDisk CSI driver add-on enabled, and points to
5868
# the local K8s repository to get the e2e test binary.
5969
# ${PKGDIR}/bin/k8s-integration-test --run-in-prow=false --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \
6070
# --test-focus="External.Storage" --local-k8s-dir=$KTOP --storageclass-files=sc-standard.yaml,sc-balanced.yaml,sc-ssd.yaml \
61-
# --snapshotclass-file=pd-volumesnapshotclass.yaml --do-driver-build=false --teardown-driver=false \
71+
# --snapshotclass-files=pd-volumesnapshotclass.yaml --do-driver-build=false --teardown-driver=false \
6272
# --gce-zone="us-central1-c" --num-nodes=${NUM_NODES:-3} --gke-cluster-version="latest" --deployment-strategy="gke" \
6373
# --use-gke-managed-driver=true --teardown-cluster=true
6474

@@ -67,7 +77,7 @@ make -C "${PKGDIR}" test-k8s-integration
6777
# the local K8s repository to get the e2e test binary.
6878
# ${PKGDIR}/bin/k8s-integration-test --run-in-prow=false --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \
6979
# --test-focus="External.Storage" --local-k8s-dir=$KTOP --storageclass-files=sc-standard.yaml,sc-balanced.yaml,sc-ssd.yaml \
70-
# --snapshotclass-file=pd-volumesnapshotclass.yaml --do-driver-build=false --teardown-driver=false \
80+
# --snapshotclass-files=pd-volumesnapshotclass.yaml --do-driver-build=false --teardown-driver=false \
7181
# --gce-zone="us-central1-c" --num-nodes=${NUM_NODES:-3} --gke-release-channel="rapid" --deployment-strategy="gke" \
7282
# --use-gke-managed-driver=true --teardown-cluster=true
7383

@@ -96,4 +106,4 @@ make -C "${PKGDIR}" test-k8s-integration
96106
# --staging-image="${GCE_PD_CSI_STAGING_IMAGE}" --service-account-file="${GCE_PD_SA_DIR}/cloud-sa.json" \
97107
# --deploy-overlay-name=dev --bringup-cluster=false --teardown-cluster=false --local-k8s-dir="$KTOP" \
98108
# --storageclass-files=sc-standard.yaml --do-driver-build=false --test-focus='External.Storage' \
99-
# --gce-zone="us-central1-b" --num-nodes="${NUM_NODES:-3}"
109+
# --gce-zone="us-central1-b" --num-nodes="${NUM_NODES:-3}"

‎test/run-k8s-integration.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ fi
4444
base_cmd="${PKGDIR}/bin/k8s-integration-test \
4545
--run-in-prow=true --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \
4646
--do-driver-build=${do_driver_build} --teardown-driver=${teardown_driver} --boskos-resource-type=${boskos_resource_type} \
47-
--storageclass-files=sc-standard.yaml --snapshotclass-file=pd-volumesnapshotclass.yaml \
47+
--storageclass-files=sc-standard.yaml --snapshotclass-files=pd-volumesnapshotclass.yaml \
4848
--deployment-strategy=${deployment_strategy} --test-version=${test_version} \
4949
--num-nodes=3 --image-type=${image_type} --use-kubetest2=${use_kubetest2}"
5050

‎test/run-windows-k8s-integration.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,6 @@ ${PKGDIR}/bin/k8s-integration-test \
5151
--test-version="${test_version}" \
5252
--kube-version="${kube_version}" \
5353
--storageclass-files=sc-windows.yaml \
54-
--snapshotclass-file=pd-volumesnapshotclass.yaml \
54+
--snapshotclass-files=pd-volumesnapshotclass.yaml \
5555
--test-focus='External.Storage' \
5656
--use-kubetest2="${use_kubetest2}"

‎test/run-windows-k8s-migration.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,6 @@ ${PKGDIR}/bin/k8s-integration-test \
6060
--kube-version="${kube_version}" \
6161
--kube-feature-gates="${feature_gates}" \
6262
--storageclass-files=sc-windows.yaml \
63-
--snapshotclass-file=pd-volumesnapshotclass.yaml \
63+
--snapshotclass-files=pd-volumesnapshotclass.yaml \
6464
--test-focus="${GCE_PD_TEST_FOCUS}" \
6565
--use-kubetest2="${use_kubetest2}"

0 commit comments

Comments
 (0)