diff --git a/pkg/gce-cloud-provider/compute/cloud-disk.go b/pkg/gce-cloud-provider/compute/cloud-disk.go index 86243d986..e6a4cd6b7 100644 --- a/pkg/gce-cloud-provider/compute/cloud-disk.go +++ b/pkg/gce-cloud-provider/compute/cloud-disk.go @@ -177,6 +177,17 @@ func (d *CloudDisk) GetSnapshotId() string { } } +func (d *CloudDisk) GetSourceDiskId() string { + switch { + case d.disk != nil: + return d.disk.SourceDiskId + case d.betaDisk != nil: + return d.betaDisk.SourceDiskId + default: + return "" + } +} + func (d *CloudDisk) GetKMSKeyName() string { switch { case d.disk != nil: diff --git a/pkg/gce-cloud-provider/compute/fake-gce.go b/pkg/gce-cloud-provider/compute/fake-gce.go index 67b2d8afe..6a7adeff6 100644 --- a/pkg/gce-cloud-provider/compute/fake-gce.go +++ b/pkg/gce-cloud-provider/compute/fake-gce.go @@ -220,7 +220,7 @@ func (cloud *FakeCloudProvider) ValidateExistingDisk(ctx context.Context, resp * return ValidateDiskParameters(resp, params) } -func (cloud *FakeCloudProvider) InsertDisk(ctx context.Context, project string, volKey *meta.Key, params common.DiskParameters, capBytes int64, capacityRange *csi.CapacityRange, replicaZones []string, snapshotID string, multiWriter bool) error { +func (cloud *FakeCloudProvider) InsertDisk(ctx context.Context, project string, volKey *meta.Key, params common.DiskParameters, capBytes int64, capacityRange *csi.CapacityRange, replicaZones []string, snapshotID string, volumeContentSourceVolumeID string, multiWriter bool) error { if disk, ok := cloud.disks[volKey.Name]; ok { err := cloud.ValidateExistingDisk(ctx, disk, params, int64(capacityRange.GetRequiredBytes()), @@ -237,6 +237,7 @@ func (cloud *FakeCloudProvider) InsertDisk(ctx context.Context, project string, Description: "Disk created by GCE-PD CSI Driver", Type: cloud.GetDiskTypeURI(project, volKey, params.DiskType), SourceSnapshotId: snapshotID, + SourceDiskId: volumeContentSourceVolumeID, Status: cloud.mockDiskStatus, Labels: params.Labels, } diff --git a/pkg/gce-cloud-provider/compute/gce-compute.go b/pkg/gce-cloud-provider/compute/gce-compute.go index 73787c679..8e672baaa 100644 --- a/pkg/gce-cloud-provider/compute/gce-compute.go +++ b/pkg/gce-cloud-provider/compute/gce-compute.go @@ -56,7 +56,7 @@ type GCECompute interface { GetDisk(ctx context.Context, project string, volumeKey *meta.Key, gceAPIVersion GCEAPIVersion) (*CloudDisk, error) RepairUnderspecifiedVolumeKey(ctx context.Context, project string, volumeKey *meta.Key) (string, *meta.Key, error) ValidateExistingDisk(ctx context.Context, disk *CloudDisk, params common.DiskParameters, reqBytes, limBytes int64, multiWriter bool) error - InsertDisk(ctx context.Context, project string, volKey *meta.Key, params common.DiskParameters, capBytes int64, capacityRange *csi.CapacityRange, replicaZones []string, snapshotID string, multiWriter bool) error + InsertDisk(ctx context.Context, project string, volKey *meta.Key, params common.DiskParameters, capBytes int64, capacityRange *csi.CapacityRange, replicaZones []string, snapshotID string, volumeContentSourceVolumeID string, multiWriter bool) error DeleteDisk(ctx context.Context, project string, volumeKey *meta.Key) error AttachDisk(ctx context.Context, project string, volKey *meta.Key, readWrite, diskType, instanceZone, instanceName string) error DetachDisk(ctx context.Context, project, deviceName, instanceZone, instanceName string) error @@ -315,7 +315,7 @@ func ValidateDiskParameters(disk *CloudDisk, params common.DiskParameters) error return nil } -func (cloud *CloudProvider) InsertDisk(ctx context.Context, project string, volKey *meta.Key, params common.DiskParameters, capBytes int64, capacityRange *csi.CapacityRange, replicaZones []string, snapshotID string, multiWriter bool) error { +func (cloud *CloudProvider) InsertDisk(ctx context.Context, project string, volKey *meta.Key, params common.DiskParameters, capBytes int64, capacityRange *csi.CapacityRange, replicaZones []string, snapshotID string, volumeContentSourceVolumeID string, multiWriter bool) error { klog.V(5).Infof("Inserting disk %v", volKey) description, err := encodeDiskTags(params.Tags) @@ -328,12 +328,12 @@ func (cloud *CloudProvider) InsertDisk(ctx context.Context, project string, volK if description == "" { description = "Disk created by GCE-PD CSI Driver" } - return cloud.insertZonalDisk(ctx, project, volKey, params, capBytes, capacityRange, snapshotID, description, multiWriter) + return cloud.insertZonalDisk(ctx, project, volKey, params, capBytes, capacityRange, snapshotID, volumeContentSourceVolumeID, description, multiWriter) case meta.Regional: if description == "" { description = "Regional disk created by GCE-PD CSI Driver" } - return cloud.insertRegionalDisk(ctx, project, volKey, params, capBytes, capacityRange, replicaZones, snapshotID, description, multiWriter) + return cloud.insertRegionalDisk(ctx, project, volKey, params, capBytes, capacityRange, replicaZones, snapshotID, volumeContentSourceVolumeID, description, multiWriter) default: return fmt.Errorf("could not insert disk, key was neither zonal nor regional, instead got: %v", volKey.String()) } @@ -377,6 +377,7 @@ func (cloud *CloudProvider) insertRegionalDisk( capacityRange *csi.CapacityRange, replicaZones []string, snapshotID string, + volumeContentSourceVolumeID string, description string, multiWriter bool) error { var ( @@ -399,6 +400,9 @@ func (cloud *CloudProvider) insertRegionalDisk( if snapshotID != "" { diskToCreate.SourceSnapshot = snapshotID } + if volumeContentSourceVolumeID != "" { + diskToCreate.SourceDisk = volumeContentSourceVolumeID + } if len(replicaZones) != 0 { diskToCreate.ReplicaZones = replicaZones } @@ -472,6 +476,7 @@ func (cloud *CloudProvider) insertZonalDisk( capBytes int64, capacityRange *csi.CapacityRange, snapshotID string, + volumeContentSourceVolumeID string, description string, multiWriter bool) error { var ( @@ -495,6 +500,9 @@ func (cloud *CloudProvider) insertZonalDisk( if snapshotID != "" { diskToCreate.SourceSnapshot = snapshotID } + if volumeContentSourceVolumeID != "" { + diskToCreate.SourceDisk = volumeContentSourceVolumeID + } if params.DiskEncryptionKMSKey != "" { diskToCreate.DiskEncryptionKey = &computev1.CustomerEncryptionKey{ diff --git a/pkg/gce-pd-csi-driver/controller.go b/pkg/gce-pd-csi-driver/controller.go index 7b1434075..3bbc664c4 100644 --- a/pkg/gce-pd-csi-driver/controller.go +++ b/pkg/gce-pd-csi-driver/controller.go @@ -179,9 +179,8 @@ func (gceCS *GCEControllerServer) CreateVolume(ctx context.Context, req *csi.Cre if err != nil { return nil, status.Error(codes.Internal, fmt.Sprintf("CreateVolume disk %v had error checking ready status: %v", volKey, err)) } - if !ready { - return nil, status.Error(codes.Internal, fmt.Sprintf("CreateVolume disk %v is not ready", volKey)) + return nil, status.Error(codes.Internal, fmt.Sprintf("CreateVolume existing disk %v is not ready", volKey)) } // If there is no validation error, immediately return success @@ -190,10 +189,10 @@ func (gceCS *GCEControllerServer) CreateVolume(ctx context.Context, req *csi.Cre } snapshotID := "" + volumeContentSourceVolumeID := "" content := req.GetVolumeContentSource() if content != nil { if content.GetSnapshot() != nil { - // TODO(#161): Add support for Volume Source (cloning) introduced in CSI v1.0.0 snapshotID = content.GetSnapshot().GetSnapshotId() // Verify that snapshot exists @@ -204,8 +203,38 @@ func (gceCS *GCEControllerServer) CreateVolume(ctx context.Context, req *csi.Cre return nil, status.Errorf(codes.NotFound, "CreateVolume source snapshot %s does not exist", snapshotID) } } - } + if content.GetVolume() != nil { + volumeContentSourceVolumeID = content.GetVolume().GetVolumeId() + // Verify that the source VolumeID is in the correct format. + project, sourceVolKey, err := common.VolumeIDToKey(volumeContentSourceVolumeID) + if err != nil { + return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("CreateVolume source volume id is invalid: %v", err)) + } + + // Verify that the volume in VolumeContentSource exists. + diskFromSourceVolume, err := gceCS.CloudProvider.GetDisk(ctx, project, sourceVolKey, gceAPIVersion) + if err != nil { + if gce.IsGCEError(err, "notFound") { + return nil, status.Errorf(codes.NotFound, "CreateVolume source volume %s does not exist", volumeContentSourceVolumeID) + } else { + return nil, status.Error(codes.Internal, fmt.Sprintf("CreateVolume unknown get disk error when validating: %v", err)) + } + } + // Verify the zone, region, and disk type of the clone must be the same as that of the source disk. + if err := gce.ValidateDiskParameters(diskFromSourceVolume, params); err != nil { + return nil, status.Errorf(codes.InvalidArgument, `CreateVolume source volume parameters do not match CreateVolumeRequest Parameters: %v`, err) + } + // Verify the source disk is ready. + ready, err := isDiskReady(diskFromSourceVolume) + if err != nil { + return nil, status.Error(codes.Internal, fmt.Sprintf("CreateVolume disk from source volume %v had error checking ready status: %v", sourceVolKey, err)) + } + if !ready { + return nil, status.Error(codes.Internal, fmt.Sprintf("CreateVolume disk from source volume %v is not ready", sourceVolKey)) + } + } + } // Create the disk var disk *gce.CloudDisk switch params.ReplicationType { @@ -213,7 +242,7 @@ func (gceCS *GCEControllerServer) CreateVolume(ctx context.Context, req *csi.Cre if len(zones) != 1 { return nil, status.Error(codes.Internal, fmt.Sprintf("CreateVolume failed to get a single zone for creating zonal disk, instead got: %v", zones)) } - disk, err = createSingleZoneDisk(ctx, gceCS.CloudProvider, name, zones, params, capacityRange, capBytes, snapshotID, multiWriter) + disk, err = createSingleZoneDisk(ctx, gceCS.CloudProvider, name, zones, params, capacityRange, capBytes, snapshotID, volumeContentSourceVolumeID, multiWriter) if err != nil { return nil, status.Error(codes.Internal, fmt.Sprintf("CreateVolume failed to create single zonal disk %#v: %v", name, err)) } @@ -221,7 +250,7 @@ func (gceCS *GCEControllerServer) CreateVolume(ctx context.Context, req *csi.Cre if len(zones) != 2 { return nil, status.Errorf(codes.Internal, fmt.Sprintf("CreateVolume failed to get a 2 zones for creating regional disk, instead got: %v", zones)) } - disk, err = createRegionalDisk(ctx, gceCS.CloudProvider, name, zones, params, capacityRange, capBytes, snapshotID, multiWriter) + disk, err = createRegionalDisk(ctx, gceCS.CloudProvider, name, zones, params, capacityRange, capBytes, snapshotID, volumeContentSourceVolumeID, multiWriter) if err != nil { return nil, status.Error(codes.Internal, fmt.Sprintf("CreateVolume failed to create regional disk %#v: %v", name, err)) } @@ -1053,15 +1082,28 @@ func generateCreateVolumeResponse(disk *gce.CloudDisk, zones []string) *csi.Crea }, } snapshotID := disk.GetSnapshotId() - if snapshotID != "" { - source := &csi.VolumeContentSource{ - Type: &csi.VolumeContentSource_Snapshot{ - Snapshot: &csi.VolumeContentSource_SnapshotSource{ - SnapshotId: snapshotID, + diskID := disk.GetSourceDiskId() + if diskID != "" || snapshotID != "" { + contentSource := &csi.VolumeContentSource{} + if snapshotID != "" { + contentSource = &csi.VolumeContentSource{ + Type: &csi.VolumeContentSource_Snapshot{ + Snapshot: &csi.VolumeContentSource_SnapshotSource{ + SnapshotId: snapshotID, + }, }, - }, + } + } + if diskID != "" { + contentSource = &csi.VolumeContentSource{ + Type: &csi.VolumeContentSource_Volume{ + Volume: &csi.VolumeContentSource_VolumeSource{ + VolumeId: diskID, + }, + }, + } } - createResp.Volume.ContentSource = source + createResp.Volume.ContentSource = contentSource } return createResp } @@ -1072,7 +1114,7 @@ func cleanSelfLink(selfLink string) string { return strings.TrimPrefix(temp, gce.GCEComputeAlphaAPIEndpoint) } -func createRegionalDisk(ctx context.Context, cloudProvider gce.GCECompute, name string, zones []string, params common.DiskParameters, capacityRange *csi.CapacityRange, capBytes int64, snapshotID string, multiWriter bool) (*gce.CloudDisk, error) { +func createRegionalDisk(ctx context.Context, cloudProvider gce.GCECompute, name string, zones []string, params common.DiskParameters, capacityRange *csi.CapacityRange, capBytes int64, snapshotID string, volumeContentSourceVolumeID string, multiWriter bool) (*gce.CloudDisk, error) { project := cloudProvider.GetDefaultProject() region, err := common.GetRegionFromZones(zones) if err != nil { @@ -1085,7 +1127,7 @@ func createRegionalDisk(ctx context.Context, cloudProvider gce.GCECompute, name fullyQualifiedReplicaZones, cloudProvider.GetReplicaZoneURI(project, replicaZone)) } - err = cloudProvider.InsertDisk(ctx, project, meta.RegionalKey(name, region), params, capBytes, capacityRange, fullyQualifiedReplicaZones, snapshotID, multiWriter) + err = cloudProvider.InsertDisk(ctx, project, meta.RegionalKey(name, region), params, capBytes, capacityRange, fullyQualifiedReplicaZones, snapshotID, volumeContentSourceVolumeID, multiWriter) if err != nil { return nil, fmt.Errorf("failed to insert regional disk: %v", err) } @@ -1102,13 +1144,13 @@ func createRegionalDisk(ctx context.Context, cloudProvider gce.GCECompute, name return disk, nil } -func createSingleZoneDisk(ctx context.Context, cloudProvider gce.GCECompute, name string, zones []string, params common.DiskParameters, capacityRange *csi.CapacityRange, capBytes int64, snapshotID string, multiWriter bool) (*gce.CloudDisk, error) { +func createSingleZoneDisk(ctx context.Context, cloudProvider gce.GCECompute, name string, zones []string, params common.DiskParameters, capacityRange *csi.CapacityRange, capBytes int64, snapshotID string, volumeContentSourceVolumeID string, multiWriter bool) (*gce.CloudDisk, error) { project := cloudProvider.GetDefaultProject() if len(zones) != 1 { return nil, fmt.Errorf("got wrong number of zones for zonal create volume: %v", len(zones)) } diskZone := zones[0] - err := cloudProvider.InsertDisk(ctx, project, meta.ZonalKey(name, diskZone), params, capBytes, capacityRange, nil, snapshotID, multiWriter) + err := cloudProvider.InsertDisk(ctx, project, meta.ZonalKey(name, diskZone), params, capBytes, capacityRange, nil, snapshotID, volumeContentSourceVolumeID, multiWriter) if err != nil { return nil, fmt.Errorf("failed to insert zonal disk: %v", err) } diff --git a/pkg/gce-pd-csi-driver/controller_test.go b/pkg/gce-pd-csi-driver/controller_test.go index fe671ba78..fbba1058d 100644 --- a/pkg/gce-pd-csi-driver/controller_test.go +++ b/pkg/gce-pd-csi-driver/controller_test.go @@ -824,7 +824,7 @@ func TestListVolumeArgs(t *testing.T) { } } -func TestCreateVolumeWithVolumeSource(t *testing.T) { +func TestCreateVolumeWithVolumeSourceFromSnapshot(t *testing.T) { // Define test cases testCases := []struct { name string @@ -900,6 +900,162 @@ func TestCreateVolumeWithVolumeSource(t *testing.T) { } } +func TestCreateVolumeWithVolumeSourceFromVolume(t *testing.T) { + testSourceVolumeName := "test-volume-source-name" + testZonalVolumeSourceID := fmt.Sprintf("projects/%s/zones/%s/disks/%s", project, zone, testSourceVolumeName) + testRegionalVolumeSourceID := fmt.Sprintf("projects/%s/regions/%s/disks/%s", project, region, testSourceVolumeName) + testVolumeSourceIDDifferentZone := fmt.Sprintf("projects/%s/zones/%s/disks/%s", project, "different-zone", testSourceVolumeName) + topology := &csi.TopologyRequirement{ + Requisite: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: region + "-b"}, + }, + { + Segments: map[string]string{common.TopologyKeyZone: region + "-c"}, + }, + }, + } + regionalParams := map[string]string{ + common.ParameterKeyType: "test-type", common.ParameterKeyReplicationType: "regional-pd", + } + // Define test cases + testCases := []struct { + name string + volumeOnCloud bool + expErrCode codes.Code + sourceVolumeID string + reqParameters map[string]string + sourceReqParameters map[string]string + topology *csi.TopologyRequirement + }{ + { + name: "success with data source of zonal volume type", + volumeOnCloud: true, + sourceVolumeID: testZonalVolumeSourceID, + reqParameters: stdParams, + sourceReqParameters: stdParams, + }, + { + name: "success with data source of regional volume type", + volumeOnCloud: true, + sourceVolumeID: testRegionalVolumeSourceID, + reqParameters: regionalParams, + sourceReqParameters: regionalParams, + topology: topology, + }, + { + name: "fail with with data source of replication-type different from CreateVolumeRequest", + volumeOnCloud: true, + expErrCode: codes.InvalidArgument, + sourceVolumeID: testZonalVolumeSourceID, + reqParameters: stdParams, + sourceReqParameters: regionalParams, + topology: topology, + }, + { + name: "fail with data source of zonal volume type that doesn't exist", + volumeOnCloud: false, + expErrCode: codes.NotFound, + sourceVolumeID: testZonalVolumeSourceID, + reqParameters: stdParams, + sourceReqParameters: stdParams, + }, + { + name: "fail with data source of zonal volume type with invalid volume id format", + volumeOnCloud: false, + expErrCode: codes.InvalidArgument, + sourceVolumeID: testZonalVolumeSourceID + "invalid/format", + reqParameters: stdParams, + sourceReqParameters: stdParams, + }, + { + name: "fail with data source of zonal volume type with invalid disk parameters", + volumeOnCloud: true, + expErrCode: codes.InvalidArgument, + sourceVolumeID: testVolumeSourceIDDifferentZone, + reqParameters: stdParams, + sourceReqParameters: map[string]string{ + common.ParameterKeyType: "different-type", + }, + }, + { + name: "fail with data source of zonal volume type with invalid replication type", + volumeOnCloud: true, + expErrCode: codes.InvalidArgument, + sourceVolumeID: testZonalVolumeSourceID, + reqParameters: regionalParams, + sourceReqParameters: stdParams, + }, + } + + for _, tc := range testCases { + t.Logf("test case: %s", tc.name) + gceDriver := initGCEDriver(t, nil) + + req := &csi.CreateVolumeRequest{ + Name: name, + CapacityRange: stdCapRange, + VolumeCapabilities: stdVolCaps, + Parameters: tc.reqParameters, + VolumeContentSource: &csi.VolumeContentSource{ + Type: &csi.VolumeContentSource_Volume{ + Volume: &csi.VolumeContentSource_VolumeSource{ + VolumeId: tc.sourceVolumeID, + }, + }, + }, + } + + sourceVolumeRequest := &csi.CreateVolumeRequest{ + Name: testSourceVolumeName, + CapacityRange: stdCapRange, + VolumeCapabilities: stdVolCaps, + Parameters: tc.sourceReqParameters, + } + + if tc.topology != nil { + // req.AccessibilityRequirements = tc.topology + sourceVolumeRequest.AccessibilityRequirements = tc.topology + } + + if tc.volumeOnCloud { + // Create the source volume. + sourceVolume, _ := gceDriver.cs.CreateVolume(context.Background(), sourceVolumeRequest) + req.VolumeContentSource = &csi.VolumeContentSource{ + Type: &csi.VolumeContentSource_Volume{ + Volume: &csi.VolumeContentSource_VolumeSource{ + VolumeId: sourceVolume.GetVolume().VolumeId, + }, + }, + } + } + + resp, err := gceDriver.cs.CreateVolume(context.Background(), req) + t.Logf("response: %v err: %v", resp, err) + if err != nil { + serverError, ok := status.FromError(err) + if !ok { + t.Fatalf("Could not get error status code from err: %v", serverError) + } + if serverError.Code() != tc.expErrCode { + t.Fatalf("Expected error code: %v, got: %v. err : %v", tc.expErrCode, serverError.Code(), err) + } + continue + } + if tc.expErrCode != codes.OK { + t.Fatalf("Expected error: %v, got no error", tc.expErrCode) + } + + // Make sure the response has the source volume. + sourceVolume := resp.GetVolume() + t.Logf("response has source volume: %v ", sourceVolume) + if sourceVolume.ContentSource == nil || sourceVolume.ContentSource.Type == nil || + sourceVolume.ContentSource.GetVolume() == nil || sourceVolume.ContentSource.GetVolume().VolumeId == "" { + t.Fatalf("Expected volume content source to have volume ID, got none") + } + } +} + func TestCreateVolumeRandomRequisiteTopology(t *testing.T) { req := &csi.CreateVolumeRequest{ Name: "test-name", diff --git a/pkg/gce-pd-csi-driver/gce-pd-driver.go b/pkg/gce-pd-csi-driver/gce-pd-driver.go index 62189861a..4dcb34703 100644 --- a/pkg/gce-pd-csi-driver/gce-pd-driver.go +++ b/pkg/gce-pd-csi-driver/gce-pd-driver.go @@ -67,6 +67,7 @@ func (gceDriver *GCEDriver) SetupGCEDriver(name, vendorVersion string, extraVolu csi.ControllerServiceCapability_RPC_EXPAND_VOLUME, csi.ControllerServiceCapability_RPC_LIST_VOLUMES, csi.ControllerServiceCapability_RPC_LIST_VOLUMES_PUBLISHED_NODES, + csi.ControllerServiceCapability_RPC_CLONE_VOLUME, } gceDriver.AddControllerServiceCapabilities(csc) ns := []csi.NodeServiceCapability_RPC_Type{ diff --git a/test/k8s-integration/driver-config.go b/test/k8s-integration/driver-config.go index c612ea918..97d7289e6 100644 --- a/test/k8s-integration/driver-config.go +++ b/test/k8s-integration/driver-config.go @@ -76,7 +76,6 @@ func generateDriverConfigFile(testParams *testParameters, storageClassFile strin } /* Unsupported Capabilities: - pvcDataSource RWX volumeLimits # PD Supports volume limits but test is very slow singleNodeVolume @@ -117,6 +116,7 @@ func generateDriverConfigFile(testParams *testParameters, storageClassFile strin absSnapshotClassFilePath = filepath.Join(testParams.pkgDir, testConfigDir, testParams.snapshotClassFile) } + caps = append(caps, "pvcDataSource") minimumVolumeSize := "5Gi" numAllowedTopologies := 1 if storageClassFile == regionalPDStorageClass { diff --git a/test/k8s-integration/main.go b/test/k8s-integration/main.go index c6646070e..a456e1559 100644 --- a/test/k8s-integration/main.go +++ b/test/k8s-integration/main.go @@ -543,6 +543,9 @@ func generateGCETestSkip(testParams *testParameters) string { if testParams.platform == "windows" { skipString = skipString + "|\\[LinuxOnly\\]" } + // Volume cloning has timeouts due to GCE disk cloning rate limits, and operation serialization + // race conditions where the cloning begins while the source disk is still being created. + skipString = skipString + "|provisioning\\sshould\\sprovision\\sstorage\\swith\\spvc\\sdata\\ssource[^|]*" return skipString } @@ -585,7 +588,9 @@ func generateGKETestSkip(testParams *testParameters) string { (!testParams.useGKEManagedDriver && (*curVer).lessThan(mustParseVersion("1.17.0"))) { skipString = skipString + "|VolumeSnapshotDataSource" } - + // Volume cloning has timeouts due to GCE disk cloning rate limits, and operation serialization + // race conditions where the cloning begins while the source disk is still being created. + skipString = skipString + "|provisioning\\sshould\\sprovision\\sstorage\\swith\\spvc\\sdata\\ssource[^|]*" return skipString }