diff --git a/pkg/common/parameters.go b/pkg/common/parameters.go index 8ef4e3ede..ea36f6276 100644 --- a/pkg/common/parameters.go +++ b/pkg/common/parameters.go @@ -30,8 +30,11 @@ const ( // Parameters for VolumeSnapshotClass ParameterKeyStorageLocations = "storage-locations" - - replicationTypeNone = "none" + ParameterKeySnapshotType = "snapshot-type" + ParameterKeyImageFamily = "image-family" + DiskSnapshotType = "snapshots" + DiskImageType = "images" + replicationTypeNone = "none" // Keys for PV and PVC parameters as reported by external-provisioner ParameterKeyPVCName = "csi.storage.k8s.io/pvc/name" @@ -67,6 +70,8 @@ type DiskParameters struct { // SnapshotParameters contains normalized and defaulted parameters for snapshots type SnapshotParameters struct { StorageLocations []string + SnapshotType string + ImageFamily string } // ExtractAndDefaultParameters will take the relevant parameters from a map and @@ -131,6 +136,7 @@ func ExtractAndDefaultParameters(parameters map[string]string, driverName string func ExtractAndDefaultSnapshotParameters(parameters map[string]string) (SnapshotParameters, error) { p := SnapshotParameters{ StorageLocations: []string{}, + SnapshotType: DiskSnapshotType, } for k, v := range parameters { switch strings.ToLower(k) { @@ -140,6 +146,14 @@ func ExtractAndDefaultSnapshotParameters(parameters map[string]string) (Snapshot return p, err } p.StorageLocations = normalizedStorageLocations + case ParameterKeySnapshotType: + err := ValidateSnapshotType(v) + if err != nil { + return p, err + } + p.SnapshotType = v + case ParameterKeyImageFamily: + p.ImageFamily = v default: return p, fmt.Errorf("parameters contains invalid option %q", k) } diff --git a/pkg/common/parameters_test.go b/pkg/common/parameters_test.go index d0fb8f8e8..5f7eff78a 100644 --- a/pkg/common/parameters_test.go +++ b/pkg/common/parameters_test.go @@ -165,37 +165,50 @@ func TestExtractAndDefaultParameters(t *testing.T) { } } -// Currently the only parameter is storage-locations, which is already tested -// in utils_test/TestSnapshotStorageLocations. Here we just test the case where -// no parameter is set in the snapshot class. +// Currently the storage-locations parameter is tested in utils_test/TestSnapshotStorageLocations. +// Here we just test other parameters. func TestSnapshotParameters(t *testing.T) { tests := []struct { desc string parameters map[string]string expectedSnapshotParames SnapshotParameters + expectError bool }{ { desc: "valid parameter", - parameters: map[string]string{ParameterKeyStorageLocations: "ASIA "}, + parameters: map[string]string{ParameterKeyStorageLocations: "ASIA ", ParameterKeySnapshotType: "images", ParameterKeyImageFamily: "test-family"}, expectedSnapshotParames: SnapshotParameters{ StorageLocations: []string{"asia"}, + SnapshotType: DiskImageType, + ImageFamily: "test-family", }, + expectError: false, }, { desc: "nil parameter", parameters: nil, expectedSnapshotParames: SnapshotParameters{ StorageLocations: []string{}, + SnapshotType: DiskSnapshotType, }, + expectError: false, + }, + { + desc: "invalid snapshot type", + parameters: map[string]string{ParameterKeySnapshotType: "invalid-type"}, + expectError: true, }, } for _, tc := range tests { t.Run(tc.desc, func(t *testing.T) { p, err := ExtractAndDefaultSnapshotParameters(tc.parameters) - if err != nil { - t.Errorf("Got error processing snapshot parameters: %v; expect no error", err) + if err != nil && !tc.expectError { + t.Errorf("Got error %v; expect no error", err) + } + if err == nil && tc.expectError { + t.Error("Got no error; expect an error") } - if !reflect.DeepEqual(p, tc.expectedSnapshotParames) { + if err == nil && !reflect.DeepEqual(p, tc.expectedSnapshotParames) { t.Errorf("Got ExtractAndDefaultSnapshotParameters(%+v) = %+v; expect %+v", tc.parameters, p, tc.expectedSnapshotParames) } }) diff --git a/pkg/common/utils.go b/pkg/common/utils.go index 6b71feb78..d5842ea36 100644 --- a/pkg/common/utils.go +++ b/pkg/common/utils.go @@ -114,15 +114,15 @@ func GenerateUnderspecifiedVolumeID(diskName string, isZonal bool) string { return fmt.Sprintf(volIDRegionalFmt, UnspecifiedValue, UnspecifiedValue, diskName) } -func SnapshotIDToProjectKey(id string) (string, string, error) { +func SnapshotIDToProjectKey(id string) (string, string, string, error) { splitId := strings.Split(id, "/") if len(splitId) != snapshotTotalElements { - return "", "", fmt.Errorf("failed to get id components. Expected projects/{project}/global/snapshot/{name}. Got: %s", id) + return "", "", "", fmt.Errorf("failed to get id components. Expected projects/{project}/global/{snapshots|images}/{name}. Got: %s", id) } if splitId[snapshotTopologyKey] == "global" { - return splitId[snapshotProjectKey], splitId[snapshotTotalElements-1], nil + return splitId[snapshotProjectKey], splitId[snapshotTotalElements-2], splitId[snapshotTotalElements-1], nil } else { - return "", "", fmt.Errorf("could not get id components, expected global, got: %v", splitId[snapshotTopologyKey]) + return "", "", "", fmt.Errorf("could not get id components, expected global, got: %v", splitId[snapshotTopologyKey]) } } @@ -238,3 +238,13 @@ func ProcessStorageLocations(storageLocations string) ([]string, error) { } return []string{normalizedLoc}, nil } + +// ValidateSnapshotType validates the type +func ValidateSnapshotType(snapshotType string) error { + switch snapshotType { + case DiskSnapshotType, DiskImageType: + return nil + default: + return fmt.Errorf("invalid snapshot type %s", snapshotType) + } +} diff --git a/pkg/gce-cloud-provider/compute/fake-gce.go b/pkg/gce-cloud-provider/compute/fake-gce.go index 6a7adeff6..dd9953167 100644 --- a/pkg/gce-cloud-provider/compute/fake-gce.go +++ b/pkg/gce-cloud-provider/compute/fake-gce.go @@ -17,7 +17,6 @@ package gcecloudprovider import ( "context" "fmt" - "strconv" "strings" "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta" @@ -37,6 +36,7 @@ const ( Timestamp = "2018-09-05T15:17:08.270-07:00" BasePath = "https://www.googleapis.com/compute/v1/projects/" snapshotURITemplateGlobal = "%s/global/snapshots/%s" //{gce.projectID}/global/snapshots/{snapshot.Name}" + imageURITemplateGlobal = "%s/global/images/%s" //{gce.projectID}/global/images/{image.Name}" ) type FakeCloudProvider struct { @@ -47,6 +47,7 @@ type FakeCloudProvider struct { pageTokens map[string]sets.String instances map[string]*computev1.Instance snapshots map[string]*computev1.Snapshot + images map[string]*computev1.Image // marker to set disk status during InsertDisk operation. mockDiskStatus string @@ -61,6 +62,7 @@ func CreateFakeCloudProvider(project, zone string, cloudDisks []*CloudDisk) (*Fa disks: map[string]*CloudDisk{}, instances: map[string]*computev1.Instance{}, snapshots: map[string]*computev1.Snapshot{}, + images: map[string]*computev1.Image{}, pageTokens: map[string]sets.String{}, // A newly created disk is marked READY by default. mockDiskStatus: "READY", @@ -122,7 +124,7 @@ func (cloud *FakeCloudProvider) ListDisks(ctx context.Context) ([]*computev1.Dis return d, "", nil } -func (cloud *FakeCloudProvider) ListSnapshots(ctx context.Context, filter string, maxEntries int64, pageToken string) ([]*computev1.Snapshot, string, error) { +func (cloud *FakeCloudProvider) ListSnapshots(ctx context.Context, filter string) ([]*computev1.Snapshot, string, error) { var sourceDisk string snapshots := []*computev1.Snapshot{} if len(filter) > 0 { @@ -141,45 +143,7 @@ func (cloud *FakeCloudProvider) ListSnapshots(ctx context.Context, filter string snapshots = append(snapshots, snapshot) } - var ( - ulenSnapshots = len(snapshots) - startingToken int - ) - - if len(pageToken) > 0 { - i, err := strconv.ParseUint(pageToken, 10, 32) - if err != nil { - return nil, "", invalidError() - } - startingToken = int(i) - } - - if startingToken > ulenSnapshots { - return nil, "", invalidError() - } - - // Discern the number of remaining entries. - rem := ulenSnapshots - startingToken - - // If maxEntries is 0 or greater than the number of remaining entries then - // set maxEntries to the number of remaining entries. - max := int(maxEntries) - if max == 0 || max > rem { - max = rem - } - - results := []*computev1.Snapshot{} - j := startingToken - for i := 0; i < max; i++ { - results = append(results, snapshots[j]) - j++ - } - - var nextToken string - if j < ulenSnapshots { - nextToken = fmt.Sprintf("%d", j) - } - return results, nextToken, nil + return snapshots, "", nil } // Disk Methods @@ -238,6 +202,7 @@ func (cloud *FakeCloudProvider) InsertDisk(ctx context.Context, project string, Type: cloud.GetDiskTypeURI(project, volKey, params.DiskType), SourceSnapshotId: snapshotID, SourceDiskId: volumeContentSourceVolumeID, + SourceImageId: snapshotID, Status: cloud.mockDiskStatus, Labels: params.Labels, } @@ -399,6 +364,71 @@ func (cloud *FakeCloudProvider) DeleteSnapshot(ctx context.Context, project, sna return nil } +func (cloud *FakeCloudProvider) ListImages(ctx context.Context, filter string) ([]*computev1.Image, string, error) { + var sourceDisk string + images := []*computev1.Image{} + if len(filter) > 0 { + filterSplits := strings.Fields(filter) + if len(filterSplits) != 3 || filterSplits[0] != "sourceDisk" { + return nil, "", invalidError() + } + sourceDisk = filterSplits[2] + } + for _, image := range cloud.images { + if len(sourceDisk) > 0 { + if image.SourceDisk == sourceDisk { + continue + } + } + images = append(images, image) + } + + return images, "", nil +} + +func (cloud *FakeCloudProvider) GetImage(ctx context.Context, project, imageName string) (*computev1.Image, error) { + image, ok := cloud.images[imageName] + if !ok { + return nil, notFoundError() + } + image.Status = "READY" + return image, nil +} + +func (cloud *FakeCloudProvider) CreateImage(ctx context.Context, project string, volKey *meta.Key, imageName string, snapshotParams common.SnapshotParameters) (*computev1.Image, error) { + if image, ok := cloud.images[imageName]; ok { + return image, nil + } + + imageToCreate := &computev1.Image{ + CreationTimestamp: Timestamp, + DiskSizeGb: int64(DiskSizeGb), + Family: snapshotParams.ImageFamily, + Name: imageName, + SelfLink: cloud.getGlobalImageURI(project, imageName), + SourceType: "RAW", + Status: "PENDING", + StorageLocations: snapshotParams.StorageLocations, + } + + switch volKey.Type() { + case meta.Zonal: + imageToCreate.SourceDisk = cloud.getZonalDiskSourceURI(project, volKey.Name, volKey.Zone) + case meta.Regional: + imageToCreate.SourceDisk = cloud.getRegionalDiskSourceURI(project, volKey.Name, volKey.Region) + default: + return nil, fmt.Errorf("could not create image, disk key was neither zonal nor regional, instead got: %v", volKey.String()) + } + + cloud.images[imageName] = imageToCreate + return imageToCreate, nil +} + +func (cloud *FakeCloudProvider) DeleteImage(ctx context.Context, project, imageName string) error { + delete(cloud.images, imageName) + return nil +} + func (cloud *FakeCloudProvider) ValidateExistingSnapshot(resp *computev1.Snapshot, volKey *meta.Key) error { if resp == nil { return fmt.Errorf("disk does not exist") @@ -447,6 +477,13 @@ func (cloud *FakeCloudProvider) getGlobalSnapshotURI(project, snapshotName strin snapshotName) } +func (cloud *FakeCloudProvider) getGlobalImageURI(project, imageName string) string { + return BasePath + fmt.Sprintf( + imageURITemplateGlobal, + project, + imageName) +} + func (cloud *FakeCloudProvider) UpdateDiskStatus(s string) { cloud.mockDiskStatus = s } @@ -467,6 +504,13 @@ func (cloud *FakeBlockingCloudProvider) CreateSnapshot(ctx context.Context, proj return cloud.FakeCloudProvider.CreateSnapshot(ctx, project, volKey, snapshotName, snapshotParams) } +func (cloud *FakeBlockingCloudProvider) CreateImage(ctx context.Context, project string, volKey *meta.Key, imageName string, snapshotParams common.SnapshotParameters) (*computev1.Image, error) { + executeCreateSnapshot := make(chan struct{}) + cloud.ReadyToExecute <- executeCreateSnapshot + <-executeCreateSnapshot + return cloud.FakeCloudProvider.CreateImage(ctx, project, volKey, imageName, snapshotParams) +} + func notFoundError() *googleapi.Error { return &googleapi.Error{ Errors: []googleapi.ErrorItem{ diff --git a/pkg/gce-cloud-provider/compute/gce-compute.go b/pkg/gce-cloud-provider/compute/gce-compute.go index f89499502..8d1818c50 100644 --- a/pkg/gce-cloud-provider/compute/gce-compute.go +++ b/pkg/gce-cloud-provider/compute/gce-compute.go @@ -35,6 +35,7 @@ import ( const ( operationStatusDone = "DONE" waitForSnapshotCreationTimeOut = 2 * time.Minute + waitForImageCreationTimeOut = 5 * time.Minute diskKind = "compute#disk" cryptoKeyVerDelimiter = "/cryptoKeyVersions" ) @@ -71,10 +72,14 @@ type GCECompute interface { GetInstanceOrError(ctx context.Context, instanceZone, instanceName string) (*computev1.Instance, error) // Zone Methods ListZones(ctx context.Context, region string) ([]string, error) - ListSnapshots(ctx context.Context, filter string, maxEntries int64, pageToken string) ([]*computev1.Snapshot, string, error) + ListSnapshots(ctx context.Context, filter string) ([]*computev1.Snapshot, string, error) GetSnapshot(ctx context.Context, project, snapshotName string) (*computev1.Snapshot, error) CreateSnapshot(ctx context.Context, project string, volKey *meta.Key, snapshotName string, snapshotParams common.SnapshotParameters) (*computev1.Snapshot, error) DeleteSnapshot(ctx context.Context, project, snapshotName string) error + ListImages(ctx context.Context, filter string) ([]*computev1.Image, string, error) + GetImage(ctx context.Context, project, imageName string) (*computev1.Image, error) + CreateImage(ctx context.Context, project string, volKey *meta.Key, imageName string, snapshotParams common.SnapshotParameters) (*computev1.Image, error) + DeleteImage(ctx context.Context, project, imageName string) error } // GetDefaultProject returns the project that was used to instantiate this GCE client. @@ -189,18 +194,19 @@ func (cloud *CloudProvider) ListZones(ctx context.Context, region string) ([]str } -func (cloud *CloudProvider) ListSnapshots(ctx context.Context, filter string, maxEntries int64, pageToken string) ([]*computev1.Snapshot, string, error) { - klog.V(5).Infof("Listing snapshots with filter: %s, max entries: %v, page token: %s", filter, maxEntries, pageToken) - snapshots := []*computev1.Snapshot{} - snapshotList, err := cloud.service.Snapshots.List(cloud.project).Filter(filter).MaxResults(maxEntries).PageToken(pageToken).Do() - if err != nil { - return snapshots, "", err - } - for _, snapshot := range snapshotList.Items { - snapshots = append(snapshots, snapshot) +func (cloud *CloudProvider) ListSnapshots(ctx context.Context, filter string) ([]*computev1.Snapshot, string, error) { + klog.V(5).Infof("Listing snapshots with filter: %s", filter) + items := []*computev1.Snapshot{} + lCall := cloud.service.Snapshots.List(cloud.project).Filter(filter) + nextPageToken := "pageToken" + for nextPageToken != "" { + snapshotList, err := lCall.Do() + if err != nil { + return nil, "", err + } + items = append(items, snapshotList.Items...) } - return snapshots, snapshotList.NextPageToken, nil - + return items, "", nil } func (cloud *CloudProvider) GetDisk(ctx context.Context, project string, key *meta.Key, gceAPIVersion GCEAPIVersion) (*CloudDisk, error) { @@ -398,11 +404,23 @@ func (cloud *CloudProvider) insertRegionalDisk( Labels: params.Labels, } if snapshotID != "" { - diskToCreate.SourceSnapshot = snapshotID + _, snapshotType, _, err := common.SnapshotIDToProjectKey(snapshotID) + if err != nil { + return err + } + switch snapshotType { + case common.DiskSnapshotType: + diskToCreate.SourceSnapshot = snapshotID + case common.DiskImageType: + diskToCreate.SourceImage = snapshotID + default: + return fmt.Errorf("invalid snapshot type in snapshot ID: %s", snapshotType) + } } if volumeContentSourceVolumeID != "" { diskToCreate.SourceDisk = volumeContentSourceVolumeID } + if len(replicaZones) != 0 { diskToCreate.ReplicaZones = replicaZones } @@ -499,7 +517,18 @@ func (cloud *CloudProvider) insertZonalDisk( } if snapshotID != "" { - diskToCreate.SourceSnapshot = snapshotID + _, snapshotType, _, err := common.SnapshotIDToProjectKey(snapshotID) + if err != nil { + return err + } + switch snapshotType { + case common.DiskSnapshotType: + diskToCreate.SourceSnapshot = snapshotID + case common.DiskImageType: + diskToCreate.SourceImage = snapshotID + default: + return fmt.Errorf("invalid snapshot type in snapshot ID: %s", snapshotType) + } } if volumeContentSourceVolumeID != "" { diskToCreate.SourceDisk = volumeContentSourceVolumeID @@ -829,6 +858,94 @@ func (cloud *CloudProvider) CreateSnapshot(ctx context.Context, project string, } } +func (cloud *CloudProvider) CreateImage(ctx context.Context, project string, volKey *meta.Key, imageName string, snapshotParams common.SnapshotParameters) (*computev1.Image, error) { + klog.V(5).Infof("Creating image %s for source %v", imageName, volKey) + diskID, err := common.KeyToVolumeID(volKey, project) + if err != nil { + return nil, err + } + image := &computev1.Image{ + SourceDisk: diskID, + Family: snapshotParams.ImageFamily, + Name: imageName, + StorageLocations: snapshotParams.StorageLocations, + } + + _, err = cloud.service.Images.Insert(project, image).Context(ctx).Do() + if err != nil { + return nil, err + } + + return cloud.waitForImageCreation(ctx, project, imageName) +} + +func (cloud *CloudProvider) waitForImageCreation(ctx context.Context, project, imageName string) (*computev1.Image, error) { + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + timer := time.NewTimer(waitForImageCreationTimeOut) + defer timer.Stop() + + for { + select { + case <-ticker.C: + klog.V(6).Infof("Checking GCE Image %s.", imageName) + image, err := cloud.GetImage(ctx, project, imageName) + if err != nil { + klog.Warningf("Error in getting image %s, %v", imageName, err) + } else if image != nil { + if image.Status != "PENDING" { + klog.V(6).Infof("Image %s status is %s", imageName, image.Status) + return image, nil + } else { + klog.V(6).Infof("Image %s is still pending", imageName) + } + } + case <-timer.C: + return nil, fmt.Errorf("timeout waiting for image %s to be created", imageName) + } + } +} + +func (cloud *CloudProvider) GetImage(ctx context.Context, project, imageName string) (*computev1.Image, error) { + klog.V(5).Infof("Getting image %v", imageName) + image, err := cloud.service.Images.Get(project, imageName).Context(ctx).Do() + if err != nil { + return nil, err + } + return image, nil +} + +func (cloud *CloudProvider) ListImages(ctx context.Context, filter string) ([]*computev1.Image, string, error) { + klog.V(5).Infof("Listing images with filter: %s", filter) + var items []*computev1.Image + lCall := cloud.service.Images.List(cloud.project).Context(ctx).Filter(filter) + nextPageToken := "pageToken" + for nextPageToken != "" { + imageList, err := lCall.Do() + if err != nil { + return nil, "", err + } + items = append(items, imageList.Items...) + } + return items, "", nil +} + +func (cloud *CloudProvider) DeleteImage(ctx context.Context, project, imageName string) error { + klog.V(5).Infof("Deleting image %v", imageName) + op, err := cloud.service.Images.Delete(cloud.project, imageName).Context(ctx).Do() + if err != nil { + if IsGCEError(err, "notFound") { + return nil + } + return err + } + err = cloud.waitForGlobalOp(ctx, project, op.Name) + if err != nil { + return err + } + return nil +} + // ResizeDisk takes in the requested disk size in bytes and returns the resized // size in Gi // TODO(#461) The whole driver could benefit from standardized usage of the diff --git a/pkg/gce-pd-csi-driver/controller.go b/pkg/gce-pd-csi-driver/controller.go index 2ffa96e87..705ac41e3 100644 --- a/pkg/gce-pd-csi-driver/controller.go +++ b/pkg/gce-pd-csi-driver/controller.go @@ -25,6 +25,7 @@ import ( "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta" csi "github.com/container-storage-interface/spec/lib/go/csi" "github.com/golang/protobuf/ptypes" + "github.com/golang/protobuf/ptypes/timestamp" compute "google.golang.org/api/compute/v1" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -45,6 +46,9 @@ type GCEControllerServer struct { disks []*compute.Disk seen map[string]int + snapshots []*csi.ListSnapshotsResponse_Entry + snapshotTokens map[string]int + // A map storing all volumes with ongoing operations so that additional // operations for that same volume (as defined by Volume Key) return an // Aborted error @@ -807,19 +811,46 @@ func (gceCS *GCEControllerServer) CreateSnapshot(ctx context.Context, req *csi.C return nil, status.Error(codes.Internal, fmt.Sprintf("CreateSnapshot unknown get disk error: %v", err)) } - // Check if snapshot already exists + snapshotParams, err := common.ExtractAndDefaultSnapshotParameters(req.GetParameters()) + if err != nil { + return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("Invalid snapshot parameters: %v", err)) + } + + var snapshot *csi.Snapshot + switch snapshotParams.SnapshotType { + case common.DiskSnapshotType: + snapshot, err = gceCS.createPDSnapshot(ctx, project, volKey, req.Name, snapshotParams) + if err != nil { + return nil, err + } + case common.DiskImageType: + snapshot, err = gceCS.createImage(ctx, project, volKey, req.Name, snapshotParams) + if err != nil { + return nil, err + } + default: + return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("Invalid snapshot type: %s", snapshotParams.SnapshotType)) + } + + klog.V(4).Infof("CreateSnapshot succeeded for snapshot %s on volume %s", snapshot.SnapshotId, volumeID) + return &csi.CreateSnapshotResponse{Snapshot: snapshot}, nil +} + +func (gceCS *GCEControllerServer) createPDSnapshot(ctx context.Context, project string, volKey *meta.Key, snapshotName string, snapshotParams common.SnapshotParameters) (*csi.Snapshot, error) { + volumeID, err := common.KeyToVolumeID(volKey, project) + if err != nil { + return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("Invalid volume key: %v", volKey)) + } + + // Check if PD snapshot already exists var snapshot *compute.Snapshot - snapshot, err = gceCS.CloudProvider.GetSnapshot(ctx, project, req.Name) + snapshot, err = gceCS.CloudProvider.GetSnapshot(ctx, project, snapshotName) if err != nil { if !gce.IsGCEError(err, "notFound") { return nil, status.Error(codes.Internal, fmt.Sprintf("Unknown get snapshot error: %v", err)) } // If we could not find the snapshot, we create a new one - snapshotParams, err := common.ExtractAndDefaultSnapshotParameters(req.GetParameters()) - if err != nil { - return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("Invalid snapshot parameters: %v", err)) - } - snapshot, err = gceCS.CloudProvider.CreateSnapshot(ctx, project, volKey, req.Name, snapshotParams) + snapshot, err = gceCS.CloudProvider.CreateSnapshot(ctx, project, volKey, snapshotName, snapshotParams) if err != nil { if gce.IsGCEError(err, "notFound") { return nil, status.Error(codes.NotFound, fmt.Sprintf("Could not find volume with ID %v: %v", volKey.String(), err)) @@ -832,32 +863,122 @@ func (gceCS *GCEControllerServer) CreateSnapshot(ctx context.Context, req *csi.C if err != nil { return nil, status.Error(codes.AlreadyExists, fmt.Sprintf("Error in creating snapshot: %v", err)) } - t, err := time.Parse(time.RFC3339, snapshot.CreationTimestamp) + + timestamp, err := parseTimestamp(snapshot.CreationTimestamp) if err != nil { return nil, status.Error(codes.Internal, fmt.Sprintf("Failed to covert creation timestamp: %v", err)) } - tp, err := ptypes.TimestampProto(t) + ready, err := isCSISnapshotReady(snapshot.Status) + if err != nil { + return nil, status.Error(codes.Internal, fmt.Sprintf("Snapshot had error checking ready status: %v", err)) + } + + return &csi.Snapshot{ + SizeBytes: common.GbToBytes(snapshot.DiskSizeGb), + SnapshotId: cleanSelfLink(snapshot.SelfLink), + SourceVolumeId: volumeID, + CreationTime: timestamp, + ReadyToUse: ready, + }, nil +} + +func (gceCS *GCEControllerServer) createImage(ctx context.Context, project string, volKey *meta.Key, imageName string, snapshotParams common.SnapshotParameters) (*csi.Snapshot, error) { + volumeID, err := common.KeyToVolumeID(volKey, project) + if err != nil { + return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("Invalid volume key: %v", volKey)) + } + + // Check if image already exists + var image *compute.Image + image, err = gceCS.CloudProvider.GetImage(ctx, project, imageName) + if err != nil { + if !gce.IsGCEError(err, "notFound") { + return nil, status.Error(codes.Internal, fmt.Sprintf("Unknown get image error: %v", err)) + } + // create a new image + image, err = gceCS.CloudProvider.CreateImage(ctx, project, volKey, imageName, snapshotParams) + if err != nil { + if gce.IsGCEError(err, "notFound") { + return nil, status.Error(codes.NotFound, fmt.Sprintf("Could not find volume with ID %v: %v", volKey.String(), err)) + } + return nil, status.Error(codes.Internal, fmt.Sprintf("Unknown create image error: %v", err)) + } + } + + err = gceCS.validateExistingImage(image, volKey) + if err != nil { + return nil, status.Errorf(codes.AlreadyExists, fmt.Sprintf("Error in creating snapshot: %v", err)) + } + + timestamp, err := parseTimestamp(image.CreationTimestamp) if err != nil { return nil, status.Error(codes.Internal, fmt.Sprintf("Failed to covert creation timestamp: %v", err)) } - ready, err := isCSISnapshotReady(snapshot.Status) + ready, err := isImageReady(image.Status) if err != nil { - return nil, status.Error(codes.Internal, fmt.Sprintf("Snapshot had error checking ready status: %v", err)) + return nil, status.Errorf(codes.Internal, fmt.Sprintf("Failed to check image status: %v", err)) } - createResp := &csi.CreateSnapshotResponse{ - Snapshot: &csi.Snapshot{ - SizeBytes: common.GbToBytes(snapshot.DiskSizeGb), - SnapshotId: cleanSelfLink(snapshot.SelfLink), - SourceVolumeId: volumeID, - CreationTime: tp, - ReadyToUse: ready, - }, + return &csi.Snapshot{ + SizeBytes: common.GbToBytes(image.DiskSizeGb), + SnapshotId: cleanSelfLink(image.SelfLink), + SourceVolumeId: volumeID, + CreationTime: timestamp, + ReadyToUse: ready, + }, nil +} + +func (gceCS *GCEControllerServer) validateExistingImage(image *compute.Image, volKey *meta.Key) error { + if image == nil { + return fmt.Errorf("disk does not exist") + } + + _, sourceKey, err := common.VolumeIDToKey(cleanSelfLink(image.SourceDisk)) + if err != nil { + return fmt.Errorf("fail to get source disk key %s, %v", image.SourceDisk, err) + } + + if sourceKey.String() != volKey.String() { + return fmt.Errorf("image already exists with same name but with a different disk source %s, expected disk source %s", sourceKey.String(), volKey.String()) + } + + klog.V(5).Infof("Compatible image %s exists with source disk %s.", image.Name, image.SourceDisk) + return nil +} + +func parseTimestamp(creationTimestamp string) (*timestamp.Timestamp, error) { + t, err := time.Parse(time.RFC3339, creationTimestamp) + if err != nil { + return nil, err + } + + timestamp, err := ptypes.TimestampProto(t) + if err != nil { + return nil, err + } + return timestamp, nil +} +func isImageReady(status string) (bool, error) { + // Possible status: + // "DELETING" + // "FAILED" + // "PENDING" + // "READY" + switch status { + case "DELETING": + klog.V(4).Infof("image status is DELETING") + return true, nil + case "FAILED": + return false, fmt.Errorf("image status is FAILED") + case "PENDING": + return false, nil + case "READY": + return true, nil + default: + return false, fmt.Errorf("unknown image status %s", status) } - klog.V(4).Infof("CreateSnapshot succeeded for snapshot %s on volume %s", cleanSelfLink(snapshot.SelfLink), volumeID) - return createResp, nil } func (gceCS *GCEControllerServer) validateExistingSnapshot(snapshot *compute.Snapshot, volKey *meta.Key) error { @@ -899,7 +1020,7 @@ func (gceCS *GCEControllerServer) DeleteSnapshot(ctx context.Context, req *csi.D return nil, status.Error(codes.InvalidArgument, "DeleteSnapshot Snapshot ID must be provided") } - project, key, err := common.SnapshotIDToProjectKey(snapshotID) + project, snapshotType, key, err := common.SnapshotIDToProjectKey(snapshotID) if err != nil { // Cannot get snapshot ID from the passing request // This is a success according to the spec @@ -907,9 +1028,19 @@ func (gceCS *GCEControllerServer) DeleteSnapshot(ctx context.Context, req *csi.D return &csi.DeleteSnapshotResponse{}, nil } - err = gceCS.CloudProvider.DeleteSnapshot(ctx, project, key) - if err != nil { - return nil, status.Error(codes.Internal, fmt.Sprintf("unknown Delete snapshot error: %v", err)) + switch snapshotType { + case common.DiskSnapshotType: + err = gceCS.CloudProvider.DeleteSnapshot(ctx, project, key) + if err != nil { + return nil, status.Error(codes.Internal, fmt.Sprintf("unknown Delete snapshot error: %v", err)) + } + case common.DiskImageType: + err = gceCS.CloudProvider.DeleteImage(ctx, project, key) + if err != nil { + return nil, status.Error(codes.Internal, fmt.Sprintf("unknown Delete image error: %v", err)) + } + default: + return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("unknown snapshot type %s", snapshotType)) } return &csi.DeleteSnapshotResponse{}, nil @@ -922,7 +1053,48 @@ func (gceCS *GCEControllerServer) ListSnapshots(ctx context.Context, req *csi.Li } // case 2: no SnapshotId is set, so we return all the snapshots that satify the reqeust. - return gceCS.getSnapshots(ctx, req) + var maxEntries int = int(req.MaxEntries) + if maxEntries < 0 { + return nil, status.Error(codes.InvalidArgument, fmt.Sprintf( + "ListSnapshots got max entries request %v. GCE only supports values >0", maxEntries)) + } + + var offset int + var length int + var ok bool + var nextToken string + if req.StartingToken == "" { + snapshotList, err := gceCS.getSnapshots(ctx, req) + if err != nil { + if gce.IsGCEInvalidError(err) { + return nil, status.Error(codes.Aborted, fmt.Sprintf("ListSnapshots error with invalid request: %v", err)) + } + return nil, status.Error(codes.Internal, fmt.Sprintf("Unknown list snapshots error: %v", err)) + } + gceCS.snapshots = snapshotList + gceCS.snapshotTokens = map[string]int{} + } else { + offset, ok = gceCS.snapshotTokens[req.StartingToken] + if !ok { + return nil, status.Error(codes.Aborted, fmt.Sprintf("ListSnapshots error with invalid startingToken: %s", req.StartingToken)) + } + } + + if maxEntries == 0 { + maxEntries = len(gceCS.snapshots) + } + if maxEntries < len(gceCS.snapshots)-offset { + length = maxEntries + nextToken = string(uuid.NewUUID()) + gceCS.snapshotTokens[nextToken] = length + offset + } else { + length = len(gceCS.snapshots) - offset + } + + return &csi.ListSnapshotsResponse{ + Entries: gceCS.snapshots[offset : offset+length], + NextToken: nextToken, + }, nil } func (gceCS *GCEControllerServer) ControllerExpandVolume(ctx context.Context, req *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) { @@ -961,59 +1133,90 @@ func (gceCS *GCEControllerServer) ControllerExpandVolume(ctx context.Context, re }, nil } -func (gceCS *GCEControllerServer) getSnapshots(ctx context.Context, req *csi.ListSnapshotsRequest) (*csi.ListSnapshotsResponse, error) { +func (gceCS *GCEControllerServer) getSnapshots(ctx context.Context, req *csi.ListSnapshotsRequest) ([]*csi.ListSnapshotsResponse_Entry, error) { var snapshots []*compute.Snapshot - var nextToken string + var images []*compute.Image + var filter string var err error if len(req.GetSourceVolumeId()) != 0 { - snapshots, nextToken, err = gceCS.CloudProvider.ListSnapshots(ctx, fmt.Sprintf("sourceDisk eq .*%s$", req.SourceVolumeId), int64(req.MaxEntries), req.StartingToken) - } else { - snapshots, nextToken, err = gceCS.CloudProvider.ListSnapshots(ctx, "", int64(req.MaxEntries), req.StartingToken) + filter = fmt.Sprintf("sourceDisk eq .*%s$", req.SourceVolumeId) } + snapshots, _, err = gceCS.CloudProvider.ListSnapshots(ctx, filter) if err != nil { if gce.IsGCEError(err, "invalid") { return nil, status.Error(codes.Aborted, fmt.Sprintf("Invalid error: %v", err)) } return nil, status.Error(codes.Internal, fmt.Sprintf("Unknown list snapshot error: %v", err)) } + + images, _, err = gceCS.CloudProvider.ListImages(ctx, filter) + if err != nil { + if gce.IsGCEError(err, "invalid") { + return nil, status.Error(codes.Aborted, fmt.Sprintf("Invalid error: %v", err)) + } + return nil, status.Error(codes.Internal, fmt.Sprintf("Unknown list image error: %v", err)) + } + entries := []*csi.ListSnapshotsResponse_Entry{} for _, snapshot := range snapshots { - entry, err := generateSnapshotEntry(snapshot) + entry, err := generateDiskSnapshotEntry(snapshot) if err != nil { return nil, fmt.Errorf("failed to generate snapshot entry: %v", err) } entries = append(entries, entry) } - listSnapshotResp := &csi.ListSnapshotsResponse{ - Entries: entries, - NextToken: nextToken, + + for _, image := range images { + entry, err := generateDiskImageEntry(image) + if err != nil { + return nil, fmt.Errorf("failed to generate image entry: %v", err) + } + entries = append(entries, entry) } - return listSnapshotResp, nil + + return entries, nil } func (gceCS *GCEControllerServer) getSnapshotByID(ctx context.Context, snapshotID string) (*csi.ListSnapshotsResponse, error) { - project, key, err := common.SnapshotIDToProjectKey(snapshotID) + project, snapshotType, key, err := common.SnapshotIDToProjectKey(snapshotID) if err != nil { // Cannot get snapshot ID from the passing request klog.Warningf("invalid snapshot id format %s", snapshotID) return &csi.ListSnapshotsResponse{}, nil } - snapshot, err := gceCS.CloudProvider.GetSnapshot(ctx, project, key) - if err != nil { - if gce.IsGCEError(err, "notFound") { - // return empty list if no snapshot is found - return &csi.ListSnapshotsResponse{}, nil + var entries []*csi.ListSnapshotsResponse_Entry + switch snapshotType { + case common.DiskSnapshotType: + snapshot, err := gceCS.CloudProvider.GetSnapshot(ctx, project, key) + if err != nil { + if gce.IsGCEError(err, "notFound") { + // return empty list if no snapshot is found + return &csi.ListSnapshotsResponse{}, nil + } + return nil, status.Error(codes.Internal, fmt.Sprintf("Unknown list snapshot error: %v", err)) } - return nil, status.Error(codes.Internal, fmt.Sprintf("Unknown list snapshot error: %v", err)) - } - e, err := generateSnapshotEntry(snapshot) - if err != nil { - return nil, fmt.Errorf("failed to generate snapshot entry: %v", err) + e, err := generateDiskSnapshotEntry(snapshot) + if err != nil { + return nil, fmt.Errorf("failed to generate snapshot entry: %v", err) + } + entries = []*csi.ListSnapshotsResponse_Entry{e} + case common.DiskImageType: + image, err := gceCS.CloudProvider.GetImage(ctx, project, key) + if err != nil { + if gce.IsGCEError(err, "notFound") { + // return empty list if no snapshot is found + return &csi.ListSnapshotsResponse{}, nil + } + } + e, err := generateImageEntry(image) + if err != nil { + return nil, fmt.Errorf("failed to generate image entry: %v", err) + } + entries = []*csi.ListSnapshotsResponse_Entry{e} } - entries := []*csi.ListSnapshotsResponse_Entry{e} //entries[0] = entry listSnapshotResp := &csi.ListSnapshotsResponse{ Entries: entries, @@ -1021,7 +1224,7 @@ func (gceCS *GCEControllerServer) getSnapshotByID(ctx context.Context, snapshotI return listSnapshotResp, nil } -func generateSnapshotEntry(snapshot *compute.Snapshot) (*csi.ListSnapshotsResponse_Entry, error) { +func generateDiskSnapshotEntry(snapshot *compute.Snapshot) (*csi.ListSnapshotsResponse_Entry, error) { t, _ := time.Parse(time.RFC3339, snapshot.CreationTimestamp) tp, err := ptypes.TimestampProto(t) @@ -1046,6 +1249,49 @@ func generateSnapshotEntry(snapshot *compute.Snapshot) (*csi.ListSnapshotsRespon return entry, nil } +func generateDiskImageEntry(image *compute.Image) (*csi.ListSnapshotsResponse_Entry, error) { + t, _ := time.Parse(time.RFC3339, image.CreationTimestamp) + + tp, err := ptypes.TimestampProto(t) + if err != nil { + return nil, fmt.Errorf("Failed to covert creation timestamp: %v", err) + } + + ready, _ := isImageReady(image.Status) + + entry := &csi.ListSnapshotsResponse_Entry{ + Snapshot: &csi.Snapshot{ + SizeBytes: common.GbToBytes(image.DiskSizeGb), + SnapshotId: cleanSelfLink(image.SelfLink), + SourceVolumeId: cleanSelfLink(image.SourceDisk), + CreationTime: tp, + ReadyToUse: ready, + }, + } + return entry, nil +} + +func generateImageEntry(image *compute.Image) (*csi.ListSnapshotsResponse_Entry, error) { + timestamp, err := parseTimestamp(image.CreationTimestamp) + if err != nil { + return nil, fmt.Errorf("Failed to covert creation timestamp: %v", err) + } + + // ignore the error intentionally here since we are just listing images + ready, _ := isImageReady(image.Status) + + entry := &csi.ListSnapshotsResponse_Entry{ + Snapshot: &csi.Snapshot{ + SizeBytes: common.GbToBytes(image.DiskSizeGb), + SnapshotId: cleanSelfLink(image.SelfLink), + SourceVolumeId: cleanSelfLink(image.SourceDisk), + CreationTime: timestamp, + ReadyToUse: ready, + }, + } + return entry, nil +} + func getRequestCapacity(capRange *csi.CapacityRange) (int64, error) { var capBytes int64 // Default case where nothing is set diff --git a/pkg/gce-pd-csi-driver/controller_test.go b/pkg/gce-pd-csi-driver/controller_test.go index 14f203614..5ac0215f4 100644 --- a/pkg/gce-pd-csi-driver/controller_test.go +++ b/pkg/gce-pd-csi-driver/controller_test.go @@ -64,6 +64,7 @@ var ( region, _ = common.GetRegionFromZones([]string{zone}) testRegionalID = fmt.Sprintf("projects/%s/regions/%s/disks/%s", project, region, name) testSnapshotID = fmt.Sprintf("projects/%s/global/snapshots/%s", project, name) + testImageID = fmt.Sprintf("projects/%s/global/images/%s", project, name) testNodeID = fmt.Sprintf("projects/%s/zones/%s/instances/%s", project, zone, node) ) @@ -99,6 +100,24 @@ func TestCreateSnapshotArguments(t *testing.T) { ReadyToUse: false, }, }, + { + name: "success disk image of zonal disk", + req: &csi.CreateSnapshotRequest{ + Name: name, + SourceVolumeId: testVolumeID, + Parameters: map[string]string{common.ParameterKeyStorageLocations: " US-WEST2", common.ParameterKeySnapshotType: "images"}, + }, + seedDisks: []*gce.CloudDisk{ + createZonalCloudDisk(name), + }, + expSnapshot: &csi.Snapshot{ + SnapshotId: testImageID, + SourceVolumeId: testVolumeID, + CreationTime: tp, + SizeBytes: common.GbToBytes(gce.DiskSizeGb), + ReadyToUse: false, + }, + }, { name: "fail no name", req: &csi.CreateSnapshotRequest{ @@ -199,11 +218,17 @@ func TestDeleteSnapshot(t *testing.T) { expErrCode codes.Code }{ { - name: "valid", + name: "valid snapshot delete", req: &csi.DeleteSnapshotRequest{ SnapshotId: testSnapshotID, }, }, + { + name: "valid image delete", + req: &csi.DeleteSnapshotRequest{ + SnapshotId: testImageID, + }, + }, { name: "invalid id", req: &csi.DeleteSnapshotRequest{ @@ -246,31 +271,62 @@ func TestDeleteSnapshot(t *testing.T) { func TestListSnapshotsArguments(t *testing.T) { // Define test cases testCases := []struct { - name string - req *csi.ListSnapshotsRequest - numSnapshots int - expErrCode codes.Code + name string + req *csi.ListSnapshotsRequest + numSnapshots int + numImages int + expectedCount int + expErrCode codes.Code }{ { name: "valid", req: &csi.ListSnapshotsRequest{ SnapshotId: testSnapshotID + "0", }, - numSnapshots: 1, + numSnapshots: 3, + numImages: 2, + expectedCount: 1, }, { name: "invalid id", req: &csi.ListSnapshotsRequest{ SnapshotId: testSnapshotID + "/foo", }, - numSnapshots: 0, + expectedCount: 0, }, { name: "no id", req: &csi.ListSnapshotsRequest{ SnapshotId: "", }, - numSnapshots: 5, + numSnapshots: 2, + numImages: 3, + expectedCount: 5, + }, + { + name: "with invalid token", + req: &csi.ListSnapshotsRequest{ + StartingToken: "invalid", + }, + expectedCount: 0, + expErrCode: codes.Aborted, + }, + { + name: "negative entries", + req: &csi.ListSnapshotsRequest{ + MaxEntries: -1, + }, + + expErrCode: codes.InvalidArgument, + }, + { + name: "max enries", + req: &csi.ListSnapshotsRequest{ + MaxEntries: 4, + }, + numSnapshots: 2, + numImages: 3, + expectedCount: 4, }, } @@ -278,7 +334,7 @@ func TestListSnapshotsArguments(t *testing.T) { t.Logf("test case: %s", tc.name) disks := []*gce.CloudDisk{} - for i := 0; i < tc.numSnapshots; i++ { + for i := 0; i < tc.numSnapshots+tc.numImages; i++ { sname := fmt.Sprintf("%s%d", name, i) disks = append(disks, createZonalCloudDisk(sname)) } @@ -292,6 +348,21 @@ func TestListSnapshotsArguments(t *testing.T) { createReq := &csi.CreateSnapshotRequest{ Name: nameID, SourceVolumeId: volumeID, + Parameters: map[string]string{common.ParameterKeySnapshotType: common.DiskSnapshotType}, + } + _, err := gceDriver.cs.CreateSnapshot(context.Background(), createReq) + if err != nil { + t.Errorf("error %v", err) + } + } + + for i := 0; i < tc.numImages; i++ { + volumeID := fmt.Sprintf("%s%d", testVolumeID, i) + nameID := fmt.Sprintf("%s%d", name, i) + createReq := &csi.CreateSnapshotRequest{ + Name: nameID, + SourceVolumeId: volumeID, + Parameters: map[string]string{common.ParameterKeySnapshotType: common.DiskImageType}, } _, err := gceDriver.cs.CreateSnapshot(context.Background(), createReq) if err != nil { @@ -327,8 +398,8 @@ func TestListSnapshotsArguments(t *testing.T) { // If one is nil or empty but not both t.Fatalf("Expected snapshots number %v, got no snapshot", tc.numSnapshots) } - if len(snapshots) != tc.numSnapshots { - errStr := fmt.Sprintf("Expected snapshot: %#v\n to equal snapshot: %#v\n", snapshots[0].Snapshot, tc.numSnapshots) + if len(snapshots) != tc.expectedCount { + errStr := fmt.Sprintf("Expected snapshot number to equal: %v", tc.numSnapshots) t.Errorf(errStr) } } @@ -827,6 +898,7 @@ func TestCreateVolumeWithVolumeSourceFromSnapshot(t *testing.T) { name string project string volKey *meta.Key + snapshotType string snapshotOnCloud bool expErrCode codes.Code }{ @@ -834,12 +906,29 @@ func TestCreateVolumeWithVolumeSourceFromSnapshot(t *testing.T) { name: "success with data source of snapshot type", project: "test-project", volKey: meta.ZonalKey("my-disk", zone), + snapshotType: common.DiskSnapshotType, snapshotOnCloud: true, }, { name: "fail with data source of snapshot type that doesn't exist", project: "test-project", volKey: meta.ZonalKey("my-disk", zone), + snapshotType: common.DiskSnapshotType, + snapshotOnCloud: false, + expErrCode: codes.NotFound, + }, + { + name: "success with data source of snapshot type", + project: "test-project", + volKey: meta.ZonalKey("my-disk", zone), + snapshotType: common.DiskImageType, + snapshotOnCloud: true, + }, + { + name: "fail with data source of snapshot type that doesn't exist", + project: "test-project", + volKey: meta.ZonalKey("my-disk", zone), + snapshotType: common.DiskImageType, snapshotOnCloud: false, expErrCode: codes.NotFound, }, @@ -851,7 +940,28 @@ func TestCreateVolumeWithVolumeSourceFromSnapshot(t *testing.T) { // Setup new driver each time so no interference gceDriver := initGCEDriver(t, nil) + snapshotParams, err := common.ExtractAndDefaultSnapshotParameters(nil) + if err != nil { + t.Errorf("Got error extracting snapshot parameters: %v", err) + } + // Start Test + var snapshotID string + switch tc.snapshotType { + case common.DiskSnapshotType: + snapshotID = testSnapshotID + if tc.snapshotOnCloud { + gceDriver.cs.CloudProvider.CreateSnapshot(context.Background(), tc.project, tc.volKey, name, snapshotParams) + } + case common.DiskImageType: + snapshotID = testImageID + if tc.snapshotOnCloud { + gceDriver.cs.CloudProvider.CreateImage(context.Background(), tc.project, tc.volKey, name, snapshotParams) + } + default: + t.Errorf("Unknown snapshot type: %v", tc.snapshotType) + } + req := &csi.CreateVolumeRequest{ Name: "test-name", CapacityRange: stdCapRange, @@ -859,19 +969,12 @@ func TestCreateVolumeWithVolumeSourceFromSnapshot(t *testing.T) { VolumeContentSource: &csi.VolumeContentSource{ Type: &csi.VolumeContentSource_Snapshot{ Snapshot: &csi.VolumeContentSource_SnapshotSource{ - SnapshotId: testSnapshotID, + SnapshotId: snapshotID, }, }, }, } - if tc.snapshotOnCloud { - snapshotParams, err := common.ExtractAndDefaultSnapshotParameters(req.GetParameters()) - if err != nil { - t.Errorf("Got error extracting snapshot parameters: %v", err) - } - gceDriver.cs.CloudProvider.CreateSnapshot(context.Background(), tc.project, tc.volKey, name, snapshotParams) - } resp, err := gceDriver.cs.CreateVolume(context.Background(), req) //check response if err != nil { @@ -1717,6 +1820,7 @@ func TestVolumeOperationConcurrency(t *testing.T) { response := make(chan error) go func() { _, err := cs.CreateSnapshot(context.Background(), req) + t.Log(err) response <- err }() return response