diff --git a/pkg/common/parameters.go b/pkg/common/parameters.go index e7f0104dc..f91d915ee 100644 --- a/pkg/common/parameters.go +++ b/pkg/common/parameters.go @@ -29,6 +29,7 @@ const ( ParameterKeyLabels = "labels" ParameterKeyProvisionedIOPSOnCreate = "provisioned-iops-on-create" ParameterKeyProvisionedThroughputOnCreate = "provisioned-throughput-on-create" + ParameterAvailabilityClass = "availability-class" // Parameters for VolumeSnapshotClass ParameterKeyStorageLocations = "storage-locations" @@ -38,6 +39,10 @@ const ( DiskImageType = "images" replicationTypeNone = "none" + // Parameters for AvailabilityClass + ParameterNoAvailabilityClass = "none" + ParameterRegionalHardFailoverClass = "regional-hard-failover" + // Keys for PV and PVC parameters as reported by external-provisioner ParameterKeyPVCName = "csi.storage.k8s.io/pvc/name" ParameterKeyPVCNamespace = "csi.storage.k8s.io/pvc/namespace" @@ -83,6 +88,8 @@ type DiskParameters struct { // Values: {int64} // Default: none ProvisionedThroughputOnCreate int64 + // Default: false + ForceAttach bool } // SnapshotParameters contains normalized and defaulted parameters for snapshots @@ -155,6 +162,14 @@ func ExtractAndDefaultParameters(parameters map[string]string, driverName string return p, fmt.Errorf("parameters contain invalid provisionedThroughputOnCreate parameter: %w", err) } p.ProvisionedThroughputOnCreate = paramProvisionedThroughputOnCreate + case ParameterAvailabilityClass: + paramAvailabilityClass, err := ConvertStringToAvailabilityClass(v) + if err != nil { + return p, fmt.Errorf("parameters contain invalid availability class parameter: %w", err) + } + if paramAvailabilityClass == ParameterRegionalHardFailoverClass { + p.ForceAttach = true + } default: return p, fmt.Errorf("parameters contains invalid option %q", k) } diff --git a/pkg/common/parameters_test.go b/pkg/common/parameters_test.go index 2d1a4b6b0..acffe5a53 100644 --- a/pkg/common/parameters_test.go +++ b/pkg/common/parameters_test.go @@ -178,6 +178,27 @@ func TestExtractAndDefaultParameters(t *testing.T) { Labels: map[string]string{"key1": "value1", "label-1": "value-a", "label-2": "label-value-2"}, }, }, + { + name: "availability class parameters", + parameters: map[string]string{ParameterAvailabilityClass: ParameterRegionalHardFailoverClass}, + expectParams: DiskParameters{ + DiskType: "pd-standard", + ReplicationType: "none", + ForceAttach: true, + Tags: map[string]string{}, + Labels: map[string]string{}, + }, + }, + { + name: "no force attach parameters", + parameters: map[string]string{ParameterAvailabilityClass: ParameterNoAvailabilityClass}, + expectParams: DiskParameters{ + DiskType: "pd-standard", + ReplicationType: "none", + Tags: map[string]string{}, + Labels: map[string]string{}, + }, + }, } for _, tc := range tests { diff --git a/pkg/common/utils.go b/pkg/common/utils.go index af4e5d210..26ecb5a8d 100644 --- a/pkg/common/utils.go +++ b/pkg/common/utils.go @@ -284,6 +284,28 @@ func ConvertMiStringToInt64(str string) (int64, error) { return volumehelpers.RoundUpToMiB(quantity) } +// ConvertStringToBool converts a string to a boolean. +func ConvertStringToBool(str string) (bool, error) { + switch strings.ToLower(str) { + case "true": + return true, nil + case "false": + return false, nil + } + return false, fmt.Errorf("Unexpected boolean string %s", str) +} + +// ConvertStringToAvailabilityClass converts a string to an availability class string. +func ConvertStringToAvailabilityClass(str string) (string, error) { + switch strings.ToLower(str) { + case ParameterNoAvailabilityClass: + return ParameterNoAvailabilityClass, nil + case ParameterRegionalHardFailoverClass: + return ParameterRegionalHardFailoverClass, nil + } + return "", fmt.Errorf("Unexpected boolean string %s", str) +} + // ParseMachineType returns an extracted machineType from a URL, or empty if not found. // machineTypeUrl: Full or partial URL of the machine type resource, in the format: // diff --git a/pkg/common/utils_test.go b/pkg/common/utils_test.go index edf49e28c..0fbb5a0b0 100644 --- a/pkg/common/utils_test.go +++ b/pkg/common/utils_test.go @@ -806,6 +806,114 @@ func TestConvertMiStringToInt64(t *testing.T) { } } +func TestConvertStringToBool(t *testing.T) { + tests := []struct { + desc string + inputStr string + expected bool + expectError bool + }{ + { + desc: "valid true", + inputStr: "true", + expected: true, + expectError: false, + }, + { + desc: "valid mixed case true", + inputStr: "True", + expected: true, + expectError: false, + }, + { + desc: "valid false", + inputStr: "false", + expected: false, + expectError: false, + }, + { + desc: "valid mixed case false", + inputStr: "False", + expected: false, + expectError: false, + }, + { + desc: "invalid", + inputStr: "yes", + expected: false, + expectError: true, + }, + } + for _, tc := range tests { + t.Run(tc.desc, func(t *testing.T) { + got, err := ConvertStringToBool(tc.inputStr) + if err != nil && !tc.expectError { + t.Errorf("Got error %v converting string to bool %s; expect no error", err, tc.inputStr) + } + if err == nil && tc.expectError { + t.Errorf("Got no error converting string to bool %s; expect an error", tc.inputStr) + } + if err == nil && got != tc.expected { + t.Errorf("Got %v for converting string to bool; expect %v", got, tc.expected) + } + }) + } +} + +func TestConvertStringToAvailabilityClass(t *testing.T) { + tests := []struct { + desc string + inputStr string + expected string + expectError bool + }{ + { + desc: "valid none", + inputStr: "none", + expected: ParameterNoAvailabilityClass, + expectError: false, + }, + { + desc: "valid mixed case none", + inputStr: "None", + expected: ParameterNoAvailabilityClass, + expectError: false, + }, + { + desc: "valid failover", + inputStr: "regional-hard-failover", + expected: ParameterRegionalHardFailoverClass, + expectError: false, + }, + { + desc: "valid mixed case failover", + inputStr: "Regional-Hard-Failover", + expected: ParameterRegionalHardFailoverClass, + expectError: false, + }, + { + desc: "invalid", + inputStr: "yes", + expected: "", + expectError: true, + }, + } + for _, tc := range tests { + t.Run(tc.desc, func(t *testing.T) { + got, err := ConvertStringToAvailabilityClass(tc.inputStr) + if err != nil && !tc.expectError { + t.Errorf("Got error %v converting string to availablity class %s; expect no error", err, tc.inputStr) + } + if err == nil && tc.expectError { + t.Errorf("Got no error converting string to availablity class %s; expect an error", tc.inputStr) + } + if err == nil && got != tc.expected { + t.Errorf("Got %v for converting string to availablity class; expect %v", got, tc.expected) + } + }) + } +} + func TestParseMachineType(t *testing.T) { tests := []struct { desc string diff --git a/pkg/gce-cloud-provider/compute/fake-gce.go b/pkg/gce-cloud-provider/compute/fake-gce.go index e67311400..82a5e18fb 100644 --- a/pkg/gce-cloud-provider/compute/fake-gce.go +++ b/pkg/gce-cloud-provider/compute/fake-gce.go @@ -246,15 +246,16 @@ func (cloud *FakeCloudProvider) DeleteDisk(ctx context.Context, project string, return nil } -func (cloud *FakeCloudProvider) AttachDisk(ctx context.Context, project string, volKey *meta.Key, readWrite, diskType, instanceZone, instanceName string) error { +func (cloud *FakeCloudProvider) AttachDisk(ctx context.Context, project string, volKey *meta.Key, readWrite, diskType, instanceZone, instanceName string, forceAttach bool) error { source := cloud.GetDiskSourceURI(project, volKey) attachedDiskV1 := &computev1.AttachedDisk{ - DeviceName: volKey.Name, - Kind: diskKind, - Mode: readWrite, - Source: source, - Type: diskType, + DeviceName: volKey.Name, + Kind: diskKind, + Mode: readWrite, + Source: source, + Type: diskType, + ForceAttach: forceAttach, } instance, ok := cloud.instances[instanceName] if !ok { @@ -538,14 +539,14 @@ func (cloud *FakeBlockingCloudProvider) DetachDisk(ctx context.Context, project, return cloud.FakeCloudProvider.DetachDisk(ctx, project, deviceName, instanceZone, instanceName) } -func (cloud *FakeBlockingCloudProvider) AttachDisk(ctx context.Context, project string, volKey *meta.Key, readWrite, diskType, instanceZone, instanceName string) error { +func (cloud *FakeBlockingCloudProvider) AttachDisk(ctx context.Context, project string, volKey *meta.Key, readWrite, diskType, instanceZone, instanceName string, forceAttach bool) error { execute := make(chan Signal) cloud.ReadyToExecute <- execute val := <-execute if val.ReportError { return fmt.Errorf("force mock error for AttachDisk: volkey %s", volKey) } - return cloud.FakeCloudProvider.AttachDisk(ctx, project, volKey, readWrite, diskType, instanceZone, instanceName) + return cloud.FakeCloudProvider.AttachDisk(ctx, project, volKey, readWrite, diskType, instanceZone, instanceName, forceAttach) } func notFoundError() *googleapi.Error { diff --git a/pkg/gce-cloud-provider/compute/gce-compute.go b/pkg/gce-cloud-provider/compute/gce-compute.go index 6631855a9..82f154314 100644 --- a/pkg/gce-cloud-provider/compute/gce-compute.go +++ b/pkg/gce-cloud-provider/compute/gce-compute.go @@ -92,7 +92,7 @@ type GCECompute interface { ValidateExistingDisk(ctx context.Context, disk *CloudDisk, params common.DiskParameters, reqBytes, limBytes int64, multiWriter bool) error InsertDisk(ctx context.Context, project string, volKey *meta.Key, params common.DiskParameters, capBytes int64, capacityRange *csi.CapacityRange, replicaZones []string, snapshotID string, volumeContentSourceVolumeID string, multiWriter bool) error DeleteDisk(ctx context.Context, project string, volumeKey *meta.Key) error - AttachDisk(ctx context.Context, project string, volKey *meta.Key, readWrite, diskType, instanceZone, instanceName string) error + AttachDisk(ctx context.Context, project string, volKey *meta.Key, readWrite, diskType, instanceZone, instanceName string, forceAttach bool) error DetachDisk(ctx context.Context, project, deviceName, instanceZone, instanceName string) error GetDiskSourceURI(project string, volKey *meta.Key) string GetDiskTypeURI(project string, volKey *meta.Key, diskType string) string @@ -700,7 +700,7 @@ func (cloud *CloudProvider) deleteRegionalDisk(ctx context.Context, project, reg return nil } -func (cloud *CloudProvider) AttachDisk(ctx context.Context, project string, volKey *meta.Key, readWrite, diskType, instanceZone, instanceName string) error { +func (cloud *CloudProvider) AttachDisk(ctx context.Context, project string, volKey *meta.Key, readWrite, diskType, instanceZone, instanceName string, forceAttach bool) error { klog.V(5).Infof("Attaching disk %v to %s", volKey, instanceName) source := cloud.GetDiskSourceURI(project, volKey) @@ -714,9 +714,13 @@ func (cloud *CloudProvider) AttachDisk(ctx context.Context, project string, volK Mode: readWrite, Source: source, Type: diskType, + // This parameter is ignored in the call, the ForceAttach decorator + // (query parameter) is the important one. We'll set it in both places + // in case that behavior changes. + ForceAttach: forceAttach, } - op, err := cloud.service.Instances.AttachDisk(project, instanceZone, instanceName, attachedDiskV1).Context(ctx).Do() + op, err := cloud.service.Instances.AttachDisk(project, instanceZone, instanceName, attachedDiskV1).Context(ctx).ForceAttach(forceAttach).Do() if err != nil { return fmt.Errorf("failed cloud service attach disk call: %w", err) } diff --git a/pkg/gce-pd-csi-driver/controller.go b/pkg/gce-pd-csi-driver/controller.go index b2157960f..ff7a0858f 100644 --- a/pkg/gce-pd-csi-driver/controller.go +++ b/pkg/gce-pd-csi-driver/controller.go @@ -111,6 +111,11 @@ type locationRequirements struct { cloneReplicationType string } +// PDCSIContext is the extracted VolumeContext from controller requests. +type PDCSIContext struct { + ForceAttach bool +} + var _ csi.ControllerServer = &GCEControllerServer{} const ( @@ -130,6 +135,9 @@ const ( // but 500 is a good proxy (gives ~8KB of data per ListVolumesResponse#Entry) // See https://github.com/grpc/grpc/blob/master/include/grpc/impl/codegen/grpc_types.h#L503) maxListVolumesResponseEntries = 500 + + // Keys in the volume context. + contextForceAttach = "force-attach" ) func isDiskReady(disk *gce.CloudDisk) (bool, error) { @@ -236,6 +244,11 @@ func (gceCS *GCEControllerServer) CreateVolume(ctx context.Context, req *csi.Cre gceAPIVersion = gce.GCEAPIVersionBeta } + // Verify that the regional availability class is only used on regional disks. + if params.ForceAttach && params.ReplicationType != replicationTypeRegionalPD { + return nil, status.Errorf(codes.InvalidArgument, "invalid availabilty class for zonal disk") + } + var locationTopReq *locationRequirements if useVolumeCloning(req) { locationTopReq, err = cloningLocationRequirements(req, params.ReplicationType) @@ -309,7 +322,7 @@ func (gceCS *GCEControllerServer) CreateVolume(ctx context.Context, req *csi.Cre // If there is no validation error, immediately return success klog.V(4).Infof("CreateVolume succeeded for disk %v, it already exists and was compatible", volKey) - return generateCreateVolumeResponse(existingDisk, zones), nil + return generateCreateVolumeResponse(existingDisk, zones, params), nil } snapshotID := "" @@ -424,7 +437,7 @@ func (gceCS *GCEControllerServer) CreateVolume(ctx context.Context, req *csi.Cre } klog.V(4).Infof("CreateVolume succeeded for disk %v", volKey) - return generateCreateVolumeResponse(disk, zones), nil + return generateCreateVolumeResponse(disk, zones, params), nil } @@ -483,7 +496,7 @@ func (gceCS *GCEControllerServer) ControllerPublishVolume(ctx context.Context, r } }() // Only valid requests will be accepted - _, _, err = gceCS.validateControllerPublishVolumeRequest(ctx, req) + _, _, _, err = gceCS.validateControllerPublishVolumeRequest(ctx, req) if err != nil { return nil, err } @@ -504,32 +517,37 @@ func (gceCS *GCEControllerServer) ControllerPublishVolume(ctx context.Context, r return resp, err } -func (gceCS *GCEControllerServer) validateControllerPublishVolumeRequest(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (string, *meta.Key, error) { +func (gceCS *GCEControllerServer) validateControllerPublishVolumeRequest(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (string, *meta.Key, *PDCSIContext, error) { // Validate arguments volumeID := req.GetVolumeId() nodeID := req.GetNodeId() volumeCapability := req.GetVolumeCapability() if len(volumeID) == 0 { - return "", nil, status.Error(codes.InvalidArgument, "ControllerPublishVolume Volume ID must be provided") + return "", nil, nil, status.Error(codes.InvalidArgument, "ControllerPublishVolume Volume ID must be provided") } if len(nodeID) == 0 { - return "", nil, status.Error(codes.InvalidArgument, "ControllerPublishVolume Node ID must be provided") + return "", nil, nil, status.Error(codes.InvalidArgument, "ControllerPublishVolume Node ID must be provided") } if volumeCapability == nil { - return "", nil, status.Error(codes.InvalidArgument, "ControllerPublishVolume Volume capability must be provided") + return "", nil, nil, status.Error(codes.InvalidArgument, "ControllerPublishVolume Volume capability must be provided") } project, volKey, err := common.VolumeIDToKey(volumeID) if err != nil { - return "", nil, status.Errorf(codes.InvalidArgument, "ControllerPublishVolume volume ID is invalid: %v", err.Error()) + return "", nil, nil, status.Errorf(codes.InvalidArgument, "ControllerPublishVolume volume ID is invalid: %v", err.Error()) } // TODO(#253): Check volume capability matches for ALREADY_EXISTS if err = validateVolumeCapability(volumeCapability); err != nil { - return "", nil, status.Errorf(codes.InvalidArgument, "VolumeCapabilities is invalid: %v", err.Error()) + return "", nil, nil, status.Errorf(codes.InvalidArgument, "VolumeCapabilities is invalid: %v", err.Error()) } - return project, volKey, nil + var pdcsiContext *PDCSIContext + if pdcsiContext, err = extractVolumeContext(req.VolumeContext); err != nil { + return "", nil, nil, status.Errorf(codes.InvalidArgument, "Invalid volume context: %v", err.Error()) + } + + return project, volKey, pdcsiContext, nil } func parseMachineType(machineTypeUrl string) string { @@ -543,7 +561,7 @@ func parseMachineType(machineTypeUrl string) string { func (gceCS *GCEControllerServer) executeControllerPublishVolume(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error, string) { diskType := "" - project, volKey, err := gceCS.validateControllerPublishVolumeRequest(ctx, req) + project, volKey, pdcsiContext, err := gceCS.validateControllerPublishVolumeRequest(ctx, req) if err != nil { return nil, err, diskType } @@ -615,7 +633,7 @@ func (gceCS *GCEControllerServer) executeControllerPublishVolume(ctx context.Con if err != nil { return nil, status.Errorf(codes.InvalidArgument, "could not split nodeID: %v", err.Error()), diskType } - err = gceCS.CloudProvider.AttachDisk(ctx, project, volKey, readWrite, attachableDiskTypePersistent, instanceZone, instanceName) + err = gceCS.CloudProvider.AttachDisk(ctx, project, volKey, readWrite, attachableDiskTypePersistent, instanceZone, instanceName, pdcsiContext.ForceAttach) if err != nil { var udErr *gce.UnsupportedDiskError if errors.As(err, &udErr) { @@ -788,11 +806,6 @@ func (gceCS *GCEControllerServer) ValidateVolumeCapabilities(ctx context.Context return nil, common.LoggedError("Failed to getDisk: ", err) } - // Check Volume Context is Empty - if len(req.GetVolumeContext()) != 0 { - return generateFailedValidationMessage("VolumeContext expected to be empty but got %v", req.GetVolumeContext()), nil - } - // Check volume capabilities supported by PD. These are the same for any PD if err := validateVolumeCapabilities(req.GetVolumeCapabilities()); err != nil { return generateFailedValidationMessage("VolumeCapabilities not valid: %v", err.Error()), nil @@ -1667,7 +1680,35 @@ func getDefaultZonesInRegion(ctx context.Context, gceCS *GCEControllerServer, ex return ret, nil } -func generateCreateVolumeResponse(disk *gce.CloudDisk, zones []string) *csi.CreateVolumeResponse { +func paramsToVolumeContext(params common.DiskParameters) map[string]string { + context := map[string]string{} + if params.ForceAttach { + context[contextForceAttach] = "true" + } + if len(context) > 0 { + return context + } + return nil +} + +func extractVolumeContext(context map[string]string) (*PDCSIContext, error) { + info := &PDCSIContext{} + // Note that sidecars may inject values in the context (eg, + // csiProvisionerIdentity). So we don't validate that all keys are known. + for key, val := range context { + switch key { + case contextForceAttach: + b, err := common.ConvertStringToBool(val) + if err != nil { + return nil, fmt.Errorf("Bad volume context force attach: %v", err) + } + info.ForceAttach = b + } + } + return info, nil +} + +func generateCreateVolumeResponse(disk *gce.CloudDisk, zones []string, params common.DiskParameters) *csi.CreateVolumeResponse { tops := []*csi.Topology{} for _, zone := range zones { tops = append(tops, &csi.Topology{ @@ -1679,7 +1720,7 @@ func generateCreateVolumeResponse(disk *gce.CloudDisk, zones []string) *csi.Crea Volume: &csi.Volume{ CapacityBytes: realDiskSizeBytes, VolumeId: cleanSelfLink(disk.GetSelfLink()), - VolumeContext: nil, + VolumeContext: paramsToVolumeContext(params), AccessibleTopology: tops, }, } diff --git a/pkg/gce-pd-csi-driver/controller_test.go b/pkg/gce-pd-csi-driver/controller_test.go index d49809c77..d97815f10 100644 --- a/pkg/gce-pd-csi-driver/controller_test.go +++ b/pkg/gce-pd-csi-driver/controller_test.go @@ -3117,9 +3117,11 @@ func TestControllerUnpublishSucceedsIfNotFound(t *testing.T) { func TestControllerPublishBackoff(t *testing.T) { for desc, tc := range map[string]struct { - config *backoffDriverConfig + config *backoffDriverConfig + forceAttach bool }{ - "success": {}, + "success": {}, + "force attach": {forceAttach: true}, "missing instance": { config: &backoffDriverConfig{ mockMissingInstance: true, @@ -3148,6 +3150,12 @@ func TestControllerPublishBackoff(t *testing.T) { t.Errorf("expected no error on different unpublish, got %v", err) } + var volumeContext map[string]string + if tc.forceAttach { + volumeContext = map[string]string{ + contextForceAttach: "true", + } + } pubreq := &csi.ControllerPublishVolumeRequest{ VolumeId: testVolumeID, NodeId: testNodeID, @@ -3159,6 +3167,7 @@ func TestControllerPublishBackoff(t *testing.T) { Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, }, }, + VolumeContext: volumeContext, } // For the first 199 ms, the backoff condition is true. All controller publish request will be denied with 'Unavailable' error code. for i := 0; i < 199; i++ { @@ -3222,6 +3231,18 @@ func TestControllerPublishBackoff(t *testing.T) { t.Errorf("unexpected error") } + if tc.forceAttach { + instance, err := driver.cs.CloudProvider.GetInstanceOrError(context.Background(), zone, node) + if err != nil { + t.Fatalf("%s instance not found: %v", node, err) + } + for _, disk := range instance.Disks { + if !disk.ForceAttach { + t.Errorf("Expected %s to be force attached", disk.DeviceName) + } + } + } + // Driver is expected to remove the node key from the backoff map. t1 := driver.cs.errorBackoff.backoff.Get(string(backoffId)) if t1 != 0 { diff --git a/test/e2e/tests/multi_zone_e2e_test.go b/test/e2e/tests/multi_zone_e2e_test.go index 5149c4d81..bb54356c4 100644 --- a/test/e2e/tests/multi_zone_e2e_test.go +++ b/test/e2e/tests/multi_zone_e2e_test.go @@ -31,6 +31,14 @@ import ( remote "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/remote" ) +type verifyArgs struct { + publishDir, stageDir string +} + +type verifyFunc func(*verifyArgs) error + +type detacherFunc func() + var _ = Describe("GCE PD CSI Driver Multi-Zone", func() { BeforeEach(func() { Expect(len(testContexts)).To(BeNumerically(">", 1)) @@ -77,7 +85,7 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() { } } - Expect(len(zoneToContext)).To(Equal(2), "Must have instances in exactly 2 zones") + Expect(len(zoneToContext)).To(Equal(2), "Must have instances in 2 zones") controllerContext := zoneToContext[zones[0]] controllerClient := controllerContext.Client @@ -90,7 +98,7 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() { // Create Disk volName := testNamePrefix + string(uuid.NewUUID()) - volID, err := controllerClient.CreateVolume(volName, map[string]string{ + volume, err := controllerClient.CreateVolume(volName, map[string]string{ common.ParameterKeyReplicationType: "regional-pd", }, defaultRepdSizeGb, &csi.TopologyRequirement{ Requisite: []*csi.Topology{ @@ -121,7 +129,7 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() { defer func() { // Delete Disk - controllerClient.DeleteVolume(volID) + controllerClient.DeleteVolume(volume.VolumeId) Expect(err).To(BeNil(), "DeleteVolume failed") // Validate Disk Deleted @@ -136,22 +144,121 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() { if i >= 1 { readOnly = true } - err = testAttachWriteReadDetach(volID, volName, testContext.Instance, testContext.Client, readOnly) + err = testAttachWriteReadDetach(volume.VolumeId, volName, testContext.Instance, testContext.Client, readOnly) Expect(err).To(BeNil(), "failed volume lifecycle checks") i = i + 1 } }) -}) -type verifyArgs struct { - publishDir string -} + It("Should create a RePD instance, write to it, force-attach it to another instance, and read the same data", func() { + Expect(testContexts).NotTo(BeEmpty()) + + zoneToContext := map[string]*remote.TestContext{} + zones := []string{} + + for _, tc := range testContexts { + _, z, _ := tc.Instance.GetIdentity() + // Zone hasn't been seen before + if _, ok := zoneToContext[z]; !ok { + zoneToContext[z] = tc + zones = append(zones, z) + } + if len(zoneToContext) == 2 { + break + } + } + + Expect(len(zoneToContext)).To(Equal(2), "Must have instances in 2 zones") + + controllerContext := zoneToContext[zones[0]] + controllerClient := controllerContext.Client + controllerInstance := controllerContext.Instance -type verifyFunc func(verifyArgs) error + p, _, _ := controllerInstance.GetIdentity() + + region, err := common.GetRegionFromZones(zones) + Expect(err).To(BeNil(), "Failed to get region from zones") + + // Create Disk + volName := testNamePrefix + string(uuid.NewUUID()) + volume, err := controllerClient.CreateVolume(volName, map[string]string{ + common.ParameterKeyReplicationType: "regional-pd", + common.ParameterAvailabilityClass: common.ParameterRegionalHardFailoverClass, + }, defaultRepdSizeGb, &csi.TopologyRequirement{ + Requisite: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: zones[0]}, + }, + { + Segments: map[string]string{common.TopologyKeyZone: zones[1]}, + }, + }, + }, nil) + Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err) + + // Validate Disk Created + cloudDisk, err := computeService.RegionDisks.Get(p, region, volName).Do() + Expect(err).To(BeNil(), "Could not get disk from cloud directly") + Expect(cloudDisk.Type).To(ContainSubstring(standardDiskType)) + Expect(cloudDisk.Status).To(Equal(readyState)) + Expect(cloudDisk.SizeGb).To(Equal(defaultRepdSizeGb)) + Expect(cloudDisk.Name).To(Equal(volName)) + Expect(len(cloudDisk.ReplicaZones)).To(Equal(2)) + zonesSet := sets.NewString(zones...) + for _, replicaZone := range cloudDisk.ReplicaZones { + tokens := strings.Split(replicaZone, "/") + actualZone := tokens[len(tokens)-1] + Expect(zonesSet.Has(actualZone)).To(BeTrue(), "Expected zone %v to exist in zone set %v", actualZone, zones) + } + Expect(volume.VolumeContext).To(HaveKeyWithValue("force-attach", "true")) + + detachers := []detacherFunc{} + + defer func() { + // Perform any detaches + for _, fn := range detachers { + fn() + } + + // Delete Disk + controllerClient.DeleteVolume(volume.VolumeId) + Expect(err).To(BeNil(), "DeleteVolume failed") + + // Validate Disk Deleted + _, err = computeService.RegionDisks.Get(p, region, volName).Do() + Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") + }() + + // Attach disk to instance in the first zone. + tc0 := zoneToContext[zones[0]] + err, detacher, args := testAttachAndMount(volume.VolumeId, volName, tc0.Instance, tc0.Client, false /* useBlock */, false /* forceAttach */) + detachers = append(detachers, detacher) + Expect(err).To(BeNil(), "failed attach in zone 0") + testFileName := filepath.Join(args.publishDir, "force-attach-test") + testFileContents := "force attach test" + err = testutils.WriteFile(tc0.Instance, testFileName, testFileContents) + Expect(err).To(BeNil(), "failed write in zone 0") + _, err = tc0.Instance.SSH("sync") // Sync so force detach doesn't lose data. + Expect(err).To(BeNil(), "failed sync") + + readContents, err := testutils.ReadFile(tc0.Instance, testFileName) + Expect(err).To(BeNil(), "failed read in zone 0") + Expect(strings.TrimSpace(string(readContents))).To(BeIdenticalTo(testFileContents), "content mismatch in zone 0") + + // Now force attach to the second instance without detaching. + tc1 := zoneToContext[zones[1]] + err, detacher, args = testAttachAndMount(volume.VolumeId, volName, tc1.Instance, tc1.Client, false /* useBlock */, true /* forceAttach */) + detachers = append(detachers, detacher) + Expect(err).To(BeNil(), "failed force attach in zone 1") + readContents, err = testutils.ReadFile(tc1.Instance, testFileName) + Expect(err).To(BeNil(), "failed read in zone 1") + Expect(strings.TrimSpace(string(readContents))).To(BeIdenticalTo(testFileContents), "content mismatch in zone 1") + }) +}) func testAttachWriteReadDetach(volID string, volName string, instance *remote.InstanceInfo, client *remote.CsiClient, readOnly bool) error { var testFileContents = "test" - writeFile := func(a verifyArgs) error { + writeFile := func(a *verifyArgs) error { if !readOnly { // Write a file testFile := filepath.Join(a.publishDir, "testfile") @@ -163,7 +270,7 @@ func testAttachWriteReadDetach(volID string, volName string, instance *remote.In return nil } - verifyReadFile := func(a verifyArgs) error { + verifyReadFile := func(a *verifyArgs) error { // Read File secondTestFile := filepath.Join(a.publishDir, "testfile") readContents, err := testutils.ReadFile(instance, secondTestFile) @@ -178,23 +285,20 @@ func testAttachWriteReadDetach(volID string, volName string, instance *remote.In return testLifecycleWithVerify(volID, volName, instance, client, readOnly, false /* fs */, writeFile, verifyReadFile) } -func testLifecycleWithVerify(volID string, volName string, instance *remote.InstanceInfo, client *remote.CsiClient, readOnly, useBlock bool, firstMountVerify, secondMountVerify verifyFunc) error { - var err error - klog.Infof("Starting testAttachWriteReadDetach with volume %v node %v with readonly %v\n", volID, instance.GetNodeID(), readOnly) +func testAttachAndMount(volID string, volName string, instance *remote.InstanceInfo, client *remote.CsiClient, useBlock, forceAttach bool) (error, func(), *verifyArgs) { // Attach Disk - err = client.ControllerPublishVolume(volID, instance.GetNodeID()) + err := client.ControllerPublishVolume(volID, instance.GetNodeID(), forceAttach) if err != nil { - return fmt.Errorf("ControllerPublishVolume failed with error for disk %v on node %v: %v", volID, instance.GetNodeID(), err.Error()) + return fmt.Errorf("ControllerPublishVolume failed with error for disk %v on node %v: %v", volID, instance.GetNodeID(), err.Error()), nil, nil } - defer func() { + detach := func() { // Detach Disk err = client.ControllerUnpublishVolume(volID, instance.GetNodeID()) if err != nil { klog.Errorf("Failed to detach disk: %w", err) } - - }() + } // Stage Disk stageDir := filepath.Join("/tmp/", volName, "stage") @@ -204,12 +308,12 @@ func testLifecycleWithVerify(volID string, volName string, instance *remote.Inst err = client.NodeStageExt4Volume(volID, stageDir) } - //err = client.NodeStageExt4Volume(volID, stageDir) if err != nil { - return fmt.Errorf("NodeStageExt4Volume failed with error: %w", err) + detach() + return fmt.Errorf("NodeStageExt4Volume failed with error: %w", err), nil, nil } - defer func() { + unstageAndDetach := func() { // Unstage Disk err = client.NodeUnstageVolume(volID, stageDir) if err != nil { @@ -220,7 +324,9 @@ func testLifecycleWithVerify(volID string, volName string, instance *remote.Inst if err != nil { klog.Errorf("Failed to rm file path %s: %w", fp, err) } - }() + + detach() + } // Mount Disk publishDir := filepath.Join("/tmp/", volName, "mount") @@ -232,24 +338,38 @@ func testLifecycleWithVerify(volID string, volName string, instance *remote.Inst } if err != nil { - return fmt.Errorf("NodePublishVolume failed with error: %v", err.Error()) + unstageAndDetach() + return fmt.Errorf("NodePublishVolume failed with error: %v", err.Error()), nil, nil } err = testutils.ForceChmod(instance, filepath.Join("/tmp/", volName), "777") if err != nil { - return fmt.Errorf("Chmod failed with error: %v", err.Error()) + unstageAndDetach() + return fmt.Errorf("Chmod failed with error: %v", err.Error()), nil, nil } - a := verifyArgs{ + args := &verifyArgs{ publishDir: publishDir, + stageDir: stageDir, + } + + return nil, unstageAndDetach, args +} + +func testLifecycleWithVerify(volID string, volName string, instance *remote.InstanceInfo, client *remote.CsiClient, readOnly, useBlock bool, firstMountVerify, secondMountVerify verifyFunc) error { + klog.Infof("Starting testAttachWriteReadDetach with volume %v node %v with readonly %v\n", volID, instance.GetNodeID(), readOnly) + err, detacher, args := testAttachAndMount(volID, volName, instance, client, useBlock, false /* forceAttach */) + if err != nil { + return fmt.Errorf("failed to attach and mount: %w", err) } + defer detacher() - err = firstMountVerify(a) + err = firstMountVerify(args) if err != nil { - return fmt.Errorf("failed to verify after first mount to %s: %v", publishDir, err) + return fmt.Errorf("failed to verify after first mount to %s: %w", args.publishDir, err) } // Unmount Disk - err = client.NodeUnpublishVolume(volID, publishDir) + err = client.NodeUnpublishVolume(volID, args.publishDir) if err != nil { return fmt.Errorf("NodeUnpublishVolume failed with error: %v", err.Error()) } @@ -258,9 +378,9 @@ func testLifecycleWithVerify(volID string, volName string, instance *remote.Inst // Mount disk somewhere else secondPublishDir := filepath.Join("/tmp/", volName, "secondmount") if useBlock { - err = client.NodePublishBlockVolume(volID, stageDir, secondPublishDir) + err = client.NodePublishBlockVolume(volID, args.stageDir, secondPublishDir) } else { - err = client.NodePublishVolume(volID, stageDir, secondPublishDir) + err = client.NodePublishVolume(volID, args.stageDir, secondPublishDir) } if err != nil { return fmt.Errorf("NodePublishVolume failed with error: %v", err.Error()) @@ -273,9 +393,9 @@ func testLifecycleWithVerify(volID string, volName string, instance *remote.Inst b := verifyArgs{ publishDir: secondPublishDir, } - err = secondMountVerify(b) + err = secondMountVerify(&b) if err != nil { - return fmt.Errorf("failed to verify after second mount to %s: %v", publishDir, err.Error()) + return fmt.Errorf("failed to verify after second mount to %s: %v", args.publishDir, err.Error()) } // Unmount Disk diff --git a/test/e2e/tests/resize_e2e_test.go b/test/e2e/tests/resize_e2e_test.go index f6126475c..2b04047d4 100644 --- a/test/e2e/tests/resize_e2e_test.go +++ b/test/e2e/tests/resize_e2e_test.go @@ -38,7 +38,7 @@ var _ = Describe("GCE PD CSI Driver", func() { // Create Disk volName := testNamePrefix + string(uuid.NewUUID()) - volID, err := client.CreateVolume(volName, nil, defaultSizeGb, + volume, err := client.CreateVolume(volName, nil, defaultSizeGb, &csi.TopologyRequirement{ Requisite: []*csi.Topology{ { @@ -58,7 +58,7 @@ var _ = Describe("GCE PD CSI Driver", func() { defer func() { // Delete Disk - client.DeleteVolume(volID) + client.DeleteVolume(volume.VolumeId) Expect(err).To(BeNil(), "DeleteVolume failed") // Validate Disk Deleted @@ -67,12 +67,12 @@ var _ = Describe("GCE PD CSI Driver", func() { }() // Attach Disk - err = client.ControllerPublishVolume(volID, instance.GetNodeID()) + err = client.ControllerPublishVolume(volume.VolumeId, instance.GetNodeID(), false /* forceAttach */) Expect(err).To(BeNil(), "Controller publish volume failed") defer func() { // Detach Disk - err = client.ControllerUnpublishVolume(volID, instance.GetNodeID()) + err = client.ControllerUnpublishVolume(volume.VolumeId, instance.GetNodeID()) if err != nil { klog.Errorf("Failed to detach disk: %w", err) } @@ -80,12 +80,12 @@ var _ = Describe("GCE PD CSI Driver", func() { // Stage Disk stageDir := filepath.Join("/tmp/", volName, "stage") - err = client.NodeStageExt4Volume(volID, stageDir) + err = client.NodeStageExt4Volume(volume.VolumeId, stageDir) Expect(err).To(BeNil(), "Node Stage volume failed") defer func() { // Unstage Disk - err = client.NodeUnstageVolume(volID, stageDir) + err = client.NodeUnstageVolume(volume.VolumeId, stageDir) if err != nil { klog.Errorf("Failed to unstage volume: %w", err) } @@ -98,12 +98,12 @@ var _ = Describe("GCE PD CSI Driver", func() { // Mount Disk publishDir := filepath.Join("/tmp/", volName, "mount") - err = client.NodePublishVolume(volID, stageDir, publishDir) + err = client.NodePublishVolume(volume.VolumeId, stageDir, publishDir) Expect(err).To(BeNil(), "Node publish volume failed") defer func() { // Unmount Disk - err = client.NodeUnpublishVolume(volID, publishDir) + err = client.NodeUnpublishVolume(volume.VolumeId, publishDir) if err != nil { klog.Errorf("NodeUnpublishVolume failed with error: %w", err) } @@ -116,7 +116,7 @@ var _ = Describe("GCE PD CSI Driver", func() { // Resize controller var newSizeGb int64 = 10 - err = client.ControllerExpandVolume(volID, newSizeGb) + err = client.ControllerExpandVolume(volume.VolumeId, newSizeGb) Expect(err).To(BeNil(), "Controller expand volume failed") @@ -126,7 +126,7 @@ var _ = Describe("GCE PD CSI Driver", func() { Expect(cloudDisk.SizeGb).To(Equal(newSizeGb)) // Resize node - _, err = client.NodeExpandVolume(volID, publishDir, newSizeGb) + _, err = client.NodeExpandVolume(volume.VolumeId, publishDir, newSizeGb) Expect(err).To(BeNil(), "Node expand volume failed") // Verify disk size @@ -145,7 +145,7 @@ var _ = Describe("GCE PD CSI Driver", func() { // Create Disk volName := testNamePrefix + string(uuid.NewUUID()) - volID, err := client.CreateVolume(volName, nil, defaultSizeGb, + volume, err := client.CreateVolume(volName, nil, defaultSizeGb, &csi.TopologyRequirement{ Requisite: []*csi.Topology{ { @@ -165,7 +165,7 @@ var _ = Describe("GCE PD CSI Driver", func() { defer func() { // Delete Disk - client.DeleteVolume(volID) + client.DeleteVolume(volume.VolumeId) Expect(err).To(BeNil(), "DeleteVolume failed") // Validate Disk Deleted @@ -174,12 +174,12 @@ var _ = Describe("GCE PD CSI Driver", func() { }() // Volume should be attached/formatted/mounted/unmounted/detached - err = testAttachWriteReadDetach(volID, volName, instance, client, false /* readOnly */) + err = testAttachWriteReadDetach(volume.VolumeId, volName, instance, client, false /* readOnly */) Expect(err).To(BeNil(), "Failed to go through volume lifecycle") // Resize controller var newSizeGb int64 = 10 - err = client.ControllerExpandVolume(volID, newSizeGb) + err = client.ControllerExpandVolume(volume.VolumeId, newSizeGb) Expect(err).To(BeNil(), "Controller expand volume failed") @@ -189,12 +189,12 @@ var _ = Describe("GCE PD CSI Driver", func() { Expect(cloudDisk.SizeGb).To(Equal(newSizeGb)) // Attach and mount again - err = client.ControllerPublishVolume(volID, instance.GetNodeID()) + err = client.ControllerPublishVolume(volume.VolumeId, instance.GetNodeID(), false /* forceAttach */) Expect(err).To(BeNil(), "Controller publish volume failed") defer func() { // Detach Disk - err = client.ControllerUnpublishVolume(volID, instance.GetNodeID()) + err = client.ControllerUnpublishVolume(volume.VolumeId, instance.GetNodeID()) if err != nil { klog.Errorf("Failed to detach disk: %w", err) } @@ -203,12 +203,12 @@ var _ = Describe("GCE PD CSI Driver", func() { // Stage Disk stageDir := filepath.Join("/tmp/", volName, "stage") - err = client.NodeStageExt4Volume(volID, stageDir) + err = client.NodeStageExt4Volume(volume.VolumeId, stageDir) Expect(err).To(BeNil(), "Node Stage volume failed") defer func() { // Unstage Disk - err = client.NodeUnstageVolume(volID, stageDir) + err = client.NodeUnstageVolume(volume.VolumeId, stageDir) if err != nil { klog.Errorf("Failed to unstage volume: %w", err) } @@ -221,19 +221,19 @@ var _ = Describe("GCE PD CSI Driver", func() { // Mount Disk publishDir := filepath.Join("/tmp/", volName, "mount") - err = client.NodePublishVolume(volID, stageDir, publishDir) + err = client.NodePublishVolume(volume.VolumeId, stageDir, publishDir) Expect(err).To(BeNil(), "Node publish volume failed") defer func() { // Unmount Disk - err = client.NodeUnpublishVolume(volID, publishDir) + err = client.NodeUnpublishVolume(volume.VolumeId, publishDir) if err != nil { klog.Errorf("NodeUnpublishVolume failed with error: %w", err) } }() // Resize node - _, err = client.NodeExpandVolume(volID, publishDir, newSizeGb) + _, err = client.NodeExpandVolume(volume.VolumeId, publishDir, newSizeGb) Expect(err).To(BeNil(), "Node expand volume failed") // Verify disk size @@ -252,7 +252,7 @@ var _ = Describe("GCE PD CSI Driver", func() { // Create Disk volName := testNamePrefix + string(uuid.NewUUID()) - volID, err := client.CreateVolume(volName, nil, defaultSizeGb, + volume, err := client.CreateVolume(volName, nil, defaultSizeGb, &csi.TopologyRequirement{ Requisite: []*csi.Topology{ { @@ -272,7 +272,7 @@ var _ = Describe("GCE PD CSI Driver", func() { defer func() { // Delete Disk - client.DeleteVolume(volID) + client.DeleteVolume(volume.VolumeId) Expect(err).To(BeNil(), "DeleteVolume failed") // Validate Disk Deleted @@ -281,12 +281,12 @@ var _ = Describe("GCE PD CSI Driver", func() { }() // Attach Disk - err = client.ControllerPublishVolume(volID, instance.GetNodeID()) + err = client.ControllerPublishVolume(volume.VolumeId, instance.GetNodeID(), false /* forceAttach */) Expect(err).To(BeNil(), "Controller publish volume failed") defer func() { // Detach Disk - err = client.ControllerUnpublishVolume(volID, instance.GetNodeID()) + err = client.ControllerUnpublishVolume(volume.VolumeId, instance.GetNodeID()) if err != nil { klog.Errorf("Failed to detach disk: %w", err) } @@ -295,12 +295,12 @@ var _ = Describe("GCE PD CSI Driver", func() { // Stage Disk stageDir := filepath.Join("/tmp/", volName, "stage") - err = client.NodeStageBlockVolume(volID, stageDir) + err = client.NodeStageBlockVolume(volume.VolumeId, stageDir) Expect(err).To(BeNil(), "Node Stage volume failed") defer func() { // Unstage Disk - err = client.NodeUnstageVolume(volID, stageDir) + err = client.NodeUnstageVolume(volume.VolumeId, stageDir) if err != nil { klog.Errorf("Failed to unstage volume: %w", err) } @@ -313,12 +313,12 @@ var _ = Describe("GCE PD CSI Driver", func() { // Mount Disk publishDir := filepath.Join("/tmp/", volName, "mount") - err = client.NodePublishBlockVolume(volID, stageDir, publishDir) + err = client.NodePublishBlockVolume(volume.VolumeId, stageDir, publishDir) Expect(err).To(BeNil(), "Node publish volume failed") defer func() { // Unmount Disk - err = client.NodeUnpublishVolume(volID, publishDir) + err = client.NodeUnpublishVolume(volume.VolumeId, publishDir) if err != nil { klog.Errorf("NodeUnpublishVolume failed with error: %w", err) } @@ -331,7 +331,7 @@ var _ = Describe("GCE PD CSI Driver", func() { // Resize controller var newSizeGb int64 = 10 - err = client.ControllerExpandVolume(volID, newSizeGb) + err = client.ControllerExpandVolume(volume.VolumeId, newSizeGb) Expect(err).To(BeNil(), "Controller expand volume failed") @@ -341,7 +341,7 @@ var _ = Describe("GCE PD CSI Driver", func() { Expect(cloudDisk.SizeGb).To(Equal(newSizeGb)) // Resize node - resp, err := client.NodeExpandVolume(volID, publishDir, newSizeGb) + resp, err := client.NodeExpandVolume(volume.VolumeId, publishDir, newSizeGb) Expect(err).To(BeNil(), "Node expand volume failed") Expect(resp.CapacityBytes).To(Equal(common.GbToBytes(newSizeGb)), "Node expand should not do anything") diff --git a/test/e2e/tests/single_zone_e2e_test.go b/test/e2e/tests/single_zone_e2e_test.go index 960757995..c115d3e19 100644 --- a/test/e2e/tests/single_zone_e2e_test.go +++ b/test/e2e/tests/single_zone_e2e_test.go @@ -113,7 +113,7 @@ var _ = Describe("GCE PD CSI Driver", func() { }() // Attach Disk - err := client.ControllerPublishVolume(volID, instance.GetNodeID()) + err := client.ControllerPublishVolume(volID, instance.GetNodeID(), false /* forceAttach */) Expect(err).To(BeNil(), "ControllerPublishVolume failed with error for disk %v on node %v: %v", volID, instance.GetNodeID()) defer func() { @@ -185,7 +185,7 @@ var _ = Describe("GCE PD CSI Driver", func() { }() // Attach Disk - err := client.ControllerPublishVolume(volID, instance.GetNodeID()) + err := client.ControllerPublishVolume(volID, instance.GetNodeID(), false /* forceAttach */) Expect(err).To(BeNil(), "ControllerPublishVolume failed with error for disk %v on node %v: %v", volID, instance.GetNodeID()) defer func() { @@ -251,10 +251,10 @@ var _ = Describe("GCE PD CSI Driver", func() { }, }, } - volID, err := testContext.Client.CreateVolume(volName, nil, defaultSizeGb, topReq, nil) + volume, err := testContext.Client.CreateVolume(volName, nil, defaultSizeGb, topReq, nil) Expect(err).To(BeNil(), "Failed to create volume") defer func() { - err = testContext.Client.DeleteVolume(volID) + err = testContext.Client.DeleteVolume(volume.VolumeId) Expect(err).To(BeNil(), "Failed to delete volume") }() @@ -324,7 +324,7 @@ var _ = Describe("GCE PD CSI Driver", func() { }() // Attach Disk - err := client.ControllerPublishVolume(underSpecifiedID, instance.GetNodeID()) + err := client.ControllerPublishVolume(underSpecifiedID, instance.GetNodeID(), false /* forceAttach */) Expect(err).To(BeNil(), "ControllerPublishVolume failed") }, Entry("on pd-standard", standardDiskType), @@ -345,7 +345,7 @@ var _ = Describe("GCE PD CSI Driver", func() { // Create Disk volName := testNamePrefix + string(uuid.NewUUID()) - volID, err := controllerClient.CreateVolume(volName, map[string]string{ + volume, err := controllerClient.CreateVolume(volName, map[string]string{ common.ParameterKeyReplicationType: "regional-pd", }, defaultRepdSizeGb, nil, nil) Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err) @@ -366,7 +366,7 @@ var _ = Describe("GCE PD CSI Driver", func() { } defer func() { // Delete Disk - controllerClient.DeleteVolume(volID) + controllerClient.DeleteVolume(volume.VolumeId) Expect(err).To(BeNil(), "DeleteVolume failed") // Validate Disk Deleted @@ -386,7 +386,7 @@ var _ = Describe("GCE PD CSI Driver", func() { // Create Disk disk := typeToDisk[diskType] volName := testNamePrefix + string(uuid.NewUUID()) - volID, err := client.CreateVolume(volName, disk.params, defaultSizeGb, nil, nil) + volume, err := client.CreateVolume(volName, disk.params, defaultSizeGb, nil, nil) Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err) // Validate Disk Created @@ -399,7 +399,7 @@ var _ = Describe("GCE PD CSI Driver", func() { defer func() { // Delete Disk - client.DeleteVolume(volID) + client.DeleteVolume(volume.VolumeId) Expect(err).To(BeNil(), "DeleteVolume failed") // Validate Disk Deleted @@ -425,7 +425,7 @@ var _ = Describe("GCE PD CSI Driver", func() { params := merge(disk.params, map[string]string{ common.ParameterKeyLabels: "key1=value1,key2=value2", }) - volID, err := client.CreateVolume(volName, params, defaultSizeGb, nil, nil) + volume, err := client.CreateVolume(volName, params, defaultSizeGb, nil, nil) Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err) // Validate Disk Created @@ -444,7 +444,7 @@ var _ = Describe("GCE PD CSI Driver", func() { defer func() { // Delete Disk - err := client.DeleteVolume(volID) + err := client.DeleteVolume(volume.VolumeId) Expect(err).To(BeNil(), "DeleteVolume failed") // Validate Disk Deleted @@ -547,7 +547,7 @@ var _ = Describe("GCE PD CSI Driver", func() { }, }, } - volID, err := controllerClient.CreateVolume(volName, params, defaultSizeGb, topology, nil) + volume, err := controllerClient.CreateVolume(volName, params, defaultSizeGb, topology, nil) Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err) // Validate Disk Created @@ -560,7 +560,7 @@ var _ = Describe("GCE PD CSI Driver", func() { defer func() { // Delete Disk - err = controllerClient.DeleteVolume(volID) + err = controllerClient.DeleteVolume(volume.VolumeId) Expect(err).To(BeNil(), "DeleteVolume failed") // Validate Disk Deleted @@ -569,7 +569,7 @@ var _ = Describe("GCE PD CSI Driver", func() { }() // Test disk works - err = testAttachWriteReadDetach(volID, volName, controllerInstance, controllerClient, false /* readOnly */) + err = testAttachWriteReadDetach(volume.VolumeId, volName, controllerInstance, controllerClient, false /* readOnly */) Expect(err).To(BeNil(), "Failed to go through volume lifecycle before revoking CMEK key") // Revoke CMEK key @@ -590,7 +590,7 @@ var _ = Describe("GCE PD CSI Driver", func() { } // Make sure attach of PD fails - err = testAttachWriteReadDetach(volID, volName, controllerInstance, controllerClient, false /* readOnly */) + err = testAttachWriteReadDetach(volume.VolumeId, volName, controllerInstance, controllerClient, false /* readOnly */) Expect(err).ToNot(BeNil(), "Volume lifecycle should have failed, but succeeded") // Restore CMEK key @@ -611,7 +611,7 @@ var _ = Describe("GCE PD CSI Driver", func() { // The controller publish failure in above step would set a backoff condition on the node. Wait suffcient amount of time for the driver to accept new controller publish requests. time.Sleep(time.Second) // Make sure attach of PD succeeds - err = testAttachWriteReadDetach(volID, volName, controllerInstance, controllerClient, false /* readOnly */) + err = testAttachWriteReadDetach(volume.VolumeId, volName, controllerInstance, controllerClient, false /* readOnly */) Expect(err).To(BeNil(), "Failed to go through volume lifecycle after restoring CMEK key") }, Entry("on pd-standard", standardDiskType), @@ -634,7 +634,7 @@ var _ = Describe("GCE PD CSI Driver", func() { defer deleteVolumeOrError(client, secondVolID) // Attach volID to current instance - err := client.ControllerPublishVolume(volID, nodeID) + err := client.ControllerPublishVolume(volID, nodeID, false /* forceAttach */) Expect(err).To(BeNil(), "Failed ControllerPublishVolume") defer client.ControllerUnpublishVolume(volID, nodeID) @@ -662,7 +662,7 @@ var _ = Describe("GCE PD CSI Driver", func() { // Create Disk volName := testNamePrefix + string(uuid.NewUUID()) - volID, err := controllerClient.CreateVolume(volName, map[string]string{ + volume, err := controllerClient.CreateVolume(volName, map[string]string{ common.ParameterKeyReplicationType: "regional-pd", }, defaultRepdSizeGb, nil, nil) Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err) @@ -684,7 +684,7 @@ var _ = Describe("GCE PD CSI Driver", func() { // Create Snapshot snapshotName := testNamePrefix + string(uuid.NewUUID()) - snapshotID, err := controllerClient.CreateSnapshot(snapshotName, volID, nil) + snapshotID, err := controllerClient.CreateSnapshot(snapshotName, volume.VolumeId, nil) Expect(err).To(BeNil(), "CreateSnapshot failed with error: %v", err) // Validate Snapshot Created @@ -704,7 +704,7 @@ var _ = Describe("GCE PD CSI Driver", func() { defer func() { // Delete Disk - err := controllerClient.DeleteVolume(volID) + err := controllerClient.DeleteVolume(volume.VolumeId) Expect(err).To(BeNil(), "DeleteVolume failed") // Validate Disk Deleted @@ -740,7 +740,7 @@ var _ = Describe("GCE PD CSI Driver", func() { Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") }() - verifyVolumeStats := func(a verifyArgs) error { + verifyVolumeStats := func(a *verifyArgs) error { available, capacity, used, inodesFree, inodes, inodesUsed, err := client.NodeGetVolumeStats(volID, a.publishDir) if err != nil { return fmt.Errorf("failed to get node volume stats: %v", err.Error()) @@ -777,7 +777,7 @@ var _ = Describe("GCE PD CSI Driver", func() { Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") }() - verifyVolumeStats := func(a verifyArgs) error { + verifyVolumeStats := func(a *verifyArgs) error { available, capacity, used, inodesFree, inodes, inodesUsed, err := client.NodeGetVolumeStats(volID, a.publishDir) if err != nil { return fmt.Errorf("failed to get node volume stats: %v", err.Error()) @@ -843,14 +843,14 @@ var _ = Describe("GCE PD CSI Driver", func() { // Attach Disk testFileContents := "test" - writeFunc := func(a verifyArgs) error { + writeFunc := func(a *verifyArgs) error { err := testutils.WriteBlock(instance, a.publishDir, testFileContents) if err != nil { return fmt.Errorf("Failed to write file: %v", err.Error()) } return nil } - verifyReadFunc := func(a verifyArgs) error { + verifyReadFunc := func(a *verifyArgs) error { readContents, err := testutils.ReadBlock(instance, a.publishDir, len(testFileContents)) if err != nil { return fmt.Errorf("ReadFile failed with error: %v", err.Error()) @@ -882,7 +882,7 @@ var _ = Describe("GCE PD CSI Driver", func() { common.ParameterKeyPVCNamespace: "test-pvc-namespace", common.ParameterKeyPVName: "test-pv-name", }) - volID, err := controllerClient.CreateVolume(volName, params, defaultSizeGb, nil /* topReq */, nil) + volume, err := controllerClient.CreateVolume(volName, params, defaultSizeGb, nil /* topReq */, nil) Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err) // Validate Disk Created @@ -896,7 +896,7 @@ var _ = Describe("GCE PD CSI Driver", func() { defer func() { // Delete Disk - controllerClient.DeleteVolume(volID) + controllerClient.DeleteVolume(volume.VolumeId) Expect(err).To(BeNil(), "DeleteVolume failed") // Validate Disk Deleted @@ -1042,7 +1042,7 @@ var _ = Describe("GCE PD CSI Driver", func() { // Create Disk volName := testNamePrefix + string(uuid.NewUUID()) - volID, err := controllerClient.CreateVolume(volName, map[string]string{ + volume, err := controllerClient.CreateVolume(volName, map[string]string{ common.ParameterKeyReplicationType: "none", }, defaultSizeGb, &csi.TopologyRequirement{ @@ -1075,7 +1075,7 @@ var _ = Describe("GCE PD CSI Driver", func() { Expect(zoneFromURL(cloudDisk.Zone)).To(Equal(srcKey.Zone)) defer func() { // Delete Disk - controllerClient.DeleteVolume(volID) + controllerClient.DeleteVolume(volume.VolumeId) Expect(err).To(BeNil(), "DeleteVolume failed") // Validate Disk Deleted @@ -1098,18 +1098,18 @@ var _ = Describe("GCE PD CSI Driver", func() { // Create Source Disk srcVolName := testNamePrefix + string(uuid.NewUUID()) - srcVolID, err := controllerClient.CreateVolume(srcVolName, map[string]string{ + srcVolume, err := controllerClient.CreateVolume(srcVolName, map[string]string{ common.ParameterKeyReplicationType: "none", }, defaultRepdSizeGb, nil, nil) // Create Disk volName := testNamePrefix + string(uuid.NewUUID()) - volID, err := controllerClient.CreateVolume(volName, map[string]string{ + volume, err := controllerClient.CreateVolume(volName, map[string]string{ common.ParameterKeyReplicationType: "regional-pd", }, defaultRepdSizeGb, nil, &csi.VolumeContentSource{ Type: &csi.VolumeContentSource_Volume{ Volume: &csi.VolumeContentSource_VolumeSource{ - VolumeId: srcVolID, + VolumeId: srcVolume.VolumeId, }, }, }) @@ -1125,7 +1125,7 @@ var _ = Describe("GCE PD CSI Driver", func() { Expect(cloudDisk.Name).To(Equal(volName)) Expect(len(cloudDisk.ReplicaZones)).To(Equal(2)) replicaZonesCompatible := false - _, srcKey, err := common.VolumeIDToKey(srcVolID) + _, srcKey, err := common.VolumeIDToKey(srcVolume.VolumeId) Expect(err).To(BeNil(), "Could not get source volume key from id") for _, replicaZone := range cloudDisk.ReplicaZones { actualZone := zoneFromURL(replicaZone) @@ -1140,7 +1140,7 @@ var _ = Describe("GCE PD CSI Driver", func() { Expect(replicaZonesCompatible).To(Equal(true)) defer func() { // Delete Disk - controllerClient.DeleteVolume(volID) + controllerClient.DeleteVolume(volume.VolumeId) Expect(err).To(BeNil(), "DeleteVolume failed") // Validate Disk Deleted @@ -1163,18 +1163,18 @@ var _ = Describe("GCE PD CSI Driver", func() { // Create Source Disk srcVolName := testNamePrefix + string(uuid.NewUUID()) - srcVolID, err := controllerClient.CreateVolume(srcVolName, map[string]string{ + srcVolume, err := controllerClient.CreateVolume(srcVolName, map[string]string{ common.ParameterKeyReplicationType: "regional-pd", }, defaultRepdSizeGb, nil, nil) // Create Disk volName := testNamePrefix + string(uuid.NewUUID()) - volID, err := controllerClient.CreateVolume(volName, map[string]string{ + volume, err := controllerClient.CreateVolume(volName, map[string]string{ common.ParameterKeyReplicationType: "regional-pd", }, defaultRepdSizeGb, nil, &csi.VolumeContentSource{ Type: &csi.VolumeContentSource_Volume{ Volume: &csi.VolumeContentSource_VolumeSource{ - VolumeId: srcVolID, + VolumeId: srcVolume.VolumeId, }, }, }) @@ -1201,7 +1201,7 @@ var _ = Describe("GCE PD CSI Driver", func() { } defer func() { // Delete Disk - controllerClient.DeleteVolume(volID) + controllerClient.DeleteVolume(volume.VolumeId) Expect(err).To(BeNil(), "DeleteVolume failed") // Validate Disk Deleted @@ -1256,7 +1256,7 @@ func createAndValidateUniqueZonalDisk(client *remote.CsiClient, project, zone st // Create Disk disk := typeToDisk[diskType] volName := testNamePrefix + string(uuid.NewUUID()) - volID, err := client.CreateVolume(volName, disk.params, defaultSizeGb, + volume, err := client.CreateVolume(volName, disk.params, defaultSizeGb, &csi.TopologyRequirement{ Requisite: []*csi.Topology{ { @@ -1274,7 +1274,7 @@ func createAndValidateUniqueZonalDisk(client *remote.CsiClient, project, zone st Expect(cloudDisk.Name).To(Equal(volName)) disk.validate(cloudDisk) - return volName, volID + return volName, volume.VolumeId } func deleteVolumeOrError(client *remote.CsiClient, volID string) { @@ -1293,7 +1293,7 @@ func createAndValidateUniqueZonalMultiWriterDisk(client *remote.CsiClient, proje // Create Disk disk := typeToDisk[diskType] volName := testNamePrefix + string(uuid.NewUUID()) - volID, err := client.CreateVolumeWithCaps(volName, disk.params, defaultMwSizeGb, + volume, err := client.CreateVolumeWithCaps(volName, disk.params, defaultMwSizeGb, &csi.TopologyRequirement{ Requisite: []*csi.Topology{ { @@ -1325,7 +1325,7 @@ func createAndValidateUniqueZonalMultiWriterDisk(client *remote.CsiClient, proje Expect(err).To(BeNil(), "Failed to get cloud disk using alpha API") Expect(alphaDisk.MultiWriter).To(Equal(true)) - return volName, volID + return volName, volume.VolumeId } func cleanSelfLink(selfLink string) string { diff --git a/test/e2e/utils/utils.go b/test/e2e/utils/utils.go index 8a80501e1..f57c09d9d 100644 --- a/test/e2e/utils/utils.go +++ b/test/e2e/utils/utils.go @@ -59,7 +59,9 @@ func GCEClientAndDriverSetup(instance *remote.InstanceInfo, computeEndpoint stri } workspace := remote.NewWorkspaceDir("gce-pd-e2e-") - driverRunCmd := fmt.Sprintf("sh -c '/usr/bin/nohup %s/gce-pd-csi-driver -v=4 --endpoint=%s %s --extra-labels=%s=%s 2> %s/prog.out < /dev/null > /dev/null &'", + // Log at V(6) as the compute API calls are emitted at that level and it's + // useful to see what's happening when debugging tests. + driverRunCmd := fmt.Sprintf("sh -c '/usr/bin/nohup %s/gce-pd-csi-driver -v=6 --endpoint=%s %s --extra-labels=%s=%s 2> %s/prog.out < /dev/null > /dev/null &'", workspace, endpoint, computeFlag, DiskLabelKey, DiskLabelValue, workspace) config := &remote.ClientConfig{ diff --git a/test/remote/client-wrappers.go b/test/remote/client-wrappers.go index 748c09183..7c86e657d 100644 --- a/test/remote/client-wrappers.go +++ b/test/remote/client-wrappers.go @@ -99,7 +99,7 @@ func (c *CsiClient) CloseConn() error { return c.conn.Close() } -func (c *CsiClient) CreateVolumeWithCaps(volName string, params map[string]string, sizeInGb int64, topReq *csipb.TopologyRequirement, caps []*csipb.VolumeCapability, volContentSrc *csipb.VolumeContentSource) (string, error) { +func (c *CsiClient) CreateVolumeWithCaps(volName string, params map[string]string, sizeInGb int64, topReq *csipb.TopologyRequirement, caps []*csipb.VolumeCapability, volContentSrc *csipb.VolumeContentSource) (*csipb.Volume, error) { capRange := &csipb.CapacityRange{ RequiredBytes: common.GbToBytes(sizeInGb), } @@ -117,12 +117,12 @@ func (c *CsiClient) CreateVolumeWithCaps(volName string, params map[string]strin } cresp, err := c.ctrlClient.CreateVolume(context.Background(), cvr) if err != nil { - return "", err + return nil, err } - return cresp.GetVolume().GetVolumeId(), nil + return cresp.Volume, nil } -func (c *CsiClient) CreateVolume(volName string, params map[string]string, sizeInGb int64, topReq *csipb.TopologyRequirement, volContentSrc *csipb.VolumeContentSource) (string, error) { +func (c *CsiClient) CreateVolume(volName string, params map[string]string, sizeInGb int64, topReq *csipb.TopologyRequirement, volContentSrc *csipb.VolumeContentSource) (*csipb.Volume, error) { return c.CreateVolumeWithCaps(volName, params, sizeInGb, topReq, stdVolCaps, volContentSrc) } @@ -134,13 +134,18 @@ func (c *CsiClient) DeleteVolume(volId string) error { return err } -func (c *CsiClient) ControllerPublishVolume(volId, nodeId string) error { +func (c *CsiClient) ControllerPublishVolume(volId, nodeId string, forceAttach bool) error { cpreq := &csipb.ControllerPublishVolumeRequest{ VolumeId: volId, NodeId: nodeId, VolumeCapability: stdVolCap, Readonly: false, } + if forceAttach { + cpreq.VolumeContext = map[string]string{ + "force-attach": "true", + } + } _, err := c.ctrlClient.ControllerPublishVolume(context.Background(), cpreq) return err } diff --git a/test/run-e2e-local.sh b/test/run-e2e-local.sh index be0455a1e..6e60ccc88 100755 --- a/test/run-e2e-local.sh +++ b/test/run-e2e-local.sh @@ -2,7 +2,13 @@ set -o nounset set -o errexit +set -x + +echo Using GOPATH $GOPATH readonly PKGDIR=sigs.k8s.io/gcp-compute-persistent-disk-csi-driver -ginkgo --v "test/e2e/tests" -- --project "${PROJECT}" --service-account "${IAM_NAME}" --v=4 --logtostderr +# This requires application default credentials to be set up, eg by +# `gcloud auth application-default login` + +ginkgo --v "test/e2e/tests" -- --project "${PROJECT}" --service-account "${IAM_NAME}" --v=6 --logtostderr