Skip to content

Commit 96182f2

Browse files
davidz627davidz627
committed
Add block device handling for NodeGetVolumeStats and e2e test
1 parent 328583e commit 96182f2

File tree

4 files changed

+130
-86
lines changed

4 files changed

+130
-86
lines changed

‎pkg/gce-pd-csi-driver/node.go

+19
Original file line numberDiff line numberDiff line change
@@ -379,6 +379,25 @@ func (ns *GCENodeServer) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGe
379379
return nil, status.Errorf(codes.NotFound, "path %s does not exist", req.VolumePath)
380380
}
381381

382+
isBlock, err := ns.VolumeStatter.IsBlockDevice(req.VolumePath)
383+
if err != nil {
384+
return nil, status.Errorf(codes.Internal, "failed to determine whether %s is block device: %v", req.VolumePath, err)
385+
}
386+
if isBlock {
387+
bcap, err := ns.getBlockSizeBytes(req.VolumePath)
388+
if err != nil {
389+
return nil, status.Errorf(codes.Internal, "failed to get block capacity on path %s: %v", req.VolumePath, err)
390+
}
391+
return &csi.NodeGetVolumeStatsResponse{
392+
Usage: []*csi.VolumeUsage{
393+
{
394+
Unit: csi.VolumeUsage_BYTES,
395+
Total: bcap,
396+
},
397+
},
398+
}, nil
399+
}
400+
382401
available, capacity, used, inodesFree, inodes, inodesUsed, err := ns.VolumeStatter.StatFS(req.VolumePath)
383402
if err != nil {
384403
return nil, status.Errorf(codes.Internal, "failed to get fs info on path %s: %v", req.VolumePath, err)

‎pkg/mount-manager/statter.go

+16
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222

2323
type Statter interface {
2424
StatFS(path string) (int64, int64, int64, int64, int64, int64, error)
25+
IsBlockDevice(string) (bool, error)
2526
}
2627

2728
var _ Statter = realStatter{}
@@ -33,6 +34,17 @@ func NewStatter() realStatter {
3334
return realStatter{}
3435
}
3536

37+
// IsBlock checks if the given path is a block device
38+
func (realStatter) IsBlockDevice(fullPath string) (bool, error) {
39+
var st unix.Stat_t
40+
err := unix.Stat(fullPath, &st)
41+
if err != nil {
42+
return false, err
43+
}
44+
45+
return (st.Mode & unix.S_IFMT) == unix.S_IFBLK, nil
46+
}
47+
3648
func (realStatter) StatFS(path string) (available, capacity, used, inodesFree, inodes, inodesUsed int64, err error) {
3749
statfs := &unix.Statfs_t{}
3850
err = unix.Statfs(path, statfs)
@@ -63,3 +75,7 @@ func (fakeStatter) StatFS(path string) (available, capacity, used, inodesFree, i
6375
// Assume the file exists and give some dummy values back
6476
return 1, 1, 1, 1, 1, 1, nil
6577
}
78+
79+
func (fakeStatter) IsBlockDevice(fullPath string) (bool, error) {
80+
return false, nil
81+
}

‎test/e2e/tests/multi_zone_e2e_test.go

+16-4
Original file line numberDiff line numberDiff line change
@@ -174,10 +174,10 @@ func testAttachWriteReadDetach(volID string, volName string, instance *remote.In
174174
}
175175
return nil
176176
}
177-
return testLifecycleWithVerify(volID, volName, instance, client, readOnly, writeFile, verifyReadFile)
177+
return testLifecycleWithVerify(volID, volName, instance, client, readOnly, false /* fs */, writeFile, verifyReadFile)
178178
}
179179

180-
func testLifecycleWithVerify(volID string, volName string, instance *remote.InstanceInfo, client *remote.CsiClient, readOnly bool, firstMountVerify, secondMountVerify verifyFunc) error {
180+
func testLifecycleWithVerify(volID string, volName string, instance *remote.InstanceInfo, client *remote.CsiClient, readOnly, useBlock bool, firstMountVerify, secondMountVerify verifyFunc) error {
181181
var err error
182182
klog.Infof("Starting testAttachWriteReadDetach with volume %v node %v with readonly %v\n", volID, instance.GetNodeID(), readOnly)
183183
// Attach Disk
@@ -197,7 +197,13 @@ func testLifecycleWithVerify(volID string, volName string, instance *remote.Inst
197197

198198
// Stage Disk
199199
stageDir := filepath.Join("/tmp/", volName, "stage")
200-
err = client.NodeStageExt4Volume(volID, stageDir)
200+
if useBlock {
201+
err = client.NodeStageBlockVolume(volID, stageDir)
202+
} else {
203+
err = client.NodeStageExt4Volume(volID, stageDir)
204+
}
205+
206+
//err = client.NodeStageExt4Volume(volID, stageDir)
201207
if err != nil {
202208
return fmt.Errorf("NodeStageExt4Volume failed with error: %v", err)
203209
}
@@ -217,7 +223,13 @@ func testLifecycleWithVerify(volID string, volName string, instance *remote.Inst
217223

218224
// Mount Disk
219225
publishDir := filepath.Join("/tmp/", volName, "mount")
220-
err = client.NodePublishVolume(volID, stageDir, publishDir)
226+
227+
if useBlock {
228+
err = client.NodePublishBlockVolume(volID, stageDir, publishDir)
229+
} else {
230+
err = client.NodePublishVolume(volID, stageDir, publishDir)
231+
}
232+
221233
if err != nil {
222234
return fmt.Errorf("NodePublishVolume failed with error: %v", err)
223235
}

‎test/e2e/tests/single_zone_e2e_test.go

+79-82
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"k8s.io/apimachinery/pkg/util/wait"
2525
"sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common"
2626
gce "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider/compute"
27+
remote "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/remote"
2728

2829
csi "github.com/container-storage-interface/spec/lib/go/csi"
2930
. "github.com/onsi/ginkgo"
@@ -44,7 +45,7 @@ const (
4445
ssdDiskType = "pd-ssd"
4546
defaultVolumeLimit int64 = 127
4647

47-
defaultEpsiolon = 500000000 // 500M
48+
defaultEpsilon = 500000000 // 500M
4849
)
4950

5051
var _ = Describe("GCE PD CSI Driver", func() {
@@ -65,28 +66,11 @@ var _ = Describe("GCE PD CSI Driver", func() {
6566
instance := testContext.Instance
6667

6768
// Create Disk
68-
volName := testNamePrefix + string(uuid.NewUUID())
69-
volID, err := client.CreateVolume(volName, nil, defaultSizeGb,
70-
&csi.TopologyRequirement{
71-
Requisite: []*csi.Topology{
72-
{
73-
Segments: map[string]string{common.TopologyKeyZone: z},
74-
},
75-
},
76-
})
77-
Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err)
78-
79-
// Validate Disk Created
80-
cloudDisk, err := computeService.Disks.Get(p, z, volName).Do()
81-
Expect(err).To(BeNil(), "Could not get disk from cloud directly")
82-
Expect(cloudDisk.Type).To(ContainSubstring(standardDiskType))
83-
Expect(cloudDisk.Status).To(Equal(readyState))
84-
Expect(cloudDisk.SizeGb).To(Equal(defaultSizeGb))
85-
Expect(cloudDisk.Name).To(Equal(volName))
69+
volName, volID := createAndValidateUniqueZonalDisk(client, p, z)
8670

8771
defer func() {
8872
// Delete Disk
89-
client.DeleteVolume(volID)
73+
err := client.DeleteVolume(volID)
9074
Expect(err).To(BeNil(), "DeleteVolume failed")
9175

9276
// Validate Disk Deleted
@@ -95,7 +79,7 @@ var _ = Describe("GCE PD CSI Driver", func() {
9579
}()
9680

9781
// Attach Disk
98-
err = testAttachWriteReadDetach(volID, volName, instance, client, false /* readOnly */)
82+
err := testAttachWriteReadDetach(volID, volName, instance, client, false /* readOnly */)
9983
Expect(err).To(BeNil(), "Failed to go through volume lifecycle")
10084

10185
})
@@ -137,31 +121,13 @@ var _ = Describe("GCE PD CSI Driver", func() {
137121
client := testContext.Client
138122
instance := testContext.Instance
139123

140-
// Create Disk
141-
volName := testNamePrefix + string(uuid.NewUUID())
142-
_, err := client.CreateVolume(volName, nil, defaultSizeGb,
143-
&csi.TopologyRequirement{
144-
Requisite: []*csi.Topology{
145-
{
146-
Segments: map[string]string{common.TopologyKeyZone: z},
147-
},
148-
},
149-
})
150-
Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err)
151-
152-
// Validate Disk Created
153-
cloudDisk, err := computeService.Disks.Get(p, z, volName).Do()
154-
Expect(err).To(BeNil(), "Could not get disk from cloud directly")
155-
Expect(cloudDisk.Type).To(ContainSubstring(standardDiskType))
156-
Expect(cloudDisk.Status).To(Equal(readyState))
157-
Expect(cloudDisk.SizeGb).To(Equal(defaultSizeGb))
158-
Expect(cloudDisk.Name).To(Equal(volName))
124+
volName, _ := createAndValidateUniqueZonalDisk(client, p, z)
159125

160126
underSpecifiedID := common.GenerateUnderspecifiedVolumeID(volName, true /* isZonal */)
161127

162128
defer func() {
163129
// Delete Disk
164-
client.DeleteVolume(underSpecifiedID)
130+
err := client.DeleteVolume(underSpecifiedID)
165131
Expect(err).To(BeNil(), "DeleteVolume failed")
166132

167133
// Validate Disk Deleted
@@ -170,7 +136,7 @@ var _ = Describe("GCE PD CSI Driver", func() {
170136
}()
171137

172138
// Attach Disk
173-
err = testAttachWriteReadDetach(underSpecifiedID, volName, instance, client, false /* readOnly */)
139+
err := testAttachWriteReadDetach(underSpecifiedID, volName, instance, client, false /* readOnly */)
174140
Expect(err).To(BeNil(), "Failed to go through volume lifecycle")
175141

176142
})
@@ -262,22 +228,11 @@ var _ = Describe("GCE PD CSI Driver", func() {
262228
p, z, _ := testContext.Instance.GetIdentity()
263229
client := testContext.Client
264230

265-
// Create Disk
266-
volName := testNamePrefix + string(uuid.NewUUID())
267-
volId, err := client.CreateVolume(volName, nil, defaultSizeGb, nil)
268-
Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err)
269-
270-
// Validate Disk Created
271-
cloudDisk, err := computeService.Disks.Get(p, z, volName).Do()
272-
Expect(err).To(BeNil(), "Could not get disk from cloud directly")
273-
Expect(cloudDisk.Type).To(ContainSubstring(standardDiskType))
274-
Expect(cloudDisk.Status).To(Equal(readyState))
275-
Expect(cloudDisk.SizeGb).To(Equal(defaultSizeGb))
276-
Expect(cloudDisk.Name).To(Equal(volName))
231+
volName, volID := createAndValidateUniqueZonalDisk(client, p, z)
277232

278233
// Create Snapshot
279234
snapshotName := testNamePrefix + string(uuid.NewUUID())
280-
snapshotID, err := client.CreateSnapshot(snapshotName, volId, nil)
235+
snapshotID, err := client.CreateSnapshot(snapshotName, volID, nil)
281236
Expect(err).To(BeNil(), "CreateSnapshot failed with error: %v", err)
282237

283238
// Validate Snapshot Created
@@ -297,7 +252,7 @@ var _ = Describe("GCE PD CSI Driver", func() {
297252

298253
defer func() {
299254
// Delete Disk
300-
err := client.DeleteVolume(volId)
255+
err := client.DeleteVolume(volID)
301256
Expect(err).To(BeNil(), "DeleteVolume failed")
302257

303258
// Validate Disk Deleted
@@ -482,7 +437,7 @@ var _ = Describe("GCE PD CSI Driver", func() {
482437

483438
// Create Disk
484439
volName := testNamePrefix + string(uuid.NewUUID())
485-
volId, err := controllerClient.CreateVolume(volName, map[string]string{
440+
volID, err := controllerClient.CreateVolume(volName, map[string]string{
486441
common.ParameterKeyReplicationType: "regional-pd",
487442
}, defaultRepdSizeGb, nil)
488443
Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err)
@@ -505,7 +460,7 @@ var _ = Describe("GCE PD CSI Driver", func() {
505460

506461
// Create Snapshot
507462
snapshotName := testNamePrefix + string(uuid.NewUUID())
508-
snapshotID, err := controllerClient.CreateSnapshot(snapshotName, volId, nil)
463+
snapshotID, err := controllerClient.CreateSnapshot(snapshotName, volID, nil)
509464
Expect(err).To(BeNil(), "CreateSnapshot failed with error: %v", err)
510465

511466
// Validate Snapshot Created
@@ -525,7 +480,7 @@ var _ = Describe("GCE PD CSI Driver", func() {
525480

526481
defer func() {
527482
// Delete Disk
528-
err := controllerClient.DeleteVolume(volId)
483+
err := controllerClient.DeleteVolume(volID)
529484
Expect(err).To(BeNil(), "DeleteVolume failed")
530485

531486
// Validate Disk Deleted
@@ -542,36 +497,55 @@ var _ = Describe("GCE PD CSI Driver", func() {
542497
}()
543498
})
544499

545-
It("Should get correct VolumeStats", func() {
500+
It("Should get correct VolumeStats for Block", func() {
546501
testContext := getRandomTestContext()
547502

548503
p, z, _ := testContext.Instance.GetIdentity()
549504
client := testContext.Client
550505
instance := testContext.Instance
551506

552-
// Create Disk
553-
volName := testNamePrefix + string(uuid.NewUUID())
554-
volID, err := client.CreateVolume(volName, nil, defaultSizeGb,
555-
&csi.TopologyRequirement{
556-
Requisite: []*csi.Topology{
557-
{
558-
Segments: map[string]string{common.TopologyKeyZone: z},
559-
},
560-
},
561-
})
562-
Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err)
507+
volName, volID := createAndValidateUniqueZonalDisk(client, p, z)
563508

564-
// Validate Disk Created
565-
cloudDisk, err := computeService.Disks.Get(p, z, volName).Do()
566-
Expect(err).To(BeNil(), "Could not get disk from cloud directly")
567-
Expect(cloudDisk.Type).To(ContainSubstring(standardDiskType))
568-
Expect(cloudDisk.Status).To(Equal(readyState))
569-
Expect(cloudDisk.SizeGb).To(Equal(defaultSizeGb))
570-
Expect(cloudDisk.Name).To(Equal(volName))
509+
defer func() {
510+
// Delete Disk
511+
err := client.DeleteVolume(volID)
512+
Expect(err).To(BeNil(), "DeleteVolume failed")
513+
514+
// Validate Disk Deleted
515+
_, err = computeService.Disks.Get(p, z, volName).Do()
516+
Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found")
517+
}()
518+
519+
verifyVolumeStats := func(a verifyArgs) error {
520+
available, capacity, used, inodesFree, inodes, inodesUsed, err := client.NodeGetVolumeStats(volID, a.publishDir)
521+
if err != nil {
522+
return fmt.Errorf("failed to get node volume stats: %v", err)
523+
}
524+
if available != 0 || capacity != common.GbToBytes(defaultSizeGb) || used != 0 ||
525+
inodesFree != 0 || inodes != 0 || inodesUsed != 0 {
526+
return fmt.Errorf("got: available %v, capacity %v, used %v, inodesFree %v, inodes %v, inodesUsed %v -- expected: capacity = %v, available = 0, used = 0, inodesFree = 0, inodes = 0 , inodesUsed = 0",
527+
available, capacity, used, inodesFree, inodes, inodesUsed, common.GbToBytes(defaultSizeGb))
528+
}
529+
return nil
530+
}
531+
532+
// Attach Disk
533+
err := testLifecycleWithVerify(volID, volName, instance, client, false /* readOnly */, true /* block */, verifyVolumeStats, nil)
534+
Expect(err).To(BeNil(), "Failed to go through volume lifecycle")
535+
})
536+
537+
It("Should get correct VolumeStats", func() {
538+
testContext := getRandomTestContext()
539+
540+
p, z, _ := testContext.Instance.GetIdentity()
541+
client := testContext.Client
542+
instance := testContext.Instance
543+
544+
volName, volID := createAndValidateUniqueZonalDisk(client, p, z)
571545

572546
defer func() {
573547
// Delete Disk
574-
client.DeleteVolume(volID)
548+
err := client.DeleteVolume(volID)
575549
Expect(err).To(BeNil(), "DeleteVolume failed")
576550

577551
// Validate Disk Deleted
@@ -584,7 +558,7 @@ var _ = Describe("GCE PD CSI Driver", func() {
584558
if err != nil {
585559
return fmt.Errorf("failed to get node volume stats: %v", err)
586560
}
587-
if !equalWithinEpsilon(available, common.GbToBytes(defaultSizeGb), defaultEpsiolon) || !equalWithinEpsilon(capacity, common.GbToBytes(defaultSizeGb), defaultEpsiolon) || !equalWithinEpsilon(used, 0, defaultEpsiolon) ||
561+
if !equalWithinEpsilon(available, common.GbToBytes(defaultSizeGb), defaultEpsilon) || !equalWithinEpsilon(capacity, common.GbToBytes(defaultSizeGb), defaultEpsilon) || !equalWithinEpsilon(used, 0, defaultEpsilon) ||
588562
inodesFree == 0 || inodes == 0 || inodesUsed == 0 {
589563
return fmt.Errorf("got: available %v, capacity %v, used %v, inodesFree %v, inodes %v, inodesUsed %v -- expected: available ~= %v, capacity ~= %v, used = 0, inodesFree != 0, inodes != 0 , inodesUsed != 0",
590564
available, capacity, used, inodesFree, inodes, inodesUsed, common.GbToBytes(defaultSizeGb), common.GbToBytes(defaultSizeGb))
@@ -593,9 +567,8 @@ var _ = Describe("GCE PD CSI Driver", func() {
593567
}
594568

595569
// Attach Disk
596-
err = testLifecycleWithVerify(volID, volName, instance, client, false /* readOnly */, verifyVolumeStats, nil)
570+
err := testLifecycleWithVerify(volID, volName, instance, client, false /* readOnly */, false /* fs */, verifyVolumeStats, nil)
597571
Expect(err).To(BeNil(), "Failed to go through volume lifecycle")
598-
599572
})
600573

601574
})
@@ -606,3 +579,27 @@ func equalWithinEpsilon(a, b, epsiolon int64) bool {
606579
}
607580
return b-a < epsiolon
608581
}
582+
583+
func createAndValidateUniqueZonalDisk(client *remote.CsiClient, project, zone string) (string, string) {
584+
// Create Disk
585+
volName := testNamePrefix + string(uuid.NewUUID())
586+
volID, err := client.CreateVolume(volName, nil, defaultSizeGb,
587+
&csi.TopologyRequirement{
588+
Requisite: []*csi.Topology{
589+
{
590+
Segments: map[string]string{common.TopologyKeyZone: zone},
591+
},
592+
},
593+
})
594+
Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err)
595+
596+
// Validate Disk Created
597+
cloudDisk, err := computeService.Disks.Get(project, zone, volName).Do()
598+
Expect(err).To(BeNil(), "Could not get disk from cloud directly")
599+
Expect(cloudDisk.Type).To(ContainSubstring(standardDiskType))
600+
Expect(cloudDisk.Status).To(Equal(readyState))
601+
Expect(cloudDisk.SizeGb).To(Equal(defaultSizeGb))
602+
Expect(cloudDisk.Name).To(Equal(volName))
603+
604+
return volName, volID
605+
}

0 commit comments

Comments
 (0)