Skip to content

Commit 569e33f

Browse files
davidz627davidz627
committed
Add implementation and unit tests for Expand capability
1 parent 0c48475 commit 569e33f

File tree

14 files changed

+1001
-11
lines changed

14 files changed

+1001
-11
lines changed

‎deploy/kubernetes/overlays/alpha/controller_add_resizer.yaml

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
kind: StatefulSet
2+
apiVersion: apps/v1
3+
metadata:
4+
name: csi-gce-pd-controller
5+
spec:
6+
template:
7+
spec:
8+
containers:
9+
- name: csi-resizer
10+
imagePullPolicy: Always
11+
image: quay.io/k8scsi/csi-resizer:canary
12+
args:
13+
- "--v=5"
14+
- "--csi-address=/csi/csi.sock"
15+
volumeMounts:
16+
- name: socket-dir
17+
mountPath: /csi

‎deploy/kubernetes/overlays/alpha/kustomization.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ bases:
44
- ../stable
55
patches:
66
- controller_add_snapshotter.yaml
7+
- controller_add_resizer.yaml
78
patchesJson6902:
89
- target:
910
group: rbac.authorization.k8s.io
@@ -13,3 +14,4 @@ patchesJson6902:
1314
path: rbac_add_snapshots_to_provisioner.yaml
1415
resources:
1516
- rbac_add_snapshotter.yaml
17+
- rbac_add_resizer.yaml

‎deploy/kubernetes/overlays/alpha/rbac_add_resizer.yaml

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
# Resizer must be able to work with PVCs, PVs, SCs.
2+
kind: ClusterRole
3+
apiVersion: rbac.authorization.k8s.io/v1
4+
metadata:
5+
name: external-resizer-role
6+
rules:
7+
- apiGroups: [""]
8+
resources: ["persistentvolumes"]
9+
verbs: ["get", "list", "watch", "update", "patch"]
10+
- apiGroups: [""]
11+
resources: ["persistentvolumeclaims"]
12+
verbs: ["get", "list", "watch"]
13+
- apiGroups: [""]
14+
resources: ["persistentvolumeclaims/status"]
15+
verbs: ["update", "patch"]
16+
- apiGroups: [""]
17+
resources: ["events"]
18+
verbs: ["list", "watch", "create", "update", "patch"]
19+
20+
---
21+
kind: ClusterRoleBinding
22+
apiVersion: rbac.authorization.k8s.io/v1
23+
metadata:
24+
name: csi-resizer-role
25+
subjects:
26+
- kind: ServiceAccount
27+
name: csi-controller-sa
28+
namespace: default
29+
roleRef:
30+
kind: ClusterRole
31+
name: external-resizer-role
32+
apiGroup: rbac.authorization.k8s.io

‎pkg/gce-cloud-provider/compute/cloud-disk.go

+11
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,17 @@ func (d *CloudDisk) GetSizeGb() int64 {
124124
}
125125
}
126126

127+
// setSizeGb sets the size of the disk used ONLY
128+
// for testing purposes.
129+
func (d *CloudDisk) setSizeGb(size int64) {
130+
switch d.Type() {
131+
case Zonal:
132+
d.ZonalDisk.SizeGb = size
133+
case Regional:
134+
d.RegionalDisk.SizeGb = size
135+
}
136+
}
137+
127138
func (d *CloudDisk) GetZone() string {
128139
switch d.Type() {
129140
case Zonal:

‎pkg/gce-cloud-provider/compute/fake-gce.go

+12
Original file line numberDiff line numberDiff line change
@@ -367,6 +367,18 @@ func (cloud *FakeCloudProvider) CreateSnapshot(ctx context.Context, volKey *meta
367367
return snapshotToCreate, nil
368368
}
369369

370+
func (cloud *FakeCloudProvider) ResizeDisk(ctx context.Context, volKey *meta.Key, requestBytes int64) (int64, error) {
371+
disk, ok := cloud.disks[volKey.Name]
372+
if !ok {
373+
return -1, notFoundError()
374+
}
375+
376+
disk.setSizeGb(common.BytesToGb(requestBytes))
377+
378+
return requestBytes, nil
379+
380+
}
381+
370382
// Snapshot Methods
371383
func (cloud *FakeCloudProvider) DeleteSnapshot(ctx context.Context, snapshotName string) error {
372384
delete(cloud.snapshots, snapshotName)

‎pkg/gce-cloud-provider/compute/gce-compute.go

+60
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ type GCECompute interface {
5050
GetDiskSourceURI(volKey *meta.Key) string
5151
GetDiskTypeURI(volKey *meta.Key, diskType string) string
5252
WaitForAttach(ctx context.Context, volKey *meta.Key, instanceZone, instanceName string) error
53+
ResizeDisk(ctx context.Context, volKey *meta.Key, requestBytes int64) (int64, error)
5354
// Regional Disk Methods
5455
GetReplicaZoneURI(zone string) string
5556
// Instance Methods
@@ -593,6 +594,65 @@ func (cloud *CloudProvider) CreateSnapshot(ctx context.Context, volKey *meta.Key
593594
}
594595
}
595596

597+
func (cloud *CloudProvider) ResizeDisk(ctx context.Context, volKey *meta.Key, requestBytes int64) (int64, error) {
598+
cloudDisk, err := cloud.GetDisk(ctx, volKey)
599+
if err != nil {
600+
return -1, fmt.Errorf("failed to get disk: %v", err)
601+
}
602+
603+
sizeGb := cloudDisk.GetSizeGb()
604+
requestGb := common.BytesToGb(requestBytes)
605+
606+
// If disk is already of size equal or greater than requested size, we simply return
607+
if sizeGb >= requestGb {
608+
return requestBytes, nil
609+
}
610+
611+
switch volKey.Type() {
612+
case meta.Zonal:
613+
return cloud.resizeZonalDisk(ctx, volKey, requestGb)
614+
case meta.Regional:
615+
return cloud.resizeRegionalDisk(ctx, volKey, requestGb)
616+
default:
617+
return -1, fmt.Errorf("could not resize disk, key was neither zonal nor regional, instead got: %v", volKey.String())
618+
}
619+
}
620+
621+
func (cloud *CloudProvider) resizeZonalDisk(ctx context.Context, volKey *meta.Key, requestGb int64) (int64, error) {
622+
resizeReq := &compute.DisksResizeRequest{
623+
SizeGb: requestGb,
624+
}
625+
op, err := cloud.service.Disks.Resize(cloud.project, volKey.Zone, volKey.Name, resizeReq).Context(ctx).Do()
626+
if err != nil {
627+
return -1, fmt.Errorf("failed to resize zonal volume %v: %v", volKey.String(), err)
628+
}
629+
630+
err = cloud.waitForZonalOp(ctx, op, volKey.Zone)
631+
if err != nil {
632+
return -1, fmt.Errorf("failed waiting for op for zonal resize for %s: %v", volKey.String(), err)
633+
}
634+
635+
return requestGb, nil
636+
}
637+
638+
func (cloud *CloudProvider) resizeRegionalDisk(ctx context.Context, volKey *meta.Key, requestGb int64) (int64, error) {
639+
resizeReq := &computebeta.RegionDisksResizeRequest{
640+
SizeGb: requestGb,
641+
}
642+
643+
op, err := cloud.betaService.RegionDisks.Resize(cloud.project, volKey.Region, volKey.Name, resizeReq).Context(ctx).Do()
644+
if err != nil {
645+
return -1, fmt.Errorf("failed to resize regional volume %v: %v", volKey.String(), err)
646+
}
647+
648+
err = cloud.waitForRegionalOp(ctx, op, volKey.Region)
649+
if err != nil {
650+
return -1, fmt.Errorf("failed waiting for op for regional resize for %s: %v", volKey.String(), err)
651+
}
652+
653+
return requestGb, nil
654+
}
655+
596656
func (cloud *CloudProvider) createZonalDiskSnapshot(ctx context.Context, volKey *meta.Key, snapshotName string) (*compute.Snapshot, error) {
597657
snapshotToCreate := &compute.Snapshot{
598658
Name: snapshotName,

‎pkg/gce-pd-csi-driver/controller.go

+24-1
Original file line numberDiff line numberDiff line change
@@ -659,7 +659,30 @@ func (gceCS *GCEControllerServer) ListSnapshots(ctx context.Context, req *csi.Li
659659
}
660660

661661
func (gceCS *GCEControllerServer) ControllerExpandVolume(ctx context.Context, req *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) {
662-
return nil, status.Error(codes.Unimplemented, "ControllerExpandVolume is not yet implemented")
662+
volumeID := req.GetVolumeId()
663+
if len(volumeID) == 0 {
664+
return nil, status.Error(codes.InvalidArgument, "ControllerExpandVolume volume ID must be provided")
665+
}
666+
capacityRange := req.GetCapacityRange()
667+
reqBytes, err := getRequestCapacity(capacityRange)
668+
if err != nil {
669+
return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("ControllerExpandVolume capacity range is invalid: %v", err))
670+
}
671+
672+
volKey, err := common.VolumeIDToKey(volumeID)
673+
if err != nil {
674+
return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("ControllerExpandVolume volume ID is invalid: %v", err))
675+
}
676+
677+
resizedGb, err := gceCS.CloudProvider.ResizeDisk(ctx, volKey, reqBytes)
678+
if err != nil {
679+
return nil, status.Error(codes.Internal, fmt.Sprintf("ControllerExpandVolume failed to resize disk: %v", err))
680+
}
681+
682+
return &csi.ControllerExpandVolumeResponse{
683+
CapacityBytes: common.GbToBytes(resizedGb),
684+
NodeExpansionRequired: true,
685+
}, nil
663686
}
664687

665688
func (gceCS *GCEControllerServer) getSnapshots(ctx context.Context, req *csi.ListSnapshotsRequest) (*csi.ListSnapshotsResponse, error) {

‎pkg/gce-pd-csi-driver/gce-pd-driver.go

+2
Original file line numberDiff line numberDiff line change
@@ -66,10 +66,12 @@ func (gceDriver *GCEDriver) SetupGCEDriver(cloudProvider gce.GCECompute, mounter
6666
csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT,
6767
csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS,
6868
csi.ControllerServiceCapability_RPC_PUBLISH_READONLY,
69+
csi.ControllerServiceCapability_RPC_EXPAND_VOLUME,
6970
}
7071
gceDriver.AddControllerServiceCapabilities(csc)
7172
ns := []csi.NodeServiceCapability_RPC_Type{
7273
csi.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME,
74+
csi.NodeServiceCapability_RPC_EXPAND_VOLUME,
7375
}
7476
gceDriver.AddNodeServiceCapabilities(ns)
7577

‎pkg/gce-pd-csi-driver/identity.go

+15
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ package gceGCEDriver
1616

1717
import (
1818
"context"
19+
1920
csi "github.com/container-storage-interface/spec/lib/go/csi"
2021
"google.golang.org/grpc/codes"
2122
"google.golang.org/grpc/status"
@@ -58,6 +59,20 @@ func (gceIdentity *GCEIdentityServer) GetPluginCapabilities(ctx context.Context,
5859
},
5960
},
6061
},
62+
{
63+
Type: &csi.PluginCapability_VolumeExpansion_{
64+
VolumeExpansion: &csi.PluginCapability_VolumeExpansion{
65+
Type: csi.PluginCapability_VolumeExpansion_ONLINE,
66+
},
67+
},
68+
},
69+
{
70+
Type: &csi.PluginCapability_VolumeExpansion_{
71+
VolumeExpansion: &csi.PluginCapability_VolumeExpansion{
72+
Type: csi.PluginCapability_VolumeExpansion_OFFLINE,
73+
},
74+
},
75+
},
6176
},
6277
}, nil
6378
}

‎pkg/gce-pd-csi-driver/identity_test.go

+16-5
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
"testing"
1919

2020
"context"
21+
2122
csi "github.com/container-storage-interface/spec/lib/go/csi"
2223
metadataservice "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider/metadata"
2324
)
@@ -58,11 +59,21 @@ func TestGetPluginCapabilities(t *testing.T) {
5859
}
5960

6061
for _, capability := range resp.GetCapabilities() {
61-
switch capability.GetService().GetType() {
62-
case csi.PluginCapability_Service_CONTROLLER_SERVICE:
63-
case csi.PluginCapability_Service_VOLUME_ACCESSIBILITY_CONSTRAINTS:
64-
default:
65-
t.Fatalf("Unknown capability: %v", capability.GetService().GetType())
62+
if capability.GetVolumeExpansion() != nil {
63+
switch capability.GetVolumeExpansion().GetType() {
64+
case csi.PluginCapability_VolumeExpansion_ONLINE:
65+
case csi.PluginCapability_VolumeExpansion_OFFLINE:
66+
default:
67+
t.Fatalf("Unknown capability: %v", capability.GetVolumeExpansion().GetType())
68+
}
69+
}
70+
if capability.GetService() != nil {
71+
switch capability.GetService().GetType() {
72+
case csi.PluginCapability_Service_CONTROLLER_SERVICE:
73+
case csi.PluginCapability_Service_VOLUME_ACCESSIBILITY_CONSTRAINTS:
74+
default:
75+
t.Fatalf("Unknown capability: %v", capability.GetService().GetType())
76+
}
6677
}
6778
}
6879
}

‎pkg/gce-pd-csi-driver/node.go

+83-1
Original file line numberDiff line numberDiff line change
@@ -17,14 +17,17 @@ package gceGCEDriver
1717
import (
1818
"fmt"
1919
"os"
20+
"strconv"
2021
"strings"
2122

2223
"context"
24+
2325
csi "github.com/container-storage-interface/spec/lib/go/csi"
2426
"google.golang.org/grpc/codes"
2527
"google.golang.org/grpc/status"
2628
"k8s.io/klog"
2729
"k8s.io/kubernetes/pkg/util/mount"
30+
"k8s.io/kubernetes/pkg/util/resizefs"
2831
"sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common"
2932
metadataservice "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider/metadata"
3033
mountmanager "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/mount-manager"
@@ -364,7 +367,73 @@ func (ns *GCENodeServer) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGe
364367
}
365368

366369
func (ns *GCENodeServer) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandVolumeRequest) (*csi.NodeExpandVolumeResponse, error) {
367-
return nil, status.Error(codes.Unimplemented, fmt.Sprintf("NodeExpandVolume is not yet implemented"))
370+
volumeID := req.GetVolumeId()
371+
if len(volumeID) == 0 {
372+
return nil, status.Error(codes.InvalidArgument, "ControllerExpandVolume volume ID must be provided")
373+
}
374+
capacityRange := req.GetCapacityRange()
375+
reqBytes, err := getRequestCapacity(capacityRange)
376+
if err != nil {
377+
return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("ControllerExpandVolume capacity range is invalid: %v", err))
378+
}
379+
380+
volumePath := req.GetVolumePath()
381+
if len(volumePath) == 0 {
382+
return nil, status.Error(codes.InvalidArgument, "ControllerExpandVolume volume path must be provided")
383+
}
384+
385+
volKey, err := common.VolumeIDToKey(volumeID)
386+
if err != nil {
387+
return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("ControllerExpandVolume volume ID is invalid: %v", err))
388+
}
389+
390+
devicePath, err := ns.getDevicePath(volumeID, "")
391+
if err != nil {
392+
return nil, status.Error(codes.Internal, fmt.Sprintf("ControllerExpandVolume error when getting device path for %s: %v", volumeID, err))
393+
}
394+
395+
// TODO(#328): Use requested size in resize if provided
396+
resizer := resizefs.NewResizeFs(ns.Mounter)
397+
_, err = resizer.Resize(devicePath, volumePath)
398+
if err != nil {
399+
return nil, status.Error(codes.Internal, fmt.Sprintf("ControllerExpandVolume error when resizing volume %s: %v", volKey.String(), err))
400+
401+
}
402+
403+
// Check the block size
404+
gotBlockSizeBytes, err := ns.getBlockSizeBytes(devicePath)
405+
if err != nil {
406+
return nil, status.Error(codes.Internal, fmt.Sprintf("ControllerExpandVolume error when getting size of block volume at path %s: %v", devicePath, err))
407+
}
408+
if gotBlockSizeBytes < reqBytes {
409+
// It's possible that the somewhere the volume size was rounded up, getting more size than requested is a success :)
410+
return nil, status.Errorf(codes.Internal, "ControllerExpandVolume resize requested for %v but after resize volume was size %v", reqBytes, gotBlockSizeBytes)
411+
}
412+
413+
// TODO(dyzz) Some sort of formatted volume could also check the fs size.
414+
// Issue is that we don't know how to account for filesystem overhead, it
415+
// could be proportional to fs size and different for xfs, ext4 and we don't
416+
// know the proportions
417+
418+
/*
419+
format, err := ns.Mounter.GetDiskFormat(devicePath)
420+
if err != nil {
421+
return nil, status.Error(codes.Internal, fmt.Sprintf("ControllerExpandVolume error checking format for device %s: %v", devicePath, err))
422+
}
423+
gotSizeBytes, err = ns.getFSSizeBytes(devicePath)
424+
425+
if err != nil {
426+
return nil, status.Errorf(codes.Internal, "ControllerExpandVolume resize could not get fs size of %s: %v", volumePath, err)
427+
}
428+
if gotSizeBytes != reqBytes {
429+
return nil, status.Errorf(codes.Internal, "ControllerExpandVolume resize requested for size %v but after resize volume was size %v", reqBytes, gotSizeBytes)
430+
}
431+
*/
432+
433+
// Respond
434+
return &csi.NodeExpandVolumeResponse{
435+
CapacityBytes: reqBytes,
436+
}, nil
368437
}
369438

370439
func (ns *GCENodeServer) GetVolumeLimits() (int64, error) {
@@ -401,3 +470,16 @@ func (ns *GCENodeServer) getDevicePath(volumeID string, partition string) (strin
401470
}
402471
return devicePath, nil
403472
}
473+
474+
func (ns *GCENodeServer) getBlockSizeBytes(devicePath string) (int64, error) {
475+
output, err := ns.Mounter.Exec.Run("blockdev", "--getsize64", devicePath)
476+
if err != nil {
477+
return -1, fmt.Errorf("error when getting size of block volume at path %s: output: %s, err: %v", devicePath, string(output), err)
478+
}
479+
strOut := strings.TrimSpace(string(output))
480+
gotSizeBytes, err := strconv.ParseInt(strOut, 10, 64)
481+
if err != nil {
482+
return -1, fmt.Errorf("failed to parse size %s into int a size", strOut)
483+
}
484+
return gotSizeBytes, nil
485+
}

0 commit comments

Comments
 (0)