Merge pull request kubernetes-sigs#317 from davidz627/feature/resize

Add support for Resize (Expand Volume) on the driver

Author: k8s-ci-robot

Gopkg.lock

@@ -34,14 +34,14 @@ This plugin is compatible with CSI versions [v1.0.0](https://github.com/containe
 
 ### Kubernetes Compatibility
 
-| GCE PD CSI Driver\Kubernetes Version | 1.10.5 - 1.11 | 1.12 | 1.13 | 1.14+
-|--------------------------------------|---------------|------|------|------|
-| v0.1.x.alpha                         | yes           | no   | no   | no   |
-| v0.2.x (alpha)                       | no            | yes  | no   | no   |
-| v0.3.x (beta)                        | no            | no   | yes  | yes  |
-| v0.4.x (beta)                        | no            | no   | yes  | yes  |
-| v0.5.x (beta)                        | no            | no   | no   | yes  |
-| dev                                  | no            | no   | no   | yes  |
+| GCE PD CSI Driver\Kubernetes Version | 1.10.5 - 1.11 | 1.12 | 1.13 | 1.14 | 1.15+|
+|--------------------------------------|---------------|------|------|------|------|
+| v0.1.x.alpha                         | yes           | no   | no   | no   | no   |
+| v0.2.x (alpha)                       | no            | yes  | no   | no   | no   |
+| v0.3.x (beta)                        | no            | no   | yes  | yes  | yes  |
+| v0.4.x (beta)                        | no            | no   | yes  | yes  | yes  |
+| v0.5.x (beta)                        | no            | no   | no   | yes  | yes  |
+| dev                                  | no            | no   | no   | no   | yes  |
 
 ### Known Issues
 
@@ -62,14 +62,13 @@ This driver supports only one topology key:
 `topology.gke.io/zone`
 that represents availability by zone.
 
-### Kubernetes Beta Features
+### Features in Development
 
-* Topology: Requires K8s 1.14+ on Master and Nodes and PD driver v0.5.0+
-
-### Kubernetes Alpha Features
-
-* Snapshots: Requires K8s 1.13+ on Master and PD driver v0.3.0+ with the alpha
-  overlay
+| Feature         | Stage | Min Kubernetes Master Version | Min Kubernetes Nodes Version | Min Driver Version | Deployment Overlay |
+|-----------------|-------|-------------------------------|------------------------------|--------------------|--------------------|
+| Topology        | Beta  | 1.14                          | 1.14                         | v0.5.0             | Stable             |
+| Snapshots       | Alpha | 1.13                          | Any                          | v0.3.0             | Alpha              |
+| Resize (Expand) | Alpha | 1.14                          | 1.14                         | dev                | Alpha              |
 
 ### Future Features
 

README.md

@@ -0,0 +1,17 @@
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: csi-gce-pd-controller
+spec:
+  template:
+    spec:
+      containers:
+        - name: csi-resizer
+          imagePullPolicy: Always
+          image: quay.io/k8scsi/csi-resizer:canary
+          args:
+            - "--v=5"
+            - "--csi-address=/csi/csi.sock"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi

deploy/kubernetes/overlays/alpha/controller_add_resizer.yaml

@@ -4,6 +4,7 @@ bases:
 - ../stable
 patches:
 - controller_add_snapshotter.yaml
+- controller_add_resizer.yaml
 patchesJson6902:
 - target:
     group: rbac.authorization.k8s.io
@@ -13,3 +14,4 @@ patchesJson6902:
   path: rbac_add_snapshots_to_provisioner.yaml
 resources:
 - rbac_add_snapshotter.yaml
+- rbac_add_resizer.yaml

deploy/kubernetes/overlays/alpha/kustomization.yaml

@@ -0,0 +1,32 @@
+# Resizer must be able to work with PVCs, PVs, SCs.
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: external-resizer-role
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims/status"]
+    verbs: ["update", "patch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-resizer-role
+subjects:
+  - kind: ServiceAccount
+    name: csi-controller-sa
+    namespace: default
+roleRef:
+  kind: ClusterRole
+  name: external-resizer-role
+  apiGroup: rbac.authorization.k8s.io

deploy/kubernetes/overlays/alpha/rbac_add_resizer.yaml

@@ -0,0 +1,101 @@
+# Kubernetes Resize User Guide (Alpha)
+
+>**Attention:** Volume Resize is an alpha feature. Make sure you have enabled it in Kubernetes API server using `--feature-gates=ExpandCSIVolumes=true` flag.
+>**Attention:** Volume Resize is only available in the driver version v0.6.0+
+
+### Install Driver with alpha resize feature
+
+1. [One-time per project] Create GCP service account for the CSI driver and set required roles
+
+```
+$ PROJECT=your-project-here                       # GCP project
+$ GCE_PD_SA_NAME=my-gce-pd-csi-sa                 # Name of the service account to create
+$ GCE_PD_SA_DIR=/my/safe/credentials/directory    # Directory to save the service account key
+$ ./deploy/setup-project.sh
+```
+
+2. Deploy driver to Kubernetes Cluster
+
+```
+$ GCE_PD_SA_DIR=/my/safe/credentials/directory    # Directory to get the service account key
+$ GCE_PD_DRIVER_VERSION=alpha                     # Driver version to deploy
+$ ./deploy/kubernetes/deploy-driver.sh
+```
+
+### Resize Example
+
+This example provisions a zonal PD in both single-zone and regional clusters.
+
+1. Add resize field to example Zonal Storage Class
+```
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: csi-gce-pd
+provisioner: pd.csi.storage.gke.io
+parameters:
+  type: pd-standard
+volumeBindingMode: WaitForFirstConsumer
+allowVolumeExpansion: true
+```
+
+2. Create example Zonal Storage Class with resize enabled
+```
+$ kubectl apply -f ./examples/kubernetes/demo-zonal-sc.yaml
+```
+
+3. Create example PVC and Pod
+```
+$ kubectl apply -f ./examples/kubernetes/demo-pod.yaml
+```
+
+4. Verify PV is created and bound to PVC
+```
+$ kubectl get pvc
+NAME      STATUS    VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
+podpvc     Bound     pvc-e36abf50-84f3-11e8-8538-42010a800002   10Gi       RWO            csi-gce-pd     9s
+```
+
+5. Verify pod is created and in `RUNNING` state (it may take a few minutes to get to running state)
+```
+$ kubectl get pods
+NAME                      READY     STATUS    RESTARTS   AGE
+web-server                1/1       Running   0          1m
+```
+
+8. Check current filesystem size on the running pod
+```
+$ kubectl exec web-server -- df -h /var/lib/www/html
+Filesystem      Size  Used Avail Use% Mounted on
+/dev/sdb        5.9G   24M  5.9G   1% /var/lib/www/html
+```
+
+6. Resize volume by modifying the field `spec -> resources -> requests -> storage`
+```
+$ kubectl edit pvc podpvc
+apiVersion: v1
+kind: PersistentVolumeClaim
+...
+spec:
+  resources:
+    requests:
+      storage: 9Gi
+  ...
+...
+```
+
+7. Verify actual disk resized on cloud
+```
+$ gcloud compute disks describe ${disk_name} --zone=${zone}
+description: Disk created by GCE-PD CSI Driver
+name: pvc-10ea155f-e5a4-4a82-a171-21481742c80c
+...
+sizeGb: '9'
+```
+
+8. Verify filesystem resized on the running pod
+```
+$ kubectl exec web-server -- df -h /var/lib/www/html
+Filesystem      Size  Used Avail Use% Mounted on
+/dev/sdb        8.8G   27M  8.8G   1% /var/lib/www/html
+```

docs/kubernetes/user-guides/resize.md

@@ -124,6 +124,17 @@ func (d *CloudDisk) GetSizeGb() int64 {
 	}
 }
 
+// setSizeGb sets the size of the disk used ONLY
+// for testing purposes.
+func (d *CloudDisk) setSizeGb(size int64) {
+	switch d.Type() {
+	case Zonal:
+		d.ZonalDisk.SizeGb = size
+	case Regional:
+		d.RegionalDisk.SizeGb = size
+	}
+}
+
 func (d *CloudDisk) GetZone() string {
 	switch d.Type() {
 	case Zonal:

pkg/gce-cloud-provider/compute/cloud-disk.go

@@ -367,6 +367,18 @@ func (cloud *FakeCloudProvider) CreateSnapshot(ctx context.Context, volKey *meta
 	return snapshotToCreate, nil
 }
 
+func (cloud *FakeCloudProvider) ResizeDisk(ctx context.Context, volKey *meta.Key, requestBytes int64) (int64, error) {
+	disk, ok := cloud.disks[volKey.Name]
+	if !ok {
+		return -1, notFoundError()
+	}
+
+	disk.setSizeGb(common.BytesToGb(requestBytes))
+
+	return requestBytes, nil
+
+}
+
 // Snapshot Methods
 func (cloud *FakeCloudProvider) DeleteSnapshot(ctx context.Context, snapshotName string) error {
 	delete(cloud.snapshots, snapshotName)

pkg/gce-cloud-provider/compute/fake-gce.go

@@ -50,6 +50,7 @@ type GCECompute interface {
 	GetDiskSourceURI(volKey *meta.Key) string
 	GetDiskTypeURI(volKey *meta.Key, diskType string) string
 	WaitForAttach(ctx context.Context, volKey *meta.Key, instanceZone, instanceName string) error
+	ResizeDisk(ctx context.Context, volKey *meta.Key, requestBytes int64) (int64, error)
 	// Regional Disk Methods
 	GetReplicaZoneURI(zone string) string
 	// Instance Methods
@@ -593,6 +594,65 @@ func (cloud *CloudProvider) CreateSnapshot(ctx context.Context, volKey *meta.Key
 	}
 }
 
+func (cloud *CloudProvider) ResizeDisk(ctx context.Context, volKey *meta.Key, requestBytes int64) (int64, error) {
+	cloudDisk, err := cloud.GetDisk(ctx, volKey)
+	if err != nil {
+		return -1, fmt.Errorf("failed to get disk: %v", err)
+	}
+
+	sizeGb := cloudDisk.GetSizeGb()
+	requestGb := common.BytesToGb(requestBytes)
+
+	// If disk is already of size equal or greater than requested size, we simply return
+	if sizeGb >= requestGb {
+		return requestBytes, nil
+	}
+
+	switch volKey.Type() {
+	case meta.Zonal:
+		return cloud.resizeZonalDisk(ctx, volKey, requestGb)
+	case meta.Regional:
+		return cloud.resizeRegionalDisk(ctx, volKey, requestGb)
+	default:
+		return -1, fmt.Errorf("could not resize disk, key was neither zonal nor regional, instead got: %v", volKey.String())
+	}
+}
+
+func (cloud *CloudProvider) resizeZonalDisk(ctx context.Context, volKey *meta.Key, requestGb int64) (int64, error) {
+	resizeReq := &compute.DisksResizeRequest{
+		SizeGb: requestGb,
+	}
+	op, err := cloud.service.Disks.Resize(cloud.project, volKey.Zone, volKey.Name, resizeReq).Context(ctx).Do()
+	if err != nil {
+		return -1, fmt.Errorf("failed to resize zonal volume %v: %v", volKey.String(), err)
+	}
+
+	err = cloud.waitForZonalOp(ctx, op, volKey.Zone)
+	if err != nil {
+		return -1, fmt.Errorf("failed waiting for op for zonal resize for %s: %v", volKey.String(), err)
+	}
+
+	return requestGb, nil
+}
+
+func (cloud *CloudProvider) resizeRegionalDisk(ctx context.Context, volKey *meta.Key, requestGb int64) (int64, error) {
+	resizeReq := &computebeta.RegionDisksResizeRequest{
+		SizeGb: requestGb,
+	}
+
+	op, err := cloud.betaService.RegionDisks.Resize(cloud.project, volKey.Region, volKey.Name, resizeReq).Context(ctx).Do()
+	if err != nil {
+		return -1, fmt.Errorf("failed to resize regional volume %v: %v", volKey.String(), err)
+	}
+
+	err = cloud.waitForRegionalOp(ctx, op, volKey.Region)
+	if err != nil {
+		return -1, fmt.Errorf("failed waiting for op for regional resize for %s: %v", volKey.String(), err)
+	}
+
+	return requestGb, nil
+}
+
 func (cloud *CloudProvider) createZonalDiskSnapshot(ctx context.Context, volKey *meta.Key, snapshotName string) (*compute.Snapshot, error) {
 	snapshotToCreate := &compute.Snapshot{
 		Name: snapshotName,

pkg/gce-cloud-provider/compute/gce-compute.go

@@ -659,7 +659,30 @@ func (gceCS *GCEControllerServer) ListSnapshots(ctx context.Context, req *csi.Li
 }
 
 func (gceCS *GCEControllerServer) ControllerExpandVolume(ctx context.Context, req *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) {
-	return nil, status.Error(codes.Unimplemented, "ControllerExpandVolume is not yet implemented")
+	volumeID := req.GetVolumeId()
+	if len(volumeID) == 0 {
+		return nil, status.Error(codes.InvalidArgument, "ControllerExpandVolume volume ID must be provided")
+	}
+	capacityRange := req.GetCapacityRange()
+	reqBytes, err := getRequestCapacity(capacityRange)
+	if err != nil {
+		return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("ControllerExpandVolume capacity range is invalid: %v", err))
+	}
+
+	volKey, err := common.VolumeIDToKey(volumeID)
+	if err != nil {
+		return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("ControllerExpandVolume volume ID is invalid: %v", err))
+	}
+
+	resizedGb, err := gceCS.CloudProvider.ResizeDisk(ctx, volKey, reqBytes)
+	if err != nil {
+		return nil, status.Error(codes.Internal, fmt.Sprintf("ControllerExpandVolume failed to resize disk: %v", err))
+	}
+
+	return &csi.ControllerExpandVolumeResponse{
+		CapacityBytes:         common.GbToBytes(resizedGb),
+		NodeExpansionRequired: true,
+	}, nil
 }
 
 func (gceCS *GCEControllerServer) getSnapshots(ctx context.Context, req *csi.ListSnapshotsRequest) (*csi.ListSnapshotsResponse, error) {

pkg/gce-pd-csi-driver/controller.go

@@ -66,10 +66,12 @@ func (gceDriver *GCEDriver) SetupGCEDriver(cloudProvider gce.GCECompute, mounter
 		csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT,
 		csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS,
 		csi.ControllerServiceCapability_RPC_PUBLISH_READONLY,
+		csi.ControllerServiceCapability_RPC_EXPAND_VOLUME,
 	}
 	gceDriver.AddControllerServiceCapabilities(csc)
 	ns := []csi.NodeServiceCapability_RPC_Type{
 		csi.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME,
+		csi.NodeServiceCapability_RPC_EXPAND_VOLUME,
 	}
 	gceDriver.AddNodeServiceCapabilities(ns)