Merge pull request #1801 from travisyx/master

Add ControllerModifyVolume functionality

Author: mattcary

cmd/gce-pd-csi-driver/main.go

@@ -84,6 +84,9 @@ var (
 	useInstanceAPIForListVolumesPublishedNodesFlag = flag.Bool("use-instance-api-to-list-volumes-published-nodes", false, "Enables using the instances.list API to determine published_node_ids in ListVolumes. When false (default), the disks.list API is used")
 	instancesListFiltersFlag                       = flag.String("instances-list-filters", "", "Comma separated list of filters to use when calling the instances.list API. By default instances.list fetches all instances in a region")
 
+	diskSupportsIopsChangeFlag       = flag.String("supports-dynamic-iops-provisioning", "", "Comma separated list of disk types that support dynamic IOPS provisioning")
+	diskSupportsThroughputChangeFlag = flag.String("supports-dynamic-throughput-provisioning", "", "Comma separated list of disk types that support dynamic throughput provisioning")
+
 	extraTagsStr = flag.String("extra-tags", "", "Extra tags to attach to each Compute Disk, Image, Snapshot created. It is a comma separated list of parent id, key and value like '<parent_id1>/<tag_key1>/<tag_value1>,...,<parent_idN>/<tag_keyN>/<tag_valueN>'. parent_id is the Organization or the Project ID or Project name where the tag key and the tag value resources exist. A maximum of 50 tags bindings is allowed for a resource. See https://cloud.google.com/resource-manager/docs/tags/tags-overview, https://cloud.google.com/resource-manager/docs/tags/tags-creating-and-managing for details")
 
 	version string
@@ -200,6 +203,14 @@ func handle() {
 		UseInstancesAPIForPublishedNodes: *useInstanceAPIForListVolumesPublishedNodesFlag,
 	}
 
+	// Initialize provisionableDisks config
+	supportsIopsChange := parseCSVFlag(*diskSupportsIopsChangeFlag)
+	supportsThroughputChange := parseCSVFlag(*diskSupportsThroughputChangeFlag)
+	provisionableDisksConfig := driver.ProvisionableDisksConfig{
+		SupportsIopsChange:       supportsIopsChange,
+		SupportsThroughputChange: supportsThroughputChange,
+	}
+
 	// Initialize requirements for the controller service
 	var controllerServer *driver.GCEControllerServer
 	if *runControllerService {
@@ -209,7 +220,7 @@ func handle() {
 		}
 		initialBackoffDuration := time.Duration(*errorBackoffInitialDurationMs) * time.Millisecond
 		maxBackoffDuration := time.Duration(*errorBackoffMaxDurationMs) * time.Millisecond
-		controllerServer = driver.NewControllerServer(gceDriver, cloudProvider, initialBackoffDuration, maxBackoffDuration, fallbackRequisiteZones, *enableStoragePoolsFlag, multiZoneVolumeHandleConfig, listVolumesConfig)
+		controllerServer = driver.NewControllerServer(gceDriver, cloudProvider, initialBackoffDuration, maxBackoffDuration, fallbackRequisiteZones, *enableStoragePoolsFlag, multiZoneVolumeHandleConfig, listVolumesConfig, provisionableDisksConfig)
 	} else if *cloudConfigFilePath != "" {
 		klog.Warningf("controller service is disabled but cloud config given - it has no effect")
 	}

deploy/kubernetes/base/controller/cluster_setup.yaml

@@ -34,6 +34,9 @@ rules:
   - apiGroups: ["storage.k8s.io"]
     resources: ["storageclasses"]
     verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattributesclasses"]
+    verbs: ["get", "list", "watch"]
   - apiGroups: [""]
     resources: ["events"]
     verbs: ["list", "watch", "create", "update", "patch"]
@@ -69,7 +72,7 @@ roleRef:
   kind: ClusterRole
   name: csi-gce-pd-provisioner-role
   apiGroup: rbac.authorization.k8s.io
-  
+
 ---
 # xref: https://github.com/kubernetes-csi/external-attacher/blob/master/deploy/kubernetes/rbac.yaml
 kind: ClusterRole
@@ -143,6 +146,9 @@ rules:
   - apiGroups: [""]
     resources: ["persistentvolumeclaims/status"]
     verbs: ["update", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattributesclasses"]
+    verbs: ["get", "list", "watch"]
   - apiGroups: [""]
     resources: ["events"]
     verbs: ["list", "watch", "create", "update", "patch"]
@@ -312,4 +318,4 @@ subjects:
 roleRef:
   kind: Role
   name: csi-gce-pd-leaderelection-role
-  apiGroup: rbac.authorization.k8s.io
+  apiGroup: rbac.authorization.k8s.io

deploy/kubernetes/base/controller/controller.yaml

@@ -37,6 +37,7 @@ spec:
             - "--leader-election"
             - "--default-fstype=ext4"
             - "--controller-publish-readonly=true"
+            - "--feature-gates=VolumeAttributesClass=true"
           env:
             - name: PDCSI_NAMESPACE
               valueFrom:
@@ -95,6 +96,7 @@ spec:
             - "--leader-election"
             - "--leader-election-namespace=$(PDCSI_NAMESPACE)"
             - "--handle-volume-inuse-error=false"
+            - "--feature-gates=VolumeAttributesClass=true"
           env:
             - name: PDCSI_NAMESPACE
               valueFrom:

deploy/kubernetes/images/stable-master/image.yaml

@@ -4,7 +4,7 @@ metadata:
   name: imagetag-csi-provisioner
 imageTag:
   name: registry.k8s.io/sig-storage/csi-provisioner
-  newTag: "v3.6.3"
+  newTag: "v5.1.0"
 
 ---
 apiVersion: builtin
@@ -22,7 +22,7 @@ metadata:
   name: imagetag-csi-resizer
 imageTag:
   name: registry.k8s.io/sig-storage/csi-resizer
-  newTag: "v1.9.3"
+  newTag: "v1.11.1"
 ---
 
 apiVersion: builtin

deploy/kubernetes/overlays/dev/controller_always_pull.yaml

@@ -7,5 +7,4 @@ spec:
     spec:
       containers:
         - name: gce-pd-driver
-          imagePullPolicy: Always
-
+          imagePullPolicy: Always

deploy/kubernetes/overlays/dev/driver-args.yaml

@@ -0,0 +1,7 @@
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: --supports-dynamic-throughput-provisioning=hyperdisk-balanced,hyperdisk-throughput,hyperdisk-ml
+
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: --supports-dynamic-iops-provisioning=hyperdisk-balanced,hyperdisk-extreme

deploy/kubernetes/overlays/dev/kustomization.yaml

@@ -9,6 +9,14 @@ resources:
 # Here dev overlay is using the same image as alpha
 transformers:
 - ../../images/stable-master
+# Apply patches to support dynamic provisioning for hyperdisks
+patches:
+- path: ./driver-args.yaml
+  target:
+    group: apps
+    version: v1
+    kind: Deployment
+    name: csi-gce-pd-controller
 # To change the dev image, add something like the following.
 #images:
 #- name: gke.gcr.io/gcp-compute-persistent-disk-csi-driver

examples/kubernetes/demo-vol-update.yml

@@ -0,0 +1,99 @@
+apiVersion: storage.k8s.io/v1beta1
+kind: VolumeAttributesClass
+metadata:
+  name: silver
+driverName: pd.csi.storage.gke.io
+parameters:
+  throughput: "350"
+  iops: "6000"
+---
+apiVersion: storage.k8s.io/v1beta1
+kind: VolumeAttributesClass
+metadata:
+  name: gold
+driverName: pd.csi.storage.gke.io
+parameters:
+  throughput: "550"
+  iops: "15000"
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: balanced
+provisioner: pd.csi.storage.gke.io
+allowVolumeExpansion: true
+volumeBindingMode: WaitForFirstConsumer
+parameters:
+  type: hyperdisk-balanced
+  provisioned-throughput-on-create: "300Mi"
+  provisioned-iops-on-create: "5000"
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: throughput-optimized
+provisioner: pd.csi.storage.gke.io
+volumeBindingMode: WaitForFirstConsumer
+allowVolumeExpansion: true
+parameters:
+  type: hyperdisk-balanced
+  provisioned-throughput-on-create: "500Mi"
+  provisioned-iops-on-create: "10000"
+
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: balanced-pvc
+spec:
+  volumeAttributesClassName: silver
+  storageClassName: balanced
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 256Gi
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: throughput-optimized-pvc
+spec:
+  volumeAttributesClassName: silver
+  storageClassName: throughput-optimized
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 256Gi
+---
+kind: Pod
+apiVersion: v1
+metadata:
+  name: pod-demo
+spec:
+  volumes:
+    - name: pvc-demo-vol
+      persistentVolumeClaim:
+       claimName: balanced-pvc
+    - name: data-vol
+      persistentVolumeClaim:
+        claimName: throughput-optimized-pvc
+  containers:
+    - name: pod-demo
+      image: nginx:latest
+      resources:
+        limits:
+          cpu: 10m
+          memory: 80Mi
+        requests:
+          cpu: 10m
+          memory: 80Mi
+      ports:
+        - containerPort: 80
+          name: "http-server"
+      volumeMounts:
+        - mountPath: "/usr/share/nginx/html"
+          name: pvc-demo-vol
+        - mountPath: "/data"
+          name: data-vol

pkg/common/parameters.go

@@ -131,6 +131,11 @@ type ParameterProcessor struct {
 	EnableMultiZone    bool
 }
 
+type ModifyVolumeParameters struct {
+	IOPS       *int64
+	Throughput *int64
+}
+
 // ExtractAndDefaultParameters will take the relevant parameters from a map and
 // put them into a well defined struct making sure to default unspecified fields.
 // extraVolumeLabels are added as labels; if there are also labels specified in
@@ -324,3 +329,28 @@ func extractResourceTagsParameter(tagsString string, resourceTags map[string]str
 	}
 	return nil
 }
+
+func ExtractModifyVolumeParameters(parameters map[string]string) (ModifyVolumeParameters, error) {
+
+	modifyVolumeParams := ModifyVolumeParameters{}
+
+	for key, value := range parameters {
+		switch strings.ToLower(key) {
+		case "iops":
+			iops, err := ConvertStringToInt64(value)
+			if err != nil {
+				return ModifyVolumeParameters{}, fmt.Errorf("parameters contain invalid iops parameter: %w", err)
+			}
+			modifyVolumeParams.IOPS = &iops
+		case "throughput":
+			throughput, err := ConvertStringToInt64(value)
+			if err != nil {
+				return ModifyVolumeParameters{}, fmt.Errorf("parameters contain invalid throughput parameter: %w", err)
+			}
+			modifyVolumeParams.Throughput = &throughput
+		default:
+			return ModifyVolumeParameters{}, fmt.Errorf("parameters contain unknown parameter: %s", key)
+		}
+	}
+	return modifyVolumeParams, nil
+}

pkg/common/parameters_test.go

@@ -482,3 +482,25 @@ func TestSnapshotParameters(t *testing.T) {
 		})
 	}
 }
+func TestExtractModifyVolumeParameters(t *testing.T) {
+	parameters := map[string]string{
+		"iops":       "1000",
+		"throughput": "500",
+	}
+
+	iops := int64(1000)
+	throughput := int64(500)
+	expected := ModifyVolumeParameters{
+		IOPS:       &iops,
+		Throughput: &throughput,
+	}
+
+	result, err := ExtractModifyVolumeParameters(parameters)
+	if err != nil {
+		t.Errorf("Unexpected error: %v", err)
+	}
+
+	if !reflect.DeepEqual(result, expected) {
+		t.Errorf("Got ExtractModifyVolumeParameters(%+v) = %+v; want: %v", parameters, result, expected)
+	}
+}

pkg/gce-cloud-provider/compute/cloud-disk.go

@@ -267,3 +267,25 @@ func (d *CloudDisk) GetAccessMode() string {
 		return ""
 	}
 }
+
+func (d *CloudDisk) GetProvisionedIops() int64 {
+	switch {
+	case d.disk != nil:
+		return d.disk.ProvisionedIops
+	case d.betaDisk != nil:
+		return d.betaDisk.ProvisionedIops
+	default:
+		return 0
+	}
+}
+
+func (d *CloudDisk) GetProvisionedThroughput() int64 {
+	switch {
+	case d.disk != nil:
+		return d.disk.ProvisionedThroughput
+	case d.betaDisk != nil:
+		return d.betaDisk.ProvisionedThroughput
+	default:
+		return 0
+	}
+}