[WIP] KEP 3751 volume attribute class

.vscode/launch.json

@@ -0,0 +1,33 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Launch Package",
+            "type": "go",
+            "request": "launch",
+            "mode": "auto",
+            "program": "${fileDirname}",
+            "args" : ["--run-in-prow", "false", "--gke-cluster-prefix", "csitest-", 
+            "--gke-is-alpha", "true", "--machine-type", "n4-standard-8", "--deploy-overlay-name", "noauth-debug",
+            "--staging-image", "europe-west3-docker.pkg.dev/kimambo-sandbox/csi-dev/gce-pd-csi-driver", 
+            "--service-account-file", "/usr/local/google/home/kimambo/dev/go/src/github.com/maxkimambo/creds/cloud-sa.json", 
+            "--deploy-overlay-name", "noauth-debug", "--storageclass-files", "sc-standard.yaml,sc-balanced.yaml,sc-ssd.yaml", 
+            "--test-focus", "External.Storage", "--gce-zone", "us-central1-b", "--deployment-strategy", "gke", "--gke-cluster-version", "${gke_cluster_version}", "--test-version", "${test_version}", "--num-nodes", "3"]
+        },
+        {
+            "name": "Cluster",
+            "type": "go",
+            "request": "attach",
+            "mode": "remote",
+            "debugAdapter": "dlv-dap",
+            "substitutePath": [
+                {"from": "${workspaceFolder}", "to": "/go/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver"}
+            ],
+            "port": 2345,
+            "host": "127.0.0.1"
+        }, 
+    ]
+}

deploy/kubernetes/base/controller/cluster_setup.yaml

@@ -313,3 +313,28 @@ roleRef:
   kind: Role
   name: csi-gce-pd-leaderelection-role
   apiGroup: rbac.authorization.k8s.io
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: cluster-admin
+rules:
+- apiGroups: ["*"]
+  resources: ["*"]
+  verbs: ["*"]
+- nonResourceURLs: ["*"]
+  verbs: ["*"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: csi-gce-pd-controller-sa-cluster-admin
+subjects:
+- kind: ServiceAccount
+  name: csi-gce-pd-controller-sa   
+  namespace: gce-pd-csi-driver
+roleRef:
+  kind: ClusterRole
+  name: cluster-admin
+  apiGroup: rbac.authorization.k8s.io

deploy/kubernetes/images/stable-master/image.yaml

@@ -22,7 +22,7 @@ metadata:
   name: imagetag-csi-resizer
 imageTag:
   name: registry.k8s.io/sig-storage/csi-resizer
-  newTag: "v1.9.3"
+  newTag: "v1.11.1"
 ---
 
 apiVersion: builtin

deploy/kubernetes/install-kustomize.sh

@@ -31,11 +31,11 @@ if [[ ! "$tmpDir" || ! -d "$tmpDir" ]]; then
   exit 1
 fi
 
-function cleanup {
-  rm -rf "$tmpDir"
-}
+# function cleanup {
+#   rm -rf "$tmpDir"
+# }
 
-trap cleanup EXIT
+# trap cleanup EXIT
 
 pushd $tmpDir >& /dev/null
 

deploy/kubernetes/overlays/noauth-debug/controller-overlay.yaml

@@ -8,6 +8,7 @@ metadata:
 spec:
   template:
     spec:
+
       containers:
         - name: gce-pd-driver
           imagePullPolicy: Always
@@ -32,3 +33,36 @@ spec:
               add:
               - SYS_PTRACE
 
+        - name: csi-provisioner
+          image: gcr.io/k8s-staging-sig-storage/csi-provisioner:canary
+          args:
+            - "--v=5"
+            - "--csi-address=/csi/csi.sock"
+            - "--feature-gates=Topology=true"
+            - "--http-endpoint=:22011"
+            - "--leader-election-namespace=$(PDCSI_NAMESPACE)"
+            - "--timeout=250s"
+            - "--extra-create-metadata"
+          # - "--run-controller-service=false"  # disable the controller service of the CSI driver
+          # - "--run-node-service=false"        # disable the node service of the CSI driver
+            - "--leader-election"
+            - "--default-fstype=ext4"
+            - "--controller-publish-readonly=true"
+            - "--feature-gates=VolumeAttributesClass=true"
+
+        - name: csi-resizer
+          image: registry.k8s.io/sig-storage/csi-resizer
+          imagePullPolicy: Always
+          args:
+            - "--v=5"
+            - "--csi-address=/csi/csi.sock"
+            - "--http-endpoint=:22013"
+            - "--leader-election"
+            - "--leader-election-namespace=$(PDCSI_NAMESPACE)"
+            - "--handle-volume-inuse-error=false"
+            - "--feature-gates=VolumeAttributesClass=true"
+
+      # used with vanilla K8s 
+      # imagePullSecrets:
+      #   - name: artifactory-cred
+

deploy/kubernetes/overlays/noauth-debug/kustomization.yaml

@@ -10,8 +10,7 @@ patchesStrategicMerge:
 - controller-overlay.yaml
 - node-overlay.yaml
 namespace: gce-pd-csi-driver
-# To change the dev image, add something like the following.
-# images:
-# - name: gke.gcr.io/gcp-compute-persistent-disk-csi-driver
-#   newName: gcr.io/mauriciopoppe-gke-dev/gcp-compute-persistent-disk-csi-driver
-#   newTag: latest
+images:
+- name: gke.gcr.io/gcp-compute-persistent-disk-csi-driver
+  newName: europe-west3-docker.pkg.dev/kimambo-sandbox/csi-dev/gce-pd-csi-driver
+  newTag: dev_linux

examples/kubernetes/demo-vol-update.yml

@@ -0,0 +1,99 @@
+apiVersion: storage.k8s.io/v1alpha1
+kind: VolumeAttributesClass
+metadata:
+  name: silver
+driverName: pd.csi.storage.gke.io
+parameters:
+  throughput: "350"
+  iops: "6000"
+---
+apiVersion: storage.k8s.io/v1alpha1
+kind: VolumeAttributesClass
+metadata:
+  name: gold
+driverName: pd.csi.storage.gke.io
+parameters:
+  throughput: "550"
+  iops: "15000"
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: balanced
+provisioner: pd.csi.storage.gke.io
+allowVolumeExpansion: true 
+volumeBindingMode: WaitForFirstConsumer
+parameters:
+  type: hyperdisk-balanced
+  provisioned-throughput-on-create: "300Mi"
+  provisioned-iops-on-create: "5000"
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: throughput-optimized
+provisioner: pd.csi.storage.gke.io
+volumeBindingMode: WaitForFirstConsumer
+allowVolumeExpansion: true
+parameters:
+  type: hyperdisk-balanced
+  provisioned-throughput-on-create: "500Mi"
+  provisioned-iops-on-create: "10000"
+
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: balanced-pvc
+spec:
+  volumeAttributesClassName: silver
+  storageClassName: balanced
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 256Gi
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: throughput-optimized-pvc
+spec:
+  volumeAttributesClassName: silver
+  storageClassName: throughput-optimized
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 256Gi
+---
+kind: Pod
+apiVersion: v1
+metadata:
+  name: pod-demo
+spec:
+  volumes:
+    - name: pvc-demo-vol
+      persistentVolumeClaim:
+       claimName: balanced-pvc
+    - name: data-vol
+      persistentVolumeClaim:
+        claimName: throughput-optimized-pvc
+  containers:
+    - name: pod-demo
+      image: nginx:latest
+      resources:
+        limits:
+          cpu: 10m
+          memory: 80Mi
+        requests:
+          cpu: 10m
+          memory: 80Mi
+      ports:
+        - containerPort: 80
+          name: "http-server"
+      volumeMounts:
+        - mountPath: "/usr/share/nginx/html"
+          name: pvc-demo-vol
+        - mountPath: "/data"
+          name: data-vol

go.mod

@@ -9,7 +9,7 @@ require (
 	cloud.google.com/go/kms v1.17.1
 	cloud.google.com/go/resourcemanager v1.9.7
 	github.com/GoogleCloudPlatform/k8s-cloud-provider v1.24.0
-	github.com/container-storage-interface/spec v1.6.0
+	github.com/container-storage-interface/spec v1.9.0
 	github.com/google/go-cmp v0.6.0
 	github.com/google/uuid v1.6.0
 	github.com/googleapis/gax-go/v2 v2.12.4

go.sum

@@ -895,8 +895,9 @@ github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:z
 github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo=
 github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA=
 github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
-github.com/container-storage-interface/spec v1.6.0 h1:vwN9uCciKygX/a0toYryoYD5+qI9ZFeAMuhEEKO+JBA=
 github.com/container-storage-interface/spec v1.6.0/go.mod h1:8K96oQNkJ7pFcC2R9Z1ynGGBB1I93kcS6PGg3SsOk8s=
+github.com/container-storage-interface/spec v1.9.0 h1:zKtX4STsq31Knz3gciCYCi1SXtO2HJDecIjDVboYavY=
+github.com/container-storage-interface/spec v1.9.0/go.mod h1:ZfDu+3ZRyeVqxZM0Ds19MVLkN2d1XJ5MAfi1L3VjlT0=
 github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko=
 github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=
 github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=

pkg/common/parameters.go

@@ -131,6 +131,11 @@ type ParameterProcessor struct {
 	EnableMultiZone    bool
 }
 
+type ModifyVolumeParameters struct {
+	IOPS       int64
+	Throughput int64
+}
+
 // ExtractAndDefaultParameters will take the relevant parameters from a map and
 // put them into a well defined struct making sure to default unspecified fields.
 // extraVolumeLabels are added as labels; if there are also labels specified in
@@ -324,3 +329,26 @@ func extractResourceTagsParameter(tagsString string, resourceTags map[string]str
 	}
 	return nil
 }
+
+func ExtractModifyVolumeParameters(parameters map[string]string) (ModifyVolumeParameters, error) {
+
+	modifyVolumeParams := ModifyVolumeParameters{}
+
+	for key, value := range parameters {
+		switch strings.ToLower(key) {
+		case "iops":
+			iops, err := ConvertStringToInt64(value)
+			if err != nil {
+				return ModifyVolumeParameters{}, fmt.Errorf("parameters contain invalid iops parameter: %w", err)
+			}
+			modifyVolumeParams.IOPS = iops
+		case "throughput":
+			throughput, err := ConvertStringToInt64(value)
+			if err != nil {
+				return ModifyVolumeParameters{}, fmt.Errorf("parameters contain invalid throughput parameter: %w", err)
+			}
+			modifyVolumeParams.Throughput = throughput
+		}
+	}
+	return modifyVolumeParams, nil
+}

pkg/common/parameters_test.go

@@ -482,3 +482,23 @@ func TestSnapshotParameters(t *testing.T) {
 		})
 	}
 }
+func TestExtractModifyVolumeParameters(t *testing.T) {
+	parameters := map[string]string{
+		"iops":       "1000",
+		"throughput": "500",
+	}
+
+	expected := ModifyVolumeParameters{
+		IOPS:       1000,
+		Throughput: 500,
+	}
+
+	result, err := ExtractModifyVolumeParameters(parameters)
+	if err != nil {
+		t.Errorf("Unexpected error: %v", err)
+	}
+
+	if !reflect.DeepEqual(result, expected) {
+		t.Errorf("Expected: %v, but got: %v", expected, result)
+	}
+}

pkg/gce-cloud-provider/compute/cloud-disk.go

@@ -267,3 +267,25 @@ func (d *CloudDisk) GetAccessMode() string {
 		return ""
 	}
 }
+
+func (d *CloudDisk) GetProvisionedIops() int64 {
+	switch {
+	case d.disk != nil:
+		return d.disk.ProvisionedIops
+	case d.betaDisk != nil:
+		return d.betaDisk.ProvisionedIops
+	default:
+		return 0
+	}
+}
+
+func (d *CloudDisk) GetProvisionedThroughput() int64 {
+	switch {
+	case d.disk != nil:
+		return d.disk.ProvisionedThroughput
+	case d.betaDisk != nil:
+		return d.betaDisk.ProvisionedThroughput
+	default:
+		return 0
+	}
+}