Squash 20 commits

Author: travisyx

Dockerfile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM --platform=$BUILDPLATFORM golang:1.22.4 as builder
+FROM --platform=$BUILDPLATFORM golang:1.23.0 as builder
 
 ARG STAGINGVERSION
 ARG TARGETPLATFORM

Dockerfile.Windows

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 ARG BASE_IMAGE
-FROM --platform=$BUILDPLATFORM golang:1.22.4 AS builder
+FROM --platform=$BUILDPLATFORM golang:1.23.0 AS builder
 
 ARG TARGETPLATFORM
 ARG STAGINGVERSION

Dockerfile.debug

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM golang:1.22.4 as builder
+FROM golang:1.23.0 as builder
 WORKDIR /go/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver
 ADD . .
 

cmd/gce-pd-csi-driver/main.go

@@ -84,6 +84,9 @@ var (
 	useInstanceAPIForListVolumesPublishedNodesFlag = flag.Bool("use-instance-api-to-list-volumes-published-nodes", false, "Enables using the instances.list API to determine published_node_ids in ListVolumes. When false (default), the disks.list API is used")
 	instancesListFiltersFlag                       = flag.String("instances-list-filters", "", "Comma separated list of filters to use when calling the instances.list API. By default instances.list fetches all instances in a region")
 
+	diskSupportsIopsChangeFlag       = flag.String("supports-dynamic-iops-provisioning", "", "Comma separated list of disk types that support dynamic IOPS provisioning")
+	diskSupportsThroughputChangeFlag = flag.String("supports-dynamic-throughput-provisioning", "", "Comma separated list of disk types that support dynamic throughput provisioning")
+
 	extraTagsStr = flag.String("extra-tags", "", "Extra tags to attach to each Compute Disk, Image, Snapshot created. It is a comma separated list of parent id, key and value like '<parent_id1>/<tag_key1>/<tag_value1>,...,<parent_idN>/<tag_keyN>/<tag_valueN>'. parent_id is the Organization or the Project ID or Project name where the tag key and the tag value resources exist. A maximum of 50 tags bindings is allowed for a resource. See https://cloud.google.com/resource-manager/docs/tags/tags-overview, https://cloud.google.com/resource-manager/docs/tags/tags-creating-and-managing for details")
 
 	version string
@@ -200,6 +203,14 @@ func handle() {
 		UseInstancesAPIForPublishedNodes: *useInstanceAPIForListVolumesPublishedNodesFlag,
 	}
 
+	// Initialize provisionableDisks config
+	supportsIopsChange := parseCSVFlag(*diskSupportsIopsChangeFlag)
+	supportsThroughputChange := parseCSVFlag(*diskSupportsThroughputChangeFlag)
+	provisionableDisksConfig := driver.ProvisionableDisksConfig{
+		SupportsIopsChange:       supportsIopsChange,
+		SupportsThroughputChange: supportsThroughputChange,
+	}
+
 	// Initialize requirements for the controller service
 	var controllerServer *driver.GCEControllerServer
 	if *runControllerService {
@@ -209,7 +220,7 @@ func handle() {
 		}
 		initialBackoffDuration := time.Duration(*errorBackoffInitialDurationMs) * time.Millisecond
 		maxBackoffDuration := time.Duration(*errorBackoffMaxDurationMs) * time.Millisecond
-		controllerServer = driver.NewControllerServer(gceDriver, cloudProvider, initialBackoffDuration, maxBackoffDuration, fallbackRequisiteZones, *enableStoragePoolsFlag, multiZoneVolumeHandleConfig, listVolumesConfig)
+		controllerServer = driver.NewControllerServer(gceDriver, cloudProvider, initialBackoffDuration, maxBackoffDuration, fallbackRequisiteZones, *enableStoragePoolsFlag, multiZoneVolumeHandleConfig, listVolumesConfig, provisionableDisksConfig)
 	} else if *cloudConfigFilePath != "" {
 		klog.Warningf("controller service is disabled but cloud config given - it has no effect")
 	}

deploy/kubernetes/base/controller/cluster_setup.yaml

@@ -34,8 +34,8 @@ rules:
   - apiGroups: ["storage.k8s.io"]
     resources: ["storageclasses"]
     verbs: ["get", "list", "watch"]
-  - apiGroups: ["storage.k8s.io/v1alpha1"]
-    resources: ["volumeattributesclass"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattributesclasses"]
     verbs: ["get", "list", "watch"]
   - apiGroups: [""]
     resources: ["events"]
@@ -146,8 +146,8 @@ rules:
   - apiGroups: [""]
     resources: ["persistentvolumeclaims/status"]
     verbs: ["update", "patch"]
-  - apiGroups: ["storage.k8s.io/v1alpha1"]
-    resources: ["volumeattributesclass"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattributesclasses"]
     verbs: ["get", "list", "watch"]
   - apiGroups: [""]
     resources: ["events"]
@@ -318,18 +318,4 @@ subjects:
 roleRef:
   kind: Role
   name: csi-gce-pd-leaderelection-role
-  apiGroup: rbac.authorization.k8s.io
-
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: csi-gce-pd-controller-sa-cluster-admin
-subjects:
-- kind: ServiceAccount
-  name: csi-gce-pd-controller-sa
-  namespace: gce-pd-csi-driver
-roleRef:
-  kind: ClusterRole
-  name: cluster-admin
-  apiGroup: rbac.authorization.k8s.io
+  apiGroup: rbac.authorization.k8s.io

deploy/kubernetes/base/controller/controller.yaml

@@ -37,6 +37,7 @@ spec:
             - "--leader-election"
             - "--default-fstype=ext4"
             - "--controller-publish-readonly=true"
+            - "--feature-gates=VolumeAttributesClass=true"
           env:
             - name: PDCSI_NAMESPACE
               valueFrom:
@@ -95,6 +96,7 @@ spec:
             - "--leader-election"
             - "--leader-election-namespace=$(PDCSI_NAMESPACE)"
             - "--handle-volume-inuse-error=false"
+            - "--feature-gates=VolumeAttributesClass=true"
           env:
             - name: PDCSI_NAMESPACE
               valueFrom:

deploy/kubernetes/images/prow-stable-sidecar-rc-master/image.yaml

@@ -48,6 +48,6 @@ metadata:
 imageTag:
   name: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver
   newName: gcr.io/k8s-staging-cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver
-  newTag: "v1.13.3-rc1"
+  newTag: "v1.14.2-rc1"
 ---
 

deploy/kubernetes/images/stable-master/image.yaml

@@ -4,7 +4,7 @@ metadata:
   name: imagetag-csi-provisioner
 imageTag:
   name: registry.k8s.io/sig-storage/csi-provisioner
-  newTag: "v3.6.3"
+  newTag: "v5.1.0"
 
 ---
 apiVersion: builtin

deploy/kubernetes/install-kustomize.sh

@@ -31,11 +31,11 @@ if [[ ! "$tmpDir" || ! -d "$tmpDir" ]]; then
   exit 1
 fi
 
-# function cleanup {
-#   rm -rf "$tmpDir"
-# }
+function cleanup {
+  rm -rf "$tmpDir"
+}
 
-# trap cleanup EXIT
+trap cleanup EXIT
 
 pushd $tmpDir >& /dev/null
 

deploy/kubernetes/overlays/dev/controller_always_pull.yaml

@@ -7,5 +7,4 @@ spec:
     spec:
       containers:
         - name: gce-pd-driver
-          imagePullPolicy: Always
-
+          imagePullPolicy: Always

deploy/kubernetes/overlays/dev/driver-args.yaml

@@ -0,0 +1,7 @@
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: --supports-dynamic-throughput-provisioning=hyperdisk-balanced,hyperdisk-throughput,hyperdisk-ml
+
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: --supports-dynamic-iops-provisioning=hyperdisk-balanced,hyperdisk-extreme

deploy/kubernetes/overlays/dev/kustomization.yaml

@@ -9,6 +9,14 @@ resources:
 # Here dev overlay is using the same image as alpha
 transformers:
 - ../../images/stable-master
+# Apply patches to support dynamic provisioning for hyperdisks
+patches:
+- path: ./driver-args.yaml
+  target:
+    group: apps
+    version: v1
+    kind: Deployment
+    name: csi-gce-pd-controller
 # To change the dev image, add something like the following.
 #images:
 #- name: gke.gcr.io/gcp-compute-persistent-disk-csi-driver

deploy/kubernetes/overlays/noauth-debug/controller-overlay.yaml

@@ -8,85 +8,27 @@ metadata:
 spec:
   template:
     spec:
-      
       containers:
         - name: gce-pd-driver
           imagePullPolicy: Always
-          command: ["/go/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/bin/gce-pd-csi-driver"]
+          command: ["/go/bin/dlv"]
           args:
+            - "--listen=:2345"
+            - "--headless=true"
+            - "--api-version=2"
+            # https://github.com/go-delve/delve/blob/master/Documentation/usage/dlv_exec.md#options
+            - "--accept-multiclient"
+            - "--continue"
+            - "--log"
+            - "exec"
+            - "/go/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/bin/gce-pd-csi-driver"
+            - "--"
             - "--v=5"
             - "--endpoint=unix:/csi/csi.sock"
           ports:
             - containerPort: 2345
           securityContext:
             capabilities:
               add:
-                - SYS_PTRACE
-
-        - name: csi-provisioner
-          image: gcr.io/k8s-staging-sig-storage/csi-provisioner:canary
-          args:
-            - "--v=5"
-            - "--csi-address=/csi/csi.sock"
-            - "--feature-gates=Topology=true"
-            - "--http-endpoint=:22011"
-            - "--leader-election-namespace=$(PDCSI_NAMESPACE)"
-            - "--timeout=250s"
-            - "--extra-create-metadata"
-          # - "--run-controller-service=false"  # disable the controller service of the CSI driver
-          # - "--run-node-service=false"        # disable the node service of the CSI driver
-            - "--leader-election"
-            - "--default-fstype=ext4"
-            - "--controller-publish-readonly=true"
-            - "--feature-gates=VolumeAttributesClass=true"
-
-        - name: csi-resizer
-          image: registry.k8s.io/sig-storage/csi-resizer
-          imagePullPolicy: Always
-          args:
-            - "--v=5"
-            - "--csi-address=/csi/csi.sock"
-            - "--http-endpoint=:22013"
-            - "--leader-election"
-            - "--leader-election-namespace=$(PDCSI_NAMESPACE)"
-            - "--handle-volume-inuse-error=false"
-            - "--feature-gates=VolumeAttributesClass=true"
-
-      # used with vanilla K8s
-      # imagePullSecrets:
-      #   - name: artifactory-cred
-
-
-        - name: csi-provisioner
-          image: gcr.io/k8s-staging-sig-storage/csi-provisioner:canary
-          args:
-            - "--v=5"
-            - "--csi-address=/csi/csi.sock"
-            - "--feature-gates=Topology=true"
-            - "--http-endpoint=:22011"
-            - "--leader-election-namespace=$(PDCSI_NAMESPACE)"
-            - "--timeout=250s"
-            - "--extra-create-metadata"
-          # - "--run-controller-service=false"  # disable the controller service of the CSI driver
-          # - "--run-node-service=false"        # disable the node service of the CSI driver
-            - "--leader-election"
-            - "--default-fstype=ext4"
-            - "--controller-publish-readonly=true"
-            - "--feature-gates=VolumeAttributesClass=true"
-
-        - name: csi-resizer
-          image: registry.k8s.io/sig-storage/csi-resizer
-          imagePullPolicy: Always
-          args:
-            - "--v=5"
-            - "--csi-address=/csi/csi.sock"
-            - "--http-endpoint=:22013"
-            - "--leader-election"
-            - "--leader-election-namespace=$(PDCSI_NAMESPACE)"
-            - "--handle-volume-inuse-error=false"
-            - "--feature-gates=VolumeAttributesClass=true"
-      
-      # used with vanilla K8s 
-      # imagePullSecrets:
-      #   - name: artifactory-cred
+              - SYS_PTRACE
 

deploy/kubernetes/overlays/noauth-debug/kustomization.yaml

@@ -10,7 +10,8 @@ patchesStrategicMerge:
 - controller-overlay.yaml
 - node-overlay.yaml
 namespace: gce-pd-csi-driver
-images:
-- name: gke.gcr.io/gcp-compute-persistent-disk-csi-driver
-  newName: us-central1-docker.pkg.dev/travisx-joonix/csi-dev/gce-pd-csi-driver
-  newTag: dev_linux
+# To change the dev image, add something like the following.
+# images:
+# - name: gke.gcr.io/gcp-compute-persistent-disk-csi-driver
+#   newName: gcr.io/mauriciopoppe-gke-dev/gcp-compute-persistent-disk-csi-driver
+#   newTag: latest

docs/kubernetes/user-guides/storage-pools.md

@@ -0,0 +1,37 @@
+# Hyperdisk Storage Pools User Guide
+
+Provisioning attached disks in Storage Pools is supported in the managed version of our driver which is automatically enabled on new GKE clusters, and in a manually deployed GCE PD CSI Driver. We recommend using this feature with the managed GCE PD CSI Driver. See the public documentation [here](https://cloud.google.com/kubernetes-engine/docs/how-to/persistent-volumes/hyperdisk-storage-pools). 
+
+If you would like to use Storage Pools with a manual deployment of the GCE PD CSI driver, you will need to do a couple additional things to enable the feature as described below.
+
+### Enabling Storage Pools for a Manual Deployment of GCE PD CSI driver.
+
+>**Attention:** Note that Storage Pools is only available in the driver version [v1.13.0+](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/releases/tag/v1.13.0).
+
+In addition to the install instructions [here](driver-install.md), you need to specify the CSI driver command line flags, `--enable-storage-pools=true`. 
+
+### Provision Volumes in a Storage Pool
+
+To provision volumes in a Storage Pool, follow the instructions [here](https://cloud.google.com/kubernetes-engine/docs/how-to/persistent-volumes/hyperdisk-storage-pools#provision-attached-disk). When using the feature in a manual deployment of the GCE PD CSI Driver, in the [Create a StorageClass](https://cloud.google.com/kubernetes-engine/docs/how-to/persistent-volumes/hyperdisk-storage-pools#create-storageclass) section, you must additionally set [`allowedTopologies`](https://kubernetes.io/docs/concepts/storage/storage-classes/#allowed-topologies) to restrict the topology of provisioned volumes to specific zones where Storage Pools exist as specified in the `storage-pools` StorageClass parameter. 
+
+For example, looking at the example in [Create a StorageClass](https://cloud.google.com/kubernetes-engine/docs/how-to/persistent-volumes/hyperdisk-storage-pools#create-storageclass), assuming you already created a Storage Pool in `us-east4-c` according to [Create a Hyperdisk Storage Pool](https://cloud.google.com/kubernetes-engine/docs/how-to/persistent-volumes/hyperdisk-storage-pools#create-sp), your `StorageClass` would need to specify `allowedTopologies` to restrict the topology of provisioned volumes to us-east4-c, where the Storage Pool exists.
+
+    ```yaml
+    apiVersion: storage.k8s.io/v1
+    kind: StorageClass
+    metadata:
+        name: storage-pools-sc
+    provisioner: pd.csi.storage.gke.io
+    volumeBindingMode: WaitForFirstConsumer
+    allowVolumeExpansion: true
+    parameters:
+        type: hyperdisk-balanced
+        provisioned-throughput-on-create: "140Mi"
+        provisioned-iops-on-create: "3000"
+        storage-pools: projects/my-project/zones/us-east4-c/storagePools/pool-us-east4-c
+    allowedTopologies:
+    - matchLabelExpressions:
+        - key: topology.gke.io/zone
+            values:
+            - us-east4-c
+    ```

examples/kubernetes/demo-vol-update.yml

@@ -1,4 +1,4 @@
-apiVersion: storage.k8s.io/v1alpha1
+apiVersion: storage.k8s.io/v1beta1
 kind: VolumeAttributesClass
 metadata:
   name: silver
@@ -7,7 +7,7 @@ parameters:
   throughput: "350"
   iops: "6000"
 ---
-apiVersion: storage.k8s.io/v1alpha1
+apiVersion: storage.k8s.io/v1beta1
 kind: VolumeAttributesClass
 metadata:
   name: gold