Cleanup ControllerModifyVolume logic and run upstream E2E tests

Author: travisyx

.gitignore

@@ -49,6 +49,3 @@ test/k8s-integration/config/test-config.yaml
 *.un~
 Session.vim
 .netrwhist
-
-# Credentials
-creds/

.vscode/launch.json

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM --platform=$BUILDPLATFORM golang:1.22.4 as builder
+FROM --platform=$BUILDPLATFORM golang:1.23.0 as builder
 
 ARG STAGINGVERSION
 ARG TARGETPLATFORM
@@ -96,6 +96,7 @@ COPY --from=debian /lib/${LIB_DIR_PREFIX}-linux-gnu/libselinux.so.1 \
                    /lib/${LIB_DIR_PREFIX}-linux-gnu/libnvme-mi.so.1 \
                    /lib/${LIB_DIR_PREFIX}-linux-gnu/libnvme.so.1 \
                    /lib/${LIB_DIR_PREFIX}-linux-gnu/libsystemd.so.0 \
+                   /lib/${LIB_DIR_PREFIX}-linux-gnu/libgpg-error.so.0 \
                    /lib/${LIB_DIR_PREFIX}-linux-gnu/libzstd.so.1 /lib/${LIB_DIR_PREFIX}-linux-gnu/
 
 COPY --from=debian /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libblkid.so.1 \

Dockerfile

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 ARG BASE_IMAGE
-FROM --platform=$BUILDPLATFORM golang:1.22.4 AS builder
+FROM --platform=$BUILDPLATFORM golang:1.23.0 AS builder
 
 ARG TARGETPLATFORM
 ARG STAGINGVERSION

Dockerfile.Windows

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM golang:1.22.4 as builder
+FROM golang:1.23.0 as builder
 WORKDIR /go/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver
 ADD . .
 

Dockerfile.debug

@@ -126,7 +126,7 @@ init-buildx:
 	$(DOCKER) run --rm --privileged multiarch/qemu-user-static --reset --credential yes --persistent yes
 	# Ensure we use a builder that can leverage it (the default on linux will not)
 	-$(DOCKER) buildx rm multiarch-multiplatform-builder
-	$(DOCKER) buildx create --use --name=multiarch-multiplatform-builder --driver-opt network=host --driver-opt image=moby/buildkit:v0.11.3
+	$(DOCKER) buildx create --use --name=multiarch-multiplatform-builder --driver-opt network=host --driver-opt image=moby/buildkit:v0.14.1
 	# Register gcloud as a Docker credential helper.
 	# Required for "docker buildx build --push".
 	gcloud auth configure-docker --quiet

Makefile

@@ -84,6 +84,9 @@ var (
 	useInstanceAPIForListVolumesPublishedNodesFlag = flag.Bool("use-instance-api-to-list-volumes-published-nodes", false, "Enables using the instances.list API to determine published_node_ids in ListVolumes. When false (default), the disks.list API is used")
 	instancesListFiltersFlag                       = flag.String("instances-list-filters", "", "Comma separated list of filters to use when calling the instances.list API. By default instances.list fetches all instances in a region")
 
+	diskSupportsIopsChangeFlag       = flag.String("supports-dynamic-iops-provisioning", "", "Comma separated list of disk types that support dynamic IOPS provisioning")
+	diskSupportsThroughputChangeFlag = flag.String("supports-dynamic-throughput-provisioning", "", "Comma separated list of disk types that support dynamic throughput provisioning")
+
 	extraTagsStr = flag.String("extra-tags", "", "Extra tags to attach to each Compute Disk, Image, Snapshot created. It is a comma separated list of parent id, key and value like '<parent_id1>/<tag_key1>/<tag_value1>,...,<parent_idN>/<tag_keyN>/<tag_valueN>'. parent_id is the Organization or the Project ID or Project name where the tag key and the tag value resources exist. A maximum of 50 tags bindings is allowed for a resource. See https://cloud.google.com/resource-manager/docs/tags/tags-overview, https://cloud.google.com/resource-manager/docs/tags/tags-creating-and-managing for details")
 
 	version string
@@ -200,6 +203,14 @@ func handle() {
 		UseInstancesAPIForPublishedNodes: *useInstanceAPIForListVolumesPublishedNodesFlag,
 	}
 
+	// Initialize provisionableDisks config
+	supportsIopsChange := parseCSVFlag(*diskSupportsIopsChangeFlag)
+	supportsThroughputChange := parseCSVFlag(*diskSupportsThroughputChangeFlag)
+	provisionableDisksConfig := driver.ProvisionableDisksConfig{
+		SupportsIopsChange:       supportsIopsChange,
+		SupportsThroughputChange: supportsThroughputChange,
+	}
+
 	// Initialize requirements for the controller service
 	var controllerServer *driver.GCEControllerServer
 	if *runControllerService {
@@ -209,7 +220,7 @@ func handle() {
 		}
 		initialBackoffDuration := time.Duration(*errorBackoffInitialDurationMs) * time.Millisecond
 		maxBackoffDuration := time.Duration(*errorBackoffMaxDurationMs) * time.Millisecond
-		controllerServer = driver.NewControllerServer(gceDriver, cloudProvider, initialBackoffDuration, maxBackoffDuration, fallbackRequisiteZones, *enableStoragePoolsFlag, multiZoneVolumeHandleConfig, listVolumesConfig)
+		controllerServer = driver.NewControllerServer(gceDriver, cloudProvider, initialBackoffDuration, maxBackoffDuration, fallbackRequisiteZones, *enableStoragePoolsFlag, multiZoneVolumeHandleConfig, listVolumesConfig, provisionableDisksConfig)
 	} else if *cloudConfigFilePath != "" {
 		klog.Warningf("controller service is disabled but cloud config given - it has no effect")
 	}

cmd/gce-pd-csi-driver/main.go

@@ -34,6 +34,9 @@ rules:
   - apiGroups: ["storage.k8s.io"]
     resources: ["storageclasses"]
     verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattributesclasses"]
+    verbs: ["get", "list", "watch"]
   - apiGroups: [""]
     resources: ["events"]
     verbs: ["list", "watch", "create", "update", "patch"]
@@ -69,7 +72,7 @@ roleRef:
   kind: ClusterRole
   name: csi-gce-pd-provisioner-role
   apiGroup: rbac.authorization.k8s.io
-  
+
 ---
 # xref: https://github.com/kubernetes-csi/external-attacher/blob/master/deploy/kubernetes/rbac.yaml
 kind: ClusterRole
@@ -143,6 +146,9 @@ rules:
   - apiGroups: [""]
     resources: ["persistentvolumeclaims/status"]
     verbs: ["update", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattributesclasses"]
+    verbs: ["get", "list", "watch"]
   - apiGroups: [""]
     resources: ["events"]
     verbs: ["list", "watch", "create", "update", "patch"]
@@ -312,29 +318,4 @@ subjects:
 roleRef:
   kind: Role
   name: csi-gce-pd-leaderelection-role
-  apiGroup: rbac.authorization.k8s.io
-
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: cluster-admin
-rules:
-- apiGroups: ["*"]
-  resources: ["*"]
-  verbs: ["*"]
-- nonResourceURLs: ["*"]
-  verbs: ["*"]
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: csi-gce-pd-controller-sa-cluster-admin
-subjects:
-- kind: ServiceAccount
-  name: csi-gce-pd-controller-sa
-  namespace: gce-pd-csi-driver
-roleRef:
-  kind: ClusterRole
-  name: cluster-admin
-  apiGroup: rbac.authorization.k8s.io
+  apiGroup: rbac.authorization.k8s.io

deploy/kubernetes/base/controller/cluster_setup.yaml

@@ -37,6 +37,7 @@ spec:
             - "--leader-election"
             - "--default-fstype=ext4"
             - "--controller-publish-readonly=true"
+            - "--feature-gates=VolumeAttributesClass=true"
           env:
             - name: PDCSI_NAMESPACE
               valueFrom:
@@ -95,6 +96,7 @@ spec:
             - "--leader-election"
             - "--leader-election-namespace=$(PDCSI_NAMESPACE)"
             - "--handle-volume-inuse-error=false"
+            - "--feature-gates=VolumeAttributesClass=true"
           env:
             - name: PDCSI_NAMESPACE
               valueFrom:

deploy/kubernetes/base/controller/controller.yaml

@@ -48,6 +48,6 @@ metadata:
 imageTag:
   name: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver
   newName: gcr.io/k8s-staging-cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver
-  newTag: "v1.13.3-rc1"
+  newTag: "v1.14.2-rc1"
 ---
 

deploy/kubernetes/images/prow-stable-sidecar-rc-master/image.yaml

@@ -4,7 +4,7 @@ metadata:
   name: imagetag-csi-provisioner
 imageTag:
   name: registry.k8s.io/sig-storage/csi-provisioner
-  newTag: "v3.6.3"
+  newTag: "v5.1.0"
 
 ---
 apiVersion: builtin

deploy/kubernetes/images/stable-master/image.yaml

@@ -31,11 +31,11 @@ if [[ ! "$tmpDir" || ! -d "$tmpDir" ]]; then
   exit 1
 fi
 
-# function cleanup {
-#   rm -rf "$tmpDir"
-# }
+function cleanup {
+  rm -rf "$tmpDir"
+}
 
-# trap cleanup EXIT
+trap cleanup EXIT
 
 pushd $tmpDir >& /dev/null
 

deploy/kubernetes/install-kustomize.sh

@@ -7,5 +7,4 @@ spec:
     spec:
       containers:
         - name: gce-pd-driver
-          imagePullPolicy: Always
-
+          imagePullPolicy: Always

deploy/kubernetes/overlays/dev/controller_always_pull.yaml

@@ -0,0 +1,7 @@
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: --supports-dynamic-throughput-provisioning=hyperdisk-balanced,hyperdisk-throughput,hyperdisk-ml
+
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: --supports-dynamic-iops-provisioning=hyperdisk-balanced,hyperdisk-extreme

deploy/kubernetes/overlays/dev/driver-args.yaml

@@ -9,6 +9,14 @@ resources:
 # Here dev overlay is using the same image as alpha
 transformers:
 - ../../images/stable-master
+# Apply patches to support dynamic provisioning for hyperdisks
+patches:
+- path: ./driver-args.yaml
+  target:
+    group: apps
+    version: v1
+    kind: Deployment
+    name: csi-gce-pd-controller
 # To change the dev image, add something like the following.
 #images:
 #- name: gke.gcr.io/gcp-compute-persistent-disk-csi-driver

deploy/kubernetes/overlays/dev/kustomization.yaml

@@ -11,48 +11,24 @@ spec:
       containers:
         - name: gce-pd-driver
           imagePullPolicy: Always
-          command: ["/go/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/bin/gce-pd-csi-driver"]
+          command: ["/go/bin/dlv"]
           args:
+            - "--listen=:2345"
+            - "--headless=true"
+            - "--api-version=2"
+            # https://github.com/go-delve/delve/blob/master/Documentation/usage/dlv_exec.md#options
+            - "--accept-multiclient"
+            - "--continue"
+            - "--log"
+            - "exec"
+            - "/go/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/bin/gce-pd-csi-driver"
+            - "--"
             - "--v=5"
             - "--endpoint=unix:/csi/csi.sock"
           ports:
             - containerPort: 2345
           securityContext:
             capabilities:
               add:
-                - SYS_PTRACE
-
-        - name: csi-provisioner
-          image: gcr.io/k8s-staging-sig-storage/csi-provisioner:canary
-          args:
-            - "--v=5"
-            - "--csi-address=/csi/csi.sock"
-            - "--feature-gates=Topology=true"
-            - "--http-endpoint=:22011"
-            - "--leader-election-namespace=$(PDCSI_NAMESPACE)"
-            - "--timeout=250s"
-            - "--extra-create-metadata"
-          # - "--run-controller-service=false"  # disable the controller service of the CSI driver
-          # - "--run-node-service=false"        # disable the node service of the CSI driver
-            - "--leader-election"
-            - "--default-fstype=ext4"
-            - "--controller-publish-readonly=true"
-            - "--feature-gates=VolumeAttributesClass=true"
-
-        - name: csi-resizer
-          image: registry.k8s.io/sig-storage/csi-resizer
-          imagePullPolicy: Always
-          args:
-            - "--v=5"
-            - "--csi-address=/csi/csi.sock"
-            - "--http-endpoint=:22013"
-            - "--leader-election"
-            - "--leader-election-namespace=$(PDCSI_NAMESPACE)"
-            - "--handle-volume-inuse-error=false"
-            - "--feature-gates=VolumeAttributesClass=true"
-
-      # used with vanilla K8s
-      # imagePullSecrets:
-      #   - name: artifactory-cred
-
+              - SYS_PTRACE
 

deploy/kubernetes/overlays/noauth-debug/controller-overlay.yaml

@@ -10,7 +10,8 @@ patchesStrategicMerge:
 - controller-overlay.yaml
 - node-overlay.yaml
 namespace: gce-pd-csi-driver
-images:
-- name: gke.gcr.io/gcp-compute-persistent-disk-csi-driver
-  newName: us-central1-docker.pkg.dev/travisx-joonix/csi-dev/gce-pd-csi-driver
-  newTag: dev_linux
+# To change the dev image, add something like the following.
+# images:
+# - name: gke.gcr.io/gcp-compute-persistent-disk-csi-driver
+#   newName: gcr.io/mauriciopoppe-gke-dev/gcp-compute-persistent-disk-csi-driver
+#   newTag: latest

deploy/kubernetes/overlays/noauth-debug/kustomization.yaml

@@ -1,4 +1,4 @@
-apiVersion: storage.k8s.io/v1alpha1
+apiVersion: storage.k8s.io/v1beta1
 kind: VolumeAttributesClass
 metadata:
   name: silver
@@ -7,7 +7,7 @@ parameters:
   throughput: "350"
   iops: "6000"
 ---
-apiVersion: storage.k8s.io/v1alpha1
+apiVersion: storage.k8s.io/v1beta1
 kind: VolumeAttributesClass
 metadata:
   name: gold