From c72a2906086395d72bfbf14d36cf284ce3675c7d Mon Sep 17 00:00:00 2001
From: Max Kimambo <max@kimambo.de>
Date: Fri, 14 Jun 2024 09:23:42 +0000
Subject: [PATCH 01/20] Add Controller modify volume functionality

---
 pkg/common/parameters.go                      | 28 +++++++++++++
 pkg/gce-cloud-provider/compute/fake-gce.go    |  5 +++
 pkg/gce-cloud-provider/compute/gce-compute.go | 40 ++++++++++++++++++
 pkg/gce-pd-csi-driver/controller.go           | 42 +++++++++++++++++++
 pkg/gce-pd-csi-driver/gce-pd-driver.go        |  1 +
 vendor/modules.txt                            |  4 +-
 6 files changed, 118 insertions(+), 2 deletions(-)

diff --git a/pkg/common/parameters.go b/pkg/common/parameters.go
index 3b07f0779..3b4ee3bb1 100644
--- a/pkg/common/parameters.go
+++ b/pkg/common/parameters.go
@@ -121,6 +121,11 @@ type StoragePool struct {
 	ResourceName string
 }
 
+type ModifyVolumeParameters struct {
+	IOPS       int64
+	Throughput int64
+}
+
 // ExtractAndDefaultParameters will take the relevant parameters from a map and
 // put them into a well defined struct making sure to default unspecified fields.
 // extraVolumeLabels are added as labels; if there are also labels specified in
@@ -301,3 +306,26 @@ func extractResourceTagsParameter(tagsString string, resourceTags map[string]str
 	}
 	return nil
 }
+
+func ExtractModifyVolumeParameters(parameters map[string]string) (ModifyVolumeParameters, error) {
+
+	modifyVolumeParams := ModifyVolumeParameters{}
+
+	for key, value := range parameters {
+		switch strings.ToLower(key) {
+		case "iops":
+			iops, err := ConvertStringToInt64(value)
+			if err != nil {
+				return ModifyVolumeParameters{}, fmt.Errorf("parameters contain invalid iops parameter: %w", err)
+			}
+			modifyVolumeParams.IOPS = iops
+		case "throughput":
+			throughput, err := ConvertStringToInt64(value)
+			if err != nil {
+				return ModifyVolumeParameters{}, fmt.Errorf("parameters contain invalid throughput parameter: %w", err)
+			}
+			modifyVolumeParams.Throughput = throughput
+		}
+	}
+	return modifyVolumeParams, nil
+}
diff --git a/pkg/gce-cloud-provider/compute/fake-gce.go b/pkg/gce-cloud-provider/compute/fake-gce.go
index 3b7416b18..bac819cec 100644
--- a/pkg/gce-cloud-provider/compute/fake-gce.go
+++ b/pkg/gce-cloud-provider/compute/fake-gce.go
@@ -266,6 +266,11 @@ func (cloud *FakeCloudProvider) InsertDisk(ctx context.Context, project string,
 	return nil
 }
 
+func (cloud *FakeCloudProvider) UpdateDisk(ctx context.Context, project string, volKey *meta.Key, existingDisk *CloudDisk, params common.ModifyVolumeParameters) error {
+
+	return nil
+}
+
 func (cloud *FakeCloudProvider) DeleteDisk(ctx context.Context, project string, volKey *meta.Key) error {
 	delete(cloud.disks, volKey.Name)
 	return nil
diff --git a/pkg/gce-cloud-provider/compute/gce-compute.go b/pkg/gce-cloud-provider/compute/gce-compute.go
index eaa0c9953..3b58f0133 100644
--- a/pkg/gce-cloud-provider/compute/gce-compute.go
+++ b/pkg/gce-cloud-provider/compute/gce-compute.go
@@ -101,6 +101,7 @@ type GCECompute interface {
 	ValidateExistingDisk(ctx context.Context, disk *CloudDisk, params common.DiskParameters, reqBytes, limBytes int64, multiWriter bool) error
 	InsertDisk(ctx context.Context, project string, volKey *meta.Key, params common.DiskParameters, capBytes int64, capacityRange *csi.CapacityRange, replicaZones []string, snapshotID string, volumeContentSourceVolumeID string, multiWriter bool) error
 	DeleteDisk(ctx context.Context, project string, volumeKey *meta.Key) error
+	UpdateDisk(ctx context.Context, project string, volKey *meta.Key, existingDisk *CloudDisk, params common.ModifyVolumeParameters) error
 	AttachDisk(ctx context.Context, project string, volKey *meta.Key, readWrite, diskType, instanceZone, instanceName string, forceAttach bool) error
 	DetachDisk(ctx context.Context, project, deviceName, instanceZone, instanceName string) error
 	GetDiskSourceURI(project string, volKey *meta.Key) string
@@ -442,6 +443,45 @@ func (cloud *CloudProvider) InsertDisk(ctx context.Context, project string, volK
 	}
 }
 
+func (cloud *CloudProvider) UpdateDisk(ctx context.Context, project string, volKey *meta.Key, existingDisk *CloudDisk, params common.ModifyVolumeParameters) error {
+
+	klog.V(5).Infof("Updating disk %v", volKey)
+
+	switch volKey.Type() {
+	case meta.Zonal:
+		return cloud.updateZonalDisk(ctx, project, volKey, existingDisk, params)
+	case meta.Regional:
+		return cloud.updateRegionalDisk(ctx, project, volKey, existingDisk, params)
+	default:
+		return fmt.Errorf("could not update disk, key was neither zonal nor regional, instead got: %v", volKey.String())
+	}
+}
+
+func (cloud *CloudProvider) updateZonalDisk(ctx context.Context, project string, volKey *meta.Key, existingDisk *CloudDisk, params common.ModifyVolumeParameters) error {
+
+	updatedDisk := &computev1.Disk{
+		Name:                  existingDisk.GetName(),
+		ProvisionedIops:       params.IOPS,
+		ProvisionedThroughput: params.Throughput,
+	}
+
+	diskUpdateOp := cloud.service.Disks.Update(project, volKey.Zone, volKey.Name, updatedDisk)
+	diskUpdateOp.Paths("provisionedIops", "provisionedThroughput")
+	updateOpResult, err := diskUpdateOp.Context(ctx).Do()
+
+	if err != nil {
+		return fmt.Errorf("error updating disk %v: %w", volKey, err)
+	}
+
+	fmt.Printf("http status : %d", updateOpResult.HTTPStatusCode)
+
+	return nil
+}
+
+func (cloud *CloudProvider) updateRegionalDisk(ctx context.Context, project string, volKey *meta.Key, existingDisk *CloudDisk, params common.ModifyVolumeParameters) error {
+	// TODO : Implement this
+	return nil
+}
 func convertV1CustomerEncryptionKeyToBeta(v1Key *computev1.CustomerEncryptionKey) *computebeta.CustomerEncryptionKey {
 	return &computebeta.CustomerEncryptionKey{
 		KmsKeyName:      v1Key.KmsKeyName,
diff --git a/pkg/gce-pd-csi-driver/controller.go b/pkg/gce-pd-csi-driver/controller.go
index 81c0426a3..37eaaf257 100644
--- a/pkg/gce-pd-csi-driver/controller.go
+++ b/pkg/gce-pd-csi-driver/controller.go
@@ -527,6 +527,48 @@ func (gceCS *GCEControllerServer) CreateVolume(ctx context.Context, req *csi.Cre
 
 }
 
+func (gceCS *GCEControllerServer) ControllerModifyVolume(ctx context.Context, req *csi.ControllerModifyVolumeRequest) (*csi.ControllerModifyVolumeResponse, error) {
+
+	volumeID := req.GetVolumeId()
+	klog.V(4).Infof("Modifying Volume ID: %s", volumeID)
+
+	if volumeID == "" {
+		return nil, status.Error(codes.InvalidArgument, "volume ID must be provided")
+	}
+
+	project, volKey, err := common.VolumeIDToKey(volumeID)
+
+	if err != nil {
+		return nil, status.Errorf(codes.InvalidArgument, "volume ID is invalid: %v", err.Error())
+	}
+
+	volumeModifyParams, err := common.ExtractModifyVolumeParameters(req.GetMutableParameters())
+	if err != nil {
+		klog.Errorf("Failed to extract parameters for volume %s: %v", volumeID, err)
+		return nil, status.Errorf(codes.InvalidArgument, "Invalid parameters: %v", err)
+	}
+	klog.V(4).Infof("Modify Volume Parameters for %s: %v", volumeID, volumeModifyParams)
+
+	existingDisk, err := gceCS.CloudProvider.GetDisk(ctx, project, volKey, gce.GCEAPIVersionBeta)
+
+	if err != nil {
+		return nil, status.Errorf(codes.NotFound, "failed to get volume: %v", err)
+	}
+
+	if existingDisk != nil || existingDisk.GetSelfLink() == "" {
+
+		return nil, status.Errorf(codes.Internal, "failed to get volume : %s", volumeID)
+	}
+
+	err = gceCS.CloudProvider.UpdateDisk(ctx, project, volKey, existingDisk, volumeModifyParams)
+	if err != nil {
+		klog.Errorf("Failed to modify volume %s: %v", volumeID, err)
+		return nil, status.Errorf(codes.Internal, "Failed to modify volume %s: %v", volumeID, err)
+	}
+
+	return &csi.ControllerModifyVolumeResponse{}, nil
+}
+
 func (gceCS *GCEControllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) {
 	var err error
 	diskTypeForMetric := metrics.DefaultDiskTypeForMetric
diff --git a/pkg/gce-pd-csi-driver/gce-pd-driver.go b/pkg/gce-pd-csi-driver/gce-pd-driver.go
index ba1a0c0ba..4d12850c5 100644
--- a/pkg/gce-pd-csi-driver/gce-pd-driver.go
+++ b/pkg/gce-pd-csi-driver/gce-pd-driver.go
@@ -73,6 +73,7 @@ func (gceDriver *GCEDriver) SetupGCEDriver(name, vendorVersion string, extraVolu
 		csi.ControllerServiceCapability_RPC_LIST_VOLUMES,
 		csi.ControllerServiceCapability_RPC_LIST_VOLUMES_PUBLISHED_NODES,
 		csi.ControllerServiceCapability_RPC_CLONE_VOLUME,
+		csi.ControllerServiceCapability_RPC_MODIFY_VOLUME,
 	}
 	gceDriver.AddControllerServiceCapabilities(csc)
 	ns := []csi.NodeServiceCapability_RPC_Type{
diff --git a/vendor/modules.txt b/vendor/modules.txt
index a1793c71c..b5db16497 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -70,8 +70,8 @@ github.com/cenkalti/backoff/v4
 # github.com/cespare/xxhash/v2 v2.2.0
 ## explicit; go 1.11
 github.com/cespare/xxhash/v2
-# github.com/container-storage-interface/spec v1.6.0
-## explicit; go 1.16
+# github.com/container-storage-interface/spec v1.9.0
+## explicit; go 1.18
 github.com/container-storage-interface/spec/lib/go/csi
 # github.com/davecgh/go-spew v1.1.1
 ## explicit

From ac50923fa8f654442a7bbdd930c8cfd90c376b10 Mon Sep 17 00:00:00 2001
From: Max Kimambo <max@kimambo.de>
Date: Fri, 14 Jun 2024 09:44:54 +0000
Subject: [PATCH 02/20] updated csi version

---
 go.mod | 2 +-
 go.sum | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/go.mod b/go.mod
index 0430ed2af..aa544f916 100644
--- a/go.mod
+++ b/go.mod
@@ -9,7 +9,7 @@ require (
 	cloud.google.com/go/kms v1.17.1
 	cloud.google.com/go/resourcemanager v1.9.7
 	github.com/GoogleCloudPlatform/k8s-cloud-provider v1.24.0
-	github.com/container-storage-interface/spec v1.6.0
+	github.com/container-storage-interface/spec v1.9.0
 	github.com/google/go-cmp v0.6.0
 	github.com/google/uuid v1.6.0
 	github.com/googleapis/gax-go/v2 v2.12.4
diff --git a/go.sum b/go.sum
index 841ea000d..a5bca44f5 100644
--- a/go.sum
+++ b/go.sum
@@ -895,8 +895,9 @@ github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:z
 github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo=
 github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA=
 github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
-github.com/container-storage-interface/spec v1.6.0 h1:vwN9uCciKygX/a0toYryoYD5+qI9ZFeAMuhEEKO+JBA=
 github.com/container-storage-interface/spec v1.6.0/go.mod h1:8K96oQNkJ7pFcC2R9Z1ynGGBB1I93kcS6PGg3SsOk8s=
+github.com/container-storage-interface/spec v1.9.0 h1:zKtX4STsq31Knz3gciCYCi1SXtO2HJDecIjDVboYavY=
+github.com/container-storage-interface/spec v1.9.0/go.mod h1:ZfDu+3ZRyeVqxZM0Ds19MVLkN2d1XJ5MAfi1L3VjlT0=
 github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko=
 github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=
 github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=

From e6f9f4aa9e570382ed4903b94e0eef3be8a7185b Mon Sep 17 00:00:00 2001
From: Max Kimambo <max@kimambo.de>
Date: Fri, 14 Jun 2024 09:45:22 +0000
Subject: [PATCH 03/20] Add regional disk update capability

---
 pkg/gce-cloud-provider/compute/gce-compute.go | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/pkg/gce-cloud-provider/compute/gce-compute.go b/pkg/gce-cloud-provider/compute/gce-compute.go
index 3b58f0133..cb7d4020f 100644
--- a/pkg/gce-cloud-provider/compute/gce-compute.go
+++ b/pkg/gce-cloud-provider/compute/gce-compute.go
@@ -479,9 +479,24 @@ func (cloud *CloudProvider) updateZonalDisk(ctx context.Context, project string,
 }
 
 func (cloud *CloudProvider) updateRegionalDisk(ctx context.Context, project string, volKey *meta.Key, existingDisk *CloudDisk, params common.ModifyVolumeParameters) error {
-	// TODO : Implement this
+
+	updatedDisk := &computev1.Disk{
+		Name:                  existingDisk.GetName(),
+		ProvisionedIops:       params.IOPS,
+		ProvisionedThroughput: params.Throughput,
+	}
+
+	diskUpdateOp := cloud.service.RegionDisks.Update(project, volKey.Region, volKey.Name, updatedDisk)
+	diskUpdateOp.Paths("provisionedIops", "provisionedThroughput")
+	updateOpResult, err := diskUpdateOp.Context(ctx).Do()
+
+	if err != nil {
+		return fmt.Errorf("error updating disk %v: %w", volKey, err)
+	}
+	fmt.Printf("http status : %d", updateOpResult.HTTPStatusCode)
 	return nil
 }
+
 func convertV1CustomerEncryptionKeyToBeta(v1Key *computev1.CustomerEncryptionKey) *computebeta.CustomerEncryptionKey {
 	return &computebeta.CustomerEncryptionKey{
 		KmsKeyName:      v1Key.KmsKeyName,

From f310525e5d9c76e9b19a1fb7b6d79d8603fecede Mon Sep 17 00:00:00 2001
From: Max Kimambo <max@kimambo.de>
Date: Fri, 14 Jun 2024 11:56:49 +0000
Subject: [PATCH 04/20] Add mutable parameters to create volume

---
 pkg/gce-pd-csi-driver/controller.go | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/pkg/gce-pd-csi-driver/controller.go b/pkg/gce-pd-csi-driver/controller.go
index 37eaaf257..b8d22bc72 100644
--- a/pkg/gce-pd-csi-driver/controller.go
+++ b/pkg/gce-pd-csi-driver/controller.go
@@ -317,6 +317,19 @@ func (gceCS *GCEControllerServer) CreateVolume(ctx context.Context, req *csi.Cre
 	if err != nil {
 		return nil, status.Errorf(codes.InvalidArgument, "failed to extract parameters: %v", err.Error())
 	}
+
+	// https://github.com/container-storage-interface/spec/blob/master/spec.md#createvolume
+	// mutable_parameters MUST take precedence over the values from parameters.
+	mutableParams := req.GetMutableParameters()
+	if mutableParams != nil {
+		p, err := common.ExtractModifyVolumeParameters(mutableParams)
+		if err != nil {
+			return nil, status.Errorf(codes.InvalidArgument, "Invalid mutable parameters: %v", err)
+		}
+		params.ProvisionedIOPSOnCreate = p.IOPS
+		params.ProvisionedThroughputOnCreate = p.Throughput
+	}
+
 	// Determine multiWriter
 	gceAPIVersion := gce.GCEAPIVersionV1
 	multiWriter, _ := getMultiWriterFromCapabilities(volumeCapabilities)

From 97ee9511cb1afcb101c2f8f57771f0462deef8b6 Mon Sep 17 00:00:00 2001
From: Max Kimambo <max@kimambo.de>
Date: Fri, 14 Jun 2024 12:17:26 +0000
Subject: [PATCH 05/20] modify update to remove regional disk check

---
 pkg/gce-cloud-provider/compute/gce-compute.go | 31 ++-----------------
 1 file changed, 3 insertions(+), 28 deletions(-)

diff --git a/pkg/gce-cloud-provider/compute/gce-compute.go b/pkg/gce-cloud-provider/compute/gce-compute.go
index cb7d4020f..7b4eef000 100644
--- a/pkg/gce-cloud-provider/compute/gce-compute.go
+++ b/pkg/gce-cloud-provider/compute/gce-compute.go
@@ -446,15 +446,9 @@ func (cloud *CloudProvider) InsertDisk(ctx context.Context, project string, volK
 func (cloud *CloudProvider) UpdateDisk(ctx context.Context, project string, volKey *meta.Key, existingDisk *CloudDisk, params common.ModifyVolumeParameters) error {
 
 	klog.V(5).Infof("Updating disk %v", volKey)
-
-	switch volKey.Type() {
-	case meta.Zonal:
-		return cloud.updateZonalDisk(ctx, project, volKey, existingDisk, params)
-	case meta.Regional:
-		return cloud.updateRegionalDisk(ctx, project, volKey, existingDisk, params)
-	default:
-		return fmt.Errorf("could not update disk, key was neither zonal nor regional, instead got: %v", volKey.String())
-	}
+	// hyperdisks are zonal disks
+	// pd-disks do not support modification of IOPS and Throughput
+	return cloud.updateZonalDisk(ctx, project, volKey, existingDisk, params)
 }
 
 func (cloud *CloudProvider) updateZonalDisk(ctx context.Context, project string, volKey *meta.Key, existingDisk *CloudDisk, params common.ModifyVolumeParameters) error {
@@ -478,25 +472,6 @@ func (cloud *CloudProvider) updateZonalDisk(ctx context.Context, project string,
 	return nil
 }
 
-func (cloud *CloudProvider) updateRegionalDisk(ctx context.Context, project string, volKey *meta.Key, existingDisk *CloudDisk, params common.ModifyVolumeParameters) error {
-
-	updatedDisk := &computev1.Disk{
-		Name:                  existingDisk.GetName(),
-		ProvisionedIops:       params.IOPS,
-		ProvisionedThroughput: params.Throughput,
-	}
-
-	diskUpdateOp := cloud.service.RegionDisks.Update(project, volKey.Region, volKey.Name, updatedDisk)
-	diskUpdateOp.Paths("provisionedIops", "provisionedThroughput")
-	updateOpResult, err := diskUpdateOp.Context(ctx).Do()
-
-	if err != nil {
-		return fmt.Errorf("error updating disk %v: %w", volKey, err)
-	}
-	fmt.Printf("http status : %d", updateOpResult.HTTPStatusCode)
-	return nil
-}
-
 func convertV1CustomerEncryptionKeyToBeta(v1Key *computev1.CustomerEncryptionKey) *computebeta.CustomerEncryptionKey {
 	return &computebeta.CustomerEncryptionKey{
 		KmsKeyName:      v1Key.KmsKeyName,

From e53aee4ee68611c802066277223ba187227c62ad Mon Sep 17 00:00:00 2001
From: Max Kimambo <max@kimambo.de>
Date: Fri, 14 Jun 2024 13:35:17 +0000
Subject: [PATCH 06/20] clean up

---
 pkg/gce-cloud-provider/compute/gce-compute.go | 4 +---
 pkg/gce-pd-csi-driver/controller.go           | 2 +-
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/pkg/gce-cloud-provider/compute/gce-compute.go b/pkg/gce-cloud-provider/compute/gce-compute.go
index 7b4eef000..1e3d355c8 100644
--- a/pkg/gce-cloud-provider/compute/gce-compute.go
+++ b/pkg/gce-cloud-provider/compute/gce-compute.go
@@ -461,14 +461,12 @@ func (cloud *CloudProvider) updateZonalDisk(ctx context.Context, project string,
 
 	diskUpdateOp := cloud.service.Disks.Update(project, volKey.Zone, volKey.Name, updatedDisk)
 	diskUpdateOp.Paths("provisionedIops", "provisionedThroughput")
-	updateOpResult, err := diskUpdateOp.Context(ctx).Do()
+	_, err := diskUpdateOp.Context(ctx).Do()
 
 	if err != nil {
 		return fmt.Errorf("error updating disk %v: %w", volKey, err)
 	}
 
-	fmt.Printf("http status : %d", updateOpResult.HTTPStatusCode)
-
 	return nil
 }
 
diff --git a/pkg/gce-pd-csi-driver/controller.go b/pkg/gce-pd-csi-driver/controller.go
index b8d22bc72..5a8be7a99 100644
--- a/pkg/gce-pd-csi-driver/controller.go
+++ b/pkg/gce-pd-csi-driver/controller.go
@@ -568,7 +568,7 @@ func (gceCS *GCEControllerServer) ControllerModifyVolume(ctx context.Context, re
 		return nil, status.Errorf(codes.NotFound, "failed to get volume: %v", err)
 	}
 
-	if existingDisk != nil || existingDisk.GetSelfLink() == "" {
+	if existingDisk == nil || existingDisk.GetSelfLink() == "" {
 
 		return nil, status.Errorf(codes.Internal, "failed to get volume : %s", volumeID)
 	}

From 36779354c8e5cc2289abd4438181a08f9d5244a6 Mon Sep 17 00:00:00 2001
From: Max Kimambo <max@kimambo.de>
Date: Fri, 14 Jun 2024 13:59:17 +0000
Subject: [PATCH 07/20] Add controller modify volume metric

---
 pkg/gce-pd-csi-driver/controller.go | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/pkg/gce-pd-csi-driver/controller.go b/pkg/gce-pd-csi-driver/controller.go
index 5a8be7a99..74b4069a8 100644
--- a/pkg/gce-pd-csi-driver/controller.go
+++ b/pkg/gce-pd-csi-driver/controller.go
@@ -541,6 +541,7 @@ func (gceCS *GCEControllerServer) CreateVolume(ctx context.Context, req *csi.Cre
 }
 
 func (gceCS *GCEControllerServer) ControllerModifyVolume(ctx context.Context, req *csi.ControllerModifyVolumeRequest) (*csi.ControllerModifyVolumeResponse, error) {
+	var err error
 
 	volumeID := req.GetVolumeId()
 	klog.V(4).Infof("Modifying Volume ID: %s", volumeID)
@@ -549,6 +550,14 @@ func (gceCS *GCEControllerServer) ControllerModifyVolume(ctx context.Context, re
 		return nil, status.Error(codes.InvalidArgument, "volume ID must be provided")
 	}
 
+	diskTypeForMetric := metrics.DefaultDiskTypeForMetric
+	enableConfidentialCompute := metrics.DefaultEnableConfidentialCompute
+	enableStoragePools := metrics.DefaultEnableStoragePools
+
+	defer func() {
+		gceCS.Metrics.RecordOperationErrorMetrics("ModifyVolume", err, diskTypeForMetric, enableConfidentialCompute, enableStoragePools)
+	}()
+
 	project, volKey, err := common.VolumeIDToKey(volumeID)
 
 	if err != nil {
@@ -572,6 +581,12 @@ func (gceCS *GCEControllerServer) ControllerModifyVolume(ctx context.Context, re
 
 		return nil, status.Errorf(codes.Internal, "failed to get volume : %s", volumeID)
 	}
+	diskTypeForMetric = existingDisk.GetPDType()
+	if existingDisk.GetEnableStoragePools() {
+		enableStoragePools = "enabled"
+	} else {
+		enableStoragePools = "disabled"
+	}
 
 	err = gceCS.CloudProvider.UpdateDisk(ctx, project, volKey, existingDisk, volumeModifyParams)
 	if err != nil {

From 38715f3bd3b6eeaa73d2ecc76c92808156cce4fb Mon Sep 17 00:00:00 2001
From: Max Kimambo <max@kimambo.de>
Date: Fri, 14 Jun 2024 14:12:58 +0000
Subject: [PATCH 08/20] add modify volume capability to sanity tests

---
 .../kubernetes-csi/csi-test/v4/pkg/sanity/controller.go          | 1 +
 1 file changed, 1 insertion(+)

diff --git a/vendor/github.com/kubernetes-csi/csi-test/v4/pkg/sanity/controller.go b/vendor/github.com/kubernetes-csi/csi-test/v4/pkg/sanity/controller.go
index 4dfaeb943..e7174d6c0 100644
--- a/vendor/github.com/kubernetes-csi/csi-test/v4/pkg/sanity/controller.go
+++ b/vendor/github.com/kubernetes-csi/csi-test/v4/pkg/sanity/controller.go
@@ -148,6 +148,7 @@ var _ = DescribeSanity("Controller Service [Controller Server]", func(sc *TestCo
 				case csi.ControllerServiceCapability_RPC_GET_VOLUME:
 				case csi.ControllerServiceCapability_RPC_VOLUME_CONDITION:
 				case csi.ControllerServiceCapability_RPC_SINGLE_NODE_MULTI_WRITER:
+				case csi.ControllerServiceCapability_RPC_MODIFY_VOLUME:
 				default:
 					Fail(fmt.Sprintf("Unknown capability: %v\n", cap.GetRpc().GetType()))
 				}

From 799ca648886c7373d6c11c4027121561f68c0217 Mon Sep 17 00:00:00 2001
From: Max Kimambo <max@kimambo.de>
Date: Mon, 17 Jun 2024 13:52:56 +0000
Subject: [PATCH 09/20] ensure mutable params are not applied to pd- disk types

---
 pkg/gce-pd-csi-driver/controller.go | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/pkg/gce-pd-csi-driver/controller.go b/pkg/gce-pd-csi-driver/controller.go
index 74b4069a8..caad07d2c 100644
--- a/pkg/gce-pd-csi-driver/controller.go
+++ b/pkg/gce-pd-csi-driver/controller.go
@@ -321,13 +321,16 @@ func (gceCS *GCEControllerServer) CreateVolume(ctx context.Context, req *csi.Cre
 	// https://github.com/container-storage-interface/spec/blob/master/spec.md#createvolume
 	// mutable_parameters MUST take precedence over the values from parameters.
 	mutableParams := req.GetMutableParameters()
-	if mutableParams != nil {
+	// If the disk type is pd-*, the IOPS and Throughput parameters are ignored.
+	if mutableParams != nil && !strings.HasPrefix(params.DiskType, "pd") {
 		p, err := common.ExtractModifyVolumeParameters(mutableParams)
 		if err != nil {
 			return nil, status.Errorf(codes.InvalidArgument, "Invalid mutable parameters: %v", err)
 		}
 		params.ProvisionedIOPSOnCreate = p.IOPS
 		params.ProvisionedThroughputOnCreate = p.Throughput
+	} else {
+		klog.V(4).Infof("Ignoring IOPS and throughput parameters for unsupported disk type %s", params.DiskType)
 	}
 
 	// Determine multiWriter
@@ -550,12 +553,12 @@ func (gceCS *GCEControllerServer) ControllerModifyVolume(ctx context.Context, re
 		return nil, status.Error(codes.InvalidArgument, "volume ID must be provided")
 	}
 
-	diskTypeForMetric := metrics.DefaultDiskTypeForMetric
+	diskType := metrics.DefaultDiskTypeForMetric
 	enableConfidentialCompute := metrics.DefaultEnableConfidentialCompute
 	enableStoragePools := metrics.DefaultEnableStoragePools
 
 	defer func() {
-		gceCS.Metrics.RecordOperationErrorMetrics("ModifyVolume", err, diskTypeForMetric, enableConfidentialCompute, enableStoragePools)
+		gceCS.Metrics.RecordOperationErrorMetrics("ModifyVolume", err, diskType, enableConfidentialCompute, enableStoragePools)
 	}()
 
 	project, volKey, err := common.VolumeIDToKey(volumeID)
@@ -581,7 +584,8 @@ func (gceCS *GCEControllerServer) ControllerModifyVolume(ctx context.Context, re
 
 		return nil, status.Errorf(codes.Internal, "failed to get volume : %s", volumeID)
 	}
-	diskTypeForMetric = existingDisk.GetPDType()
+	diskType = existingDisk.GetPDType()
+	// TODO : not sure how this metric should be reported
 	if existingDisk.GetEnableStoragePools() {
 		enableStoragePools = "enabled"
 	} else {

From 8c533de96a5489c37eb92e8a0d033b95485185ef Mon Sep 17 00:00:00 2001
From: Max Kimambo <max@kimambo.de>
Date: Thu, 20 Jun 2024 16:33:17 +0000
Subject: [PATCH 10/20] Adding tests for controller modify volume

---
 pkg/gce-cloud-provider/compute/cloud-disk.go  | 22 ++++++
 pkg/gce-cloud-provider/compute/fake-gce.go    | 13 +++-
 pkg/gce-cloud-provider/compute/gce-compute.go |  3 +
 pkg/gce-pd-csi-driver/controller_test.go      | 78 +++++++++++++++++++
 4 files changed, 115 insertions(+), 1 deletion(-)

diff --git a/pkg/gce-cloud-provider/compute/cloud-disk.go b/pkg/gce-cloud-provider/compute/cloud-disk.go
index 02b70d5df..6790992c4 100644
--- a/pkg/gce-cloud-provider/compute/cloud-disk.go
+++ b/pkg/gce-cloud-provider/compute/cloud-disk.go
@@ -267,3 +267,25 @@ func (d *CloudDisk) GetAccessMode() string {
 		return ""
 	}
 }
+
+func (d *CloudDisk) GetProvisionedIops() int64 {
+	switch {
+	case d.disk != nil:
+		return d.disk.ProvisionedIops
+	case d.betaDisk != nil:
+		return d.betaDisk.ProvisionedIops
+	default:
+		return 0
+	}
+}
+
+func (d *CloudDisk) GetProvisionedThroughput() int64 {
+	switch {
+	case d.disk != nil:
+		return d.disk.ProvisionedThroughput
+	case d.betaDisk != nil:
+		return d.betaDisk.ProvisionedThroughput
+	default:
+		return 0
+	}
+}
diff --git a/pkg/gce-cloud-provider/compute/fake-gce.go b/pkg/gce-cloud-provider/compute/fake-gce.go
index fadd35dd6..d29226afa 100644
--- a/pkg/gce-cloud-provider/compute/fake-gce.go
+++ b/pkg/gce-cloud-provider/compute/fake-gce.go
@@ -281,7 +281,18 @@ func (cloud *FakeCloudProvider) InsertDisk(ctx context.Context, project string,
 
 func (cloud *FakeCloudProvider) UpdateDisk(ctx context.Context, project string, volKey *meta.Key, existingDisk *CloudDisk, params common.ModifyVolumeParameters) error {
 
-	return nil
+	if params.IOPS == 0 || params.Throughput == 0 {
+		return fmt.Errorf("no IOPS or Throughput specified for disk %v", existingDisk.GetSelfLink())
+	}
+
+	if _, ok := cloud.disks[volKey.String()]; ok {
+		existingDisk.betaDisk.ProvisionedIops = params.IOPS
+		existingDisk.betaDisk.ProvisionedThroughput = params.Throughput
+		cloud.disks[volKey.String()] = existingDisk
+		return nil
+	}
+
+	return fmt.Errorf("disk %v not found", volKey)
 }
 
 func (cloud *FakeCloudProvider) DeleteDisk(ctx context.Context, project string, volKey *meta.Key) error {
diff --git a/pkg/gce-cloud-provider/compute/gce-compute.go b/pkg/gce-cloud-provider/compute/gce-compute.go
index ac95a1fec..21ec3ed3b 100644
--- a/pkg/gce-cloud-provider/compute/gce-compute.go
+++ b/pkg/gce-cloud-provider/compute/gce-compute.go
@@ -470,6 +470,9 @@ func (cloud *CloudProvider) UpdateDisk(ctx context.Context, project string, volK
 
 func (cloud *CloudProvider) updateZonalDisk(ctx context.Context, project string, volKey *meta.Key, existingDisk *CloudDisk, params common.ModifyVolumeParameters) error {
 
+	if params.IOPS == 0 && params.Throughput == 0 {
+		return fmt.Errorf("no IOPS or Throughput specified for disk %v", existingDisk.GetSelfLink())
+	}
 	updatedDisk := &computev1.Disk{
 		Name:                  existingDisk.GetName(),
 		ProvisionedIops:       params.IOPS,
diff --git a/pkg/gce-pd-csi-driver/controller_test.go b/pkg/gce-pd-csi-driver/controller_test.go
index e196a1d3c..3197c7c0d 100644
--- a/pkg/gce-pd-csi-driver/controller_test.go
+++ b/pkg/gce-pd-csi-driver/controller_test.go
@@ -22,6 +22,7 @@ import (
 	"reflect"
 	"sort"
 	"strconv"
+	"strings"
 	"testing"
 	"time"
 
@@ -1711,6 +1712,83 @@ func TestMultiZoneVolumeCreationErrHandling(t *testing.T) {
 	}
 }
 
+func TestVolumeUpdateOperation(t *testing.T) {
+
+	testCases := []struct {
+		name          string
+		req           *csi.ControllerModifyVolumeRequest
+		expIops       int64
+		expThroughput int64
+		expErrMessage string
+	}{
+		{
+			name: "Update volume with valid parameters",
+			req: &csi.ControllerModifyVolumeRequest{
+				VolumeId:          testVolumeID,
+				MutableParameters: map[string]string{"iops": "20000", "throughput": "600"},
+			},
+			expIops:       20000,
+			expThroughput: 600,
+			expErrMessage: "",
+		},
+		{
+			name: "Update volume with invalid parameters",
+			req: &csi.ControllerModifyVolumeRequest{
+				VolumeId:          testVolumeID,
+				MutableParameters: map[string]string{"iops": "0", "throughput": "0"},
+			},
+			expIops:       10000,
+			expThroughput: 500,
+			expErrMessage: "no IOPS or Throughput specified for disk",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Logf("test case: %s", tc.name)
+
+		fcp, err := gce.CreateFakeCloudProvider(project, zone, nil)
+
+		if err != nil {
+			t.Fatalf("Failed to create mock cloud provider: %v", err)
+		}
+
+		gceDriver := initGCEDriverWithCloudProvider(t, fcp)
+		// Arrange
+		project, volKey, err := common.VolumeIDToKey(testVolumeID)
+		if err != nil {
+			t.Fatalf("Failed convert key: %v", err)
+		}
+
+		testDiskParams := &common.DiskParameters{
+			DiskType:                      "hyperdisk-balanced",
+			ProvisionedIOPSOnCreate:       10000,
+			ProvisionedThroughputOnCreate: 500,
+		}
+
+		err = fcp.InsertDisk(context.Background(), project, volKey, *testDiskParams, 20, nil, nil, "", "", false, "")
+		if err != nil {
+			t.Fatalf("Failed to insert disk: %v", err)
+		}
+		_, err = gceDriver.cs.ControllerModifyVolume(context.Background(), tc.req)
+
+		if err != nil && !strings.Contains(err.Error(), tc.expErrMessage) {
+			t.Errorf("Failed to modify volume: %v", err)
+		}
+
+		modifiedVol, err := fcp.GetDisk(context.Background(), project, volKey, gce.GCEAPIVersionBeta)
+
+		if err != nil {
+			t.Errorf("Failed to get volume: %v", err)
+		}
+
+		diskIops := modifiedVol.GetProvisionedIops()
+		throughput := modifiedVol.GetProvisionedThroughput()
+
+		if diskIops != tc.expIops && throughput != tc.expThroughput {
+			t.Errorf("Failed to modify volume: %v", err)
+		}
+	}
+}
 func TestListVolumePagination(t *testing.T) {
 	testCases := []struct {
 		name            string

From cd1c0f23b200f649ec3c2112d0daac1a5cd96ec9 Mon Sep 17 00:00:00 2001
From: Max Kimambo <max@kimambo.de>
Date: Thu, 20 Jun 2024 16:33:17 +0000
Subject: [PATCH 11/20] Adding tests for controller modify volume

---
 pkg/common/parameters_test.go | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/pkg/common/parameters_test.go b/pkg/common/parameters_test.go
index 75f1c2a16..f188c02d0 100644
--- a/pkg/common/parameters_test.go
+++ b/pkg/common/parameters_test.go
@@ -482,3 +482,23 @@ func TestSnapshotParameters(t *testing.T) {
 		})
 	}
 }
+func TestExtractModifyVolumeParameters(t *testing.T) {
+	parameters := map[string]string{
+		"iops":       "1000",
+		"throughput": "500",
+	}
+
+	expected := ModifyVolumeParameters{
+		IOPS:       1000,
+		Throughput: 500,
+	}
+
+	result, err := ExtractModifyVolumeParameters(parameters)
+	if err != nil {
+		t.Errorf("Unexpected error: %v", err)
+	}
+
+	if !reflect.DeepEqual(result, expected) {
+		t.Errorf("Expected: %v, but got: %v", expected, result)
+	}
+}

From fe328941f7a414f7fe1b927fb0733169a61187eb Mon Sep 17 00:00:00 2001
From: Max Kimambo <max@kimambo.de>
Date: Tue, 9 Jul 2024 18:13:32 +0000
Subject: [PATCH 12/20] current state

---
 .vscode/launch.json                           |   33 +
 deploy/kubernetes/install-kustomize.sh        |    8 +-
 .../overlays/noauth-debug/kustomization.yaml  |    9 +-
 pkg/gce-pd-csi-driver/controller.go           |   25 +-
 test/k8s-integration/cluster.go               |   12 +-
 test/k8s-integration/driver.go                |    5 +-
 test/k8s-integration/main.go                  |    4 +-
 test/run-k8s-integration-local.sh             |   22 +-
 .../spec/lib/go/csi/csi.pb.go                 | 1939 +++++++++++++----
 .../csi-test/v4/pkg/sanity/controller.go      |    1 -
 10 files changed, 1610 insertions(+), 448 deletions(-)
 create mode 100644 .vscode/launch.json

diff --git a/.vscode/launch.json b/.vscode/launch.json
new file mode 100644
index 000000000..379925b42
--- /dev/null
+++ b/.vscode/launch.json
@@ -0,0 +1,33 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Launch Package",
+            "type": "go",
+            "request": "launch",
+            "mode": "auto",
+            "program": "${fileDirname}",
+            "args" : ["--run-in-prow", "false", "--gke-cluster-prefix", "csitest-", 
+            "--gke-is-alpha", "true", "--machine-type", "n4-standard-8", "--deploy-overlay-name", "noauth-debug",
+            "--staging-image", "europe-west3-docker.pkg.dev/kimambo-sandbox/csi-dev/gce-pd-csi-driver", 
+            "--service-account-file", "/usr/local/google/home/kimambo/dev/go/src/github.com/maxkimambo/creds/cloud-sa.json", 
+            "--deploy-overlay-name", "noauth-debug", "--storageclass-files", "sc-standard.yaml,sc-balanced.yaml,sc-ssd.yaml", 
+            "--test-focus", "External.Storage", "--gce-zone", "us-central1-b", "--deployment-strategy", "gke", "--gke-cluster-version", "${gke_cluster_version}", "--test-version", "${test_version}", "--num-nodes", "3"]
+        },
+        {
+            "name": "Cluster",
+            "type": "go",
+            "request": "attach",
+            "mode": "remote",
+            "debugAdapter": "dlv-dap",
+            "substitutePath": [
+                {"from": "${workspaceFolder}", "to": "/go/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver"}
+            ],
+            "port": 2345,
+            "host": "127.0.0.1"
+        }, 
+    ]
+}
\ No newline at end of file
diff --git a/deploy/kubernetes/install-kustomize.sh b/deploy/kubernetes/install-kustomize.sh
index eaeb8ebbe..aaedfa164 100755
--- a/deploy/kubernetes/install-kustomize.sh
+++ b/deploy/kubernetes/install-kustomize.sh
@@ -31,11 +31,11 @@ if [[ ! "$tmpDir" || ! -d "$tmpDir" ]]; then
   exit 1
 fi
 
-function cleanup {
-  rm -rf "$tmpDir"
-}
+# function cleanup {
+#   rm -rf "$tmpDir"
+# }
 
-trap cleanup EXIT
+# trap cleanup EXIT
 
 pushd $tmpDir >& /dev/null
 
diff --git a/deploy/kubernetes/overlays/noauth-debug/kustomization.yaml b/deploy/kubernetes/overlays/noauth-debug/kustomization.yaml
index d85ab5e71..881acc6ca 100644
--- a/deploy/kubernetes/overlays/noauth-debug/kustomization.yaml
+++ b/deploy/kubernetes/overlays/noauth-debug/kustomization.yaml
@@ -10,8 +10,7 @@ patchesStrategicMerge:
 - controller-overlay.yaml
 - node-overlay.yaml
 namespace: gce-pd-csi-driver
-# To change the dev image, add something like the following.
-# images:
-# - name: gke.gcr.io/gcp-compute-persistent-disk-csi-driver
-#   newName: gcr.io/mauriciopoppe-gke-dev/gcp-compute-persistent-disk-csi-driver
-#   newTag: latest
+images:
+- name: gke.gcr.io/gcp-compute-persistent-disk-csi-driver
+  newName: europe-west3-docker.pkg.dev/kimambo-sandbox/csi-dev/gce-pd-csi-driver
+  newTag: dev_linux
diff --git a/pkg/gce-pd-csi-driver/controller.go b/pkg/gce-pd-csi-driver/controller.go
index 64687272d..d7a7aa263 100644
--- a/pkg/gce-pd-csi-driver/controller.go
+++ b/pkg/gce-pd-csi-driver/controller.go
@@ -331,17 +331,21 @@ func (gceCS *GCEControllerServer) createVolumeInternal(ctx context.Context, req
 	// https://github.com/container-storage-interface/spec/blob/master/spec.md#createvolume
 	// mutable_parameters MUST take precedence over the values from parameters.
 	mutableParams := req.GetMutableParameters()
+
+	klog.V(4).Infof("CreateVolume parameters: %v", params)
+	klog.V(4).Infof("CreateVolume mutable parameters: %v", mutableParams)
+	klog.V(4).Infof("CreateVolume disk type: %v", diskTypeForMetric)
 	// If the disk type is pd-*, the IOPS and Throughput parameters are ignored.
-	if mutableParams != nil && !strings.HasPrefix(params.DiskType, "pd") {
-		p, err := common.ExtractModifyVolumeParameters(mutableParams)
-		if err != nil {
-			return nil, status.Errorf(codes.InvalidArgument, "Invalid mutable parameters: %v", err)
-		}
-		params.ProvisionedIOPSOnCreate = p.IOPS
-		params.ProvisionedThroughputOnCreate = p.Throughput
-	} else {
-		klog.V(4).Infof("Ignoring IOPS and throughput parameters for unsupported disk type %s", params.DiskType)
-	}
+	// if !strings.HasPrefix(params.DiskType, "pd-") {
+	// 	p, err := common.ExtractModifyVolumeParameters(mutableParams)
+	// 	if err != nil {
+	// 		return nil, status.Errorf(codes.InvalidArgument, "Invalid mutable parameters: %v", err)
+	// 	}
+	// 	params.ProvisionedIOPSOnCreate = p.IOPS
+	// 	params.ProvisionedThroughputOnCreate = p.Throughput
+	// } else {
+	// 	klog.V(4).Infof("Ignoring IOPS and throughput parameters for unsupported disk type %s", params.DiskType)
+	// }
 
 	if _, err := getMultiWriterFromCapabilities(volumeCapabilities); err != nil {
 		return nil, status.Errorf(codes.InvalidArgument, "VolumeCapabilities is invalid: %v", err.Error())
@@ -754,6 +758,7 @@ func (gceCS *GCEControllerServer) ControllerModifyVolume(ctx context.Context, re
 	}
 
 	volumeModifyParams, err := common.ExtractModifyVolumeParameters(req.GetMutableParameters())
+	klog.V(4).Info("Volume Modify Parameters: ", volumeModifyParams)
 	if err != nil {
 		klog.Errorf("Failed to extract parameters for volume %s: %v", volumeID, err)
 		return nil, status.Errorf(codes.InvalidArgument, "Invalid parameters: %v", err)
diff --git a/test/k8s-integration/cluster.go b/test/k8s-integration/cluster.go
index 5f63cbaaf..12bcffd6d 100644
--- a/test/k8s-integration/cluster.go
+++ b/test/k8s-integration/cluster.go
@@ -7,6 +7,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"slices"
 	"strconv"
 	"strings"
 
@@ -159,7 +160,7 @@ func setImageTypeEnvs(imageType string) error {
 	return nil
 }
 
-func clusterUpGKE(gceZone, gceRegion string, numNodes int, numWindowsNodes int, imageType string, useManagedDriver bool) error {
+func clusterUpGKE(gceZone, gceRegion string, numNodes int, numWindowsNodes int, machineType string, isAlpha bool, imageType string, useManagedDriver bool) error {
 	locationArg, locationVal, err := gkeLocationArgs(gceZone, gceRegion)
 	if err != nil {
 		return err
@@ -181,9 +182,16 @@ func clusterUpGKE(gceZone, gceRegion string, numNodes int, numWindowsNodes int,
 	}
 
 	var cmd *exec.Cmd
+
 	cmdParams := []string{"container", "clusters", "create", *gkeTestClusterName,
 		locationArg, locationVal, "--num-nodes", strconv.Itoa(numNodes),
-		"--quiet", "--machine-type", "n1-standard-2", "--image-type", imageType, "--no-enable-autoupgrade"}
+		"--quiet", "--machine-type", machineType, "--image-type", imageType, "--no-enable-autoupgrade", "--service-account", "csi-dev@kimambo-sandbox.iam.gserviceaccount.com"}
+
+	if isAlpha {
+		// prepend beta to be used with gcloud beta container clusters create
+		cmdParams = slices.Insert(cmdParams, 0, "beta")
+		cmdParams = append(cmdParams, "--enable-kubernetes-alpha")
+	}
 	if isVariableSet(gkeClusterVer) {
 		cmdParams = append(cmdParams, "--cluster-version", *gkeClusterVer)
 	} else {
diff --git a/test/k8s-integration/driver.go b/test/k8s-integration/driver.go
index 0481e8ad6..e4dbebfb2 100644
--- a/test/k8s-integration/driver.go
+++ b/test/k8s-integration/driver.go
@@ -17,14 +17,17 @@ func getOverlayDir(pkgDir, deployOverlayName string) string {
 }
 
 func installDriver(testParams *testParameters, stagingImage, deployOverlayName string, doDriverBuild bool) error {
+	fmt.Println("Doing driver build and install")
 	if doDriverBuild {
 		// Install kustomize
 		klog.Infof("Installing kustomize")
 		out, err := exec.Command(filepath.Join(testParams.pkgDir, "deploy", "kubernetes", "install-kustomize.sh")).CombinedOutput()
+		klog.Infof("Done Installing kustomize")
 		if err != nil {
 			return fmt.Errorf("failed to install kustomize: %s, err: %v", out, err.Error())
 		}
-
+		
+		klog.Infof("Applying overlays")
 		// Edit ci kustomization to use given image tag
 		overlayDir := getOverlayDir(testParams.pkgDir, deployOverlayName)
 		err = os.Chdir(overlayDir)
diff --git a/test/k8s-integration/main.go b/test/k8s-integration/main.go
index b23815e3e..a1bde9151 100644
--- a/test/k8s-integration/main.go
+++ b/test/k8s-integration/main.go
@@ -54,7 +54,9 @@ var (
 	gkeTestClusterPrefix = flag.String("gke-cluster-prefix", "pdcsi", "Prefix of GKE cluster names. A random suffix will be appended to form the full name.")
 	gkeTestClusterName   = flag.String("gke-cluster-name", "", "Name of existing cluster")
 	gkeNodeVersion       = flag.String("gke-node-version", "", "GKE cluster worker node version")
+	gkeIsAlpha           = flag.Bool("gke-is-alpha", false, "Deploy alpha GKE cluster")
 	isRegionalCluster    = flag.Bool("is-regional-cluster", false, "tell the test that a regional cluster is being used. Should be used for running on an existing regional cluster (ie, --bringup-cluster=false). The test will fail if a zonal GKE cluster is created when this flag is true")
+	machineType          = flag.String("machine-type", "n1-standard-2", "machine type to use for the cluster")
 
 	// Test infrastructure flags
 	boskosResourceType = flag.String("boskos-resource-type", "gce-project", "name of the boskos resource type to reserve")
@@ -366,7 +368,7 @@ func handle() error {
 		case "gce":
 			err = clusterUpGCE(testParams.k8sSourceDir, *gceZone, *numNodes, *numWindowsNodes, testParams.imageType)
 		case "gke":
-			err = clusterUpGKE(*gceZone, *gceRegion, *numNodes, *numWindowsNodes, testParams.imageType, testParams.useGKEManagedDriver)
+			err = clusterUpGKE(*gceZone, *gceRegion, *numNodes, *numWindowsNodes, *machineType, *gkeIsAlpha, testParams.imageType, testParams.useGKEManagedDriver)
 		default:
 			err = fmt.Errorf("deployment-strategy must be set to 'gce' or 'gke', but is: %s", testParams.deploymentStrategy)
 		}
diff --git a/test/run-k8s-integration-local.sh b/test/run-k8s-integration-local.sh
index c551b0212..c30f00c61 100755
--- a/test/run-k8s-integration-local.sh
+++ b/test/run-k8s-integration-local.sh
@@ -15,12 +15,14 @@ make -C "${PKGDIR}" test-k8s-integration
 # This version of the command creates a GKE cluster. It also downloads and builds a k8s release
 # so that it can run the test specified
 
-# ${PKGDIR}/bin/k8s-integration-test --run-in-prow=false \
-# --staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \
-# --deploy-overlay-name=dev --storageclass-files=sc-standard.yaml,sc-balanced.yaml,sc-ssd.yaml \
-# --test-focus="External.Storage" --gce-zone="us-central1-b" \
-# --deployment-strategy=gke --gke-cluster-version=${gke_cluster_version} \
-# --test-version=${test_version} --num-nodes=3
+${PKGDIR}/bin/k8s-integration-test --run-in-prow=false --do-driver-build=true \
+--gke-cluster-prefix="csitest-" --gke-is-alpha=true --machine-type "n4-standard-8" \
+--gke-cluster-version=1.30.1-gke.1329000 \
+--staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \
+--deploy-overlay-name=noauth-debug --storageclass-files=sc-standard.yaml,sc-balanced.yaml,sc-ssd.yaml \
+--test-focus="External.Storage" --gce-zone="europe-west1-c" \
+--deployment-strategy=gke --gke-cluster-version=${gke_cluster_version} \
+--test-version=${TEST_VERSION} --num-nodes=3 --teardown-driver=false 
 
 # This version of the command creates a GCE cluster. It downloads and builds two k8s releases,
 # one for the cluster and one for the tests, unless the cluster and test versioning is the same.
@@ -94,10 +96,10 @@ make -C "${PKGDIR}" test-k8s-integration
 #
 # As with all other methods local credentials must be set by running 
 #  gcloud auth application-default login
-"${PKGDIR}/bin/k8s-integration-test" --run-in-prow=false \
---deploy-overlay-name=noauth --bringup-cluster=false --teardown-cluster=false --local-k8s-dir="$KTOP" \
---storageclass-files=sc-standard.yaml --do-driver-build=false --test-focus='External.Storage' \
---gce-zone="us-central1-b" --num-nodes="${NUM_NODES:-3}"
+# "${PKGDIR}/bin/k8s-integration-test" --run-in-prow=false \
+# --deploy-overlay-name=noauth --bringup-cluster=false --teardown-cluster=false --local-k8s-dir="$KTOP" \
+# --storageclass-files=sc-standard.yaml --do-driver-build=false --test-focus='External.Storage' \
+# --gce-zone="us-central1-b" --num-nodes="${NUM_NODES:-3}"
 
 
 # This version of the command does not build the driver or K8s, points to a
diff --git a/vendor/github.com/container-storage-interface/spec/lib/go/csi/csi.pb.go b/vendor/github.com/container-storage-interface/spec/lib/go/csi/csi.pb.go
index 3df30fa82..ad6cc651c 100644
--- a/vendor/github.com/container-storage-interface/spec/lib/go/csi/csi.pb.go
+++ b/vendor/github.com/container-storage-interface/spec/lib/go/csi/csi.pb.go
@@ -47,18 +47,28 @@ const (
 	// returned by NodeGetInfo to ensure that a given volume is
 	// accessible from a given node when scheduling workloads.
 	PluginCapability_Service_VOLUME_ACCESSIBILITY_CONSTRAINTS PluginCapability_Service_Type = 2
+	// GROUP_CONTROLLER_SERVICE indicates that the Plugin provides
+	// RPCs for operating on groups of volumes. Plugins MAY provide
+	// this capability.
+	// The presence of this capability determines whether the CO will
+	// attempt to invoke the REQUIRED GroupController service RPCs, as
+	// well as specific RPCs as indicated by
+	// GroupControllerGetCapabilities.
+	PluginCapability_Service_GROUP_CONTROLLER_SERVICE PluginCapability_Service_Type = 3
 )
 
 var PluginCapability_Service_Type_name = map[int32]string{
 	0: "UNKNOWN",
 	1: "CONTROLLER_SERVICE",
 	2: "VOLUME_ACCESSIBILITY_CONSTRAINTS",
+	3: "GROUP_CONTROLLER_SERVICE",
 }
 
 var PluginCapability_Service_Type_value = map[string]int32{
 	"UNKNOWN":                          0,
 	"CONTROLLER_SERVICE":               1,
 	"VOLUME_ACCESSIBILITY_CONSTRAINTS": 2,
+	"GROUP_CONTROLLER_SERVICE":         3,
 }
 
 func (x PluginCapability_Service_Type) String() string {
@@ -85,17 +95,20 @@ const (
 	// expansion of node-published volume via NodeExpandVolume.
 	//
 	// Example 1: Given a shared filesystem volume (e.g. GlusterFs),
-	//   the Plugin may set the ONLINE volume expansion capability and
-	//   implement ControllerExpandVolume but not NodeExpandVolume.
+	//
+	//	the Plugin may set the ONLINE volume expansion capability and
+	//	implement ControllerExpandVolume but not NodeExpandVolume.
 	//
 	// Example 2: Given a block storage volume type (e.g. EBS), the
-	//   Plugin may set the ONLINE volume expansion capability and
-	//   implement both ControllerExpandVolume and NodeExpandVolume.
+	//
+	//	Plugin may set the ONLINE volume expansion capability and
+	//	implement both ControllerExpandVolume and NodeExpandVolume.
 	//
 	// Example 3: Given a Plugin that supports volume expansion only
-	//   upon a node, the Plugin may set the ONLINE volume
-	//   expansion capability and implement NodeExpandVolume but not
-	//   ControllerExpandVolume.
+	//
+	//	upon a node, the Plugin may set the ONLINE volume
+	//	expansion capability and implement NodeExpandVolume but not
+	//	ControllerExpandVolume.
 	PluginCapability_VolumeExpansion_ONLINE PluginCapability_VolumeExpansion_Type = 1
 	// OFFLINE indicates that volumes currently published and
 	// available on a node SHALL NOT be expanded via
@@ -105,10 +118,11 @@ const (
 	// the EXPAND_VOLUME node capability.
 	//
 	// Example 1: Given a block storage volume type (e.g. Azure Disk)
-	//   that does not support expansion of "node-attached" (i.e.
-	//   controller-published) volumes, the Plugin may indicate
-	//   OFFLINE volume expansion support and implement both
-	//   ControllerExpandVolume and NodeExpandVolume.
+	//
+	//	that does not support expansion of "node-attached" (i.e.
+	//	controller-published) volumes, the Plugin may indicate
+	//	OFFLINE volume expansion support and implement both
+	//	ControllerExpandVolume and NodeExpandVolume.
 	PluginCapability_VolumeExpansion_OFFLINE PluginCapability_VolumeExpansion_Type = 2
 )
 
@@ -246,6 +260,9 @@ const (
 	// SINGLE_NODE_SINGLE_WRITER and/or SINGLE_NODE_MULTI_WRITER are
 	// supported, in order to permit older COs to continue working.
 	ControllerServiceCapability_RPC_SINGLE_NODE_MULTI_WRITER ControllerServiceCapability_RPC_Type = 13
+	// Indicates the SP supports modifying volume with mutable
+	// parameters. See ControllerModifyVolume for details.
+	ControllerServiceCapability_RPC_MODIFY_VOLUME ControllerServiceCapability_RPC_Type = 14
 )
 
 var ControllerServiceCapability_RPC_Type_name = map[int32]string{
@@ -263,6 +280,7 @@ var ControllerServiceCapability_RPC_Type_name = map[int32]string{
 	11: "VOLUME_CONDITION",
 	12: "GET_VOLUME",
 	13: "SINGLE_NODE_MULTI_WRITER",
+	14: "MODIFY_VOLUME",
 }
 
 var ControllerServiceCapability_RPC_Type_value = map[string]int32{
@@ -280,6 +298,7 @@ var ControllerServiceCapability_RPC_Type_value = map[string]int32{
 	"VOLUME_CONDITION":             11,
 	"GET_VOLUME":                   12,
 	"SINGLE_NODE_MULTI_WRITER":     13,
+	"MODIFY_VOLUME":                14,
 }
 
 func (x ControllerServiceCapability_RPC_Type) String() string {
@@ -287,7 +306,7 @@ func (x ControllerServiceCapability_RPC_Type) String() string {
 }
 
 func (ControllerServiceCapability_RPC_Type) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptor_9cdb00adce470e01, []int{31, 0, 0}
+	return fileDescriptor_9cdb00adce470e01, []int{33, 0, 0}
 }
 
 type VolumeUsage_Unit int32
@@ -315,7 +334,7 @@ func (x VolumeUsage_Unit) String() string {
 }
 
 func (VolumeUsage_Unit) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptor_9cdb00adce470e01, []int{51, 0}
+	return fileDescriptor_9cdb00adce470e01, []int{53, 0}
 }
 
 type NodeServiceCapability_RPC_Type int32
@@ -382,7 +401,35 @@ func (x NodeServiceCapability_RPC_Type) String() string {
 }
 
 func (NodeServiceCapability_RPC_Type) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptor_9cdb00adce470e01, []int{55, 0, 0}
+	return fileDescriptor_9cdb00adce470e01, []int{57, 0, 0}
+}
+
+type GroupControllerServiceCapability_RPC_Type int32
+
+const (
+	GroupControllerServiceCapability_RPC_UNKNOWN GroupControllerServiceCapability_RPC_Type = 0
+	// Indicates that the group controller plugin supports
+	// creating, deleting, and getting details of a volume
+	// group snapshot.
+	GroupControllerServiceCapability_RPC_CREATE_DELETE_GET_VOLUME_GROUP_SNAPSHOT GroupControllerServiceCapability_RPC_Type = 1
+)
+
+var GroupControllerServiceCapability_RPC_Type_name = map[int32]string{
+	0: "UNKNOWN",
+	1: "CREATE_DELETE_GET_VOLUME_GROUP_SNAPSHOT",
+}
+
+var GroupControllerServiceCapability_RPC_Type_value = map[string]int32{
+	"UNKNOWN": 0,
+	"CREATE_DELETE_GET_VOLUME_GROUP_SNAPSHOT": 1,
+}
+
+func (x GroupControllerServiceCapability_RPC_Type) String() string {
+	return proto.EnumName(GroupControllerServiceCapability_RPC_Type_name, int32(x))
+}
+
+func (GroupControllerServiceCapability_RPC_Type) EnumDescriptor() ([]byte, []int) {
+	return fileDescriptor_9cdb00adce470e01, []int{64, 0, 0}
 }
 
 type GetPluginInfoRequest struct {
@@ -555,6 +602,7 @@ func (m *GetPluginCapabilitiesResponse) GetCapabilities() []*PluginCapability {
 // Specifies a capability of the plugin.
 type PluginCapability struct {
 	// Types that are valid to be assigned to Type:
+	//
 	//	*PluginCapability_Service_
 	//	*PluginCapability_VolumeExpansion_
 	Type                 isPluginCapability_Type `protobuf_oneof:"type"`
@@ -748,16 +796,16 @@ type ProbeResponse struct {
 	// and it is important for a CO to distinguish between the following
 	// cases:
 	//
-	// 1) The plugin is in an unhealthy state and MAY need restarting. In
-	//    this case a gRPC error code SHALL be returned.
-	// 2) The plugin is still initializing, but is otherwise perfectly
-	//    healthy. In this case a successful response SHALL be returned
-	//    with a readiness value of `false`. Calls to the plugin's
-	//    Controller and/or Node services MAY fail due to an incomplete
-	//    initialization state.
-	// 3) The plugin has finished initializing and is ready to service
-	//    calls to its Controller and/or Node services. A successful
-	//    response is returned with a readiness value of `true`.
+	//  1. The plugin is in an unhealthy state and MAY need restarting. In
+	//     this case a gRPC error code SHALL be returned.
+	//  2. The plugin is still initializing, but is otherwise perfectly
+	//     healthy. In this case a successful response SHALL be returned
+	//     with a readiness value of `false`. Calls to the plugin's
+	//     Controller and/or Node services MAY fail due to an incomplete
+	//     initialization state.
+	//  3. The plugin has finished initializing and is ready to service
+	//     calls to its Controller and/or Node services. A successful
+	//     response is returned with a readiness value of `true`.
 	//
 	// This field is OPTIONAL. If not present, the caller SHALL assume
 	// that the plugin is in a ready state and is accepting calls to its
@@ -804,26 +852,27 @@ func (m *ProbeResponse) GetReady() *wrappers.BoolValue {
 type CreateVolumeRequest struct {
 	// The suggested name for the storage space. This field is REQUIRED.
 	// It serves two purposes:
-	// 1) Idempotency - This name is generated by the CO to achieve
-	//    idempotency.  The Plugin SHOULD ensure that multiple
-	//    `CreateVolume` calls for the same name do not result in more
-	//    than one piece of storage provisioned corresponding to that
-	//    name. If a Plugin is unable to enforce idempotency, the CO's
-	//    error recovery logic could result in multiple (unused) volumes
-	//    being provisioned.
-	//    In the case of error, the CO MUST handle the gRPC error codes
-	//    per the recovery behavior defined in the "CreateVolume Errors"
-	//    section below.
-	//    The CO is responsible for cleaning up volumes it provisioned
-	//    that it no longer needs. If the CO is uncertain whether a volume
-	//    was provisioned or not when a `CreateVolume` call fails, the CO
-	//    MAY call `CreateVolume` again, with the same name, to ensure the
-	//    volume exists and to retrieve the volume's `volume_id` (unless
-	//    otherwise prohibited by "CreateVolume Errors").
-	// 2) Suggested name - Some storage systems allow callers to specify
-	//    an identifier by which to refer to the newly provisioned
-	//    storage. If a storage system supports this, it can optionally
-	//    use this name as the identifier for the new volume.
+	//  1. Idempotency - This name is generated by the CO to achieve
+	//     idempotency.  The Plugin SHOULD ensure that multiple
+	//     `CreateVolume` calls for the same name do not result in more
+	//     than one piece of storage provisioned corresponding to that
+	//     name. If a Plugin is unable to enforce idempotency, the CO's
+	//     error recovery logic could result in multiple (unused) volumes
+	//     being provisioned.
+	//     In the case of error, the CO MUST handle the gRPC error codes
+	//     per the recovery behavior defined in the "CreateVolume Errors"
+	//     section below.
+	//     The CO is responsible for cleaning up volumes it provisioned
+	//     that it no longer needs. If the CO is uncertain whether a volume
+	//     was provisioned or not when a `CreateVolume` call fails, the CO
+	//     MAY call `CreateVolume` again, with the same name, to ensure the
+	//     volume exists and to retrieve the volume's `volume_id` (unless
+	//     otherwise prohibited by "CreateVolume Errors").
+	//  2. Suggested name - Some storage systems allow callers to specify
+	//     an identifier by which to refer to the newly provisioned
+	//     storage. If a storage system supports this, it can optionally
+	//     use this name as the identifier for the new volume.
+	//
 	// Any Unicode string that conforms to the length limit is allowed
 	// except those containing the following banned characters:
 	// U+0000-U+0008, U+000B, U+000C, U+000E-U+001F, U+007F-U+009F.
@@ -851,9 +900,10 @@ type CreateVolumeRequest struct {
 	// MUST return the appropriate gRPC error code.
 	// This field is REQUIRED.
 	VolumeCapabilities []*VolumeCapability `protobuf:"bytes,3,rep,name=volume_capabilities,json=volumeCapabilities,proto3" json:"volume_capabilities,omitempty"`
-	// Plugin specific parameters passed in as opaque key-value pairs.
-	// This field is OPTIONAL. The Plugin is responsible for parsing and
-	// validating these parameters. COs will treat these as opaque.
+	// Plugin specific creation-time parameters passed in as opaque
+	// key-value pairs. This field is OPTIONAL. The Plugin is responsible
+	// for parsing and validating these parameters. COs will treat
+	// these as opaque.
 	Parameters map[string]string `protobuf:"bytes,4,rep,name=parameters,proto3" json:"parameters,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
 	// Secrets required by plugin to complete volume creation request.
 	// This field is OPTIONAL. Refer to the `Secrets Requirements`
@@ -874,9 +924,14 @@ type CreateVolumeRequest struct {
 	// VOLUME_ACCESSIBILITY_CONSTRAINTS plugin capability, the SP MAY
 	// choose where the provisioned volume is accessible from.
 	AccessibilityRequirements *TopologyRequirement `protobuf:"bytes,7,opt,name=accessibility_requirements,json=accessibilityRequirements,proto3" json:"accessibility_requirements,omitempty"`
-	XXX_NoUnkeyedLiteral      struct{}             `json:"-"`
-	XXX_unrecognized          []byte               `json:"-"`
-	XXX_sizecache             int32                `json:"-"`
+	// Plugins MUST treat these
+	// as if they take precedence over the parameters field.
+	// This field SHALL NOT be specified unless the SP has the
+	// MODIFY_VOLUME plugin capability.
+	MutableParameters    map[string]string `protobuf:"bytes,8,rep,name=mutable_parameters,json=mutableParameters,proto3" json:"mutable_parameters,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+	XXX_NoUnkeyedLiteral struct{}          `json:"-"`
+	XXX_unrecognized     []byte            `json:"-"`
+	XXX_sizecache        int32             `json:"-"`
 }
 
 func (m *CreateVolumeRequest) Reset()         { *m = CreateVolumeRequest{} }
@@ -953,10 +1008,18 @@ func (m *CreateVolumeRequest) GetAccessibilityRequirements() *TopologyRequiremen
 	return nil
 }
 
+func (m *CreateVolumeRequest) GetMutableParameters() map[string]string {
+	if m != nil {
+		return m.MutableParameters
+	}
+	return nil
+}
+
 // Specifies what source the volume will be created from. One of the
 // type fields MUST be specified.
 type VolumeContentSource struct {
 	// Types that are valid to be assigned to Type:
+	//
 	//	*VolumeContentSource_Snapshot
 	//	*VolumeContentSource_Volume
 	Type                 isVolumeContentSource_Type `protobuf_oneof:"type"`
@@ -1168,6 +1231,7 @@ type VolumeCapability struct {
 	// following fields MUST be specified.
 	//
 	// Types that are valid to be assigned to AccessType:
+	//
 	//	*VolumeCapability_Block
 	//	*VolumeCapability_Mount
 	AccessType isVolumeCapability_AccessType `protobuf_oneof:"access_type"`
@@ -1509,14 +1573,18 @@ type Volume struct {
 	// node.
 	//
 	// Example 1:
-	//   accessible_topology = {"region": "R1", "zone": "Z2"}
+	//
+	//	accessible_topology = {"region": "R1", "zone": "Z2"}
+	//
 	// Indicates a volume accessible only from the "region" "R1" and the
 	// "zone" "Z2".
 	//
 	// Example 2:
-	//   accessible_topology =
-	//     {"region": "R1", "zone": "Z2"},
-	//     {"region": "R1", "zone": "Z3"}
+	//
+	//	accessible_topology =
+	//	  {"region": "R1", "zone": "Z2"},
+	//	  {"region": "R1", "zone": "Z3"}
+	//
 	// Indicates a volume accessible from both "zone" "Z2" and "zone" "Z3"
 	// in the "region" "R1".
 	AccessibleTopology   []*Topology `protobuf:"bytes,5,rep,name=accessible_topology,json=accessibleTopology,proto3" json:"accessible_topology,omitempty"`
@@ -1595,21 +1663,27 @@ type TopologyRequirement struct {
 	// accessible from at least one of the requisite topologies.
 	//
 	// Given
-	//   x = number of topologies provisioned volume is accessible from
-	//   n = number of requisite topologies
+	//
+	//	x = number of topologies provisioned volume is accessible from
+	//	n = number of requisite topologies
+	//
 	// The CO MUST ensure n >= 1. The SP MUST ensure x >= 1
 	// If x==n, then the SP MUST make the provisioned volume available to
 	// all topologies from the list of requisite topologies. If it is
 	// unable to do so, the SP MUST fail the CreateVolume call.
 	// For example, if a volume should be accessible from a single zone,
 	// and requisite =
-	//   {"region": "R1", "zone": "Z2"}
+	//
+	//	{"region": "R1", "zone": "Z2"}
+	//
 	// then the provisioned volume MUST be accessible from the "region"
 	// "R1" and the "zone" "Z2".
 	// Similarly, if a volume should be accessible from two zones, and
 	// requisite =
-	//   {"region": "R1", "zone": "Z2"},
-	//   {"region": "R1", "zone": "Z3"}
+	//
+	//	{"region": "R1", "zone": "Z2"},
+	//	{"region": "R1", "zone": "Z3"}
+	//
 	// then the provisioned volume MUST be accessible from the "region"
 	// "R1" and both "zone" "Z2" and "zone" "Z3".
 	//
@@ -1618,18 +1692,23 @@ type TopologyRequirement struct {
 	// the CreateVolume call.
 	// For example, if a volume should be accessible from a single zone,
 	// and requisite =
-	//   {"region": "R1", "zone": "Z2"},
-	//   {"region": "R1", "zone": "Z3"}
+	//
+	//	{"region": "R1", "zone": "Z2"},
+	//	{"region": "R1", "zone": "Z3"}
+	//
 	// then the SP may choose to make the provisioned volume available in
 	// either the "zone" "Z2" or the "zone" "Z3" in the "region" "R1".
 	// Similarly, if a volume should be accessible from two zones, and
 	// requisite =
-	//   {"region": "R1", "zone": "Z2"},
-	//   {"region": "R1", "zone": "Z3"},
-	//   {"region": "R1", "zone": "Z4"}
+	//
+	//	{"region": "R1", "zone": "Z2"},
+	//	{"region": "R1", "zone": "Z3"},
+	//	{"region": "R1", "zone": "Z4"}
+	//
 	// then the provisioned volume MUST be accessible from any combination
 	// of two unique topologies: e.g. "R1/Z2" and "R1/Z3", or "R1/Z2" and
-	//  "R1/Z4", or "R1/Z3" and "R1/Z4".
+	//
+	//	"R1/Z4", or "R1/Z3" and "R1/Z4".
 	//
 	// If x>n, then the SP MUST make the provisioned volume available from
 	// all topologies from the list of requisite topologies and MAY choose
@@ -1638,7 +1717,9 @@ type TopologyRequirement struct {
 	// CreateVolume call.
 	// For example, if a volume should be accessible from two zones, and
 	// requisite =
-	//   {"region": "R1", "zone": "Z2"}
+	//
+	//	{"region": "R1", "zone": "Z2"}
+	//
 	// then the provisioned volume MUST be accessible from the "region"
 	// "R1" and the "zone" "Z2" and the SP may select the second zone
 	// independently, e.g. "R1/Z4".
@@ -1667,25 +1748,33 @@ type TopologyRequirement struct {
 	// Example 1:
 	// Given a volume should be accessible from a single zone, and
 	// requisite =
-	//   {"region": "R1", "zone": "Z2"},
-	//   {"region": "R1", "zone": "Z3"}
+	//
+	//	{"region": "R1", "zone": "Z2"},
+	//	{"region": "R1", "zone": "Z3"}
+	//
 	// preferred =
-	//   {"region": "R1", "zone": "Z3"}
-	// then the the SP SHOULD first attempt to make the provisioned volume
+	//
+	//	{"region": "R1", "zone": "Z3"}
+	//
+	// then the SP SHOULD first attempt to make the provisioned volume
 	// available from "zone" "Z3" in the "region" "R1" and fall back to
 	// "zone" "Z2" in the "region" "R1" if that is not possible.
 	//
 	// Example 2:
 	// Given a volume should be accessible from a single zone, and
 	// requisite =
-	//   {"region": "R1", "zone": "Z2"},
-	//   {"region": "R1", "zone": "Z3"},
-	//   {"region": "R1", "zone": "Z4"},
-	//   {"region": "R1", "zone": "Z5"}
+	//
+	//	{"region": "R1", "zone": "Z2"},
+	//	{"region": "R1", "zone": "Z3"},
+	//	{"region": "R1", "zone": "Z4"},
+	//	{"region": "R1", "zone": "Z5"}
+	//
 	// preferred =
-	//   {"region": "R1", "zone": "Z4"},
-	//   {"region": "R1", "zone": "Z2"}
-	// then the the SP SHOULD first attempt to make the provisioned volume
+	//
+	//	{"region": "R1", "zone": "Z4"},
+	//	{"region": "R1", "zone": "Z2"}
+	//
+	// then the SP SHOULD first attempt to make the provisioned volume
 	// accessible from "zone" "Z4" in the "region" "R1" and fall back to
 	// "zone" "Z2" in the "region" "R1" if that is not possible. If that
 	// is not possible, the SP may choose between either the "zone"
@@ -1697,14 +1786,18 @@ type TopologyRequirement struct {
 	// the volume is accessible from two zones, aka synchronously
 	// replicated), and
 	// requisite =
-	//   {"region": "R1", "zone": "Z2"},
-	//   {"region": "R1", "zone": "Z3"},
-	//   {"region": "R1", "zone": "Z4"},
-	//   {"region": "R1", "zone": "Z5"}
+	//
+	//	{"region": "R1", "zone": "Z2"},
+	//	{"region": "R1", "zone": "Z3"},
+	//	{"region": "R1", "zone": "Z4"},
+	//	{"region": "R1", "zone": "Z5"}
+	//
 	// preferred =
-	//   {"region": "R1", "zone": "Z5"},
-	//   {"region": "R1", "zone": "Z3"}
-	// then the the SP SHOULD first attempt to make the provisioned volume
+	//
+	//	{"region": "R1", "zone": "Z5"},
+	//	{"region": "R1", "zone": "Z3"}
+	//
+	// then the SP SHOULD first attempt to make the provisioned volume
 	// accessible from the combination of the two "zones" "Z5" and "Z3" in
 	// the "region" "R1". If that's not possible, it should fall back to
 	// a combination of "Z5" and other possibilities from the list of
@@ -2173,7 +2266,10 @@ type ValidateVolumeCapabilitiesRequest struct {
 	// Secrets required by plugin to complete volume validation request.
 	// This field is OPTIONAL. Refer to the `Secrets Requirements`
 	// section on how to use this field.
-	Secrets              map[string]string `protobuf:"bytes,5,rep,name=secrets,proto3" json:"secrets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+	Secrets map[string]string `protobuf:"bytes,5,rep,name=secrets,proto3" json:"secrets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+	// See CreateVolumeRequest.mutable_parameters.
+	// This field is OPTIONAL.
+	MutableParameters    map[string]string `protobuf:"bytes,6,rep,name=mutable_parameters,json=mutableParameters,proto3" json:"mutable_parameters,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
 	XXX_NoUnkeyedLiteral struct{}          `json:"-"`
 	XXX_unrecognized     []byte            `json:"-"`
 	XXX_sizecache        int32             `json:"-"`
@@ -2239,6 +2335,13 @@ func (m *ValidateVolumeCapabilitiesRequest) GetSecrets() map[string]string {
 	return nil
 }
 
+func (m *ValidateVolumeCapabilitiesRequest) GetMutableParameters() map[string]string {
+	if m != nil {
+		return m.MutableParameters
+	}
+	return nil
+}
+
 type ValidateVolumeCapabilitiesResponse struct {
 	// Confirmed indicates to the CO the set of capabilities that the
 	// plugin has validated. This field SHALL only be set to a non-empty
@@ -2306,7 +2409,10 @@ type ValidateVolumeCapabilitiesResponse_Confirmed struct {
 	VolumeCapabilities []*VolumeCapability `protobuf:"bytes,2,rep,name=volume_capabilities,json=volumeCapabilities,proto3" json:"volume_capabilities,omitempty"`
 	// The volume creation parameters validated by the plugin.
 	// This field is OPTIONAL.
-	Parameters           map[string]string `protobuf:"bytes,3,rep,name=parameters,proto3" json:"parameters,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+	Parameters map[string]string `protobuf:"bytes,3,rep,name=parameters,proto3" json:"parameters,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+	// The volume creation mutable_parameters validated by the plugin.
+	// This field is OPTIONAL.
+	MutableParameters    map[string]string `protobuf:"bytes,4,rep,name=mutable_parameters,json=mutableParameters,proto3" json:"mutable_parameters,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
 	XXX_NoUnkeyedLiteral struct{}          `json:"-"`
 	XXX_unrecognized     []byte            `json:"-"`
 	XXX_sizecache        int32             `json:"-"`
@@ -2362,6 +2468,13 @@ func (m *ValidateVolumeCapabilitiesResponse_Confirmed) GetParameters() map[strin
 	return nil
 }
 
+func (m *ValidateVolumeCapabilitiesResponse_Confirmed) GetMutableParameters() map[string]string {
+	if m != nil {
+		return m.MutableParameters
+	}
+	return nil
+}
+
 type ListVolumesRequest struct {
 	// If specified (non-zero value), the Plugin MUST NOT return more
 	// entries than this number in the response. If the actual number of
@@ -2736,6 +2849,105 @@ func (m *ControllerGetVolumeResponse_VolumeStatus) GetVolumeCondition() *VolumeC
 	return nil
 }
 
+type ControllerModifyVolumeRequest struct {
+	// Contains identity information for the existing volume.
+	// This field is REQUIRED.
+	VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"`
+	// Secrets required by plugin to complete modify volume request.
+	// This field is OPTIONAL. Refer to the `Secrets Requirements`
+	// section on how to use this field.
+	Secrets map[string]string `protobuf:"bytes,2,rep,name=secrets,proto3" json:"secrets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+	// Plugin specific volume attributes to mutate, passed in as
+	// opaque key-value pairs.
+	// This field is REQUIRED. The Plugin is responsible for
+	// parsing and validating these parameters. COs will treat these
+	// as opaque. The CO SHOULD specify the intended values of all mutable
+	// parameters it intends to modify. SPs MUST NOT modify volumes based
+	// on the absence of keys, only keys that are specified should result
+	// in modifications to the volume.
+	MutableParameters    map[string]string `protobuf:"bytes,3,rep,name=mutable_parameters,json=mutableParameters,proto3" json:"mutable_parameters,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+	XXX_NoUnkeyedLiteral struct{}          `json:"-"`
+	XXX_unrecognized     []byte            `json:"-"`
+	XXX_sizecache        int32             `json:"-"`
+}
+
+func (m *ControllerModifyVolumeRequest) Reset()         { *m = ControllerModifyVolumeRequest{} }
+func (m *ControllerModifyVolumeRequest) String() string { return proto.CompactTextString(m) }
+func (*ControllerModifyVolumeRequest) ProtoMessage()    {}
+func (*ControllerModifyVolumeRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_9cdb00adce470e01, []int{27}
+}
+
+func (m *ControllerModifyVolumeRequest) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_ControllerModifyVolumeRequest.Unmarshal(m, b)
+}
+func (m *ControllerModifyVolumeRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_ControllerModifyVolumeRequest.Marshal(b, m, deterministic)
+}
+func (m *ControllerModifyVolumeRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_ControllerModifyVolumeRequest.Merge(m, src)
+}
+func (m *ControllerModifyVolumeRequest) XXX_Size() int {
+	return xxx_messageInfo_ControllerModifyVolumeRequest.Size(m)
+}
+func (m *ControllerModifyVolumeRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_ControllerModifyVolumeRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_ControllerModifyVolumeRequest proto.InternalMessageInfo
+
+func (m *ControllerModifyVolumeRequest) GetVolumeId() string {
+	if m != nil {
+		return m.VolumeId
+	}
+	return ""
+}
+
+func (m *ControllerModifyVolumeRequest) GetSecrets() map[string]string {
+	if m != nil {
+		return m.Secrets
+	}
+	return nil
+}
+
+func (m *ControllerModifyVolumeRequest) GetMutableParameters() map[string]string {
+	if m != nil {
+		return m.MutableParameters
+	}
+	return nil
+}
+
+type ControllerModifyVolumeResponse struct {
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *ControllerModifyVolumeResponse) Reset()         { *m = ControllerModifyVolumeResponse{} }
+func (m *ControllerModifyVolumeResponse) String() string { return proto.CompactTextString(m) }
+func (*ControllerModifyVolumeResponse) ProtoMessage()    {}
+func (*ControllerModifyVolumeResponse) Descriptor() ([]byte, []int) {
+	return fileDescriptor_9cdb00adce470e01, []int{28}
+}
+
+func (m *ControllerModifyVolumeResponse) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_ControllerModifyVolumeResponse.Unmarshal(m, b)
+}
+func (m *ControllerModifyVolumeResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_ControllerModifyVolumeResponse.Marshal(b, m, deterministic)
+}
+func (m *ControllerModifyVolumeResponse) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_ControllerModifyVolumeResponse.Merge(m, src)
+}
+func (m *ControllerModifyVolumeResponse) XXX_Size() int {
+	return xxx_messageInfo_ControllerModifyVolumeResponse.Size(m)
+}
+func (m *ControllerModifyVolumeResponse) XXX_DiscardUnknown() {
+	xxx_messageInfo_ControllerModifyVolumeResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_ControllerModifyVolumeResponse proto.InternalMessageInfo
+
 type GetCapacityRequest struct {
 	// If specified, the Plugin SHALL report the capacity of the storage
 	// that can be used to provision volumes that satisfy ALL of the
@@ -2764,7 +2976,7 @@ func (m *GetCapacityRequest) Reset()         { *m = GetCapacityRequest{} }
 func (m *GetCapacityRequest) String() string { return proto.CompactTextString(m) }
 func (*GetCapacityRequest) ProtoMessage()    {}
 func (*GetCapacityRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_9cdb00adce470e01, []int{27}
+	return fileDescriptor_9cdb00adce470e01, []int{29}
 }
 
 func (m *GetCapacityRequest) XXX_Unmarshal(b []byte) error {
@@ -2855,7 +3067,7 @@ func (m *GetCapacityResponse) Reset()         { *m = GetCapacityResponse{} }
 func (m *GetCapacityResponse) String() string { return proto.CompactTextString(m) }
 func (*GetCapacityResponse) ProtoMessage()    {}
 func (*GetCapacityResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_9cdb00adce470e01, []int{28}
+	return fileDescriptor_9cdb00adce470e01, []int{30}
 }
 
 func (m *GetCapacityResponse) XXX_Unmarshal(b []byte) error {
@@ -2907,7 +3119,7 @@ func (m *ControllerGetCapabilitiesRequest) Reset()         { *m = ControllerGetC
 func (m *ControllerGetCapabilitiesRequest) String() string { return proto.CompactTextString(m) }
 func (*ControllerGetCapabilitiesRequest) ProtoMessage()    {}
 func (*ControllerGetCapabilitiesRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_9cdb00adce470e01, []int{29}
+	return fileDescriptor_9cdb00adce470e01, []int{31}
 }
 
 func (m *ControllerGetCapabilitiesRequest) XXX_Unmarshal(b []byte) error {
@@ -2941,7 +3153,7 @@ func (m *ControllerGetCapabilitiesResponse) Reset()         { *m = ControllerGet
 func (m *ControllerGetCapabilitiesResponse) String() string { return proto.CompactTextString(m) }
 func (*ControllerGetCapabilitiesResponse) ProtoMessage()    {}
 func (*ControllerGetCapabilitiesResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_9cdb00adce470e01, []int{30}
+	return fileDescriptor_9cdb00adce470e01, []int{32}
 }
 
 func (m *ControllerGetCapabilitiesResponse) XXX_Unmarshal(b []byte) error {
@@ -2972,6 +3184,7 @@ func (m *ControllerGetCapabilitiesResponse) GetCapabilities() []*ControllerServi
 // Specifies a capability of the controller service.
 type ControllerServiceCapability struct {
 	// Types that are valid to be assigned to Type:
+	//
 	//	*ControllerServiceCapability_Rpc
 	Type                 isControllerServiceCapability_Type `protobuf_oneof:"type"`
 	XXX_NoUnkeyedLiteral struct{}                           `json:"-"`
@@ -2983,7 +3196,7 @@ func (m *ControllerServiceCapability) Reset()         { *m = ControllerServiceCa
 func (m *ControllerServiceCapability) String() string { return proto.CompactTextString(m) }
 func (*ControllerServiceCapability) ProtoMessage()    {}
 func (*ControllerServiceCapability) Descriptor() ([]byte, []int) {
-	return fileDescriptor_9cdb00adce470e01, []int{31}
+	return fileDescriptor_9cdb00adce470e01, []int{33}
 }
 
 func (m *ControllerServiceCapability) XXX_Unmarshal(b []byte) error {
@@ -3046,7 +3259,7 @@ func (m *ControllerServiceCapability_RPC) Reset()         { *m = ControllerServi
 func (m *ControllerServiceCapability_RPC) String() string { return proto.CompactTextString(m) }
 func (*ControllerServiceCapability_RPC) ProtoMessage()    {}
 func (*ControllerServiceCapability_RPC) Descriptor() ([]byte, []int) {
-	return fileDescriptor_9cdb00adce470e01, []int{31, 0}
+	return fileDescriptor_9cdb00adce470e01, []int{33, 0}
 }
 
 func (m *ControllerServiceCapability_RPC) XXX_Unmarshal(b []byte) error {
@@ -3093,12 +3306,12 @@ type CreateSnapshotRequest struct {
 	// This field is OPTIONAL. The Plugin is responsible for parsing and
 	// validating these parameters. COs will treat these as opaque.
 	// Use cases for opaque parameters:
-	// - Specify a policy to automatically clean up the snapshot.
-	// - Specify an expiration date for the snapshot.
-	// - Specify whether the snapshot is readonly or read/write.
-	// - Specify if the snapshot should be replicated to some place.
-	// - Specify primary or secondary for replication systems that
-	//   support snapshotting only on primary.
+	//   - Specify a policy to automatically clean up the snapshot.
+	//   - Specify an expiration date for the snapshot.
+	//   - Specify whether the snapshot is readonly or read/write.
+	//   - Specify if the snapshot should be replicated to some place.
+	//   - Specify primary or secondary for replication systems that
+	//     support snapshotting only on primary.
 	Parameters           map[string]string `protobuf:"bytes,4,rep,name=parameters,proto3" json:"parameters,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
 	XXX_NoUnkeyedLiteral struct{}          `json:"-"`
 	XXX_unrecognized     []byte            `json:"-"`
@@ -3109,7 +3322,7 @@ func (m *CreateSnapshotRequest) Reset()         { *m = CreateSnapshotRequest{} }
 func (m *CreateSnapshotRequest) String() string { return proto.CompactTextString(m) }
 func (*CreateSnapshotRequest) ProtoMessage()    {}
 func (*CreateSnapshotRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_9cdb00adce470e01, []int{32}
+	return fileDescriptor_9cdb00adce470e01, []int{34}
 }
 
 func (m *CreateSnapshotRequest) XXX_Unmarshal(b []byte) error {
@@ -3172,7 +3385,7 @@ func (m *CreateSnapshotResponse) Reset()         { *m = CreateSnapshotResponse{}
 func (m *CreateSnapshotResponse) String() string { return proto.CompactTextString(m) }
 func (*CreateSnapshotResponse) ProtoMessage()    {}
 func (*CreateSnapshotResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_9cdb00adce470e01, []int{33}
+	return fileDescriptor_9cdb00adce470e01, []int{35}
 }
 
 func (m *CreateSnapshotResponse) XXX_Unmarshal(b []byte) error {
@@ -3230,7 +3443,21 @@ type Snapshot struct {
 	// Indicates if a snapshot is ready to use as a
 	// `volume_content_source` in a `CreateVolumeRequest`. The default
 	// value is false. This field is REQUIRED.
-	ReadyToUse           bool     `protobuf:"varint,5,opt,name=ready_to_use,json=readyToUse,proto3" json:"ready_to_use,omitempty"`
+	ReadyToUse bool `protobuf:"varint,5,opt,name=ready_to_use,json=readyToUse,proto3" json:"ready_to_use,omitempty"`
+	// The ID of the volume group snapshot that this snapshot is part of.
+	// It uniquely identifies the group snapshot on the storage system.
+	// This field is OPTIONAL.
+	// If this snapshot is a member of a volume group snapshot, and it
+	// MUST NOT be deleted as a stand alone snapshot, then the SP
+	// MUST provide the ID of the volume group snapshot in this field.
+	// If provided, CO MUST use this field in subsequent volume group
+	// snapshot operations to indicate that this snapshot is part of the
+	// specified group snapshot.
+	// If not provided, CO SHALL treat the snapshot as independent,
+	// and SP SHALL allow it to be deleted separately.
+	// If this message is inside a VolumeGroupSnapshot message, the value
+	// MUST be the same as the group_snapshot_id in that message.
+	GroupSnapshotId      string   `protobuf:"bytes,6,opt,name=group_snapshot_id,json=groupSnapshotId,proto3" json:"group_snapshot_id,omitempty"`
 	XXX_NoUnkeyedLiteral struct{} `json:"-"`
 	XXX_unrecognized     []byte   `json:"-"`
 	XXX_sizecache        int32    `json:"-"`
@@ -3240,7 +3467,7 @@ func (m *Snapshot) Reset()         { *m = Snapshot{} }
 func (m *Snapshot) String() string { return proto.CompactTextString(m) }
 func (*Snapshot) ProtoMessage()    {}
 func (*Snapshot) Descriptor() ([]byte, []int) {
-	return fileDescriptor_9cdb00adce470e01, []int{34}
+	return fileDescriptor_9cdb00adce470e01, []int{36}
 }
 
 func (m *Snapshot) XXX_Unmarshal(b []byte) error {
@@ -3296,6 +3523,13 @@ func (m *Snapshot) GetReadyToUse() bool {
 	return false
 }
 
+func (m *Snapshot) GetGroupSnapshotId() string {
+	if m != nil {
+		return m.GroupSnapshotId
+	}
+	return ""
+}
+
 type DeleteSnapshotRequest struct {
 	// The ID of the snapshot to be deleted.
 	// This field is REQUIRED.
@@ -3313,7 +3547,7 @@ func (m *DeleteSnapshotRequest) Reset()         { *m = DeleteSnapshotRequest{} }
 func (m *DeleteSnapshotRequest) String() string { return proto.CompactTextString(m) }
 func (*DeleteSnapshotRequest) ProtoMessage()    {}
 func (*DeleteSnapshotRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_9cdb00adce470e01, []int{35}
+	return fileDescriptor_9cdb00adce470e01, []int{37}
 }
 
 func (m *DeleteSnapshotRequest) XXX_Unmarshal(b []byte) error {
@@ -3358,7 +3592,7 @@ func (m *DeleteSnapshotResponse) Reset()         { *m = DeleteSnapshotResponse{}
 func (m *DeleteSnapshotResponse) String() string { return proto.CompactTextString(m) }
 func (*DeleteSnapshotResponse) ProtoMessage()    {}
 func (*DeleteSnapshotResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_9cdb00adce470e01, []int{36}
+	return fileDescriptor_9cdb00adce470e01, []int{38}
 }
 
 func (m *DeleteSnapshotResponse) XXX_Unmarshal(b []byte) error {
@@ -3418,7 +3652,7 @@ func (m *ListSnapshotsRequest) Reset()         { *m = ListSnapshotsRequest{} }
 func (m *ListSnapshotsRequest) String() string { return proto.CompactTextString(m) }
 func (*ListSnapshotsRequest) ProtoMessage()    {}
 func (*ListSnapshotsRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_9cdb00adce470e01, []int{37}
+	return fileDescriptor_9cdb00adce470e01, []int{39}
 }
 
 func (m *ListSnapshotsRequest) XXX_Unmarshal(b []byte) error {
@@ -3492,7 +3726,7 @@ func (m *ListSnapshotsResponse) Reset()         { *m = ListSnapshotsResponse{} }
 func (m *ListSnapshotsResponse) String() string { return proto.CompactTextString(m) }
 func (*ListSnapshotsResponse) ProtoMessage()    {}
 func (*ListSnapshotsResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_9cdb00adce470e01, []int{38}
+	return fileDescriptor_9cdb00adce470e01, []int{40}
 }
 
 func (m *ListSnapshotsResponse) XXX_Unmarshal(b []byte) error {
@@ -3538,7 +3772,7 @@ func (m *ListSnapshotsResponse_Entry) Reset()         { *m = ListSnapshotsRespon
 func (m *ListSnapshotsResponse_Entry) String() string { return proto.CompactTextString(m) }
 func (*ListSnapshotsResponse_Entry) ProtoMessage()    {}
 func (*ListSnapshotsResponse_Entry) Descriptor() ([]byte, []int) {
-	return fileDescriptor_9cdb00adce470e01, []int{38, 0}
+	return fileDescriptor_9cdb00adce470e01, []int{40, 0}
 }
 
 func (m *ListSnapshotsResponse_Entry) XXX_Unmarshal(b []byte) error {
@@ -3592,7 +3826,7 @@ func (m *ControllerExpandVolumeRequest) Reset()         { *m = ControllerExpandV
 func (m *ControllerExpandVolumeRequest) String() string { return proto.CompactTextString(m) }
 func (*ControllerExpandVolumeRequest) ProtoMessage()    {}
 func (*ControllerExpandVolumeRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_9cdb00adce470e01, []int{39}
+	return fileDescriptor_9cdb00adce470e01, []int{41}
 }
 
 func (m *ControllerExpandVolumeRequest) XXX_Unmarshal(b []byte) error {
@@ -3657,7 +3891,7 @@ func (m *ControllerExpandVolumeResponse) Reset()         { *m = ControllerExpand
 func (m *ControllerExpandVolumeResponse) String() string { return proto.CompactTextString(m) }
 func (*ControllerExpandVolumeResponse) ProtoMessage()    {}
 func (*ControllerExpandVolumeResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_9cdb00adce470e01, []int{40}
+	return fileDescriptor_9cdb00adce470e01, []int{42}
 }
 
 func (m *ControllerExpandVolumeResponse) XXX_Unmarshal(b []byte) error {
@@ -3738,7 +3972,7 @@ func (m *NodeStageVolumeRequest) Reset()         { *m = NodeStageVolumeRequest{}
 func (m *NodeStageVolumeRequest) String() string { return proto.CompactTextString(m) }
 func (*NodeStageVolumeRequest) ProtoMessage()    {}
 func (*NodeStageVolumeRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_9cdb00adce470e01, []int{41}
+	return fileDescriptor_9cdb00adce470e01, []int{43}
 }
 
 func (m *NodeStageVolumeRequest) XXX_Unmarshal(b []byte) error {
@@ -3811,7 +4045,7 @@ func (m *NodeStageVolumeResponse) Reset()         { *m = NodeStageVolumeResponse
 func (m *NodeStageVolumeResponse) String() string { return proto.CompactTextString(m) }
 func (*NodeStageVolumeResponse) ProtoMessage()    {}
 func (*NodeStageVolumeResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_9cdb00adce470e01, []int{42}
+	return fileDescriptor_9cdb00adce470e01, []int{44}
 }
 
 func (m *NodeStageVolumeResponse) XXX_Unmarshal(b []byte) error {
@@ -3852,7 +4086,7 @@ func (m *NodeUnstageVolumeRequest) Reset()         { *m = NodeUnstageVolumeReque
 func (m *NodeUnstageVolumeRequest) String() string { return proto.CompactTextString(m) }
 func (*NodeUnstageVolumeRequest) ProtoMessage()    {}
 func (*NodeUnstageVolumeRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_9cdb00adce470e01, []int{43}
+	return fileDescriptor_9cdb00adce470e01, []int{45}
 }
 
 func (m *NodeUnstageVolumeRequest) XXX_Unmarshal(b []byte) error {
@@ -3897,7 +4131,7 @@ func (m *NodeUnstageVolumeResponse) Reset()         { *m = NodeUnstageVolumeResp
 func (m *NodeUnstageVolumeResponse) String() string { return proto.CompactTextString(m) }
 func (*NodeUnstageVolumeResponse) ProtoMessage()    {}
 func (*NodeUnstageVolumeResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_9cdb00adce470e01, []int{44}
+	return fileDescriptor_9cdb00adce470e01, []int{46}
 }
 
 func (m *NodeUnstageVolumeResponse) XXX_Unmarshal(b []byte) error {
@@ -3981,7 +4215,7 @@ func (m *NodePublishVolumeRequest) Reset()         { *m = NodePublishVolumeReque
 func (m *NodePublishVolumeRequest) String() string { return proto.CompactTextString(m) }
 func (*NodePublishVolumeRequest) ProtoMessage()    {}
 func (*NodePublishVolumeRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_9cdb00adce470e01, []int{45}
+	return fileDescriptor_9cdb00adce470e01, []int{47}
 }
 
 func (m *NodePublishVolumeRequest) XXX_Unmarshal(b []byte) error {
@@ -4068,7 +4302,7 @@ func (m *NodePublishVolumeResponse) Reset()         { *m = NodePublishVolumeResp
 func (m *NodePublishVolumeResponse) String() string { return proto.CompactTextString(m) }
 func (*NodePublishVolumeResponse) ProtoMessage()    {}
 func (*NodePublishVolumeResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_9cdb00adce470e01, []int{46}
+	return fileDescriptor_9cdb00adce470e01, []int{48}
 }
 
 func (m *NodePublishVolumeResponse) XXX_Unmarshal(b []byte) error {
@@ -4110,7 +4344,7 @@ func (m *NodeUnpublishVolumeRequest) Reset()         { *m = NodeUnpublishVolumeR
 func (m *NodeUnpublishVolumeRequest) String() string { return proto.CompactTextString(m) }
 func (*NodeUnpublishVolumeRequest) ProtoMessage()    {}
 func (*NodeUnpublishVolumeRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_9cdb00adce470e01, []int{47}
+	return fileDescriptor_9cdb00adce470e01, []int{49}
 }
 
 func (m *NodeUnpublishVolumeRequest) XXX_Unmarshal(b []byte) error {
@@ -4155,7 +4389,7 @@ func (m *NodeUnpublishVolumeResponse) Reset()         { *m = NodeUnpublishVolume
 func (m *NodeUnpublishVolumeResponse) String() string { return proto.CompactTextString(m) }
 func (*NodeUnpublishVolumeResponse) ProtoMessage()    {}
 func (*NodeUnpublishVolumeResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_9cdb00adce470e01, []int{48}
+	return fileDescriptor_9cdb00adce470e01, []int{50}
 }
 
 func (m *NodeUnpublishVolumeResponse) XXX_Unmarshal(b []byte) error {
@@ -4208,7 +4442,7 @@ func (m *NodeGetVolumeStatsRequest) Reset()         { *m = NodeGetVolumeStatsReq
 func (m *NodeGetVolumeStatsRequest) String() string { return proto.CompactTextString(m) }
 func (*NodeGetVolumeStatsRequest) ProtoMessage()    {}
 func (*NodeGetVolumeStatsRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_9cdb00adce470e01, []int{49}
+	return fileDescriptor_9cdb00adce470e01, []int{51}
 }
 
 func (m *NodeGetVolumeStatsRequest) XXX_Unmarshal(b []byte) error {
@@ -4267,7 +4501,7 @@ func (m *NodeGetVolumeStatsResponse) Reset()         { *m = NodeGetVolumeStatsRe
 func (m *NodeGetVolumeStatsResponse) String() string { return proto.CompactTextString(m) }
 func (*NodeGetVolumeStatsResponse) ProtoMessage()    {}
 func (*NodeGetVolumeStatsResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_9cdb00adce470e01, []int{50}
+	return fileDescriptor_9cdb00adce470e01, []int{52}
 }
 
 func (m *NodeGetVolumeStatsResponse) XXX_Unmarshal(b []byte) error {
@@ -4323,7 +4557,7 @@ func (m *VolumeUsage) Reset()         { *m = VolumeUsage{} }
 func (m *VolumeUsage) String() string { return proto.CompactTextString(m) }
 func (*VolumeUsage) ProtoMessage()    {}
 func (*VolumeUsage) Descriptor() ([]byte, []int) {
-	return fileDescriptor_9cdb00adce470e01, []int{51}
+	return fileDescriptor_9cdb00adce470e01, []int{53}
 }
 
 func (m *VolumeUsage) XXX_Unmarshal(b []byte) error {
@@ -4390,7 +4624,7 @@ func (m *VolumeCondition) Reset()         { *m = VolumeCondition{} }
 func (m *VolumeCondition) String() string { return proto.CompactTextString(m) }
 func (*VolumeCondition) ProtoMessage()    {}
 func (*VolumeCondition) Descriptor() ([]byte, []int) {
-	return fileDescriptor_9cdb00adce470e01, []int{52}
+	return fileDescriptor_9cdb00adce470e01, []int{54}
 }
 
 func (m *VolumeCondition) XXX_Unmarshal(b []byte) error {
@@ -4435,7 +4669,7 @@ func (m *NodeGetCapabilitiesRequest) Reset()         { *m = NodeGetCapabilitiesR
 func (m *NodeGetCapabilitiesRequest) String() string { return proto.CompactTextString(m) }
 func (*NodeGetCapabilitiesRequest) ProtoMessage()    {}
 func (*NodeGetCapabilitiesRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_9cdb00adce470e01, []int{53}
+	return fileDescriptor_9cdb00adce470e01, []int{55}
 }
 
 func (m *NodeGetCapabilitiesRequest) XXX_Unmarshal(b []byte) error {
@@ -4469,7 +4703,7 @@ func (m *NodeGetCapabilitiesResponse) Reset()         { *m = NodeGetCapabilities
 func (m *NodeGetCapabilitiesResponse) String() string { return proto.CompactTextString(m) }
 func (*NodeGetCapabilitiesResponse) ProtoMessage()    {}
 func (*NodeGetCapabilitiesResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_9cdb00adce470e01, []int{54}
+	return fileDescriptor_9cdb00adce470e01, []int{56}
 }
 
 func (m *NodeGetCapabilitiesResponse) XXX_Unmarshal(b []byte) error {
@@ -4500,6 +4734,7 @@ func (m *NodeGetCapabilitiesResponse) GetCapabilities() []*NodeServiceCapability
 // Specifies a capability of the node service.
 type NodeServiceCapability struct {
 	// Types that are valid to be assigned to Type:
+	//
 	//	*NodeServiceCapability_Rpc
 	Type                 isNodeServiceCapability_Type `protobuf_oneof:"type"`
 	XXX_NoUnkeyedLiteral struct{}                     `json:"-"`
@@ -4511,7 +4746,7 @@ func (m *NodeServiceCapability) Reset()         { *m = NodeServiceCapability{} }
 func (m *NodeServiceCapability) String() string { return proto.CompactTextString(m) }
 func (*NodeServiceCapability) ProtoMessage()    {}
 func (*NodeServiceCapability) Descriptor() ([]byte, []int) {
-	return fileDescriptor_9cdb00adce470e01, []int{55}
+	return fileDescriptor_9cdb00adce470e01, []int{57}
 }
 
 func (m *NodeServiceCapability) XXX_Unmarshal(b []byte) error {
@@ -4574,7 +4809,7 @@ func (m *NodeServiceCapability_RPC) Reset()         { *m = NodeServiceCapability
 func (m *NodeServiceCapability_RPC) String() string { return proto.CompactTextString(m) }
 func (*NodeServiceCapability_RPC) ProtoMessage()    {}
 func (*NodeServiceCapability_RPC) Descriptor() ([]byte, []int) {
-	return fileDescriptor_9cdb00adce470e01, []int{55, 0}
+	return fileDescriptor_9cdb00adce470e01, []int{57, 0}
 }
 
 func (m *NodeServiceCapability_RPC) XXX_Unmarshal(b []byte) error {
@@ -4612,7 +4847,7 @@ func (m *NodeGetInfoRequest) Reset()         { *m = NodeGetInfoRequest{} }
 func (m *NodeGetInfoRequest) String() string { return proto.CompactTextString(m) }
 func (*NodeGetInfoRequest) ProtoMessage()    {}
 func (*NodeGetInfoRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_9cdb00adce470e01, []int{56}
+	return fileDescriptor_9cdb00adce470e01, []int{58}
 }
 
 func (m *NodeGetInfoRequest) XXX_Unmarshal(b []byte) error {
@@ -4666,8 +4901,10 @@ type NodeGetInfoResponse struct {
 	// no topological constraints declared for V.
 	//
 	// Example 1:
-	//   accessible_topology =
-	//     {"region": "R1", "zone": "Z2"}
+	//
+	//	accessible_topology =
+	//	  {"region": "R1", "zone": "Z2"}
+	//
 	// Indicates the node exists within the "region" "R1" and the "zone"
 	// "Z2".
 	AccessibleTopology   *Topology `protobuf:"bytes,3,opt,name=accessible_topology,json=accessibleTopology,proto3" json:"accessible_topology,omitempty"`
@@ -4680,7 +4917,7 @@ func (m *NodeGetInfoResponse) Reset()         { *m = NodeGetInfoResponse{} }
 func (m *NodeGetInfoResponse) String() string { return proto.CompactTextString(m) }
 func (*NodeGetInfoResponse) ProtoMessage()    {}
 func (*NodeGetInfoResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_9cdb00adce470e01, []int{57}
+	return fileDescriptor_9cdb00adce470e01, []int{59}
 }
 
 func (m *NodeGetInfoResponse) XXX_Unmarshal(b []byte) error {
@@ -4771,7 +5008,7 @@ func (m *NodeExpandVolumeRequest) Reset()         { *m = NodeExpandVolumeRequest
 func (m *NodeExpandVolumeRequest) String() string { return proto.CompactTextString(m) }
 func (*NodeExpandVolumeRequest) ProtoMessage()    {}
 func (*NodeExpandVolumeRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_9cdb00adce470e01, []int{58}
+	return fileDescriptor_9cdb00adce470e01, []int{60}
 }
 
 func (m *NodeExpandVolumeRequest) XXX_Unmarshal(b []byte) error {
@@ -4846,7 +5083,7 @@ func (m *NodeExpandVolumeResponse) Reset()         { *m = NodeExpandVolumeRespon
 func (m *NodeExpandVolumeResponse) String() string { return proto.CompactTextString(m) }
 func (*NodeExpandVolumeResponse) ProtoMessage()    {}
 func (*NodeExpandVolumeResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_9cdb00adce470e01, []int{59}
+	return fileDescriptor_9cdb00adce470e01, []int{61}
 }
 
 func (m *NodeExpandVolumeResponse) XXX_Unmarshal(b []byte) error {
@@ -4874,61 +5111,663 @@ func (m *NodeExpandVolumeResponse) GetCapacityBytes() int64 {
 	return 0
 }
 
-var E_AlphaEnum = &proto.ExtensionDesc{
-	ExtendedType:  (*descriptor.EnumOptions)(nil),
-	ExtensionType: (*bool)(nil),
-	Field:         1060,
-	Name:          "csi.v1.alpha_enum",
-	Tag:           "varint,1060,opt,name=alpha_enum",
-	Filename:      "github.com/container-storage-interface/spec/csi.proto",
+type GroupControllerGetCapabilitiesRequest struct {
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
 }
 
-var E_AlphaEnumValue = &proto.ExtensionDesc{
-	ExtendedType:  (*descriptor.EnumValueOptions)(nil),
-	ExtensionType: (*bool)(nil),
-	Field:         1060,
-	Name:          "csi.v1.alpha_enum_value",
-	Tag:           "varint,1060,opt,name=alpha_enum_value",
-	Filename:      "github.com/container-storage-interface/spec/csi.proto",
+func (m *GroupControllerGetCapabilitiesRequest) Reset()         { *m = GroupControllerGetCapabilitiesRequest{} }
+func (m *GroupControllerGetCapabilitiesRequest) String() string { return proto.CompactTextString(m) }
+func (*GroupControllerGetCapabilitiesRequest) ProtoMessage()    {}
+func (*GroupControllerGetCapabilitiesRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_9cdb00adce470e01, []int{62}
 }
 
-var E_CsiSecret = &proto.ExtensionDesc{
-	ExtendedType:  (*descriptor.FieldOptions)(nil),
-	ExtensionType: (*bool)(nil),
-	Field:         1059,
-	Name:          "csi.v1.csi_secret",
-	Tag:           "varint,1059,opt,name=csi_secret",
-	Filename:      "github.com/container-storage-interface/spec/csi.proto",
+func (m *GroupControllerGetCapabilitiesRequest) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_GroupControllerGetCapabilitiesRequest.Unmarshal(m, b)
 }
-
-var E_AlphaField = &proto.ExtensionDesc{
-	ExtendedType:  (*descriptor.FieldOptions)(nil),
-	ExtensionType: (*bool)(nil),
-	Field:         1060,
-	Name:          "csi.v1.alpha_field",
-	Tag:           "varint,1060,opt,name=alpha_field",
-	Filename:      "github.com/container-storage-interface/spec/csi.proto",
+func (m *GroupControllerGetCapabilitiesRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_GroupControllerGetCapabilitiesRequest.Marshal(b, m, deterministic)
+}
+func (m *GroupControllerGetCapabilitiesRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_GroupControllerGetCapabilitiesRequest.Merge(m, src)
+}
+func (m *GroupControllerGetCapabilitiesRequest) XXX_Size() int {
+	return xxx_messageInfo_GroupControllerGetCapabilitiesRequest.Size(m)
+}
+func (m *GroupControllerGetCapabilitiesRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_GroupControllerGetCapabilitiesRequest.DiscardUnknown(m)
 }
 
-var E_AlphaMessage = &proto.ExtensionDesc{
-	ExtendedType:  (*descriptor.MessageOptions)(nil),
-	ExtensionType: (*bool)(nil),
-	Field:         1060,
-	Name:          "csi.v1.alpha_message",
-	Tag:           "varint,1060,opt,name=alpha_message",
-	Filename:      "github.com/container-storage-interface/spec/csi.proto",
+var xxx_messageInfo_GroupControllerGetCapabilitiesRequest proto.InternalMessageInfo
+
+type GroupControllerGetCapabilitiesResponse struct {
+	// All the capabilities that the group controller service supports.
+	// This field is OPTIONAL.
+	Capabilities         []*GroupControllerServiceCapability `protobuf:"bytes,1,rep,name=capabilities,proto3" json:"capabilities,omitempty"`
+	XXX_NoUnkeyedLiteral struct{}                            `json:"-"`
+	XXX_unrecognized     []byte                              `json:"-"`
+	XXX_sizecache        int32                               `json:"-"`
 }
 
-var E_AlphaMethod = &proto.ExtensionDesc{
-	ExtendedType:  (*descriptor.MethodOptions)(nil),
-	ExtensionType: (*bool)(nil),
-	Field:         1060,
-	Name:          "csi.v1.alpha_method",
-	Tag:           "varint,1060,opt,name=alpha_method",
-	Filename:      "github.com/container-storage-interface/spec/csi.proto",
+func (m *GroupControllerGetCapabilitiesResponse) Reset() {
+	*m = GroupControllerGetCapabilitiesResponse{}
+}
+func (m *GroupControllerGetCapabilitiesResponse) String() string { return proto.CompactTextString(m) }
+func (*GroupControllerGetCapabilitiesResponse) ProtoMessage()    {}
+func (*GroupControllerGetCapabilitiesResponse) Descriptor() ([]byte, []int) {
+	return fileDescriptor_9cdb00adce470e01, []int{63}
 }
 
-var E_AlphaService = &proto.ExtensionDesc{
+func (m *GroupControllerGetCapabilitiesResponse) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_GroupControllerGetCapabilitiesResponse.Unmarshal(m, b)
+}
+func (m *GroupControllerGetCapabilitiesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_GroupControllerGetCapabilitiesResponse.Marshal(b, m, deterministic)
+}
+func (m *GroupControllerGetCapabilitiesResponse) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_GroupControllerGetCapabilitiesResponse.Merge(m, src)
+}
+func (m *GroupControllerGetCapabilitiesResponse) XXX_Size() int {
+	return xxx_messageInfo_GroupControllerGetCapabilitiesResponse.Size(m)
+}
+func (m *GroupControllerGetCapabilitiesResponse) XXX_DiscardUnknown() {
+	xxx_messageInfo_GroupControllerGetCapabilitiesResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_GroupControllerGetCapabilitiesResponse proto.InternalMessageInfo
+
+func (m *GroupControllerGetCapabilitiesResponse) GetCapabilities() []*GroupControllerServiceCapability {
+	if m != nil {
+		return m.Capabilities
+	}
+	return nil
+}
+
+// Specifies a capability of the group controller service.
+type GroupControllerServiceCapability struct {
+	// Types that are valid to be assigned to Type:
+	//
+	//	*GroupControllerServiceCapability_Rpc
+	Type                 isGroupControllerServiceCapability_Type `protobuf_oneof:"type"`
+	XXX_NoUnkeyedLiteral struct{}                                `json:"-"`
+	XXX_unrecognized     []byte                                  `json:"-"`
+	XXX_sizecache        int32                                   `json:"-"`
+}
+
+func (m *GroupControllerServiceCapability) Reset()         { *m = GroupControllerServiceCapability{} }
+func (m *GroupControllerServiceCapability) String() string { return proto.CompactTextString(m) }
+func (*GroupControllerServiceCapability) ProtoMessage()    {}
+func (*GroupControllerServiceCapability) Descriptor() ([]byte, []int) {
+	return fileDescriptor_9cdb00adce470e01, []int{64}
+}
+
+func (m *GroupControllerServiceCapability) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_GroupControllerServiceCapability.Unmarshal(m, b)
+}
+func (m *GroupControllerServiceCapability) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_GroupControllerServiceCapability.Marshal(b, m, deterministic)
+}
+func (m *GroupControllerServiceCapability) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_GroupControllerServiceCapability.Merge(m, src)
+}
+func (m *GroupControllerServiceCapability) XXX_Size() int {
+	return xxx_messageInfo_GroupControllerServiceCapability.Size(m)
+}
+func (m *GroupControllerServiceCapability) XXX_DiscardUnknown() {
+	xxx_messageInfo_GroupControllerServiceCapability.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_GroupControllerServiceCapability proto.InternalMessageInfo
+
+type isGroupControllerServiceCapability_Type interface {
+	isGroupControllerServiceCapability_Type()
+}
+
+type GroupControllerServiceCapability_Rpc struct {
+	Rpc *GroupControllerServiceCapability_RPC `protobuf:"bytes,1,opt,name=rpc,proto3,oneof"`
+}
+
+func (*GroupControllerServiceCapability_Rpc) isGroupControllerServiceCapability_Type() {}
+
+func (m *GroupControllerServiceCapability) GetType() isGroupControllerServiceCapability_Type {
+	if m != nil {
+		return m.Type
+	}
+	return nil
+}
+
+func (m *GroupControllerServiceCapability) GetRpc() *GroupControllerServiceCapability_RPC {
+	if x, ok := m.GetType().(*GroupControllerServiceCapability_Rpc); ok {
+		return x.Rpc
+	}
+	return nil
+}
+
+// XXX_OneofWrappers is for the internal use of the proto package.
+func (*GroupControllerServiceCapability) XXX_OneofWrappers() []interface{} {
+	return []interface{}{
+		(*GroupControllerServiceCapability_Rpc)(nil),
+	}
+}
+
+type GroupControllerServiceCapability_RPC struct {
+	Type                 GroupControllerServiceCapability_RPC_Type `protobuf:"varint,1,opt,name=type,proto3,enum=csi.v1.GroupControllerServiceCapability_RPC_Type" json:"type,omitempty"`
+	XXX_NoUnkeyedLiteral struct{}                                  `json:"-"`
+	XXX_unrecognized     []byte                                    `json:"-"`
+	XXX_sizecache        int32                                     `json:"-"`
+}
+
+func (m *GroupControllerServiceCapability_RPC) Reset()         { *m = GroupControllerServiceCapability_RPC{} }
+func (m *GroupControllerServiceCapability_RPC) String() string { return proto.CompactTextString(m) }
+func (*GroupControllerServiceCapability_RPC) ProtoMessage()    {}
+func (*GroupControllerServiceCapability_RPC) Descriptor() ([]byte, []int) {
+	return fileDescriptor_9cdb00adce470e01, []int{64, 0}
+}
+
+func (m *GroupControllerServiceCapability_RPC) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_GroupControllerServiceCapability_RPC.Unmarshal(m, b)
+}
+func (m *GroupControllerServiceCapability_RPC) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_GroupControllerServiceCapability_RPC.Marshal(b, m, deterministic)
+}
+func (m *GroupControllerServiceCapability_RPC) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_GroupControllerServiceCapability_RPC.Merge(m, src)
+}
+func (m *GroupControllerServiceCapability_RPC) XXX_Size() int {
+	return xxx_messageInfo_GroupControllerServiceCapability_RPC.Size(m)
+}
+func (m *GroupControllerServiceCapability_RPC) XXX_DiscardUnknown() {
+	xxx_messageInfo_GroupControllerServiceCapability_RPC.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_GroupControllerServiceCapability_RPC proto.InternalMessageInfo
+
+func (m *GroupControllerServiceCapability_RPC) GetType() GroupControllerServiceCapability_RPC_Type {
+	if m != nil {
+		return m.Type
+	}
+	return GroupControllerServiceCapability_RPC_UNKNOWN
+}
+
+type CreateVolumeGroupSnapshotRequest struct {
+	// The suggested name for the group snapshot. This field is REQUIRED
+	// for idempotency.
+	// Any Unicode string that conforms to the length limit is allowed
+	// except those containing the following banned characters:
+	// U+0000-U+0008, U+000B, U+000C, U+000E-U+001F, U+007F-U+009F.
+	// (These are control characters other than commonly used whitespace.)
+	Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
+	// volume IDs of the source volumes to be snapshotted together.
+	// This field is REQUIRED.
+	SourceVolumeIds []string `protobuf:"bytes,2,rep,name=source_volume_ids,json=sourceVolumeIds,proto3" json:"source_volume_ids,omitempty"`
+	// Secrets required by plugin to complete
+	// ControllerCreateVolumeGroupSnapshot request.
+	// This field is OPTIONAL. Refer to the `Secrets Requirements`
+	// section on how to use this field.
+	// The secrets provided in this field SHOULD be the same for
+	// all group snapshot operations on the same group snapshot.
+	Secrets map[string]string `protobuf:"bytes,3,rep,name=secrets,proto3" json:"secrets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+	// Plugin specific parameters passed in as opaque key-value pairs.
+	// This field is OPTIONAL. The Plugin is responsible for parsing and
+	// validating these parameters. COs will treat these as opaque.
+	Parameters           map[string]string `protobuf:"bytes,4,rep,name=parameters,proto3" json:"parameters,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+	XXX_NoUnkeyedLiteral struct{}          `json:"-"`
+	XXX_unrecognized     []byte            `json:"-"`
+	XXX_sizecache        int32             `json:"-"`
+}
+
+func (m *CreateVolumeGroupSnapshotRequest) Reset()         { *m = CreateVolumeGroupSnapshotRequest{} }
+func (m *CreateVolumeGroupSnapshotRequest) String() string { return proto.CompactTextString(m) }
+func (*CreateVolumeGroupSnapshotRequest) ProtoMessage()    {}
+func (*CreateVolumeGroupSnapshotRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_9cdb00adce470e01, []int{65}
+}
+
+func (m *CreateVolumeGroupSnapshotRequest) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_CreateVolumeGroupSnapshotRequest.Unmarshal(m, b)
+}
+func (m *CreateVolumeGroupSnapshotRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_CreateVolumeGroupSnapshotRequest.Marshal(b, m, deterministic)
+}
+func (m *CreateVolumeGroupSnapshotRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_CreateVolumeGroupSnapshotRequest.Merge(m, src)
+}
+func (m *CreateVolumeGroupSnapshotRequest) XXX_Size() int {
+	return xxx_messageInfo_CreateVolumeGroupSnapshotRequest.Size(m)
+}
+func (m *CreateVolumeGroupSnapshotRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_CreateVolumeGroupSnapshotRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_CreateVolumeGroupSnapshotRequest proto.InternalMessageInfo
+
+func (m *CreateVolumeGroupSnapshotRequest) GetName() string {
+	if m != nil {
+		return m.Name
+	}
+	return ""
+}
+
+func (m *CreateVolumeGroupSnapshotRequest) GetSourceVolumeIds() []string {
+	if m != nil {
+		return m.SourceVolumeIds
+	}
+	return nil
+}
+
+func (m *CreateVolumeGroupSnapshotRequest) GetSecrets() map[string]string {
+	if m != nil {
+		return m.Secrets
+	}
+	return nil
+}
+
+func (m *CreateVolumeGroupSnapshotRequest) GetParameters() map[string]string {
+	if m != nil {
+		return m.Parameters
+	}
+	return nil
+}
+
+type CreateVolumeGroupSnapshotResponse struct {
+	// Contains all attributes of the newly created group snapshot.
+	// This field is REQUIRED.
+	GroupSnapshot        *VolumeGroupSnapshot `protobuf:"bytes,1,opt,name=group_snapshot,json=groupSnapshot,proto3" json:"group_snapshot,omitempty"`
+	XXX_NoUnkeyedLiteral struct{}             `json:"-"`
+	XXX_unrecognized     []byte               `json:"-"`
+	XXX_sizecache        int32                `json:"-"`
+}
+
+func (m *CreateVolumeGroupSnapshotResponse) Reset()         { *m = CreateVolumeGroupSnapshotResponse{} }
+func (m *CreateVolumeGroupSnapshotResponse) String() string { return proto.CompactTextString(m) }
+func (*CreateVolumeGroupSnapshotResponse) ProtoMessage()    {}
+func (*CreateVolumeGroupSnapshotResponse) Descriptor() ([]byte, []int) {
+	return fileDescriptor_9cdb00adce470e01, []int{66}
+}
+
+func (m *CreateVolumeGroupSnapshotResponse) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_CreateVolumeGroupSnapshotResponse.Unmarshal(m, b)
+}
+func (m *CreateVolumeGroupSnapshotResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_CreateVolumeGroupSnapshotResponse.Marshal(b, m, deterministic)
+}
+func (m *CreateVolumeGroupSnapshotResponse) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_CreateVolumeGroupSnapshotResponse.Merge(m, src)
+}
+func (m *CreateVolumeGroupSnapshotResponse) XXX_Size() int {
+	return xxx_messageInfo_CreateVolumeGroupSnapshotResponse.Size(m)
+}
+func (m *CreateVolumeGroupSnapshotResponse) XXX_DiscardUnknown() {
+	xxx_messageInfo_CreateVolumeGroupSnapshotResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_CreateVolumeGroupSnapshotResponse proto.InternalMessageInfo
+
+func (m *CreateVolumeGroupSnapshotResponse) GetGroupSnapshot() *VolumeGroupSnapshot {
+	if m != nil {
+		return m.GroupSnapshot
+	}
+	return nil
+}
+
+type VolumeGroupSnapshot struct {
+	// The identifier for this group snapshot, generated by the plugin.
+	// This field MUST contain enough information to uniquely identify
+	// this specific snapshot vs all other group snapshots supported by
+	// this plugin.
+	// This field SHALL be used by the CO in subsequent calls to refer to
+	// this group snapshot.
+	// The SP is NOT responsible for global uniqueness of
+	// group_snapshot_id across multiple SPs.
+	// This field is REQUIRED.
+	GroupSnapshotId string `protobuf:"bytes,1,opt,name=group_snapshot_id,json=groupSnapshotId,proto3" json:"group_snapshot_id,omitempty"`
+	// A list of snapshots belonging to this group.
+	// This field is REQUIRED.
+	Snapshots []*Snapshot `protobuf:"bytes,2,rep,name=snapshots,proto3" json:"snapshots,omitempty"`
+	// Timestamp of when the volume group snapshot was taken.
+	// This field is REQUIRED.
+	CreationTime *timestamp.Timestamp `protobuf:"bytes,3,opt,name=creation_time,json=creationTime,proto3" json:"creation_time,omitempty"`
+	// Indicates if all individual snapshots in the group snapshot
+	// are ready to use as a `volume_content_source` in a
+	// `CreateVolumeRequest`. The default value is false.
+	// If any snapshot in the list of snapshots in this message have
+	// ready_to_use set to false, the SP MUST set this field to false.
+	// If all of the snapshots in the list of snapshots in this message
+	// have ready_to_use set to true, the SP SHOULD set this field to
+	// true.
+	// This field is REQUIRED.
+	ReadyToUse           bool     `protobuf:"varint,4,opt,name=ready_to_use,json=readyToUse,proto3" json:"ready_to_use,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *VolumeGroupSnapshot) Reset()         { *m = VolumeGroupSnapshot{} }
+func (m *VolumeGroupSnapshot) String() string { return proto.CompactTextString(m) }
+func (*VolumeGroupSnapshot) ProtoMessage()    {}
+func (*VolumeGroupSnapshot) Descriptor() ([]byte, []int) {
+	return fileDescriptor_9cdb00adce470e01, []int{67}
+}
+
+func (m *VolumeGroupSnapshot) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_VolumeGroupSnapshot.Unmarshal(m, b)
+}
+func (m *VolumeGroupSnapshot) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_VolumeGroupSnapshot.Marshal(b, m, deterministic)
+}
+func (m *VolumeGroupSnapshot) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_VolumeGroupSnapshot.Merge(m, src)
+}
+func (m *VolumeGroupSnapshot) XXX_Size() int {
+	return xxx_messageInfo_VolumeGroupSnapshot.Size(m)
+}
+func (m *VolumeGroupSnapshot) XXX_DiscardUnknown() {
+	xxx_messageInfo_VolumeGroupSnapshot.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_VolumeGroupSnapshot proto.InternalMessageInfo
+
+func (m *VolumeGroupSnapshot) GetGroupSnapshotId() string {
+	if m != nil {
+		return m.GroupSnapshotId
+	}
+	return ""
+}
+
+func (m *VolumeGroupSnapshot) GetSnapshots() []*Snapshot {
+	if m != nil {
+		return m.Snapshots
+	}
+	return nil
+}
+
+func (m *VolumeGroupSnapshot) GetCreationTime() *timestamp.Timestamp {
+	if m != nil {
+		return m.CreationTime
+	}
+	return nil
+}
+
+func (m *VolumeGroupSnapshot) GetReadyToUse() bool {
+	if m != nil {
+		return m.ReadyToUse
+	}
+	return false
+}
+
+type DeleteVolumeGroupSnapshotRequest struct {
+	// The ID of the group snapshot to be deleted.
+	// This field is REQUIRED.
+	GroupSnapshotId string `protobuf:"bytes,1,opt,name=group_snapshot_id,json=groupSnapshotId,proto3" json:"group_snapshot_id,omitempty"`
+	// A list of snapshot IDs that are part of this group snapshot.
+	// If SP does not need to rely on this field to delete the snapshots
+	// in the group, it SHOULD check this field and report an error
+	// if it has the ability to detect a mismatch.
+	// Some SPs require this list to delete the snapshots in the group.
+	// If SP needs to use this field to delete the snapshots in the
+	// group, it MUST report an error if it has the ability to detect
+	// a mismatch.
+	// This field is REQUIRED.
+	SnapshotIds []string `protobuf:"bytes,2,rep,name=snapshot_ids,json=snapshotIds,proto3" json:"snapshot_ids,omitempty"`
+	// Secrets required by plugin to complete group snapshot deletion
+	// request.
+	// This field is OPTIONAL. Refer to the `Secrets Requirements`
+	// section on how to use this field.
+	// The secrets provided in this field SHOULD be the same for
+	// all group snapshot operations on the same group snapshot.
+	Secrets              map[string]string `protobuf:"bytes,3,rep,name=secrets,proto3" json:"secrets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+	XXX_NoUnkeyedLiteral struct{}          `json:"-"`
+	XXX_unrecognized     []byte            `json:"-"`
+	XXX_sizecache        int32             `json:"-"`
+}
+
+func (m *DeleteVolumeGroupSnapshotRequest) Reset()         { *m = DeleteVolumeGroupSnapshotRequest{} }
+func (m *DeleteVolumeGroupSnapshotRequest) String() string { return proto.CompactTextString(m) }
+func (*DeleteVolumeGroupSnapshotRequest) ProtoMessage()    {}
+func (*DeleteVolumeGroupSnapshotRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_9cdb00adce470e01, []int{68}
+}
+
+func (m *DeleteVolumeGroupSnapshotRequest) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_DeleteVolumeGroupSnapshotRequest.Unmarshal(m, b)
+}
+func (m *DeleteVolumeGroupSnapshotRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_DeleteVolumeGroupSnapshotRequest.Marshal(b, m, deterministic)
+}
+func (m *DeleteVolumeGroupSnapshotRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_DeleteVolumeGroupSnapshotRequest.Merge(m, src)
+}
+func (m *DeleteVolumeGroupSnapshotRequest) XXX_Size() int {
+	return xxx_messageInfo_DeleteVolumeGroupSnapshotRequest.Size(m)
+}
+func (m *DeleteVolumeGroupSnapshotRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_DeleteVolumeGroupSnapshotRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_DeleteVolumeGroupSnapshotRequest proto.InternalMessageInfo
+
+func (m *DeleteVolumeGroupSnapshotRequest) GetGroupSnapshotId() string {
+	if m != nil {
+		return m.GroupSnapshotId
+	}
+	return ""
+}
+
+func (m *DeleteVolumeGroupSnapshotRequest) GetSnapshotIds() []string {
+	if m != nil {
+		return m.SnapshotIds
+	}
+	return nil
+}
+
+func (m *DeleteVolumeGroupSnapshotRequest) GetSecrets() map[string]string {
+	if m != nil {
+		return m.Secrets
+	}
+	return nil
+}
+
+type DeleteVolumeGroupSnapshotResponse struct {
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *DeleteVolumeGroupSnapshotResponse) Reset()         { *m = DeleteVolumeGroupSnapshotResponse{} }
+func (m *DeleteVolumeGroupSnapshotResponse) String() string { return proto.CompactTextString(m) }
+func (*DeleteVolumeGroupSnapshotResponse) ProtoMessage()    {}
+func (*DeleteVolumeGroupSnapshotResponse) Descriptor() ([]byte, []int) {
+	return fileDescriptor_9cdb00adce470e01, []int{69}
+}
+
+func (m *DeleteVolumeGroupSnapshotResponse) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_DeleteVolumeGroupSnapshotResponse.Unmarshal(m, b)
+}
+func (m *DeleteVolumeGroupSnapshotResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_DeleteVolumeGroupSnapshotResponse.Marshal(b, m, deterministic)
+}
+func (m *DeleteVolumeGroupSnapshotResponse) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_DeleteVolumeGroupSnapshotResponse.Merge(m, src)
+}
+func (m *DeleteVolumeGroupSnapshotResponse) XXX_Size() int {
+	return xxx_messageInfo_DeleteVolumeGroupSnapshotResponse.Size(m)
+}
+func (m *DeleteVolumeGroupSnapshotResponse) XXX_DiscardUnknown() {
+	xxx_messageInfo_DeleteVolumeGroupSnapshotResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_DeleteVolumeGroupSnapshotResponse proto.InternalMessageInfo
+
+type GetVolumeGroupSnapshotRequest struct {
+	// The ID of the group snapshot to fetch current group snapshot
+	// information for.
+	// This field is REQUIRED.
+	GroupSnapshotId string `protobuf:"bytes,1,opt,name=group_snapshot_id,json=groupSnapshotId,proto3" json:"group_snapshot_id,omitempty"`
+	// A list of snapshot IDs that are part of this group snapshot.
+	// If SP does not need to rely on this field to get the snapshots
+	// in the group, it SHOULD check this field and report an error
+	// if it has the ability to detect a mismatch.
+	// Some SPs require this list to get the snapshots in the group.
+	// If SP needs to use this field to get the snapshots in the
+	// group, it MUST report an error if it has the ability to detect
+	// a mismatch.
+	// This field is REQUIRED.
+	SnapshotIds []string `protobuf:"bytes,2,rep,name=snapshot_ids,json=snapshotIds,proto3" json:"snapshot_ids,omitempty"`
+	// Secrets required by plugin to complete
+	// GetVolumeGroupSnapshot request.
+	// This field is OPTIONAL. Refer to the `Secrets Requirements`
+	// section on how to use this field.
+	// The secrets provided in this field SHOULD be the same for
+	// all group snapshot operations on the same group snapshot.
+	Secrets              map[string]string `protobuf:"bytes,3,rep,name=secrets,proto3" json:"secrets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+	XXX_NoUnkeyedLiteral struct{}          `json:"-"`
+	XXX_unrecognized     []byte            `json:"-"`
+	XXX_sizecache        int32             `json:"-"`
+}
+
+func (m *GetVolumeGroupSnapshotRequest) Reset()         { *m = GetVolumeGroupSnapshotRequest{} }
+func (m *GetVolumeGroupSnapshotRequest) String() string { return proto.CompactTextString(m) }
+func (*GetVolumeGroupSnapshotRequest) ProtoMessage()    {}
+func (*GetVolumeGroupSnapshotRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_9cdb00adce470e01, []int{70}
+}
+
+func (m *GetVolumeGroupSnapshotRequest) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_GetVolumeGroupSnapshotRequest.Unmarshal(m, b)
+}
+func (m *GetVolumeGroupSnapshotRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_GetVolumeGroupSnapshotRequest.Marshal(b, m, deterministic)
+}
+func (m *GetVolumeGroupSnapshotRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_GetVolumeGroupSnapshotRequest.Merge(m, src)
+}
+func (m *GetVolumeGroupSnapshotRequest) XXX_Size() int {
+	return xxx_messageInfo_GetVolumeGroupSnapshotRequest.Size(m)
+}
+func (m *GetVolumeGroupSnapshotRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_GetVolumeGroupSnapshotRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_GetVolumeGroupSnapshotRequest proto.InternalMessageInfo
+
+func (m *GetVolumeGroupSnapshotRequest) GetGroupSnapshotId() string {
+	if m != nil {
+		return m.GroupSnapshotId
+	}
+	return ""
+}
+
+func (m *GetVolumeGroupSnapshotRequest) GetSnapshotIds() []string {
+	if m != nil {
+		return m.SnapshotIds
+	}
+	return nil
+}
+
+func (m *GetVolumeGroupSnapshotRequest) GetSecrets() map[string]string {
+	if m != nil {
+		return m.Secrets
+	}
+	return nil
+}
+
+type GetVolumeGroupSnapshotResponse struct {
+	// This field is REQUIRED
+	GroupSnapshot        *VolumeGroupSnapshot `protobuf:"bytes,1,opt,name=group_snapshot,json=groupSnapshot,proto3" json:"group_snapshot,omitempty"`
+	XXX_NoUnkeyedLiteral struct{}             `json:"-"`
+	XXX_unrecognized     []byte               `json:"-"`
+	XXX_sizecache        int32                `json:"-"`
+}
+
+func (m *GetVolumeGroupSnapshotResponse) Reset()         { *m = GetVolumeGroupSnapshotResponse{} }
+func (m *GetVolumeGroupSnapshotResponse) String() string { return proto.CompactTextString(m) }
+func (*GetVolumeGroupSnapshotResponse) ProtoMessage()    {}
+func (*GetVolumeGroupSnapshotResponse) Descriptor() ([]byte, []int) {
+	return fileDescriptor_9cdb00adce470e01, []int{71}
+}
+
+func (m *GetVolumeGroupSnapshotResponse) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_GetVolumeGroupSnapshotResponse.Unmarshal(m, b)
+}
+func (m *GetVolumeGroupSnapshotResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_GetVolumeGroupSnapshotResponse.Marshal(b, m, deterministic)
+}
+func (m *GetVolumeGroupSnapshotResponse) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_GetVolumeGroupSnapshotResponse.Merge(m, src)
+}
+func (m *GetVolumeGroupSnapshotResponse) XXX_Size() int {
+	return xxx_messageInfo_GetVolumeGroupSnapshotResponse.Size(m)
+}
+func (m *GetVolumeGroupSnapshotResponse) XXX_DiscardUnknown() {
+	xxx_messageInfo_GetVolumeGroupSnapshotResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_GetVolumeGroupSnapshotResponse proto.InternalMessageInfo
+
+func (m *GetVolumeGroupSnapshotResponse) GetGroupSnapshot() *VolumeGroupSnapshot {
+	if m != nil {
+		return m.GroupSnapshot
+	}
+	return nil
+}
+
+var E_AlphaEnum = &proto.ExtensionDesc{
+	ExtendedType:  (*descriptor.EnumOptions)(nil),
+	ExtensionType: (*bool)(nil),
+	Field:         1060,
+	Name:          "csi.v1.alpha_enum",
+	Tag:           "varint,1060,opt,name=alpha_enum",
+	Filename:      "github.com/container-storage-interface/spec/csi.proto",
+}
+
+var E_AlphaEnumValue = &proto.ExtensionDesc{
+	ExtendedType:  (*descriptor.EnumValueOptions)(nil),
+	ExtensionType: (*bool)(nil),
+	Field:         1060,
+	Name:          "csi.v1.alpha_enum_value",
+	Tag:           "varint,1060,opt,name=alpha_enum_value",
+	Filename:      "github.com/container-storage-interface/spec/csi.proto",
+}
+
+var E_CsiSecret = &proto.ExtensionDesc{
+	ExtendedType:  (*descriptor.FieldOptions)(nil),
+	ExtensionType: (*bool)(nil),
+	Field:         1059,
+	Name:          "csi.v1.csi_secret",
+	Tag:           "varint,1059,opt,name=csi_secret",
+	Filename:      "github.com/container-storage-interface/spec/csi.proto",
+}
+
+var E_AlphaField = &proto.ExtensionDesc{
+	ExtendedType:  (*descriptor.FieldOptions)(nil),
+	ExtensionType: (*bool)(nil),
+	Field:         1060,
+	Name:          "csi.v1.alpha_field",
+	Tag:           "varint,1060,opt,name=alpha_field",
+	Filename:      "github.com/container-storage-interface/spec/csi.proto",
+}
+
+var E_AlphaMessage = &proto.ExtensionDesc{
+	ExtendedType:  (*descriptor.MessageOptions)(nil),
+	ExtensionType: (*bool)(nil),
+	Field:         1060,
+	Name:          "csi.v1.alpha_message",
+	Tag:           "varint,1060,opt,name=alpha_message",
+	Filename:      "github.com/container-storage-interface/spec/csi.proto",
+}
+
+var E_AlphaMethod = &proto.ExtensionDesc{
+	ExtendedType:  (*descriptor.MethodOptions)(nil),
+	ExtensionType: (*bool)(nil),
+	Field:         1060,
+	Name:          "csi.v1.alpha_method",
+	Tag:           "varint,1060,opt,name=alpha_method",
+	Filename:      "github.com/container-storage-interface/spec/csi.proto",
+}
+
+var E_AlphaService = &proto.ExtensionDesc{
 	ExtendedType:  (*descriptor.ServiceOptions)(nil),
 	ExtensionType: (*bool)(nil),
 	Field:         1060,
@@ -4944,6 +5783,7 @@ func init() {
 	proto.RegisterEnum("csi.v1.ControllerServiceCapability_RPC_Type", ControllerServiceCapability_RPC_Type_name, ControllerServiceCapability_RPC_Type_value)
 	proto.RegisterEnum("csi.v1.VolumeUsage_Unit", VolumeUsage_Unit_name, VolumeUsage_Unit_value)
 	proto.RegisterEnum("csi.v1.NodeServiceCapability_RPC_Type", NodeServiceCapability_RPC_Type_name, NodeServiceCapability_RPC_Type_value)
+	proto.RegisterEnum("csi.v1.GroupControllerServiceCapability_RPC_Type", GroupControllerServiceCapability_RPC_Type_name, GroupControllerServiceCapability_RPC_Type_value)
 	proto.RegisterType((*GetPluginInfoRequest)(nil), "csi.v1.GetPluginInfoRequest")
 	proto.RegisterType((*GetPluginInfoResponse)(nil), "csi.v1.GetPluginInfoResponse")
 	proto.RegisterMapType((map[string]string)(nil), "csi.v1.GetPluginInfoResponse.ManifestEntry")
@@ -4955,6 +5795,7 @@ func init() {
 	proto.RegisterType((*ProbeRequest)(nil), "csi.v1.ProbeRequest")
 	proto.RegisterType((*ProbeResponse)(nil), "csi.v1.ProbeResponse")
 	proto.RegisterType((*CreateVolumeRequest)(nil), "csi.v1.CreateVolumeRequest")
+	proto.RegisterMapType((map[string]string)(nil), "csi.v1.CreateVolumeRequest.MutableParametersEntry")
 	proto.RegisterMapType((map[string]string)(nil), "csi.v1.CreateVolumeRequest.ParametersEntry")
 	proto.RegisterMapType((map[string]string)(nil), "csi.v1.CreateVolumeRequest.SecretsEntry")
 	proto.RegisterType((*VolumeContentSource)(nil), "csi.v1.VolumeContentSource")
@@ -4983,11 +5824,13 @@ func init() {
 	proto.RegisterMapType((map[string]string)(nil), "csi.v1.ControllerUnpublishVolumeRequest.SecretsEntry")
 	proto.RegisterType((*ControllerUnpublishVolumeResponse)(nil), "csi.v1.ControllerUnpublishVolumeResponse")
 	proto.RegisterType((*ValidateVolumeCapabilitiesRequest)(nil), "csi.v1.ValidateVolumeCapabilitiesRequest")
+	proto.RegisterMapType((map[string]string)(nil), "csi.v1.ValidateVolumeCapabilitiesRequest.MutableParametersEntry")
 	proto.RegisterMapType((map[string]string)(nil), "csi.v1.ValidateVolumeCapabilitiesRequest.ParametersEntry")
 	proto.RegisterMapType((map[string]string)(nil), "csi.v1.ValidateVolumeCapabilitiesRequest.SecretsEntry")
 	proto.RegisterMapType((map[string]string)(nil), "csi.v1.ValidateVolumeCapabilitiesRequest.VolumeContextEntry")
 	proto.RegisterType((*ValidateVolumeCapabilitiesResponse)(nil), "csi.v1.ValidateVolumeCapabilitiesResponse")
 	proto.RegisterType((*ValidateVolumeCapabilitiesResponse_Confirmed)(nil), "csi.v1.ValidateVolumeCapabilitiesResponse.Confirmed")
+	proto.RegisterMapType((map[string]string)(nil), "csi.v1.ValidateVolumeCapabilitiesResponse.Confirmed.MutableParametersEntry")
 	proto.RegisterMapType((map[string]string)(nil), "csi.v1.ValidateVolumeCapabilitiesResponse.Confirmed.ParametersEntry")
 	proto.RegisterMapType((map[string]string)(nil), "csi.v1.ValidateVolumeCapabilitiesResponse.Confirmed.VolumeContextEntry")
 	proto.RegisterType((*ListVolumesRequest)(nil), "csi.v1.ListVolumesRequest")
@@ -4997,6 +5840,10 @@ func init() {
 	proto.RegisterType((*ControllerGetVolumeRequest)(nil), "csi.v1.ControllerGetVolumeRequest")
 	proto.RegisterType((*ControllerGetVolumeResponse)(nil), "csi.v1.ControllerGetVolumeResponse")
 	proto.RegisterType((*ControllerGetVolumeResponse_VolumeStatus)(nil), "csi.v1.ControllerGetVolumeResponse.VolumeStatus")
+	proto.RegisterType((*ControllerModifyVolumeRequest)(nil), "csi.v1.ControllerModifyVolumeRequest")
+	proto.RegisterMapType((map[string]string)(nil), "csi.v1.ControllerModifyVolumeRequest.MutableParametersEntry")
+	proto.RegisterMapType((map[string]string)(nil), "csi.v1.ControllerModifyVolumeRequest.SecretsEntry")
+	proto.RegisterType((*ControllerModifyVolumeResponse)(nil), "csi.v1.ControllerModifyVolumeResponse")
 	proto.RegisterType((*GetCapacityRequest)(nil), "csi.v1.GetCapacityRequest")
 	proto.RegisterMapType((map[string]string)(nil), "csi.v1.GetCapacityRequest.ParametersEntry")
 	proto.RegisterType((*GetCapacityResponse)(nil), "csi.v1.GetCapacityResponse")
@@ -5046,6 +5893,21 @@ func init() {
 	proto.RegisterType((*NodeExpandVolumeRequest)(nil), "csi.v1.NodeExpandVolumeRequest")
 	proto.RegisterMapType((map[string]string)(nil), "csi.v1.NodeExpandVolumeRequest.SecretsEntry")
 	proto.RegisterType((*NodeExpandVolumeResponse)(nil), "csi.v1.NodeExpandVolumeResponse")
+	proto.RegisterType((*GroupControllerGetCapabilitiesRequest)(nil), "csi.v1.GroupControllerGetCapabilitiesRequest")
+	proto.RegisterType((*GroupControllerGetCapabilitiesResponse)(nil), "csi.v1.GroupControllerGetCapabilitiesResponse")
+	proto.RegisterType((*GroupControllerServiceCapability)(nil), "csi.v1.GroupControllerServiceCapability")
+	proto.RegisterType((*GroupControllerServiceCapability_RPC)(nil), "csi.v1.GroupControllerServiceCapability.RPC")
+	proto.RegisterType((*CreateVolumeGroupSnapshotRequest)(nil), "csi.v1.CreateVolumeGroupSnapshotRequest")
+	proto.RegisterMapType((map[string]string)(nil), "csi.v1.CreateVolumeGroupSnapshotRequest.ParametersEntry")
+	proto.RegisterMapType((map[string]string)(nil), "csi.v1.CreateVolumeGroupSnapshotRequest.SecretsEntry")
+	proto.RegisterType((*CreateVolumeGroupSnapshotResponse)(nil), "csi.v1.CreateVolumeGroupSnapshotResponse")
+	proto.RegisterType((*VolumeGroupSnapshot)(nil), "csi.v1.VolumeGroupSnapshot")
+	proto.RegisterType((*DeleteVolumeGroupSnapshotRequest)(nil), "csi.v1.DeleteVolumeGroupSnapshotRequest")
+	proto.RegisterMapType((map[string]string)(nil), "csi.v1.DeleteVolumeGroupSnapshotRequest.SecretsEntry")
+	proto.RegisterType((*DeleteVolumeGroupSnapshotResponse)(nil), "csi.v1.DeleteVolumeGroupSnapshotResponse")
+	proto.RegisterType((*GetVolumeGroupSnapshotRequest)(nil), "csi.v1.GetVolumeGroupSnapshotRequest")
+	proto.RegisterMapType((map[string]string)(nil), "csi.v1.GetVolumeGroupSnapshotRequest.SecretsEntry")
+	proto.RegisterType((*GetVolumeGroupSnapshotResponse)(nil), "csi.v1.GetVolumeGroupSnapshotResponse")
 	proto.RegisterExtension(E_AlphaEnum)
 	proto.RegisterExtension(E_AlphaEnumValue)
 	proto.RegisterExtension(E_CsiSecret)
@@ -5060,245 +5922,278 @@ func init() {
 }
 
 var fileDescriptor_9cdb00adce470e01 = []byte{
-	// 3802 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe4, 0x3b, 0x4b, 0x6c, 0x23, 0x47,
-	0x76, 0x6a, 0xfe, 0x24, 0x3d, 0x4a, 0x1a, 0xaa, 0x28, 0x69, 0x38, 0x2d, 0x69, 0xa4, 0xe9, 0xf1,
-	0x78, 0xe5, 0xf1, 0x0c, 0x67, 0xad, 0xb5, 0x8d, 0x58, 0x1e, 0xef, 0x9a, 0xa4, 0x38, 0x12, 0x77,
-	0x28, 0x52, 0x6e, 0x52, 0x33, 0x3b, 0x93, 0x18, 0xed, 0x16, 0x59, 0xe2, 0x34, 0x4c, 0x76, 0xd3,
-	0xdd, 0x4d, 0x45, 0xf2, 0x25, 0x41, 0x82, 0x1c, 0x82, 0x5c, 0x72, 0x5b, 0xe7, 0xb6, 0x48, 0xf6,
-	0x98, 0xc5, 0x22, 0x08, 0x82, 0x1c, 0x03, 0xe4, 0x18, 0x20, 0x9b, 0xdc, 0x12, 0xe4, 0xb2, 0x87,
-	0x00, 0x39, 0x2c, 0x12, 0xc0, 0x97, 0x5c, 0x72, 0x08, 0x82, 0xae, 0xaa, 0x6e, 0xf6, 0x97, 0x9f,
-	0x91, 0x06, 0x3e, 0xe4, 0x24, 0xf6, 0xab, 0xf7, 0x5e, 0xbd, 0xaa, 0x7a, 0xef, 0xd5, 0xfb, 0x94,
-	0xe0, 0x83, 0x8e, 0x62, 0xbe, 0x1a, 0x9c, 0xe6, 0x5b, 0x5a, 0xef, 0x51, 0x4b, 0x53, 0x4d, 0x59,
-	0x51, 0xb1, 0xfe, 0xd0, 0x30, 0x35, 0x5d, 0xee, 0xe0, 0x87, 0x8a, 0x6a, 0x62, 0xfd, 0x4c, 0x6e,
-	0xe1, 0x47, 0x46, 0x1f, 0xb7, 0x1e, 0xb5, 0x0c, 0x25, 0xdf, 0xd7, 0x35, 0x53, 0x43, 0x29, 0xeb,
-	0xe7, 0xf9, 0x7b, 0xfc, 0x76, 0x47, 0xd3, 0x3a, 0x5d, 0xfc, 0x88, 0x40, 0x4f, 0x07, 0x67, 0x8f,
-	0xda, 0xd8, 0x68, 0xe9, 0x4a, 0xdf, 0xd4, 0x74, 0x8a, 0xc9, 0x6f, 0xf9, 0x31, 0x4c, 0xa5, 0x87,
-	0x0d, 0x53, 0xee, 0xf5, 0x19, 0xc2, 0x6d, 0x3f, 0xc2, 0xef, 0xea, 0x72, 0xbf, 0x8f, 0x75, 0x83,
-	0x8e, 0x0b, 0x6b, 0xb0, 0x72, 0x80, 0xcd, 0xe3, 0xee, 0xa0, 0xa3, 0xa8, 0x15, 0xf5, 0x4c, 0x13,
-	0xf1, 0x57, 0x03, 0x6c, 0x98, 0xc2, 0xbf, 0x72, 0xb0, 0xea, 0x1b, 0x30, 0xfa, 0x9a, 0x6a, 0x60,
-	0x84, 0x20, 0xa1, 0xca, 0x3d, 0x9c, 0xe3, 0xb6, 0xb9, 0x9d, 0x79, 0x91, 0xfc, 0x46, 0xf7, 0x60,
-	0xe9, 0x1c, 0xab, 0x6d, 0x4d, 0x97, 0xce, 0xb1, 0x6e, 0x28, 0x9a, 0x9a, 0x8b, 0x91, 0xd1, 0x45,
-	0x0a, 0x7d, 0x46, 0x81, 0xe8, 0x00, 0xe6, 0x7a, 0xb2, 0xaa, 0x9c, 0x61, 0xc3, 0xcc, 0xc5, 0xb7,
-	0xe3, 0x3b, 0xe9, 0xdd, 0x77, 0xf3, 0x74, 0xa9, 0xf9, 0xd0, 0xb9, 0xf2, 0x47, 0x0c, 0xbb, 0xac,
-	0x9a, 0xfa, 0xa5, 0xe8, 0x10, 0xf3, 0x1f, 0xc3, 0xa2, 0x67, 0x08, 0x65, 0x20, 0xfe, 0x25, 0xbe,
-	0x64, 0x32, 0x59, 0x3f, 0xd1, 0x0a, 0x24, 0xcf, 0xe5, 0xee, 0x00, 0x33, 0x49, 0xe8, 0xc7, 0x5e,
-	0xec, 0xb7, 0x38, 0xe1, 0x36, 0x6c, 0x38, 0xb3, 0x95, 0xe4, 0xbe, 0x7c, 0xaa, 0x74, 0x15, 0x53,
-	0xc1, 0x86, 0xbd, 0xf4, 0xcf, 0x61, 0x33, 0x62, 0x9c, 0xed, 0xc0, 0x63, 0x58, 0x68, 0xb9, 0xe0,
-	0x39, 0x8e, 0x2c, 0x25, 0x67, 0x2f, 0xc5, 0x47, 0x79, 0x29, 0x7a, 0xb0, 0x85, 0x5f, 0xc5, 0x21,
-	0xe3, 0x47, 0x41, 0x8f, 0x61, 0xd6, 0xc0, 0xfa, 0xb9, 0xd2, 0xa2, 0xfb, 0x9a, 0xde, 0xdd, 0x8e,
-	0xe2, 0x96, 0x6f, 0x50, 0xbc, 0xc3, 0x19, 0xd1, 0x26, 0x41, 0x27, 0x90, 0x39, 0xd7, 0xba, 0x83,
-	0x1e, 0x96, 0xf0, 0x45, 0x5f, 0x56, 0x9d, 0x03, 0x48, 0xef, 0xee, 0x44, 0xb2, 0x79, 0x46, 0x08,
-	0xca, 0x36, 0xfe, 0xe1, 0x8c, 0x78, 0xe3, 0xdc, 0x0b, 0xe2, 0x7f, 0xca, 0xc1, 0x2c, 0x9b, 0x0d,
-	0x7d, 0x04, 0x09, 0xf3, 0xb2, 0x4f, 0xa5, 0x5b, 0xda, 0xbd, 0x37, 0x4e, 0xba, 0x7c, 0xf3, 0xb2,
-	0x8f, 0x45, 0x42, 0x22, 0x7c, 0x06, 0x09, 0xeb, 0x0b, 0xa5, 0x61, 0xf6, 0xa4, 0xf6, 0xb4, 0x56,
-	0x7f, 0x5e, 0xcb, 0xcc, 0xa0, 0x35, 0x40, 0xa5, 0x7a, 0xad, 0x29, 0xd6, 0xab, 0xd5, 0xb2, 0x28,
-	0x35, 0xca, 0xe2, 0xb3, 0x4a, 0xa9, 0x9c, 0xe1, 0xd0, 0x5b, 0xb0, 0xfd, 0xac, 0x5e, 0x3d, 0x39,
-	0x2a, 0x4b, 0x85, 0x52, 0xa9, 0xdc, 0x68, 0x54, 0x8a, 0x95, 0x6a, 0xa5, 0xf9, 0x42, 0x2a, 0xd5,
-	0x6b, 0x8d, 0xa6, 0x58, 0xa8, 0xd4, 0x9a, 0x8d, 0x4c, 0x8c, 0xff, 0x03, 0x0e, 0x6e, 0xf8, 0x16,
-	0x80, 0x0a, 0x1e, 0x09, 0x1f, 0x4e, 0xba, 0x70, 0xb7, 0xa4, 0x0f, 0xc2, 0x24, 0x05, 0x48, 0xd5,
-	0x6b, 0xd5, 0x4a, 0xcd, 0x92, 0x2e, 0x0d, 0xb3, 0xf5, 0x27, 0x4f, 0xc8, 0x47, 0xac, 0x98, 0xa2,
-	0x13, 0x0a, 0x4b, 0xb0, 0x70, 0xac, 0x6b, 0xa7, 0xd8, 0xd6, 0x9f, 0x02, 0x2c, 0xb2, 0x6f, 0xa6,
-	0x2f, 0xdf, 0x87, 0xa4, 0x8e, 0xe5, 0xf6, 0x25, 0x3b, 0x5a, 0x3e, 0x4f, 0x6d, 0x32, 0x6f, 0xdb,
-	0x64, 0xbe, 0xa8, 0x69, 0xdd, 0x67, 0x96, 0x7e, 0x8a, 0x14, 0x51, 0xf8, 0x36, 0x01, 0xd9, 0x92,
-	0x8e, 0x65, 0x13, 0x53, 0x69, 0x19, 0xeb, 0x50, 0xdb, 0x7b, 0x0c, 0x4b, 0x96, 0x7e, 0xb5, 0x14,
-	0xf3, 0x52, 0xd2, 0x65, 0xb5, 0x83, 0xd9, 0xd1, 0xaf, 0xda, 0x3b, 0x50, 0x62, 0xa3, 0xa2, 0x35,
-	0x28, 0x2e, 0xb6, 0xdc, 0x9f, 0xa8, 0x02, 0x59, 0xa6, 0x3a, 0x1e, 0x95, 0x8e, 0x7b, 0x55, 0x9a,
-	0x4a, 0xe1, 0x52, 0x69, 0x74, 0xee, 0x85, 0x28, 0xd8, 0x40, 0x4f, 0x01, 0xfa, 0xb2, 0x2e, 0xf7,
-	0xb0, 0x89, 0x75, 0x23, 0x97, 0xf0, 0xda, 0x77, 0xc8, 0x6a, 0xf2, 0xc7, 0x0e, 0x36, 0xb5, 0x6f,
-	0x17, 0x39, 0x3a, 0xb0, 0x0c, 0xa2, 0xa5, 0x63, 0xd3, 0xc8, 0x25, 0x09, 0xa7, 0x9d, 0x51, 0x9c,
-	0x1a, 0x14, 0x95, 0xb0, 0x29, 0xc6, 0xbf, 0x29, 0x72, 0xa2, 0x4d, 0x8d, 0xea, 0xb0, 0x6a, 0x2f,
-	0x50, 0x53, 0x4d, 0xac, 0x9a, 0x92, 0xa1, 0x0d, 0xf4, 0x16, 0xce, 0xa5, 0xc8, 0x2e, 0xad, 0xfb,
-	0x96, 0x48, 0x71, 0x1a, 0x04, 0x45, 0x64, 0x5b, 0xe3, 0x01, 0xa2, 0x97, 0xc0, 0xcb, 0xad, 0x16,
-	0x36, 0x0c, 0x85, 0xee, 0x85, 0xa4, 0xe3, 0xaf, 0x06, 0x8a, 0x8e, 0x7b, 0x58, 0x35, 0x8d, 0xdc,
-	0xac, 0x97, 0x6b, 0x53, 0xeb, 0x6b, 0x5d, 0xad, 0x73, 0x29, 0x0e, 0x71, 0xc4, 0x5b, 0x1e, 0x72,
-	0xd7, 0x88, 0xc1, 0x7f, 0x02, 0x37, 0x7c, 0x9b, 0x32, 0x8d, 0x67, 0xe3, 0xf7, 0x60, 0xc1, 0xbd,
-	0x13, 0x53, 0x79, 0xc5, 0x3f, 0x89, 0x41, 0x36, 0x64, 0x0f, 0xd0, 0x21, 0xcc, 0x19, 0xaa, 0xdc,
-	0x37, 0x5e, 0x69, 0x26, 0xd3, 0xdf, 0xfb, 0x23, 0xb6, 0x2c, 0xdf, 0x60, 0xb8, 0xf4, 0xf3, 0x70,
-	0x46, 0x74, 0xa8, 0x51, 0x11, 0x52, 0x74, 0x3f, 0xfd, 0xbe, 0x29, 0x8c, 0x0f, 0x85, 0x39, 0x5c,
-	0x18, 0x25, 0xff, 0x1e, 0x2c, 0x79, 0x67, 0x40, 0x5b, 0x90, 0xb6, 0x67, 0x90, 0x94, 0x36, 0x5b,
-	0x2b, 0xd8, 0xa0, 0x4a, 0x9b, 0x7f, 0x17, 0x16, 0xdc, 0xcc, 0xd0, 0x3a, 0xcc, 0x33, 0x85, 0x70,
-	0xd0, 0xe7, 0x28, 0xa0, 0xd2, 0x76, 0x6c, 0xfa, 0x87, 0xb0, 0xe2, 0xd5, 0x33, 0x66, 0xca, 0x6f,
-	0x3b, 0x6b, 0xa0, 0x7b, 0xb1, 0xe4, 0x5d, 0x83, 0x2d, 0xa7, 0xf0, 0xf3, 0x24, 0x64, 0xfc, 0x46,
-	0x83, 0x1e, 0x43, 0xf2, 0xb4, 0xab, 0xb5, 0xbe, 0x64, 0xb4, 0x6f, 0x45, 0x59, 0x57, 0xbe, 0x68,
-	0x61, 0x51, 0xe8, 0xe1, 0x8c, 0x48, 0x89, 0x2c, 0xea, 0x9e, 0x36, 0x50, 0x4d, 0xb6, 0x7b, 0xd1,
-	0xd4, 0x47, 0x16, 0xd6, 0x90, 0x9a, 0x10, 0xa1, 0x7d, 0x48, 0x53, 0xb5, 0x93, 0x7a, 0x5a, 0x1b,
-	0xe7, 0xe2, 0x84, 0xc7, 0xdd, 0x48, 0x1e, 0x05, 0x82, 0x7b, 0xa4, 0xb5, 0xb1, 0x08, 0xb2, 0xf3,
-	0x9b, 0x5f, 0x84, 0xb4, 0x4b, 0x36, 0xfe, 0x6b, 0x48, 0xbb, 0x26, 0x43, 0x37, 0x61, 0xf6, 0xcc,
-	0x90, 0x1c, 0x27, 0x3c, 0x2f, 0xa6, 0xce, 0x0c, 0xe2, 0x4f, 0xb7, 0x20, 0x4d, 0xa4, 0x90, 0xce,
-	0xba, 0x72, 0xc7, 0xc8, 0xc5, 0xb6, 0xe3, 0xd6, 0x19, 0x11, 0xd0, 0x13, 0x0b, 0x82, 0xde, 0x03,
-	0xe6, 0x50, 0x24, 0x8a, 0xd7, 0xd1, 0xb5, 0x41, 0x9f, 0x08, 0x39, 0x5f, 0x8c, 0xff, 0xac, 0xc8,
-	0x89, 0xec, 0x7e, 0x23, 0xb3, 0x1d, 0x58, 0x83, 0xfc, 0x5f, 0xc7, 0x00, 0x86, 0x52, 0xa2, 0xc7,
-	0x90, 0x20, 0x0b, 0xa3, 0xde, 0x7f, 0x67, 0x82, 0x85, 0xe5, 0xc9, 0xea, 0x08, 0x95, 0xf0, 0x1f,
-	0x1c, 0x24, 0x08, 0x1b, 0xff, 0x1d, 0xd5, 0xa8, 0xd4, 0x0e, 0xaa, 0x65, 0xa9, 0x56, 0xdf, 0x2f,
-	0x4b, 0xcf, 0xc5, 0x4a, 0xb3, 0x2c, 0x66, 0x38, 0xb4, 0x0e, 0x37, 0xdd, 0x70, 0xb1, 0x5c, 0xd8,
-	0x2f, 0x8b, 0x52, 0xbd, 0x56, 0x7d, 0x91, 0x89, 0x21, 0x1e, 0xd6, 0x8e, 0x4e, 0xaa, 0xcd, 0x4a,
-	0x70, 0x2c, 0x8e, 0x36, 0x20, 0xe7, 0x1a, 0x63, 0x3c, 0x18, 0xdb, 0x84, 0xc5, 0xd6, 0x35, 0x4a,
-	0x7f, 0xb2, 0xc1, 0x24, 0x12, 0xe0, 0x96, 0x7b, 0x4e, 0x2f, 0x6d, 0x8a, 0xb7, 0x36, 0x09, 0xdd,
-	0x81, 0x9c, 0x1b, 0xc7, 0xc3, 0x61, 0x96, 0xa0, 0x14, 0x17, 0x1d, 0x35, 0x20, 0x6a, 0xfe, 0x1c,
-	0x16, 0x3d, 0xb7, 0x83, 0x15, 0xc8, 0x31, 0x77, 0xd6, 0x96, 0x4e, 0x2f, 0x4d, 0x12, 0xdc, 0x70,
-	0x3b, 0x71, 0x71, 0xd1, 0x86, 0x16, 0x2d, 0xa0, 0x75, 0xa0, 0x5d, 0xa5, 0xa7, 0x98, 0x0c, 0x27,
-	0x46, 0x70, 0x80, 0x80, 0x08, 0x82, 0xf0, 0xeb, 0x18, 0xa4, 0x98, 0x56, 0xdc, 0x73, 0xdd, 0x4f,
-	0x1e, 0x96, 0x36, 0x94, 0xb2, 0xf4, 0x98, 0x65, 0xcc, 0x6b, 0x96, 0xe8, 0x10, 0x96, 0xdc, 0x4e,
-	0xfc, 0xc2, 0x0e, 0x1f, 0xef, 0x78, 0xcf, 0xd9, 0xed, 0x49, 0x2e, 0x58, 0xd0, 0xb8, 0x78, 0xee,
-	0x86, 0xa1, 0x22, 0x2c, 0xf9, 0xee, 0x81, 0xc4, 0xf8, 0x7b, 0x60, 0xb1, 0xe5, 0x71, 0x89, 0x05,
-	0xc8, 0xda, 0x2e, 0xbc, 0x8b, 0x25, 0x93, 0xb9, 0x78, 0x76, 0x4f, 0x65, 0x02, 0xae, 0x1f, 0x0d,
-	0x91, 0x6d, 0x18, 0xff, 0x29, 0xa0, 0xa0, 0xac, 0x53, 0xf9, 0xeb, 0x01, 0x64, 0x43, 0x2e, 0x17,
-	0x94, 0x87, 0x79, 0x72, 0x54, 0x86, 0x62, 0x62, 0x16, 0x98, 0x06, 0x25, 0x1a, 0xa2, 0x58, 0xf8,
-	0x7d, 0x1d, 0x9f, 0x61, 0x5d, 0xc7, 0x6d, 0x62, 0x98, 0xa1, 0xf8, 0x0e, 0x8a, 0xf0, 0x87, 0x1c,
-	0xcc, 0xd9, 0x70, 0xb4, 0x07, 0x73, 0x06, 0xee, 0xd0, 0x8b, 0x8f, 0xce, 0x75, 0xdb, 0x4f, 0x9b,
-	0x6f, 0x30, 0x04, 0x16, 0xc2, 0xdb, 0xf8, 0x56, 0x08, 0xef, 0x19, 0x9a, 0x6a, 0xf1, 0x7f, 0xcb,
-	0x41, 0x76, 0x1f, 0x77, 0xb1, 0x3f, 0x3e, 0x1a, 0xe5, 0xdb, 0xdd, 0x21, 0x45, 0xcc, 0x1b, 0x52,
-	0x84, 0xb0, 0x1a, 0x11, 0x52, 0x5c, 0xe9, 0x9a, 0x5d, 0x83, 0x15, 0xef, 0x6c, 0xf4, 0x62, 0x11,
-	0xfe, 0x2b, 0x0e, 0xb7, 0x2d, 0x5d, 0xd0, 0xb5, 0x6e, 0x17, 0xeb, 0xc7, 0x83, 0xd3, 0xae, 0x62,
-	0xbc, 0x9a, 0x62, 0x71, 0x37, 0x61, 0x56, 0xd5, 0xda, 0x2e, 0xe3, 0x49, 0x59, 0x9f, 0x95, 0x36,
-	0x2a, 0xc3, 0xb2, 0x3f, 0xc0, 0xbb, 0x64, 0xee, 0x3f, 0x3a, 0xbc, 0xcb, 0x9c, 0xfb, 0xef, 0x2e,
-	0x1e, 0xe6, 0xac, 0xd0, 0x54, 0x53, 0xbb, 0x97, 0xc4, 0x62, 0xe6, 0x44, 0xe7, 0x1b, 0x89, 0xfe,
-	0x58, 0xed, 0x07, 0x4e, 0xac, 0x36, 0x72, 0x45, 0xa3, 0xc2, 0xb6, 0x2f, 0x02, 0x16, 0x9f, 0x22,
-	0xac, 0x3f, 0x9a, 0x90, 0xf5, 0x58, 0x4f, 0x70, 0x95, 0x53, 0xbc, 0x06, 0xf3, 0xfd, 0x07, 0x0e,
-	0xb6, 0x22, 0x97, 0xc0, 0x82, 0x8d, 0x36, 0xdc, 0xe8, 0xd3, 0x01, 0x67, 0x13, 0xa8, 0x95, 0x7d,
-	0x3c, 0x76, 0x13, 0x58, 0xfe, 0xcc, 0xa0, 0x9e, 0x6d, 0x58, 0xea, 0x7b, 0x80, 0x7c, 0x01, 0xb2,
-	0x21, 0x68, 0x53, 0x2d, 0xe6, 0x37, 0x1c, 0x6c, 0x0f, 0x45, 0x39, 0x51, 0xfb, 0xd7, 0xa7, 0xbe,
-	0xcd, 0xa1, 0x6e, 0x51, 0x97, 0xff, 0x41, 0x70, 0xed, 0xe1, 0x13, 0xbe, 0x29, 0x0b, 0xbe, 0x0b,
-	0x77, 0x46, 0x4c, 0xcd, 0xcc, 0xf9, 0xd7, 0x09, 0xb8, 0xf3, 0x4c, 0xee, 0x2a, 0x6d, 0x27, 0x84,
-	0x0c, 0xa9, 0x34, 0x8c, 0xde, 0x92, 0x56, 0xc0, 0x02, 0xa8, 0xd7, 0x7a, 0xec, 0x58, 0xed, 0x38,
-	0xfe, 0x13, 0x5c, 0x87, 0xd7, 0x98, 0xfe, 0xbd, 0x08, 0x49, 0xff, 0x3e, 0x9a, 0x5c, 0xd6, 0x51,
-	0xc9, 0xe0, 0x89, 0xdf, 0xc1, 0x7c, 0x38, 0x39, 0xdf, 0x11, 0x5a, 0x70, 0x65, 0x2b, 0xfe, 0x2e,
-	0xf3, 0xb5, 0xbf, 0x4f, 0x80, 0x30, 0x6a, 0xf5, 0xcc, 0x87, 0x88, 0x30, 0xdf, 0xd2, 0xd4, 0x33,
-	0x45, 0xef, 0xe1, 0x36, 0xcb, 0x3b, 0xde, 0x9f, 0x64, 0xf3, 0x98, 0x03, 0x29, 0xd9, 0xb4, 0xe2,
-	0x90, 0x0d, 0xca, 0xc1, 0x6c, 0x0f, 0x1b, 0x86, 0xdc, 0xb1, 0xc5, 0xb2, 0x3f, 0xf9, 0x5f, 0xc4,
-	0x61, 0xde, 0x21, 0x41, 0x6a, 0x40, 0x83, 0xa9, 0xfb, 0x3a, 0x78, 0x1d, 0x01, 0x5e, 0x5f, 0x99,
-	0x63, 0xaf, 0xa1, 0xcc, 0x6d, 0x8f, 0x32, 0x53, 0x73, 0xd8, 0x7f, 0x2d, 0xb1, 0x47, 0xe8, 0xf5,
-	0x77, 0xae, 0x80, 0xc2, 0xef, 0x00, 0xaa, 0x2a, 0x06, 0xcb, 0xdf, 0x1c, 0xb7, 0x64, 0xa5, 0x6b,
-	0xf2, 0x85, 0x84, 0x55, 0x53, 0x57, 0x58, 0xb8, 0x9e, 0x14, 0xa1, 0x27, 0x5f, 0x94, 0x29, 0xc4,
-	0x0a, 0xe9, 0x0d, 0x53, 0xd6, 0x4d, 0x45, 0xed, 0x48, 0xa6, 0xf6, 0x25, 0x76, 0xca, 0xbd, 0x36,
-	0xb4, 0x69, 0x01, 0x85, 0xff, 0x8c, 0x41, 0xd6, 0xc3, 0x9e, 0xe9, 0xe4, 0xc7, 0x30, 0x3b, 0xe4,
-	0xed, 0x09, 0xe3, 0x43, 0xb0, 0xf3, 0x74, 0xdb, 0x6c, 0x0a, 0xb4, 0x09, 0xa0, 0xe2, 0x0b, 0xd3,
-	0x33, 0xef, 0xbc, 0x05, 0x21, 0x73, 0xf2, 0x7f, 0xc4, 0x39, 0xe9, 0xbe, 0x29, 0x9b, 0x03, 0x03,
-	0x3d, 0x00, 0xc4, 0x5c, 0x34, 0x6e, 0x4b, 0xec, 0x8e, 0xa1, 0xf3, 0xce, 0x8b, 0x19, 0x67, 0xa4,
-	0x46, 0x6e, 0x1b, 0x03, 0x1d, 0x38, 0x95, 0xd4, 0x96, 0xa6, 0xb6, 0x15, 0x73, 0x58, 0x49, 0xbd,
-	0x19, 0x48, 0x10, 0xe8, 0x30, 0xcd, 0x4f, 0x6f, 0x9c, 0x7b, 0xa1, 0xfc, 0x57, 0x90, 0xa4, 0xc7,
-	0x31, 0x61, 0xc5, 0x00, 0x7d, 0x0a, 0x29, 0x83, 0x48, 0xec, 0xaf, 0x8e, 0x84, 0xed, 0x89, 0x7b,
-	0x85, 0x22, 0xa3, 0x13, 0x7e, 0x08, 0xfc, 0xf0, 0x62, 0x3a, 0xc0, 0xe6, 0xe4, 0xd7, 0xef, 0x9e,
-	0xb5, 0x06, 0xe1, 0xa7, 0x31, 0x58, 0x0f, 0x65, 0x30, 0x5d, 0xed, 0x03, 0x1d, 0xfa, 0x56, 0xf2,
-	0xfd, 0xe0, 0x8d, 0x1d, 0x60, 0x1e, 0xba, 0x22, 0xfe, 0xf7, 0xaf, 0x76, 0x98, 0xc5, 0xa9, 0x0f,
-	0x33, 0x70, 0x8e, 0x74, 0x67, 0x7e, 0x11, 0x03, 0x74, 0x80, 0x4d, 0x27, 0x55, 0x66, 0x5b, 0x1a,
-	0xe1, 0x6f, 0xb8, 0xd7, 0xf0, 0x37, 0x3f, 0xf6, 0xf8, 0x1b, 0xea, 0xb1, 0xee, 0xbb, 0x7a, 0x23,
-	0xbe, 0xa9, 0x47, 0xde, 0x96, 0x11, 0xe9, 0x29, 0x8d, 0xf9, 0x27, 0x4b, 0x4f, 0xaf, 0xe8, 0x56,
-	0xfe, 0x9d, 0x83, 0xac, 0x47, 0x68, 0xa6, 0x41, 0x0f, 0x01, 0xc9, 0xe7, 0xb2, 0xd2, 0x95, 0x2d,
-	0xc1, 0xec, 0xf4, 0x9f, 0x95, 0x03, 0x96, 0x9d, 0x11, 0x9b, 0x0c, 0x3d, 0x85, 0x6c, 0x4f, 0xbe,
-	0x50, 0x7a, 0x83, 0x9e, 0xc4, 0xf6, 0xd9, 0x50, 0xbe, 0xb6, 0xab, 0x87, 0xeb, 0x81, 0x2a, 0x7a,
-	0x45, 0x35, 0x3f, 0x7c, 0x9f, 0x96, 0xd1, 0x97, 0x19, 0x1d, 0x53, 0x1e, 0xe5, 0x6b, 0x8c, 0x8e,
-	0x21, 0xdb, 0x53, 0xd4, 0x00, 0xb3, 0xf8, 0x58, 0x66, 0xd4, 0xc0, 0x97, 0x19, 0xf1, 0x90, 0xa3,
-	0x20, 0xb8, 0x83, 0x5e, 0xb6, 0x5c, 0x7f, 0x2f, 0xa9, 0xeb, 0x0e, 0x16, 0x03, 0x38, 0x6c, 0x5b,
-	0x0e, 0x42, 0xfb, 0x49, 0x77, 0x83, 0x66, 0xc3, 0x9a, 0x2b, 0x91, 0xad, 0xa5, 0xff, 0x8d, 0xbb,
-	0x2d, 0x38, 0x80, 0x8d, 0x3e, 0x86, 0xb8, 0xde, 0x6f, 0x31, 0xf3, 0xfd, 0xde, 0x04, 0xfc, 0xf3,
-	0xe2, 0x71, 0xe9, 0x70, 0x46, 0xb4, 0xa8, 0xf8, 0x3f, 0x8b, 0x43, 0x5c, 0x3c, 0x2e, 0xa1, 0x4f,
-	0x3d, 0x7d, 0x96, 0x07, 0x13, 0x72, 0x71, 0xb7, 0x59, 0xfe, 0x29, 0x16, 0xd6, 0x67, 0xc9, 0xc1,
-	0x4a, 0x49, 0x2c, 0x17, 0x9a, 0x65, 0x69, 0xbf, 0x5c, 0x2d, 0x37, 0xcb, 0x12, 0xed, 0x03, 0x65,
-	0x38, 0xb4, 0x01, 0xb9, 0xe3, 0x93, 0x62, 0xb5, 0xd2, 0x38, 0x94, 0x4e, 0x6a, 0xf6, 0x2f, 0x36,
-	0x1a, 0x43, 0x19, 0x58, 0xa8, 0x56, 0x1a, 0x4d, 0x06, 0x68, 0x64, 0xe2, 0x16, 0xe4, 0xa0, 0xdc,
-	0x94, 0x4a, 0x85, 0xe3, 0x42, 0xa9, 0xd2, 0x7c, 0x91, 0x49, 0x20, 0x1e, 0xd6, 0xbc, 0xbc, 0x1b,
-	0xb5, 0xc2, 0x71, 0xe3, 0xb0, 0xde, 0xcc, 0x24, 0x11, 0x82, 0x25, 0x42, 0x6f, 0x83, 0x1a, 0x99,
-	0x94, 0xc5, 0xa1, 0x54, 0xad, 0xd7, 0x1c, 0x19, 0x66, 0xd1, 0x0a, 0x64, 0xec, 0x99, 0xc5, 0x72,
-	0x61, 0x9f, 0x14, 0xf4, 0xe6, 0xd0, 0x32, 0x2c, 0x96, 0x7f, 0x72, 0x5c, 0xa8, 0xed, 0xdb, 0x88,
-	0xf3, 0x68, 0x1b, 0x36, 0xdc, 0xe2, 0x48, 0x8c, 0xaa, 0xbc, 0x4f, 0x8a, 0x72, 0x8d, 0x0c, 0xa0,
-	0x5b, 0x90, 0x61, 0x2d, 0xae, 0x52, 0xbd, 0xb6, 0x5f, 0x69, 0x56, 0xea, 0xb5, 0x4c, 0x9a, 0x56,
-	0xf0, 0xb2, 0x00, 0x96, 0xe4, 0x8c, 0xd9, 0xc2, 0xf8, 0xb2, 0xde, 0x22, 0x2d, 0xeb, 0xd9, 0x65,
-	0xeb, 0xdf, 0xc4, 0x60, 0x95, 0xd6, 0xad, 0xed, 0x2a, 0xb9, 0xed, 0xab, 0x76, 0x20, 0x43, 0xeb,
-	0x5d, 0x92, 0xff, 0x16, 0x58, 0xa2, 0xf0, 0x67, 0x76, 0xde, 0x61, 0xf7, 0x98, 0x62, 0xae, 0x1e,
-	0x53, 0xc5, 0x9f, 0x85, 0xdd, 0xf7, 0x76, 0x63, 0x7c, 0xb3, 0x8d, 0x4a, 0xec, 0x8f, 0x42, 0xd2,
-	0x84, 0x87, 0xa3, 0xb9, 0x8d, 0x0a, 0xa1, 0xae, 0x92, 0xc5, 0x5f, 0xd1, 0xcb, 0x3d, 0x81, 0x35,
-	0xbf, 0xbc, 0xcc, 0xa0, 0x1f, 0x04, 0x7a, 0x26, 0x8e, 0xdb, 0x75, 0x70, 0x1d, 0x0c, 0xe1, 0x5f,
-	0x38, 0x98, 0xb3, 0xc1, 0x56, 0x78, 0x63, 0xf9, 0x25, 0x4f, 0xa5, 0x74, 0xde, 0x82, 0x38, 0x85,
-	0x57, 0x77, 0xb7, 0x23, 0xe6, 0xef, 0x76, 0x84, 0x9e, 0x73, 0x3c, 0xf4, 0x9c, 0x7f, 0x04, 0x8b,
-	0x2d, 0x4b, 0x7c, 0x45, 0x53, 0x25, 0x53, 0xe9, 0xd9, 0x85, 0xd0, 0x60, 0x77, 0xb2, 0x69, 0x3f,
-	0x29, 0x10, 0x17, 0x6c, 0x02, 0x0b, 0x84, 0xb6, 0x61, 0x81, 0x74, 0x2b, 0x25, 0x53, 0x93, 0x06,
-	0x06, 0xce, 0x25, 0x49, 0x59, 0x08, 0x08, 0xac, 0xa9, 0x9d, 0x18, 0x58, 0xf8, 0x3b, 0x0e, 0x56,
-	0x69, 0xb5, 0xcb, 0xaf, 0x8e, 0xe3, 0xba, 0x36, 0x6e, 0x8d, 0xf3, 0xdd, 0x86, 0xa1, 0x0c, 0xdf,
-	0x54, 0xb2, 0x9f, 0x83, 0x35, 0xff, 0x7c, 0x2c, 0xc3, 0xff, 0x65, 0x0c, 0x56, 0xac, 0xd0, 0xcc,
-	0x1e, 0xb8, 0xee, 0xe8, 0x79, 0x8a, 0x93, 0xf4, 0x6d, 0x66, 0x22, 0xb0, 0x99, 0x87, 0xfe, 0xfc,
-	0xf9, 0x1d, 0x77, 0x70, 0xe9, 0x5f, 0xc1, 0x9b, 0xda, 0xcb, 0xbf, 0xe4, 0x60, 0xd5, 0x37, 0x1f,
-	0xb3, 0x97, 0x4f, 0xfc, 0x09, 0xc1, 0xdd, 0x08, 0xf9, 0x5e, 0x2b, 0x25, 0xf8, 0xc0, 0x0e, 0xc5,
-	0xa7, 0x33, 0xcb, 0x7f, 0x8e, 0xc1, 0xe6, 0xf0, 0x52, 0x23, 0xef, 0x05, 0xda, 0x53, 0x54, 0xb4,
-	0xae, 0xd6, 0x96, 0xff, 0xcc, 0xef, 0x70, 0x77, 0x83, 0xf7, 0x6c, 0x88, 0x48, 0xa3, 0x1c, 0x6f,
-	0x68, 0x21, 0x38, 0x31, 0x6d, 0x21, 0xf8, 0x4a, 0x1a, 0xf0, 0x7b, 0xee, 0x1a, 0xb7, 0x57, 0x7c,
-	0xa6, 0x09, 0x13, 0x36, 0x8b, 0x3e, 0x84, 0x9b, 0x24, 0xfa, 0x77, 0x9e, 0xbb, 0xd8, 0x4d, 0x78,
-	0xea, 0x12, 0xe7, 0xc4, 0x55, 0x6b, 0xd8, 0x79, 0xe3, 0xc1, 0x1a, 0x24, 0x6d, 0xe1, 0xdb, 0x04,
-	0xac, 0x59, 0xd9, 0x41, 0xc3, 0x94, 0x3b, 0xd3, 0xb4, 0x0e, 0x7e, 0x3b, 0x58, 0x89, 0x8d, 0x79,
-	0x8f, 0x25, 0x9c, 0xeb, 0x24, 0x05, 0x58, 0x94, 0x87, 0xac, 0x61, 0xca, 0x1d, 0xe2, 0x0e, 0x64,
-	0xbd, 0x83, 0x4d, 0xa9, 0x2f, 0x9b, 0xaf, 0x98, 0xad, 0x2f, 0xb3, 0xa1, 0x26, 0x19, 0x39, 0x96,
-	0xcd, 0x57, 0xd7, 0x74, 0x90, 0xe8, 0xc7, 0x7e, 0xa7, 0xf0, 0xee, 0x98, 0xb5, 0x8c, 0xd0, 0xad,
-	0x9f, 0x44, 0x54, 0xeb, 0xdf, 0x1b, 0xc3, 0x72, 0x7c, 0x95, 0xfe, 0xea, 0xd5, 0xe9, 0xef, 0xb8,
-	0xd0, 0x7f, 0x0b, 0x6e, 0x06, 0x16, 0xcf, 0xae, 0x90, 0x0e, 0xe4, 0xac, 0xa1, 0x13, 0xd5, 0x98,
-	0x52, 0x1d, 0x23, 0x34, 0x26, 0x16, 0xa1, 0x31, 0xc2, 0x3a, 0xdc, 0x0a, 0x99, 0x88, 0x49, 0xf1,
-	0x37, 0x49, 0x2a, 0xc6, 0xf4, 0x3d, 0xa7, 0xcf, 0xa3, 0xac, 0xe2, 0x7d, 0xf7, 0xb1, 0x87, 0xb6,
-	0x67, 0xde, 0x84, 0x5d, 0x6c, 0x41, 0xda, 0x8d, 0xc7, 0xae, 0x41, 0x73, 0x8c, 0xe1, 0x24, 0xaf,
-	0xd4, 0x0a, 0x4b, 0xf9, 0x5a, 0x61, 0xd5, 0xa1, 0x51, 0xcd, 0x7a, 0x43, 0xdb, 0xc8, 0xad, 0x18,
-	0x61, 0x56, 0x2f, 0x03, 0x66, 0x35, 0xe7, 0xed, 0xaf, 0x45, 0x32, 0xfd, 0x7f, 0x60, 0x58, 0x4c,
-	0xa9, 0x43, 0x1b, 0x5f, 0xc2, 0x4b, 0xe0, 0xa9, 0xc6, 0x4f, 0xdf, 0x8a, 0xf2, 0xa9, 0x51, 0xcc,
-	0xaf, 0x46, 0xc2, 0x26, 0xac, 0x87, 0xf2, 0x66, 0x53, 0xff, 0x31, 0x47, 0x05, 0x73, 0x6a, 0x5c,
-	0x0d, 0x53, 0x36, 0x8d, 0x49, 0xa7, 0x66, 0x83, 0xee, 0xa9, 0x29, 0x88, 0x68, 0xf0, 0x94, 0x26,
-	0x21, 0xfc, 0x29, 0x47, 0xf7, 0xc1, 0x2f, 0x0b, 0xbb, 0x6d, 0xdf, 0x81, 0xe4, 0x80, 0x94, 0xf1,
-	0x69, 0xd4, 0x95, 0xf5, 0x1a, 0xc1, 0x89, 0x35, 0x24, 0x52, 0x8c, 0x6b, 0x2b, 0x8c, 0x0a, 0xbf,
-	0xe4, 0x20, 0xed, 0xe2, 0x8f, 0x36, 0x60, 0xde, 0xa9, 0xfc, 0xd8, 0xf9, 0x8e, 0x03, 0xb0, 0x8e,
-	0xdf, 0xd4, 0x4c, 0xb9, 0xcb, 0x9e, 0x98, 0xd0, 0x0f, 0x2b, 0x45, 0x1d, 0x18, 0x98, 0x86, 0xc3,
-	0x71, 0x91, 0xfc, 0x46, 0x0f, 0x20, 0x31, 0x50, 0x15, 0x93, 0x98, 0xfd, 0x92, 0xdf, 0x9e, 0xc9,
-	0x54, 0xf9, 0x13, 0x55, 0x31, 0x45, 0x82, 0x25, 0xdc, 0x87, 0x84, 0xf5, 0xe5, 0xad, 0x40, 0xcc,
-	0x43, 0xb2, 0xf8, 0xa2, 0x59, 0x6e, 0x64, 0x38, 0x04, 0x90, 0xaa, 0xd0, 0x7c, 0x3d, 0x26, 0x54,
-	0xed, 0xb7, 0xa6, 0xce, 0x22, 0x2c, 0x17, 0x20, 0x9f, 0xaa, 0x9a, 0xde, 0x93, 0xbb, 0x44, 0xe6,
-	0x39, 0xd1, 0xf9, 0x8e, 0xee, 0x8e, 0xd0, 0x5a, 0xe2, 0x86, 0x73, 0x22, 0x61, 0xf5, 0xa2, 0x2f,
-	0xa8, 0x6e, 0x45, 0x55, 0x8a, 0x0a, 0xa1, 0x95, 0xa2, 0x4d, 0xcf, 0x2d, 0x3b, 0xa6, 0x46, 0xf4,
-	0xab, 0x18, 0xac, 0x86, 0xe2, 0xa1, 0x0f, 0xdc, 0xd5, 0xa1, 0x3b, 0x23, 0x79, 0xba, 0xeb, 0x42,
-	0xff, 0xcd, 0xd1, 0xba, 0xd0, 0x9e, 0xa7, 0x2e, 0xf4, 0xf6, 0x58, 0x7a, 0x77, 0x45, 0xe8, 0xaf,
-	0xb8, 0x88, 0x8a, 0x50, 0xa3, 0x59, 0x38, 0x28, 0x4b, 0x27, 0x35, 0xfa, 0xd7, 0xa9, 0x08, 0xad,
-	0x40, 0x66, 0x58, 0x27, 0x91, 0x1a, 0xcd, 0x42, 0xb3, 0x91, 0x89, 0x05, 0xab, 0x31, 0xf1, 0xd0,
-	0x5a, 0x4b, 0x62, 0x7c, 0x59, 0x25, 0x49, 0x51, 0xd6, 0x01, 0x31, 0xea, 0xa3, 0xfa, 0x49, 0xad,
-	0x29, 0x1d, 0x88, 0xf5, 0x93, 0x63, 0xf6, 0xda, 0xca, 0xa9, 0xb9, 0xac, 0x00, 0x62, 0x47, 0xe6,
-	0x7e, 0x3f, 0xff, 0xe7, 0x1c, 0x64, 0x3d, 0x60, 0x76, 0x82, 0xae, 0x46, 0x37, 0xe7, 0x69, 0x74,
-	0x3f, 0x82, 0x15, 0x2b, 0x6d, 0xa4, 0xe6, 0x62, 0x48, 0x7d, 0xac, 0x93, 0x02, 0x37, 0x53, 0xfc,
-	0xe5, 0x9e, 0x7c, 0xc1, 0x9a, 0x00, 0xc7, 0x58, 0xb7, 0x18, 0x5f, 0x43, 0x99, 0x57, 0xf8, 0x26,
-	0x4e, 0x83, 0x93, 0xa9, 0x93, 0x9b, 0xb1, 0x8e, 0x2a, 0x98, 0xfd, 0xc4, 0xa7, 0xc8, 0x7e, 0x22,
-	0xdc, 0x5c, 0x62, 0xaa, 0x88, 0x78, 0xfa, 0x8b, 0xbd, 0x36, 0xbc, 0xbc, 0x69, 0xf8, 0xfa, 0xc0,
-	0xad, 0xc4, 0x63, 0xd3, 0xad, 0xd4, 0x37, 0x45, 0xee, 0x67, 0xd7, 0x95, 0x2c, 0x17, 0x68, 0x50,
-	0x76, 0x85, 0x24, 0x69, 0xf7, 0x7f, 0x38, 0x98, 0xab, 0xb4, 0xb1, 0x6a, 0xd2, 0xb5, 0x2d, 0x7a,
-	0xfe, 0xc5, 0x02, 0x6d, 0x44, 0xfc, 0xe7, 0x05, 0x59, 0x18, 0xbf, 0x39, 0xf2, 0xff, 0x32, 0x84,
-	0x19, 0x74, 0xe6, 0xfa, 0xf7, 0x10, 0x4f, 0x27, 0xe3, 0xad, 0x00, 0x65, 0x88, 0x9f, 0xe3, 0xef,
-	0x8d, 0xc1, 0x72, 0xe6, 0xf9, 0x10, 0x92, 0xe4, 0x31, 0x3d, 0x5a, 0x71, 0x1e, 0xf4, 0xbb, 0xde,
-	0xda, 0xf3, 0xab, 0x3e, 0xa8, 0x4d, 0xb7, 0xfb, 0x8f, 0xf3, 0x00, 0xc3, 0x5c, 0x13, 0x3d, 0x85,
-	0x05, 0xf7, 0x7b, 0x5e, 0xb4, 0x3e, 0xe2, 0x35, 0x39, 0xbf, 0x11, 0x3e, 0xe8, 0xc8, 0xf4, 0x14,
-	0x16, 0xdc, 0x6f, 0xb8, 0x86, 0xcc, 0x42, 0xde, 0x91, 0x0d, 0x99, 0x85, 0x3e, 0xfb, 0x9a, 0x41,
-	0x5d, 0xb8, 0x19, 0xf1, 0x8a, 0x07, 0xbd, 0x3d, 0xd9, 0x5b, 0x27, 0xfe, 0x7b, 0x13, 0x3e, 0x07,
-	0x12, 0x66, 0x90, 0x0e, 0xb7, 0x22, 0x1f, 0xaf, 0xa0, 0x9d, 0x49, 0x9f, 0xd6, 0xf0, 0xef, 0x4c,
-	0x80, 0xe9, 0xcc, 0x39, 0x00, 0x3e, 0xba, 0x63, 0x8e, 0xde, 0x99, 0xf8, 0x29, 0x07, 0x7f, 0x7f,
-	0xf2, 0x06, 0xbc, 0x30, 0x83, 0x0e, 0x21, 0xed, 0x6a, 0x9d, 0x22, 0x3e, 0xb4, 0x9f, 0x4a, 0x19,
-	0xaf, 0x8f, 0xe8, 0xb5, 0x52, 0x4e, 0xae, 0x6e, 0xd6, 0x90, 0x53, 0xb0, 0x2f, 0x37, 0xe4, 0x14,
-	0xd2, 0xfe, 0xf2, 0x6f, 0xbf, 0xef, 0x92, 0x0f, 0xdb, 0xfe, 0xf0, 0x28, 0x21, 0x6c, 0xfb, 0x23,
-	0x22, 0x06, 0x61, 0x06, 0x7d, 0x06, 0x4b, 0xde, 0x32, 0x35, 0xda, 0x1c, 0x59, 0x6e, 0xe7, 0x6f,
-	0x47, 0x0d, 0xbb, 0x59, 0x7a, 0xab, 0xa2, 0x43, 0x96, 0xa1, 0xd5, 0xd9, 0x21, 0xcb, 0x88, 0x62,
-	0xea, 0x8c, 0xe5, 0x9f, 0x3c, 0xb5, 0xbe, 0xa1, 0x7f, 0x0a, 0x2b, 0x51, 0x0e, 0xfd, 0x53, 0x68,
-	0x81, 0x50, 0x98, 0x41, 0x0a, 0xac, 0x85, 0x97, 0x9a, 0xd0, 0xbd, 0x89, 0x2a, 0x69, 0xfc, 0xdb,
-	0xe3, 0xd0, 0x9c, 0xa9, 0x5a, 0x90, 0x0d, 0xe9, 0x6c, 0x23, 0x61, 0x64, 0xdb, 0x9b, 0x4e, 0x72,
-	0x77, 0x82, 0xd6, 0xb8, 0x60, 0x45, 0x1b, 0xbb, 0xff, 0x96, 0x84, 0x04, 0xb9, 0xf6, 0x9b, 0x70,
-	0xc3, 0x57, 0x4f, 0x40, 0xb7, 0x47, 0x57, 0x59, 0xf8, 0xad, 0xc8, 0x71, 0x67, 0x0d, 0x2f, 0x61,
-	0x39, 0x50, 0x21, 0x40, 0xdb, 0x6e, 0xba, 0xb0, 0x2a, 0x05, 0x7f, 0x67, 0x04, 0x86, 0x9f, 0xb7,
-	0xd7, 0xb7, 0x6d, 0x8f, 0x4b, 0x61, 0xbd, 0xbc, 0xa3, 0xfc, 0xd9, 0x17, 0x34, 0xca, 0xf2, 0x7b,
-	0x32, 0xc1, 0x2b, 0x57, 0xa8, 0x0f, 0xbb, 0x3b, 0x12, 0xc7, 0x99, 0xe1, 0x73, 0x27, 0xbc, 0x73,
-	0x65, 0x50, 0xc8, 0x23, 0x5c, 0x68, 0xa6, 0xc7, 0x0b, 0xa3, 0x50, 0x1c, 0xf6, 0xcf, 0x21, 0xe3,
-	0xbf, 0xe7, 0xd1, 0xd6, 0x98, 0xb0, 0x83, 0xdf, 0x8e, 0x46, 0xf0, 0xef, 0x8c, 0xdf, 0xc9, 0xf8,
-	0xa5, 0x0a, 0x73, 0x2f, 0x77, 0x47, 0xe2, 0xb8, 0xdd, 0xa2, 0x2b, 0xc2, 0x1d, 0xba, 0xc5, 0x60,
-	0x34, 0x3c, 0x74, 0x8b, 0x21, 0x21, 0xb1, 0x30, 0xb3, 0xf7, 0x18, 0x40, 0xee, 0xf6, 0x5f, 0xc9,
-	0x12, 0x56, 0x07, 0x3d, 0xb4, 0x11, 0xe8, 0x40, 0x95, 0xd5, 0x41, 0xaf, 0xde, 0xb7, 0x32, 0x2f,
-	0x23, 0xf7, 0xf3, 0x39, 0x92, 0x6f, 0xcd, 0x13, 0x02, 0x6b, 0x60, 0xaf, 0x0a, 0x99, 0x21, 0xb5,
-	0x44, 0x42, 0x28, 0x74, 0x27, 0x94, 0x07, 0xe9, 0xe7, 0xfb, 0x18, 0x2d, 0x39, 0x8c, 0xc8, 0xe8,
-	0xde, 0x27, 0x00, 0x2d, 0x43, 0x91, 0x68, 0x0c, 0x87, 0x36, 0x03, 0x7c, 0x9e, 0x28, 0xb8, 0xdb,
-	0xb6, 0x79, 0xfc, 0x05, 0x13, 0xa6, 0x65, 0x28, 0x34, 0xd2, 0xdb, 0xfb, 0x11, 0xa4, 0xa9, 0x30,
-	0x67, 0x16, 0xde, 0x38, 0x7a, 0x26, 0x03, 0x5d, 0x3d, 0x19, 0xd9, 0x2b, 0xc3, 0x22, 0x65, 0xc0,
-	0xb2, 0x46, 0xb4, 0x15, 0x60, 0x71, 0x44, 0x47, 0x7c, 0x4c, 0x16, 0x08, 0x19, 0x1b, 0xdb, 0x2b,
-	0xc2, 0x82, 0xcd, 0xc6, 0x7c, 0xa5, 0xb5, 0xd1, 0xed, 0x10, 0x2e, 0xd6, 0x80, 0x8f, 0x49, 0x9a,
-	0x31, 0xb1, 0x86, 0x86, 0xa2, 0xd8, 0xff, 0x67, 0x1a, 0x14, 0x85, 0x65, 0x76, 0xa1, 0xa2, 0xb0,
-	0xb1, 0x62, 0xf2, 0x65, 0xbc, 0x65, 0x28, 0xa7, 0x29, 0x42, 0xf4, 0x83, 0xff, 0x0b, 0x00, 0x00,
-	0xff, 0xff, 0x2e, 0xde, 0xed, 0x1d, 0x14, 0x3d, 0x00, 0x00,
+	// 4327 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe4, 0x3c, 0x4d, 0x6c, 0x1c, 0x59,
+	0x5a, 0xae, 0xfe, 0xb3, 0xfd, 0xf9, 0x27, 0xed, 0x67, 0xc7, 0xe9, 0x54, 0x12, 0xc7, 0xae, 0x6c,
+	0x32, 0x9e, 0x4c, 0xd2, 0x99, 0xf1, 0xce, 0x8c, 0x76, 0x3c, 0x99, 0xdd, 0x74, 0xdb, 0x1d, 0xbb,
+	0x37, 0x76, 0xb7, 0xa7, 0xba, 0x9d, 0xd9, 0x04, 0x46, 0x35, 0xe5, 0xee, 0xb2, 0x53, 0x9a, 0xee,
+	0xaa, 0x9e, 0xaa, 0x6a, 0x13, 0x2f, 0x87, 0x85, 0x5d, 0x21, 0x76, 0xc5, 0x05, 0x81, 0x04, 0x23,
+	0x21, 0xc1, 0x0a, 0x38, 0x2e, 0x5a, 0x21, 0x84, 0x90, 0xe0, 0xc0, 0x19, 0x4e, 0x1c, 0x81, 0x0b,
+	0x07, 0x24, 0x90, 0x56, 0x20, 0x2d, 0x12, 0xe2, 0xc0, 0x09, 0xd5, 0x7b, 0xaf, 0xaa, 0x5f, 0x55,
+	0xbd, 0xaa, 0xee, 0x8e, 0x1d, 0x96, 0x15, 0xa7, 0xb8, 0xdf, 0xfb, 0xfe, 0xde, 0xcf, 0xf7, 0xff,
+	0x2a, 0xf0, 0xde, 0x89, 0xee, 0xbc, 0xe8, 0x1f, 0x15, 0x5b, 0x66, 0xf7, 0x41, 0xcb, 0x34, 0x1c,
+	0x55, 0x37, 0x34, 0xeb, 0xbe, 0xed, 0x98, 0x96, 0x7a, 0xa2, 0xdd, 0xd7, 0x0d, 0x47, 0xb3, 0x8e,
+	0xd5, 0x96, 0xf6, 0xc0, 0xee, 0x69, 0xad, 0x07, 0x2d, 0x5b, 0x2f, 0xf6, 0x2c, 0xd3, 0x31, 0x51,
+	0xce, 0xfd, 0xf3, 0xf4, 0x1d, 0x71, 0xf5, 0xc4, 0x34, 0x4f, 0x3a, 0xda, 0x03, 0x3c, 0x7a, 0xd4,
+	0x3f, 0x7e, 0xd0, 0xd6, 0xec, 0x96, 0xa5, 0xf7, 0x1c, 0xd3, 0x22, 0x90, 0xe2, 0xcd, 0x30, 0x84,
+	0xa3, 0x77, 0x35, 0xdb, 0x51, 0xbb, 0x3d, 0x0a, 0xb0, 0x12, 0x06, 0xf8, 0x25, 0x4b, 0xed, 0xf5,
+	0x34, 0xcb, 0x26, 0xf3, 0xd2, 0x32, 0x2c, 0xed, 0x68, 0xce, 0x41, 0xa7, 0x7f, 0xa2, 0x1b, 0x55,
+	0xe3, 0xd8, 0x94, 0xb5, 0x2f, 0xfa, 0x9a, 0xed, 0x48, 0x7f, 0x2f, 0xc0, 0xe5, 0xd0, 0x84, 0xdd,
+	0x33, 0x0d, 0x5b, 0x43, 0x08, 0x32, 0x86, 0xda, 0xd5, 0x0a, 0xc2, 0xaa, 0xb0, 0x3e, 0x2d, 0xe3,
+	0xbf, 0xd1, 0x6d, 0x98, 0x3f, 0xd5, 0x8c, 0xb6, 0x69, 0x29, 0xa7, 0x9a, 0x65, 0xeb, 0xa6, 0x51,
+	0x48, 0xe1, 0xd9, 0x39, 0x32, 0xfa, 0x94, 0x0c, 0xa2, 0x1d, 0x98, 0xea, 0xaa, 0x86, 0x7e, 0xac,
+	0xd9, 0x4e, 0x21, 0xbd, 0x9a, 0x5e, 0x9f, 0xd9, 0x78, 0xab, 0x48, 0x96, 0x5a, 0xe4, 0xf2, 0x2a,
+	0xee, 0x53, 0xe8, 0x8a, 0xe1, 0x58, 0x67, 0xb2, 0x8f, 0x2c, 0x7e, 0x08, 0x73, 0x81, 0x29, 0x94,
+	0x87, 0xf4, 0xe7, 0xda, 0x19, 0x95, 0xc9, 0xfd, 0x13, 0x2d, 0x41, 0xf6, 0x54, 0xed, 0xf4, 0x35,
+	0x2a, 0x09, 0xf9, 0xb1, 0x99, 0xfa, 0x9a, 0x20, 0xad, 0xc0, 0x75, 0x9f, 0xdb, 0x96, 0xda, 0x53,
+	0x8f, 0xf4, 0x8e, 0xee, 0xe8, 0x9a, 0xed, 0x2d, 0xfd, 0x53, 0xb8, 0x11, 0x33, 0x4f, 0x77, 0xe0,
+	0x21, 0xcc, 0xb6, 0x98, 0xf1, 0x82, 0x80, 0x97, 0x52, 0xf0, 0x96, 0x12, 0xc2, 0x3c, 0x93, 0x03,
+	0xd0, 0xd2, 0x7f, 0xa4, 0x21, 0x1f, 0x06, 0x41, 0x0f, 0x61, 0xd2, 0xd6, 0xac, 0x53, 0xbd, 0x45,
+	0xf6, 0x75, 0x66, 0x63, 0x35, 0x8e, 0x5a, 0xb1, 0x41, 0xe0, 0x76, 0x27, 0x64, 0x0f, 0x05, 0x1d,
+	0x42, 0xfe, 0xd4, 0xec, 0xf4, 0xbb, 0x9a, 0xa2, 0xbd, 0xec, 0xa9, 0x86, 0x7f, 0x00, 0x33, 0x1b,
+	0xeb, 0xb1, 0x64, 0x9e, 0x62, 0x84, 0x8a, 0x07, 0xbf, 0x3b, 0x21, 0x5f, 0x3a, 0x0d, 0x0e, 0x89,
+	0x7f, 0x29, 0xc0, 0x24, 0xe5, 0x86, 0x3e, 0x80, 0x8c, 0x73, 0xd6, 0x23, 0xd2, 0xcd, 0x6f, 0xdc,
+	0x1e, 0x26, 0x5d, 0xb1, 0x79, 0xd6, 0xd3, 0x64, 0x8c, 0x22, 0x39, 0x90, 0x71, 0x7f, 0xa1, 0x19,
+	0x98, 0x3c, 0xac, 0x3d, 0xa9, 0xd5, 0x3f, 0xa9, 0xe5, 0x27, 0xd0, 0x32, 0xa0, 0xad, 0x7a, 0xad,
+	0x29, 0xd7, 0xf7, 0xf6, 0x2a, 0xb2, 0xd2, 0xa8, 0xc8, 0x4f, 0xab, 0x5b, 0x95, 0xbc, 0x80, 0xbe,
+	0x02, 0xab, 0x4f, 0xeb, 0x7b, 0x87, 0xfb, 0x15, 0xa5, 0xb4, 0xb5, 0x55, 0x69, 0x34, 0xaa, 0xe5,
+	0xea, 0x5e, 0xb5, 0xf9, 0x4c, 0xd9, 0xaa, 0xd7, 0x1a, 0x4d, 0xb9, 0x54, 0xad, 0x35, 0x1b, 0xf9,
+	0x14, 0x5a, 0x83, 0xc2, 0x8e, 0x5c, 0x3f, 0x3c, 0x50, 0x38, 0x34, 0xd2, 0x62, 0xfa, 0x87, 0x65,
+	0x41, 0xfc, 0xae, 0x00, 0x97, 0x42, 0x6b, 0x44, 0xa5, 0xc0, 0x22, 0xee, 0x8f, 0xba, 0x37, 0xec,
+	0x62, 0xee, 0xf1, 0x16, 0x03, 0x90, 0xab, 0xd7, 0xf6, 0xaa, 0x35, 0x77, 0x01, 0x33, 0x30, 0x59,
+	0x7f, 0xfc, 0x18, 0xff, 0x48, 0x95, 0x73, 0x84, 0xa1, 0x34, 0x0f, 0xb3, 0x07, 0x96, 0x79, 0xa4,
+	0x79, 0x57, 0xac, 0x04, 0x73, 0xf4, 0x37, 0xbd, 0x52, 0x6f, 0x43, 0xd6, 0xd2, 0xd4, 0xf6, 0x19,
+	0x3d, 0x7d, 0xb1, 0x48, 0xd4, 0xb6, 0xe8, 0xa9, 0x6d, 0xb1, 0x6c, 0x9a, 0x9d, 0xa7, 0xee, 0x15,
+	0x96, 0x09, 0xa0, 0xf4, 0xfb, 0x39, 0x58, 0xdc, 0xb2, 0x34, 0xd5, 0xd1, 0x88, 0xb4, 0x94, 0x34,
+	0x57, 0x3d, 0x1f, 0xc2, 0xbc, 0x7b, 0x05, 0x5b, 0xba, 0x73, 0xa6, 0x58, 0xaa, 0x71, 0xa2, 0xd1,
+	0xdb, 0x71, 0xd9, 0xdb, 0x81, 0x2d, 0x3a, 0x2b, 0xbb, 0x93, 0xf2, 0x5c, 0x8b, 0xfd, 0x89, 0xaa,
+	0xb0, 0x48, 0x6f, 0x57, 0xe0, 0xd6, 0xa7, 0x83, 0xb7, 0x9e, 0x48, 0xc1, 0xdc, 0x7a, 0x74, 0x1a,
+	0x1c, 0xd1, 0x35, 0x1b, 0x3d, 0x01, 0xe8, 0xa9, 0x96, 0xda, 0xd5, 0x1c, 0xcd, 0xb2, 0x0b, 0x99,
+	0xa0, 0x09, 0xe0, 0xac, 0xa6, 0x78, 0xe0, 0x43, 0x13, 0x13, 0xc0, 0xa0, 0xa3, 0x1d, 0x57, 0x67,
+	0x5a, 0x96, 0xe6, 0xd8, 0x85, 0x2c, 0xa6, 0xb4, 0x9e, 0x44, 0xa9, 0x41, 0x40, 0x31, 0x99, 0x72,
+	0xfa, 0xcb, 0xb2, 0x20, 0x7b, 0xd8, 0xa8, 0x0e, 0x97, 0xbd, 0x05, 0x9a, 0x86, 0xa3, 0x19, 0x8e,
+	0x62, 0x9b, 0x7d, 0xab, 0xa5, 0x15, 0x72, 0x78, 0x97, 0xae, 0x85, 0x96, 0x48, 0x60, 0x1a, 0x18,
+	0x44, 0xa6, 0x5b, 0x13, 0x18, 0x44, 0xcf, 0x41, 0x54, 0x5b, 0x2d, 0xcd, 0xb6, 0x75, 0xb2, 0x17,
+	0x8a, 0xa5, 0x7d, 0xd1, 0xd7, 0x2d, 0xad, 0xab, 0x19, 0x8e, 0x5d, 0x98, 0x0c, 0x52, 0x6d, 0x9a,
+	0x3d, 0xb3, 0x63, 0x9e, 0x9c, 0xc9, 0x03, 0x18, 0xf9, 0x6a, 0x00, 0x9d, 0x99, 0xb1, 0xd1, 0x31,
+	0xa0, 0x6e, 0xdf, 0x51, 0x8f, 0x3a, 0x9a, 0xc2, 0x6c, 0xe5, 0x14, 0xde, 0x80, 0x8d, 0xa4, 0x0d,
+	0xd8, 0x27, 0x58, 0xa1, 0x1d, 0x2d, 0xbb, 0xca, 0x22, 0x2f, 0x74, 0xc3, 0x93, 0xe2, 0x47, 0x70,
+	0x29, 0x04, 0x3a, 0x8e, 0x91, 0x15, 0x37, 0x61, 0x96, 0xdd, 0xf1, 0xb1, 0x70, 0xb7, 0x61, 0x99,
+	0x2f, 0xec, 0x58, 0x66, 0xfe, 0x37, 0x52, 0xb0, 0xc8, 0x39, 0x31, 0xb4, 0x0b, 0x53, 0xb6, 0xa1,
+	0xf6, 0xec, 0x17, 0xa6, 0x43, 0xb5, 0xed, 0x6e, 0xc2, 0x01, 0x17, 0x1b, 0x14, 0x96, 0xfc, 0xdc,
+	0x9d, 0x90, 0x7d, 0x6c, 0x54, 0x86, 0x1c, 0x39, 0xfd, 0xb0, 0xb1, 0xe5, 0xd1, 0x21, 0x63, 0x3e,
+	0x15, 0x8a, 0x29, 0xbe, 0x03, 0xf3, 0x41, 0x0e, 0xe8, 0x26, 0xcc, 0x78, 0x1c, 0x14, 0xbd, 0x4d,
+	0xd7, 0x0a, 0xde, 0x50, 0xb5, 0x2d, 0xbe, 0x05, 0xb3, 0x2c, 0x31, 0x74, 0x0d, 0xa6, 0xe9, 0xf5,
+	0xf5, 0xc1, 0xa7, 0xc8, 0x40, 0xb5, 0xed, 0x5b, 0xa0, 0xaf, 0xc3, 0x52, 0xf0, 0x52, 0x50, 0xc3,
+	0x73, 0xc7, 0x5f, 0x03, 0xd9, 0x8b, 0xf9, 0xe0, 0x1a, 0x3c, 0x39, 0xa5, 0x3f, 0xc8, 0x42, 0x3e,
+	0xac, 0xe2, 0xe8, 0x21, 0x64, 0x8f, 0x3a, 0x66, 0xeb, 0x73, 0x8a, 0xfb, 0x95, 0x38, 0x5b, 0x50,
+	0x2c, 0xbb, 0x50, 0x64, 0x74, 0x77, 0x42, 0x26, 0x48, 0x2e, 0x76, 0xd7, 0xec, 0x1b, 0x0e, 0xdd,
+	0xbd, 0x78, 0xec, 0x7d, 0x17, 0x6a, 0x80, 0x8d, 0x91, 0xd0, 0x36, 0xcc, 0x10, 0x25, 0x51, 0xba,
+	0x66, 0x5b, 0x2b, 0xa4, 0x31, 0x8d, 0x5b, 0xb1, 0x34, 0x4a, 0x18, 0x76, 0xdf, 0x6c, 0x6b, 0x32,
+	0xa8, 0xfe, 0xdf, 0xe2, 0x1c, 0xcc, 0x30, 0xb2, 0x89, 0x7d, 0x98, 0x61, 0x98, 0xa1, 0x2b, 0x30,
+	0x79, 0x6c, 0x2b, 0xbe, 0xcb, 0x98, 0x96, 0x73, 0xc7, 0x36, 0xb6, 0xfe, 0x37, 0x61, 0x06, 0x4b,
+	0xa1, 0x1c, 0x77, 0xd4, 0x13, 0xbb, 0x90, 0x5a, 0x4d, 0xbb, 0x67, 0x84, 0x87, 0x1e, 0xbb, 0x23,
+	0xe8, 0x1e, 0x50, 0xf3, 0xa7, 0x10, 0xb8, 0x13, 0xcb, 0xec, 0xf7, 0xb0, 0x90, 0xd3, 0x32, 0xf5,
+	0xd5, 0x98, 0xd1, 0x8e, 0x3b, 0x2e, 0xfe, 0x59, 0x0a, 0x60, 0x20, 0x20, 0x7a, 0x08, 0x19, 0xbc,
+	0x26, 0xe2, 0xa6, 0xd6, 0x47, 0x58, 0x53, 0x11, 0x2f, 0x0c, 0x63, 0x49, 0xff, 0x22, 0x40, 0x06,
+	0x93, 0x09, 0xfb, 0xdb, 0x46, 0xb5, 0xb6, 0xb3, 0x57, 0x51, 0x6a, 0xf5, 0xed, 0x8a, 0xf2, 0x89,
+	0x5c, 0x6d, 0x56, 0xe4, 0xbc, 0x80, 0xae, 0xc1, 0x15, 0x76, 0x5c, 0xae, 0x94, 0xb6, 0x2b, 0xb2,
+	0x52, 0xaf, 0xed, 0x3d, 0xcb, 0xa7, 0x90, 0x08, 0xcb, 0xfb, 0x87, 0x7b, 0xcd, 0x6a, 0x74, 0x2e,
+	0x8d, 0xae, 0x43, 0x81, 0x99, 0xa3, 0x34, 0x28, 0xd9, 0x8c, 0x4b, 0x96, 0x99, 0x25, 0x7f, 0xd2,
+	0xc9, 0x2c, 0x92, 0xe0, 0x2a, 0xcb, 0x33, 0x88, 0x9b, 0xc3, 0xee, 0xdb, 0xf5, 0xf0, 0x2c, 0x4c,
+	0x80, 0xc2, 0x24, 0x06, 0x29, 0xcf, 0xf9, 0x37, 0x00, 0xdf, 0xf0, 0x4f, 0x60, 0x2e, 0xe0, 0xc6,
+	0xdc, 0xa0, 0x94, 0xda, 0xdd, 0xb6, 0x72, 0x74, 0xe6, 0xe0, 0x40, 0x4d, 0x58, 0x4f, 0xcb, 0x73,
+	0xde, 0x68, 0xd9, 0x1d, 0x74, 0xcf, 0xb2, 0xa3, 0x77, 0x75, 0x87, 0xc2, 0xa4, 0x30, 0x0c, 0xe0,
+	0x21, 0x0c, 0x20, 0xfd, 0x53, 0x0a, 0x72, 0xf4, 0x42, 0xdc, 0x66, 0x1c, 0x69, 0x80, 0xa4, 0x37,
+	0x4a, 0x48, 0x06, 0x34, 0x32, 0x15, 0xd4, 0x48, 0xb4, 0x0b, 0xf3, 0xac, 0xb7, 0x79, 0xe9, 0x85,
+	0xc2, 0x6b, 0xc1, 0x73, 0x66, 0x8d, 0xc8, 0x4b, 0x1a, 0x00, 0xcf, 0x9d, 0xb2, 0x63, 0xa8, 0x0c,
+	0xf3, 0x21, 0x87, 0x95, 0x19, 0xee, 0xb0, 0xe6, 0x5a, 0x01, 0x6b, 0x58, 0x82, 0x45, 0xcf, 0xd7,
+	0x74, 0x34, 0xc5, 0xa1, 0xbe, 0x88, 0x3a, 0xd4, 0x7c, 0xc4, 0x47, 0xa1, 0x01, 0xb0, 0x37, 0x26,
+	0x3e, 0x02, 0x14, 0x95, 0x75, 0x2c, 0x53, 0xdd, 0x87, 0x45, 0x8e, 0x17, 0x44, 0x45, 0x98, 0xc6,
+	0x47, 0x65, 0xeb, 0x8e, 0x46, 0x83, 0xec, 0xa8, 0x44, 0x03, 0x10, 0x17, 0xbe, 0x67, 0x69, 0xc7,
+	0x9a, 0x65, 0x69, 0x6d, 0xac, 0x93, 0x5c, 0x78, 0x1f, 0x44, 0xfa, 0x9e, 0x00, 0x53, 0xde, 0x38,
+	0xda, 0x84, 0x29, 0x5b, 0x3b, 0x21, 0x1e, 0x9a, 0xf0, 0x5a, 0x09, 0xe3, 0x16, 0x1b, 0x14, 0x80,
+	0xa6, 0x23, 0x1e, 0xbc, 0x9b, 0x8e, 0x04, 0xa6, 0xc6, 0x5a, 0xfc, 0x5f, 0x08, 0xb0, 0xb8, 0xad,
+	0x75, 0xb4, 0x70, 0x20, 0x97, 0x64, 0xd6, 0xd9, 0xd8, 0x27, 0x15, 0x8c, 0x7d, 0x38, 0xa4, 0x12,
+	0x62, 0x9f, 0xf3, 0xf8, 0x69, 0x37, 0x77, 0x0c, 0x72, 0x23, 0x3e, 0x45, 0xfa, 0xf7, 0x34, 0xac,
+	0xb8, 0x77, 0xc1, 0x32, 0x3b, 0x1d, 0xcd, 0x3a, 0xe8, 0x1f, 0x75, 0x74, 0xfb, 0xc5, 0x18, 0x8b,
+	0xbb, 0x02, 0x93, 0x86, 0xd9, 0x66, 0x94, 0x27, 0xe7, 0xfe, 0xac, 0xb6, 0x51, 0x05, 0x16, 0xc2,
+	0x91, 0xe8, 0x19, 0xb5, 0xfc, 0xf1, 0x71, 0x68, 0xfe, 0x34, 0xec, 0xb6, 0x44, 0x98, 0x72, 0x63,
+	0x68, 0xd3, 0xe8, 0x9c, 0x61, 0x8d, 0x99, 0x92, 0xfd, 0xdf, 0x48, 0x0e, 0x07, 0x95, 0x5f, 0xf5,
+	0x63, 0xaa, 0xc4, 0x15, 0x25, 0xc5, 0x97, 0x9f, 0x45, 0x34, 0x3e, 0x87, 0x49, 0x7f, 0x30, 0x22,
+	0xe9, 0xa1, 0x96, 0xe0, 0x5c, 0xd1, 0xd6, 0xf9, 0xd5, 0xf7, 0x6f, 0x05, 0xb8, 0x19, 0xbb, 0x04,
+	0x1a, 0x67, 0xb4, 0xe1, 0x52, 0x8f, 0x4c, 0xf8, 0x9b, 0x40, 0xb4, 0xec, 0xc3, 0xa1, 0x9b, 0x40,
+	0x6b, 0x01, 0x74, 0x34, 0xb0, 0x0d, 0xf3, 0xbd, 0xc0, 0xa0, 0x58, 0x82, 0x45, 0x0e, 0xd8, 0x58,
+	0x8b, 0xf9, 0x89, 0x00, 0xab, 0x03, 0x51, 0x0e, 0x8d, 0xde, 0xc5, 0x5d, 0xdf, 0xe6, 0xe0, 0x6e,
+	0x11, 0x93, 0xff, 0x5e, 0x74, 0xed, 0x7c, 0x86, 0xaf, 0x4b, 0x83, 0x6f, 0xc1, 0x5a, 0x02, 0x6b,
+	0xaa, 0xce, 0xbf, 0x97, 0x83, 0xb5, 0xa7, 0x6a, 0x47, 0x6f, 0xfb, 0xd1, 0x23, 0xa7, 0x6a, 0x92,
+	0xbc, 0x25, 0xad, 0x88, 0x06, 0x10, 0xab, 0xf5, 0xd0, 0xd7, 0xda, 0x61, 0xf4, 0x47, 0x70, 0x87,
+	0x17, 0x98, 0xa7, 0x3e, 0xe3, 0xe4, 0xa9, 0x1f, 0x8c, 0x2e, 0x6b, 0x52, 0xd6, 0x7a, 0x18, 0x36,
+	0x30, 0xef, 0x8f, 0x4e, 0x37, 0xc1, 0xc6, 0x38, 0xdc, 0xb4, 0x90, 0xd8, 0x99, 0x47, 0xa3, 0x73,
+	0x18, 0x33, 0x49, 0x3c, 0xb7, 0xed, 0xf8, 0xf9, 0x4f, 0x33, 0x7f, 0x90, 0x03, 0x29, 0x69, 0x5f,
+	0xa9, 0xfd, 0x93, 0x61, 0xba, 0x65, 0x1a, 0xc7, 0xba, 0xd5, 0xd5, 0xda, 0x34, 0x5d, 0x7a, 0x77,
+	0x94, 0x63, 0xa1, 0xc6, 0x6f, 0xcb, 0xc3, 0x95, 0x07, 0x64, 0x50, 0x01, 0x26, 0xbb, 0x9a, 0x6d,
+	0xab, 0x27, 0x9e, 0x58, 0xde, 0x4f, 0xf1, 0x3f, 0x33, 0x30, 0xed, 0xa3, 0x20, 0x23, 0xa2, 0x7d,
+	0xc4, 0xf4, 0xee, 0xbc, 0x8a, 0x00, 0xaf, 0xae, 0x88, 0xa9, 0x57, 0x50, 0xc4, 0x76, 0x40, 0x11,
+	0x89, 0x2a, 0x6f, 0xbf, 0x92, 0xd8, 0x49, 0x3a, 0xf9, 0x1d, 0xae, 0xf2, 0x10, 0xb5, 0x7f, 0xf2,
+	0x4a, 0xdc, 0x7e, 0xde, 0xf4, 0xe8, 0x62, 0x74, 0xe1, 0x17, 0x01, 0xed, 0xe9, 0x36, 0xcd, 0x9e,
+	0x7d, 0xcf, 0xe0, 0x26, 0xcb, 0xea, 0x4b, 0x45, 0x33, 0x1c, 0x4b, 0xa7, 0x19, 0x53, 0x56, 0x86,
+	0xae, 0xfa, 0xb2, 0x42, 0x46, 0xdc, 0xac, 0xca, 0x76, 0x54, 0xcb, 0xd1, 0x8d, 0x13, 0xc5, 0x31,
+	0x3f, 0xd7, 0xfc, 0xee, 0x81, 0x37, 0xda, 0x74, 0x07, 0xa5, 0x7f, 0x4b, 0xc1, 0x62, 0x80, 0x3c,
+	0x55, 0xad, 0x0f, 0x61, 0x72, 0x40, 0x3b, 0x90, 0x49, 0x71, 0xa0, 0x8b, 0xe4, 0xf4, 0x3d, 0x0c,
+	0x74, 0x03, 0xc0, 0xd0, 0x5e, 0x3a, 0x01, 0xbe, 0xd3, 0xee, 0x08, 0xe6, 0x29, 0xfe, 0x9a, 0xe0,
+	0x17, 0x5b, 0x1c, 0xd5, 0xe9, 0xe3, 0xc4, 0x9e, 0x7a, 0x49, 0xad, 0xad, 0x50, 0x37, 0x4f, 0xf8,
+	0x4e, 0xcb, 0x79, 0x7f, 0xa6, 0x86, 0x1d, 0xbe, 0x8d, 0x76, 0xfc, 0xc2, 0x7c, 0xcb, 0x34, 0xda,
+	0xba, 0x33, 0x28, 0xcc, 0x5f, 0x89, 0xe4, 0x68, 0x64, 0x9a, 0x5c, 0x91, 0x4b, 0xa7, 0xc1, 0x51,
+	0xf1, 0x0b, 0xc8, 0x92, 0xe3, 0x18, 0xb1, 0x5e, 0x83, 0x1e, 0x41, 0xce, 0xc6, 0x12, 0x87, 0x6b,
+	0x53, 0xbc, 0x3d, 0x61, 0x57, 0x28, 0x53, 0x3c, 0xe9, 0xeb, 0x20, 0x0e, 0x62, 0x83, 0x1d, 0xcd,
+	0x19, 0x3d, 0x02, 0xda, 0x74, 0xd7, 0x20, 0xfd, 0x6e, 0x0a, 0xae, 0x71, 0x09, 0x8c, 0x57, 0x79,
+	0x42, 0xbb, 0xa1, 0x95, 0xbc, 0x1d, 0x0d, 0x9a, 0x22, 0xc4, 0xb9, 0x2b, 0x12, 0x7f, 0xe5, 0x7c,
+	0x87, 0x59, 0x1e, 0xfb, 0x30, 0x23, 0xe7, 0x48, 0x76, 0xe6, 0x7b, 0x69, 0xb8, 0x31, 0x10, 0x7e,
+	0xdf, 0x6c, 0xeb, 0xc7, 0x67, 0x63, 0xc4, 0x97, 0x1f, 0x87, 0x73, 0xbf, 0x8d, 0xe8, 0x8e, 0x70,
+	0x88, 0x26, 0x45, 0x0f, 0x9f, 0x73, 0x0d, 0x60, 0x3a, 0x18, 0xa3, 0x25, 0x53, 0xe7, 0x9b, 0x0f,
+	0x9e, 0xb1, 0xfb, 0x99, 0xfb, 0x6c, 0x72, 0x0a, 0xb7, 0xd9, 0x24, 0x35, 0xb8, 0x22, 0x72, 0x89,
+	0x08, 0xd8, 0x9f, 0xa4, 0x00, 0xed, 0x68, 0x8e, 0x5f, 0x5a, 0xa2, 0x27, 0x14, 0xe3, 0xe3, 0x84,
+	0x57, 0xf0, 0x71, 0xdf, 0x0c, 0xf8, 0x38, 0x72, 0xa4, 0x77, 0x99, 0xbe, 0x68, 0x88, 0x75, 0xa2,
+	0x27, 0x8b, 0x29, 0xe7, 0x90, 0x1c, 0x79, 0xb4, 0x72, 0xce, 0xf9, 0x3c, 0x89, 0xf4, 0xcf, 0x02,
+	0x2c, 0x06, 0x84, 0xa6, 0xea, 0x7e, 0x1f, 0x90, 0x7a, 0xaa, 0xea, 0x1d, 0x7c, 0xc9, 0xbc, 0x72,
+	0x19, 0x2d, 0x9f, 0x2d, 0xf8, 0x33, 0x1e, 0x1a, 0x7a, 0x02, 0x8b, 0x5d, 0xf5, 0xa5, 0xde, 0xed,
+	0x77, 0x15, 0xba, 0xcf, 0xb6, 0xfe, 0x6d, 0xaf, 0xd0, 0x7e, 0x2d, 0xd2, 0x1e, 0xab, 0x1a, 0xce,
+	0xfb, 0xef, 0x92, 0xfe, 0xd8, 0x02, 0xc5, 0xa3, 0x9a, 0xae, 0x7f, 0x5b, 0x43, 0x07, 0xb0, 0xd8,
+	0xd5, 0x8d, 0x08, 0xb1, 0xf4, 0x50, 0x62, 0x9e, 0xc3, 0x26, 0xc8, 0x03, 0x8a, 0x92, 0xc4, 0x26,
+	0x89, 0x74, 0xb9, 0xe1, 0x3e, 0x72, 0x87, 0x4d, 0xae, 0x22, 0x30, 0x74, 0x5b, 0x76, 0xb8, 0xbd,
+	0xe4, 0x5b, 0x51, 0x9d, 0xa3, 0x8d, 0xd5, 0xd8, 0xb6, 0xf2, 0xef, 0x64, 0x58, 0x73, 0x1b, 0x81,
+	0x46, 0x1f, 0x42, 0xda, 0xea, 0xb5, 0xa8, 0xad, 0x7d, 0x63, 0x04, 0xfa, 0x45, 0xf9, 0x60, 0x6b,
+	0x77, 0x42, 0x76, 0xb1, 0xc4, 0x3f, 0x4d, 0x43, 0x5a, 0x3e, 0xd8, 0x42, 0x8f, 0x02, 0x0d, 0xd4,
+	0x7b, 0x23, 0x52, 0x61, 0xfb, 0xa7, 0xff, 0x9a, 0xe2, 0x35, 0x50, 0x0b, 0xb0, 0xb4, 0x25, 0x57,
+	0x4a, 0xcd, 0x8a, 0xb2, 0x5d, 0xd9, 0xab, 0x34, 0x2b, 0x0a, 0xe9, 0x01, 0xe7, 0x05, 0x74, 0x1d,
+	0x0a, 0x07, 0x87, 0xe5, 0xbd, 0x6a, 0x63, 0x57, 0x39, 0xac, 0x79, 0x7f, 0xd1, 0xd9, 0x14, 0xca,
+	0xc3, 0xec, 0x5e, 0xb5, 0xd1, 0xa4, 0x03, 0x8d, 0x7c, 0xda, 0x1d, 0xd9, 0xa9, 0x34, 0x95, 0xad,
+	0xd2, 0x41, 0x69, 0xab, 0xda, 0x7c, 0x96, 0xcf, 0x20, 0x11, 0x96, 0x83, 0xb4, 0x1b, 0xb5, 0xd2,
+	0x41, 0x63, 0xb7, 0xde, 0xcc, 0x67, 0x11, 0x82, 0x79, 0x8c, 0xef, 0x0d, 0x35, 0xf2, 0x39, 0x97,
+	0xc2, 0xd6, 0x5e, 0xbd, 0xe6, 0xcb, 0x30, 0x89, 0x96, 0x20, 0xef, 0x71, 0x96, 0x2b, 0xa5, 0x6d,
+	0x5c, 0x00, 0x9f, 0x42, 0x0b, 0x30, 0x57, 0xf9, 0xd6, 0x41, 0xa9, 0xb6, 0xed, 0x01, 0x4e, 0xa3,
+	0x55, 0xb8, 0xce, 0x8a, 0xa3, 0x50, 0xac, 0xca, 0x36, 0x2e, 0x62, 0x37, 0xf2, 0x80, 0xae, 0x42,
+	0x9e, 0xb6, 0xb7, 0xb7, 0xea, 0xb5, 0xed, 0x6a, 0xb3, 0x5a, 0xaf, 0xe5, 0x67, 0x48, 0xc5, 0x7b,
+	0x11, 0xc0, 0x95, 0x9c, 0x12, 0x9b, 0x1d, 0x5e, 0x06, 0x9f, 0x23, 0x20, 0xcb, 0x30, 0xb7, 0x5f,
+	0xdf, 0xae, 0x3e, 0x7e, 0xe6, 0xa1, 0xce, 0x93, 0xf2, 0xb8, 0xd7, 0xf9, 0xf9, 0x49, 0x0a, 0x2e,
+	0x93, 0xd6, 0x8f, 0xd7, 0x68, 0xf2, 0x6c, 0xd8, 0x3a, 0xe4, 0x49, 0xdd, 0x58, 0x09, 0x3b, 0x9b,
+	0x79, 0x32, 0xfe, 0xd4, 0x73, 0x39, 0x5e, 0x53, 0x39, 0xc5, 0x34, 0x95, 0xab, 0xe1, 0x6a, 0xc6,
+	0xdd, 0x60, 0xf7, 0x31, 0xc4, 0x2d, 0xc9, 0xfd, 0xec, 0x73, 0xd2, 0xed, 0xfb, 0xc9, 0xd4, 0x12,
+	0x8c, 0xe0, 0xb9, 0x1c, 0xcc, 0x39, 0xad, 0xdf, 0x63, 0x58, 0x0e, 0xcb, 0x4b, 0x15, 0xfd, 0x5e,
+	0xa4, 0xed, 0xe8, 0x9b, 0x63, 0x1f, 0xd6, 0x87, 0x90, 0xbe, 0x9f, 0x82, 0x29, 0x6f, 0xd8, 0x8d,
+	0x51, 0x5d, 0x7b, 0x15, 0xe8, 0x38, 0x4c, 0xbb, 0x23, 0x7e, 0x03, 0x83, 0x6d, 0x18, 0xa6, 0xc2,
+	0x0d, 0x43, 0xee, 0x39, 0xa7, 0xb9, 0xe7, 0xfc, 0x0d, 0x98, 0x6b, 0xb9, 0xe2, 0xeb, 0xa6, 0xa1,
+	0x38, 0x7a, 0xd7, 0x6b, 0x28, 0x44, 0x9f, 0x23, 0x34, 0xbd, 0x67, 0x46, 0xf2, 0xac, 0x87, 0xe0,
+	0x0e, 0xa1, 0x55, 0x98, 0xc5, 0xcf, 0x13, 0x14, 0xc7, 0x54, 0xfa, 0xb6, 0x56, 0xc8, 0xe2, 0xf2,
+	0x2a, 0xe0, 0xb1, 0xa6, 0x79, 0x68, 0x6b, 0xe8, 0x01, 0x2c, 0xe0, 0x66, 0x98, 0xc2, 0xca, 0x9c,
+	0x73, 0xa5, 0xa1, 0xa1, 0x2f, 0x9e, 0x6d, 0xf8, 0xd2, 0x4b, 0x7f, 0x2d, 0xc0, 0x65, 0x52, 0x66,
+	0x0e, 0xdf, 0xdf, 0x61, 0x9d, 0x52, 0xf6, 0x8a, 0x86, 0xdc, 0x2a, 0x97, 0xe0, 0xeb, 0xaa, 0xb2,
+	0x15, 0x60, 0x39, 0xcc, 0x8f, 0x96, 0xd6, 0x7e, 0x9c, 0x82, 0x25, 0x37, 0x20, 0xf7, 0x26, 0x2e,
+	0x3a, 0x67, 0x1a, 0xe3, 0xe8, 0x43, 0x9b, 0x99, 0x89, 0x6c, 0xe6, 0x6e, 0xb8, 0x70, 0xf5, 0x26,
+	0x9b, 0x52, 0x84, 0x57, 0xf0, 0xba, 0xf6, 0xf2, 0x47, 0x02, 0x5c, 0x0e, 0xf1, 0xa3, 0x0a, 0xf6,
+	0x51, 0x38, 0x0d, 0xbc, 0x15, 0x23, 0xdf, 0x2b, 0x25, 0x82, 0xef, 0x79, 0x09, 0xd8, 0x78, 0x7a,
+	0xfc, 0x77, 0x29, 0x36, 0xd4, 0xc7, 0x2f, 0x8a, 0xda, 0x63, 0x84, 0xfa, 0xe7, 0x7b, 0xb8, 0xf3,
+	0x71, 0xd8, 0x42, 0x73, 0x12, 0x05, 0x8e, 0x48, 0x49, 0x96, 0x9a, 0xdb, 0x81, 0xc9, 0x8c, 0xdb,
+	0x81, 0x39, 0xd7, 0x0d, 0xf8, 0x0e, 0x1b, 0xb7, 0x07, 0xc5, 0xa7, 0x37, 0x61, 0xc4, 0x2e, 0xed,
+	0xfb, 0x70, 0x05, 0xe7, 0x7c, 0xfe, 0x9b, 0x39, 0xef, 0x99, 0x0e, 0xb1, 0xa1, 0x53, 0xf2, 0x65,
+	0x77, 0xda, 0x7f, 0x05, 0x46, 0x3b, 0x93, 0x6d, 0xe9, 0xa7, 0x19, 0x58, 0x76, 0x73, 0xc2, 0x86,
+	0xa3, 0x9e, 0x8c, 0xd3, 0xb3, 0xfb, 0x85, 0x68, 0x0b, 0x24, 0x94, 0xbf, 0xf1, 0xa9, 0x8e, 0xd2,
+	0xf9, 0x40, 0x45, 0x58, 0xb4, 0x1d, 0xf5, 0x04, 0x9b, 0x03, 0xd5, 0x3a, 0xd1, 0x1c, 0xa5, 0xa7,
+	0x3a, 0x2f, 0xa8, 0xae, 0x2f, 0xd0, 0xa9, 0x26, 0x9e, 0x39, 0x50, 0x9d, 0x17, 0x17, 0x74, 0x90,
+	0xe8, 0x9b, 0x61, 0xa3, 0xf0, 0xd6, 0x90, 0xb5, 0x24, 0xdc, 0xad, 0x6f, 0xc5, 0xb4, 0xc9, 0xde,
+	0x19, 0x42, 0x72, 0x78, 0x7b, 0xec, 0xfc, 0x6d, 0xa1, 0x9f, 0x71, 0x87, 0xed, 0x2a, 0x5c, 0x89,
+	0x2c, 0x9e, 0xba, 0x90, 0x13, 0x28, 0xb8, 0x53, 0x87, 0x86, 0x3d, 0xe6, 0x75, 0x8c, 0xb9, 0x31,
+	0xa9, 0x98, 0x1b, 0x23, 0x5d, 0x83, 0xab, 0x1c, 0x46, 0x54, 0x8a, 0x3f, 0xcf, 0x12, 0x31, 0xc6,
+	0x6f, 0xf6, 0x7e, 0x1a, 0xa7, 0x15, 0xef, 0xb2, 0xc7, 0xce, 0xed, 0x8b, 0xbe, 0x0e, 0xbd, 0xb8,
+	0x09, 0x33, 0x2c, 0x1c, 0x75, 0x83, 0xce, 0x10, 0xc5, 0xc9, 0x9e, 0xab, 0x07, 0x9d, 0x0b, 0xf5,
+	0xa0, 0xf7, 0x06, 0x4a, 0x35, 0x19, 0x8c, 0x85, 0x63, 0xb7, 0x22, 0x41, 0xad, 0x9e, 0x47, 0xd4,
+	0x6a, 0x2a, 0xd8, 0xd8, 0x8e, 0x25, 0xfa, 0xff, 0x40, 0xb1, 0xe8, 0xa5, 0xe6, 0x76, 0x9c, 0xa5,
+	0xe7, 0x20, 0x92, 0x1b, 0x3f, 0x7e, 0x0f, 0x38, 0x74, 0x8d, 0x52, 0xe1, 0x6b, 0x24, 0xdd, 0x80,
+	0x6b, 0x5c, 0xda, 0x94, 0xf5, 0x0f, 0x04, 0x22, 0x98, 0x5f, 0xd9, 0x6c, 0x38, 0xaa, 0x63, 0x8f,
+	0xca, 0x9a, 0x4e, 0xb2, 0xac, 0xc9, 0x10, 0xbe, 0xc1, 0x63, 0xaa, 0x84, 0xf4, 0x9b, 0x02, 0xd9,
+	0x87, 0xb0, 0x2c, 0xd4, 0xdb, 0xbe, 0x09, 0xd9, 0x3e, 0xee, 0x41, 0x91, 0xa8, 0x6b, 0x31, 0xa8,
+	0x04, 0x87, 0xee, 0x94, 0x4c, 0x20, 0x2e, 0xac, 0x1c, 0x2e, 0xfd, 0x58, 0x80, 0x19, 0x86, 0x3e,
+	0xba, 0x0e, 0xd3, 0x7e, 0x09, 0xc9, 0x4b, 0x90, 0xfc, 0x01, 0xf7, 0xf8, 0x1d, 0xd3, 0x51, 0x3b,
+	0xf4, 0x6d, 0x17, 0xf9, 0xe1, 0xe6, 0xb4, 0x7d, 0x5b, 0x23, 0xe1, 0x70, 0x5a, 0xc6, 0x7f, 0xa3,
+	0x7b, 0x90, 0xe9, 0x1b, 0xba, 0x83, 0xd5, 0x7e, 0x3e, 0xac, 0xcf, 0x98, 0x55, 0xf1, 0xd0, 0xd0,
+	0x1d, 0x19, 0x43, 0x49, 0x77, 0x21, 0xe3, 0xfe, 0x0a, 0x96, 0x32, 0xa6, 0x21, 0x5b, 0x7e, 0xd6,
+	0xac, 0x34, 0xf2, 0x02, 0x02, 0xc8, 0x55, 0x49, 0xe2, 0x9f, 0x92, 0xf6, 0xbc, 0xd7, 0xe8, 0xfe,
+	0x22, 0x5c, 0x13, 0xa0, 0x1e, 0x19, 0xa6, 0xd5, 0x55, 0x3b, 0x58, 0xe6, 0x29, 0xd9, 0xff, 0x1d,
+	0xdf, 0xda, 0x23, 0x45, 0xc9, 0xeb, 0xfe, 0x89, 0xf0, 0x0a, 0x4f, 0x9f, 0x91, 0xbb, 0x15, 0x57,
+	0x72, 0x2a, 0x71, 0x4b, 0x4e, 0x37, 0x02, 0x5e, 0x76, 0x48, 0xb1, 0xe9, 0x6f, 0x52, 0x70, 0x99,
+	0x0b, 0x87, 0xde, 0x63, 0xcb, 0x4c, 0x6b, 0x89, 0x34, 0xd9, 0x02, 0xd3, 0x4f, 0x05, 0x52, 0x60,
+	0xda, 0x0c, 0x14, 0x98, 0xee, 0x0c, 0xc5, 0x67, 0x4b, 0x4b, 0x3f, 0x12, 0x62, 0x4a, 0x4b, 0x8d,
+	0x66, 0x69, 0xa7, 0xa2, 0x1c, 0xd6, 0xc8, 0xbf, 0x7e, 0x69, 0x69, 0x09, 0xf2, 0x83, 0x82, 0x8b,
+	0xd2, 0x68, 0x96, 0xf0, 0xa7, 0x05, 0x91, 0xb2, 0x4e, 0x9a, 0x5b, 0xb4, 0xc9, 0x0c, 0xaf, 0xcf,
+	0x64, 0xbd, 0xfa, 0x0c, 0xa2, 0xd8, 0xfb, 0xf5, 0xc3, 0x5a, 0x53, 0xc1, 0x1f, 0x2e, 0xe4, 0x73,
+	0x7e, 0x7d, 0x66, 0x09, 0x10, 0x3d, 0x2d, 0xf6, 0xfb, 0x9b, 0x3f, 0x14, 0x60, 0x31, 0x30, 0x4c,
+	0x0f, 0x8f, 0x79, 0x5c, 0x22, 0x04, 0x1e, 0x97, 0x3c, 0x80, 0x25, 0x37, 0x63, 0x24, 0x9a, 0x62,
+	0x2b, 0x3d, 0xcd, 0xc2, 0x1d, 0x0d, 0x7a, 0xe7, 0x17, 0xba, 0xea, 0x4b, 0xda, 0xf5, 0x39, 0xd0,
+	0x2c, 0x97, 0xf0, 0x05, 0x94, 0x8a, 0xa5, 0x2f, 0xd3, 0x24, 0x2e, 0x19, 0x3b, 0xaf, 0x19, 0x6a,
+	0xa3, 0xa2, 0x89, 0x4f, 0x7a, 0x8c, 0xc4, 0x27, 0xc6, 0xc2, 0x65, 0xc6, 0x0a, 0x86, 0xc7, 0xf7,
+	0xe9, 0xb5, 0x81, 0xdf, 0x26, 0x91, 0xeb, 0x3d, 0xf6, 0xfe, 0x0e, 0xcd, 0xb4, 0x72, 0x5f, 0x96,
+	0x85, 0x1f, 0x5e, 0x54, 0x9e, 0x5c, 0x22, 0xf1, 0xd8, 0x39, 0xf2, 0x23, 0xe9, 0x1e, 0xdc, 0xc6,
+	0xcf, 0x93, 0x87, 0x15, 0xba, 0x89, 0x49, 0xfa, 0x65, 0xb8, 0x33, 0x0c, 0x9a, 0xb2, 0xdf, 0xe3,
+	0xda, 0x1f, 0xbf, 0x41, 0x19, 0xa2, 0x32, 0xc4, 0x14, 0x11, 0xe6, 0xbf, 0x9e, 0x82, 0xd5, 0x61,
+	0x78, 0xe8, 0x11, 0x6b, 0x9a, 0xee, 0x8d, 0xca, 0x8e, 0xb5, 0x52, 0xbf, 0x4d, 0xad, 0x54, 0x25,
+	0x60, 0xa5, 0xde, 0x19, 0x87, 0x14, 0x6b, 0xb0, 0x2a, 0x3c, 0x7b, 0xf5, 0x36, 0xbc, 0x11, 0x2c,
+	0x57, 0x33, 0x36, 0x8a, 0x7c, 0xf3, 0xe4, 0xd7, 0xaf, 0x05, 0x6c, 0x60, 0x36, 0x03, 0xd5, 0xde,
+	0xdf, 0x4a, 0xc3, 0x2a, 0xfb, 0xd0, 0x7f, 0x87, 0xad, 0xa6, 0x25, 0x7d, 0x23, 0x74, 0x17, 0x16,
+	0xc2, 0x95, 0x22, 0xef, 0x61, 0xfb, 0xa5, 0x60, 0xa9, 0xc8, 0x4e, 0x7a, 0xc8, 0x36, 0x84, 0x75,
+	0x72, 0xfe, 0x17, 0xad, 0x02, 0x7f, 0x6d, 0x64, 0xc2, 0xff, 0x37, 0x0b, 0xc2, 0xe4, 0x7a, 0x76,
+	0x60, 0x2d, 0x41, 0x7e, 0xaa, 0x16, 0x65, 0x98, 0x0f, 0x16, 0x46, 0xe9, 0x4d, 0x0d, 0xbd, 0xe6,
+	0x0e, 0x22, 0xcf, 0x05, 0xaa, 0xa5, 0x84, 0xdb, 0x3f, 0x08, 0xde, 0x87, 0x2f, 0x01, 0x58, 0xf7,
+	0x84, 0xa3, 0x95, 0x57, 0xb2, 0x88, 0x70, 0xd1, 0x15, 0x15, 0x61, 0xda, 0x83, 0xb2, 0xc3, 0x4f,
+	0xa9, 0x7d, 0xe6, 0x03, 0x90, 0x68, 0xe1, 0x38, 0x7d, 0xce, 0xc2, 0x71, 0x26, 0x5c, 0x38, 0x26,
+	0x6b, 0xfb, 0x7e, 0x0a, 0x56, 0xd9, 0x37, 0xc7, 0xdc, 0xeb, 0x3d, 0xce, 0x42, 0xd7, 0x60, 0x96,
+	0x81, 0xf2, 0x6e, 0xfc, 0xcc, 0xa0, 0xee, 0x99, 0x74, 0xdb, 0x87, 0x49, 0xf2, 0x9a, 0x8a, 0xa0,
+	0x64, 0x2b, 0xd6, 0x61, 0x2d, 0x81, 0x3f, 0xdb, 0xc2, 0xfe, 0x6e, 0x0a, 0x7f, 0xd1, 0xfa, 0xbf,
+	0xb7, 0x63, 0xf1, 0x85, 0xc7, 0x44, 0x31, 0x5e, 0xeb, 0x76, 0xe9, 0xb0, 0x12, 0xc7, 0xfc, 0x82,
+	0x15, 0x70, 0xe3, 0xbf, 0x05, 0x98, 0xaa, 0xb6, 0x35, 0xc3, 0x21, 0x41, 0xc1, 0x5c, 0xe0, 0xdb,
+	0x66, 0x74, 0x3d, 0xe6, 0x93, 0x67, 0xbc, 0x05, 0xe2, 0x8d, 0xc4, 0x0f, 0xa2, 0xa5, 0x09, 0x74,
+	0xcc, 0x7c, 0x97, 0x1d, 0x78, 0x46, 0xf0, 0x95, 0x08, 0x26, 0xc7, 0x57, 0x8b, 0xb7, 0x87, 0x40,
+	0xf9, 0x7c, 0xde, 0x87, 0x2c, 0xfe, 0x44, 0x15, 0x2d, 0xf9, 0x9f, 0xc9, 0x32, 0x5f, 0xb0, 0x8a,
+	0x97, 0x43, 0xa3, 0x1e, 0xde, 0xc6, 0x5f, 0x01, 0xc0, 0xc0, 0x07, 0xa2, 0x27, 0x30, 0xcb, 0x9a,
+	0x3e, 0x74, 0x2d, 0xe1, 0x13, 0x45, 0xf1, 0x3a, 0x7f, 0xd2, 0x97, 0xe9, 0x09, 0xcc, 0xb2, 0x57,
+	0x7e, 0x40, 0x8c, 0xf3, 0xd1, 0xc3, 0x80, 0x18, 0xf7, 0x1b, 0x85, 0x09, 0xd4, 0x81, 0x2b, 0x31,
+	0x4f, 0xce, 0xd1, 0x9d, 0xd1, 0x1e, 0xe6, 0x8b, 0x6f, 0x8c, 0xf8, 0x76, 0x5d, 0x9a, 0x40, 0x16,
+	0x5c, 0x8d, 0x7d, 0x69, 0x8d, 0xd6, 0x47, 0x7d, 0x07, 0x2e, 0xbe, 0x39, 0x02, 0xa4, 0xcf, 0xb3,
+	0x0f, 0x62, 0xfc, 0xa3, 0x45, 0xf4, 0xe6, 0xc8, 0xaf, 0x82, 0xc5, 0xbb, 0xa3, 0xbf, 0x81, 0x94,
+	0x26, 0xd0, 0x2e, 0xcc, 0x30, 0x8f, 0xcc, 0x90, 0xc8, 0x7d, 0x79, 0x46, 0x08, 0x5f, 0x4b, 0x78,
+	0x95, 0x46, 0x28, 0x31, 0x4f, 0x49, 0x06, 0x94, 0xa2, 0x8f, 0x62, 0x06, 0x94, 0x38, 0x6f, 0x4f,
+	0xc2, 0xdb, 0x1f, 0x0a, 0x4c, 0x79, 0xdb, 0xcf, 0x8f, 0x74, 0x79, 0xdb, 0x1f, 0x13, 0xe5, 0x4a,
+	0x13, 0xe8, 0x63, 0x98, 0x0f, 0xf6, 0x82, 0xd1, 0x8d, 0xc4, 0x9e, 0xb6, 0xb8, 0x12, 0x37, 0xcd,
+	0x92, 0x0c, 0x76, 0x12, 0x07, 0x24, 0xb9, 0x1d, 0xcd, 0x01, 0xc9, 0x98, 0x06, 0xe4, 0x84, 0x6b,
+	0x9f, 0x02, 0xfd, 0xb1, 0x81, 0x7d, 0xe2, 0xb5, 0xf5, 0x06, 0xf6, 0x89, 0xdb, 0x54, 0x93, 0x26,
+	0x90, 0x0e, 0xcb, 0xfc, 0xf6, 0x0c, 0xba, 0x3d, 0x52, 0xf7, 0x49, 0xbc, 0x33, 0x0c, 0xcc, 0x67,
+	0xd5, 0x82, 0x45, 0xce, 0x1b, 0x40, 0x24, 0x25, 0x3e, 0x10, 0x24, 0x4c, 0x6e, 0x8d, 0xf0, 0x88,
+	0x50, 0xc2, 0xe9, 0x7b, 0x87, 0x5d, 0x0f, 0xfb, 0x4c, 0x8c, 0xb7, 0x1e, 0xce, 0xc3, 0x38, 0xde,
+	0x7a, 0x78, 0xaf, 0xcd, 0x30, 0xb7, 0x8d, 0xff, 0x4a, 0xc3, 0xa5, 0x50, 0x1a, 0x81, 0x7e, 0x55,
+	0x80, 0x95, 0xe4, 0xd4, 0x0a, 0xdd, 0x8f, 0x49, 0x41, 0x62, 0xae, 0x71, 0x71, 0x54, 0x70, 0xc6,
+	0x94, 0x5c, 0x8d, 0x8d, 0x60, 0xd1, 0xfa, 0xa8, 0x41, 0x3a, 0xa3, 0x3f, 0xc3, 0xc2, 0x61, 0xb2,
+	0xf9, 0x7d, 0xb8, 0x1a, 0x1b, 0xe3, 0xa0, 0xf5, 0x51, 0xc3, 0xb0, 0x01, 0xdb, 0xa1, 0x01, 0x93,
+	0x7f, 0xe6, 0xfc, 0x58, 0x01, 0xdd, 0x1e, 0x29, 0x90, 0x19, 0x9c, 0x79, 0x72, 0xc8, 0x81, 0xb9,
+	0xe1, 0x24, 0x6e, 0xe3, 0x1f, 0xb3, 0x90, 0xc1, 0x65, 0x99, 0x26, 0x5c, 0x0a, 0xb5, 0x7a, 0xd0,
+	0x4a, 0x72, 0x03, 0x4c, 0xbc, 0x19, 0x3b, 0xef, 0x9f, 0xdf, 0x73, 0x58, 0x88, 0x34, 0x6f, 0xd0,
+	0x2a, 0x8b, 0xc7, 0x6b, 0x20, 0x89, 0x6b, 0x09, 0x10, 0x61, 0xda, 0x41, 0x17, 0xba, 0x3a, 0xac,
+	0xbb, 0x10, 0xa4, 0x1d, 0xe7, 0x36, 0x3f, 0x23, 0x55, 0xb0, 0xb0, 0xc3, 0x94, 0x82, 0x72, 0x71,
+	0x5d, 0xe5, 0xad, 0x44, 0x18, 0x9f, 0xc3, 0xa7, 0x7e, 0xf9, 0x8d, 0x29, 0x6e, 0xa3, 0x80, 0x70,
+	0xdc, 0x22, 0xbc, 0x28, 0x25, 0x81, 0xf8, 0xe4, 0x3f, 0x81, 0x7c, 0xb8, 0x0e, 0x83, 0x6e, 0x0e,
+	0x29, 0x0b, 0x89, 0xab, 0xf1, 0x00, 0xe1, 0x9d, 0x09, 0x5b, 0x82, 0xb0, 0x54, 0x3c, 0xf5, 0xbf,
+	0x95, 0x08, 0xc3, 0x7a, 0x5f, 0xa6, 0x02, 0x39, 0xf0, 0xbe, 0xd1, 0x6a, 0xe5, 0xc0, 0xfb, 0x72,
+	0x4a, 0x96, 0xd2, 0xc4, 0xe6, 0x43, 0x00, 0xb5, 0xd3, 0x7b, 0xa1, 0x2a, 0x9a, 0xd1, 0xef, 0xa2,
+	0xeb, 0x91, 0xa4, 0xb0, 0x62, 0xf4, 0xbb, 0xf5, 0x9e, 0x9b, 0x0b, 0xda, 0x85, 0x3f, 0x9e, 0xc2,
+	0x99, 0xdf, 0x34, 0x46, 0x70, 0x27, 0x36, 0xf7, 0x20, 0x3f, 0xc0, 0x56, 0x70, 0x58, 0x8f, 0xd6,
+	0xb8, 0x34, 0xf0, 0x9b, 0xcd, 0x10, 0xa1, 0x79, 0x9f, 0x10, 0x9e, 0xdd, 0xfc, 0x08, 0xa0, 0x65,
+	0xeb, 0x0a, 0xc9, 0x2b, 0xd0, 0x8d, 0x08, 0x9d, 0xc7, 0xba, 0xd6, 0x69, 0x7b, 0x34, 0xfe, 0x88,
+	0x0a, 0xd3, 0xb2, 0x75, 0x92, 0x7d, 0x6c, 0x7e, 0x03, 0x66, 0x88, 0x30, 0xc7, 0x2e, 0xdc, 0x30,
+	0x7c, 0x2a, 0x03, 0x59, 0x3d, 0x9e, 0xd9, 0xac, 0xc0, 0x1c, 0x21, 0x40, 0x0b, 0xfa, 0xe8, 0x66,
+	0x84, 0xc4, 0x3e, 0x99, 0x09, 0x11, 0x99, 0xc5, 0x68, 0x74, 0x6e, 0xb3, 0x0c, 0xb3, 0x1e, 0x19,
+	0xe7, 0x85, 0xd9, 0x46, 0x2b, 0x1c, 0x2a, 0xee, 0x44, 0x88, 0xc8, 0x0c, 0x25, 0xe2, 0x4e, 0x0d,
+	0x44, 0xf1, 0xfe, 0x1f, 0xa1, 0xa8, 0x28, 0xb4, 0x86, 0xc5, 0x15, 0x85, 0xce, 0x95, 0xb3, 0xcf,
+	0xd3, 0x2d, 0x5b, 0x3f, 0xca, 0x61, 0xa4, 0xaf, 0xfe, 0x4f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x88,
+	0xc2, 0x11, 0x67, 0xf4, 0x4a, 0x00, 0x00,
 }
 
 // Reference imports to suppress errors if they are not otherwise used.
@@ -5470,6 +6365,7 @@ type ControllerClient interface {
 	ListSnapshots(ctx context.Context, in *ListSnapshotsRequest, opts ...grpc.CallOption) (*ListSnapshotsResponse, error)
 	ControllerExpandVolume(ctx context.Context, in *ControllerExpandVolumeRequest, opts ...grpc.CallOption) (*ControllerExpandVolumeResponse, error)
 	ControllerGetVolume(ctx context.Context, in *ControllerGetVolumeRequest, opts ...grpc.CallOption) (*ControllerGetVolumeResponse, error)
+	ControllerModifyVolume(ctx context.Context, in *ControllerModifyVolumeRequest, opts ...grpc.CallOption) (*ControllerModifyVolumeResponse, error)
 }
 
 type controllerClient struct {
@@ -5597,6 +6493,15 @@ func (c *controllerClient) ControllerGetVolume(ctx context.Context, in *Controll
 	return out, nil
 }
 
+func (c *controllerClient) ControllerModifyVolume(ctx context.Context, in *ControllerModifyVolumeRequest, opts ...grpc.CallOption) (*ControllerModifyVolumeResponse, error) {
+	out := new(ControllerModifyVolumeResponse)
+	err := c.cc.Invoke(ctx, "/csi.v1.Controller/ControllerModifyVolume", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 // ControllerServer is the server API for Controller service.
 type ControllerServer interface {
 	CreateVolume(context.Context, *CreateVolumeRequest) (*CreateVolumeResponse, error)
@@ -5612,6 +6517,7 @@ type ControllerServer interface {
 	ListSnapshots(context.Context, *ListSnapshotsRequest) (*ListSnapshotsResponse, error)
 	ControllerExpandVolume(context.Context, *ControllerExpandVolumeRequest) (*ControllerExpandVolumeResponse, error)
 	ControllerGetVolume(context.Context, *ControllerGetVolumeRequest) (*ControllerGetVolumeResponse, error)
+	ControllerModifyVolume(context.Context, *ControllerModifyVolumeRequest) (*ControllerModifyVolumeResponse, error)
 }
 
 // UnimplementedControllerServer can be embedded to have forward compatible implementations.
@@ -5657,6 +6563,9 @@ func (*UnimplementedControllerServer) ControllerExpandVolume(ctx context.Context
 func (*UnimplementedControllerServer) ControllerGetVolume(ctx context.Context, req *ControllerGetVolumeRequest) (*ControllerGetVolumeResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method ControllerGetVolume not implemented")
 }
+func (*UnimplementedControllerServer) ControllerModifyVolume(ctx context.Context, req *ControllerModifyVolumeRequest) (*ControllerModifyVolumeResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method ControllerModifyVolume not implemented")
+}
 
 func RegisterControllerServer(s *grpc.Server, srv ControllerServer) {
 	s.RegisterService(&_Controller_serviceDesc, srv)
@@ -5896,6 +6805,24 @@ func _Controller_ControllerGetVolume_Handler(srv interface{}, ctx context.Contex
 	return interceptor(ctx, in, info, handler)
 }
 
+func _Controller_ControllerModifyVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(ControllerModifyVolumeRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(ControllerServer).ControllerModifyVolume(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/csi.v1.Controller/ControllerModifyVolume",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(ControllerServer).ControllerModifyVolume(ctx, req.(*ControllerModifyVolumeRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
 var _Controller_serviceDesc = grpc.ServiceDesc{
 	ServiceName: "csi.v1.Controller",
 	HandlerType: (*ControllerServer)(nil),
@@ -5952,6 +6879,190 @@ var _Controller_serviceDesc = grpc.ServiceDesc{
 			MethodName: "ControllerGetVolume",
 			Handler:    _Controller_ControllerGetVolume_Handler,
 		},
+		{
+			MethodName: "ControllerModifyVolume",
+			Handler:    _Controller_ControllerModifyVolume_Handler,
+		},
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "github.com/container-storage-interface/spec/csi.proto",
+}
+
+// GroupControllerClient is the client API for GroupController service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
+type GroupControllerClient interface {
+	GroupControllerGetCapabilities(ctx context.Context, in *GroupControllerGetCapabilitiesRequest, opts ...grpc.CallOption) (*GroupControllerGetCapabilitiesResponse, error)
+	CreateVolumeGroupSnapshot(ctx context.Context, in *CreateVolumeGroupSnapshotRequest, opts ...grpc.CallOption) (*CreateVolumeGroupSnapshotResponse, error)
+	DeleteVolumeGroupSnapshot(ctx context.Context, in *DeleteVolumeGroupSnapshotRequest, opts ...grpc.CallOption) (*DeleteVolumeGroupSnapshotResponse, error)
+	GetVolumeGroupSnapshot(ctx context.Context, in *GetVolumeGroupSnapshotRequest, opts ...grpc.CallOption) (*GetVolumeGroupSnapshotResponse, error)
+}
+
+type groupControllerClient struct {
+	cc *grpc.ClientConn
+}
+
+func NewGroupControllerClient(cc *grpc.ClientConn) GroupControllerClient {
+	return &groupControllerClient{cc}
+}
+
+func (c *groupControllerClient) GroupControllerGetCapabilities(ctx context.Context, in *GroupControllerGetCapabilitiesRequest, opts ...grpc.CallOption) (*GroupControllerGetCapabilitiesResponse, error) {
+	out := new(GroupControllerGetCapabilitiesResponse)
+	err := c.cc.Invoke(ctx, "/csi.v1.GroupController/GroupControllerGetCapabilities", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *groupControllerClient) CreateVolumeGroupSnapshot(ctx context.Context, in *CreateVolumeGroupSnapshotRequest, opts ...grpc.CallOption) (*CreateVolumeGroupSnapshotResponse, error) {
+	out := new(CreateVolumeGroupSnapshotResponse)
+	err := c.cc.Invoke(ctx, "/csi.v1.GroupController/CreateVolumeGroupSnapshot", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *groupControllerClient) DeleteVolumeGroupSnapshot(ctx context.Context, in *DeleteVolumeGroupSnapshotRequest, opts ...grpc.CallOption) (*DeleteVolumeGroupSnapshotResponse, error) {
+	out := new(DeleteVolumeGroupSnapshotResponse)
+	err := c.cc.Invoke(ctx, "/csi.v1.GroupController/DeleteVolumeGroupSnapshot", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *groupControllerClient) GetVolumeGroupSnapshot(ctx context.Context, in *GetVolumeGroupSnapshotRequest, opts ...grpc.CallOption) (*GetVolumeGroupSnapshotResponse, error) {
+	out := new(GetVolumeGroupSnapshotResponse)
+	err := c.cc.Invoke(ctx, "/csi.v1.GroupController/GetVolumeGroupSnapshot", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// GroupControllerServer is the server API for GroupController service.
+type GroupControllerServer interface {
+	GroupControllerGetCapabilities(context.Context, *GroupControllerGetCapabilitiesRequest) (*GroupControllerGetCapabilitiesResponse, error)
+	CreateVolumeGroupSnapshot(context.Context, *CreateVolumeGroupSnapshotRequest) (*CreateVolumeGroupSnapshotResponse, error)
+	DeleteVolumeGroupSnapshot(context.Context, *DeleteVolumeGroupSnapshotRequest) (*DeleteVolumeGroupSnapshotResponse, error)
+	GetVolumeGroupSnapshot(context.Context, *GetVolumeGroupSnapshotRequest) (*GetVolumeGroupSnapshotResponse, error)
+}
+
+// UnimplementedGroupControllerServer can be embedded to have forward compatible implementations.
+type UnimplementedGroupControllerServer struct {
+}
+
+func (*UnimplementedGroupControllerServer) GroupControllerGetCapabilities(ctx context.Context, req *GroupControllerGetCapabilitiesRequest) (*GroupControllerGetCapabilitiesResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GroupControllerGetCapabilities not implemented")
+}
+func (*UnimplementedGroupControllerServer) CreateVolumeGroupSnapshot(ctx context.Context, req *CreateVolumeGroupSnapshotRequest) (*CreateVolumeGroupSnapshotResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method CreateVolumeGroupSnapshot not implemented")
+}
+func (*UnimplementedGroupControllerServer) DeleteVolumeGroupSnapshot(ctx context.Context, req *DeleteVolumeGroupSnapshotRequest) (*DeleteVolumeGroupSnapshotResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method DeleteVolumeGroupSnapshot not implemented")
+}
+func (*UnimplementedGroupControllerServer) GetVolumeGroupSnapshot(ctx context.Context, req *GetVolumeGroupSnapshotRequest) (*GetVolumeGroupSnapshotResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetVolumeGroupSnapshot not implemented")
+}
+
+func RegisterGroupControllerServer(s *grpc.Server, srv GroupControllerServer) {
+	s.RegisterService(&_GroupController_serviceDesc, srv)
+}
+
+func _GroupController_GroupControllerGetCapabilities_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GroupControllerGetCapabilitiesRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(GroupControllerServer).GroupControllerGetCapabilities(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/csi.v1.GroupController/GroupControllerGetCapabilities",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(GroupControllerServer).GroupControllerGetCapabilities(ctx, req.(*GroupControllerGetCapabilitiesRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _GroupController_CreateVolumeGroupSnapshot_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(CreateVolumeGroupSnapshotRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(GroupControllerServer).CreateVolumeGroupSnapshot(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/csi.v1.GroupController/CreateVolumeGroupSnapshot",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(GroupControllerServer).CreateVolumeGroupSnapshot(ctx, req.(*CreateVolumeGroupSnapshotRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _GroupController_DeleteVolumeGroupSnapshot_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(DeleteVolumeGroupSnapshotRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(GroupControllerServer).DeleteVolumeGroupSnapshot(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/csi.v1.GroupController/DeleteVolumeGroupSnapshot",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(GroupControllerServer).DeleteVolumeGroupSnapshot(ctx, req.(*DeleteVolumeGroupSnapshotRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _GroupController_GetVolumeGroupSnapshot_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetVolumeGroupSnapshotRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(GroupControllerServer).GetVolumeGroupSnapshot(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/csi.v1.GroupController/GetVolumeGroupSnapshot",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(GroupControllerServer).GetVolumeGroupSnapshot(ctx, req.(*GetVolumeGroupSnapshotRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+var _GroupController_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "csi.v1.GroupController",
+	HandlerType: (*GroupControllerServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "GroupControllerGetCapabilities",
+			Handler:    _GroupController_GroupControllerGetCapabilities_Handler,
+		},
+		{
+			MethodName: "CreateVolumeGroupSnapshot",
+			Handler:    _GroupController_CreateVolumeGroupSnapshot_Handler,
+		},
+		{
+			MethodName: "DeleteVolumeGroupSnapshot",
+			Handler:    _GroupController_DeleteVolumeGroupSnapshot_Handler,
+		},
+		{
+			MethodName: "GetVolumeGroupSnapshot",
+			Handler:    _GroupController_GetVolumeGroupSnapshot_Handler,
+		},
 	},
 	Streams:  []grpc.StreamDesc{},
 	Metadata: "github.com/container-storage-interface/spec/csi.proto",
diff --git a/vendor/github.com/kubernetes-csi/csi-test/v4/pkg/sanity/controller.go b/vendor/github.com/kubernetes-csi/csi-test/v4/pkg/sanity/controller.go
index e7174d6c0..4dfaeb943 100644
--- a/vendor/github.com/kubernetes-csi/csi-test/v4/pkg/sanity/controller.go
+++ b/vendor/github.com/kubernetes-csi/csi-test/v4/pkg/sanity/controller.go
@@ -148,7 +148,6 @@ var _ = DescribeSanity("Controller Service [Controller Server]", func(sc *TestCo
 				case csi.ControllerServiceCapability_RPC_GET_VOLUME:
 				case csi.ControllerServiceCapability_RPC_VOLUME_CONDITION:
 				case csi.ControllerServiceCapability_RPC_SINGLE_NODE_MULTI_WRITER:
-				case csi.ControllerServiceCapability_RPC_MODIFY_VOLUME:
 				default:
 					Fail(fmt.Sprintf("Unknown capability: %v\n", cap.GetRpc().GetType()))
 				}

From 8c180b71798cc65f8a6b0cdb6c50c4b288f30a4f Mon Sep 17 00:00:00 2001
From: Max Kimambo <max@kimambo.de>
Date: Tue, 9 Jul 2024 18:16:56 +0000
Subject: [PATCH 13/20] added vol-update example

---
 examples/kubernetes/demo-vol-update.yml | 99 +++++++++++++++++++++++++
 1 file changed, 99 insertions(+)
 create mode 100644 examples/kubernetes/demo-vol-update.yml

diff --git a/examples/kubernetes/demo-vol-update.yml b/examples/kubernetes/demo-vol-update.yml
new file mode 100644
index 000000000..921b527cc
--- /dev/null
+++ b/examples/kubernetes/demo-vol-update.yml
@@ -0,0 +1,99 @@
+apiVersion: storage.k8s.io/v1alpha1
+kind: VolumeAttributesClass
+metadata:
+  name: silver
+driverName: pd.csi.storage.gke.io
+parameters:
+  throughput: "350"
+  iops: "6000"
+---
+apiVersion: storage.k8s.io/v1alpha1
+kind: VolumeAttributesClass
+metadata:
+  name: gold
+driverName: pd.csi.storage.gke.io
+parameters:
+  throughput: "550"
+  iops: "15000"
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: balanced
+provisioner: pd.csi.storage.gke.io
+allowVolumeExpansion: true 
+volumeBindingMode: WaitForFirstConsumer
+parameters:
+  type: hyperdisk-balanced
+  provisioned-throughput-on-create: "300Mi"
+  provisioned-iops-on-create: "5000"
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: throughput-optimized
+provisioner: pd.csi.storage.gke.io
+volumeBindingMode: WaitForFirstConsumer
+allowVolumeExpansion: true
+parameters:
+  type: hyperdisk-balanced
+  provisioned-throughput-on-create: "500Mi"
+  provisioned-iops-on-create: "10000"
+
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: balanced-pvc
+spec:
+  volumeAttributesClassName: silver
+  storageClassName: balanced
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 256Gi
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: throughput-optimized-pvc
+spec:
+  volumeAttributesClassName: silver
+  storageClassName: throughput-optimized
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 256Gi
+---
+kind: Pod
+apiVersion: v1
+metadata:
+  name: pod-demo
+spec:
+  volumes:
+    - name: pvc-demo-vol
+      persistentVolumeClaim:
+       claimName: balanced-pvc
+    - name: data-vol
+      persistentVolumeClaim:
+        claimName: throughput-optimized-pvc
+  containers:
+    - name: pod-demo
+      image: nginx:latest
+      resources:
+        limits:
+          cpu: 10m
+          memory: 80Mi
+        requests:
+          cpu: 10m
+          memory: 80Mi
+      ports:
+        - containerPort: 80
+          name: "http-server"
+      volumeMounts:
+        - mountPath: "/usr/share/nginx/html"
+          name: pvc-demo-vol
+        - mountPath: "/data"
+          name: data-vol
\ No newline at end of file

From fc0f5dbf0b54262461fd68a9fa82e52f91f2917a Mon Sep 17 00:00:00 2001
From: Max Kimambo <max@kimambo.de>
Date: Wed, 10 Jul 2024 09:42:21 +0000
Subject: [PATCH 14/20] corrected the overlay

---
 .../noauth-debug/controller-overlay.yaml      | 34 +++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/deploy/kubernetes/overlays/noauth-debug/controller-overlay.yaml b/deploy/kubernetes/overlays/noauth-debug/controller-overlay.yaml
index adc68a679..9f9a5ddf4 100644
--- a/deploy/kubernetes/overlays/noauth-debug/controller-overlay.yaml
+++ b/deploy/kubernetes/overlays/noauth-debug/controller-overlay.yaml
@@ -8,6 +8,7 @@ metadata:
 spec:
   template:
     spec:
+      
       containers:
         - name: gce-pd-driver
           imagePullPolicy: Always
@@ -32,3 +33,36 @@ spec:
               add:
               - SYS_PTRACE
 
+        - name: csi-provisioner
+          image: gcr.io/k8s-staging-sig-storage/csi-provisioner:canary
+          args:
+            - "--v=5"
+            - "--csi-address=/csi/csi.sock"
+            - "--feature-gates=Topology=true"
+            - "--http-endpoint=:22011"
+            - "--leader-election-namespace=$(PDCSI_NAMESPACE)"
+            - "--timeout=250s"
+            - "--extra-create-metadata"
+          # - "--run-controller-service=false"  # disable the controller service of the CSI driver
+          # - "--run-node-service=false"        # disable the node service of the CSI driver
+            - "--leader-election"
+            - "--default-fstype=ext4"
+            - "--controller-publish-readonly=true"
+            - "--feature-gates=VolumeAttributesClass=true"
+
+        - name: csi-resizer
+          image: registry.k8s.io/sig-storage/csi-resizer
+          imagePullPolicy: Always
+          args:
+            - "--v=5"
+            - "--csi-address=/csi/csi.sock"
+            - "--http-endpoint=:22013"
+            - "--leader-election"
+            - "--leader-election-namespace=$(PDCSI_NAMESPACE)"
+            - "--handle-volume-inuse-error=false"
+            - "--feature-gates=VolumeAttributesClass=true"
+      
+      # used with vanilla K8s 
+      # imagePullSecrets:
+      #   - name: artifactory-cred
+

From 13b4ca43ac24417bfb093e612bd8cc8ff45d6dad Mon Sep 17 00:00:00 2001
From: Max Kimambo <max@kimambo.de>
Date: Wed, 10 Jul 2024 13:30:31 +0000
Subject: [PATCH 15/20] missing configuration

---
 .../base/controller/cluster_setup.yaml        | 25 +++++++++++++++++++
 .../images/stable-master/image.yaml           |  2 +-
 .../overlays/noauth-debug/kustomization.yaml  |  2 +-
 3 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/deploy/kubernetes/base/controller/cluster_setup.yaml b/deploy/kubernetes/base/controller/cluster_setup.yaml
index e22c46b55..b9258ca62 100644
--- a/deploy/kubernetes/base/controller/cluster_setup.yaml
+++ b/deploy/kubernetes/base/controller/cluster_setup.yaml
@@ -313,3 +313,28 @@ roleRef:
   kind: Role
   name: csi-gce-pd-leaderelection-role
   apiGroup: rbac.authorization.k8s.io
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: cluster-admin
+rules:
+- apiGroups: ["*"]
+  resources: ["*"]
+  verbs: ["*"]
+- nonResourceURLs: ["*"]
+  verbs: ["*"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: csi-gce-pd-controller-sa-cluster-admin
+subjects:
+- kind: ServiceAccount
+  name: csi-gce-pd-controller-sa   
+  namespace: gce-pd-csi-driver
+roleRef:
+  kind: ClusterRole
+  name: cluster-admin
+  apiGroup: rbac.authorization.k8s.io
\ No newline at end of file
diff --git a/deploy/kubernetes/images/stable-master/image.yaml b/deploy/kubernetes/images/stable-master/image.yaml
index 56a0ac6ae..8ed57736b 100644
--- a/deploy/kubernetes/images/stable-master/image.yaml
+++ b/deploy/kubernetes/images/stable-master/image.yaml
@@ -22,7 +22,7 @@ metadata:
   name: imagetag-csi-resizer
 imageTag:
   name: registry.k8s.io/sig-storage/csi-resizer
-  newTag: "v1.9.3"
+  newTag: "v1.11.1"
 ---
 
 apiVersion: builtin
diff --git a/deploy/kubernetes/overlays/noauth-debug/kustomization.yaml b/deploy/kubernetes/overlays/noauth-debug/kustomization.yaml
index 881acc6ca..02cfdc9c7 100644
--- a/deploy/kubernetes/overlays/noauth-debug/kustomization.yaml
+++ b/deploy/kubernetes/overlays/noauth-debug/kustomization.yaml
@@ -13,4 +13,4 @@ namespace: gce-pd-csi-driver
 images:
 - name: gke.gcr.io/gcp-compute-persistent-disk-csi-driver
   newName: europe-west3-docker.pkg.dev/kimambo-sandbox/csi-dev/gce-pd-csi-driver
-  newTag: dev_linux
+  newTag: dev_linux
\ No newline at end of file

From ad56d4c4127c1d8085ad2380a86482c81641de8d Mon Sep 17 00:00:00 2001
From: Max Kimambo <max@kimambo.de>
Date: Thu, 11 Jul 2024 13:14:04 +0000
Subject: [PATCH 16/20] add explicit check for supported disktypes

---
 creds/cloud-sa.json                 |  0
 pkg/gce-pd-csi-driver/controller.go | 44 ++++++++++++++++++-----------
 2 files changed, 28 insertions(+), 16 deletions(-)
 delete mode 100644 creds/cloud-sa.json

diff --git a/creds/cloud-sa.json b/creds/cloud-sa.json
deleted file mode 100644
index e69de29bb..000000000
diff --git a/pkg/gce-pd-csi-driver/controller.go b/pkg/gce-pd-csi-driver/controller.go
index d7a7aa263..2da9acb86 100644
--- a/pkg/gce-pd-csi-driver/controller.go
+++ b/pkg/gce-pd-csi-driver/controller.go
@@ -201,7 +201,17 @@ const (
 
 var (
 	validResourceApiVersions = map[string]bool{"v1": true, "alpha": true, "beta": true, "staging_v1": true, "staging_beta": true, "staging_alpha": true}
-
+	// disk types that support IOPS and Throughput parameters setting / modifications
+	diskSupportsIopsAndThroughput = map[string]bool{
+		"pd-balanced":          false,
+		"pd-standard":          false,
+		"pd-ssd":               false,
+		"pd-extreme":           true,
+		"hyperdisk-ml":         true,
+		"hyperdisk-balanced":   true,
+		"hyperdisk-throughput": true,
+		"hyperdisk-extreme":    true,
+	}
 	// By default GCE returns a lot of data for each instance. Request only a subset of the fields.
 	listInstancesFields = []googleapi.Field{
 		"items/disks/deviceName",
@@ -331,21 +341,18 @@ func (gceCS *GCEControllerServer) createVolumeInternal(ctx context.Context, req
 	// https://github.com/container-storage-interface/spec/blob/master/spec.md#createvolume
 	// mutable_parameters MUST take precedence over the values from parameters.
 	mutableParams := req.GetMutableParameters()
-
-	klog.V(4).Infof("CreateVolume parameters: %v", params)
-	klog.V(4).Infof("CreateVolume mutable parameters: %v", mutableParams)
-	klog.V(4).Infof("CreateVolume disk type: %v", diskTypeForMetric)
-	// If the disk type is pd-*, the IOPS and Throughput parameters are ignored.
-	// if !strings.HasPrefix(params.DiskType, "pd-") {
-	// 	p, err := common.ExtractModifyVolumeParameters(mutableParams)
-	// 	if err != nil {
-	// 		return nil, status.Errorf(codes.InvalidArgument, "Invalid mutable parameters: %v", err)
-	// 	}
-	// 	params.ProvisionedIOPSOnCreate = p.IOPS
-	// 	params.ProvisionedThroughputOnCreate = p.Throughput
-	// } else {
-	// 	klog.V(4).Infof("Ignoring IOPS and throughput parameters for unsupported disk type %s", params.DiskType)
-	// }
+	// If the disk type is pd-balanced, pd-ssd, pd-standard except for pd-extreme,
+	// the IOPS and Throughput parameters are ignored.
+	if diskSupportsIopsAndThroughput[params.DiskType] && len(mutableParams) > 0 {
+		p, err := common.ExtractModifyVolumeParameters(mutableParams)
+		if err != nil {
+			return nil, status.Errorf(codes.InvalidArgument, "Invalid mutable parameters: %v", err)
+		}
+		params.ProvisionedIOPSOnCreate = p.IOPS
+		params.ProvisionedThroughputOnCreate = p.Throughput
+	} else {
+		klog.V(4).Infof("Ignoring IOPS and throughput parameters for unsupported disk type %s", params.DiskType)
+	}
 
 	if _, err := getMultiWriterFromCapabilities(volumeCapabilities); err != nil {
 		return nil, status.Errorf(codes.InvalidArgument, "VolumeCapabilities is invalid: %v", err.Error())
@@ -776,6 +783,11 @@ func (gceCS *GCEControllerServer) ControllerModifyVolume(ctx context.Context, re
 		return nil, status.Errorf(codes.Internal, "failed to get volume : %s", volumeID)
 	}
 	diskType = existingDisk.GetPDType()
+
+	if !diskSupportsIopsAndThroughput[diskType] {
+		return nil, status.Errorf(codes.InvalidArgument, "ModifyVolume is not supported for disk type %s", diskType)
+	}
+
 	// TODO : not sure how this metric should be reported
 	if existingDisk.GetEnableStoragePools() {
 		enableStoragePools = "enabled"

From 282eaebfef476986b69e2b306b5468c4ee356ea6 Mon Sep 17 00:00:00 2001
From: Max Kimambo <max@kimambo.de>
Date: Thu, 11 Jul 2024 16:15:17 +0000
Subject: [PATCH 17/20] add: test for vac parameter override storage class
 params

---
 pkg/gce-cloud-provider/compute/fake-gce.go | 17 +++----
 pkg/gce-pd-csi-driver/controller_test.go   | 53 +++++++++++++++++++++-
 2 files changed, 61 insertions(+), 9 deletions(-)

diff --git a/pkg/gce-cloud-provider/compute/fake-gce.go b/pkg/gce-cloud-provider/compute/fake-gce.go
index d29226afa..ebf8818a6 100644
--- a/pkg/gce-cloud-provider/compute/fake-gce.go
+++ b/pkg/gce-cloud-provider/compute/fake-gce.go
@@ -228,14 +228,15 @@ func (cloud *FakeCloudProvider) InsertDisk(ctx context.Context, project string,
 	}
 
 	computeDisk := &computev1.Disk{
-		Name:            volKey.Name,
-		SizeGb:          common.BytesToGbRoundUp(capBytes),
-		Description:     "Disk created by GCE-PD CSI Driver",
-		Type:            cloud.GetDiskTypeURI(project, volKey, params.DiskType),
-		SourceDiskId:    volumeContentSourceVolumeID,
-		Status:          cloud.mockDiskStatus,
-		Labels:          params.Labels,
-		ProvisionedIops: params.ProvisionedIOPSOnCreate,
+		Name:                  volKey.Name,
+		SizeGb:                common.BytesToGbRoundUp(capBytes),
+		Description:           "Disk created by GCE-PD CSI Driver",
+		Type:                  cloud.GetDiskTypeURI(project, volKey, params.DiskType),
+		SourceDiskId:          volumeContentSourceVolumeID,
+		Status:                cloud.mockDiskStatus,
+		Labels:                params.Labels,
+		ProvisionedIops:       params.ProvisionedIOPSOnCreate,
+		ProvisionedThroughput: params.ProvisionedThroughputOnCreate,
 	}
 
 	if snapshotID != "" {
diff --git a/pkg/gce-pd-csi-driver/controller_test.go b/pkg/gce-pd-csi-driver/controller_test.go
index 3197c7c0d..b3f19208f 100644
--- a/pkg/gce-pd-csi-driver/controller_test.go
+++ b/pkg/gce-pd-csi-driver/controller_test.go
@@ -1711,8 +1711,59 @@ func TestMultiZoneVolumeCreationErrHandling(t *testing.T) {
 		}
 	}
 }
+func TestCreateVolumeWithVolumeAttributeClassParameters(t *testing.T) {
+	// When volume attribute class specifies iops / throughput they should take precedence over storage class parameters
+	var d []*gce.CloudDisk
+	fcp, err := gce.CreateFakeCloudProvider(project, zone, d)
+	gceDriver := initGCEDriverWithCloudProvider(t, fcp)
 
-func TestVolumeUpdateOperation(t *testing.T) {
+	if err != nil {
+		t.Fatalf("Failed to create fake cloud provider: %v", err)
+	}
+
+	createVolReq := &csi.CreateVolumeRequest{
+		Name:          "test-name",
+		CapacityRange: stdCapRange,
+		VolumeCapabilities: []*csi.VolumeCapability{
+			{
+				AccessType: &csi.VolumeCapability_Mount{
+					Mount: &csi.VolumeCapability_MountVolume{},
+				},
+				AccessMode: &csi.VolumeCapability_AccessMode{
+					Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER,
+				},
+			},
+		},
+		Parameters: map[string]string{
+			common.ParameterKeyType:                          "hyperdisk-balanced",
+			common.ParameterKeyProvisionedIOPSOnCreate:       "10000",
+			common.ParameterKeyProvisionedThroughputOnCreate: "500",
+		},
+		MutableParameters: map[string]string{"iops": "20000", "throughput": "600"},
+	}
+
+	_, err = gceDriver.cs.CreateVolume(context.Background(), createVolReq)
+	if err != nil {
+		t.Fatalf("Failed to create volume: %v", err)
+	}
+
+	volumeKey := meta.ZonalKey(name, zone)
+	disk, err := fcp.GetDisk(context.Background(), project, volumeKey, gce.GCEAPIVersionBeta)
+
+	if err != nil {
+		t.Fatalf("Failed to get disk: %v", err)
+	}
+	if disk != nil {
+		if disk.GetProvisionedIops() != 20000 {
+			t.Errorf("Expected IOPS to be 20000, got: %v", disk.GetProvisionedIops())
+		}
+		if disk.GetProvisionedThroughput() != 600 {
+			t.Errorf("Expected Throughput to be 600, got: %v", disk.GetProvisionedThroughput())
+		}
+	}
+
+}
+func TestVolumeModifyOperation(t *testing.T) {
 
 	testCases := []struct {
 		name          string

From 27eed2c4fffc002603d310d51ed86abb7c573e4d Mon Sep 17 00:00:00 2001
From: Max Kimambo <max@kimambo.de>
Date: Fri, 12 Jul 2024 13:49:28 +0000
Subject: [PATCH 18/20] add: unit tests

---
 pkg/gce-pd-csi-driver/controller.go      |   2 +-
 pkg/gce-pd-csi-driver/controller_test.go | 182 +++++++++++++++++------
 2 files changed, 137 insertions(+), 47 deletions(-)

diff --git a/pkg/gce-pd-csi-driver/controller.go b/pkg/gce-pd-csi-driver/controller.go
index 2da9acb86..2ec14d669 100644
--- a/pkg/gce-pd-csi-driver/controller.go
+++ b/pkg/gce-pd-csi-driver/controller.go
@@ -785,7 +785,7 @@ func (gceCS *GCEControllerServer) ControllerModifyVolume(ctx context.Context, re
 	diskType = existingDisk.GetPDType()
 
 	if !diskSupportsIopsAndThroughput[diskType] {
-		return nil, status.Errorf(codes.InvalidArgument, "ModifyVolume is not supported for disk type %s", diskType)
+		return nil, status.Errorf(codes.InvalidArgument, "Failed to modify volume: modifications not supported for disk type %s", diskType)
 	}
 
 	// TODO : not sure how this metric should be reported
diff --git a/pkg/gce-pd-csi-driver/controller_test.go b/pkg/gce-pd-csi-driver/controller_test.go
index b3f19208f..0d01f6ed5 100644
--- a/pkg/gce-pd-csi-driver/controller_test.go
+++ b/pkg/gce-pd-csi-driver/controller_test.go
@@ -1713,52 +1713,116 @@ func TestMultiZoneVolumeCreationErrHandling(t *testing.T) {
 }
 func TestCreateVolumeWithVolumeAttributeClassParameters(t *testing.T) {
 	// When volume attribute class specifies iops / throughput they should take precedence over storage class parameters
-	var d []*gce.CloudDisk
-	fcp, err := gce.CreateFakeCloudProvider(project, zone, d)
-	gceDriver := initGCEDriverWithCloudProvider(t, fcp)
 
-	if err != nil {
-		t.Fatalf("Failed to create fake cloud provider: %v", err)
-	}
-
-	createVolReq := &csi.CreateVolumeRequest{
-		Name:          "test-name",
-		CapacityRange: stdCapRange,
-		VolumeCapabilities: []*csi.VolumeCapability{
-			{
-				AccessType: &csi.VolumeCapability_Mount{
-					Mount: &csi.VolumeCapability_MountVolume{},
+	testCases := []struct {
+		name          string
+		req           *csi.CreateVolumeRequest
+		expIops       int64
+		expThroughput int64
+		expError      string
+	}{
+		{
+			name: "Volume attribute class parameters should take precedence over storage class parameters",
+			req: &csi.CreateVolumeRequest{
+				Name:          name,
+				CapacityRange: stdCapRange,
+				VolumeCapabilities: []*csi.VolumeCapability{
+					{
+						AccessType: &csi.VolumeCapability_Mount{
+							Mount: &csi.VolumeCapability_MountVolume{},
+						},
+						AccessMode: &csi.VolumeCapability_AccessMode{
+							Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER,
+						},
+					},
 				},
-				AccessMode: &csi.VolumeCapability_AccessMode{
-					Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER,
+				Parameters: map[string]string{
+					common.ParameterKeyType:                          "hyperdisk-balanced",
+					common.ParameterKeyProvisionedIOPSOnCreate:       "10000",
+					common.ParameterKeyProvisionedThroughputOnCreate: "500",
 				},
+				AccessibilityRequirements: &csi.TopologyRequirement{
+					Preferred: []*csi.Topology{
+						{
+							Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"},
+						},
+					},
+				},
+				MutableParameters: map[string]string{"iops": "20000", "throughput": "600"},
 			},
+			expIops:       20000,
+			expThroughput: 600,
+			expError:      "",
 		},
-		Parameters: map[string]string{
-			common.ParameterKeyType:                          "hyperdisk-balanced",
-			common.ParameterKeyProvisionedIOPSOnCreate:       "10000",
-			common.ParameterKeyProvisionedThroughputOnCreate: "500",
+		{
+			name: "Volume attribute class parameters should be ignored for incompatible disk types",
+			req: &csi.CreateVolumeRequest{
+				Name:          "pd-ssd-vol",
+				CapacityRange: stdCapRange,
+				VolumeCapabilities: []*csi.VolumeCapability{
+					{
+						AccessType: &csi.VolumeCapability_Mount{
+							Mount: &csi.VolumeCapability_MountVolume{},
+						},
+						AccessMode: &csi.VolumeCapability_AccessMode{
+							Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER,
+						},
+					},
+				},
+				Parameters: map[string]string{
+					common.ParameterKeyType: "pd-ssd",
+					// common.ParameterKeyProvisionedIOPSOnCreate:       "10000",
+					// common.ParameterKeyProvisionedThroughputOnCreate: "500",
+				},
+				AccessibilityRequirements: &csi.TopologyRequirement{
+					Preferred: []*csi.Topology{
+						{
+							Segments: map[string]string{common.TopologyKeyZone: "us-central1-a"},
+						},
+					},
+				},
+				MutableParameters: map[string]string{"iops": "20000", "throughput": "600"},
+			},
+			expIops:       0,
+			expThroughput: 0,
+			expError:      "",
 		},
-		MutableParameters: map[string]string{"iops": "20000", "throughput": "600"},
 	}
 
-	_, err = gceDriver.cs.CreateVolume(context.Background(), createVolReq)
-	if err != nil {
-		t.Fatalf("Failed to create volume: %v", err)
-	}
+	for _, tc := range testCases {
+		var d []*gce.CloudDisk
+		fcp, err := gce.CreateFakeCloudProvider(project, zone, d)
+		gceDriver := initGCEDriverWithCloudProvider(t, fcp)
 
-	volumeKey := meta.ZonalKey(name, zone)
-	disk, err := fcp.GetDisk(context.Background(), project, volumeKey, gce.GCEAPIVersionBeta)
+		if err != nil {
+			t.Fatalf("Failed to create fake cloud provider: %v", err)
+		}
 
-	if err != nil {
-		t.Fatalf("Failed to get disk: %v", err)
-	}
-	if disk != nil {
-		if disk.GetProvisionedIops() != 20000 {
-			t.Errorf("Expected IOPS to be 20000, got: %v", disk.GetProvisionedIops())
+		createVolReq := tc.req
+
+		resp, err := gceDriver.cs.CreateVolume(context.Background(), createVolReq)
+		if err != nil {
+			t.Fatalf("Failed to create volume: %v", err)
 		}
-		if disk.GetProvisionedThroughput() != 600 {
-			t.Errorf("Expected Throughput to be 600, got: %v", disk.GetProvisionedThroughput())
+
+		volumeId := resp.GetVolume().VolumeId
+		project, volumeKey, err := common.VolumeIDToKey(volumeId)
+		if err != nil {
+			t.Fatalf("Failed to convert volume id to key: %v", err)
+		}
+
+		disk, err := fcp.GetDisk(context.Background(), project, volumeKey, gce.GCEAPIVersionBeta)
+
+		if err != nil {
+			t.Fatalf("Failed to get disk: %v", err)
+		}
+		if disk != nil {
+			if disk.GetProvisionedIops() != tc.expIops {
+				t.Errorf("Expected IOPS to be %d, got: %v", tc.expIops, disk.GetProvisionedIops())
+			}
+			if disk.GetProvisionedThroughput() != tc.expThroughput {
+				t.Errorf("Expected Throughput to be %d, got: %v", tc.expThroughput, disk.GetProvisionedThroughput())
+			}
 		}
 	}
 
@@ -1768,6 +1832,8 @@ func TestVolumeModifyOperation(t *testing.T) {
 	testCases := []struct {
 		name          string
 		req           *csi.ControllerModifyVolumeRequest
+		diskType      string
+		params        *common.DiskParameters
 		expIops       int64
 		expThroughput int64
 		expErrMessage string
@@ -1778,6 +1844,12 @@ func TestVolumeModifyOperation(t *testing.T) {
 				VolumeId:          testVolumeID,
 				MutableParameters: map[string]string{"iops": "20000", "throughput": "600"},
 			},
+			diskType: "hyperdisk-balanced",
+			params: &common.DiskParameters{
+				DiskType:                      "hyperdisk-balanced",
+				ProvisionedIOPSOnCreate:       10000,
+				ProvisionedThroughputOnCreate: 500,
+			},
 			expIops:       20000,
 			expThroughput: 600,
 			expErrMessage: "",
@@ -1788,15 +1860,35 @@ func TestVolumeModifyOperation(t *testing.T) {
 				VolumeId:          testVolumeID,
 				MutableParameters: map[string]string{"iops": "0", "throughput": "0"},
 			},
+			diskType: "hyperdisk-balanced",
+			params: &common.DiskParameters{
+				DiskType:                      "hyperdisk-balanced",
+				ProvisionedIOPSOnCreate:       10000,
+				ProvisionedThroughputOnCreate: 500,
+			},
 			expIops:       10000,
 			expThroughput: 500,
 			expErrMessage: "no IOPS or Throughput specified for disk",
 		},
+		{
+			name: "Update volume with valid parameters but invalid disk type",
+			req: &csi.ControllerModifyVolumeRequest{
+				VolumeId:          testVolumeID,
+				MutableParameters: map[string]string{"iops": "20000", "throughput": "600"},
+			},
+			diskType: "pd-ssd",
+			params: &common.DiskParameters{
+				DiskType: "pd-ssd",
+			},
+			expIops:       0,
+			expThroughput: 0,
+			expErrMessage: fmt.Sprintf("modifications not supported for disk type %s", "pd-ssd"),
+		},
 	}
 
 	for _, tc := range testCases {
 		t.Logf("test case: %s", tc.name)
-
+		// Arrange
 		fcp, err := gce.CreateFakeCloudProvider(project, zone, nil)
 
 		if err != nil {
@@ -1804,26 +1896,24 @@ func TestVolumeModifyOperation(t *testing.T) {
 		}
 
 		gceDriver := initGCEDriverWithCloudProvider(t, fcp)
-		// Arrange
 		project, volKey, err := common.VolumeIDToKey(testVolumeID)
 		if err != nil {
 			t.Fatalf("Failed convert key: %v", err)
 		}
 
-		testDiskParams := &common.DiskParameters{
-			DiskType:                      "hyperdisk-balanced",
-			ProvisionedIOPSOnCreate:       10000,
-			ProvisionedThroughputOnCreate: 500,
-		}
-
-		err = fcp.InsertDisk(context.Background(), project, volKey, *testDiskParams, 20, nil, nil, "", "", false, "")
+		err = fcp.InsertDisk(context.Background(), project, volKey, *tc.params, 200000, nil, nil, "", "", false, "")
 		if err != nil {
 			t.Fatalf("Failed to insert disk: %v", err)
 		}
+		// Act
 		_, err = gceDriver.cs.ControllerModifyVolume(context.Background(), tc.req)
 
-		if err != nil && !strings.Contains(err.Error(), tc.expErrMessage) {
-			t.Errorf("Failed to modify volume: %v", err)
+		// Assert
+		if err != nil {
+			msg := err.Error()
+			if !strings.ContainsAny(msg, tc.expErrMessage) {
+				t.Errorf("Failed to modify volume: %v", err)
+			}
 		}
 
 		modifiedVol, err := fcp.GetDisk(context.Background(), project, volKey, gce.GCEAPIVersionBeta)

From 0a87a40d046de59cb233ed0001b47287579f3dfc Mon Sep 17 00:00:00 2001
From: Max Kimambo <max@kimambo.de>
Date: Thu, 20 Jun 2024 16:11:56 +0000
Subject: [PATCH 19/20] Adding controller tests

---
 pkg/gce-cloud-provider/compute/cloud-disk.go  | 22 ++++++
 pkg/gce-cloud-provider/compute/fake-gce.go    | 16 ++++
 pkg/gce-cloud-provider/compute/gce-compute.go | 31 ++++++++
 pkg/gce-pd-csi-driver/controller_test.go      | 78 +++++++++++++++++++
 .../csi-test/v4/pkg/sanity/controller.go      |  1 +
 5 files changed, 148 insertions(+)

diff --git a/pkg/gce-cloud-provider/compute/cloud-disk.go b/pkg/gce-cloud-provider/compute/cloud-disk.go
index 02b70d5df..6790992c4 100644
--- a/pkg/gce-cloud-provider/compute/cloud-disk.go
+++ b/pkg/gce-cloud-provider/compute/cloud-disk.go
@@ -267,3 +267,25 @@ func (d *CloudDisk) GetAccessMode() string {
 		return ""
 	}
 }
+
+func (d *CloudDisk) GetProvisionedIops() int64 {
+	switch {
+	case d.disk != nil:
+		return d.disk.ProvisionedIops
+	case d.betaDisk != nil:
+		return d.betaDisk.ProvisionedIops
+	default:
+		return 0
+	}
+}
+
+func (d *CloudDisk) GetProvisionedThroughput() int64 {
+	switch {
+	case d.disk != nil:
+		return d.disk.ProvisionedThroughput
+	case d.betaDisk != nil:
+		return d.betaDisk.ProvisionedThroughput
+	default:
+		return 0
+	}
+}
diff --git a/pkg/gce-cloud-provider/compute/fake-gce.go b/pkg/gce-cloud-provider/compute/fake-gce.go
index 77dbf0751..d29226afa 100644
--- a/pkg/gce-cloud-provider/compute/fake-gce.go
+++ b/pkg/gce-cloud-provider/compute/fake-gce.go
@@ -279,6 +279,22 @@ func (cloud *FakeCloudProvider) InsertDisk(ctx context.Context, project string,
 	return nil
 }
 
+func (cloud *FakeCloudProvider) UpdateDisk(ctx context.Context, project string, volKey *meta.Key, existingDisk *CloudDisk, params common.ModifyVolumeParameters) error {
+
+	if params.IOPS == 0 || params.Throughput == 0 {
+		return fmt.Errorf("no IOPS or Throughput specified for disk %v", existingDisk.GetSelfLink())
+	}
+
+	if _, ok := cloud.disks[volKey.String()]; ok {
+		existingDisk.betaDisk.ProvisionedIops = params.IOPS
+		existingDisk.betaDisk.ProvisionedThroughput = params.Throughput
+		cloud.disks[volKey.String()] = existingDisk
+		return nil
+	}
+
+	return fmt.Errorf("disk %v not found", volKey)
+}
+
 func (cloud *FakeCloudProvider) DeleteDisk(ctx context.Context, project string, volKey *meta.Key) error {
 	delete(cloud.disks, volKey.String())
 	return nil
diff --git a/pkg/gce-cloud-provider/compute/gce-compute.go b/pkg/gce-cloud-provider/compute/gce-compute.go
index 13f00fcc2..21ec3ed3b 100644
--- a/pkg/gce-cloud-provider/compute/gce-compute.go
+++ b/pkg/gce-cloud-provider/compute/gce-compute.go
@@ -104,6 +104,7 @@ type GCECompute interface {
 	ValidateExistingDisk(ctx context.Context, disk *CloudDisk, params common.DiskParameters, reqBytes, limBytes int64, multiWriter bool) error
 	InsertDisk(ctx context.Context, project string, volKey *meta.Key, params common.DiskParameters, capBytes int64, capacityRange *csi.CapacityRange, replicaZones []string, snapshotID string, volumeContentSourceVolumeID string, multiWriter bool, accessMode string) error
 	DeleteDisk(ctx context.Context, project string, volumeKey *meta.Key) error
+	UpdateDisk(ctx context.Context, project string, volKey *meta.Key, existingDisk *CloudDisk, params common.ModifyVolumeParameters) error
 	AttachDisk(ctx context.Context, project string, volKey *meta.Key, readWrite, diskType, instanceZone, instanceName string, forceAttach bool) error
 	DetachDisk(ctx context.Context, project, deviceName, instanceZone, instanceName string) error
 	SetDiskAccessMode(ctx context.Context, project string, volKey *meta.Key, accessMode string) error
@@ -459,6 +460,36 @@ func (cloud *CloudProvider) InsertDisk(ctx context.Context, project string, volK
 	}
 }
 
+func (cloud *CloudProvider) UpdateDisk(ctx context.Context, project string, volKey *meta.Key, existingDisk *CloudDisk, params common.ModifyVolumeParameters) error {
+
+	klog.V(5).Infof("Updating disk %v", volKey)
+	// hyperdisks are zonal disks
+	// pd-disks do not support modification of IOPS and Throughput
+	return cloud.updateZonalDisk(ctx, project, volKey, existingDisk, params)
+}
+
+func (cloud *CloudProvider) updateZonalDisk(ctx context.Context, project string, volKey *meta.Key, existingDisk *CloudDisk, params common.ModifyVolumeParameters) error {
+
+	if params.IOPS == 0 && params.Throughput == 0 {
+		return fmt.Errorf("no IOPS or Throughput specified for disk %v", existingDisk.GetSelfLink())
+	}
+	updatedDisk := &computev1.Disk{
+		Name:                  existingDisk.GetName(),
+		ProvisionedIops:       params.IOPS,
+		ProvisionedThroughput: params.Throughput,
+	}
+
+	diskUpdateOp := cloud.service.Disks.Update(project, volKey.Zone, volKey.Name, updatedDisk)
+	diskUpdateOp.Paths("provisionedIops", "provisionedThroughput")
+	_, err := diskUpdateOp.Context(ctx).Do()
+
+	if err != nil {
+		return fmt.Errorf("error updating disk %v: %w", volKey, err)
+	}
+
+	return nil
+}
+
 func convertV1CustomerEncryptionKeyToBeta(v1Key *computev1.CustomerEncryptionKey) *computebeta.CustomerEncryptionKey {
 	return &computebeta.CustomerEncryptionKey{
 		KmsKeyName:      v1Key.KmsKeyName,
diff --git a/pkg/gce-pd-csi-driver/controller_test.go b/pkg/gce-pd-csi-driver/controller_test.go
index e196a1d3c..699ba3c46 100644
--- a/pkg/gce-pd-csi-driver/controller_test.go
+++ b/pkg/gce-pd-csi-driver/controller_test.go
@@ -22,6 +22,7 @@ import (
 	"reflect"
 	"sort"
 	"strconv"
+	"strings"
 	"testing"
 	"time"
 
@@ -1711,6 +1712,83 @@ func TestMultiZoneVolumeCreationErrHandling(t *testing.T) {
 	}
 }
 
+func TestVolumeUpdateOperation(t *testing.T) {
+
+	testCases := []struct {
+		name          string
+		req           *csi.ControllerModifyVolumeRequest
+		expIops       int64
+		expThroughput int64
+		expErrMessage string
+	}{
+		{
+			name: "Update volume with valid parameters",
+			req: &csi.ControllerModifyVolumeRequest{
+				VolumeId:          testVolumeID,
+				MutableParameters: map[string]string{"iops": "20000", "throughput": "600"},
+			},
+			expIops:       20000,
+			expThroughput: 600,
+			expErrMessage: "",
+		},
+		{
+			name: "Update volume with invalid parameters",
+			req: &csi.ControllerModifyVolumeRequest{
+				VolumeId:          testVolumeID,
+				MutableParameters: map[string]string{"iops": "0", "throughput": "0"},
+			},
+			expIops:       10000,
+			expThroughput: 500,
+			expErrMessage: "no IOPS or Throughput specified for disk",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Logf("test case: %s", tc.name)
+
+		fcp, err := gce.CreateFakeCloudProvider(project, zone, nil)
+
+		if err != nil {
+			t.Fatalf("Failed to create mock cloud provider: %v", err)
+		}
+
+		gceDriver := initGCEDriverWithCloudProvider(t, fcp)
+		// Arrange
+		project, volKey, err := common.VolumeIDToKey(testVolumeID)
+		if err != nil {
+			t.Fatalf("Failed convert key: %v", err)
+		}
+
+		testDiskParams := &common.DiskParameters{
+			DiskType:                      "hyperdisk-balanced",
+			ProvisionedIOPSOnCreate:       10000,
+			ProvisionedThroughputOnCreate: 500,
+		}
+
+		err = fcp.InsertDisk(context.Background(), project, volKey, *testDiskParams, 20, nil, nil, "", "", false, "")
+		if err != nil {
+			t.Errorf("Failed to insert disk: %v", err)
+		}
+		_, err = gceDriver.cs.ControllerModifyVolume(context.Background(), tc.req)
+
+		if err != nil && !strings.Contains(err.Error(), tc.expErrMessage) {
+			t.Errorf("Failed to modify volume: %v", err)
+		}
+
+		modifiedVol, err := fcp.GetDisk(context.Background(), project, volKey, gce.GCEAPIVersionBeta)
+
+		if err != nil {
+			t.Errorf("Failed to get volume: %v", err)
+		}
+
+		diskIops := modifiedVol.GetProvisionedIops()
+		throughput := modifiedVol.GetProvisionedThroughput()
+
+		if diskIops != tc.expIops && throughput != tc.expThroughput {
+			t.Errorf("Failed to modify volume: %v", err)
+		}
+	}
+}
 func TestListVolumePagination(t *testing.T) {
 	testCases := []struct {
 		name            string
diff --git a/vendor/github.com/kubernetes-csi/csi-test/v4/pkg/sanity/controller.go b/vendor/github.com/kubernetes-csi/csi-test/v4/pkg/sanity/controller.go
index 4dfaeb943..e7174d6c0 100644
--- a/vendor/github.com/kubernetes-csi/csi-test/v4/pkg/sanity/controller.go
+++ b/vendor/github.com/kubernetes-csi/csi-test/v4/pkg/sanity/controller.go
@@ -148,6 +148,7 @@ var _ = DescribeSanity("Controller Service [Controller Server]", func(sc *TestCo
 				case csi.ControllerServiceCapability_RPC_GET_VOLUME:
 				case csi.ControllerServiceCapability_RPC_VOLUME_CONDITION:
 				case csi.ControllerServiceCapability_RPC_SINGLE_NODE_MULTI_WRITER:
+				case csi.ControllerServiceCapability_RPC_MODIFY_VOLUME:
 				default:
 					Fail(fmt.Sprintf("Unknown capability: %v\n", cap.GetRpc().GetType()))
 				}

From 6cc24f1e97cc9162d397ee25a1ab999ffb71aae2 Mon Sep 17 00:00:00 2001
From: Max Kimambo <max@kimambo.de>
Date: Fri, 12 Jul 2024 16:58:24 +0000
Subject: [PATCH 20/20] remove duplicate declaration

---
 pkg/gce-pd-csi-driver/controller_test.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/pkg/gce-pd-csi-driver/controller_test.go b/pkg/gce-pd-csi-driver/controller_test.go
index 7ebf97e12..0d01f6ed5 100644
--- a/pkg/gce-pd-csi-driver/controller_test.go
+++ b/pkg/gce-pd-csi-driver/controller_test.go
@@ -23,7 +23,6 @@ import (
 	"sort"
 	"strconv"
 	"strings"
-	"strings"
 	"testing"
 	"time"