From 3f3ed63d53ead534ef1a1a38957aa3a93b01472b Mon Sep 17 00:00:00 2001 From: Sunny Song Date: Mon, 14 Nov 2022 10:13:41 -0800 Subject: [PATCH 1/3] Add provisionedIops for pd-extreme --- README.md | 14 +- go.mod | 1 + go.sum | 1 + pkg/common/parameters.go | 18 +- pkg/common/parameters_test.go | 16 + pkg/common/utils.go | 13 + pkg/common/utils_test.go | 90 +++++ pkg/gce-cloud-provider/compute/fake-gce.go | 15 +- pkg/gce-cloud-provider/compute/gce-compute.go | 11 +- pkg/gce-pd-csi-driver/controller_test.go | 25 ++ vendor/k8s.io/cloud-provider/LICENSE | 201 +++++++++++ .../k8s.io/cloud-provider/volume/constants.go | 26 ++ .../cloud-provider/volume/helpers/rounding.go | 165 +++++++++ .../cloud-provider/volume/helpers/zones.go | 313 ++++++++++++++++++ vendor/modules.txt | 4 + 15 files changed, 891 insertions(+), 22 deletions(-) create mode 100644 vendor/k8s.io/cloud-provider/LICENSE create mode 100644 vendor/k8s.io/cloud-provider/volume/constants.go create mode 100644 vendor/k8s.io/cloud-provider/volume/helpers/rounding.go create mode 100644 vendor/k8s.io/cloud-provider/volume/helpers/zones.go diff --git a/README.md b/README.md index d67626902..4375f236f 100644 --- a/README.md +++ b/README.md @@ -58,12 +58,14 @@ See Github [Issues](https://github.com/kubernetes-sigs/gcp-compute-persistent-di ### CreateVolume Parameters -| Parameter | Values | Default | Description | -|------------------|---------------------------|---------------|----------------------------------------------------------------------------------------------------| -| type | Any PD type (see [GCP documentation](https://cloud.google.com/compute/docs/disks#disk-types)), eg `pd-ssd` `pd-balanced` | `pd-standard` | Type allows you to choose between standard Persistent Disks or Solid State Drive Persistent Disks | -| replication-type | `none` OR `regional-pd` | `none` | Replication type allows you to choose between Zonal Persistent Disks or Regional Persistent Disks | -| disk-encryption-kms-key | Fully qualified resource identifier for the key to use to encrypt new disks. | Empty string. | Encrypt disk using Customer Managed Encryption Key (CMEK). See [GKE Docs](https://cloud.google.com/kubernetes-engine/docs/how-to/using-cmek#create_a_cmek_protected_attached_disk) for details. | -| labels | `key1=value1,key2=value2` | | Labels allow you to assign custom [GCE Disk labels](https://cloud.google.com/compute/docs/labeling-resources). | +| Parameter | Values | Default | Description | +|-----------------------------|---------------------------|---------------|----------------------------------------------------------------------------------------------------| +| type | Any PD type (see [GCP documentation](https://cloud.google.com/compute/docs/disks#disk-types)), eg `pd-ssd` `pd-balanced` | `pd-standard` | Type allows you to choose between standard Persistent Disks or Solid State Drive Persistent Disks | +| replication-type | `none` OR `regional-pd` | `none` | Replication type allows you to choose between Zonal Persistent Disks or Regional Persistent Disks | +| disk-encryption-kms-key | Fully qualified resource identifier for the key to use to encrypt new disks. | Empty string. | Encrypt disk using Customer Managed Encryption Key (CMEK). See [GKE Docs](https://cloud.google.com/kubernetes-engine/docs/how-to/using-cmek#create_a_cmek_protected_attached_disk) for details. | +| labels | `key1=value1,key2=value2` | | Labels allow you to assign custom [GCE Disk labels](https://cloud.google.com/compute/docs/labeling-resources). | +| provisioned-iops-on-create | string (int64 format). Values typically between 10,000 and 120,000 | | Indicates how many IOPS to provision for the disk. See the [Extreme persistent disk documentation](https://cloud.google.com/compute/docs/disks/extreme-persistent-disk) for details, including valid ranges for IOPS. | + ### Topology diff --git a/go.mod b/go.mod index aa16e28cc..d01dff99f 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( gopkg.in/gcfg.v1 v1.2.3 k8s.io/apimachinery v0.24.1 k8s.io/client-go v11.0.1-0.20190805182717-6502b5e7b1b5+incompatible + k8s.io/cloud-provider v0.24.1 k8s.io/component-base v0.24.1 k8s.io/klog/v2 v2.60.1 k8s.io/kubernetes v1.24.1 diff --git a/go.sum b/go.sum index b56f2f4a2..211f77635 100644 --- a/go.sum +++ b/go.sum @@ -2432,6 +2432,7 @@ k8s.io/apiserver v0.24.1/go.mod h1:dQWNMx15S8NqJMp0gpYfssyvhYnkilc1LpExd/dkLh0= k8s.io/cli-runtime v0.24.1/go.mod h1:14aVvCTqkA7dNXY51N/6hRY3GUjchyWDOwW84qmR3bs= k8s.io/client-go v0.24.1 h1:w1hNdI9PFrzu3OlovVeTnf4oHDt+FJLd9Ndluvnb42E= k8s.io/client-go v0.24.1/go.mod h1:f1kIDqcEYmwXS/vTbbhopMUbhKp2JhOeVTfxgaCIlF8= +k8s.io/cloud-provider v0.24.1 h1:SaQNq2Ax+epdY9wFngwN9GWpOVnM72hUqr2qy20cOvg= k8s.io/cloud-provider v0.24.1/go.mod h1:h5m/KIiwiQ76hpUBsgrwm/rxteIfJG9kJQ/+/w1as2M= k8s.io/cluster-bootstrap v0.24.1/go.mod h1:uq2PiYfKh8ZLb6DBU/3/2Z1DkMqXkTOHLemalC4tOgE= k8s.io/code-generator v0.24.1/go.mod h1:dpVhs00hTuTdTY6jvVxvTFCk6gSMrtfRydbhZwHI15w= diff --git a/pkg/common/parameters.go b/pkg/common/parameters.go index 8dc20d3c1..4649e758c 100644 --- a/pkg/common/parameters.go +++ b/pkg/common/parameters.go @@ -23,10 +23,11 @@ import ( const ( // Parameters for StorageClass - ParameterKeyType = "type" - ParameterKeyReplicationType = "replication-type" - ParameterKeyDiskEncryptionKmsKey = "disk-encryption-kms-key" - ParameterKeyLabels = "labels" + ParameterKeyType = "type" + ParameterKeyReplicationType = "replication-type" + ParameterKeyDiskEncryptionKmsKey = "disk-encryption-kms-key" + ParameterKeyLabels = "labels" + ParameterKeyProvisionedIOPSOnCreate = "provisioned-iops-on-create" // Parameters for VolumeSnapshotClass ParameterKeyStorageLocations = "storage-locations" @@ -80,6 +81,9 @@ type DiskParameters struct { // Values: {map[string]string} // Default: "" Labels map[string]string + // Values: {int64} + // Default: none + ProvisionedIOPSOnCreate int64 } // SnapshotParameters contains normalized and defaulted parameters for snapshots @@ -143,6 +147,12 @@ func ExtractAndDefaultParameters(parameters map[string]string, driverName string for labelKey, labelValue := range paramLabels { p.Labels[labelKey] = labelValue } + case ParameterKeyProvisionedIOPSOnCreate: + paramProvisionedIOPSOnCreate, err := ConvertGiBStringToInt64(v) + if err != nil { + return p, fmt.Errorf("parameters contain invalid provisionedIOPSOnCreate parameter: %w", err) + } + p.ProvisionedIOPSOnCreate = paramProvisionedIOPSOnCreate default: return p, fmt.Errorf("parameters contains invalid option %q", k) } diff --git a/pkg/common/parameters_test.go b/pkg/common/parameters_test.go index b896d0b9a..631ea8854 100644 --- a/pkg/common/parameters_test.go +++ b/pkg/common/parameters_test.go @@ -74,6 +74,22 @@ func TestExtractAndDefaultParameters(t *testing.T) { }, }, }, + { + name: "values from parameters, checking pd-extreme", + parameters: map[string]string{ParameterKeyType: "pd-extreme", ParameterKeyReplicationType: "none", ParameterKeyDiskEncryptionKmsKey: "foo/key", ParameterKeyLabels: "key1=value1,key2=value2", ParameterKeyProvisionedIOPSOnCreate: "10000Gi"}, + labels: map[string]string{}, + expectParams: DiskParameters{ + DiskType: "pd-extreme", + ReplicationType: "none", + DiskEncryptionKMSKey: "foo/key", + Tags: map[string]string{}, + Labels: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + ProvisionedIOPSOnCreate: 10000, + }, + }, { name: "values from parameters, checking balanced pd", parameters: map[string]string{ParameterKeyType: "pd-balanced", ParameterKeyReplicationType: "regional-pd", ParameterKeyDiskEncryptionKmsKey: "foo/key"}, diff --git a/pkg/common/utils.go b/pkg/common/utils.go index d5842ea36..2f02ea3ca 100644 --- a/pkg/common/utils.go +++ b/pkg/common/utils.go @@ -22,7 +22,9 @@ import ( "strings" "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/util/sets" + volumehelpers "k8s.io/cloud-provider/volume/helpers" ) const ( @@ -248,3 +250,14 @@ func ValidateSnapshotType(snapshotType string) error { return fmt.Errorf("invalid snapshot type %s", snapshotType) } } + +// ConvertGiBStringToInt64 converts a GiB string to int64 +func ConvertGiBStringToInt64(str string) (int64, error) { + // Verify regex before + match, _ := regexp.MatchString("^([+-]?[0-9.]+)([eEinumkKMGTP]*[-+]?[0-9]*)$", str) + if !match { + return 0, fmt.Errorf("invalid string %s", str) + } + quantity := resource.MustParse(str) + return volumehelpers.RoundUpToGiB(quantity) +} diff --git a/pkg/common/utils_test.go b/pkg/common/utils_test.go index ca435caf3..74c1be8d3 100644 --- a/pkg/common/utils_test.go +++ b/pkg/common/utils_test.go @@ -577,3 +577,93 @@ func TestSnapshotStorageLocations(t *testing.T) { }) } } + +func TestConvertGiBStringToInt64(t *testing.T) { + tests := []struct { + desc string + inputStr string + expInt64 int64 + expectError bool + }{ + { + "valid number string", + "10000", + 1, + false, + }, + { + "round Ki to GiB", + "1000Ki", + 1, + false, + }, + { + "round k to GiB", + "1000k", + 1, + false, + }, + { + "round Mi to GiB", + "1000Mi", + 1, + false, + }, + { + "round M to GiB", + "1000M", + 1, + false, + }, + { + "round G to GiB", + "1000G", + 932, + false, + }, + { + "round Gi to GiB", + "10000Gi", + 10000, + false, + }, + { + "round decimal to GiB", + "1.2Gi", + 2, + false, + }, + { + "round big value to GiB", + "8191Pi", + 8588886016, + false, + }, + { + "invalid empty string", + "", + 10000, + true, + }, + { + "invalid string", + "ew%65", + 10000, + true, + }, + } + for _, tc := range tests { + t.Run(tc.desc, func(t *testing.T) { + actualInt64, err := ConvertGiBStringToInt64(tc.inputStr) + if err != nil && !tc.expectError { + t.Errorf("Got error %v converting string to int64 %s; expect no error", err, tc.inputStr) + } + if err == nil && tc.expectError { + t.Errorf("Got no error converting string to int64 %s; expect an error", tc.inputStr) + } + if err == nil && actualInt64 != tc.expInt64 { + t.Errorf("Got %d for converting string to int64; expect %d", actualInt64, tc.expInt64) + } + }) + } +} diff --git a/pkg/gce-cloud-provider/compute/fake-gce.go b/pkg/gce-cloud-provider/compute/fake-gce.go index 3b3893288..0a10f5463 100644 --- a/pkg/gce-cloud-provider/compute/fake-gce.go +++ b/pkg/gce-cloud-provider/compute/fake-gce.go @@ -196,13 +196,14 @@ func (cloud *FakeCloudProvider) InsertDisk(ctx context.Context, project string, } computeDisk := &computev1.Disk{ - Name: volKey.Name, - SizeGb: common.BytesToGbRoundUp(capBytes), - Description: "Disk created by GCE-PD CSI Driver", - Type: cloud.GetDiskTypeURI(project, volKey, params.DiskType), - SourceDiskId: volumeContentSourceVolumeID, - Status: cloud.mockDiskStatus, - Labels: params.Labels, + Name: volKey.Name, + SizeGb: common.BytesToGbRoundUp(capBytes), + Description: "Disk created by GCE-PD CSI Driver", + Type: cloud.GetDiskTypeURI(project, volKey, params.DiskType), + SourceDiskId: volumeContentSourceVolumeID, + Status: cloud.mockDiskStatus, + Labels: params.Labels, + ProvisionedIops: params.ProvisionedIOPSOnCreate, } if snapshotID != "" { diff --git a/pkg/gce-cloud-provider/compute/gce-compute.go b/pkg/gce-cloud-provider/compute/gce-compute.go index aa0f67c02..61ac36afb 100644 --- a/pkg/gce-cloud-provider/compute/gce-compute.go +++ b/pkg/gce-cloud-provider/compute/gce-compute.go @@ -429,11 +429,12 @@ func (cloud *CloudProvider) insertRegionalDisk( } diskToCreate := &computev1.Disk{ - Name: volKey.Name, - SizeGb: common.BytesToGbRoundUp(capBytes), - Description: description, - Type: cloud.GetDiskTypeURI(cloud.project, volKey, params.DiskType), - Labels: params.Labels, + Name: volKey.Name, + SizeGb: common.BytesToGbRoundUp(capBytes), + Description: description, + Type: cloud.GetDiskTypeURI(cloud.project, volKey, params.DiskType), + Labels: params.Labels, + ProvisionedIops: params.ProvisionedIOPSOnCreate, } if snapshotID != "" { _, snapshotType, _, err := common.SnapshotIDToProjectKey(snapshotID) diff --git a/pkg/gce-pd-csi-driver/controller_test.go b/pkg/gce-pd-csi-driver/controller_test.go index 665ebb4af..ac6664577 100644 --- a/pkg/gce-pd-csi-driver/controller_test.go +++ b/pkg/gce-pd-csi-driver/controller_test.go @@ -792,6 +792,31 @@ func TestCreateVolumeArguments(t *testing.T) { }, expErrCode: codes.InvalidArgument, }, + { + name: "success with provisionedIops parameter", + req: &csi.CreateVolumeRequest{ + Name: name, + CapacityRange: stdCapRange, + VolumeCapabilities: stdVolCaps, + Parameters: map[string]string{"labels": "key1=value1,key2=value2", "provisioned-iops-on-create": "10000"}, + }, + expVol: &csi.Volume{ + CapacityBytes: common.GbToBytes(20), + VolumeId: testVolumeID, + VolumeContext: nil, + AccessibleTopology: stdTopology, + }, + }, + { + name: "fail with malformed provisionedIops parameter", + req: &csi.CreateVolumeRequest{ + Name: name, + CapacityRange: stdCapRange, + VolumeCapabilities: stdVolCaps, + Parameters: map[string]string{"labels": "key1=value1,key2=value2", "provisioned-iops-on-create": "dsfo3"}, + }, + expErrCode: codes.InvalidArgument, + }, } // Run test cases diff --git a/vendor/k8s.io/cloud-provider/LICENSE b/vendor/k8s.io/cloud-provider/LICENSE new file mode 100644 index 000000000..8dada3eda --- /dev/null +++ b/vendor/k8s.io/cloud-provider/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/k8s.io/cloud-provider/volume/constants.go b/vendor/k8s.io/cloud-provider/volume/constants.go new file mode 100644 index 000000000..d05f64ae2 --- /dev/null +++ b/vendor/k8s.io/cloud-provider/volume/constants.go @@ -0,0 +1,26 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package volume + +const ( + // ProvisionedVolumeName is the name of a volume in an external cloud + // that is being provisioned and thus should be ignored by rest of Kubernetes. + ProvisionedVolumeName = "placeholder-for-provisioning" + + // LabelMultiZoneDelimiter separates zones for volumes + LabelMultiZoneDelimiter = "__" +) diff --git a/vendor/k8s.io/cloud-provider/volume/helpers/rounding.go b/vendor/k8s.io/cloud-provider/volume/helpers/rounding.go new file mode 100644 index 000000000..101815738 --- /dev/null +++ b/vendor/k8s.io/cloud-provider/volume/helpers/rounding.go @@ -0,0 +1,165 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package helpers + +import ( + "fmt" + "math" + "math/bits" + + "k8s.io/apimachinery/pkg/api/resource" +) + +/* +The Cloud Provider's volume plugins provision disks for corresponding +PersistentVolumeClaims. Cloud Providers use different allocation unit for their +disk sizes. AWS allows you to specify the size as an integer amount of GiB, +while Portworx expects bytes for example. On AWS, if you want a volume of +1500MiB, the actual call to the AWS API should therefore be for a 2GiB disk. +This file contains functions that help rounding a storage request based on a +Cloud Provider's allocation unit. +*/ + +const ( + // GB - GigaByte size + GB = 1000 * 1000 * 1000 + // GiB - GibiByte size + GiB = 1024 * 1024 * 1024 + + // MB - MegaByte size + MB = 1000 * 1000 + // MiB - MebiByte size + MiB = 1024 * 1024 + + // KB - KiloByte size + KB = 1000 + // KiB - KibiByte size + KiB = 1024 +) + +// RoundUpToGiB rounds up given quantity upto chunks of GiB +func RoundUpToGiB(size resource.Quantity) (int64, error) { + return roundUpSizeInt64(size, GiB) +} + +// RoundUpToMB rounds up given quantity to chunks of MB +func RoundUpToMB(size resource.Quantity) (int64, error) { + return roundUpSizeInt64(size, MB) +} + +// RoundUpToMiB rounds up given quantity upto chunks of MiB +func RoundUpToMiB(size resource.Quantity) (int64, error) { + return roundUpSizeInt64(size, MiB) +} + +// RoundUpToKB rounds up given quantity to chunks of KB +func RoundUpToKB(size resource.Quantity) (int64, error) { + return roundUpSizeInt64(size, KB) +} + +// RoundUpToKiB rounds up given quantity to chunks of KiB +func RoundUpToKiB(size resource.Quantity) (int64, error) { + return roundUpSizeInt64(size, KiB) +} + +// RoundUpToB rounds up given quantity to chunks of bytes +func RoundUpToB(size resource.Quantity) (int64, error) { + return roundUpSizeInt64(size, 1) +} + +// RoundUpToGiBInt rounds up given quantity upto chunks of GiB. It returns an +// int instead of an int64 and an error if there's overflow +func RoundUpToGiBInt(size resource.Quantity) (int, error) { + return roundUpSizeInt(size, GiB) +} + +// RoundUpToMBInt rounds up given quantity to chunks of MB. It returns an +// int instead of an int64 and an error if there's overflow +func RoundUpToMBInt(size resource.Quantity) (int, error) { + return roundUpSizeInt(size, MB) +} + +// RoundUpToMiBInt rounds up given quantity upto chunks of MiB. It returns an +// int instead of an int64 and an error if there's overflow +func RoundUpToMiBInt(size resource.Quantity) (int, error) { + return roundUpSizeInt(size, MiB) +} + +// RoundUpToKBInt rounds up given quantity to chunks of KB. It returns an +// int instead of an int64 and an error if there's overflow +func RoundUpToKBInt(size resource.Quantity) (int, error) { + return roundUpSizeInt(size, KB) +} + +// RoundUpToKiBInt rounds up given quantity upto chunks of KiB. It returns an +// int instead of an int64 and an error if there's overflow +func RoundUpToKiBInt(size resource.Quantity) (int, error) { + return roundUpSizeInt(size, KiB) +} + +// RoundUpToGiBInt32 rounds up given quantity up to chunks of GiB. It returns an +// int32 instead of an int64 and an error if there's overflow +func RoundUpToGiBInt32(size resource.Quantity) (int32, error) { + return roundUpSizeInt32(size, GiB) +} + +// roundUpSizeInt calculates how many allocation units are needed to accommodate +// a volume of a given size. It returns an int and an error if there's overflow +func roundUpSizeInt(size resource.Quantity, allocationUnitBytes int64) (int, error) { + if bits.UintSize == 32 { + res, err := roundUpSizeInt32(size, allocationUnitBytes) + return int(res), err + } + res, err := roundUpSizeInt64(size, allocationUnitBytes) + return int(res), err +} + +// roundUpSizeInt32 calculates how many allocation units are needed to accommodate +// a volume of a given size. It returns an int32 and an error if there's overflow +func roundUpSizeInt32(size resource.Quantity, allocationUnitBytes int64) (int32, error) { + roundedUpInt32, err := roundUpSizeInt64(size, allocationUnitBytes) + if err != nil { + return 0, err + } + if roundedUpInt32 > math.MaxInt32 { + return 0, fmt.Errorf("quantity %s is too great, overflows int32", size.String()) + } + return int32(roundedUpInt32), nil +} + +// roundUpSizeInt64 calculates how many allocation units are needed to accommodate +// a volume of a given size. It returns an int64 and an error if there's overflow +func roundUpSizeInt64(size resource.Quantity, allocationUnitBytes int64) (int64, error) { + // Use CmpInt64() to find out if the value of "size" would overflow an + // int64 and therefore have Value() return a wrong result. Then, retrieve + // the value as int64 and perform the rounding. + // It's not convenient to use AsScale() and related functions as they don't + // support BinarySI format, nor can we use AsInt64() directly since it's + // only implemented for int64 scaled numbers (int64Amount). + + // CmpInt64() actually returns 0 when comparing an amount bigger than MaxInt64. + if size.CmpInt64(math.MaxInt64) >= 0 { + return 0, fmt.Errorf("quantity %s is too great, overflows int64", size.String()) + } + volumeSizeBytes := size.Value() + + roundedUp := volumeSizeBytes / allocationUnitBytes + if volumeSizeBytes%allocationUnitBytes > 0 { + roundedUp++ + } + return roundedUp, nil +} diff --git a/vendor/k8s.io/cloud-provider/volume/helpers/zones.go b/vendor/k8s.io/cloud-provider/volume/helpers/zones.go new file mode 100644 index 000000000..ff3a39284 --- /dev/null +++ b/vendor/k8s.io/cloud-provider/volume/helpers/zones.go @@ -0,0 +1,313 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package helpers + +import ( + "fmt" + "hash/fnv" + "math/rand" + "strconv" + "strings" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/sets" + cloudvolume "k8s.io/cloud-provider/volume" + "k8s.io/klog/v2" +) + +// LabelZonesToSet converts a PV label value from string containing a delimited list of zones to set +func LabelZonesToSet(labelZonesValue string) (sets.String, error) { + return stringToSet(labelZonesValue, cloudvolume.LabelMultiZoneDelimiter) +} + +// ZonesSetToLabelValue converts zones set to label value +func ZonesSetToLabelValue(strSet sets.String) string { + return strings.Join(strSet.UnsortedList(), cloudvolume.LabelMultiZoneDelimiter) +} + +// ZonesToSet converts a string containing a comma separated list of zones to set +func ZonesToSet(zonesString string) (sets.String, error) { + zones, err := stringToSet(zonesString, ",") + if err != nil { + return nil, fmt.Errorf("error parsing zones %s, must be strings separated by commas: %v", zonesString, err) + } + return zones, nil +} + +// StringToSet converts a string containing list separated by specified delimiter to a set +func stringToSet(str, delimiter string) (sets.String, error) { + zonesSlice := strings.Split(str, delimiter) + zonesSet := make(sets.String) + for _, zone := range zonesSlice { + trimmedZone := strings.TrimSpace(zone) + if trimmedZone == "" { + return make(sets.String), fmt.Errorf( + "%q separated list (%q) must not contain an empty string", + delimiter, + str) + } + zonesSet.Insert(trimmedZone) + } + return zonesSet, nil +} + +// LabelZonesToList converts a PV label value from string containing a delimited list of zones to list +func LabelZonesToList(labelZonesValue string) ([]string, error) { + return stringToList(labelZonesValue, cloudvolume.LabelMultiZoneDelimiter) +} + +// StringToList converts a string containing list separated by specified delimiter to a list +func stringToList(str, delimiter string) ([]string, error) { + zonesSlice := make([]string, 0) + for _, zone := range strings.Split(str, delimiter) { + trimmedZone := strings.TrimSpace(zone) + if trimmedZone == "" { + return nil, fmt.Errorf( + "%q separated list (%q) must not contain an empty string", + delimiter, + str) + } + zonesSlice = append(zonesSlice, trimmedZone) + } + return zonesSlice, nil +} + +// SelectZoneForVolume is a wrapper around SelectZonesForVolume +// to select a single zone for a volume based on parameters +func SelectZoneForVolume(zoneParameterPresent, zonesParameterPresent bool, zoneParameter string, zonesParameter, zonesWithNodes sets.String, node *v1.Node, allowedTopologies []v1.TopologySelectorTerm, pvcName string) (string, error) { + zones, err := SelectZonesForVolume(zoneParameterPresent, zonesParameterPresent, zoneParameter, zonesParameter, zonesWithNodes, node, allowedTopologies, pvcName, 1) + if err != nil { + return "", err + } + zone, ok := zones.PopAny() + if !ok { + return "", fmt.Errorf("could not determine a zone to provision volume in") + } + return zone, nil +} + +// SelectZonesForVolume selects zones for a volume based on several factors: +// node.zone, allowedTopologies, zone/zones parameters from storageclass, +// zones with active nodes from the cluster. The number of zones = replicas. +func SelectZonesForVolume(zoneParameterPresent, zonesParameterPresent bool, zoneParameter string, zonesParameter, zonesWithNodes sets.String, node *v1.Node, allowedTopologies []v1.TopologySelectorTerm, pvcName string, numReplicas uint32) (sets.String, error) { + if zoneParameterPresent && zonesParameterPresent { + return nil, fmt.Errorf("both zone and zones StorageClass parameters must not be used at the same time") + } + + var zoneFromNode string + // pick one zone from node if present + if node != nil { + // VolumeScheduling implicit since node is not nil + if zoneParameterPresent || zonesParameterPresent { + return nil, fmt.Errorf("zone[s] cannot be specified in StorageClass if VolumeBindingMode is set to WaitForFirstConsumer. Please specify allowedTopologies in StorageClass for constraining zones") + } + + // pick node's zone for one of the replicas + var ok bool + zoneFromNode, ok = node.ObjectMeta.Labels[v1.LabelTopologyZone] + if !ok { + zoneFromNode, ok = node.ObjectMeta.Labels[v1.LabelFailureDomainBetaZone] + if !ok { + return nil, fmt.Errorf("Either %s or %s Label for node missing", v1.LabelTopologyZone, v1.LabelFailureDomainBetaZone) + } + } + // if single replica volume and node with zone found, return immediately + if numReplicas == 1 { + return sets.NewString(zoneFromNode), nil + } + } + + // pick zone from allowedZones if specified + allowedZones, err := ZonesFromAllowedTopologies(allowedTopologies) + if err != nil { + return nil, err + } + + if (len(allowedTopologies) > 0) && (allowedZones.Len() == 0) { + return nil, fmt.Errorf("no matchLabelExpressions with %s key found in allowedTopologies. Please specify matchLabelExpressions with %s key", v1.LabelTopologyZone, v1.LabelTopologyZone) + } + + if allowedZones.Len() > 0 { + // VolumeScheduling implicit since allowedZones present + if zoneParameterPresent || zonesParameterPresent { + return nil, fmt.Errorf("zone[s] cannot be specified in StorageClass if allowedTopologies specified") + } + // scheduler will guarantee if node != null above, zoneFromNode is member of allowedZones. + // so if zoneFromNode != "", we can safely assume it is part of allowedZones. + zones, err := chooseZonesForVolumeIncludingZone(allowedZones, pvcName, zoneFromNode, numReplicas) + if err != nil { + return nil, fmt.Errorf("cannot process zones in allowedTopologies: %v", err) + } + return zones, nil + } + + // pick zone from parameters if present + if zoneParameterPresent { + if numReplicas > 1 { + return nil, fmt.Errorf("zone cannot be specified if desired number of replicas for pv is greather than 1. Please specify zones or allowedTopologies to specify desired zones") + } + return sets.NewString(zoneParameter), nil + } + + if zonesParameterPresent { + if uint32(zonesParameter.Len()) < numReplicas { + return nil, fmt.Errorf("not enough zones found in zones parameter to provision a volume with %d replicas. Found %d zones, need %d zones", numReplicas, zonesParameter.Len(), numReplicas) + } + // directly choose from zones parameter; no zone from node need to be considered + return ChooseZonesForVolume(zonesParameter, pvcName, numReplicas), nil + } + + // pick zone from zones with nodes + if zonesWithNodes.Len() > 0 { + // If node != null (and thus zoneFromNode != ""), zoneFromNode will be member of zonesWithNodes + zones, err := chooseZonesForVolumeIncludingZone(zonesWithNodes, pvcName, zoneFromNode, numReplicas) + if err != nil { + return nil, fmt.Errorf("cannot process zones where nodes exist in the cluster: %v", err) + } + return zones, nil + } + return nil, fmt.Errorf("cannot determine zones to provision volume in") +} + +// ZonesFromAllowedTopologies returns a list of zones specified in allowedTopologies +func ZonesFromAllowedTopologies(allowedTopologies []v1.TopologySelectorTerm) (sets.String, error) { + zones := make(sets.String) + for _, term := range allowedTopologies { + for _, exp := range term.MatchLabelExpressions { + if exp.Key == v1.LabelTopologyZone || exp.Key == v1.LabelFailureDomainBetaZone { + for _, value := range exp.Values { + zones.Insert(value) + } + } else { + return nil, fmt.Errorf("unsupported key found in matchLabelExpressions: %s", exp.Key) + } + } + } + return zones, nil +} + +// chooseZonesForVolumeIncludingZone is a wrapper around ChooseZonesForVolume that ensures zoneToInclude is chosen +// zoneToInclude can either be empty in which case it is ignored. If non-empty, zoneToInclude is expected to be member of zones. +// numReplicas is expected to be > 0 and <= zones.Len() +func chooseZonesForVolumeIncludingZone(zones sets.String, pvcName, zoneToInclude string, numReplicas uint32) (sets.String, error) { + if numReplicas == 0 { + return nil, fmt.Errorf("invalid number of replicas passed") + } + if uint32(zones.Len()) < numReplicas { + return nil, fmt.Errorf("not enough zones found to provision a volume with %d replicas. Need at least %d distinct zones for a volume with %d replicas", numReplicas, numReplicas, numReplicas) + } + if zoneToInclude != "" && !zones.Has(zoneToInclude) { + return nil, fmt.Errorf("zone to be included: %s needs to be member of set: %v", zoneToInclude, zones) + } + if uint32(zones.Len()) == numReplicas { + return zones, nil + } + if zoneToInclude != "" { + zones.Delete(zoneToInclude) + numReplicas = numReplicas - 1 + } + zonesChosen := ChooseZonesForVolume(zones, pvcName, numReplicas) + if zoneToInclude != "" { + zonesChosen.Insert(zoneToInclude) + } + return zonesChosen, nil +} + +// ChooseZonesForVolume is identical to ChooseZoneForVolume, but selects a multiple zones, for multi-zone disks. +func ChooseZonesForVolume(zones sets.String, pvcName string, numZones uint32) sets.String { + // No zones available, return empty set. + replicaZones := sets.NewString() + if zones.Len() == 0 { + return replicaZones + } + + // We create the volume in a zone determined by the name + // Eventually the scheduler will coordinate placement into an available zone + hash, index := getPVCNameHashAndIndexOffset(pvcName) + + // Zones.List returns zones in a consistent order (sorted) + // We do have a potential failure case where volumes will not be properly spread, + // if the set of zones changes during StatefulSet volume creation. However, this is + // probably relatively unlikely because we expect the set of zones to be essentially + // static for clusters. + // Hopefully we can address this problem if/when we do full scheduler integration of + // PVC placement (which could also e.g. avoid putting volumes in overloaded or + // unhealthy zones) + zoneSlice := zones.List() + + startingIndex := index * numZones + for index = startingIndex; index < startingIndex+numZones; index++ { + zone := zoneSlice[(hash+index)%uint32(len(zoneSlice))] + replicaZones.Insert(zone) + } + + klog.V(2).Infof("Creating volume for replicated PVC %q; chosen zones=%q from zones=%q", + pvcName, replicaZones.UnsortedList(), zoneSlice) + return replicaZones +} + +func getPVCNameHashAndIndexOffset(pvcName string) (hash uint32, index uint32) { + if pvcName == "" { + // We should always be called with a name; this shouldn't happen + klog.Warningf("No name defined during volume create; choosing random zone") + + hash = rand.Uint32() + } else { + hashString := pvcName + + // Heuristic to make sure that volumes in a StatefulSet are spread across zones + // StatefulSet PVCs are (currently) named ClaimName-StatefulSetName-Id, + // where Id is an integer index. + // Note though that if a StatefulSet pod has multiple claims, we need them to be + // in the same zone, because otherwise the pod will be unable to mount both volumes, + // and will be unschedulable. So we hash _only_ the "StatefulSetName" portion when + // it looks like `ClaimName-StatefulSetName-Id`. + // We continue to round-robin volume names that look like `Name-Id` also; this is a useful + // feature for users that are creating statefulset-like functionality without using statefulsets. + lastDash := strings.LastIndexByte(pvcName, '-') + if lastDash != -1 { + statefulsetIDString := pvcName[lastDash+1:] + statefulsetID, err := strconv.ParseUint(statefulsetIDString, 10, 32) + if err == nil { + // Offset by the statefulsetID, so we round-robin across zones + index = uint32(statefulsetID) + // We still hash the volume name, but only the prefix + hashString = pvcName[:lastDash] + + // In the special case where it looks like `ClaimName-StatefulSetName-Id`, + // hash only the StatefulSetName, so that different claims on the same StatefulSet + // member end up in the same zone. + // Note that StatefulSetName (and ClaimName) might themselves both have dashes. + // We actually just take the portion after the final - of ClaimName-StatefulSetName. + // For our purposes it doesn't much matter (just suboptimal spreading). + lastDash := strings.LastIndexByte(hashString, '-') + if lastDash != -1 { + hashString = hashString[lastDash+1:] + } + + klog.V(2).Infof("Detected StatefulSet-style volume name %q; index=%d", pvcName, index) + } + } + + // We hash the (base) volume name, so we don't bias towards the first N zones + h := fnv.New32() + h.Write([]byte(hashString)) + hash = h.Sum32() + } + + return hash, index +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 3b1dce51a..cc70f94e6 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -674,6 +674,10 @@ k8s.io/client-go/util/homedir k8s.io/client-go/util/jsonpath k8s.io/client-go/util/keyutil k8s.io/client-go/util/workqueue +# k8s.io/cloud-provider v0.24.1 => k8s.io/cloud-provider v0.24.1 +## explicit; go 1.16 +k8s.io/cloud-provider/volume +k8s.io/cloud-provider/volume/helpers # k8s.io/component-base v0.24.1 => k8s.io/component-base v0.24.1 ## explicit; go 1.16 k8s.io/component-base/metrics From a11cd4b71b6a27774749fe9b2618b5881ebaa591 Mon Sep 17 00:00:00 2001 From: Sunny Song Date: Thu, 1 Dec 2022 16:08:40 -0800 Subject: [PATCH 2/3] Test pd-extreme e2e test case --- test/e2e/tests/single_zone_e2e_test.go | 135 ++++----- .../tests/single_zone_pd_extreme_e2e_test.go | 275 ++++++++++++++++++ test/k8s-integration/config/sc-extreme.yaml | 11 + test/k8s-integration/main.go | 2 +- 4 files changed, 357 insertions(+), 66 deletions(-) create mode 100644 test/e2e/tests/single_zone_pd_extreme_e2e_test.go create mode 100644 test/k8s-integration/config/sc-extreme.yaml diff --git a/test/e2e/tests/single_zone_e2e_test.go b/test/e2e/tests/single_zone_e2e_test.go index 802cf488b..f7e709be5 100644 --- a/test/e2e/tests/single_zone_e2e_test.go +++ b/test/e2e/tests/single_zone_e2e_test.go @@ -71,7 +71,7 @@ var _ = Describe("GCE PD CSI Driver", func() { instance := testContext.Instance // Create Disk - volName, volID := createAndValidateUniqueZonalDisk(client, p, z) + volName, volID := createAndValidateUniqueZonalDisk(client, p, z, standardDiskType) defer func() { // Delete Disk @@ -96,7 +96,7 @@ var _ = Describe("GCE PD CSI Driver", func() { instance := testContext.Instance // Create Disk - volName, volID := createAndValidateUniqueZonalDisk(client, p, z) + volName, volID := createAndValidateUniqueZonalDisk(client, p, z, standardDiskType) defer func() { // Delete Disk @@ -168,7 +168,7 @@ var _ = Describe("GCE PD CSI Driver", func() { instance := testContext.Instance // Create Disk - volName, volID := createAndValidateUniqueZonalDisk(client, p, z) + volName, volID := createAndValidateUniqueZonalDisk(client, p, z, standardDiskType) defer func() { // Delete Disk @@ -267,7 +267,7 @@ var _ = Describe("GCE PD CSI Driver", func() { client := testContext.Client instance := testContext.Instance - volName, _ := createAndValidateUniqueZonalDisk(client, p, z) + volName, _ := createAndValidateUniqueZonalDisk(client, p, z, standardDiskType) underSpecifiedID := common.GenerateUnderspecifiedVolumeID(volName, true /* isZonal */) @@ -414,7 +414,7 @@ var _ = Describe("GCE PD CSI Driver", func() { p, z, _ := testContext.Instance.GetIdentity() client := testContext.Client - volName, volID := createAndValidateUniqueZonalDisk(client, p, z) + volName, volID := createAndValidateUniqueZonalDisk(client, p, z, standardDiskType) // Create Snapshot snapshotName := testNamePrefix + string(uuid.NewUUID()) @@ -470,52 +470,7 @@ var _ = Describe("GCE PD CSI Driver", func() { parentName := fmt.Sprintf("projects/%s/locations/%s", p, locationID) keyRingId := "gce-pd-csi-test-ring" - // Create KeyRing - ringReq := &kmspb.CreateKeyRingRequest{ - Parent: parentName, - KeyRingId: keyRingId, - } - keyRing, err := kmsClient.CreateKeyRing(ctx, ringReq) - if !gce.IsGCEError(err, "alreadyExists") { - getKeyRingReq := &kmspb.GetKeyRingRequest{ - Name: fmt.Sprintf("%s/keyRings/%s", parentName, keyRingId), - } - keyRing, err = kmsClient.GetKeyRing(ctx, getKeyRingReq) - - } - Expect(err).To(BeNil(), "Failed to create or get key ring %v", keyRingId) - - // Create CryptoKey in KeyRing - keyId := "test-key-" + string(uuid.NewUUID()) - keyReq := &kmspb.CreateCryptoKeyRequest{ - Parent: keyRing.Name, - CryptoKeyId: keyId, - CryptoKey: &kmspb.CryptoKey{ - Purpose: kmspb.CryptoKey_ENCRYPT_DECRYPT, - VersionTemplate: &kmspb.CryptoKeyVersionTemplate{ - Algorithm: kmspb.CryptoKeyVersion_GOOGLE_SYMMETRIC_ENCRYPTION, - }, - }, - } - key, err := kmsClient.CreateCryptoKey(ctx, keyReq) - Expect(err).To(BeNil(), "Failed to create crypto key %v in key ring %v", keyId, keyRing.Name) - - keyVersions := []string{} - keyVersionReq := &kmspb.ListCryptoKeyVersionsRequest{ - Parent: key.Name, - } - - it := kmsClient.ListCryptoKeyVersions(ctx, keyVersionReq) - - for { - keyVersion, err := it.Next() - if err == iterator.Done { - break - } - Expect(err).To(BeNil(), "Failed to list crypto key versions") - - keyVersions = append(keyVersions, keyVersion.Name) - } + key, keyVersions := setupKeyRing(ctx, parentName, keyRingId) // Defer deletion of all key versions // https://cloud.google.com/kms/docs/destroy-restore @@ -525,7 +480,7 @@ var _ = Describe("GCE PD CSI Driver", func() { destroyKeyReq := &kmspb.DestroyCryptoKeyVersionRequest{ Name: keyVersion, } - _, err = kmsClient.DestroyCryptoKeyVersion(ctx, destroyKeyReq) + _, err := kmsClient.DestroyCryptoKeyVersion(ctx, destroyKeyReq) Expect(err).To(BeNil(), "Failed to destroy crypto key version: %v", keyVersion) } @@ -620,10 +575,10 @@ var _ = Describe("GCE PD CSI Driver", func() { nodeID := testContext.Instance.GetNodeID() - _, volID := createAndValidateUniqueZonalDisk(client, p, z) + _, volID := createAndValidateUniqueZonalDisk(client, p, z, standardDiskType) defer deleteVolumeOrError(client, volID) - _, secondVolID := createAndValidateUniqueZonalDisk(client, p, z) + _, secondVolID := createAndValidateUniqueZonalDisk(client, p, z, standardDiskType) defer deleteVolumeOrError(client, secondVolID) // Attach volID to current instance @@ -722,7 +677,7 @@ var _ = Describe("GCE PD CSI Driver", func() { client := testContext.Client instance := testContext.Instance - volName, volID := createAndValidateUniqueZonalDisk(client, p, z) + volName, volID := createAndValidateUniqueZonalDisk(client, p, z, standardDiskType) defer func() { // Delete Disk @@ -759,7 +714,7 @@ var _ = Describe("GCE PD CSI Driver", func() { client := testContext.Client instance := testContext.Instance - volName, volID := createAndValidateUniqueZonalDisk(client, p, z) + volName, volID := createAndValidateUniqueZonalDisk(client, p, z, standardDiskType) defer func() { // Delete Disk @@ -801,7 +756,7 @@ var _ = Describe("GCE PD CSI Driver", func() { zone := "us-east1-a" // Create and Validate Disk - volName, volID := createAndValidateUniqueZonalMultiWriterDisk(client, p, zone) + volName, volID := createAndValidateUniqueZonalMultiWriterDisk(client, p, zone, standardDiskType) defer func() { // Delete Disk @@ -823,7 +778,7 @@ var _ = Describe("GCE PD CSI Driver", func() { instance := testContext.Instance // Create and Validate Disk - volName, volID := createAndValidateUniqueZonalMultiWriterDisk(client, p, z) + volName, volID := createAndValidateUniqueZonalMultiWriterDisk(client, p, z, standardDiskType) defer func() { // Delete Disk @@ -903,7 +858,7 @@ var _ = Describe("GCE PD CSI Driver", func() { client := testContext.Client // Create Disk - volName, volID := createAndValidateUniqueZonalDisk(client, p, z) + volName, volID := createAndValidateUniqueZonalDisk(client, p, z, standardDiskType) // Create Snapshot snapshotName := testNamePrefix + string(uuid.NewUUID()) @@ -964,7 +919,7 @@ var _ = Describe("GCE PD CSI Driver", func() { client := testContext.Client // Create Disk - volName, volID := createAndValidateUniqueZonalDisk(client, p, z) + volName, volID := createAndValidateUniqueZonalDisk(client, p, z, standardDiskType) // Create Snapshot snapshotName := testNamePrefix + string(uuid.NewUUID()) @@ -1025,7 +980,7 @@ var _ = Describe("GCE PD CSI Driver", func() { p, z, _ := controllerInstance.GetIdentity() // Create Source Disk - _, srcVolID := createAndValidateUniqueZonalDisk(controllerClient, p, z) + _, srcVolID := createAndValidateUniqueZonalDisk(controllerClient, p, z, standardDiskType) // Create Disk volName := testNamePrefix + string(uuid.NewUUID()) @@ -1167,7 +1122,7 @@ func equalWithinEpsilon(a, b, epsiolon int64) bool { return b-a < epsiolon } -func createAndValidateUniqueZonalDisk(client *remote.CsiClient, project, zone string) (volName, volID string) { +func createAndValidateUniqueZonalDisk(client *remote.CsiClient, project, zone string, diskType string) (volName, volID string) { // Create Disk var err error volName = testNamePrefix + string(uuid.NewUUID()) @@ -1184,7 +1139,7 @@ func createAndValidateUniqueZonalDisk(client *remote.CsiClient, project, zone st // Validate Disk Created cloudDisk, err := computeService.Disks.Get(project, zone, volName).Do() Expect(err).To(BeNil(), "Could not get disk from cloud directly") - Expect(cloudDisk.Type).To(ContainSubstring(standardDiskType)) + Expect(cloudDisk.Type).To(ContainSubstring(diskType)) Expect(cloudDisk.Status).To(Equal(readyState)) Expect(cloudDisk.SizeGb).To(Equal(defaultSizeGb)) Expect(cloudDisk.Name).To(Equal(volName)) @@ -1203,7 +1158,7 @@ func deleteVolumeOrError(client *remote.CsiClient, volID string) { Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") } -func createAndValidateUniqueZonalMultiWriterDisk(client *remote.CsiClient, project, zone string) (string, string) { +func createAndValidateUniqueZonalMultiWriterDisk(client *remote.CsiClient, project, zone string, diskType string) (string, string) { // Create Disk volName := testNamePrefix + string(uuid.NewUUID()) volID, err := client.CreateVolumeWithCaps(volName, nil, defaultMwSizeGb, @@ -1229,7 +1184,7 @@ func createAndValidateUniqueZonalMultiWriterDisk(client *remote.CsiClient, proje // Validate Disk Created cloudDisk, err := computeAlphaService.Disks.Get(project, zone, volName).Do() Expect(err).To(BeNil(), "Could not get disk from cloud directly") - Expect(cloudDisk.Type).To(ContainSubstring(standardDiskType)) + Expect(cloudDisk.Type).To(ContainSubstring(diskType)) Expect(cloudDisk.Status).To(Equal(readyState)) Expect(cloudDisk.SizeGb).To(Equal(defaultMwSizeGb)) Expect(cloudDisk.Name).To(Equal(volName)) @@ -1242,3 +1197,53 @@ func cleanSelfLink(selfLink string) string { r, _ := regexp.Compile("https:\\/\\/www.*apis.com\\/.*(v1|beta|alpha)\\/") return r.ReplaceAllString(selfLink, "") } + +func setupKeyRing(ctx context.Context, parentName string, keyRingId string) (*kmspb.CryptoKey, []string) { + // Create KeyRing + ringReq := &kmspb.CreateKeyRingRequest{ + Parent: parentName, + KeyRingId: keyRingId, + } + keyRing, err := kmsClient.CreateKeyRing(ctx, ringReq) + if !gce.IsGCEError(err, "alreadyExists") { + getKeyRingReq := &kmspb.GetKeyRingRequest{ + Name: fmt.Sprintf("%s/keyRings/%s", parentName, keyRingId), + } + keyRing, err = kmsClient.GetKeyRing(ctx, getKeyRingReq) + + } + Expect(err).To(BeNil(), "Failed to create or get key ring %v", keyRingId) + + // Create CryptoKey in KeyRing + keyId := "test-key-" + string(uuid.NewUUID()) + keyReq := &kmspb.CreateCryptoKeyRequest{ + Parent: keyRing.Name, + CryptoKeyId: keyId, + CryptoKey: &kmspb.CryptoKey{ + Purpose: kmspb.CryptoKey_ENCRYPT_DECRYPT, + VersionTemplate: &kmspb.CryptoKeyVersionTemplate{ + Algorithm: kmspb.CryptoKeyVersion_GOOGLE_SYMMETRIC_ENCRYPTION, + }, + }, + } + key, err := kmsClient.CreateCryptoKey(ctx, keyReq) + Expect(err).To(BeNil(), "Failed to create crypto key %v in key ring %v", keyId, keyRing.Name) + + keyVersions := []string{} + keyVersionReq := &kmspb.ListCryptoKeyVersionsRequest{ + Parent: key.Name, + } + + it := kmsClient.ListCryptoKeyVersions(ctx, keyVersionReq) + + for { + keyVersion, err := it.Next() + if err == iterator.Done { + break + } + Expect(err).To(BeNil(), "Failed to list crypto key versions") + + keyVersions = append(keyVersions, keyVersion.Name) + } + return key, keyVersions +} diff --git a/test/e2e/tests/single_zone_pd_extreme_e2e_test.go b/test/e2e/tests/single_zone_pd_extreme_e2e_test.go new file mode 100644 index 000000000..2bd447042 --- /dev/null +++ b/test/e2e/tests/single_zone_pd_extreme_e2e_test.go @@ -0,0 +1,275 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tests + +import ( + "context" + "fmt" + "time" + + "k8s.io/apimachinery/pkg/util/uuid" + "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common" + gce "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider/compute" + testutils "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/e2e/utils" + + csi "github.com/container-storage-interface/spec/lib/go/csi" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + kmspb "google.golang.org/genproto/googleapis/cloud/kms/v1" + fieldmask "google.golang.org/genproto/protobuf/field_mask" +) + +const ( + extremeDiskType = "pd-extreme" + provisionedIOPSOnCreate = "100000Gi" + provisionedIOPSOnCreateInt = int64(100000) +) + +var _ = Describe("GCE PD CSI Driver pd-extreme", func() { + + It("Should create and delete pd-extreme disk", func() { + Expect(testContexts).ToNot(BeEmpty()) + testContext := getRandomTestContext() + + p, z, _ := testContext.Instance.GetIdentity() + client := testContext.Client + + // Create Disk + volName := testNamePrefix + string(uuid.NewUUID()) + params := map[string]string{ + common.ParameterKeyType: extremeDiskType, + common.ParameterKeyProvisionedIOPSOnCreate: provisionedIOPSOnCreate, + } + volID, err := client.CreateVolume(volName, params, defaultSizeGb, nil, nil) + Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err) + + // Validate Disk Created + cloudDisk, err := computeService.Disks.Get(p, z, volName).Do() + Expect(err).To(BeNil(), "Could not get disk from cloud directly") + Expect(cloudDisk.Type).To(ContainSubstring(extremeDiskType)) + Expect(cloudDisk.ProvisionedIops).To(Equal(provisionedIOPSOnCreateInt)) + Expect(cloudDisk.Status).To(Equal(readyState)) + Expect(cloudDisk.SizeGb).To(Equal(defaultSizeGb)) + Expect(cloudDisk.Name).To(Equal(volName)) + + defer func() { + // Delete Disk + err := client.DeleteVolume(volID) + Expect(err).To(BeNil(), "DeleteVolume failed") + + // Validate Disk Deleted + _, err = computeService.Disks.Get(p, z, volName).Do() + Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") + }() + }) + + It("Should create and delete pd-extreme disk with labels", func() { + Expect(testContexts).ToNot(BeEmpty()) + testContext := getRandomTestContext() + + p, z, _ := testContext.Instance.GetIdentity() + client := testContext.Client + + // Create Disk + volName := testNamePrefix + string(uuid.NewUUID()) + params := map[string]string{ + common.ParameterKeyLabels: "key1=value1,key2=value2", + common.ParameterKeyType: extremeDiskType, + common.ParameterKeyProvisionedIOPSOnCreate: provisionedIOPSOnCreate, + } + volID, err := client.CreateVolume(volName, params, defaultSizeGb, nil, nil) + Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err) + + // Validate Disk Created + cloudDisk, err := computeService.Disks.Get(p, z, volName).Do() + Expect(err).To(BeNil(), "Could not get disk from cloud directly") + Expect(cloudDisk.Type).To(ContainSubstring(extremeDiskType)) + Expect(cloudDisk.ProvisionedIops).To(Equal(provisionedIOPSOnCreateInt)) + Expect(cloudDisk.Status).To(Equal(readyState)) + Expect(cloudDisk.SizeGb).To(Equal(defaultSizeGb)) + Expect(cloudDisk.Labels).To(Equal(map[string]string{ + "key1": "value1", + "key2": "value2", + // The label below is added as an --extra-label driver command line argument. + testutils.DiskLabelKey: testutils.DiskLabelValue, + })) + Expect(cloudDisk.Name).To(Equal(volName)) + + defer func() { + // Delete Disk + err := client.DeleteVolume(volID) + Expect(err).To(BeNil(), "DeleteVolume failed") + + // Validate Disk Deleted + _, err = computeService.Disks.Get(p, z, volName).Do() + Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") + }() + }) + + It("Should create CMEK key, go through volume lifecycle, validate behavior on key revoke and restore for pd-extreme", func() { + ctx := context.Background() + Expect(testContexts).ToNot(BeEmpty()) + testContext := getRandomTestContext() + + controllerInstance := testContext.Instance + controllerClient := testContext.Client + + p, z, _ := controllerInstance.GetIdentity() + locationID := "global" + + // The resource name of the key rings. + parentName := fmt.Sprintf("projects/%s/locations/%s", p, locationID) + keyRingId := "gce-pd-csi-test-ring" + + key, keyVersions := setupKeyRing(ctx, parentName, keyRingId) + + // Defer deletion of all key versions + // https://cloud.google.com/kms/docs/destroy-restore + defer func() { + + for _, keyVersion := range keyVersions { + destroyKeyReq := &kmspb.DestroyCryptoKeyVersionRequest{ + Name: keyVersion, + } + _, err := kmsClient.DestroyCryptoKeyVersion(ctx, destroyKeyReq) + Expect(err).To(BeNil(), "Failed to destroy crypto key version: %v", keyVersion) + } + + }() + + // Go through volume lifecycle using CMEK-ed PD + // Create Disk + volName := testNamePrefix + string(uuid.NewUUID()) + volID, err := controllerClient.CreateVolume(volName, map[string]string{ + common.ParameterKeyDiskEncryptionKmsKey: key.Name, + common.ParameterKeyType: extremeDiskType, + common.ParameterKeyProvisionedIOPSOnCreate: provisionedIOPSOnCreate, + }, defaultSizeGb, + &csi.TopologyRequirement{ + Requisite: []*csi.Topology{ + { + Segments: map[string]string{common.TopologyKeyZone: z}, + }, + }, + }, nil) + Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err) + + // Validate Disk Created + cloudDisk, err := computeService.Disks.Get(p, z, volName).Do() + Expect(err).To(BeNil(), "Could not get disk from cloud directly") + Expect(cloudDisk.Type).To(ContainSubstring(extremeDiskType)) + Expect(cloudDisk.ProvisionedIops).To(Equal(provisionedIOPSOnCreateInt)) + Expect(cloudDisk.Status).To(Equal(readyState)) + Expect(cloudDisk.SizeGb).To(Equal(defaultSizeGb)) + Expect(cloudDisk.Name).To(Equal(volName)) + + defer func() { + // Delete Disk + err = controllerClient.DeleteVolume(volID) + Expect(err).To(BeNil(), "DeleteVolume failed") + + // Validate Disk Deleted + _, err = computeService.Disks.Get(p, z, volName).Do() + Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") + }() + + // Test disk works + err = testAttachWriteReadDetach(volID, volName, controllerInstance, controllerClient, false /* readOnly */) + Expect(err).To(BeNil(), "Failed to go through volume lifecycle before revoking CMEK key") + + // Revoke CMEK key + // https://cloud.google.com/kms/docs/enable-disable + + for _, keyVersion := range keyVersions { + disableReq := &kmspb.UpdateCryptoKeyVersionRequest{ + CryptoKeyVersion: &kmspb.CryptoKeyVersion{ + Name: keyVersion, + State: kmspb.CryptoKeyVersion_DISABLED, + }, + UpdateMask: &fieldmask.FieldMask{ + Paths: []string{"state"}, + }, + } + _, err = kmsClient.UpdateCryptoKeyVersion(ctx, disableReq) + Expect(err).To(BeNil(), "Failed to disable crypto key") + } + + // Make sure attach of PD fails + err = testAttachWriteReadDetach(volID, volName, controllerInstance, controllerClient, false /* readOnly */) + Expect(err).ToNot(BeNil(), "Volume lifecycle should have failed, but succeeded") + + // Restore CMEK key + for _, keyVersion := range keyVersions { + enableReq := &kmspb.UpdateCryptoKeyVersionRequest{ + CryptoKeyVersion: &kmspb.CryptoKeyVersion{ + Name: keyVersion, + State: kmspb.CryptoKeyVersion_ENABLED, + }, + UpdateMask: &fieldmask.FieldMask{ + Paths: []string{"state"}, + }, + } + _, err = kmsClient.UpdateCryptoKeyVersion(ctx, enableReq) + Expect(err).To(BeNil(), "Failed to enable crypto key") + } + + // The controller publish failure in above step would set a backoff condition on the node. Wait suffcient amount of time for the driver to accept new controller publish requests. + time.Sleep(time.Second) + // Make sure attach of PD succeeds + err = testAttachWriteReadDetach(volID, volName, controllerInstance, controllerClient, false /* readOnly */) + Expect(err).To(BeNil(), "Failed to go through volume lifecycle after restoring CMEK key") + }) + + It("Should successfully create pd-extreme disk with PVC/PV tags", func() { + Expect(testContexts).ToNot(BeEmpty()) + testContext := getRandomTestContext() + + controllerInstance := testContext.Instance + controllerClient := testContext.Client + + p, z, _ := controllerInstance.GetIdentity() + + // Create Disk + volName := testNamePrefix + string(uuid.NewUUID()) + volID, err := controllerClient.CreateVolume(volName, map[string]string{ + common.ParameterKeyPVCName: "test-pvc", + common.ParameterKeyPVCNamespace: "test-pvc-namespace", + common.ParameterKeyPVName: "test-pv-name", + common.ParameterKeyType: extremeDiskType, + common.ParameterKeyProvisionedIOPSOnCreate: provisionedIOPSOnCreate, + }, defaultSizeGb, nil /* topReq */, nil) + Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err) + + // Validate Disk Created + cloudDisk, err := computeService.Disks.Get(p, z, volName).Do() + Expect(err).To(BeNil(), "Could not get disk from cloud directly") + Expect(cloudDisk.Type).To(ContainSubstring(extremeDiskType)) + Expect(cloudDisk.ProvisionedIops).To(Equal(provisionedIOPSOnCreateInt)) + Expect(cloudDisk.Status).To(Equal(readyState)) + Expect(cloudDisk.SizeGb).To(Equal(defaultSizeGb)) + Expect(cloudDisk.Name).To(Equal(volName)) + Expect(cloudDisk.Description).To(Equal("{\"kubernetes.io/created-for/pv/name\":\"test-pv-name\",\"kubernetes.io/created-for/pvc/name\":\"test-pvc\",\"kubernetes.io/created-for/pvc/namespace\":\"test-pvc-namespace\",\"storage.gke.io/created-by\":\"pd.csi.storage.gke.io\"}")) + defer func() { + // Delete Disk + controllerClient.DeleteVolume(volID) + Expect(err).To(BeNil(), "DeleteVolume failed") + + // Validate Disk Deleted + _, err = computeService.Disks.Get(p, z, volName).Do() + Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found") + }() + }) +}) diff --git a/test/k8s-integration/config/sc-extreme.yaml b/test/k8s-integration/config/sc-extreme.yaml new file mode 100644 index 000000000..2505c420a --- /dev/null +++ b/test/k8s-integration/config/sc-extreme.yaml @@ -0,0 +1,11 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: csi-gcepd +provisioner: pd.csi.storage.gke.io +parameters: + type: pd-extreme + provisioned-iops-on-create: '10000Gi' + # Add labels for testing. + labels: key1=value1,key2=value2 +volumeBindingMode: WaitForFirstConsumer \ No newline at end of file diff --git a/test/k8s-integration/main.go b/test/k8s-integration/main.go index 4549c6a8b..d14b39efc 100644 --- a/test/k8s-integration/main.go +++ b/test/k8s-integration/main.go @@ -479,7 +479,7 @@ func handle() error { } skipDiskImageSnapshots := false - if mustParseVersion(testParams.clusterVersion).lessThan(mustParseVersion("1.22.0")) { + if mustParseVersion(testParams.clusterVersion).lessThan(mustParseVersion("1.22.0")) || strings.Contains(testParams.storageClassFile, "sc-extreme") { // Disk image cloning in only supported from 1.22 on. skipDiskImageSnapshots = true } From 163455549b481745aafcb172306bd30702e2cff1 Mon Sep 17 00:00:00 2001 From: Sunny Song Date: Thu, 8 Dec 2022 11:03:48 -0800 Subject: [PATCH 3/3] Test pd-extreme k8s integration --- test/k8s-integration/driver-config.go | 4 +++- test/k8s-integration/main.go | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/test/k8s-integration/driver-config.go b/test/k8s-integration/driver-config.go index b3a8cf9b3..25cdef47d 100644 --- a/test/k8s-integration/driver-config.go +++ b/test/k8s-integration/driver-config.go @@ -127,7 +127,9 @@ func generateDriverConfigFile(testParams *testParameters) (string, error) { snapshotClassName = "no-volumesnapshotclass" } - caps = append(caps, "pvcDataSource") + if !strings.Contains(testParams.storageClassFile, "sc-extreme") { + caps = append(caps, "pvcDataSource") + } minimumVolumeSize := "5Gi" numAllowedTopologies := 1 if testParams.storageClassFile == regionalPDStorageClass { diff --git a/test/k8s-integration/main.go b/test/k8s-integration/main.go index d14b39efc..4549c6a8b 100644 --- a/test/k8s-integration/main.go +++ b/test/k8s-integration/main.go @@ -479,7 +479,7 @@ func handle() error { } skipDiskImageSnapshots := false - if mustParseVersion(testParams.clusterVersion).lessThan(mustParseVersion("1.22.0")) || strings.Contains(testParams.storageClassFile, "sc-extreme") { + if mustParseVersion(testParams.clusterVersion).lessThan(mustParseVersion("1.22.0")) { // Disk image cloning in only supported from 1.22 on. skipDiskImageSnapshots = true }