Skip to content

Commit c272900

Browse files
arkadeepsenarkadeepsen
committed
Support to add ResoureManagerTags to GCP Compute Disk, Image, Snapshot resources
Signed-off-by: arkadeepsen <[email protected]>
1 parent d1f263d commit c272900

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

68 files changed

+32456
-24
lines changed

‎cmd/gce-pd-csi-driver/main.go

+11-1
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,8 @@ var (
6666
maxConcurrentFormatAndMount = flag.Int("max-concurrent-format-and-mount", 1, "If set then format and mount operations are serialized on each node. This is stronger than max-concurrent-format as it includes fsck and other mount operations")
6767
formatAndMountTimeout = flag.Duration("format-and-mount-timeout", 1*time.Minute, "The maximum duration of a format and mount operation before another such operation will be started. Used only if --serialize-format-and-mount")
6868

69+
extraTagsStr = flag.String("extra-tags", "", "Extra tags to attach to each PD created. It is a comma separated list of parent id, key and value like '<parent_id1>/<tag_key1>/<tag_value1>,...,<parent_idN>/<tag_keyN>/<tag_valueN>'. parent_id is the Organization or the Project ID where the tag key and the tag value resources exist. A maximum of 50 tags bindings is allowed for a resource. See https://cloud.google.com/resource-manager/docs/tags/tags-overview, https://cloud.google.com/resource-manager/docs/tags/tags-creating-and-managing for details")
70+
6971
version string
7072
)
7173

@@ -119,6 +121,14 @@ func handle() {
119121
klog.Fatalf("Bad extra volume labels: %v", err.Error())
120122
}
121123

124+
if len(*extraTagsStr) > 0 && !*runControllerService {
125+
klog.Fatalf("Extra tags provided but not running controller")
126+
}
127+
extraTags, err := common.ConvertTagsStringToSlice(*extraTagsStr)
128+
if err != nil {
129+
klog.Fatalf("Bad extra tags: %v", err.Error())
130+
}
131+
122132
ctx, cancel := context.WithCancel(context.Background())
123133
defer cancel()
124134

@@ -161,7 +171,7 @@ func handle() {
161171
}
162172
}
163173

164-
err = gceDriver.SetupGCEDriver(driverName, version, extraVolumeLabels, identityServer, controllerServer, nodeServer)
174+
err = gceDriver.SetupGCEDriver(driverName, version, extraVolumeLabels, extraTags, identityServer, controllerServer, nodeServer)
165175
if err != nil {
166176
klog.Fatalf("Failed to initialize GCE CSI Driver: %v", err.Error())
167177
}

‎go.mod

+5-2
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,18 @@ go 1.20
55
require (
66
cloud.google.com/go/compute/metadata v0.2.3
77
cloud.google.com/go/kms v1.12.1
8+
cloud.google.com/go/resourcemanager v1.9.1
89
github.com/GoogleCloudPlatform/k8s-cloud-provider v1.24.0
910
github.com/container-storage-interface/spec v1.6.0
1011
github.com/google/uuid v1.3.0
12+
github.com/googleapis/gax-go/v2 v2.12.0
1113
github.com/kubernetes-csi/csi-proxy/client v1.1.1
1214
github.com/kubernetes-csi/csi-test/v4 v4.4.0
1315
github.com/onsi/ginkgo/v2 v2.7.1
1416
github.com/onsi/gomega v1.25.0
1517
golang.org/x/oauth2 v0.10.0
1618
golang.org/x/sys v0.10.0
19+
golang.org/x/time v0.0.0-20220210224613-90d013bbcef8
1720
google.golang.org/api v0.134.0
1821
google.golang.org/genproto v0.0.0-20230706204954-ccb25ca9f130
1922
google.golang.org/grpc v1.56.2
@@ -31,8 +34,10 @@ require (
3134
)
3235

3336
require (
37+
cloud.google.com/go v0.110.4 // indirect
3438
cloud.google.com/go/compute v1.20.1 // indirect
3539
cloud.google.com/go/iam v1.1.0 // indirect
40+
cloud.google.com/go/longrunning v0.5.1 // indirect
3641
github.com/Microsoft/go-winio v0.4.17 // indirect
3742
github.com/PuerkitoBio/purell v1.1.1 // indirect
3843
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
@@ -54,7 +59,6 @@ require (
5459
github.com/google/gofuzz v1.2.1-0.20210504230335-f78f29fc09ea // indirect
5560
github.com/google/s2a-go v0.1.4 // indirect
5661
github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect
57-
github.com/googleapis/gax-go/v2 v2.12.0 // indirect
5862
github.com/hashicorp/errwrap v1.0.0 // indirect
5963
github.com/hashicorp/go-multierror v1.1.0 // indirect
6064
github.com/imdario/mergo v0.3.12 // indirect
@@ -81,7 +85,6 @@ require (
8185
golang.org/x/net v0.12.0 // indirect
8286
golang.org/x/term v0.10.0 // indirect
8387
golang.org/x/text v0.11.0 // indirect
84-
golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
8588
google.golang.org/appengine v1.6.7 // indirect
8689
google.golang.org/genproto/googleapis/api v0.0.0-20230706204954-ccb25ca9f130 // indirect
8790
google.golang.org/genproto/googleapis/rpc v0.0.0-20230720185612-659f7aaaa771 // indirect

‎go.sum

+5
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECH
3131
cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8=
3232
cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0=
3333
cloud.google.com/go v0.110.4 h1:1JYyxKMN9hd5dR2MYTPWkGUgcoxVVhg0LKNKEo0qvmk=
34+
cloud.google.com/go v0.110.4/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI=
3435
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
3536
cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
3637
cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
@@ -49,11 +50,15 @@ cloud.google.com/go/iam v1.1.0/go.mod h1:nxdHjaKfCr7fNYx/HJMM8LgiMugmveWlkatear5
4950
cloud.google.com/go/kms v1.12.1 h1:xZmZuwy2cwzsocmKDOPu4BL7umg8QXagQx6fKVmf45U=
5051
cloud.google.com/go/kms v1.12.1/go.mod h1:c9J991h5DTl+kg7gi3MYomh12YEENGrf48ee/N/2CDM=
5152
cloud.google.com/go/logging v1.0.0/go.mod h1:V1cc3ogwobYzQq5f2R7DS/GvRIrI4FKj01Gs5glwAls=
53+
cloud.google.com/go/longrunning v0.5.1 h1:Fr7TXftcqTudoyRJa113hyaqlGdiBQkp0Gq7tErFDWI=
54+
cloud.google.com/go/longrunning v0.5.1/go.mod h1:spvimkwdz6SPWKEt/XBij79E9fiTkHSQl/fRUUQJYJc=
5255
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
5356
cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
5457
cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
5558
cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
5659
cloud.google.com/go/pubsub v1.4.0/go.mod h1:LFrqilwgdw4X2cJS9ALgzYmMu+ULyrUN6IHV3CPK4TM=
60+
cloud.google.com/go/resourcemanager v1.9.1 h1:QIAMfndPOHR6yTmMUB0ZN+HSeRmPjR/21Smq5/xwghI=
61+
cloud.google.com/go/resourcemanager v1.9.1/go.mod h1:dVCuosgrh1tINZ/RwBufr8lULmWGOkPS8gL5gqyjdT8=
5762
cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
5863
cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
5964
cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=

‎pkg/common/parameters.go

+23-2
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,9 @@ type DiskParameters struct {
9494
EnableConfidentialCompute bool
9595
// Default: false
9696
ForceAttach bool
97+
// Values: {[]Tag}
98+
// Default: ""
99+
ExtraTags []Tag
97100
}
98101

99102
// SnapshotParameters contains normalized and defaulted parameters for snapshots
@@ -103,13 +106,26 @@ type SnapshotParameters struct {
103106
ImageFamily string
104107
Tags map[string]string
105108
Labels map[string]string
109+
// Values: {[]Tag}
110+
// Default: ""
111+
ExtraTags []Tag
112+
}
113+
114+
// Tag stores the details of a GCP tag
115+
type Tag struct {
116+
// ParentID is the Organization or the Project ID where the tag key and the tag value resources exist
117+
ParentID string
118+
// TagKey is the short name of the tag key
119+
TagKey string
120+
// TagValue is the short name of the tag value
121+
TagValue string
106122
}
107123

108124
// ExtractAndDefaultParameters will take the relevant parameters from a map and
109125
// put them into a well defined struct making sure to default unspecified fields.
110126
// extraVolumeLabels are added as labels; if there are also labels specified in
111127
// parameters, any matching extraVolumeLabels will be overridden.
112-
func ExtractAndDefaultParameters(parameters map[string]string, driverName string, extraVolumeLabels map[string]string) (DiskParameters, error) {
128+
func ExtractAndDefaultParameters(parameters map[string]string, driverName string, extraVolumeLabels map[string]string, extraTags []Tag) (DiskParameters, error) {
113129
p := DiskParameters{
114130
DiskType: "pd-standard", // Default
115131
ReplicationType: replicationTypeNone, // Default
@@ -122,6 +138,8 @@ func ExtractAndDefaultParameters(parameters map[string]string, driverName string
122138
p.Labels[k] = v
123139
}
124140

141+
p.ExtraTags = append(p.ExtraTags, extraTags...)
142+
125143
for k, v := range parameters {
126144
if k == "csiProvisionerSecretName" || k == "csiProvisionerSecretNamespace" {
127145
// These are hardcoded secrets keys required to function but not needed by GCE PD
@@ -199,13 +217,16 @@ func ExtractAndDefaultParameters(parameters map[string]string, driverName string
199217
return p, nil
200218
}
201219

202-
func ExtractAndDefaultSnapshotParameters(parameters map[string]string, driverName string) (SnapshotParameters, error) {
220+
func ExtractAndDefaultSnapshotParameters(parameters map[string]string, driverName string, extraTags []Tag) (SnapshotParameters, error) {
203221
p := SnapshotParameters{
204222
StorageLocations: []string{},
205223
SnapshotType: DiskSnapshotType,
206224
Tags: make(map[string]string), // Default
207225
Labels: make(map[string]string), // Default
208226
}
227+
228+
p.ExtraTags = append(p.ExtraTags, extraTags...)
229+
209230
for k, v := range parameters {
210231
switch strings.ToLower(k) {
211232
case ParameterKeyStorageLocations:

‎pkg/common/parameters_test.go

+71-2
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ func TestExtractAndDefaultParameters(t *testing.T) {
2626
name string
2727
parameters map[string]string
2828
labels map[string]string
29+
extraTags []Tag
2930
expectParams DiskParameters
3031
expectErr bool
3132
}{
@@ -199,11 +200,45 @@ func TestExtractAndDefaultParameters(t *testing.T) {
199200
Labels: map[string]string{},
200201
},
201202
},
203+
{
204+
name: "extra tags",
205+
parameters: map[string]string{},
206+
labels: map[string]string{},
207+
extraTags: []Tag{
208+
{
209+
ParentID: "parent-id1",
210+
TagKey: "key1",
211+
TagValue: "value1",
212+
}, {
213+
ParentID: "parent-id2",
214+
TagKey: "key2",
215+
TagValue: "value2",
216+
},
217+
},
218+
expectParams: DiskParameters{
219+
DiskType: "pd-standard",
220+
ReplicationType: "none",
221+
DiskEncryptionKMSKey: "",
222+
Tags: map[string]string{},
223+
Labels: map[string]string{},
224+
ExtraTags: []Tag{
225+
{
226+
ParentID: "parent-id1",
227+
TagKey: "key1",
228+
TagValue: "value1",
229+
}, {
230+
ParentID: "parent-id2",
231+
TagKey: "key2",
232+
TagValue: "value2",
233+
},
234+
},
235+
},
236+
},
202237
}
203238

204239
for _, tc := range tests {
205240
t.Run(tc.name, func(t *testing.T) {
206-
p, err := ExtractAndDefaultParameters(tc.parameters, "testDriver", tc.labels)
241+
p, err := ExtractAndDefaultParameters(tc.parameters, "testDriver", tc.labels, tc.extraTags)
207242
if gotErr := err != nil; gotErr != tc.expectErr {
208243
t.Fatalf("ExtractAndDefaultParameters(%+v) = %v; expectedErr: %v", tc.parameters, err, tc.expectErr)
209244
}
@@ -224,6 +259,7 @@ func TestSnapshotParameters(t *testing.T) {
224259
tests := []struct {
225260
desc string
226261
parameters map[string]string
262+
extraTags []Tag
227263
expectedSnapshotParames SnapshotParameters
228264
expectError bool
229265
}{
@@ -268,10 +304,43 @@ func TestSnapshotParameters(t *testing.T) {
268304
parameters: map[string]string{ParameterKeySnapshotType: "invalid-type"},
269305
expectError: true,
270306
},
307+
{
308+
desc: "extra tags",
309+
parameters: nil,
310+
extraTags: []Tag{
311+
{
312+
ParentID: "parent-id1",
313+
TagKey: "key1",
314+
TagValue: "value1",
315+
}, {
316+
ParentID: "parent-id2",
317+
TagKey: "key2",
318+
TagValue: "value2",
319+
},
320+
},
321+
expectedSnapshotParames: SnapshotParameters{
322+
StorageLocations: []string{},
323+
SnapshotType: DiskSnapshotType,
324+
Tags: make(map[string]string),
325+
Labels: map[string]string{},
326+
ExtraTags: []Tag{
327+
{
328+
ParentID: "parent-id1",
329+
TagKey: "key1",
330+
TagValue: "value1",
331+
}, {
332+
ParentID: "parent-id2",
333+
TagKey: "key2",
334+
TagValue: "value2",
335+
},
336+
},
337+
},
338+
expectError: false,
339+
},
271340
}
272341
for _, tc := range tests {
273342
t.Run(tc.desc, func(t *testing.T) {
274-
p, err := ExtractAndDefaultSnapshotParameters(tc.parameters, "test-driver")
343+
p, err := ExtractAndDefaultSnapshotParameters(tc.parameters, "test-driver", tc.extraTags)
275344
if err != nil && !tc.expectError {
276345
t.Errorf("Got error %v; expect no error", err)
277346
}

‎pkg/common/utils.go

+100
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,10 @@ import (
2323
"net/http"
2424
"regexp"
2525
"strings"
26+
"time"
2627

2728
"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta"
29+
"golang.org/x/time/rate"
2830
"google.golang.org/api/googleapi"
2931
"google.golang.org/grpc/codes"
3032
"google.golang.org/grpc/status"
@@ -255,6 +257,91 @@ func ConvertLabelsStringToMap(labels string) (map[string]string, error) {
255257
return labelsMap, nil
256258
}
257259

260+
// ConvertTagsStringToSlice converts the tags from string to Tag slice
261+
// example: "parent_id1/tag_key1/tag_value1,parent_id2/tag_key2/tag_value2" gets
262+
// converted into {{ParentID: "parent_id1", TagKey: "tag_key1", TagValue: "tag_value1"},
263+
// {ParentID: "parent_id2", TagKey: "tag_key2", TagValue: "tag_value2"}}
264+
// See https://cloud.google.com/resource-manager/docs/tags/tags-overview,
265+
// https://cloud.google.com/resource-manager/docs/tags/tags-creating-and-managing for details
266+
func ConvertTagsStringToSlice(tags string) ([]Tag, error) {
267+
const tagsDelimiter = ","
268+
const tagsParentIDKeyValueDelimiter = "/"
269+
270+
tagsSlice := []Tag{}
271+
if tags == "" {
272+
return tagsSlice, nil
273+
}
274+
275+
regexParent, _ := regexp.Compile(`(^[1-9][0-9]{0,31}$)|(^[a-z][a-z0-9-]{4,28}[a-z0-9]$)`)
276+
checkTagParentIDFn := func(parentID string) error {
277+
if !regexParent.MatchString(parentID) {
278+
return fmt.Errorf("tag parent_id %q is invalid. parent_id can have a maximum of 32 characters and cannot be empty. parent_id can be either OrganizationID or ProjectID. OrganizationID must consist of decimal numbers, and cannot have leading zeroes and ProjectID must be 6 to 30 characters in length, can only contain lowercase letters, numbers, and hyphens, and must start with a letter, and cannot end with a hyphen", parentID)
279+
}
280+
return nil
281+
}
282+
283+
regexKey, _ := regexp.Compile(`^[a-zA-Z0-9]([0-9A-Za-z_.-]{0,61}[a-zA-Z0-9])?$`)
284+
checkTagKeyFn := func(key string) error {
285+
if !regexKey.MatchString(key) {
286+
return fmt.Errorf("tag key %q is invalid. Tag key can have a maximum of 63 characters and cannot be empty. Tag key must begin and end with an alphanumeric character, and must contain only uppercase, lowercase alphanumeric characters, and the following special characters `._-`", key)
287+
}
288+
return nil
289+
}
290+
291+
regexValue, _ := regexp.Compile(`^[a-zA-Z0-9]([0-9A-Za-z_.@%=+:,*#&()\[\]{}\-\s]{0,61}[a-zA-Z0-9])?$`)
292+
checkTagValueFn := func(value string) error {
293+
if !regexValue.MatchString(value) {
294+
return fmt.Errorf("tag value %q is invalid. Tag value can have a maximum of 63 characters and cannot be empty. Tag value must begin and end with an alphanumeric character, and must contain only uppercase, lowercase alphanumeric characters, and the following special characters `_-.@%%=+:,*#&(){}[]` and spaces", value)
295+
}
296+
297+
return nil
298+
}
299+
300+
checkTagParentIDKey := sets.String{}
301+
parentIDkeyValueStrings := strings.Split(tags, tagsDelimiter)
302+
for _, parentIDkeyValueString := range parentIDkeyValueStrings {
303+
parentIDKeyValue := strings.Split(parentIDkeyValueString, tagsParentIDKeyValueDelimiter)
304+
305+
if len(parentIDKeyValue) != 3 {
306+
return nil, fmt.Errorf("tags %q are invalid, correct format: 'parent_id1/key1/value1,parent_id2/key2/value2'", tags)
307+
}
308+
309+
parentIDKeyStr := parentIDkeyValueString[:strings.LastIndex(parentIDkeyValueString, tagsParentIDKeyValueDelimiter)]
310+
if checkTagParentIDKey.Has(parentIDKeyStr) {
311+
return nil, fmt.Errorf("tag parent_id & key combination %q exists more than once", parentIDKeyStr)
312+
}
313+
checkTagParentIDKey.Insert(parentIDKeyStr)
314+
315+
parentID := strings.TrimSpace(parentIDKeyValue[0])
316+
if err := checkTagParentIDFn(parentID); err != nil {
317+
return nil, err
318+
}
319+
320+
key := strings.TrimSpace(parentIDKeyValue[1])
321+
if err := checkTagKeyFn(key); err != nil {
322+
return nil, err
323+
}
324+
325+
value := strings.TrimSpace(parentIDKeyValue[2])
326+
if err := checkTagValueFn(value); err != nil {
327+
return nil, err
328+
}
329+
330+
tagsSlice = append(tagsSlice, Tag{
331+
ParentID: parentID,
332+
TagKey: key,
333+
TagValue: value,
334+
})
335+
}
336+
337+
const maxNumberOfTags = 50
338+
if len(tagsSlice) > maxNumberOfTags {
339+
return nil, fmt.Errorf("more than %d tags is not allowed, given: %d", maxNumberOfTags, len(tagsSlice))
340+
}
341+
342+
return tagsSlice, nil
343+
}
344+
258345
// ProcessStorageLocations trims and normalizes storage location to lower letters.
259346
func ProcessStorageLocations(storageLocations string) ([]string, error) {
260347
normalizedLoc := strings.ToLower(strings.TrimSpace(storageLocations))
@@ -392,3 +479,16 @@ func isValidDiskEncryptionKmsKey(DiskEncryptionKmsKey string) bool {
392479
kmsKeyPattern := regexp.MustCompile("projects/[^/]+/locations/([^/]+)/keyRings/[^/]+/cryptoKeys/[^/]+")
393480
return kmsKeyPattern.MatchString(DiskEncryptionKmsKey)
394481
}
482+
483+
// NewLimiter returns a token bucket based request rate limiter after initializing
484+
// the passed values for limit, burst (or token bucket) size. If opted for emptyBucket
485+
// all initial tokens are reserved for the first burst.
486+
func NewLimiter(limit, burst int, emptyBucket bool) *rate.Limiter {
487+
limiter := rate.NewLimiter(rate.Every(time.Second/time.Duration(limit)), burst)
488+
489+
if emptyBucket {
490+
limiter.AllowN(time.Now(), burst)
491+
}
492+
493+
return limiter
494+
}

0 commit comments

Comments
 (0)