Support to add ResoureManagerTags to GCP Compute Disk, Image, Snapshot resources

Signed-off-by: arkadeepsen <[email protected]>

Author: arkadeepsen

cmd/gce-pd-csi-driver/main.go

@@ -66,6 +66,8 @@ var (
 	maxConcurrentFormatAndMount = flag.Int("max-concurrent-format-and-mount", 1, "If set then format and mount operations are serialized on each node. This is stronger than max-concurrent-format as it includes fsck and other mount operations")
 	formatAndMountTimeout       = flag.Duration("format-and-mount-timeout", 1*time.Minute, "The maximum duration of a format and mount operation before another such operation will be started. Used only if --serialize-format-and-mount")
 
+	extraTagsStr = flag.String("extra-tags", "", "Extra tags to attach to each Compute Disk, Image, Snapshot created. It is a comma separated list of parent id, key and value like '<parent_id1>/<tag_key1>/<tag_value1>,...,<parent_idN>/<tag_keyN>/<tag_valueN>'. parent_id is the Organization or the Project ID where the tag key and the tag value resources exist. A maximum of 50 tags bindings is allowed for a resource. See https://cloud.google.com/resource-manager/docs/tags/tags-overview, https://cloud.google.com/resource-manager/docs/tags/tags-creating-and-managing for details")
+
 	version string
 )
 
@@ -119,6 +121,14 @@ func handle() {
 		klog.Fatalf("Bad extra volume labels: %v", err.Error())
 	}
 
+	if len(*extraTagsStr) > 0 && !*runControllerService {
+		klog.Fatalf("Extra tags provided but not running controller")
+	}
+	extraTags, err := common.ConvertTagsStringToSlice(*extraTagsStr)
+	if err != nil {
+		klog.Fatalf("Bad extra tags: %v", err.Error())
+	}
+
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
@@ -131,7 +141,7 @@ func handle() {
 	// Initialize requirements for the controller service
 	var controllerServer *driver.GCEControllerServer
 	if *runControllerService {
-		cloudProvider, err := gce.CreateCloudProvider(ctx, version, *cloudConfigFilePath, *computeEndpoint)
+		cloudProvider, err := gce.CreateCloudProvider(ctx, version, *cloudConfigFilePath, *computeEndpoint, extraTags)
 		if err != nil {
 			klog.Fatalf("Failed to get cloud provider: %v", err.Error())
 		}

go.mod

@@ -5,15 +5,18 @@ go 1.20
 require (
 	cloud.google.com/go/compute/metadata v0.2.3
 	cloud.google.com/go/kms v1.12.1
+	cloud.google.com/go/resourcemanager v1.9.1
 	github.com/GoogleCloudPlatform/k8s-cloud-provider v1.24.0
 	github.com/container-storage-interface/spec v1.6.0
 	github.com/google/uuid v1.3.0
+	github.com/googleapis/gax-go/v2 v2.12.0
 	github.com/kubernetes-csi/csi-proxy/client v1.1.1
 	github.com/kubernetes-csi/csi-test/v4 v4.4.0
 	github.com/onsi/ginkgo/v2 v2.7.1
 	github.com/onsi/gomega v1.25.0
 	golang.org/x/oauth2 v0.10.0
 	golang.org/x/sys v0.10.0
+	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8
 	google.golang.org/api v0.134.0
 	google.golang.org/genproto v0.0.0-20230706204954-ccb25ca9f130
 	google.golang.org/grpc v1.56.2
@@ -31,8 +34,10 @@ require (
 )
 
 require (
+	cloud.google.com/go v0.110.4 // indirect
 	cloud.google.com/go/compute v1.20.1 // indirect
 	cloud.google.com/go/iam v1.1.0 // indirect
+	cloud.google.com/go/longrunning v0.5.1 // indirect
 	github.com/Microsoft/go-winio v0.4.17 // indirect
 	github.com/PuerkitoBio/purell v1.1.1 // indirect
 	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
@@ -54,7 +59,6 @@ require (
 	github.com/google/gofuzz v1.2.1-0.20210504230335-f78f29fc09ea // indirect
 	github.com/google/s2a-go v0.1.4 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect
-	github.com/googleapis/gax-go/v2 v2.12.0 // indirect
 	github.com/hashicorp/errwrap v1.0.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.0 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
@@ -81,7 +85,6 @@ require (
 	golang.org/x/net v0.12.0 // indirect
 	golang.org/x/term v0.10.0 // indirect
 	golang.org/x/text v0.11.0 // indirect
-	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20230706204954-ccb25ca9f130 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20230720185612-659f7aaaa771 // indirect

go.sum

@@ -31,6 +31,7 @@ cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECH
 cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8=
 cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0=
 cloud.google.com/go v0.110.4 h1:1JYyxKMN9hd5dR2MYTPWkGUgcoxVVhg0LKNKEo0qvmk=
+cloud.google.com/go v0.110.4/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
 cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
 cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
@@ -49,11 +50,15 @@ cloud.google.com/go/iam v1.1.0/go.mod h1:nxdHjaKfCr7fNYx/HJMM8LgiMugmveWlkatear5
 cloud.google.com/go/kms v1.12.1 h1:xZmZuwy2cwzsocmKDOPu4BL7umg8QXagQx6fKVmf45U=
 cloud.google.com/go/kms v1.12.1/go.mod h1:c9J991h5DTl+kg7gi3MYomh12YEENGrf48ee/N/2CDM=
 cloud.google.com/go/logging v1.0.0/go.mod h1:V1cc3ogwobYzQq5f2R7DS/GvRIrI4FKj01Gs5glwAls=
+cloud.google.com/go/longrunning v0.5.1 h1:Fr7TXftcqTudoyRJa113hyaqlGdiBQkp0Gq7tErFDWI=
+cloud.google.com/go/longrunning v0.5.1/go.mod h1:spvimkwdz6SPWKEt/XBij79E9fiTkHSQl/fRUUQJYJc=
 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
 cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
 cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
 cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
 cloud.google.com/go/pubsub v1.4.0/go.mod h1:LFrqilwgdw4X2cJS9ALgzYmMu+ULyrUN6IHV3CPK4TM=
+cloud.google.com/go/resourcemanager v1.9.1 h1:QIAMfndPOHR6yTmMUB0ZN+HSeRmPjR/21Smq5/xwghI=
+cloud.google.com/go/resourcemanager v1.9.1/go.mod h1:dVCuosgrh1tINZ/RwBufr8lULmWGOkPS8gL5gqyjdT8=
 cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
 cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
 cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=

pkg/common/utils.go

@@ -23,8 +23,10 @@ import (
 	"net/http"
 	"regexp"
 	"strings"
+	"time"
 
 	"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta"
+	"golang.org/x/time/rate"
 	"google.golang.org/api/googleapi"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
@@ -255,6 +257,86 @@ func ConvertLabelsStringToMap(labels string) (map[string]string, error) {
 	return labelsMap, nil
 }
 
+// ConvertTagsStringToSlice converts the tags from string to Tag slice
+// example: "parent_id1/tag_key1/tag_value1,parent_id2/tag_key2/tag_value2" gets
+// converted into {"parent_id1/tag_key1/tag_value1", "parent_id2/tag_key2/tag_value2"}
+// See https://cloud.google.com/resource-manager/docs/tags/tags-overview,
+// https://cloud.google.com/resource-manager/docs/tags/tags-creating-and-managing for details
+func ConvertTagsStringToSlice(tags string) ([]string, error) {
+	const tagsDelimiter = ","
+	const tagsParentIDKeyValueDelimiter = "/"
+
+	tagsSlice := []string{}
+	if tags == "" {
+		return tagsSlice, nil
+	}
+
+	regexParent, _ := regexp.Compile(`(^[1-9][0-9]{0,31}$)|(^[a-z][a-z0-9-]{4,28}[a-z0-9]$)`)
+	checkTagParentIDFn := func(parentID string) error {
+		if !regexParent.MatchString(parentID) {
+			return fmt.Errorf("tag parent_id %q is invalid. parent_id can have a maximum of 32 characters and cannot be empty. parent_id can be either OrganizationID or ProjectID. OrganizationID must consist of decimal numbers, and cannot have leading zeroes and ProjectID must be 6 to 30 characters in length, can only contain lowercase letters, numbers, and hyphens, and must start with a letter, and cannot end with a hyphen", parentID)
+		}
+		return nil
+	}
+
+	regexKey, _ := regexp.Compile(`^[a-zA-Z0-9]([0-9A-Za-z_.-]{0,61}[a-zA-Z0-9])?$`)
+	checkTagKeyFn := func(key string) error {
+		if !regexKey.MatchString(key) {
+			return fmt.Errorf("tag key %q is invalid. Tag key can have a maximum of 63 characters and cannot be empty. Tag key must begin and end with an alphanumeric character, and must contain only uppercase, lowercase alphanumeric characters, and the following special characters `._-`", key)
+		}
+		return nil
+	}
+
+	regexValue, _ := regexp.Compile(`^[a-zA-Z0-9]([0-9A-Za-z_.@%=+:,*#&()\[\]{}\-\s]{0,61}[a-zA-Z0-9])?$`)
+	checkTagValueFn := func(value string) error {
+		if !regexValue.MatchString(value) {
+			return fmt.Errorf("tag value %q is invalid. Tag value can have a maximum of 63 characters and cannot be empty. Tag value must begin and end with an alphanumeric character, and must contain only uppercase, lowercase alphanumeric characters, and the following special characters `_-.@%%=+:,*#&(){}[]` and spaces", value)
+		}
+
+		return nil
+	}
+
+	checkTagParentIDKey := sets.String{}
+	parentIDkeyValueStrings := strings.Split(tags, tagsDelimiter)
+	for _, parentIDkeyValueString := range parentIDkeyValueStrings {
+		parentIDKeyValue := strings.Split(parentIDkeyValueString, tagsParentIDKeyValueDelimiter)
+
+		if len(parentIDKeyValue) != 3 {
+			return nil, fmt.Errorf("tags %q are invalid, correct format: 'parent_id1/key1/value1,parent_id2/key2/value2'", tags)
+		}
+
+		parentIDKeyStr := parentIDkeyValueString[:strings.LastIndex(parentIDkeyValueString, tagsParentIDKeyValueDelimiter)]
+		if checkTagParentIDKey.Has(parentIDKeyStr) {
+			return nil, fmt.Errorf("tag parent_id & key combination %q exists more than once", parentIDKeyStr)
+		}
+		checkTagParentIDKey.Insert(parentIDKeyStr)
+
+		parentID := strings.TrimSpace(parentIDKeyValue[0])
+		if err := checkTagParentIDFn(parentID); err != nil {
+			return nil, err
+		}
+
+		key := strings.TrimSpace(parentIDKeyValue[1])
+		if err := checkTagKeyFn(key); err != nil {
+			return nil, err
+		}
+
+		value := strings.TrimSpace(parentIDKeyValue[2])
+		if err := checkTagValueFn(value); err != nil {
+			return nil, err
+		}
+
+		tagsSlice = append(tagsSlice, strings.TrimSpace(parentIDkeyValueString))
+	}
+
+	const maxNumberOfTags = 50
+	if len(tagsSlice) > maxNumberOfTags {
+		return nil, fmt.Errorf("more than %d tags is not allowed, given: %d", maxNumberOfTags, len(tagsSlice))
+	}
+
+	return tagsSlice, nil
+}
+
 // ProcessStorageLocations trims and normalizes storage location to lower letters.
 func ProcessStorageLocations(storageLocations string) ([]string, error) {
 	normalizedLoc := strings.ToLower(strings.TrimSpace(storageLocations))
@@ -392,3 +474,16 @@ func isValidDiskEncryptionKmsKey(DiskEncryptionKmsKey string) bool {
 	kmsKeyPattern := regexp.MustCompile("projects/[^/]+/locations/([^/]+)/keyRings/[^/]+/cryptoKeys/[^/]+")
 	return kmsKeyPattern.MatchString(DiskEncryptionKmsKey)
 }
+
+// NewLimiter returns a token bucket based request rate limiter after initializing
+// the passed values for limit, burst (or token bucket) size. If opted for emptyBucket
+// all initial tokens are reserved for the first burst.
+func NewLimiter(limit, burst int, emptyBucket bool) *rate.Limiter {
+	limiter := rate.NewLimiter(rate.Every(time.Second/time.Duration(limit)), burst)
+
+	if emptyBucket {
+		limiter.AllowN(time.Now(), burst)
+	}
+
+	return limiter
+}