From 3e947e05c4c2d2811728ee3c737548a3bcff757b Mon Sep 17 00:00:00 2001 From: Alexis MacAskill Date: Mon, 20 Nov 2023 18:55:41 +0000 Subject: [PATCH] Use GCE Alpha API to create disks when Storage Pools is enabled --- pkg/common/parameters.go | 13 +++- pkg/common/utils.go | 13 ++++ pkg/gce-cloud-provider/compute/cloud-disk.go | 12 +++- pkg/gce-cloud-provider/compute/gce-compute.go | 71 ++++++++++++++++++- pkg/gce-cloud-provider/compute/gce.go | 45 +++++++++--- 5 files changed, 138 insertions(+), 16 deletions(-) diff --git a/pkg/common/parameters.go b/pkg/common/parameters.go index 4c7e5e441..a9eea224c 100644 --- a/pkg/common/parameters.go +++ b/pkg/common/parameters.go @@ -31,6 +31,7 @@ const ( ParameterKeyProvisionedThroughputOnCreate = "provisioned-throughput-on-create" ParameterAvailabilityClass = "availability-class" ParameterKeyEnableConfidentialCompute = "enable-confidential-storage" + ParameterKeyStoragePools = "storage-pools" // Parameters for VolumeSnapshotClass ParameterKeyStorageLocations = "storage-locations" @@ -94,6 +95,9 @@ type DiskParameters struct { EnableConfidentialCompute bool // Default: false ForceAttach bool + // Values: {[]string} + // Default: "" + StoragePools []string } // SnapshotParameters contains normalized and defaulted parameters for snapshots @@ -183,12 +187,17 @@ func ExtractAndDefaultParameters(parameters map[string]string, driverName string if paramEnableConfidentialCompute { // DiskEncryptionKmsKey is needed to enable confidentialStorage if val, ok := parameters[ParameterKeyDiskEncryptionKmsKey]; !ok || !isValidDiskEncryptionKmsKey(val) { - return p, fmt.Errorf("Valid %v is required to enbale ConfidentialStorage", ParameterKeyDiskEncryptionKmsKey) + return p, fmt.Errorf("Valid %v is required to enable ConfidentialStorage", ParameterKeyDiskEncryptionKmsKey) } } p.EnableConfidentialCompute = paramEnableConfidentialCompute - + case ParameterKeyStoragePools: + storagePools, err := ParseStoragePools(v) + if err != nil { + return p, fmt.Errorf("parameters contain invalid value for %s parameter: %w", ParameterKeyStoragePools, err) + } + p.StoragePools = storagePools default: return p, fmt.Errorf("parameters contains invalid option %q", k) } diff --git a/pkg/common/utils.go b/pkg/common/utils.go index 368a853a4..813e58387 100644 --- a/pkg/common/utils.go +++ b/pkg/common/utils.go @@ -392,3 +392,16 @@ func isValidDiskEncryptionKmsKey(DiskEncryptionKmsKey string) bool { kmsKeyPattern := regexp.MustCompile("projects/[^/]+/locations/([^/]+)/keyRings/[^/]+/cryptoKeys/[^/]+") return kmsKeyPattern.MatchString(DiskEncryptionKmsKey) } + +// TODO(amacaskill): Implement this function. +// ParseStoragePools returns an error if none of the given storagePools +// (delimited by a comma) are in the format +// projects/project/zones/zone/storagePools/storagePool. +func ParseStoragePools(storagePools string) ([]string, error) { + return nil, status.Errorf(codes.Unimplemented, "") +} + +// TODO(amacaskill): Implement this function. +func StoragePoolInZone(storagePools []string, zone string) string { + return "" +} diff --git a/pkg/gce-cloud-provider/compute/cloud-disk.go b/pkg/gce-cloud-provider/compute/cloud-disk.go index 1d59cdd0e..749b43241 100644 --- a/pkg/gce-cloud-provider/compute/cloud-disk.go +++ b/pkg/gce-cloud-provider/compute/cloud-disk.go @@ -18,13 +18,15 @@ import ( "strings" "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta" + computealpha "google.golang.org/api/compute/v0.alpha" computebeta "google.golang.org/api/compute/v0.beta" computev1 "google.golang.org/api/compute/v1" ) type CloudDisk struct { - disk *computev1.Disk - betaDisk *computebeta.Disk + disk *computev1.Disk + betaDisk *computebeta.Disk + alphaDisk *computealpha.Disk } type CloudDiskType string @@ -41,6 +43,12 @@ func CloudDiskFromBeta(disk *computebeta.Disk) *CloudDisk { } } +func CloudDiskFromAlpha(disk *computealpha.Disk) *CloudDisk { + return &CloudDisk{ + alphaDisk: disk, + } +} + func (d *CloudDisk) LocationType() meta.KeyType { var zone, region string switch { diff --git a/pkg/gce-cloud-provider/compute/gce-compute.go b/pkg/gce-cloud-provider/compute/gce-compute.go index e7962cffc..61477bd70 100644 --- a/pkg/gce-cloud-provider/compute/gce-compute.go +++ b/pkg/gce-cloud-provider/compute/gce-compute.go @@ -24,6 +24,7 @@ import ( "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta" csi "github.com/container-storage-interface/spec/lib/go/csi" + computealpha "google.golang.org/api/compute/v0.alpha" computebeta "google.golang.org/api/compute/v0.beta" computev1 "google.golang.org/api/compute/v1" "google.golang.org/grpc/codes" @@ -51,8 +52,10 @@ type GCEAPIVersion string const ( // V1 key type GCEAPIVersionV1 GCEAPIVersion = "v1" - // Alpha key type + // Beta key type GCEAPIVersionBeta GCEAPIVersion = "beta" + // Alpha key type + GCEAPIVersionAlpha GCEAPIVersion = "alpha" ) // AttachDiskBackoff is backoff used to wait for AttachDisk to complete. @@ -266,13 +269,16 @@ func (cloud *CloudProvider) GetDisk(ctx context.Context, project string, key *me if gceAPIVersion == GCEAPIVersionBeta { disk, err := cloud.getZonalBetaDiskOrError(ctx, project, key.Zone, key.Name) return CloudDiskFromBeta(disk), err + } else if gceAPIVersion == GCEAPIVersionAlpha { + disk, err := cloud.getZonalAlphaDiskOrError(ctx, project, key.Zone, key.Name) + return CloudDiskFromAlpha(disk), err } else { disk, err := cloud.getZonalDiskOrError(ctx, project, key.Zone, key.Name) return CloudDiskFromV1(disk), err } case meta.Regional: if gceAPIVersion == GCEAPIVersionBeta { - disk, err := cloud.getRegionalAlphaDiskOrError(ctx, project, key.Region, key.Name) + disk, err := cloud.getRegionalBetaDiskOrError(ctx, project, key.Region, key.Name) return CloudDiskFromBeta(disk), err } else { disk, err := cloud.getRegionalDiskOrError(ctx, project, key.Region, key.Name) @@ -299,6 +305,14 @@ func (cloud *CloudProvider) getRegionalDiskOrError(ctx context.Context, project, return disk, nil } +func (cloud *CloudProvider) getZonalAlphaDiskOrError(ctx context.Context, project, volumeZone, volumeName string) (*computealpha.Disk, error) { + disk, err := cloud.alphaService.Disks.Get(project, volumeZone, volumeName).Context(ctx).Do() + if err != nil { + return nil, err + } + return disk, nil +} + func (cloud *CloudProvider) getZonalBetaDiskOrError(ctx context.Context, project, volumeZone, volumeName string) (*computebeta.Disk, error) { disk, err := cloud.betaService.Disks.Get(project, volumeZone, volumeName).Context(ctx).Do() if err != nil { @@ -307,7 +321,7 @@ func (cloud *CloudProvider) getZonalBetaDiskOrError(ctx context.Context, project return disk, nil } -func (cloud *CloudProvider) getRegionalAlphaDiskOrError(ctx context.Context, project, volumeRegion, volumeName string) (*computebeta.Disk, error) { +func (cloud *CloudProvider) getRegionalBetaDiskOrError(ctx context.Context, project, volumeRegion, volumeName string) (*computebeta.Disk, error) { disk, err := cloud.betaService.RegionDisks.Get(project, volumeRegion, volumeName).Context(ctx).Do() if err != nil { return nil, err @@ -440,6 +454,37 @@ func convertV1DiskToBetaDisk(v1Disk *computev1.Disk, provisionedThroughputOnCrea return betaDisk } +func convertV1DiskToAlphaDisk(v1Disk *computev1.Disk, provisionedThroughputOnCreate int64, storagePool string) *computealpha.Disk { + // Note: this is an incomplete list. It only includes the fields we use for disk creation. + alphaDisk := &computealpha.Disk{ + Name: v1Disk.Name, + SizeGb: v1Disk.SizeGb, + Description: v1Disk.Description, + Type: v1Disk.Type, + SourceSnapshot: v1Disk.SourceSnapshot, + SourceImage: v1Disk.SourceImage, + SourceImageId: v1Disk.SourceImageId, + SourceSnapshotId: v1Disk.SourceSnapshotId, + SourceDisk: v1Disk.SourceDisk, + ReplicaZones: v1Disk.ReplicaZones, + Zone: v1Disk.Zone, + Region: v1Disk.Region, + Status: v1Disk.Status, + SelfLink: v1Disk.SelfLink, + } + if v1Disk.ProvisionedIops > 0 { + alphaDisk.ProvisionedIops = v1Disk.ProvisionedIops + } + if provisionedThroughputOnCreate > 0 { + alphaDisk.ProvisionedThroughput = provisionedThroughputOnCreate + } + if storagePool != "" { + alphaDisk.StoragePool = storagePool + } + + return alphaDisk +} + func (cloud *CloudProvider) insertRegionalDisk( ctx context.Context, project string, @@ -572,6 +617,10 @@ func (cloud *CloudProvider) insertZonalDisk( if multiWriter || containsBetaDiskType(hyperdiskTypes, params.DiskType) { gceAPIVersion = GCEAPIVersionBeta } + storagePoolsEnabled := params.StoragePools != nil + if storagePoolsEnabled { + gceAPIVersion = GCEAPIVersionAlpha + } diskToCreate := &computev1.Disk{ Name: volKey.Name, @@ -618,6 +667,22 @@ func (cloud *CloudProvider) insertZonalDisk( if insertOp != nil { opName = insertOp.Name } + } else if gceAPIVersion == GCEAPIVersionAlpha { + var insertOp *computealpha.Operation + var storagePool string + if storagePoolsEnabled { + storagePool = common.StoragePoolInZone(params.StoragePools, diskToCreate.Zone) + if storagePool == "" { + return status.Errorf(codes.InvalidArgument, "cannot create disk in zone %q: no Storage Pools exist in zone", diskToCreate.Zone) + } + } + alphaDiskToCreate := convertV1DiskToAlphaDisk(diskToCreate, params.ProvisionedThroughputOnCreate, storagePool) + alphaDiskToCreate.MultiWriter = multiWriter + alphaDiskToCreate.EnableConfidentialCompute = params.EnableConfidentialCompute + insertOp, err = cloud.alphaService.Disks.Insert(project, volKey.Zone, alphaDiskToCreate).Context(ctx).Do() + if insertOp != nil { + opName = insertOp.Name + } } else { var insertOp *computev1.Operation insertOp, err = cloud.service.Disks.Insert(project, volKey.Zone, diskToCreate).Context(ctx).Do() diff --git a/pkg/gce-cloud-provider/compute/gce.go b/pkg/gce-cloud-provider/compute/gce.go index 1d9cb051d..e2a998b16 100644 --- a/pkg/gce-cloud-provider/compute/gce.go +++ b/pkg/gce-cloud-provider/compute/gce.go @@ -29,6 +29,7 @@ import ( "cloud.google.com/go/compute/metadata" "golang.org/x/oauth2" + computealpha "google.golang.org/api/compute/v0.alpha" computebeta "google.golang.org/api/compute/v0.beta" "google.golang.org/api/compute/v1" "google.golang.org/api/googleapi" @@ -49,10 +50,11 @@ const ( ) type CloudProvider struct { - service *compute.Service - betaService *computebeta.Service - project string - zone string + service *compute.Service + betaService *computebeta.Service + alphaService *computealpha.Service + project string + zone string zonesCache map[string][]string } @@ -95,17 +97,23 @@ func CreateCloudProvider(ctx context.Context, vendorVersion string, configPath s return nil, err } + alphasvc, err := createAlphaCloudService(ctx, vendorVersion, tokenSource, computeEndpoint) + if err != nil { + return nil, err + } + project, zone, err := getProjectAndZone(configFile) if err != nil { return nil, fmt.Errorf("Failed getting Project and Zone: %w", err) } return &CloudProvider{ - service: svc, - betaService: betasvc, - project: project, - zone: zone, - zonesCache: make(map[string]([]string)), + service: svc, + betaService: betasvc, + alphaService: alphasvc, + project: project, + zone: zone, + zonesCache: make(map[string]([]string)), }, nil } @@ -175,6 +183,25 @@ func createBetaCloudService(ctx context.Context, vendorVersion string, tokenSour return service, nil } +func createAlphaCloudService(ctx context.Context, vendorVersion string, tokenSource oauth2.TokenSource, computeEndpoint string) (*computealpha.Service, error) { + client, err := newOauthClient(ctx, tokenSource) + if err != nil { + return nil, err + } + + computeOpts := []option.ClientOption{option.WithHTTPClient(client)} + if computeEndpoint != "" { + alphaEndpoint := fmt.Sprintf("%s/compute/alpha/", computeEndpoint) + computeOpts = append(computeOpts, option.WithEndpoint(alphaEndpoint)) + } + service, err := computealpha.NewService(ctx, computeOpts...) + if err != nil { + return nil, err + } + service.UserAgent = fmt.Sprintf("GCE CSI Driver/%s (%s %s)", vendorVersion, runtime.GOOS, runtime.GOARCH) + return service, nil +} + func createCloudService(ctx context.Context, vendorVersion string, tokenSource oauth2.TokenSource, computeEndpoint string) (*compute.Service, error) { svc, err := createCloudServiceWithDefaultServiceAccount(ctx, vendorVersion, tokenSource, computeEndpoint) return svc, err