Skip to content

Use GCE Alpha API to create zonal disks when Storage Pools is enabled #1524

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Nov 21, 2023
Merged

Use GCE Alpha API to create zonal disks when Storage Pools is enabled #1524

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 11 additions & 2 deletions pkg/common/parameters.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ const (
ParameterKeyProvisionedThroughputOnCreate = "provisioned-throughput-on-create"
ParameterAvailabilityClass = "availability-class"
ParameterKeyEnableConfidentialCompute = "enable-confidential-storage"
ParameterKeyStoragePools = "storage-pools"

// Parameters for VolumeSnapshotClass
ParameterKeyStorageLocations = "storage-locations"
Expand Down Expand Up @@ -94,6 +95,9 @@ type DiskParameters struct {
EnableConfidentialCompute bool
// Default: false
ForceAttach bool
// Values: {[]string}
// Default: ""
StoragePools []string
}

// SnapshotParameters contains normalized and defaulted parameters for snapshots
Expand Down Expand Up @@ -183,12 +187,17 @@ func ExtractAndDefaultParameters(parameters map[string]string, driverName string
if paramEnableConfidentialCompute {
// DiskEncryptionKmsKey is needed to enable confidentialStorage
if val, ok := parameters[ParameterKeyDiskEncryptionKmsKey]; !ok || !isValidDiskEncryptionKmsKey(val) {
return p, fmt.Errorf("Valid %v is required to enbale ConfidentialStorage", ParameterKeyDiskEncryptionKmsKey)
return p, fmt.Errorf("Valid %v is required to enable ConfidentialStorage", ParameterKeyDiskEncryptionKmsKey)
}
}

p.EnableConfidentialCompute = paramEnableConfidentialCompute

case ParameterKeyStoragePools:
storagePools, err := ParseStoragePools(v)
if err != nil {
return p, fmt.Errorf("parameters contain invalid value for %s parameter: %w", ParameterKeyStoragePools, err)
}
p.StoragePools = storagePools
default:
return p, fmt.Errorf("parameters contains invalid option %q", k)
}
Expand Down
13 changes: 13 additions & 0 deletions pkg/common/utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -392,3 +392,16 @@ func isValidDiskEncryptionKmsKey(DiskEncryptionKmsKey string) bool {
kmsKeyPattern := regexp.MustCompile("projects/[^/]+/locations/([^/]+)/keyRings/[^/]+/cryptoKeys/[^/]+")
return kmsKeyPattern.MatchString(DiskEncryptionKmsKey)
}

// TODO(amacaskill): Implement this function.
// ParseStoragePools returns an error if none of the given storagePools
// (delimited by a comma) are in the format
// projects/project/zones/zone/storagePools/storagePool.
func ParseStoragePools(storagePools string) ([]string, error) {
return nil, status.Errorf(codes.Unimplemented, "")
}

// TODO(amacaskill): Implement this function.
func StoragePoolInZone(storagePools []string, zone string) string {
return ""
}
12 changes: 10 additions & 2 deletions pkg/gce-cloud-provider/compute/cloud-disk.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,15 @@ import (
"strings"

"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta"
computealpha "google.golang.org/api/compute/v0.alpha"
computebeta "google.golang.org/api/compute/v0.beta"
computev1 "google.golang.org/api/compute/v1"
)

type CloudDisk struct {
disk *computev1.Disk
betaDisk *computebeta.Disk
disk *computev1.Disk
betaDisk *computebeta.Disk
alphaDisk *computealpha.Disk
}

type CloudDiskType string
Expand All @@ -41,6 +43,12 @@ func CloudDiskFromBeta(disk *computebeta.Disk) *CloudDisk {
}
}

func CloudDiskFromAlpha(disk *computealpha.Disk) *CloudDisk {
return &CloudDisk{
alphaDisk: disk,
}
}

func (d *CloudDisk) LocationType() meta.KeyType {
var zone, region string
switch {
Expand Down
71 changes: 68 additions & 3 deletions pkg/gce-cloud-provider/compute/gce-compute.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (

"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta"
csi "github.com/container-storage-interface/spec/lib/go/csi"
computealpha "google.golang.org/api/compute/v0.alpha"
computebeta "google.golang.org/api/compute/v0.beta"
computev1 "google.golang.org/api/compute/v1"
"google.golang.org/grpc/codes"
Expand Down Expand Up @@ -51,8 +52,10 @@ type GCEAPIVersion string
const (
// V1 key type
GCEAPIVersionV1 GCEAPIVersion = "v1"
// Alpha key type
// Beta key type
GCEAPIVersionBeta GCEAPIVersion = "beta"
// Alpha key type
GCEAPIVersionAlpha GCEAPIVersion = "alpha"
)

// AttachDiskBackoff is backoff used to wait for AttachDisk to complete.
Expand Down Expand Up @@ -266,13 +269,16 @@ func (cloud *CloudProvider) GetDisk(ctx context.Context, project string, key *me
if gceAPIVersion == GCEAPIVersionBeta {
disk, err := cloud.getZonalBetaDiskOrError(ctx, project, key.Zone, key.Name)
return CloudDiskFromBeta(disk), err
} else if gceAPIVersion == GCEAPIVersionAlpha {
disk, err := cloud.getZonalAlphaDiskOrError(ctx, project, key.Zone, key.Name)
return CloudDiskFromAlpha(disk), err
} else {
disk, err := cloud.getZonalDiskOrError(ctx, project, key.Zone, key.Name)
return CloudDiskFromV1(disk), err
}
case meta.Regional:
if gceAPIVersion == GCEAPIVersionBeta {
disk, err := cloud.getRegionalAlphaDiskOrError(ctx, project, key.Region, key.Name)
disk, err := cloud.getRegionalBetaDiskOrError(ctx, project, key.Region, key.Name)
return CloudDiskFromBeta(disk), err
} else {
disk, err := cloud.getRegionalDiskOrError(ctx, project, key.Region, key.Name)
Expand All @@ -299,6 +305,14 @@ func (cloud *CloudProvider) getRegionalDiskOrError(ctx context.Context, project,
return disk, nil
}

func (cloud *CloudProvider) getZonalAlphaDiskOrError(ctx context.Context, project, volumeZone, volumeName string) (*computealpha.Disk, error) {
disk, err := cloud.alphaService.Disks.Get(project, volumeZone, volumeName).Context(ctx).Do()
if err != nil {
return nil, err
}
return disk, nil
}

func (cloud *CloudProvider) getZonalBetaDiskOrError(ctx context.Context, project, volumeZone, volumeName string) (*computebeta.Disk, error) {
disk, err := cloud.betaService.Disks.Get(project, volumeZone, volumeName).Context(ctx).Do()
if err != nil {
Expand All @@ -307,7 +321,7 @@ func (cloud *CloudProvider) getZonalBetaDiskOrError(ctx context.Context, project
return disk, nil
}

func (cloud *CloudProvider) getRegionalAlphaDiskOrError(ctx context.Context, project, volumeRegion, volumeName string) (*computebeta.Disk, error) {
func (cloud *CloudProvider) getRegionalBetaDiskOrError(ctx context.Context, project, volumeRegion, volumeName string) (*computebeta.Disk, error) {
disk, err := cloud.betaService.RegionDisks.Get(project, volumeRegion, volumeName).Context(ctx).Do()
if err != nil {
return nil, err
Expand Down Expand Up @@ -440,6 +454,37 @@ func convertV1DiskToBetaDisk(v1Disk *computev1.Disk, provisionedThroughputOnCrea
return betaDisk
}

func convertV1DiskToAlphaDisk(v1Disk *computev1.Disk, provisionedThroughputOnCreate int64, storagePool string) *computealpha.Disk {
// Note: this is an incomplete list. It only includes the fields we use for disk creation.
alphaDisk := &computealpha.Disk{
Name: v1Disk.Name,
SizeGb: v1Disk.SizeGb,
Description: v1Disk.Description,
Type: v1Disk.Type,
SourceSnapshot: v1Disk.SourceSnapshot,
SourceImage: v1Disk.SourceImage,
SourceImageId: v1Disk.SourceImageId,
SourceSnapshotId: v1Disk.SourceSnapshotId,
SourceDisk: v1Disk.SourceDisk,
ReplicaZones: v1Disk.ReplicaZones,
Zone: v1Disk.Zone,
Region: v1Disk.Region,
Status: v1Disk.Status,
SelfLink: v1Disk.SelfLink,
}
if v1Disk.ProvisionedIops > 0 {
alphaDisk.ProvisionedIops = v1Disk.ProvisionedIops
}
if provisionedThroughputOnCreate > 0 {
alphaDisk.ProvisionedThroughput = provisionedThroughputOnCreate
}
if storagePool != "" {
alphaDisk.StoragePool = storagePool
}

return alphaDisk
}

func (cloud *CloudProvider) insertRegionalDisk(
ctx context.Context,
project string,
Expand Down Expand Up @@ -572,6 +617,10 @@ func (cloud *CloudProvider) insertZonalDisk(
if multiWriter || containsBetaDiskType(hyperdiskTypes, params.DiskType) {
gceAPIVersion = GCEAPIVersionBeta
}
storagePoolsEnabled := params.StoragePools != nil
if storagePoolsEnabled {
gceAPIVersion = GCEAPIVersionAlpha
}

diskToCreate := &computev1.Disk{
Name: volKey.Name,
Expand Down Expand Up @@ -618,6 +667,22 @@ func (cloud *CloudProvider) insertZonalDisk(
if insertOp != nil {
opName = insertOp.Name
}
} else if gceAPIVersion == GCEAPIVersionAlpha {
var insertOp *computealpha.Operation
var storagePool string
if storagePoolsEnabled {
storagePool = common.StoragePoolInZone(params.StoragePools, diskToCreate.Zone)
if storagePool == "" {
return status.Errorf(codes.InvalidArgument, "cannot create disk in zone %q: no Storage Pools exist in zone", diskToCreate.Zone)
}
}
alphaDiskToCreate := convertV1DiskToAlphaDisk(diskToCreate, params.ProvisionedThroughputOnCreate, storagePool)
alphaDiskToCreate.MultiWriter = multiWriter
alphaDiskToCreate.EnableConfidentialCompute = params.EnableConfidentialCompute
insertOp, err = cloud.alphaService.Disks.Insert(project, volKey.Zone, alphaDiskToCreate).Context(ctx).Do()
if insertOp != nil {
opName = insertOp.Name
}
} else {
var insertOp *computev1.Operation
insertOp, err = cloud.service.Disks.Insert(project, volKey.Zone, diskToCreate).Context(ctx).Do()
Expand Down
45 changes: 36 additions & 9 deletions pkg/gce-cloud-provider/compute/gce.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (

"cloud.google.com/go/compute/metadata"
"golang.org/x/oauth2"
computealpha "google.golang.org/api/compute/v0.alpha"
computebeta "google.golang.org/api/compute/v0.beta"
"google.golang.org/api/compute/v1"
"google.golang.org/api/googleapi"
Expand All @@ -49,10 +50,11 @@ const (
)

type CloudProvider struct {
service *compute.Service
betaService *computebeta.Service
project string
zone string
service *compute.Service
betaService *computebeta.Service
alphaService *computealpha.Service
project string
zone string

zonesCache map[string][]string
}
Expand Down Expand Up @@ -95,17 +97,23 @@ func CreateCloudProvider(ctx context.Context, vendorVersion string, configPath s
return nil, err
}

alphasvc, err := createAlphaCloudService(ctx, vendorVersion, tokenSource, computeEndpoint)
if err != nil {
return nil, err
}

project, zone, err := getProjectAndZone(configFile)
if err != nil {
return nil, fmt.Errorf("Failed getting Project and Zone: %w", err)
}

return &CloudProvider{
service: svc,
betaService: betasvc,
project: project,
zone: zone,
zonesCache: make(map[string]([]string)),
service: svc,
betaService: betasvc,
alphaService: alphasvc,
project: project,
zone: zone,
zonesCache: make(map[string]([]string)),
}, nil

}
Expand Down Expand Up @@ -175,6 +183,25 @@ func createBetaCloudService(ctx context.Context, vendorVersion string, tokenSour
return service, nil
}

func createAlphaCloudService(ctx context.Context, vendorVersion string, tokenSource oauth2.TokenSource, computeEndpoint string) (*computealpha.Service, error) {
client, err := newOauthClient(ctx, tokenSource)
if err != nil {
return nil, err
}

computeOpts := []option.ClientOption{option.WithHTTPClient(client)}
if computeEndpoint != "" {
alphaEndpoint := fmt.Sprintf("%s/compute/alpha/", computeEndpoint)
computeOpts = append(computeOpts, option.WithEndpoint(alphaEndpoint))
}
service, err := computealpha.NewService(ctx, computeOpts...)
if err != nil {
return nil, err
}
service.UserAgent = fmt.Sprintf("GCE CSI Driver/%s (%s %s)", vendorVersion, runtime.GOOS, runtime.GOARCH)
return service, nil
}

func createCloudService(ctx context.Context, vendorVersion string, tokenSource oauth2.TokenSource, computeEndpoint string) (*compute.Service, error) {
svc, err := createCloudServiceWithDefaultServiceAccount(ctx, vendorVersion, tokenSource, computeEndpoint)
return svc, err
Expand Down