Add Cloud KMS encrypted disk lifecycle test

Author: davidz627

test/e2e/tests/multi_zone_e2e_test.go

@@ -15,6 +15,7 @@ limitations under the License.
 package tests
 
 import (
+	"fmt"
 	"path/filepath"
 	"strings"
 
@@ -143,68 +144,85 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() {
 
 })
 
-func testAttachWriteReadDetach(volID string, volName string, instance *remote.InstanceInfo, client *remote.CsiClient, readOnly bool) {
+func testAttachWriteReadDetach(volID string, volName string, instance *remote.InstanceInfo, client *remote.CsiClient, readOnly bool) error {
 	var err error
 
 	glog.Infof("Starting testAttachWriteReadDetach with volume %v node %v with readonly %v\n", volID, instance.GetNodeID(), readOnly)
 	// Attach Disk
 	err = client.ControllerPublishVolume(volID, instance.GetNodeID())
-	Expect(err).To(BeNil(), "ControllerPublishVolume failed with error for disk %v on node %v", volID, instance.GetNodeID())
+	if err != nil {
+		return fmt.Errorf("ControllerPublishVolume failed with error for disk %v on node %v: %v", volID, instance.GetNodeID(), err)
+	}
 
 	defer func() {
-
 		// Detach Disk
-		err = client.ControllerUnpublishVolume(volID, instance.GetNodeID())
-		Expect(err).To(BeNil(), "ControllerUnpublishVolume failed with error")
+		_ = client.ControllerUnpublishVolume(volID, instance.GetNodeID())
 	}()
 
 	// Stage Disk
 	stageDir := filepath.Join("/tmp/", volName, "stage")
-	client.NodeStageVolume(volID, stageDir)
-	Expect(err).To(BeNil(), "NodeStageVolume failed with error")
+	err = client.NodeStageVolume(volID, stageDir)
+	if err != nil {
+		return fmt.Errorf("NodeStageVolume failed with error: %v", err)
+	}
 
 	defer func() {
 		// Unstage Disk
-		err = client.NodeUnstageVolume(volID, stageDir)
-		Expect(err).To(BeNil(), "NodeUnstageVolume failed with error")
-		err = testutils.RmAll(instance, filepath.Join("/tmp/", volName))
-		Expect(err).To(BeNil(), "Failed to remove temp directory")
+		_ = client.NodeUnstageVolume(volID, stageDir)
+		_ = testutils.RmAll(instance, filepath.Join("/tmp/", volName))
 	}()
 
 	// Mount Disk
 	publishDir := filepath.Join("/tmp/", volName, "mount")
 	err = client.NodePublishVolume(volID, stageDir, publishDir)
-	Expect(err).To(BeNil(), "NodePublishVolume failed with error")
+	if err != nil {
+		return fmt.Errorf("NodePublishVolume failed with error: %v", err)
+	}
 	err = testutils.ForceChmod(instance, filepath.Join("/tmp/", volName), "777")
-	Expect(err).To(BeNil(), "Chmod failed with error")
+	if err != nil {
+		return fmt.Errorf("Chmod failed with error: %v", err)
+	}
 	testFileContents := "test"
 	if !readOnly {
 		// Write a file
 		testFile := filepath.Join(publishDir, "testfile")
 		err = testutils.WriteFile(instance, testFile, testFileContents)
-		Expect(err).To(BeNil(), "Failed to write file")
+		if err != nil {
+			return fmt.Errorf("Failed to write file: %v", err)
+		}
 	}
 
 	// Unmount Disk
 	err = client.NodeUnpublishVolume(volID, publishDir)
-	Expect(err).To(BeNil(), "NodeUnpublishVolume failed with error")
+	if err != nil {
+		return fmt.Errorf("NodeUnpublishVolume failed with error: %v", err)
+	}
 
 	// Mount disk somewhere else
 	secondPublishDir := filepath.Join("/tmp/", volName, "secondmount")
 	err = client.NodePublishVolume(volID, stageDir, secondPublishDir)
-	Expect(err).To(BeNil(), "NodePublishVolume failed with error")
+	if err != nil {
+		return fmt.Errorf("NodePublishVolume failed with error: %v", err)
+	}
 	err = testutils.ForceChmod(instance, filepath.Join("/tmp/", volName), "777")
-	Expect(err).To(BeNil(), "Chmod failed with error")
+	if err != nil {
+		return fmt.Errorf("Chmod failed with error: %v", err)
+	}
 
 	// Read File
 	secondTestFile := filepath.Join(secondPublishDir, "testfile")
 	readContents, err := testutils.ReadFile(instance, secondTestFile)
-	Expect(err).To(BeNil(), "ReadFile failed with error")
+	if err != nil {
+		return fmt.Errorf("ReadFile failed with error: %v", err)
+	}
 	Expect(strings.TrimSpace(string(readContents))).To(Equal(testFileContents))
 
 	// Unmount Disk
 	err = client.NodeUnpublishVolume(volID, secondPublishDir)
-	Expect(err).To(BeNil(), "NodeUnpublishVolume failed with error")
+	if err != nil {
+		return fmt.Errorf("NodeUnpublishVolume failed with error: %v", err)
+	}
 
 	glog.Infof("Completed testAttachWriteReadDetach with volume %v node %v\n", volID, instance.GetNodeID())
+	return nil
 }

test/e2e/tests/setup_e2e_test.go

@@ -15,12 +15,14 @@ limitations under the License.
 package tests
 
 import (
+	"context"
 	"flag"
 	"fmt"
 	"math/rand"
 	"testing"
 	"time"
 
+	cloudkms "cloud.google.com/go/kms/apiv1"
 	"github.com/golang/glog"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -39,6 +41,7 @@ var (
 	testContexts       = []*remote.TestContext{}
 	computeService     *compute.Service
 	betaComputeService *computebeta.Service
+	kmsClient          *cloudkms.KeyManagementClient
 )
 
 func TestE2E(t *testing.T) {
@@ -62,6 +65,10 @@ var _ = BeforeSuite(func() {
 	betaComputeService, err = remote.GetBetaComputeClient()
 	Expect(err).To(BeNil())
 
+	// Create the KMS client.
+	kmsClient, err = cloudkms.NewKeyManagementClient(context.Background())
+	Expect(err).To(BeNil())
+
 	if *runInProw {
 		*project, *serviceAccount = testutils.SetupProwConfig("gce-project")
 	}

test/e2e/tests/single_zone_e2e_test.go

@@ -15,6 +15,8 @@ limitations under the License.
 package tests
 
 import (
+	"context"
+	"fmt"
 	"strings"
 	"time"
 
@@ -26,6 +28,10 @@ import (
 	csi "github.com/container-storage-interface/spec/lib/go/csi"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+
+	"google.golang.org/api/iterator"
+	kmspb "google.golang.org/genproto/googleapis/cloud/kms/v1"
+	fieldmask "google.golang.org/genproto/protobuf/field_mask"
 )
 
 const (
@@ -77,7 +83,8 @@ var _ = Describe("GCE PD CSI Driver", func() {
 		}()
 
 		// Attach Disk
-		testAttachWriteReadDetach(volID, volName, instance, client, false /* readOnly */)
+		err = testAttachWriteReadDetach(volID, volName, instance, client, false /* readOnly */)
+		Expect(err).To(BeNil(), "Failed to go through volume lifecycle")
 
 	})
 
@@ -151,7 +158,8 @@ var _ = Describe("GCE PD CSI Driver", func() {
 		}()
 
 		// Attach Disk
-		testAttachWriteReadDetach(underSpecifiedID, volName, instance, client, false /* readOnly */)
+		err = testAttachWriteReadDetach(underSpecifiedID, volName, instance, client, false /* readOnly */)
+		Expect(err).To(BeNil(), "Failed to go through volume lifecycle")
 
 	})
 
@@ -294,6 +302,165 @@ var _ = Describe("GCE PD CSI Driver", func() {
 		}()
 	})
 
+	It("Should create CMEK key, go through volume lifecycle, validate behavior on key revoke and restore", func() {
+		ctx := context.Background()
+		Expect(testContexts).ToNot(BeEmpty())
+		testContext := getRandomTestContext()
+
+		controllerInstance := testContext.Instance
+		controllerClient := testContext.Client
+
+		p, z, _ := controllerInstance.GetIdentity()
+		locationID := "global"
+
+		// The resource name of the key rings.
+		parentName := fmt.Sprintf("projects/%s/locations/%s", p, locationID)
+		keyRingId := "gce-pd-csi-test-ring"
+
+		// Create KeyRing
+		ringReq := &kmspb.CreateKeyRingRequest{
+			Parent:    parentName,
+			KeyRingId: keyRingId,
+		}
+		keyRing, err := kmsClient.CreateKeyRing(ctx, ringReq)
+		if !gce.IsGCEError(err, "alreadyExists") {
+			getKeyRingReq := &kmspb.GetKeyRingRequest{
+				Name: fmt.Sprintf("%s/keyRings/%s", parentName, keyRingId),
+			}
+			keyRing, err = kmsClient.GetKeyRing(ctx, getKeyRingReq)
+
+		}
+		Expect(err).To(BeNil(), "Failed to create or get key ring %v", keyRingId)
+
+		// Create CryptoKey in KeyRing
+		keyId := "test-key-" + string(uuid.NewUUID())
+		keyReq := &kmspb.CreateCryptoKeyRequest{
+			Parent:      keyRing.Name,
+			CryptoKeyId: keyId,
+			CryptoKey: &kmspb.CryptoKey{
+				Purpose: kmspb.CryptoKey_ENCRYPT_DECRYPT,
+				VersionTemplate: &kmspb.CryptoKeyVersionTemplate{
+					Algorithm: kmspb.CryptoKeyVersion_GOOGLE_SYMMETRIC_ENCRYPTION,
+				},
+			},
+		}
+		key, err := kmsClient.CreateCryptoKey(ctx, keyReq)
+		if !gce.IsGCEError(err, "alreadyExists") {
+			getKeyReq := &kmspb.GetCryptoKeyRequest{
+				Name: fmt.Sprintf("%s/cryptoKeys/%s", keyRing.Name, keyId),
+			}
+			key, err = kmsClient.GetCryptoKey(ctx, getKeyReq)
+
+		}
+		Expect(err).To(BeNil(), "Failed to create crypto key %v in key ring %v", keyId, keyRing.Name)
+
+		keyVersions := []string{}
+		keyVersionReq := &kmspb.ListCryptoKeyVersionsRequest{
+			Parent: key.Name,
+		}
+
+		it := kmsClient.ListCryptoKeyVersions(ctx, keyVersionReq)
+
+		for {
+			keyVersion, err := it.Next()
+			if err == iterator.Done {
+				break
+			}
+			Expect(err).To(BeNil(), "Failed to list crypto key versions")
+
+			keyVersions = append(keyVersions, keyVersion.Name)
+		}
+
+		// Defer deletion of all key versions
+		// https://cloud.google.com/kms/docs/destroy-restore
+		defer func() {
+			for _, keyVersion := range keyVersions {
+				destroyKeyReq := &kmspb.DestroyCryptoKeyVersionRequest{
+					Name: keyVersion,
+				}
+				_, err = kmsClient.DestroyCryptoKeyVersion(ctx, destroyKeyReq)
+				Expect(err).To(BeNil(), "Failed to destroy crypto key version: %v", keyVersion)
+			}
+
+		}()
+
+		// Go through volume lifecycle using CMEK-ed PD
+		// Create Disk
+		volName := testNamePrefix + string(uuid.NewUUID())
+		volID, err := controllerClient.CreateVolume(volName, map[string]string{
+			common.ParameterKeyDiskEncryptionKmsKey: key.Name,
+		}, defaultSizeGb,
+			&csi.TopologyRequirement{
+				Requisite: []*csi.Topology{
+					{
+						Segments: map[string]string{common.TopologyKeyZone: z},
+					},
+				},
+			})
+		Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err)
+
+		// Validate Disk Created
+		cloudDisk, err := computeService.Disks.Get(p, z, volName).Do()
+		Expect(err).To(BeNil(), "Could not get disk from cloud directly")
+		Expect(cloudDisk.Type).To(ContainSubstring(standardDiskType))
+		Expect(cloudDisk.Status).To(Equal(readyState))
+		Expect(cloudDisk.SizeGb).To(Equal(defaultSizeGb))
+		Expect(cloudDisk.Name).To(Equal(volName))
+
+		defer func() {
+			// Delete Disk
+			controllerClient.DeleteVolume(volID)
+			Expect(err).To(BeNil(), "DeleteVolume failed")
+
+			// Validate Disk Deleted
+			_, err = computeService.Disks.Get(p, z, volName).Do()
+			Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found")
+		}()
+
+		// Test disk works
+		err = testAttachWriteReadDetach(volID, volName, controllerInstance, controllerClient, false /* readOnly */)
+		Expect(err).To(BeNil(), "Failed to go through volume lifecycle before revoking CMEK key")
+
+		// Revoke CMEK key
+		// https://cloud.google.com/kms/docs/enable-disable
+		for _, keyVersion := range keyVersions {
+			disableReq := &kmspb.UpdateCryptoKeyVersionRequest{
+				CryptoKeyVersion: &kmspb.CryptoKeyVersion{
+					Name:  keyVersion,
+					State: kmspb.CryptoKeyVersion_DISABLED,
+				},
+				UpdateMask: &fieldmask.FieldMask{
+					Paths: []string{"state"},
+				},
+			}
+			_, err = kmsClient.UpdateCryptoKeyVersion(ctx, disableReq)
+			Expect(err).To(BeNil(), "Failed to disable crypto key")
+		}
+
+		// Make sure attach of PD fails
+		err = testAttachWriteReadDetach(volID, volName, controllerInstance, controllerClient, false /* readOnly */)
+		Expect(err).ToNot(BeNil(), "Volume lifecycle should have failed, but succeeded")
+
+		// Restore CMEK key
+		for _, keyVersion := range keyVersions {
+			enableReq := &kmspb.UpdateCryptoKeyVersionRequest{
+				CryptoKeyVersion: &kmspb.CryptoKeyVersion{
+					Name:  keyVersion,
+					State: kmspb.CryptoKeyVersion_ENABLED,
+				},
+				UpdateMask: &fieldmask.FieldMask{
+					Paths: []string{"state"},
+				},
+			}
+			_, err = kmsClient.UpdateCryptoKeyVersion(ctx, enableReq)
+			Expect(err).To(BeNil(), "Failed to enable crypto key")
+		}
+
+		// Make sure attach of PD succeeds
+		err = testAttachWriteReadDetach(volID, volName, controllerInstance, controllerClient, false /* readOnly */)
+		Expect(err).To(BeNil(), "Failed to go through volume lifecycle after restoring CMEK key")
+	})
+
 	It("Should create and delete snapshot for RePD in two zones ", func() {
 		Expect(testContexts).ToNot(BeEmpty())
 		testContext := getRandomTestContext()

test/e2e/utils/utils.go

@@ -27,6 +27,7 @@ import (
 	cloudresourcemanager "google.golang.org/api/cloudresourcemanager/v1"
 	boskosclient "k8s.io/test-infra/boskos/client"
 	"k8s.io/test-infra/boskos/common"
+	"k8s.io/utils/exec"
 	remote "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/remote"
 )
 
@@ -128,6 +129,23 @@ func SetupProwConfig(resourceType string) (project, serviceAccount string) {
 	// [PROJECT_NUMBER][email protected]
 	serviceAccount = fmt.Sprintf("%[email protected]", resp.ProjectNumber)
 	glog.Infof("Using project %v and service account %v", project, serviceAccount)
+
+	// Set Cloud KMS permissions on compute service account
+	// TODO: Use the API to make this call instead of exec-ing gcloud command
+	computeSystemSA := fmt.Sprintf("service-%[email protected]", resp.ProjectNumber)
+	_, err = exec.New().Command(
+		"gcloud",
+		"projects",
+		"add-iam-policy-binding",
+		project,
+		"--member",
+		fmt.Sprintf("serviceAccount:%v", computeSystemSA),
+		"--role",
+		"roles/cloudkms.cryptoKeyEncrypterDecrypter").CombinedOutput()
+	if err != nil {
+		glog.Fatalf("Failed to set iam policy binding roles/cloudkms.cryptoKeyEncrypterDecrypter on %s: %v", computeSystemSA, err)
+	}
+
 	return project, serviceAccount
 }