Skip to content

Commit 5b8b082

Browse files
davidz627davidz627
committed
Add Cloud KMS encrypted disk lifecycle test
1 parent df44c92 commit 5b8b082

File tree

4 files changed

+221
-20
lines changed

4 files changed

+221
-20
lines changed

‎test/e2e/tests/multi_zone_e2e_test.go

+46-17
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ limitations under the License.
1515
package tests
1616

1717
import (
18+
"fmt"
1819
"path/filepath"
1920
"strings"
2021

@@ -143,68 +144,96 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() {
143144

144145
})
145146

146-
func testAttachWriteReadDetach(volID string, volName string, instance *remote.InstanceInfo, client *remote.CsiClient, readOnly bool) {
147+
func testAttachWriteReadDetach(volID string, volName string, instance *remote.InstanceInfo, client *remote.CsiClient, readOnly bool) error {
147148
var err error
148149

149150
glog.Infof("Starting testAttachWriteReadDetach with volume %v node %v with readonly %v\n", volID, instance.GetNodeID(), readOnly)
150151
// Attach Disk
151152
err = client.ControllerPublishVolume(volID, instance.GetNodeID())
152-
Expect(err).To(BeNil(), "ControllerPublishVolume failed with error for disk %v on node %v", volID, instance.GetNodeID())
153+
if err != nil {
154+
return fmt.Errorf("ControllerPublishVolume failed with error for disk %v on node %v: %v", volID, instance.GetNodeID(), err)
155+
}
153156

154157
defer func() {
155-
156158
// Detach Disk
157159
err = client.ControllerUnpublishVolume(volID, instance.GetNodeID())
158-
Expect(err).To(BeNil(), "ControllerUnpublishVolume failed with error")
160+
if err != nil {
161+
glog.Errorf("Failed to detach disk: %v", err)
162+
}
163+
159164
}()
160165

161166
// Stage Disk
162167
stageDir := filepath.Join("/tmp/", volName, "stage")
163-
client.NodeStageVolume(volID, stageDir)
164-
Expect(err).To(BeNil(), "NodeStageVolume failed with error")
168+
err = client.NodeStageVolume(volID, stageDir)
169+
if err != nil {
170+
return fmt.Errorf("NodeStageVolume failed with error: %v", err)
171+
}
165172

166173
defer func() {
167174
// Unstage Disk
168175
err = client.NodeUnstageVolume(volID, stageDir)
169-
Expect(err).To(BeNil(), "NodeUnstageVolume failed with error")
170-
err = testutils.RmAll(instance, filepath.Join("/tmp/", volName))
171-
Expect(err).To(BeNil(), "Failed to remove temp directory")
176+
if err != nil {
177+
glog.Errorf("Failed to unstage volume: %v", err)
178+
}
179+
fp := filepath.Join("/tmp/", volName)
180+
err = testutils.RmAll(instance, fp)
181+
if err != nil {
182+
glog.Errorf("Failed to rm file path %s: %v", fp, err)
183+
}
172184
}()
173185

174186
// Mount Disk
175187
publishDir := filepath.Join("/tmp/", volName, "mount")
176188
err = client.NodePublishVolume(volID, stageDir, publishDir)
177-
Expect(err).To(BeNil(), "NodePublishVolume failed with error")
189+
if err != nil {
190+
return fmt.Errorf("NodePublishVolume failed with error: %v", err)
191+
}
178192
err = testutils.ForceChmod(instance, filepath.Join("/tmp/", volName), "777")
179-
Expect(err).To(BeNil(), "Chmod failed with error")
193+
if err != nil {
194+
return fmt.Errorf("Chmod failed with error: %v", err)
195+
}
180196
testFileContents := "test"
181197
if !readOnly {
182198
// Write a file
183199
testFile := filepath.Join(publishDir, "testfile")
184200
err = testutils.WriteFile(instance, testFile, testFileContents)
185-
Expect(err).To(BeNil(), "Failed to write file")
201+
if err != nil {
202+
return fmt.Errorf("Failed to write file: %v", err)
203+
}
186204
}
187205

188206
// Unmount Disk
189207
err = client.NodeUnpublishVolume(volID, publishDir)
190-
Expect(err).To(BeNil(), "NodeUnpublishVolume failed with error")
208+
if err != nil {
209+
return fmt.Errorf("NodeUnpublishVolume failed with error: %v", err)
210+
}
191211

192212
// Mount disk somewhere else
193213
secondPublishDir := filepath.Join("/tmp/", volName, "secondmount")
194214
err = client.NodePublishVolume(volID, stageDir, secondPublishDir)
195-
Expect(err).To(BeNil(), "NodePublishVolume failed with error")
215+
if err != nil {
216+
return fmt.Errorf("NodePublishVolume failed with error: %v", err)
217+
}
196218
err = testutils.ForceChmod(instance, filepath.Join("/tmp/", volName), "777")
197-
Expect(err).To(BeNil(), "Chmod failed with error")
219+
if err != nil {
220+
return fmt.Errorf("Chmod failed with error: %v", err)
221+
}
198222

199223
// Read File
200224
secondTestFile := filepath.Join(secondPublishDir, "testfile")
201225
readContents, err := testutils.ReadFile(instance, secondTestFile)
202-
Expect(err).To(BeNil(), "ReadFile failed with error")
226+
if err != nil {
227+
return fmt.Errorf("ReadFile failed with error: %v", err)
228+
}
203229
Expect(strings.TrimSpace(string(readContents))).To(Equal(testFileContents))
204230

205231
// Unmount Disk
206232
err = client.NodeUnpublishVolume(volID, secondPublishDir)
207-
Expect(err).To(BeNil(), "NodeUnpublishVolume failed with error")
233+
if err != nil {
234+
return fmt.Errorf("NodeUnpublishVolume failed with error: %v", err)
235+
}
208236

209237
glog.Infof("Completed testAttachWriteReadDetach with volume %v node %v\n", volID, instance.GetNodeID())
238+
return nil
210239
}

‎test/e2e/tests/setup_e2e_test.go

+7
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,14 @@ limitations under the License.
1515
package tests
1616

1717
import (
18+
"context"
1819
"flag"
1920
"fmt"
2021
"math/rand"
2122
"testing"
2223
"time"
2324

25+
cloudkms "cloud.google.com/go/kms/apiv1"
2426
"github.com/golang/glog"
2527
. "github.com/onsi/ginkgo"
2628
. "github.com/onsi/gomega"
@@ -39,6 +41,7 @@ var (
3941
testContexts = []*remote.TestContext{}
4042
computeService *compute.Service
4143
betaComputeService *computebeta.Service
44+
kmsClient *cloudkms.KeyManagementClient
4245
)
4346

4447
func TestE2E(t *testing.T) {
@@ -62,6 +65,10 @@ var _ = BeforeSuite(func() {
6265
betaComputeService, err = remote.GetBetaComputeClient()
6366
Expect(err).To(BeNil())
6467

68+
// Create the KMS client.
69+
kmsClient, err = cloudkms.NewKeyManagementClient(context.Background())
70+
Expect(err).To(BeNil())
71+
6572
if *runInProw {
6673
*project, *serviceAccount = testutils.SetupProwConfig("gce-project")
6774
}

‎test/e2e/tests/single_zone_e2e_test.go

+167-2
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ limitations under the License.
1515
package tests
1616

1717
import (
18+
"context"
19+
"fmt"
1820
"strings"
1921
"time"
2022

@@ -26,6 +28,10 @@ import (
2628
csi "github.com/container-storage-interface/spec/lib/go/csi"
2729
. "github.com/onsi/ginkgo"
2830
. "github.com/onsi/gomega"
31+
32+
"google.golang.org/api/iterator"
33+
kmspb "google.golang.org/genproto/googleapis/cloud/kms/v1"
34+
fieldmask "google.golang.org/genproto/protobuf/field_mask"
2935
)
3036

3137
const (
@@ -77,7 +83,8 @@ var _ = Describe("GCE PD CSI Driver", func() {
7783
}()
7884

7985
// Attach Disk
80-
testAttachWriteReadDetach(volID, volName, instance, client, false /* readOnly */)
86+
err = testAttachWriteReadDetach(volID, volName, instance, client, false /* readOnly */)
87+
Expect(err).To(BeNil(), "Failed to go through volume lifecycle")
8188

8289
})
8390

@@ -151,7 +158,8 @@ var _ = Describe("GCE PD CSI Driver", func() {
151158
}()
152159

153160
// Attach Disk
154-
testAttachWriteReadDetach(underSpecifiedID, volName, instance, client, false /* readOnly */)
161+
err = testAttachWriteReadDetach(underSpecifiedID, volName, instance, client, false /* readOnly */)
162+
Expect(err).To(BeNil(), "Failed to go through volume lifecycle")
155163

156164
})
157165

@@ -294,6 +302,163 @@ var _ = Describe("GCE PD CSI Driver", func() {
294302
}()
295303
})
296304

305+
It("Should create CMEK key, go through volume lifecycle, validate behavior on key revoke and restore", func() {
306+
ctx := context.Background()
307+
Expect(testContexts).ToNot(BeEmpty())
308+
testContext := getRandomTestContext()
309+
310+
controllerInstance := testContext.Instance
311+
controllerClient := testContext.Client
312+
313+
p, z, _ := controllerInstance.GetIdentity()
314+
locationID := "global"
315+
316+
// The resource name of the key rings.
317+
parentName := fmt.Sprintf("projects/%s/locations/%s", p, locationID)
318+
keyRingId := "gce-pd-csi-test-ring"
319+
320+
// Create KeyRing
321+
ringReq := &kmspb.CreateKeyRingRequest{
322+
Parent: parentName,
323+
KeyRingId: keyRingId,
324+
}
325+
keyRing, err := kmsClient.CreateKeyRing(ctx, ringReq)
326+
if !gce.IsGCEError(err, "alreadyExists") {
327+
getKeyRingReq := &kmspb.GetKeyRingRequest{
328+
Name: fmt.Sprintf("%s/keyRings/%s", parentName, keyRingId),
329+
}
330+
keyRing, err = kmsClient.GetKeyRing(ctx, getKeyRingReq)
331+
332+
}
333+
Expect(err).To(BeNil(), "Failed to create or get key ring %v", keyRingId)
334+
335+
// Create CryptoKey in KeyRing
336+
keyId := "test-key-" + string(uuid.NewUUID())
337+
keyReq := &kmspb.CreateCryptoKeyRequest{
338+
Parent: keyRing.Name,
339+
CryptoKeyId: keyId,
340+
CryptoKey: &kmspb.CryptoKey{
341+
Purpose: kmspb.CryptoKey_ENCRYPT_DECRYPT,
342+
VersionTemplate: &kmspb.CryptoKeyVersionTemplate{
343+
Algorithm: kmspb.CryptoKeyVersion_GOOGLE_SYMMETRIC_ENCRYPTION,
344+
},
345+
},
346+
}
347+
key, err := kmsClient.CreateCryptoKey(ctx, keyReq)
348+
Expect(err).To(BeNil(), "Failed to create crypto key %v in key ring %v", keyId, keyRing.Name)
349+
350+
keyVersions := []string{}
351+
keyVersionReq := &kmspb.ListCryptoKeyVersionsRequest{
352+
Parent: key.Name,
353+
}
354+
355+
it := kmsClient.ListCryptoKeyVersions(ctx, keyVersionReq)
356+
357+
for {
358+
keyVersion, err := it.Next()
359+
if err == iterator.Done {
360+
break
361+
}
362+
Expect(err).To(BeNil(), "Failed to list crypto key versions")
363+
364+
keyVersions = append(keyVersions, keyVersion.Name)
365+
}
366+
367+
// Defer deletion of all key versions
368+
// https://cloud.google.com/kms/docs/destroy-restore
369+
// Temporarily disable revokation of CMEK key because of test infra CI permissions
370+
/*
371+
defer func() {
372+
373+
for _, keyVersion := range keyVersions {
374+
destroyKeyReq := &kmspb.DestroyCryptoKeyVersionRequest{
375+
Name: keyVersion,
376+
}
377+
_, err = kmsClient.DestroyCryptoKeyVersion(ctx, destroyKeyReq)
378+
Expect(err).To(BeNil(), "Failed to destroy crypto key version: %v", keyVersion)
379+
}
380+
381+
}()
382+
*/
383+
384+
// Go through volume lifecycle using CMEK-ed PD
385+
// Create Disk
386+
volName := testNamePrefix + string(uuid.NewUUID())
387+
volID, err := controllerClient.CreateVolume(volName, map[string]string{
388+
common.ParameterKeyDiskEncryptionKmsKey: key.Name,
389+
}, defaultSizeGb,
390+
&csi.TopologyRequirement{
391+
Requisite: []*csi.Topology{
392+
{
393+
Segments: map[string]string{common.TopologyKeyZone: z},
394+
},
395+
},
396+
})
397+
Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err)
398+
399+
// Validate Disk Created
400+
cloudDisk, err := computeService.Disks.Get(p, z, volName).Do()
401+
Expect(err).To(BeNil(), "Could not get disk from cloud directly")
402+
Expect(cloudDisk.Type).To(ContainSubstring(standardDiskType))
403+
Expect(cloudDisk.Status).To(Equal(readyState))
404+
Expect(cloudDisk.SizeGb).To(Equal(defaultSizeGb))
405+
Expect(cloudDisk.Name).To(Equal(volName))
406+
407+
defer func() {
408+
// Delete Disk
409+
err = controllerClient.DeleteVolume(volID)
410+
Expect(err).To(BeNil(), "DeleteVolume failed")
411+
412+
// Validate Disk Deleted
413+
_, err = computeService.Disks.Get(p, z, volName).Do()
414+
Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found")
415+
}()
416+
417+
// Test disk works
418+
err = testAttachWriteReadDetach(volID, volName, controllerInstance, controllerClient, false /* readOnly */)
419+
Expect(err).To(BeNil(), "Failed to go through volume lifecycle before revoking CMEK key")
420+
421+
// Revoke CMEK key
422+
// https://cloud.google.com/kms/docs/enable-disable
423+
424+
for _, keyVersion := range keyVersions {
425+
disableReq := &kmspb.UpdateCryptoKeyVersionRequest{
426+
CryptoKeyVersion: &kmspb.CryptoKeyVersion{
427+
Name: keyVersion,
428+
State: kmspb.CryptoKeyVersion_DISABLED,
429+
},
430+
UpdateMask: &fieldmask.FieldMask{
431+
Paths: []string{"state"},
432+
},
433+
}
434+
_, err = kmsClient.UpdateCryptoKeyVersion(ctx, disableReq)
435+
Expect(err).To(BeNil(), "Failed to disable crypto key")
436+
}
437+
438+
// Make sure attach of PD fails
439+
err = testAttachWriteReadDetach(volID, volName, controllerInstance, controllerClient, false /* readOnly */)
440+
Expect(err).ToNot(BeNil(), "Volume lifecycle should have failed, but succeeded")
441+
442+
// Restore CMEK key
443+
for _, keyVersion := range keyVersions {
444+
enableReq := &kmspb.UpdateCryptoKeyVersionRequest{
445+
CryptoKeyVersion: &kmspb.CryptoKeyVersion{
446+
Name: keyVersion,
447+
State: kmspb.CryptoKeyVersion_ENABLED,
448+
},
449+
UpdateMask: &fieldmask.FieldMask{
450+
Paths: []string{"state"},
451+
},
452+
}
453+
_, err = kmsClient.UpdateCryptoKeyVersion(ctx, enableReq)
454+
Expect(err).To(BeNil(), "Failed to enable crypto key")
455+
}
456+
457+
// Make sure attach of PD succeeds
458+
err = testAttachWriteReadDetach(volID, volName, controllerInstance, controllerClient, false /* readOnly */)
459+
Expect(err).To(BeNil(), "Failed to go through volume lifecycle after restoring CMEK key")
460+
})
461+
297462
It("Should create and delete snapshot for RePD in two zones ", func() {
298463
Expect(testContexts).ToNot(BeEmpty())
299464
testContext := getRandomTestContext()

‎test/run-e2e-local.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,4 +5,4 @@ set -o errexit
55

66
readonly PKGDIR=sigs.k8s.io/gcp-compute-persistent-disk-csi-driver
77

8-
ginkgo -v "test/e2e/tests" --logtostderr -- --project ${PROJECT} --service-account ${IAM_NAME} --v=4
8+
ginkgo -v -focus=CMEK "test/e2e/tests" --logtostderr -- --project ${PROJECT} --service-account ${IAM_NAME} --v=4

0 commit comments

Comments
 (0)