Skip to content

Commit d042fcf

Browse files
davidz627davidz627
committed
Add Cloud KMS encrypted disk lifecycle test
1 parent df44c92 commit d042fcf

File tree

3 files changed

+217
-19
lines changed

3 files changed

+217
-19
lines changed

‎test/e2e/tests/multi_zone_e2e_test.go

+46-17
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ limitations under the License.
1515
package tests
1616

1717
import (
18+
"fmt"
1819
"path/filepath"
1920
"strings"
2021

@@ -143,68 +144,96 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() {
143144

144145
})
145146

146-
func testAttachWriteReadDetach(volID string, volName string, instance *remote.InstanceInfo, client *remote.CsiClient, readOnly bool) {
147+
func testAttachWriteReadDetach(volID string, volName string, instance *remote.InstanceInfo, client *remote.CsiClient, readOnly bool) error {
147148
var err error
148149

149150
glog.Infof("Starting testAttachWriteReadDetach with volume %v node %v with readonly %v\n", volID, instance.GetNodeID(), readOnly)
150151
// Attach Disk
151152
err = client.ControllerPublishVolume(volID, instance.GetNodeID())
152-
Expect(err).To(BeNil(), "ControllerPublishVolume failed with error for disk %v on node %v", volID, instance.GetNodeID())
153+
if err != nil {
154+
return fmt.Errorf("ControllerPublishVolume failed with error for disk %v on node %v: %v", volID, instance.GetNodeID(), err)
155+
}
153156

154157
defer func() {
155-
156158
// Detach Disk
157159
err = client.ControllerUnpublishVolume(volID, instance.GetNodeID())
158-
Expect(err).To(BeNil(), "ControllerUnpublishVolume failed with error")
160+
if err != nil {
161+
glog.Errorf("Failed to detach disk: %v", err)
162+
}
163+
159164
}()
160165

161166
// Stage Disk
162167
stageDir := filepath.Join("/tmp/", volName, "stage")
163-
client.NodeStageVolume(volID, stageDir)
164-
Expect(err).To(BeNil(), "NodeStageVolume failed with error")
168+
err = client.NodeStageVolume(volID, stageDir)
169+
if err != nil {
170+
return fmt.Errorf("NodeStageVolume failed with error: %v", err)
171+
}
165172

166173
defer func() {
167174
// Unstage Disk
168175
err = client.NodeUnstageVolume(volID, stageDir)
169-
Expect(err).To(BeNil(), "NodeUnstageVolume failed with error")
170-
err = testutils.RmAll(instance, filepath.Join("/tmp/", volName))
171-
Expect(err).To(BeNil(), "Failed to remove temp directory")
176+
if err != nil {
177+
glog.Errorf("Failed to unstage volume: %v", err)
178+
}
179+
fp := filepath.Join("/tmp/", volName)
180+
err = testutils.RmAll(instance, fp)
181+
if err != nil {
182+
glog.Errorf("Failed to rm file path %s: %v", fp, err)
183+
}
172184
}()
173185

174186
// Mount Disk
175187
publishDir := filepath.Join("/tmp/", volName, "mount")
176188
err = client.NodePublishVolume(volID, stageDir, publishDir)
177-
Expect(err).To(BeNil(), "NodePublishVolume failed with error")
189+
if err != nil {
190+
return fmt.Errorf("NodePublishVolume failed with error: %v", err)
191+
}
178192
err = testutils.ForceChmod(instance, filepath.Join("/tmp/", volName), "777")
179-
Expect(err).To(BeNil(), "Chmod failed with error")
193+
if err != nil {
194+
return fmt.Errorf("Chmod failed with error: %v", err)
195+
}
180196
testFileContents := "test"
181197
if !readOnly {
182198
// Write a file
183199
testFile := filepath.Join(publishDir, "testfile")
184200
err = testutils.WriteFile(instance, testFile, testFileContents)
185-
Expect(err).To(BeNil(), "Failed to write file")
201+
if err != nil {
202+
return fmt.Errorf("Failed to write file: %v", err)
203+
}
186204
}
187205

188206
// Unmount Disk
189207
err = client.NodeUnpublishVolume(volID, publishDir)
190-
Expect(err).To(BeNil(), "NodeUnpublishVolume failed with error")
208+
if err != nil {
209+
return fmt.Errorf("NodeUnpublishVolume failed with error: %v", err)
210+
}
191211

192212
// Mount disk somewhere else
193213
secondPublishDir := filepath.Join("/tmp/", volName, "secondmount")
194214
err = client.NodePublishVolume(volID, stageDir, secondPublishDir)
195-
Expect(err).To(BeNil(), "NodePublishVolume failed with error")
215+
if err != nil {
216+
return fmt.Errorf("NodePublishVolume failed with error: %v", err)
217+
}
196218
err = testutils.ForceChmod(instance, filepath.Join("/tmp/", volName), "777")
197-
Expect(err).To(BeNil(), "Chmod failed with error")
219+
if err != nil {
220+
return fmt.Errorf("Chmod failed with error: %v", err)
221+
}
198222

199223
// Read File
200224
secondTestFile := filepath.Join(secondPublishDir, "testfile")
201225
readContents, err := testutils.ReadFile(instance, secondTestFile)
202-
Expect(err).To(BeNil(), "ReadFile failed with error")
226+
if err != nil {
227+
return fmt.Errorf("ReadFile failed with error: %v", err)
228+
}
203229
Expect(strings.TrimSpace(string(readContents))).To(Equal(testFileContents))
204230

205231
// Unmount Disk
206232
err = client.NodeUnpublishVolume(volID, secondPublishDir)
207-
Expect(err).To(BeNil(), "NodeUnpublishVolume failed with error")
233+
if err != nil {
234+
return fmt.Errorf("NodeUnpublishVolume failed with error: %v", err)
235+
}
208236

209237
glog.Infof("Completed testAttachWriteReadDetach with volume %v node %v\n", volID, instance.GetNodeID())
238+
return nil
210239
}

‎test/e2e/tests/setup_e2e_test.go

+7
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,14 @@ limitations under the License.
1515
package tests
1616

1717
import (
18+
"context"
1819
"flag"
1920
"fmt"
2021
"math/rand"
2122
"testing"
2223
"time"
2324

25+
cloudkms "cloud.google.com/go/kms/apiv1"
2426
"github.com/golang/glog"
2527
. "github.com/onsi/ginkgo"
2628
. "github.com/onsi/gomega"
@@ -39,6 +41,7 @@ var (
3941
testContexts = []*remote.TestContext{}
4042
computeService *compute.Service
4143
betaComputeService *computebeta.Service
44+
kmsClient *cloudkms.KeyManagementClient
4245
)
4346

4447
func TestE2E(t *testing.T) {
@@ -62,6 +65,10 @@ var _ = BeforeSuite(func() {
6265
betaComputeService, err = remote.GetBetaComputeClient()
6366
Expect(err).To(BeNil())
6467

68+
// Create the KMS client.
69+
kmsClient, err = cloudkms.NewKeyManagementClient(context.Background())
70+
Expect(err).To(BeNil())
71+
6572
if *runInProw {
6673
*project, *serviceAccount = testutils.SetupProwConfig("gce-project")
6774
}

‎test/e2e/tests/single_zone_e2e_test.go

+164-2
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ limitations under the License.
1515
package tests
1616

1717
import (
18+
"context"
19+
"fmt"
1820
"strings"
1921
"time"
2022

@@ -26,6 +28,10 @@ import (
2628
csi "github.com/container-storage-interface/spec/lib/go/csi"
2729
. "github.com/onsi/ginkgo"
2830
. "github.com/onsi/gomega"
31+
32+
"google.golang.org/api/iterator"
33+
kmspb "google.golang.org/genproto/googleapis/cloud/kms/v1"
34+
fieldmask "google.golang.org/genproto/protobuf/field_mask"
2935
)
3036

3137
const (
@@ -77,7 +83,8 @@ var _ = Describe("GCE PD CSI Driver", func() {
7783
}()
7884

7985
// Attach Disk
80-
testAttachWriteReadDetach(volID, volName, instance, client, false /* readOnly */)
86+
err = testAttachWriteReadDetach(volID, volName, instance, client, false /* readOnly */)
87+
Expect(err).To(BeNil(), "Failed to go through volume lifecycle")
8188

8289
})
8390

@@ -151,7 +158,8 @@ var _ = Describe("GCE PD CSI Driver", func() {
151158
}()
152159

153160
// Attach Disk
154-
testAttachWriteReadDetach(underSpecifiedID, volName, instance, client, false /* readOnly */)
161+
err = testAttachWriteReadDetach(underSpecifiedID, volName, instance, client, false /* readOnly */)
162+
Expect(err).To(BeNil(), "Failed to go through volume lifecycle")
155163

156164
})
157165

@@ -294,6 +302,160 @@ var _ = Describe("GCE PD CSI Driver", func() {
294302
}()
295303
})
296304

305+
It("Should create CMEK key, go through volume lifecycle, validate behavior on key revoke and restore", func() {
306+
ctx := context.Background()
307+
Expect(testContexts).ToNot(BeEmpty())
308+
testContext := getRandomTestContext()
309+
310+
controllerInstance := testContext.Instance
311+
controllerClient := testContext.Client
312+
313+
p, z, _ := controllerInstance.GetIdentity()
314+
locationID := "global"
315+
316+
// The resource name of the key rings.
317+
parentName := fmt.Sprintf("projects/%s/locations/%s", p, locationID)
318+
keyRingId := "gce-pd-csi-test-ring"
319+
320+
// Create KeyRing
321+
ringReq := &kmspb.CreateKeyRingRequest{
322+
Parent: parentName,
323+
KeyRingId: keyRingId,
324+
}
325+
keyRing, err := kmsClient.CreateKeyRing(ctx, ringReq)
326+
if !gce.IsGCEError(err, "alreadyExists") {
327+
getKeyRingReq := &kmspb.GetKeyRingRequest{
328+
Name: fmt.Sprintf("%s/keyRings/%s", parentName, keyRingId),
329+
}
330+
keyRing, err = kmsClient.GetKeyRing(ctx, getKeyRingReq)
331+
332+
}
333+
Expect(err).To(BeNil(), "Failed to create or get key ring %v", keyRingId)
334+
335+
// Create CryptoKey in KeyRing
336+
keyId := "test-key-" + string(uuid.NewUUID())
337+
keyReq := &kmspb.CreateCryptoKeyRequest{
338+
Parent: keyRing.Name,
339+
CryptoKeyId: keyId,
340+
CryptoKey: &kmspb.CryptoKey{
341+
Purpose: kmspb.CryptoKey_ENCRYPT_DECRYPT,
342+
VersionTemplate: &kmspb.CryptoKeyVersionTemplate{
343+
Algorithm: kmspb.CryptoKeyVersion_GOOGLE_SYMMETRIC_ENCRYPTION,
344+
},
345+
},
346+
}
347+
key, err := kmsClient.CreateCryptoKey(ctx, keyReq)
348+
Expect(err).To(BeNil(), "Failed to create crypto key %v in key ring %v", keyId, keyRing.Name)
349+
350+
keyVersions := []string{}
351+
keyVersionReq := &kmspb.ListCryptoKeyVersionsRequest{
352+
Parent: key.Name,
353+
}
354+
355+
it := kmsClient.ListCryptoKeyVersions(ctx, keyVersionReq)
356+
357+
for {
358+
keyVersion, err := it.Next()
359+
if err == iterator.Done {
360+
break
361+
}
362+
Expect(err).To(BeNil(), "Failed to list crypto key versions")
363+
364+
keyVersions = append(keyVersions, keyVersion.Name)
365+
}
366+
367+
// Defer deletion of all key versions
368+
// https://cloud.google.com/kms/docs/destroy-restore
369+
defer func() {
370+
371+
for _, keyVersion := range keyVersions {
372+
destroyKeyReq := &kmspb.DestroyCryptoKeyVersionRequest{
373+
Name: keyVersion,
374+
}
375+
_, err = kmsClient.DestroyCryptoKeyVersion(ctx, destroyKeyReq)
376+
Expect(err).To(BeNil(), "Failed to destroy crypto key version: %v", keyVersion)
377+
}
378+
379+
}()
380+
381+
// Go through volume lifecycle using CMEK-ed PD
382+
// Create Disk
383+
volName := testNamePrefix + string(uuid.NewUUID())
384+
volID, err := controllerClient.CreateVolume(volName, map[string]string{
385+
common.ParameterKeyDiskEncryptionKmsKey: key.Name,
386+
}, defaultSizeGb,
387+
&csi.TopologyRequirement{
388+
Requisite: []*csi.Topology{
389+
{
390+
Segments: map[string]string{common.TopologyKeyZone: z},
391+
},
392+
},
393+
})
394+
Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err)
395+
396+
// Validate Disk Created
397+
cloudDisk, err := computeService.Disks.Get(p, z, volName).Do()
398+
Expect(err).To(BeNil(), "Could not get disk from cloud directly")
399+
Expect(cloudDisk.Type).To(ContainSubstring(standardDiskType))
400+
Expect(cloudDisk.Status).To(Equal(readyState))
401+
Expect(cloudDisk.SizeGb).To(Equal(defaultSizeGb))
402+
Expect(cloudDisk.Name).To(Equal(volName))
403+
404+
defer func() {
405+
// Delete Disk
406+
err = controllerClient.DeleteVolume(volID)
407+
Expect(err).To(BeNil(), "DeleteVolume failed")
408+
409+
// Validate Disk Deleted
410+
_, err = computeService.Disks.Get(p, z, volName).Do()
411+
Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found")
412+
}()
413+
414+
// Test disk works
415+
err = testAttachWriteReadDetach(volID, volName, controllerInstance, controllerClient, false /* readOnly */)
416+
Expect(err).To(BeNil(), "Failed to go through volume lifecycle before revoking CMEK key")
417+
418+
// Revoke CMEK key
419+
// https://cloud.google.com/kms/docs/enable-disable
420+
421+
for _, keyVersion := range keyVersions {
422+
disableReq := &kmspb.UpdateCryptoKeyVersionRequest{
423+
CryptoKeyVersion: &kmspb.CryptoKeyVersion{
424+
Name: keyVersion,
425+
State: kmspb.CryptoKeyVersion_DISABLED,
426+
},
427+
UpdateMask: &fieldmask.FieldMask{
428+
Paths: []string{"state"},
429+
},
430+
}
431+
_, err = kmsClient.UpdateCryptoKeyVersion(ctx, disableReq)
432+
Expect(err).To(BeNil(), "Failed to disable crypto key")
433+
}
434+
435+
// Make sure attach of PD fails
436+
err = testAttachWriteReadDetach(volID, volName, controllerInstance, controllerClient, false /* readOnly */)
437+
Expect(err).ToNot(BeNil(), "Volume lifecycle should have failed, but succeeded")
438+
439+
// Restore CMEK key
440+
for _, keyVersion := range keyVersions {
441+
enableReq := &kmspb.UpdateCryptoKeyVersionRequest{
442+
CryptoKeyVersion: &kmspb.CryptoKeyVersion{
443+
Name: keyVersion,
444+
State: kmspb.CryptoKeyVersion_ENABLED,
445+
},
446+
UpdateMask: &fieldmask.FieldMask{
447+
Paths: []string{"state"},
448+
},
449+
}
450+
_, err = kmsClient.UpdateCryptoKeyVersion(ctx, enableReq)
451+
Expect(err).To(BeNil(), "Failed to enable crypto key")
452+
}
453+
454+
// Make sure attach of PD succeeds
455+
err = testAttachWriteReadDetach(volID, volName, controllerInstance, controllerClient, false /* readOnly */)
456+
Expect(err).To(BeNil(), "Failed to go through volume lifecycle after restoring CMEK key")
457+
})
458+
297459
It("Should create and delete snapshot for RePD in two zones ", func() {
298460
Expect(testContexts).ToNot(BeEmpty())
299461
testContext := getRandomTestContext()

0 commit comments

Comments
 (0)