Skip to content

Commit 7721771

Browse files
davidz627davidz627
committed
Add Cloud KMS encrypted disk lifecycle test
1 parent df44c92 commit 7721771

File tree

3 files changed

+208
-19
lines changed

3 files changed

+208
-19
lines changed

‎test/e2e/tests/multi_zone_e2e_test.go

+39-17
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ limitations under the License.
1515
package tests
1616

1717
import (
18+
"fmt"
1819
"path/filepath"
1920
"strings"
2021

@@ -143,68 +144,89 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() {
143144

144145
})
145146

146-
func testAttachWriteReadDetach(volID string, volName string, instance *remote.InstanceInfo, client *remote.CsiClient, readOnly bool) {
147+
func testAttachWriteReadDetach(volID string, volName string, instance *remote.InstanceInfo, client *remote.CsiClient, readOnly bool) error {
147148
var err error
148149

149150
glog.Infof("Starting testAttachWriteReadDetach with volume %v node %v with readonly %v\n", volID, instance.GetNodeID(), readOnly)
150151
// Attach Disk
151152
err = client.ControllerPublishVolume(volID, instance.GetNodeID())
152-
Expect(err).To(BeNil(), "ControllerPublishVolume failed with error for disk %v on node %v", volID, instance.GetNodeID())
153+
if err != nil {
154+
return fmt.Errorf("ControllerPublishVolume failed with error for disk %v on node %v: %v", volID, instance.GetNodeID(), err)
155+
}
153156

154157
defer func() {
155-
156158
// Detach Disk
157159
err = client.ControllerUnpublishVolume(volID, instance.GetNodeID())
158-
Expect(err).To(BeNil(), "ControllerUnpublishVolume failed with error")
160+
glog.Errorf("Failed to detach disk: %v", err)
159161
}()
160162

161163
// Stage Disk
162164
stageDir := filepath.Join("/tmp/", volName, "stage")
163-
client.NodeStageVolume(volID, stageDir)
164-
Expect(err).To(BeNil(), "NodeStageVolume failed with error")
165+
err = client.NodeStageVolume(volID, stageDir)
166+
if err != nil {
167+
return fmt.Errorf("NodeStageVolume failed with error: %v", err)
168+
}
165169

166170
defer func() {
167171
// Unstage Disk
168172
err = client.NodeUnstageVolume(volID, stageDir)
169-
Expect(err).To(BeNil(), "NodeUnstageVolume failed with error")
170-
err = testutils.RmAll(instance, filepath.Join("/tmp/", volName))
171-
Expect(err).To(BeNil(), "Failed to remove temp directory")
173+
glog.Errorf("Failed to unstage volume: %v", err)
174+
fp := filepath.Join("/tmp/", volName)
175+
err = testutils.RmAll(instance, fp)
176+
glog.Errorf("Failed to rm file path %s: %v", fp, err)
172177
}()
173178

174179
// Mount Disk
175180
publishDir := filepath.Join("/tmp/", volName, "mount")
176181
err = client.NodePublishVolume(volID, stageDir, publishDir)
177-
Expect(err).To(BeNil(), "NodePublishVolume failed with error")
182+
if err != nil {
183+
return fmt.Errorf("NodePublishVolume failed with error: %v", err)
184+
}
178185
err = testutils.ForceChmod(instance, filepath.Join("/tmp/", volName), "777")
179-
Expect(err).To(BeNil(), "Chmod failed with error")
186+
if err != nil {
187+
return fmt.Errorf("Chmod failed with error: %v", err)
188+
}
180189
testFileContents := "test"
181190
if !readOnly {
182191
// Write a file
183192
testFile := filepath.Join(publishDir, "testfile")
184193
err = testutils.WriteFile(instance, testFile, testFileContents)
185-
Expect(err).To(BeNil(), "Failed to write file")
194+
if err != nil {
195+
return fmt.Errorf("Failed to write file: %v", err)
196+
}
186197
}
187198

188199
// Unmount Disk
189200
err = client.NodeUnpublishVolume(volID, publishDir)
190-
Expect(err).To(BeNil(), "NodeUnpublishVolume failed with error")
201+
if err != nil {
202+
return fmt.Errorf("NodeUnpublishVolume failed with error: %v", err)
203+
}
191204

192205
// Mount disk somewhere else
193206
secondPublishDir := filepath.Join("/tmp/", volName, "secondmount")
194207
err = client.NodePublishVolume(volID, stageDir, secondPublishDir)
195-
Expect(err).To(BeNil(), "NodePublishVolume failed with error")
208+
if err != nil {
209+
return fmt.Errorf("NodePublishVolume failed with error: %v", err)
210+
}
196211
err = testutils.ForceChmod(instance, filepath.Join("/tmp/", volName), "777")
197-
Expect(err).To(BeNil(), "Chmod failed with error")
212+
if err != nil {
213+
return fmt.Errorf("Chmod failed with error: %v", err)
214+
}
198215

199216
// Read File
200217
secondTestFile := filepath.Join(secondPublishDir, "testfile")
201218
readContents, err := testutils.ReadFile(instance, secondTestFile)
202-
Expect(err).To(BeNil(), "ReadFile failed with error")
219+
if err != nil {
220+
return fmt.Errorf("ReadFile failed with error: %v", err)
221+
}
203222
Expect(strings.TrimSpace(string(readContents))).To(Equal(testFileContents))
204223

205224
// Unmount Disk
206225
err = client.NodeUnpublishVolume(volID, secondPublishDir)
207-
Expect(err).To(BeNil(), "NodeUnpublishVolume failed with error")
226+
if err != nil {
227+
return fmt.Errorf("NodeUnpublishVolume failed with error: %v", err)
228+
}
208229

209230
glog.Infof("Completed testAttachWriteReadDetach with volume %v node %v\n", volID, instance.GetNodeID())
231+
return nil
210232
}

‎test/e2e/tests/setup_e2e_test.go

+7
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,14 @@ limitations under the License.
1515
package tests
1616

1717
import (
18+
"context"
1819
"flag"
1920
"fmt"
2021
"math/rand"
2122
"testing"
2223
"time"
2324

25+
cloudkms "cloud.google.com/go/kms/apiv1"
2426
"github.com/golang/glog"
2527
. "github.com/onsi/ginkgo"
2628
. "github.com/onsi/gomega"
@@ -39,6 +41,7 @@ var (
3941
testContexts = []*remote.TestContext{}
4042
computeService *compute.Service
4143
betaComputeService *computebeta.Service
44+
kmsClient *cloudkms.KeyManagementClient
4245
)
4346

4447
func TestE2E(t *testing.T) {
@@ -62,6 +65,10 @@ var _ = BeforeSuite(func() {
6265
betaComputeService, err = remote.GetBetaComputeClient()
6366
Expect(err).To(BeNil())
6467

68+
// Create the KMS client.
69+
kmsClient, err = cloudkms.NewKeyManagementClient(context.Background())
70+
Expect(err).To(BeNil())
71+
6572
if *runInProw {
6673
*project, *serviceAccount = testutils.SetupProwConfig("gce-project")
6774
}

‎test/e2e/tests/single_zone_e2e_test.go

+162-2
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ limitations under the License.
1515
package tests
1616

1717
import (
18+
"context"
19+
"fmt"
1820
"strings"
1921
"time"
2022

@@ -26,6 +28,10 @@ import (
2628
csi "github.com/container-storage-interface/spec/lib/go/csi"
2729
. "github.com/onsi/ginkgo"
2830
. "github.com/onsi/gomega"
31+
32+
"google.golang.org/api/iterator"
33+
kmspb "google.golang.org/genproto/googleapis/cloud/kms/v1"
34+
fieldmask "google.golang.org/genproto/protobuf/field_mask"
2935
)
3036

3137
const (
@@ -77,7 +83,8 @@ var _ = Describe("GCE PD CSI Driver", func() {
7783
}()
7884

7985
// Attach Disk
80-
testAttachWriteReadDetach(volID, volName, instance, client, false /* readOnly */)
86+
err = testAttachWriteReadDetach(volID, volName, instance, client, false /* readOnly */)
87+
Expect(err).To(BeNil(), "Failed to go through volume lifecycle")
8188

8289
})
8390

@@ -151,7 +158,8 @@ var _ = Describe("GCE PD CSI Driver", func() {
151158
}()
152159

153160
// Attach Disk
154-
testAttachWriteReadDetach(underSpecifiedID, volName, instance, client, false /* readOnly */)
161+
err = testAttachWriteReadDetach(underSpecifiedID, volName, instance, client, false /* readOnly */)
162+
Expect(err).To(BeNil(), "Failed to go through volume lifecycle")
155163

156164
})
157165

@@ -294,6 +302,158 @@ var _ = Describe("GCE PD CSI Driver", func() {
294302
}()
295303
})
296304

305+
It("Should create CMEK key, go through volume lifecycle, validate behavior on key revoke and restore", func() {
306+
ctx := context.Background()
307+
Expect(testContexts).ToNot(BeEmpty())
308+
testContext := getRandomTestContext()
309+
310+
controllerInstance := testContext.Instance
311+
controllerClient := testContext.Client
312+
313+
p, z, _ := controllerInstance.GetIdentity()
314+
locationID := "global"
315+
316+
// The resource name of the key rings.
317+
parentName := fmt.Sprintf("projects/%s/locations/%s", p, locationID)
318+
keyRingId := "gce-pd-csi-test-ring"
319+
320+
// Create KeyRing
321+
ringReq := &kmspb.CreateKeyRingRequest{
322+
Parent: parentName,
323+
KeyRingId: keyRingId,
324+
}
325+
keyRing, err := kmsClient.CreateKeyRing(ctx, ringReq)
326+
if !gce.IsGCEError(err, "alreadyExists") {
327+
getKeyRingReq := &kmspb.GetKeyRingRequest{
328+
Name: fmt.Sprintf("%s/keyRings/%s", parentName, keyRingId),
329+
}
330+
keyRing, err = kmsClient.GetKeyRing(ctx, getKeyRingReq)
331+
332+
}
333+
Expect(err).To(BeNil(), "Failed to create or get key ring %v", keyRingId)
334+
335+
// Create CryptoKey in KeyRing
336+
keyId := "test-key-" + string(uuid.NewUUID())
337+
keyReq := &kmspb.CreateCryptoKeyRequest{
338+
Parent: keyRing.Name,
339+
CryptoKeyId: keyId,
340+
CryptoKey: &kmspb.CryptoKey{
341+
Purpose: kmspb.CryptoKey_ENCRYPT_DECRYPT,
342+
VersionTemplate: &kmspb.CryptoKeyVersionTemplate{
343+
Algorithm: kmspb.CryptoKeyVersion_GOOGLE_SYMMETRIC_ENCRYPTION,
344+
},
345+
},
346+
}
347+
key, err := kmsClient.CreateCryptoKey(ctx, keyReq)
348+
Expect(err).To(BeNil(), "Failed to create crypto key %v in key ring %v", keyId, keyRing.Name)
349+
350+
keyVersions := []string{}
351+
keyVersionReq := &kmspb.ListCryptoKeyVersionsRequest{
352+
Parent: key.Name,
353+
}
354+
355+
it := kmsClient.ListCryptoKeyVersions(ctx, keyVersionReq)
356+
357+
for {
358+
keyVersion, err := it.Next()
359+
if err == iterator.Done {
360+
break
361+
}
362+
Expect(err).To(BeNil(), "Failed to list crypto key versions")
363+
364+
keyVersions = append(keyVersions, keyVersion.Name)
365+
}
366+
367+
// Defer deletion of all key versions
368+
// https://cloud.google.com/kms/docs/destroy-restore
369+
defer func() {
370+
for _, keyVersion := range keyVersions {
371+
destroyKeyReq := &kmspb.DestroyCryptoKeyVersionRequest{
372+
Name: keyVersion,
373+
}
374+
_, err = kmsClient.DestroyCryptoKeyVersion(ctx, destroyKeyReq)
375+
Expect(err).To(BeNil(), "Failed to destroy crypto key version: %v", keyVersion)
376+
}
377+
378+
}()
379+
380+
// Go through volume lifecycle using CMEK-ed PD
381+
// Create Disk
382+
volName := testNamePrefix + string(uuid.NewUUID())
383+
volID, err := controllerClient.CreateVolume(volName, map[string]string{
384+
common.ParameterKeyDiskEncryptionKmsKey: key.Name,
385+
}, defaultSizeGb,
386+
&csi.TopologyRequirement{
387+
Requisite: []*csi.Topology{
388+
{
389+
Segments: map[string]string{common.TopologyKeyZone: z},
390+
},
391+
},
392+
})
393+
Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err)
394+
395+
// Validate Disk Created
396+
cloudDisk, err := computeService.Disks.Get(p, z, volName).Do()
397+
Expect(err).To(BeNil(), "Could not get disk from cloud directly")
398+
Expect(cloudDisk.Type).To(ContainSubstring(standardDiskType))
399+
Expect(cloudDisk.Status).To(Equal(readyState))
400+
Expect(cloudDisk.SizeGb).To(Equal(defaultSizeGb))
401+
Expect(cloudDisk.Name).To(Equal(volName))
402+
403+
defer func() {
404+
// Delete Disk
405+
err = controllerClient.DeleteVolume(volID)
406+
Expect(err).To(BeNil(), "DeleteVolume failed")
407+
408+
// Validate Disk Deleted
409+
_, err = computeService.Disks.Get(p, z, volName).Do()
410+
Expect(gce.IsGCEError(err, "notFound")).To(BeTrue(), "Expected disk to not be found")
411+
}()
412+
413+
// Test disk works
414+
err = testAttachWriteReadDetach(volID, volName, controllerInstance, controllerClient, false /* readOnly */)
415+
Expect(err).To(BeNil(), "Failed to go through volume lifecycle before revoking CMEK key")
416+
417+
// Revoke CMEK key
418+
// https://cloud.google.com/kms/docs/enable-disable
419+
for _, keyVersion := range keyVersions {
420+
disableReq := &kmspb.UpdateCryptoKeyVersionRequest{
421+
CryptoKeyVersion: &kmspb.CryptoKeyVersion{
422+
Name: keyVersion,
423+
State: kmspb.CryptoKeyVersion_DISABLED,
424+
},
425+
UpdateMask: &fieldmask.FieldMask{
426+
Paths: []string{"state"},
427+
},
428+
}
429+
_, err = kmsClient.UpdateCryptoKeyVersion(ctx, disableReq)
430+
Expect(err).To(BeNil(), "Failed to disable crypto key")
431+
}
432+
433+
// Make sure attach of PD fails
434+
err = testAttachWriteReadDetach(volID, volName, controllerInstance, controllerClient, false /* readOnly */)
435+
Expect(err).ToNot(BeNil(), "Volume lifecycle should have failed, but succeeded")
436+
437+
// Restore CMEK key
438+
for _, keyVersion := range keyVersions {
439+
enableReq := &kmspb.UpdateCryptoKeyVersionRequest{
440+
CryptoKeyVersion: &kmspb.CryptoKeyVersion{
441+
Name: keyVersion,
442+
State: kmspb.CryptoKeyVersion_ENABLED,
443+
},
444+
UpdateMask: &fieldmask.FieldMask{
445+
Paths: []string{"state"},
446+
},
447+
}
448+
_, err = kmsClient.UpdateCryptoKeyVersion(ctx, enableReq)
449+
Expect(err).To(BeNil(), "Failed to enable crypto key")
450+
}
451+
452+
// Make sure attach of PD succeeds
453+
err = testAttachWriteReadDetach(volID, volName, controllerInstance, controllerClient, false /* readOnly */)
454+
Expect(err).To(BeNil(), "Failed to go through volume lifecycle after restoring CMEK key")
455+
})
456+
297457
It("Should create and delete snapshot for RePD in two zones ", func() {
298458
Expect(testContexts).ToNot(BeEmpty())
299459
testContext := getRandomTestContext()

0 commit comments

Comments
 (0)