From b6f703a67ab01229bb4e7537796037381ea268c0 Mon Sep 17 00:00:00 2001 From: David Zhu Date: Mon, 27 Aug 2018 12:20:15 -0700 Subject: [PATCH 1/2] Test contexts are now initialized once on startup instead of per-test --- test/e2e/tests/multi_zone_e2e_test.go | 26 ++++------------ test/e2e/tests/setup_e2e_test.go | 30 +++++++++++-------- test/e2e/tests/single_zone_e2e_test.go | 41 +++++--------------------- test/remote/instance.go | 3 +- 4 files changed, 32 insertions(+), 68 deletions(-) diff --git a/test/e2e/tests/multi_zone_e2e_test.go b/test/e2e/tests/multi_zone_e2e_test.go index c559af3fc..454477b87 100644 --- a/test/e2e/tests/multi_zone_e2e_test.go +++ b/test/e2e/tests/multi_zone_e2e_test.go @@ -31,18 +31,11 @@ import ( var _ = Describe("GCE PD CSI Driver Multi-Zone", func() { BeforeEach(func() { - Expect(len(testInstances)).To(BeNumerically(">", 1)) + Expect(len(testContexts)).To(BeNumerically(">", 1)) }) It("Should get reasonable topology from nodes with NodeGetInfo", func() { - for _, instance := range testInstances { - testContext, err := testutils.GCEClientAndDriverSetup(instance) - Expect(err).To(BeNil(), "Set up new Driver and Client failed with error") - defer func() { - err := remote.TeardownDriverAndClient(testContext) - Expect(err).To(BeNil(), "Teardown Driver and Client failed with error") - }() - + for _, testContext := range testContexts { resp, err := testContext.Client.NodeGetInfo() Expect(err).To(BeNil()) @@ -65,24 +58,17 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() { It("Should successfully run through entire lifecycle of an RePD volume on instances in 2 zones", func() { // Create new driver and client - Expect(testInstances).NotTo(BeEmpty()) + Expect(testContexts).NotTo(BeEmpty()) zoneToContext := map[string]*remote.TestContext{} zones := []string{} - for _, i := range testInstances { - _, z, _ := i.GetIdentity() + for _, tc := range testContexts { + _, z, _ := tc.Instance.GetIdentity() // Zone hasn't been seen before if _, ok := zoneToContext[z]; !ok { - c, err := testutils.GCEClientAndDriverSetup(i) - Expect(err).To(BeNil(), "Set up new Driver and Client failed with error") - zoneToContext[z] = c + zoneToContext[z] = tc zones = append(zones, z) - - defer func() { - err := remote.TeardownDriverAndClient(c) - Expect(err).To(BeNil(), "Teardown Driver and Client failed with error") - }() } if len(zoneToContext) == 2 { break diff --git a/test/e2e/tests/setup_e2e_test.go b/test/e2e/tests/setup_e2e_test.go index a0f882a7a..09ec9309e 100644 --- a/test/e2e/tests/setup_e2e_test.go +++ b/test/e2e/tests/setup_e2e_test.go @@ -35,7 +35,7 @@ var ( runInProw = flag.Bool("run-in-prow", false, "If true, use a Boskos loaned project and special CI service accounts and ssh keys") deleteInstances = flag.Bool("delete-instances", false, "Delete the instances after tests run") - testInstances = []*remote.InstanceInfo{} + testContexts = []*remote.TestContext{} computeService *compute.Service betaComputeService *computebeta.Service ) @@ -73,22 +73,28 @@ var _ = BeforeSuite(func() { i, err := remote.SetupInstance(*project, zone, nodeID, *serviceAccount, computeService) Expect(err).To(BeNil()) - testInstances = append(testInstances, i) - } + // Create new driver and client + testContext, err := testutils.GCEClientAndDriverSetup(i) + Expect(err).To(BeNil(), "Set up new Driver and Client failed with error") + + testContexts = append(testContexts, testContext) + } }) var _ = AfterSuite(func() { - /* - err := node.client.CloseConn() - if err != nil { - Logf("Failed to close the client") - } else { - Logf("Closed the client") - */ - for _, i := range testInstances { + + for _, tc := range testContexts { + err := remote.TeardownDriverAndClient(tc) + Expect(err).To(BeNil(), "Teardown Driver and Client failed with error") if *deleteInstances { - i.DeleteInstance() + tc.Instance.DeleteInstance() } } }) + +func getRandomTestContext() *remote.TestContext { + Expect(testContexts).ToNot(BeEmpty()) + rn := rand.Intn(len(testContexts)) + return testContexts[rn] +} diff --git a/test/e2e/tests/single_zone_e2e_test.go b/test/e2e/tests/single_zone_e2e_test.go index c3600aee7..f53ac7ec7 100644 --- a/test/e2e/tests/single_zone_e2e_test.go +++ b/test/e2e/tests/single_zone_e2e_test.go @@ -21,8 +21,6 @@ import ( "k8s.io/apimachinery/pkg/util/uuid" "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common" gce "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider/compute" - testutils "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/e2e/utils" - remote "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/remote" csi "github.com/container-storage-interface/spec/lib/go/csi/v0" . "github.com/onsi/ginkgo" @@ -41,14 +39,7 @@ const ( var _ = Describe("GCE PD CSI Driver", func() { It("Should create->attach->stage->mount volume and check if it is writable, then unmount->unstage->detach->delete and check disk is deleted", func() { - // Create new driver and client - Expect(testInstances).NotTo(BeEmpty()) - testContext, err := testutils.GCEClientAndDriverSetup(testInstances[0]) - Expect(err).To(BeNil(), "Set up new Driver and Client failed with error") - defer func() { - err := remote.TeardownDriverAndClient(testContext) - Expect(err).To(BeNil(), "Teardown Driver and Client failed with error") - }() + testContext := getRandomTestContext() p, z, _ := testContext.Instance.GetIdentity() client := testContext.Client @@ -90,14 +81,8 @@ var _ = Describe("GCE PD CSI Driver", func() { }) It("Should create disks in correct zones when topology is specified", func() { - /// - Expect(testInstances).NotTo(BeEmpty()) - testContext, err := testutils.GCEClientAndDriverSetup(testInstances[0]) - Expect(err).To(BeNil(), "Failed to set up new driver and client") - defer func() { - err := remote.TeardownDriverAndClient(testContext) - Expect(err).To(BeNil(), "Teardown Driver and Client failed with error") - }() + Expect(testContexts).ToNot(BeEmpty()) + testContext := getRandomTestContext() p, _, _ := testContext.Instance.GetIdentity() @@ -126,14 +111,8 @@ var _ = Describe("GCE PD CSI Driver", func() { }) It("Should successfully create RePD in two zones in the drivers region when none are specified", func() { - // Create new driver and client - Expect(testInstances).NotTo(BeEmpty()) - testContext, err := testutils.GCEClientAndDriverSetup(testInstances[0]) - Expect(err).To(BeNil(), "Failed to set up new driver and client") - defer func() { - err := remote.TeardownDriverAndClient(testContext) - Expect(err).To(BeNil(), "Teardown Driver and Client failed with error") - }() + Expect(testContexts).ToNot(BeEmpty()) + testContext := getRandomTestContext() controllerInstance := testContext.Instance controllerClient := testContext.Client @@ -177,14 +156,8 @@ var _ = Describe("GCE PD CSI Driver", func() { }) It("Should create and delete disk with default zone", func() { - // Create new driver and client - Expect(testInstances).NotTo(BeEmpty()) - testContext, err := testutils.GCEClientAndDriverSetup(testInstances[0]) - Expect(err).To(BeNil(), "Set up new Driver and Client failed with error") - defer func() { - err := remote.TeardownDriverAndClient(testContext) - Expect(err).To(BeNil(), "Teardown Driver and Client failed with error") - }() + Expect(testContexts).ToNot(BeEmpty()) + testContext := getRandomTestContext() p, z, _ := testContext.Instance.GetIdentity() client := testContext.Client diff --git a/test/remote/instance.go b/test/remote/instance.go index 607d2c5c0..8c252a727 100644 --- a/test/remote/instance.go +++ b/test/remote/instance.go @@ -88,8 +88,7 @@ func (i *InstanceInfo) CreateOrGetInstance(serviceAccount string) error { return fmt.Errorf("Failed to create firewall rule: %v", err) } - // TODO(#97): Pick a better boot disk image - imageURL := "projects/ml-images/global/images/family/tf-1-9" + imageURL := "projects/ubuntu-os-cloud/global/images/family/ubuntu-minimal-1804-lts" inst := &compute.Instance{ Name: i.name, MachineType: machineType(i.zone, ""), From 4198d90fa035044432083bb7c2892f5409985b97 Mon Sep 17 00:00:00 2001 From: David Zhu Date: Mon, 27 Aug 2018 13:25:44 -0700 Subject: [PATCH 2/2] Enable test setup to run in parallel --- test/e2e/tests/multi_zone_e2e_test.go | 30 ++++++++++++------------- test/e2e/tests/setup_e2e_test.go | 31 +++++++++++++++++--------- test/e2e/tests/single_zone_e2e_test.go | 14 ++++++------ test/remote/setup-teardown.go | 6 ++--- 4 files changed, 45 insertions(+), 36 deletions(-) diff --git a/test/e2e/tests/multi_zone_e2e_test.go b/test/e2e/tests/multi_zone_e2e_test.go index 454477b87..b842f4d59 100644 --- a/test/e2e/tests/multi_zone_e2e_test.go +++ b/test/e2e/tests/multi_zone_e2e_test.go @@ -88,7 +88,7 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() { // Create Disk volName := testNamePrefix + string(uuid.NewUUID()) - volId, err := controllerClient.CreateVolume(volName, map[string]string{ + volID, err := controllerClient.CreateVolume(volName, map[string]string{ common.ParameterKeyReplicationType: "regional-pd", }, defaultSizeGb, &csi.TopologyRequirement{ Requisite: []*csi.Topology{ @@ -119,7 +119,7 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() { defer func() { // Delete Disk - controllerClient.DeleteVolume(volId) + controllerClient.DeleteVolume(volID) Expect(err).To(BeNil(), "DeleteVolume failed") // Validate Disk Deleted @@ -134,7 +134,7 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() { if i >= 1 { readOnly = true } - testAttachWriteReadDetach(volId, volName, testContext.Instance, testContext.Client, readOnly) + testAttachWriteReadDetach(volID, volName, testContext.Instance, testContext.Client, readOnly) i = i + 1 } @@ -142,29 +142,29 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() { }) -func testAttachWriteReadDetach(volId string, volName string, instance *remote.InstanceInfo, client *remote.CsiClient, readOnly bool) { +func testAttachWriteReadDetach(volID string, volName string, instance *remote.InstanceInfo, client *remote.CsiClient, readOnly bool) { var err error - Logf("Starting testAttachWriteReadDetach with volume %v node %v with readonly %v\n", volId, instance.GetNodeID(), readOnly) + Logf("Starting testAttachWriteReadDetach with volume %v node %v with readonly %v\n", volID, instance.GetNodeID(), readOnly) // Attach Disk - err = client.ControllerPublishVolume(volId, instance.GetNodeID()) - Expect(err).To(BeNil(), "ControllerPublishVolume failed with error for disk %v on node %v", volId, instance.GetNodeID()) + err = client.ControllerPublishVolume(volID, instance.GetNodeID()) + Expect(err).To(BeNil(), "ControllerPublishVolume failed with error for disk %v on node %v", volID, instance.GetNodeID()) defer func() { // Detach Disk - err = client.ControllerUnpublishVolume(volId, instance.GetNodeID()) + err = client.ControllerUnpublishVolume(volID, instance.GetNodeID()) Expect(err).To(BeNil(), "ControllerUnpublishVolume failed with error") }() // Stage Disk stageDir := filepath.Join("/tmp/", volName, "stage") - client.NodeStageVolume(volId, stageDir) + client.NodeStageVolume(volID, stageDir) Expect(err).To(BeNil(), "NodeStageVolume failed with error") defer func() { // Unstage Disk - err = client.NodeUnstageVolume(volId, stageDir) + err = client.NodeUnstageVolume(volID, stageDir) Expect(err).To(BeNil(), "NodeUnstageVolume failed with error") err = testutils.RmAll(instance, filepath.Join("/tmp/", volName)) Expect(err).To(BeNil(), "Failed to remove temp directory") @@ -172,7 +172,7 @@ func testAttachWriteReadDetach(volId string, volName string, instance *remote.In // Mount Disk publishDir := filepath.Join("/tmp/", volName, "mount") - err = client.NodePublishVolume(volId, stageDir, publishDir) + err = client.NodePublishVolume(volID, stageDir, publishDir) Expect(err).To(BeNil(), "NodePublishVolume failed with error") err = testutils.ForceChmod(instance, filepath.Join("/tmp/", volName), "777") Expect(err).To(BeNil(), "Chmod failed with error") @@ -185,12 +185,12 @@ func testAttachWriteReadDetach(volId string, volName string, instance *remote.In } // Unmount Disk - err = client.NodeUnpublishVolume(volId, publishDir) + err = client.NodeUnpublishVolume(volID, publishDir) Expect(err).To(BeNil(), "NodeUnpublishVolume failed with error") // Mount disk somewhere else secondPublishDir := filepath.Join("/tmp/", volName, "secondmount") - err = client.NodePublishVolume(volId, stageDir, secondPublishDir) + err = client.NodePublishVolume(volID, stageDir, secondPublishDir) Expect(err).To(BeNil(), "NodePublishVolume failed with error") err = testutils.ForceChmod(instance, filepath.Join("/tmp/", volName), "777") Expect(err).To(BeNil(), "Chmod failed with error") @@ -202,8 +202,8 @@ func testAttachWriteReadDetach(volId string, volName string, instance *remote.In Expect(strings.TrimSpace(string(readContents))).To(Equal(testFileContents)) // Unmount Disk - err = client.NodeUnpublishVolume(volId, secondPublishDir) + err = client.NodeUnpublishVolume(volID, secondPublishDir) Expect(err).To(BeNil(), "NodeUnpublishVolume failed with error") - Logf("Completed testAttachWriteReadDetach with volume %v node %v\n", volId, instance.GetNodeID()) + Logf("Completed testAttachWriteReadDetach with volume %v node %v\n", volID, instance.GetNodeID()) } diff --git a/test/e2e/tests/setup_e2e_test.go b/test/e2e/tests/setup_e2e_test.go index 09ec9309e..a7f1aeed8 100644 --- a/test/e2e/tests/setup_e2e_test.go +++ b/test/e2e/tests/setup_e2e_test.go @@ -48,6 +48,9 @@ func TestE2E(t *testing.T) { var _ = BeforeSuite(func() { var err error + tcc := make(chan *remote.TestContext) + defer close(tcc) + zones := []string{"us-central1-c", "us-central1-b"} rand.Seed(time.Now().UnixNano()) @@ -68,17 +71,25 @@ var _ = BeforeSuite(func() { Logf("Running in project %v with service account %v\n\n", *project, *serviceAccount) for _, zone := range zones { - nodeID := fmt.Sprintf("gce-pd-csi-e2e-%s", zone) - - i, err := remote.SetupInstance(*project, zone, nodeID, *serviceAccount, computeService) - Expect(err).To(BeNil()) - - // Create new driver and client - testContext, err := testutils.GCEClientAndDriverSetup(i) - Expect(err).To(BeNil(), "Set up new Driver and Client failed with error") - - testContexts = append(testContexts, testContext) + go func(curZone string) { + defer GinkgoRecover() + nodeID := fmt.Sprintf("gce-pd-csi-e2e-%s", curZone) + Logf("Setting up node %s\n", nodeID) + + i, err := remote.SetupInstance(*project, curZone, nodeID, *serviceAccount, computeService) + Expect(err).To(BeNil()) + + // Create new driver and client + testContext, err := testutils.GCEClientAndDriverSetup(i) + Expect(err).To(BeNil(), "Set up new Driver and Client failed with error") + tcc <- testContext + }(zone) + } + for i := 0; i < len(zones); i++ { + tc := <-tcc + Logf("Test Context for node %s set up\n", tc.Instance.GetName()) + testContexts = append(testContexts, tc) } }) diff --git a/test/e2e/tests/single_zone_e2e_test.go b/test/e2e/tests/single_zone_e2e_test.go index f53ac7ec7..f8d853f80 100644 --- a/test/e2e/tests/single_zone_e2e_test.go +++ b/test/e2e/tests/single_zone_e2e_test.go @@ -47,7 +47,7 @@ var _ = Describe("GCE PD CSI Driver", func() { // Create Disk volName := testNamePrefix + string(uuid.NewUUID()) - volId, err := client.CreateVolume(volName, nil, defaultSizeGb, + volID, err := client.CreateVolume(volName, nil, defaultSizeGb, &csi.TopologyRequirement{ Requisite: []*csi.Topology{ { @@ -67,7 +67,7 @@ var _ = Describe("GCE PD CSI Driver", func() { defer func() { // Delete Disk - client.DeleteVolume(volId) + client.DeleteVolume(volID) Expect(err).To(BeNil(), "DeleteVolume failed") // Validate Disk Deleted @@ -76,7 +76,7 @@ var _ = Describe("GCE PD CSI Driver", func() { }() // Attach Disk - testAttachWriteReadDetach(volId, volName, instance, client, false /* readOnly */) + testAttachWriteReadDetach(volID, volName, instance, client, false /* readOnly */) }) @@ -124,7 +124,7 @@ var _ = Describe("GCE PD CSI Driver", func() { // Create Disk volName := testNamePrefix + string(uuid.NewUUID()) - volId, err := controllerClient.CreateVolume(volName, map[string]string{ + volID, err := controllerClient.CreateVolume(volName, map[string]string{ common.ParameterKeyReplicationType: "regional-pd", }, defaultSizeGb, nil) Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err) @@ -146,7 +146,7 @@ var _ = Describe("GCE PD CSI Driver", func() { } defer func() { // Delete Disk - controllerClient.DeleteVolume(volId) + controllerClient.DeleteVolume(volID) Expect(err).To(BeNil(), "DeleteVolume failed") // Validate Disk Deleted @@ -164,7 +164,7 @@ var _ = Describe("GCE PD CSI Driver", func() { // Create Disk volName := testNamePrefix + string(uuid.NewUUID()) - volId, err := client.CreateVolume(volName, nil, defaultSizeGb, nil) + volID, err := client.CreateVolume(volName, nil, defaultSizeGb, nil) Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err) // Validate Disk Created @@ -177,7 +177,7 @@ var _ = Describe("GCE PD CSI Driver", func() { defer func() { // Delete Disk - client.DeleteVolume(volId) + client.DeleteVolume(volID) Expect(err).To(BeNil(), "DeleteVolume failed") // Validate Disk Deleted diff --git a/test/remote/setup-teardown.go b/test/remote/setup-teardown.go index 575e1715d..9b9055616 100644 --- a/test/remote/setup-teardown.go +++ b/test/remote/setup-teardown.go @@ -22,10 +22,7 @@ import ( "github.com/golang/glog" compute "google.golang.org/api/compute/v1" -) - -const ( - archiveName = "e2e_driver_binaries.tar.gz" + "k8s.io/apimachinery/pkg/util/uuid" ) // TestContext holds the CSI Client handle to a remotely connected Driver @@ -75,6 +72,7 @@ func SetupInstance(instanceProject, instanceZone, instanceName, instanceServiceA // a CSI client to it through SHH tunnelling. It returns a TestContext with both a handle to the instance // that the driver is on and the CSI Client object to make CSI calls to the remote driver. func SetupNewDriverAndClient(instance *InstanceInfo, config *ClientConfig) (*TestContext, error) { + archiveName := fmt.Sprintf("e2e_driver_binaries_%s.tar.gz", uuid.NewUUID()) archivePath, err := CreateDriverArchive(archiveName, config.PkgPath, config.BinPath) if err != nil { return nil, err