Merge pull request #104 from davidz627/fix/testImage

Test contexts are now initialized once on startup in parallel instead of per-test, Chose a better boot disk image.

Author: k8s-ci-robot

test/e2e/tests/multi_zone_e2e_test.go

@@ -31,18 +31,11 @@ import (
 
 var _ = Describe("GCE PD CSI Driver Multi-Zone", func() {
 	BeforeEach(func() {
-		Expect(len(testInstances)).To(BeNumerically(">", 1))
+		Expect(len(testContexts)).To(BeNumerically(">", 1))
 	})
 
 	It("Should get reasonable topology from nodes with NodeGetInfo", func() {
-		for _, instance := range testInstances {
-			testContext, err := testutils.GCEClientAndDriverSetup(instance)
-			Expect(err).To(BeNil(), "Set up new Driver and Client failed with error")
-			defer func() {
-				err := remote.TeardownDriverAndClient(testContext)
-				Expect(err).To(BeNil(), "Teardown Driver and Client failed with error")
-			}()
-
+		for _, testContext := range testContexts {
 			resp, err := testContext.Client.NodeGetInfo()
 			Expect(err).To(BeNil())
 
@@ -65,24 +58,17 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() {
 	It("Should successfully run through entire lifecycle of an RePD volume on instances in 2 zones", func() {
 		// Create new driver and client
 
-		Expect(testInstances).NotTo(BeEmpty())
+		Expect(testContexts).NotTo(BeEmpty())
 
 		zoneToContext := map[string]*remote.TestContext{}
 		zones := []string{}
 
-		for _, i := range testInstances {
-			_, z, _ := i.GetIdentity()
+		for _, tc := range testContexts {
+			_, z, _ := tc.Instance.GetIdentity()
 			// Zone hasn't been seen before
 			if _, ok := zoneToContext[z]; !ok {
-				c, err := testutils.GCEClientAndDriverSetup(i)
-				Expect(err).To(BeNil(), "Set up new Driver and Client failed with error")
-				zoneToContext[z] = c
+				zoneToContext[z] = tc
 				zones = append(zones, z)
-
-				defer func() {
-					err := remote.TeardownDriverAndClient(c)
-					Expect(err).To(BeNil(), "Teardown Driver and Client failed with error")
-				}()
 			}
 			if len(zoneToContext) == 2 {
 				break
@@ -102,7 +88,7 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() {
 
 		// Create Disk
 		volName := testNamePrefix + string(uuid.NewUUID())
-		volId, err := controllerClient.CreateVolume(volName, map[string]string{
+		volID, err := controllerClient.CreateVolume(volName, map[string]string{
 			common.ParameterKeyReplicationType: "regional-pd",
 		}, defaultSizeGb, &csi.TopologyRequirement{
 			Requisite: []*csi.Topology{
@@ -133,7 +119,7 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() {
 
 		defer func() {
 			// Delete Disk
-			controllerClient.DeleteVolume(volId)
+			controllerClient.DeleteVolume(volID)
 			Expect(err).To(BeNil(), "DeleteVolume failed")
 
 			// Validate Disk Deleted
@@ -148,45 +134,45 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() {
 			if i >= 1 {
 				readOnly = true
 			}
-			testAttachWriteReadDetach(volId, volName, testContext.Instance, testContext.Client, readOnly)
+			testAttachWriteReadDetach(volID, volName, testContext.Instance, testContext.Client, readOnly)
 			i = i + 1
 		}
 
 	})
 
 })
 
-func testAttachWriteReadDetach(volId string, volName string, instance *remote.InstanceInfo, client *remote.CsiClient, readOnly bool) {
+func testAttachWriteReadDetach(volID string, volName string, instance *remote.InstanceInfo, client *remote.CsiClient, readOnly bool) {
 	var err error
 
-	Logf("Starting testAttachWriteReadDetach with volume %v node %v with readonly %v\n", volId, instance.GetNodeID(), readOnly)
+	Logf("Starting testAttachWriteReadDetach with volume %v node %v with readonly %v\n", volID, instance.GetNodeID(), readOnly)
 	// Attach Disk
-	err = client.ControllerPublishVolume(volId, instance.GetNodeID())
-	Expect(err).To(BeNil(), "ControllerPublishVolume failed with error for disk %v on node %v", volId, instance.GetNodeID())
+	err = client.ControllerPublishVolume(volID, instance.GetNodeID())
+	Expect(err).To(BeNil(), "ControllerPublishVolume failed with error for disk %v on node %v", volID, instance.GetNodeID())
 
 	defer func() {
 
 		// Detach Disk
-		err = client.ControllerUnpublishVolume(volId, instance.GetNodeID())
+		err = client.ControllerUnpublishVolume(volID, instance.GetNodeID())
 		Expect(err).To(BeNil(), "ControllerUnpublishVolume failed with error")
 	}()
 
 	// Stage Disk
 	stageDir := filepath.Join("/tmp/", volName, "stage")
-	client.NodeStageVolume(volId, stageDir)
+	client.NodeStageVolume(volID, stageDir)
 	Expect(err).To(BeNil(), "NodeStageVolume failed with error")
 
 	defer func() {
 		// Unstage Disk
-		err = client.NodeUnstageVolume(volId, stageDir)
+		err = client.NodeUnstageVolume(volID, stageDir)
 		Expect(err).To(BeNil(), "NodeUnstageVolume failed with error")
 		err = testutils.RmAll(instance, filepath.Join("/tmp/", volName))
 		Expect(err).To(BeNil(), "Failed to remove temp directory")
 	}()
 
 	// Mount Disk
 	publishDir := filepath.Join("/tmp/", volName, "mount")
-	err = client.NodePublishVolume(volId, stageDir, publishDir)
+	err = client.NodePublishVolume(volID, stageDir, publishDir)
 	Expect(err).To(BeNil(), "NodePublishVolume failed with error")
 	err = testutils.ForceChmod(instance, filepath.Join("/tmp/", volName), "777")
 	Expect(err).To(BeNil(), "Chmod failed with error")
@@ -199,12 +185,12 @@ func testAttachWriteReadDetach(volId string, volName string, instance *remote.In
 	}
 
 	// Unmount Disk
-	err = client.NodeUnpublishVolume(volId, publishDir)
+	err = client.NodeUnpublishVolume(volID, publishDir)
 	Expect(err).To(BeNil(), "NodeUnpublishVolume failed with error")
 
 	// Mount disk somewhere else
 	secondPublishDir := filepath.Join("/tmp/", volName, "secondmount")
-	err = client.NodePublishVolume(volId, stageDir, secondPublishDir)
+	err = client.NodePublishVolume(volID, stageDir, secondPublishDir)
 	Expect(err).To(BeNil(), "NodePublishVolume failed with error")
 	err = testutils.ForceChmod(instance, filepath.Join("/tmp/", volName), "777")
 	Expect(err).To(BeNil(), "Chmod failed with error")
@@ -216,8 +202,8 @@ func testAttachWriteReadDetach(volId string, volName string, instance *remote.In
 	Expect(strings.TrimSpace(string(readContents))).To(Equal(testFileContents))
 
 	// Unmount Disk
-	err = client.NodeUnpublishVolume(volId, secondPublishDir)
+	err = client.NodeUnpublishVolume(volID, secondPublishDir)
 	Expect(err).To(BeNil(), "NodeUnpublishVolume failed with error")
 
-	Logf("Completed testAttachWriteReadDetach with volume %v node %v\n", volId, instance.GetNodeID())
+	Logf("Completed testAttachWriteReadDetach with volume %v node %v\n", volID, instance.GetNodeID())
 }

test/e2e/tests/setup_e2e_test.go

@@ -35,7 +35,7 @@ var (
 	runInProw       = flag.Bool("run-in-prow", false, "If true, use a Boskos loaned project and special CI service accounts and ssh keys")
 	deleteInstances = flag.Bool("delete-instances", false, "Delete the instances after tests run")
 
-	testInstances      = []*remote.InstanceInfo{}
+	testContexts       = []*remote.TestContext{}
 	computeService     *compute.Service
 	betaComputeService *computebeta.Service
 )
@@ -48,6 +48,9 @@ func TestE2E(t *testing.T) {
 
 var _ = BeforeSuite(func() {
 	var err error
+	tcc := make(chan *remote.TestContext)
+	defer close(tcc)
+
 	zones := []string{"us-central1-c", "us-central1-b"}
 
 	rand.Seed(time.Now().UnixNano())
@@ -68,27 +71,41 @@ var _ = BeforeSuite(func() {
 	Logf("Running in project %v with service account %v\n\n", *project, *serviceAccount)
 
 	for _, zone := range zones {
-		nodeID := fmt.Sprintf("gce-pd-csi-e2e-%s", zone)
-
-		i, err := remote.SetupInstance(*project, zone, nodeID, *serviceAccount, computeService)
-		Expect(err).To(BeNil())
-
-		testInstances = append(testInstances, i)
+		go func(curZone string) {
+			defer GinkgoRecover()
+			nodeID := fmt.Sprintf("gce-pd-csi-e2e-%s", curZone)
+			Logf("Setting up node %s\n", nodeID)
+
+			i, err := remote.SetupInstance(*project, curZone, nodeID, *serviceAccount, computeService)
+			Expect(err).To(BeNil())
+
+			// Create new driver and client
+			testContext, err := testutils.GCEClientAndDriverSetup(i)
+			Expect(err).To(BeNil(), "Set up new Driver and Client failed with error")
+			tcc <- testContext
+		}(zone)
 	}
 
+	for i := 0; i < len(zones); i++ {
+		tc := <-tcc
+		Logf("Test Context for node %s set up\n", tc.Instance.GetName())
+		testContexts = append(testContexts, tc)
+	}
 })
 
 var _ = AfterSuite(func() {
-	/*
-		err := node.client.CloseConn()
-		if err != nil {
-			Logf("Failed to close the client")
-		} else {
-			Logf("Closed the client")
-	*/
-	for _, i := range testInstances {
+
+	for _, tc := range testContexts {
+		err := remote.TeardownDriverAndClient(tc)
+		Expect(err).To(BeNil(), "Teardown Driver and Client failed with error")
 		if *deleteInstances {
-			i.DeleteInstance()
+			tc.Instance.DeleteInstance()
 		}
 	}
 })
+
+func getRandomTestContext() *remote.TestContext {
+	Expect(testContexts).ToNot(BeEmpty())
+	rn := rand.Intn(len(testContexts))
+	return testContexts[rn]
+}

test/e2e/tests/single_zone_e2e_test.go

@@ -21,8 +21,6 @@ import (
 	"k8s.io/apimachinery/pkg/util/uuid"
 	"sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common"
 	gce "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider/compute"
-	testutils "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/e2e/utils"
-	remote "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/remote"
 
 	csi "github.com/container-storage-interface/spec/lib/go/csi/v0"
 	. "github.com/onsi/ginkgo"
@@ -41,22 +39,15 @@ const (
 var _ = Describe("GCE PD CSI Driver", func() {
 
 	It("Should create->attach->stage->mount volume and check if it is writable, then unmount->unstage->detach->delete and check disk is deleted", func() {
-		// Create new driver and client
-		Expect(testInstances).NotTo(BeEmpty())
-		testContext, err := testutils.GCEClientAndDriverSetup(testInstances[0])
-		Expect(err).To(BeNil(), "Set up new Driver and Client failed with error")
-		defer func() {
-			err := remote.TeardownDriverAndClient(testContext)
-			Expect(err).To(BeNil(), "Teardown Driver and Client failed with error")
-		}()
+		testContext := getRandomTestContext()
 
 		p, z, _ := testContext.Instance.GetIdentity()
 		client := testContext.Client
 		instance := testContext.Instance
 
 		// Create Disk
 		volName := testNamePrefix + string(uuid.NewUUID())
-		volId, err := client.CreateVolume(volName, nil, defaultSizeGb,
+		volID, err := client.CreateVolume(volName, nil, defaultSizeGb,
 			&csi.TopologyRequirement{
 				Requisite: []*csi.Topology{
 					{
@@ -76,7 +67,7 @@ var _ = Describe("GCE PD CSI Driver", func() {
 
 		defer func() {
 			// Delete Disk
-			client.DeleteVolume(volId)
+			client.DeleteVolume(volID)
 			Expect(err).To(BeNil(), "DeleteVolume failed")
 
 			// Validate Disk Deleted
@@ -85,19 +76,13 @@ var _ = Describe("GCE PD CSI Driver", func() {
 		}()
 
 		// Attach Disk
-		testAttachWriteReadDetach(volId, volName, instance, client, false /* readOnly */)
+		testAttachWriteReadDetach(volID, volName, instance, client, false /* readOnly */)
 
 	})
 
 	It("Should create disks in correct zones when topology is specified", func() {
-		///
-		Expect(testInstances).NotTo(BeEmpty())
-		testContext, err := testutils.GCEClientAndDriverSetup(testInstances[0])
-		Expect(err).To(BeNil(), "Failed to set up new driver and client")
-		defer func() {
-			err := remote.TeardownDriverAndClient(testContext)
-			Expect(err).To(BeNil(), "Teardown Driver and Client failed with error")
-		}()
+		Expect(testContexts).ToNot(BeEmpty())
+		testContext := getRandomTestContext()
 
 		p, _, _ := testContext.Instance.GetIdentity()
 
@@ -126,14 +111,8 @@ var _ = Describe("GCE PD CSI Driver", func() {
 	})
 
 	It("Should successfully create RePD in two zones in the drivers region when none are specified", func() {
-		// Create new driver and client
-		Expect(testInstances).NotTo(BeEmpty())
-		testContext, err := testutils.GCEClientAndDriverSetup(testInstances[0])
-		Expect(err).To(BeNil(), "Failed to set up new driver and client")
-		defer func() {
-			err := remote.TeardownDriverAndClient(testContext)
-			Expect(err).To(BeNil(), "Teardown Driver and Client failed with error")
-		}()
+		Expect(testContexts).ToNot(BeEmpty())
+		testContext := getRandomTestContext()
 
 		controllerInstance := testContext.Instance
 		controllerClient := testContext.Client
@@ -145,7 +124,7 @@ var _ = Describe("GCE PD CSI Driver", func() {
 
 		// Create Disk
 		volName := testNamePrefix + string(uuid.NewUUID())
-		volId, err := controllerClient.CreateVolume(volName, map[string]string{
+		volID, err := controllerClient.CreateVolume(volName, map[string]string{
 			common.ParameterKeyReplicationType: "regional-pd",
 		}, defaultSizeGb, nil)
 		Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err)
@@ -167,7 +146,7 @@ var _ = Describe("GCE PD CSI Driver", func() {
 		}
 		defer func() {
 			// Delete Disk
-			controllerClient.DeleteVolume(volId)
+			controllerClient.DeleteVolume(volID)
 			Expect(err).To(BeNil(), "DeleteVolume failed")
 
 			// Validate Disk Deleted
@@ -177,21 +156,15 @@ var _ = Describe("GCE PD CSI Driver", func() {
 	})
 
 	It("Should create and delete disk with default zone", func() {
-		// Create new driver and client
-		Expect(testInstances).NotTo(BeEmpty())
-		testContext, err := testutils.GCEClientAndDriverSetup(testInstances[0])
-		Expect(err).To(BeNil(), "Set up new Driver and Client failed with error")
-		defer func() {
-			err := remote.TeardownDriverAndClient(testContext)
-			Expect(err).To(BeNil(), "Teardown Driver and Client failed with error")
-		}()
+		Expect(testContexts).ToNot(BeEmpty())
+		testContext := getRandomTestContext()
 
 		p, z, _ := testContext.Instance.GetIdentity()
 		client := testContext.Client
 
 		// Create Disk
 		volName := testNamePrefix + string(uuid.NewUUID())
-		volId, err := client.CreateVolume(volName, nil, defaultSizeGb, nil)
+		volID, err := client.CreateVolume(volName, nil, defaultSizeGb, nil)
 		Expect(err).To(BeNil(), "CreateVolume failed with error: %v", err)
 
 		// Validate Disk Created
@@ -204,7 +177,7 @@ var _ = Describe("GCE PD CSI Driver", func() {
 
 		defer func() {
 			// Delete Disk
-			client.DeleteVolume(volId)
+			client.DeleteVolume(volID)
 			Expect(err).To(BeNil(), "DeleteVolume failed")
 
 			// Validate Disk Deleted

test/remote/instance.go

@@ -88,8 +88,7 @@ func (i *InstanceInfo) CreateOrGetInstance(serviceAccount string) error {
 		return fmt.Errorf("Failed to create firewall rule: %v", err)
 	}
 
-	// TODO(#97): Pick a better boot disk image
-	imageURL := "projects/ml-images/global/images/family/tf-1-9"
+	imageURL := "projects/ubuntu-os-cloud/global/images/family/ubuntu-minimal-1804-lts"
 	inst := &compute.Instance{
 		Name:        i.name,
 		MachineType: machineType(i.zone, ""),

test/remote/setup-teardown.go

@@ -22,10 +22,7 @@ import (
 
 	"github.com/golang/glog"
 	compute "google.golang.org/api/compute/v1"
-)
-
-const (
-	archiveName = "e2e_driver_binaries.tar.gz"
+	"k8s.io/apimachinery/pkg/util/uuid"
 )
 
 // TestContext holds the CSI Client handle to a remotely connected Driver
@@ -75,6 +72,7 @@ func SetupInstance(instanceProject, instanceZone, instanceName, instanceServiceA
 // a CSI client to it through SHH tunnelling. It returns a TestContext with both a handle to the instance
 // that the driver is on and the CSI Client object to make CSI calls to the remote driver.
 func SetupNewDriverAndClient(instance *InstanceInfo, config *ClientConfig) (*TestContext, error) {
+	archiveName := fmt.Sprintf("e2e_driver_binaries_%s.tar.gz", uuid.NewUUID())
 	archivePath, err := CreateDriverArchive(archiveName, config.PkgPath, config.BinPath)
 	if err != nil {
 		return nil, err