Test infrastructure changes to support multizone and topology

Author: davidz627

test/binremote/instance.go

@@ -46,27 +46,33 @@ type InstanceInfo struct {
 	zone    string
 	name    string
 
+	// External IP is filled in after instance creation
+	// TODO: Maybe combine create instance and create instance info so all fields are set up at the same time...
 	externalIP string
 
 	computeService *compute.Service
 }
 
-func CreateInstanceInfo(project, zone, name string) (*InstanceInfo, error) {
-	cs, err := getComputeClient()
-	if err != nil {
-		return nil, err
-	}
+func (i *InstanceInfo) GetIdentity() (string, string, string) {
+	return i.project, i.zone, i.name
+}
+
+func (i *InstanceInfo) GetName() string {
+	return i.name
+}
+
+func CreateInstanceInfo(project, instanceZone, name string, cs *compute.Service) (*InstanceInfo, error) {
 	return &InstanceInfo{
 		project: project,
-		zone:    zone,
+		zone:    instanceZone,
 		name:    name,
 
 		computeService: cs,
 	}, nil
 }
 
 // Provision a gce instance using image
-func (i *InstanceInfo) CreateInstance(serviceAccount string) error {
+func (i *InstanceInfo) CreateOrGetInstance(serviceAccount string) error {
 	var err error
 	var instance *compute.Instance
 	glog.V(4).Infof("Creating instance: %v", i.name)
@@ -236,7 +242,7 @@ func (i *InstanceInfo) createDefaultFirewallRule() error {
 	return nil
 }
 
-func getComputeClient() (*compute.Service, error) {
+func GetComputeClient() (*compute.Service, error) {
 	const retries = 10
 	const backoff = time.Second * 6
 

test/e2e/utils/client_wrappers.go

@@ -87,11 +87,14 @@ func (c *CsiClient) CloseConn() error {
 	return c.conn.Close()
 }
 
-func (c *CsiClient) CreateVolume(volName string) (string, error) {
+func (c *CsiClient) CreateVolume(volName string, topReq *csipb.TopologyRequirement) (string, error) {
 	cvr := &csipb.CreateVolumeRequest{
 		Name:               volName,
 		VolumeCapabilities: stdVolCaps,
 	}
+	if topReq != nil {
+		cvr.AccessibilityRequirements = topReq
+	}
 	cresp, err := c.ctrlClient.CreateVolume(context.Background(), cvr)
 	if err != nil {
 		return "", err
@@ -166,3 +169,8 @@ func (c *CsiClient) NodePublishVolume(volumeId, stageDir, publishDir string) err
 	_, err := c.nodeClient.NodePublishVolume(context.Background(), nodePublishReq)
 	return err
 }
+
+func (c *CsiClient) NodeGetInfo() (*csipb.NodeGetInfoResponse, error) {
+	resp, err := c.nodeClient.NodeGetInfo(context.Background(), &csipb.NodeGetInfoRequest{})
+	return resp, err
+}

test/e2e/utils/utils.go

@@ -17,13 +17,15 @@ package utils
 import (
 	"context"
 	"fmt"
+	"math/rand"
 	"os"
 	"path"
 	"time"
 
 	"github.com/golang/glog"
 	"golang.org/x/oauth2/google"
 	cloudresourcemanager "google.golang.org/api/cloudresourcemanager/v1"
+	compute "google.golang.org/api/compute/v1"
 	boskosclient "k8s.io/test-infra/boskos/client"
 	remote "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/binremote"
 )
@@ -36,21 +38,28 @@ const (
 	archiveName = "e2e_gce_pd_test.tar.gz"
 )
 
-func SetupInstanceAndDriver(instanceProject, instanceZone, instanceName, port, instanceServiceAccount string) (*remote.InstanceInfo, error) {
+type TestContext struct {
+	Instance *remote.InstanceInfo
+	Client   *CsiClient
+}
+
+func SetupInstance(instanceProject, instanceZone, instanceName, instanceServiceAccount string, cs *compute.Service) (*remote.InstanceInfo, error) {
 	// Create the instance in the requisite zone
-	instance, err := remote.CreateInstanceInfo(instanceProject, instanceZone, instanceName)
+	instance, err := remote.CreateInstanceInfo(instanceProject, instanceZone, instanceName, cs)
 	if err != nil {
 		return nil, err
 	}
 
-	err = instance.CreateInstance(instanceServiceAccount)
-
+	err = instance.CreateOrGetInstance(instanceServiceAccount)
 	if err != nil {
 		return nil, err
 	}
+	return instance, nil
+}
 
-	// Create Driver Archive
-
+// TODO: Need a function to clean up this driver from the instance
+func SetupNewDriverAndClient(instance *remote.InstanceInfo) (*TestContext, error) {
+	port := fmt.Sprintf("%v", 1024+rand.Intn(10000))
 	goPath, ok := os.LookupEnv("GOPATH")
 	if !ok {
 		return nil, fmt.Errorf("Could not find environment variable GOPATH")
@@ -72,7 +81,7 @@ func SetupInstanceAndDriver(instanceProject, instanceZone, instanceName, port, i
 	endpoint := fmt.Sprintf("tcp://localhost:%s", port)
 	workspace := remote.NewWorkspaceDir("gce-pd-e2e-")
 	driverRunCmd := fmt.Sprintf("sh -c '/usr/bin/nohup %s/gce-pd-csi-driver --endpoint=%s --nodeid=%s > %s/prog.out 2> %s/prog.err < /dev/null &'",
-		workspace, endpoint, instanceName, workspace, workspace)
+		workspace, endpoint, instance.GetName(), workspace, workspace)
 	err = instance.UploadAndRun(archivePath, workspace, driverRunCmd)
 	if err != nil {
 		return nil, err
@@ -84,7 +93,13 @@ func SetupInstanceAndDriver(instanceProject, instanceZone, instanceName, port, i
 		return nil, fmt.Errorf("SSH Tunnel pid %v encountered error: %v", res, err)
 	}
 
-	return instance, nil
+	client := CreateCSIClient(fmt.Sprintf("localhost:%s", port))
+	err = client.AssertCSIConnection()
+	if err != nil {
+		return nil, fmt.Errorf("asserting csi connection failed with: %v", err)
+	}
+
+	return &TestContext{Instance: instance, Client: client}, nil
 }
 
 func SetupProwConfig() (project, serviceAccount string) {

test/run-e2e-local.sh

@@ -5,4 +5,4 @@ set -o errexit
 
 readonly PKGDIR=sigs.k8s.io/gcp-compute-persistent-disk-csi-driver
 
-go test --v=true "${PKGDIR}/test/e2e/tests" --logtostderr --project ${PROJECT} --service-account ${IAM_NAME}
+ginkgo -v "test/e2e/tests" --logtostderr -- --project ${PROJECT} --service-account ${IAM_NAME}

test/run-e2e.sh

@@ -5,4 +5,4 @@ set -x
 
 readonly PKGDIR=sigs.k8s.io/gcp-compute-persistent-disk-csi-driver
 
-go test --v=true "${PKGDIR}/test/e2e/tests" --logtostderr --run-in-prow=true
+go test --v=true "${PKGDIR}/test/e2e/tests" --logtostderr --run-in-prow=true --delete-instances=true