Refactored remote test framework for easy usage. Added teardown methods

test/e2e/tests/multi_zone_e2e_test.go

@@ -19,6 +19,7 @@ import (
 	. "github.com/onsi/gomega"
 	"sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common"
 	testutils "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/e2e/utils"
+	remote "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/remote"
 )
 
 var _ = Describe("GCE PD CSI Driver Multi-Zone", func() {
@@ -30,8 +31,12 @@ var _ = Describe("GCE PD CSI Driver Multi-Zone", func() {
 
 	It("Should get reasonable topology from nodes with NodeGetInfo", func() {
 		for _, instance := range testInstances {
-			testContext, err := testutils.SetupNewDriverAndClient(instance)
+			testContext, err := testutils.GCEClientAndDriverSetup(instance)
 			Expect(err).To(BeNil(), "Set up new Driver and Client failed with error")
+			defer func() {
+				err := remote.TeardownDriverAndClient(testContext)
+				Expect(err).To(BeNil(), "Teardown Driver and Client failed with error")
+			}()
 
 			resp, err := testContext.Client.NodeGetInfo()
 			Expect(err).To(BeNil())

test/e2e/tests/setup_e2e_test.go

@@ -24,8 +24,8 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	compute "google.golang.org/api/compute/v1"
-	remote "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/binremote"
 	testutils "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/e2e/utils"
+	remote "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/remote"
 )
 
 var (
@@ -63,7 +63,7 @@ var _ = BeforeSuite(func() {
 		Expect(*project).ToNot(BeEmpty(), "Project should not be empty")
 		Expect(*serviceAccount).ToNot(BeEmpty(), "Service account should not be empty")
 
-		i, err := testutils.SetupInstance(*project, zone, nodeID, *serviceAccount, computeService)
+		i, err := remote.SetupInstance(*project, zone, nodeID, *serviceAccount, computeService)
 		Expect(err).To(BeNil())
 
 		testInstances = append(testInstances, i)

test/e2e/tests/single_zone_e2e_test.go

@@ -23,6 +23,7 @@ import (
 	"sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common"
 	gce "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider/compute"
 	testutils "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/e2e/utils"
+	remote "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/remote"
 
 	csi "github.com/container-storage-interface/spec/lib/go/csi/v0"
 	. "github.com/onsi/ginkgo"
@@ -44,8 +45,13 @@ var _ = Describe("GCE PD CSI Driver", func() {
 		// Create new driver and client
 		// TODO: Should probably actual have some object that includes both client and instance so we can relate the two??
 		Expect(testInstances).NotTo(BeEmpty())
-		testContext, err := testutils.SetupNewDriverAndClient(testInstances[0])
+		testContext, err := testutils.GCEClientAndDriverSetup(testInstances[0])
 		Expect(err).To(BeNil(), "Set up new Driver and Client failed with error")
+		defer func() {
+			err := remote.TeardownDriverAndClient(testContext)
+			Expect(err).To(BeNil(), "Teardown Driver and Client failed with error")
+		}()
+
 		p, z, _ := testContext.Instance.GetIdentity()
 		client := testContext.Client
 		instance := testContext.Instance
@@ -135,8 +141,13 @@ var _ = Describe("GCE PD CSI Driver", func() {
 	It("Should create disks in correct zones when topology is specified", func() {
 		///
 		Expect(testInstances).NotTo(BeEmpty())
-		testContext, err := testutils.SetupNewDriverAndClient(testInstances[0])
+		testContext, err := testutils.GCEClientAndDriverSetup(testInstances[0])
 		Expect(err).To(BeNil(), "Failed to set up new driver and client")
+		defer func() {
+			err := remote.TeardownDriverAndClient(testContext)
+			Expect(err).To(BeNil(), "Teardown Driver and Client failed with error")
+		}()
+
 		p, _, _ := testContext.Instance.GetIdentity()
 
 		zones := []string{"us-central1-c", "us-central1-b", "us-central1-a"}

test/e2e/utils/utils.go

@@ -25,81 +25,38 @@ import (
 	"github.com/golang/glog"
 	"golang.org/x/oauth2/google"
 	cloudresourcemanager "google.golang.org/api/cloudresourcemanager/v1"
-	compute "google.golang.org/api/compute/v1"
 	boskosclient "k8s.io/test-infra/boskos/client"
-	remote "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/binremote"
+	remote "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/remote"
 )
 
 var (
 	boskos = boskosclient.NewClient(os.Getenv("JOB_NAME"), "http://boskos")
 )
 
-const (
-	archiveName = "e2e_gce_pd_test.tar.gz"
-)
-
-type TestContext struct {
-	Instance *remote.InstanceInfo
-	Client   *CsiClient
-}
-
-func SetupInstance(instanceProject, instanceZone, instanceName, instanceServiceAccount string, cs *compute.Service) (*remote.InstanceInfo, error) {
-	// Create the instance in the requisite zone
-	instance, err := remote.CreateInstanceInfo(instanceProject, instanceZone, instanceName, cs)
-	if err != nil {
-		return nil, err
-	}
-
-	err = instance.CreateOrGetInstance(instanceServiceAccount)
-	if err != nil {
-		return nil, err
-	}
-	return instance, nil
-}
-
-// TODO: Need a function to clean up this driver from the instance
-func SetupNewDriverAndClient(instance *remote.InstanceInfo) (*TestContext, error) {
+func GCEClientAndDriverSetup(instance *remote.InstanceInfo) (*remote.TestContext, error) {
 	port := fmt.Sprintf("%v", 1024+rand.Intn(10000))
 	goPath, ok := os.LookupEnv("GOPATH")
 	if !ok {
 		return nil, fmt.Errorf("Could not find environment variable GOPATH")
 	}
 	pkgPath := path.Join(goPath, "src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/")
 	binPath := path.Join(pkgPath, "bin/gce-pd-csi-driver")
-	archivePath, err := remote.CreateDriverArchive(archiveName, pkgPath, binPath)
-	if err != nil {
-		return nil, err
-	}
-	defer func() {
-		err = os.Remove(archivePath)
-		if err != nil {
-			glog.Warningf("Failed to remove archive file %s: %v", archivePath, err)
-		}
-	}()
 
-	// Upload archive to instance and run binaries
 	endpoint := fmt.Sprintf("tcp://localhost:%s", port)
+
 	workspace := remote.NewWorkspaceDir("gce-pd-e2e-")
 	driverRunCmd := fmt.Sprintf("sh -c '/usr/bin/nohup %s/gce-pd-csi-driver --endpoint=%s --nodeid=%s > %s/prog.out 2> %s/prog.err < /dev/null &'",
 		workspace, endpoint, instance.GetName(), workspace, workspace)
-	err = instance.UploadAndRun(archivePath, workspace, driverRunCmd)
-	if err != nil {
-		return nil, err
-	}
 
-	// Create an SSH tunnel from port to port
-	res, err := instance.CreateSSHTunnel(port, port)
-	if err != nil {
-		return nil, fmt.Errorf("SSH Tunnel pid %v encountered error: %v", res, err)
-	}
-
-	client := CreateCSIClient(fmt.Sprintf("localhost:%s", port))
-	err = client.AssertCSIConnection()
-	if err != nil {
-		return nil, fmt.Errorf("asserting csi connection failed with: %v", err)
+	config := &remote.ClientConfig{
+		PkgPath:      pkgPath,
+		BinPath:      binPath,
+		WorkspaceDir: workspace,
+		RunDriverCmd: driverRunCmd,
+		Port:         port,
 	}
 
-	return &TestContext{Instance: instance, Client: client}, nil
+	return remote.SetupNewDriverAndClient(instance, config)
 }
 
 func SetupProwConfig() (project, serviceAccount string) {

test/binremote/archiver.go → test/remote/archiver.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package binremote
+package remote
 
 import (
 	"fmt"

test/e2e/utils/client_wrappers.go → test/remote/client-wrappers.go

@@ -12,7 +12,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package utils
+package remote
 
 import (
 	"context"

test/binremote/instance.go → test/remote/instance.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package binremote
+package remote
 
 import (
 	"context"

test/binremote/runner.go → test/remote/runner.go

@@ -14,31 +14,33 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package binremote
+package remote
 
 import (
 	"fmt"
 	"path"
 	"path/filepath"
+	"strconv"
+	"strings"
 
 	"github.com/golang/glog"
 )
 
-func (i *InstanceInfo) UploadAndRun(archivePath, remoteWorkspace, driverRunCmd string) error {
+func (i *InstanceInfo) UploadAndRun(archivePath, remoteWorkspace, driverRunCmd string) (int, error) {
 
 	// Create the temp staging directory
 	glog.V(4).Infof("Staging test binaries on %q", i.name)
 
 	// Do not sudo here, so that we can use scp to copy test archive to the directdory.
 	if output, err := i.SSHNoSudo("mkdir", remoteWorkspace); err != nil {
 		// Exit failure with the error
-		return fmt.Errorf("failed to create remoteWorkspace directory %q on i.name %q: %v output: %q", remoteWorkspace, i.name, err, output)
+		return -1, fmt.Errorf("failed to create remoteWorkspace directory %q on i.name %q: %v output: %q", remoteWorkspace, i.name, err, output)
 	}
 
 	// Copy the archive to the staging directory
 	if output, err := runSSHCommand("scp", archivePath, fmt.Sprintf("%s:%s/", i.GetSSHTarget(), remoteWorkspace)); err != nil {
 		// Exit failure with the error
-		return fmt.Errorf("failed to copy test archive: %v, output: %q", err, output)
+		return -1, fmt.Errorf("failed to copy test archive: %v, output: %q", err, output)
 	}
 
 	// Extract the archive
@@ -52,22 +54,40 @@ func (i *InstanceInfo) UploadAndRun(archivePath, remoteWorkspace, driverRunCmd s
 	// we want the extracted files to be owned by the current user.
 	if output, err := i.SSHNoSudo("sh", "-c", cmd); err != nil {
 		// Exit failure with the error
-		return fmt.Errorf("failed to extract test archive: %v, output: %q", err, output)
+		return -1, fmt.Errorf("failed to extract test archive: %v, output: %q", err, output)
 	}
 
 	glog.V(4).Infof("Starting driver on %q", i.name)
 	// When the process is killed the driver should close the TCP endpoint, then we want to download the logs
 	output, err := i.SSH(driverRunCmd)
+	if err != nil {
+		// Exit failure with the error
+		return -1, fmt.Errorf("failed start driver, got output: %v, error: %v", output, err)
+	}
 
+	// Get the driver PID
+	// ps -aux | grep  /tmp/gce-pd-e2e-0180801T114407/gce-pd-csi-driver | awk '{print $2}'
+	driverPIDCmd := getSSHCommand(" | ",
+		"ps -aux",
+		fmt.Sprintf("grep %s", remoteWorkspace),
+		"grep -v grep",
+		// All ye who try to deal with escaped/non-escaped quotes with exec beware.
+		//`awk "{print \$2}"`,
+	)
+	driverPIDString, err := i.SSHNoSudo("sh", "-c", driverPIDCmd)
 	if err != nil {
 		// Exit failure with the error
-		return fmt.Errorf("failed start GCE PD driver, got output: %v, error: %v", output, err)
+		return -1, fmt.Errorf("failed to get PID of driver, got output: %v, error: %v", output, err)
 	}
 
+	driverPID, err := strconv.Atoi(strings.Fields(driverPIDString)[1])
+	if err != nil {
+		return -1, fmt.Errorf("failed to convert driver PID from string %s to int: %v", driverPIDString, err)
+	}
 	// TODO: return the PID so that we can kill the driver later
 	// Actually just do a pkill -f my_pattern
 
-	return nil
+	return driverPID, nil
 }
 
 func NewWorkspaceDir(workspaceDirPrefix string) string {