Made framework more configurable by removing constants and gce-pd specgcepd specific code into variables

Author: davidz627

Makefile

@@ -23,7 +23,6 @@ all: gce-pd-driver
 gce-pd-driver:
 	mkdir -p bin
 	go build -ldflags "-X main.vendorVersion=${STAGINGVERSION}" -o bin/gce-pd-csi-driver ./cmd/
-	#go test -c sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/e2e -o bin/e2e.test
 
 build-container:
 	docker build --build-arg TAG=$(STAGINGVERSION) -t $(STAGINGIMAGE):$(STAGINGVERSION) .

pkg/gce-pd-csi-driver/server.go

@@ -114,7 +114,7 @@ func (s *nonBlockingGRPCServer) serve(endpoint string, ids csi.IdentityServer, c
 	glog.Infof("Listening for connections on address: %#v", listener.Addr())
 
 	if err := server.Serve(listener); err != nil {
-		glog.Warningf("Failed to serve: %v", err)
+		glog.Fatalf("Failed to serve: %v", err)
 	}
 
 }

test/binremote/binremote/archiver.go renamed to test/binremote/archiver.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2016 The Kubernetes Authors.
+Copyright 2018 The Kubernetes Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -21,22 +21,21 @@ import (
 	"io/ioutil"
 	"os"
 	"os/exec"
-	"path"
 	"path/filepath"
 
 	"github.com/golang/glog"
 )
 
-func CreateDriverArchive(archiveName string) (string, error) {
+func CreateDriverArchive(archiveName, pkgPath, binPath string) (string, error) {
 	glog.V(2).Infof("Building archive...")
-	tarDir, err := ioutil.TempDir("", "gce-pd-archive")
+	tarDir, err := ioutil.TempDir("", "driver-temp-archive")
 	if err != nil {
 		return "", fmt.Errorf("failed to create temporary directory %v", err)
 	}
 	defer os.RemoveAll(tarDir)
 
 	// Call the suite function to setup the test package.
-	err = setupBinaries(tarDir)
+	err = setupBinaries(tarDir, pkgPath, binPath)
 	if err != nil {
 		return "", fmt.Errorf("failed to setup test package %q: %v", tarDir, err)
 	}
@@ -54,28 +53,20 @@ func CreateDriverArchive(archiveName string) (string, error) {
 	return filepath.Join(dir, archiveName), nil
 }
 
-func setupBinaries(tarDir string) error {
-	// TODO(dyzz): build the gce driver tests instead
-	goPath, ok := os.LookupEnv("GOPATH")
-	if !ok {
-		return fmt.Errorf("Could not find environment variable GOPATH")
-	}
-	binPath := path.Join(goPath, "src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/bin/gce-pd-csi-driver")
-
-	// TODO make the driver but from the base dir
-	out, err := exec.Command("go", "build", "-ldflags", "-X main.vendorVersion=latest", "-o", binPath, "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/cmd").CombinedOutput()
+func setupBinaries(tarDir, pkgPath, binPath string) error {
+	glog.V(4).Infof("Making binaries and copying to temp dir...")
+	out, err := exec.Command("make", "-C", pkgPath).CombinedOutput()
 	if err != nil {
-		return fmt.Errorf("Failed to make gce-pd-driver: %v: %v", string(out), err)
+		return fmt.Errorf("Failed to make at %s: %v: %v", pkgPath, string(out), err)
 	}
 
 	// Copy binaries
-
 	if _, err := os.Stat(binPath); err != nil {
-		return fmt.Errorf("failed to locate test binary %s: %v", "gce-pd-csi-driver", err)
+		return fmt.Errorf("failed to locate test binary %s: %v", binPath, err)
 	}
-	out, err = exec.Command("cp", binPath, filepath.Join(tarDir, "gce-pd-csi-driver")).CombinedOutput()
+	out, err = exec.Command("cp", binPath, tarDir).CombinedOutput()
 	if err != nil {
-		return fmt.Errorf("failed to copy %q: %v Output: %q", "gce-pd-csi-driver", err, out)
+		return fmt.Errorf("failed to copy %q: %v Output: %q", binPath, err, out)
 	}
 
 	return nil

test/binremote/binremote/instance.go renamed to test/binremote/instance.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2016 The Kubernetes Authors.
+Copyright 2018 The Kubernetes Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -36,6 +36,9 @@ import (
 const (
 	defaultMachine      = "n1-standard-1"
 	defaultFirewallRule = "default-allow-ssh"
+
+	// timestampFormat is the timestamp format used in the e2e directory name.
+	timestampFormat = "20060102T150405"
 )
 
 type InstanceInfo struct {
@@ -66,16 +69,15 @@ func CreateInstanceInfo(project, zone, name string) (*InstanceInfo, error) {
 func (i *InstanceInfo) CreateInstance(serviceAccount string) error {
 	var err error
 	var instance *compute.Instance
+	glog.V(4).Infof("Creating instance: %v", i.name)
 
 	myuuid := string(uuid.NewUUID())
 
-	err = i.CreateDefaultFirewallRule()
+	err = i.createDefaultFirewallRule()
 	if err != nil {
 		return fmt.Errorf("Failed to create firewall rule: %v", err)
 	}
 
-	glog.V(4).Infof("Creating instance: %v", i.name)
-
 	// TODO: Pick a better boot disk image
 	imageURL := "projects/ml-images/global/images/family/tf-1-9"
 	inst := &compute.Instance{
@@ -197,13 +199,6 @@ func getexternalIP(instance *compute.Instance) string {
 	return ""
 }
 
-const (
-	// workspaceDirPrefix is the string prefix used in the workspace directory name.
-	workspaceDirPrefix = "gce-pd-e2e-"
-	// timestampFormat is the timestamp format used in the e2e directory name.
-	timestampFormat = "20060102T150405"
-)
-
 func getTimestamp() string {
 	return fmt.Sprintf(time.Now().Format(timestampFormat))
 }
@@ -216,8 +211,10 @@ func machineType(zone, machine string) string {
 }
 
 // Create default SSH filewall rule if it does not exist
-func (i *InstanceInfo) CreateDefaultFirewallRule() error {
+func (i *InstanceInfo) createDefaultFirewallRule() error {
 	var err error
+	glog.V(4).Infof("Creating default firewall rule %s...", defaultFirewallRule)
+
 	if _, err = i.computeService.Firewalls.Get(i.project, defaultFirewallRule).Do(); err != nil {
 		glog.Infof("Default firewall rule %v does not exist, creating", defaultFirewallRule)
 		f := &compute.Firewall{
@@ -243,6 +240,8 @@ func getComputeClient() (*compute.Service, error) {
 	const retries = 10
 	const backoff = time.Second * 6
 
+	glog.V(4).Infof("Getting compute client...")
+
 	// Setup the gce client for provisioning instances
 	// Getting credentials on gce jenkins is flaky, so try a couple times
 	var err error

test/binremote/binremote/runner.go renamed to test/binremote/runner.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2016 The Kubernetes Authors.
+Copyright 2018 The Kubernetes Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -18,46 +18,46 @@ package binremote
 
 import (
 	"fmt"
+	"path"
 	"path/filepath"
 
 	"github.com/golang/glog"
 )
 
-func (i *InstanceInfo) UploadAndRun(archivePath, archiveName, instanceName, port string) error {
-	endpoint := fmt.Sprintf("tcp://localhost:%s", port)
-	workspace := newWorkspaceDir()
+func (i *InstanceInfo) UploadAndRun(archivePath, remoteWorkspace, driverRunCmd string) error {
 
 	// Create the temp staging directory
-	glog.V(2).Infof("Staging test binaries on %q", i.name)
+	glog.V(4).Infof("Staging test binaries on %q", i.name)
 
 	// Do not sudo here, so that we can use scp to copy test archive to the directdory.
-	if output, err := i.SSHNoSudo("mkdir", workspace); err != nil {
+	if output, err := i.SSHNoSudo("mkdir", remoteWorkspace); err != nil {
 		// Exit failure with the error
-		return fmt.Errorf("failed to create workspace directory %q on i.name %q: %v output: %q", workspace, i.name, err, output)
+		return fmt.Errorf("failed to create remoteWorkspace directory %q on i.name %q: %v output: %q", remoteWorkspace, i.name, err, output)
 	}
 
 	// Copy the archive to the staging directory
-	if output, err := runSSHCommand("scp", archivePath, fmt.Sprintf("%s:%s/", i.GetSSHTarget(), workspace)); err != nil {
+	if output, err := runSSHCommand("scp", archivePath, fmt.Sprintf("%s:%s/", i.GetSSHTarget(), remoteWorkspace)); err != nil {
 		// Exit failure with the error
 		return fmt.Errorf("failed to copy test archive: %v, output: %q", err, output)
 	}
 
 	// Extract the archive
+	archiveName := path.Base(archivePath)
 	cmd := getSSHCommand(" && ",
-		fmt.Sprintf("cd %s", workspace),
+		fmt.Sprintf("cd %s", remoteWorkspace),
 		fmt.Sprintf("tar -xzvf ./%s", archiveName),
 	)
-	glog.V(2).Infof("Extracting tar on %q", i.name)
+	glog.V(4).Infof("Extracting tar on %q", i.name)
 	// Do not use sudo here, because `sudo tar -x` will recover the file ownership inside the tar ball, but
 	// we want the extracted files to be owned by the current user.
 	if output, err := i.SSHNoSudo("sh", "-c", cmd); err != nil {
 		// Exit failure with the error
 		return fmt.Errorf("failed to extract test archive: %v, output: %q", err, output)
 	}
 
-	glog.V(2).Infof("Starting driver on %q", i.name)
+	glog.V(4).Infof("Starting driver on %q", i.name)
 	// When the process is killed the driver should close the TCP endpoint, then we want to download the logs
-	output, err := i.SSH(fmt.Sprintf("sh -c '/usr/bin/nohup %s/gce-pd-csi-driver --endpoint=%s --nodeid=%s > %s/prog.out 2> %s/prog.err < /dev/null &'", workspace, endpoint, instanceName, workspace, workspace))
+	output, err := i.SSH(driverRunCmd)
 
 	if err != nil {
 		// Exit failure with the error
@@ -70,6 +70,6 @@ func (i *InstanceInfo) UploadAndRun(archivePath, archiveName, instanceName, port
 	return nil
 }
 
-func newWorkspaceDir() string {
+func NewWorkspaceDir(workspaceDirPrefix string) string {
 	return filepath.Join("/tmp", workspaceDirPrefix+getTimestamp())
 }

test/binremote/binremote/ssh.go renamed to test/binremote/ssh.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2016 The Kubernetes Authors.
+Copyright 2018 The Kubernetes Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -102,7 +102,7 @@ func runSSHCommand(cmd string, args ...string) (string, error) {
 	}
 	args = append(strings.Split(sshOption, " "), args...)
 
-	glog.V(4).Infof("Executing command: %v %v", cmd, args)
+	glog.V(4).Infof("Executing SSH command: %v %v", cmd, args)
 
 	output, err := exec.Command(cmd, args...).CombinedOutput()
 	if err != nil {

test/e2e/setup_e2e_test.go

@@ -0,0 +1,35 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"flag"
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var (
+	project        = flag.String("project", "", "Project to run tests in")
+	serviceAccount = flag.String("service-account", "", "Service account to bring up instance with")
+	runInProw      = flag.Bool("run-in-prow", false, "If true, use a Boskos loaned project and special CI service accounts and ssh keys")
+)
+
+func TestE2E(t *testing.T) {
+	flag.Parse()
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Google Compute Engine Persistent Disk Container Storage Interface Driver Tests")
+}

test/e2e/single_zone_e2e_test.go

@@ -15,14 +15,12 @@ limitations under the License.
 package e2e
 
 import (
-	"flag"
 	"fmt"
 	"path/filepath"
 	"strings"
-	"testing"
 
 	"k8s.io/apimachinery/pkg/util/uuid"
-	remote "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/binremote/binremote"
+	remote "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/binremote"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -43,19 +41,8 @@ var (
 	instance *remote.InstanceInfo
 	//gceCloud *gce.CloudProvider
 	nodeID string
-
-	project        = flag.String("project", "", "Project to run tests in")
-	serviceAccount = flag.String("service-account", "", "Service account to bring up instance with")
-	runInProw      = flag.Bool("run-in-prow", false, "If true, use a Boskos loaned project and special CI service accounts and ssh keys")
 )
 
-func TestE2E(t *testing.T) {
-	flag.Parse()
-	RegisterFailHandler(Fail)
-	RunSpecs(t, "Google Compute Engine Persistent Disk Container Storage Interface Driver Tests")
-
-}
-
 var _ = BeforeSuite(func() {
 	var err error
 	// TODO(dyzz): better defaults

test/e2e/utils.go

@@ -18,19 +18,24 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"path"
 	"time"
 
 	"github.com/golang/glog"
 	"golang.org/x/oauth2/google"
 	cloudresourcemanager "google.golang.org/api/cloudresourcemanager/v1"
 	boskosclient "k8s.io/test-infra/boskos/client"
-	remote "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/binremote/binremote"
+	remote "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/binremote"
 )
 
 var (
 	boskos = boskosclient.NewClient(os.Getenv("JOB_NAME"), "http://boskos")
 )
 
+const (
+	archiveName = "e2e_gce_pd_test.tar.gz"
+)
+
 func setupInstanceAndDriver(instanceProject, instanceZone, instanceName, port, instanceServiceAccount string) (*remote.InstanceInfo, error) {
 	// Create the instance in the requisite zone
 	instance, err := remote.CreateInstanceInfo(instanceProject, instanceZone, instanceName)
@@ -45,8 +50,14 @@ func setupInstanceAndDriver(instanceProject, instanceZone, instanceName, port, i
 	}
 
 	// Create Driver Archive
-	archiveName := "e2e_gce_pd_test.tar.gz"
-	archivePath, err := remote.CreateDriverArchive(archiveName)
+
+	goPath, ok := os.LookupEnv("GOPATH")
+	if !ok {
+		return nil, fmt.Errorf("Could not find environment variable GOPATH")
+	}
+	pkgPath := path.Join(goPath, "src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/")
+	binPath := path.Join(pkgPath, "bin/gce-pd-csi-driver")
+	archivePath, err := remote.CreateDriverArchive(archiveName, pkgPath, binPath)
 	if err != nil {
 		return nil, err
 	}
@@ -58,7 +69,11 @@ func setupInstanceAndDriver(instanceProject, instanceZone, instanceName, port, i
 	}()
 
 	// Upload archive to instance and run binaries
-	err = instance.UploadAndRun(archivePath, archiveName, instanceName, port)
+	endpoint := fmt.Sprintf("tcp://localhost:%s", port)
+	workspace := remote.NewWorkspaceDir("gce-pd-e2e-")
+	driverRunCmd := fmt.Sprintf("sh -c '/usr/bin/nohup %s/gce-pd-csi-driver --endpoint=%s --nodeid=%s > %s/prog.out 2> %s/prog.err < /dev/null &'",
+		workspace, endpoint, instanceName, workspace, workspace)
+	err = instance.UploadAndRun(archivePath, workspace, driverRunCmd)
 	if err != nil {
 		return nil, err
 	}