Test framework changes to run properly on PROW

Author: davidz627

Makefile

@@ -22,6 +22,7 @@ all: gce-pd-driver
 gce-pd-driver:
 	mkdir -p bin
 	go build -o bin/gce-pd-csi-driver ./cmd/
+	go test -c sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/e2e -o bin/e2e.test
 
 build-container: gce-pd-driver
 	docker build -t $(STAGINGIMAGE):$(STAGINGVERSION) .

test/remote/remote/e2e.go

@@ -36,33 +36,30 @@ func InitE2ERemote() TestSuite {
 
 // SetupTestPackage sets up the test package with binaries k8s required for node e2e tests
 func (n *E2ERemote) SetupTestPackage(tardir string) error {
-	// Make sure we can find the newly built binaries
-	gopath, ok := os.LookupEnv("GOPATH")
-	if !ok {
-		return fmt.Errorf("Could not find gopath")
-	}
-
 	// TODO(dyzz): build the gce driver tests instead.
 
-	cmd := exec.Command("ginkgo", "build", "test/e2e")
+	cmd := exec.Command("go", "test", "-c", "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/e2e", "-o", "test/e2e/e2e.test")
 	err := cmd.Run()
 
 	if err != nil {
-		return fmt.Errorf("Failed to ginkgo build test: %v", err)
+		return fmt.Errorf("Failed to build test: %v", err)
 	}
 
-	cmd = exec.Command("cp", "test/e2e/e2e.test", "bin")
+	cmd = exec.Command("mkdir", "-p", "bin")
 	err = cmd.Run()
 	if err != nil {
-		return fmt.Errorf("Failed to copy: %v", err)
+		return fmt.Errorf("Failed to mkdir bin/: %v", err)
 	}
 
-	buildOutputDir := filepath.Join(gopath, "src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/bin")
-
+	cmd = exec.Command("cp", "test/e2e/e2e.test", "bin/e2e.test")
+	err = cmd.Run()
+	if err != nil {
+		return fmt.Errorf("Failed to copy: %v", err)
+	}
 	// Copy binaries
 	requiredBins := []string{"e2e.test"}
 	for _, bin := range requiredBins {
-		source := filepath.Join(buildOutputDir, bin)
+		source := filepath.Join("bin", bin)
 		if _, err := os.Stat(source); err != nil {
 			return fmt.Errorf("failed to locate test binary %s: %v", bin, err)
 		}

test/remote/remote/ssh.go

@@ -19,6 +19,7 @@ package remote
 import (
 	"flag"
 	"fmt"
+	"os"
 	"os/exec"
 	"os/user"
 	"strings"
@@ -69,6 +70,8 @@ func GetHostnameOrIP(hostname string) string {
 	}
 	if *sshUser != "" {
 		host = fmt.Sprintf("%s@%s", *sshUser, host)
+	} else if _, ok := os.LookupEnv("JENKINS_GCE_SSH_PRIVATE_KEY_FILE"); ok {
+		host = fmt.Sprintf("prow@%s", host)
 	}
 	return host
 }
@@ -92,7 +95,10 @@ func SSHNoSudo(host string, cmd ...string) (string, error) {
 
 // runSSHCommand executes the ssh or scp command, adding the flag provided --ssh-options
 func runSSHCommand(cmd string, args ...string) (string, error) {
-	if *sshKey != "" {
+	if pk, ok := os.LookupEnv("JENKINS_GCE_SSH_PRIVATE_KEY_FILE"); ok {
+		glog.V(4).Infof("Running on Jenkins, using special private key file at %v", pk)
+		args = append([]string{"-i", pk}, args...)
+	} else if *sshKey != "" {
 		args = append([]string{"-i", *sshKey}, args...)
 	} else if key, found := sshDefaultKeyMap[*sshEnv]; found {
 		args = append([]string{"-i", key}, args...)

test/remote/run_remote/run_remote.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"io/ioutil"
 	"math/rand"
 	"net/http"
 	"os"
@@ -29,6 +30,7 @@ import (
 
 	"k8s.io/apimachinery/pkg/util/uuid"
 	"k8s.io/apimachinery/pkg/util/wait"
+	gce "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider"
 	"sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/remote/remote"
 
 	"github.com/golang/glog"
@@ -268,7 +270,7 @@ func createInstance(serviceAccount string) (string, error) {
 	myuuid := string(uuid.NewUUID())
 	glog.V(2).Infof("Creating instance: %v", name)
 
-	imageURL := "https://www.googleapis.com/compute/v1/projects/eip-images/global/images/debian-9-drawfork-v20180423"
+	imageURL := "https://www.googleapis.com/compute/v1/projects/ml-images/global/images/debian-9-tf-1-9-v20180626"
 	i := &compute.Instance{
 		Name:        name,
 		MachineType: machineType(""),
@@ -303,8 +305,9 @@ func createInstance(serviceAccount string) (string, error) {
 	}
 
 	var err error
-	if _, err = computeService.Instances.Get(*project, *zone, i.Name).Do(); err != nil {
+	if gotInstance, err := computeService.Instances.Get(*project, *zone, i.Name).Do(); err != nil {
 		op, err := computeService.Instances.Insert(*project, *zone, i).Do()
+		glog.V(4).Infof("Inserted instance in project %v, zone %v: %#v", *project, *zone, i)
 		if err != nil {
 			ret := fmt.Sprintf("could not create instance %s: API error: %v", name, err)
 			if op != nil {
@@ -314,10 +317,24 @@ func createInstance(serviceAccount string) (string, error) {
 		} else if op.Error != nil {
 			return "", fmt.Errorf("could not create instance %s: %+v", name, op.Error)
 		}
+	} else {
+		glog.V(4).Infof("Compute service GOT instance %v, skipping instance creation: %#v", i.Name, gotInstance)
+	}
+
+	pubkey, ok := os.LookupEnv("JENKINS_GCE_SSH_PUBLIC_KEY_FILE")
+	if ok {
+		glog.Infof("Running on Jenkins and JENKINS_GCE_SSH_PUBLIC_KEY_FILE set")
+		// If we're on CI add public SSH keys to the instance
+		err = addPubKeyToInstance(*project, *zone, i.Name, pubkey)
+		if err != nil {
+			return "", fmt.Errorf("could not add Jenkins public key %v to instance %v: %v", pubkey, i.Name, err)
+		}
+	} else {
+		glog.V(4).Infof("JENKINS_GCE_SSH_PUBLIC_KEY_FILE not set, not adding SSH public key to instance")
 	}
 
 	then := time.Now()
-	err = wait.Poll(15*time.Second, 10*time.Minute, func() (bool, error) {
+	err = wait.Poll(10*time.Second, 5*time.Minute, func() (bool, error) {
 		glog.V(2).Infof("Waiting for instance %v to come up. %v elapsed", name, time.Since(then))
 		var instance *compute.Instance
 		instance, err = computeService.Instances.Get(*project, *zone, name).Do()
@@ -337,9 +354,9 @@ func createInstance(serviceAccount string) (string, error) {
 			remote.AddHostnameIP(name, externalIP)
 		}
 
-		if _, err = remote.SSHNoSudo(name, "echo"); err != nil {
+		if sshOut, err := remote.SSHNoSudo(name, "echo"); err != nil {
 			err = fmt.Errorf("Instance %v in state RUNNING but not available by SSH: %v", name, err)
-			glog.Error(err)
+			glog.Errorf("SSH encountered an error: %v, output: %v", err, sshOut)
 			return false, nil
 		}
 
@@ -355,6 +372,54 @@ func createInstance(serviceAccount string) (string, error) {
 	return name, nil
 }
 
+func addPubKeyToInstance(project, zone, name, pubKeyFile string) error {
+	newKeys := ""
+	i, err := computeService.Instances.Get(project, zone, name).Do()
+	if err != nil {
+		return err
+	}
+	fingerprint := i.Metadata.Fingerprint
+	items := i.Metadata.Items
+	for _, item := range items {
+		if item.Key == "ssh-keys" {
+			glog.V(2).Infof("Found existing ssh-keys, prepending to new key string")
+			newKeys += *item.Value
+			break
+		}
+	}
+	publicKeyByte, err := ioutil.ReadFile(pubKeyFile)
+	if err != nil {
+		return err
+	}
+
+	publicKey := string(publicKeyByte)
+
+	// Take username and prepend it to the public key
+	tokens := strings.Split(publicKey, " ")
+	if len(tokens) != 3 {
+		return fmt.Errorf("Public key not comprised of 3 parts, instead was: %v", publicKey)
+	}
+	publicKey = strings.TrimSpace(tokens[2]) + ":" + publicKey
+
+	newKeys = newKeys + publicKey
+	glog.V(4).Infof("New ssh-keys for instance %v: %v", name, newKeys)
+	newMeta := &compute.Metadata{
+		Fingerprint: fingerprint,
+		Items: []*compute.MetadataItems{
+			&compute.MetadataItems{
+				Key:   "ssh-keys",
+				Value: &newKeys,
+			},
+		},
+	}
+	_, err = computeService.Instances.SetMetadata(project, zone, name, newMeta).Do()
+	if err != nil {
+		return err
+	}
+	return nil
+
+}
+
 func getexternalIP(instance *compute.Instance) string {
 	for i := range instance.NetworkInterfaces {
 		ni := instance.NetworkInterfaces[i]
@@ -400,6 +465,9 @@ func deleteInstance(host string) {
 	glog.Infof("Deleting instance %q", host)
 	_, err := computeService.Instances.Delete(*project, *zone, host).Do()
 	if err != nil {
+		if gce.IsGCEError(err, "notFound") {
+			return
+		}
 		glog.Errorf("Error deleting instance %q: %v", host, err)
 	}
 }