add functionality to use gke with kubetest

Author: Hantao (Will) Wang

test/k8s-integration/cluster.go

@@ -155,3 +155,42 @@ func downloadKubernetesSource(pkgDir, k8sIoDir, kubeVersion string) error {
 
 	return nil
 }
+
+func getGKEKubeTestArgs() ([]string, error) {
+	var gkeEnv string
+	switch gkeURL := os.Getenv("CLOUDSDK_API_ENDPOINT_OVERRIDES_CONTAINER"); gkeURL {
+	case "https://staging-container.sandbox.googleapis.com/":
+		gkeEnv = "staging"
+	case "https://test-container.sandbox.googleapis.com/":
+		gkeEnv = "test"
+	case "":
+		gkeEnv = "prod"
+	default:
+		// if the URL does not match to an option, assume it is a custom GKE backend
+		// URL and pass that to kubetest
+		gkeEnv = gkeURL
+	}
+
+	cmd := exec.Command("gcloud", "config", "get-value", "project")
+	project, err := cmd.Output()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get current project: %v", err)
+	}
+
+	args := []string{
+		"--up=false",
+		"--down=false",
+		"--provider=gke",
+		"--gcp-network=default",
+		"--check-version-skew=false",
+		"--deployment=gke",
+		"--gcp-node-image=cos",
+		"--gcp-network=default",
+		fmt.Sprintf("--cluster=%s", gkeTestClusterName),
+		fmt.Sprintf("--gke-environment=%s", gkeEnv),
+		fmt.Sprintf("--gcp-zone=%s", *gceZone),
+		fmt.Sprintf("--gcp-project=%s", project[:len(project)-1]),
+	}
+
+	return args, nil
+}

test/k8s-integration/main.go

@@ -216,10 +216,17 @@ func handle() error {
 		if err != nil {
 			return fmt.Errorf("failed to build Kubernetes: %v", err)
 		}
+		// kubetest relies on ginkgo already built in the test k8s directory
+		err = buildKubernetes(testDir, "ginkgo")
+		if err != nil {
+			return fmt.Errorf("failed to build gingko: %v", err)
+		}
 	} else {
 		testDir = k8sDir
 	}
 
+	var cloudProviderArgs []string
+
 	// Create a cluster either through GKE or GCE
 	if *bringupCluster {
 		var err error = nil
@@ -228,6 +235,10 @@ func handle() error {
 			err = clusterUpGCE(k8sDir, *gceZone, *numNodes)
 		case "gke":
 			err = clusterUpGKE(*gceZone, *numNodes)
+			cloudProviderArgs, err = getGKEKubeTestArgs()
+			if err != nil {
+				return fmt.Errorf("failed to build GKE kubetest args: %v", err)
+			}
 		default:
 			err = fmt.Errorf("deployment-strategy must be set to 'gce' or 'gke', but is: %s", *deploymentStrat)
 		}
@@ -272,9 +283,9 @@ func handle() error {
 
 	// Run the tests using the testDir kubernetes
 	if len(*storageClassFile) != 0 {
-		err = runCSITests(pkgDir, testDir, *testFocus, *storageClassFile)
+		err = runCSITests(pkgDir, testDir, *testFocus, *storageClassFile, cloudProviderArgs)
 	} else if *migrationTest {
-		err = runMigrationTests(pkgDir, testDir, *testFocus)
+		err = runMigrationTests(pkgDir, testDir, *testFocus, cloudProviderArgs)
 	} else {
 		return fmt.Errorf("did not run either CSI or Migration test")
 	}
@@ -299,20 +310,20 @@ func setEnvProject(project string) error {
 	return nil
 }
 
-func runMigrationTests(pkgDir, testDir, testFocus string) error {
-	return runTestsWithConfig(testDir, testFocus, "--storage.migratedPlugins=kubernetes.io/gce-pd")
+func runMigrationTests(pkgDir, testDir, testFocus string, cloudProviderArgs []string) error {
+	return runTestsWithConfig(testDir, testFocus, "--storage.migratedPlugins=kubernetes.io/gce-pd", cloudProviderArgs)
 }
 
-func runCSITests(pkgDir, testDir, testFocus, storageClassFile string) error {
+func runCSITests(pkgDir, testDir, testFocus, storageClassFile string, cloudProviderArgs []string) error {
 	testDriverConfigFile, err := generateDriverConfigFile(pkgDir, storageClassFile)
 	if err != nil {
 		return err
 	}
 	testConfigArg := fmt.Sprintf("--storage.testdriver=%s", testDriverConfigFile)
-	return runTestsWithConfig(testDir, testFocus, testConfigArg)
+	return runTestsWithConfig(testDir, testFocus, testConfigArg, cloudProviderArgs)
 }
 
-func runTestsWithConfig(testDir, testFocus, testConfigArg string) error {
+func runTestsWithConfig(testDir, testFocus, testConfigArg string, cloudProviderArgs []string) error {
 	err := os.Chdir(testDir)
 	if err != nil {
 		return err
@@ -324,18 +335,21 @@ func runTestsWithConfig(testDir, testFocus, testConfigArg string) error {
 	artifactsDir, _ := os.LookupEnv("ARTIFACTS")
 	reportArg := fmt.Sprintf("-report-dir=%s", artifactsDir)
 
-	kubetestArgs := fmt.Sprintf("--ginkgo.focus=%s --ginkgo.skip=%s %s %s",
+	testArgs := fmt.Sprintf("--ginkgo.focus=%s --ginkgo.skip=%s %s %s",
 		testFocus,
 		"\\[Disruptive\\]|\\[Serial\\]|\\[Feature:.+\\]",
 		testConfigArg,
 		reportArg)
 
-	cmd := exec.Command("kubetest",
+	kubeTestArgs := []string{
 		"--test",
 		"--ginkgo-parallel",
-		fmt.Sprintf("--test_args=%s", kubetestArgs),
-	)
-	err = runCommand("Running Tests", cmd)
+		fmt.Sprintf("--test_args=%s", testArgs),
+	}
+
+	kubeTestArgs = append(kubeTestArgs, cloudProviderArgs...)
+
+	err = runCommand("Running Tests", exec.Command("kubetest", kubeTestArgs...))
 	if err != nil {
 		return fmt.Errorf("failed to run tests on e2e cluster: %v", err)
 	}