Add a mode to run e2e test using kubetest2.

Engprod plans to end kubetest support. This diff adds a --use-kubetest2 flag to the
integration test driver.

Next step:
1. Add kubetest2 installation to test scripts.
2. Add a canary test config to validate kubetest2.

Author: TeweiLuo

test/k8s-integration/cluster.go

@@ -260,14 +260,16 @@ func downloadKubernetesSource(pkgDir, k8sIoDir, kubeVersion string) error {
 	return nil
 }
 
-func getGKEKubeTestArgs(gceZone, gceRegion, imageType string) ([]string, error) {
-	var locationArg, locationVal string
+func getGKEKubeTestArgs(gceZone, gceRegion, imageType string, useKubetest2 bool) ([]string, error) {
+	var locationArg, locationVal, locationArgK2 string
 	switch {
 	case len(gceZone) > 0:
 		locationArg = "--gcp-zone"
+		locationArgK2 = "--zone"
 		locationVal = gceZone
 	case len(gceRegion) > 0:
 		locationArg = "--gcp-region"
+		locationArgK2 = "--region"
 		locationVal = gceRegion
 	}
 
@@ -291,6 +293,7 @@ func getGKEKubeTestArgs(gceZone, gceRegion, imageType string) ([]string, error)
 		return nil, fmt.Errorf("failed to get current project: %v", err)
 	}
 
+	// kubetest arguments
 	args := []string{
 		"--up=false",
 		"--down=false",
@@ -306,7 +309,21 @@ func getGKEKubeTestArgs(gceZone, gceRegion, imageType string) ([]string, error)
 		fmt.Sprintf("--gcp-project=%s", project[:len(project)-1]),
 	}
 
-	return args, nil
+	// kubetest2 arguments
+	argsK2 := []string{
+		"--up=false",
+		"--down=false",
+		fmt.Sprintf("--cluster-name=%s", *gkeTestClusterName),
+		fmt.Sprintf("--environment=%s", gkeEnv),
+		fmt.Sprintf("%s=%s", locationArgK2, locationVal),
+		fmt.Sprintf("--project=%s", project[:len(project)-1]),
+	}
+
+	if useKubetest2 {
+		return argsK2, nil
+	} else {
+		return args, nil
+	}
 }
 
 func getNormalizedVersion(kubeVersion, gkeVersion string) (string, error) {

test/k8s-integration/main.go

@@ -67,6 +67,8 @@ var (
 	// Test flags
 	migrationTest = flag.Bool("migration-test", false, "sets the flag on the e2e binary signalling migration")
 	testFocus     = flag.String("test-focus", "", "test focus for Kubernetes e2e")
+
+	useKubeTest2 = flag.Bool("use-kubetest2", false, "use kubetest2 to run e2e tests")
 )
 
 const (
@@ -89,6 +91,7 @@ type testParameters struct {
 	testSkip             string
 	snapshotClassFile    string
 	cloudProviderArgs    []string
+	cloudProviderArgsK2  []string
 	deploymentStrategy   string
 	outputDir            string
 	allowedNotReadyNodes int
@@ -436,10 +439,23 @@ func handle() error {
 
 	switch testParams.deploymentStrategy {
 	case "gke":
-		testParams.cloudProviderArgs, err = getGKEKubeTestArgs(*gceZone, *gceRegion, testParams.imageType)
-		if err != nil {
+		// Build params for both kubetest and kubetest2 for debugging.
+		testParams.cloudProviderArgs, err = getGKEKubeTestArgs(*gceZone, *gceRegion, testParams.imageType, false /* useKubeTest2 */)
+		if err != nil && !*useKubeTest2 {
 			return fmt.Errorf("failed to build GKE kubetest args: %v", err)
 		}
+		testParams.cloudProviderArgsK2, err = getGKEKubeTestArgs(*gceZone, *gceRegion, testParams.imageType, true /* useKubeTest2 */)
+		if err != nil && *useKubeTest2 {
+			return fmt.Errorf("failed to build GKE kubetest2 args: %v", err)
+		}
+	case "gce":
+		testParams.cloudProviderArgsK2 = []string{
+			// This flag tells kubetest2 what "repo-root" is.
+			// If --legacy-mode is set, kubernetes/kubernetes is used;
+			// otherwise kubernetes/cloud-provider-gcp is used.
+			"--legacy-mode",
+			fmt.Sprintf("--repo-root=%s", *localK8sDir),
+		}
 	}
 
 	// Kubernetes version of GKE deployments are expected to be of the pattern x.y.z-gke.k,
@@ -634,12 +650,32 @@ func runTestsWithConfig(testParams *testParameters, testConfigArg, reportPrefix
 		"--check-version-skew=false",
 		fmt.Sprintf("--test_args=%s", testArgs),
 	}
+
+	kubeTest2Args := []string{
+		*deploymentStrat,
+		"--test=ginkgo",
+	}
+	kubeTest2Args = append(kubeTest2Args, testParams.cloudProviderArgsK2...)
+	if kubetestDumpDir != "" {
+		kubeTest2Args = append(kubeTest2Args, fmt.Sprintf("--log_dir=%s", kubetestDumpDir))
+	}
+	kubeTest2Args = append(kubeTest2Args, "--")
+	kubeTest2Args = append(kubeTest2Args, fmt.Sprintf("--focus-regex=%s", testParams.testFocus))
+	kubeTest2Args = append(kubeTest2Args, fmt.Sprintf("--skip-regex=%s", testParams.testSkip))
+	// kubetest uses 25 as default value for ginkgo parallelism (--nodes).
+	kubeTest2Args = append(kubeTest2Args, "--parallel=25")
+	kubeTest2Args = append(kubeTest2Args, fmt.Sprintf("--test-args=%s %s", testConfigArg, reportArg))
+
 	if kubetestDumpDir != "" {
 		kubeTestArgs = append(kubeTestArgs, fmt.Sprintf("--dump=%s", kubetestDumpDir))
 	}
 	kubeTestArgs = append(kubeTestArgs, testParams.cloudProviderArgs...)
 
-	err = runCommand("Running Tests", exec.Command("kubetest", kubeTestArgs...))
+	if *useKubeTest2 {
+		err = runCommand("Running Tests", exec.Command("kubetest2", kubeTest2Args...))
+	} else {
+		err = runCommand("Running Tests", exec.Command("kubetest", kubeTestArgs...))
+	}
 	if err != nil {
 		return fmt.Errorf("failed to run tests on e2e cluster: %v", err)
 	}