Migrate k8s integration tests back to kubetest to pick up kubetest niceties that I think will fix a lot of GCE config issues

Author: davidz627

test/k8s-integration/main.go

@@ -187,7 +187,6 @@ func handle() error {
 	k8sDir := filepath.Join(k8sParentDir, "kubernetes")
 	testParentDir := generateUniqueTmpDir()
 	testDir := filepath.Join(testParentDir, "kubernetes")
-	k8sBuildBinDir := k8sInDockerBuildBinDir
 	defer removeDir(k8sParentDir)
 	defer removeDir(testParentDir)
 
@@ -217,11 +216,6 @@ func handle() error {
 		if err != nil {
 			return fmt.Errorf("failed to build Kubernetes: %v", err)
 		}
-		err = buildKubernetes(testDir, "ginkgo")
-		if err != nil {
-			return fmt.Errorf("failed to build Gingko: %v", err)
-		}
-		k8sBuildBinDir = k8sOutOfDockerBuildBinDir
 	} else {
 		testDir = k8sDir
 	}
@@ -277,11 +271,10 @@ func handle() error {
 	}
 
 	// Run the tests using the testDir kubernetes
-	fullK8sBuildBinPath := filepath.Join(testDir, k8sBuildBinDir)
 	if len(*storageClassFile) != 0 {
-		err = runCSITests(pkgDir, fullK8sBuildBinPath, *testFocus, *storageClassFile, *gceZone)
+		err = runCSITests(pkgDir, k8sDir, *testFocus, *storageClassFile)
 	} else if *migrationTest {
-		err = runMigrationTests(pkgDir, fullK8sBuildBinPath, *testFocus, *gceZone)
+		err = runMigrationTests(pkgDir, k8sDir, *testFocus)
 	} else {
 		return fmt.Errorf("did not run either CSI or Migration test")
 	}
@@ -306,21 +299,21 @@ func setEnvProject(project string) error {
 	return nil
 }
 
-func runMigrationTests(pkgDir, k8sBinDir, testFocus, gceZone string) error {
-	return runTestsWithConfig(k8sBinDir, gceZone, testFocus, "-storage.migratedPlugins=kubernetes.io/gce-pd")
+func runMigrationTests(pkgDir, k8sDir, testFocus string) error {
+	return runTestsWithConfig(k8sDir, testFocus, "--storage.migratedPlugins=kubernetes.io/gce-pd")
 }
 
-func runCSITests(pkgDir, k8sBinDir, testFocus, storageClassFile, gceZone string) error {
+func runCSITests(pkgDir, k8sDir, testFocus, storageClassFile string) error {
 	testDriverConfigFile, err := generateDriverConfigFile(pkgDir, storageClassFile)
 	if err != nil {
 		return err
 	}
-	testConfigArg := fmt.Sprintf("-storage.testdriver=%s", testDriverConfigFile)
-	return runTestsWithConfig(k8sBinDir, gceZone, testFocus, testConfigArg)
+	testConfigArg := fmt.Sprintf("--storage.testdriver=%s", testDriverConfigFile)
+	return runTestsWithConfig(k8sDir, testFocus, testConfigArg)
 }
 
-func runTestsWithConfig(k8sBinDir, gceZone, testFocus, testConfigArg string) error {
-	err := os.Chdir(k8sBinDir)
+func runTestsWithConfig(k8sDir, testFocus, testConfigArg string) error {
+	err := os.Chdir(k8sDir)
 	if err != nil {
 		return err
 	}
@@ -331,23 +324,17 @@ func runTestsWithConfig(k8sBinDir, gceZone, testFocus, testConfigArg string) err
 	artifactsDir, _ := os.LookupEnv("ARTIFACTS")
 	reportArg := fmt.Sprintf("-report-dir=%s", artifactsDir)
 
-	testFocusArg := fmt.Sprintf("-focus=%s", testFocus)
-
-	cmd := exec.Command("./ginkgo",
-		"-p",
-		"-v",
-		testFocusArg,
-		"-skip=\\[Disruptive\\]|\\[Serial\\]|\\[Feature:.+\\]",
-		"e2e.test",
-		"--",
-		"-v=5",
-		reportArg,
-		"-provider=gce",
-		"-node-os-distro=cos",
-		fmt.Sprintf("-num-nodes=%v", *numNodes),
-		fmt.Sprintf("-gce-zone=%s", gceZone),
-		testConfigArg)
-
+	kubetestArgs := fmt.Sprintf("--ginkgo.focus=%s --ginkgo.skip=%s %s %s",
+		testFocus,
+		"\\[Disruptive\\]|\\[Serial\\]|\\[Feature:.+\\]",
+		testConfigArg,
+		reportArg)
+
+	cmd := exec.Command("kubetest",
+		"--test",
+		"--ginkgo-parallel",
+		fmt.Sprintf("--test_args=%s", kubetestArgs),
+	)
 	err = runCommand("Running Tests", cmd)
 	if err != nil {
 		return fmt.Errorf("failed to run tests on e2e cluster: %v", err)

test/run-k8s-integration-local.sh

@@ -38,7 +38,7 @@ make -C ${PKGDIR} test-k8s-integration
 # This version of the command does not build the driver or K8s, points to a
 # local K8s repo to get the e2e.test binary, and does not bring up or down the cluster
 
-${PKGDIR}/bin/k8s-integration-test --kube-version=master --run-in-prow=false \
+${PKGDIR}/bin/k8s-integration-test --run-in-prow=false \
 --staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \
 --deploy-overlay-name=dev --bringup-cluster=false --teardown-cluster=false --local-k8s-dir=$KTOP \
 --storageclass-file=sc-standard.yaml --do-driver-build=true  --test-focus="External.Storage" \

test/run-k8s-integration-migration-local.sh

@@ -15,18 +15,18 @@ ensure_var GCE_PD_SA_DIR
 
 make -C ${PKGDIR} test-k8s-integration
 
-${PKGDIR}/bin/k8s-integration-test --kube-version=master --run-in-prow=false \
---staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \
---deploy-overlay-name=dev --test-focus=${GCE_PD_TEST_FOCUS} \
---kube-feature-gates="CSIMigration=true,CSIMigrationGCE=true" --migration-test=true --gce-zone="us-central1-b" \
---deployment-strategy=gce --test-version=${test_version} --num-nodes=${NUM_NODES:-3}
+# ${PKGDIR}/bin/k8s-integration-test --kube-version=master --run-in-prow=false \
+# --staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \
+# --deploy-overlay-name=dev --test-focus=${GCE_PD_TEST_FOCUS} \
+# --kube-feature-gates="CSIMigration=true,CSIMigrationGCE=true" --migration-test=true --gce-zone="us-central1-b" \
+# --deployment-strategy=gce --test-version=${test_version} --gce-zone=${GCE_PD_ZONE} \
+# --num-nodes=${NUM_NODES:-3}
 
 # This version of the command does not build the driver or K8s, points to a
 # local K8s repo to get the e2e.test binary, and does not bring up or down the cluster
 #
-# ensure_var GCE_PD_ZONE
-# ${PKGDIR}/bin/k8s-integration-test --run-in-prow=false \
-# --staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \
-# --deploy-overlay-name=dev --test-focus=${GCE_PD_TEST_FOCUS} \
-# --bringup-cluster=false --teardown-cluster=false --local-k8s-dir=$KTOP --migration-test=true \
-# --do-driver-build=true --gce-zone=${GCE_PD_ZONE} --num-nodes=${NUM_NODES:-3}
+${PKGDIR}/bin/k8s-integration-test --run-in-prow=false \
+--staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \
+--deploy-overlay-name=dev --test-focus=${GCE_PD_TEST_FOCUS} \
+--bringup-cluster=false --teardown-cluster=false --local-k8s-dir=$KTOP --migration-test=true \
+--do-driver-build=false --gce-zone=${GCE_PD_ZONE} --num-nodes=${NUM_NODES:-3}