Merge pull request #323 from hantaowang/separate-test-k8s

k8s-ci-robot · web-flow · commit b42a3a991bd0 · 2019-06-27T15:13:21.000-07:00
Separate test k8s
diff --git a/test/k8s-integration/main.go b/test/k8s-integration/main.go
@@ -37,12 +37,12 @@ var (
 	teardownDriver   = flag.Bool("teardown-driver", true, "teardown the driver after the e2e test")
 	bringupCluster   = flag.Bool("bringup-cluster", true, "build kubernetes and bringup a cluster")
 	gceZone          = flag.String("gce-zone", "", "zone that the gce k8s cluster is created/found in")
-	kubeVersion      = flag.String("kube-version", "master", "version of Kubernetes to download and use")
+	kubeVersion      = flag.String("kube-version", "", "version of Kubernetes to download and use for the cluster")
+	testVersion      = flag.String("test-version", "", "version of Kubernetes to download and use for tests")
 	kubeFeatureGates = flag.String("kube-feature-gates", "", "feature gates to set on new kubernetes cluster")
-	localK8sDir      = flag.String("local-k8s-dir", "", "local kubernetes/kubernetes directory to run e2e tests from")
-	deploymentStrat  = flag.String("deployment-strategy", "gce", "choose between deploying on gce or gke")
-	gkeClusterVer    = flag.String("gke-cluster-version", "latest", "version of Kubernetes master and node for gke")
-
+	localK8sDir      = flag.String("local-k8s-dir", "", "local prebuilt kubernetes/kubernetes directory to use for cluster and test binaries")
+	deploymentStrat  = flag.String("deployment-strategy", "", "choose between deploying on gce or gke")
+	gkeClusterVer    = flag.String("gke-cluster-version", "", "version of Kubernetes master and node for gke")
 	// Test infrastructure flags
 	boskosResourceType = flag.String("boskos-resource-type", "gce-project", "name of the boskos resource type to reserve")
 	storageClassFile   = flag.String("storageclass-file", "", "name of storageclass yaml file to use for test relative to test/k8s-integration/config")
@@ -68,6 +68,7 @@ const (
 func init() {
 	flag.Set("logtostderr", "true")
 }
+
 func main() {
 	flag.Parse()
 
@@ -101,10 +102,14 @@ func main() {
 		ensureVariable(kubeVersion, false, "Cannot set kube-version when using deployment strategy 'gke'. Use gke-cluster-version.")
 		ensureVariable(gkeClusterVer, true, "Must set gke-cluster-version when using deployment strategy 'gke'.")
 		ensureVariable(kubeFeatureGates, false, "Cannot set feature gates when using deployment strategy 'gke'.")
+		if len(*localK8sDir) == 0 {
+			ensureVariable(testVersion, true, "Must set either test-version or local k8s dir when using deployment strategy 'gke'.")
+		}
 	}
 
 	if len(*localK8sDir) != 0 {
 		ensureVariable(kubeVersion, false, "Cannot set a kube version when using a local k8s dir.")
+		ensureVariable(testVersion, false, "Cannot set a test version when using a local k8s dir.")
 	}
 }
 
@@ -118,11 +123,10 @@ func handle() error {
 	if !ok {
 		return fmt.Errorf("Could not find env variable GOPATH")
 	}
+
 	pkgDir := filepath.Join(goPath, "src", "sigs.k8s.io", "gcp-compute-persistent-disk-csi-driver")
-	k8sParentDir := generateUniqueTmpDir()
-	k8sDir := filepath.Join(k8sParentDir, "kubernetes")
-	defer removeDir(k8sParentDir)
 
+	// If running in Prow, then acquire and set up a project through Boskos
 	if *inProw {
 		project, _ := testutils.SetupProwConfig(*boskosResourceType)
 
@@ -154,69 +158,77 @@ func handle() error {
 		}
 	}
 
+	// Build and push the driver, if required. Defer the driver image deletion.
 	if *doDriverBuild {
 		err := pushImage(pkgDir, *stagingImage, stagingVersion)
 		if err != nil {
 			return fmt.Errorf("failed pushing image: %v", err)
 		}
 		defer func() {
 			if *teardownCluster {
-				err = deleteImage(*stagingImage, stagingVersion)
+				err := deleteImage(*stagingImage, stagingVersion)
 				if err != nil {
 					klog.Errorf("failed to delete image: %v", err)
 				}
 			}
 		}()
 	}
 
-	if *bringupCluster {
+	// Create temporary directories for kubernetes builds
+	k8sParentDir := generateUniqueTmpDir()
+	k8sDir := filepath.Join(k8sParentDir, "kubernetes")
+	testParentDir := generateUniqueTmpDir()
+	testDir := filepath.Join(testParentDir, "kubernetes")
+	defer removeDir(k8sParentDir)
+	defer removeDir(testParentDir)
+
+	// If kube version is set, then download and build Kubernetes for cluster creation
+	// Otherwise, either GKE or a prebuild local K8s dir is being used
+	if len(*kubeVersion) != 0 {
 		err := downloadKubernetesSource(pkgDir, k8sParentDir, *kubeVersion)
 		if err != nil {
 			return fmt.Errorf("failed to download Kubernetes source: %v", err)
 		}
-
-		err = buildKubernetes(k8sDir)
+		err = buildKubernetes(k8sDir, "quick-release")
 		if err != nil {
 			return fmt.Errorf("failed to build Kubernetes: %v", err)
 		}
+	} else {
+		k8sDir = *localK8sDir
+	}
 
-		kshPath := filepath.Join(k8sDir, "cluster", "kubectl.sh")
-		_, err = os.Stat(kshPath)
-		if err == nil {
-			// Set kubectl to the one bundled in the k8s tar for versioning
-			err = os.Setenv("GCE_PD_KUBECTL", kshPath)
-			if err != nil {
-				return fmt.Errorf("failed to set cluster specific kubectl: %v", err)
-			}
-		} else {
-			klog.Errorf("could not find cluster kubectl at %s, falling back to default kubectl", kshPath)
+	// If test version is set, then download and build Kubernetes to run K8s tests
+	// Otherwise, either kube version is set (which implies GCE) or a local K8s dir is being used
+	if len(*testVersion) != 0 && *testVersion != *kubeVersion {
+		err := downloadKubernetesSource(pkgDir, testParentDir, *testVersion)
+		if err != nil {
+			return fmt.Errorf("failed to download Kubernetes source: %v", err)
 		}
-
-		if len(*kubeFeatureGates) != 0 {
-			err = os.Setenv("KUBE_FEATURE_GATES", *kubeFeatureGates)
-			if err != nil {
-				return fmt.Errorf("failed to set kubernetes feature gates: %v", err)
-			}
-			klog.V(4).Infof("Set Kubernetes feature gates: %v", *kubeFeatureGates)
+		err = buildKubernetes(testDir, "WHAT=test/e2e/e2e.test")
+		if err != nil {
+			return fmt.Errorf("failed to build Kubernetes: %v", err)
 		}
+	} else {
+		testDir = k8sDir
+	}
 
+	// Create a cluster either through GKE or GCE
+	if *bringupCluster {
+		var err error = nil
 		switch *deploymentStrat {
 		case "gce":
 			err = clusterUpGCE(k8sDir, *gceZone)
-			if err != nil {
-				return fmt.Errorf("failed to cluster up: %v", err)
-			}
 		case "gke":
 			err = clusterUpGKE(*gceZone)
-			if err != nil {
-				return fmt.Errorf("failed to cluster up: %v", err)
-			}
 		default:
-			return fmt.Errorf("deployment-strategy must be set to 'gce' or 'gke', but is: %s", *deploymentStrat)
+			err = fmt.Errorf("deployment-strategy must be set to 'gce' or 'gke', but is: %s", *deploymentStrat)
+		}
+		if err != nil {
+			return fmt.Errorf("failed to cluster up: %v", err)
 		}
-
 	}
 
+	// Defer the tear down of the cluster through GKE or GCE
 	if *teardownCluster {
 		defer func() {
 			switch *deploymentStrat {
@@ -236,6 +248,7 @@ func handle() error {
 		}()
 	}
 
+	// Install the driver and defer its teardown
 	err := installDriver(goPath, pkgDir, *stagingImage, stagingVersion, *deployOverlayName, *doDriverBuild)
 	if *teardownDriver {
 		defer func() {
@@ -249,14 +262,11 @@ func handle() error {
 		return fmt.Errorf("failed to install CSI Driver: %v", err)
 	}
 
-	if len(*localK8sDir) != 0 {
-		k8sDir = *localK8sDir
-	}
-
+	// Run the tests using the testDir kubernetes
 	if len(*storageClassFile) != 0 {
-		err = runCSITests(pkgDir, k8sDir, *testFocus, *storageClassFile, *gceZone)
+		err = runCSITests(pkgDir, testDir, *testFocus, *storageClassFile, *gceZone)
 	} else if *migrationTest {
-		err = runMigrationTests(pkgDir, k8sDir, *testFocus, *gceZone)
+		err = runMigrationTests(pkgDir, testDir, *testFocus, *gceZone)
 	} else {
 		return fmt.Errorf("Did not run either CSI or Migration test")
 	}
@@ -367,8 +377,8 @@ func clusterDownGKE(gceZone string) error {
 	return nil
 }
 
-func buildKubernetes(k8sDir string) error {
-	cmd := exec.Command("make", "-C", k8sDir, "quick-release")
+func buildKubernetes(k8sDir, command string) error {
+	cmd := exec.Command("make", "-C", k8sDir, command)
 	err := runCommand("Building Kubernetes", cmd)
 	if err != nil {
 		return fmt.Errorf("failed to build Kubernetes: %v", err)
@@ -377,7 +387,27 @@ func buildKubernetes(k8sDir string) error {
 }
 
 func clusterUpGCE(k8sDir, gceZone string) error {
-	err := os.Setenv("KUBE_GCE_ZONE", gceZone)
+	kshPath := filepath.Join(k8sDir, "cluster", "kubectl.sh")
+	_, err := os.Stat(kshPath)
+	if err == nil {
+		// Set kubectl to the one bundled in the k8s tar for versioning
+		err = os.Setenv("GCE_PD_KUBECTL", kshPath)
+		if err != nil {
+			return fmt.Errorf("failed to set cluster specific kubectl: %v", err)
+		}
+	} else {
+		klog.Errorf("could not find cluster kubectl at %s, falling back to default kubectl", kshPath)
+	}
+
+	if len(*kubeFeatureGates) != 0 {
+		err = os.Setenv("KUBE_FEATURE_GATES", *kubeFeatureGates)
+		if err != nil {
+			return fmt.Errorf("failed to set kubernetes feature gates: %v", err)
+		}
+		klog.V(4).Infof("Set Kubernetes feature gates: %v", *kubeFeatureGates)
+	}
+
+	err = os.Setenv("KUBE_GCE_ZONE", gceZone)
 	if err != nil {
 		return err
 	}
diff --git a/test/run-k8s-integration-local.sh b/test/run-k8s-integration-local.sh
@@ -4,7 +4,9 @@ set -o nounset
 set -o errexit
 
 readonly PKGDIR=${GOPATH}/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver
-readonly deployment_strategy=${DEPLOYMENT_STRATEGY:-gce}
+readonly gke_cluster_version=${GKE_CLUSTER_VERSION:-latest}
+readonly kube_version=${KUBE_VERSION:-master}
+readonly test_version=${TEST_VERSION:-master}
 
 source "${PKGDIR}/deploy/common.sh"
 
@@ -13,15 +15,29 @@ ensure_var GCE_PD_SA_DIR
 
 make -C ${PKGDIR} test-k8s-integration
 
-# ${PKGDIR}/bin/k8s-integration-test --kube-version=master --run-in-prow=false \
+# This version of the command creates a GKE cluster. It also downloads and builds a k8s release
+# so that it can run the test specified
+
+# ${PKGDIR}/bin/k8s-integration-test --run-in-prow=false \
 # --staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \
 # --deploy-overlay-name=dev --storageclass-file=sc-standard.yaml \
 # --test-focus="External.Storage" --gce-zone="us-central1-b" \
-# --deployment-strategy=${deployment_strategy}
+# --deployment-strategy=gke --gke-cluster-version=${gke_cluster_version} \
+# --test-version=${test_version}
+
+# This version of the command creates a GCE cluster. It downloads and builds two k8s releases,
+# one for the cluster and one for the tests, unless the cluster and test versioning is the same.
+
+# ${PKGDIR}/bin/k8s-integration-test --run-in-prow=false \
+# --staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \
+# --deploy-overlay-name=dev --storageclass-file=sc-standard.yaml \
+# --test-focus="External.Storage" --gce-zone="us-central1-b" \
+# --deployment-strategy=gce --kube-version=${kube_version} \
+# --test-version=${test_version}
 
 # This version of the command does not build the driver or K8s, points to a
 # local K8s repo to get the e2e.test binary, and does not bring up or down the cluster
-#
+
 ${PKGDIR}/bin/k8s-integration-test --kube-version=master --run-in-prow=false \
 --staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \
 --deploy-overlay-name=dev --bringup-cluster=false --teardown-cluster=false --local-k8s-dir=$KTOP \
diff --git a/test/run-k8s-integration.sh b/test/run-k8s-integration.sh
@@ -14,11 +14,24 @@ readonly overlay_name="${GCE_PD_OVERLAY_NAME:-stable}"
 readonly boskos_resource_type="${GCE_PD_BOSKOS_RESOURCE_TYPE:-gce-project}"
 readonly do_driver_build="${GCE_PD_DO_DRIVER_BUILD:-true}"
 readonly deployment_strategy=${DEPLOYMENT_STRATEGY:-gce}
+readonly gke_cluster_version=${GKE_CLUSTER_VERSION:-latest}
+readonly kube_version=${GCE_PD_KUBE_VERSION:-master}
+readonly test_version=${TEST_VERSION:-master}
+
 export GCE_PD_VERBOSITY=9
 
 make -C ${PKGDIR} test-k8s-integration
-${PKGDIR}/bin/k8s-integration-test --kube-version=${GCE_PD_KUBE_VERSION:-master} \
---run-in-prow=true --deploy-overlay-name=${overlay_name} --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \
---do-driver-build=${do_driver_build} --boskos-resource-type=${boskos_resource_type} \
---storageclass-file=sc-standard.yaml --test-focus="External.Storage" --gce-zone="us-central1-b" \
---deployment-strategy=${deployment_strategy}
+
+base_cmd="${PKGDIR}/bin/k8s-integration-test \
+            --run-in-prow=true --deploy-overlay-name=${overlay_name} --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \
+            --do-driver-build=${do_driver_build} --boskos-resource-type=${boskos_resource_type} \
+            --storageclass-file=sc-standard.yaml --test-focus="External.Storage" --gce-zone="us-central1-b" \
+            --deployment-strategy=${deployment_strategy} --test-version=${test_version}"
+
+if [ "$deployment_strategy" = "gke" ]; then
+  base_cmd="${base_cmd} --gke-cluster-version=${gke_cluster_version}"
+else
+  base_cmd="${base_cmd} --kube-version=${kube_version}"
+fi
+
+eval $base_cmd
