Merge pull request #345 from davidz627/fix/numNodes

Add forced variable num-nodes so that we can give it to kubetest to force tests to run that have high node requirements

Author: k8s-ci-robot

test/k8s-integration/cluster.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strconv"
 
 	"k8s.io/klog"
 )
@@ -37,7 +38,7 @@ func buildKubernetes(k8sDir, command string) error {
 	return nil
 }
 
-func clusterUpGCE(k8sDir, gceZone string) error {
+func clusterUpGCE(k8sDir, gceZone string, numNodes int) error {
 	kshPath := filepath.Join(k8sDir, "cluster", "kubectl.sh")
 	_, err := os.Stat(kshPath)
 	if err == nil {
@@ -58,6 +59,11 @@ func clusterUpGCE(k8sDir, gceZone string) error {
 		klog.V(4).Infof("Set Kubernetes feature gates: %v", *kubeFeatureGates)
 	}
 
+	err = os.Setenv("NUM_NODES", strconv.Itoa(numNodes))
+	if err != nil {
+		return err
+	}
+
 	err = os.Setenv("KUBE_GCE_ZONE", gceZone)
 	if err != nil {
 		return err
@@ -71,7 +77,7 @@ func clusterUpGCE(k8sDir, gceZone string) error {
 	return nil
 }
 
-func clusterUpGKE(gceZone string) error {
+func clusterUpGKE(gceZone string, numNodes int) error {
 	out, err := exec.Command("gcloud", "container", "clusters", "list", "--zone", gceZone,
 		"--filter", fmt.Sprintf("name=%s", gkeTestClusterName)).CombinedOutput()
 	if err != nil {
@@ -85,7 +91,7 @@ func clusterUpGKE(gceZone string) error {
 		}
 	}
 	cmd := exec.Command("gcloud", "container", "clusters", "create", gkeTestClusterName,
-		"--zone", gceZone, "--cluster-version", *gkeClusterVer, "--quiet")
+		"--zone", gceZone, "--cluster-version", *gkeClusterVer, "--num-nodes", strconv.Itoa(numNodes), "--quiet")
 	err = runCommand("Staring E2E Cluster on GKE", cmd)
 	if err != nil {
 		return fmt.Errorf("failed to bring up kubernetes e2e cluster on gke: %v", err)

test/k8s-integration/main.go

@@ -40,6 +40,7 @@ var (
 	localK8sDir      = flag.String("local-k8s-dir", "", "local prebuilt kubernetes/kubernetes directory to use for cluster and test binaries")
 	deploymentStrat  = flag.String("deployment-strategy", "", "choose between deploying on gce or gke")
 	gkeClusterVer    = flag.String("gke-cluster-version", "", "version of Kubernetes master and node for gke")
+	numNodes         = flag.Int("num-nodes", -1, "the number of nodes in the test cluster")
 	// Test infrastructure flags
 	boskosResourceType = flag.String("boskos-resource-type", "gce-project", "name of the boskos resource type to reserve")
 	storageClassFile   = flag.String("storageclass-file", "", "name of storageclass yaml file to use for test relative to test/k8s-integration/config")
@@ -110,6 +111,10 @@ func main() {
 		ensureVariable(testVersion, false, "Cannot set a test version when using a local k8s dir.")
 	}
 
+	if *numNodes == -1 {
+		klog.Fatalf("num-nodes must be set to number of nodes in cluster")
+	}
+
 	err := handle()
 	if err != nil {
 		klog.Fatalf("Failed to run integration test: %v", err)
@@ -226,9 +231,9 @@ func handle() error {
 		var err error = nil
 		switch *deploymentStrat {
 		case "gce":
-			err = clusterUpGCE(k8sDir, *gceZone)
+			err = clusterUpGCE(k8sDir, *gceZone, *numNodes)
 		case "gke":
-			err = clusterUpGKE(*gceZone)
+			err = clusterUpGKE(*gceZone, *numNodes)
 		default:
 			err = fmt.Errorf("deployment-strategy must be set to 'gce' or 'gke', but is: %s", *deploymentStrat)
 		}
@@ -339,6 +344,7 @@ func runTestsWithConfig(k8sBinDir, gceZone, testFocus, testConfigArg string) err
 		reportArg,
 		"-provider=gce",
 		"-node-os-distro=cos",
+		fmt.Sprintf("-num-nodes=%v", *numNodes),
 		fmt.Sprintf("-gce-zone=%s", gceZone),
 		testConfigArg)
 

test/run-k8s-integration-local.sh

@@ -23,7 +23,7 @@ make -C ${PKGDIR} test-k8s-integration
 # --deploy-overlay-name=dev --storageclass-file=sc-standard.yaml \
 # --test-focus="External.Storage" --gce-zone="us-central1-b" \
 # --deployment-strategy=gke --gke-cluster-version=${gke_cluster_version} \
-# --test-version=${test_version}
+# --test-version=${test_version} --num-nodes=3
 
 # This version of the command creates a GCE cluster. It downloads and builds two k8s releases,
 # one for the cluster and one for the tests, unless the cluster and test versioning is the same.
@@ -33,7 +33,7 @@ make -C ${PKGDIR} test-k8s-integration
 # --deploy-overlay-name=dev --storageclass-file=sc-standard.yaml \
 # --test-focus="External.Storage" --gce-zone="us-central1-b" \
 # --deployment-strategy=gce --kube-version=${kube_version} \
-# --test-version=${test_version}
+# --test-version=${test_version} --num-nodes=3
 
 # This version of the command does not build the driver or K8s, points to a
 # local K8s repo to get the e2e.test binary, and does not bring up or down the cluster
@@ -42,4 +42,4 @@ ${PKGDIR}/bin/k8s-integration-test --kube-version=master --run-in-prow=false \
 --staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \
 --deploy-overlay-name=dev --bringup-cluster=false --teardown-cluster=false --local-k8s-dir=$KTOP \
 --storageclass-file=sc-standard.yaml --do-driver-build=true  --test-focus="External.Storage" \
---gce-zone="us-central1-b"
+--gce-zone="us-central1-b" --num-nodes=${NUM_NODES:-3}

test/run-k8s-integration-migration-local.sh

@@ -19,7 +19,7 @@ ${PKGDIR}/bin/k8s-integration-test --kube-version=master --run-in-prow=false \
 --staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \
 --deploy-overlay-name=dev --test-focus=${GCE_PD_TEST_FOCUS} \
 --kube-feature-gates="CSIMigration=true,CSIMigrationGCE=true" --migration-test=true --gce-zone="us-central1-b" \
---deployment-strategy=gce --test-version=${test_version}
+--deployment-strategy=gce --test-version=${test_version} --num-nodes=${NUM_NODES:-3}
 
 # This version of the command does not build the driver or K8s, points to a
 # local K8s repo to get the e2e.test binary, and does not bring up or down the cluster
@@ -29,4 +29,4 @@ ${PKGDIR}/bin/k8s-integration-test --kube-version=master --run-in-prow=false \
 # --staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \
 # --deploy-overlay-name=dev --test-focus=${GCE_PD_TEST_FOCUS} \
 # --bringup-cluster=false --teardown-cluster=false --local-k8s-dir=$KTOP --migration-test=true \
-# --do-driver-build=true --gce-zone=${GCE_PD_ZONE}
+# --do-driver-build=true --gce-zone=${GCE_PD_ZONE} --num-nodes=${NUM_NODES:-3}

test/run-k8s-integration-migration.sh

@@ -38,4 +38,5 @@ ${PKGDIR}/bin/k8s-integration-test --kube-version=${kube_version} \
 --deploy-overlay-name=${overlay_name} --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \
 --do-driver-build=${do_driver_build} --boskos-resource-type=${boskos_resource_type} \
 --migration-test=true --test-focus=${GCE_PD_TEST_FOCUS} \
---gce-zone="us-central1-b" --deployment-strategy=${deployment_strategy} --test-version=${test_version}
+--gce-zone="us-central1-b" --deployment-strategy=${deployment_strategy} --test-version=${test_version} \
+--num-nodes=3

test/run-k8s-integration.sh

@@ -26,7 +26,7 @@ base_cmd="${PKGDIR}/bin/k8s-integration-test \
             --run-in-prow=true --deploy-overlay-name=${overlay_name} --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \
             --do-driver-build=${do_driver_build} --boskos-resource-type=${boskos_resource_type} \
             --storageclass-file=sc-standard.yaml --test-focus="External.Storage" --gce-zone="us-central1-b" \
-            --deployment-strategy=${deployment_strategy} --test-version=${test_version}"
+            --deployment-strategy=${deployment_strategy} --test-version=${test_version} --num-nodes=3"
 
 if [ "$deployment_strategy" = "gke" ]; then
   base_cmd="${base_cmd} --gke-cluster-version=${gke_cluster_version}"