From af0fe08e6411113ed346f20c07a536dcb9818ff5 Mon Sep 17 00:00:00 2001 From: David Zhu Date: Tue, 16 Jul 2019 11:51:31 -0700 Subject: [PATCH] Add forced variable num-nodes so that we can give it to ginkgo to force tests to run that have high node requirements --- test/k8s-integration/cluster.go | 12 +++++++++--- test/k8s-integration/main.go | 10 ++++++++-- test/run-k8s-integration-local.sh | 6 +++--- test/run-k8s-integration-migration-local.sh | 4 ++-- test/run-k8s-integration-migration.sh | 3 ++- test/run-k8s-integration.sh | 2 +- 6 files changed, 25 insertions(+), 12 deletions(-) diff --git a/test/k8s-integration/cluster.go b/test/k8s-integration/cluster.go index b549ae26e..30858959f 100644 --- a/test/k8s-integration/cluster.go +++ b/test/k8s-integration/cluster.go @@ -5,6 +5,7 @@ import ( "os" "os/exec" "path/filepath" + "strconv" "k8s.io/klog" ) @@ -37,7 +38,7 @@ func buildKubernetes(k8sDir, command string) error { return nil } -func clusterUpGCE(k8sDir, gceZone string) error { +func clusterUpGCE(k8sDir, gceZone string, numNodes int) error { kshPath := filepath.Join(k8sDir, "cluster", "kubectl.sh") _, err := os.Stat(kshPath) if err == nil { @@ -58,6 +59,11 @@ func clusterUpGCE(k8sDir, gceZone string) error { klog.V(4).Infof("Set Kubernetes feature gates: %v", *kubeFeatureGates) } + err = os.Setenv("NUM_NODES", strconv.Itoa(numNodes)) + if err != nil { + return err + } + err = os.Setenv("KUBE_GCE_ZONE", gceZone) if err != nil { return err @@ -71,7 +77,7 @@ func clusterUpGCE(k8sDir, gceZone string) error { return nil } -func clusterUpGKE(gceZone string) error { +func clusterUpGKE(gceZone string, numNodes int) error { out, err := exec.Command("gcloud", "container", "clusters", "list", "--zone", gceZone, "--filter", fmt.Sprintf("name=%s", gkeTestClusterName)).CombinedOutput() if err != nil { @@ -85,7 +91,7 @@ func clusterUpGKE(gceZone string) error { } } cmd := exec.Command("gcloud", "container", "clusters", "create", gkeTestClusterName, - "--zone", gceZone, "--cluster-version", *gkeClusterVer, "--quiet") + "--zone", gceZone, "--cluster-version", *gkeClusterVer, "--num-nodes", strconv.Itoa(numNodes), "--quiet") err = runCommand("Staring E2E Cluster on GKE", cmd) if err != nil { return fmt.Errorf("failed to bring up kubernetes e2e cluster on gke: %v", err) diff --git a/test/k8s-integration/main.go b/test/k8s-integration/main.go index cebf13ef9..e23cf2cd4 100644 --- a/test/k8s-integration/main.go +++ b/test/k8s-integration/main.go @@ -40,6 +40,7 @@ var ( localK8sDir = flag.String("local-k8s-dir", "", "local prebuilt kubernetes/kubernetes directory to use for cluster and test binaries") deploymentStrat = flag.String("deployment-strategy", "", "choose between deploying on gce or gke") gkeClusterVer = flag.String("gke-cluster-version", "", "version of Kubernetes master and node for gke") + numNodes = flag.Int("num-nodes", -1, "the number of nodes in the test cluster") // Test infrastructure flags boskosResourceType = flag.String("boskos-resource-type", "gce-project", "name of the boskos resource type to reserve") storageClassFile = flag.String("storageclass-file", "", "name of storageclass yaml file to use for test relative to test/k8s-integration/config") @@ -110,6 +111,10 @@ func main() { ensureVariable(testVersion, false, "Cannot set a test version when using a local k8s dir.") } + if *numNodes == -1 { + klog.Fatalf("num-nodes must be set to number of nodes in cluster") + } + err := handle() if err != nil { klog.Fatalf("Failed to run integration test: %v", err) @@ -226,9 +231,9 @@ func handle() error { var err error = nil switch *deploymentStrat { case "gce": - err = clusterUpGCE(k8sDir, *gceZone) + err = clusterUpGCE(k8sDir, *gceZone, *numNodes) case "gke": - err = clusterUpGKE(*gceZone) + err = clusterUpGKE(*gceZone, *numNodes) default: err = fmt.Errorf("deployment-strategy must be set to 'gce' or 'gke', but is: %s", *deploymentStrat) } @@ -339,6 +344,7 @@ func runTestsWithConfig(k8sBinDir, gceZone, testFocus, testConfigArg string) err reportArg, "-provider=gce", "-node-os-distro=cos", + fmt.Sprintf("-num-nodes=%v", *numNodes), fmt.Sprintf("-gce-zone=%s", gceZone), testConfigArg) diff --git a/test/run-k8s-integration-local.sh b/test/run-k8s-integration-local.sh index 7c8dee861..b9c1a01ae 100755 --- a/test/run-k8s-integration-local.sh +++ b/test/run-k8s-integration-local.sh @@ -23,7 +23,7 @@ make -C ${PKGDIR} test-k8s-integration # --deploy-overlay-name=dev --storageclass-file=sc-standard.yaml \ # --test-focus="External.Storage" --gce-zone="us-central1-b" \ # --deployment-strategy=gke --gke-cluster-version=${gke_cluster_version} \ -# --test-version=${test_version} +# --test-version=${test_version} --num-nodes=3 # This version of the command creates a GCE cluster. It downloads and builds two k8s releases, # one for the cluster and one for the tests, unless the cluster and test versioning is the same. @@ -33,7 +33,7 @@ make -C ${PKGDIR} test-k8s-integration # --deploy-overlay-name=dev --storageclass-file=sc-standard.yaml \ # --test-focus="External.Storage" --gce-zone="us-central1-b" \ # --deployment-strategy=gce --kube-version=${kube_version} \ -# --test-version=${test_version} +# --test-version=${test_version} --num-nodes=3 # This version of the command does not build the driver or K8s, points to a # local K8s repo to get the e2e.test binary, and does not bring up or down the cluster @@ -42,4 +42,4 @@ ${PKGDIR}/bin/k8s-integration-test --kube-version=master --run-in-prow=false \ --staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \ --deploy-overlay-name=dev --bringup-cluster=false --teardown-cluster=false --local-k8s-dir=$KTOP \ --storageclass-file=sc-standard.yaml --do-driver-build=true --test-focus="External.Storage" \ ---gce-zone="us-central1-b" +--gce-zone="us-central1-b" --num-nodes=${NUM_NODES:-3} diff --git a/test/run-k8s-integration-migration-local.sh b/test/run-k8s-integration-migration-local.sh index 607f0c5e3..bf60078ff 100755 --- a/test/run-k8s-integration-migration-local.sh +++ b/test/run-k8s-integration-migration-local.sh @@ -19,7 +19,7 @@ ${PKGDIR}/bin/k8s-integration-test --kube-version=master --run-in-prow=false \ --staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \ --deploy-overlay-name=dev --test-focus=${GCE_PD_TEST_FOCUS} \ --kube-feature-gates="CSIMigration=true,CSIMigrationGCE=true" --migration-test=true --gce-zone="us-central1-b" \ ---deployment-strategy=gce --test-version=${test_version} +--deployment-strategy=gce --test-version=${test_version} --num-nodes=${NUM_NODES:-3} # This version of the command does not build the driver or K8s, points to a # local K8s repo to get the e2e.test binary, and does not bring up or down the cluster @@ -29,4 +29,4 @@ ${PKGDIR}/bin/k8s-integration-test --kube-version=master --run-in-prow=false \ # --staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \ # --deploy-overlay-name=dev --test-focus=${GCE_PD_TEST_FOCUS} \ # --bringup-cluster=false --teardown-cluster=false --local-k8s-dir=$KTOP --migration-test=true \ -# --do-driver-build=true --gce-zone=${GCE_PD_ZONE} \ No newline at end of file +# --do-driver-build=true --gce-zone=${GCE_PD_ZONE} --num-nodes=${NUM_NODES:-3} \ No newline at end of file diff --git a/test/run-k8s-integration-migration.sh b/test/run-k8s-integration-migration.sh index a49a1bddb..c488ed2c7 100755 --- a/test/run-k8s-integration-migration.sh +++ b/test/run-k8s-integration-migration.sh @@ -38,4 +38,5 @@ ${PKGDIR}/bin/k8s-integration-test --kube-version=${kube_version} \ --deploy-overlay-name=${overlay_name} --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \ --do-driver-build=${do_driver_build} --boskos-resource-type=${boskos_resource_type} \ --migration-test=true --test-focus=${GCE_PD_TEST_FOCUS} \ ---gce-zone="us-central1-b" --deployment-strategy=${deployment_strategy} --test-version=${test_version} +--gce-zone="us-central1-b" --deployment-strategy=${deployment_strategy} --test-version=${test_version} \ +--num-nodes=3 diff --git a/test/run-k8s-integration.sh b/test/run-k8s-integration.sh index 13fb60e39..b93e3f3d5 100755 --- a/test/run-k8s-integration.sh +++ b/test/run-k8s-integration.sh @@ -26,7 +26,7 @@ base_cmd="${PKGDIR}/bin/k8s-integration-test \ --run-in-prow=true --deploy-overlay-name=${overlay_name} --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \ --do-driver-build=${do_driver_build} --boskos-resource-type=${boskos_resource_type} \ --storageclass-file=sc-standard.yaml --test-focus="External.Storage" --gce-zone="us-central1-b" \ - --deployment-strategy=${deployment_strategy} --test-version=${test_version}" + --deployment-strategy=${deployment_strategy} --test-version=${test_version} --num-nodes=3" if [ "$deployment_strategy" = "gke" ]; then base_cmd="${base_cmd} --gke-cluster-version=${gke_cluster_version}"