From 1184d0d9480982159dc5284b4302992e474164d2 Mon Sep 17 00:00:00 2001 From: Mauricio Poppe Date: Fri, 14 Jan 2022 22:20:18 +0000 Subject: [PATCH 1/3] Move run-windows-k8s-integration.sh script to use kubetest2 Please keep this commit as reference --- test/run-windows-k8s-integration.sh | 44 +++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/test/run-windows-k8s-integration.sh b/test/run-windows-k8s-integration.sh index 08b204dd8..f40e9e6e7 100755 --- a/test/run-windows-k8s-integration.sh +++ b/test/run-windows-k8s-integration.sh @@ -6,17 +6,23 @@ # use the driver version from the overlay # GCE_PD_BOSKOS_RESOURCE_TYPE: name of the boskos resource type to reserve +set -o xtrace set -o nounset set -o errexit readonly PKGDIR=${GOPATH}/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver -readonly overlay_name="${GCE_PD_OVERLAY_NAME:-stable-master}" +readonly overlay_name="${GCE_PD_OVERLAY_NAME:-noauth}" readonly do_driver_build="${GCE_PD_DO_DRIVER_BUILD:-true}" readonly deployment_strategy=${DEPLOYMENT_STRATEGY:-gce} +readonly kube_version=${GCE_PD_KUBE_VERSION:-master} readonly test_version=${TEST_VERSION:-master} readonly gce_zone=${GCE_CLUSTER_ZONE:-us-central1-b} readonly teardown_driver=${GCE_PD_TEARDOWN_DRIVER:-true} readonly use_kubetest2=${USE_KUBETEST2:-true} +readonly num_windows_nodes=${NUM_WINDOWS_NODES:-3} + +# build platforms for `make quick-release` +export KUBE_BUILD_PLATFORMS="linux/amd64 windows/amd64" make -C "${PKGDIR}" test-k8s-integration @@ -27,11 +33,37 @@ if [ "$use_kubetest2" = true ]; then go install sigs.k8s.io/kubetest2/kubetest2-tester-ginkgo@latest; fi +# TODO(mauriciopoppe): remove this assignment +E2E_GOOGLE_APPLICATION_CREDENTIALS=sa + +# TODO(mauriciopoppe): change run-in-prow=true + +# TODO(mauriciopoppe): change overlay back to stable-master + +# TODO(mauriciopoppe): remove --staging-image and this flag +GCE_PD_CSI_STAGING_IMAGE=gcr.io/mauriciopoppe-gke-dev/gcp-compute-persistent-disk-csi-driver + +# TODO(mauriciopoppe): change to --do-driver-build=${do_driver_build} \ + base_cmd="${PKGDIR}/bin/k8s-integration-test \ - --platform=windows --bringup-cluster=false --teardown-cluster=false --teardown-driver=${teardown_driver}\ - --run-in-prow=true --deploy-overlay-name=${overlay_name} --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \ - --do-driver-build=${do_driver_build} --gce-zone=${gce_zone} --test-version=${test_version}\ - --storageclass-files=sc-windows.yaml --snapshotclass-file=pd-volumesnapshotclass.yaml --test-focus='External.Storage' \ - --deployment-strategy=${deployment_strategy} --use-kubetest2=${use_kubetest2}" + --staging-image="${GCE_PD_CSI_STAGING_IMAGE}" \ + --run-in-prow=false \ + --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \ + --deployment-strategy=${deployment_strategy} \ + --gce-zone=${gce_zone} \ + --platform=windows \ + --bringup-cluster=true \ + --teardown-cluster=true \ + --num-nodes=1 \ + --num-windows-nodes=${num_windows_nodes} \ + --teardown-driver=${teardown_driver} \ + --do-driver-build=true \ + --deploy-overlay-name=${overlay_name} \ + --test-version=${test_version} \ + --kube-version=${kube_version} \ + --storageclass-files=sc-windows.yaml \ + --snapshotclass-file=pd-volumesnapshotclass.yaml \ + --test-focus='External.Storage' \ + --use-kubetest2=${use_kubetest2}" eval "$base_cmd" From 35cce116e21a62826b34a53dd1a4612fe49c535a Mon Sep 17 00:00:00 2001 From: Mauricio Poppe Date: Fri, 14 Jan 2022 22:28:38 +0000 Subject: [PATCH 2/3] kubernetes bootstrap through k8s-integration for both linux & windows --- test/k8s-integration/cluster.go | 19 ++++++++-- test/k8s-integration/main.go | 59 ++++++++++++++++------------- test/k8s-integration/utils.go | 6 +-- test/run-k8s-integration-ci.sh | 3 +- test/run-k8s-integration.sh | 1 + test/run-windows-k8s-integration.sh | 21 ++-------- 6 files changed, 57 insertions(+), 52 deletions(-) diff --git a/test/k8s-integration/cluster.go b/test/k8s-integration/cluster.go index d30103f03..5a2c2b0c8 100644 --- a/test/k8s-integration/cluster.go +++ b/test/k8s-integration/cluster.go @@ -36,6 +36,7 @@ func isRegionalGKECluster(gceZone, gceRegion string) bool { func clusterDownGCE(k8sDir string) error { cmd := exec.Command(filepath.Join(k8sDir, "hack", "e2e-internal", "e2e-down.sh")) + cmd.Env = os.Environ() err := runCommand("Bringing Down E2E Cluster on GCE", cmd) if err != nil { return fmt.Errorf("failed to bring down kubernetes e2e cluster on gce: %v", err) @@ -60,14 +61,15 @@ func clusterDownGKE(gceZone, gceRegion string) error { func buildKubernetes(k8sDir, command string) error { cmd := exec.Command("make", "-C", k8sDir, command) - err := runCommand("Building Kubernetes", cmd) + cmd.Env = os.Environ() + err := runCommand(fmt.Sprintf("Running command in kubernetes/kubernetes path=%s", k8sDir), cmd) if err != nil { return fmt.Errorf("failed to build Kubernetes: %v", err) } return nil } -func clusterUpGCE(k8sDir, gceZone string, numNodes int, imageType string) error { +func clusterUpGCE(k8sDir, gceZone string, numNodes int, numWindowsNodes int, imageType string) error { kshPath := filepath.Join(k8sDir, "cluster", "kubectl.sh") _, err := os.Stat(kshPath) if err == nil { @@ -98,6 +100,14 @@ func clusterUpGCE(k8sDir, gceZone string, numNodes int, imageType string) error return err } + // the chain is NUM_WINDOWS_NODES -> --num-windows-nodes -> NUM_WINDOWS_NODES + // runCommand runs e2e-up.sh inheriting env vars so the `--num-windows-nodes` + // flags might not be needed, added to be similar to the setup of NUM_NODES + err = os.Setenv("NUM_WINDOWS_NODES", strconv.Itoa(numWindowsNodes)) + if err != nil { + return err + } + // The default master size with few nodes is too small; the tests must hit the API server // more than usual. The main issue seems to be memory, to reduce GC times that stall the // api server. For defaults, get-master-size in k/k/cluster/gce/config-common.sh. @@ -113,6 +123,7 @@ func clusterUpGCE(k8sDir, gceZone string, numNodes int, imageType string) error return err } cmd := exec.Command(filepath.Join(k8sDir, "hack", "e2e-internal", "e2e-up.sh")) + cmd.Env = os.Environ() err = runCommand("Starting E2E Cluster on GCE", cmd) if err != nil { return fmt.Errorf("failed to bring up kubernetes e2e cluster on gce: %v", err) @@ -147,7 +158,7 @@ func setImageTypeEnvs(imageType string) error { return nil } -func clusterUpGKE(gceZone, gceRegion string, numNodes int, imageType string, useManagedDriver bool) error { +func clusterUpGKE(gceZone, gceRegion string, numNodes int, numWindowsNodes int, imageType string, useManagedDriver bool) error { locationArg, locationVal, err := gkeLocationArgs(gceZone, gceRegion) if err != nil { return err @@ -213,7 +224,7 @@ func clusterUpGKE(gceZone, gceRegion string, numNodes int, imageType string, use func downloadKubernetesSource(pkgDir, k8sIoDir, kubeVersion string) error { k8sDir := filepath.Join(k8sIoDir, "kubernetes") - klog.Infof("Downloading Kubernetes source for %s", kubeVersion) + klog.Infof("Downloading Kubernetes source v=%s to path=%s", kubeVersion, k8sIoDir) if err := os.MkdirAll(k8sIoDir, 0777); err != nil { return err diff --git a/test/k8s-integration/main.go b/test/k8s-integration/main.go index 162f0dfe8..5a29aefeb 100644 --- a/test/k8s-integration/main.go +++ b/test/k8s-integration/main.go @@ -45,7 +45,8 @@ var ( localK8sDir = flag.String("local-k8s-dir", "", "local prebuilt kubernetes/kubernetes directory to use for cluster and test binaries") deploymentStrat = flag.String("deployment-strategy", "gce", "choose between deploying on gce or gke") gkeClusterVer = flag.String("gke-cluster-version", "", "version of Kubernetes master and node for gke") - numNodes = flag.Int("num-nodes", -1, "the number of nodes in the test cluster") + numNodes = flag.Int("num-nodes", 0, "the number of nodes in the test cluster") + numWindowsNodes = flag.Int("num-windows-nodes", 0, "the number of Windows nodes in the test cluster") imageType = flag.String("image-type", "cos", "the image type to use for the cluster") gkeReleaseChannel = flag.String("gke-release-channel", "", "GKE release channel to be used for cluster deploy. One of 'rapid', 'stable' or 'regular'") gkeTestClusterPrefix = flag.String("gke-cluster-prefix", "pdcsi", "Prefix of GKE cluster names. A random suffix will be appended to form the full name.") @@ -160,10 +161,6 @@ func main() { } } - if *platform == "windows" { - ensureFlag(bringupCluster, false, "bringupCluster is set to false if it is for testing in windows cluster") - } - if *deploymentStrat == "gke" { ensureVariable(kubeVersion, false, "Cannot set kube-version when using deployment strategy 'gke'. Use gke-cluster-version.") ensureExactlyOneVariableSet([]*string{gkeClusterVer, gkeReleaseChannel}, @@ -186,9 +183,12 @@ func main() { ensureVariable(testVersion, false, "Cannot set a test version when using a local k8s dir.") } - if *numNodes == -1 && *bringupCluster { + if *numNodes == 0 && *bringupCluster { klog.Fatalf("num-nodes must be set to number of nodes in cluster") } + if *numWindowsNodes == 0 && *bringupCluster && *platform == "windows" { + klog.Fatalf("num-windows-nodes must be set if the platform is windows") + } err := handle() if err != nil { @@ -225,24 +225,19 @@ func handle() error { if err != nil { return fmt.Errorf("failed to get gcloud project: %s, err: %v", oldProject, err) } - // TODO: Currently for prow tests with linux cluster, here it manually sets up a project from Boskos. - // For Windows, we used kubernetes_e2e.py which already set up the project and kubernetes automatically. - // Will update Linux in the future to use the same way as Windows test. - if *platform != "windows" { - newproject, _ := testutils.SetupProwConfig(*boskosResourceType) - err = setEnvProject(newproject) + newproject, _ := testutils.SetupProwConfig(*boskosResourceType) + err = setEnvProject(newproject) + if err != nil { + return fmt.Errorf("failed to set project environment to %s: %v", newproject, err) + } + + defer func() { + err = setEnvProject(string(oldProject)) if err != nil { - return fmt.Errorf("failed to set project environment to %s: %v", newproject, err) + klog.Errorf("failed to set project environment to %s: %v", oldProject, err) } - - defer func() { - err = setEnvProject(string(oldProject)) - if err != nil { - klog.Errorf("failed to set project environment to %s: %v", oldProject, err) - } - }() - project = newproject - } + }() + project = newproject if *doDriverBuild { *stagingImage = fmt.Sprintf("gcr.io/%s/gcp-persistent-disk-csi-driver", strings.TrimSpace(string(project))) } @@ -256,6 +251,7 @@ func handle() error { // Build and push the driver, if required. Defer the driver image deletion. if *doDriverBuild { + klog.Infof("Building GCE PD CSI Driver") err := pushImage(testParams.pkgDir, *stagingImage, testParams.stagingVersion, testParams.platform) if err != nil { return fmt.Errorf("failed pushing image: %v", err) @@ -326,9 +322,9 @@ func handle() error { var err error = nil switch *deploymentStrat { case "gce": - err = clusterUpGCE(testParams.k8sSourceDir, *gceZone, *numNodes, testParams.imageType) + err = clusterUpGCE(testParams.k8sSourceDir, *gceZone, *numNodes, *numWindowsNodes, testParams.imageType) case "gke": - err = clusterUpGKE(*gceZone, *gceRegion, *numNodes, testParams.imageType, testParams.useGKEManagedDriver) + err = clusterUpGKE(*gceZone, *gceRegion, *numNodes, *numWindowsNodes, testParams.imageType, testParams.useGKEManagedDriver) default: err = fmt.Errorf("deployment-strategy must be set to 'gce' or 'gke', but is: %s", testParams.deploymentStrategy) } @@ -360,6 +356,8 @@ func handle() error { // For windows cluster, when cluster is up, all Windows nodes are tainted with NoSchedule to avoid linux pods // being scheduled to Windows nodes. When running windows tests, we need to remove the taint. if testParams.platform == "windows" { + klog.Infof("Removing taints from all windows nodes.") + nodesCmd := exec.Command("kubectl", "get", "nodes", "-l", "kubernetes.io/os=windows", "-o", "name") out, err := nodesCmd.CombinedOutput() if err != nil { @@ -428,6 +426,7 @@ func handle() error { // unschedulable. testParams.allowedNotReadyNodes = 0 if *platform == "windows" { + klog.Infof("Tainting linux nodes") nodesCmd := exec.Command("kubectl", "get", "nodes", "-l", "kubernetes.io/os=linux", "-o", "name") out, err := nodesCmd.CombinedOutput() if err != nil { @@ -664,7 +663,6 @@ func runTestsWithConfig(testParams *testParameters, testConfigArg, reportPrefix testArgs := fmt.Sprintf("%s %s", ginkgoArgs, testConfigArg) // kubetest2 flags - var runID string if uid, exists := os.LookupEnv("PROW_JOB_ID"); exists && uid != "" { // reuse uid for CI use cases @@ -673,20 +671,27 @@ func runTestsWithConfig(testParams *testParameters, testConfigArg, reportPrefix runID = string(uuid.NewUUID()) } + // Usage: kubetest2 [Flags] [DeployerFlags] -- [TesterArgs] + // [Flags] kubeTest2Args := []string{ *deploymentStrat, fmt.Sprintf("--run-id=%s", runID), "--test=ginkgo", } + + // [DeployerFlags] kubeTest2Args = append(kubeTest2Args, testParams.cloudProviderArgs...) if kubetestDumpDir != "" { kubeTest2Args = append(kubeTest2Args, fmt.Sprintf("--artifacts=%s", kubetestDumpDir)) } + kubeTest2Args = append(kubeTest2Args, "--") + + // [TesterArgs] if len(*testVersion) != 0 { if *testVersion == "master" { - // the kubernetes binaries should've already been built above - // or by the user if --localK8sDir was set, these binaries should be copied to the + // the kubernetes binaries should've already been built above because of `--kube-version` + // or by the user if --local-k8s-dir was set, these binaries should be copied to the // path sent to kubetest2 through its --artifacts path // pkg/_artifacts is the default value that kubetests uses for --artifacts diff --git a/test/k8s-integration/utils.go b/test/k8s-integration/utils.go index ecc7ebb62..201baa346 100644 --- a/test/k8s-integration/utils.go +++ b/test/k8s-integration/utils.go @@ -1,7 +1,6 @@ package main import ( - "fmt" "io/ioutil" "os" "os/exec" @@ -14,8 +13,9 @@ func runCommand(action string, cmd *exec.Cmd) error { cmd.Stdin = os.Stdin cmd.Stderr = os.Stderr - fmt.Printf("%s\n", action) - fmt.Printf("%s\n", cmd.Args) + klog.Infof("%s", action) + klog.Infof("cmd env=%v", cmd.Env) + klog.Infof("cmd args=%s", cmd.Args) err := cmd.Start() if err != nil { diff --git a/test/run-k8s-integration-ci.sh b/test/run-k8s-integration-ci.sh index c10f7ca70..6027898b6 100755 --- a/test/run-k8s-integration-ci.sh +++ b/test/run-k8s-integration-ci.sh @@ -1,4 +1,4 @@ -#!/bin/bash -x +#!/bin/bash # Optional environment variables # GCE_PD_OVERLAY_NAME: which Kustomize overlay to deploy with @@ -6,6 +6,7 @@ # use the driver version from the overlay # GCE_PD_BOSKOS_RESOURCE_TYPE: name of the boskos resource type to reserve +set -o xtrace set -o nounset set -o errexit diff --git a/test/run-k8s-integration.sh b/test/run-k8s-integration.sh index 822bda8fd..3ade9fc19 100755 --- a/test/run-k8s-integration.sh +++ b/test/run-k8s-integration.sh @@ -6,6 +6,7 @@ # use the driver version from the overlay # GCE_PD_BOSKOS_RESOURCE_TYPE: name of the boskos resource type to reserve +set -o xtrace set -o nounset set -o errexit diff --git a/test/run-windows-k8s-integration.sh b/test/run-windows-k8s-integration.sh index f40e9e6e7..51dd41c9f 100755 --- a/test/run-windows-k8s-integration.sh +++ b/test/run-windows-k8s-integration.sh @@ -11,7 +11,7 @@ set -o nounset set -o errexit readonly PKGDIR=${GOPATH}/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver -readonly overlay_name="${GCE_PD_OVERLAY_NAME:-noauth}" +readonly overlay_name="${GCE_PD_OVERLAY_NAME:-stable-master}" readonly do_driver_build="${GCE_PD_DO_DRIVER_BUILD:-true}" readonly deployment_strategy=${DEPLOYMENT_STRATEGY:-gce} readonly kube_version=${GCE_PD_KUBE_VERSION:-master} @@ -22,7 +22,7 @@ readonly use_kubetest2=${USE_KUBETEST2:-true} readonly num_windows_nodes=${NUM_WINDOWS_NODES:-3} # build platforms for `make quick-release` -export KUBE_BUILD_PLATFORMS="linux/amd64 windows/amd64" +export KUBE_BUILD_PLATFORMS=${KUBE_BUILD_PLATFORMS:-"linux/amd64 windows/amd64"} make -C "${PKGDIR}" test-k8s-integration @@ -33,21 +33,8 @@ if [ "$use_kubetest2" = true ]; then go install sigs.k8s.io/kubetest2/kubetest2-tester-ginkgo@latest; fi -# TODO(mauriciopoppe): remove this assignment -E2E_GOOGLE_APPLICATION_CREDENTIALS=sa - -# TODO(mauriciopoppe): change run-in-prow=true - -# TODO(mauriciopoppe): change overlay back to stable-master - -# TODO(mauriciopoppe): remove --staging-image and this flag -GCE_PD_CSI_STAGING_IMAGE=gcr.io/mauriciopoppe-gke-dev/gcp-compute-persistent-disk-csi-driver - -# TODO(mauriciopoppe): change to --do-driver-build=${do_driver_build} \ - base_cmd="${PKGDIR}/bin/k8s-integration-test \ - --staging-image="${GCE_PD_CSI_STAGING_IMAGE}" \ - --run-in-prow=false \ + --run-in-prow=true \ --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \ --deployment-strategy=${deployment_strategy} \ --gce-zone=${gce_zone} \ @@ -57,7 +44,7 @@ base_cmd="${PKGDIR}/bin/k8s-integration-test \ --num-nodes=1 \ --num-windows-nodes=${num_windows_nodes} \ --teardown-driver=${teardown_driver} \ - --do-driver-build=true \ + --do-driver-build=${do_driver_build} \ --deploy-overlay-name=${overlay_name} \ --test-version=${test_version} \ --kube-version=${kube_version} \ From 72c1f4bca70cd0eca2502130bb5ce9b36ee6d31d Mon Sep 17 00:00:00 2001 From: Mauricio Poppe Date: Fri, 21 Jan 2022 22:42:53 +0000 Subject: [PATCH 3/3] Upgrade windows migration job to the new structure --- test/run-windows-k8s-integration.sh | 3 +- ...ration.sh => run-windows-k8s-migration.sh} | 39 +++++++++++++------ 2 files changed, 29 insertions(+), 13 deletions(-) rename test/{run-k8s-windows-migration.sh => run-windows-k8s-migration.sh} (59%) diff --git a/test/run-windows-k8s-integration.sh b/test/run-windows-k8s-integration.sh index 51dd41c9f..81f0c6ad7 100755 --- a/test/run-windows-k8s-integration.sh +++ b/test/run-windows-k8s-integration.sh @@ -12,12 +12,12 @@ set -o errexit readonly PKGDIR=${GOPATH}/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver readonly overlay_name="${GCE_PD_OVERLAY_NAME:-stable-master}" +readonly boskos_resource_type="${GCE_PD_BOSKOS_RESOURCE_TYPE:-gce-project}" readonly do_driver_build="${GCE_PD_DO_DRIVER_BUILD:-true}" readonly deployment_strategy=${DEPLOYMENT_STRATEGY:-gce} readonly kube_version=${GCE_PD_KUBE_VERSION:-master} readonly test_version=${TEST_VERSION:-master} readonly gce_zone=${GCE_CLUSTER_ZONE:-us-central1-b} -readonly teardown_driver=${GCE_PD_TEARDOWN_DRIVER:-true} readonly use_kubetest2=${USE_KUBETEST2:-true} readonly num_windows_nodes=${NUM_WINDOWS_NODES:-3} @@ -36,6 +36,7 @@ fi base_cmd="${PKGDIR}/bin/k8s-integration-test \ --run-in-prow=true \ --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \ + --boskos-resource-type=${boskos_resource_type} \ --deployment-strategy=${deployment_strategy} \ --gce-zone=${gce_zone} \ --platform=windows \ diff --git a/test/run-k8s-windows-migration.sh b/test/run-windows-k8s-migration.sh similarity index 59% rename from test/run-k8s-windows-migration.sh rename to test/run-windows-k8s-migration.sh index 8e974166a..67edbf4c1 100755 --- a/test/run-k8s-windows-migration.sh +++ b/test/run-windows-k8s-migration.sh @@ -6,12 +6,12 @@ # use the driver version from the overlay # GCE_PD_BOSKOS_RESOURCE_TYPE: name of the boskos resource type to reserve +set -o xtrace set -o nounset set -o errexit export GCE_PD_VERBOSITY=9 readonly PKGDIR=${GOPATH}/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver - readonly overlay_name="${GCE_PD_OVERLAY_NAME:-stable-master}" readonly boskos_resource_type="${GCE_PD_BOSKOS_RESOURCE_TYPE:-gce-project}" readonly do_driver_build="${GCE_PD_DO_DRIVER_BUILD:-true}" @@ -21,6 +21,10 @@ readonly test_version=${TEST_VERSION:-master} readonly gce_zone=${GCE_CLUSTER_ZONE:-us-central1-b} readonly feature_gates="CSIMigration=true,CSIMigrationGCE=true,ExpandCSIVolumes=true" readonly use_kubetest2=${USE_KUBETEST2:-false} +readonly num_windows_nodes=${NUM_WINDOWS_NODES:-3} + +# build platforms for `make quick-release` +export KUBE_BUILD_PLATFORMS=${KUBE_BUILD_PLATFORMS:-"linux/amd64 windows/amd64"} make -C "${PKGDIR}" test-k8s-integration @@ -36,15 +40,26 @@ readonly GCE_PD_TEST_FOCUS="PersistentVolumes\sGCEPD|[V|v]olume\sexpand|\[sig-st # TODO(#167): Enable reconstructions tests -${PKGDIR}/bin/k8s-integration-test \ - --platform=windows --bringup-cluster=false --teardown-cluster=false \ - --run-in-prow=true \ - --kube-feature-gates=${feature_gates} --run-in-prow=true \ - --deploy-overlay-name=${overlay_name} --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \ - --do-driver-build=${do_driver_build} --boskos-resource-type=${boskos_resource_type} \ - --migration-test=true --test-focus=${GCE_PD_TEST_FOCUS} \ - --gce-zone=${gce_zone} --deployment-strategy=${deployment_strategy} --test-version=${test_version} \ - --use-kubetest2=${use_kubetest2} - -#eval "$base_cmd" +base_cmd="${PKGDIR}/bin/k8s-integration-test \ + --run-in-prow=true \ + --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \ + --boskos-resource-type=${boskos_resource_type} \ + --deployment-strategy=${deployment_strategy} \ + --gce-zone=${gce_zone} \ + --platform=windows \ + --bringup-cluster=true \ + --teardown-cluster=true \ + --num-nodes=1 \ + --num-windows-nodes=${num_windows_nodes} \ + --teardown-driver=${teardown_driver} \ + --do-driver-build=${do_driver_build} \ + --deploy-overlay-name=${overlay_name} \ + --test-version=${test_version} \ + --kube-version=${kube_version} \ + --kube-feature-gates=${feature_gates} + --storageclass-files=sc-windows.yaml \ + --snapshotclass-file=pd-volumesnapshotclass.yaml \ + --test-focus=${GCE_PD_TEST_FOCUS} \ + --use-kubetest2=${use_kubetest2}" +eval "$base_cmd"