From 531f5c7724968a741d1bb31afe17345bfbbab18d Mon Sep 17 00:00:00 2001 From: "Hantao (Will) Wang" Date: Wed, 5 Jun 2019 14:11:21 -0700 Subject: [PATCH] add ability to use gcloud to boot a gke cluster for e2e integration tests --- test/k8s-integration/main.go | 85 ++++++++++++++++++++++++++----- test/run-k8s-integration-local.sh | 5 +- test/run-k8s-integration.sh | 4 +- 3 files changed, 80 insertions(+), 14 deletions(-) diff --git a/test/k8s-integration/main.go b/test/k8s-integration/main.go index 571a29131..c3dbc8a8b 100644 --- a/test/k8s-integration/main.go +++ b/test/k8s-integration/main.go @@ -39,6 +39,8 @@ var ( kubeVersion = flag.String("kube-version", "master", "version of Kubernetes to download and use") kubeFeatureGates = flag.String("kube-feature-gates", "", "feature gates to set on new kubernetes cluster") localK8sDir = flag.String("local-k8s-dir", "", "local kubernetes/kubernetes directory to run e2e tests from") + deploymentStrat = flag.String("deployment-strategy", "gce", "choose between deploying on gce or gke") + gkeClusterVer = flag.String("gke-cluster-version", "latest", "version of Kubernetes master and node for gke") // Test infrastructure flags boskosResourceType = flag.String("boskos-resource-type", "gce-project", "name of the boskos resource type to reserve") @@ -59,6 +61,7 @@ var ( const ( pdImagePlaceholder = "gcr.io/gke-release/gcp-compute-persistent-disk-csi-driver" k8sBuildBinDir = "_output/dockerized/bin/linux/amd64" + gkeTestClusterName = "gcp-pd-csi-driver-test-cluster" ) func init() { @@ -99,6 +102,10 @@ func main() { glog.Fatalf("gce-zone is a required flag") } + if *deploymentStrat == "gke" && *migrationTest { + glog.Fatalf("Cannot set deployment strategy to 'gke' for migration tests.") + } + err := handle() if err != nil { glog.Fatalf("Failed to run integration test: %v", err) @@ -196,17 +203,38 @@ func handle() error { glog.V(4).Infof("Set Kubernetes feature gates: %v", *kubeFeatureGates) } - err = clusterUp(k8sDir, *gceZone) - if err != nil { - return fmt.Errorf("failed to cluster up: %v", err) + switch *deploymentStrat { + case "gce": + err = clusterUpGCE(k8sDir, *gceZone) + if err != nil { + return fmt.Errorf("failed to cluster up: %v", err) + } + case "gke": + err = clusterUpGKE(*gceZone) + if err != nil { + return fmt.Errorf("failed to cluster up: %v", err) + } + default: + return fmt.Errorf("deployment-strategy must be set to 'gce' or 'gke', but is: %s", *deploymentStrat) } + } if *teardownCluster { defer func() { - err := clusterDown(k8sDir) - if err != nil { - glog.Errorf("failed to cluster down: %v", err) + switch *deploymentStrat { + case "gce": + err := clusterDownGCE(k8sDir) + if err != nil { + glog.Errorf("failed to cluster down: %v", err) + } + case "gke": + err := clusterDownGKE(*gceZone) + if err != nil { + glog.Errorf("failed to cluster down: %v", err) + } + default: + glog.Errorf("deployment-strategy must be set to 'gce' or 'gke', but is: %s", *deploymentStrat) } }() } @@ -322,11 +350,21 @@ func runCommand(action string, cmd *exec.Cmd) error { return nil } -func clusterDown(k8sDir string) error { +func clusterDownGCE(k8sDir string) error { cmd := exec.Command(filepath.Join(k8sDir, "hack", "e2e-internal", "e2e-down.sh")) - err := runCommand("Bringing Down E2E Cluster", cmd) + err := runCommand("Bringing Down E2E Cluster on GCE", cmd) + if err != nil { + return fmt.Errorf("failed to bring down kubernetes e2e cluster on gce: %v", err) + } + return nil +} + +func clusterDownGKE(gceZone string) error { + cmd := exec.Command("gcloud", "container", "clusters", "delete", gkeTestClusterName, + "--zone", gceZone, "--quiet") + err := runCommand("Bringing Down E2E Cluster on GKE", cmd) if err != nil { - return fmt.Errorf("failed to bring down kubernetes e2e cluster: %v", err) + return fmt.Errorf("failed to bring down kubernetes e2e cluster on gke: %v", err) } return nil } @@ -340,15 +378,38 @@ func buildKubernetes(k8sDir string) error { return nil } -func clusterUp(k8sDir, gceZone string) error { +func clusterUpGCE(k8sDir, gceZone string) error { err := os.Setenv("KUBE_GCE_ZONE", gceZone) if err != nil { return err } cmd := exec.Command(filepath.Join(k8sDir, "hack", "e2e-internal", "e2e-up.sh")) - err = runCommand("Starting E2E Cluster", cmd) + err = runCommand("Starting E2E Cluster on GCE", cmd) + if err != nil { + return fmt.Errorf("failed to bring up kubernetes e2e cluster on gce: %v", err) + } + + return nil +} + +func clusterUpGKE(gceZone string) error { + out, err := exec.Command("gcloud", "container", "clusters", "list", "--zone", gceZone, + "--filter", fmt.Sprintf("name=%s", gkeTestClusterName)).CombinedOutput() + if err != nil { + return fmt.Errorf("failed to check for previous test cluster: %v %s", err, out) + } + if len(out) > 0 { + glog.Infof("Detected previous cluster %s. Deleting so a new one can be created...", gkeTestClusterName) + err = clusterDownGKE(gceZone) + if err != nil { + return err + } + } + cmd := exec.Command("gcloud", "container", "clusters", "create", gkeTestClusterName, + "--zone", gceZone, "--cluster-version", *gkeClusterVer, "--quiet") + err = runCommand("Staring E2E Cluster on GKE", cmd) if err != nil { - return fmt.Errorf("failed to bring up kubernetes e2e cluster: %v", err) + return fmt.Errorf("failed to bring up kubernetes e2e cluster on gke: %v", err) } return nil diff --git a/test/run-k8s-integration-local.sh b/test/run-k8s-integration-local.sh index 14377f70c..a913c8eb5 100755 --- a/test/run-k8s-integration-local.sh +++ b/test/run-k8s-integration-local.sh @@ -4,6 +4,8 @@ set -o nounset set -o errexit readonly PKGDIR=${GOPATH}/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver +readonly deployment_strategy=${DEPLOYMENT_STRATEGY:-gce} + source "${PKGDIR}/deploy/common.sh" ensure_var GCE_PD_CSI_STAGING_IMAGE @@ -14,7 +16,8 @@ make -C ${PKGDIR} test-k8s-integration # ${PKGDIR}/bin/k8s-integration-test --kube-version=master --run-in-prow=false \ # --staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \ # --deploy-overlay-name=dev --storageclass-file=sc-standard.yaml \ -# --test-focus="External.Storage" --gce-zone="us-central1-b" +# --test-focus="External.Storage" --gce-zone="us-central1-b" \ +# --deployment-strategy=${deployment_strategy} # This version of the command does not build the driver or K8s, points to a # local K8s repo to get the e2e.test binary, and does not bring up or down the cluster diff --git a/test/run-k8s-integration.sh b/test/run-k8s-integration.sh index f074aed23..3ba42b0de 100755 --- a/test/run-k8s-integration.sh +++ b/test/run-k8s-integration.sh @@ -13,10 +13,12 @@ readonly PKGDIR=${GOPATH}/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver readonly overlay_name="${GCE_PD_OVERLAY_NAME:-stable}" readonly boskos_resource_type="${GCE_PD_BOSKOS_RESOURCE_TYPE:-gce-project}" readonly do_driver_build="${GCE_PD_DO_DRIVER_BUILD:-true}" +readonly deployment_strategy=${DEPLOYMENT_STRATEGY:-gce} export GCE_PD_VERBOSITY=9 make -C ${PKGDIR} test-k8s-integration ${PKGDIR}/bin/k8s-integration-test --kube-version=${GCE_PD_KUBE_VERSION:-master} \ --run-in-prow=true --deploy-overlay-name=${overlay_name} --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \ --do-driver-build=${do_driver_build} --boskos-resource-type=${boskos_resource_type} \ ---storageclass-file=sc-standard.yaml --test-focus="External.Storage" --gce-zone="us-central1-b" +--storageclass-file=sc-standard.yaml --test-focus="External.Storage" --gce-zone="us-central1-b" \ +--deployment-strategy=${deployment_strategy}