From 3249fa2c02ee8137c6b72e0ff51cb77e34a36011 Mon Sep 17 00:00:00 2001 From: Saikat Roychowdhury Date: Wed, 27 May 2020 18:19:52 +0000 Subject: [PATCH] Add option to deploy GKE managed PD CSI driver for integration tests --- test/k8s-integration/cluster.go | 18 ++++++++--- test/k8s-integration/main.go | 51 +++++++++++++++++++------------ test/k8s-integration/utils.go | 6 ++++ test/run-k8s-integration-local.sh | 10 +++++- test/run-k8s-integration.sh | 6 ++-- 5 files changed, 65 insertions(+), 26 deletions(-) diff --git a/test/k8s-integration/cluster.go b/test/k8s-integration/cluster.go index 92c5a8fae..d26f3d57e 100644 --- a/test/k8s-integration/cluster.go +++ b/test/k8s-integration/cluster.go @@ -131,7 +131,7 @@ func setImageTypeEnvs(imageType string) error { return nil } -func clusterUpGKE(gceZone, gceRegion string, numNodes int, imageType string) error { +func clusterUpGKE(gceZone, gceRegion string, numNodes int, imageType string, useManagedDriver bool) error { locationArg, locationVal, err := gkeLocationArgs(gceZone, gceRegion) if err != nil { return err @@ -150,9 +150,19 @@ func clusterUpGKE(gceZone, gceRegion string, numNodes int, imageType string) err return err } } - cmd := exec.Command("gcloud", "container", "clusters", "create", gkeTestClusterName, - locationArg, locationVal, "--cluster-version", *gkeClusterVer, "--num-nodes", strconv.Itoa(numNodes), - "--quiet", "--machine-type", "n1-standard-2", "--image-type", imageType) + + var cmd *exec.Cmd + if useManagedDriver { + // PD CSI Driver add on is enabled only in gcloud beta. + cmd = exec.Command("gcloud", "beta", "container", "clusters", "create", gkeTestClusterName, + locationArg, locationVal, "--cluster-version", *gkeClusterVer, "--num-nodes", strconv.Itoa(numNodes), + "--quiet", "--machine-type", "n1-standard-2", "--image-type", imageType, "--addons", "GcePersistentDiskCsiDriver") + } else { + cmd = exec.Command("gcloud", "container", "clusters", "create", gkeTestClusterName, + locationArg, locationVal, "--cluster-version", *gkeClusterVer, "--num-nodes", strconv.Itoa(numNodes), + "--quiet", "--machine-type", "n1-standard-2", "--image-type", imageType) + } + err = runCommand("Staring E2E Cluster on GKE", cmd) if err != nil { return fmt.Errorf("failed to bring up kubernetes e2e cluster on gke: %v", err) diff --git a/test/k8s-integration/main.go b/test/k8s-integration/main.go index b807e776b..df3c79cba 100644 --- a/test/k8s-integration/main.go +++ b/test/k8s-integration/main.go @@ -51,10 +51,11 @@ var ( inProw = flag.Bool("run-in-prow", false, "is the test running in PROW") // Driver flags - stagingImage = flag.String("staging-image", "", "name of image to stage to") - saFile = flag.String("service-account-file", "", "path of service account file") - deployOverlayName = flag.String("deploy-overlay-name", "", "which kustomize overlay to deploy the driver with") - doDriverBuild = flag.Bool("do-driver-build", true, "building the driver from source") + stagingImage = flag.String("staging-image", "", "name of image to stage to") + saFile = flag.String("service-account-file", "", "path of service account file") + deployOverlayName = flag.String("deploy-overlay-name", "", "which kustomize overlay to deploy the driver with") + doDriverBuild = flag.Bool("do-driver-build", true, "building the driver from source") + useGKEManagedDriver = flag.Bool("use-gke-managed-driver", false, "use GKE managed PD CSI driver for the tests") // Test flags migrationTest = flag.Bool("migration-test", false, "sets the flag on the e2e binary signalling migration") @@ -75,12 +76,21 @@ func init() { func main() { flag.Parse() - if !*inProw { + if !*inProw && !*useGKEManagedDriver { ensureVariable(stagingImage, true, "staging-image is a required flag, please specify the name of image to stage to") } + if *useGKEManagedDriver { + ensureVariableVal(deploymentStrat, "gke", "deployment strategy must be GKE for using managed driver") + ensureFlag(doDriverBuild, false, "driver build flag will be ignored when using GKE managed driver") + ensureFlag(teardownDriver, false, "driver teardown flag will be ignored when using GKE managed driver") + } + ensureVariable(saFile, true, "service-account-file is a required flag") - ensureVariable(deployOverlayName, true, "deploy-overlay-name is a required flag") + if !*useGKEManagedDriver { + ensureVariable(deployOverlayName, true, "deploy-overlay-name is a required flag") + } + ensureVariable(testFocus, true, "test-focus is a required flag") ensureVariable(imageType, true, "image type is a required flag. Available options include 'cos' and 'ubuntu'") @@ -243,7 +253,7 @@ func handle() error { case "gce": err = clusterUpGCE(k8sDir, *gceZone, *numNodes, *imageType) case "gke": - err = clusterUpGKE(*gceZone, *gceRegion, *numNodes, *imageType) + err = clusterUpGKE(*gceZone, *gceRegion, *numNodes, *imageType, *useGKEManagedDriver) default: err = fmt.Errorf("deployment-strategy must be set to 'gce' or 'gke', but is: %s", *deploymentStrat) } @@ -272,21 +282,24 @@ func handle() error { }() } - // Install the driver and defer its teardown - err := installDriver(goPath, pkgDir, *stagingImage, stagingVersion, *deployOverlayName, *doDriverBuild) - if *teardownDriver { - defer func() { - // TODO (#140): collect driver logs - if teardownErr := deleteDriver(goPath, pkgDir, *deployOverlayName); teardownErr != nil { - klog.Errorf("failed to delete driver: %v", teardownErr) - } - }() - } - if err != nil { - return fmt.Errorf("failed to install CSI Driver: %v", err) + if !*useGKEManagedDriver { + // Install the driver and defer its teardown + err := installDriver(goPath, pkgDir, *stagingImage, stagingVersion, *deployOverlayName, *doDriverBuild) + if *teardownDriver { + defer func() { + // TODO (#140): collect driver logs + if teardownErr := deleteDriver(goPath, pkgDir, *deployOverlayName); teardownErr != nil { + klog.Errorf("failed to delete driver: %v", teardownErr) + } + }() + } + if err != nil { + return fmt.Errorf("failed to install CSI Driver: %v", err) + } } var cloudProviderArgs []string + var err error switch *deploymentStrat { case "gke": cloudProviderArgs, err = getGKEKubeTestArgs(*gceZone, *gceRegion, *imageType) diff --git a/test/k8s-integration/utils.go b/test/k8s-integration/utils.go index ea981ceac..d41ccb685 100644 --- a/test/k8s-integration/utils.go +++ b/test/k8s-integration/utils.go @@ -79,3 +79,9 @@ func shredFile(filePath string) { klog.V(4).Infof("Failed to remove service account file %s: %v", filePath, err) } } + +func ensureVariableVal(v *string, val string, msgOnError string) { + if *v != val { + klog.Fatal(msgOnError) + } +} diff --git a/test/run-k8s-integration-local.sh b/test/run-k8s-integration-local.sh index dd507dda4..c318db007 100755 --- a/test/run-k8s-integration-local.sh +++ b/test/run-k8s-integration-local.sh @@ -10,7 +10,6 @@ readonly test_version=${TEST_VERSION:-master} source "${PKGDIR}/deploy/common.sh" -ensure_var GCE_PD_CSI_STAGING_IMAGE ensure_var GCE_PD_SA_DIR make -C ${PKGDIR} test-k8s-integration @@ -56,6 +55,15 @@ make -C ${PKGDIR} test-k8s-integration #--storageclass-file=sc-standard.yaml --snapshotclass-file=pd-volumesnapshotclass.yaml --do-driver-build=true \ #--gce-zone="us-central1-b" --num-nodes=${NUM_NODES:-3} +# This version of the command brings up (and subsequently tears down) a GKE +# cluster with managed GCE PersistentDisk CSI driver add-on enabled, and points to +# the local K8s repository to get the e2e test binary. +# ${PKGDIR}/bin/k8s-integration-test --run-in-prow=false --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \ +# --test-focus="External.*Storage.*snapshot" --local-k8s-dir=$KTOP --storageclass-file=sc-standard.yaml \ +# --snapshotclass-file=pd-volumesnapshotclass.yaml --do-driver-build=false --teardown-driver=false \ +# --gce-zone="us-central1-c" --num-nodes=${NUM_NODES:-3} --gke-cluster-version="latest" --deployment-strategy="gke" \ +# --use-gke-managed-driver=true + # This version of the command does not build the driver or K8s, points to a # local K8s repo to get the e2e.test binary, and does not bring up or down the cluster diff --git a/test/run-k8s-integration.sh b/test/run-k8s-integration.sh index 625d0ecc6..c43572cf3 100755 --- a/test/run-k8s-integration.sh +++ b/test/run-k8s-integration.sh @@ -20,6 +20,8 @@ readonly test_version=${TEST_VERSION:-master} readonly gce_zone=${GCE_CLUSTER_ZONE:-us-central1-b} readonly gce_region=${GCE_CLUSTER_REGION:-} readonly image_type=${IMAGE_TYPE:-cos} +readonly use_gke_managed_driver=${GCE_PD_USE_MANAGED_DRIVER:-false} +readonly teardown_driver=${GCE_PD_TEARDOWN_DRIVER:-true} export GCE_PD_VERBOSITY=9 @@ -27,10 +29,10 @@ make -C ${PKGDIR} test-k8s-integration base_cmd="${PKGDIR}/bin/k8s-integration-test \ --run-in-prow=true --deploy-overlay-name=${overlay_name} --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \ - --do-driver-build=${do_driver_build} --boskos-resource-type=${boskos_resource_type} \ + --do-driver-build=${do_driver_build} --teardown-driver=${teardown_driver} --boskos-resource-type=${boskos_resource_type} \ --storageclass-file=sc-standard.yaml --snapshotclass-file=pd-volumesnapshotclass.yaml --test-focus="External.Storage" \ --deployment-strategy=${deployment_strategy} --test-version=${test_version} --num-nodes=3 \ - --image-type=${image_type}" + --image-type=${image_type} --use-gke-managed-driver=${use_gke_managed_driver}" if [ "$deployment_strategy" = "gke" ]; then base_cmd="${base_cmd} --gke-cluster-version=${gke_cluster_version}"