Add option to deploy GKE managed PD CSI driver for integration tests

test/k8s-integration/cluster.go

@@ -131,7 +131,7 @@ func setImageTypeEnvs(imageType string) error {
 	return nil
 }
 
-func clusterUpGKE(gceZone, gceRegion string, numNodes int, imageType string) error {
+func clusterUpGKE(gceZone, gceRegion string, numNodes int, imageType string, useManagedDriver bool) error {
 	locationArg, locationVal, err := gkeLocationArgs(gceZone, gceRegion)
 	if err != nil {
 		return err
@@ -150,9 +150,19 @@ func clusterUpGKE(gceZone, gceRegion string, numNodes int, imageType string) err
 			return err
 		}
 	}
-	cmd := exec.Command("gcloud", "container", "clusters", "create", gkeTestClusterName,
-		locationArg, locationVal, "--cluster-version", *gkeClusterVer, "--num-nodes", strconv.Itoa(numNodes),
-		"--quiet", "--machine-type", "n1-standard-2", "--image-type", imageType)
+
+	var cmd *exec.Cmd
+	if useManagedDriver {
+		// PD CSI Driver add on is enabled only in gcloud beta.
+		cmd = exec.Command("gcloud", "beta", "container", "clusters", "create", gkeTestClusterName,
+			locationArg, locationVal, "--cluster-version", *gkeClusterVer, "--num-nodes", strconv.Itoa(numNodes),
+			"--quiet", "--machine-type", "n1-standard-2", "--image-type", imageType, "--addons", "GcePersistentDiskCsiDriver")
+	} else {
+		cmd = exec.Command("gcloud", "container", "clusters", "create", gkeTestClusterName,
+			locationArg, locationVal, "--cluster-version", *gkeClusterVer, "--num-nodes", strconv.Itoa(numNodes),
+			"--quiet", "--machine-type", "n1-standard-2", "--image-type", imageType)
+	}
+
 	err = runCommand("Staring E2E Cluster on GKE", cmd)
 	if err != nil {
 		return fmt.Errorf("failed to bring up kubernetes e2e cluster on gke: %v", err)

test/k8s-integration/main.go

@@ -51,10 +51,11 @@ var (
 	inProw             = flag.Bool("run-in-prow", false, "is the test running in PROW")
 
 	// Driver flags
-	stagingImage      = flag.String("staging-image", "", "name of image to stage to")
-	saFile            = flag.String("service-account-file", "", "path of service account file")
-	deployOverlayName = flag.String("deploy-overlay-name", "", "which kustomize overlay to deploy the driver with")
-	doDriverBuild     = flag.Bool("do-driver-build", true, "building the driver from source")
+	stagingImage        = flag.String("staging-image", "", "name of image to stage to")
+	saFile              = flag.String("service-account-file", "", "path of service account file")
+	deployOverlayName   = flag.String("deploy-overlay-name", "", "which kustomize overlay to deploy the driver with")
+	doDriverBuild       = flag.Bool("do-driver-build", true, "building the driver from source")
+	useGKEManagedDriver = flag.Bool("use-gke-managed-driver", false, "use GKE managed PD CSI driver for the tests")
 
 	// Test flags
 	migrationTest = flag.Bool("migration-test", false, "sets the flag on the e2e binary signalling migration")
@@ -75,12 +76,21 @@ func init() {
 func main() {
 	flag.Parse()
 
-	if !*inProw {
+	if !*inProw && !*useGKEManagedDriver {
 		ensureVariable(stagingImage, true, "staging-image is a required flag, please specify the name of image to stage to")
 	}
 
+	if *useGKEManagedDriver {
+		ensureVariableVal(deploymentStrat, "gke", "deployment strategy must be GKE for using managed driver")
+		ensureFlag(doDriverBuild, false, "driver build flag will be ignored when using GKE managed driver")
+		ensureFlag(teardownDriver, false, "driver teardown flag will be ignored when using GKE managed driver")
+	}
+
 	ensureVariable(saFile, true, "service-account-file is a required flag")
-	ensureVariable(deployOverlayName, true, "deploy-overlay-name is a required flag")
+	if !*useGKEManagedDriver {
+		ensureVariable(deployOverlayName, true, "deploy-overlay-name is a required flag")
+	}
+
 	ensureVariable(testFocus, true, "test-focus is a required flag")
 	ensureVariable(imageType, true, "image type is a required flag. Available options include 'cos' and 'ubuntu'")
 
@@ -243,7 +253,7 @@ func handle() error {
 		case "gce":
 			err = clusterUpGCE(k8sDir, *gceZone, *numNodes, *imageType)
 		case "gke":
-			err = clusterUpGKE(*gceZone, *gceRegion, *numNodes, *imageType)
+			err = clusterUpGKE(*gceZone, *gceRegion, *numNodes, *imageType, *useGKEManagedDriver)
 		default:
 			err = fmt.Errorf("deployment-strategy must be set to 'gce' or 'gke', but is: %s", *deploymentStrat)
 		}
@@ -272,21 +282,24 @@ func handle() error {
 		}()
 	}
 
-	// Install the driver and defer its teardown
-	err := installDriver(goPath, pkgDir, *stagingImage, stagingVersion, *deployOverlayName, *doDriverBuild)
-	if *teardownDriver {
-		defer func() {
-			// TODO (#140): collect driver logs
-			if teardownErr := deleteDriver(goPath, pkgDir, *deployOverlayName); teardownErr != nil {
-				klog.Errorf("failed to delete driver: %v", teardownErr)
-			}
-		}()
-	}
-	if err != nil {
-		return fmt.Errorf("failed to install CSI Driver: %v", err)
+	if !*useGKEManagedDriver {
+		// Install the driver and defer its teardown
+		err := installDriver(goPath, pkgDir, *stagingImage, stagingVersion, *deployOverlayName, *doDriverBuild)
+		if *teardownDriver {
+			defer func() {
+				// TODO (#140): collect driver logs
+				if teardownErr := deleteDriver(goPath, pkgDir, *deployOverlayName); teardownErr != nil {
+					klog.Errorf("failed to delete driver: %v", teardownErr)
+				}
+			}()
+		}
+		if err != nil {
+			return fmt.Errorf("failed to install CSI Driver: %v", err)
+		}
 	}
 
 	var cloudProviderArgs []string
+	var err error
 	switch *deploymentStrat {
 	case "gke":
 		cloudProviderArgs, err = getGKEKubeTestArgs(*gceZone, *gceRegion, *imageType)

test/k8s-integration/utils.go

@@ -79,3 +79,9 @@ func shredFile(filePath string) {
 		klog.V(4).Infof("Failed to remove service account file %s: %v", filePath, err)
 	}
 }
+
+func ensureVariableVal(v *string, val string, msgOnError string) {
+	if *v != val {
+		klog.Fatal(msgOnError)
+	}
+}

test/run-k8s-integration-local.sh

@@ -10,7 +10,6 @@ readonly test_version=${TEST_VERSION:-master}
 
 source "${PKGDIR}/deploy/common.sh"
 
-ensure_var GCE_PD_CSI_STAGING_IMAGE
 ensure_var GCE_PD_SA_DIR
 
 make -C ${PKGDIR} test-k8s-integration
@@ -56,6 +55,15 @@ make -C ${PKGDIR} test-k8s-integration
 #--storageclass-file=sc-standard.yaml --snapshotclass-file=pd-volumesnapshotclass.yaml --do-driver-build=true \
 #--gce-zone="us-central1-b" --num-nodes=${NUM_NODES:-3}
 
+# This version of the command brings up (and subsequently tears down) a GKE
+# cluster with managed GCE PersistentDisk CSI driver add-on enabled, and points to
+# the local K8s repository to get the e2e test binary.
+# ${PKGDIR}/bin/k8s-integration-test --run-in-prow=false --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \
+# --test-focus="External.*Storage.*snapshot" --local-k8s-dir=$KTOP --storageclass-file=sc-standard.yaml \
+# --snapshotclass-file=pd-volumesnapshotclass.yaml --do-driver-build=false --teardown-driver=false \
+# --gce-zone="us-central1-c" --num-nodes=${NUM_NODES:-3} --gke-cluster-version="latest" --deployment-strategy="gke" \
+# --use-gke-managed-driver=true
+
 # This version of the command does not build the driver or K8s, points to a
 # local K8s repo to get the e2e.test binary, and does not bring up or down the cluster
 

test/run-k8s-integration.sh

@@ -20,17 +20,19 @@ readonly test_version=${TEST_VERSION:-master}
 readonly gce_zone=${GCE_CLUSTER_ZONE:-us-central1-b}
 readonly gce_region=${GCE_CLUSTER_REGION:-}
 readonly image_type=${IMAGE_TYPE:-cos}
+readonly use_gke_managed_driver=${GCE_PD_USE_MANAGED_DRIVER:-false}
+readonly teardown_driver=${GCE_PD_TEARDOWN_DRIVER:-true}
 
 export GCE_PD_VERBOSITY=9
 
 make -C ${PKGDIR} test-k8s-integration
 
 base_cmd="${PKGDIR}/bin/k8s-integration-test \
             --run-in-prow=true --deploy-overlay-name=${overlay_name} --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \
-            --do-driver-build=${do_driver_build} --boskos-resource-type=${boskos_resource_type} \
+            --do-driver-build=${do_driver_build} --teardown-driver=${teardown_driver} --boskos-resource-type=${boskos_resource_type} \
             --storageclass-file=sc-standard.yaml --snapshotclass-file=pd-volumesnapshotclass.yaml --test-focus="External.Storage" \
             --deployment-strategy=${deployment_strategy} --test-version=${test_version} --num-nodes=3 \
-            --image-type=${image_type}"
+            --image-type=${image_type} --use-gke-managed-driver=${use_gke_managed_driver}"
 
 if [ "$deployment_strategy" = "gke" ]; then
   base_cmd="${base_cmd} --gke-cluster-version=${gke_cluster_version}"