Skip to content

Commit 0b4cb88

Browse files
saikat-roycsaikat-royc
committed
Add option to deploy GKE managed PD CSI driver for integration tests
1 parent 3324f67 commit 0b4cb88

File tree

5 files changed

+65
-26
lines changed

5 files changed

+65
-26
lines changed

‎test/k8s-integration/cluster.go

+14-4
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,7 @@ func setImageTypeEnvs(imageType string) error {
131131
return nil
132132
}
133133

134-
func clusterUpGKE(gceZone, gceRegion string, numNodes int, imageType string) error {
134+
func clusterUpGKE(gceZone, gceRegion string, numNodes int, imageType string, useManagedDriver bool) error {
135135
locationArg, locationVal, err := gkeLocationArgs(gceZone, gceRegion)
136136
if err != nil {
137137
return err
@@ -150,9 +150,19 @@ func clusterUpGKE(gceZone, gceRegion string, numNodes int, imageType string) err
150150
return err
151151
}
152152
}
153-
cmd := exec.Command("gcloud", "container", "clusters", "create", gkeTestClusterName,
154-
locationArg, locationVal, "--cluster-version", *gkeClusterVer, "--num-nodes", strconv.Itoa(numNodes),
155-
"--quiet", "--machine-type", "n1-standard-2", "--image-type", imageType)
153+
154+
var cmd *exec.Cmd
155+
if useManagedDriver {
156+
// PD CSI Driver add on is enabled only in gcloud beta.
157+
cmd = exec.Command("gcloud", "beta", "container", "clusters", "create", gkeTestClusterName,
158+
locationArg, locationVal, "--cluster-version", *gkeClusterVer, "--num-nodes", strconv.Itoa(numNodes),
159+
"--quiet", "--machine-type", "n1-standard-2", "--image-type", imageType, "--addons", "GcePersistentDiskCsiDriver")
160+
} else {
161+
cmd = exec.Command("gcloud", "container", "clusters", "create", gkeTestClusterName,
162+
locationArg, locationVal, "--cluster-version", *gkeClusterVer, "--num-nodes", strconv.Itoa(numNodes),
163+
"--quiet", "--machine-type", "n1-standard-2", "--image-type", imageType)
164+
}
165+
156166
err = runCommand("Staring E2E Cluster on GKE", cmd)
157167
if err != nil {
158168
return fmt.Errorf("failed to bring up kubernetes e2e cluster on gke: %v", err)

‎test/k8s-integration/main.go

+32-19
Original file line numberDiff line numberDiff line change
@@ -51,10 +51,11 @@ var (
5151
inProw = flag.Bool("run-in-prow", false, "is the test running in PROW")
5252

5353
// Driver flags
54-
stagingImage = flag.String("staging-image", "", "name of image to stage to")
55-
saFile = flag.String("service-account-file", "", "path of service account file")
56-
deployOverlayName = flag.String("deploy-overlay-name", "", "which kustomize overlay to deploy the driver with")
57-
doDriverBuild = flag.Bool("do-driver-build", true, "building the driver from source")
54+
stagingImage = flag.String("staging-image", "", "name of image to stage to")
55+
saFile = flag.String("service-account-file", "", "path of service account file")
56+
deployOverlayName = flag.String("deploy-overlay-name", "", "which kustomize overlay to deploy the driver with")
57+
doDriverBuild = flag.Bool("do-driver-build", true, "building the driver from source")
58+
useGKEManagedDriver = flag.Bool("use-gke-managed-driver", false, "use GKE managed PD CSI driver for the tests")
5859

5960
// Test flags
6061
migrationTest = flag.Bool("migration-test", false, "sets the flag on the e2e binary signalling migration")
@@ -75,12 +76,21 @@ func init() {
7576
func main() {
7677
flag.Parse()
7778

78-
if !*inProw {
79+
if !*inProw && !*useGKEManagedDriver {
7980
ensureVariable(stagingImage, true, "staging-image is a required flag, please specify the name of image to stage to")
8081
}
8182

83+
if *useGKEManagedDriver {
84+
ensureVariableVal(deploymentStrat, "gke", "deployment strategy must be GKE for using managed driver")
85+
ensureFlag(doDriverBuild, false, "driver build flag will be ignored when using GKE managed driver")
86+
ensureFlag(teardownDriver, false, "driver teardown flag will be ignored when using GKE managed driver")
87+
}
88+
8289
ensureVariable(saFile, true, "service-account-file is a required flag")
83-
ensureVariable(deployOverlayName, true, "deploy-overlay-name is a required flag")
90+
if !*useGKEManagedDriver {
91+
ensureVariable(deployOverlayName, true, "deploy-overlay-name is a required flag")
92+
}
93+
8494
ensureVariable(testFocus, true, "test-focus is a required flag")
8595
ensureVariable(imageType, true, "image type is a required flag. Available options include 'cos' and 'ubuntu'")
8696

@@ -243,7 +253,7 @@ func handle() error {
243253
case "gce":
244254
err = clusterUpGCE(k8sDir, *gceZone, *numNodes, *imageType)
245255
case "gke":
246-
err = clusterUpGKE(*gceZone, *gceRegion, *numNodes, *imageType)
256+
err = clusterUpGKE(*gceZone, *gceRegion, *numNodes, *imageType, *useGKEManagedDriver)
247257
default:
248258
err = fmt.Errorf("deployment-strategy must be set to 'gce' or 'gke', but is: %s", *deploymentStrat)
249259
}
@@ -272,21 +282,24 @@ func handle() error {
272282
}()
273283
}
274284

275-
// Install the driver and defer its teardown
276-
err := installDriver(goPath, pkgDir, *stagingImage, stagingVersion, *deployOverlayName, *doDriverBuild)
277-
if *teardownDriver {
278-
defer func() {
279-
// TODO (#140): collect driver logs
280-
if teardownErr := deleteDriver(goPath, pkgDir, *deployOverlayName); teardownErr != nil {
281-
klog.Errorf("failed to delete driver: %v", teardownErr)
282-
}
283-
}()
284-
}
285-
if err != nil {
286-
return fmt.Errorf("failed to install CSI Driver: %v", err)
285+
if !*useGKEManagedDriver {
286+
// Install the driver and defer its teardown
287+
err := installDriver(goPath, pkgDir, *stagingImage, stagingVersion, *deployOverlayName, *doDriverBuild)
288+
if *teardownDriver {
289+
defer func() {
290+
// TODO (#140): collect driver logs
291+
if teardownErr := deleteDriver(goPath, pkgDir, *deployOverlayName); teardownErr != nil {
292+
klog.Errorf("failed to delete driver: %v", teardownErr)
293+
}
294+
}()
295+
}
296+
if err != nil {
297+
return fmt.Errorf("failed to install CSI Driver: %v", err)
298+
}
287299
}
288300

289301
var cloudProviderArgs []string
302+
var err error
290303
switch *deploymentStrat {
291304
case "gke":
292305
cloudProviderArgs, err = getGKEKubeTestArgs(*gceZone, *gceRegion, *imageType)

‎test/k8s-integration/utils.go

+6
Original file line numberDiff line numberDiff line change
@@ -79,3 +79,9 @@ func shredFile(filePath string) {
7979
klog.V(4).Infof("Failed to remove service account file %s: %v", filePath, err)
8080
}
8181
}
82+
83+
func ensureVariableVal(v *string, val string, msgOnError string) {
84+
if *v != val {
85+
klog.Fatal(msgOnError)
86+
}
87+
}

‎test/run-k8s-integration-local.sh

+9-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ readonly test_version=${TEST_VERSION:-master}
1010

1111
source "${PKGDIR}/deploy/common.sh"
1212

13-
ensure_var GCE_PD_CSI_STAGING_IMAGE
1413
ensure_var GCE_PD_SA_DIR
1514

1615
make -C ${PKGDIR} test-k8s-integration
@@ -56,6 +55,15 @@ make -C ${PKGDIR} test-k8s-integration
5655
#--storageclass-file=sc-standard.yaml --snapshotclass-file=pd-volumesnapshotclass.yaml --do-driver-build=true \
5756
#--gce-zone="us-central1-b" --num-nodes=${NUM_NODES:-3}
5857

58+
# This version of the command brings up (and subsequently tears down) a GKE
59+
# cluster with managed GCE PersistentDisk CSI driver add-on enabled, and points to
60+
# the local K8s repository to get the e2e test binary.
61+
# ${PKGDIR}/bin/k8s-integration-test --run-in-prow=false --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \
62+
# --test-focus="External.*Storage.*VolumeSnapshotDataSource" --local-k8s-dir=$KTOP --storageclass-file=sc-standard.yaml \
63+
# --snapshotclass-file=pd-volumesnapshotclass.yaml --do-driver-build=false --teardown-driver=false \
64+
# --gce-zone="us-central1-c" --num-nodes=${NUM_NODES:-3} --gke-cluster-version="latest" --deployment-strategy="gke" \
65+
# --use-gke-managed-driver=true
66+
5967
# This version of the command does not build the driver or K8s, points to a
6068
# local K8s repo to get the e2e.test binary, and does not bring up or down the cluster
6169

‎test/run-k8s-integration.sh

+4-2
Original file line numberDiff line numberDiff line change
@@ -20,17 +20,19 @@ readonly test_version=${TEST_VERSION:-master}
2020
readonly gce_zone=${GCE_CLUSTER_ZONE:-us-central1-b}
2121
readonly gce_region=${GCE_CLUSTER_REGION:-}
2222
readonly image_type=${IMAGE_TYPE:-cos}
23+
readonly use_gke_managed_driver=${GCE_PD_USE_MANAGED_DRIVER:-false}
24+
readonly teardown_driver=${GCE_PD_TEARDOWN_DRIVER:-true}
2325

2426
export GCE_PD_VERBOSITY=9
2527

2628
make -C ${PKGDIR} test-k8s-integration
2729

2830
base_cmd="${PKGDIR}/bin/k8s-integration-test \
2931
--run-in-prow=true --deploy-overlay-name=${overlay_name} --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \
30-
--do-driver-build=${do_driver_build} --boskos-resource-type=${boskos_resource_type} \
32+
--do-driver-build=${do_driver_build} --teardown-driver=${teardown_driver} --boskos-resource-type=${boskos_resource_type} \
3133
--storageclass-file=sc-standard.yaml --snapshotclass-file=pd-volumesnapshotclass.yaml --test-focus="External.Storage" \
3234
--deployment-strategy=${deployment_strategy} --test-version=${test_version} --num-nodes=3 \
33-
--image-type=${image_type}"
35+
--image-type=${image_type} --use-gke-managed-driver=${use_gke_managed_driver}"
3436

3537
if [ "$deployment_strategy" = "gke" ]; then
3638
base_cmd="${base_cmd} --gke-cluster-version=${gke_cluster_version}"

0 commit comments

Comments
 (0)