Skip to content

Commit 7a3ea7b

Browse files
saikat-roycsaikat-royc
committed
Add option to deploy GKE managed PD CSI driver for integration tests
Summary of changes: 1. Add a new hook to pass release-channel for GKE cluster deployment. 2. Add a new hook to deploy GKE managed PD CSI driver. 3. Added a GKE version comparision util with supporting UTs.
1 parent 3324f67 commit 7a3ea7b

File tree

7 files changed

+644
-68
lines changed

7 files changed

+644
-68
lines changed

‎test/k8s-integration/cluster.go

+47-4
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package main
22

33
import (
4+
"encoding/json"
45
"errors"
56
"fmt"
67
"os"
@@ -9,6 +10,7 @@ import (
910
"strconv"
1011
"strings"
1112

13+
apimachineryversion "k8s.io/apimachinery/pkg/version"
1214
"k8s.io/klog"
1315
)
1416

@@ -131,7 +133,7 @@ func setImageTypeEnvs(imageType string) error {
131133
return nil
132134
}
133135

134-
func clusterUpGKE(gceZone, gceRegion string, numNodes int, imageType string) error {
136+
func clusterUpGKE(gceZone, gceRegion string, numNodes int, imageType string, useManagedDriver bool) error {
135137
locationArg, locationVal, err := gkeLocationArgs(gceZone, gceRegion)
136138
if err != nil {
137139
return err
@@ -150,9 +152,23 @@ func clusterUpGKE(gceZone, gceRegion string, numNodes int, imageType string) err
150152
return err
151153
}
152154
}
153-
cmd := exec.Command("gcloud", "container", "clusters", "create", gkeTestClusterName,
154-
locationArg, locationVal, "--cluster-version", *gkeClusterVer, "--num-nodes", strconv.Itoa(numNodes),
155-
"--quiet", "--machine-type", "n1-standard-2", "--image-type", imageType)
155+
156+
var cmd *exec.Cmd
157+
cmdParams := []string{"container", "clusters", "create", gkeTestClusterName,
158+
locationArg, locationVal, "--num-nodes", strconv.Itoa(numNodes)}
159+
if isVariableSet(gkeClusterVer) {
160+
cmdParams = append(cmdParams, "--cluster-version", *gkeClusterVer)
161+
} else {
162+
cmdParams = append(cmdParams, "--release-channel", *gkeReleaseChannel)
163+
}
164+
165+
if useManagedDriver {
166+
// PD CSI Driver add on is enabled only in gcloud beta.
167+
cmdParams = append([]string{"beta"}, cmdParams...)
168+
cmdParams = append(cmdParams, "--addons", "GcePersistentDiskCsiDriver")
169+
}
170+
171+
cmd = exec.Command("gcloud", cmdParams...)
156172
err = runCommand("Staring E2E Cluster on GKE", cmd)
157173
if err != nil {
158174
return fmt.Errorf("failed to bring up kubernetes e2e cluster on gke: %v", err)
@@ -290,3 +306,30 @@ func getNormalizedVersion(kubeVersion, gkeVersion string) (string, error) {
290306
return strings.Join(toks[:2], "."), nil
291307

292308
}
309+
310+
func getKubeClusterVersion() (string, error) {
311+
out, err := exec.Command("kubectl", "version", "-o=json").CombinedOutput()
312+
if err != nil {
313+
return "", fmt.Errorf("failed to obtain cluster version, error: %v", err)
314+
}
315+
type version struct {
316+
ClientVersion *apimachineryversion.Info `json:"clientVersion,omitempty" yaml:"clientVersion,omitempty"`
317+
ServerVersion *apimachineryversion.Info `json:"serverVersion,omitempty" yaml:"serverVersion,omitempty"`
318+
}
319+
320+
var v version
321+
err = json.Unmarshal(out, &v)
322+
if err != nil {
323+
return "", fmt.Errorf("Failed to parse kubectl version output, error: %v", err)
324+
}
325+
326+
return v.ServerVersion.GitVersion, nil
327+
}
328+
329+
func mustGetKubeClusterVersion() string {
330+
ver, err := getKubeClusterVersion()
331+
if err != nil {
332+
klog.Fatalf("Error: %v", err)
333+
}
334+
return ver
335+
}

‎test/k8s-integration/main.go

+94-60
Original file line numberDiff line numberDiff line change
@@ -22,27 +22,28 @@ import (
2222
"path/filepath"
2323
"syscall"
2424

25-
testutils "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/e2e/utils"
26-
2725
"k8s.io/apimachinery/pkg/util/uuid"
26+
apimachineryversion "k8s.io/apimachinery/pkg/util/version"
2827
"k8s.io/klog"
28+
testutils "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/e2e/utils"
2929
)
3030

3131
var (
3232
// Kubernetes cluster flags
33-
teardownCluster = flag.Bool("teardown-cluster", true, "teardown the cluster after the e2e test")
34-
teardownDriver = flag.Bool("teardown-driver", true, "teardown the driver after the e2e test")
35-
bringupCluster = flag.Bool("bringup-cluster", true, "build kubernetes and bringup a cluster")
36-
gceZone = flag.String("gce-zone", "", "zone that the gce k8s cluster is created/found in")
37-
gceRegion = flag.String("gce-region", "", "region that gke regional cluster should be created in")
38-
kubeVersion = flag.String("kube-version", "", "version of Kubernetes to download and use for the cluster")
39-
testVersion = flag.String("test-version", "", "version of Kubernetes to download and use for tests")
40-
kubeFeatureGates = flag.String("kube-feature-gates", "", "feature gates to set on new kubernetes cluster")
41-
localK8sDir = flag.String("local-k8s-dir", "", "local prebuilt kubernetes/kubernetes directory to use for cluster and test binaries")
42-
deploymentStrat = flag.String("deployment-strategy", "gce", "choose between deploying on gce or gke")
43-
gkeClusterVer = flag.String("gke-cluster-version", "", "version of Kubernetes master and node for gke")
44-
numNodes = flag.Int("num-nodes", -1, "the number of nodes in the test cluster")
45-
imageType = flag.String("image-type", "cos", "the image type to use for the cluster")
33+
teardownCluster = flag.Bool("teardown-cluster", true, "teardown the cluster after the e2e test")
34+
teardownDriver = flag.Bool("teardown-driver", true, "teardown the driver after the e2e test")
35+
bringupCluster = flag.Bool("bringup-cluster", true, "build kubernetes and bringup a cluster")
36+
gceZone = flag.String("gce-zone", "", "zone that the gce k8s cluster is created/found in")
37+
gceRegion = flag.String("gce-region", "", "region that gke regional cluster should be created in")
38+
kubeVersion = flag.String("kube-version", "", "version of Kubernetes to download and use for the cluster")
39+
testVersion = flag.String("test-version", "", "version of Kubernetes to download and use for tests")
40+
kubeFeatureGates = flag.String("kube-feature-gates", "", "feature gates to set on new kubernetes cluster")
41+
localK8sDir = flag.String("local-k8s-dir", "", "local prebuilt kubernetes/kubernetes directory to use for cluster and test binaries")
42+
deploymentStrat = flag.String("deployment-strategy", "gce", "choose between deploying on gce or gke")
43+
gkeClusterVer = flag.String("gke-cluster-version", "", "version of Kubernetes master and node for gke")
44+
numNodes = flag.Int("num-nodes", -1, "the number of nodes in the test cluster")
45+
imageType = flag.String("image-type", "cos", "the image type to use for the cluster")
46+
gkeReleaseChannel = flag.String("gke-release-channel", "", "GKE release channel to be used for cluster deploy. One of 'rapid', 'stable' or 'regular'")
4647

4748
// Test infrastructure flags
4849
boskosResourceType = flag.String("boskos-resource-type", "gce-project", "name of the boskos resource type to reserve")
@@ -51,10 +52,11 @@ var (
5152
inProw = flag.Bool("run-in-prow", false, "is the test running in PROW")
5253

5354
// Driver flags
54-
stagingImage = flag.String("staging-image", "", "name of image to stage to")
55-
saFile = flag.String("service-account-file", "", "path of service account file")
56-
deployOverlayName = flag.String("deploy-overlay-name", "", "which kustomize overlay to deploy the driver with")
57-
doDriverBuild = flag.Bool("do-driver-build", true, "building the driver from source")
55+
stagingImage = flag.String("staging-image", "", "name of image to stage to")
56+
saFile = flag.String("service-account-file", "", "path of service account file")
57+
deployOverlayName = flag.String("deploy-overlay-name", "", "which kustomize overlay to deploy the driver with")
58+
doDriverBuild = flag.Bool("do-driver-build", true, "building the driver from source")
59+
useGKEManagedDriver = flag.Bool("use-gke-managed-driver", false, "use GKE managed PD CSI driver for the tests")
5860

5961
// Test flags
6062
migrationTest = flag.Bool("migration-test", false, "sets the flag on the e2e binary signalling migration")
@@ -75,12 +77,23 @@ func init() {
7577
func main() {
7678
flag.Parse()
7779

78-
if !*inProw {
80+
if !*inProw && !*useGKEManagedDriver {
7981
ensureVariable(stagingImage, true, "staging-image is a required flag, please specify the name of image to stage to")
8082
}
8183

84+
if *useGKEManagedDriver {
85+
ensureVariableVal(deploymentStrat, "gke", "deployment strategy must be GKE for using managed driver")
86+
ensureFlag(doDriverBuild, false, "'do-driver-build' must be false when using GKE managed driver")
87+
ensureFlag(teardownDriver, false, "'teardown-driver' must be false when using GKE managed driver")
88+
ensureVariable(stagingImage, false, "'staging-image' must not be set when using GKE managed driver")
89+
ensureVariable(deployOverlayName, false, "'deploy-overlay-name' must not be set when using GKE managed driver")
90+
}
91+
8292
ensureVariable(saFile, true, "service-account-file is a required flag")
83-
ensureVariable(deployOverlayName, true, "deploy-overlay-name is a required flag")
93+
if !*useGKEManagedDriver {
94+
ensureVariable(deployOverlayName, true, "deploy-overlay-name is a required flag")
95+
}
96+
8497
ensureVariable(testFocus, true, "test-focus is a required flag")
8598
ensureVariable(imageType, true, "image type is a required flag. Available options include 'cos' and 'ubuntu'")
8699

@@ -103,7 +116,8 @@ func main() {
103116
if *deploymentStrat == "gke" {
104117
ensureFlag(migrationTest, false, "Cannot set deployment strategy to 'gke' for migration tests.")
105118
ensureVariable(kubeVersion, false, "Cannot set kube-version when using deployment strategy 'gke'. Use gke-cluster-version.")
106-
ensureVariable(gkeClusterVer, true, "Must set gke-cluster-version when using deployment strategy 'gke'.")
119+
ensureExactlyOneVariableSet([]*string{gkeClusterVer, gkeReleaseChannel},
120+
"For GKE cluster deployment, exactly one of 'gke-cluster-version' or 'gke-release-channel' must be set")
107121
ensureVariable(kubeFeatureGates, false, "Cannot set feature gates when using deployment strategy 'gke'.")
108122
if len(*localK8sDir) == 0 {
109123
ensureVariable(testVersion, true, "Must set either test-version or local k8s dir when using deployment strategy 'gke'.")
@@ -243,7 +257,7 @@ func handle() error {
243257
case "gce":
244258
err = clusterUpGCE(k8sDir, *gceZone, *numNodes, *imageType)
245259
case "gke":
246-
err = clusterUpGKE(*gceZone, *gceRegion, *numNodes, *imageType)
260+
err = clusterUpGKE(*gceZone, *gceRegion, *numNodes, *imageType, *useGKEManagedDriver)
247261
default:
248262
err = fmt.Errorf("deployment-strategy must be set to 'gce' or 'gke', but is: %s", *deploymentStrat)
249263
}
@@ -272,21 +286,24 @@ func handle() error {
272286
}()
273287
}
274288

275-
// Install the driver and defer its teardown
276-
err := installDriver(goPath, pkgDir, *stagingImage, stagingVersion, *deployOverlayName, *doDriverBuild)
277-
if *teardownDriver {
278-
defer func() {
279-
// TODO (#140): collect driver logs
280-
if teardownErr := deleteDriver(goPath, pkgDir, *deployOverlayName); teardownErr != nil {
281-
klog.Errorf("failed to delete driver: %v", teardownErr)
282-
}
283-
}()
284-
}
285-
if err != nil {
286-
return fmt.Errorf("failed to install CSI Driver: %v", err)
289+
if !*useGKEManagedDriver {
290+
// Install the driver and defer its teardown
291+
err := installDriver(goPath, pkgDir, *stagingImage, stagingVersion, *deployOverlayName, *doDriverBuild)
292+
if *teardownDriver {
293+
defer func() {
294+
// TODO (#140): collect driver logs
295+
if teardownErr := deleteDriver(goPath, pkgDir, *deployOverlayName); teardownErr != nil {
296+
klog.Errorf("failed to delete driver: %v", teardownErr)
297+
}
298+
}()
299+
}
300+
if err != nil {
301+
return fmt.Errorf("failed to install CSI Driver: %v", err)
302+
}
287303
}
288304

289305
var cloudProviderArgs []string
306+
var err error
290307
switch *deploymentStrat {
291308
case "gke":
292309
cloudProviderArgs, err = getGKEKubeTestArgs(*gceZone, *gceRegion, *imageType)
@@ -295,12 +312,22 @@ func handle() error {
295312
}
296313
}
297314

298-
normalizedVersion, err := getNormalizedVersion(*kubeVersion, *gkeClusterVer)
299-
if err != nil {
300-
return fmt.Errorf("failed to get cluster minor version: %v", err)
315+
// Kubernetes version of GKE deployments are expected to be of the pattern x.y.z-gke.k,
316+
// hence we use the main.Version utils to parse and compare GKE managed cluster versions.
317+
// For clusters deployed on GCE, use the apimachinery version utils (which supports non-gke based semantic versioning).
318+
clusterVersion := mustGetKubeClusterVersion()
319+
fmt.Printf("Kubernetes cluster version %q\n", clusterVersion)
320+
var testSkip string
321+
switch *deploymentStrat {
322+
case "gce":
323+
testSkip = generateGCETestSkip(clusterVersion)
324+
case "gke":
325+
testSkip = generateGKETestSkip(clusterVersion, *useGKEManagedDriver)
326+
default:
327+
return fmt.Errorf("Unknown deployment strategy %s", *deploymentStrat)
301328
}
302329

303-
testSkip := generateTestSkip(normalizedVersion)
330+
fmt.Printf("Test skip string %s\n", testSkip)
304331
// Run the tests using the testDir kubernetes
305332
if len(*storageClassFile) != 0 {
306333
err = runCSITests(pkgDir, testDir, *testFocus, testSkip, *storageClassFile, *snapshotClassFile, cloudProviderArgs, *deploymentStrat)
@@ -317,30 +344,37 @@ func handle() error {
317344
return nil
318345
}
319346

320-
func generateTestSkip(normalizedVersion string) string {
347+
func generateGCETestSkip(clusterVersion string) string {
321348
skipString := "\\[Disruptive\\]|\\[Serial\\]"
322-
switch normalizedVersion {
323-
// Fall-through versioning since all test cases we want to skip in 1.15
324-
// should also be skipped in 1.14
325-
case "1.13":
326-
fallthrough
327-
case "1.14":
328-
fallthrough
329-
case "1.15":
330-
fallthrough
331-
case "1.16":
332-
// "volumeMode should not mount / map unused volumes in a pod" tests a
333-
// bug-fix introduced in 1.17
334-
// (https://github.com/kubernetes/kubernetes/pull/81163)
349+
v := apimachineryversion.MustParseSemantic(clusterVersion)
350+
351+
// "volumeMode should not mount / map unused volumes in a pod" tests a
352+
// (https://github.com/kubernetes/kubernetes/pull/81163)
353+
if v.LessThan(apimachineryversion.MustParseSemantic("1.16.0")) {
335354
skipString = skipString + "|volumeMode\\sshould\\snot\\smount\\s/\\smap\\sunused\\svolumes\\sin\\sa\\spod"
336-
// Skip Snapshot tests pre 1.17
337-
skipString = skipString + "|snapshot"
338-
fallthrough
339-
case "1.17":
340-
case "latest":
341-
case "master":
342-
default:
343355
}
356+
357+
if v.LessThan(apimachineryversion.MustParseSemantic("1.17.0")) {
358+
skipString = skipString + "|VolumeSnapshotDataSource"
359+
}
360+
return skipString
361+
}
362+
363+
func generateGKETestSkip(clusterVersion string, use_gke_managed_driver bool) string {
364+
skipString := "\\[Disruptive\\]|\\[Serial\\]"
365+
curVer := mustParseVersion(clusterVersion)
366+
if curVer.lessThan(mustParseVersion("1.16.0")) {
367+
skipString = skipString + "|volumeMode\\sshould\\snot\\smount\\s/\\smap\\sunused\\svolumes\\sin\\sa\\spod"
368+
}
369+
370+
// For GKE deployed PD CSI snapshot is enabled in 1.17.5-gke.9(and higher), 1.17.6-gke.4(and higher), 1.18.3(and higher).
371+
if (use_gke_managed_driver && curVer.lessThan(mustParseVersion("1.17.5-gke.9"))) ||
372+
(use_gke_managed_driver && curVer.major() == 1 && curVer.minor() == 17 && curVer.patch() == 6 && curVer.lessThan(mustParseVersion("1.17.6-gke.4"))) ||
373+
(use_gke_managed_driver && curVer.major() == 1 && curVer.minor() == 18 && curVer.lessThan(mustParseVersion("1.18.3-gke.0"))) ||
374+
(!use_gke_managed_driver && (*curVer).lessThan(mustParseVersion("1.17.0"))) {
375+
skipString = skipString + "|VolumeSnapshotDataSource"
376+
}
377+
344378
return skipString
345379
}
346380

‎test/k8s-integration/utils.go

+23
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,19 @@ func ensureFlag(v *bool, setTo bool, msgOnError string) {
5858
}
5959
}
6060

61+
func ensureExactlyOneVariableSet(vars []*string, msgOnError string) {
62+
var count int
63+
for _, v := range vars {
64+
if len(*v) != 0 {
65+
count++
66+
}
67+
}
68+
69+
if count != 1 {
70+
klog.Fatal(msgOnError)
71+
}
72+
}
73+
6174
func shredFile(filePath string) {
6275
if _, err := os.Stat(filePath); os.IsNotExist(err) {
6376
klog.V(4).Infof("File %v was not found, skipping shredding", filePath)
@@ -79,3 +92,13 @@ func shredFile(filePath string) {
7992
klog.V(4).Infof("Failed to remove service account file %s: %v", filePath, err)
8093
}
8194
}
95+
96+
func ensureVariableVal(v *string, val string, msgOnError string) {
97+
if *v != val {
98+
klog.Fatal(msgOnError)
99+
}
100+
}
101+
102+
func isVariableSet(v *string) bool {
103+
return len(*v) != 0
104+
}

0 commit comments

Comments
 (0)