Add option to deploy GKE managed PD CSI driver for integration tests

Summary of changes:
1. Add a new hook to pass release-channel for GKE cluster deployment.
2. Add a new hook to deploy GKE managed PD CSI driver.
3. Added a version comparision util with supporting UTs.

Author: saikat-royc

test/k8s-integration/cluster.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"os"
@@ -9,6 +10,7 @@ import (
 	"strconv"
 	"strings"
 
+	apimachineryversion "k8s.io/apimachinery/pkg/version"
 	"k8s.io/klog"
 )
 
@@ -152,17 +154,24 @@ func clusterUpGKE(gceZone, gceRegion string, numNodes int, imageType string, use
 	}
 
 	var cmd *exec.Cmd
+	cmdParams := []string{"container", "clusters", "create", gkeTestClusterName,
+		locationArg, locationVal, "--num-nodes", strconv.Itoa(numNodes)}
+
+	ensureExactlyOneVariableSet([]*string{gkeClusterVer, gkeReleaseChannel},
+		"For GKE cluster deployment, exactly one of 'gke-cluster-version' or 'gke-release-channel' must be set")
+	if isVariableSet(gkeClusterVer) {
+		cmdParams = append(cmdParams, "--cluster-version", *gkeClusterVer)
+	} else {
+		cmdParams = append(cmdParams, "--release-channel", *gkeReleaseChannel)
+	}
+
 	if useManagedDriver {
 		// PD CSI Driver add on is enabled only in gcloud beta.
-		cmd = exec.Command("gcloud", "beta", "container", "clusters", "create", gkeTestClusterName,
-			locationArg, locationVal, "--cluster-version", *gkeClusterVer, "--num-nodes", strconv.Itoa(numNodes),
-			"--quiet", "--machine-type", "n1-standard-2", "--image-type", imageType, "--addons", "GcePersistentDiskCsiDriver")
-	} else {
-		cmd = exec.Command("gcloud", "container", "clusters", "create", gkeTestClusterName,
-			locationArg, locationVal, "--cluster-version", *gkeClusterVer, "--num-nodes", strconv.Itoa(numNodes),
-			"--quiet", "--machine-type", "n1-standard-2", "--image-type", imageType)
+		cmdParams = append([]string{"beta"}, cmdParams...)
+		cmdParams = append(cmdParams, "--addons", "GcePersistentDiskCsiDriver")
 	}
 
+	cmd = exec.Command("gcloud", cmdParams...)
 	err = runCommand("Staring E2E Cluster on GKE", cmd)
 	if err != nil {
 		return fmt.Errorf("failed to bring up kubernetes e2e cluster on gke: %v", err)
@@ -300,3 +309,30 @@ func getNormalizedVersion(kubeVersion, gkeVersion string) (string, error) {
 	return strings.Join(toks[:2], "."), nil
 
 }
+
+func getKubClusterVersion() (string, error) {
+	out, err := exec.Command("kubectl", "version", "-o=json").CombinedOutput()
+	if err != nil {
+		return "", fmt.Errorf("failed to obtain cluster version, error: %v", err)
+	}
+	type version struct {
+		ClientVersion *apimachineryversion.Info `json:"clientVersion,omitempty" yaml:"clientVersion,omitempty"`
+		ServerVersion *apimachineryversion.Info `json:"serverVersion,omitempty" yaml:"serverVersion,omitempty"`
+	}
+
+	var v version
+	err = json.Unmarshal(out, &v)
+	if err != nil {
+		return "", fmt.Errorf("Failed to parse kubectl version output, error: %v", err)
+	}
+
+	return v.ServerVersion.GitVersion, nil
+}
+
+func MustGetKubClusterVersion() string {
+	ver, err := getKubClusterVersion()
+	if err != nil {
+		klog.Fatalf("Error: %v", err)
+	}
+	return ver
+}

test/k8s-integration/main.go

@@ -30,19 +30,20 @@ import (
 
 var (
 	// Kubernetes cluster flags
-	teardownCluster  = flag.Bool("teardown-cluster", true, "teardown the cluster after the e2e test")
-	teardownDriver   = flag.Bool("teardown-driver", true, "teardown the driver after the e2e test")
-	bringupCluster   = flag.Bool("bringup-cluster", true, "build kubernetes and bringup a cluster")
-	gceZone          = flag.String("gce-zone", "", "zone that the gce k8s cluster is created/found in")
-	gceRegion        = flag.String("gce-region", "", "region that gke regional cluster should be created in")
-	kubeVersion      = flag.String("kube-version", "", "version of Kubernetes to download and use for the cluster")
-	testVersion      = flag.String("test-version", "", "version of Kubernetes to download and use for tests")
-	kubeFeatureGates = flag.String("kube-feature-gates", "", "feature gates to set on new kubernetes cluster")
-	localK8sDir      = flag.String("local-k8s-dir", "", "local prebuilt kubernetes/kubernetes directory to use for cluster and test binaries")
-	deploymentStrat  = flag.String("deployment-strategy", "gce", "choose between deploying on gce or gke")
-	gkeClusterVer    = flag.String("gke-cluster-version", "", "version of Kubernetes master and node for gke")
-	numNodes         = flag.Int("num-nodes", -1, "the number of nodes in the test cluster")
-	imageType        = flag.String("image-type", "cos", "the image type to use for the cluster")
+	teardownCluster   = flag.Bool("teardown-cluster", true, "teardown the cluster after the e2e test")
+	teardownDriver    = flag.Bool("teardown-driver", true, "teardown the driver after the e2e test")
+	bringupCluster    = flag.Bool("bringup-cluster", true, "build kubernetes and bringup a cluster")
+	gceZone           = flag.String("gce-zone", "", "zone that the gce k8s cluster is created/found in")
+	gceRegion         = flag.String("gce-region", "", "region that gke regional cluster should be created in")
+	kubeVersion       = flag.String("kube-version", "", "version of Kubernetes to download and use for the cluster")
+	testVersion       = flag.String("test-version", "", "version of Kubernetes to download and use for tests")
+	kubeFeatureGates  = flag.String("kube-feature-gates", "", "feature gates to set on new kubernetes cluster")
+	localK8sDir       = flag.String("local-k8s-dir", "", "local prebuilt kubernetes/kubernetes directory to use for cluster and test binaries")
+	deploymentStrat   = flag.String("deployment-strategy", "gce", "choose between deploying on gce or gke")
+	gkeClusterVer     = flag.String("gke-cluster-version", "", "version of Kubernetes master and node for gke")
+	numNodes          = flag.Int("num-nodes", -1, "the number of nodes in the test cluster")
+	imageType         = flag.String("image-type", "cos", "the image type to use for the cluster")
+	gkeReleaseChannel = flag.String("gke-release-channel", "", "GKE release channel to be used for cluster deploy. One of 'rapid', 'stable' or 'regular'")
 
 	// Test infrastructure flags
 	boskosResourceType = flag.String("boskos-resource-type", "gce-project", "name of the boskos resource type to reserve")
@@ -82,8 +83,8 @@ func main() {
 
 	if *useGKEManagedDriver {
 		ensureVariableVal(deploymentStrat, "gke", "deployment strategy must be GKE for using managed driver")
-		ensureFlag(doDriverBuild, false, "driver build flag will be ignored when using GKE managed driver")
-		ensureFlag(teardownDriver, false, "driver teardown flag will be ignored when using GKE managed driver")
+		ensureFlag(doDriverBuild, false, "'do-driver-build' must be false when using GKE managed driver")
+		ensureFlag(teardownDriver, false, "'teardown-driver' must be false when using GKE managed driver")
 	}
 
 	ensureVariable(saFile, true, "service-account-file is a required flag")
@@ -113,7 +114,8 @@ func main() {
 	if *deploymentStrat == "gke" {
 		ensureFlag(migrationTest, false, "Cannot set deployment strategy to 'gke' for migration tests.")
 		ensureVariable(kubeVersion, false, "Cannot set kube-version when using deployment strategy 'gke'. Use gke-cluster-version.")
-		ensureVariable(gkeClusterVer, true, "Must set gke-cluster-version when using deployment strategy 'gke'.")
+		ensureExactlyOneVariableSet([]*string{gkeClusterVer, gkeReleaseChannel},
+			"For GKE cluster deployment, exactly one of 'gke-cluster-version' or 'gke-release-channel' must be set")
 		ensureVariable(kubeFeatureGates, false, "Cannot set feature gates when using deployment strategy 'gke'.")
 		if len(*localK8sDir) == 0 {
 			ensureVariable(testVersion, true, "Must set either test-version or local k8s dir when using deployment strategy 'gke'.")
@@ -308,12 +310,20 @@ func handle() error {
 		}
 	}
 
-	normalizedVersion, err := getNormalizedVersion(*kubeVersion, *gkeClusterVer)
-	if err != nil {
-		return fmt.Errorf("failed to get cluster minor version: %v", err)
+	var testSkip string
+	if *useGKEManagedDriver {
+		clusterVersion := MustGetKubClusterVersion()
+		fmt.Printf("GKE cluster version %q\n", clusterVersion)
+		testSkip = generateGKEBasedTestSkip(clusterVersion)
+	} else {
+		normalizedVersion, err := getNormalizedVersion(*kubeVersion, *gkeClusterVer)
+		if err != nil {
+			return fmt.Errorf("failed to get cluster minor version: %v", err)
+		}
+		testSkip = generateTestSkip(normalizedVersion)
 	}
 
-	testSkip := generateTestSkip(normalizedVersion)
+	fmt.Println("test skip string ", testSkip)
 	// Run the tests using the testDir kubernetes
 	if len(*storageClassFile) != 0 {
 		err = runCSITests(pkgDir, testDir, *testFocus, testSkip, *storageClassFile, *snapshotClassFile, cloudProviderArgs, *deploymentStrat)
@@ -357,6 +367,21 @@ func generateTestSkip(normalizedVersion string) string {
 	return skipString
 }
 
+// Helper function to generate skip string when using GKE managed PD CSI driver.
+func generateGKEBasedTestSkip(clusterVersion string) string {
+	skipString := "\\[Disruptive\\]|\\[Serial\\]"
+	curVer := MustParseVersion(clusterVersion)
+	if (*curVer).IsLessThan(MustParseVersion("1.17.0")) {
+		skipString = skipString + "|volumeMode\\sshould\\snot\\smount\\s/\\smap\\sunused\\svolumes\\sin\\sa\\spod"
+	}
+
+	if (*curVer).IsLessThan(MustParseVersion("1.18.3-gke.0")) {
+		skipString = skipString + "|snapshot"
+	}
+
+	return skipString
+}
+
 func setEnvProject(project string) error {
 	out, err := exec.Command("gcloud", "config", "set", "project", project).CombinedOutput()
 	if err != nil {

test/k8s-integration/utils.go

@@ -58,6 +58,19 @@ func ensureFlag(v *bool, setTo bool, msgOnError string) {
 	}
 }
 
+func ensureExactlyOneVariableSet(vars []*string, msgOnError string) {
+	var count int
+	for _, v := range vars {
+		if len(*v) != 0 {
+			count++
+		}
+	}
+
+	if count != 1 {
+		klog.Fatal(msgOnError)
+	}
+}
+
 func shredFile(filePath string) {
 	if _, err := os.Stat(filePath); os.IsNotExist(err) {
 		klog.V(4).Infof("File %v was not found, skipping shredding", filePath)
@@ -85,3 +98,7 @@ func ensureVariableVal(v *string, val string, msgOnError string) {
 		klog.Fatal(msgOnError)
 	}
 }
+
+func isVariableSet(v *string) bool {
+	return len(*v) != 0
+}

test/k8s-integration/version.go

@@ -0,0 +1,113 @@
+package main
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"k8s.io/klog"
+)
+
+var (
+	versionNum           = `(0|[1-9][0-9]*)`
+	internalPatchVersion = `(\-[a-zA-Z0-9_.+-]+)`
+
+	versionRegex         = regexp.MustCompile(`^` + versionNum + `\.` + versionNum + `\.` + versionNum + internalPatchVersion + "?$")
+	gkeExtraVersionRegex = regexp.MustCompile(`^(?:gke)\.(0|[1-9][0-9]*)$`)
+)
+
+type Version struct {
+	version [4]int
+	extra   string
+}
+
+func (v Version) String() string {
+	return fmt.Sprintf("%d.%d.%d-gke.%d", v.version[0], v.version[1], v.version[2], v.version[3])
+}
+
+func (v Version) isGKEExtraVersion() bool {
+	return gkeExtraVersionRegex.MatchString(v.extra)
+}
+
+func extractGKEExtraVersion(extra string) (int, error) {
+	m := gkeExtraVersionRegex.FindStringSubmatch(extra)
+	if len(m) != 2 {
+		return -1, fmt.Errorf("Invalid GKE Patch version %q", extra)
+	}
+
+	v, err := strconv.Atoi(m[1])
+	if err != nil {
+		return -1, fmt.Errorf("GKE extra version atoi failed %q", extra)
+	}
+
+	if v < 0 {
+		return -1, fmt.Errorf("GKE extra version check failed %q", extra)
+	}
+	return v, nil
+}
+
+func ParseVersion(version string) (*Version, error) {
+	// If version has a prefix 'v', remove it before parsing.
+	if strings.HasPrefix(version, "v") {
+		version = version[1:]
+	}
+
+	submatches := versionRegex.FindStringSubmatch(version)
+	if submatches == nil {
+		return nil, fmt.Errorf("version %q is invalid", version)
+	}
+
+	var v Version
+	// submatches[0] is the whole match, [1]..[3] are the version bits, [4] is the extra
+	for i, sm := range submatches[1:4] {
+		var err error
+		if v.version[i], err = strconv.Atoi(sm); err != nil {
+			return nil, fmt.Errorf("submatch %q failed atoi conversion", sm)
+		}
+	}
+	if submatches[4] != "" {
+		v.extra = submatches[4][1:]
+	}
+
+	// Ensure 1.X.Y < 1.X.Y-gke.0
+	v.version[3] = -1
+	if v.isGKEExtraVersion() {
+		ver, err := extractGKEExtraVersion(v.extra)
+		if err != nil {
+			return nil, err
+		}
+		v.version[3] = ver
+	}
+	return &v, nil
+}
+
+func MustParseVersion(version string) *Version {
+	v, err := ParseVersion(version)
+	if err != nil {
+		klog.Fatalf("Failed to parse GKE version: %q", version)
+	}
+	return v
+}
+
+// Helper function to compare versions.
+//  -1 -- if left  < right
+//   0 -- if left == right
+//   1 -- if left  > right
+func (v Version) compare(right *Version) int {
+	for i, b := range v.version {
+		if b > right.version[i] {
+			return 1
+		}
+		if b < right.version[i] {
+			return -1
+		}
+	}
+
+	return 0
+}
+
+// Compare versions if left is strictly less than right.
+func (v Version) IsLessThan(right *Version) bool {
+	return v.compare(right) < 0
+}