Merge pull request kubernetes-sigs#414 from davidz627/feature/conditionalSkip

Add test skipping based on cluster version since some tests added are fixing bugs in newer versions

Author: k8s-ci-robot

test/k8s-integration/cluster.go

@@ -1,11 +1,13 @@
 package main
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strconv"
+	"strings"
 
 	"k8s.io/klog"
 )
@@ -230,3 +232,28 @@ func getGKEKubeTestArgs(gceZone, gceRegion string) ([]string, error) {
 
 	return args, nil
 }
+
+func getNormalizedVersion(kubeVersion, gkeVersion string) (string, error) {
+	if kubeVersion != "" && gkeVersion != "" {
+		return "", fmt.Errorf("both kube version (%s) and gke version (%s) specified", kubeVersion, gkeVersion)
+	}
+	if kubeVersion == "" && gkeVersion == "" {
+		return "", errors.New("neither kube verison nor gke verison specified")
+	}
+	var v string
+	if kubeVersion != "" {
+		v = kubeVersion
+	} else if gkeVersion != "" {
+		v = gkeVersion
+	}
+	if v == "master" || v == "latest" {
+		// Ugh
+		return v, nil
+	}
+	toks := strings.Split(v, ".")
+	if len(toks) < 2 || len(toks) > 3 {
+		return "", fmt.Errorf("got unexpected number of tokens in version string %s - wanted 2 or 3", v)
+	}
+	return strings.Join(toks[:2], "."), nil
+
+}

test/k8s-integration/cluster_test.go

@@ -0,0 +1,71 @@
+package main
+
+import "testing"
+
+func TestGetNormalizedVersion(t *testing.T) {
+	tests := []struct {
+		name      string
+		kubeV     string
+		gkeV      string
+		expectV   string
+		expectErr bool
+	}{
+		{
+			name:    "kube",
+			kubeV:   "1.13.5",
+			expectV: "1.13",
+		},
+		{
+			name:    "kube master",
+			kubeV:   "master",
+			expectV: "master",
+		},
+		{
+			name:    "gke",
+			gkeV:    "1.14.3",
+			expectV: "1.14",
+		},
+		{
+			name:    "gke minor",
+			gkeV:    "1.14",
+			expectV: "1.14",
+		},
+		{
+			name:    "gke latest",
+			gkeV:    "latest",
+			expectV: "latest",
+		},
+		{
+			name:    "kube minor",
+			kubeV:   "1.13",
+			expectV: "1.13",
+		},
+		{
+			name:      "kube err",
+			kubeV:     "1",
+			expectErr: true,
+		},
+		{
+			name:      "neither err",
+			expectErr: true,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			gotV, err := getNormalizedVersion(tc.kubeV, tc.gkeV)
+			if err != nil {
+				if !tc.expectErr {
+					t.Fatalf("Got unexpected err: %v", err)
+				}
+				return
+			}
+			if err == nil && tc.expectErr {
+				t.Fatal("Got no error but expected one")
+			}
+			if gotV != tc.expectV {
+				t.Errorf("Got version: %s, expected: %s", gotV, tc.expectV)
+			}
+		})
+	}
+}

test/k8s-integration/main.go

@@ -292,11 +292,18 @@ func handle() error {
 		}
 	}
 
+	normalizedVersion, err := getNormalizedVersion(*kubeVersion, *gkeClusterVer)
+	if err != nil {
+		return fmt.Errorf("failed to get cluster minor version: %v", err)
+	}
+
+	testSkip := generateTestSkip(normalizedVersion)
+
 	// Run the tests using the testDir kubernetes
 	if len(*storageClassFile) != 0 {
-		err = runCSITests(pkgDir, testDir, *testFocus, *storageClassFile, cloudProviderArgs, *deploymentStrat)
+		err = runCSITests(pkgDir, testDir, *testFocus, testSkip, *storageClassFile, cloudProviderArgs, *deploymentStrat)
 	} else if *migrationTest {
-		err = runMigrationTests(pkgDir, testDir, *testFocus, cloudProviderArgs)
+		err = runMigrationTests(pkgDir, testDir, *testFocus, testSkip, cloudProviderArgs)
 	} else {
 		return fmt.Errorf("did not run either CSI or Migration test")
 	}
@@ -308,6 +315,31 @@ func handle() error {
 	return nil
 }
 
+func generateTestSkip(normalizedVersion string) string {
+	skipString := "\\[Disruptive\\]|\\[Serial\\]|\\[Feature:.+\\]"
+	switch normalizedVersion {
+	// Fall-through versioning since all test cases we want to skip in 1.15
+	// should also be skipped in 1.14
+	case "1.13":
+		fallthrough
+	case "1.14":
+		fallthrough
+	case "1.15":
+		fallthrough
+	case "1.16":
+		// "volumeMode should not mount / map unused volumes in a pod" tests a
+		// bug-fix introduced in 1.17
+		// (https://github.com/kubernetes/kubernetes/pull/81163)
+		skipString = skipString + "|volumeMode\\sshould\\snot\\smount\\s/\\smap\\sunused\\svolumes\\sin\\sa\\spod"
+		fallthrough
+	case "1.17":
+	case "latest":
+	case "master":
+	default:
+	}
+	return skipString
+}
+
 func setEnvProject(project string) error {
 	out, err := exec.Command("gcloud", "config", "set", "project", project).CombinedOutput()
 	if err != nil {
@@ -321,20 +353,20 @@ func setEnvProject(project string) error {
 	return nil
 }
 
-func runMigrationTests(pkgDir, testDir, testFocus string, cloudProviderArgs []string) error {
-	return runTestsWithConfig(testDir, testFocus, "--storage.migratedPlugins=kubernetes.io/gce-pd", cloudProviderArgs)
+func runMigrationTests(pkgDir, testDir, testFocus, testSkip string, cloudProviderArgs []string) error {
+	return runTestsWithConfig(testDir, testFocus, testSkip, "--storage.migratedPlugins=kubernetes.io/gce-pd", cloudProviderArgs)
 }
 
-func runCSITests(pkgDir, testDir, testFocus, storageClassFile string, cloudProviderArgs []string, deploymentStrat string) error {
+func runCSITests(pkgDir, testDir, testFocus, testSkip, storageClassFile string, cloudProviderArgs []string, deploymentStrat string) error {
 	testDriverConfigFile, err := generateDriverConfigFile(pkgDir, storageClassFile, deploymentStrat)
 	if err != nil {
 		return err
 	}
 	testConfigArg := fmt.Sprintf("--storage.testdriver=%s", testDriverConfigFile)
-	return runTestsWithConfig(testDir, testFocus, testConfigArg, cloudProviderArgs)
+	return runTestsWithConfig(testDir, testFocus, testSkip, testConfigArg, cloudProviderArgs)
 }
 
-func runTestsWithConfig(testDir, testFocus, testConfigArg string, cloudProviderArgs []string) error {
+func runTestsWithConfig(testDir, testFocus, testSkip, testConfigArg string, cloudProviderArgs []string) error {
 	err := os.Chdir(testDir)
 	if err != nil {
 		return err
@@ -348,7 +380,7 @@ func runTestsWithConfig(testDir, testFocus, testConfigArg string, cloudProviderA
 
 	testArgs := fmt.Sprintf("--ginkgo.focus=%s --ginkgo.skip=%s %s %s",
 		testFocus,
-		"\\[Disruptive\\]|\\[Serial\\]|\\[Feature:.+\\]",
+		testSkip,
 		testConfigArg,
 		reportArg)