Merge pull request kubernetes-sigs#249 from msau42/use-external-tests

Use K8s external-storage test framework

Author: k8s-ci-robot

test/k8s-integration/config/sc-standard.yaml

@@ -0,0 +1,8 @@
+apiVersion: storage.k8s.io/v1beta1
+kind: StorageClass
+metadata:
+  name: csi-gcepd
+provisioner: pd.csi.storage.gke.io
+parameters:
+  type: pd-standard
+volumeBindingMode: Immediate

test/k8s-integration/config/test-config-template.in

@@ -0,0 +1,13 @@
+ShortName: pdtest
+StorageClass:
+  FromFile: {{.StorageClassFile}}
+DriverInfo:
+  Name: csi-gcepd
+  Capabilities:
+    persistence: true
+    multipods: true
+    fsGroup: true
+    exec: true
+    # block: true
+    # dataSource: true
+    # RWX: true

test/k8s-integration/driver-config.go

@@ -0,0 +1,52 @@
+package main
+
+import (
+	"bufio"
+	"os"
+	"path/filepath"
+	"text/template"
+)
+
+type driverConfig struct {
+	StorageClassFile string
+}
+
+const (
+	testConfigDir      = "test/k8s-integration/config"
+	configTemplateFile = "test-config-template.in"
+	configFile         = "test-config.yaml"
+)
+
+// generateDriverConfigFile loads a testdriver config template and creates a file
+// with the test-specific configuration
+func generateDriverConfigFile(pkgDir, storageClassFile string) (string, error) {
+	// Load template
+	t, err := template.ParseFiles(filepath.Join(pkgDir, testConfigDir, configTemplateFile))
+	if err != nil {
+		return "", err
+	}
+
+	// Create destination
+	configFilePath := filepath.Join(pkgDir, testConfigDir, configFile)
+	f, err := os.Create(configFilePath)
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+
+	w := bufio.NewWriter(f)
+	defer w.Flush()
+
+	// Fill in template parameters
+	params := driverConfig{
+		StorageClassFile: filepath.Join(pkgDir, testConfigDir, storageClassFile),
+	}
+
+	// Write config file
+	err = t.Execute(w, params)
+	if err != nil {
+		return "", err
+	}
+
+	return configFilePath, nil
+}

test/k8s-integration/main.go

@@ -42,10 +42,12 @@ var (
 	localK8sDir        = flag.String("local-k8s-dir", "", "local kubernetes/kubernetes directory to run e2e tests from")
 	doDriverBuild      = flag.Bool("do-driver-build", true, "building the driver from source")
 	boskosResourceType = flag.String("boskos-resource-type", "gce-project", "name of the boskos resource type to reserve")
+	storageClassFile   = flag.String("storageclass-file", "", "name of storageclass yaml file to use for test relative to test/k8s-integration/config")
 )
 
 const (
 	pdImagePlaceholder = "REPLACEME/gcp-compute-persistent-disk-csi-driver"
+	k8sBuildBinDir     = "_output/dockerized/bin/linux/amd64"
 )
 
 func init() {
@@ -66,6 +68,10 @@ func main() {
 		glog.Fatalf("deploy-overlay-name is a required flag")
 	}
 
+	if len(*storageClassFile) == 0 {
+		glog.Fatalf("storageclass-file is a required flag")
+	}
+
 	err := handle()
 	if err != nil {
 		glog.Fatalf("Failed to run integration test: %v", err)
@@ -186,7 +192,7 @@ func handle() error {
 	if len(*localK8sDir) != 0 {
 		k8sDir = *localK8sDir
 	}
-	err = runTests(k8sDir)
+	err = runTests(pkgDir, k8sDir, *storageClassFile)
 	if err != nil {
 		return fmt.Errorf("failed to run tests: %v", err)
 	}
@@ -207,20 +213,34 @@ func setEnvProject(project string) error {
 	return nil
 }
 
-func runTests(k8sDir string) error {
-	err := os.Chdir(k8sDir)
+func runTests(pkgDir, k8sDir, storageClassFile string) error {
+	testDriverConfigFile, err := generateDriverConfigFile(pkgDir, storageClassFile)
+	if err != nil {
+		return err
+	}
+
+	err = os.Chdir(k8sDir)
 	if err != nil {
 		return err
 	}
+
+	homeDir, _ := os.LookupEnv("HOME")
+	os.Setenv("KUBECONFIG", filepath.Join(homeDir, ".kube/config"))
+
 	artifactsDir, _ := os.LookupEnv("ARTIFACTS")
-	reportArg := fmt.Sprintf("--report-dir=%s", artifactsDir)
-	testArgs := fmt.Sprintf("--test_args=--ginkgo.focus=CSI.*gcePD-external --ginkgo.skip=\\[Disruptive\\]|\\[Serial\\]|kubelet.*down %s", reportArg)
-	cmd := exec.Command("go", "run", "hack/e2e.go",
+	reportArg := fmt.Sprintf("-report-dir=%s", artifactsDir)
+
+	driverConfigArg := fmt.Sprintf("-storage.testdriver=%s", testDriverConfigFile)
+
+	cmd := exec.Command(filepath.Join(k8sBuildBinDir, "ginkgo"),
+		"-p",
+		"-focus=External.Storage",
+		"-skip=\\[Disruptive\\]|\\[Serial\\]|\\[Feature:.+\\]",
+		filepath.Join(k8sBuildBinDir, "e2e.test"),
 		"--",
-		"--check-version-skew=false",
-		"--test",
-		"--ginkgo-parallel",
-		testArgs)
+		reportArg,
+		driverConfigArg)
+
 	err = runCommand("Running Tests", cmd)
 	if err != nil {
 		return fmt.Errorf("failed to run tests on e2e cluster: %v", err)
@@ -235,6 +255,7 @@ func runCommand(action string, cmd *exec.Cmd) error {
 	cmd.Stderr = os.Stderr
 
 	fmt.Printf("%s\n", action)
+	fmt.Printf("%s\n", cmd.Args)
 
 	err := cmd.Start()
 	if err != nil {

test/run-k8s-integration-local.sh

@@ -10,5 +10,9 @@ ensure_var GCE_PD_CSI_STAGING_IMAGE
 ensure_var GCE_PD_SA_DIR
 
 make -C ${PKGDIR} test-k8s-integration
-${PKGDIR}/bin/k8s-integration-test --kube-version=master --run-in-prow=false --staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json --deploy-overlay-name=dev
-#${PKGDIR}/bin/k8s-integration-test --kube-version=master --run-in-prow=false --staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json --deploy-overlay-name=dev --bringup-cluster=false --teardown-cluster=false --local-k8s-dir=$KTOP
+${PKGDIR}/bin/k8s-integration-test --kube-version=master --run-in-prow=false --staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json --deploy-overlay-name=dev --storageclass-file=sc-standard.yaml
+
+# This version of the command does not build the driver or K8s, points to a
+# local K8s repo to get the e2e.test binary, and does not bring up or down the cluster
+#
+# ${PKGDIR}/bin/k8s-integration-test --kube-version=master --run-in-prow=false --staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json --deploy-overlay-name=dev --bringup-cluster=false --teardown-cluster=false --local-k8s-dir=$KTOP --storageclass-file=sc-standard.yaml --do-driver-build=false

test/run-k8s-integration.sh

@@ -16,4 +16,4 @@ readonly do_driver_build="${GCE_PD_DO_DRIVER_BUILD:-true}"
 export GCE_PD_VERBOSITY=9
 
 make -C ${PKGDIR} test-k8s-integration
-${PKGDIR}/bin/k8s-integration-test --kube-version=master --run-in-prow=true --deploy-overlay-name=${overlay_name} --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} --do-driver-build=${do_driver_build} --boskos-resource-type=${boskos_resource_type}
+${PKGDIR}/bin/k8s-integration-test --kube-version=master --run-in-prow=true --deploy-overlay-name=${overlay_name} --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} --do-driver-build=${do_driver_build} --boskos-resource-type=${boskos_resource_type} --storageclass-file=sc-standard.yaml