From ee485c4d1d3a0a1bed48ad81d63230f96dddb249 Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Fri, 19 Apr 2019 19:24:48 -0700 Subject: [PATCH] Use K8s external-storage test framework --- test/k8s-integration/config/sc-standard.yaml | 8 +++ .../config/test-config-template.in | 13 +++++ test/k8s-integration/driver-config.go | 52 +++++++++++++++++++ test/k8s-integration/main.go | 41 +++++++++++---- test/run-k8s-integration-local.sh | 8 ++- test/run-k8s-integration.sh | 2 +- 6 files changed, 111 insertions(+), 13 deletions(-) create mode 100644 test/k8s-integration/config/sc-standard.yaml create mode 100644 test/k8s-integration/config/test-config-template.in create mode 100644 test/k8s-integration/driver-config.go diff --git a/test/k8s-integration/config/sc-standard.yaml b/test/k8s-integration/config/sc-standard.yaml new file mode 100644 index 000000000..9ccd5f5dc --- /dev/null +++ b/test/k8s-integration/config/sc-standard.yaml @@ -0,0 +1,8 @@ +apiVersion: storage.k8s.io/v1beta1 +kind: StorageClass +metadata: + name: csi-gcepd +provisioner: pd.csi.storage.gke.io +parameters: + type: pd-standard +volumeBindingMode: Immediate diff --git a/test/k8s-integration/config/test-config-template.in b/test/k8s-integration/config/test-config-template.in new file mode 100644 index 000000000..5abe6765d --- /dev/null +++ b/test/k8s-integration/config/test-config-template.in @@ -0,0 +1,13 @@ +ShortName: pdtest +StorageClass: + FromFile: {{.StorageClassFile}} +DriverInfo: + Name: csi-gcepd + Capabilities: + persistence: true + multipods: true + fsGroup: true + exec: true + # block: true + # dataSource: true + # RWX: true diff --git a/test/k8s-integration/driver-config.go b/test/k8s-integration/driver-config.go new file mode 100644 index 000000000..422d32508 --- /dev/null +++ b/test/k8s-integration/driver-config.go @@ -0,0 +1,52 @@ +package main + +import ( + "bufio" + "os" + "path/filepath" + "text/template" +) + +type driverConfig struct { + StorageClassFile string +} + +const ( + testConfigDir = "test/k8s-integration/config" + configTemplateFile = "test-config-template.in" + configFile = "test-config.yaml" +) + +// generateDriverConfigFile loads a testdriver config template and creates a file +// with the test-specific configuration +func generateDriverConfigFile(pkgDir, storageClassFile string) (string, error) { + // Load template + t, err := template.ParseFiles(filepath.Join(pkgDir, testConfigDir, configTemplateFile)) + if err != nil { + return "", err + } + + // Create destination + configFilePath := filepath.Join(pkgDir, testConfigDir, configFile) + f, err := os.Create(configFilePath) + if err != nil { + return "", err + } + defer f.Close() + + w := bufio.NewWriter(f) + defer w.Flush() + + // Fill in template parameters + params := driverConfig{ + StorageClassFile: filepath.Join(pkgDir, testConfigDir, storageClassFile), + } + + // Write config file + err = t.Execute(w, params) + if err != nil { + return "", err + } + + return configFilePath, nil +} diff --git a/test/k8s-integration/main.go b/test/k8s-integration/main.go index 93c021c75..5e1e1c494 100644 --- a/test/k8s-integration/main.go +++ b/test/k8s-integration/main.go @@ -42,10 +42,12 @@ var ( localK8sDir = flag.String("local-k8s-dir", "", "local kubernetes/kubernetes directory to run e2e tests from") doDriverBuild = flag.Bool("do-driver-build", true, "building the driver from source") boskosResourceType = flag.String("boskos-resource-type", "gce-project", "name of the boskos resource type to reserve") + storageClassFile = flag.String("storageclass-file", "", "name of storageclass yaml file to use for test relative to test/k8s-integration/config") ) const ( pdImagePlaceholder = "REPLACEME/gcp-compute-persistent-disk-csi-driver" + k8sBuildBinDir = "_output/dockerized/bin/linux/amd64" ) func init() { @@ -66,6 +68,10 @@ func main() { glog.Fatalf("deploy-overlay-name is a required flag") } + if len(*storageClassFile) == 0 { + glog.Fatalf("storageclass-file is a required flag") + } + err := handle() if err != nil { glog.Fatalf("Failed to run integration test: %v", err) @@ -186,7 +192,7 @@ func handle() error { if len(*localK8sDir) != 0 { k8sDir = *localK8sDir } - err = runTests(k8sDir) + err = runTests(pkgDir, k8sDir, *storageClassFile) if err != nil { return fmt.Errorf("failed to run tests: %v", err) } @@ -207,20 +213,34 @@ func setEnvProject(project string) error { return nil } -func runTests(k8sDir string) error { - err := os.Chdir(k8sDir) +func runTests(pkgDir, k8sDir, storageClassFile string) error { + testDriverConfigFile, err := generateDriverConfigFile(pkgDir, storageClassFile) + if err != nil { + return err + } + + err = os.Chdir(k8sDir) if err != nil { return err } + + homeDir, _ := os.LookupEnv("HOME") + os.Setenv("KUBECONFIG", filepath.Join(homeDir, ".kube/config")) + artifactsDir, _ := os.LookupEnv("ARTIFACTS") - reportArg := fmt.Sprintf("--report-dir=%s", artifactsDir) - testArgs := fmt.Sprintf("--test_args=--ginkgo.focus=CSI.*gcePD-external --ginkgo.skip=\\[Disruptive\\]|\\[Serial\\]|kubelet.*down %s", reportArg) - cmd := exec.Command("go", "run", "hack/e2e.go", + reportArg := fmt.Sprintf("-report-dir=%s", artifactsDir) + + driverConfigArg := fmt.Sprintf("-storage.testdriver=%s", testDriverConfigFile) + + cmd := exec.Command(filepath.Join(k8sBuildBinDir, "ginkgo"), + "-p", + "-focus=External.Storage", + "-skip=\\[Disruptive\\]|\\[Serial\\]|\\[Feature:.+\\]", + filepath.Join(k8sBuildBinDir, "e2e.test"), "--", - "--check-version-skew=false", - "--test", - "--ginkgo-parallel", - testArgs) + reportArg, + driverConfigArg) + err = runCommand("Running Tests", cmd) if err != nil { return fmt.Errorf("failed to run tests on e2e cluster: %v", err) @@ -235,6 +255,7 @@ func runCommand(action string, cmd *exec.Cmd) error { cmd.Stderr = os.Stderr fmt.Printf("%s\n", action) + fmt.Printf("%s\n", cmd.Args) err := cmd.Start() if err != nil { diff --git a/test/run-k8s-integration-local.sh b/test/run-k8s-integration-local.sh index 57fdb6013..03f1a35a9 100755 --- a/test/run-k8s-integration-local.sh +++ b/test/run-k8s-integration-local.sh @@ -10,5 +10,9 @@ ensure_var GCE_PD_CSI_STAGING_IMAGE ensure_var GCE_PD_SA_DIR make -C ${PKGDIR} test-k8s-integration -${PKGDIR}/bin/k8s-integration-test --kube-version=master --run-in-prow=false --staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json --deploy-overlay-name=dev -#${PKGDIR}/bin/k8s-integration-test --kube-version=master --run-in-prow=false --staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json --deploy-overlay-name=dev --bringup-cluster=false --teardown-cluster=false --local-k8s-dir=$KTOP +${PKGDIR}/bin/k8s-integration-test --kube-version=master --run-in-prow=false --staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json --deploy-overlay-name=dev --storageclass-file=sc-standard.yaml + +# This version of the command does not build the driver or K8s, points to a +# local K8s repo to get the e2e.test binary, and does not bring up or down the cluster +# +# ${PKGDIR}/bin/k8s-integration-test --kube-version=master --run-in-prow=false --staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json --deploy-overlay-name=dev --bringup-cluster=false --teardown-cluster=false --local-k8s-dir=$KTOP --storageclass-file=sc-standard.yaml --do-driver-build=false diff --git a/test/run-k8s-integration.sh b/test/run-k8s-integration.sh index d36067e5a..dacf75c0d 100755 --- a/test/run-k8s-integration.sh +++ b/test/run-k8s-integration.sh @@ -16,4 +16,4 @@ readonly do_driver_build="${GCE_PD_DO_DRIVER_BUILD:-true}" export GCE_PD_VERBOSITY=9 make -C ${PKGDIR} test-k8s-integration -${PKGDIR}/bin/k8s-integration-test --kube-version=master --run-in-prow=true --deploy-overlay-name=${overlay_name} --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} --do-driver-build=${do_driver_build} --boskos-resource-type=${boskos_resource_type} +${PKGDIR}/bin/k8s-integration-test --kube-version=master --run-in-prow=true --deploy-overlay-name=${overlay_name} --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} --do-driver-build=${do_driver_build} --boskos-resource-type=${boskos_resource_type} --storageclass-file=sc-standard.yaml