Skip to content

Commit ca37ad6

Browse files
msau42msau42
committed
Use K8s external-storage test framework
1 parent ac6c3e9 commit ca37ad6

File tree

6 files changed

+111
-13
lines changed

6 files changed

+111
-13
lines changed

‎test/k8s-integration/config/sc-standard.yaml

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
apiVersion: storage.k8s.io/v1beta1
2+
kind: StorageClass
3+
metadata:
4+
name: csi-gcepd
5+
provisioner: pd.csi.storage.gke.io
6+
parameters:
7+
type: pd-standard
8+
volumeBindingMode: Immediate

‎test/k8s-integration/config/test-config-template.in

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
ShortName: pdtest
2+
StorageClass:
3+
FromFile: {{.StorageClassFile}}
4+
DriverInfo:
5+
Name: csi-gcepd
6+
Capabilities:
7+
persistence: true
8+
multipods: true
9+
fsGroup: true
10+
exec: true
11+
# block: true
12+
# dataSource: true
13+
# RWX: true

‎test/k8s-integration/driver-config.go

+52
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
package main
2+
3+
import (
4+
"bufio"
5+
"os"
6+
"path/filepath"
7+
"text/template"
8+
)
9+
10+
type driverConfig struct {
11+
StorageClassFile string
12+
}
13+
14+
const (
15+
testConfigDir = "test/k8s-integration/config"
16+
configTemplateFile = "test-config-template.in"
17+
configFile = "test-config.yaml"
18+
)
19+
20+
// generateDriverConfigFile loads a testdriver config template and creates a file
21+
// with the test-specific configuration
22+
func generateDriverConfigFile(pkgDir, storageClassFile string) (string, error) {
23+
// Load template
24+
t, err := template.ParseFiles(filepath.Join(pkgDir, testConfigDir, configTemplateFile))
25+
if err != nil {
26+
return "", err
27+
}
28+
29+
// Create destination
30+
configFilePath := filepath.Join(pkgDir, testConfigDir, configFile)
31+
f, err := os.Create(configFilePath)
32+
if err != nil {
33+
return "", err
34+
}
35+
defer f.Close()
36+
37+
w := bufio.NewWriter(f)
38+
defer w.Flush()
39+
40+
// Fill in template parameters
41+
params := driverConfig{
42+
StorageClassFile: filepath.Join(pkgDir, testConfigDir, storageClassFile),
43+
}
44+
45+
// Write config file
46+
err = t.Execute(w, params)
47+
if err != nil {
48+
return "", err
49+
}
50+
51+
return configFilePath, nil
52+
}

‎test/k8s-integration/main.go

+31-10
Original file line numberDiff line numberDiff line change
@@ -42,10 +42,12 @@ var (
4242
localK8sDir = flag.String("local-k8s-dir", "", "local kubernetes/kubernetes directory to run e2e tests from")
4343
doDriverBuild = flag.Bool("do-driver-build", true, "building the driver from source")
4444
boskosResourceType = flag.String("boskos-resource-type", "gce-project", "name of the boskos resource type to reserve")
45+
storageClassFile = flag.String("storageclass-file", "", "name of storageclass yaml file to use for test relative to test/k8s-integration/config")
4546
)
4647

4748
const (
4849
pdImagePlaceholder = "REPLACEME/gcp-compute-persistent-disk-csi-driver"
50+
k8sBuildBinDir = "_output/dockerized/bin/linux/amd64"
4951
)
5052

5153
func init() {
@@ -66,6 +68,10 @@ func main() {
6668
glog.Fatalf("deploy-overlay-name is a required flag")
6769
}
6870

71+
if len(*storageClassFile) == 0 {
72+
glog.Fatalf("storageclass-file is a required flag")
73+
}
74+
6975
err := handle()
7076
if err != nil {
7177
glog.Fatalf("Failed to run integration test: %v", err)
@@ -186,7 +192,7 @@ func handle() error {
186192
if len(*localK8sDir) != 0 {
187193
k8sDir = *localK8sDir
188194
}
189-
err = runTests(k8sDir)
195+
err = runTests(pkgDir, k8sDir, *storageClassFile)
190196
if err != nil {
191197
return fmt.Errorf("failed to run tests: %v", err)
192198
}
@@ -207,20 +213,34 @@ func setEnvProject(project string) error {
207213
return nil
208214
}
209215

210-
func runTests(k8sDir string) error {
211-
err := os.Chdir(k8sDir)
216+
func runTests(pkgDir, k8sDir, storageClassFile string) error {
217+
testDriverConfigFile, err := generateDriverConfigFile(pkgDir, storageClassFile)
218+
if err != nil {
219+
return err
220+
}
221+
222+
err = os.Chdir(k8sDir)
212223
if err != nil {
213224
return err
214225
}
226+
227+
homeDir, _ := os.LookupEnv("HOME")
228+
os.Setenv("KUBECONFIG", filepath.Join(homeDir, ".kube/config"))
229+
215230
artifactsDir, _ := os.LookupEnv("ARTIFACTS")
216-
reportArg := fmt.Sprintf("--report-dir=%s", artifactsDir)
217-
testArgs := fmt.Sprintf("--test_args=--ginkgo.focus=CSI.*gcePD-external --ginkgo.skip=\\[Disruptive\\]|\\[Serial\\]|kubelet.*down %s", reportArg)
218-
cmd := exec.Command("go", "run", "hack/e2e.go",
231+
reportArg := fmt.Sprintf("-report-dir=%s", artifactsDir)
232+
233+
driverConfigArg := fmt.Sprintf("-storage.testdriver=%s", testDriverConfigFile)
234+
235+
cmd := exec.Command(filepath.Join(k8sBuildBinDir, "ginkgo"),
236+
"-p",
237+
"-focus='External.Storage'",
238+
"-skip='\\[Disruptive\\]|\\[Serial\\]|\\[Feature:.+\\]'",
239+
filepath.Join(k8sBuildBinDir, "e2e.test"),
219240
"--",
220-
"--check-version-skew=false",
221-
"--test",
222-
"--ginkgo-parallel",
223-
testArgs)
241+
reportArg,
242+
driverConfigArg)
243+
224244
err = runCommand("Running Tests", cmd)
225245
if err != nil {
226246
return fmt.Errorf("failed to run tests on e2e cluster: %v", err)
@@ -235,6 +255,7 @@ func runCommand(action string, cmd *exec.Cmd) error {
235255
cmd.Stderr = os.Stderr
236256

237257
fmt.Printf("%s\n", action)
258+
fmt.Printf("%s\n", cmd.Args)
238259

239260
err := cmd.Start()
240261
if err != nil {

‎test/run-k8s-integration-local.sh

+6-2
Original file line numberDiff line numberDiff line change
@@ -10,5 +10,9 @@ ensure_var GCE_PD_CSI_STAGING_IMAGE
1010
ensure_var GCE_PD_SA_DIR
1111

1212
make -C ${PKGDIR} test-k8s-integration
13-
${PKGDIR}/bin/k8s-integration-test --kube-version=master --run-in-prow=false --staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json --deploy-overlay-name=dev
14-
#${PKGDIR}/bin/k8s-integration-test --kube-version=master --run-in-prow=false --staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json --deploy-overlay-name=dev --bringup-cluster=false --teardown-cluster=false --local-k8s-dir=$KTOP
13+
${PKGDIR}/bin/k8s-integration-test --kube-version=master --run-in-prow=false --staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json --deploy-overlay-name=dev --storageclass-file=sc-standard.yaml
14+
15+
# This version of the command does not build the driver or K8s, points to a
16+
# local K8s repo to get the e2e.test binary, and does not bring up or down the cluster
17+
#
18+
# ${PKGDIR}/bin/k8s-integration-test --kube-version=master --run-in-prow=false --staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json --deploy-overlay-name=dev --bringup-cluster=false --teardown-cluster=false --local-k8s-dir=$KTOP --storageclass-file=sc-standard.yaml --do-driver-build=false

‎test/run-k8s-integration.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,4 +16,4 @@ readonly do_driver_build="${GCE_PD_DO_DRIVER_BUILD:-true}"
1616
export GCE_PD_VERBOSITY=9
1717

1818
make -C ${PKGDIR} test-k8s-integration
19-
${PKGDIR}/bin/k8s-integration-test --kube-version=master --run-in-prow=true --deploy-overlay-name=${overlay_name} --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} --do-driver-build=${do_driver_build} --boskos-resource-type=${boskos_resource_type}
19+
${PKGDIR}/bin/k8s-integration-test --kube-version=master --run-in-prow=true --deploy-overlay-name=${overlay_name} --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} --do-driver-build=${do_driver_build} --boskos-resource-type=${boskos_resource_type} --storageclass-file=sc-standard.yaml

0 commit comments

Comments
 (0)