Skip to content

Commit 4a5a0d3

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #712 from TeweiLuo/kubetest2
Add a mode to run e2e tests using kubetest2.
2 parents c8aa168 + 652f4db commit 4a5a0d3

File tree

7 files changed

+105
-10
lines changed

7 files changed

+105
-10
lines changed

‎test/k8s-integration/cluster.go

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -260,14 +260,16 @@ func downloadKubernetesSource(pkgDir, k8sIoDir, kubeVersion string) error {
260260
return nil
261261
}
262262

263-
func getGKEKubeTestArgs(gceZone, gceRegion, imageType string) ([]string, error) {
264-
var locationArg, locationVal string
263+
func getGKEKubeTestArgs(gceZone, gceRegion, imageType string, useKubetest2 bool) ([]string, error) {
264+
var locationArg, locationVal, locationArgK2 string
265265
switch {
266266
case len(gceZone) > 0:
267267
locationArg = "--gcp-zone"
268+
locationArgK2 = "--zone"
268269
locationVal = gceZone
269270
case len(gceRegion) > 0:
270271
locationArg = "--gcp-region"
272+
locationArgK2 = "--region"
271273
locationVal = gceRegion
272274
}
273275

@@ -291,6 +293,7 @@ func getGKEKubeTestArgs(gceZone, gceRegion, imageType string) ([]string, error)
291293
return nil, fmt.Errorf("failed to get current project: %v", err)
292294
}
293295

296+
// kubetest arguments
294297
args := []string{
295298
"--up=false",
296299
"--down=false",
@@ -306,7 +309,21 @@ func getGKEKubeTestArgs(gceZone, gceRegion, imageType string) ([]string, error)
306309
fmt.Sprintf("--gcp-project=%s", project[:len(project)-1]),
307310
}
308311

309-
return args, nil
312+
// kubetest2 arguments
313+
argsK2 := []string{
314+
"--up=false",
315+
"--down=false",
316+
fmt.Sprintf("--cluster-name=%s", *gkeTestClusterName),
317+
fmt.Sprintf("--environment=%s", gkeEnv),
318+
fmt.Sprintf("%s=%s", locationArgK2, locationVal),
319+
fmt.Sprintf("--project=%s", project[:len(project)-1]),
320+
}
321+
322+
if useKubetest2 {
323+
return argsK2, nil
324+
} else {
325+
return args, nil
326+
}
310327
}
311328

312329
func getNormalizedVersion(kubeVersion, gkeVersion string) (string, error) {

‎test/k8s-integration/main.go

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@ var (
6767
// Test flags
6868
migrationTest = flag.Bool("migration-test", false, "sets the flag on the e2e binary signalling migration")
6969
testFocus = flag.String("test-focus", "", "test focus for Kubernetes e2e")
70+
71+
useKubeTest2 = flag.Bool("use-kubetest2", false, "use kubetest2 to run e2e tests")
7072
)
7173

7274
const (
@@ -436,10 +438,20 @@ func handle() error {
436438

437439
switch testParams.deploymentStrategy {
438440
case "gke":
439-
testParams.cloudProviderArgs, err = getGKEKubeTestArgs(*gceZone, *gceRegion, testParams.imageType)
441+
testParams.cloudProviderArgs, err = getGKEKubeTestArgs(*gceZone, *gceRegion, testParams.imageType, *useKubeTest2)
440442
if err != nil {
441443
return fmt.Errorf("failed to build GKE kubetest args: %v", err)
442444
}
445+
case "gce":
446+
if *useKubeTest2 {
447+
testParams.cloudProviderArgs = []string{
448+
// This flag tells kubetest2 what "repo-root" is.
449+
// If --legacy-mode is set, kubernetes/kubernetes is used;
450+
// otherwise kubernetes/cloud-provider-gcp is used.
451+
"--legacy-mode",
452+
fmt.Sprintf("--repo-root=%s", *localK8sDir),
453+
}
454+
}
443455
}
444456

445457
// Kubernetes version of GKE deployments are expected to be of the pattern x.y.z-gke.k,
@@ -634,12 +646,32 @@ func runTestsWithConfig(testParams *testParameters, testConfigArg, reportPrefix
634646
"--check-version-skew=false",
635647
fmt.Sprintf("--test_args=%s", testArgs),
636648
}
649+
650+
kubeTest2Args := []string{
651+
*deploymentStrat,
652+
"--test=ginkgo",
653+
}
654+
kubeTest2Args = append(kubeTest2Args, testParams.cloudProviderArgs...)
655+
if kubetestDumpDir != "" {
656+
kubeTest2Args = append(kubeTest2Args, fmt.Sprintf("--log_dir=%s", kubetestDumpDir))
657+
}
658+
kubeTest2Args = append(kubeTest2Args, "--")
659+
kubeTest2Args = append(kubeTest2Args, fmt.Sprintf("--focus-regex=%s", testParams.testFocus))
660+
kubeTest2Args = append(kubeTest2Args, fmt.Sprintf("--skip-regex=%s", testParams.testSkip))
661+
// kubetest uses 25 as default value for ginkgo parallelism (--nodes).
662+
kubeTest2Args = append(kubeTest2Args, "--parallel=25")
663+
kubeTest2Args = append(kubeTest2Args, fmt.Sprintf("--test-args=%s %s", testConfigArg, reportArg))
664+
637665
if kubetestDumpDir != "" {
638666
kubeTestArgs = append(kubeTestArgs, fmt.Sprintf("--dump=%s", kubetestDumpDir))
639667
}
640668
kubeTestArgs = append(kubeTestArgs, testParams.cloudProviderArgs...)
641669

642-
err = runCommand("Running Tests", exec.Command("kubetest", kubeTestArgs...))
670+
if *useKubeTest2 {
671+
err = runCommand("Running Tests", exec.Command("kubetest2", kubeTest2Args...))
672+
} else {
673+
err = runCommand("Running Tests", exec.Command("kubetest", kubeTestArgs...))
674+
}
643675
if err != nil {
644676
return fmt.Errorf("failed to run tests on e2e cluster: %v", err)
645677
}

‎test/run-k8s-integration-ci.sh

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ readonly gke_release_channel=${GKE_RELEASE_CHANNEL:-""}
2525
readonly teardown_driver=${GCE_PD_TEARDOWN_DRIVER:-true}
2626
readonly gke_node_version=${GKE_NODE_VERSION:-}
2727
readonly run_intree_plugin_tests=${RUN_INTREE_PLUGIN_TESTS:-false}
28+
readonly use_kubetest2=${USE_KUBETEST2:-false}
2829

2930
storage_classes=sc-standard.yaml,sc-balanced.yaml,sc-ssd.yaml
3031

@@ -36,12 +37,20 @@ export GCE_PD_VERBOSITY=9
3637

3738
make -C "${PKGDIR}" test-k8s-integration
3839

40+
if [ "$use_kubetest2" = true ]; then
41+
export GO111MODULE=on;
42+
go get sigs.k8s.io/kubetest2@latest;
43+
go get sigs.k8s.io/kubetest2/kubetest2-gce@latest;
44+
go get sigs.k8s.io/kubetest2/kubetest2-gke@latest;
45+
go get sigs.k8s.io/kubetest2/kubetest2-tester-ginkgo@latest;
46+
fi
47+
3948
base_cmd="${PKGDIR}/bin/k8s-integration-test \
4049
--run-in-prow=true --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \
4150
--do-driver-build=${do_driver_build} --teardown-driver=${teardown_driver} --boskos-resource-type=${boskos_resource_type} \
4251
--storageclass-files="${storage_classes}" --snapshotclass-file=pd-volumesnapshotclass.yaml \
4352
--deployment-strategy=${deployment_strategy} --test-version=${test_version} \
44-
--num-nodes=3 --image-type=${image_type}"
53+
--num-nodes=3 --image-type=${image_type} --use-kubetest2=${use_kubetest2}"
4554

4655
if [ "$run_intree_plugin_tests" = true ]; then
4756
base_cmd="${base_cmd} --test-focus='External.Storage|In-tree.*Driver.*gcepd'"

‎test/run-k8s-integration-migration.sh

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,15 @@ readonly do_driver_build="${GCE_PD_DO_DRIVER_BUILD:-true}"
1919
readonly deployment_strategy=${DEPLOYMENT_STRATEGY:-gce}
2020
readonly kube_version=${GCE_PD_KUBE_VERSION:-master}
2121
readonly test_version=${TEST_VERSION:-master}
22+
readonly use_kubetest2=${USE_KUBETEST2:-false}
23+
24+
if [ "$use_kubetest2" = true ]; then
25+
export GO111MODULE=on;
26+
go get sigs.k8s.io/kubetest2@latest;
27+
go get sigs.k8s.io/kubetest2/kubetest2-gce@latest;
28+
go get sigs.k8s.io/kubetest2/kubetest2-gke@latest;
29+
go get sigs.k8s.io/kubetest2/kubetest2-tester-ginkgo@latest;
30+
fi
2231

2332
readonly GCE_PD_TEST_FOCUS="PersistentVolumes\sGCEPD|[V|v]olume\sexpand|\[sig-storage\]\sIn-tree\sVolumes\s\[Driver:\sgcepd\]|allowedTopologies|Pod\sDisks|PersistentVolumes\sDefault"
2433

@@ -31,4 +40,4 @@ make -C "${PKGDIR}" test-k8s-integration
3140
--do-driver-build="${do_driver_build}" --boskos-resource-type="${boskos_resource_type}" \
3241
--migration-test=true --test-focus="${GCE_PD_TEST_FOCUS}" \
3342
--gce-zone="us-central1-b" --deployment-strategy="${deployment_strategy}" --test-version="${test_version}" \
34-
--num-nodes=3
43+
--num-nodes=3 --use-kubetest2=${use_kubetest2}

‎test/run-k8s-integration.sh

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,17 +24,26 @@ readonly use_gke_managed_driver=${USE_GKE_MANAGED_DRIVER:-false}
2424
readonly gke_release_channel=${GKE_RELEASE_CHANNEL:-""}
2525
readonly teardown_driver=${GCE_PD_TEARDOWN_DRIVER:-true}
2626
readonly gke_node_version=${GKE_NODE_VERSION:-}
27+
readonly use_kubetest2=${USE_KUBETEST2:-false}
2728

2829
export GCE_PD_VERBOSITY=9
2930

3031
make -C "${PKGDIR}" test-k8s-integration
3132

33+
if [ "$use_kubetest2" = true ]; then
34+
export GO111MODULE=on;
35+
go get sigs.k8s.io/kubetest2@latest;
36+
go get sigs.k8s.io/kubetest2/kubetest2-gce@latest;
37+
go get sigs.k8s.io/kubetest2/kubetest2-gke@latest;
38+
go get sigs.k8s.io/kubetest2/kubetest2-tester-ginkgo@latest;
39+
fi
40+
3241
base_cmd="${PKGDIR}/bin/k8s-integration-test \
3342
--run-in-prow=true --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \
3443
--do-driver-build=${do_driver_build} --teardown-driver=${teardown_driver} --boskos-resource-type=${boskos_resource_type} \
3544
--storageclass-files=sc-standard.yaml --snapshotclass-file=pd-volumesnapshotclass.yaml \
3645
--test-focus='External.Storage' --deployment-strategy=${deployment_strategy} --test-version=${test_version} \
37-
--num-nodes=3 --image-type=${image_type}"
46+
--num-nodes=3 --image-type=${image_type} --use-kubetest2=${use_kubetest2}"
3847

3948
if [ "$use_gke_managed_driver" = false ]; then
4049
base_cmd="${base_cmd} --deploy-overlay-name=${overlay_name}"

‎test/run-k8s-windows-migration.sh

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,15 @@ readonly kube_version=${GCE_PD_KUBE_VERSION:-master}
2020
readonly test_version=${TEST_VERSION:-master}
2121
readonly gce_zone=${GCE_CLUSTER_ZONE:-us-central1-b}
2222
readonly feature_gates="CSIMigration=true,CSIMigrationGCE=true,ExpandCSIVolumes=true"
23+
readonly use_kubetest2=${USE_KUBETEST2:-false}
24+
25+
if [ "$use_kubetest2" = true ]; then
26+
export GO111MODULE=on;
27+
go get sigs.k8s.io/kubetest2@latest;
28+
go get sigs.k8s.io/kubetest2/kubetest2-gce@latest;
29+
go get sigs.k8s.io/kubetest2/kubetest2-gke@latest;
30+
go get sigs.k8s.io/kubetest2/kubetest2-tester-ginkgo@latest;
31+
fi
2332

2433
readonly GCE_PD_TEST_FOCUS="PersistentVolumes\sGCEPD|[V|v]olume\sexpand|\[sig-storage\]\sIn-tree\sVolumes\s\[Driver:\swindows-gcepd\]|allowedTopologies|Pod\sDisks|PersistentVolumes\sDefault"
2534

@@ -34,7 +43,8 @@ ${PKGDIR}/bin/k8s-integration-test \
3443
--deploy-overlay-name=${overlay_name} --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \
3544
--do-driver-build=${do_driver_build} --boskos-resource-type=${boskos_resource_type} \
3645
--migration-test=true --test-focus=${GCE_PD_TEST_FOCUS} \
37-
--gce-zone=${gce_zone} --deployment-strategy=${deployment_strategy} --test-version=${test_version}
46+
--gce-zone=${gce_zone} --deployment-strategy=${deployment_strategy} --test-version=${test_version} \
47+
--use-kubetest2=${use_kubetest2}
3848

3949
#eval "$base_cmd"
4050

‎test/run-windows-k8s-integration.sh

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,15 @@ readonly deployment_strategy=${DEPLOYMENT_STRATEGY:-gce}
1616
readonly test_version=${TEST_VERSION:-master}
1717
readonly gce_zone=${GCE_CLUSTER_ZONE:-us-central1-b}
1818
readonly teardown_driver=${GCE_PD_TEARDOWN_DRIVER:-true}
19+
readonly use_kubetest2=${USE_KUBETEST2:-false}
20+
21+
if [ "$use_kubetest2" = true ]; then
22+
export GO111MODULE=on;
23+
go get sigs.k8s.io/kubetest2@latest;
24+
go get sigs.k8s.io/kubetest2/kubetest2-gce@latest;
25+
go get sigs.k8s.io/kubetest2/kubetest2-gke@latest;
26+
go get sigs.k8s.io/kubetest2/kubetest2-tester-ginkgo@latest;
27+
fi
1928

2029
make -C "${PKGDIR}" test-k8s-integration
2130

@@ -24,6 +33,6 @@ base_cmd="${PKGDIR}/bin/k8s-integration-test \
2433
--run-in-prow=true --deploy-overlay-name=${overlay_name} --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \
2534
--do-driver-build=${do_driver_build} --gce-zone=${gce_zone} --test-version=${test_version}\
2635
--storageclass-files=sc-windows.yaml --snapshotclass-file=pd-volumesnapshotclass.yaml --test-focus='External.Storage' \
27-
--deployment-strategy=${deployment_strategy}"
36+
--deployment-strategy=${deployment_strategy} --use-kubetest2=${use_kubetest2}"
2837

2938
eval "$base_cmd"

0 commit comments

Comments
 (0)