Skip to content

Commit 70bcfed

Browse files
TeweiLuoTeweiLuo
committed
Add a mode to run e2e test using kubetest2.
Engprod plans to end kubetest support. This diff adds a --use-kubetest2 flag to the integration test driver. Next step: 1. Add kubetest2 installation to test scripts. 2. Add a canary test config to validate kubetest2.
1 parent a15bae1 commit 70bcfed

File tree

4 files changed

+72
-8
lines changed

4 files changed

+72
-8
lines changed

‎test/k8s-integration/cluster.go

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -260,14 +260,16 @@ func downloadKubernetesSource(pkgDir, k8sIoDir, kubeVersion string) error {
260260
return nil
261261
}
262262

263-
func getGKEKubeTestArgs(gceZone, gceRegion, imageType string) ([]string, error) {
264-
var locationArg, locationVal string
263+
func getGKEKubeTestArgs(gceZone, gceRegion, imageType string, useKubetest2 bool) ([]string, error) {
264+
var locationArg, locationVal, locationArgK2 string
265265
switch {
266266
case len(gceZone) > 0:
267267
locationArg = "--gcp-zone"
268+
locationArgK2 = "--zone"
268269
locationVal = gceZone
269270
case len(gceRegion) > 0:
270271
locationArg = "--gcp-region"
272+
locationArgK2 = "--region"
271273
locationVal = gceRegion
272274
}
273275

@@ -291,6 +293,7 @@ func getGKEKubeTestArgs(gceZone, gceRegion, imageType string) ([]string, error)
291293
return nil, fmt.Errorf("failed to get current project: %v", err)
292294
}
293295

296+
// kubetest arguments
294297
args := []string{
295298
"--up=false",
296299
"--down=false",
@@ -306,7 +309,21 @@ func getGKEKubeTestArgs(gceZone, gceRegion, imageType string) ([]string, error)
306309
fmt.Sprintf("--gcp-project=%s", project[:len(project)-1]),
307310
}
308311

309-
return args, nil
312+
// kubetest2 arguments
313+
argsK2 := []string{
314+
"--up=false",
315+
"--down=false",
316+
fmt.Sprintf("--cluster-name=%s", *gkeTestClusterName),
317+
fmt.Sprintf("--environment=%s", gkeEnv),
318+
fmt.Sprintf("%s=%s", locationArgK2, locationVal),
319+
fmt.Sprintf("--project=%s", project[:len(project)-1]),
320+
}
321+
322+
if useKubetest2 {
323+
return argsK2, nil
324+
} else {
325+
return args, nil
326+
}
310327
}
311328

312329
func getNormalizedVersion(kubeVersion, gkeVersion string) (string, error) {

‎test/k8s-integration/main.go

Lines changed: 40 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@ var (
6767
// Test flags
6868
migrationTest = flag.Bool("migration-test", false, "sets the flag on the e2e binary signalling migration")
6969
testFocus = flag.String("test-focus", "", "test focus for Kubernetes e2e")
70+
71+
useKubeTest2 = flag.Bool("use-kubetest2", false, "use kubetest2 to run e2e tests")
7072
)
7173

7274
const (
@@ -89,6 +91,7 @@ type testParameters struct {
8991
testSkip string
9092
snapshotClassFile string
9193
cloudProviderArgs []string
94+
cloudProviderArgsK2 []string
9295
deploymentStrategy string
9396
outputDir string
9497
allowedNotReadyNodes int
@@ -436,10 +439,23 @@ func handle() error {
436439

437440
switch testParams.deploymentStrategy {
438441
case "gke":
439-
testParams.cloudProviderArgs, err = getGKEKubeTestArgs(*gceZone, *gceRegion, testParams.imageType)
440-
if err != nil {
442+
// Build params for both kubetest and kubetest2 for debugging.
443+
testParams.cloudProviderArgs, err = getGKEKubeTestArgs(*gceZone, *gceRegion, testParams.imageType, false /* useKubeTest2 */)
444+
if err != nil && !*useKubeTest2 {
441445
return fmt.Errorf("failed to build GKE kubetest args: %v", err)
442446
}
447+
testParams.cloudProviderArgsK2, err = getGKEKubeTestArgs(*gceZone, *gceRegion, testParams.imageType, true /* useKubeTest2 */)
448+
if err != nil && *useKubeTest2 {
449+
return fmt.Errorf("failed to build GKE kubetest2 args: %v", err)
450+
}
451+
case "gce":
452+
testParams.cloudProviderArgsK2 = []string{
453+
// This flag tells kubetest2 what "repo-root" is.
454+
// If --legacy-mode is set, kubernetes/kubernetes is used;
455+
// otherwise kubernetes/cloud-provider-gcp is used.
456+
"--legacy-mode",
457+
fmt.Sprintf("--repo-root=%s", *localK8sDir),
458+
}
443459
}
444460

445461
// Kubernetes version of GKE deployments are expected to be of the pattern x.y.z-gke.k,
@@ -634,12 +650,33 @@ func runTestsWithConfig(testParams *testParameters, testConfigArg, reportPrefix
634650
"--check-version-skew=false",
635651
fmt.Sprintf("--test_args=%s", testArgs),
636652
}
653+
654+
kubeTest2Args := []string{
655+
*deploymentStrat,
656+
"--test=ginkgo",
657+
}
658+
kubeTest2Args = append(kubeTest2Args, testParams.cloudProviderArgsK2...)
659+
if kubetestDumpDir != "" {
660+
kubeTest2Args = append(kubeTest2Args, fmt.Sprintf("--log_dir=%s", kubetestDumpDir))
661+
}
662+
kubeTest2Args = append(kubeTest2Args, "--")
663+
kubeTest2Args = append(kubeTest2Args, fmt.Sprintf("--focus-regex=%s", testParams.testFocus))
664+
kubeTest2Args = append(kubeTest2Args, fmt.Sprintf("--skip-regex=%s", testParams.testSkip))
665+
// kubetest uses 25 as default value for ginkgo parallelism (--nodes).
666+
kubeTest2Args = append(kubeTest2Args, "--parallel=25")
667+
kubeTest2Args = append(kubeTest2Args, fmt.Sprintf("--test-args=%s %s", testConfigArg, reportArg))
668+
637669
if kubetestDumpDir != "" {
638670
kubeTestArgs = append(kubeTestArgs, fmt.Sprintf("--dump=%s", kubetestDumpDir))
639671
}
640672
kubeTestArgs = append(kubeTestArgs, testParams.cloudProviderArgs...)
641673

642-
err = runCommand("Running Tests", exec.Command("kubetest", kubeTestArgs...))
674+
if *useKubeTest2 {
675+
klog.Info("Using kubetest2: %+v", kubeTest2Args)
676+
err = runCommand("Running Tests", exec.Command("kubetest2", kubeTest2Args...))
677+
} else {
678+
err = runCommand("Running Tests", exec.Command("kubetest", kubeTestArgs...))
679+
}
643680
if err != nil {
644681
return fmt.Errorf("failed to run tests on e2e cluster: %v", err)
645682
}

‎test/run-k8s-integration-ci.sh

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ readonly gke_release_channel=${GKE_RELEASE_CHANNEL:-""}
2525
readonly teardown_driver=${GCE_PD_TEARDOWN_DRIVER:-true}
2626
readonly gke_node_version=${GKE_NODE_VERSION:-}
2727
readonly run_intree_plugin_tests=${RUN_INTREE_PLUGIN_TESTS:-false}
28+
readonly use_kubetest2=${USE_KUBETEST2:-false}
2829

2930
storage_classes=sc-standard.yaml,sc-balanced.yaml,sc-ssd.yaml
3031

@@ -41,7 +42,7 @@ base_cmd="${PKGDIR}/bin/k8s-integration-test \
4142
--do-driver-build=${do_driver_build} --teardown-driver=${teardown_driver} --boskos-resource-type=${boskos_resource_type} \
4243
--storageclass-files="${storage_classes}" --snapshotclass-file=pd-volumesnapshotclass.yaml \
4344
--deployment-strategy=${deployment_strategy} --test-version=${test_version} \
44-
--num-nodes=3 --image-type=${image_type}"
45+
--num-nodes=3 --image-type=${image_type}" --use-kubetest2=${use_kubetest2}
4546

4647
if [ "$run_intree_plugin_tests" = true ]; then
4748
base_cmd="${base_cmd} --test-focus='External.Storage|In-tree.*Driver.*gcepd'"

‎test/run-k8s-integration.sh

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,17 +24,26 @@ readonly use_gke_managed_driver=${USE_GKE_MANAGED_DRIVER:-false}
2424
readonly gke_release_channel=${GKE_RELEASE_CHANNEL:-""}
2525
readonly teardown_driver=${GCE_PD_TEARDOWN_DRIVER:-true}
2626
readonly gke_node_version=${GKE_NODE_VERSION:-}
27+
readonly use_kubetest2=${USE_KUBETEST2:-true}
2728

2829
export GCE_PD_VERBOSITY=9
2930

3031
make -C "${PKGDIR}" test-k8s-integration
3132

33+
if [ "$use_kubetest2" = true ]; then
34+
export GO111MODULE=on;
35+
go get sigs.k8s.io/kubetest2@latest;
36+
go get sigs.k8s.io/kubetest2/kubetest2-gce@latest;
37+
go get sigs.k8s.io/kubetest2/kubetest2-gke@latest;
38+
go get sigs.k8s.io/kubetest2/kubetest2-tester-ginkgo@latest;
39+
fi
40+
3241
base_cmd="${PKGDIR}/bin/k8s-integration-test \
3342
--run-in-prow=true --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \
3443
--do-driver-build=${do_driver_build} --teardown-driver=${teardown_driver} --boskos-resource-type=${boskos_resource_type} \
3544
--storageclass-files=sc-standard.yaml --snapshotclass-file=pd-volumesnapshotclass.yaml \
3645
--test-focus='External.Storage' --deployment-strategy=${deployment_strategy} --test-version=${test_version} \
37-
--num-nodes=3 --image-type=${image_type}"
46+
--num-nodes=3 --image-type=${image_type}" --use-kubetest2=${use_kubetest2}
3847

3948
if [ "$use_gke_managed_driver" = false ]; then
4049
base_cmd="${base_cmd} --deploy-overlay-name=${overlay_name}"

0 commit comments

Comments
 (0)