Skip to content

Commit cfd432d

Browse files
TeweiLuoTeweiLuo
committed
Add a mode to run e2e test using kubetest2.
Engprod plans to end kubetest support. This diff adds a --use-kubetest2 flag to the integration test driver. Next step: 1. Add kubetest2 installation to test scripts. 2. Add a canary test config to validate kubetest2.
1 parent c8aa168 commit cfd432d

File tree

7 files changed

+78
-11
lines changed

7 files changed

+78
-11
lines changed

‎test/k8s-integration/cluster.go

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -260,14 +260,16 @@ func downloadKubernetesSource(pkgDir, k8sIoDir, kubeVersion string) error {
260260
return nil
261261
}
262262

263-
func getGKEKubeTestArgs(gceZone, gceRegion, imageType string) ([]string, error) {
264-
var locationArg, locationVal string
263+
func getGKEKubeTestArgs(gceZone, gceRegion, imageType string, useKubetest2 bool) ([]string, error) {
264+
var locationArg, locationVal, locationArgK2 string
265265
switch {
266266
case len(gceZone) > 0:
267267
locationArg = "--gcp-zone"
268+
locationArgK2 = "--zone"
268269
locationVal = gceZone
269270
case len(gceRegion) > 0:
270271
locationArg = "--gcp-region"
272+
locationArgK2 = "--region"
271273
locationVal = gceRegion
272274
}
273275

@@ -291,6 +293,7 @@ func getGKEKubeTestArgs(gceZone, gceRegion, imageType string) ([]string, error)
291293
return nil, fmt.Errorf("failed to get current project: %v", err)
292294
}
293295

296+
// kubetest arguments
294297
args := []string{
295298
"--up=false",
296299
"--down=false",
@@ -306,7 +309,21 @@ func getGKEKubeTestArgs(gceZone, gceRegion, imageType string) ([]string, error)
306309
fmt.Sprintf("--gcp-project=%s", project[:len(project)-1]),
307310
}
308311

309-
return args, nil
312+
// kubetest2 arguments
313+
argsK2 := []string{
314+
"--up=false",
315+
"--down=false",
316+
fmt.Sprintf("--cluster-name=%s", *gkeTestClusterName),
317+
fmt.Sprintf("--environment=%s", gkeEnv),
318+
fmt.Sprintf("%s=%s", locationArgK2, locationVal),
319+
fmt.Sprintf("--project=%s", project[:len(project)-1]),
320+
}
321+
322+
if useKubetest2 {
323+
return argsK2, nil
324+
} else {
325+
return args, nil
326+
}
310327
}
311328

312329
func getNormalizedVersion(kubeVersion, gkeVersion string) (string, error) {

‎test/k8s-integration/main.go

Lines changed: 39 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@ var (
6767
// Test flags
6868
migrationTest = flag.Bool("migration-test", false, "sets the flag on the e2e binary signalling migration")
6969
testFocus = flag.String("test-focus", "", "test focus for Kubernetes e2e")
70+
71+
useKubeTest2 = flag.Bool("use-kubetest2", false, "use kubetest2 to run e2e tests")
7072
)
7173

7274
const (
@@ -89,6 +91,7 @@ type testParameters struct {
8991
testSkip string
9092
snapshotClassFile string
9193
cloudProviderArgs []string
94+
cloudProviderArgsK2 []string
9295
deploymentStrategy string
9396
outputDir string
9497
allowedNotReadyNodes int
@@ -436,10 +439,23 @@ func handle() error {
436439

437440
switch testParams.deploymentStrategy {
438441
case "gke":
439-
testParams.cloudProviderArgs, err = getGKEKubeTestArgs(*gceZone, *gceRegion, testParams.imageType)
440-
if err != nil {
442+
// Build params for both kubetest and kubetest2 for debugging.
443+
testParams.cloudProviderArgs, err = getGKEKubeTestArgs(*gceZone, *gceRegion, testParams.imageType, false /* useKubeTest2 */)
444+
if err != nil && !*useKubeTest2 {
441445
return fmt.Errorf("failed to build GKE kubetest args: %v", err)
442446
}
447+
testParams.cloudProviderArgsK2, err = getGKEKubeTestArgs(*gceZone, *gceRegion, testParams.imageType, true /* useKubeTest2 */)
448+
if err != nil && *useKubeTest2 {
449+
return fmt.Errorf("failed to build GKE kubetest2 args: %v", err)
450+
}
451+
case "gce":
452+
testParams.cloudProviderArgsK2 = []string{
453+
// This flag tells kubetest2 what "repo-root" is.
454+
// If --legacy-mode is set, kubernetes/kubernetes is used;
455+
// otherwise kubernetes/cloud-provider-gcp is used.
456+
"--legacy-mode",
457+
fmt.Sprintf("--repo-root=%s", *localK8sDir),
458+
}
443459
}
444460

445461
// Kubernetes version of GKE deployments are expected to be of the pattern x.y.z-gke.k,
@@ -634,12 +650,32 @@ func runTestsWithConfig(testParams *testParameters, testConfigArg, reportPrefix
634650
"--check-version-skew=false",
635651
fmt.Sprintf("--test_args=%s", testArgs),
636652
}
653+
654+
kubeTest2Args := []string{
655+
*deploymentStrat,
656+
"--test=ginkgo",
657+
}
658+
kubeTest2Args = append(kubeTest2Args, testParams.cloudProviderArgsK2...)
659+
if kubetestDumpDir != "" {
660+
kubeTest2Args = append(kubeTest2Args, fmt.Sprintf("--log_dir=%s", kubetestDumpDir))
661+
}
662+
kubeTest2Args = append(kubeTest2Args, "--")
663+
kubeTest2Args = append(kubeTest2Args, fmt.Sprintf("--focus-regex=%s", testParams.testFocus))
664+
kubeTest2Args = append(kubeTest2Args, fmt.Sprintf("--skip-regex=%s", testParams.testSkip))
665+
// kubetest uses 25 as default value for ginkgo parallelism (--nodes).
666+
kubeTest2Args = append(kubeTest2Args, "--parallel=25")
667+
kubeTest2Args = append(kubeTest2Args, fmt.Sprintf("--test-args=%s %s", testConfigArg, reportArg))
668+
637669
if kubetestDumpDir != "" {
638670
kubeTestArgs = append(kubeTestArgs, fmt.Sprintf("--dump=%s", kubetestDumpDir))
639671
}
640672
kubeTestArgs = append(kubeTestArgs, testParams.cloudProviderArgs...)
641673

642-
err = runCommand("Running Tests", exec.Command("kubetest", kubeTestArgs...))
674+
if *useKubeTest2 {
675+
err = runCommand("Running Tests", exec.Command("kubetest2", kubeTest2Args...))
676+
} else {
677+
err = runCommand("Running Tests", exec.Command("kubetest", kubeTestArgs...))
678+
}
643679
if err != nil {
644680
return fmt.Errorf("failed to run tests on e2e cluster: %v", err)
645681
}

‎test/run-k8s-integration-ci.sh

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ readonly gke_release_channel=${GKE_RELEASE_CHANNEL:-""}
2525
readonly teardown_driver=${GCE_PD_TEARDOWN_DRIVER:-true}
2626
readonly gke_node_version=${GKE_NODE_VERSION:-}
2727
readonly run_intree_plugin_tests=${RUN_INTREE_PLUGIN_TESTS:-false}
28+
readonly use_kubetest2=${USE_KUBETEST2:-false}
2829

2930
storage_classes=sc-standard.yaml,sc-balanced.yaml,sc-ssd.yaml
3031

@@ -41,7 +42,7 @@ base_cmd="${PKGDIR}/bin/k8s-integration-test \
4142
--do-driver-build=${do_driver_build} --teardown-driver=${teardown_driver} --boskos-resource-type=${boskos_resource_type} \
4243
--storageclass-files="${storage_classes}" --snapshotclass-file=pd-volumesnapshotclass.yaml \
4344
--deployment-strategy=${deployment_strategy} --test-version=${test_version} \
44-
--num-nodes=3 --image-type=${image_type}"
45+
--num-nodes=3 --image-type=${image_type} --use-kubetest2=${use_kubetest2}"
4546

4647
if [ "$run_intree_plugin_tests" = true ]; then
4748
base_cmd="${base_cmd} --test-focus='External.Storage|In-tree.*Driver.*gcepd'"

‎test/run-k8s-integration-migration.sh

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ readonly do_driver_build="${GCE_PD_DO_DRIVER_BUILD:-true}"
1919
readonly deployment_strategy=${DEPLOYMENT_STRATEGY:-gce}
2020
readonly kube_version=${GCE_PD_KUBE_VERSION:-master}
2121
readonly test_version=${TEST_VERSION:-master}
22+
readonly use_kubetest2=${USE_KUBETEST2:-false}
2223

2324
readonly GCE_PD_TEST_FOCUS="PersistentVolumes\sGCEPD|[V|v]olume\sexpand|\[sig-storage\]\sIn-tree\sVolumes\s\[Driver:\sgcepd\]|allowedTopologies|Pod\sDisks|PersistentVolumes\sDefault"
2425

@@ -31,4 +32,4 @@ make -C "${PKGDIR}" test-k8s-integration
3132
--do-driver-build="${do_driver_build}" --boskos-resource-type="${boskos_resource_type}" \
3233
--migration-test=true --test-focus="${GCE_PD_TEST_FOCUS}" \
3334
--gce-zone="us-central1-b" --deployment-strategy="${deployment_strategy}" --test-version="${test_version}" \
34-
--num-nodes=3
35+
--num-nodes=3 --use-kubetest2=${use_kubetest2}

‎test/run-k8s-integration.sh

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,17 +24,26 @@ readonly use_gke_managed_driver=${USE_GKE_MANAGED_DRIVER:-false}
2424
readonly gke_release_channel=${GKE_RELEASE_CHANNEL:-""}
2525
readonly teardown_driver=${GCE_PD_TEARDOWN_DRIVER:-true}
2626
readonly gke_node_version=${GKE_NODE_VERSION:-}
27+
readonly use_kubetest2=${USE_KUBETEST2:-false}
2728

2829
export GCE_PD_VERBOSITY=9
2930

3031
make -C "${PKGDIR}" test-k8s-integration
3132

33+
if [ "$use_kubetest2" = true ]; then
34+
export GO111MODULE=on;
35+
go get sigs.k8s.io/kubetest2@latest;
36+
go get sigs.k8s.io/kubetest2/kubetest2-gce@latest;
37+
go get sigs.k8s.io/kubetest2/kubetest2-gke@latest;
38+
go get sigs.k8s.io/kubetest2/kubetest2-tester-ginkgo@latest;
39+
fi
40+
3241
base_cmd="${PKGDIR}/bin/k8s-integration-test \
3342
--run-in-prow=true --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \
3443
--do-driver-build=${do_driver_build} --teardown-driver=${teardown_driver} --boskos-resource-type=${boskos_resource_type} \
3544
--storageclass-files=sc-standard.yaml --snapshotclass-file=pd-volumesnapshotclass.yaml \
3645
--test-focus='External.Storage' --deployment-strategy=${deployment_strategy} --test-version=${test_version} \
37-
--num-nodes=3 --image-type=${image_type}"
46+
--num-nodes=3 --image-type=${image_type} --use-kubetest2=${use_kubetest2}"
3847

3948
if [ "$use_gke_managed_driver" = false ]; then
4049
base_cmd="${base_cmd} --deploy-overlay-name=${overlay_name}"

‎test/run-k8s-windows-migration.sh

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ readonly kube_version=${GCE_PD_KUBE_VERSION:-master}
2020
readonly test_version=${TEST_VERSION:-master}
2121
readonly gce_zone=${GCE_CLUSTER_ZONE:-us-central1-b}
2222
readonly feature_gates="CSIMigration=true,CSIMigrationGCE=true,ExpandCSIVolumes=true"
23+
readonly use_kubetest2=${USE_KUBETEST2:-false}
2324

2425
readonly GCE_PD_TEST_FOCUS="PersistentVolumes\sGCEPD|[V|v]olume\sexpand|\[sig-storage\]\sIn-tree\sVolumes\s\[Driver:\swindows-gcepd\]|allowedTopologies|Pod\sDisks|PersistentVolumes\sDefault"
2526

@@ -34,7 +35,8 @@ ${PKGDIR}/bin/k8s-integration-test \
3435
--deploy-overlay-name=${overlay_name} --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \
3536
--do-driver-build=${do_driver_build} --boskos-resource-type=${boskos_resource_type} \
3637
--migration-test=true --test-focus=${GCE_PD_TEST_FOCUS} \
37-
--gce-zone=${gce_zone} --deployment-strategy=${deployment_strategy} --test-version=${test_version}
38+
--gce-zone=${gce_zone} --deployment-strategy=${deployment_strategy} --test-version=${test_version} \
39+
--use-kubetest2=${use_kubetest2}
3840

3941
#eval "$base_cmd"
4042

‎test/run-windows-k8s-integration.sh

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ readonly deployment_strategy=${DEPLOYMENT_STRATEGY:-gce}
1616
readonly test_version=${TEST_VERSION:-master}
1717
readonly gce_zone=${GCE_CLUSTER_ZONE:-us-central1-b}
1818
readonly teardown_driver=${GCE_PD_TEARDOWN_DRIVER:-true}
19+
readonly use_kubetest2=${USE_KUBETEST2:-false}
1920

2021
make -C "${PKGDIR}" test-k8s-integration
2122

@@ -24,6 +25,6 @@ base_cmd="${PKGDIR}/bin/k8s-integration-test \
2425
--run-in-prow=true --deploy-overlay-name=${overlay_name} --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \
2526
--do-driver-build=${do_driver_build} --gce-zone=${gce_zone} --test-version=${test_version}\
2627
--storageclass-files=sc-windows.yaml --snapshotclass-file=pd-volumesnapshotclass.yaml --test-focus='External.Storage' \
27-
--deployment-strategy=${deployment_strategy}"
28+
--deployment-strategy=${deployment_strategy} --use-kubetest2=${use_kubetest2}"
2829

2930
eval "$base_cmd"

0 commit comments

Comments
 (0)