From 8c91563038c84988033d7dd1036f4b2ea47650a4 Mon Sep 17 00:00:00 2001 From: Travis Xiang Date: Fri, 20 Sep 2024 22:08:54 +0000 Subject: [PATCH] Fix ci tests --- .../driver-args.yaml | 7 +++++++ .../kustomization.yaml | 6 ++++++ .../config/sc-balanced-no-labels.yaml | 8 ++++++++ test/k8s-integration/config/sc-balanced.yaml | 2 ++ test/k8s-integration/main.go | 18 ++++++++++++++++-- test/run-k8s-integration-ci.sh | 11 ++++++++--- test/run-k8s-integration.sh | 3 ++- 7 files changed, 49 insertions(+), 6 deletions(-) create mode 100644 deploy/kubernetes/overlays/prow-stable-sidecar-rc-master/driver-args.yaml create mode 100644 test/k8s-integration/config/sc-balanced-no-labels.yaml diff --git a/deploy/kubernetes/overlays/prow-stable-sidecar-rc-master/driver-args.yaml b/deploy/kubernetes/overlays/prow-stable-sidecar-rc-master/driver-args.yaml new file mode 100644 index 000000000..b3b858d58 --- /dev/null +++ b/deploy/kubernetes/overlays/prow-stable-sidecar-rc-master/driver-args.yaml @@ -0,0 +1,7 @@ +- op: add + path: /spec/template/spec/containers/4/args/- + value: --supports-dynamic-throughput-provisioning=hyperdisk-balanced,hyperdisk-throughput,hyperdisk-ml + +- op: add + path: /spec/template/spec/containers/4/args/- + value: --supports-dynamic-iops-provisioning=hyperdisk-balanced,hyperdisk-extreme \ No newline at end of file diff --git a/deploy/kubernetes/overlays/prow-stable-sidecar-rc-master/kustomization.yaml b/deploy/kubernetes/overlays/prow-stable-sidecar-rc-master/kustomization.yaml index 2063caa67..0d99a52cf 100644 --- a/deploy/kubernetes/overlays/prow-stable-sidecar-rc-master/kustomization.yaml +++ b/deploy/kubernetes/overlays/prow-stable-sidecar-rc-master/kustomization.yaml @@ -24,5 +24,11 @@ patchesJson6902: kind: Deployment name: csi-gce-pd-controller version: v1 +- path: driver-args.yaml + target: + group: apps + kind: Deployment + name: csi-gce-pd-controller + version: v1 transformers: - ../../images/prow-stable-sidecar-rc-master diff --git a/test/k8s-integration/config/sc-balanced-no-labels.yaml b/test/k8s-integration/config/sc-balanced-no-labels.yaml new file mode 100644 index 000000000..642c692f1 --- /dev/null +++ b/test/k8s-integration/config/sc-balanced-no-labels.yaml @@ -0,0 +1,8 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: csi-gcepd-balanced +provisioner: pd.csi.storage.gke.io +parameters: + type: pd-balanced +volumeBindingMode: WaitForFirstConsumer diff --git a/test/k8s-integration/config/sc-balanced.yaml b/test/k8s-integration/config/sc-balanced.yaml index 642c692f1..ac6a1c020 100644 --- a/test/k8s-integration/config/sc-balanced.yaml +++ b/test/k8s-integration/config/sc-balanced.yaml @@ -5,4 +5,6 @@ metadata: provisioner: pd.csi.storage.gke.io parameters: type: pd-balanced + # Add labels for testing. + labels: key1=value1,key2=value2 volumeBindingMode: WaitForFirstConsumer diff --git a/test/k8s-integration/main.go b/test/k8s-integration/main.go index 90f954453..092526a8b 100644 --- a/test/k8s-integration/main.go +++ b/test/k8s-integration/main.go @@ -62,6 +62,7 @@ var ( storageClassFiles = flag.String("storageclass-files", "", "name of storageclass yaml file to use for test relative to test/k8s-integration/config. This may be a comma-separated list to test multiple storage classes") snapshotClassFiles = flag.String("snapshotclass-files", "", "name of snapshotclass yaml file to use for test relative to test/k8s-integration/config. This may be a comma-separated list to test multiple storage classes") vacFiles = flag.String("volumeattributesclass-files", "", "name of volumeattributesclass yaml file to use for test relative to test/k8s-integration/config. This may be a comma-separated list to test multiple volumeattributesclasses.") + scVacFile = flag.String("storageclass-for-vac-file", "", "name of storageclass yaml file to use for test relative to test/k8s-integraion/config against the volumeattributesclass-files.") inProw = flag.Bool("run-in-prow", false, "is the test running in PROW") // Driver flags @@ -160,6 +161,10 @@ func main() { ensureVariable(storageClassFiles, true, "One of storageclass-file and migration-test must be set") } + if len(*vacFiles) != 0 { + ensureVariable(scVacFile, true, "storageclass-for-vac-file must be set when volumeattributesclass-files is set") + } + if !*bringupCluster && *platform != "windows" { ensureVariable(kubeFeatureGates, false, "kube-feature-gates set but not bringing up new cluster") } else { @@ -563,6 +568,13 @@ func handle() error { applicableVacFiles = append(applicableVacFiles, vacFile) } } + scForVac := "" + if len(*scVacFile) > 0 { + storageClassFile := strings.TrimSpace(*scVacFile) + if len(storageClassFile) != 0 { + scForVac = storageClassFile + } + } var ginkgoErrors []string var testOutputDirs []string @@ -577,8 +589,8 @@ func handle() error { } } // Run volume modify tests - if len(applicableStorageClassFiles) > 0 { - testParams.storageClassFile = applicableStorageClassFiles[0] + if len(scForVac) > 0 { + testParams.storageClassFile = scForVac for _, vacFile := range applicableVacFiles { outputDir := strings.TrimSuffix(vacFile, ".yaml") testOutputDirs = append(testOutputDirs, outputDir) @@ -587,6 +599,8 @@ func handle() error { ginkgoErrors = append(ginkgoErrors, err.Error()) } } + // Unset the VAC file for future tests + testParams.volumeAttributesClassFile = "" } // Run snapshot tests, if there are applicable files, using the first storage class. if len(applicableStorageClassFiles) > 0 { diff --git a/test/run-k8s-integration-ci.sh b/test/run-k8s-integration-ci.sh index 90a1051df..b7ab7dc82 100755 --- a/test/run-k8s-integration-ci.sh +++ b/test/run-k8s-integration-ci.sh @@ -30,6 +30,7 @@ readonly use_kubetest2=${USE_KUBETEST2:-true} readonly test_pd_labels=${TEST_PD_LABELS:-true} readonly migration_test=${MIGRATION_TEST:-false} readonly test_disk_image_snapshot=${TEST_DISK_IMAGE_SNAPSHOT:-true} +readonly test_volumeattributesclass=${TEST_VOLUMEATTRIBUTESCLASS:-true} readonly GCE_PD_TEST_FOCUS="PersistentVolumes\sGCEPD|[V|v]olume\sexpand|\[sig-storage\]\sIn-tree\sVolumes\s\[Driver:\sgcepd\]|allowedTopologies|Pod\sDisks|PersistentVolumes\sDefault" @@ -42,12 +43,12 @@ rm -rf /usr/local/go && tar -xzf go_tar.tar.gz -C /usr/local # to run go commands with this go version. export PATH=$PATH:/usr/local/go/bin && go version && rm go_tar.tar.gz -storage_classes=sc-balanced.yaml,sc-ssd.yaml,sc-xfs.yaml +storage_classes=sc-ssd.yaml,sc-xfs.yaml if [[ $test_pd_labels = true ]] ; then - storage_classes=${storage_classes},sc-standard.yaml + storage_classes=${storage_classes},sc-balanced.yaml else - storage_classes=${storage_classes},sc-standard-no-labels.yaml + storage_classes=${storage_classes},sc-balanced-no-labels.yaml fi if [[ -n $gce_region ]] ; then @@ -117,4 +118,8 @@ else base_cmd="${base_cmd} --snapshotclass-files=pd-volumesnapshotclass.yaml" fi +if [ "$test_volumeattributesclass" = true]; then + base_cmd="${base_cmd} --volumeattributesclass-files=hdb-volumeattributesclass.yaml --storageclass-for-vac-file=sc-hdb.yaml --kube-runtime-config=api/all=true" +fi + eval "$base_cmd" diff --git a/test/run-k8s-integration.sh b/test/run-k8s-integration.sh index 160cea662..c57ac0dcc 100755 --- a/test/run-k8s-integration.sh +++ b/test/run-k8s-integration.sh @@ -47,8 +47,9 @@ base_cmd="${PKGDIR}/bin/k8s-integration-test \ --run-in-prow=true --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \ --do-driver-build=${do_driver_build} --teardown-driver=${teardown_driver} \ --do-k8s-build=${do_k8s_build} --boskos-resource-type=${boskos_resource_type} \ - --storageclass-files=sc-hdb.yaml --snapshotclass-files=pd-volumesnapshotclass.yaml \ + --storageclass-files=sc-balanced.yaml --snapshotclass-files=pd-volumesnapshotclass.yaml \ --volumeattributesclass-files=hdb-volumeattributesclass.yaml \ + --storageclass-for-vac-file=sc-hdb.yaml \ --kube-runtime-config=api/all=true \ --deployment-strategy=${deployment_strategy} --test-version=${test_version} \ --num-nodes=3 --image-type=${image_type} --use-kubetest2=${use_kubetest2}"