From 12707e11dda94f6c52f816339486cb6abf6a5c4d Mon Sep 17 00:00:00 2001 From: Hanchi Zhang Date: Tue, 18 Mar 2025 15:02:47 +0000 Subject: [PATCH] Make integration tests support windows 2022. --- Makefile | 19 +++++++++++++------ test/k8s-integration/cluster.go | 15 ++++++++++++++- test/k8s-integration/driver.go | 3 +-- test/k8s-integration/main.go | 2 +- test/run-windows-k8s-integration.sh | 9 ++++++++- 5 files changed, 37 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index a827ca7e9..8a375b916 100644 --- a/Makefile +++ b/Makefile @@ -24,6 +24,7 @@ DRIVERWINDOWSBINARY=${DRIVERBINARY}.exe DOCKER=DOCKER_CLI_EXPERIMENTAL=enabled docker BASE_IMAGE_LTSC2019=mcr.microsoft.com/windows/servercore:ltsc2019 +BASE_IMAGE_LTSC2022=mcr.microsoft.com/windows/servercore:ltsc2022 # Both arrays MUST be index aligned. WINDOWS_IMAGE_TAGS=ltsc2019 @@ -60,14 +61,20 @@ build-and-push-windows-container-ltsc2019: require-GCE_PD_CSI_STAGING_IMAGE init --build-arg BASE_IMAGE=$(BASE_IMAGE_LTSC2019) \ --build-arg STAGINGVERSION=$(STAGINGVERSION) --push --provenance=false . -build-and-push-multi-arch: build-and-push-container-linux-amd64 build-and-push-container-linux-arm64 build-and-push-windows-container-ltsc2019 - $(DOCKER) manifest create $(STAGINGIMAGE):$(STAGINGVERSION) $(STAGINGIMAGE):$(STAGINGVERSION)_linux_amd64 $(STAGINGIMAGE):$(STAGINGVERSION)_linux_arm64 $(STAGINGIMAGE):$(STAGINGVERSION)_ltsc2019 - STAGINGIMAGE="$(STAGINGIMAGE)" STAGINGVERSION="$(STAGINGVERSION)" WINDOWS_IMAGE_TAGS="$(WINDOWS_IMAGE_TAGS)" WINDOWS_BASE_IMAGES="$(WINDOWS_BASE_IMAGES)" ./manifest_osversion.sh +build-and-push-windows-container-ltsc2022: require-GCE_PD_CSI_STAGING_IMAGE init-buildx + $(DOCKER) buildx build --file=Dockerfile.Windows --platform=windows/amd64 \ + -t $(STAGINGIMAGE):$(STAGINGVERSION)_ltsc2022 \ + --build-arg BASE_IMAGE=$(BASE_IMAGE_LTSC2022) \ + --build-arg STAGINGVERSION=$(STAGINGVERSION) --push --provenance=false . + +build-and-push-multi-arch: build-and-push-container-linux-amd64 build-and-push-container-linux-arm64 build-and-push-windows-container-ltsc2019 build-and-push-windows-container-ltsc2022 + $(DOCKER) manifest create $(STAGINGIMAGE):$(STAGINGVERSION) $(STAGINGIMAGE):$(STAGINGVERSION)_linux_amd64 $(STAGINGIMAGE):$(STAGINGVERSION)_linux_arm64 $(STAGINGIMAGE):$(STAGINGVERSION)_ltsc2019 $(STAGINGIMAGE):$(STAGINGVERSION)_ltsc2022 + STAGINGIMAGE="$(STAGINGIMAGE)" STAGINGVERSION="$(STAGINGVERSION)" WINDOWS_IMAGE_TAGS="ltsc2019, ltsc2022" WINDOWS_BASE_IMAGES="$(BASE_IMAGE_LTSC2019), $(BASE_IMAGE_LTSC2022)" ./manifest_osversion.sh $(DOCKER) manifest push -p $(STAGINGIMAGE):$(STAGINGVERSION) -build-and-push-multi-arch-debug: build-and-push-container-linux-debug build-and-push-windows-container-ltsc2019 - $(DOCKER) manifest create $(STAGINGIMAGE):$(STAGINGVERSION) $(STAGINGIMAGE):$(STAGINGVERSION)_linux $(STAGINGIMAGE):$(STAGINGVERSION)_ltsc2019 - STAGINGIMAGE="$(STAGINGIMAGE)" STAGINGVERSION="$(STAGINGVERSION)" WINDOWS_IMAGE_TAGS="ltsc2019" WINDOWS_BASE_IMAGES="$(BASE_IMAGE_LTSC2019)" ./manifest_osversion.sh +build-and-push-multi-arch-debug: build-and-push-container-linux-debug build-and-push-windows-container-ltsc2019 build-and-push-windows-container-ltsc2022 + $(DOCKER) manifest create $(STAGINGIMAGE):$(STAGINGVERSION) $(STAGINGIMAGE):$(STAGINGVERSION)_linux $(STAGINGIMAGE):$(STAGINGVERSION)_ltsc2019 $(STAGINGIMAGE):$(STAGINGVERSION)_ltsc2022 + STAGINGIMAGE="$(STAGINGIMAGE)" STAGINGVERSION="$(STAGINGVERSION)" WINDOWS_IMAGE_TAGS="ltsc2019, ltsc2022" WINDOWS_BASE_IMAGES="$(BASE_IMAGE_LTSC2019), $(BASE_IMAGE_LTSC2022)" ./manifest_osversion.sh $(DOCKER) manifest push -p $(STAGINGIMAGE):$(STAGINGVERSION) push-container: build-container diff --git a/test/k8s-integration/cluster.go b/test/k8s-integration/cluster.go index 88343c530..b411dd003 100644 --- a/test/k8s-integration/cluster.go +++ b/test/k8s-integration/cluster.go @@ -155,6 +155,9 @@ func setImageTypeEnvs(imageType string) error { case "cos": case "cos_containerd": case "gci": // GCI/COS is default type and does not need env vars set + case "win2019", "win2022": + // These are handled by the WINDOWS_NODE_OS_DISTRIBUTION env var and + // ingested in https://github.com/kubernetes/kubernetes/blob/ded2956c832502da8a0678f5392c24af1cc9dfc0/cluster/gce/util.sh#L112. case "ubuntu", "ubuntu_containerd": return errors.New("setting environment vars for bringing up *ubuntu* cluster on GCE is unimplemented") /* TODO(dyzz) figure out how to bring up a Ubuntu cluster on GCE. The below doesn't work. @@ -201,7 +204,17 @@ func clusterUpGKE(gceZone, gceRegion string, numNodes int, numWindowsNodes int, var cmd *exec.Cmd cmdParams := []string{"container", "clusters", "create", *gkeTestClusterName, locationArg, locationVal, "--num-nodes", strconv.Itoa(numNodes), - "--quiet", "--machine-type", "n1-standard-2", "--image-type", imageType, "--no-enable-autoupgrade"} + "--quiet", "--machine-type", "n1-standard-2", "--no-enable-autoupgrade"} + if imageType == "win2019" || imageType == "win2022" { + cmdParams = append(cmdParams, "--image-type", "WINDOWS_LTSC_CONTAINERD") + if imageType == "win2019" { + cmdParams = append(cmdParams, "--windows-os-version", "ltsc2019") + } else { + cmdParams = append(cmdParams, "--windows-os-version", "ltsc2022") + } + } else { + cmdParams = append(cmdParams, "--image-type", imageType) + } if isVariableSet(gkeClusterVer) { cmdParams = append(cmdParams, "--cluster-version", *gkeClusterVer) } else { diff --git a/test/k8s-integration/driver.go b/test/k8s-integration/driver.go index 4b9032348..6f51df9dc 100644 --- a/test/k8s-integration/driver.go +++ b/test/k8s-integration/driver.go @@ -126,7 +126,7 @@ func deleteDriver(testParams *testParameters, deployOverlayName string) error { return nil } -func pushImage(pkgDir, stagingImage, stagingVersion, platform string) error { +func pushImage(pkgDir, stagingImage, stagingVersion, platform, imageType string) error { err := os.Setenv("GCE_PD_CSI_STAGING_VERSION", stagingVersion) if err != nil { return err @@ -138,7 +138,6 @@ func pushImage(pkgDir, stagingImage, stagingVersion, platform string) error { var cmd *exec.Cmd if platform == "windows" { - // build multi-arch image which can work for both Linux and Windows cmd = exec.Command("make", "-C", pkgDir, "build-and-push-multi-arch", fmt.Sprintf("GCE_PD_CSI_STAGING_VERSION=%s", stagingVersion), fmt.Sprintf("GCE_PD_CSI_STAGING_IMAGE=%s", stagingImage)) diff --git a/test/k8s-integration/main.go b/test/k8s-integration/main.go index 9edb0ef52..a3d826684 100644 --- a/test/k8s-integration/main.go +++ b/test/k8s-integration/main.go @@ -300,7 +300,7 @@ func handle() error { // Build and push the driver, if required. Defer the driver image deletion. if *doDriverBuild { klog.Infof("Building GCE PD CSI Driver") - err := pushImage(testParams.pkgDir, *stagingImage, testParams.stagingVersion, testParams.platform) + err := pushImage(testParams.pkgDir, *stagingImage, testParams.stagingVersion, testParams.platform, *imageType) if err != nil { return fmt.Errorf("failed pushing image: %v", err.Error()) } diff --git a/test/run-windows-k8s-integration.sh b/test/run-windows-k8s-integration.sh index e10654cdc..01f7153d4 100755 --- a/test/run-windows-k8s-integration.sh +++ b/test/run-windows-k8s-integration.sh @@ -21,6 +21,7 @@ readonly test_version=${TEST_VERSION:-master} readonly gce_zone=${GCE_CLUSTER_ZONE:-us-central1-b} readonly use_kubetest2=${USE_KUBETEST2:-true} readonly num_windows_nodes=${NUM_WINDOWS_NODES:-3} +readonly windows_distribution=${WINDOWS_NODE_OS_DISTRIBUTION:-win2019} # build platforms for `make quick-release` export KUBE_BUILD_PLATFORMS=${KUBE_BUILD_PLATFORMS:-"linux/amd64 windows/amd64"} @@ -35,6 +36,11 @@ if [ "$use_kubetest2" = true ]; then go install sigs.k8s.io/kubetest2/kubetest2-tester-ginkgo@${kt2_version} fi +if [ "$windows_distribution" != "win2019" -a "$windows_distribution" != "win2022"]; then + print "Invalid windows distribution $windows_distribution provided. Exiting." + exit 1 +fi + ${PKGDIR}/bin/k8s-integration-test \ --run-in-prow=true \ --service-account-file="${E2E_GOOGLE_APPLICATION_CREDENTIALS}" \ @@ -54,4 +60,5 @@ ${PKGDIR}/bin/k8s-integration-test \ --storageclass-files=sc-windows.yaml \ --snapshotclass-files=pd-volumesnapshotclass.yaml \ --test-focus='External.Storage' \ - --use-kubetest2="${use_kubetest2}" + --use-kubetest2="${use_kubetest2}" \ + --image-type="${windows_distribution}"