Merge pull request #529 from jingxu97/June/windowstest

Add e2e CI tests for Windows

Author: k8s-ci-robot

Dockerfile.Windows

@@ -1,9 +1,17 @@
 ARG BASE_IMAGE
 ARG BASE_IMAGE_TAG
+FROM --platform=$BUILDPLATFORM golang:1.13.3 AS builder
+
+ARG TARGETPLATFORM
+ARG BUILDPLATFORM
+ARG STAGINGVERSION
+WORKDIR /code
+ADD . /code/
+RUN cd /code/ && GOARCH=$(echo $TARGETPLATFORM | cut -f2 -d '/') GCE_PD_CSI_STAGING_VERSION=${STAGINGVERSION} make gce-pd-driver-windows
 
 FROM mcr.microsoft.com/windows/${BASE_IMAGE}:${BASE_IMAGE_TAG}
 LABEL description="PD CSI driver"
-COPY bin/gce-pd-csi-driver.exe /gce-pd-csi-driver.exe
+COPY --from=builder /code/bin/gce-pd-csi-driver.exe /gce-pd-csi-driver.exe
 
 USER ContainerAdministrator
 ENTRYPOINT ["/gce-pd-csi-driver.exe"]

Makefile

@@ -45,15 +45,15 @@ ifndef GCE_PD_CSI_STAGING_IMAGE
 endif
 	@sh init-buildx.sh; \
 	DOCKER_CLI_EXPERIMENTAL=enabled docker buildx build --file=Dockerfile.Windows --platform=windows \
-	-t $(STAGINGIMAGE):$(STAGINGVERSION) --build-arg BASE_IMAGE=servercore --build-arg BASE_IMAGE_TAG=ltsc2019 --push .
+	-t $(STAGINGIMAGE):$(STAGINGVERSION) --build-arg BASE_IMAGE=servercore --build-arg BASE_IMAGE_TAG=ltsc2019 --build-arg STAGINGVERSION=$(STAGINGVERSION) --push .
 
 build-and-push-windows-container-1909:
 ifndef GCE_PD_CSI_STAGING_IMAGE
 	$(error "Must set environment variable GCE_PD_CSI_STAGING_IMAGE to staging image repository")
 endif
 	@sh init-buildx.sh; \
 	DOCKER_CLI_EXPERIMENTAL=enabled docker buildx build --file=Dockerfile.Windows --platform=windows \
-	-t $(STAGINGIMAGE):$(STAGINGVERSION) --build-arg BASE_IMAGE=servercore --build-arg BASE_IMAGE_TAG=1909 --push .
+	-t $(STAGINGIMAGE):$(STAGINGVERSION) --build-arg BASE_IMAGE=servercore --build-arg BASE_IMAGE_TAG=1909 --build-arg STAGINGVERSION=$(STAGINGVERSION) --push .
 
 push-container: build-container
 	gcloud docker -- push $(STAGINGIMAGE):$(STAGINGVERSION)

deploy/kubernetes/base/controller/cluster_setup.yaml

@@ -272,4 +272,3 @@ roleRef:
   kind: Role
   name: csi-gce-pd-leaderelection-role
   apiGroup: rbac.authorization.k8s.io
-

deploy/kubernetes/images/alpha/image.yaml

@@ -4,8 +4,8 @@ metadata:
   name: imagetag-gcepd-driver-alpha-win
 imageTag:
   name: gke.gcr.io/gcp-compute-persistent-disk-csi-driver-win
-  newName: gcr.io/jing-k8s-dev/gce-pd-windows-2019
-  newTag: "0.2.0"
+  newName: gcr.io/gke-release-staging/gcp-compute-persistent-disk-csi-driver-amd64-windows-1909
+  newTag: "cca3c14"
 ---
 
 apiVersion: builtin

deploy/kubernetes/overlays/alpha/kustomization.yaml

@@ -1,7 +1,6 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
-namespace:
-  gce-pd-csi-driver
+namespace: gce-pd-csi-driver
 resources:
 - ../../base/
 transformers:

test/k8s-integration/config/sc-windows.yaml

@@ -0,0 +1,9 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: csi-gcepd
+provisioner: pd.csi.storage.gke.io
+parameters:
+  type: pd-standard
+  csi.storage.k8s.io/fstype: ntfs
+volumeBindingMode: WaitForFirstConsumer

test/k8s-integration/config/test-config-template.in

@@ -7,13 +7,8 @@ SnapshotClass:
 DriverInfo:
   Name: csi-gcepd
   SupportedFsType:
-    ext2:
-    ext3:
-    ext4:
-    xfs:
-    # The following FS types supported by GCE PD but
-    # currently we do not test the CSI Driver on Windows
-    # ntfs: NTFS only available on Windows
+  {{range .SupportedFsType}}  {{ . }}:
+  {{end}}
   Capabilities:
   {{range .Capabilities}}  {{ . }}: true
   {{end}}

test/k8s-integration/driver-config.go

@@ -12,6 +12,7 @@ type driverConfig struct {
 	StorageClassFile  string
 	SnapshotClassFile string
 	Capabilities      []string
+	SupportedFsType   []string
 }
 
 const (
@@ -22,7 +23,7 @@ const (
 
 // generateDriverConfigFile loads a testdriver config template and creates a file
 // with the test-specific configuration
-func generateDriverConfigFile(pkgDir, storageClassFile, snapshotClassFile, deploymentStrat string) (string, error) {
+func generateDriverConfigFile(platform, pkgDir, storageClassFile, snapshotClassFile, deploymentStrat string) (string, error) {
 	// Load template
 	t, err := template.ParseFiles(filepath.Join(pkgDir, testConfigDir, configTemplateFile))
 	if err != nil {
@@ -50,6 +51,17 @@ func generateDriverConfigFile(pkgDir, storageClassFile, snapshotClassFile, deplo
 		"multipods",
 		"topology",
 	}
+	var fsTypes []string
+	if platform == "windows" {
+		fsTypes = []string{"ntfs"}
+	} else {
+		fsTypes = []string{
+			"ext2",
+			"ext3",
+			"ext4",
+			"xfs",
+		}
+	}
 
 	/* Unsupported Capabilities:
 	   pvcDataSource
@@ -83,6 +95,7 @@ func generateDriverConfigFile(pkgDir, storageClassFile, snapshotClassFile, deplo
 	params := driverConfig{
 		StorageClassFile:  filepath.Join(pkgDir, testConfigDir, storageClassFile),
 		SnapshotClassFile: absSnapshotClassFilePath,
+		SupportedFsType:   fsTypes,
 		Capabilities:      caps,
 	}
 

test/k8s-integration/driver.go

@@ -87,7 +87,7 @@ func deleteDriver(goPath, pkgDir, deployOverlayName string) error {
 	return nil
 }
 
-func pushImage(pkgDir, stagingImage, stagingVersion string) error {
+func pushImage(pkgDir, stagingImage, stagingVersion, platform string) error {
 	err := os.Setenv("GCE_PD_CSI_STAGING_VERSION", stagingVersion)
 	if err != nil {
 		return err
@@ -96,9 +96,16 @@ func pushImage(pkgDir, stagingImage, stagingVersion string) error {
 	if err != nil {
 		return err
 	}
-	cmd := exec.Command("make", "-C", pkgDir, "push-container",
-		fmt.Sprintf("GCE_PD_CSI_STAGING_VERSION=%s", stagingVersion),
-		fmt.Sprintf("GCE_PD_CSI_STAGING_IMAGE=%s", stagingImage))
+	var cmd *exec.Cmd
+	if platform != "windows" {
+		cmd = exec.Command("make", "-C", pkgDir, "push-container",
+			fmt.Sprintf("GCE_PD_CSI_STAGING_VERSION=%s", stagingVersion),
+			fmt.Sprintf("GCE_PD_CSI_STAGING_IMAGE=%s", stagingImage))
+	} else {
+		cmd = exec.Command("make", "-C", pkgDir, "build-and-push-windows-container-1909",
+			fmt.Sprintf("GCE_PD_CSI_STAGING_VERSION=%s", stagingVersion),
+			fmt.Sprintf("GCE_PD_CSI_STAGING_IMAGE=%s", stagingImage))
+	}
 	err = runCommand("Pushing GCP Container", cmd)
 	if err != nil {
 		return fmt.Errorf("failed to run make command: err: %v", err)

test/k8s-integration/main.go

@@ -20,6 +20,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strings"
 	"syscall"
 
 	"k8s.io/apimachinery/pkg/util/uuid"
@@ -33,6 +34,7 @@ var (
 	teardownCluster   = flag.Bool("teardown-cluster", true, "teardown the cluster after the e2e test")
 	teardownDriver    = flag.Bool("teardown-driver", true, "teardown the driver after the e2e test")
 	bringupCluster    = flag.Bool("bringup-cluster", true, "build kubernetes and bringup a cluster")
+	platform          = flag.String("platform", "linux", "platform that the tests will be run, either linux or windows")
 	gceZone           = flag.String("gce-zone", "", "zone that the gce k8s cluster is created/found in")
 	gceRegion         = flag.String("gce-region", "", "region that gke regional cluster should be created in")
 	kubeVersion       = flag.String("kube-version", "", "version of Kubernetes to download and use for the cluster")
@@ -95,7 +97,6 @@ func main() {
 	}
 
 	ensureVariable(testFocus, true, "test-focus is a required flag")
-	ensureVariable(imageType, true, "image type is a required flag. Available options include 'cos' and 'ubuntu'")
 
 	if len(*gceRegion) != 0 {
 		ensureVariable(gceZone, false, "gce-zone and gce-region cannot both be set")
@@ -111,6 +112,12 @@ func main() {
 
 	if !*bringupCluster {
 		ensureVariable(kubeFeatureGates, false, "kube-feature-gates set but not bringing up new cluster")
+	} else {
+		ensureVariable(imageType, true, "image type is a required flag. Available options include 'cos' and 'ubuntu'")
+	}
+
+	if *platform == "windows" {
+		ensureFlag(bringupCluster, false, "bringupCluster is set to false if it is for testing in windows cluster")
 	}
 
 	if *deploymentStrat == "gke" {
@@ -132,7 +139,7 @@ func main() {
 		ensureVariable(testVersion, false, "Cannot set a test version when using a local k8s dir.")
 	}
 
-	if *numNodes == -1 {
+	if *numNodes == -1 && *bringupCluster {
 		klog.Fatalf("num-nodes must be set to number of nodes in cluster")
 	}
 
@@ -157,28 +164,32 @@ func handle() error {
 
 	// If running in Prow, then acquire and set up a project through Boskos
 	if *inProw {
-		project, _ := testutils.SetupProwConfig(*boskosResourceType)
-
 		oldProject, err := exec.Command("gcloud", "config", "get-value", "project").CombinedOutput()
+		project := strings.TrimSpace(string(oldProject))
 		if err != nil {
 			return fmt.Errorf("failed to get gcloud project: %s, err: %v", oldProject, err)
 		}
-
-		err = setEnvProject(project)
-		if err != nil {
-			return fmt.Errorf("failed to set project environment to %s: %v", project, err)
-		}
-		defer func() {
-			err = setEnvProject(string(oldProject))
+		// TODO: Currently for prow tests with linux cluster, here it manually sets up a project from Boskos.
+		// For Windows, we used kubernetes_e2e.py which already set up the project and kubernetes automatically.
+		// Will update Linux in the future to use the same way as Windows test.
+		if *platform != "windows" {
+			newproject, _ := testutils.SetupProwConfig(*boskosResourceType)
+			err = setEnvProject(newproject)
 			if err != nil {
-				klog.Errorf("failed to set project environment to %s: %v", oldProject, err)
+				return fmt.Errorf("failed to set project environment to %s: %v", newproject, err)
 			}
-		}()
 
+			defer func() {
+				err = setEnvProject(string(oldProject))
+				if err != nil {
+					klog.Errorf("failed to set project environment to %s: %v", oldProject, err)
+				}
+			}()
+			project = newproject
+		}
 		if *doDriverBuild {
-			*stagingImage = fmt.Sprintf("gcr.io/%s/gcp-persistent-disk-csi-driver", project)
+			*stagingImage = fmt.Sprintf("gcr.io/%s/gcp-persistent-disk-csi-driver", strings.TrimSpace(string(project)))
 		}
-
 		if _, ok := os.LookupEnv("USER"); !ok {
 			err = os.Setenv("USER", "prow")
 			if err != nil {
@@ -189,7 +200,7 @@ func handle() error {
 
 	// Build and push the driver, if required. Defer the driver image deletion.
 	if *doDriverBuild {
-		err := pushImage(pkgDir, *stagingImage, stagingVersion)
+		err := pushImage(pkgDir, *stagingImage, stagingVersion, *platform)
 		if err != nil {
 			return fmt.Errorf("failed pushing image: %v", err)
 		}
@@ -319,7 +330,7 @@ func handle() error {
 	var testSkip string
 	switch *deploymentStrat {
 	case "gce":
-		testSkip = generateGCETestSkip(clusterVersion)
+		testSkip = generateGCETestSkip(clusterVersion, *platform)
 	case "gke":
 		testSkip = generateGKETestSkip(clusterVersion, *useGKEManagedDriver)
 	default:
@@ -328,7 +339,7 @@ func handle() error {
 
 	// Run the tests using the testDir kubernetes
 	if len(*storageClassFile) != 0 {
-		err = runCSITests(pkgDir, testDir, *testFocus, testSkip, *storageClassFile, *snapshotClassFile, cloudProviderArgs, *deploymentStrat)
+		err = runCSITests(*platform, pkgDir, testDir, *testFocus, testSkip, *storageClassFile, *snapshotClassFile, cloudProviderArgs, *deploymentStrat)
 	} else if *migrationTest {
 		err = runMigrationTests(pkgDir, testDir, *testFocus, testSkip, cloudProviderArgs)
 	} else {
@@ -342,7 +353,7 @@ func handle() error {
 	return nil
 }
 
-func generateGCETestSkip(clusterVersion string) string {
+func generateGCETestSkip(clusterVersion, platform string) string {
 	skipString := "\\[Disruptive\\]|\\[Serial\\]"
 	v := apimachineryversion.MustParseSemantic(clusterVersion)
 
@@ -352,10 +363,12 @@ func generateGCETestSkip(clusterVersion string) string {
 	if v.LessThan(apimachineryversion.MustParseSemantic("1.16.0")) {
 		skipString = skipString + "|volumeMode\\sshould\\snot\\smount\\s/\\smap\\sunused\\svolumes\\sin\\sa\\spod"
 	}
-
 	if v.LessThan(apimachineryversion.MustParseSemantic("1.17.0")) {
 		skipString = skipString + "|VolumeSnapshotDataSource"
 	}
+	if platform == "windows" {
+		skipString = skipString + "|\\[LinuxOnly\\]"
+	}
 	return skipString
 }
 
@@ -375,7 +388,6 @@ func generateGKETestSkip(clusterVersion string, use_gke_managed_driver bool) str
 		(!use_gke_managed_driver && (*curVer).lessThan(mustParseVersion("1.17.0"))) {
 		skipString = skipString + "|VolumeSnapshotDataSource"
 	}
-
 	return skipString
 }
 
@@ -396,8 +408,8 @@ func runMigrationTests(pkgDir, testDir, testFocus, testSkip string, cloudProvide
 	return runTestsWithConfig(testDir, testFocus, testSkip, "--storage.migratedPlugins=kubernetes.io/gce-pd", cloudProviderArgs)
 }
 
-func runCSITests(pkgDir, testDir, testFocus, testSkip, storageClassFile, snapshotClassFile string, cloudProviderArgs []string, deploymentStrat string) error {
-	testDriverConfigFile, err := generateDriverConfigFile(pkgDir, storageClassFile, snapshotClassFile, deploymentStrat)
+func runCSITests(platform, pkgDir, testDir, testFocus, testSkip, storageClassFile, snapshotClassFile string, cloudProviderArgs []string, deploymentStrat string) error {
+	testDriverConfigFile, err := generateDriverConfigFile(platform, pkgDir, storageClassFile, snapshotClassFile, deploymentStrat)
 	if err != nil {
 		return err
 	}
@@ -419,10 +431,12 @@ func runTestsWithConfig(testDir, testFocus, testSkip, testConfigArg string, clou
 	if ok {
 		reportArg = fmt.Sprintf("-report-dir=%s", artifactsDir)
 	}
-
-	testArgs := fmt.Sprintf("--ginkgo.focus=%s --ginkgo.skip=%s %s %s",
-		testFocus,
-		testSkip,
+	ginkgoArgs := fmt.Sprintf("--ginkgo.focus=%s --ginkgo.skip=%s", testFocus, testSkip)
+	if *platform == "windows" {
+		ginkgoArgs = ginkgoArgs + fmt.Sprintf(" --node-os-distro=%s", *platform)
+	}
+	testArgs := fmt.Sprintf("%s %s %s",
+		ginkgoArgs,
 		testConfigArg,
 		reportArg)
 
@@ -432,7 +446,6 @@ func runTestsWithConfig(testDir, testFocus, testSkip, testConfigArg string, clou
 		"--check-version-skew=false",
 		fmt.Sprintf("--test_args=%s", testArgs),
 	}
-
 	kubeTestArgs = append(kubeTestArgs, cloudProviderArgs...)
 
 	err = runCommand("Running Tests", exec.Command("kubetest", kubeTestArgs...))

test/run-windows-k8s-integration.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# Optional environment variables
+# GCE_PD_OVERLAY_NAME: which Kustomize overlay to deploy with
+# GCE_PD_DO_DRIVER_BUILD: if set, don't build the driver from source and just
+#   use the driver version from the overlay
+# GCE_PD_BOSKOS_RESOURCE_TYPE: name of the boskos resource type to reserve
+
+set -o nounset
+set -o errexit
+
+readonly GCE_PD_DO_DRIVER_BUILD=false
+readonly PKGDIR=${GOPATH}/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver
+readonly LOCAL_K8S_DIR=${GOPATH}/src/k8s.io/kubernetes
+readonly overlay_name="${GCE_PD_OVERLAY_NAME:-alpha}"
+readonly do_driver_build="${GCE_PD_DO_DRIVER_BUILD:-true}"
+readonly deployment_strategy=${DEPLOYMENT_STRATEGY:-gce}
+readonly test_version=${TEST_VERSION:-master}
+readonly gce_zone=${GCE_CLUSTER_ZONE:-us-central1-b}
+readonly teardown_driver=${GCE_CLUSTER_ZONE:-true}
+
+make -C ${PKGDIR} test-k8s-integration
+
+base_cmd="${PKGDIR}/bin/k8s-integration-test \
+            --platform=windows --local-k8s-dir=${LOCAL_K8S_DIR} --bringup-cluster=false --teardown-cluster=false --teardown-driver=${teardown_driver}\
+            --run-in-prow=true --deploy-overlay-name=${overlay_name} --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \
+            --do-driver-build=${do_driver_build} --gce-zone=${gce_zone}\
+            --storageclass-file=sc-windows.yaml --snapshotclass-file=pd-volumesnapshotclass.yaml --test-focus="External.Storage" \
+            --deployment-strategy=${deployment_strategy}"
+
+eval $base_cmd