Add e2e CI tests for Windows

This PR adds e2e CI tests for Windows.

Author: jingxu97

Dockerfile.Windows

@@ -1,9 +1,17 @@
 ARG BASE_IMAGE
 ARG BASE_IMAGE_TAG
+FROM --platform=$BUILDPLATFORM golang:1.13.3 AS builder
+
+ARG TARGETPLATFORM
+ARG BUILDPLATFORM
+ARG STAGINGVERSION
+WORKDIR /code
+ADD . /code/
+RUN cd /code/ && GOARCH=$(echo $TARGETPLATFORM | cut -f2 -d '/') GCE_PD_CSI_STAGING_VERSION=${STAGINGVERSION} make gce-pd-driver-windows
 
 FROM mcr.microsoft.com/windows/${BASE_IMAGE}:${BASE_IMAGE_TAG}
 LABEL description="PD CSI driver"
-COPY bin/gce-pd-csi-driver.exe /gce-pd-csi-driver.exe
+COPY --from=builder /code/bin/gce-pd-csi-driver.exe /gce-pd-csi-driver.exe
 
 USER ContainerAdministrator
 ENTRYPOINT ["/gce-pd-csi-driver.exe"]

Makefile

@@ -45,15 +45,15 @@ ifndef GCE_PD_CSI_STAGING_IMAGE
 endif
 	@sh init-buildx.sh; \
 	DOCKER_CLI_EXPERIMENTAL=enabled docker buildx build --file=Dockerfile.Windows --platform=windows \
-	-t $(STAGINGIMAGE):$(STAGINGVERSION) --build-arg BASE_IMAGE=servercore --build-arg BASE_IMAGE_TAG=ltsc2019 --push .
+	-t $(STAGINGIMAGE):$(STAGINGVERSION) --build-arg BASE_IMAGE=servercore --build-arg BASE_IMAGE_TAG=ltsc2019 --build-arg STAGINGVERSION=$(STAGINGVERSION) --push .
 
 build-and-push-windows-container-1909:
 ifndef GCE_PD_CSI_STAGING_IMAGE
 	$(error "Must set environment variable GCE_PD_CSI_STAGING_IMAGE to staging image repository")
 endif
 	@sh init-buildx.sh; \
 	DOCKER_CLI_EXPERIMENTAL=enabled docker buildx build --file=Dockerfile.Windows --platform=windows \
-	-t $(STAGINGIMAGE):$(STAGINGVERSION) --build-arg BASE_IMAGE=servercore --build-arg BASE_IMAGE_TAG=1909 --push .
+	-t $(STAGINGIMAGE):$(STAGINGVERSION) --build-arg BASE_IMAGE=servercore --build-arg BASE_IMAGE_TAG=1909 --build-arg STAGINGVERSION=$(STAGINGVERSION) --push .
 
 push-container: build-container
 	gcloud docker -- push $(STAGINGIMAGE):$(STAGINGVERSION)

deploy/kubernetes/base/controller/cluster_setup.yaml

@@ -272,4 +272,3 @@ roleRef:
   kind: Role
   name: csi-gce-pd-leaderelection-role
   apiGroup: rbac.authorization.k8s.io
-

deploy/kubernetes/overlays/alpha/kustomization.yaml

@@ -1,7 +1,6 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
-namespace:
-  gce-pd-csi-driver
+namespace: gce-pd-csi-driver
 resources:
 - ../../base/
 transformers:

test/k8s-integration/config/test-config-template.in

@@ -7,13 +7,8 @@ SnapshotClass:
 DriverInfo:
   Name: csi-gcepd
   SupportedFsType:
-    ext2:
-    ext3:
-    ext4:
-    xfs:
-    # The following FS types supported by GCE PD but
-    # currently we do not test the CSI Driver on Windows
-    # ntfs: NTFS only available on Windows
+  {{range .SupportedFsType}}  {{ . }}
+  {{end}}
   Capabilities:
   {{range .Capabilities}}  {{ . }}: true
   {{end}}

test/k8s-integration/config/test-driver-windows.yaml

@@ -0,0 +1,20 @@
+StorageClass:
+  FromName: false
+  FromFile: sc-windows.yaml
+SnapshotClass:
+  FromFile: pd-volumesnapshotclass.yaml 
+DriverInfo:
+  Name: csi.gcepd
+  SupportedFsType:
+    ntfs:
+  Capabilities:
+    block: true
+    controllerExpansion: false 
+    exec: true
+    multipods: true
+    persistence: true
+    pvcDataSource: false
+    snapshotDataSource: true
+  SupportedSizeRange:
+    Min: 5Gi
+    Max: 64Ti

test/k8s-integration/driver-config.go

@@ -12,6 +12,7 @@ type driverConfig struct {
 	StorageClassFile  string
 	SnapshotClassFile string
 	Capabilities      []string
+	SupportedFsType   []string
 }
 
 const (
@@ -22,7 +23,7 @@ const (
 
 // generateDriverConfigFile loads a testdriver config template and creates a file
 // with the test-specific configuration
-func generateDriverConfigFile(pkgDir, storageClassFile, snapshotClassFile, deploymentStrat string) (string, error) {
+func generateDriverConfigFile(platform, pkgDir, storageClassFile, snapshotClassFile, deploymentStrat string) (string, error) {
 	// Load template
 	t, err := template.ParseFiles(filepath.Join(pkgDir, testConfigDir, configTemplateFile))
 	if err != nil {
@@ -50,6 +51,17 @@ func generateDriverConfigFile(pkgDir, storageClassFile, snapshotClassFile, deplo
 		"multipods",
 		"topology",
 	}
+	var fsTypes []string
+	if platform == "windows" {
+		fsTypes = []string{"ntfs"}
+	} else {
+		fsTypes = []string{
+			"ext2",
+			"ext3",
+			"ext4",
+			"xfs",
+		}
+	}
 
 	/* Unsupported Capabilities:
 	   pvcDataSource
@@ -83,6 +95,7 @@ func generateDriverConfigFile(pkgDir, storageClassFile, snapshotClassFile, deplo
 	params := driverConfig{
 		StorageClassFile:  filepath.Join(pkgDir, testConfigDir, storageClassFile),
 		SnapshotClassFile: absSnapshotClassFilePath,
+		SupportedFsType:   fsTypes,
 		Capabilities:      caps,
 	}
 

test/k8s-integration/driver.go

@@ -87,7 +87,7 @@ func deleteDriver(goPath, pkgDir, deployOverlayName string) error {
 	return nil
 }
 
-func pushImage(pkgDir, stagingImage, stagingVersion string) error {
+func pushImage(pkgDir, stagingImage, stagingVersion, platform string) error {
 	err := os.Setenv("GCE_PD_CSI_STAGING_VERSION", stagingVersion)
 	if err != nil {
 		return err
@@ -96,9 +96,16 @@ func pushImage(pkgDir, stagingImage, stagingVersion string) error {
 	if err != nil {
 		return err
 	}
-	cmd := exec.Command("make", "-C", pkgDir, "push-container",
-		fmt.Sprintf("GCE_PD_CSI_STAGING_VERSION=%s", stagingVersion),
-		fmt.Sprintf("GCE_PD_CSI_STAGING_IMAGE=%s", stagingImage))
+	var cmd *exec.Cmd
+	if platform != "windows" {
+		cmd = exec.Command("make", "-C", pkgDir, "push-container",
+			fmt.Sprintf("GCE_PD_CSI_STAGING_VERSION=%s", stagingVersion),
+			fmt.Sprintf("GCE_PD_CSI_STAGING_IMAGE=%s", stagingImage))
+	} else {
+		cmd = exec.Command("make", "-C", pkgDir, "build-and-push-windows-container-1909",
+			fmt.Sprintf("GCE_PD_CSI_STAGING_VERSION=%s", stagingVersion),
+			fmt.Sprintf("GCE_PD_CSI_STAGING_IMAGE=%s", stagingImage))
+	}
 	err = runCommand("Pushing GCP Container", cmd)
 	if err != nil {
 		return fmt.Errorf("failed to run make command: err: %v", err)

test/k8s-integration/main.go

@@ -20,6 +20,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strings"
 	"syscall"
 
 	"k8s.io/apimachinery/pkg/util/uuid"
@@ -33,6 +34,7 @@ var (
 	teardownCluster   = flag.Bool("teardown-cluster", true, "teardown the cluster after the e2e test")
 	teardownDriver    = flag.Bool("teardown-driver", true, "teardown the driver after the e2e test")
 	bringupCluster    = flag.Bool("bringup-cluster", true, "build kubernetes and bringup a cluster")
+	platform          = flag.String("platform", "linux", "platform that the tests will be run, either linux or windows")
 	gceZone           = flag.String("gce-zone", "", "zone that the gce k8s cluster is created/found in")
 	gceRegion         = flag.String("gce-region", "", "region that gke regional cluster should be created in")
 	kubeVersion       = flag.String("kube-version", "", "version of Kubernetes to download and use for the cluster")
@@ -157,26 +159,27 @@ func handle() error {
 
 	// If running in Prow, then acquire and set up a project through Boskos
 	if *inProw {
-		project, _ := testutils.SetupProwConfig(*boskosResourceType)
-
 		oldProject, err := exec.Command("gcloud", "config", "get-value", "project").CombinedOutput()
+		*stagingImage = fmt.Sprintf("gcr.io/%s/gcp-persistent-disk-csi-driver", strings.TrimSpace(string(oldProject)))
 		if err != nil {
 			return fmt.Errorf("failed to get gcloud project: %s, err: %v", oldProject, err)
 		}
-
-		err = setEnvProject(project)
-		if err != nil {
-			return fmt.Errorf("failed to set project environment to %s: %v", project, err)
-		}
-		defer func() {
-			err = setEnvProject(string(oldProject))
+		if *platform != "windows" {
+			project, _ := testutils.SetupProwConfig(*boskosResourceType)
+			err = setEnvProject(project)
 			if err != nil {
-				klog.Errorf("failed to set project environment to %s: %v", oldProject, err)
+				return fmt.Errorf("failed to set project environment to %s: %v", project, err)
+			}
+			if *doDriverBuild {
+				*stagingImage = fmt.Sprintf("gcr.io/%s/gcp-persistent-disk-csi-driver", strings.TrimSpace(string(project)))
 			}
-		}()
 
-		if *doDriverBuild {
-			*stagingImage = fmt.Sprintf("gcr.io/%s/gcp-persistent-disk-csi-driver", project)
+			defer func() {
+				err = setEnvProject(string(oldProject))
+				if err != nil {
+					klog.Errorf("failed to set project environment to %s: %v", oldProject, err)
+				}
+			}()
 		}
 
 		if _, ok := os.LookupEnv("USER"); !ok {
@@ -189,7 +192,7 @@ func handle() error {
 
 	// Build and push the driver, if required. Defer the driver image deletion.
 	if *doDriverBuild {
-		err := pushImage(pkgDir, *stagingImage, stagingVersion)
+		err := pushImage(pkgDir, *stagingImage, stagingVersion, *platform)
 		if err != nil {
 			return fmt.Errorf("failed pushing image: %v", err)
 		}
@@ -319,7 +322,7 @@ func handle() error {
 	var testSkip string
 	switch *deploymentStrat {
 	case "gce":
-		testSkip = generateGCETestSkip(clusterVersion)
+		testSkip = generateGCETestSkip(clusterVersion, *platform)
 	case "gke":
 		testSkip = generateGKETestSkip(clusterVersion, *useGKEManagedDriver)
 	default:
@@ -328,7 +331,7 @@ func handle() error {
 
 	// Run the tests using the testDir kubernetes
 	if len(*storageClassFile) != 0 {
-		err = runCSITests(pkgDir, testDir, *testFocus, testSkip, *storageClassFile, *snapshotClassFile, cloudProviderArgs, *deploymentStrat)
+		err = runCSITests(*platform, pkgDir, testDir, *testFocus, testSkip, *storageClassFile, *snapshotClassFile, cloudProviderArgs, *deploymentStrat)
 	} else if *migrationTest {
 		err = runMigrationTests(pkgDir, testDir, *testFocus, testSkip, cloudProviderArgs)
 	} else {
@@ -342,7 +345,7 @@ func handle() error {
 	return nil
 }
 
-func generateGCETestSkip(clusterVersion string) string {
+func generateGCETestSkip(clusterVersion, platform string) string {
 	skipString := "\\[Disruptive\\]|\\[Serial\\]"
 	v := apimachineryversion.MustParseSemantic(clusterVersion)
 
@@ -352,10 +355,12 @@ func generateGCETestSkip(clusterVersion string) string {
 	if v.LessThan(apimachineryversion.MustParseSemantic("1.16.0")) {
 		skipString = skipString + "|volumeMode\\sshould\\snot\\smount\\s/\\smap\\sunused\\svolumes\\sin\\sa\\spod"
 	}
-
 	if v.LessThan(apimachineryversion.MustParseSemantic("1.17.0")) {
 		skipString = skipString + "|VolumeSnapshotDataSource"
 	}
+	if platform == "windows" {
+		skipString = skipString + "|\\[LinuxOnly\\]|\\[Feature:.+\\][Ephemeral]"
+	}
 	return skipString
 }
 
@@ -375,7 +380,6 @@ func generateGKETestSkip(clusterVersion string, use_gke_managed_driver bool) str
 		(!use_gke_managed_driver && (*curVer).lessThan(mustParseVersion("1.17.0"))) {
 		skipString = skipString + "|VolumeSnapshotDataSource"
 	}
-
 	return skipString
 }
 
@@ -396,8 +400,8 @@ func runMigrationTests(pkgDir, testDir, testFocus, testSkip string, cloudProvide
 	return runTestsWithConfig(testDir, testFocus, testSkip, "--storage.migratedPlugins=kubernetes.io/gce-pd", cloudProviderArgs)
 }
 
-func runCSITests(pkgDir, testDir, testFocus, testSkip, storageClassFile, snapshotClassFile string, cloudProviderArgs []string, deploymentStrat string) error {
-	testDriverConfigFile, err := generateDriverConfigFile(pkgDir, storageClassFile, snapshotClassFile, deploymentStrat)
+func runCSITests(platform, pkgDir, testDir, testFocus, testSkip, storageClassFile, snapshotClassFile string, cloudProviderArgs []string, deploymentStrat string) error {
+	testDriverConfigFile, err := generateDriverConfigFile(platform, pkgDir, storageClassFile, snapshotClassFile, deploymentStrat)
 	if err != nil {
 		return err
 	}
@@ -419,10 +423,12 @@ func runTestsWithConfig(testDir, testFocus, testSkip, testConfigArg string, clou
 	if ok {
 		reportArg = fmt.Sprintf("-report-dir=%s", artifactsDir)
 	}
-
-	testArgs := fmt.Sprintf("--ginkgo.focus=%s --ginkgo.skip=%s %s %s",
-		testFocus,
-		testSkip,
+	ginkgoArgs := fmt.Sprintf("--ginkgo.focus=%s --ginkgo.skip=%s", testFocus, testSkip)
+	if *platform == "windows" {
+		ginkgoArgs = ginkgoArgs + fmt.Sprintf(" --node-os-distro=%s", *platform)
+	}
+	testArgs := fmt.Sprintf("%s %s %s",
+		ginkgoArgs,
 		testConfigArg,
 		reportArg)
 
@@ -432,7 +438,6 @@ func runTestsWithConfig(testDir, testFocus, testSkip, testConfigArg string, clou
 		"--check-version-skew=false",
 		fmt.Sprintf("--test_args=%s", testArgs),
 	}
-
 	kubeTestArgs = append(kubeTestArgs, cloudProviderArgs...)
 
 	err = runCommand("Running Tests", exec.Command("kubetest", kubeTestArgs...))

test/run-windows-k8s-integration.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Optional environment variables
+# GCE_PD_OVERLAY_NAME: which Kustomize overlay to deploy with
+# GCE_PD_DO_DRIVER_BUILD: if set, don't build the driver from source and just
+#   use the driver version from the overlay
+# GCE_PD_BOSKOS_RESOURCE_TYPE: name of the boskos resource type to reserve
+
+set -o nounset
+set -o errexit
+
+readonly GCE_PD_DO_DRIVER_BUILD=false
+readonly PKGDIR=${GOPATH}/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver
+readonly LOCAL_K8S_DIR=${GOPATH}/src/k8s.io/kubernetes
+readonly overlay_name="${GCE_PD_OVERLAY_NAME:-alpha}"
+readonly do_driver_build="${GCE_PD_DO_DRIVER_BUILD:-true}"
+readonly deployment_strategy=${DEPLOYMENT_STRATEGY:-gce}
+readonly test_version=${TEST_VERSION:-master}
+readonly gce_zone=${GCE_CLUSTER_ZONE:-us-central1-b}
+#readonly gce_region=${GCE_CLUSTER_REGION:-}
+readonly image_type=${IMAGE_TYPE:-cos}
+
+export GCE_PD_VERBOSITY=9
+
+make -C ${PKGDIR} test-k8s-integration
+
+base_cmd="${PKGDIR}/bin/k8s-integration-test \
+            --platform=windows --local-k8s-dir=${LOCAL_K8S_DIR} --bringup-cluster=false --teardown-cluster=false --teardown-driver=true\
+            --run-in-prow=true --deploy-overlay-name=${overlay_name} --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \
+            --do-driver-build=${do_driver_build} --gce-zone=${gce_zone}\
+            --storageclass-file=sc-windows.yaml --snapshotclass-file=pd-volumesnapshotclass.yaml --test-focus="External.Storage.*ntfs" \
+            --deployment-strategy=${deployment_strategy} --num-nodes=4 \
+            --image-type=${image_type}"
+
+eval $base_cmd

test/run-windows-script.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+set -o nounset
+set -o pipefail
+set -o xtrace
+
+# When running in prow, the working directory is the root of the test-infra
+# repository.
+
+# Taint the Linux nodes to prevent the test workloads from landing on them.
+# TODO: remove this once the issue is resolved:
+# https://github.com/kubernetes/kubernetes/issues/69892
+#LINUX_NODES=$(kubectl get nodes -l beta.kubernetes.io/os=linux -o name)
+#LINUX_NODE_COUNT=$(echo ${LINUX_NODES} | wc -w)
+#for node in $LINUX_NODES; do
+#  kubectl taint node $node node-under-test=false:NoSchedule
+#done
+
+# Untaint the windows nodes to allow test workloads without tolerations to be
+# scheduled onto them.
+WINDOWS_NODES=$(kubectl get nodes -l beta.kubernetes.io/os=windows -o name)
+for node in $WINDOWS_NODES; do
+  kubectl taint node $node node.kubernetes.io/os:NoSchedule-
+done
+
+# Download and set the list of test image repositories to use.
+curl \
+  https://raw.githubusercontent.com/kubernetes-sigs/windows-testing/master/images/image-repo-list \
+  -o repo-list.yaml
+export KUBE_TEST_REPO_LIST=repo-list.yaml
+export KUBECONFIG=~/.kube/config
+export GCE_PD_SA_DIR="${GCE_PD_SA_DIR:-${E2E_GOOGLE_APPLICATION_CREDENTIALS}}"
+export GCE_PD_DRIVER_VERSION=alpha
+export PROJECT=$(gcloud config get-value project)
+
+$GOPATH/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/deploy/kubernetes/deploy-driver.sh
+cp $GOPATH/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/k8s-integration/config/sc-windows.yaml .
+cp $GOPATH/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/k8s-integration/config/pd-volumesnapshotclass.yaml .
+
+# When using customized test command (which we are now), report-dir is not set
+# by default, so set it here.
+# The test framework will not proceed to run tests unless all nodes are ready
+# AND schedulable. Allow not-ready nodes since we make Linux nodes
+# unschedulable.
+# Do not set --disable-log-dump because upstream cannot handle dumping logs
+# from windows nodes yet.
+./hack/ginkgo-e2e.sh $@ --storage.testdriver=$GOPATH/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/k8s-integration/config/test-driver-windows.yaml --report-dir=${ARTIFACTS} --allowed-not-ready-nodes=${LINUX_NODE_COUNT}
+