Add e2e CI tests for Windows

This PR adds e2e CI tests for Windows.

Author: jingxu97

Dockerfile.Windows

@@ -1,9 +1,17 @@
 ARG BASE_IMAGE
 ARG BASE_IMAGE_TAG
+FROM --platform=$BUILDPLATFORM golang:1.13.3 AS builder
+
+ARG TARGETPLATFORM
+ARG BUILDPLATFORM
+WORKDIR /code
+ADD . /code/
+
+RUN cd /code/ && GOARCH=$(echo $TARGETPLATFORM | cut -f2 -d '/') GCE_PD_CSI_STAGING_VERSION=dev make gce-pd-driver-windows
 
 FROM mcr.microsoft.com/windows/${BASE_IMAGE}:${BASE_IMAGE_TAG}
 LABEL description="PD CSI driver"
-COPY bin/gce-pd-csi-driver.exe /gce-pd-csi-driver.exe
+COPY --from=builder /code/bin/gce-pd-csi-driver.exe /gce-pd-csi-driver.exe
 
 USER ContainerAdministrator
 ENTRYPOINT ["/gce-pd-csi-driver.exe"]

deploy/kubernetes/base/controller/cluster_setup.yaml

@@ -1,3 +1,10 @@
+kind: Namespace
+apiVersion: v1
+metadata:
+  name: gce-pd-csi-driver
+
+---
+
 ##### Node Service Account, Roles, RoleBindings
 apiVersion: v1
 kind: ServiceAccount
@@ -272,4 +279,3 @@ roleRef:
   kind: Role
   name: csi-gce-pd-leaderelection-role
   apiGroup: rbac.authorization.k8s.io
-

deploy/kubernetes/overlays/alpha/kustomization.yaml

@@ -1,7 +1,6 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
-namespace:
-  gce-pd-csi-driver
+namespace: gce-pd-csi-driver
 resources:
 - ../../base/
 transformers:

test/k8s-integration/config/test-driver-windows.yaml

@@ -0,0 +1,20 @@
+StorageClass:
+  FromName: false
+  FromFile: sc-windows.yaml
+SnapshotClass:
+  FromFile: pd-volumesnapshotclass.yaml 
+DriverInfo:
+  Name: csi.gcepd
+  SupportedFsType:
+    ntfs:
+  Capabilities:
+    block: true
+    controllerExpansion: false 
+    exec: true
+    multipods: true
+    persistence: true
+    pvcDataSource: false
+    snapshotDataSource: true
+  SupportedSizeRange:
+    Min: 5Gi
+    Max: 64Ti

test/k8s-integration/driver-config.go

@@ -15,16 +15,21 @@ type driverConfig struct {
 }
 
 const (
-	testConfigDir      = "test/k8s-integration/config"
-	configTemplateFile = "test-config-template.in"
-	configFile         = "test-config.yaml"
+	testConfigDir             = "test/k8s-integration/config"
+	configTemplateFile        = "test-config-template.in"
+	configTemplateWindowsFile = "test-config-template-windows.in"
+	configFile                = "test-config.yaml"
 )
 
 // generateDriverConfigFile loads a testdriver config template and creates a file
 // with the test-specific configuration
-func generateDriverConfigFile(pkgDir, storageClassFile, snapshotClassFile, deploymentStrat string) (string, error) {
+func generateDriverConfigFile(platform, pkgDir, storageClassFile, snapshotClassFile, deploymentStrat string) (string, error) {
 	// Load template
-	t, err := template.ParseFiles(filepath.Join(pkgDir, testConfigDir, configTemplateFile))
+	configFile := configTemplateFile
+	if platform == "windows" {
+		configFile = configTemplateWindowsFile
+	}
+	t, err := template.ParseFiles(filepath.Join(pkgDir, testConfigDir, configFile))
 	if err != nil {
 		return "", err
 	}

test/k8s-integration/driver.go

@@ -87,7 +87,7 @@ func deleteDriver(goPath, pkgDir, deployOverlayName string) error {
 	return nil
 }
 
-func pushImage(pkgDir, stagingImage, stagingVersion string) error {
+func pushImage(pkgDir, stagingImage, stagingVersion, platform string) error {
 	err := os.Setenv("GCE_PD_CSI_STAGING_VERSION", stagingVersion)
 	if err != nil {
 		return err
@@ -96,9 +96,16 @@ func pushImage(pkgDir, stagingImage, stagingVersion string) error {
 	if err != nil {
 		return err
 	}
-	cmd := exec.Command("make", "-C", pkgDir, "push-container",
-		fmt.Sprintf("GCE_PD_CSI_STAGING_VERSION=%s", stagingVersion),
-		fmt.Sprintf("GCE_PD_CSI_STAGING_IMAGE=%s", stagingImage))
+	var cmd *exec.Cmd
+	if platform != "windows" {
+		cmd = exec.Command("make", "-C", pkgDir, "push-container",
+			fmt.Sprintf("GCE_PD_CSI_STAGING_VERSION=%s", stagingVersion),
+			fmt.Sprintf("GCE_PD_CSI_STAGING_IMAGE=%s", stagingImage))
+	} else {
+		cmd = exec.Command("make", "-C", pkgDir, "build-and-push-windows-container-1909",
+			fmt.Sprintf("GCE_PD_CSI_STAGING_VERSION=%s", stagingVersion),
+			fmt.Sprintf("GCE_PD_CSI_STAGING_IMAGE=%s", stagingImage))
+	}
 	err = runCommand("Pushing GCP Container", cmd)
 	if err != nil {
 		return fmt.Errorf("failed to run make command: err: %v", err)

test/k8s-integration/main.go

@@ -20,6 +20,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strings"
 	"syscall"
 
 	testutils "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/e2e/utils"
@@ -33,6 +34,7 @@ var (
 	teardownCluster  = flag.Bool("teardown-cluster", true, "teardown the cluster after the e2e test")
 	teardownDriver   = flag.Bool("teardown-driver", true, "teardown the driver after the e2e test")
 	bringupCluster   = flag.Bool("bringup-cluster", true, "build kubernetes and bringup a cluster")
+	platform         = flag.String("platform", "linux", "platform that the tests will be run, either linux or windows")
 	gceZone          = flag.String("gce-zone", "", "zone that the gce k8s cluster is created/found in")
 	gceRegion        = flag.String("gce-region", "", "region that gke regional cluster should be created in")
 	kubeVersion      = flag.String("kube-version", "", "version of Kubernetes to download and use for the cluster")
@@ -74,7 +76,6 @@ func init() {
 
 func main() {
 	flag.Parse()
-
 	if !*inProw {
 		ensureVariable(stagingImage, true, "staging-image is a required flag, please specify the name of image to stage to")
 	}
@@ -143,26 +144,26 @@ func handle() error {
 
 	// If running in Prow, then acquire and set up a project through Boskos
 	if *inProw {
-		project, _ := testutils.SetupProwConfig(*boskosResourceType)
-
 		oldProject, err := exec.Command("gcloud", "config", "get-value", "project").CombinedOutput()
 		if err != nil {
 			return fmt.Errorf("failed to get gcloud project: %s, err: %v", oldProject, err)
 		}
-
-		err = setEnvProject(project)
-		if err != nil {
-			return fmt.Errorf("failed to set project environment to %s: %v", project, err)
-		}
-		defer func() {
-			err = setEnvProject(string(oldProject))
+		if *platform != "windows" {
+			project, _ := testutils.SetupProwConfig(*boskosResourceType)
+			err = setEnvProject(project)
 			if err != nil {
-				klog.Errorf("failed to set project environment to %s: %v", oldProject, err)
+				return fmt.Errorf("failed to set project environment to %s: %v", project, err)
 			}
-		}()
-
+			defer func() {
+				err = setEnvProject(string(oldProject))
+				if err != nil {
+					klog.Errorf("failed to set project environment to %s: %v", oldProject, err)
+				}
+			}()
+		}
 		if *doDriverBuild {
-			*stagingImage = fmt.Sprintf("gcr.io/%s/gcp-persistent-disk-csi-driver", project)
+			*stagingImage = fmt.Sprintf("gcr.io/%s/gcp-persistent-disk-csi-driver", strings.TrimSpace(string(oldProject)))
+			fmt.Printf("stagingImage %s \n", *stagingImage)
 		}
 
 		if _, ok := os.LookupEnv("USER"); !ok {
@@ -175,7 +176,7 @@ func handle() error {
 
 	// Build and push the driver, if required. Defer the driver image deletion.
 	if *doDriverBuild {
-		err := pushImage(pkgDir, *stagingImage, stagingVersion)
+		err := pushImage(pkgDir, *stagingImage, stagingVersion, *platform)
 		if err != nil {
 			return fmt.Errorf("failed pushing image: %v", err)
 		}
@@ -295,15 +296,15 @@ func handle() error {
 		}
 	}
 
-	normalizedVersion, err := getNormalizedVersion(*kubeVersion, *gkeClusterVer)
+	normalizedVersion, err := getNormalizedVersion("master", *gkeClusterVer)
 	if err != nil {
 		return fmt.Errorf("failed to get cluster minor version: %v", err)
 	}
 
-	testSkip := generateTestSkip(normalizedVersion)
+	testSkip := generateTestSkip(normalizedVersion, *platform)
 	// Run the tests using the testDir kubernetes
 	if len(*storageClassFile) != 0 {
-		err = runCSITests(pkgDir, testDir, *testFocus, testSkip, *storageClassFile, *snapshotClassFile, cloudProviderArgs, *deploymentStrat)
+		err = runCSITests(*platform, pkgDir, testDir, *testFocus, testSkip, *storageClassFile, *snapshotClassFile, cloudProviderArgs, *deploymentStrat)
 	} else if *migrationTest {
 		err = runMigrationTests(pkgDir, testDir, *testFocus, testSkip, cloudProviderArgs)
 	} else {
@@ -317,7 +318,7 @@ func handle() error {
 	return nil
 }
 
-func generateTestSkip(normalizedVersion string) string {
+func generateTestSkip(normalizedVersion, platform string) string {
 	skipString := "\\[Disruptive\\]|\\[Serial\\]"
 	switch normalizedVersion {
 	// Fall-through versioning since all test cases we want to skip in 1.15
@@ -341,6 +342,9 @@ func generateTestSkip(normalizedVersion string) string {
 	case "master":
 	default:
 	}
+	if platform == "windows" {
+		skipString = skipString + "|\\[Slow\\]|\\[LinuxOnly\\]|\\[Feature:.+\\][Ephemeral]"
+	}
 	return skipString
 }
 
@@ -361,8 +365,8 @@ func runMigrationTests(pkgDir, testDir, testFocus, testSkip string, cloudProvide
 	return runTestsWithConfig(testDir, testFocus, testSkip, "--storage.migratedPlugins=kubernetes.io/gce-pd", cloudProviderArgs)
 }
 
-func runCSITests(pkgDir, testDir, testFocus, testSkip, storageClassFile, snapshotClassFile string, cloudProviderArgs []string, deploymentStrat string) error {
-	testDriverConfigFile, err := generateDriverConfigFile(pkgDir, storageClassFile, snapshotClassFile, deploymentStrat)
+func runCSITests(platform, pkgDir, testDir, testFocus, testSkip, storageClassFile, snapshotClassFile string, cloudProviderArgs []string, deploymentStrat string) error {
+	testDriverConfigFile, err := generateDriverConfigFile(platform, pkgDir, storageClassFile, snapshotClassFile, deploymentStrat)
 	if err != nil {
 		return err
 	}
@@ -385,9 +389,10 @@ func runTestsWithConfig(testDir, testFocus, testSkip, testConfigArg string, clou
 		reportArg = fmt.Sprintf("-report-dir=%s", artifactsDir)
 	}
 
-	testArgs := fmt.Sprintf("--ginkgo.focus=%s --ginkgo.skip=%s %s %s",
+	testArgs := fmt.Sprintf("--ginkgo.focus=%s --ginkgo.skip=%s --node-os-distro=%s %s %s",
 		testFocus,
 		testSkip,
+		*platform,
 		testConfigArg,
 		reportArg)
 
@@ -397,7 +402,6 @@ func runTestsWithConfig(testDir, testFocus, testSkip, testConfigArg string, clou
 		"--check-version-skew=false",
 		fmt.Sprintf("--test_args=%s", testArgs),
 	}
-
 	kubeTestArgs = append(kubeTestArgs, cloudProviderArgs...)
 
 	err = runCommand("Running Tests", exec.Command("kubetest", kubeTestArgs...))

test/run-windows-k8s-integration.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Optional environment variables
+# GCE_PD_OVERLAY_NAME: which Kustomize overlay to deploy with
+# GCE_PD_DO_DRIVER_BUILD: if set, don't build the driver from source and just
+#   use the driver version from the overlay
+# GCE_PD_BOSKOS_RESOURCE_TYPE: name of the boskos resource type to reserve
+
+set -o nounset
+set -o errexit
+
+readonly GCE_PD_DO_DRIVER_BUILD=false
+readonly PKGDIR=${GOPATH}/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver
+readonly LOCAL_K8S_DIR=${GOPATH}/src/k8s.io/kubernetes
+readonly overlay_name="${GCE_PD_OVERLAY_NAME:-alpha}"
+readonly do_driver_build="${GCE_PD_DO_DRIVER_BUILD:-true}"
+readonly deployment_strategy=${DEPLOYMENT_STRATEGY:-gce}
+readonly test_version=${TEST_VERSION:-master}
+readonly gce_zone=${GCE_CLUSTER_ZONE:-us-central1-b}
+#readonly gce_region=${GCE_CLUSTER_REGION:-}
+readonly image_type=${IMAGE_TYPE:-cos}
+
+export GCE_PD_VERBOSITY=9
+
+make -C ${PKGDIR} test-k8s-integration
+
+base_cmd="${PKGDIR}/bin/k8s-integration-test \
+            --platform=windows --local-k8s-dir=${LOCAL_K8S_DIR} --bringup-cluster=false --teardown-cluster=false --teardown-driver=true\
+            --run-in-prow=true --deploy-overlay-name=${overlay_name} --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \
+            --do-driver-build=${do_driver_build} --gce-zone=${gce_zone}\
+            --storageclass-file=sc-windows.yaml --snapshotclass-file=pd-volumesnapshotclass.yaml --test-focus="External.Storage.*ntfs" \
+            --deployment-strategy=${deployment_strategy} --num-nodes=4 \
+            --image-type=${image_type}"
+
+eval $base_cmd

test/run-windows-script.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+set -o nounset
+set -o pipefail
+set -o xtrace
+
+# When running in prow, the working directory is the root of the test-infra
+# repository.
+
+# Taint the Linux nodes to prevent the test workloads from landing on them.
+# TODO: remove this once the issue is resolved:
+# https://github.com/kubernetes/kubernetes/issues/69892
+#LINUX_NODES=$(kubectl get nodes -l beta.kubernetes.io/os=linux -o name)
+#LINUX_NODE_COUNT=$(echo ${LINUX_NODES} | wc -w)
+#for node in $LINUX_NODES; do
+#  kubectl taint node $node node-under-test=false:NoSchedule
+#done
+
+# Untaint the windows nodes to allow test workloads without tolerations to be
+# scheduled onto them.
+WINDOWS_NODES=$(kubectl get nodes -l beta.kubernetes.io/os=windows -o name)
+for node in $WINDOWS_NODES; do
+  kubectl taint node $node node.kubernetes.io/os:NoSchedule-
+done
+
+# Download and set the list of test image repositories to use.
+curl \
+  https://raw.githubusercontent.com/kubernetes-sigs/windows-testing/master/images/image-repo-list \
+  -o repo-list.yaml
+export KUBE_TEST_REPO_LIST=repo-list.yaml
+export KUBECONFIG=~/.kube/config
+export GCE_PD_SA_DIR="${GCE_PD_SA_DIR:-${E2E_GOOGLE_APPLICATION_CREDENTIALS}}"
+export GCE_PD_DRIVER_VERSION=alpha
+export PROJECT=$(gcloud config get-value project)
+
+$GOPATH/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/deploy/kubernetes/deploy-driver.sh
+cp $GOPATH/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/k8s-integration/config/sc-windows.yaml .
+cp $GOPATH/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/k8s-integration/config/pd-volumesnapshotclass.yaml .
+
+# When using customized test command (which we are now), report-dir is not set
+# by default, so set it here.
+# The test framework will not proceed to run tests unless all nodes are ready
+# AND schedulable. Allow not-ready nodes since we make Linux nodes
+# unschedulable.
+# Do not set --disable-log-dump because upstream cannot handle dumping logs
+# from windows nodes yet.
+./hack/ginkgo-e2e.sh $@ --storage.testdriver=$GOPATH/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/k8s-integration/config/test-driver-windows.yaml --report-dir=${ARTIFACTS} --allowed-not-ready-nodes=${LINUX_NODE_COUNT}
+