Skip to content

Commit ec1ccd6

Browse files
committed
Add e2e CI tests for Windows
This PR adds e2e CI tests for Windows.
1 parent c16e7d1 commit ec1ccd6

File tree

9 files changed

+168
-35
lines changed

9 files changed

+168
-35
lines changed

Dockerfile.Windows

+9-1
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,17 @@
11
ARG BASE_IMAGE
22
ARG BASE_IMAGE_TAG
3+
FROM --platform=$BUILDPLATFORM golang:1.13.3 AS builder
4+
5+
ARG TARGETPLATFORM
6+
ARG BUILDPLATFORM
7+
WORKDIR /code
8+
ADD . /code/
9+
10+
RUN cd /code/ && GOARCH=$(echo $TARGETPLATFORM | cut -f2 -d '/') GCE_PD_CSI_STAGING_VERSION=dev make gce-pd-driver-windows
311

412
FROM mcr.microsoft.com/windows/${BASE_IMAGE}:${BASE_IMAGE_TAG}
513
LABEL description="PD CSI driver"
6-
COPY bin/gce-pd-csi-driver.exe /gce-pd-csi-driver.exe
14+
COPY --from=builder /code/bin/gce-pd-csi-driver.exe /gce-pd-csi-driver.exe
715

816
USER ContainerAdministrator
917
ENTRYPOINT ["/gce-pd-csi-driver.exe"]

deploy/kubernetes/base/controller/cluster_setup.yaml

+7-1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,10 @@
1+
kind: Namespace
2+
apiVersion: v1
3+
metadata:
4+
name: gce-pd-csi-driver
5+
6+
---
7+
18
##### Node Service Account, Roles, RoleBindings
29
apiVersion: v1
310
kind: ServiceAccount
@@ -272,4 +279,3 @@ roleRef:
272279
kind: Role
273280
name: csi-gce-pd-leaderelection-role
274281
apiGroup: rbac.authorization.k8s.io
275-

deploy/kubernetes/overlays/alpha/kustomization.yaml

+1-2
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
apiVersion: kustomize.config.k8s.io/v1beta1
22
kind: Kustomization
3-
namespace:
4-
gce-pd-csi-driver
3+
namespace: gce-pd-csi-driver
54
resources:
65
- ../../base/
76
transformers:
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
StorageClass:
2+
FromName: false
3+
FromFile: sc-windows.yaml
4+
SnapshotClass:
5+
FromFile: pd-volumesnapshotclass.yaml
6+
DriverInfo:
7+
Name: csi.gcepd
8+
SupportedFsType:
9+
ntfs:
10+
Capabilities:
11+
block: true
12+
controllerExpansion: false
13+
exec: true
14+
multipods: true
15+
persistence: true
16+
pvcDataSource: false
17+
snapshotDataSource: true
18+
SupportedSizeRange:
19+
Min: 5Gi
20+
Max: 64Ti

test/k8s-integration/driver-config.go

+10-5
Original file line numberDiff line numberDiff line change
@@ -15,16 +15,21 @@ type driverConfig struct {
1515
}
1616

1717
const (
18-
testConfigDir = "test/k8s-integration/config"
19-
configTemplateFile = "test-config-template.in"
20-
configFile = "test-config.yaml"
18+
testConfigDir = "test/k8s-integration/config"
19+
configTemplateFile = "test-config-template.in"
20+
configTemplateWindowsFile = "test-config-template-windows.in"
21+
configFile = "test-config.yaml"
2122
)
2223

2324
// generateDriverConfigFile loads a testdriver config template and creates a file
2425
// with the test-specific configuration
25-
func generateDriverConfigFile(pkgDir, storageClassFile, snapshotClassFile, deploymentStrat string) (string, error) {
26+
func generateDriverConfigFile(platform, pkgDir, storageClassFile, snapshotClassFile, deploymentStrat string) (string, error) {
2627
// Load template
27-
t, err := template.ParseFiles(filepath.Join(pkgDir, testConfigDir, configTemplateFile))
28+
configFile := configTemplateFile
29+
if platform == "windows" {
30+
configFile = configTemplateWindowsFile
31+
}
32+
t, err := template.ParseFiles(filepath.Join(pkgDir, testConfigDir, configFile))
2833
if err != nil {
2934
return "", err
3035
}

test/k8s-integration/driver.go

+11-4
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ func deleteDriver(goPath, pkgDir, deployOverlayName string) error {
8787
return nil
8888
}
8989

90-
func pushImage(pkgDir, stagingImage, stagingVersion string) error {
90+
func pushImage(pkgDir, stagingImage, stagingVersion, platform string) error {
9191
err := os.Setenv("GCE_PD_CSI_STAGING_VERSION", stagingVersion)
9292
if err != nil {
9393
return err
@@ -96,9 +96,16 @@ func pushImage(pkgDir, stagingImage, stagingVersion string) error {
9696
if err != nil {
9797
return err
9898
}
99-
cmd := exec.Command("make", "-C", pkgDir, "push-container",
100-
fmt.Sprintf("GCE_PD_CSI_STAGING_VERSION=%s", stagingVersion),
101-
fmt.Sprintf("GCE_PD_CSI_STAGING_IMAGE=%s", stagingImage))
99+
var cmd *exec.Cmd
100+
if platform != "windows" {
101+
cmd = exec.Command("make", "-C", pkgDir, "push-container",
102+
fmt.Sprintf("GCE_PD_CSI_STAGING_VERSION=%s", stagingVersion),
103+
fmt.Sprintf("GCE_PD_CSI_STAGING_IMAGE=%s", stagingImage))
104+
} else {
105+
cmd = exec.Command("make", "-C", pkgDir, "build-and-push-windows-container-1909",
106+
fmt.Sprintf("GCE_PD_CSI_STAGING_VERSION=%s", stagingVersion),
107+
fmt.Sprintf("GCE_PD_CSI_STAGING_IMAGE=%s", stagingImage))
108+
}
102109
err = runCommand("Pushing GCP Container", cmd)
103110
if err != nil {
104111
return fmt.Errorf("failed to run make command: err: %v", err)

test/k8s-integration/main.go

+27-22
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"os"
2121
"os/exec"
2222
"path/filepath"
23+
"strings"
2324
"syscall"
2425

2526
"k8s.io/apimachinery/pkg/util/uuid"
@@ -33,6 +34,7 @@ var (
3334
teardownCluster = flag.Bool("teardown-cluster", true, "teardown the cluster after the e2e test")
3435
teardownDriver = flag.Bool("teardown-driver", true, "teardown the driver after the e2e test")
3536
bringupCluster = flag.Bool("bringup-cluster", true, "build kubernetes and bringup a cluster")
37+
platform = flag.String("platform", "linux", "platform that the tests will be run, either linux or windows")
3638
gceZone = flag.String("gce-zone", "", "zone that the gce k8s cluster is created/found in")
3739
gceRegion = flag.String("gce-region", "", "region that gke regional cluster should be created in")
3840
kubeVersion = flag.String("kube-version", "", "version of Kubernetes to download and use for the cluster")
@@ -157,26 +159,27 @@ func handle() error {
157159

158160
// If running in Prow, then acquire and set up a project through Boskos
159161
if *inProw {
160-
project, _ := testutils.SetupProwConfig(*boskosResourceType)
161-
162162
oldProject, err := exec.Command("gcloud", "config", "get-value", "project").CombinedOutput()
163+
*stagingImage = fmt.Sprintf("gcr.io/%s/gcp-persistent-disk-csi-driver", strings.TrimSpace(string(oldProject)))
163164
if err != nil {
164165
return fmt.Errorf("failed to get gcloud project: %s, err: %v", oldProject, err)
165166
}
166-
167-
err = setEnvProject(project)
168-
if err != nil {
169-
return fmt.Errorf("failed to set project environment to %s: %v", project, err)
170-
}
171-
defer func() {
172-
err = setEnvProject(string(oldProject))
167+
if *platform != "windows" {
168+
project, _ := testutils.SetupProwConfig(*boskosResourceType)
169+
err = setEnvProject(project)
173170
if err != nil {
174-
klog.Errorf("failed to set project environment to %s: %v", oldProject, err)
171+
return fmt.Errorf("failed to set project environment to %s: %v", project, err)
172+
}
173+
if *doDriverBuild {
174+
*stagingImage = fmt.Sprintf("gcr.io/%s/gcp-persistent-disk-csi-driver", strings.TrimSpace(string(project)))
175175
}
176-
}()
177176

178-
if *doDriverBuild {
179-
*stagingImage = fmt.Sprintf("gcr.io/%s/gcp-persistent-disk-csi-driver", project)
177+
defer func() {
178+
err = setEnvProject(string(oldProject))
179+
if err != nil {
180+
klog.Errorf("failed to set project environment to %s: %v", oldProject, err)
181+
}
182+
}()
180183
}
181184

182185
if _, ok := os.LookupEnv("USER"); !ok {
@@ -189,7 +192,7 @@ func handle() error {
189192

190193
// Build and push the driver, if required. Defer the driver image deletion.
191194
if *doDriverBuild {
192-
err := pushImage(pkgDir, *stagingImage, stagingVersion)
195+
err := pushImage(pkgDir, *stagingImage, stagingVersion, *platform)
193196
if err != nil {
194197
return fmt.Errorf("failed pushing image: %v", err)
195198
}
@@ -319,7 +322,7 @@ func handle() error {
319322
var testSkip string
320323
switch *deploymentStrat {
321324
case "gce":
322-
testSkip = generateGCETestSkip(clusterVersion)
325+
testSkip = generateGCETestSkip(clusterVersion, *platform)
323326
case "gke":
324327
testSkip = generateGKETestSkip(clusterVersion, *useGKEManagedDriver)
325328
default:
@@ -328,7 +331,7 @@ func handle() error {
328331

329332
// Run the tests using the testDir kubernetes
330333
if len(*storageClassFile) != 0 {
331-
err = runCSITests(pkgDir, testDir, *testFocus, testSkip, *storageClassFile, *snapshotClassFile, cloudProviderArgs, *deploymentStrat)
334+
err = runCSITests(*platform, pkgDir, testDir, *testFocus, testSkip, *storageClassFile, *snapshotClassFile, cloudProviderArgs, *deploymentStrat)
332335
} else if *migrationTest {
333336
err = runMigrationTests(pkgDir, testDir, *testFocus, testSkip, cloudProviderArgs)
334337
} else {
@@ -342,7 +345,7 @@ func handle() error {
342345
return nil
343346
}
344347

345-
func generateGCETestSkip(clusterVersion string) string {
348+
func generateGCETestSkip(clusterVersion, platform string) string {
346349
skipString := "\\[Disruptive\\]|\\[Serial\\]"
347350
v := apimachineryversion.MustParseSemantic(clusterVersion)
348351

@@ -352,6 +355,9 @@ func generateGCETestSkip(clusterVersion string) string {
352355
if v.LessThan(apimachineryversion.MustParseSemantic("1.16.0")) {
353356
skipString = skipString + "|volumeMode\\sshould\\snot\\smount\\s/\\smap\\sunused\\svolumes\\sin\\sa\\spod"
354357
}
358+
if platform == "windows" {
359+
skipString = skipString + "|\\[Slow\\]|\\[LinuxOnly\\]|\\[Feature:.+\\][Ephemeral]"
360+
}
355361

356362
if v.LessThan(apimachineryversion.MustParseSemantic("1.17.0")) {
357363
skipString = skipString + "|VolumeSnapshotDataSource"
@@ -375,7 +381,6 @@ func generateGKETestSkip(clusterVersion string, use_gke_managed_driver bool) str
375381
(!use_gke_managed_driver && (*curVer).lessThan(mustParseVersion("1.17.0"))) {
376382
skipString = skipString + "|VolumeSnapshotDataSource"
377383
}
378-
379384
return skipString
380385
}
381386

@@ -396,8 +401,8 @@ func runMigrationTests(pkgDir, testDir, testFocus, testSkip string, cloudProvide
396401
return runTestsWithConfig(testDir, testFocus, testSkip, "--storage.migratedPlugins=kubernetes.io/gce-pd", cloudProviderArgs)
397402
}
398403

399-
func runCSITests(pkgDir, testDir, testFocus, testSkip, storageClassFile, snapshotClassFile string, cloudProviderArgs []string, deploymentStrat string) error {
400-
testDriverConfigFile, err := generateDriverConfigFile(pkgDir, storageClassFile, snapshotClassFile, deploymentStrat)
404+
func runCSITests(platform, pkgDir, testDir, testFocus, testSkip, storageClassFile, snapshotClassFile string, cloudProviderArgs []string, deploymentStrat string) error {
405+
testDriverConfigFile, err := generateDriverConfigFile(platform, pkgDir, storageClassFile, snapshotClassFile, deploymentStrat)
401406
if err != nil {
402407
return err
403408
}
@@ -420,9 +425,10 @@ func runTestsWithConfig(testDir, testFocus, testSkip, testConfigArg string, clou
420425
reportArg = fmt.Sprintf("-report-dir=%s", artifactsDir)
421426
}
422427

423-
testArgs := fmt.Sprintf("--ginkgo.focus=%s --ginkgo.skip=%s %s %s",
428+
testArgs := fmt.Sprintf("--ginkgo.focus=%s --ginkgo.skip=%s --node-os-distro=%s %s %s",
424429
testFocus,
425430
testSkip,
431+
*platform,
426432
testConfigArg,
427433
reportArg)
428434

@@ -432,7 +438,6 @@ func runTestsWithConfig(testDir, testFocus, testSkip, testConfigArg string, clou
432438
"--check-version-skew=false",
433439
fmt.Sprintf("--test_args=%s", testArgs),
434440
}
435-
436441
kubeTestArgs = append(kubeTestArgs, cloudProviderArgs...)
437442

438443
err = runCommand("Running Tests", exec.Command("kubetest", kubeTestArgs...))

test/run-windows-k8s-integration.sh

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
#!/bin/bash
2+
3+
# Optional environment variables
4+
# GCE_PD_OVERLAY_NAME: which Kustomize overlay to deploy with
5+
# GCE_PD_DO_DRIVER_BUILD: if set, don't build the driver from source and just
6+
# use the driver version from the overlay
7+
# GCE_PD_BOSKOS_RESOURCE_TYPE: name of the boskos resource type to reserve
8+
9+
set -o nounset
10+
set -o errexit
11+
12+
readonly GCE_PD_DO_DRIVER_BUILD=false
13+
readonly PKGDIR=${GOPATH}/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver
14+
readonly LOCAL_K8S_DIR=${GOPATH}/src/k8s.io/kubernetes
15+
readonly overlay_name="${GCE_PD_OVERLAY_NAME:-alpha}"
16+
readonly do_driver_build="${GCE_PD_DO_DRIVER_BUILD:-true}"
17+
readonly deployment_strategy=${DEPLOYMENT_STRATEGY:-gce}
18+
readonly test_version=${TEST_VERSION:-master}
19+
readonly gce_zone=${GCE_CLUSTER_ZONE:-us-central1-b}
20+
#readonly gce_region=${GCE_CLUSTER_REGION:-}
21+
readonly image_type=${IMAGE_TYPE:-cos}
22+
23+
export GCE_PD_VERBOSITY=9
24+
25+
make -C ${PKGDIR} test-k8s-integration
26+
27+
base_cmd="${PKGDIR}/bin/k8s-integration-test \
28+
--platform=windows --local-k8s-dir=${LOCAL_K8S_DIR} --bringup-cluster=false --teardown-cluster=false --teardown-driver=true\
29+
--run-in-prow=true --deploy-overlay-name=${overlay_name} --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \
30+
--do-driver-build=${do_driver_build} --gce-zone=${gce_zone}\
31+
--storageclass-file=sc-windows.yaml --snapshotclass-file=pd-volumesnapshotclass.yaml --test-focus="External.Storage.*ntfs" \
32+
--deployment-strategy=${deployment_strategy} --num-nodes=4 \
33+
--image-type=${image_type}"
34+
35+
eval $base_cmd

test/run-windows-script.sh

+48
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
#!/bin/bash
2+
3+
set -o nounset
4+
set -o pipefail
5+
set -o xtrace
6+
7+
# When running in prow, the working directory is the root of the test-infra
8+
# repository.
9+
10+
# Taint the Linux nodes to prevent the test workloads from landing on them.
11+
# TODO: remove this once the issue is resolved:
12+
# https://github.com/kubernetes/kubernetes/issues/69892
13+
#LINUX_NODES=$(kubectl get nodes -l beta.kubernetes.io/os=linux -o name)
14+
#LINUX_NODE_COUNT=$(echo ${LINUX_NODES} | wc -w)
15+
#for node in $LINUX_NODES; do
16+
# kubectl taint node $node node-under-test=false:NoSchedule
17+
#done
18+
19+
# Untaint the windows nodes to allow test workloads without tolerations to be
20+
# scheduled onto them.
21+
WINDOWS_NODES=$(kubectl get nodes -l beta.kubernetes.io/os=windows -o name)
22+
for node in $WINDOWS_NODES; do
23+
kubectl taint node $node node.kubernetes.io/os:NoSchedule-
24+
done
25+
26+
# Download and set the list of test image repositories to use.
27+
curl \
28+
https://raw.githubusercontent.com/kubernetes-sigs/windows-testing/master/images/image-repo-list \
29+
-o repo-list.yaml
30+
export KUBE_TEST_REPO_LIST=repo-list.yaml
31+
export KUBECONFIG=~/.kube/config
32+
export GCE_PD_SA_DIR="${GCE_PD_SA_DIR:-${E2E_GOOGLE_APPLICATION_CREDENTIALS}}"
33+
export GCE_PD_DRIVER_VERSION=alpha
34+
export PROJECT=$(gcloud config get-value project)
35+
36+
$GOPATH/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/deploy/kubernetes/deploy-driver.sh
37+
cp $GOPATH/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/k8s-integration/config/sc-windows.yaml .
38+
cp $GOPATH/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/k8s-integration/config/pd-volumesnapshotclass.yaml .
39+
40+
# When using customized test command (which we are now), report-dir is not set
41+
# by default, so set it here.
42+
# The test framework will not proceed to run tests unless all nodes are ready
43+
# AND schedulable. Allow not-ready nodes since we make Linux nodes
44+
# unschedulable.
45+
# Do not set --disable-log-dump because upstream cannot handle dumping logs
46+
# from windows nodes yet.
47+
./hack/ginkgo-e2e.sh $@ --storage.testdriver=$GOPATH/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/k8s-integration/config/test-driver-windows.yaml --report-dir=${ARTIFACTS} --allowed-not-ready-nodes=${LINUX_NODE_COUNT}
48+

0 commit comments

Comments
 (0)