diff --git a/Dockerfile.Windows b/Dockerfile.Windows index 12587e170..73e5e4589 100644 --- a/Dockerfile.Windows +++ b/Dockerfile.Windows @@ -1,9 +1,17 @@ ARG BASE_IMAGE ARG BASE_IMAGE_TAG +FROM --platform=$BUILDPLATFORM golang:1.13.3 AS builder + +ARG TARGETPLATFORM +ARG BUILDPLATFORM +ARG STAGINGVERSION +WORKDIR /code +ADD . /code/ +RUN cd /code/ && GOARCH=$(echo $TARGETPLATFORM | cut -f2 -d '/') GCE_PD_CSI_STAGING_VERSION=${STAGINGVERSION} make gce-pd-driver-windows FROM mcr.microsoft.com/windows/${BASE_IMAGE}:${BASE_IMAGE_TAG} LABEL description="PD CSI driver" -COPY bin/gce-pd-csi-driver.exe /gce-pd-csi-driver.exe +COPY --from=builder /code/bin/gce-pd-csi-driver.exe /gce-pd-csi-driver.exe USER ContainerAdministrator ENTRYPOINT ["/gce-pd-csi-driver.exe"] diff --git a/Makefile b/Makefile index 0ae631f29..59a5297bd 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,7 @@ ifndef GCE_PD_CSI_STAGING_IMAGE endif @sh init-buildx.sh; \ DOCKER_CLI_EXPERIMENTAL=enabled docker buildx build --file=Dockerfile.Windows --platform=windows \ - -t $(STAGINGIMAGE):$(STAGINGVERSION) --build-arg BASE_IMAGE=servercore --build-arg BASE_IMAGE_TAG=ltsc2019 --push . + -t $(STAGINGIMAGE):$(STAGINGVERSION) --build-arg BASE_IMAGE=servercore --build-arg BASE_IMAGE_TAG=ltsc2019 --build-arg STAGINGVERSION=$(STAGINGVERSION) --push . build-and-push-windows-container-1909: ifndef GCE_PD_CSI_STAGING_IMAGE @@ -53,7 +53,7 @@ ifndef GCE_PD_CSI_STAGING_IMAGE endif @sh init-buildx.sh; \ DOCKER_CLI_EXPERIMENTAL=enabled docker buildx build --file=Dockerfile.Windows --platform=windows \ - -t $(STAGINGIMAGE):$(STAGINGVERSION) --build-arg BASE_IMAGE=servercore --build-arg BASE_IMAGE_TAG=1909 --push . + -t $(STAGINGIMAGE):$(STAGINGVERSION) --build-arg BASE_IMAGE=servercore --build-arg BASE_IMAGE_TAG=1909 --build-arg STAGINGVERSION=$(STAGINGVERSION) --push . push-container: build-container gcloud docker -- push $(STAGINGIMAGE):$(STAGINGVERSION) diff --git a/deploy/kubernetes/base/controller/cluster_setup.yaml b/deploy/kubernetes/base/controller/cluster_setup.yaml index 8f8953665..a4a2a66cc 100644 --- a/deploy/kubernetes/base/controller/cluster_setup.yaml +++ b/deploy/kubernetes/base/controller/cluster_setup.yaml @@ -272,4 +272,3 @@ roleRef: kind: Role name: csi-gce-pd-leaderelection-role apiGroup: rbac.authorization.k8s.io - diff --git a/deploy/kubernetes/images/alpha/image.yaml b/deploy/kubernetes/images/alpha/image.yaml index a339045e5..b74d32280 100644 --- a/deploy/kubernetes/images/alpha/image.yaml +++ b/deploy/kubernetes/images/alpha/image.yaml @@ -4,8 +4,8 @@ metadata: name: imagetag-gcepd-driver-alpha-win imageTag: name: gke.gcr.io/gcp-compute-persistent-disk-csi-driver-win - newName: gcr.io/jing-k8s-dev/gce-pd-windows-2019 - newTag: "0.2.0" + newName: gcr.io/gke-release-staging/gcp-compute-persistent-disk-csi-driver-amd64-windows-1909 + newTag: "cca3c14" --- apiVersion: builtin diff --git a/deploy/kubernetes/overlays/alpha/kustomization.yaml b/deploy/kubernetes/overlays/alpha/kustomization.yaml index 3e4e4a3a7..ebf7e4566 100644 --- a/deploy/kubernetes/overlays/alpha/kustomization.yaml +++ b/deploy/kubernetes/overlays/alpha/kustomization.yaml @@ -1,7 +1,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -namespace: - gce-pd-csi-driver +namespace: gce-pd-csi-driver resources: - ../../base/ transformers: diff --git a/test/k8s-integration/config/sc-windows.yaml b/test/k8s-integration/config/sc-windows.yaml new file mode 100644 index 000000000..9a88a6edf --- /dev/null +++ b/test/k8s-integration/config/sc-windows.yaml @@ -0,0 +1,9 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: csi-gcepd +provisioner: pd.csi.storage.gke.io +parameters: + type: pd-standard + csi.storage.k8s.io/fstype: ntfs +volumeBindingMode: WaitForFirstConsumer diff --git a/test/k8s-integration/config/test-config-template.in b/test/k8s-integration/config/test-config-template.in index 971986a0b..566521cea 100644 --- a/test/k8s-integration/config/test-config-template.in +++ b/test/k8s-integration/config/test-config-template.in @@ -7,13 +7,8 @@ SnapshotClass: DriverInfo: Name: csi-gcepd SupportedFsType: - ext2: - ext3: - ext4: - xfs: - # The following FS types supported by GCE PD but - # currently we do not test the CSI Driver on Windows - # ntfs: NTFS only available on Windows + {{range .SupportedFsType}} {{ . }}: + {{end}} Capabilities: {{range .Capabilities}} {{ . }}: true {{end}} diff --git a/test/k8s-integration/driver-config.go b/test/k8s-integration/driver-config.go index 00fdc9daf..4d3a8bdd6 100644 --- a/test/k8s-integration/driver-config.go +++ b/test/k8s-integration/driver-config.go @@ -12,6 +12,7 @@ type driverConfig struct { StorageClassFile string SnapshotClassFile string Capabilities []string + SupportedFsType []string } const ( @@ -22,7 +23,7 @@ const ( // generateDriverConfigFile loads a testdriver config template and creates a file // with the test-specific configuration -func generateDriverConfigFile(pkgDir, storageClassFile, snapshotClassFile, deploymentStrat string) (string, error) { +func generateDriverConfigFile(platform, pkgDir, storageClassFile, snapshotClassFile, deploymentStrat string) (string, error) { // Load template t, err := template.ParseFiles(filepath.Join(pkgDir, testConfigDir, configTemplateFile)) if err != nil { @@ -50,6 +51,17 @@ func generateDriverConfigFile(pkgDir, storageClassFile, snapshotClassFile, deplo "multipods", "topology", } + var fsTypes []string + if platform == "windows" { + fsTypes = []string{"ntfs"} + } else { + fsTypes = []string{ + "ext2", + "ext3", + "ext4", + "xfs", + } + } /* Unsupported Capabilities: pvcDataSource @@ -83,6 +95,7 @@ func generateDriverConfigFile(pkgDir, storageClassFile, snapshotClassFile, deplo params := driverConfig{ StorageClassFile: filepath.Join(pkgDir, testConfigDir, storageClassFile), SnapshotClassFile: absSnapshotClassFilePath, + SupportedFsType: fsTypes, Capabilities: caps, } diff --git a/test/k8s-integration/driver.go b/test/k8s-integration/driver.go index 28ac51186..7a446797d 100644 --- a/test/k8s-integration/driver.go +++ b/test/k8s-integration/driver.go @@ -87,7 +87,7 @@ func deleteDriver(goPath, pkgDir, deployOverlayName string) error { return nil } -func pushImage(pkgDir, stagingImage, stagingVersion string) error { +func pushImage(pkgDir, stagingImage, stagingVersion, platform string) error { err := os.Setenv("GCE_PD_CSI_STAGING_VERSION", stagingVersion) if err != nil { return err @@ -96,9 +96,16 @@ func pushImage(pkgDir, stagingImage, stagingVersion string) error { if err != nil { return err } - cmd := exec.Command("make", "-C", pkgDir, "push-container", - fmt.Sprintf("GCE_PD_CSI_STAGING_VERSION=%s", stagingVersion), - fmt.Sprintf("GCE_PD_CSI_STAGING_IMAGE=%s", stagingImage)) + var cmd *exec.Cmd + if platform != "windows" { + cmd = exec.Command("make", "-C", pkgDir, "push-container", + fmt.Sprintf("GCE_PD_CSI_STAGING_VERSION=%s", stagingVersion), + fmt.Sprintf("GCE_PD_CSI_STAGING_IMAGE=%s", stagingImage)) + } else { + cmd = exec.Command("make", "-C", pkgDir, "build-and-push-windows-container-1909", + fmt.Sprintf("GCE_PD_CSI_STAGING_VERSION=%s", stagingVersion), + fmt.Sprintf("GCE_PD_CSI_STAGING_IMAGE=%s", stagingImage)) + } err = runCommand("Pushing GCP Container", cmd) if err != nil { return fmt.Errorf("failed to run make command: err: %v", err) diff --git a/test/k8s-integration/main.go b/test/k8s-integration/main.go index d5c1c3f04..a9cb0086b 100644 --- a/test/k8s-integration/main.go +++ b/test/k8s-integration/main.go @@ -20,6 +20,7 @@ import ( "os" "os/exec" "path/filepath" + "strings" "syscall" "k8s.io/apimachinery/pkg/util/uuid" @@ -33,6 +34,7 @@ var ( teardownCluster = flag.Bool("teardown-cluster", true, "teardown the cluster after the e2e test") teardownDriver = flag.Bool("teardown-driver", true, "teardown the driver after the e2e test") bringupCluster = flag.Bool("bringup-cluster", true, "build kubernetes and bringup a cluster") + platform = flag.String("platform", "linux", "platform that the tests will be run, either linux or windows") gceZone = flag.String("gce-zone", "", "zone that the gce k8s cluster is created/found in") gceRegion = flag.String("gce-region", "", "region that gke regional cluster should be created in") kubeVersion = flag.String("kube-version", "", "version of Kubernetes to download and use for the cluster") @@ -95,7 +97,6 @@ func main() { } ensureVariable(testFocus, true, "test-focus is a required flag") - ensureVariable(imageType, true, "image type is a required flag. Available options include 'cos' and 'ubuntu'") if len(*gceRegion) != 0 { ensureVariable(gceZone, false, "gce-zone and gce-region cannot both be set") @@ -111,6 +112,12 @@ func main() { if !*bringupCluster { ensureVariable(kubeFeatureGates, false, "kube-feature-gates set but not bringing up new cluster") + } else { + ensureVariable(imageType, true, "image type is a required flag. Available options include 'cos' and 'ubuntu'") + } + + if *platform == "windows" { + ensureFlag(bringupCluster, false, "bringupCluster is set to false if it is for testing in windows cluster") } if *deploymentStrat == "gke" { @@ -132,7 +139,7 @@ func main() { ensureVariable(testVersion, false, "Cannot set a test version when using a local k8s dir.") } - if *numNodes == -1 { + if *numNodes == -1 && *bringupCluster { klog.Fatalf("num-nodes must be set to number of nodes in cluster") } @@ -157,28 +164,32 @@ func handle() error { // If running in Prow, then acquire and set up a project through Boskos if *inProw { - project, _ := testutils.SetupProwConfig(*boskosResourceType) - oldProject, err := exec.Command("gcloud", "config", "get-value", "project").CombinedOutput() + project := strings.TrimSpace(string(oldProject)) if err != nil { return fmt.Errorf("failed to get gcloud project: %s, err: %v", oldProject, err) } - - err = setEnvProject(project) - if err != nil { - return fmt.Errorf("failed to set project environment to %s: %v", project, err) - } - defer func() { - err = setEnvProject(string(oldProject)) + // TODO: Currently for prow tests with linux cluster, here it manually sets up a project from Boskos. + // For Windows, we used kubernetes_e2e.py which already set up the project and kubernetes automatically. + // Will update Linux in the future to use the same way as Windows test. + if *platform != "windows" { + newproject, _ := testutils.SetupProwConfig(*boskosResourceType) + err = setEnvProject(newproject) if err != nil { - klog.Errorf("failed to set project environment to %s: %v", oldProject, err) + return fmt.Errorf("failed to set project environment to %s: %v", newproject, err) } - }() + defer func() { + err = setEnvProject(string(oldProject)) + if err != nil { + klog.Errorf("failed to set project environment to %s: %v", oldProject, err) + } + }() + project = newproject + } if *doDriverBuild { - *stagingImage = fmt.Sprintf("gcr.io/%s/gcp-persistent-disk-csi-driver", project) + *stagingImage = fmt.Sprintf("gcr.io/%s/gcp-persistent-disk-csi-driver", strings.TrimSpace(string(project))) } - if _, ok := os.LookupEnv("USER"); !ok { err = os.Setenv("USER", "prow") if err != nil { @@ -189,7 +200,7 @@ func handle() error { // Build and push the driver, if required. Defer the driver image deletion. if *doDriverBuild { - err := pushImage(pkgDir, *stagingImage, stagingVersion) + err := pushImage(pkgDir, *stagingImage, stagingVersion, *platform) if err != nil { return fmt.Errorf("failed pushing image: %v", err) } @@ -319,7 +330,7 @@ func handle() error { var testSkip string switch *deploymentStrat { case "gce": - testSkip = generateGCETestSkip(clusterVersion) + testSkip = generateGCETestSkip(clusterVersion, *platform) case "gke": testSkip = generateGKETestSkip(clusterVersion, *useGKEManagedDriver) default: @@ -328,7 +339,7 @@ func handle() error { // Run the tests using the testDir kubernetes if len(*storageClassFile) != 0 { - err = runCSITests(pkgDir, testDir, *testFocus, testSkip, *storageClassFile, *snapshotClassFile, cloudProviderArgs, *deploymentStrat) + err = runCSITests(*platform, pkgDir, testDir, *testFocus, testSkip, *storageClassFile, *snapshotClassFile, cloudProviderArgs, *deploymentStrat) } else if *migrationTest { err = runMigrationTests(pkgDir, testDir, *testFocus, testSkip, cloudProviderArgs) } else { @@ -342,7 +353,7 @@ func handle() error { return nil } -func generateGCETestSkip(clusterVersion string) string { +func generateGCETestSkip(clusterVersion, platform string) string { skipString := "\\[Disruptive\\]|\\[Serial\\]" v := apimachineryversion.MustParseSemantic(clusterVersion) @@ -352,10 +363,12 @@ func generateGCETestSkip(clusterVersion string) string { if v.LessThan(apimachineryversion.MustParseSemantic("1.16.0")) { skipString = skipString + "|volumeMode\\sshould\\snot\\smount\\s/\\smap\\sunused\\svolumes\\sin\\sa\\spod" } - if v.LessThan(apimachineryversion.MustParseSemantic("1.17.0")) { skipString = skipString + "|VolumeSnapshotDataSource" } + if platform == "windows" { + skipString = skipString + "|\\[LinuxOnly\\]" + } return skipString } @@ -375,7 +388,6 @@ func generateGKETestSkip(clusterVersion string, use_gke_managed_driver bool) str (!use_gke_managed_driver && (*curVer).lessThan(mustParseVersion("1.17.0"))) { skipString = skipString + "|VolumeSnapshotDataSource" } - return skipString } @@ -396,8 +408,8 @@ func runMigrationTests(pkgDir, testDir, testFocus, testSkip string, cloudProvide return runTestsWithConfig(testDir, testFocus, testSkip, "--storage.migratedPlugins=kubernetes.io/gce-pd", cloudProviderArgs) } -func runCSITests(pkgDir, testDir, testFocus, testSkip, storageClassFile, snapshotClassFile string, cloudProviderArgs []string, deploymentStrat string) error { - testDriverConfigFile, err := generateDriverConfigFile(pkgDir, storageClassFile, snapshotClassFile, deploymentStrat) +func runCSITests(platform, pkgDir, testDir, testFocus, testSkip, storageClassFile, snapshotClassFile string, cloudProviderArgs []string, deploymentStrat string) error { + testDriverConfigFile, err := generateDriverConfigFile(platform, pkgDir, storageClassFile, snapshotClassFile, deploymentStrat) if err != nil { return err } @@ -419,10 +431,12 @@ func runTestsWithConfig(testDir, testFocus, testSkip, testConfigArg string, clou if ok { reportArg = fmt.Sprintf("-report-dir=%s", artifactsDir) } - - testArgs := fmt.Sprintf("--ginkgo.focus=%s --ginkgo.skip=%s %s %s", - testFocus, - testSkip, + ginkgoArgs := fmt.Sprintf("--ginkgo.focus=%s --ginkgo.skip=%s", testFocus, testSkip) + if *platform == "windows" { + ginkgoArgs = ginkgoArgs + fmt.Sprintf(" --node-os-distro=%s", *platform) + } + testArgs := fmt.Sprintf("%s %s %s", + ginkgoArgs, testConfigArg, reportArg) @@ -432,7 +446,6 @@ func runTestsWithConfig(testDir, testFocus, testSkip, testConfigArg string, clou "--check-version-skew=false", fmt.Sprintf("--test_args=%s", testArgs), } - kubeTestArgs = append(kubeTestArgs, cloudProviderArgs...) err = runCommand("Running Tests", exec.Command("kubetest", kubeTestArgs...)) diff --git a/test/run-windows-k8s-integration.sh b/test/run-windows-k8s-integration.sh new file mode 100755 index 000000000..ab9244d46 --- /dev/null +++ b/test/run-windows-k8s-integration.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# Optional environment variables +# GCE_PD_OVERLAY_NAME: which Kustomize overlay to deploy with +# GCE_PD_DO_DRIVER_BUILD: if set, don't build the driver from source and just +# use the driver version from the overlay +# GCE_PD_BOSKOS_RESOURCE_TYPE: name of the boskos resource type to reserve + +set -o nounset +set -o errexit + +readonly GCE_PD_DO_DRIVER_BUILD=false +readonly PKGDIR=${GOPATH}/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver +readonly LOCAL_K8S_DIR=${GOPATH}/src/k8s.io/kubernetes +readonly overlay_name="${GCE_PD_OVERLAY_NAME:-alpha}" +readonly do_driver_build="${GCE_PD_DO_DRIVER_BUILD:-true}" +readonly deployment_strategy=${DEPLOYMENT_STRATEGY:-gce} +readonly test_version=${TEST_VERSION:-master} +readonly gce_zone=${GCE_CLUSTER_ZONE:-us-central1-b} +readonly teardown_driver=${GCE_CLUSTER_ZONE:-true} + +make -C ${PKGDIR} test-k8s-integration + +base_cmd="${PKGDIR}/bin/k8s-integration-test \ + --platform=windows --local-k8s-dir=${LOCAL_K8S_DIR} --bringup-cluster=false --teardown-cluster=false --teardown-driver=${teardown_driver}\ + --run-in-prow=true --deploy-overlay-name=${overlay_name} --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \ + --do-driver-build=${do_driver_build} --gce-zone=${gce_zone}\ + --storageclass-file=sc-windows.yaml --snapshotclass-file=pd-volumesnapshotclass.yaml --test-focus="External.Storage" \ + --deployment-strategy=${deployment_strategy}" + +eval $base_cmd