Enable all the e2e in Windows, retry for SSH

Author: mauriciopoppe

hack/e2e.sh

@@ -323,9 +323,9 @@ if [ "$PROVIDER" == "gke" ]; then
 fi
 
 # legacy path
-    # --up
-    # --down
 go run $ROOT/hack/e2e.go -- "${kubetest_args[@]}" \
     --deployment="$DEPLOYMENT" \
+    --up \
+    --down \
     --test-cmd="$ROOT/hack/run-e2e.sh" \
     "${@:-}"

hack/release.sh

@@ -91,7 +91,7 @@ ALLOW_OVERRIDE=${ALLOW_OVERRIDE:-}
 SKIP_BUILD=${SKIP_BUILD:-}
 SKIP_PUSH_LATEST=${SKIP_PUSH_LATEST:-}
 LINUX_ARCH=${LINUX_ARCH:-amd64 arm arm64 ppc64le s390x}
-WINDOWS_DISTROS=${WINDOWS_DISTROS:-ltsc2019 1909 2004 20H2}
+WINDOWS_DISTROS=${WINDOWS_DISTROS:-ltsc2019 1909 2004 20H2 ltsc2022}
 
 echo "REGISTRY: $REGISTRY"
 echo "VERSION: $VERSION"

pkg/common/common.go

@@ -329,8 +329,8 @@ func ConfigMapDataToVolumeConfig(data map[string]string, provisionerConfig *Prov
 		if config.MountDir == "" || config.HostDir == "" {
 			return fmt.Errorf("Storage Class %v is misconfigured, missing HostDir or MountDir parameter", class)
 		}
-		config.MountDir = normalizePath(config.MountDir)
-		config.HostDir = normalizePath(config.HostDir)
+		config.MountDir = NormalizePath(config.MountDir)
+		config.HostDir = NormalizePath(config.HostDir)
 
 		if config.VolumeMode == "" {
 			config.VolumeMode = DefaultVolumeMode
@@ -357,10 +357,10 @@ func ConfigMapDataToVolumeConfig(data map[string]string, provisionerConfig *Prov
 	return nil
 }
 
-// normalizePath makes sure the given path is a valid path on Windows too
+// NormalizePath makes sure the given path is a valid path on Windows too
 // by making sure all instances of `/` are replaced with `\\`, and the
 // path beings with `c:`
-func normalizePath(path string) string {
+func NormalizePath(path string) string {
 	if runtime.GOOS != "windows" {
 		return path
 	}

pkg/discovery/discovery.go

@@ -267,7 +267,6 @@ func (d *Discoverer) discoverVolumesAtPath(class string, config common.MountConf
 
 	// Retrieve list of mount points to iterate through discovered paths (aka files) below
 	mountPoints, err := d.RuntimeConfig.Mounter.List()
-	klog.V(4).Infof("[DEBUG] the mount points are %v", mountPoints)
 	if err != nil {
 		return fmt.Errorf("error retrieving mountpoints: %v", err)
 	}

test/e2e/e2e_test.go

@@ -85,7 +85,7 @@ const (
 var (
 	// default provisioner image used for e2e tests
 	provisionerImageName                     = "quay.io/external_storage/local-volume-provisioner:latest"
-	provisionerImagePullPolicy v1.PullPolicy = "Never"
+	provisionerImagePullPolicy v1.PullPolicy = "Always"
 	// storage class volume binding modes
 	waitMode      = storagev1.VolumeBindingWaitForFirstConsumer
 	immediateMode = storagev1.VolumeBindingImmediate
@@ -232,9 +232,9 @@ var _ = utils.SIGDescribe("PersistentVolumes-local ", func() {
 			})
 
 			AfterEach(func() {
+				deleteProvisionerDaemonset(config)
 				cleanupLocalVolumeProvisioner(config)
 				cleanupStorageClass(config)
-				deleteProvisionerDaemonset(config)
 			})
 
 			It("should create and recreate local persistent volume", func() {
@@ -290,15 +290,18 @@ var _ = utils.SIGDescribe("PersistentVolumes-local ", func() {
 		})
 
 		AfterEach(func() {
+			deleteProvisionerDaemonset(config)
 			cleanupLocalVolumeProvisioner(config)
 			cleanupStorageClass(config)
-			deleteProvisionerDaemonset(config)
 		})
 
 		It("should not create local persistent volume for filesystem volume that was not bind mounted", func() {
 			directoryPath := filepath.Join(config.discoveryDir, "notbindmount")
 			By("Creating a directory, not bind mounted, in discovery directory")
 			mkdirCmd := fmt.Sprintf("mkdir -p %v -m 777", directoryPath)
+			if nodeOSDistroIs("windows") {
+				mkdirCmd = fmt.Sprintf("mkdir -Force %v", directoryPath)
+			}
 			execResult, err := config.hostExec.Execute(mkdirCmd, config.node0)
 			utils.LogResult(execResult)
 			Expect(err).NotTo(HaveOccurred())
@@ -313,6 +316,11 @@ var _ = utils.SIGDescribe("PersistentVolumes-local ", func() {
 				"Error getting logs from pod %s in namespace %s", provisionerPodName, config.ns)
 
 			expectedLogMessage := "path \"/mnt/local-storage/notbindmount\" is not an actual mountpoint"
+			// the expected log message above is thrown in Linux if the mount path doesn't exist in /proc/mounts
+			// in Windows we instead look for the error `path "%q" fs stats error`
+			if nodeOSDistroIs("windows") {
+				expectedLogMessage = `path "c:\\mnt\\local-storage\\notbindmount" fs stats error`
+			}
 			Expect(strings.Contains(logs, expectedLogMessage)).To(BeTrue())
 		})
 	})
@@ -321,11 +329,19 @@ var _ = utils.SIGDescribe("PersistentVolumes-local ", func() {
 	Context("Stress with local volume provisioner [Serial]", func() {
 		var testVols [][]*localVolume
 
-		const (
+		var (
 			volsPerNode = 10 // Make this non-divisable by volsPerPod to increase changes of partial binding failure
 			volsPerPod  = 3
 			podsFactor  = 4
 		)
+		if nodeOSDistroIs("windows") {
+			// the Windows image is big and having `volsPerNode/volsPerPod + 1` pods pulling it
+			// at the same time consumes all the bandwidth, in Windows these parameters are
+			// changed so that the stress test is with fewer parallel pods
+			volsPerNode = 7
+			volsPerPod = 3
+			podsFactor = 3
+		}
 
 		BeforeEach(func() {
 			setupStorageClass(config, &waitMode)
@@ -343,14 +359,11 @@ var _ = utils.SIGDescribe("PersistentVolumes-local ", func() {
 				testVols = append(testVols, vols)
 			}
 
-			By("Starting the local volume provisioner")
 			createProvisionerDaemonset(config)
 		})
 
 		AfterEach(func() {
-			By("Deleting provisioner daemonset")
 			deleteProvisionerDaemonset(config)
-
 			for i, vols := range testVols {
 				for _, vol := range vols {
 					cleanupLocalVolumeProvisionerMountPoint(config, vol, &config.nodes[i])
@@ -393,7 +406,8 @@ var _ = utils.SIGDescribe("PersistentVolumes-local ", func() {
 				for i := 0; i < numConcurrentPods; i++ {
 					pvcs := []*v1.PersistentVolumeClaim{}
 					for j := 0; j < volsPerPod; j++ {
-						pvc := e2epv.MakePersistentVolumeClaim(makeLocalPVCConfig(config, DirectoryLocalVolumeType), config.ns)
+						pvcConfig := makeLocalPVCConfig(config, DirectoryLocalVolumeType)
+						pvc := e2epv.MakePersistentVolumeClaim(pvcConfig, config.ns)
 						pvc, err := e2epv.CreatePVC(config.client, config.ns, pvc)
 						framework.ExpectNoError(err)
 						pvcs = append(pvcs, pvc)
@@ -429,7 +443,8 @@ var _ = utils.SIGDescribe("PersistentVolumes-local ", func() {
 			}()
 
 			By("Waiting for all pods to complete successfully")
-			err := wait.PollImmediate(time.Second, 5*time.Minute, func() (done bool, err error) {
+			timeout := 10 * time.Minute
+			err := wait.PollImmediate(time.Second, timeout, func() (done bool, err error) {
 				podsList, err := config.client.CoreV1().Pods(config.ns).List(context.TODO(), metav1.ListOptions{})
 				if err != nil {
 					return false, err
@@ -475,6 +490,7 @@ func setupStorageClass(config *localTestConfig, mode *storagev1.VolumeBindingMod
 }
 
 func cleanupStorageClass(config *localTestConfig) {
+	By("Cleanup StorageClass")
 	framework.ExpectNoError(config.client.StorageV1().StorageClasses().Delete(context.TODO(), config.scName, metav1.DeleteOptions{}))
 }
 
@@ -505,6 +521,9 @@ func cleanupLocalVolumeProvisioner(config *localTestConfig) {
 	for _, node := range config.nodes {
 		By(fmt.Sprintf("Removing the test discovery directory on node %v", node.Name))
 		removeCmd := fmt.Sprintf("[ ! -e %v ] || rm -r %v", config.discoveryDir, config.discoveryDir)
+		if nodeOSDistroIs("windows") {
+			removeCmd = fmt.Sprintf("rmdir -Recurse -Force %v", config.discoveryDir)
+		}
 		execResult, err := config.hostExec.Execute(removeCmd, &node)
 		utils.LogResult(execResult)
 		Expect(err).NotTo(HaveOccurred())
@@ -629,8 +648,22 @@ func findLoopDevice(config *localTestConfig, file string, node *v1.Node) string
 	return strings.TrimSpace(loopDevResult)
 }
 
+// normalizePath makes sure the given path is a valid path on Windows too
+// by making sure all instances of `/` are replaced with `\\`, and the
+// path beings with `c:`
+func normalizePath(path string) string {
+	if !nodeOSDistroIs("windows") {
+		return path
+	}
+	normalizedPath := strings.Replace(path, "/", "\\", -1)
+	if strings.HasPrefix(normalizedPath, "\\") {
+		normalizedPath = "c:" + normalizedPath
+	}
+	return normalizedPath
+}
+
 func setupLocalVolumeProvisionerMountPoint(config *localTestConfig, node *v1.Node, volumeType localVolumeType) *localVolume {
-	volumePath := path.Join(config.discoveryDir, fmt.Sprintf("vol-%v", string(uuid.NewUUID())))
+	volumePath := normalizePath(path.Join(config.discoveryDir, fmt.Sprintf("vol-%v", string(uuid.NewUUID()))))
 	if volumeType == DirectoryLocalVolumeType {
 		lv := &localVolume{
 			volumePath: volumePath,
@@ -648,6 +681,7 @@ func setupLocalVolumeProvisionerMountPoint(config *localTestConfig, node *v1.Nod
 
 		By(fmt.Sprintf("Mounting local directory at path %q", volumePath))
 		if nodeOSDistroIs("windows") {
+			// NOTE: the value must be greater than the PVC config ClaimSize
 			vhd := setupVHD(config, node, volumePath, 1024*1024*1024 /* 1GB */)
 			lv.vhd = vhd
 		} else {
@@ -807,10 +841,7 @@ func createProvisionerDaemonset(config *localTestConfig) {
 	// mounted as volumes in the pod
 	// see helm/provisioner/templates/daemonset_windows.yaml for more details
 	if nodeOSDistroIs("windows") {
-		apiGroups := []string{
-			"disk-v1", "volume-v1", "filesystem-v1",
-			"disk-v1beta2", "volume-v1beta2", "filesystem-v1beta1",
-		}
+		apiGroups := []string{"volume-v1", "volume-v1beta2"}
 
 		for _, apiGroup := range apiGroups {
 			additionalVolumes = append(
@@ -1017,7 +1048,7 @@ func createWriteCmd(testDir string, testFile string, writeTestFileContent string
 func createFileDoesntExistCmd(testFileDir string, testFile string) string {
 	testFilePath := filepath.Join(testFileDir, testFile)
 	if nodeOSDistroIs("windows") {
-		return fmt.Sprintf("if(-not(Test-Path -Path %s)) { throw 'File exists' }", testFilePath)
+		return fmt.Sprintf("if (Test-Path -Path %s) { throw 'File exists' }", testFilePath)
 	}
 	return fmt.Sprintf("[ ! -e %s ]", testFilePath)
 }
@@ -1054,6 +1085,7 @@ func (c *localTestConfig) isNodeInList(name string) bool {
 }
 
 func deleteProvisionerDaemonset(config *localTestConfig) {
+	By("Cleanup Provisioner daemonset")
 	ds, err := config.client.AppsV1().DaemonSets(config.ns).Get(context.TODO(), daemonSetName, metav1.GetOptions{})
 	if ds == nil {
 		return
@@ -1125,6 +1157,7 @@ func makeLocalPVCConfig(config *localTestConfig, volumeType localVolumeType) e2e
 	pvcConfig := e2epv.PersistentVolumeClaimConfig{
 		AccessModes:      []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
 		StorageClassName: &config.scName,
+		ClaimSize:        "100M",
 	}
 	if volumeType == BlockLocalVolumeType {
 		pvcVolumeMode := v1.PersistentVolumeBlock

test/e2e/windows/host_exec.go

@@ -21,8 +21,11 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
+	"strings"
+	"time"
 
 	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
 	e2estorageutils "k8s.io/kubernetes/test/e2e/storage/utils"
 )
 
@@ -53,21 +56,40 @@ func (h *hostExecutor) Execute(command string, node *v1.Node) (e2estorageutils.R
 	cmd.Stdout = &outBuffer
 	cmd.Stderr = &errBuffer
 
-	err := cmd.Run()
-	result := e2estorageutils.Result{
-		Host:   node.Name,
-		Cmd:    cmd.String(),
-		Stdout: outBuffer.String(),
-		Stderr: errBuffer.String(),
-	}
-	if err != nil {
-		if exitError, ok := err.(*exec.ExitError); ok {
-			result.Code = exitError.ExitCode()
+	var result e2estorageutils.Result
+
+	// sometimes `gcloud compute ssh` fails with
+	//
+	//   Permission denied (publickey,keyboard-interactive).
+	//   ERROR: (gcloud.compute.ssh) [/usr/bin/ssh] exited with return code [255]
+	//
+	// in that case let's retry with backoff
+	backoff := wait.Backoff{Duration: 1 * time.Second, Factor: 1, Steps: 3}
+	err := wait.ExponentialBackoff(backoff, func() (bool, error) {
+		err := cmd.Run()
+		result = e2estorageutils.Result{
+			Host:   node.Name,
+			Cmd:    cmd.String(),
+			Stdout: outBuffer.String(),
+			Stderr: errBuffer.String(),
 		}
-		return result, err
-	}
+		if err != nil {
+			if strings.Contains(result.Stderr, "Permission denied") {
+				return false, nil
+			}
+
+			// the command failed with an error that's not transient
+			if exitError, ok := err.(*exec.ExitError); ok {
+				result.Code = exitError.ExitCode()
+			}
+			return true, err
+		}
+
+		// request succeeded!
+		return true, nil
+	})
 
-	return result, nil
+	return result, err
 }
 
 // IssueCommandWithResult issues command on the given node and returns stdout as

test/e2e/windows/vhd.go

@@ -41,7 +41,7 @@ func NewVHD(path string, sizeBytes int) *VHD {
 func (vhd *VHD) StageScript() (string, error) {
 	script := strings.Replace(`"& {
 $global:progressPreference = 'SilentlyContinue'
-$vhdPath = {{.Path}}
+$vhdPath = '{{.Path}}'
 New-VHD -Path $vhdPath -SizeBytes {{.SizeBytes}}
 Mount-VHD -Path $vhdPath -PassThru | Initialize-Disk -PartitionStyle GPT -PassThru | New-Partition -UseMaximumSize
 Get-VHD -Path $vhdPath | Get-Partition | Get-Volume | Format-Volume -Filesystem ntfs -Confirm:$false