kubernetes bootstrap through k8s-integration for both linux & windows

Author: mauriciopoppe

test/k8s-integration/cluster.go

@@ -36,6 +36,7 @@ func isRegionalGKECluster(gceZone, gceRegion string) bool {
 
 func clusterDownGCE(k8sDir string) error {
 	cmd := exec.Command(filepath.Join(k8sDir, "hack", "e2e-internal", "e2e-down.sh"))
+	cmd.Env = os.Environ()
 	err := runCommand("Bringing Down E2E Cluster on GCE", cmd)
 	if err != nil {
 		return fmt.Errorf("failed to bring down kubernetes e2e cluster on gce: %v", err)
@@ -60,14 +61,15 @@ func clusterDownGKE(gceZone, gceRegion string) error {
 
 func buildKubernetes(k8sDir, command string) error {
 	cmd := exec.Command("make", "-C", k8sDir, command)
-	err := runCommand("Building Kubernetes", cmd)
+	cmd.Env = os.Environ()
+	err := runCommand(fmt.Sprintf("Running command in kubernetes/kubernetes path=%s", k8sDir), cmd)
 	if err != nil {
 		return fmt.Errorf("failed to build Kubernetes: %v", err)
 	}
 	return nil
 }
 
-func clusterUpGCE(k8sDir, gceZone string, numNodes int, imageType string) error {
+func clusterUpGCE(k8sDir, gceZone string, numNodes int, numWindowsNodes int, imageType string) error {
 	kshPath := filepath.Join(k8sDir, "cluster", "kubectl.sh")
 	_, err := os.Stat(kshPath)
 	if err == nil {
@@ -98,6 +100,14 @@ func clusterUpGCE(k8sDir, gceZone string, numNodes int, imageType string) error
 		return err
 	}
 
+	// the chain is NUM_WINDOWS_NODES -> --num-windows-nodes -> NUM_WINDOWS_NODES
+	// runCommand runs e2e-up.sh inheriting env vars so the `--num-windows-nodes`
+	// flags might not be needed, added to be similar to the setup of NUM_NODES
+	err = os.Setenv("NUM_WINDOWS_NODES", strconv.Itoa(numWindowsNodes))
+	if err != nil {
+		return err
+	}
+
 	// The default master size with few nodes is too small; the tests must hit the API server
 	// more than usual. The main issue seems to be memory, to reduce GC times that stall the
 	// api server. For defaults, get-master-size in k/k/cluster/gce/config-common.sh.
@@ -113,6 +123,7 @@ func clusterUpGCE(k8sDir, gceZone string, numNodes int, imageType string) error
 		return err
 	}
 	cmd := exec.Command(filepath.Join(k8sDir, "hack", "e2e-internal", "e2e-up.sh"))
+	cmd.Env = os.Environ()
 	err = runCommand("Starting E2E Cluster on GCE", cmd)
 	if err != nil {
 		return fmt.Errorf("failed to bring up kubernetes e2e cluster on gce: %v", err)
@@ -147,7 +158,7 @@ func setImageTypeEnvs(imageType string) error {
 	return nil
 }
 
-func clusterUpGKE(gceZone, gceRegion string, numNodes int, imageType string, useManagedDriver bool) error {
+func clusterUpGKE(gceZone, gceRegion string, numNodes int, numWindowsNodes int, imageType string, useManagedDriver bool) error {
 	locationArg, locationVal, err := gkeLocationArgs(gceZone, gceRegion)
 	if err != nil {
 		return err
@@ -213,7 +224,7 @@ func clusterUpGKE(gceZone, gceRegion string, numNodes int, imageType string, use
 
 func downloadKubernetesSource(pkgDir, k8sIoDir, kubeVersion string) error {
 	k8sDir := filepath.Join(k8sIoDir, "kubernetes")
-	klog.Infof("Downloading Kubernetes source for %s", kubeVersion)
+	klog.Infof("Downloading Kubernetes source v=%s to path=%s", kubeVersion, k8sIoDir)
 
 	if err := os.MkdirAll(k8sIoDir, 0777); err != nil {
 		return err

test/k8s-integration/main.go

@@ -45,7 +45,8 @@ var (
 	localK8sDir          = flag.String("local-k8s-dir", "", "local prebuilt kubernetes/kubernetes directory to use for cluster and test binaries")
 	deploymentStrat      = flag.String("deployment-strategy", "gce", "choose between deploying on gce or gke")
 	gkeClusterVer        = flag.String("gke-cluster-version", "", "version of Kubernetes master and node for gke")
-	numNodes             = flag.Int("num-nodes", -1, "the number of nodes in the test cluster")
+	numNodes             = flag.Int("num-nodes", 0, "the number of nodes in the test cluster")
+	numWindowsNodes      = flag.Int("num-windows-nodes", 0, "the number of Windows nodes in the test cluster")
 	imageType            = flag.String("image-type", "cos", "the image type to use for the cluster")
 	gkeReleaseChannel    = flag.String("gke-release-channel", "", "GKE release channel to be used for cluster deploy. One of 'rapid', 'stable' or 'regular'")
 	gkeTestClusterPrefix = flag.String("gke-cluster-prefix", "pdcsi", "Prefix of GKE cluster names. A random suffix will be appended to form the full name.")
@@ -160,10 +161,6 @@ func main() {
 		}
 	}
 
-	if *platform == "windows" {
-		ensureFlag(bringupCluster, false, "bringupCluster is set to false if it is for testing in windows cluster")
-	}
-
 	if *deploymentStrat == "gke" {
 		ensureVariable(kubeVersion, false, "Cannot set kube-version when using deployment strategy 'gke'. Use gke-cluster-version.")
 		ensureExactlyOneVariableSet([]*string{gkeClusterVer, gkeReleaseChannel},
@@ -186,9 +183,12 @@ func main() {
 		ensureVariable(testVersion, false, "Cannot set a test version when using a local k8s dir.")
 	}
 
-	if *numNodes == -1 && *bringupCluster {
+	if *numNodes == 0 && *bringupCluster {
 		klog.Fatalf("num-nodes must be set to number of nodes in cluster")
 	}
+	if *numWindowsNodes == 0 && *bringupCluster && *platform == "windows" {
+		klog.Fatalf("num-windows-nodes must be set if the platform is windows")
+	}
 
 	err := handle()
 	if err != nil {
@@ -225,24 +225,19 @@ func handle() error {
 		if err != nil {
 			return fmt.Errorf("failed to get gcloud project: %s, err: %v", oldProject, err)
 		}
-		// TODO: Currently for prow tests with linux cluster, here it manually sets up a project from Boskos.
-		// For Windows, we used kubernetes_e2e.py which already set up the project and kubernetes automatically.
-		// Will update Linux in the future to use the same way as Windows test.
-		if *platform != "windows" {
-			newproject, _ := testutils.SetupProwConfig(*boskosResourceType)
-			err = setEnvProject(newproject)
+		newproject, _ := testutils.SetupProwConfig(*boskosResourceType)
+		err = setEnvProject(newproject)
+		if err != nil {
+			return fmt.Errorf("failed to set project environment to %s: %v", newproject, err)
+		}
+
+		defer func() {
+			err = setEnvProject(string(oldProject))
 			if err != nil {
-				return fmt.Errorf("failed to set project environment to %s: %v", newproject, err)
+				klog.Errorf("failed to set project environment to %s: %v", oldProject, err)
 			}
-
-			defer func() {
-				err = setEnvProject(string(oldProject))
-				if err != nil {
-					klog.Errorf("failed to set project environment to %s: %v", oldProject, err)
-				}
-			}()
-			project = newproject
-		}
+		}()
+		project = newproject
 		if *doDriverBuild {
 			*stagingImage = fmt.Sprintf("gcr.io/%s/gcp-persistent-disk-csi-driver", strings.TrimSpace(string(project)))
 		}
@@ -256,6 +251,7 @@ func handle() error {
 
 	// Build and push the driver, if required. Defer the driver image deletion.
 	if *doDriverBuild {
+		klog.Infof("Building GCE PD CSI Driver")
 		err := pushImage(testParams.pkgDir, *stagingImage, testParams.stagingVersion, testParams.platform)
 		if err != nil {
 			return fmt.Errorf("failed pushing image: %v", err)
@@ -326,9 +322,9 @@ func handle() error {
 		var err error = nil
 		switch *deploymentStrat {
 		case "gce":
-			err = clusterUpGCE(testParams.k8sSourceDir, *gceZone, *numNodes, testParams.imageType)
+			err = clusterUpGCE(testParams.k8sSourceDir, *gceZone, *numNodes, *numWindowsNodes, testParams.imageType)
 		case "gke":
-			err = clusterUpGKE(*gceZone, *gceRegion, *numNodes, testParams.imageType, testParams.useGKEManagedDriver)
+			err = clusterUpGKE(*gceZone, *gceRegion, *numNodes, *numWindowsNodes, testParams.imageType, testParams.useGKEManagedDriver)
 		default:
 			err = fmt.Errorf("deployment-strategy must be set to 'gce' or 'gke', but is: %s", testParams.deploymentStrategy)
 		}
@@ -360,6 +356,8 @@ func handle() error {
 	// For windows cluster, when cluster is up, all Windows nodes are tainted with NoSchedule to avoid linux pods
 	// being scheduled to Windows nodes. When running windows tests, we need to remove the taint.
 	if testParams.platform == "windows" {
+		klog.Infof("Removing taints from all windows nodes.")
+
 		nodesCmd := exec.Command("kubectl", "get", "nodes", "-l", "kubernetes.io/os=windows", "-o", "name")
 		out, err := nodesCmd.CombinedOutput()
 		if err != nil {
@@ -428,6 +426,7 @@ func handle() error {
 	// unschedulable.
 	testParams.allowedNotReadyNodes = 0
 	if *platform == "windows" {
+		klog.Infof("Tainting linux nodes")
 		nodesCmd := exec.Command("kubectl", "get", "nodes", "-l", "kubernetes.io/os=linux", "-o", "name")
 		out, err := nodesCmd.CombinedOutput()
 		if err != nil {
@@ -664,7 +663,6 @@ func runTestsWithConfig(testParams *testParameters, testConfigArg, reportPrefix
 	testArgs := fmt.Sprintf("%s %s", ginkgoArgs, testConfigArg)
 
 	// kubetest2 flags
-
 	var runID string
 	if uid, exists := os.LookupEnv("PROW_JOB_ID"); exists && uid != "" {
 		// reuse uid for CI use cases
@@ -673,20 +671,27 @@ func runTestsWithConfig(testParams *testParameters, testConfigArg, reportPrefix
 		runID = string(uuid.NewUUID())
 	}
 
+	// Usage: kubetest2 <deployer> [Flags] [DeployerFlags] -- [TesterArgs]
+	// [Flags]
 	kubeTest2Args := []string{
 		*deploymentStrat,
 		fmt.Sprintf("--run-id=%s", runID),
 		"--test=ginkgo",
 	}
+
+	// [DeployerFlags]
 	kubeTest2Args = append(kubeTest2Args, testParams.cloudProviderArgs...)
 	if kubetestDumpDir != "" {
 		kubeTest2Args = append(kubeTest2Args, fmt.Sprintf("--artifacts=%s", kubetestDumpDir))
 	}
+
 	kubeTest2Args = append(kubeTest2Args, "--")
+
+	// [TesterArgs]
 	if len(*testVersion) != 0 {
 		if *testVersion == "master" {
-			// the kubernetes binaries should've already been built above
-			// or by the user if --localK8sDir was set, these binaries should be copied to the
+			// the kubernetes binaries should've already been built above because of `--kube-version`
+			// or by the user if --local-k8s-dir was set, these binaries should be copied to the
 			// path sent to kubetest2 through its --artifacts path
 
 			// pkg/_artifacts is the default value that kubetests uses for --artifacts

test/k8s-integration/utils.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"fmt"
 	"io/ioutil"
 	"os"
 	"os/exec"
@@ -14,8 +13,9 @@ func runCommand(action string, cmd *exec.Cmd) error {
 	cmd.Stdin = os.Stdin
 	cmd.Stderr = os.Stderr
 
-	fmt.Printf("%s\n", action)
-	fmt.Printf("%s\n", cmd.Args)
+	klog.Infof("%s", action)
+	klog.Infof("cmd env=%v", cmd.Env)
+	klog.Infof("cmd args=%s", cmd.Args)
 
 	err := cmd.Start()
 	if err != nil {

test/run-k8s-integration-ci.sh

@@ -1,11 +1,12 @@
-#!/bin/bash -x
+#!/bin/bash
 
 # Optional environment variables
 # GCE_PD_OVERLAY_NAME: which Kustomize overlay to deploy with
 # GCE_PD_DO_DRIVER_BUILD: if set, don't build the driver from source and just
 #   use the driver version from the overlay
 # GCE_PD_BOSKOS_RESOURCE_TYPE: name of the boskos resource type to reserve
 
+set -o xtrace
 set -o nounset
 set -o errexit
 

test/run-k8s-integration.sh

@@ -6,6 +6,7 @@
 #   use the driver version from the overlay
 # GCE_PD_BOSKOS_RESOURCE_TYPE: name of the boskos resource type to reserve
 
+set -o xtrace
 set -o nounset
 set -o errexit
 

test/run-windows-k8s-integration.sh

@@ -11,7 +11,7 @@ set -o nounset
 set -o errexit
 
 readonly PKGDIR=${GOPATH}/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver
-readonly overlay_name="${GCE_PD_OVERLAY_NAME:-noauth}"
+readonly overlay_name="${GCE_PD_OVERLAY_NAME:-stable-master}"
 readonly do_driver_build="${GCE_PD_DO_DRIVER_BUILD:-true}"
 readonly deployment_strategy=${DEPLOYMENT_STRATEGY:-gce}
 readonly kube_version=${GCE_PD_KUBE_VERSION:-master}
@@ -22,7 +22,7 @@ readonly use_kubetest2=${USE_KUBETEST2:-true}
 readonly num_windows_nodes=${NUM_WINDOWS_NODES:-3}
 
 # build platforms for `make quick-release`
-export KUBE_BUILD_PLATFORMS="linux/amd64 windows/amd64"
+export KUBE_BUILD_PLATFORMS=${KUBE_BUILD_PLATFORMS:-"linux/amd64 windows/amd64"}
 
 make -C "${PKGDIR}" test-k8s-integration
 
@@ -33,21 +33,8 @@ if [ "$use_kubetest2" = true ]; then
     go install sigs.k8s.io/kubetest2/kubetest2-tester-ginkgo@latest;
 fi
 
-# TODO(mauriciopoppe): remove this assignment
-E2E_GOOGLE_APPLICATION_CREDENTIALS=sa
-
-# TODO(mauriciopoppe): change run-in-prow=true
-
-# TODO(mauriciopoppe): change overlay back to stable-master
-
-# TODO(mauriciopoppe): remove --staging-image and this flag
-GCE_PD_CSI_STAGING_IMAGE=gcr.io/mauriciopoppe-gke-dev/gcp-compute-persistent-disk-csi-driver
-
-# TODO(mauriciopoppe): change to --do-driver-build=${do_driver_build} \
-
 base_cmd="${PKGDIR}/bin/k8s-integration-test \
-    --staging-image="${GCE_PD_CSI_STAGING_IMAGE}" \
-    --run-in-prow=false \
+    --run-in-prow=true \
     --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \
     --deployment-strategy=${deployment_strategy} \
     --gce-zone=${gce_zone} \
@@ -57,7 +44,7 @@ base_cmd="${PKGDIR}/bin/k8s-integration-test \
     --num-nodes=1 \
     --num-windows-nodes=${num_windows_nodes} \
     --teardown-driver=${teardown_driver} \
-    --do-driver-build=true \
+    --do-driver-build=${do_driver_build} \
     --deploy-overlay-name=${overlay_name} \
     --test-version=${test_version} \
     --kube-version=${kube_version} \