Use precompiled stable release of Kubernetes by default when running integration test

test/k8s-integration/cluster.go

@@ -223,11 +223,81 @@ func clusterUpGKE(gceZone, gceRegion string, numNodes int, numWindowsNodes int,
 	return nil
 }
 
-func downloadKubernetesSource(pkgDir, k8sIoDir, kubeVersion string) error {
-	k8sDir := filepath.Join(k8sIoDir, "kubernetes")
-	klog.Infof("Downloading Kubernetes source v=%s to path=%s", kubeVersion, k8sIoDir)
+func downloadTarball(k8sDir, releaseVersion, subDir, tarballName string) error {
+	tarballPath := filepath.Join(k8sDir, subDir)
+	if err := os.MkdirAll(tarballPath, 0777); err != nil {
+		return err
+	}
+	tarballOutput := filepath.Join(tarballPath, tarballName)
+	downloadUrl := fmt.Sprintf("https://dl.k8s.io/release/%s/%s", releaseVersion, tarballName)
+	klog.Infof("Downloading tarball %s to path=%s from url=%s", tarballName, tarballPath, downloadUrl)
+	_, err := exec.Command("curl", "-Lsf", "--output", tarballOutput, downloadUrl).CombinedOutput()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func downloadAndExtractSrcTarball(k8sDir, releaseVersion, tarballName string) error {
+	tarballPath := filepath.Join(k8sDir, tarballName)
+	if err := downloadTarball(k8sDir, releaseVersion, "", tarballName); err != nil {
+		return err
+	}
+	out, err := exec.Command("tar", "xf", tarballPath, "-C", k8sDir).CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("Failed to extract src tarball %s: %s: %s", tarballName, out, err)
+	}
+	return nil
+}
+
+func getReleaseVersionFromKubeVersion(kubeVersion string) (string, error) {
+	releaseVersion := fmt.Sprintf("v%s", kubeVersion)
+	if kubeVersion == "stable" || kubeVersion == "latest" {
+		// See https://kubernetes.io/releases/download/
+		out, err := exec.Command("curl", "-Lsf", "https://dl.k8s.io/release/stable.txt").CombinedOutput()
+		if err != nil {
+			return "", err
+		}
+		releaseVersion = string(out)
+	}
+	return releaseVersion, nil
+}
+
+func downloadKubernetesRelease(k8sDir, kubeVersion, platform, arch string) error {
+	releaseVersion, err := getReleaseVersionFromKubeVersion(kubeVersion)
+	if err != nil {
+		return err
+	}
+
+	// Download precompiled tarballs
+	serverTarballName := fmt.Sprintf("kubernetes-server-%s-%s.tar.gz", platform, arch)
+	if err := downloadTarball(k8sDir, releaseVersion, "server", serverTarballName); err != nil {
+		return err
+	}
+
+	manifestsTarballName := fmt.Sprintf("kubernetes-manifests.tar.gz")
+	if err := downloadTarball(k8sDir, releaseVersion, "server", manifestsTarballName); err != nil {
+		return err
+	}
+
+	nodeTarballName := fmt.Sprintf("kubernetes-node-%s-%s.tar.gz", platform, arch)
+	if err := downloadTarball(k8sDir, releaseVersion, "node", nodeTarballName); err != nil {
+		return err
+	}
 
-	if err := os.MkdirAll(k8sIoDir, 0777); err != nil {
+	clientTarballName := fmt.Sprintf("kubernetes-client-%s-%s.tar.gz", platform, arch)
+	if err := downloadTarball(k8sDir, releaseVersion, "client", clientTarballName); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func downloadKubernetesSource(k8sDir, kubeVersion string) error {
+	klog.Infof("Downloading Kubernetes source v=%s to path=%s", kubeVersion, k8sDir)
+
+	if err := os.MkdirAll(k8sDir, 0777); err != nil {
 		return err
 	}
 	if err := os.RemoveAll(k8sDir); err != nil {
@@ -248,12 +318,17 @@ func downloadKubernetesSource(pkgDir, k8sIoDir, kubeVersion string) error {
 			return fmt.Errorf("failed to clone kubernetes master: %s, err: %v", out, err.Error())
 		}
 	} else {
-		// Shallow clone of a release branch.
-		vKubeVersion := "v" + kubeVersion
-		klog.Infof("shallow clone of k8s %s", vKubeVersion)
-		out, err := exec.Command("git", "clone", "--depth", "1", "https://github.com/kubernetes/kubernetes", k8sDir).CombinedOutput()
+		releaseVersion, err := getReleaseVersionFromKubeVersion(kubeVersion)
 		if err != nil {
-			return fmt.Errorf("failed to clone kubernetes %s: %s, err: %v", vKubeVersion, out, err.Error())
+			return err
+		}
+
+		// Download versioned source.
+		if err := downloadAndExtractSrcTarball(k8sDir, releaseVersion, "kubernetes-src.tar.gz"); err != nil {
+			return err
+		}
+		if err := downloadAndExtractSrcTarball(k8sDir, releaseVersion, "kubernetes.tar.gz"); err != nil {
+			return err
 		}
 	}
 	return nil

test/k8s-integration/main.go

@@ -28,6 +28,7 @@ import (
 	apimachineryversion "k8s.io/apimachinery/pkg/util/version"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 	"k8s.io/klog/v2"
+	"k8s.io/utils/strings/slices"
 	testutils "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/e2e/utils"
 )
 
@@ -36,7 +37,8 @@ var (
 	teardownCluster      = flag.Bool("teardown-cluster", true, "teardown the cluster after the e2e test")
 	teardownDriver       = flag.Bool("teardown-driver", true, "teardown the driver after the e2e test")
 	bringupCluster       = flag.Bool("bringup-cluster", true, "build kubernetes and bringup a cluster")
-	platform             = flag.String("platform", "linux", "platform that the tests will be run, either linux or windows")
+	platform             = flag.String("platform", "linux", "platform that the tests will be run on, either linux or windows")
+	arch                 = flag.String("arch", "amd64", "architecture that the tests will be run on (eg: amd64/arm64)")
 	gceZone              = flag.String("gce-zone", "", "zone that the gce k8s cluster is created/found in")
 	gceRegion            = flag.String("gce-region", "", "region that gke regional cluster should be created in")
 	kubeVersion          = flag.String("kube-version", "", "version of Kubernetes to download and use for the cluster")
@@ -65,6 +67,7 @@ var (
 	saFile              = flag.String("service-account-file", "", "path of service account file")
 	deployOverlayName   = flag.String("deploy-overlay-name", "", "which kustomize overlay to deploy the driver with")
 	doDriverBuild       = flag.Bool("do-driver-build", true, "building the driver from source")
+	doK8sBuild          = flag.Bool("do-k8s-build", true, "building the driver from source. If false, will fetch precompiled artifacts")
 	useGKEManagedDriver = flag.Bool("use-gke-managed-driver", false, "use GKE managed PD CSI driver for the tests")
 
 	// Test flags
@@ -73,6 +76,8 @@ var (
 
 	useKubeTest2 = flag.Bool("use-kubetest2", false, "use kubetest2 to run e2e tests")
 	parallel     = flag.Int("parallel", 4, "the number of parallel tests setting for ginkgo parallelism")
+
+	allowedVersionTags = []string{"master", "stable", "latest"}
 )
 
 const (
@@ -191,12 +196,48 @@ func main() {
 		klog.Fatalf("num-windows-nodes must be set if the platform is windows")
 	}
 
+	if *kubeVersion == "master" && !*doK8sBuild {
+		klog.Fatalf("must set do-k8s-build=true when kubeVersion=%s", *kubeVersion)
+	}
+
+	if len(*kubeVersion) != 0 {
+		if !slices.Contains(allowedVersionTags, *kubeVersion) {
+			if _, err := parseVersion(*kubeVersion); err != nil {
+				klog.Fatalf("invalid kube-version %s", *kubeVersion)
+			}
+		}
+	}
+
+	if len(*testVersion) != 0 {
+		if !slices.Contains(allowedVersionTags, *testVersion) {
+			if _, err := parseVersion(*testVersion); err != nil {
+				klog.Fatalf("invalid test-version %s", *testVersion)
+			}
+		}
+	}
+
 	err := handle()
 	if err != nil {
 		klog.Fatalf("Failed to run integration test: %w", err)
 	}
 }
 
+func buildTestingBinaries(k8sDir string) error {
+	if err := buildKubernetes(k8sDir, "WHAT=test/e2e/e2e.test"); err != nil {
+		return fmt.Errorf("failed to build Kubernetes e2e: %v", err.Error())
+	}
+
+	// kubetest relies on ginkgo and kubectl already built in the test k8s directory
+	if err := buildKubernetes(k8sDir, "ginkgo"); err != nil {
+		return fmt.Errorf("failed to build gingko: %v", err.Error())
+	}
+
+	if err := buildKubernetes(k8sDir, "kubectl"); err != nil {
+		return fmt.Errorf("failed to build kubectl: %v", err.Error())
+	}
+	return nil
+}
+
 func handle() error {
 	oldmask := syscall.Umask(0000)
 	defer syscall.Umask(oldmask)
@@ -266,21 +307,32 @@ func handle() error {
 		}()
 	}
 
-	// Create temporary directories for kubernetes builds
+	// Create temporary directories for kubernetes source/build artifacts
 	testParams.testParentDir = generateUniqueTmpDir()
 	defer removeDir(testParams.testParentDir)
 
 	// If kube version is set, then download and build Kubernetes for cluster creation
 	// Otherwise, either GKE or a prebuild local K8s dir is being used
 	if len(*kubeVersion) != 0 {
 		testParams.k8sSourceDir = filepath.Join(testParams.testParentDir, "kubernetes")
-		err := downloadKubernetesSource(testParams.pkgDir, testParams.testParentDir, *kubeVersion)
-		if err != nil {
+		klog.Infof("Downloading Kubernetes source from: %s", *kubeVersion)
+		if err := downloadKubernetesSource(testParams.k8sSourceDir, *kubeVersion); err != nil {
 			return fmt.Errorf("failed to download Kubernetes source: %v", err.Error())
 		}
-		err = buildKubernetes(testParams.k8sSourceDir, "quick-release")
-		if err != nil {
-			return fmt.Errorf("failed to build Kubernetes: %v", err.Error())
+
+		if *doK8sBuild {
+			klog.Info("Building Kubernetes source")
+			if err := buildKubernetes(testParams.k8sSourceDir, "quick-release"); err != nil {
+				return fmt.Errorf("failed to build Kubernetes: %v", err.Error())
+			}
+		} else {
+			klog.Info("Fetching precompiled Kubernetes artifacts for %s/%s", *platform, *arch)
+			if err := downloadKubernetesRelease(testParams.k8sSourceDir, *kubeVersion, *platform, *arch); err != nil {
+				return fmt.Errorf("failed to download Kubernetes release: %v", err.Error())
+			}
+			if err := buildTestingBinaries(testParams.k8sSourceDir); err != nil {
+				return err
+			}
 		}
 	} else {
 		testParams.k8sSourceDir = *localK8sDir
@@ -290,22 +342,13 @@ func handle() error {
 	// Otherwise, either kube version is set (which implies GCE) or a local K8s dir is being used.
 	if !*useKubeTest2 && len(*testVersion) != 0 && *testVersion != *kubeVersion {
 		testParams.k8sSourceDir = filepath.Join(testParams.testParentDir, "kubernetes")
-		err := downloadKubernetesSource(testParams.pkgDir, testParams.testParentDir, *testVersion)
-		if err != nil {
+		// Overlay the Kubernetes source
+		if err := downloadKubernetesSource(testParams.k8sSourceDir, *testVersion); err != nil {
 			return fmt.Errorf("failed to download Kubernetes source: %v", err.Error())
 		}
-		err = buildKubernetes(testParams.k8sSourceDir, "WHAT=test/e2e/e2e.test")
-		if err != nil {
-			return fmt.Errorf("failed to build Kubernetes e2e: %v", err.Error())
-		}
-		// kubetest relies on ginkgo and kubectl already built in the test k8s directory
-		err = buildKubernetes(testParams.k8sSourceDir, "ginkgo")
-		if err != nil {
-			return fmt.Errorf("failed to build gingko: %v", err.Error())
-		}
-		err = buildKubernetes(testParams.k8sSourceDir, "kubectl")
-		if err != nil {
-			return fmt.Errorf("failed to build kubectl: %v", err.Error())
+		klog.Infof("Building Kubernetes Testing binaries: %s", *kubeVersion)
+		if err := buildTestingBinaries(testParams.k8sSourceDir); err != nil {
+			return err
 		}
 	}
 
@@ -779,7 +822,17 @@ func runTestsWithConfig(testParams *testParameters, testConfigArg, reportPrefix
 			}
 			kubeTest2Args = append(kubeTest2Args, "--use-built-binaries")
 		} else {
-			kubeTest2Args = append(kubeTest2Args, fmt.Sprintf("--test-package-marker=latest-%s.txt", *testVersion))
+			testResourceVersion := *testVersion
+			if *testVersion != "stable" && *testVersion != "latest" {
+				// Find the minor version
+				v, err := parseVersion(*testVersion)
+				if err != nil {
+					// Note, this shouldn't happen, as we check flags in main().
+					return fmt.Errorf("failed to parse --test-version")
+				}
+				testResourceVersion = fmt.Sprintf("stable-%s", v.minorVersion())
+			}
+			kubeTest2Args = append(kubeTest2Args, fmt.Sprintf("--test-package-marker=%s.txt", testResourceVersion))
 		}
 	}
 	kubeTest2Args = append(kubeTest2Args, fmt.Sprintf("--focus-regex=%s", focus))

test/k8s-integration/version.go

@@ -54,6 +54,10 @@ func (v *version) isGKEExtraVersion(extrastr string) bool {
 	return gkeExtraVersionRegex.MatchString(extrastr)
 }
 
+func (v *version) minorVersion() string {
+	return fmt.Sprintf("%d.%d", v.major(), v.minor())
+}
+
 func extractGKEExtraVersion(extra string) (int, error) {
 	m := gkeExtraVersionRegex.FindStringSubmatch(extra)
 	if len(m) != 2 {

test/run-k8s-integration.sh

@@ -14,10 +14,11 @@ readonly PKGDIR=${GOPATH}/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver
 readonly overlay_name="${GCE_PD_OVERLAY_NAME:-stable-master}"
 readonly boskos_resource_type="${GCE_PD_BOSKOS_RESOURCE_TYPE:-gce-project}"
 readonly do_driver_build="${GCE_PD_DO_DRIVER_BUILD:-true}"
+readonly do_k8s_build="${GCE_PD_DO_K8S_BUILD:-false}"
 readonly deployment_strategy=${DEPLOYMENT_STRATEGY:-gce}
 readonly gke_cluster_version=${GKE_CLUSTER_VERSION:-latest}
-readonly kube_version=${GCE_PD_KUBE_VERSION:-master}
-readonly test_version=${TEST_VERSION:-master}
+readonly kube_version=${GCE_PD_KUBE_VERSION:-stable}
+readonly test_version=${TEST_VERSION:-stable}
 readonly gce_zone=${GCE_CLUSTER_ZONE:-us-central1-b}
 readonly gce_region=${GCE_CLUSTER_REGION:-}
 readonly image_type=${IMAGE_TYPE:-cos_containerd}
@@ -44,7 +45,8 @@ fi
 
 base_cmd="${PKGDIR}/bin/k8s-integration-test \
             --run-in-prow=true --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \
-            --do-driver-build=${do_driver_build} --teardown-driver=${teardown_driver} --boskos-resource-type=${boskos_resource_type} \
+            --do-driver-build=${do_driver_build} --teardown-driver=${teardown_driver} \
+            --do-k8s-build=${do_k8s_build} --boskos-resource-type=${boskos_resource_type} \
             --storageclass-files=sc-standard.yaml --snapshotclass-files=pd-volumesnapshotclass.yaml \
             --deployment-strategy=${deployment_strategy} --test-version=${test_version} \
             --num-nodes=3 --image-type=${image_type} --use-kubetest2=${use_kubetest2}"