From 776f51a2f3bee86d50bc412e192b3716c98c4970 Mon Sep 17 00:00:00 2001 From: Peter Schuurman Date: Fri, 11 Aug 2023 19:21:36 -0700 Subject: [PATCH] Use precompiled stable release of Kubernetes by default when running integration test --- test/k8s-integration/cluster.go | 93 ++++++++++++++++++++++++++++--- test/k8s-integration/main.go | 97 +++++++++++++++++++++++++-------- test/k8s-integration/version.go | 4 ++ test/run-k8s-integration.sh | 8 ++- 4 files changed, 168 insertions(+), 34 deletions(-) diff --git a/test/k8s-integration/cluster.go b/test/k8s-integration/cluster.go index 3d74f1594..5f63cbaaf 100644 --- a/test/k8s-integration/cluster.go +++ b/test/k8s-integration/cluster.go @@ -223,11 +223,81 @@ func clusterUpGKE(gceZone, gceRegion string, numNodes int, numWindowsNodes int, return nil } -func downloadKubernetesSource(pkgDir, k8sIoDir, kubeVersion string) error { - k8sDir := filepath.Join(k8sIoDir, "kubernetes") - klog.Infof("Downloading Kubernetes source v=%s to path=%s", kubeVersion, k8sIoDir) +func downloadTarball(k8sDir, releaseVersion, subDir, tarballName string) error { + tarballPath := filepath.Join(k8sDir, subDir) + if err := os.MkdirAll(tarballPath, 0777); err != nil { + return err + } + tarballOutput := filepath.Join(tarballPath, tarballName) + downloadUrl := fmt.Sprintf("https://dl.k8s.io/release/%s/%s", releaseVersion, tarballName) + klog.Infof("Downloading tarball %s to path=%s from url=%s", tarballName, tarballPath, downloadUrl) + _, err := exec.Command("curl", "-Lsf", "--output", tarballOutput, downloadUrl).CombinedOutput() + if err != nil { + return err + } + + return nil +} + +func downloadAndExtractSrcTarball(k8sDir, releaseVersion, tarballName string) error { + tarballPath := filepath.Join(k8sDir, tarballName) + if err := downloadTarball(k8sDir, releaseVersion, "", tarballName); err != nil { + return err + } + out, err := exec.Command("tar", "xf", tarballPath, "-C", k8sDir).CombinedOutput() + if err != nil { + return fmt.Errorf("Failed to extract src tarball %s: %s: %s", tarballName, out, err) + } + return nil +} + +func getReleaseVersionFromKubeVersion(kubeVersion string) (string, error) { + releaseVersion := fmt.Sprintf("v%s", kubeVersion) + if kubeVersion == "stable" || kubeVersion == "latest" { + // See https://kubernetes.io/releases/download/ + out, err := exec.Command("curl", "-Lsf", "https://dl.k8s.io/release/stable.txt").CombinedOutput() + if err != nil { + return "", err + } + releaseVersion = string(out) + } + return releaseVersion, nil +} + +func downloadKubernetesRelease(k8sDir, kubeVersion, platform, arch string) error { + releaseVersion, err := getReleaseVersionFromKubeVersion(kubeVersion) + if err != nil { + return err + } + + // Download precompiled tarballs + serverTarballName := fmt.Sprintf("kubernetes-server-%s-%s.tar.gz", platform, arch) + if err := downloadTarball(k8sDir, releaseVersion, "server", serverTarballName); err != nil { + return err + } + + manifestsTarballName := fmt.Sprintf("kubernetes-manifests.tar.gz") + if err := downloadTarball(k8sDir, releaseVersion, "server", manifestsTarballName); err != nil { + return err + } + + nodeTarballName := fmt.Sprintf("kubernetes-node-%s-%s.tar.gz", platform, arch) + if err := downloadTarball(k8sDir, releaseVersion, "node", nodeTarballName); err != nil { + return err + } - if err := os.MkdirAll(k8sIoDir, 0777); err != nil { + clientTarballName := fmt.Sprintf("kubernetes-client-%s-%s.tar.gz", platform, arch) + if err := downloadTarball(k8sDir, releaseVersion, "client", clientTarballName); err != nil { + return err + } + + return nil +} + +func downloadKubernetesSource(k8sDir, kubeVersion string) error { + klog.Infof("Downloading Kubernetes source v=%s to path=%s", kubeVersion, k8sDir) + + if err := os.MkdirAll(k8sDir, 0777); err != nil { return err } if err := os.RemoveAll(k8sDir); err != nil { @@ -248,12 +318,17 @@ func downloadKubernetesSource(pkgDir, k8sIoDir, kubeVersion string) error { return fmt.Errorf("failed to clone kubernetes master: %s, err: %v", out, err.Error()) } } else { - // Shallow clone of a release branch. - vKubeVersion := "v" + kubeVersion - klog.Infof("shallow clone of k8s %s", vKubeVersion) - out, err := exec.Command("git", "clone", "--depth", "1", "https://github.com/kubernetes/kubernetes", k8sDir).CombinedOutput() + releaseVersion, err := getReleaseVersionFromKubeVersion(kubeVersion) if err != nil { - return fmt.Errorf("failed to clone kubernetes %s: %s, err: %v", vKubeVersion, out, err.Error()) + return err + } + + // Download versioned source. + if err := downloadAndExtractSrcTarball(k8sDir, releaseVersion, "kubernetes-src.tar.gz"); err != nil { + return err + } + if err := downloadAndExtractSrcTarball(k8sDir, releaseVersion, "kubernetes.tar.gz"); err != nil { + return err } } return nil diff --git a/test/k8s-integration/main.go b/test/k8s-integration/main.go index db0fcfc0a..67aa5a461 100644 --- a/test/k8s-integration/main.go +++ b/test/k8s-integration/main.go @@ -28,6 +28,7 @@ import ( apimachineryversion "k8s.io/apimachinery/pkg/util/version" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" "k8s.io/klog/v2" + "k8s.io/utils/strings/slices" testutils "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/e2e/utils" ) @@ -36,7 +37,8 @@ var ( teardownCluster = flag.Bool("teardown-cluster", true, "teardown the cluster after the e2e test") teardownDriver = flag.Bool("teardown-driver", true, "teardown the driver after the e2e test") bringupCluster = flag.Bool("bringup-cluster", true, "build kubernetes and bringup a cluster") - platform = flag.String("platform", "linux", "platform that the tests will be run, either linux or windows") + platform = flag.String("platform", "linux", "platform that the tests will be run on, either linux or windows") + arch = flag.String("arch", "amd64", "architecture that the tests will be run on (eg: amd64/arm64)") gceZone = flag.String("gce-zone", "", "zone that the gce k8s cluster is created/found in") gceRegion = flag.String("gce-region", "", "region that gke regional cluster should be created in") kubeVersion = flag.String("kube-version", "", "version of Kubernetes to download and use for the cluster") @@ -65,6 +67,7 @@ var ( saFile = flag.String("service-account-file", "", "path of service account file") deployOverlayName = flag.String("deploy-overlay-name", "", "which kustomize overlay to deploy the driver with") doDriverBuild = flag.Bool("do-driver-build", true, "building the driver from source") + doK8sBuild = flag.Bool("do-k8s-build", true, "building the driver from source. If false, will fetch precompiled artifacts") useGKEManagedDriver = flag.Bool("use-gke-managed-driver", false, "use GKE managed PD CSI driver for the tests") // Test flags @@ -73,6 +76,8 @@ var ( useKubeTest2 = flag.Bool("use-kubetest2", false, "use kubetest2 to run e2e tests") parallel = flag.Int("parallel", 4, "the number of parallel tests setting for ginkgo parallelism") + + allowedVersionTags = []string{"master", "stable", "latest"} ) const ( @@ -191,12 +196,48 @@ func main() { klog.Fatalf("num-windows-nodes must be set if the platform is windows") } + if *kubeVersion == "master" && !*doK8sBuild { + klog.Fatalf("must set do-k8s-build=true when kubeVersion=%s", *kubeVersion) + } + + if len(*kubeVersion) != 0 { + if !slices.Contains(allowedVersionTags, *kubeVersion) { + if _, err := parseVersion(*kubeVersion); err != nil { + klog.Fatalf("invalid kube-version %s", *kubeVersion) + } + } + } + + if len(*testVersion) != 0 { + if !slices.Contains(allowedVersionTags, *testVersion) { + if _, err := parseVersion(*testVersion); err != nil { + klog.Fatalf("invalid test-version %s", *testVersion) + } + } + } + err := handle() if err != nil { klog.Fatalf("Failed to run integration test: %w", err) } } +func buildTestingBinaries(k8sDir string) error { + if err := buildKubernetes(k8sDir, "WHAT=test/e2e/e2e.test"); err != nil { + return fmt.Errorf("failed to build Kubernetes e2e: %v", err.Error()) + } + + // kubetest relies on ginkgo and kubectl already built in the test k8s directory + if err := buildKubernetes(k8sDir, "ginkgo"); err != nil { + return fmt.Errorf("failed to build gingko: %v", err.Error()) + } + + if err := buildKubernetes(k8sDir, "kubectl"); err != nil { + return fmt.Errorf("failed to build kubectl: %v", err.Error()) + } + return nil +} + func handle() error { oldmask := syscall.Umask(0000) defer syscall.Umask(oldmask) @@ -266,7 +307,7 @@ func handle() error { }() } - // Create temporary directories for kubernetes builds + // Create temporary directories for kubernetes source/build artifacts testParams.testParentDir = generateUniqueTmpDir() defer removeDir(testParams.testParentDir) @@ -274,13 +315,24 @@ func handle() error { // Otherwise, either GKE or a prebuild local K8s dir is being used if len(*kubeVersion) != 0 { testParams.k8sSourceDir = filepath.Join(testParams.testParentDir, "kubernetes") - err := downloadKubernetesSource(testParams.pkgDir, testParams.testParentDir, *kubeVersion) - if err != nil { + klog.Infof("Downloading Kubernetes source from: %s", *kubeVersion) + if err := downloadKubernetesSource(testParams.k8sSourceDir, *kubeVersion); err != nil { return fmt.Errorf("failed to download Kubernetes source: %v", err.Error()) } - err = buildKubernetes(testParams.k8sSourceDir, "quick-release") - if err != nil { - return fmt.Errorf("failed to build Kubernetes: %v", err.Error()) + + if *doK8sBuild { + klog.Info("Building Kubernetes source") + if err := buildKubernetes(testParams.k8sSourceDir, "quick-release"); err != nil { + return fmt.Errorf("failed to build Kubernetes: %v", err.Error()) + } + } else { + klog.Info("Fetching precompiled Kubernetes artifacts for %s/%s", *platform, *arch) + if err := downloadKubernetesRelease(testParams.k8sSourceDir, *kubeVersion, *platform, *arch); err != nil { + return fmt.Errorf("failed to download Kubernetes release: %v", err.Error()) + } + if err := buildTestingBinaries(testParams.k8sSourceDir); err != nil { + return err + } } } else { testParams.k8sSourceDir = *localK8sDir @@ -290,22 +342,13 @@ func handle() error { // Otherwise, either kube version is set (which implies GCE) or a local K8s dir is being used. if !*useKubeTest2 && len(*testVersion) != 0 && *testVersion != *kubeVersion { testParams.k8sSourceDir = filepath.Join(testParams.testParentDir, "kubernetes") - err := downloadKubernetesSource(testParams.pkgDir, testParams.testParentDir, *testVersion) - if err != nil { + // Overlay the Kubernetes source + if err := downloadKubernetesSource(testParams.k8sSourceDir, *testVersion); err != nil { return fmt.Errorf("failed to download Kubernetes source: %v", err.Error()) } - err = buildKubernetes(testParams.k8sSourceDir, "WHAT=test/e2e/e2e.test") - if err != nil { - return fmt.Errorf("failed to build Kubernetes e2e: %v", err.Error()) - } - // kubetest relies on ginkgo and kubectl already built in the test k8s directory - err = buildKubernetes(testParams.k8sSourceDir, "ginkgo") - if err != nil { - return fmt.Errorf("failed to build gingko: %v", err.Error()) - } - err = buildKubernetes(testParams.k8sSourceDir, "kubectl") - if err != nil { - return fmt.Errorf("failed to build kubectl: %v", err.Error()) + klog.Infof("Building Kubernetes Testing binaries: %s", *kubeVersion) + if err := buildTestingBinaries(testParams.k8sSourceDir); err != nil { + return err } } @@ -779,7 +822,17 @@ func runTestsWithConfig(testParams *testParameters, testConfigArg, reportPrefix } kubeTest2Args = append(kubeTest2Args, "--use-built-binaries") } else { - kubeTest2Args = append(kubeTest2Args, fmt.Sprintf("--test-package-marker=latest-%s.txt", *testVersion)) + testResourceVersion := *testVersion + if *testVersion != "stable" && *testVersion != "latest" { + // Find the minor version + v, err := parseVersion(*testVersion) + if err != nil { + // Note, this shouldn't happen, as we check flags in main(). + return fmt.Errorf("failed to parse --test-version") + } + testResourceVersion = fmt.Sprintf("stable-%s", v.minorVersion()) + } + kubeTest2Args = append(kubeTest2Args, fmt.Sprintf("--test-package-marker=%s.txt", testResourceVersion)) } } kubeTest2Args = append(kubeTest2Args, fmt.Sprintf("--focus-regex=%s", focus)) diff --git a/test/k8s-integration/version.go b/test/k8s-integration/version.go index 0cbca8ee2..a0f4c4ff2 100644 --- a/test/k8s-integration/version.go +++ b/test/k8s-integration/version.go @@ -54,6 +54,10 @@ func (v *version) isGKEExtraVersion(extrastr string) bool { return gkeExtraVersionRegex.MatchString(extrastr) } +func (v *version) minorVersion() string { + return fmt.Sprintf("%d.%d", v.major(), v.minor()) +} + func extractGKEExtraVersion(extra string) (int, error) { m := gkeExtraVersionRegex.FindStringSubmatch(extra) if len(m) != 2 { diff --git a/test/run-k8s-integration.sh b/test/run-k8s-integration.sh index 1b86f8491..a72db9eb0 100755 --- a/test/run-k8s-integration.sh +++ b/test/run-k8s-integration.sh @@ -14,10 +14,11 @@ readonly PKGDIR=${GOPATH}/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver readonly overlay_name="${GCE_PD_OVERLAY_NAME:-stable-master}" readonly boskos_resource_type="${GCE_PD_BOSKOS_RESOURCE_TYPE:-gce-project}" readonly do_driver_build="${GCE_PD_DO_DRIVER_BUILD:-true}" +readonly do_k8s_build="${GCE_PD_DO_K8S_BUILD:-false}" readonly deployment_strategy=${DEPLOYMENT_STRATEGY:-gce} readonly gke_cluster_version=${GKE_CLUSTER_VERSION:-latest} -readonly kube_version=${GCE_PD_KUBE_VERSION:-master} -readonly test_version=${TEST_VERSION:-master} +readonly kube_version=${GCE_PD_KUBE_VERSION:-stable} +readonly test_version=${TEST_VERSION:-stable} readonly gce_zone=${GCE_CLUSTER_ZONE:-us-central1-b} readonly gce_region=${GCE_CLUSTER_REGION:-} readonly image_type=${IMAGE_TYPE:-cos_containerd} @@ -44,7 +45,8 @@ fi base_cmd="${PKGDIR}/bin/k8s-integration-test \ --run-in-prow=true --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \ - --do-driver-build=${do_driver_build} --teardown-driver=${teardown_driver} --boskos-resource-type=${boskos_resource_type} \ + --do-driver-build=${do_driver_build} --teardown-driver=${teardown_driver} \ + --do-k8s-build=${do_k8s_build} --boskos-resource-type=${boskos_resource_type} \ --storageclass-files=sc-standard.yaml --snapshotclass-files=pd-volumesnapshotclass.yaml \ --deployment-strategy=${deployment_strategy} --test-version=${test_version} \ --num-nodes=3 --image-type=${image_type} --use-kubetest2=${use_kubetest2}"