Skip to content

Commit 776f51a

Browse files
pwschuurmanpwschuurman
committed
Use precompiled stable release of Kubernetes by default when running integration test
1 parent 4b5780f commit 776f51a

File tree

4 files changed

+168
-34
lines changed

4 files changed

+168
-34
lines changed

‎test/k8s-integration/cluster.go

+84-9
Original file line numberDiff line numberDiff line change
@@ -223,11 +223,81 @@ func clusterUpGKE(gceZone, gceRegion string, numNodes int, numWindowsNodes int,
223223
return nil
224224
}
225225

226-
func downloadKubernetesSource(pkgDir, k8sIoDir, kubeVersion string) error {
227-
k8sDir := filepath.Join(k8sIoDir, "kubernetes")
228-
klog.Infof("Downloading Kubernetes source v=%s to path=%s", kubeVersion, k8sIoDir)
226+
func downloadTarball(k8sDir, releaseVersion, subDir, tarballName string) error {
227+
tarballPath := filepath.Join(k8sDir, subDir)
228+
if err := os.MkdirAll(tarballPath, 0777); err != nil {
229+
return err
230+
}
231+
tarballOutput := filepath.Join(tarballPath, tarballName)
232+
downloadUrl := fmt.Sprintf("https://dl.k8s.io/release/%s/%s", releaseVersion, tarballName)
233+
klog.Infof("Downloading tarball %s to path=%s from url=%s", tarballName, tarballPath, downloadUrl)
234+
_, err := exec.Command("curl", "-Lsf", "--output", tarballOutput, downloadUrl).CombinedOutput()
235+
if err != nil {
236+
return err
237+
}
238+
239+
return nil
240+
}
241+
242+
func downloadAndExtractSrcTarball(k8sDir, releaseVersion, tarballName string) error {
243+
tarballPath := filepath.Join(k8sDir, tarballName)
244+
if err := downloadTarball(k8sDir, releaseVersion, "", tarballName); err != nil {
245+
return err
246+
}
247+
out, err := exec.Command("tar", "xf", tarballPath, "-C", k8sDir).CombinedOutput()
248+
if err != nil {
249+
return fmt.Errorf("Failed to extract src tarball %s: %s: %s", tarballName, out, err)
250+
}
251+
return nil
252+
}
253+
254+
func getReleaseVersionFromKubeVersion(kubeVersion string) (string, error) {
255+
releaseVersion := fmt.Sprintf("v%s", kubeVersion)
256+
if kubeVersion == "stable" || kubeVersion == "latest" {
257+
// See https://kubernetes.io/releases/download/
258+
out, err := exec.Command("curl", "-Lsf", "https://dl.k8s.io/release/stable.txt").CombinedOutput()
259+
if err != nil {
260+
return "", err
261+
}
262+
releaseVersion = string(out)
263+
}
264+
return releaseVersion, nil
265+
}
266+
267+
func downloadKubernetesRelease(k8sDir, kubeVersion, platform, arch string) error {
268+
releaseVersion, err := getReleaseVersionFromKubeVersion(kubeVersion)
269+
if err != nil {
270+
return err
271+
}
272+
273+
// Download precompiled tarballs
274+
serverTarballName := fmt.Sprintf("kubernetes-server-%s-%s.tar.gz", platform, arch)
275+
if err := downloadTarball(k8sDir, releaseVersion, "server", serverTarballName); err != nil {
276+
return err
277+
}
278+
279+
manifestsTarballName := fmt.Sprintf("kubernetes-manifests.tar.gz")
280+
if err := downloadTarball(k8sDir, releaseVersion, "server", manifestsTarballName); err != nil {
281+
return err
282+
}
283+
284+
nodeTarballName := fmt.Sprintf("kubernetes-node-%s-%s.tar.gz", platform, arch)
285+
if err := downloadTarball(k8sDir, releaseVersion, "node", nodeTarballName); err != nil {
286+
return err
287+
}
229288

230-
if err := os.MkdirAll(k8sIoDir, 0777); err != nil {
289+
clientTarballName := fmt.Sprintf("kubernetes-client-%s-%s.tar.gz", platform, arch)
290+
if err := downloadTarball(k8sDir, releaseVersion, "client", clientTarballName); err != nil {
291+
return err
292+
}
293+
294+
return nil
295+
}
296+
297+
func downloadKubernetesSource(k8sDir, kubeVersion string) error {
298+
klog.Infof("Downloading Kubernetes source v=%s to path=%s", kubeVersion, k8sDir)
299+
300+
if err := os.MkdirAll(k8sDir, 0777); err != nil {
231301
return err
232302
}
233303
if err := os.RemoveAll(k8sDir); err != nil {
@@ -248,12 +318,17 @@ func downloadKubernetesSource(pkgDir, k8sIoDir, kubeVersion string) error {
248318
return fmt.Errorf("failed to clone kubernetes master: %s, err: %v", out, err.Error())
249319
}
250320
} else {
251-
// Shallow clone of a release branch.
252-
vKubeVersion := "v" + kubeVersion
253-
klog.Infof("shallow clone of k8s %s", vKubeVersion)
254-
out, err := exec.Command("git", "clone", "--depth", "1", "https://github.com/kubernetes/kubernetes", k8sDir).CombinedOutput()
321+
releaseVersion, err := getReleaseVersionFromKubeVersion(kubeVersion)
255322
if err != nil {
256-
return fmt.Errorf("failed to clone kubernetes %s: %s, err: %v", vKubeVersion, out, err.Error())
323+
return err
324+
}
325+
326+
// Download versioned source.
327+
if err := downloadAndExtractSrcTarball(k8sDir, releaseVersion, "kubernetes-src.tar.gz"); err != nil {
328+
return err
329+
}
330+
if err := downloadAndExtractSrcTarball(k8sDir, releaseVersion, "kubernetes.tar.gz"); err != nil {
331+
return err
257332
}
258333
}
259334
return nil

‎test/k8s-integration/main.go

+75-22
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import (
2828
apimachineryversion "k8s.io/apimachinery/pkg/util/version"
2929
_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
3030
"k8s.io/klog/v2"
31+
"k8s.io/utils/strings/slices"
3132
testutils "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/e2e/utils"
3233
)
3334

@@ -36,7 +37,8 @@ var (
3637
teardownCluster = flag.Bool("teardown-cluster", true, "teardown the cluster after the e2e test")
3738
teardownDriver = flag.Bool("teardown-driver", true, "teardown the driver after the e2e test")
3839
bringupCluster = flag.Bool("bringup-cluster", true, "build kubernetes and bringup a cluster")
39-
platform = flag.String("platform", "linux", "platform that the tests will be run, either linux or windows")
40+
platform = flag.String("platform", "linux", "platform that the tests will be run on, either linux or windows")
41+
arch = flag.String("arch", "amd64", "architecture that the tests will be run on (eg: amd64/arm64)")
4042
gceZone = flag.String("gce-zone", "", "zone that the gce k8s cluster is created/found in")
4143
gceRegion = flag.String("gce-region", "", "region that gke regional cluster should be created in")
4244
kubeVersion = flag.String("kube-version", "", "version of Kubernetes to download and use for the cluster")
@@ -65,6 +67,7 @@ var (
6567
saFile = flag.String("service-account-file", "", "path of service account file")
6668
deployOverlayName = flag.String("deploy-overlay-name", "", "which kustomize overlay to deploy the driver with")
6769
doDriverBuild = flag.Bool("do-driver-build", true, "building the driver from source")
70+
doK8sBuild = flag.Bool("do-k8s-build", true, "building the driver from source. If false, will fetch precompiled artifacts")
6871
useGKEManagedDriver = flag.Bool("use-gke-managed-driver", false, "use GKE managed PD CSI driver for the tests")
6972

7073
// Test flags
@@ -73,6 +76,8 @@ var (
7376

7477
useKubeTest2 = flag.Bool("use-kubetest2", false, "use kubetest2 to run e2e tests")
7578
parallel = flag.Int("parallel", 4, "the number of parallel tests setting for ginkgo parallelism")
79+
80+
allowedVersionTags = []string{"master", "stable", "latest"}
7681
)
7782

7883
const (
@@ -191,12 +196,48 @@ func main() {
191196
klog.Fatalf("num-windows-nodes must be set if the platform is windows")
192197
}
193198

199+
if *kubeVersion == "master" && !*doK8sBuild {
200+
klog.Fatalf("must set do-k8s-build=true when kubeVersion=%s", *kubeVersion)
201+
}
202+
203+
if len(*kubeVersion) != 0 {
204+
if !slices.Contains(allowedVersionTags, *kubeVersion) {
205+
if _, err := parseVersion(*kubeVersion); err != nil {
206+
klog.Fatalf("invalid kube-version %s", *kubeVersion)
207+
}
208+
}
209+
}
210+
211+
if len(*testVersion) != 0 {
212+
if !slices.Contains(allowedVersionTags, *testVersion) {
213+
if _, err := parseVersion(*testVersion); err != nil {
214+
klog.Fatalf("invalid test-version %s", *testVersion)
215+
}
216+
}
217+
}
218+
194219
err := handle()
195220
if err != nil {
196221
klog.Fatalf("Failed to run integration test: %w", err)
197222
}
198223
}
199224

225+
func buildTestingBinaries(k8sDir string) error {
226+
if err := buildKubernetes(k8sDir, "WHAT=test/e2e/e2e.test"); err != nil {
227+
return fmt.Errorf("failed to build Kubernetes e2e: %v", err.Error())
228+
}
229+
230+
// kubetest relies on ginkgo and kubectl already built in the test k8s directory
231+
if err := buildKubernetes(k8sDir, "ginkgo"); err != nil {
232+
return fmt.Errorf("failed to build gingko: %v", err.Error())
233+
}
234+
235+
if err := buildKubernetes(k8sDir, "kubectl"); err != nil {
236+
return fmt.Errorf("failed to build kubectl: %v", err.Error())
237+
}
238+
return nil
239+
}
240+
200241
func handle() error {
201242
oldmask := syscall.Umask(0000)
202243
defer syscall.Umask(oldmask)
@@ -266,21 +307,32 @@ func handle() error {
266307
}()
267308
}
268309

269-
// Create temporary directories for kubernetes builds
310+
// Create temporary directories for kubernetes source/build artifacts
270311
testParams.testParentDir = generateUniqueTmpDir()
271312
defer removeDir(testParams.testParentDir)
272313

273314
// If kube version is set, then download and build Kubernetes for cluster creation
274315
// Otherwise, either GKE or a prebuild local K8s dir is being used
275316
if len(*kubeVersion) != 0 {
276317
testParams.k8sSourceDir = filepath.Join(testParams.testParentDir, "kubernetes")
277-
err := downloadKubernetesSource(testParams.pkgDir, testParams.testParentDir, *kubeVersion)
278-
if err != nil {
318+
klog.Infof("Downloading Kubernetes source from: %s", *kubeVersion)
319+
if err := downloadKubernetesSource(testParams.k8sSourceDir, *kubeVersion); err != nil {
279320
return fmt.Errorf("failed to download Kubernetes source: %v", err.Error())
280321
}
281-
err = buildKubernetes(testParams.k8sSourceDir, "quick-release")
282-
if err != nil {
283-
return fmt.Errorf("failed to build Kubernetes: %v", err.Error())
322+
323+
if *doK8sBuild {
324+
klog.Info("Building Kubernetes source")
325+
if err := buildKubernetes(testParams.k8sSourceDir, "quick-release"); err != nil {
326+
return fmt.Errorf("failed to build Kubernetes: %v", err.Error())
327+
}
328+
} else {
329+
klog.Info("Fetching precompiled Kubernetes artifacts for %s/%s", *platform, *arch)
330+
if err := downloadKubernetesRelease(testParams.k8sSourceDir, *kubeVersion, *platform, *arch); err != nil {
331+
return fmt.Errorf("failed to download Kubernetes release: %v", err.Error())
332+
}
333+
if err := buildTestingBinaries(testParams.k8sSourceDir); err != nil {
334+
return err
335+
}
284336
}
285337
} else {
286338
testParams.k8sSourceDir = *localK8sDir
@@ -290,22 +342,13 @@ func handle() error {
290342
// Otherwise, either kube version is set (which implies GCE) or a local K8s dir is being used.
291343
if !*useKubeTest2 && len(*testVersion) != 0 && *testVersion != *kubeVersion {
292344
testParams.k8sSourceDir = filepath.Join(testParams.testParentDir, "kubernetes")
293-
err := downloadKubernetesSource(testParams.pkgDir, testParams.testParentDir, *testVersion)
294-
if err != nil {
345+
// Overlay the Kubernetes source
346+
if err := downloadKubernetesSource(testParams.k8sSourceDir, *testVersion); err != nil {
295347
return fmt.Errorf("failed to download Kubernetes source: %v", err.Error())
296348
}
297-
err = buildKubernetes(testParams.k8sSourceDir, "WHAT=test/e2e/e2e.test")
298-
if err != nil {
299-
return fmt.Errorf("failed to build Kubernetes e2e: %v", err.Error())
300-
}
301-
// kubetest relies on ginkgo and kubectl already built in the test k8s directory
302-
err = buildKubernetes(testParams.k8sSourceDir, "ginkgo")
303-
if err != nil {
304-
return fmt.Errorf("failed to build gingko: %v", err.Error())
305-
}
306-
err = buildKubernetes(testParams.k8sSourceDir, "kubectl")
307-
if err != nil {
308-
return fmt.Errorf("failed to build kubectl: %v", err.Error())
349+
klog.Infof("Building Kubernetes Testing binaries: %s", *kubeVersion)
350+
if err := buildTestingBinaries(testParams.k8sSourceDir); err != nil {
351+
return err
309352
}
310353
}
311354

@@ -779,7 +822,17 @@ func runTestsWithConfig(testParams *testParameters, testConfigArg, reportPrefix
779822
}
780823
kubeTest2Args = append(kubeTest2Args, "--use-built-binaries")
781824
} else {
782-
kubeTest2Args = append(kubeTest2Args, fmt.Sprintf("--test-package-marker=latest-%s.txt", *testVersion))
825+
testResourceVersion := *testVersion
826+
if *testVersion != "stable" && *testVersion != "latest" {
827+
// Find the minor version
828+
v, err := parseVersion(*testVersion)
829+
if err != nil {
830+
// Note, this shouldn't happen, as we check flags in main().
831+
return fmt.Errorf("failed to parse --test-version")
832+
}
833+
testResourceVersion = fmt.Sprintf("stable-%s", v.minorVersion())
834+
}
835+
kubeTest2Args = append(kubeTest2Args, fmt.Sprintf("--test-package-marker=%s.txt", testResourceVersion))
783836
}
784837
}
785838
kubeTest2Args = append(kubeTest2Args, fmt.Sprintf("--focus-regex=%s", focus))

‎test/k8s-integration/version.go

+4
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,10 @@ func (v *version) isGKEExtraVersion(extrastr string) bool {
5454
return gkeExtraVersionRegex.MatchString(extrastr)
5555
}
5656

57+
func (v *version) minorVersion() string {
58+
return fmt.Sprintf("%d.%d", v.major(), v.minor())
59+
}
60+
5761
func extractGKEExtraVersion(extra string) (int, error) {
5862
m := gkeExtraVersionRegex.FindStringSubmatch(extra)
5963
if len(m) != 2 {

‎test/run-k8s-integration.sh

+5-3
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,11 @@ readonly PKGDIR=${GOPATH}/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver
1414
readonly overlay_name="${GCE_PD_OVERLAY_NAME:-stable-master}"
1515
readonly boskos_resource_type="${GCE_PD_BOSKOS_RESOURCE_TYPE:-gce-project}"
1616
readonly do_driver_build="${GCE_PD_DO_DRIVER_BUILD:-true}"
17+
readonly do_k8s_build="${GCE_PD_DO_K8S_BUILD:-false}"
1718
readonly deployment_strategy=${DEPLOYMENT_STRATEGY:-gce}
1819
readonly gke_cluster_version=${GKE_CLUSTER_VERSION:-latest}
19-
readonly kube_version=${GCE_PD_KUBE_VERSION:-master}
20-
readonly test_version=${TEST_VERSION:-master}
20+
readonly kube_version=${GCE_PD_KUBE_VERSION:-stable}
21+
readonly test_version=${TEST_VERSION:-stable}
2122
readonly gce_zone=${GCE_CLUSTER_ZONE:-us-central1-b}
2223
readonly gce_region=${GCE_CLUSTER_REGION:-}
2324
readonly image_type=${IMAGE_TYPE:-cos_containerd}
@@ -44,7 +45,8 @@ fi
4445

4546
base_cmd="${PKGDIR}/bin/k8s-integration-test \
4647
--run-in-prow=true --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \
47-
--do-driver-build=${do_driver_build} --teardown-driver=${teardown_driver} --boskos-resource-type=${boskos_resource_type} \
48+
--do-driver-build=${do_driver_build} --teardown-driver=${teardown_driver} \
49+
--do-k8s-build=${do_k8s_build} --boskos-resource-type=${boskos_resource_type} \
4850
--storageclass-files=sc-standard.yaml --snapshotclass-files=pd-volumesnapshotclass.yaml \
4951
--deployment-strategy=${deployment_strategy} --test-version=${test_version} \
5052
--num-nodes=3 --image-type=${image_type} --use-kubetest2=${use_kubetest2}"

0 commit comments

Comments
 (0)