Skip to content

Commit 1878146

Browse files
pwschuurmanpwschuurman
committed
Use precompiled stable release of Kubernetes by default when running integration test
1 parent 4b5780f commit 1878146

File tree

3 files changed

+134
-33
lines changed

3 files changed

+134
-33
lines changed

‎test/k8s-integration/cluster.go

+84-9
Original file line numberDiff line numberDiff line change
@@ -223,11 +223,81 @@ func clusterUpGKE(gceZone, gceRegion string, numNodes int, numWindowsNodes int,
223223
return nil
224224
}
225225

226-
func downloadKubernetesSource(pkgDir, k8sIoDir, kubeVersion string) error {
227-
k8sDir := filepath.Join(k8sIoDir, "kubernetes")
228-
klog.Infof("Downloading Kubernetes source v=%s to path=%s", kubeVersion, k8sIoDir)
226+
func downloadTarball(k8sDir, releaseVersion, subDir, tarballName string) error {
227+
tarballPath := filepath.Join(k8sDir, subDir)
228+
if err := os.MkdirAll(tarballPath, 0777); err != nil {
229+
return err
230+
}
231+
tarballOutput := filepath.Join(tarballPath, tarballName)
232+
downloadUrl := fmt.Sprintf("https://dl.k8s.io/release/%s/%s", releaseVersion, tarballName)
233+
klog.Infof("Downloading tarball %s to path=%s from url=%s", tarballName, tarballPath, downloadUrl)
234+
_, err := exec.Command("curl", "-Lsf", "--output", tarballOutput, downloadUrl).CombinedOutput()
235+
if err != nil {
236+
return err
237+
}
238+
239+
return nil
240+
}
241+
242+
func downloadAndExtractSrcTarball(k8sDir, releaseVersion, tarballName string) error {
243+
tarballPath := filepath.Join(k8sDir, tarballName)
244+
if err := downloadTarball(k8sDir, releaseVersion, "", tarballName); err != nil {
245+
return err
246+
}
247+
out, err := exec.Command("tar", "xf", tarballPath, "-C", k8sDir).CombinedOutput()
248+
if err != nil {
249+
return fmt.Errorf("Failed to extract src tarball %s: %s: %s", tarballName, out, err)
250+
}
251+
return nil
252+
}
253+
254+
func getReleaseVersionFromKubeVersion(kubeVersion string) (string, error) {
255+
releaseVersion := fmt.Sprintf("v%s", kubeVersion)
256+
if kubeVersion == "stable" || kubeVersion == "latest" {
257+
// See https://kubernetes.io/releases/download/
258+
out, err := exec.Command("curl", "-Lsf", "https://dl.k8s.io/release/stable.txt").CombinedOutput()
259+
if err != nil {
260+
return "", err
261+
}
262+
releaseVersion = string(out)
263+
}
264+
return releaseVersion, nil
265+
}
266+
267+
func downloadKubernetesRelease(k8sDir, kubeVersion, platform, arch string) error {
268+
releaseVersion, err := getReleaseVersionFromKubeVersion(kubeVersion)
269+
if err != nil {
270+
return err
271+
}
272+
273+
// Download precompiled tarballs
274+
serverTarballName := fmt.Sprintf("kubernetes-server-%s-%s.tar.gz", platform, arch)
275+
if err := downloadTarball(k8sDir, releaseVersion, "server", serverTarballName); err != nil {
276+
return err
277+
}
278+
279+
manifestsTarballName := fmt.Sprintf("kubernetes-manifests.tar.gz")
280+
if err := downloadTarball(k8sDir, releaseVersion, "server", manifestsTarballName); err != nil {
281+
return err
282+
}
283+
284+
nodeTarballName := fmt.Sprintf("kubernetes-node-%s-%s.tar.gz", platform, arch)
285+
if err := downloadTarball(k8sDir, releaseVersion, "node", nodeTarballName); err != nil {
286+
return err
287+
}
229288

230-
if err := os.MkdirAll(k8sIoDir, 0777); err != nil {
289+
clientTarballName := fmt.Sprintf("kubernetes-client-%s-%s.tar.gz", platform, arch)
290+
if err := downloadTarball(k8sDir, releaseVersion, "client", clientTarballName); err != nil {
291+
return err
292+
}
293+
294+
return nil
295+
}
296+
297+
func downloadKubernetesSource(k8sDir, kubeVersion string) error {
298+
klog.Infof("Downloading Kubernetes source v=%s to path=%s", kubeVersion, k8sDir)
299+
300+
if err := os.MkdirAll(k8sDir, 0777); err != nil {
231301
return err
232302
}
233303
if err := os.RemoveAll(k8sDir); err != nil {
@@ -248,12 +318,17 @@ func downloadKubernetesSource(pkgDir, k8sIoDir, kubeVersion string) error {
248318
return fmt.Errorf("failed to clone kubernetes master: %s, err: %v", out, err.Error())
249319
}
250320
} else {
251-
// Shallow clone of a release branch.
252-
vKubeVersion := "v" + kubeVersion
253-
klog.Infof("shallow clone of k8s %s", vKubeVersion)
254-
out, err := exec.Command("git", "clone", "--depth", "1", "https://github.com/kubernetes/kubernetes", k8sDir).CombinedOutput()
321+
releaseVersion, err := getReleaseVersionFromKubeVersion(kubeVersion)
255322
if err != nil {
256-
return fmt.Errorf("failed to clone kubernetes %s: %s, err: %v", vKubeVersion, out, err.Error())
323+
return err
324+
}
325+
326+
// Download versioned source.
327+
if err := downloadAndExtractSrcTarball(k8sDir, releaseVersion, "kubernetes-src.tar.gz"); err != nil {
328+
return err
329+
}
330+
if err := downloadAndExtractSrcTarball(k8sDir, releaseVersion, "kubernetes.tar.gz"); err != nil {
331+
return err
257332
}
258333
}
259334
return nil

‎test/k8s-integration/main.go

+45-21
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,8 @@ var (
3636
teardownCluster = flag.Bool("teardown-cluster", true, "teardown the cluster after the e2e test")
3737
teardownDriver = flag.Bool("teardown-driver", true, "teardown the driver after the e2e test")
3838
bringupCluster = flag.Bool("bringup-cluster", true, "build kubernetes and bringup a cluster")
39-
platform = flag.String("platform", "linux", "platform that the tests will be run, either linux or windows")
39+
platform = flag.String("platform", "linux", "platform that the tests will be run on, either linux or windows")
40+
arch = flag.String("arch", "amd64", "architecture that the tests will be run on (eg: amd64/arm64)")
4041
gceZone = flag.String("gce-zone", "", "zone that the gce k8s cluster is created/found in")
4142
gceRegion = flag.String("gce-region", "", "region that gke regional cluster should be created in")
4243
kubeVersion = flag.String("kube-version", "", "version of Kubernetes to download and use for the cluster")
@@ -65,6 +66,7 @@ var (
6566
saFile = flag.String("service-account-file", "", "path of service account file")
6667
deployOverlayName = flag.String("deploy-overlay-name", "", "which kustomize overlay to deploy the driver with")
6768
doDriverBuild = flag.Bool("do-driver-build", true, "building the driver from source")
69+
doK8sBuild = flag.Bool("do-k8s-build", true, "building the driver from source. If false, will fetch precompiled artifacts")
6870
useGKEManagedDriver = flag.Bool("use-gke-managed-driver", false, "use GKE managed PD CSI driver for the tests")
6971

7072
// Test flags
@@ -191,12 +193,32 @@ func main() {
191193
klog.Fatalf("num-windows-nodes must be set if the platform is windows")
192194
}
193195

196+
if *kubeVersion == "master" && !*doK8sBuild {
197+
klog.Fatalf("must set do-k8s-build=true when kubeVersion=%s", kubeVersion)
198+
}
199+
194200
err := handle()
195201
if err != nil {
196202
klog.Fatalf("Failed to run integration test: %w", err)
197203
}
198204
}
199205

206+
func buildTestingBinaries(k8sDir string) error {
207+
if err := buildKubernetes(k8sDir, "WHAT=test/e2e/e2e.test"); err != nil {
208+
return fmt.Errorf("failed to build Kubernetes e2e: %v", err.Error())
209+
}
210+
211+
// kubetest relies on ginkgo and kubectl already built in the test k8s directory
212+
if err := buildKubernetes(k8sDir, "ginkgo"); err != nil {
213+
return fmt.Errorf("failed to build gingko: %v", err.Error())
214+
}
215+
216+
if err := buildKubernetes(k8sDir, "kubectl"); err != nil {
217+
return fmt.Errorf("failed to build kubectl: %v", err.Error())
218+
}
219+
return nil
220+
}
221+
200222
func handle() error {
201223
oldmask := syscall.Umask(0000)
202224
defer syscall.Umask(oldmask)
@@ -266,21 +288,32 @@ func handle() error {
266288
}()
267289
}
268290

269-
// Create temporary directories for kubernetes builds
291+
// Create temporary directories for kubernetes source/build artifacts
270292
testParams.testParentDir = generateUniqueTmpDir()
271293
defer removeDir(testParams.testParentDir)
272294

273295
// If kube version is set, then download and build Kubernetes for cluster creation
274296
// Otherwise, either GKE or a prebuild local K8s dir is being used
275297
if len(*kubeVersion) != 0 {
276298
testParams.k8sSourceDir = filepath.Join(testParams.testParentDir, "kubernetes")
277-
err := downloadKubernetesSource(testParams.pkgDir, testParams.testParentDir, *kubeVersion)
278-
if err != nil {
299+
klog.Infof("Downloading Kubernetes source from: %s", *kubeVersion)
300+
if err := downloadKubernetesSource(testParams.k8sSourceDir, *kubeVersion); err != nil {
279301
return fmt.Errorf("failed to download Kubernetes source: %v", err.Error())
280302
}
281-
err = buildKubernetes(testParams.k8sSourceDir, "quick-release")
282-
if err != nil {
283-
return fmt.Errorf("failed to build Kubernetes: %v", err.Error())
303+
304+
if *doK8sBuild {
305+
klog.Info("Building Kubernetes source")
306+
if err := buildKubernetes(testParams.k8sSourceDir, "quick-release"); err != nil {
307+
return fmt.Errorf("failed to build Kubernetes: %v", err.Error())
308+
}
309+
} else {
310+
klog.Info("Fetching precompiled Kubernetes artifacts for %s/%s", *platform, *arch)
311+
if err := downloadKubernetesRelease(testParams.k8sSourceDir, *kubeVersion, *platform, *arch); err != nil {
312+
return fmt.Errorf("failed to download Kubernetes release: %v", err.Error())
313+
}
314+
if err := buildTestingBinaries(testParams.k8sSourceDir); err != nil {
315+
return err
316+
}
284317
}
285318
} else {
286319
testParams.k8sSourceDir = *localK8sDir
@@ -290,22 +323,13 @@ func handle() error {
290323
// Otherwise, either kube version is set (which implies GCE) or a local K8s dir is being used.
291324
if !*useKubeTest2 && len(*testVersion) != 0 && *testVersion != *kubeVersion {
292325
testParams.k8sSourceDir = filepath.Join(testParams.testParentDir, "kubernetes")
293-
err := downloadKubernetesSource(testParams.pkgDir, testParams.testParentDir, *testVersion)
294-
if err != nil {
326+
// Overlay the Kubernetes source
327+
if err := downloadKubernetesSource(testParams.k8sSourceDir, *testVersion); err != nil {
295328
return fmt.Errorf("failed to download Kubernetes source: %v", err.Error())
296329
}
297-
err = buildKubernetes(testParams.k8sSourceDir, "WHAT=test/e2e/e2e.test")
298-
if err != nil {
299-
return fmt.Errorf("failed to build Kubernetes e2e: %v", err.Error())
300-
}
301-
// kubetest relies on ginkgo and kubectl already built in the test k8s directory
302-
err = buildKubernetes(testParams.k8sSourceDir, "ginkgo")
303-
if err != nil {
304-
return fmt.Errorf("failed to build gingko: %v", err.Error())
305-
}
306-
err = buildKubernetes(testParams.k8sSourceDir, "kubectl")
307-
if err != nil {
308-
return fmt.Errorf("failed to build kubectl: %v", err.Error())
330+
klog.Infof("Building Kubernetes Testing binaries: %s", *kubeVersion)
331+
if err := buildTestingBinaries(testParams.k8sSourceDir); err != nil {
332+
return err
309333
}
310334
}
311335

‎test/run-k8s-integration.sh

+5-3
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,11 @@ readonly PKGDIR=${GOPATH}/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver
1414
readonly overlay_name="${GCE_PD_OVERLAY_NAME:-stable-master}"
1515
readonly boskos_resource_type="${GCE_PD_BOSKOS_RESOURCE_TYPE:-gce-project}"
1616
readonly do_driver_build="${GCE_PD_DO_DRIVER_BUILD:-true}"
17+
readonly do_k8s_build="${GCE_PD_DO_K8S_BUILD:-false}"
1718
readonly deployment_strategy=${DEPLOYMENT_STRATEGY:-gce}
1819
readonly gke_cluster_version=${GKE_CLUSTER_VERSION:-latest}
19-
readonly kube_version=${GCE_PD_KUBE_VERSION:-master}
20-
readonly test_version=${TEST_VERSION:-master}
20+
readonly kube_version=${GCE_PD_KUBE_VERSION:-stable}
21+
readonly test_version=${TEST_VERSION:-stable}
2122
readonly gce_zone=${GCE_CLUSTER_ZONE:-us-central1-b}
2223
readonly gce_region=${GCE_CLUSTER_REGION:-}
2324
readonly image_type=${IMAGE_TYPE:-cos_containerd}
@@ -44,7 +45,8 @@ fi
4445

4546
base_cmd="${PKGDIR}/bin/k8s-integration-test \
4647
--run-in-prow=true --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \
47-
--do-driver-build=${do_driver_build} --teardown-driver=${teardown_driver} --boskos-resource-type=${boskos_resource_type} \
48+
--do-driver-build=${do_driver_build} --teardown-driver=${teardown_driver} \
49+
--do-k8s-build=${do_k8s_build} --boskos-resource-type=${boskos_resource_type} \
4850
--storageclass-files=sc-standard.yaml --snapshotclass-files=pd-volumesnapshotclass.yaml \
4951
--deployment-strategy=${deployment_strategy} --test-version=${test_version} \
5052
--num-nodes=3 --image-type=${image_type} --use-kubetest2=${use_kubetest2}"

0 commit comments

Comments
 (0)