Skip to content

Commit 79bf1df

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #341 from hantaowang/gke-integration-cherrypicks
Gke integration cherrypicks
2 parents 74369d5 + 2de248c commit 79bf1df

8 files changed

+522
-321
lines changed

‎test/k8s-integration/cluster.go

+151
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
"os"
6+
"os/exec"
7+
"path/filepath"
8+
9+
"github.com/golang/glog"
10+
)
11+
12+
func clusterDownGCE(k8sDir string) error {
13+
cmd := exec.Command(filepath.Join(k8sDir, "hack", "e2e-internal", "e2e-down.sh"))
14+
err := runCommand("Bringing Down E2E Cluster on GCE", cmd)
15+
if err != nil {
16+
return fmt.Errorf("failed to bring down kubernetes e2e cluster on gce: %v", err)
17+
}
18+
return nil
19+
}
20+
21+
func clusterDownGKE(gceZone string) error {
22+
cmd := exec.Command("gcloud", "container", "clusters", "delete", gkeTestClusterName,
23+
"--zone", gceZone, "--quiet")
24+
err := runCommand("Bringing Down E2E Cluster on GKE", cmd)
25+
if err != nil {
26+
return fmt.Errorf("failed to bring down kubernetes e2e cluster on gke: %v", err)
27+
}
28+
return nil
29+
}
30+
31+
func buildKubernetes(k8sDir, command string) error {
32+
cmd := exec.Command("make", "-C", k8sDir, command)
33+
err := runCommand("Building Kubernetes", cmd)
34+
if err != nil {
35+
return fmt.Errorf("failed to build Kubernetes: %v", err)
36+
}
37+
return nil
38+
}
39+
40+
func clusterUpGCE(k8sDir, gceZone string) error {
41+
kshPath := filepath.Join(k8sDir, "cluster", "kubectl.sh")
42+
_, err := os.Stat(kshPath)
43+
if err == nil {
44+
// Set kubectl to the one bundled in the k8s tar for versioning
45+
err = os.Setenv("GCE_PD_KUBECTL", kshPath)
46+
if err != nil {
47+
return fmt.Errorf("failed to set cluster specific kubectl: %v", err)
48+
}
49+
} else {
50+
glog.Errorf("could not find cluster kubectl at %s, falling back to default kubectl", kshPath)
51+
}
52+
53+
if len(*kubeFeatureGates) != 0 {
54+
err = os.Setenv("KUBE_FEATURE_GATES", *kubeFeatureGates)
55+
if err != nil {
56+
return fmt.Errorf("failed to set kubernetes feature gates: %v", err)
57+
}
58+
glog.V(4).Infof("Set Kubernetes feature gates: %v", *kubeFeatureGates)
59+
}
60+
61+
err = os.Setenv("KUBE_GCE_ZONE", gceZone)
62+
if err != nil {
63+
return err
64+
}
65+
cmd := exec.Command(filepath.Join(k8sDir, "hack", "e2e-internal", "e2e-up.sh"))
66+
err = runCommand("Starting E2E Cluster on GCE", cmd)
67+
if err != nil {
68+
return fmt.Errorf("failed to bring up kubernetes e2e cluster on gce: %v", err)
69+
}
70+
71+
return nil
72+
}
73+
74+
func clusterUpGKE(gceZone string) error {
75+
out, err := exec.Command("gcloud", "container", "clusters", "list", "--zone", gceZone,
76+
"--filter", fmt.Sprintf("name=%s", gkeTestClusterName)).CombinedOutput()
77+
if err != nil {
78+
return fmt.Errorf("failed to check for previous test cluster: %v %s", err, out)
79+
}
80+
if len(out) > 0 {
81+
glog.Infof("Detected previous cluster %s. Deleting so a new one can be created...", gkeTestClusterName)
82+
err = clusterDownGKE(gceZone)
83+
if err != nil {
84+
return err
85+
}
86+
}
87+
cmd := exec.Command("gcloud", "container", "clusters", "create", gkeTestClusterName,
88+
"--zone", gceZone, "--cluster-version", *gkeClusterVer, "--quiet")
89+
err = runCommand("Staring E2E Cluster on GKE", cmd)
90+
if err != nil {
91+
return fmt.Errorf("failed to bring up kubernetes e2e cluster on gke: %v", err)
92+
}
93+
94+
return nil
95+
}
96+
97+
func downloadKubernetesSource(pkgDir, k8sIoDir, kubeVersion string) error {
98+
k8sDir := filepath.Join(k8sIoDir, "kubernetes")
99+
/*
100+
// TODO: Download a fresh copy every time until mutate manifests hardcoding existing image is solved.
101+
if _, err := os.Stat(k8sDir); !os.IsNotExist(err) {
102+
glog.Infof("Staging Kubernetes already found at %s, skipping download", k8sDir)
103+
return nil
104+
}
105+
*/
106+
107+
glog.V(4).Infof("Staging Kubernetes folder not found, downloading now")
108+
109+
err := os.MkdirAll(k8sIoDir, 0777)
110+
if err != nil {
111+
return err
112+
}
113+
114+
kubeTarDir := filepath.Join(k8sIoDir, fmt.Sprintf("kubernetes-%s.tar.gz", kubeVersion))
115+
116+
var vKubeVersion string
117+
if kubeVersion == "master" {
118+
vKubeVersion = kubeVersion
119+
// A hack to be able to build Kubernetes in this nested place
120+
// KUBE_GIT_VERSION_FILE set to file to load kube version from
121+
err = os.Setenv("KUBE_GIT_VERSION_FILE", filepath.Join(pkgDir, "test", "k8s-integration", ".dockerized-kube-version-defs"))
122+
if err != nil {
123+
return err
124+
}
125+
} else {
126+
vKubeVersion = "v" + kubeVersion
127+
}
128+
out, err := exec.Command("curl", "-L", fmt.Sprintf("https://github.com/kubernetes/kubernetes/archive/%s.tar.gz", vKubeVersion), "-o", kubeTarDir).CombinedOutput()
129+
if err != nil {
130+
return fmt.Errorf("failed to curl kubernetes version %s: %s, err: %v", kubeVersion, out, err)
131+
}
132+
133+
out, err = exec.Command("tar", "-C", k8sIoDir, "-xvf", kubeTarDir).CombinedOutput()
134+
if err != nil {
135+
return fmt.Errorf("failed to untar %s: %s, err: %v", kubeTarDir, out, err)
136+
}
137+
138+
err = os.RemoveAll(k8sDir)
139+
if err != nil {
140+
return err
141+
}
142+
143+
err = os.Rename(filepath.Join(k8sIoDir, fmt.Sprintf("kubernetes-%s", kubeVersion)), k8sDir)
144+
if err != nil {
145+
return err
146+
}
147+
148+
glog.V(4).Infof("Successfully downloaded Kubernetes v%s to %s", kubeVersion, k8sDir)
149+
150+
return nil
151+
}

‎test/k8s-integration/driver.go

+116
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
"os"
6+
"os/exec"
7+
"path/filepath"
8+
"time"
9+
)
10+
11+
func getOverlayDir(pkgDir, deployOverlayName string) string {
12+
return filepath.Join(pkgDir, "deploy", "kubernetes", "overlays", deployOverlayName)
13+
}
14+
15+
func installDriver(goPath, pkgDir, stagingImage, stagingVersion, deployOverlayName string, doDriverBuild bool) error {
16+
if doDriverBuild {
17+
// Install kustomize
18+
out, err := exec.Command(filepath.Join(pkgDir, "deploy", "kubernetes", "install-kustomize.sh")).CombinedOutput()
19+
if err != nil {
20+
return fmt.Errorf("failed to install kustomize: %s, err: %v", out, err)
21+
}
22+
23+
// Edit ci kustomization to use given image tag
24+
overlayDir := getOverlayDir(pkgDir, deployOverlayName)
25+
err = os.Chdir(overlayDir)
26+
if err != nil {
27+
return fmt.Errorf("failed to change to overlay directory: %s, err: %v", out, err)
28+
}
29+
30+
// TODO (#138): in a local environment this is going to modify the actual kustomize files.
31+
// maybe a copy should be made instead
32+
out, err = exec.Command(
33+
filepath.Join(pkgDir, "bin", "kustomize"),
34+
"edit",
35+
"set",
36+
"image",
37+
fmt.Sprintf("%s=%s:%s", pdImagePlaceholder, stagingImage, stagingVersion)).CombinedOutput()
38+
if err != nil {
39+
return fmt.Errorf("failed to edit kustomize: %s, err: %v", out, err)
40+
}
41+
}
42+
43+
// setup service account file for secret creation
44+
tmpSaFile := filepath.Join(generateUniqueTmpDir(), "cloud-sa.json")
45+
defer removeDir(filepath.Dir(tmpSaFile))
46+
47+
// Need to copy it to name the file "cloud-sa.json"
48+
out, err := exec.Command("cp", *saFile, tmpSaFile).CombinedOutput()
49+
if err != nil {
50+
return fmt.Errorf("error copying service account key: %s, err: %v", out, err)
51+
}
52+
defer shredFile(tmpSaFile)
53+
54+
// deploy driver
55+
deployCmd := exec.Command(filepath.Join(pkgDir, "deploy", "kubernetes", "deploy-driver.sh"), "--skip-sa-check")
56+
deployCmd.Env = append(os.Environ(),
57+
fmt.Sprintf("GOPATH=%s", goPath),
58+
fmt.Sprintf("GCE_PD_SA_DIR=%s", filepath.Dir(tmpSaFile)),
59+
fmt.Sprintf("GCE_PD_DRIVER_VERSION=%s", deployOverlayName),
60+
)
61+
err = runCommand("Deploying driver", deployCmd)
62+
if err != nil {
63+
return fmt.Errorf("failed to deploy driver: %v", err)
64+
}
65+
66+
// TODO (#139): wait for driver to be running
67+
time.Sleep(time.Minute)
68+
statusCmd := exec.Command("kubectl", "describe", "pods", "-n", "default")
69+
err = runCommand("Checking driver pods", statusCmd)
70+
if err != nil {
71+
return fmt.Errorf("failed to check driver pods: %v", err)
72+
}
73+
74+
return nil
75+
}
76+
77+
func deleteDriver(goPath, pkgDir, deployOverlayName string) error {
78+
deleteCmd := exec.Command(filepath.Join(pkgDir, "deploy", "kubernetes", "delete-driver.sh"))
79+
deleteCmd.Env = append(os.Environ(),
80+
fmt.Sprintf("GOPATH=%s", goPath),
81+
fmt.Sprintf("GCE_PD_DRIVER_VERSION=%s", deployOverlayName),
82+
)
83+
err := runCommand("Deleting driver", deleteCmd)
84+
if err != nil {
85+
return fmt.Errorf("failed to delete driver: %v", err)
86+
}
87+
return nil
88+
}
89+
90+
func pushImage(pkgDir, stagingImage, stagingVersion string) error {
91+
err := os.Setenv("GCE_PD_CSI_STAGING_VERSION", stagingVersion)
92+
if err != nil {
93+
return err
94+
}
95+
err = os.Setenv("GCE_PD_CSI_STAGING_IMAGE", stagingImage)
96+
if err != nil {
97+
return err
98+
}
99+
cmd := exec.Command("make", "-C", pkgDir, "push-container",
100+
fmt.Sprintf("GCE_PD_CSI_STAGING_VERSION=%s", stagingVersion),
101+
fmt.Sprintf("GCE_PD_CSI_STAGING_IMAGE=%s", stagingImage))
102+
err = runCommand("Pushing GCP Container", cmd)
103+
if err != nil {
104+
return fmt.Errorf("failed to run make command: err: %v", err)
105+
}
106+
return nil
107+
}
108+
109+
func deleteImage(stagingImage, stagingVersion string) error {
110+
cmd := exec.Command("gcloud", "container", "images", "delete", fmt.Sprintf("%s:%s", stagingImage, stagingVersion), "--quiet")
111+
err := runCommand("Deleting GCR Container", cmd)
112+
if err != nil {
113+
return fmt.Errorf("failed to delete container image %s:%s: %s", stagingImage, stagingVersion, err)
114+
}
115+
return nil
116+
}

0 commit comments

Comments
 (0)