[WIP] Gke integration tests

test/k8s-integration/cluster.go

@@ -0,0 +1,131 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+
+	"k8s.io/klog"
+)
+
+func clusterUpGKE(gceZone string) error {
+	out, err := exec.Command("gcloud", "container", "clusters", "list", "--zone", gceZone,
+		"--filter", fmt.Sprintf("name=%s", gkeTestClusterName)).CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("failed to check for previous test cluster: %v %s", err, out)
+	}
+	if len(out) > 0 {
+		klog.Infof("Detected previous cluster %s. Deleting so a new one can be created...", gkeTestClusterName)
+		err = clusterDownGKE(gceZone)
+		if err != nil {
+			return err
+		}
+	}
+	cmd := exec.Command("gcloud", "container", "clusters", "create", gkeTestClusterName,
+		"--zone", gceZone, "--cluster-version", *gkeClusterVer, "--quiet")
+	err = runCommand("Staring E2E Cluster on GKE", cmd)
+	if err != nil {
+		return fmt.Errorf("failed to bring up kubernetes e2e cluster on gke: %v", err)
+	}
+
+	return nil
+}
+
+func clusterDownGKE(gceZone string) error {
+	cmd := exec.Command("gcloud", "container", "clusters", "delete", gkeTestClusterName,
+		"--zone", gceZone, "--quiet")
+	err := runCommand("Bringing Down E2E Cluster on GKE", cmd)
+	if err != nil {
+		return fmt.Errorf("failed to bring down kubernetes e2e cluster on gke: %v", err)
+	}
+	return nil
+}
+
+func clusterUpGCE(k8sDir, gceZone string) error {
+	err := os.Setenv("KUBE_GCE_ZONE", gceZone)
+	if err != nil {
+		return err
+	}
+	cmd := exec.Command(filepath.Join(k8sDir, "hack", "e2e-internal", "e2e-up.sh"))
+	err = runCommand("Starting E2E Cluster on GCE", cmd)
+	if err != nil {
+		return fmt.Errorf("failed to bring up kubernetes e2e cluster on gce: %v", err)
+	}
+
+	return nil
+}
+
+func clusterDownGCE(k8sDir string) error {
+	cmd := exec.Command(filepath.Join(k8sDir, "hack", "e2e-internal", "e2e-down.sh"))
+	err := runCommand("Bringing Down E2E Cluster on GCE", cmd)
+	if err != nil {
+		return fmt.Errorf("failed to bring down kubernetes e2e cluster on gce: %v", err)
+	}
+	return nil
+}
+
+func buildKubernetes(k8sDir string) error {
+	cmd := exec.Command("make", "-C", k8sDir, "quick-release")
+	err := runCommand("Building Kubernetes", cmd)
+	if err != nil {
+		return fmt.Errorf("failed to build Kubernetes: %v", err)
+	}
+	return nil
+}
+
+func downloadKubernetesSource(pkgDir, k8sIoDir, kubeVersion string) error {
+	k8sDir := filepath.Join(k8sIoDir, "kubernetes")
+	/*
+		// TODO: Download a fresh copy every time until mutate manifests hardcoding existing image is solved.
+		if _, err := os.Stat(k8sDir); !os.IsNotExist(err) {
+			klog.Infof("Staging Kubernetes already found at %s, skipping download", k8sDir)
+			return nil
+		}
+	*/
+
+	klog.V(4).Infof("Staging Kubernetes folder not found, downloading now")
+
+	err := os.MkdirAll(k8sIoDir, 0777)
+	if err != nil {
+		return err
+	}
+
+	kubeTarDir := filepath.Join(k8sIoDir, fmt.Sprintf("kubernetes-%s.tar.gz", kubeVersion))
+
+	var vKubeVersion string
+	if kubeVersion == "master" {
+		vKubeVersion = kubeVersion
+		// A hack to be able to build Kubernetes in this nested place
+		// KUBE_GIT_VERSION_FILE set to file to load kube version from
+		err = os.Setenv("KUBE_GIT_VERSION_FILE", filepath.Join(pkgDir, "test", "k8s-integration", ".dockerized-kube-version-defs"))
+		if err != nil {
+			return err
+		}
+	} else {
+		vKubeVersion = "v" + kubeVersion
+	}
+	out, err := exec.Command("curl", "-L", fmt.Sprintf("https://github.com/kubernetes/kubernetes/archive/%s.tar.gz", vKubeVersion), "-o", kubeTarDir).CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("failed to curl kubernetes version %s: %s, err: %v", kubeVersion, out, err)
+	}
+
+	out, err = exec.Command("tar", "-C", k8sIoDir, "-xvf", kubeTarDir).CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("failed to untar %s: %s, err: %v", kubeTarDir, out, err)
+	}
+
+	err = os.RemoveAll(k8sDir)
+	if err != nil {
+		return err
+	}
+
+	err = os.Rename(filepath.Join(k8sIoDir, fmt.Sprintf("kubernetes-%s", kubeVersion)), k8sDir)
+	if err != nil {
+		return err
+	}
+
+	klog.V(4).Infof("Successfully downloaded Kubernetes v%s to %s", kubeVersion, k8sDir)
+
+	return nil
+}

test/k8s-integration/driver.go

@@ -0,0 +1,116 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"time"
+)
+
+func getOverlayDir(pkgDir, deployOverlayName string) string {
+	return filepath.Join(pkgDir, "deploy", "kubernetes", "overlays", deployOverlayName)
+}
+
+func installDriver(goPath, pkgDir, stagingImage, stagingVersion, deployOverlayName string, doDriverBuild bool) error {
+	if doDriverBuild {
+		// Install kustomize
+		out, err := exec.Command(filepath.Join(pkgDir, "deploy", "kubernetes", "install-kustomize.sh")).CombinedOutput()
+		if err != nil {
+			return fmt.Errorf("failed to install kustomize: %s, err: %v", out, err)
+		}
+
+		// Edit ci kustomization to use given image tag
+		overlayDir := getOverlayDir(pkgDir, deployOverlayName)
+		err = os.Chdir(overlayDir)
+		if err != nil {
+			return fmt.Errorf("failed to change to overlay directory: %s, err: %v", out, err)
+		}
+
+		// TODO (#138): in a local environment this is going to modify the actual kustomize files.
+		// maybe a copy should be made instead
+		out, err = exec.Command(
+			filepath.Join(pkgDir, "bin", "kustomize"),
+			"edit",
+			"set",
+			"image",
+			fmt.Sprintf("%s=%s:%s", pdImagePlaceholder, stagingImage, stagingVersion)).CombinedOutput()
+		if err != nil {
+			return fmt.Errorf("failed to edit kustomize: %s, err: %v", out, err)
+		}
+	}
+
+	// setup service account file for secret creation
+	tmpSaFile := filepath.Join(getTempDir(), "cloud-sa.json")
+	defer removeTempDir(filepath.Dir(tmpSaFile))
+
+	// Need to copy it to name the file "cloud-sa.json"
+	out, err := exec.Command("cp", *saFile, tmpSaFile).CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("error copying service account key: %s, err: %v", out, err)
+	}
+	defer shredFile(tmpSaFile)
+
+	// deploy driver
+	deployCmd := exec.Command(filepath.Join(pkgDir, "deploy", "kubernetes", "deploy-driver.sh"), "--skip-sa-check")
+	deployCmd.Env = append(os.Environ(),
+		fmt.Sprintf("GOPATH=%s", goPath),
+		fmt.Sprintf("GCE_PD_SA_DIR=%s", filepath.Dir(tmpSaFile)),
+		fmt.Sprintf("GCE_PD_DRIVER_VERSION=%s", deployOverlayName),
+	)
+	err = runCommand("Deploying driver", deployCmd)
+	if err != nil {
+		return fmt.Errorf("failed to deploy driver: %v", err)
+	}
+
+	// TODO (#139): wait for driver to be running
+	time.Sleep(10 * time.Second)
+	statusCmd := exec.Command("kubectl", "describe", "pods", "-n", "default")
+	err = runCommand("Checking driver pods", statusCmd)
+	if err != nil {
+		return fmt.Errorf("failed to check driver pods: %v", err)
+	}
+
+	return nil
+}
+
+func deleteDriver(goPath, pkgDir, deployOverlayName string) error {
+	deleteCmd := exec.Command(filepath.Join(pkgDir, "deploy", "kubernetes", "delete-driver.sh"))
+	deleteCmd.Env = append(os.Environ(),
+		fmt.Sprintf("GOPATH=%s", goPath),
+		fmt.Sprintf("GCE_PD_DRIVER_VERSION=%s", deployOverlayName),
+	)
+	err := runCommand("Deleting driver", deleteCmd)
+	if err != nil {
+		return fmt.Errorf("failed to delete driver: %v", err)
+	}
+	return nil
+}
+
+func pushImage(pkgDir, stagingImage, stagingVersion string) error {
+	err := os.Setenv("GCE_PD_CSI_STAGING_VERSION", stagingVersion)
+	if err != nil {
+		return err
+	}
+	err = os.Setenv("GCE_PD_CSI_STAGING_IMAGE", stagingImage)
+	if err != nil {
+		return err
+	}
+	cmd := exec.Command("make", "-C", pkgDir, "push-container",
+		fmt.Sprintf("GCE_PD_CSI_STAGING_VERSION=%s", stagingVersion),
+		fmt.Sprintf("GCE_PD_CSI_STAGING_IMAGE=%s", stagingImage))
+	err = runCommand("Pushing GCP Container", cmd)
+	if err != nil {
+		return fmt.Errorf("failed to run make command: err: %v", err)
+	}
+	return nil
+}
+
+func deleteImage(stagingImage, stagingVersion string) error {
+	cmd := exec.Command("gcloud", "container", "images", "delete", fmt.Sprintf("%s:%s", stagingImage, stagingVersion), "--quiet")
+	err := runCommand("Deleting GCR Container", cmd)
+	if err != nil {
+		return fmt.Errorf("failed to delete container image %s:%s: %s", stagingImage, stagingVersion, err)
+	}
+	return nil
+}