Skip to content

add ability to use gcloud to boot a gke cluster for e2e integration #288

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jun 7, 2019
Merged

add ability to use gcloud to boot a gke cluster for e2e integration #288

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 73 additions & 12 deletions test/k8s-integration/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ var (
kubeVersion = flag.String("kube-version", "master", "version of Kubernetes to download and use")
kubeFeatureGates = flag.String("kube-feature-gates", "", "feature gates to set on new kubernetes cluster")
localK8sDir = flag.String("local-k8s-dir", "", "local kubernetes/kubernetes directory to run e2e tests from")
deploymentStrat = flag.String("deployment-strategy", "gce", "choose between deploying on gce or gke")
gkeClusterVer = flag.String("gke-cluster-version", "latest", "version of Kubernetes master and node for gke")

// Test infrastructure flags
boskosResourceType = flag.String("boskos-resource-type", "gce-project", "name of the boskos resource type to reserve")
Expand All @@ -59,6 +61,7 @@ var (
const (
pdImagePlaceholder = "gcr.io/gke-release/gcp-compute-persistent-disk-csi-driver"
k8sBuildBinDir = "_output/dockerized/bin/linux/amd64"
gkeTestClusterName = "gcp-pd-csi-driver-test-cluster"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we append some random suffix to it? That will let us run multiple test instances in parallel. Or if some previous test cluster failed to cleanup properly, then subsequent test runs would not get blocked.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Unless I'm mistaken, the current kube-up script does not do this either. Are separate tests run through prow supposed to be run on separate projects, thus avoiding any concurrent and subsequent conflicts.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, technically with boskos project lending, there shouldn't be concurrent runs in the same project. However, I'm thinking more about the scenario where a previous run failed to properly clean up the cluster (for whatever reason).

The kube-up script first tries to delete any existing cluster before trying to create a new one to get around this issue. We may want to consider doing something similar for gke.

)

func init() {
Expand Down Expand Up @@ -99,6 +102,10 @@ func main() {
glog.Fatalf("gce-zone is a required flag")
}

if *deploymentStrat == "gke" && *migrationTest {
glog.Fatalf("Cannot set deployment strategy to 'gke' for migration tests.")
}

err := handle()
if err != nil {
glog.Fatalf("Failed to run integration test: %v", err)
Expand Down Expand Up @@ -196,17 +203,38 @@ func handle() error {
glog.V(4).Infof("Set Kubernetes feature gates: %v", *kubeFeatureGates)
}

err = clusterUp(k8sDir, *gceZone)
if err != nil {
return fmt.Errorf("failed to cluster up: %v", err)
switch *deploymentStrat {
case "gce":
err = clusterUpGCE(k8sDir, *gceZone)
if err != nil {
return fmt.Errorf("failed to cluster up: %v", err)
}
case "gke":
err = clusterUpGKE(*gceZone)
if err != nil {
return fmt.Errorf("failed to cluster up: %v", err)
}
default:
return fmt.Errorf("deployment-strategy must be set to 'gce' or 'gke', but is: %s", *deploymentStrat)
}

}

if *teardownCluster {
defer func() {
err := clusterDown(k8sDir)
if err != nil {
glog.Errorf("failed to cluster down: %v", err)
switch *deploymentStrat {
case "gce":
err := clusterDownGCE(k8sDir)
if err != nil {
glog.Errorf("failed to cluster down: %v", err)
}
case "gke":
err := clusterDownGKE(*gceZone)
if err != nil {
glog.Errorf("failed to cluster down: %v", err)
}
default:
glog.Errorf("deployment-strategy must be set to 'gce' or 'gke', but is: %s", *deploymentStrat)
}
}()
}
Expand Down Expand Up @@ -322,11 +350,21 @@ func runCommand(action string, cmd *exec.Cmd) error {
return nil
}

func clusterDown(k8sDir string) error {
func clusterDownGCE(k8sDir string) error {
cmd := exec.Command(filepath.Join(k8sDir, "hack", "e2e-internal", "e2e-down.sh"))
err := runCommand("Bringing Down E2E Cluster", cmd)
err := runCommand("Bringing Down E2E Cluster on GCE", cmd)
if err != nil {
return fmt.Errorf("failed to bring down kubernetes e2e cluster on gce: %v", err)
}
return nil
}

func clusterDownGKE(gceZone string) error {
cmd := exec.Command("gcloud", "container", "clusters", "delete", gkeTestClusterName,
"--zone", gceZone, "--quiet")
err := runCommand("Bringing Down E2E Cluster on GKE", cmd)
if err != nil {
return fmt.Errorf("failed to bring down kubernetes e2e cluster: %v", err)
return fmt.Errorf("failed to bring down kubernetes e2e cluster on gke: %v", err)
}
return nil
}
Expand All @@ -340,15 +378,38 @@ func buildKubernetes(k8sDir string) error {
return nil
}

func clusterUp(k8sDir, gceZone string) error {
func clusterUpGCE(k8sDir, gceZone string) error {
err := os.Setenv("KUBE_GCE_ZONE", gceZone)
if err != nil {
return err
}
cmd := exec.Command(filepath.Join(k8sDir, "hack", "e2e-internal", "e2e-up.sh"))
err = runCommand("Starting E2E Cluster", cmd)
err = runCommand("Starting E2E Cluster on GCE", cmd)
if err != nil {
return fmt.Errorf("failed to bring up kubernetes e2e cluster on gce: %v", err)
}

return nil
}

func clusterUpGKE(gceZone string) error {
out, err := exec.Command("gcloud", "container", "clusters", "list", "--zone", gceZone,
"--filter", fmt.Sprintf("name=%s", gkeTestClusterName)).CombinedOutput()
if err != nil {
return fmt.Errorf("failed to check for previous test cluster: %v %s", err, out)
}
if len(out) > 0 {
glog.Infof("Detected previous cluster %s. Deleting so a new one can be created...", gkeTestClusterName)
err = clusterDownGKE(gceZone)
if err != nil {
return err
}
}
cmd := exec.Command("gcloud", "container", "clusters", "create", gkeTestClusterName,
"--zone", gceZone, "--cluster-version", *gkeClusterVer, "--quiet")
err = runCommand("Staring E2E Cluster on GKE", cmd)
if err != nil {
return fmt.Errorf("failed to bring up kubernetes e2e cluster: %v", err)
return fmt.Errorf("failed to bring up kubernetes e2e cluster on gke: %v", err)
}

return nil
Expand Down
5 changes: 4 additions & 1 deletion test/run-k8s-integration-local.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ set -o nounset
set -o errexit

readonly PKGDIR=${GOPATH}/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver
readonly deployment_strategy=${DEPLOYMENT_STRATEGY:-gce}

source "${PKGDIR}/deploy/common.sh"

ensure_var GCE_PD_CSI_STAGING_IMAGE
Expand All @@ -14,7 +16,8 @@ make -C ${PKGDIR} test-k8s-integration
# ${PKGDIR}/bin/k8s-integration-test --kube-version=master --run-in-prow=false \
# --staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \
# --deploy-overlay-name=dev --storageclass-file=sc-standard.yaml \
# --test-focus="External.Storage" --gce-zone="us-central1-b"
# --test-focus="External.Storage" --gce-zone="us-central1-b" \
# --deployment-strategy=${deployment_strategy}

# This version of the command does not build the driver or K8s, points to a
# local K8s repo to get the e2e.test binary, and does not bring up or down the cluster
Expand Down
4 changes: 3 additions & 1 deletion test/run-k8s-integration.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,12 @@ readonly PKGDIR=${GOPATH}/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver
readonly overlay_name="${GCE_PD_OVERLAY_NAME:-stable}"
readonly boskos_resource_type="${GCE_PD_BOSKOS_RESOURCE_TYPE:-gce-project}"
readonly do_driver_build="${GCE_PD_DO_DRIVER_BUILD:-true}"
readonly deployment_strategy=${DEPLOYMENT_STRATEGY:-gce}
export GCE_PD_VERBOSITY=9

make -C ${PKGDIR} test-k8s-integration
${PKGDIR}/bin/k8s-integration-test --kube-version=${GCE_PD_KUBE_VERSION:-master} \
--run-in-prow=true --deploy-overlay-name=${overlay_name} --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \
--do-driver-build=${do_driver_build} --boskos-resource-type=${boskos_resource_type} \
--storageclass-file=sc-standard.yaml --test-focus="External.Storage" --gce-zone="us-central1-b"
--storageclass-file=sc-standard.yaml --test-focus="External.Storage" --gce-zone="us-central1-b" \
--deployment-strategy=${deployment_strategy}