Skip to content

Revert "Cherry-Pick: GKE Integration Support into Release-0.5.X" #318

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jun 25, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 12 additions & 73 deletions test/k8s-integration/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,6 @@ var (
kubeVersion = flag.String("kube-version", "master", "version of Kubernetes to download and use")
kubeFeatureGates = flag.String("kube-feature-gates", "", "feature gates to set on new kubernetes cluster")
localK8sDir = flag.String("local-k8s-dir", "", "local kubernetes/kubernetes directory to run e2e tests from")
deploymentStrat = flag.String("deployment-strategy", "gce", "choose between deploying on gce or gke")
gkeClusterVer = flag.String("gke-cluster-version", "latest", "version of Kubernetes master and node for gke")

// Test infrastructure flags
boskosResourceType = flag.String("boskos-resource-type", "gce-project", "name of the boskos resource type to reserve")
Expand All @@ -61,7 +59,6 @@ var (
const (
pdImagePlaceholder = "gke.gcr.io/gcp-compute-persistent-disk-csi-driver"
k8sBuildBinDir = "_output/dockerized/bin/linux/amd64"
gkeTestClusterName = "gcp-pd-csi-driver-test-cluster"
)

func init() {
Expand Down Expand Up @@ -102,10 +99,6 @@ func main() {
glog.Fatalf("gce-zone is a required flag")
}

if *deploymentStrat == "gke" && *migrationTest {
glog.Fatalf("Cannot set deployment strategy to 'gke' for migration tests.")
}

err := handle()
if err != nil {
glog.Fatalf("Failed to run integration test: %v", err)
Expand Down Expand Up @@ -203,38 +196,17 @@ func handle() error {
glog.V(4).Infof("Set Kubernetes feature gates: %v", *kubeFeatureGates)
}

switch *deploymentStrat {
case "gce":
err = clusterUpGCE(k8sDir, *gceZone)
if err != nil {
return fmt.Errorf("failed to cluster up: %v", err)
}
case "gke":
err = clusterUpGKE(*gceZone)
if err != nil {
return fmt.Errorf("failed to cluster up: %v", err)
}
default:
return fmt.Errorf("deployment-strategy must be set to 'gce' or 'gke', but is: %s", *deploymentStrat)
err = clusterUp(k8sDir, *gceZone)
if err != nil {
return fmt.Errorf("failed to cluster up: %v", err)
}

}

if *teardownCluster {
defer func() {
switch *deploymentStrat {
case "gce":
err := clusterDownGCE(k8sDir)
if err != nil {
glog.Errorf("failed to cluster down: %v", err)
}
case "gke":
err := clusterDownGKE(*gceZone)
if err != nil {
glog.Errorf("failed to cluster down: %v", err)
}
default:
glog.Errorf("deployment-strategy must be set to 'gce' or 'gke', but is: %s", *deploymentStrat)
err := clusterDown(k8sDir)
if err != nil {
glog.Errorf("failed to cluster down: %v", err)
}
}()
}
Expand Down Expand Up @@ -350,21 +322,11 @@ func runCommand(action string, cmd *exec.Cmd) error {
return nil
}

func clusterDownGCE(k8sDir string) error {
func clusterDown(k8sDir string) error {
cmd := exec.Command(filepath.Join(k8sDir, "hack", "e2e-internal", "e2e-down.sh"))
err := runCommand("Bringing Down E2E Cluster on GCE", cmd)
if err != nil {
return fmt.Errorf("failed to bring down kubernetes e2e cluster on gce: %v", err)
}
return nil
}

func clusterDownGKE(gceZone string) error {
cmd := exec.Command("gcloud", "container", "clusters", "delete", gkeTestClusterName,
"--zone", gceZone, "--quiet")
err := runCommand("Bringing Down E2E Cluster on GKE", cmd)
err := runCommand("Bringing Down E2E Cluster", cmd)
if err != nil {
return fmt.Errorf("failed to bring down kubernetes e2e cluster on gke: %v", err)
return fmt.Errorf("failed to bring down kubernetes e2e cluster: %v", err)
}
return nil
}
Expand All @@ -378,38 +340,15 @@ func buildKubernetes(k8sDir string) error {
return nil
}

func clusterUpGCE(k8sDir, gceZone string) error {
func clusterUp(k8sDir, gceZone string) error {
err := os.Setenv("KUBE_GCE_ZONE", gceZone)
if err != nil {
return err
}
cmd := exec.Command(filepath.Join(k8sDir, "hack", "e2e-internal", "e2e-up.sh"))
err = runCommand("Starting E2E Cluster on GCE", cmd)
if err != nil {
return fmt.Errorf("failed to bring up kubernetes e2e cluster on gce: %v", err)
}

return nil
}

func clusterUpGKE(gceZone string) error {
out, err := exec.Command("gcloud", "container", "clusters", "list", "--zone", gceZone,
"--filter", fmt.Sprintf("name=%s", gkeTestClusterName)).CombinedOutput()
if err != nil {
return fmt.Errorf("failed to check for previous test cluster: %v %s", err, out)
}
if len(out) > 0 {
glog.Infof("Detected previous cluster %s. Deleting so a new one can be created...", gkeTestClusterName)
err = clusterDownGKE(gceZone)
if err != nil {
return err
}
}
cmd := exec.Command("gcloud", "container", "clusters", "create", gkeTestClusterName,
"--zone", gceZone, "--cluster-version", *gkeClusterVer, "--quiet")
err = runCommand("Staring E2E Cluster on GKE", cmd)
err = runCommand("Starting E2E Cluster", cmd)
if err != nil {
return fmt.Errorf("failed to bring up kubernetes e2e cluster on gke: %v", err)
return fmt.Errorf("failed to bring up kubernetes e2e cluster: %v", err)
}

return nil
Expand Down
11 changes: 2 additions & 9 deletions test/run-k8s-integration-local.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,6 @@ set -o nounset
set -o errexit

readonly PKGDIR=${GOPATH}/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver
readonly deployment_strategy=${DEPLOYMENT_STRATEGY:-gce}
readonly gke_cluster_version=${GKE_CLUSTER_VERSION:-latest}
readonly kube_version=${KUBE_VERSION:-master}

source "${PKGDIR}/deploy/common.sh"

ensure_var GCE_PD_CSI_STAGING_IMAGE
Expand All @@ -18,9 +14,7 @@ make -C ${PKGDIR} test-k8s-integration
# ${PKGDIR}/bin/k8s-integration-test --kube-version=master --run-in-prow=false \
# --staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \
# --deploy-overlay-name=dev --storageclass-file=sc-standard.yaml \
# --test-focus="External.Storage" --gce-zone="us-central1-b" \
# --deployment-strategy=${deployment_strategy} --gke-cluster-version=${gke_cluster_version} \
# --kube-version=${kube_version}
# --test-focus="External.Storage" --gce-zone="us-central1-b"

# This version of the command does not build the driver or K8s, points to a
# local K8s repo to get the e2e.test binary, and does not bring up or down the cluster
Expand All @@ -29,5 +23,4 @@ ${PKGDIR}/bin/k8s-integration-test --kube-version=master --run-in-prow=false \
--staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \
--deploy-overlay-name=dev --bringup-cluster=false --teardown-cluster=false --local-k8s-dir=$KTOP \
--storageclass-file=sc-standard.yaml --do-driver-build=true --test-focus="External.Storage" \
--gce-zone="us-central1-b" --gke-cluster-version=${gke_cluster_version} \
--kube-version=${kube_version}
--gce-zone="us-central1-b"
8 changes: 1 addition & 7 deletions test/run-k8s-integration.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,10 @@ readonly PKGDIR=${GOPATH}/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver
readonly overlay_name="${GCE_PD_OVERLAY_NAME:-stable}"
readonly boskos_resource_type="${GCE_PD_BOSKOS_RESOURCE_TYPE:-gce-project}"
readonly do_driver_build="${GCE_PD_DO_DRIVER_BUILD:-true}"
readonly deployment_strategy=${DEPLOYMENT_STRATEGY:-gce}
readonly gke_cluster_version=${GKE_CLUSTER_VERSION:-latest}
readonly kube_version=${KUBE_VERSION:-master}

export GCE_PD_VERBOSITY=9

make -C ${PKGDIR} test-k8s-integration
${PKGDIR}/bin/k8s-integration-test --kube-version=${GCE_PD_KUBE_VERSION:-master} \
--run-in-prow=true --deploy-overlay-name=${overlay_name} --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \
--do-driver-build=${do_driver_build} --boskos-resource-type=${boskos_resource_type} \
--storageclass-file=sc-standard.yaml --test-focus="External.Storage" --gce-zone="us-central1-b" \
--deployment-strategy=${deployment_strategy} --gke-cluster-version=${gke_cluster_version} \
--kube-version=${kube_version}
--storageclass-file=sc-standard.yaml --test-focus="External.Storage" --gce-zone="us-central1-b"