Skip to content

Add gke regional cluster support to k8s-integration test #394

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 8, 2019
Merged

Add gke regional cluster support to k8s-integration test #394

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 44 additions & 9 deletions test/k8s-integration/cluster.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,20 @@ import (
"k8s.io/klog"
)

func gkeLocationArgs(gceZone, gceRegion string) (locationArg, locationVal string, err error) {
switch {
case len(gceZone) > 0:
locationArg = "--zone"
locationVal = gceZone
case len(gceRegion) > 0:
locationArg = "--region"
locationVal = gceRegion
default:
return "", "", fmt.Errorf("zone and region unspecified")
}
return
}

func clusterDownGCE(k8sDir string) error {
cmd := exec.Command(filepath.Join(k8sDir, "hack", "e2e-internal", "e2e-down.sh"))
err := runCommand("Bringing Down E2E Cluster on GCE", cmd)
Expand All @@ -19,10 +33,15 @@ func clusterDownGCE(k8sDir string) error {
return nil
}

func clusterDownGKE(gceZone string) error {
func clusterDownGKE(gceZone, gceRegion string) error {
locationArg, locationVal, err := gkeLocationArgs(gceZone, gceRegion)
if err != nil {
return err
}

cmd := exec.Command("gcloud", "container", "clusters", "delete", gkeTestClusterName,
"--zone", gceZone, "--quiet")
err := runCommand("Bringing Down E2E Cluster on GKE", cmd)
locationArg, locationVal, "--quiet")
err = runCommand("Bringing Down E2E Cluster on GKE", cmd)
if err != nil {
return fmt.Errorf("failed to bring down kubernetes e2e cluster on gke: %v", err)
}
Expand Down Expand Up @@ -77,21 +96,27 @@ func clusterUpGCE(k8sDir, gceZone string, numNodes int) error {
return nil
}

func clusterUpGKE(gceZone string, numNodes int) error {
out, err := exec.Command("gcloud", "container", "clusters", "list", "--zone", gceZone,
func clusterUpGKE(gceZone, gceRegion string, numNodes int) error {
locationArg, locationVal, err := gkeLocationArgs(gceZone, gceRegion)
if err != nil {
return err
}

out, err := exec.Command("gcloud", "container", "clusters", "list", locationArg, locationVal,
"--filter", fmt.Sprintf("name=%s", gkeTestClusterName)).CombinedOutput()

if err != nil {
return fmt.Errorf("failed to check for previous test cluster: %v %s", err, out)
}
if len(out) > 0 {
klog.Infof("Detected previous cluster %s. Deleting so a new one can be created...", gkeTestClusterName)
err = clusterDownGKE(gceZone)
err = clusterDownGKE(gceZone, gceRegion)
if err != nil {
return err
}
}
cmd := exec.Command("gcloud", "container", "clusters", "create", gkeTestClusterName,
"--zone", gceZone, "--cluster-version", *gkeClusterVer, "--num-nodes", strconv.Itoa(numNodes),
locationArg, locationVal, "--cluster-version", *gkeClusterVer, "--num-nodes", strconv.Itoa(numNodes),
"--quiet", "--machine-type", "n1-standard-2")
err = runCommand("Staring E2E Cluster on GKE", cmd)
if err != nil {
Expand Down Expand Up @@ -157,7 +182,17 @@ func downloadKubernetesSource(pkgDir, k8sIoDir, kubeVersion string) error {
return nil
}

func getGKEKubeTestArgs() ([]string, error) {
func getGKEKubeTestArgs(gceZone, gceRegion string) ([]string, error) {
var locationArg, locationVal string
switch {
case len(gceZone) > 0:
locationArg = "--gcp-zone"
locationVal = gceZone
case len(gceRegion) > 0:
locationArg = "--gcp-region"
locationVal = gceRegion
}

var gkeEnv string
switch gkeURL := os.Getenv("CLOUDSDK_API_ENDPOINT_OVERRIDES_CONTAINER"); gkeURL {
case "https://staging-container.sandbox.googleapis.com/":
Expand Down Expand Up @@ -189,7 +224,7 @@ func getGKEKubeTestArgs() ([]string, error) {
"--gcp-network=default",
fmt.Sprintf("--cluster=%s", gkeTestClusterName),
fmt.Sprintf("--gke-environment=%s", gkeEnv),
fmt.Sprintf("--gcp-zone=%s", *gceZone),
fmt.Sprintf("%s=%s", locationArg, locationVal),
fmt.Sprintf("--gcp-project=%s", project[:len(project)-1]),
}

Expand Down
1 change: 1 addition & 0 deletions test/k8s-integration/config/test-config-template.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ DriverInfo:
fsGroup: true
exec: true
block: true
topology: true
# dataSource: true
# RWX: true
SupportedMountOption:
Expand Down
40 changes: 24 additions & 16 deletions test/k8s-integration/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,13 +34,15 @@ var (
teardownDriver = flag.Bool("teardown-driver", true, "teardown the driver after the e2e test")
bringupCluster = flag.Bool("bringup-cluster", true, "build kubernetes and bringup a cluster")
gceZone = flag.String("gce-zone", "", "zone that the gce k8s cluster is created/found in")
gceRegion = flag.String("gce-region", "", "region that gke regional cluster should be created in")
kubeVersion = flag.String("kube-version", "", "version of Kubernetes to download and use for the cluster")
testVersion = flag.String("test-version", "", "version of Kubernetes to download and use for tests")
kubeFeatureGates = flag.String("kube-feature-gates", "", "feature gates to set on new kubernetes cluster")
localK8sDir = flag.String("local-k8s-dir", "", "local prebuilt kubernetes/kubernetes directory to use for cluster and test binaries")
deploymentStrat = flag.String("deployment-strategy", "", "choose between deploying on gce or gke")
deploymentStrat = flag.String("deployment-strategy", "gce", "choose between deploying on gce or gke")
gkeClusterVer = flag.String("gke-cluster-version", "", "version of Kubernetes master and node for gke")
numNodes = flag.Int("num-nodes", -1, "the number of nodes in the test cluster")

// Test infrastructure flags
boskosResourceType = flag.String("boskos-resource-type", "gce-project", "name of the boskos resource type to reserve")
storageClassFile = flag.String("storageclass-file", "", "name of storageclass yaml file to use for test relative to test/k8s-integration/config")
Expand Down Expand Up @@ -78,7 +80,12 @@ func main() {
ensureVariable(saFile, true, "service-account-file is a required flag")
ensureVariable(deployOverlayName, true, "deploy-overlay-name is a required flag")
ensureVariable(testFocus, true, "test-focus is a required flag")
ensureVariable(gceZone, true, "gce-zone is a required flag")

if len(*gceRegion) != 0 {
ensureVariable(gceZone, false, "gce-zone and gce-region cannot both be set")
} else {
ensureVariable(gceZone, true, "One of gce-zone or gce-region must be set")
}

if *migrationTest {
ensureVariable(storageClassFile, false, "storage-class-file and migration-test cannot both be set")
Expand All @@ -90,12 +97,6 @@ func main() {
ensureVariable(kubeFeatureGates, false, "kube-feature-gates set but not bringing up new cluster")
}

if *bringupCluster || *teardownCluster {
ensureVariable(deploymentStrat, true, "Must set the deployment strategy if bringing up or down cluster.")
} else {
ensureVariable(deploymentStrat, false, "Cannot set the deployment strategy if not bringing up or down cluster.")
}

if *deploymentStrat == "gke" {
ensureFlag(migrationTest, false, "Cannot set deployment strategy to 'gke' for migration tests.")
ensureVariable(kubeVersion, false, "Cannot set kube-version when using deployment strategy 'gke'. Use gke-cluster-version.")
Expand All @@ -104,6 +105,9 @@ func main() {
if len(*localK8sDir) == 0 {
ensureVariable(testVersion, true, "Must set either test-version or local k8s dir when using deployment strategy 'gke'.")
}
} else if *deploymentStrat == "gce" {
ensureVariable(gceRegion, false, "regional clusters not supported for 'gce' deployment")
ensureVariable(gceZone, true, "gce-zone required for 'gce' deployment")
}

if len(*localK8sDir) != 0 {
Expand Down Expand Up @@ -229,20 +233,14 @@ func handle() error {
testDir = k8sDir
}

var cloudProviderArgs []string

// Create a cluster either through GKE or GCE
if *bringupCluster {
var err error = nil
switch *deploymentStrat {
case "gce":
err = clusterUpGCE(k8sDir, *gceZone, *numNodes)
case "gke":
err = clusterUpGKE(*gceZone, *numNodes)
cloudProviderArgs, err = getGKEKubeTestArgs()
if err != nil {
return fmt.Errorf("failed to build GKE kubetest args: %v", err)
}
err = clusterUpGKE(*gceZone, *gceRegion, *numNodes)
default:
err = fmt.Errorf("deployment-strategy must be set to 'gce' or 'gke', but is: %s", *deploymentStrat)
}
Expand All @@ -261,7 +259,7 @@ func handle() error {
klog.Errorf("failed to cluster down: %v", err)
}
case "gke":
err := clusterDownGKE(*gceZone)
err := clusterDownGKE(*gceZone, *gceRegion)
if err != nil {
klog.Errorf("failed to cluster down: %v", err)
}
Expand All @@ -285,6 +283,15 @@ func handle() error {
return fmt.Errorf("failed to install CSI Driver: %v", err)
}

var cloudProviderArgs []string
switch *deploymentStrat {
case "gke":
cloudProviderArgs, err = getGKEKubeTestArgs(*gceZone, *gceRegion)
if err != nil {
return fmt.Errorf("failed to build GKE kubetest args: %v", err)
}
}

// Run the tests using the testDir kubernetes
if len(*storageClassFile) != 0 {
err = runCSITests(pkgDir, testDir, *testFocus, *storageClassFile, cloudProviderArgs)
Expand Down Expand Up @@ -348,6 +355,7 @@ func runTestsWithConfig(testDir, testFocus, testConfigArg string, cloudProviderA
kubeTestArgs := []string{
"--test",
"--ginkgo-parallel",
"--check-version-skew=false",
fmt.Sprintf("--test_args=%s", testArgs),
}

Expand Down