From 319631ba8f951626e68cf0fe497218760ab7b3a2 Mon Sep 17 00:00:00 2001 From: "Hantao (Will) Wang" Date: Mon, 24 Jun 2019 11:49:59 -0700 Subject: [PATCH 1/2] add background building and flag to specify test version spearate from kube version --- test/k8s-integration/main.go | 278 +++++++++++++++++++++--------- test/run-k8s-integration-local.sh | 26 ++- test/run-k8s-integration.sh | 23 ++- 3 files changed, 233 insertions(+), 94 deletions(-) diff --git a/test/k8s-integration/main.go b/test/k8s-integration/main.go index be6ac049f..d910392d5 100644 --- a/test/k8s-integration/main.go +++ b/test/k8s-integration/main.go @@ -17,6 +17,7 @@ package main import ( "flag" "fmt" + "io/ioutil" "os" "os/exec" "path/filepath" @@ -36,11 +37,12 @@ var ( teardownDriver = flag.Bool("teardown-driver", true, "teardown the driver after the e2e test") bringupCluster = flag.Bool("bringup-cluster", true, "build kubernetes and bringup a cluster") gceZone = flag.String("gce-zone", "", "zone that the gce k8s cluster is created/found in") - kubeVersion = flag.String("kube-version", "master", "version of Kubernetes to download and use") + kubeVersion = flag.String("kube-version", "", "version of Kubernetes to download and use for the cluster") + testVersion = flag.String("test-version", "", "version of Kubernetes to download and use for tests") kubeFeatureGates = flag.String("kube-feature-gates", "", "feature gates to set on new kubernetes cluster") localK8sDir = flag.String("local-k8s-dir", "", "local kubernetes/kubernetes directory to run e2e tests from") - deploymentStrat = flag.String("deployment-strategy", "gce", "choose between deploying on gce or gke") - gkeClusterVer = flag.String("gke-cluster-version", "latest", "version of Kubernetes master and node for gke") + deploymentStrat = flag.String("deployment-strategy", "", "choose between deploying on gce or gke") + gkeClusterVer = flag.String("gke-cluster-version", "", "version of Kubernetes master and node for gke") // Test infrastructure flags boskosResourceType = flag.String("boskos-resource-type", "gce-project", "name of the boskos resource type to reserve") @@ -67,43 +69,62 @@ const ( func init() { flag.Set("logtostderr", "true") } -func main() { - flag.Parse() - if len(*stagingImage) == 0 && !*inProw { - klog.Fatalf("staging-image is a required flag, please specify the name of image to stage to") +func ensureVariable(v *string, set bool, msgOnError string) { + if set && len(*v) == 0 { + klog.Fatal(msgOnError) + } else if !set && len(*v) != 0 { + klog.Fatal(msgOnError) } +} - if len(*saFile) == 0 { - klog.Fatalf("service-account-file is a required flag") +func ensureFlag(v *bool, setTo bool, msgOnError string) { + if *v != setTo { + klog.Fatal(msgOnError) } +} - if len(*deployOverlayName) == 0 { - klog.Fatalf("deploy-overlay-name is a required flag") - } +func main() { + flag.Parse() - if len(*storageClassFile) == 0 && !*migrationTest { - klog.Fatalf("One of storageclass-file and migration-test must be set") + if !*inProw { + ensureVariable(stagingImage, true, "staging-image is a required flag, please specify the name of image to stage to") } - if len(*storageClassFile) != 0 && *migrationTest { - klog.Fatalf("storage-class-file and migration-test cannot both be set") + ensureVariable(saFile, true, "service-account-file is a required flag") + ensureVariable(deployOverlayName, true, "deploy-overlay-name is a required flag") + ensureVariable(testFocus, true, "test-focus is a required flag") + ensureVariable(gceZone, true, "gce-zone is a required flag") + + if *migrationTest { + ensureVariable(storageClassFile, false, "storage-class-file and migration-test cannot both be set") + } else { + ensureVariable(storageClassFile, true, "One of storageclass-file and migration-test must be set") } - if !*bringupCluster && len(*kubeFeatureGates) > 0 { - klog.Fatalf("kube-feature-gates set but not bringing up new cluster") + if !*bringupCluster { + ensureVariable(kubeFeatureGates, false, "kube-feature-gates set but not bringing up new cluster") } - if len(*testFocus) == 0 { - klog.Fatalf("test-focus is a required flag") + if *bringupCluster || *teardownCluster { + ensureVariable(deploymentStrat, true, "Must set the deployment strategy if bringing up or down cluster.") + } else { + ensureVariable(deploymentStrat, false, "Cannot set the deployment strategy if not bringing up or down cluster.") } - if len(*gceZone) == 0 { - klog.Fatalf("gce-zone is a required flag") + if *deploymentStrat == "gke" { + ensureFlag(migrationTest, false, "Cannot set deployment strategy to 'gke' for migration tests.") + ensureVariable(kubeVersion, false, "Cannot set kube-version when using deployment strategy 'gke'. Use gke-cluster-version.") + ensureVariable(gkeClusterVer, true, "Must set gke-cluster-version when using deployment strategy 'gke'.") + ensureVariable(kubeFeatureGates, false, "Cannot set feature gates when using deployment strategy 'gke'.") + if len(*localK8sDir) == 0 { + ensureVariable(testVersion, true, "Must set either test-version or local k8s dir when using deployment strategy 'gke'.") + } } - if *deploymentStrat == "gke" && *migrationTest { - klog.Fatalf("Cannot set deployment strategy to 'gke' for migration tests.") + if len(*localK8sDir) != 0 { + ensureVariable(kubeVersion, false, "Cannot set a kube version when using a local k8s dir.") + ensureVariable(testVersion, false, "Cannot set a test version when using a local k8s dir.") } err := handle() @@ -122,10 +143,10 @@ func handle() error { if !ok { return fmt.Errorf("Could not find env variable GOPATH") } + pkgDir := filepath.Join(goPath, "src", "sigs.k8s.io", "gcp-compute-persistent-disk-csi-driver") - k8sIoDir := filepath.Join(pkgDir, "test", "k8s-integration", "src", "k8s.io") - k8sDir := filepath.Join(k8sIoDir, "kubernetes") + // If running in Prow, then acquire and set up a project through Boskos if *inProw { project, _ := testutils.SetupProwConfig(*boskosResourceType) @@ -157,69 +178,134 @@ func handle() error { } } + // Create temporary directories for kubernetes builds + k8sParentDir := getTempDir() + k8sDir := filepath.Join(k8sParentDir, "kubernetes") + testParentDir := getTempDir() + testDir := filepath.Join(testParentDir, "kubernetes") + defer removeTempDir(k8sParentDir) + defer removeTempDir(testParentDir) + + numTasks := 4 + errChan := make(chan error, numTasks) + k8sDependencyChan := make(chan bool, 1) + + // If kube version is set, then download and build Kubernetes for cluster creation + // Otherwise, either GKE or a prebuild local K8s dir is being used + if len(*kubeVersion) != 0 { + go func() { + err := downloadKubernetesSource(pkgDir, k8sParentDir, *kubeVersion) + if err != nil { + errChan <- fmt.Errorf("failed to download Kubernetes source: %v", err) + k8sDependencyChan <- false + return + } + err = buildKubernetes(k8sDir) + if err != nil { + errChan <- fmt.Errorf("failed to build Kubernetes: %v", err) + k8sDependencyChan <- false + return + } + k8sDependencyChan <- true + errChan <- nil + }() + } else { + errChan <- nil + k8sDir = *localK8sDir + k8sDependencyChan <- true + } + + // If test version is set, then download and build Kubernetes to run K8s tests + // Otherwise, either kube version is set (which implies GCE) or a local K8s dir is being used + if len(*testVersion) != 0 && *testVersion != *kubeVersion { + go func() { + // TODO: Build only the tests + err := downloadKubernetesSource(pkgDir, testParentDir, *testVersion) + if err != nil { + errChan <- fmt.Errorf("failed to download Kubernetes source: %v", err) + return + } + err = buildKubernetes(testDir) + if err != nil { + errChan <- fmt.Errorf("failed to build Kubernetes: %v", err) + return + } + errChan <- nil + }() + } else { + testDir = k8sDir + errChan <- nil + } + + // Build and push the driver, if required. Defer the driver image deletion. if *doDriverBuild { - err := pushImage(pkgDir, *stagingImage, stagingVersion) - if err != nil { - return fmt.Errorf("failed pushing image: %v", err) - } + go func() { + errChan <- pushImage(pkgDir, *stagingImage, stagingVersion) + }() defer func() { if *teardownCluster { - err = deleteImage(*stagingImage, stagingVersion) + err := deleteImage(*stagingImage, stagingVersion) if err != nil { klog.Errorf("failed to delete image: %v", err) } } }() + } else { + errChan <- nil } if *bringupCluster { - err := downloadKubernetesSource(pkgDir, k8sIoDir, *kubeVersion) - if err != nil { - return fmt.Errorf("failed to download Kubernetes source: %v", err) - } - - err = buildKubernetes(k8sDir) - if err != nil { - return fmt.Errorf("failed to build Kubernetes: %v", err) - } - - kshPath := filepath.Join(k8sDir, "cluster", "kubectl.sh") - _, err = os.Stat(kshPath) - if err == nil { - // Set kubectl to the one bundled in the k8s tar for versioning - err = os.Setenv("GCE_PD_KUBECTL", kshPath) - if err != nil { - return fmt.Errorf("failed to set cluster specific kubectl: %v", err) + go func() { + if !(<-k8sDependencyChan) { + errChan <- nil + return } - } else { - klog.Errorf("could not find cluster kubectl at %s, falling back to default kubectl", kshPath) - } - if len(*kubeFeatureGates) != 0 { - err = os.Setenv("KUBE_FEATURE_GATES", *kubeFeatureGates) - if err != nil { - return fmt.Errorf("failed to set kubernetes feature gates: %v", err) - } - klog.V(4).Infof("Set Kubernetes feature gates: %v", *kubeFeatureGates) - } + switch *deploymentStrat { + case "gce": + kshPath := filepath.Join(k8sDir, "cluster", "kubectl.sh") + _, err := os.Stat(kshPath) + if err == nil { + // Set kubectl to the one bundled in the k8s tar for versioning + err = os.Setenv("GCE_PD_KUBECTL", kshPath) + if err != nil { + errChan <- fmt.Errorf("failed to set cluster specific kubectl: %v", err) + return + } + } else { + klog.Errorf("could not find cluster kubectl at %s, falling back to default kubectl", kshPath) + } - switch *deploymentStrat { - case "gce": - err = clusterUpGCE(k8sDir, *gceZone) - if err != nil { - return fmt.Errorf("failed to cluster up: %v", err) - } - case "gke": - err = clusterUpGKE(*gceZone) - if err != nil { - return fmt.Errorf("failed to cluster up: %v", err) - } - default: - return fmt.Errorf("deployment-strategy must be set to 'gce' or 'gke', but is: %s", *deploymentStrat) - } + if len(*kubeFeatureGates) != 0 { + err = os.Setenv("KUBE_FEATURE_GATES", *kubeFeatureGates) + if err != nil { + errChan <- fmt.Errorf("failed to set kubernetes feature gates: %v", err) + return + } + klog.V(4).Infof("Set Kubernetes feature gates: %v", *kubeFeatureGates) + } + err = clusterUpGCE(k8sDir, *gceZone) + if err != nil { + errChan <- fmt.Errorf("failed to cluster up: %v", err) + return + } + case "gke": + err := clusterUpGKE(*gceZone) + if err != nil { + errChan <- fmt.Errorf("failed to cluster up: %v", err) + return + } + default: + errChan <- fmt.Errorf("deployment-strategy must be set to 'gce' or 'gke', but is: %s", *deploymentStrat) + } + errChan <- nil + }() + } else { + errChan <- nil } + // Defer the tear down of the cluster through GKE or GCE if *teardownCluster { defer func() { switch *deploymentStrat { @@ -239,7 +325,21 @@ func handle() error { }() } - err := installDriver(goPath, pkgDir, k8sDir, *stagingImage, stagingVersion, *deployOverlayName, *doDriverBuild) + // Block until all background operations are complete + var firstErr error = nil + for i := 0; i < 4; i++ { + if err := <-errChan; err != nil { + if firstErr == nil { + firstErr = err + } + } + } + if firstErr != nil { + return firstErr + } + + // Install the driver and defer its teardown + err := installDriver(goPath, pkgDir, *stagingImage, stagingVersion, *deployOverlayName, *doDriverBuild) if *teardownDriver { defer func() { // TODO (#140): collect driver logs @@ -252,14 +352,11 @@ func handle() error { return fmt.Errorf("failed to install CSI Driver: %v", err) } - if len(*localK8sDir) != 0 { - k8sDir = *localK8sDir - } - + // Run the tests using the testDir kubernetes if len(*storageClassFile) != 0 { - err = runCSITests(pkgDir, k8sDir, *testFocus, *storageClassFile, *gceZone) + err = runCSITests(pkgDir, testDir, *testFocus, *storageClassFile, *gceZone) } else if *migrationTest { - err = runMigrationTests(pkgDir, k8sDir, *testFocus, *gceZone) + err = runMigrationTests(pkgDir, testDir, *testFocus, *gceZone) } else { return fmt.Errorf("Did not run either CSI or Migration test") } @@ -420,7 +517,7 @@ func getOverlayDir(pkgDir, deployOverlayName string) string { return filepath.Join(pkgDir, "deploy", "kubernetes", "overlays", deployOverlayName) } -func installDriver(goPath, pkgDir, k8sDir, stagingImage, stagingVersion, deployOverlayName string, doDriverBuild bool) error { +func installDriver(goPath, pkgDir, stagingImage, stagingVersion, deployOverlayName string, doDriverBuild bool) error { if doDriverBuild { // Install kustomize out, err := exec.Command(filepath.Join(pkgDir, "deploy", "kubernetes", "install-kustomize.sh")).CombinedOutput() @@ -449,10 +546,8 @@ func installDriver(goPath, pkgDir, k8sDir, stagingImage, stagingVersion, deployO } // setup service account file for secret creation - tmpSaFile := fmt.Sprintf("/tmp/%s/cloud-sa.json", string(uuid.NewUUID())) - - os.MkdirAll(filepath.Dir(tmpSaFile), 0750) - defer os.Remove(filepath.Dir(tmpSaFile)) + tmpSaFile := filepath.Join(getTempDir(), "cloud-sa.json") + defer removeTempDir(filepath.Dir(tmpSaFile)) // Need to copy it to name the file "cloud-sa.json" out, err := exec.Command("cp", *saFile, tmpSaFile).CombinedOutput() @@ -602,3 +697,18 @@ func deleteImage(stagingImage, stagingVersion string) error { } return nil } + +func getTempDir() string { + dir, err := ioutil.TempDir("", "") + if err != nil { + klog.Fatalf("Error creating temp dir: %v", err) + } + return dir +} + +func removeTempDir(dir string) { + err := os.RemoveAll(dir) + if err != nil { + klog.Fatalf("Error removing temp dir: %v", err) + } +} diff --git a/test/run-k8s-integration-local.sh b/test/run-k8s-integration-local.sh index a913c8eb5..a63005e2d 100755 --- a/test/run-k8s-integration-local.sh +++ b/test/run-k8s-integration-local.sh @@ -4,7 +4,9 @@ set -o nounset set -o errexit readonly PKGDIR=${GOPATH}/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver -readonly deployment_strategy=${DEPLOYMENT_STRATEGY:-gce} +readonly gke_cluster_version=${GKE_CLUSTER_VERSION:-latest} +readonly kube_version=${KUBE_VERSION:-master} +readonly test_version=${TEST_VERSION:-master} source "${PKGDIR}/deploy/common.sh" @@ -13,17 +15,31 @@ ensure_var GCE_PD_SA_DIR make -C ${PKGDIR} test-k8s-integration -# ${PKGDIR}/bin/k8s-integration-test --kube-version=master --run-in-prow=false \ +# This version of the command creates a GKE cluster. It also downloads and builds a k8s release +# so that it can run the test specified + +# ${PKGDIR}/bin/k8s-integration-test --run-in-prow=false \ # --staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \ # --deploy-overlay-name=dev --storageclass-file=sc-standard.yaml \ # --test-focus="External.Storage" --gce-zone="us-central1-b" \ -# --deployment-strategy=${deployment_strategy} +# --deployment-strategy=gke --gke-cluster-version=${gke_cluster_version} \ +# --test-version=${test_version} + +# This version of the command creates a GCE cluster. It downloads and builds two k8s releases, +# one for the cluster and one for the tests, unless the cluster and test versioning is the same. + +# ${PKGDIR}/bin/k8s-integration-test --run-in-prow=false \ +# --staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \ +# --deploy-overlay-name=dev --storageclass-file=sc-standard.yaml \ +# --test-focus="External.Storage" --gce-zone="us-central1-b" \ +# --deployment-strategy=gce --kube-version=${kube_version} \ +# --test-version=${test_version} # This version of the command does not build the driver or K8s, points to a # local K8s repo to get the e2e.test binary, and does not bring up or down the cluster -# + ${PKGDIR}/bin/k8s-integration-test --kube-version=master --run-in-prow=false \ --staging-image=${GCE_PD_CSI_STAGING_IMAGE} --service-account-file=${GCE_PD_SA_DIR}/cloud-sa.json \ --deploy-overlay-name=dev --bringup-cluster=false --teardown-cluster=false --local-k8s-dir=$KTOP \ --storageclass-file=sc-standard.yaml --do-driver-build=true --test-focus="External.Storage" \ ---gce-zone="us-central1-b" +--gce-zone="us-central1-b" --test-version=${test_version} diff --git a/test/run-k8s-integration.sh b/test/run-k8s-integration.sh index 3ba42b0de..13fb60e39 100755 --- a/test/run-k8s-integration.sh +++ b/test/run-k8s-integration.sh @@ -14,11 +14,24 @@ readonly overlay_name="${GCE_PD_OVERLAY_NAME:-stable}" readonly boskos_resource_type="${GCE_PD_BOSKOS_RESOURCE_TYPE:-gce-project}" readonly do_driver_build="${GCE_PD_DO_DRIVER_BUILD:-true}" readonly deployment_strategy=${DEPLOYMENT_STRATEGY:-gce} +readonly gke_cluster_version=${GKE_CLUSTER_VERSION:-latest} +readonly kube_version=${GCE_PD_KUBE_VERSION:-master} +readonly test_version=${TEST_VERSION:-master} + export GCE_PD_VERBOSITY=9 make -C ${PKGDIR} test-k8s-integration -${PKGDIR}/bin/k8s-integration-test --kube-version=${GCE_PD_KUBE_VERSION:-master} \ ---run-in-prow=true --deploy-overlay-name=${overlay_name} --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \ ---do-driver-build=${do_driver_build} --boskos-resource-type=${boskos_resource_type} \ ---storageclass-file=sc-standard.yaml --test-focus="External.Storage" --gce-zone="us-central1-b" \ ---deployment-strategy=${deployment_strategy} + +base_cmd="${PKGDIR}/bin/k8s-integration-test \ + --run-in-prow=true --deploy-overlay-name=${overlay_name} --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \ + --do-driver-build=${do_driver_build} --boskos-resource-type=${boskos_resource_type} \ + --storageclass-file=sc-standard.yaml --test-focus="External.Storage" --gce-zone="us-central1-b" \ + --deployment-strategy=${deployment_strategy} --test-version=${test_version}" + +if [ "$deployment_strategy" = "gke" ]; then + base_cmd="${base_cmd} --gke-cluster-version=${gke_cluster_version}" +else + base_cmd="${base_cmd} --kube-version=${kube_version}" +fi + +eval $base_cmd From 2a91a29c1596038b89820ae4fbc7734915fa717e Mon Sep 17 00:00:00 2001 From: "Hantao (Will) Wang" Date: Tue, 25 Jun 2019 14:26:32 -0700 Subject: [PATCH 2/2] break up large file into smaller pieces --- test/k8s-integration/cluster.go | 131 ++++++++++++++ test/k8s-integration/driver.go | 116 ++++++++++++ test/k8s-integration/main.go | 305 +------------------------------- test/k8s-integration/utils.go | 81 +++++++++ 4 files changed, 329 insertions(+), 304 deletions(-) create mode 100644 test/k8s-integration/cluster.go create mode 100644 test/k8s-integration/driver.go create mode 100644 test/k8s-integration/utils.go diff --git a/test/k8s-integration/cluster.go b/test/k8s-integration/cluster.go new file mode 100644 index 000000000..1ed5b1094 --- /dev/null +++ b/test/k8s-integration/cluster.go @@ -0,0 +1,131 @@ +package main + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + + "k8s.io/klog" +) + +func clusterUpGKE(gceZone string) error { + out, err := exec.Command("gcloud", "container", "clusters", "list", "--zone", gceZone, + "--filter", fmt.Sprintf("name=%s", gkeTestClusterName)).CombinedOutput() + if err != nil { + return fmt.Errorf("failed to check for previous test cluster: %v %s", err, out) + } + if len(out) > 0 { + klog.Infof("Detected previous cluster %s. Deleting so a new one can be created...", gkeTestClusterName) + err = clusterDownGKE(gceZone) + if err != nil { + return err + } + } + cmd := exec.Command("gcloud", "container", "clusters", "create", gkeTestClusterName, + "--zone", gceZone, "--cluster-version", *gkeClusterVer, "--quiet") + err = runCommand("Staring E2E Cluster on GKE", cmd) + if err != nil { + return fmt.Errorf("failed to bring up kubernetes e2e cluster on gke: %v", err) + } + + return nil +} + +func clusterDownGKE(gceZone string) error { + cmd := exec.Command("gcloud", "container", "clusters", "delete", gkeTestClusterName, + "--zone", gceZone, "--quiet") + err := runCommand("Bringing Down E2E Cluster on GKE", cmd) + if err != nil { + return fmt.Errorf("failed to bring down kubernetes e2e cluster on gke: %v", err) + } + return nil +} + +func clusterUpGCE(k8sDir, gceZone string) error { + err := os.Setenv("KUBE_GCE_ZONE", gceZone) + if err != nil { + return err + } + cmd := exec.Command(filepath.Join(k8sDir, "hack", "e2e-internal", "e2e-up.sh")) + err = runCommand("Starting E2E Cluster on GCE", cmd) + if err != nil { + return fmt.Errorf("failed to bring up kubernetes e2e cluster on gce: %v", err) + } + + return nil +} + +func clusterDownGCE(k8sDir string) error { + cmd := exec.Command(filepath.Join(k8sDir, "hack", "e2e-internal", "e2e-down.sh")) + err := runCommand("Bringing Down E2E Cluster on GCE", cmd) + if err != nil { + return fmt.Errorf("failed to bring down kubernetes e2e cluster on gce: %v", err) + } + return nil +} + +func buildKubernetes(k8sDir string) error { + cmd := exec.Command("make", "-C", k8sDir, "quick-release") + err := runCommand("Building Kubernetes", cmd) + if err != nil { + return fmt.Errorf("failed to build Kubernetes: %v", err) + } + return nil +} + +func downloadKubernetesSource(pkgDir, k8sIoDir, kubeVersion string) error { + k8sDir := filepath.Join(k8sIoDir, "kubernetes") + /* + // TODO: Download a fresh copy every time until mutate manifests hardcoding existing image is solved. + if _, err := os.Stat(k8sDir); !os.IsNotExist(err) { + klog.Infof("Staging Kubernetes already found at %s, skipping download", k8sDir) + return nil + } + */ + + klog.V(4).Infof("Staging Kubernetes folder not found, downloading now") + + err := os.MkdirAll(k8sIoDir, 0777) + if err != nil { + return err + } + + kubeTarDir := filepath.Join(k8sIoDir, fmt.Sprintf("kubernetes-%s.tar.gz", kubeVersion)) + + var vKubeVersion string + if kubeVersion == "master" { + vKubeVersion = kubeVersion + // A hack to be able to build Kubernetes in this nested place + // KUBE_GIT_VERSION_FILE set to file to load kube version from + err = os.Setenv("KUBE_GIT_VERSION_FILE", filepath.Join(pkgDir, "test", "k8s-integration", ".dockerized-kube-version-defs")) + if err != nil { + return err + } + } else { + vKubeVersion = "v" + kubeVersion + } + out, err := exec.Command("curl", "-L", fmt.Sprintf("https://github.com/kubernetes/kubernetes/archive/%s.tar.gz", vKubeVersion), "-o", kubeTarDir).CombinedOutput() + if err != nil { + return fmt.Errorf("failed to curl kubernetes version %s: %s, err: %v", kubeVersion, out, err) + } + + out, err = exec.Command("tar", "-C", k8sIoDir, "-xvf", kubeTarDir).CombinedOutput() + if err != nil { + return fmt.Errorf("failed to untar %s: %s, err: %v", kubeTarDir, out, err) + } + + err = os.RemoveAll(k8sDir) + if err != nil { + return err + } + + err = os.Rename(filepath.Join(k8sIoDir, fmt.Sprintf("kubernetes-%s", kubeVersion)), k8sDir) + if err != nil { + return err + } + + klog.V(4).Infof("Successfully downloaded Kubernetes v%s to %s", kubeVersion, k8sDir) + + return nil +} diff --git a/test/k8s-integration/driver.go b/test/k8s-integration/driver.go new file mode 100644 index 000000000..607fe8aa8 --- /dev/null +++ b/test/k8s-integration/driver.go @@ -0,0 +1,116 @@ +package main + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "time" +) + +func getOverlayDir(pkgDir, deployOverlayName string) string { + return filepath.Join(pkgDir, "deploy", "kubernetes", "overlays", deployOverlayName) +} + +func installDriver(goPath, pkgDir, stagingImage, stagingVersion, deployOverlayName string, doDriverBuild bool) error { + if doDriverBuild { + // Install kustomize + out, err := exec.Command(filepath.Join(pkgDir, "deploy", "kubernetes", "install-kustomize.sh")).CombinedOutput() + if err != nil { + return fmt.Errorf("failed to install kustomize: %s, err: %v", out, err) + } + + // Edit ci kustomization to use given image tag + overlayDir := getOverlayDir(pkgDir, deployOverlayName) + err = os.Chdir(overlayDir) + if err != nil { + return fmt.Errorf("failed to change to overlay directory: %s, err: %v", out, err) + } + + // TODO (#138): in a local environment this is going to modify the actual kustomize files. + // maybe a copy should be made instead + out, err = exec.Command( + filepath.Join(pkgDir, "bin", "kustomize"), + "edit", + "set", + "image", + fmt.Sprintf("%s=%s:%s", pdImagePlaceholder, stagingImage, stagingVersion)).CombinedOutput() + if err != nil { + return fmt.Errorf("failed to edit kustomize: %s, err: %v", out, err) + } + } + + // setup service account file for secret creation + tmpSaFile := filepath.Join(getTempDir(), "cloud-sa.json") + defer removeTempDir(filepath.Dir(tmpSaFile)) + + // Need to copy it to name the file "cloud-sa.json" + out, err := exec.Command("cp", *saFile, tmpSaFile).CombinedOutput() + if err != nil { + return fmt.Errorf("error copying service account key: %s, err: %v", out, err) + } + defer shredFile(tmpSaFile) + + // deploy driver + deployCmd := exec.Command(filepath.Join(pkgDir, "deploy", "kubernetes", "deploy-driver.sh"), "--skip-sa-check") + deployCmd.Env = append(os.Environ(), + fmt.Sprintf("GOPATH=%s", goPath), + fmt.Sprintf("GCE_PD_SA_DIR=%s", filepath.Dir(tmpSaFile)), + fmt.Sprintf("GCE_PD_DRIVER_VERSION=%s", deployOverlayName), + ) + err = runCommand("Deploying driver", deployCmd) + if err != nil { + return fmt.Errorf("failed to deploy driver: %v", err) + } + + // TODO (#139): wait for driver to be running + time.Sleep(10 * time.Second) + statusCmd := exec.Command("kubectl", "describe", "pods", "-n", "default") + err = runCommand("Checking driver pods", statusCmd) + if err != nil { + return fmt.Errorf("failed to check driver pods: %v", err) + } + + return nil +} + +func deleteDriver(goPath, pkgDir, deployOverlayName string) error { + deleteCmd := exec.Command(filepath.Join(pkgDir, "deploy", "kubernetes", "delete-driver.sh")) + deleteCmd.Env = append(os.Environ(), + fmt.Sprintf("GOPATH=%s", goPath), + fmt.Sprintf("GCE_PD_DRIVER_VERSION=%s", deployOverlayName), + ) + err := runCommand("Deleting driver", deleteCmd) + if err != nil { + return fmt.Errorf("failed to delete driver: %v", err) + } + return nil +} + +func pushImage(pkgDir, stagingImage, stagingVersion string) error { + err := os.Setenv("GCE_PD_CSI_STAGING_VERSION", stagingVersion) + if err != nil { + return err + } + err = os.Setenv("GCE_PD_CSI_STAGING_IMAGE", stagingImage) + if err != nil { + return err + } + cmd := exec.Command("make", "-C", pkgDir, "push-container", + fmt.Sprintf("GCE_PD_CSI_STAGING_VERSION=%s", stagingVersion), + fmt.Sprintf("GCE_PD_CSI_STAGING_IMAGE=%s", stagingImage)) + err = runCommand("Pushing GCP Container", cmd) + if err != nil { + return fmt.Errorf("failed to run make command: err: %v", err) + } + return nil +} + +func deleteImage(stagingImage, stagingVersion string) error { + cmd := exec.Command("gcloud", "container", "images", "delete", fmt.Sprintf("%s:%s", stagingImage, stagingVersion), "--quiet") + err := runCommand("Deleting GCR Container", cmd) + if err != nil { + return fmt.Errorf("failed to delete container image %s:%s: %s", stagingImage, stagingVersion, err) + } + return nil +} diff --git a/test/k8s-integration/main.go b/test/k8s-integration/main.go index d910392d5..5da253470 100644 --- a/test/k8s-integration/main.go +++ b/test/k8s-integration/main.go @@ -17,18 +17,14 @@ package main import ( "flag" "fmt" - "io/ioutil" "os" "os/exec" "path/filepath" "syscall" - "time" + "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/klog" - testutils "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/e2e/utils" - - "k8s.io/apimachinery/pkg/util/uuid" ) var ( @@ -70,20 +66,6 @@ func init() { flag.Set("logtostderr", "true") } -func ensureVariable(v *string, set bool, msgOnError string) { - if set && len(*v) == 0 { - klog.Fatal(msgOnError) - } else if !set && len(*v) != 0 { - klog.Fatal(msgOnError) - } -} - -func ensureFlag(v *bool, setTo bool, msgOnError string) { - if *v != setTo { - klog.Fatal(msgOnError) - } -} - func main() { flag.Parse() @@ -427,288 +409,3 @@ func runTestsWithConfig(pkgDir, k8sDir, gceZone, testFocus, testConfigArg string return nil } - -func runCommand(action string, cmd *exec.Cmd) error { - cmd.Stdout = os.Stdout - cmd.Stdin = os.Stdin - cmd.Stderr = os.Stderr - - fmt.Printf("%s\n", action) - fmt.Printf("%s\n", cmd.Args) - - err := cmd.Start() - if err != nil { - return err - } - - err = cmd.Wait() - if err != nil { - return err - } - return nil -} - -func clusterDownGCE(k8sDir string) error { - cmd := exec.Command(filepath.Join(k8sDir, "hack", "e2e-internal", "e2e-down.sh")) - err := runCommand("Bringing Down E2E Cluster on GCE", cmd) - if err != nil { - return fmt.Errorf("failed to bring down kubernetes e2e cluster on gce: %v", err) - } - return nil -} - -func clusterDownGKE(gceZone string) error { - cmd := exec.Command("gcloud", "container", "clusters", "delete", gkeTestClusterName, - "--zone", gceZone, "--quiet") - err := runCommand("Bringing Down E2E Cluster on GKE", cmd) - if err != nil { - return fmt.Errorf("failed to bring down kubernetes e2e cluster on gke: %v", err) - } - return nil -} - -func buildKubernetes(k8sDir string) error { - cmd := exec.Command("make", "-C", k8sDir, "quick-release") - err := runCommand("Building Kubernetes", cmd) - if err != nil { - return fmt.Errorf("failed to build Kubernetes: %v", err) - } - return nil -} - -func clusterUpGCE(k8sDir, gceZone string) error { - err := os.Setenv("KUBE_GCE_ZONE", gceZone) - if err != nil { - return err - } - cmd := exec.Command(filepath.Join(k8sDir, "hack", "e2e-internal", "e2e-up.sh")) - err = runCommand("Starting E2E Cluster on GCE", cmd) - if err != nil { - return fmt.Errorf("failed to bring up kubernetes e2e cluster on gce: %v", err) - } - - return nil -} - -func clusterUpGKE(gceZone string) error { - out, err := exec.Command("gcloud", "container", "clusters", "list", "--zone", gceZone, - "--filter", fmt.Sprintf("name=%s", gkeTestClusterName)).CombinedOutput() - if err != nil { - return fmt.Errorf("failed to check for previous test cluster: %v %s", err, out) - } - if len(out) > 0 { - klog.Infof("Detected previous cluster %s. Deleting so a new one can be created...", gkeTestClusterName) - err = clusterDownGKE(gceZone) - if err != nil { - return err - } - } - cmd := exec.Command("gcloud", "container", "clusters", "create", gkeTestClusterName, - "--zone", gceZone, "--cluster-version", *gkeClusterVer, "--quiet") - err = runCommand("Staring E2E Cluster on GKE", cmd) - if err != nil { - return fmt.Errorf("failed to bring up kubernetes e2e cluster on gke: %v", err) - } - - return nil -} - -func getOverlayDir(pkgDir, deployOverlayName string) string { - return filepath.Join(pkgDir, "deploy", "kubernetes", "overlays", deployOverlayName) -} - -func installDriver(goPath, pkgDir, stagingImage, stagingVersion, deployOverlayName string, doDriverBuild bool) error { - if doDriverBuild { - // Install kustomize - out, err := exec.Command(filepath.Join(pkgDir, "deploy", "kubernetes", "install-kustomize.sh")).CombinedOutput() - if err != nil { - return fmt.Errorf("failed to install kustomize: %s, err: %v", out, err) - } - - // Edit ci kustomization to use given image tag - overlayDir := getOverlayDir(pkgDir, deployOverlayName) - err = os.Chdir(overlayDir) - if err != nil { - return fmt.Errorf("failed to change to overlay directory: %s, err: %v", out, err) - } - - // TODO (#138): in a local environment this is going to modify the actual kustomize files. - // maybe a copy should be made instead - out, err = exec.Command( - filepath.Join(pkgDir, "bin", "kustomize"), - "edit", - "set", - "image", - fmt.Sprintf("%s=%s:%s", pdImagePlaceholder, stagingImage, stagingVersion)).CombinedOutput() - if err != nil { - return fmt.Errorf("failed to edit kustomize: %s, err: %v", out, err) - } - } - - // setup service account file for secret creation - tmpSaFile := filepath.Join(getTempDir(), "cloud-sa.json") - defer removeTempDir(filepath.Dir(tmpSaFile)) - - // Need to copy it to name the file "cloud-sa.json" - out, err := exec.Command("cp", *saFile, tmpSaFile).CombinedOutput() - if err != nil { - return fmt.Errorf("error copying service account key: %s, err: %v", out, err) - } - defer shredFile(tmpSaFile) - - // deploy driver - deployCmd := exec.Command(filepath.Join(pkgDir, "deploy", "kubernetes", "deploy-driver.sh"), "--skip-sa-check") - deployCmd.Env = append(os.Environ(), - fmt.Sprintf("GOPATH=%s", goPath), - fmt.Sprintf("GCE_PD_SA_DIR=%s", filepath.Dir(tmpSaFile)), - fmt.Sprintf("GCE_PD_DRIVER_VERSION=%s", deployOverlayName), - ) - err = runCommand("Deploying driver", deployCmd) - if err != nil { - return fmt.Errorf("failed to deploy driver: %v", err) - } - - // TODO (#139): wait for driver to be running - time.Sleep(10 * time.Second) - statusCmd := exec.Command("kubectl", "describe", "pods", "-n", "default") - err = runCommand("Checking driver pods", statusCmd) - if err != nil { - return fmt.Errorf("failed to check driver pods: %v", err) - } - - return nil -} - -func deleteDriver(goPath, pkgDir, deployOverlayName string) error { - deleteCmd := exec.Command(filepath.Join(pkgDir, "deploy", "kubernetes", "delete-driver.sh")) - deleteCmd.Env = append(os.Environ(), - fmt.Sprintf("GOPATH=%s", goPath), - fmt.Sprintf("GCE_PD_DRIVER_VERSION=%s", deployOverlayName), - ) - err := runCommand("Deleting driver", deleteCmd) - if err != nil { - return fmt.Errorf("failed to delete driver: %v", err) - } - return nil -} - -func shredFile(filePath string) { - if _, err := os.Stat(filePath); os.IsNotExist(err) { - klog.V(4).Infof("File %v was not found, skipping shredding", filePath) - return - } - klog.V(4).Infof("Shredding file %v", filePath) - out, err := exec.Command("shred", "--remove", filePath).CombinedOutput() - if err != nil { - klog.V(4).Infof("Failed to shred file %v: %v\nOutput:%v", filePath, err, out) - } - if _, err := os.Stat(filePath); os.IsNotExist(err) { - klog.V(4).Infof("File %v successfully shredded", filePath) - return - } - - // Shred failed Try to remove the file for good meausure - err = os.Remove(filePath) - if err != nil { - klog.V(4).Infof("Failed to remove service account file %s: %v", filePath, err) - } -} - -func downloadKubernetesSource(pkgDir, k8sIoDir, kubeVersion string) error { - k8sDir := filepath.Join(k8sIoDir, "kubernetes") - /* - // TODO: Download a fresh copy every time until mutate manifests hardcoding existing image is solved. - if _, err := os.Stat(k8sDir); !os.IsNotExist(err) { - klog.Infof("Staging Kubernetes already found at %s, skipping download", k8sDir) - return nil - } - */ - - klog.V(4).Infof("Staging Kubernetes folder not found, downloading now") - - err := os.MkdirAll(k8sIoDir, 0777) - if err != nil { - return err - } - - kubeTarDir := filepath.Join(k8sIoDir, fmt.Sprintf("kubernetes-%s.tar.gz", kubeVersion)) - - var vKubeVersion string - if kubeVersion == "master" { - vKubeVersion = kubeVersion - // A hack to be able to build Kubernetes in this nested place - // KUBE_GIT_VERSION_FILE set to file to load kube version from - err = os.Setenv("KUBE_GIT_VERSION_FILE", filepath.Join(pkgDir, "test", "k8s-integration", ".dockerized-kube-version-defs")) - if err != nil { - return err - } - } else { - vKubeVersion = "v" + kubeVersion - } - out, err := exec.Command("curl", "-L", fmt.Sprintf("https://github.com/kubernetes/kubernetes/archive/%s.tar.gz", vKubeVersion), "-o", kubeTarDir).CombinedOutput() - if err != nil { - return fmt.Errorf("failed to curl kubernetes version %s: %s, err: %v", kubeVersion, out, err) - } - - out, err = exec.Command("tar", "-C", k8sIoDir, "-xvf", kubeTarDir).CombinedOutput() - if err != nil { - return fmt.Errorf("failed to untar %s: %s, err: %v", kubeTarDir, out, err) - } - - err = os.RemoveAll(k8sDir) - if err != nil { - return err - } - - err = os.Rename(filepath.Join(k8sIoDir, fmt.Sprintf("kubernetes-%s", kubeVersion)), k8sDir) - if err != nil { - return err - } - - klog.V(4).Infof("Successfully downloaded Kubernetes v%s to %s", kubeVersion, k8sDir) - - return nil -} - -func pushImage(pkgDir, stagingImage, stagingVersion string) error { - err := os.Setenv("GCE_PD_CSI_STAGING_VERSION", stagingVersion) - if err != nil { - return err - } - err = os.Setenv("GCE_PD_CSI_STAGING_IMAGE", stagingImage) - if err != nil { - return err - } - cmd := exec.Command("make", "-C", pkgDir, "push-container", - fmt.Sprintf("GCE_PD_CSI_STAGING_VERSION=%s", stagingVersion), - fmt.Sprintf("GCE_PD_CSI_STAGING_IMAGE=%s", stagingImage)) - err = runCommand("Pushing GCP Container", cmd) - if err != nil { - return fmt.Errorf("failed to run make command: err: %v", err) - } - return nil -} - -func deleteImage(stagingImage, stagingVersion string) error { - cmd := exec.Command("gcloud", "container", "images", "delete", fmt.Sprintf("%s:%s", stagingImage, stagingVersion), "--quiet") - err := runCommand("Deleting GCR Container", cmd) - if err != nil { - return fmt.Errorf("failed to delete container image %s:%s: %s", stagingImage, stagingVersion, err) - } - return nil -} - -func getTempDir() string { - dir, err := ioutil.TempDir("", "") - if err != nil { - klog.Fatalf("Error creating temp dir: %v", err) - } - return dir -} - -func removeTempDir(dir string) { - err := os.RemoveAll(dir) - if err != nil { - klog.Fatalf("Error removing temp dir: %v", err) - } -} diff --git a/test/k8s-integration/utils.go b/test/k8s-integration/utils.go new file mode 100644 index 000000000..947d181d0 --- /dev/null +++ b/test/k8s-integration/utils.go @@ -0,0 +1,81 @@ +package main + +import ( + "fmt" + "io/ioutil" + "os" + "os/exec" + + "k8s.io/klog" +) + +func runCommand(action string, cmd *exec.Cmd) error { + cmd.Stdout = os.Stdout + cmd.Stdin = os.Stdin + cmd.Stderr = os.Stderr + + fmt.Printf("%s\n", action) + fmt.Printf("%s\n", cmd.Args) + + err := cmd.Start() + if err != nil { + return err + } + + err = cmd.Wait() + if err != nil { + return err + } + return nil +} + +func shredFile(filePath string) { + if _, err := os.Stat(filePath); os.IsNotExist(err) { + klog.V(4).Infof("File %v was not found, skipping shredding", filePath) + return + } + klog.V(4).Infof("Shredding file %v", filePath) + out, err := exec.Command("shred", "--remove", filePath).CombinedOutput() + if err != nil { + klog.V(4).Infof("Failed to shred file %v: %v\nOutput:%v", filePath, err, out) + } + if _, err := os.Stat(filePath); os.IsNotExist(err) { + klog.V(4).Infof("File %v successfully shredded", filePath) + return + } + + // Shred failed Try to remove the file for good meausure + err = os.Remove(filePath) + if err != nil { + klog.V(4).Infof("Failed to remove service account file %s: %v", filePath, err) + } +} + +func getTempDir() string { + dir, err := ioutil.TempDir("", "") + if err != nil { + klog.Fatalf("Error creating temp dir: %v", err) + } + return dir +} + +func removeTempDir(dir string) { + err := os.RemoveAll(dir) + if err != nil { + klog.Fatalf("Error removing temp dir: %v", err) + } +} + +func ensureVariable(v *string, set bool, msgOnError string) { + if set && len(*v) == 0 { + klog.Fatal(msgOnError) + } else if !set && len(*v) != 0 { + klog.Fatal(msgOnError) + } +} + +func ensureFlag(v *bool, setTo bool, msgOnError string) { + if *v != setTo { + klog.Fatal(msgOnError) + } +}