Skip to content

Add XFS utils and configuration to test on GKE with ubuntu image type #447

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jan 13, 2020
Merged

Add XFS utils and configuration to test on GKE with ubuntu image type #447

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,6 @@ FROM gcr.io/google-containers/debian-base-amd64:v2.0.0
COPY --from=builder /go/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/bin/gce-pd-csi-driver /gce-pd-csi-driver

# Install necessary dependencies
RUN clean-install util-linux e2fsprogs mount ca-certificates udev
RUN clean-install util-linux e2fsprogs mount ca-certificates udev xfsprogs

ENTRYPOINT ["/gce-pd-csi-driver"]
43 changes: 38 additions & 5 deletions test/k8s-integration/cluster.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ func buildKubernetes(k8sDir, command string) error {
return nil
}

func clusterUpGCE(k8sDir, gceZone string, numNodes int) error {
func clusterUpGCE(k8sDir, gceZone string, numNodes int, imageType string) error {
kshPath := filepath.Join(k8sDir, "cluster", "kubectl.sh")
_, err := os.Stat(kshPath)
if err == nil {
Expand All @@ -80,6 +80,11 @@ func clusterUpGCE(k8sDir, gceZone string, numNodes int) error {
klog.V(4).Infof("Set Kubernetes feature gates: %v", *kubeFeatureGates)
}

err = setImageTypeEnvs(imageType)
if err != nil {
return fmt.Errorf("failed to set image type environment variables: %v", err)
}

err = os.Setenv("NUM_NODES", strconv.Itoa(numNodes))
if err != nil {
return err
Expand All @@ -98,7 +103,35 @@ func clusterUpGCE(k8sDir, gceZone string, numNodes int) error {
return nil
}

func clusterUpGKE(gceZone, gceRegion string, numNodes int) error {
func setImageTypeEnvs(imageType string) error {
//const image = "ubuntu-1804-bionic-v20191211"
//const imageProject = "ubuntu-os-cloud"
switch strings.ToLower(imageType) {
case "cos":
case "gci": // GCI/COS is default type and does not need env vars set
case "ubuntu":
return errors.New("setting environment vars for bringing up *ubuntu* cluster on GCE is unimplemented")
/* TODO(dyzz) figure out how to bring up a Ubuntu cluster on GCE. The below doesn't work.
err := os.Setenv("KUBE_OS_DISTRIBUTION", "ubuntu")
if err != nil {
return err
}
err = os.Setenv("KUBE_GCE_NODE_IMAGE", image)
if err != nil {
return err
}
err = os.Setenv("KUBE_GCE_NODE_PROJECT", imageProject)
if err != nil {
return err
}
*/
default:
return fmt.Errorf("could not set env for image type %s, only gci, cos, ubuntu supported", imageType)
}
return nil
}

func clusterUpGKE(gceZone, gceRegion string, numNodes int, imageType string) error {
locationArg, locationVal, err := gkeLocationArgs(gceZone, gceRegion)
if err != nil {
return err
Expand All @@ -119,7 +152,7 @@ func clusterUpGKE(gceZone, gceRegion string, numNodes int) error {
}
cmd := exec.Command("gcloud", "container", "clusters", "create", gkeTestClusterName,
locationArg, locationVal, "--cluster-version", *gkeClusterVer, "--num-nodes", strconv.Itoa(numNodes),
"--quiet", "--machine-type", "n1-standard-2")
"--quiet", "--machine-type", "n1-standard-2", "--image-type", imageType)
err = runCommand("Staring E2E Cluster on GKE", cmd)
if err != nil {
return fmt.Errorf("failed to bring up kubernetes e2e cluster on gke: %v", err)
Expand Down Expand Up @@ -184,7 +217,7 @@ func downloadKubernetesSource(pkgDir, k8sIoDir, kubeVersion string) error {
return nil
}

func getGKEKubeTestArgs(gceZone, gceRegion string) ([]string, error) {
func getGKEKubeTestArgs(gceZone, gceRegion, imageType string) ([]string, error) {
var locationArg, locationVal string
switch {
case len(gceZone) > 0:
Expand Down Expand Up @@ -222,7 +255,7 @@ func getGKEKubeTestArgs(gceZone, gceRegion string) ([]string, error) {
"--gcp-network=default",
"--check-version-skew=false",
"--deployment=gke",
"--gcp-node-image=cos",
fmt.Sprintf("--gcp-node-image=%s", imageType),
"--gcp-network=default",
fmt.Sprintf("--cluster=%s", gkeTestClusterName),
fmt.Sprintf("--gke-environment=%s", gkeEnv),
Expand Down
4 changes: 2 additions & 2 deletions test/k8s-integration/config/test-config-template.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@ DriverInfo:
ext2:
ext3:
ext4:
xfs:
# The following FS types supported by GCE PD but
# currently we do not test the CSI Driver on Ubuntu or Windows
# xfs: XFS only available on Ubuntu
# currently we do not test the CSI Driver on Windows
# ntfs: NTFS only available on Windows
Capabilities:
{{range .Capabilities}} {{ . }}: true
Expand Down
15 changes: 10 additions & 5 deletions test/k8s-integration/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ var (
deploymentStrat = flag.String("deployment-strategy", "gce", "choose between deploying on gce or gke")
gkeClusterVer = flag.String("gke-cluster-version", "", "version of Kubernetes master and node for gke")
numNodes = flag.Int("num-nodes", -1, "the number of nodes in the test cluster")
imageType = flag.String("image-type", "cos", "the image type to use for the cluster")

// Test infrastructure flags
boskosResourceType = flag.String("boskos-resource-type", "gce-project", "name of the boskos resource type to reserve")
Expand Down Expand Up @@ -80,6 +81,7 @@ func main() {
ensureVariable(saFile, true, "service-account-file is a required flag")
ensureVariable(deployOverlayName, true, "deploy-overlay-name is a required flag")
ensureVariable(testFocus, true, "test-focus is a required flag")
ensureVariable(imageType, true, "image type is a required flag. Available options include 'cos' and 'ubuntu'")

if len(*gceRegion) != 0 {
ensureVariable(gceZone, false, "gce-zone and gce-region cannot both be set")
Expand Down Expand Up @@ -238,9 +240,9 @@ func handle() error {
var err error = nil
switch *deploymentStrat {
case "gce":
err = clusterUpGCE(k8sDir, *gceZone, *numNodes)
err = clusterUpGCE(k8sDir, *gceZone, *numNodes, *imageType)
case "gke":
err = clusterUpGKE(*gceZone, *gceRegion, *numNodes)
err = clusterUpGKE(*gceZone, *gceRegion, *numNodes, *imageType)
default:
err = fmt.Errorf("deployment-strategy must be set to 'gce' or 'gke', but is: %s", *deploymentStrat)
}
Expand Down Expand Up @@ -286,7 +288,7 @@ func handle() error {
var cloudProviderArgs []string
switch *deploymentStrat {
case "gke":
cloudProviderArgs, err = getGKEKubeTestArgs(*gceZone, *gceRegion)
cloudProviderArgs, err = getGKEKubeTestArgs(*gceZone, *gceRegion, *imageType)
if err != nil {
return fmt.Errorf("failed to build GKE kubetest args: %v", err)
}
Expand Down Expand Up @@ -375,8 +377,11 @@ func runTestsWithConfig(testDir, testFocus, testSkip, testConfigArg string, clou
homeDir, _ := os.LookupEnv("HOME")
os.Setenv("KUBECONFIG", filepath.Join(homeDir, ".kube/config"))

artifactsDir, _ := os.LookupEnv("ARTIFACTS")
reportArg := fmt.Sprintf("-report-dir=%s", artifactsDir)
artifactsDir, ok := os.LookupEnv("ARTIFACTS")
reportArg := ""
if ok {
reportArg = fmt.Sprintf("-report-dir=%s", artifactsDir)
}

testArgs := fmt.Sprintf("--ginkgo.focus=%s --ginkgo.skip=%s %s %s",
testFocus,
Expand Down