Skip to content

Enable XFS selectively for GKE clusters with proper COS support #632

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Nov 2, 2020
Merged

Enable XFS selectively for GKE clusters with proper COS support #632

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 23 additions & 13 deletions test/k8s-integration/driver-config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,15 +27,15 @@ const (

// generateDriverConfigFile loads a testdriver config template and creates a file
// with the test-specific configuration
func generateDriverConfigFile(platform, pkgDir, storageClassFile, snapshotClassFile, deploymentStrat string) (string, error) {
func generateDriverConfigFile(testParams *testParameters, storageClassFile string) (string, error) {
// Load template
t, err := template.ParseFiles(filepath.Join(pkgDir, testConfigDir, configTemplateFile))
t, err := template.ParseFiles(filepath.Join(testParams.pkgDir, testConfigDir, configTemplateFile))
if err != nil {
return "", err
}

// Create destination
configFilePath := filepath.Join(pkgDir, testConfigDir, configFile)
configFilePath := filepath.Join(testParams.pkgDir, testConfigDir, configFile)
f, err := os.Create(configFilePath)
if err != nil {
return "", err
Expand All @@ -55,9 +55,11 @@ func generateDriverConfigFile(platform, pkgDir, storageClassFile, snapshotClassF
"exec",
"multipods",
"topology",
"controllerExpansion",
"nodeExpansion",
}
var fsTypes []string
if platform == "windows" {
if testParams.platform == "windows" {
fsTypes = []string{"ntfs"}
caps = []string{
"persistence",
Expand All @@ -70,7 +72,6 @@ func generateDriverConfigFile(platform, pkgDir, storageClassFile, snapshotClassF
"ext2",
"ext3",
"ext4",
"xfs",
}
}

Expand All @@ -82,22 +83,31 @@ func generateDriverConfigFile(platform, pkgDir, storageClassFile, snapshotClassF
dataSource
*/

// TODO: Support adding/removing capabilities based on Kubernetes version.
switch deploymentStrat {
switch testParams.deploymentStrategy {
case "gke":
fallthrough
if testParams.imageType == "cos" {
gkeVer := mustParseVersion(testParams.clusterVersion)
if gkeVer.lessThan(mustParseVersion("1.18.0")) {
// XFS is not supported on COS before 1.18.0
} else {
fsTypes = append(fsTypes, "xfs")
}
} else {
// XFS is supported on all non-COS images.
fsTypes = append(fsTypes, "xfs")
}
case "gce":
caps = append(caps, "controllerExpansion", "nodeExpansion")
fsTypes = append(fsTypes, "xfs")
default:
return "", fmt.Errorf("got unknown deployment strat %s, expected gce or gke", deploymentStrat)
return "", fmt.Errorf("got unknown deployment strat %s, expected gce or gke", testParams.deploymentStrategy)
}

var absSnapshotClassFilePath string
// If snapshot class is passed in as argument, include snapshot specific driver capabiltiites.
if snapshotClassFile != "" {
if testParams.snapshotClassFile != "" {
caps = append(caps, "snapshotDataSource")
// Update the absolute file path pointing to the snapshot class file, if it is provided as an argument.
absSnapshotClassFilePath = filepath.Join(pkgDir, testConfigDir, snapshotClassFile)
absSnapshotClassFilePath = filepath.Join(testParams.pkgDir, testConfigDir, testParams.snapshotClassFile)
}

minimumVolumeSize := "5Gi"
Expand All @@ -107,7 +117,7 @@ func generateDriverConfigFile(platform, pkgDir, storageClassFile, snapshotClassF
numAllowedTopologies = 2
}
params := driverConfig{
StorageClassFile: filepath.Join(pkgDir, testConfigDir, storageClassFile),
StorageClassFile: filepath.Join(testParams.pkgDir, testConfigDir, storageClassFile),
StorageClass: storageClassFile[:strings.LastIndex(storageClassFile, ".")],
SnapshotClassFile: absSnapshotClassFilePath,
SupportedFsType: fsTypes,
Expand Down
26 changes: 13 additions & 13 deletions test/k8s-integration/driver.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,17 +15,17 @@ func getOverlayDir(pkgDir, deployOverlayName string) string {
return filepath.Join(pkgDir, "deploy", "kubernetes", "overlays", deployOverlayName)
}

func installDriver(platform, goPath, pkgDir, stagingImage, stagingVersion, deployOverlayName string, doDriverBuild bool) error {
func installDriver(testParams *testParameters, stagingImage, deployOverlayName string, doDriverBuild bool) error {
if doDriverBuild {
// Install kustomize
klog.Infof("Installing kustomize")
out, err := exec.Command(filepath.Join(pkgDir, "deploy", "kubernetes", "install-kustomize.sh")).CombinedOutput()
out, err := exec.Command(filepath.Join(testParams.pkgDir, "deploy", "kubernetes", "install-kustomize.sh")).CombinedOutput()
if err != nil {
return fmt.Errorf("failed to install kustomize: %s, err: %v", out, err)
}

// Edit ci kustomization to use given image tag
overlayDir := getOverlayDir(pkgDir, deployOverlayName)
overlayDir := getOverlayDir(testParams.pkgDir, deployOverlayName)
err = os.Chdir(overlayDir)
if err != nil {
return fmt.Errorf("failed to change to overlay directory: %s, err: %v", out, err)
Expand All @@ -34,11 +34,11 @@ func installDriver(platform, goPath, pkgDir, stagingImage, stagingVersion, deplo
// TODO (#138): in a local environment this is going to modify the actual kustomize files.
// maybe a copy should be made instead
out, err = exec.Command(
filepath.Join(pkgDir, "bin", "kustomize"),
filepath.Join(testParams.pkgDir, "bin", "kustomize"),
"edit",
"set",
"image",
fmt.Sprintf("%s=%s:%s", pdImagePlaceholder, stagingImage, stagingVersion)).CombinedOutput()
fmt.Sprintf("%s=%s:%s", pdImagePlaceholder, stagingImage, testParams.stagingVersion)).CombinedOutput()
if err != nil {
return fmt.Errorf("failed to edit kustomize: %s, err: %v", out, err)
}
Expand All @@ -61,9 +61,9 @@ func installDriver(platform, goPath, pkgDir, stagingImage, stagingVersion, deplo
}

// deploy driver
deployCmd := exec.Command(filepath.Join(pkgDir, "deploy", "kubernetes", "deploy-driver.sh"), "--skip-sa-check")
deployCmd := exec.Command(filepath.Join(testParams.pkgDir, "deploy", "kubernetes", "deploy-driver.sh"), "--skip-sa-check")
deployEnv = append(deployEnv,
fmt.Sprintf("GOPATH=%s", goPath),
fmt.Sprintf("GOPATH=%s", testParams.goPath),
fmt.Sprintf("GCE_PD_DRIVER_VERSION=%s", deployOverlayName))
deployEnv = append(os.Environ(), deployEnv...)
deployCmd.Env = deployEnv
Expand All @@ -72,14 +72,14 @@ func installDriver(platform, goPath, pkgDir, stagingImage, stagingVersion, deplo
return fmt.Errorf("failed to deploy driver: %w", err)
}

waitScript := filepath.Join(pkgDir, "deploy", "kubernetes", "wait-for-driver.sh")
waitScript := filepath.Join(testParams.pkgDir, "deploy", "kubernetes", "wait-for-driver.sh")
waitCmd := exec.Command(waitScript)
waitCmd.Env = deployEnv
err = runCommand("Waiting for driver to start", waitCmd)
if err != nil {
return fmt.Errorf("driver failed to come up: %w", err)
}
if platform == "windows" {
if testParams.platform == "windows" {
waitCmd = exec.Command(waitScript, "--windows")
waitCmd.Env = deployEnv
err = runCommand("Waiting for windows deployment to start", waitCmd)
Expand All @@ -91,15 +91,15 @@ func installDriver(platform, goPath, pkgDir, stagingImage, stagingVersion, deplo
klog.Infof("describe pods \n %s", string(out))

if err != nil {
return fmt.Errorf("failed to describe pods: %v", err)
return fmt.Errorf("failed to describe pods: %w", err)
}
return nil
}

func deleteDriver(goPath, pkgDir, deployOverlayName string) error {
deleteCmd := exec.Command(filepath.Join(pkgDir, "deploy", "kubernetes", "delete-driver.sh"))
func deleteDriver(testParams *testParameters, deployOverlayName string) error {
deleteCmd := exec.Command(filepath.Join(testParams.pkgDir, "deploy", "kubernetes", "delete-driver.sh"))
deleteCmd.Env = append(os.Environ(),
fmt.Sprintf("GOPATH=%s", goPath),
fmt.Sprintf("GOPATH=%s", testParams.goPath),
fmt.Sprintf("GCE_PD_DRIVER_VERSION=%s", deployOverlayName),
)
err := runCommand("Deleting driver", deleteCmd)
Expand Down
Loading