Skip to content

Support GCEPD driver for Windows #483

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 3, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,23 @@

STAGINGIMAGE=${GCE_PD_CSI_STAGING_IMAGE}
STAGINGVERSION=${GCE_PD_CSI_STAGING_VERSION}
DRIVERBINARY=gce-pd-csi-driver
DRIVERWINDOWSBINARY=${DRIVERBINARY}.exe

all: gce-pd-driver

gce-pd-driver:
mkdir -p bin
ifndef GCE_PD_CSI_STAGING_VERSION
$(error "Must set environment variable GCE_PD_CSI_STAGING_VERSION to staging version")
endif
go build -ldflags "-X main.vendorVersion=${STAGINGVERSION}" -o bin/gce-pd-csi-driver ./cmd/
go build -ldflags "-X main.vendorVersion=${STAGINGVERSION}" -o bin/${DRIVERBINARY} ./cmd/

build-windows:
mkdir -p bin
ifndef GCE_PD_CSI_STAGING_VERSION
$(error "Must set environment variable GCE_PD_CSI_STAGING_VERSION to staging version")
endif
GOOS=windows go build -ldflags "-X main.vendorVersion=${STAGINGVERSION}" -o bin/${DRIVERWINDOWSBINARY} ./cmd/

build-container:
ifndef GCE_PD_CSI_STAGING_IMAGE
Expand Down
5 changes: 4 additions & 1 deletion cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,10 @@ func handle() {
//Initialize requirements for the node service
var nodeServer *driver.GCENodeServer
if *runNodeService {
mounter := mountmanager.NewSafeMounter()
mounter, err := mountmanager.NewSafeMounter()
if err != nil {
klog.Fatalf("Failed to get safe mounter: %v", err)
}
deviceUtils := mountmanager.NewDeviceUtils()
statter := mountmanager.NewStatter()
meta, err := metadataservice.NewMetadataService()
Expand Down
6 changes: 4 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -6,17 +6,19 @@ require (
cloud.google.com/go v0.45.1
github.com/GoogleCloudPlatform/k8s-cloud-provider v0.0.0-20190612171043-2e19bb35a278
github.com/container-storage-interface/spec v1.2.0
github.com/golang/protobuf v1.3.2
github.com/golang/protobuf v1.3.4
github.com/google/uuid v1.1.1
github.com/hashicorp/go-multierror v1.0.0 // indirect
github.com/kubernetes-csi/csi-proxy/client v0.0.0-20200330215040-9eff16441b2a
github.com/kubernetes-csi/csi-test/v3 v3.0.0
github.com/onsi/ginkgo v1.10.3
github.com/onsi/gomega v1.7.1
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
golang.org/x/sys v0.0.0-20191113165036-4c7a9d0fe056
golang.org/x/tools/gopls v0.3.3 // indirect
google.golang.org/api v0.10.0
google.golang.org/genproto v0.0.0-20191114150713-6bbd007550de
google.golang.org/grpc v1.25.1
google.golang.org/grpc v1.27.1
gopkg.in/gcfg.v1 v1.2.3
gopkg.in/warnings.v0 v0.1.2 // indirect
k8s.io/apimachinery v0.17.1
Expand Down
44 changes: 44 additions & 0 deletions go.sum

Large diffs are not rendered by default.

80 changes: 30 additions & 50 deletions pkg/gce-pd-csi-driver/node.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ package gceGCEDriver
import (
"fmt"
"os"
"runtime"
"strconv"
"strings"

Expand Down Expand Up @@ -57,10 +58,20 @@ var _ csi.NodeServer = &GCENodeServer{}
// node boot disk is considered an attachable disk so effective attach limit is
// one less.
const (
volumeLimitSmall int64 = 15
volumeLimitBig int64 = 127
volumeLimitSmall int64 = 15
volumeLimitBig int64 = 127
defaultLinuxFsType = "ext4"
defaultWindowsFsType = "ntfs"
)

func getDefaultFsType() string {
if runtime.GOOS == "windows" {
return defaultWindowsFsType
} else {
return defaultLinuxFsType
}
}

func (ns *GCENodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) {
// Validate Arguments
targetPath := req.GetTargetPath()
Expand Down Expand Up @@ -129,10 +140,10 @@ func (ns *GCENodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePub
}

sourcePath = stagingTargetPath

if err := os.MkdirAll(targetPath, 0750); err != nil {
if err := preparePublishPath(targetPath, ns.Mounter); err != nil {
return nil, status.Error(codes.Internal, fmt.Sprintf("mkdir failed on disk %s (%v)", targetPath, err))
}

} else if blk := volumeCapability.GetBlock(); blk != nil {
klog.V(4).Infof("NodePublishVolume with block volume mode")

Expand All @@ -141,7 +152,7 @@ func (ns *GCENodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePub
partition = part
}

sourcePath, err = ns.getDevicePath(volumeID, partition)
sourcePath, err = getDevicePath(ns, volumeID, partition)
if err != nil {
return nil, status.Error(codes.Internal, fmt.Sprintf("Error when getting device path: %v", err))
}
Expand Down Expand Up @@ -218,11 +229,9 @@ func (ns *GCENodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeU
}
defer ns.volumeLocks.Release(volumeID)

err := mount.CleanupMountPoint(targetPath, ns.Mounter.Interface, false /* bind mount */)
if err != nil {
if err := cleanupPublishPath(targetPath, ns.Mounter); err != nil {
return nil, status.Error(codes.Internal, fmt.Sprintf("Unmount failed: %v\nUnmounting arguments: %s\n", err, targetPath))
}

klog.V(4).Infof("NodeUnpublishVolume succeded on %v from %s", volumeID, targetPath)
return &csi.NodeUnpublishVolumeResponse{}, nil
}
Expand Down Expand Up @@ -264,27 +273,19 @@ func (ns *GCENodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStage
if part, ok := req.GetVolumeContext()[common.VolumeAttributePartition]; ok {
partition = part
}
devicePath, err := getDevicePath(ns, volumeID, partition)

devicePath, err := ns.getDevicePath(volumeID, partition)
if err != nil {
return nil, status.Error(codes.Internal, fmt.Sprintf("Error when getting device path: %v", err))
}

klog.V(4).Infof("Successfully found attached GCE PD %q at device path %s.", volumeKey.Name, devicePath)

// Part 2: Check if mount already exists at targetpath
// Part 2: Check if mount already exists at stagingTargetPath
notMnt, err := ns.Mounter.Interface.IsLikelyNotMountPoint(stagingTargetPath)
if err != nil {
if os.IsNotExist(err) {
if err := os.MkdirAll(stagingTargetPath, 0750); err != nil {
return nil, status.Error(codes.Internal, fmt.Sprintf("Failed to create directory (%q): %v", stagingTargetPath, err))
}
notMnt = true
} else {
return nil, status.Error(codes.Internal, fmt.Sprintf("Unknown error when checking mount point (%q): %v", stagingTargetPath, err))
}
if err != nil && !os.IsNotExist(err) {
return nil, status.Error(codes.Internal, fmt.Sprintf("cannot validate mount point: %s %v", stagingTargetPath, err))
}

if !notMnt {
// TODO(#95): Check who is mounted here. No error if its us
/*
Expand All @@ -293,15 +294,17 @@ func (ns *GCENodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStage
3) Readonly MUST match
*/

klog.V(4).Infof("NodeStageVolume succeded on %v to %s, mount already exists.", volumeID, stagingTargetPath)
return &csi.NodeStageVolumeResponse{}, nil

}
if err := prepareStagePath(stagingTargetPath, ns.Mounter); err != nil {
return nil, status.Error(codes.Internal, fmt.Sprintf("mkdir failed on disk %s (%v)", stagingTargetPath, err))
}

// Part 3: Mount device to stagingTargetPath
// Default fstype is ext4
fstype := "ext4"
fstype := getDefaultFsType()

options := []string{}
if mnt := volumeCapability.GetMount(); mnt != nil {
if mnt.FsType != "" {
Expand All @@ -316,7 +319,7 @@ func (ns *GCENodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStage
return &csi.NodeStageVolumeResponse{}, nil
}

err = ns.Mounter.FormatAndMount(devicePath, stagingTargetPath, fstype, options)
err = formatAndMount(devicePath, stagingTargetPath, fstype, options, ns.Mounter)
if err != nil {
return nil, status.Error(codes.Internal,
fmt.Sprintf("Failed to format and mount device from (%q) to (%q) with fstype (%q) and options (%q): %v",
Expand All @@ -343,9 +346,8 @@ func (ns *GCENodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUns
}
defer ns.volumeLocks.Release(volumeID)

err := mount.CleanupMountPoint(stagingTargetPath, ns.Mounter.Interface, false /* bind mount */)
if err != nil {
return nil, status.Error(codes.Internal, fmt.Sprintf("NodeUnstageVolume failed to unmount at path %s: %v", stagingTargetPath, err))
if err := cleanupStagePath(stagingTargetPath, ns.Mounter); err != nil {
return nil, status.Error(codes.Internal, fmt.Sprintf("NodeUnstageVolume failed: %v\nUnmounting arguments: %s\n", err, stagingTargetPath))
}

klog.V(4).Infof("NodeUnstageVolume succeded on %v from %s", volumeID, stagingTargetPath)
Expand Down Expand Up @@ -454,7 +456,7 @@ func (ns *GCENodeServer) NodeExpandVolume(ctx context.Context, req *csi.NodeExpa
return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("volume ID is invalid: %v", err))
}

devicePath, err := ns.getDevicePath(volumeID, "")
devicePath, err := getDevicePath(ns, volumeID, "")
if err != nil {
return nil, status.Error(codes.Internal, fmt.Sprintf("error when getting device path for %s: %v", volumeID, err))
}
Expand Down Expand Up @@ -517,28 +519,6 @@ func (ns *GCENodeServer) GetVolumeLimits() (int64, error) {
return volumeLimitBig, nil
}

func (ns *GCENodeServer) getDevicePath(volumeID string, partition string) (string, error) {
volumeKey, err := common.VolumeIDToKey(volumeID)
if err != nil {
return "", err
}
deviceName, err := common.GetDeviceName(volumeKey)
if err != nil {
return "", fmt.Errorf("error getting device name: %v", err)
}

devicePaths := ns.DeviceUtils.GetDiskByIdPaths(deviceName, partition)
devicePath, err := ns.DeviceUtils.VerifyDevicePath(devicePaths, deviceName)

if err != nil {
return "", fmt.Errorf("error verifying GCE PD (%q) is attached: %v", volumeKey.Name, err)
}
if devicePath == "" {
return "", fmt.Errorf("unable to find device path out of attempted paths: %v", devicePaths)
}
return devicePath, nil
}

func (ns *GCENodeServer) getBlockSizeBytes(devicePath string) (int64, error) {
output, err := ns.Mounter.Exec.Command("blockdev", "--getsize64", devicePath).CombinedOutput()
if err != nil {
Expand Down
66 changes: 66 additions & 0 deletions pkg/gce-pd-csi-driver/utils_linux.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
// +build !windows

/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package gceGCEDriver
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: definitely out of scope for this PR, but the convention is to avoid mixCaps: https://golang.org/doc/effective_go.html#package-names

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ack. we can do another PR for it.


import (
"fmt"
"os"

"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"k8s.io/utils/mount"
"sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common"
)

func getDevicePath(ns *GCENodeServer, volumeID, partition string) (string, error) {
volumeKey, err := common.VolumeIDToKey(volumeID)
if err != nil {
return "", err
}
deviceName, err := common.GetDeviceName(volumeKey)
if err != nil {
return "", fmt.Errorf("error getting device name: %v", err)
}
devicePaths := ns.DeviceUtils.GetDiskByIdPaths(deviceName, partition)
devicePath, err := ns.DeviceUtils.VerifyDevicePath(devicePaths, deviceName)
if err != nil {
return "", status.Error(codes.Internal, fmt.Sprintf("error verifying GCE PD (%q) is attached: %v", deviceName, err))
}
if devicePath == "" {
return "", status.Error(codes.Internal, fmt.Sprintf("Unable to find device path out of attempted paths: %v", devicePaths))
}
return devicePath, nil
}

func formatAndMount(source, target, fstype string, options []string, m *mount.SafeFormatAndMount) error {
return m.FormatAndMount(source, target, fstype, options)
}

func preparePublishPath(path string, m *mount.SafeFormatAndMount) error {
return os.MkdirAll(path, 0750)
}

func prepareStagePath(path string, m *mount.SafeFormatAndMount) error {
return os.MkdirAll(path, 0750)
}

func cleanupPublishPath(path string, m *mount.SafeFormatAndMount) error {
return mount.CleanupMountPoint(path, m, false /* bind mount */)
}

func cleanupStagePath(path string, m *mount.SafeFormatAndMount) error {
return cleanupPublishPath(path, m)
}
88 changes: 88 additions & 0 deletions pkg/gce-pd-csi-driver/utils_windows.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
// +build windows

/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package gceGCEDriver

import (
"fmt"
"k8s.io/utils/mount"
"sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common"
mounter "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/mount-manager"
)

func formatAndMount(source, target, fstype string, options []string, m *mount.SafeFormatAndMount) error {
proxy, ok := m.Interface.(*mounter.CSIProxyMounter)
if !ok {
return fmt.Errorf("could not cast to csi proxy class")
}
return proxy.FormatAndMount(source, target, fstype, options)
}

// Before mounting (which means creating symlink) in Windows, the targetPath should
// not exist. Currently kubelet creates the path beforehand, this is a workaround to
// remove the path first.
func preparePublishPath(path string, m *mount.SafeFormatAndMount) error {
proxy, ok := m.Interface.(*mounter.CSIProxyMounter)
if !ok {
return fmt.Errorf("could not cast to csi proxy class")
}
exists, err := proxy.ExistsPath(path)
if err != nil {
return err
}
if exists {
return proxy.RemovePodDir(path)
}
return nil
}

// Before staging (which means creating symlink) in Windows, the targetPath should
// not exist.
func prepareStagePath(path string, m *mount.SafeFormatAndMount) error {
return nil
}

func cleanupPublishPath(path string, m *mount.SafeFormatAndMount) error {
proxy, ok := m.Interface.(*mounter.CSIProxyMounter)
if !ok {
return fmt.Errorf("could not cast to csi proxy class")
}
return proxy.RemovePodDir(path)
}

func cleanupStagePath(path string, m *mount.SafeFormatAndMount) error {
proxy, ok := m.Interface.(*mounter.CSIProxyMounter)
if !ok {
return fmt.Errorf("could not cast to csi proxy class")
}
return proxy.RemovePluginDir(path)
}

// search Windows disk number by volumeID
func getDevicePath(ns *GCENodeServer, volumeID, partition string) (string, error) {
volumeKey, err := common.VolumeIDToKey(volumeID)
if err != nil {
return "", err
}
deviceName, err := common.GetDeviceName(volumeKey)
if err != nil {
return "", fmt.Errorf("error getting device name: %v", err)
}
proxy, ok := ns.Mounter.Interface.(*mounter.CSIProxyMounter)
if !ok {
return "", fmt.Errorf("could not cast to csi proxy class")
}
return proxy.GetDevicePath(deviceName, partition, volumeKey.Name)
}
Loading