Skip to content

Commit c8c2d8b

Browse files
jingxu97jingxu97
committed
Support GCEPD driver for Windows
This PR adds the support for gcepd driver to work on Windows node. The driver will call csi-proxy API to perform file, volume and disk operations.
1 parent 046be1c commit c8c2d8b

12 files changed

+591
-126
lines changed

‎Makefile

+10-2
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,23 @@
1717

1818
STAGINGIMAGE=${GCE_PD_CSI_STAGING_IMAGE}
1919
STAGINGVERSION=${GCE_PD_CSI_STAGING_VERSION}
20+
DRIVERBINARY=gce-pd-csi-driver
21+
DRIVERWINDOWSBINARY=${DRIVERBINARY}.exe
2022

2123
all: gce-pd-driver
22-
2324
gce-pd-driver:
2425
mkdir -p bin
2526
ifndef GCE_PD_CSI_STAGING_VERSION
2627
$(error "Must set environment variable GCE_PD_CSI_STAGING_VERSION to staging version")
2728
endif
28-
go build -ldflags "-X main.vendorVersion=${STAGINGVERSION}" -o bin/gce-pd-csi-driver ./cmd/
29+
go build -ldflags "-X main.vendorVersion=${STAGINGVERSION}" -o bin/${DRIVERBINARY} ./cmd/
30+
31+
build-windows:
32+
mkdir -p bin
33+
ifndef GCE_PD_CSI_STAGING_VERSION
34+
$(error "Must set environment variable GCE_PD_CSI_STAGING_VERSION to staging version")
35+
endif
36+
GOOS=windows go build -ldflags "-X main.vendorVersion=${STAGINGVERSION}" -o bin/${DRIVERWINDOWSBINARY} ./cmd/
2937

3038
build-container:
3139
ifndef GCE_PD_CSI_STAGING_IMAGE

‎cmd/main.go

+4-1
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,10 @@ func handle() {
9090
//Initialize requirements for the node service
9191
var nodeServer *driver.GCENodeServer
9292
if *runNodeService {
93-
mounter := mountmanager.NewSafeMounter()
93+
mounter, err := mountmanager.NewSafeMounter()
94+
if err != nil {
95+
klog.Fatalf("Failed to get safe mounter: %v", err)
96+
}
9497
deviceUtils := mountmanager.NewDeviceUtils()
9598
statter := mountmanager.NewStatter()
9699
meta, err := metadataservice.NewMetadataService()

‎go.mod

+5-2
Original file line numberDiff line numberDiff line change
@@ -6,22 +6,25 @@ require (
66
cloud.google.com/go v0.45.1
77
github.com/GoogleCloudPlatform/k8s-cloud-provider v0.0.0-20190612171043-2e19bb35a278
88
github.com/container-storage-interface/spec v1.2.0
9-
github.com/golang/protobuf v1.3.2
9+
github.com/golang/protobuf v1.3.4
1010
github.com/google/uuid v1.1.1
1111
github.com/hashicorp/go-multierror v1.0.0 // indirect
12+
github.com/kubernetes-csi/csi-proxy/client v0.0.0-20200319061913-d6ab31300107
1213
github.com/kubernetes-csi/csi-test/v3 v3.0.0
1314
github.com/onsi/ginkgo v1.10.3
1415
github.com/onsi/gomega v1.7.1
1516
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
1617
golang.org/x/sys v0.0.0-20191113165036-4c7a9d0fe056
18+
golang.org/x/tools/gopls v0.3.3 // indirect
1719
google.golang.org/api v0.10.0
1820
google.golang.org/genproto v0.0.0-20191114150713-6bbd007550de
19-
google.golang.org/grpc v1.25.1
21+
google.golang.org/grpc v1.27.1
2022
gopkg.in/gcfg.v1 v1.2.3
2123
gopkg.in/warnings.v0 v0.1.2 // indirect
2224
k8s.io/apimachinery v0.17.1
2325
k8s.io/client-go v11.0.1-0.20190805182717-6502b5e7b1b5+incompatible
2426
k8s.io/klog v1.0.0
27+
k8s.io/kubernetes v1.14.7
2528
k8s.io/test-infra v0.0.0-20200115230622-70a5174aa78d
2629
k8s.io/utils v0.0.0-20200124190032-861946025e34
2730
)

‎go.sum

+41
Large diffs are not rendered by default.

‎pkg/gce-pd-csi-driver/node.go

+46-58
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ package gceGCEDriver
1717
import (
1818
"fmt"
1919
"os"
20+
"runtime"
2021
"strconv"
2122
"strings"
2223

@@ -57,10 +58,20 @@ var _ csi.NodeServer = &GCENodeServer{}
5758
// node boot disk is considered an attachable disk so effective attach limit is
5859
// one less.
5960
const (
60-
volumeLimitSmall int64 = 15
61-
volumeLimitBig int64 = 127
61+
volumeLimitSmall int64 = 15
62+
volumeLimitBig int64 = 127
63+
defaultLinuxFsType = "ext4"
64+
defaultWindowsFsType = "ntfs"
6265
)
6366

67+
func getDefaultFsType() string {
68+
if runtime.GOOS == "windows" {
69+
return defaultWindowsFsType
70+
} else {
71+
return defaultLinuxFsType
72+
}
73+
}
74+
6475
func (ns *GCENodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) {
6576
// Validate Arguments
6677
targetPath := req.GetTargetPath()
@@ -129,10 +140,10 @@ func (ns *GCENodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePub
129140
}
130141

131142
sourcePath = stagingTargetPath
132-
133-
if err := os.MkdirAll(targetPath, 0750); err != nil {
143+
if err := preparePublishPath(targetPath, ns.Mounter); err != nil {
134144
return nil, status.Error(codes.Internal, fmt.Sprintf("mkdir failed on disk %s (%v)", targetPath, err))
135145
}
146+
136147
} else if blk := volumeCapability.GetBlock(); blk != nil {
137148
klog.V(4).Infof("NodePublishVolume with block volume mode")
138149

@@ -141,7 +152,7 @@ func (ns *GCENodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePub
141152
partition = part
142153
}
143154

144-
sourcePath, err = ns.getDevicePath(volumeID, partition)
155+
sourcePath, err = GetDevicePath(ns, volumeID, partition)
145156
if err != nil {
146157
return nil, status.Error(codes.Internal, fmt.Sprintf("Error when getting device path: %v", err))
147158
}
@@ -218,11 +229,9 @@ func (ns *GCENodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeU
218229
}
219230
defer ns.volumeLocks.Release(volumeID)
220231

221-
err := mount.CleanupMountPoint(targetPath, ns.Mounter.Interface, false /* bind mount */)
222-
if err != nil {
232+
if err := cleanupPublishPath(targetPath, ns.Mounter); err != nil {
223233
return nil, status.Error(codes.Internal, fmt.Sprintf("Unmount failed: %v\nUnmounting arguments: %s\n", err, targetPath))
224234
}
225-
226235
klog.V(4).Infof("NodeUnpublishVolume succeded on %v from %s", volumeID, targetPath)
227236
return &csi.NodeUnpublishVolumeResponse{}, nil
228237
}
@@ -264,44 +273,45 @@ func (ns *GCENodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStage
264273
if part, ok := req.GetVolumeContext()[common.VolumeAttributePartition]; ok {
265274
partition = part
266275
}
276+
devicePath, err := GetDevicePath(ns, volumeID, partition)
267277

268-
devicePath, err := ns.getDevicePath(volumeID, partition)
269278
if err != nil {
270279
return nil, status.Error(codes.Internal, fmt.Sprintf("Error when getting device path: %v", err))
271280
}
272281

273282
klog.V(4).Infof("Successfully found attached GCE PD %q at device path %s.", volumeKey.Name, devicePath)
274283

275-
// Part 2: Check if mount already exists at targetpath
276-
notMnt, err := ns.Mounter.Interface.IsLikelyNotMountPoint(stagingTargetPath)
277-
if err != nil {
278-
if os.IsNotExist(err) {
279-
if err := os.MkdirAll(stagingTargetPath, 0750); err != nil {
280-
return nil, status.Error(codes.Internal, fmt.Sprintf("Failed to create directory (%q): %v", stagingTargetPath, err))
284+
if runtime.GOOS != "windows" {
285+
// Part 2: Check if mount already exists at targetpath
286+
notMnt, err := ns.Mounter.Interface.IsLikelyNotMountPoint(stagingTargetPath)
287+
if err != nil {
288+
if os.IsNotExist(err) {
289+
if err := os.MkdirAll(stagingTargetPath, 0750); err != nil {
290+
return nil, status.Error(codes.Internal, fmt.Sprintf("Failed to create directory (%q): %v", stagingTargetPath, err))
291+
}
292+
notMnt = true
293+
} else {
294+
return nil, status.Error(codes.Internal, fmt.Sprintf("Unknown error when checking mount point (%q): %v", stagingTargetPath, err))
281295
}
282-
notMnt = true
283-
} else {
284-
return nil, status.Error(codes.Internal, fmt.Sprintf("Unknown error when checking mount point (%q): %v", stagingTargetPath, err))
285296
}
286-
}
287297

288-
if !notMnt {
289-
// TODO(#95): Check who is mounted here. No error if its us
290-
/*
291-
1) Target Path MUST be the vol referenced by vol ID
292-
2) VolumeCapability MUST match
293-
3) Readonly MUST match
298+
if !notMnt {
299+
// TODO(#95): Check who is mounted here. No error if its us
300+
/*
301+
1) Target Path MUST be the vol referenced by vol ID
302+
2) VolumeCapability MUST match
303+
3) Readonly MUST match
294304
295-
*/
305+
*/
296306

297-
klog.V(4).Infof("NodeStageVolume succeded on %v to %s, mount already exists.", volumeID, stagingTargetPath)
298-
return &csi.NodeStageVolumeResponse{}, nil
307+
klog.V(4).Infof("NodeStageVolume succeded on %v to %s, mount already exists.", volumeID, stagingTargetPath)
308+
return &csi.NodeStageVolumeResponse{}, nil
299309

310+
}
300311
}
301-
302312
// Part 3: Mount device to stagingTargetPath
303-
// Default fstype is ext4
304-
fstype := "ext4"
313+
fstype := getDefaultFsType()
314+
305315
options := []string{}
306316
if mnt := volumeCapability.GetMount(); mnt != nil {
307317
if mnt.FsType != "" {
@@ -316,7 +326,8 @@ func (ns *GCENodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStage
316326
return &csi.NodeStageVolumeResponse{}, nil
317327
}
318328

319-
err = ns.Mounter.FormatAndMount(devicePath, stagingTargetPath, fstype, options)
329+
//err = ns.Mounter.FormatAndMount(devicePath, stagingTargetPath, fstype, options)
330+
err = FormatAndMount(devicePath, stagingTargetPath, fstype, options, ns.Mounter)
320331
if err != nil {
321332
return nil, status.Error(codes.Internal,
322333
fmt.Sprintf("Failed to format and mount device from (%q) to (%q) with fstype (%q) and options (%q): %v",
@@ -343,9 +354,8 @@ func (ns *GCENodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUns
343354
}
344355
defer ns.volumeLocks.Release(volumeID)
345356

346-
err := mount.CleanupMountPoint(stagingTargetPath, ns.Mounter.Interface, false /* bind mount */)
347-
if err != nil {
348-
return nil, status.Error(codes.Internal, fmt.Sprintf("NodeUnstageVolume failed to unmount at path %s: %v", stagingTargetPath, err))
357+
if err := cleanupPublishPath(stagingTargetPath, ns.Mounter); err != nil {
358+
return nil, status.Error(codes.Internal, fmt.Sprintf("NodeUnstageVolume failed: %v\nUnmounting arguments: %s\n", err, stagingTargetPath))
349359
}
350360

351361
klog.V(4).Infof("NodeUnstageVolume succeded on %v from %s", volumeID, stagingTargetPath)
@@ -454,7 +464,7 @@ func (ns *GCENodeServer) NodeExpandVolume(ctx context.Context, req *csi.NodeExpa
454464
return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("volume ID is invalid: %v", err))
455465
}
456466

457-
devicePath, err := ns.getDevicePath(volumeID, "")
467+
devicePath, err := GetDevicePath(ns, volumeID, "")
458468
if err != nil {
459469
return nil, status.Error(codes.Internal, fmt.Sprintf("error when getting device path for %s: %v", volumeID, err))
460470
}
@@ -517,28 +527,6 @@ func (ns *GCENodeServer) GetVolumeLimits() (int64, error) {
517527
return volumeLimitBig, nil
518528
}
519529

520-
func (ns *GCENodeServer) getDevicePath(volumeID string, partition string) (string, error) {
521-
volumeKey, err := common.VolumeIDToKey(volumeID)
522-
if err != nil {
523-
return "", err
524-
}
525-
deviceName, err := common.GetDeviceName(volumeKey)
526-
if err != nil {
527-
return "", fmt.Errorf("error getting device name: %v", err)
528-
}
529-
530-
devicePaths := ns.DeviceUtils.GetDiskByIdPaths(deviceName, partition)
531-
devicePath, err := ns.DeviceUtils.VerifyDevicePath(devicePaths, deviceName)
532-
533-
if err != nil {
534-
return "", fmt.Errorf("error verifying GCE PD (%q) is attached: %v", volumeKey.Name, err)
535-
}
536-
if devicePath == "" {
537-
return "", fmt.Errorf("unable to find device path out of attempted paths: %v", devicePaths)
538-
}
539-
return devicePath, nil
540-
}
541-
542530
func (ns *GCENodeServer) getBlockSizeBytes(devicePath string) (int64, error) {
543531
output, err := ns.Mounter.Exec.Command("blockdev", "--getsize64", devicePath).CombinedOutput()
544532
if err != nil {

‎pkg/gce-pd-csi-driver/utils_linux.go

+58
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
// +build !windows
2+
3+
/*
4+
Copyright 2020 The Kubernetes Authors.
5+
Licensed under the Apache License, Version 2.0 (the "License");
6+
you may not use this file except in compliance with the License.
7+
You may obtain a copy of the License at
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
Unless required by applicable law or agreed to in writing, software
10+
distributed under the License is distributed on an "AS IS" BASIS,
11+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
See the License for the specific language governing permissions and
13+
limitations under the License.
14+
*/
15+
16+
package gceGCEDriver
17+
18+
import (
19+
"fmt"
20+
"os"
21+
22+
"google.golang.org/grpc/codes"
23+
"google.golang.org/grpc/status"
24+
"k8s.io/utils/mount"
25+
"sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common"
26+
)
27+
28+
func GetDevicePath(ns *GCENodeServer, volumeID, partition string) (string, error) {
29+
volumeKey, err := common.VolumeIDToKey(volumeID)
30+
if err != nil {
31+
return "", err
32+
}
33+
deviceName, err := common.GetDeviceName(volumeKey)
34+
if err != nil {
35+
return "", fmt.Errorf("error getting device name: %v", err)
36+
}
37+
devicePaths := ns.DeviceUtils.GetDiskByIdPaths(deviceName, partition)
38+
devicePath, err := ns.DeviceUtils.VerifyDevicePath(devicePaths, deviceName)
39+
if err != nil {
40+
return "", status.Error(codes.Internal, fmt.Sprintf("error verifying GCE PD (%q) is attached: %v\n\n volumeKey %v deviceName %q", volumeKey.Name, err, volumeKey, deviceName))
41+
}
42+
if devicePath == "" {
43+
return "", status.Error(codes.Internal, fmt.Sprintf("Unable to find device path out of attempted paths: %v", devicePaths))
44+
}
45+
return devicePath, nil
46+
}
47+
48+
func FormatAndMount(source, target, fstype string, options []string, m *mount.SafeFormatAndMount) error {
49+
return m.FormatAndMount(source, target, fstype, options)
50+
}
51+
52+
func preparePublishPath(path string, m *mount.SafeFormatAndMount) error {
53+
return os.MkdirAll(path, 0750)
54+
}
55+
56+
func cleanupPublishPath(path string, m *mount.SafeFormatAndMount) error {
57+
return mount.CleanupMountPoint(path, m, false /* bind mount */)
58+
}

‎pkg/gce-pd-csi-driver/utils_windows.go

+71
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
// +build windows
2+
3+
/*
4+
Copyright 2020 The Kubernetes Authors.
5+
Licensed under the Apache License, Version 2.0 (the "License");
6+
you may not use this file except in compliance with the License.
7+
You may obtain a copy of the License at
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
Unless required by applicable law or agreed to in writing, software
10+
distributed under the License is distributed on an "AS IS" BASIS,
11+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
See the License for the specific language governing permissions and
13+
limitations under the License.
14+
*/
15+
16+
package gceGCEDriver
17+
18+
import (
19+
"fmt"
20+
"k8s.io/utils/mount"
21+
"sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common"
22+
mounter "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/mount-manager"
23+
)
24+
25+
func FormatAndMount(source, target, fstype string, options []string, m *mount.SafeFormatAndMount) error {
26+
proxy, ok := m.Interface.(*mounter.CSIProxyMounter)
27+
if !ok {
28+
return fmt.Errorf("could not cast to csi proxy class")
29+
}
30+
return proxy.FormatAndMount(source, target, fstype, options)
31+
}
32+
33+
func preparePublishPath(path string, m *mount.SafeFormatAndMount) error {
34+
proxy, ok := m.Interface.(*mounter.CSIProxyMounter)
35+
if !ok {
36+
return fmt.Errorf("could not cast to csi proxy class")
37+
}
38+
exists, err := proxy.ExistsPath(path)
39+
if err != nil {
40+
return err
41+
}
42+
if exists {
43+
return proxy.Unmount(path)
44+
}
45+
return nil
46+
}
47+
48+
func cleanupPublishPath(path string, m *mount.SafeFormatAndMount) error {
49+
proxy, ok := m.Interface.(*mounter.CSIProxyMounter)
50+
if !ok {
51+
return fmt.Errorf("could not cast to csi proxy class")
52+
}
53+
return proxy.Unmount(path)
54+
}
55+
56+
// search Windows disk number by LUN
57+
func GetDevicePath(ns *GCENodeServer, volumeID, partition string) (string, error) {
58+
volumeKey, err := common.VolumeIDToKey(volumeID)
59+
if err != nil {
60+
return "", err
61+
}
62+
deviceName, err := common.GetDeviceName(volumeKey)
63+
if err != nil {
64+
return "", fmt.Errorf("error getting device name: %v", err)
65+
}
66+
proxy, ok := ns.Mounter.Interface.(*mounter.CSIProxyMounter)
67+
if !ok {
68+
return "", fmt.Errorf("could not cast to csi proxy class")
69+
}
70+
return proxy.GetDevicePath(deviceName, partition, volumeKey.Name)
71+
}

0 commit comments

Comments
 (0)