Skip to content

Commit 7e38ddd

Browse files
jingxu97jingxu97
committed
Support GCEPD driver for Windows
This PR adds the support for gcepd driver to work on Windows node. The driver will call csi-proxy API to perform file, volume and disk operations.
1 parent 046be1c commit 7e38ddd

13 files changed

+605
-109
lines changed

‎Makefile

+10-2
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,23 @@
1717

1818
STAGINGIMAGE=${GCE_PD_CSI_STAGING_IMAGE}
1919
STAGINGVERSION=${GCE_PD_CSI_STAGING_VERSION}
20+
DRIVERBINARY=gce-pd-csi-driver
21+
DRIVERWINDOWSBINARY=${DRIVERBINARY}.exe
2022

2123
all: gce-pd-driver
22-
2324
gce-pd-driver:
2425
mkdir -p bin
2526
ifndef GCE_PD_CSI_STAGING_VERSION
2627
$(error "Must set environment variable GCE_PD_CSI_STAGING_VERSION to staging version")
2728
endif
28-
go build -ldflags "-X main.vendorVersion=${STAGINGVERSION}" -o bin/gce-pd-csi-driver ./cmd/
29+
go build -ldflags "-X main.vendorVersion=${STAGINGVERSION}" -o bin/${DRIVERBINARY} ./cmd/
30+
31+
build-windows:
32+
mkdir -p bin
33+
ifndef GCE_PD_CSI_STAGING_VERSION
34+
$(error "Must set environment variable GCE_PD_CSI_STAGING_VERSION to staging version")
35+
endif
36+
GOOS=windows go build -ldflags "-X main.vendorVersion=${STAGINGVERSION}" -o bin/${DRIVERWINDOWSBINARY} ./cmd/
2937

3038
build-container:
3139
ifndef GCE_PD_CSI_STAGING_IMAGE

‎cmd/main.go

+4-1
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,10 @@ func handle() {
9090
//Initialize requirements for the node service
9191
var nodeServer *driver.GCENodeServer
9292
if *runNodeService {
93-
mounter := mountmanager.NewSafeMounter()
93+
mounter, err := mountmanager.NewSafeMounter()
94+
if err != nil {
95+
klog.Fatalf("Failed to get safe mounter: %v", err)
96+
}
9497
deviceUtils := mountmanager.NewDeviceUtils()
9598
statter := mountmanager.NewStatter()
9699
meta, err := metadataservice.NewMetadataService()

‎go.mod

+5-2
Original file line numberDiff line numberDiff line change
@@ -6,22 +6,25 @@ require (
66
cloud.google.com/go v0.45.1
77
github.com/GoogleCloudPlatform/k8s-cloud-provider v0.0.0-20190612171043-2e19bb35a278
88
github.com/container-storage-interface/spec v1.2.0
9-
github.com/golang/protobuf v1.3.2
9+
github.com/golang/protobuf v1.3.4
1010
github.com/google/uuid v1.1.1
1111
github.com/hashicorp/go-multierror v1.0.0 // indirect
12+
github.com/kubernetes-csi/csi-proxy/client v0.0.0-20200319061913-d6ab31300107
1213
github.com/kubernetes-csi/csi-test/v3 v3.0.0
1314
github.com/onsi/ginkgo v1.10.3
1415
github.com/onsi/gomega v1.7.1
1516
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
1617
golang.org/x/sys v0.0.0-20191113165036-4c7a9d0fe056
18+
golang.org/x/tools/gopls v0.3.3 // indirect
1719
google.golang.org/api v0.10.0
1820
google.golang.org/genproto v0.0.0-20191114150713-6bbd007550de
19-
google.golang.org/grpc v1.25.1
21+
google.golang.org/grpc v1.27.1
2022
gopkg.in/gcfg.v1 v1.2.3
2123
gopkg.in/warnings.v0 v0.1.2 // indirect
2224
k8s.io/apimachinery v0.17.1
2325
k8s.io/client-go v11.0.1-0.20190805182717-6502b5e7b1b5+incompatible
2426
k8s.io/klog v1.0.0
27+
k8s.io/kubernetes v1.14.7
2528
k8s.io/test-infra v0.0.0-20200115230622-70a5174aa78d
2629
k8s.io/utils v0.0.0-20200124190032-861946025e34
2730
)

‎go.sum

+41
Large diffs are not rendered by default.

‎pkg/gce-pd-csi-driver/node.go

+50-39
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ package gceGCEDriver
1717
import (
1818
"fmt"
1919
"os"
20+
"runtime"
2021
"strconv"
2122
"strings"
2223

@@ -57,10 +58,20 @@ var _ csi.NodeServer = &GCENodeServer{}
5758
// node boot disk is considered an attachable disk so effective attach limit is
5859
// one less.
5960
const (
60-
volumeLimitSmall int64 = 15
61-
volumeLimitBig int64 = 127
61+
volumeLimitSmall int64 = 15
62+
volumeLimitBig int64 = 127
63+
defaultLinuxFsType = "ext4"
64+
defaultWindowsFsType = "ntfs"
6265
)
6366

67+
func getDefaultFsType() string {
68+
if runtime.GOOS == "windows" {
69+
return defaultWindowsFsType
70+
} else {
71+
return defaultLinuxFsType
72+
}
73+
}
74+
6475
func (ns *GCENodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) {
6576
// Validate Arguments
6677
targetPath := req.GetTargetPath()
@@ -129,10 +140,10 @@ func (ns *GCENodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePub
129140
}
130141

131142
sourcePath = stagingTargetPath
132-
133-
if err := os.MkdirAll(targetPath, 0750); err != nil {
143+
if err := preparePublishPath(targetPath, ns.Mounter); err != nil {
134144
return nil, status.Error(codes.Internal, fmt.Sprintf("mkdir failed on disk %s (%v)", targetPath, err))
135145
}
146+
136147
} else if blk := volumeCapability.GetBlock(); blk != nil {
137148
klog.V(4).Infof("NodePublishVolume with block volume mode")
138149

@@ -141,7 +152,7 @@ func (ns *GCENodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePub
141152
partition = part
142153
}
143154

144-
sourcePath, err = ns.getDevicePath(volumeID, partition)
155+
sourcePath, err = GetDevicePath(ns, volumeID, partition)
145156
if err != nil {
146157
return nil, status.Error(codes.Internal, fmt.Sprintf("Error when getting device path: %v", err))
147158
}
@@ -218,11 +229,9 @@ func (ns *GCENodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeU
218229
}
219230
defer ns.volumeLocks.Release(volumeID)
220231

221-
err := mount.CleanupMountPoint(targetPath, ns.Mounter.Interface, false /* bind mount */)
222-
if err != nil {
232+
if err := cleanupPublishPath(targetPath, ns.Mounter); err != nil {
223233
return nil, status.Error(codes.Internal, fmt.Sprintf("Unmount failed: %v\nUnmounting arguments: %s\n", err, targetPath))
224234
}
225-
226235
klog.V(4).Infof("NodeUnpublishVolume succeded on %v from %s", volumeID, targetPath)
227236
return &csi.NodeUnpublishVolumeResponse{}, nil
228237
}
@@ -264,44 +273,46 @@ func (ns *GCENodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStage
264273
if part, ok := req.GetVolumeContext()[common.VolumeAttributePartition]; ok {
265274
partition = part
266275
}
276+
klog.Infof("get device path %v %v", volumeID, partition)
277+
devicePath, err := GetDevicePath(ns, volumeID, partition)
267278

268-
devicePath, err := ns.getDevicePath(volumeID, partition)
269279
if err != nil {
270-
return nil, status.Error(codes.Internal, fmt.Sprintf("Error when getting device path: %v", err))
280+
return nil, status.Error(codes.Internal, fmt.Sprintf("Error when getting device path volumeID %s: %v", volumeID, err))
271281
}
272282

273283
klog.V(4).Infof("Successfully found attached GCE PD %q at device path %s.", volumeKey.Name, devicePath)
274284

275-
// Part 2: Check if mount already exists at targetpath
276-
notMnt, err := ns.Mounter.Interface.IsLikelyNotMountPoint(stagingTargetPath)
277-
if err != nil {
278-
if os.IsNotExist(err) {
279-
if err := os.MkdirAll(stagingTargetPath, 0750); err != nil {
280-
return nil, status.Error(codes.Internal, fmt.Sprintf("Failed to create directory (%q): %v", stagingTargetPath, err))
285+
if runtime.GOOS != "windows" {
286+
// Part 2: Check if mount already exists at targetpath
287+
notMnt, err := ns.Mounter.Interface.IsLikelyNotMountPoint(stagingTargetPath)
288+
if err != nil {
289+
if os.IsNotExist(err) {
290+
if err := os.MkdirAll(stagingTargetPath, 0750); err != nil {
291+
return nil, status.Error(codes.Internal, fmt.Sprintf("Failed to create directory (%q): %v", stagingTargetPath, err))
292+
}
293+
notMnt = true
294+
} else {
295+
return nil, status.Error(codes.Internal, fmt.Sprintf("Unknown error when checking mount point (%q): %v", stagingTargetPath, err))
281296
}
282-
notMnt = true
283-
} else {
284-
return nil, status.Error(codes.Internal, fmt.Sprintf("Unknown error when checking mount point (%q): %v", stagingTargetPath, err))
285297
}
286-
}
287298

288-
if !notMnt {
289-
// TODO(#95): Check who is mounted here. No error if its us
290-
/*
291-
1) Target Path MUST be the vol referenced by vol ID
292-
2) VolumeCapability MUST match
293-
3) Readonly MUST match
299+
if !notMnt {
300+
// TODO(#95): Check who is mounted here. No error if its us
301+
/*
302+
1) Target Path MUST be the vol referenced by vol ID
303+
2) VolumeCapability MUST match
304+
3) Readonly MUST match
294305
295-
*/
306+
*/
296307

297-
klog.V(4).Infof("NodeStageVolume succeded on %v to %s, mount already exists.", volumeID, stagingTargetPath)
298-
return &csi.NodeStageVolumeResponse{}, nil
308+
klog.V(4).Infof("NodeStageVolume succeded on %v to %s, mount already exists.", volumeID, stagingTargetPath)
309+
return &csi.NodeStageVolumeResponse{}, nil
299310

311+
}
300312
}
301-
302313
// Part 3: Mount device to stagingTargetPath
303-
// Default fstype is ext4
304-
fstype := "ext4"
314+
fstype := getDefaultFsType()
315+
305316
options := []string{}
306317
if mnt := volumeCapability.GetMount(); mnt != nil {
307318
if mnt.FsType != "" {
@@ -316,7 +327,8 @@ func (ns *GCENodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStage
316327
return &csi.NodeStageVolumeResponse{}, nil
317328
}
318329

319-
err = ns.Mounter.FormatAndMount(devicePath, stagingTargetPath, fstype, options)
330+
//err = ns.Mounter.FormatAndMount(devicePath, stagingTargetPath, fstype, options)
331+
err = FormatAndMount(devicePath, stagingTargetPath, fstype, options, ns.Mounter)
320332
if err != nil {
321333
return nil, status.Error(codes.Internal,
322334
fmt.Sprintf("Failed to format and mount device from (%q) to (%q) with fstype (%q) and options (%q): %v",
@@ -343,9 +355,8 @@ func (ns *GCENodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUns
343355
}
344356
defer ns.volumeLocks.Release(volumeID)
345357

346-
err := mount.CleanupMountPoint(stagingTargetPath, ns.Mounter.Interface, false /* bind mount */)
347-
if err != nil {
348-
return nil, status.Error(codes.Internal, fmt.Sprintf("NodeUnstageVolume failed to unmount at path %s: %v", stagingTargetPath, err))
358+
if err := cleanupPublishPath(stagingTargetPath, ns.Mounter); err != nil {
359+
return nil, status.Error(codes.Internal, fmt.Sprintf("NodeUnstageVolume failed: %v\nUnmounting arguments: %s\n", err, stagingTargetPath))
349360
}
350361

351362
klog.V(4).Infof("NodeUnstageVolume succeded on %v from %s", volumeID, stagingTargetPath)
@@ -454,7 +465,7 @@ func (ns *GCENodeServer) NodeExpandVolume(ctx context.Context, req *csi.NodeExpa
454465
return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("volume ID is invalid: %v", err))
455466
}
456467

457-
devicePath, err := ns.getDevicePath(volumeID, "")
468+
devicePath, err := GetDevicePath(ns, volumeID, "")
458469
if err != nil {
459470
return nil, status.Error(codes.Internal, fmt.Sprintf("error when getting device path for %s: %v", volumeID, err))
460471
}
@@ -517,7 +528,7 @@ func (ns *GCENodeServer) GetVolumeLimits() (int64, error) {
517528
return volumeLimitBig, nil
518529
}
519530

520-
func (ns *GCENodeServer) getDevicePath(volumeID string, partition string) (string, error) {
531+
/*func (ns *GCENodeServer) getDevicePath(volumeID string, partition string) (string, error) {
521532
volumeKey, err := common.VolumeIDToKey(volumeID)
522533
if err != nil {
523534
return "", err
@@ -537,7 +548,7 @@ func (ns *GCENodeServer) getDevicePath(volumeID string, partition string) (strin
537548
return "", fmt.Errorf("unable to find device path out of attempted paths: %v", devicePaths)
538549
}
539550
return devicePath, nil
540-
}
551+
}*/
541552

542553
func (ns *GCENodeServer) getBlockSizeBytes(devicePath string) (int64, error) {
543554
output, err := ns.Mounter.Exec.Command("blockdev", "--getsize64", devicePath).CombinedOutput()

‎pkg/gce-pd-csi-driver/utils_linux.go

+58
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
// +build !windows
2+
3+
/*
4+
Copyright 2020 The Kubernetes Authors.
5+
Licensed under the Apache License, Version 2.0 (the "License");
6+
you may not use this file except in compliance with the License.
7+
You may obtain a copy of the License at
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
Unless required by applicable law or agreed to in writing, software
10+
distributed under the License is distributed on an "AS IS" BASIS,
11+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
See the License for the specific language governing permissions and
13+
limitations under the License.
14+
*/
15+
16+
package gceGCEDriver
17+
18+
import (
19+
"fmt"
20+
"os"
21+
22+
"google.golang.org/grpc/codes"
23+
"google.golang.org/grpc/status"
24+
"k8s.io/utils/mount"
25+
"sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common"
26+
)
27+
28+
func GetDevicePath(ns *GCENodeServer, volumeID, partition string) (string, error) {
29+
volumeKey, err := common.VolumeIDToKey(volumeID)
30+
if err != nil {
31+
return "", err
32+
}
33+
deviceName, err := common.GetDeviceName(volumeKey)
34+
if err != nil {
35+
return "", fmt.Errorf("error getting device name: %v", err)
36+
}
37+
devicePaths := ns.DeviceUtils.GetDiskByIdPaths(deviceName, partition)
38+
devicePath, err := ns.DeviceUtils.VerifyDevicePath(devicePaths, volumeKey.Name)
39+
if err != nil {
40+
return "", status.Error(codes.Internal, fmt.Sprintf("Error verifying GCE PD (%q) devicePaths %v, name %s is attached: %v", volumeKey, devicePaths, volumeKey.Name, err))
41+
}
42+
if devicePath == "" {
43+
return "", status.Error(codes.Internal, fmt.Sprintf("Unable to find device path out of attempted paths: %v", devicePaths))
44+
}
45+
return devicePath, nil
46+
}
47+
48+
func FormatAndMount(source, target, fstype string, options []string, m *mount.SafeFormatAndMount) error {
49+
return m.FormatAndMount(source, target, fstype, options)
50+
}
51+
52+
func preparePublishPath(path string, m *mount.SafeFormatAndMount) error {
53+
return os.MkdirAll(path, 0750)
54+
}
55+
56+
func cleanupPublishPath(path string, m *mount.SafeFormatAndMount) error {
57+
return mount.CleanupMountPoint(path, m, false /* bind mount */)
58+
}

‎pkg/gce-pd-csi-driver/utils_windows.go

+80
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
/*
2+
Copyright 2020 The Kubernetes Authors.
3+
Licensed under the Apache License, Version 2.0 (the "License");
4+
you may not use this file except in compliance with the License.
5+
You may obtain a copy of the License at
6+
http://www.apache.org/licenses/LICENSE-2.0
7+
Unless required by applicable law or agreed to in writing, software
8+
distributed under the License is distributed on an "AS IS" BASIS,
9+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10+
See the License for the specific language governing permissions and
11+
limitations under the License.
12+
*/
13+
14+
// +build windows
15+
16+
package gceGCEDriver
17+
18+
import (
19+
//"encoding/json"
20+
//"errors"
21+
"fmt"
22+
"os/exec"
23+
//"strconv"
24+
//"strings"
25+
//winutils "github.com/ddebroy/csi-proxy/pkg/api/v1alpha1"
26+
//"google.golang.org/grpc/codes"
27+
//"google.golang.org/grpc/status"
28+
"k8s.io/klog"
29+
"k8s.io/utils/mount"
30+
"sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common"
31+
mounter "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/mount-manager"
32+
)
33+
34+
func FormatAndMount(source, target, fstype string, options []string, m *mount.SafeFormatAndMount) error {
35+
proxy, ok := m.Interface.(*mounter.CSIProxyMounter)
36+
if !ok {
37+
return fmt.Errorf("could not cast to csi proxy class")
38+
}
39+
return proxy.FormatAndMount(source, target, fstype, options)
40+
}
41+
42+
func preparePublishPath(path string, m *mount.SafeFormatAndMount) error {
43+
proxy, ok := m.Interface.(*mounter.CSIProxyMounter)
44+
if !ok {
45+
return fmt.Errorf("could not cast to csi proxy class")
46+
}
47+
exists, err := proxy.ExistsPath(path)
48+
if err != nil {
49+
return err
50+
}
51+
if exists {
52+
return proxy.Unmount(path)
53+
}
54+
return nil
55+
}
56+
57+
func cleanupPublishPath(path string, m *mount.SafeFormatAndMount) error {
58+
proxy, ok := m.Interface.(*mounter.CSIProxyMounter)
59+
if !ok {
60+
return fmt.Errorf("could not cast to csi proxy class")
61+
}
62+
return proxy.Unmount(path)
63+
}
64+
65+
// search Windows disk number by LUN
66+
func GetDevicePath(ns *GCENodeServer, volumeID, partition string) (string, error) {
67+
volumeKey, err := common.VolumeIDToKey(volumeID)
68+
if err != nil {
69+
return "", err
70+
}
71+
deviceName, err := common.GetDeviceName(volumeKey)
72+
if err != nil {
73+
return "", fmt.Errorf("error getting device name: %v", err)
74+
}
75+
proxy, ok := ns.Mounter.Interface.(*mounter.CSIProxyMounter)
76+
if !ok {
77+
return "", fmt.Errorf("could not cast to csi proxy class")
78+
}
79+
return proxy.GetDevicePath(deviceName, partition, volumeKey.Name)
80+
}

0 commit comments

Comments
 (0)