Allow disabling controller or node service

Author: rfranzke

cmd/main.go

@@ -30,9 +30,11 @@ import (
 )
 
 var (
-	endpoint          = flag.String("endpoint", "unix:/tmp/csi.sock", "CSI endpoint")
-	gceConfigFilePath = flag.String("cloud-config", "", "Path to GCE cloud provider config")
-	vendorVersion     string
+	cloudConfigFilePath  = flag.String("cloud-config", "", "Path to GCE cloud provider config")
+	endpoint             = flag.String("endpoint", "unix:/tmp/csi.sock", "CSI endpoint")
+	runControllerService = flag.Bool("run-controller-service", true, "If set to false then the CSI driver does not activate its controller service (default: true)")
+	runNodeService       = flag.Bool("run-node-service", true, "If set to false then the CSI driver does not activate its node service (default: true)")
+	vendorVersion        string
 )
 
 const (
@@ -57,6 +59,8 @@ func main() {
 }
 
 func handle() {
+	var err error
+
 	if vendorVersion == "" {
 		klog.Fatalf("vendorVersion must be set at compile time")
 	}
@@ -68,20 +72,35 @@ func handle() {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	cloudProvider, err := gce.CreateCloudProvider(ctx, vendorVersion, *gceConfigFilePath)
-	if err != nil {
-		klog.Fatalf("Failed to get cloud provider: %v", err)
+	//Initialize identity server
+	identityServer := driver.NewIdentityServer(gceDriver)
+
+	//Initialize requirements for the controller service
+	var controllerServer *driver.GCEControllerServer
+	if *runControllerService {
+		cloudProvider, err := gce.CreateCloudProvider(ctx, vendorVersion, *cloudConfigFilePath)
+		if err != nil {
+			klog.Fatalf("Failed to get cloud provider: %v", err)
+		}
+		controllerServer = driver.NewControllerServer(gceDriver, cloudProvider)
+	} else if *cloudConfigFilePath != "" {
+		klog.Warningf("controller service is disabled but cloud config given - it has no effect")
 	}
 
-	mounter := mountmanager.NewSafeMounter()
-	deviceUtils := mountmanager.NewDeviceUtils()
-	statter := mountmanager.NewStatter()
-	ms, err := metadataservice.NewMetadataService()
-	if err != nil {
-		klog.Fatalf("Failed to set up metadata service: %v", err)
+	//Initialize requirements for the node service
+	var nodeServer *driver.GCENodeServer
+	if *runNodeService {
+		mounter := mountmanager.NewSafeMounter()
+		deviceUtils := mountmanager.NewDeviceUtils()
+		statter := mountmanager.NewStatter()
+		meta, err := metadataservice.NewMetadataService()
+		if err != nil {
+			klog.Fatalf("Failed to set up metadata service: %v", err)
+		}
+		nodeServer = driver.NewNodeServer(gceDriver, mounter, deviceUtils, meta, statter)
 	}
 
-	err = gceDriver.SetupGCEDriver(cloudProvider, mounter, deviceUtils, ms, statter, driverName, vendorVersion)
+	err = gceDriver.SetupGCEDriver(driverName, vendorVersion, identityServer, controllerServer, nodeServer)
 	if err != nil {
 		klog.Fatalf("Failed to initialize GCE CSI Driver: %v", err)
 	}

deploy/kubernetes/base/controller.yaml

@@ -17,7 +17,7 @@ spec:
       # since it replaces GCE Metadata Server with GKE Metadata Server. Remove
       # this requirement when issue is resolved and before any exposure of
       # metrics ports
-      hostNetwork: true 
+      hostNetwork: true
       serviceAccountName: csi-gce-pd-controller-sa
       priorityClassName: csi-gce-pd-controller
       containers:
@@ -27,6 +27,8 @@ spec:
             - "--v=5"
             - "--csi-address=/csi/csi.sock"
             - "--feature-gates=Topology=true"
+          # - "--run-controller-service=false" # disable the controller service of the CSI driver
+          # - "--run-node-service=false"       # disable the node service of the CSI driver
           volumeMounts:
             - name: socket-dir
               mountPath: /csi

docs/kubernetes/user-guides/driver-install.md

@@ -18,7 +18,7 @@ compute.instances.get
 compute.instances.attachDisk
 compute.instances.detachDisk
 roles/compute.storageAdmin
-roles/iam.serviceAccountUser 
+roles/iam.serviceAccountUser
 ```
 
 If there is a pre-existing service account with these roles for use then the
@@ -79,4 +79,22 @@ iam.serviceAccounts.delete
 ```
 
 These permissions are not required if you already have a service account ready
-for use by the PD Driver.
+for use by the PD Driver.
+
+## Disabling particular CSI driver services
+
+Traditionally, you run the CSI controllers with the GCE PD driver in the same Kubernetes cluster.
+Though, there may be cases where you will only want to run a subset of the available driver services (for example, one scenario is running the controllers outside of the cluster they are serving (while the GCE PD driver still runs inside the served cluster), but there might be others scenarios).
+The CSI driver consists out of these services:
+
+* The **controller** service starts the GRPC server that serves `CreateVolume`, `DeleteVolume`, etc. It is depending on the GCP service account credentials and talks with the GCP API.
+* The **identity** service is responsible to provide identity services like capability information of the CSI plugin.
+* The **node** service implements the various operations for volumes that are run locally from the node, for example `NodePublishVolume`, `NodeStageVolume`, etc. It does not do operations like `CreateVolume` or `ControllerPublish`. Also, as it runs directly on the GCE instances, it is depending on the GCE metadata service.
+
+The CSI driver has two command line flags, `--run-controller-service` and `--run-node-service` which both default to `true`.
+You can disable the individual services by setting the respective flags to `false`.
+
+Note: If you want to run the CSI controllers outside of the cluster you have to specify both the `zone` and `projectId` parameters in the GCE cloud provider config.
+The `zone` is the name of one of the availability zones the served Kubernetes cluster is deployed to.
+It is used to derive the GCP region and to discover the other availability zones in this region.
+The `project-id` is the GCP project ID in which the controller is operating.

pkg/gce-pd-csi-driver/gce-pd-driver.go

@@ -45,15 +45,11 @@ func GetGCEDriver() *GCEDriver {
 	return &GCEDriver{}
 }
 
-func (gceDriver *GCEDriver) SetupGCEDriver(cloudProvider gce.GCECompute, mounter *mount.SafeFormatAndMount,
-	deviceUtils mountmanager.DeviceUtils, meta metadataservice.MetadataService, statter mountmanager.Statter, name, vendorVersion string) error {
+func (gceDriver *GCEDriver) SetupGCEDriver(name, vendorVersion string, identityServer *GCEIdentityServer, controllerServer *GCEControllerServer, nodeServer *GCENodeServer) error {
 	if name == "" {
 		return fmt.Errorf("Driver name missing")
 	}
 
-	gceDriver.name = name
-	gceDriver.vendorVersion = vendorVersion
-
 	// Adding Capabilities
 	vcam := []csi.VolumeCapability_AccessMode_Mode{
 		csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER,
@@ -78,10 +74,11 @@ func (gceDriver *GCEDriver) SetupGCEDriver(cloudProvider gce.GCECompute, mounter
 	}
 	gceDriver.AddNodeServiceCapabilities(ns)
 
-	// Set up RPC Servers
-	gceDriver.ids = NewIdentityServer(gceDriver)
-	gceDriver.ns = NewNodeServer(gceDriver, mounter, deviceUtils, meta, statter)
-	gceDriver.cs = NewControllerServer(gceDriver, cloudProvider)
+	gceDriver.name = name
+	gceDriver.vendorVersion = vendorVersion
+	gceDriver.ids = identityServer
+	gceDriver.cs = controllerServer
+	gceDriver.ns = nodeServer
 
 	return nil
 }

pkg/gce-pd-csi-driver/gce-pd-driver_test.go

@@ -18,7 +18,6 @@ import (
 	"testing"
 
 	gce "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider/compute"
-	metadataservice "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider/metadata"
 )
 
 func initGCEDriver(t *testing.T, cloudDisks []*gce.CloudDisk) *GCEDriver {
@@ -44,7 +43,8 @@ func initBlockingGCEDriver(t *testing.T, cloudDisks []*gce.CloudDisk, readyToExe
 func initGCEDriverWithCloudProvider(t *testing.T, cloudProvider gce.GCECompute) *GCEDriver {
 	vendorVersion := "test-vendor"
 	gceDriver := GetGCEDriver()
-	err := gceDriver.SetupGCEDriver(cloudProvider, nil, nil, metadataservice.NewFakeService(), nil, driver, vendorVersion)
+	controllerServer := NewControllerServer(gceDriver, cloudProvider)
+	err := gceDriver.SetupGCEDriver(driver, vendorVersion, nil, controllerServer, nil)
 	if err != nil {
 		t.Fatalf("Failed to setup GCE Driver: %v", err)
 	}

pkg/gce-pd-csi-driver/identity_test.go

@@ -20,13 +20,13 @@ import (
 	"context"
 
 	csi "github.com/container-storage-interface/spec/lib/go/csi"
-	metadataservice "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider/metadata"
 )
 
 func TestGetPluginInfo(t *testing.T) {
 	vendorVersion := "test-vendor"
 	gceDriver := GetGCEDriver()
-	err := gceDriver.SetupGCEDriver(nil, nil, nil, metadataservice.NewFakeService(), nil, driver, vendorVersion)
+	identityServer := NewIdentityServer(gceDriver)
+	err := gceDriver.SetupGCEDriver(driver, vendorVersion, identityServer, nil, nil)
 	if err != nil {
 		t.Fatalf("Failed to setup GCE Driver: %v", err)
 	}
@@ -48,7 +48,8 @@ func TestGetPluginInfo(t *testing.T) {
 
 func TestGetPluginCapabilities(t *testing.T) {
 	gceDriver := GetGCEDriver()
-	err := gceDriver.SetupGCEDriver(nil, nil, nil, metadataservice.NewFakeService(), nil, driver, "test-vendor")
+	identityServer := NewIdentityServer(gceDriver)
+	err := gceDriver.SetupGCEDriver(driver, "test-vendor", identityServer, nil, nil)
 	if err != nil {
 		t.Fatalf("Failed to setup GCE Driver: %v", err)
 	}
@@ -80,7 +81,8 @@ func TestGetPluginCapabilities(t *testing.T) {
 
 func TestProbe(t *testing.T) {
 	gceDriver := GetGCEDriver()
-	err := gceDriver.SetupGCEDriver(nil, nil, nil, metadataservice.NewFakeService(), nil, driver, "test-vendor")
+	identityServer := NewIdentityServer(gceDriver)
+	err := gceDriver.SetupGCEDriver(driver, "test-vendor", identityServer, nil, nil)
 	if err != nil {
 		t.Fatalf("Failed to setup GCE Driver: %v", err)
 	}

pkg/gce-pd-csi-driver/node_test.go

@@ -42,7 +42,8 @@ func getTestGCEDriverWithCustomMounter(t *testing.T, mounter *mount.SafeFormatAn
 
 func getCustomTestGCEDriver(t *testing.T, mounter *mount.SafeFormatAndMount, deviceUtils mountmanager.DeviceUtils, metaService metadataservice.MetadataService) *GCEDriver {
 	gceDriver := GetGCEDriver()
-	err := gceDriver.SetupGCEDriver(nil, mounter, deviceUtils, metaService, mountmanager.NewFakeStatter(), driver, "test-vendor")
+	nodeServer := NewNodeServer(gceDriver, mounter, deviceUtils, metaService, mountmanager.NewFakeStatter())
+	err := gceDriver.SetupGCEDriver(driver, "test-vendor", nil, nil, nodeServer)
 	if err != nil {
 		t.Fatalf("Failed to setup GCE Driver: %v", err)
 	}
@@ -51,7 +52,8 @@ func getCustomTestGCEDriver(t *testing.T, mounter *mount.SafeFormatAndMount, dev
 
 func getTestBlockingGCEDriver(t *testing.T, readyToExecute chan chan struct{}) *GCEDriver {
 	gceDriver := GetGCEDriver()
-	err := gceDriver.SetupGCEDriver(nil, mountmanager.NewFakeSafeBlockingMounter(readyToExecute), mountmanager.NewFakeDeviceUtils(), metadataservice.NewFakeService(), nil, driver, "test-vendor")
+	nodeServer := NewNodeServer(gceDriver, mountmanager.NewFakeSafeBlockingMounter(readyToExecute), mountmanager.NewFakeDeviceUtils(), metadataservice.NewFakeService(), mountmanager.NewFakeStatter())
+	err := gceDriver.SetupGCEDriver(driver, "test-vendor", nil, nil, nodeServer)
 	if err != nil {
 		t.Fatalf("Failed to setup GCE Driver: %v", err)
 	}

test/sanity/sanity_test.go

@@ -54,7 +54,10 @@ func TestSanity(t *testing.T) {
 	deviceUtils := mountmanager.NewFakeDeviceUtils()
 
 	//Initialize GCE Driver
-	err = gceDriver.SetupGCEDriver(cloudProvider, mounter, deviceUtils, metadataservice.NewFakeService(), mountmanager.NewFakeStatter(), driverName, vendorVersion)
+	identityServer := driver.NewIdentityServer(gceDriver)
+	controllerServer := driver.NewControllerServer(gceDriver, cloudProvider)
+	nodeServer := driver.NewNodeServer(gceDriver, mounter, deviceUtils, metadataservice.NewFakeService(), mountmanager.NewFakeStatter())
+	err = gceDriver.SetupGCEDriver(driverName, vendorVersion, identityServer, controllerServer, nodeServer)
 	if err != nil {
 		t.Fatalf("Failed to initialize GCE CSI Driver: %v", err)
 	}