Allow running the controller service separately

Author: rfranzke

cmd/main.go

@@ -33,9 +33,12 @@ func init() {
 }
 
 var (
-	endpoint          = flag.String("endpoint", "unix:/tmp/csi.sock", "CSI endpoint")
-	gceConfigFilePath = flag.String("cloud-config", "", "Path to GCE cloud provider config")
-	vendorVersion     string
+	endpoint             = flag.String("endpoint", "unix:/tmp/csi.sock", "CSI endpoint")
+	cloudConfigFilePath  = flag.String("cloud-config", "", "Path to GCE cloud provider config")
+	runControllerService = flag.Bool("run-controller-service", true, "If set to false then the CSI driver does not activate its controller service (default: true)")
+	runIdentityService   = flag.Bool("run-identity-service", true, "If set to false then the CSI driver does not activate its identity service (default: true)")
+	runNodeService       = flag.Bool("run-node-service", true, "If set to false then the CSI driver does not activate its node service (default: true)")
+	vendorVersion        string
 )
 
 const (
@@ -66,22 +69,25 @@ func handle() {
 
 	gceDriver := driver.GetGCEDriver()
 
-	//Initialize GCE Driver (Move setup to main?)
-	cloudProvider, err := gce.CreateCloudProvider(vendorVersion, *gceConfigFilePath)
+	// Initialize GCE Driver (Move setup to main?)
+	cloudProvider, err := gce.CreateCloudProvider(vendorVersion, *cloudConfigFilePath)
 	if err != nil {
 		klog.Fatalf("Failed to get cloud provider: %v", err)
 	}
 
 	mounter := mountmanager.NewSafeMounter()
 	deviceUtils := mountmanager.NewDeviceUtils()
 	statter := mountmanager.NewStatter()
-	ms, err := metadataservice.NewMetadataService()
-	if err != nil {
-		klog.Fatalf("Failed to set up metadata service: %v", err)
+
+	var ms metadataservice.MetadataService
+	if !*runNodeService {
+		ms, err = metadataservice.NewMetadataService()
+		if err != nil {
+			klog.Fatalf("Failed to set up metadata service: %v", err)
+		}
 	}
 
-	err = gceDriver.SetupGCEDriver(cloudProvider, mounter, deviceUtils, ms, statter, driverName, vendorVersion)
-	if err != nil {
+	if err := gceDriver.SetupGCEDriver(cloudProvider, mounter, deviceUtils, ms, statter, driverName, vendorVersion, *runControllerService, *runIdentityService, *runNodeService); err != nil {
 		klog.Fatalf("Failed to initialize GCE CSI Driver: %v", err)
 	}
 

deploy/kubernetes/base/controller.yaml

@@ -17,7 +17,7 @@ spec:
       # since it replaces GCE Metadata Server with GKE Metadata Server. Remove
       # this requirement when issue is resolved and before any exposure of
       # metrics ports
-      hostNetwork: true 
+      hostNetwork: true
       serviceAccountName: csi-gce-pd-controller-sa
       priorityClassName: csi-gce-pd-controller
       containers:
@@ -27,6 +27,9 @@ spec:
             - "--v=5"
             - "--csi-address=/csi/csi.sock"
             - "--feature-gates=Topology=true"
+          # - "--run-controller-service=false" # disable the controller service of the CSI driver
+          # - "--run-identity-service=false"   # disable the identity service of the CSI driver
+          # - "--run-node-service=false"       # disable the node service of the CSI driver
           volumeMounts:
             - name: socket-dir
               mountPath: /csi

docs/kubernetes/user-guides/driver-install.md

@@ -20,4 +20,21 @@ $ ./deploy/setup-project.sh
 $ GCE_PD_SA_DIR=/my/safe/credentials/directory    # Directory to get the service account key
 $ GCE_PD_DRIVER_VERSION=stable                    # Driver version to deploy
 $ ./deploy/kubernetes/deploy-driver.sh
-```
+```
+
+## Disabling particular CSI driver services
+
+Traditionally, you run the CSI controllers with the GCE PD driver in the same Kubernetes cluster.
+Though, there may be cases where you will only want to run a subset of the available driver services (for example, one scenario is running the controllers outside of the cluster they are serving (while the GCE PD driver still runs inside the served cluster), but there might be others scenarios).
+The CSI driver consists out of these services:
+
+* The **controller** service starts the GRPC server that is used for communication between the CSI components. It is depending on the GCP service account credentials.
+* The **identity** service is responsible to provide identity services like capability information of the CSI plugin.
+* The **node** service implements the various operations for volumes like creation, deletion, etc. It is depending on the GCE metadata service.
+
+The CSI driver has three command line flags, `--run-controller-service`, `--run-identity-service`, and `--run-node-service` which all default to `true`.
+You can disable individually services by setting the respective flags to `false`.
+
+Note: If you want to run the CSI controllers outside of the cluster you might need to specify the `local-zone` parameter in the GCE cloud provider config.
+The `local-zone` is the name of one of the availability zones the served Kubernetes cluster is deployed to.
+It is used to derive the GCP region and to discover the other availability zones in this region.

pkg/gce-cloud-provider/compute/gce.go

@@ -63,9 +63,10 @@ type ConfigFile struct {
 }
 
 type ConfigGlobal struct {
+	LocalZone string `gcfg:"local-zone"`
+	ProjectId string `gcfg:"project-id"`
 	TokenURL  string `gcfg:"token-url"`
 	TokenBody string `gcfg:"token-body"`
-	ProjectId string `gcfg:"project-id"`
 }
 
 func CreateCloudProvider(vendorVersion string, configPath string) (*CloudProvider, error) {
@@ -184,9 +185,16 @@ func newOauthClient(tokenSource oauth2.TokenSource) (*http.Client, error) {
 func getProjectAndZone(config *ConfigFile) (string, string, error) {
 	var err error
 
-	zone, err := metadata.Zone()
-	if err != nil {
-		return "", "", err
+	var zone string
+	if config == nil || config.Global.LocalZone == "" {
+		zone, err = metadata.Zone()
+		if err != nil {
+			return "", "", err
+		}
+		klog.V(2).Infof("Using GCP zone from the Metadata server: %q", zone)
+	} else {
+		zone = config.Global.LocalZone
+		klog.V(2).Infof("Using GCP zone from the local GCE cloud provider config file: %q", zone)
 	}
 
 	var projectID string

pkg/gce-pd-csi-driver/gce-pd-driver.go

@@ -45,8 +45,15 @@ func GetGCEDriver() *GCEDriver {
 	return &GCEDriver{}
 }
 
-func (gceDriver *GCEDriver) SetupGCEDriver(cloudProvider gce.GCECompute, mounter *mount.SafeFormatAndMount,
-	deviceUtils mountmanager.DeviceUtils, meta metadataservice.MetadataService, statter mountmanager.Statter, name, vendorVersion string) error {
+func (gceDriver *GCEDriver) SetupGCEDriver(
+	cloudProvider gce.GCECompute,
+	mounter *mount.SafeFormatAndMount,
+	deviceUtils mountmanager.DeviceUtils,
+	meta metadataservice.MetadataService,
+	statter mountmanager.Statter,
+	name, vendorVersion string,
+	runControllerService, runIdentityService, runNodeService bool,
+) error {
 	if name == "" {
 		return fmt.Errorf("Driver name missing")
 	}
@@ -79,9 +86,15 @@ func (gceDriver *GCEDriver) SetupGCEDriver(cloudProvider gce.GCECompute, mounter
 	gceDriver.AddNodeServiceCapabilities(ns)
 
 	// Set up RPC Servers
-	gceDriver.ids = NewIdentityServer(gceDriver)
-	gceDriver.ns = NewNodeServer(gceDriver, mounter, deviceUtils, meta, statter)
-	gceDriver.cs = NewControllerServer(gceDriver, cloudProvider, meta)
+	if runControllerService {
+		gceDriver.cs = NewControllerServer(gceDriver, cloudProvider, meta)
+	}
+	if runIdentityService {
+		gceDriver.ids = NewIdentityServer(gceDriver)
+	}
+	if runNodeService {
+		gceDriver.ns = NewNodeServer(gceDriver, mounter, deviceUtils, meta, statter)
+	}
 
 	return nil
 }

pkg/gce-pd-csi-driver/gce-pd-driver_test.go

@@ -44,7 +44,7 @@ func initBlockingGCEDriver(t *testing.T, cloudDisks []*gce.CloudDisk, readyToExe
 func initGCEDriverWithCloudProvider(t *testing.T, cloudProvider gce.GCECompute) *GCEDriver {
 	vendorVersion := "test-vendor"
 	gceDriver := GetGCEDriver()
-	err := gceDriver.SetupGCEDriver(cloudProvider, nil, nil, metadataservice.NewFakeService(), nil, driver, vendorVersion)
+	err := gceDriver.SetupGCEDriver(cloudProvider, nil, nil, metadataservice.NewFakeService(), nil, driver, vendorVersion, true, true, true)
 	if err != nil {
 		t.Fatalf("Failed to setup GCE Driver: %v", err)
 	}

pkg/gce-pd-csi-driver/identity_test.go

@@ -26,7 +26,7 @@ import (
 func TestGetPluginInfo(t *testing.T) {
 	vendorVersion := "test-vendor"
 	gceDriver := GetGCEDriver()
-	err := gceDriver.SetupGCEDriver(nil, nil, nil, metadataservice.NewFakeService(), nil, driver, vendorVersion)
+	err := gceDriver.SetupGCEDriver(nil, nil, nil, metadataservice.NewFakeService(), nil, driver, vendorVersion, true, true, true)
 	if err != nil {
 		t.Fatalf("Failed to setup GCE Driver: %v", err)
 	}
@@ -48,7 +48,7 @@ func TestGetPluginInfo(t *testing.T) {
 
 func TestGetPluginCapabilities(t *testing.T) {
 	gceDriver := GetGCEDriver()
-	err := gceDriver.SetupGCEDriver(nil, nil, nil, metadataservice.NewFakeService(), nil, driver, "test-vendor")
+	err := gceDriver.SetupGCEDriver(nil, nil, nil, metadataservice.NewFakeService(), nil, driver, "test-vendor", true, true, true)
 	if err != nil {
 		t.Fatalf("Failed to setup GCE Driver: %v", err)
 	}
@@ -80,7 +80,7 @@ func TestGetPluginCapabilities(t *testing.T) {
 
 func TestProbe(t *testing.T) {
 	gceDriver := GetGCEDriver()
-	err := gceDriver.SetupGCEDriver(nil, nil, nil, metadataservice.NewFakeService(), nil, driver, "test-vendor")
+	err := gceDriver.SetupGCEDriver(nil, nil, nil, metadataservice.NewFakeService(), nil, driver, "test-vendor", true, true, true)
 	if err != nil {
 		t.Fatalf("Failed to setup GCE Driver: %v", err)
 	}

pkg/gce-pd-csi-driver/node_test.go

@@ -42,7 +42,7 @@ func getTestGCEDriverWithCustomMounter(t *testing.T, mounter *mount.SafeFormatAn
 
 func getCustomTestGCEDriver(t *testing.T, mounter *mount.SafeFormatAndMount, deviceUtils mountmanager.DeviceUtils, metaService metadataservice.MetadataService) *GCEDriver {
 	gceDriver := GetGCEDriver()
-	err := gceDriver.SetupGCEDriver(nil, mounter, deviceUtils, metaService, mountmanager.NewFakeStatter(), driver, "test-vendor")
+	err := gceDriver.SetupGCEDriver(nil, mounter, deviceUtils, metaService, mountmanager.NewFakeStatter(), driver, "test-vendor", true, true, true)
 	if err != nil {
 		t.Fatalf("Failed to setup GCE Driver: %v", err)
 	}
@@ -51,7 +51,7 @@ func getCustomTestGCEDriver(t *testing.T, mounter *mount.SafeFormatAndMount, dev
 
 func getTestBlockingGCEDriver(t *testing.T, readyToExecute chan chan struct{}) *GCEDriver {
 	gceDriver := GetGCEDriver()
-	err := gceDriver.SetupGCEDriver(nil, mountmanager.NewFakeSafeBlockingMounter(readyToExecute), mountmanager.NewFakeDeviceUtils(), metadataservice.NewFakeService(), nil, driver, "test-vendor")
+	err := gceDriver.SetupGCEDriver(nil, mountmanager.NewFakeSafeBlockingMounter(readyToExecute), mountmanager.NewFakeDeviceUtils(), metadataservice.NewFakeService(), nil, driver, "test-vendor", true, true, true)
 	if err != nil {
 		t.Fatalf("Failed to setup GCE Driver: %v", err)
 	}

test/sanity/sanity_test.go

@@ -54,7 +54,7 @@ func TestSanity(t *testing.T) {
 	deviceUtils := mountmanager.NewFakeDeviceUtils()
 
 	//Initialize GCE Driver
-	err = gceDriver.SetupGCEDriver(cloudProvider, mounter, deviceUtils, metadataservice.NewFakeService(), mountmanager.NewFakeStatter(), driverName, vendorVersion)
+	err = gceDriver.SetupGCEDriver(cloudProvider, mounter, deviceUtils, metadataservice.NewFakeService(), mountmanager.NewFakeStatter(), driverName, vendorVersion, true, true, true)
 	if err != nil {
 		t.Fatalf("Failed to initialize GCE CSI Driver: %v", err)
 	}