Allow running the controller service separately

Author: rfranzke

cmd/main.go

@@ -33,9 +33,10 @@ func init() {
 }
 
 var (
-	endpoint          = flag.String("endpoint", "unix:/tmp/csi.sock", "CSI endpoint")
-	gceConfigFilePath = flag.String("cloud-config", "", "Path to GCE cloud provider config")
-	vendorVersion     string
+	endpoint            = flag.String("endpoint", "unix:/tmp/csi.sock", "CSI endpoint")
+	cloudConfigFilePath = flag.String("cloud-config", "", "Path to GCE cloud provider config")
+	runAsController     = flag.Bool("run-as-controller", false, "If set to true then the CSI driver runs the controller only and does not start the node service.")
+	vendorVersion       string
 )
 
 const (
@@ -66,22 +67,25 @@ func handle() {
 
 	gceDriver := driver.GetGCEDriver()
 
-	//Initialize GCE Driver (Move setup to main?)
-	cloudProvider, err := gce.CreateCloudProvider(vendorVersion, *gceConfigFilePath)
+	// Initialize GCE Driver (Move setup to main?)
+	cloudProvider, err := gce.CreateCloudProvider(vendorVersion, *cloudConfigFilePath)
 	if err != nil {
 		klog.Fatalf("Failed to get cloud provider: %v", err)
 	}
 
 	mounter := mountmanager.NewSafeMounter()
 	deviceUtils := mountmanager.NewDeviceUtils()
 	statter := mountmanager.NewStatter()
-	ms, err := metadataservice.NewMetadataService()
-	if err != nil {
-		klog.Fatalf("Failed to set up metadata service: %v", err)
+
+	var ms metadataservice.MetadataService
+	if runAsController == nil || !*runAsController {
+		ms, err = metadataservice.NewMetadataService()
+		if err != nil {
+			klog.Fatalf("Failed to set up metadata service: %v", err)
+		}
 	}
 
-	err = gceDriver.SetupGCEDriver(cloudProvider, mounter, deviceUtils, ms, statter, driverName, vendorVersion)
-	if err != nil {
+	if err := gceDriver.SetupGCEDriver(cloudProvider, mounter, deviceUtils, ms, statter, driverName, vendorVersion, runAsController); err != nil {
 		klog.Fatalf("Failed to initialize GCE CSI Driver: %v", err)
 	}
 

deploy/kubernetes/base/controller.yaml

@@ -17,7 +17,7 @@ spec:
       # since it replaces GCE Metadata Server with GKE Metadata Server. Remove
       # this requirement when issue is resolved and before any exposure of
       # metrics ports
-      hostNetwork: true 
+      hostNetwork: true
       serviceAccountName: csi-gce-pd-controller-sa
       priorityClassName: csi-gce-pd-controller
       containers:
@@ -27,6 +27,7 @@ spec:
             - "--v=5"
             - "--csi-address=/csi/csi.sock"
             - "--feature-gates=Topology=true"
+          # - "--run-as-controller=true" # only run the controller service of the driver
           volumeMounts:
             - name: socket-dir
               mountPath: /csi

docs/kubernetes/user-guides/driver-install.md

@@ -20,4 +20,13 @@ $ ./deploy/setup-project.sh
 $ GCE_PD_SA_DIR=/my/safe/credentials/directory    # Directory to get the service account key
 $ GCE_PD_DRIVER_VERSION=stable                    # Driver version to deploy
 $ ./deploy/kubernetes/deploy-driver.sh
-```
+```
+
+## Running the controller separately
+
+Traditionally, you run the CSI controllers with the GCE PD driver in the same Kubernetes cluster.
+Though, in some scenarios you might want to run the CSI controllers (csi-provisioner, csi-attacher, etc.) together with the GCE PD controller service of this driver separately from the Kubernetes cluster it serves (while the GCE PD driver with an activated node service still runs inside the cluster).
+Also, this may not necessarily have to be an GCE compute instance.
+To support these cases, the GCE PD CSI driver plugin supports the following command line flag:
+
+* `--run-as-controller=true` which allows you to only start the controller service (and not the node service) of the GCE PD driver.

pkg/gce-cloud-provider/compute/gce.go

@@ -63,9 +63,10 @@ type ConfigFile struct {
 }
 
 type ConfigGlobal struct {
+	LocalZone string `gcfg:"local-zone"`
+	ProjectId string `gcfg:"project-id"`
 	TokenURL  string `gcfg:"token-url"`
 	TokenBody string `gcfg:"token-body"`
-	ProjectId string `gcfg:"project-id"`
 }
 
 func CreateCloudProvider(vendorVersion string, configPath string) (*CloudProvider, error) {
@@ -184,9 +185,16 @@ func newOauthClient(tokenSource oauth2.TokenSource) (*http.Client, error) {
 func getProjectAndZone(config *ConfigFile) (string, string, error) {
 	var err error
 
-	zone, err := metadata.Zone()
-	if err != nil {
-		return "", "", err
+	var zone string
+	if config == nil || config.Global.LocalZone == "" {
+		zone, err = metadata.Zone()
+		if err != nil {
+			return "", "", err
+		}
+		klog.V(2).Infof("Using GCP zone from the Metadata server: %q", zone)
+	} else {
+		zone = config.Global.LocalZone
+		klog.V(2).Infof("Using GCP zone from the local GCE cloud provider config file: %q", zone)
 	}
 
 	var projectID string

pkg/gce-pd-csi-driver/gce-pd-driver.go

@@ -45,8 +45,15 @@ func GetGCEDriver() *GCEDriver {
 	return &GCEDriver{}
 }
 
-func (gceDriver *GCEDriver) SetupGCEDriver(cloudProvider gce.GCECompute, mounter *mount.SafeFormatAndMount,
-	deviceUtils mountmanager.DeviceUtils, meta metadataservice.MetadataService, statter mountmanager.Statter, name, vendorVersion string) error {
+func (gceDriver *GCEDriver) SetupGCEDriver(
+	cloudProvider gce.GCECompute,
+	mounter *mount.SafeFormatAndMount,
+	deviceUtils mountmanager.DeviceUtils,
+	meta metadataservice.MetadataService,
+	statter mountmanager.Statter,
+	name, vendorVersion string,
+	runAsController *bool,
+) error {
 	if name == "" {
 		return fmt.Errorf("Driver name missing")
 	}
@@ -79,9 +86,12 @@ func (gceDriver *GCEDriver) SetupGCEDriver(cloudProvider gce.GCECompute, mounter
 	gceDriver.AddNodeServiceCapabilities(ns)
 
 	// Set up RPC Servers
-	gceDriver.ids = NewIdentityServer(gceDriver)
-	gceDriver.ns = NewNodeServer(gceDriver, mounter, deviceUtils, meta, statter)
 	gceDriver.cs = NewControllerServer(gceDriver, cloudProvider, meta)
+	gceDriver.ids = NewIdentityServer(gceDriver)
+
+	if runAsController == nil || !*runAsController {
+		gceDriver.ns = NewNodeServer(gceDriver, mounter, deviceUtils, meta, statter)
+	}
 
 	return nil
 }

pkg/gce-pd-csi-driver/gce-pd-driver_test.go

@@ -44,7 +44,8 @@ func initBlockingGCEDriver(t *testing.T, cloudDisks []*gce.CloudDisk, readyToExe
 func initGCEDriverWithCloudProvider(t *testing.T, cloudProvider gce.GCECompute) *GCEDriver {
 	vendorVersion := "test-vendor"
 	gceDriver := GetGCEDriver()
-	err := gceDriver.SetupGCEDriver(cloudProvider, nil, nil, metadataservice.NewFakeService(), nil, driver, vendorVersion)
+	runAsController := false
+	err := gceDriver.SetupGCEDriver(cloudProvider, nil, nil, metadataservice.NewFakeService(), nil, driver, vendorVersion, &runAsController)
 	if err != nil {
 		t.Fatalf("Failed to setup GCE Driver: %v", err)
 	}

pkg/gce-pd-csi-driver/identity_test.go

@@ -26,7 +26,8 @@ import (
 func TestGetPluginInfo(t *testing.T) {
 	vendorVersion := "test-vendor"
 	gceDriver := GetGCEDriver()
-	err := gceDriver.SetupGCEDriver(nil, nil, nil, metadataservice.NewFakeService(), nil, driver, vendorVersion)
+	runAsController := false
+	err := gceDriver.SetupGCEDriver(nil, nil, nil, metadataservice.NewFakeService(), nil, driver, vendorVersion, &runAsController)
 	if err != nil {
 		t.Fatalf("Failed to setup GCE Driver: %v", err)
 	}
@@ -48,7 +49,8 @@ func TestGetPluginInfo(t *testing.T) {
 
 func TestGetPluginCapabilities(t *testing.T) {
 	gceDriver := GetGCEDriver()
-	err := gceDriver.SetupGCEDriver(nil, nil, nil, metadataservice.NewFakeService(), nil, driver, "test-vendor")
+	runAsController := false
+	err := gceDriver.SetupGCEDriver(nil, nil, nil, metadataservice.NewFakeService(), nil, driver, "test-vendor", &runAsController)
 	if err != nil {
 		t.Fatalf("Failed to setup GCE Driver: %v", err)
 	}
@@ -80,7 +82,8 @@ func TestGetPluginCapabilities(t *testing.T) {
 
 func TestProbe(t *testing.T) {
 	gceDriver := GetGCEDriver()
-	err := gceDriver.SetupGCEDriver(nil, nil, nil, metadataservice.NewFakeService(), nil, driver, "test-vendor")
+	runAsController := false
+	err := gceDriver.SetupGCEDriver(nil, nil, nil, metadataservice.NewFakeService(), nil, driver, "test-vendor", &runAsController)
 	if err != nil {
 		t.Fatalf("Failed to setup GCE Driver: %v", err)
 	}

pkg/gce-pd-csi-driver/node_test.go

@@ -44,7 +44,8 @@ func getTestGCEDriverWithCustomMounter(t *testing.T, mounter *mount.SafeFormatAn
 
 func getCustomTestGCEDriver(t *testing.T, mounter *mount.SafeFormatAndMount, deviceUtils mountmanager.DeviceUtils, metaService metadataservice.MetadataService) *GCEDriver {
 	gceDriver := GetGCEDriver()
-	err := gceDriver.SetupGCEDriver(nil, mounter, deviceUtils, metaService, mountmanager.NewFakeStatter(), driver, "test-vendor")
+	runAsController := false
+	err := gceDriver.SetupGCEDriver(nil, mounter, deviceUtils, metaService, mountmanager.NewFakeStatter(), driver, "test-vendor", &runAsController)
 	if err != nil {
 		t.Fatalf("Failed to setup GCE Driver: %v", err)
 	}
@@ -53,7 +54,8 @@ func getCustomTestGCEDriver(t *testing.T, mounter *mount.SafeFormatAndMount, dev
 
 func getTestBlockingGCEDriver(t *testing.T, readyToExecute chan chan struct{}) *GCEDriver {
 	gceDriver := GetGCEDriver()
-	err := gceDriver.SetupGCEDriver(nil, mountmanager.NewFakeSafeBlockingMounter(readyToExecute), mountmanager.NewFakeDeviceUtils(), metadataservice.NewFakeService(), nil, driver, "test-vendor")
+	runAsController := false
+	err := gceDriver.SetupGCEDriver(nil, mountmanager.NewFakeSafeBlockingMounter(readyToExecute), mountmanager.NewFakeDeviceUtils(), metadataservice.NewFakeService(), nil, driver, "test-vendor", &runAsController)
 	if err != nil {
 		t.Fatalf("Failed to setup GCE Driver: %v", err)
 	}

test/sanity/sanity_test.go

@@ -43,6 +43,7 @@ func TestSanity(t *testing.T) {
 	stagePath := path.Join(tmpDir, "stage")
 	// Set up driver and env
 	gceDriver := driver.GetGCEDriver()
+	runAsController := false
 
 	cloudProvider, err := gce.CreateFakeCloudProvider(project, zone, nil)
 	if err != nil {
@@ -53,7 +54,7 @@ func TestSanity(t *testing.T) {
 	deviceUtils := mountmanager.NewFakeDeviceUtils()
 
 	//Initialize GCE Driver
-	err = gceDriver.SetupGCEDriver(cloudProvider, mounter, deviceUtils, metadataservice.NewFakeService(), mountmanager.NewFakeStatter(), driverName, vendorVersion)
+	err = gceDriver.SetupGCEDriver(cloudProvider, mounter, deviceUtils, metadataservice.NewFakeService(), mountmanager.NewFakeStatter(), driverName, vendorVersion, &runAsController)
 	if err != nil {
 		t.Fatalf("Failed to initialize GCE CSI Driver: %v", err)
 	}