Merge pull request #449 from rfranzke/feature/run-as-controller

Allow disabling particular driver services

Author: k8s-ci-robot

Makefile

@@ -40,7 +40,7 @@ push-container: build-container
 	gcloud docker -- push $(STAGINGIMAGE):$(STAGINGVERSION)
 
 test-sanity: gce-pd-driver
-	go test -timeout 30s sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/test -run ^TestSanity$
+	go test -v -timeout 30s sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/sanity -run ^TestSanity$
 
 test-k8s-integration:
 	go build -o bin/k8s-integration-test ./test/k8s-integration

cmd/main.go

@@ -30,9 +30,11 @@ import (
 )
 
 var (
-	endpoint          = flag.String("endpoint", "unix:/tmp/csi.sock", "CSI endpoint")
-	gceConfigFilePath = flag.String("cloud-config", "", "Path to GCE cloud provider config")
-	vendorVersion     string
+	cloudConfigFilePath  = flag.String("cloud-config", "", "Path to GCE cloud provider config")
+	endpoint             = flag.String("endpoint", "unix:/tmp/csi.sock", "CSI endpoint")
+	runControllerService = flag.Bool("run-controller-service", true, "If set to false then the CSI driver does not activate its controller service (default: true)")
+	runNodeService       = flag.Bool("run-node-service", true, "If set to false then the CSI driver does not activate its node service (default: true)")
+	vendorVersion        string
 )
 
 const (
@@ -57,6 +59,8 @@ func main() {
 }
 
 func handle() {
+	var err error
+
 	if vendorVersion == "" {
 		klog.Fatalf("vendorVersion must be set at compile time")
 	}
@@ -68,20 +72,35 @@ func handle() {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	cloudProvider, err := gce.CreateCloudProvider(ctx, vendorVersion, *gceConfigFilePath)
-	if err != nil {
-		klog.Fatalf("Failed to get cloud provider: %v", err)
+	//Initialize identity server
+	identityServer := driver.NewIdentityServer(gceDriver)
+
+	//Initialize requirements for the controller service
+	var controllerServer *driver.GCEControllerServer
+	if *runControllerService {
+		cloudProvider, err := gce.CreateCloudProvider(ctx, vendorVersion, *cloudConfigFilePath)
+		if err != nil {
+			klog.Fatalf("Failed to get cloud provider: %v", err)
+		}
+		controllerServer = driver.NewControllerServer(gceDriver, cloudProvider)
+	} else if *cloudConfigFilePath != "" {
+		klog.Warningf("controller service is disabled but cloud config given - it has no effect")
 	}
 
-	mounter := mountmanager.NewSafeMounter()
-	deviceUtils := mountmanager.NewDeviceUtils()
-	statter := mountmanager.NewStatter()
-	ms, err := metadataservice.NewMetadataService()
-	if err != nil {
-		klog.Fatalf("Failed to set up metadata service: %v", err)
+	//Initialize requirements for the node service
+	var nodeServer *driver.GCENodeServer
+	if *runNodeService {
+		mounter := mountmanager.NewSafeMounter()
+		deviceUtils := mountmanager.NewDeviceUtils()
+		statter := mountmanager.NewStatter()
+		meta, err := metadataservice.NewMetadataService()
+		if err != nil {
+			klog.Fatalf("Failed to set up metadata service: %v", err)
+		}
+		nodeServer = driver.NewNodeServer(gceDriver, mounter, deviceUtils, meta, statter)
 	}
 
-	err = gceDriver.SetupGCEDriver(cloudProvider, mounter, deviceUtils, ms, statter, driverName, vendorVersion)
+	err = gceDriver.SetupGCEDriver(driverName, vendorVersion, identityServer, controllerServer, nodeServer)
 	if err != nil {
 		klog.Fatalf("Failed to initialize GCE CSI Driver: %v", err)
 	}

deploy/kubernetes/base/controller.yaml

@@ -17,7 +17,7 @@ spec:
       # since it replaces GCE Metadata Server with GKE Metadata Server. Remove
       # this requirement when issue is resolved and before any exposure of
       # metrics ports
-      hostNetwork: true 
+      hostNetwork: true
       serviceAccountName: csi-gce-pd-controller-sa
       priorityClassName: csi-gce-pd-controller
       containers:
@@ -27,6 +27,8 @@ spec:
             - "--v=5"
             - "--csi-address=/csi/csi.sock"
             - "--feature-gates=Topology=true"
+          # - "--run-controller-service=false" # disable the controller service of the CSI driver
+          # - "--run-node-service=false"       # disable the node service of the CSI driver
           volumeMounts:
             - name: socket-dir
               mountPath: /csi

docs/kubernetes/user-guides/driver-install.md

@@ -18,7 +18,7 @@ compute.instances.get
 compute.instances.attachDisk
 compute.instances.detachDisk
 roles/compute.storageAdmin
-roles/iam.serviceAccountUser 
+roles/iam.serviceAccountUser
 ```
 
 If there is a pre-existing service account with these roles for use then the
@@ -79,4 +79,22 @@ iam.serviceAccounts.delete
 ```
 
 These permissions are not required if you already have a service account ready
-for use by the PD Driver.
+for use by the PD Driver.
+
+## Disabling particular CSI driver services
+
+Traditionally, you run the CSI controllers with the GCE PD driver in the same Kubernetes cluster.
+Though, there may be cases where you will only want to run a subset of the available driver services (for example, one scenario is running the controllers outside of the cluster they are serving (while the GCE PD driver still runs inside the served cluster), but there might be others scenarios).
+The CSI driver consists out of these services:
+
+* The **controller** service starts the GRPC server that serves `CreateVolume`, `DeleteVolume`, etc. It is depending on the GCP service account credentials and talks with the GCP API.
+* The **identity** service is responsible to provide identity services like capability information of the CSI plugin.
+* The **node** service implements the various operations for volumes that are run locally from the node, for example `NodePublishVolume`, `NodeStageVolume`, etc. It does not do operations like `CreateVolume` or `ControllerPublish`. Also, as it runs directly on the GCE instances, it is depending on the GCE metadata service.
+
+The CSI driver has two command line flags, `--run-controller-service` and `--run-node-service` which both default to `true`.
+You can disable the individual services by setting the respective flags to `false`.
+
+Note: If you want to run the CSI controllers outside of the cluster you have to specify both the `zone` and `projectId` parameters in the GCE cloud provider config.
+The `zone` is the name of one of the availability zones the served Kubernetes cluster is deployed to.
+It is used to derive the GCP region and to discover the other availability zones in this region.
+The `project-id` is the GCP project ID in which the controller is operating.

pkg/gce-cloud-provider/compute/fake-gce.go

@@ -67,6 +67,14 @@ func CreateFakeCloudProvider(project, zone string, cloudDisks []*CloudDisk) (*Fa
 	return fcp, nil
 }
 
+func (cloud *FakeCloudProvider) GetDefaultProject() string {
+	return cloud.project
+}
+
+func (cloud *FakeCloudProvider) GetDefaultZone() string {
+	return cloud.zone
+}
+
 func (cloud *FakeCloudProvider) RepairUnderspecifiedVolumeKey(ctx context.Context, volumeKey *meta.Key) (*meta.Key, error) {
 	switch volumeKey.Type() {
 	case meta.Zonal:

pkg/gce-cloud-provider/compute/gce-compute.go

@@ -38,6 +38,9 @@ const (
 )
 
 type GCECompute interface {
+	// Metadata information
+	GetDefaultProject() string
+	GetDefaultZone() string
 	// Disk Methods
 	GetDisk(ctx context.Context, volumeKey *meta.Key) (*CloudDisk, error)
 	RepairUnderspecifiedVolumeKey(ctx context.Context, volumeKey *meta.Key) (*meta.Key, error)
@@ -63,6 +66,16 @@ type GCECompute interface {
 	DeleteSnapshot(ctx context.Context, snapshotName string) error
 }
 
+// GetDefaultProject returns the project that was used to instantiate this GCE client.
+func (cloud *CloudProvider) GetDefaultProject() string {
+	return cloud.project
+}
+
+// GetDefaultZone returns the zone that was used to instantiate this GCE client.
+func (cloud *CloudProvider) GetDefaultZone() string {
+	return cloud.zone
+}
+
 // ListDisks lists disks based on maxEntries and pageToken only in the project
 // and zone that the driver is running in.
 func (cloud *CloudProvider) ListDisks(ctx context.Context, maxEntries int64, pageToken string) ([]*computev1.Disk, string, error) {

pkg/gce-cloud-provider/compute/gce.go

@@ -66,6 +66,7 @@ type ConfigGlobal struct {
 	TokenURL  string `gcfg:"token-url"`
 	TokenBody string `gcfg:"token-body"`
 	ProjectId string `gcfg:"project-id"`
+	Zone      string `gcfg:"zone"`
 }
 
 func CreateCloudProvider(ctx context.Context, vendorVersion string, configPath string) (*CloudProvider, error) {
@@ -103,7 +104,6 @@ func CreateCloudProvider(ctx context.Context, vendorVersion string, configPath s
 }
 
 func generateTokenSource(ctx context.Context, configFile *ConfigFile) (oauth2.TokenSource, error) {
-
 	if configFile != nil && configFile.Global.TokenURL != "" && configFile.Global.TokenURL != "nil" {
 		// configFile.Global.TokenURL is defined
 		// Use AltTokenSource
@@ -184,9 +184,16 @@ func newOauthClient(ctx context.Context, tokenSource oauth2.TokenSource) (*http.
 func getProjectAndZone(config *ConfigFile) (string, string, error) {
 	var err error
 
-	zone, err := metadata.Zone()
-	if err != nil {
-		return "", "", err
+	var zone string
+	if config == nil || config.Global.Zone == "" {
+		zone, err = metadata.Zone()
+		if err != nil {
+			return "", "", err
+		}
+		klog.V(2).Infof("Using GCP zone from the Metadata server: %q", zone)
+	} else {
+		zone = config.Global.Zone
+		klog.V(2).Infof("Using GCP zone from the local GCE cloud provider config file: %q", zone)
 	}
 
 	var projectID string

pkg/gce-cloud-provider/metadata/fake.go

@@ -21,9 +21,8 @@ type fakeServiceManager struct{}
 var _ MetadataService = &fakeServiceManager{}
 
 const (
-	FakeZone       = "country-region-zone"
-	FakeSecondZone = "country-region-zone2"
-	FakeProject    = "test-project"
+	FakeZone    = "country-region-zone"
+	FakeProject = "test-project"
 )
 
 var FakeMachineType = "n1-standard-1"

pkg/gce-pd-csi-driver/controller.go

@@ -15,18 +15,16 @@ limitations under the License.
 package gceGCEDriver
 
 import (
+	"context"
 	"fmt"
 	"math/rand"
 	"sort"
 	"strings"
 	"time"
 
-	"github.com/golang/protobuf/ptypes"
-
-	"context"
-
 	"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta"
 	csi "github.com/container-storage-interface/spec/lib/go/csi"
+	"github.com/golang/protobuf/ptypes"
 	compute "google.golang.org/api/compute/v1"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
@@ -35,13 +33,11 @@ import (
 
 	"sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common"
 	gce "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider/compute"
-	metadataservice "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider/metadata"
 )
 
 type GCEControllerServer struct {
-	Driver          *GCEDriver
-	CloudProvider   gce.GCECompute
-	MetadataService metadataservice.MetadataService
+	Driver        *GCEDriver
+	CloudProvider gce.GCECompute
 
 	// A map storing all volumes with ongoing operations so that additional
 	// operations for that same volume (as defined by Volume Key) return an
@@ -141,7 +137,7 @@ func (gceCS *GCEControllerServer) CreateVolume(ctx context.Context, req *csi.Cre
 		return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("CreateVolume replication type '%s' is not supported", replicationType))
 	}
 
-	volumeID, err := common.KeyToVolumeID(volKey, gceCS.MetadataService.GetProject())
+	volumeID, err := common.KeyToVolumeID(volKey, gceCS.CloudProvider.GetDefaultProject())
 	if err != nil {
 		return nil, status.Errorf(codes.Internal, "Failed to convert volume key to volume ID: %v", err)
 	}
@@ -950,7 +946,7 @@ func pickZones(ctx context.Context, gceCS *GCEControllerServer, top *csi.Topolog
 			return nil, fmt.Errorf("failed to pick zones from topology: %v", err)
 		}
 	} else {
-		zones, err = getDefaultZonesInRegion(ctx, gceCS, []string{gceCS.MetadataService.GetZone()}, numZones)
+		zones, err = getDefaultZonesInRegion(ctx, gceCS, []string{gceCS.CloudProvider.GetDefaultZone()}, numZones)
 		if err != nil {
 			return nil, fmt.Errorf("failed to get default %v zones in region: %v", numZones, err)
 		}

pkg/gce-pd-csi-driver/controller_test.go

@@ -35,15 +35,15 @@ import (
 	csi "github.com/container-storage-interface/spec/lib/go/csi"
 	"sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common"
 	gce "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider/compute"
-	metadataservice "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider/metadata"
 )
 
 const (
-	project = metadataservice.FakeProject
-	zone    = metadataservice.FakeZone
-	node    = "test-node"
-	driver  = "test-driver"
-	name    = "test-name"
+	project    = "test-project"
+	zone       = "country-region-zone"
+	secondZone = "country-region-fakesecondzone"
+	node       = "test-node"
+	driver     = "test-driver"
+	name       = "test-name"
 )
 
 var (
@@ -56,7 +56,7 @@ var (
 	}
 	stdTopology = []*csi.Topology{
 		{
-			Segments: map[string]string{common.TopologyKeyZone: metadataservice.FakeZone},
+			Segments: map[string]string{common.TopologyKeyZone: zone},
 		},
 	}
 	testVolumeID   = fmt.Sprintf("projects/%s/zones/%s/disks/%s", project, zone, name)
@@ -435,7 +435,7 @@ func TestCreateVolumeArguments(t *testing.T) {
 			},
 			expVol: &csi.Volume{
 				CapacityBytes: common.GbToBytes(20),
-				VolumeId:      fmt.Sprintf("projects/%s/zones/topology-zone/disks/%s", metadataservice.FakeProject, name),
+				VolumeId:      fmt.Sprintf("projects/%s/zones/topology-zone/disks/%s", project, name),
 				VolumeContext: nil,
 				AccessibleTopology: []*csi.Topology{
 					{
@@ -478,7 +478,7 @@ func TestCreateVolumeArguments(t *testing.T) {
 			},
 			expVol: &csi.Volume{
 				CapacityBytes: common.GbToBytes(20),
-				VolumeId:      fmt.Sprintf("projects/%s/zones/topology-zone2/disks/%s", metadataservice.FakeProject, name),
+				VolumeId:      fmt.Sprintf("projects/%s/zones/topology-zone2/disks/%s", project, name),
 				VolumeContext: nil,
 				AccessibleTopology: []*csi.Topology{
 					{
@@ -594,10 +594,10 @@ func TestCreateVolumeArguments(t *testing.T) {
 				VolumeContext: nil,
 				AccessibleTopology: []*csi.Topology{
 					{
-						Segments: map[string]string{common.TopologyKeyZone: metadataservice.FakeZone},
+						Segments: map[string]string{common.TopologyKeyZone: zone},
 					},
 					{
-						Segments: map[string]string{common.TopologyKeyZone: "country-region-fakesecondzone"},
+						Segments: map[string]string{common.TopologyKeyZone: secondZone},
 					},
 				},
 			},

pkg/gce-pd-csi-driver/gce-pd-driver.go

@@ -45,15 +45,11 @@ func GetGCEDriver() *GCEDriver {
 	return &GCEDriver{}
 }
 
-func (gceDriver *GCEDriver) SetupGCEDriver(cloudProvider gce.GCECompute, mounter *mount.SafeFormatAndMount,
-	deviceUtils mountmanager.DeviceUtils, meta metadataservice.MetadataService, statter mountmanager.Statter, name, vendorVersion string) error {
+func (gceDriver *GCEDriver) SetupGCEDriver(name, vendorVersion string, identityServer *GCEIdentityServer, controllerServer *GCEControllerServer, nodeServer *GCENodeServer) error {
 	if name == "" {
 		return fmt.Errorf("Driver name missing")
 	}
 
-	gceDriver.name = name
-	gceDriver.vendorVersion = vendorVersion
-
 	// Adding Capabilities
 	vcam := []csi.VolumeCapability_AccessMode_Mode{
 		csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER,
@@ -78,10 +74,11 @@ func (gceDriver *GCEDriver) SetupGCEDriver(cloudProvider gce.GCECompute, mounter
 	}
 	gceDriver.AddNodeServiceCapabilities(ns)
 
-	// Set up RPC Servers
-	gceDriver.ids = NewIdentityServer(gceDriver)
-	gceDriver.ns = NewNodeServer(gceDriver, mounter, deviceUtils, meta, statter)
-	gceDriver.cs = NewControllerServer(gceDriver, cloudProvider, meta)
+	gceDriver.name = name
+	gceDriver.vendorVersion = vendorVersion
+	gceDriver.ids = identityServer
+	gceDriver.cs = controllerServer
+	gceDriver.ns = nodeServer
 
 	return nil
 }
@@ -147,12 +144,11 @@ func NewNodeServer(gceDriver *GCEDriver, mounter *mount.SafeFormatAndMount, devi
 	}
 }
 
-func NewControllerServer(gceDriver *GCEDriver, cloudProvider gce.GCECompute, meta metadataservice.MetadataService) *GCEControllerServer {
+func NewControllerServer(gceDriver *GCEDriver, cloudProvider gce.GCECompute) *GCEControllerServer {
 	return &GCEControllerServer{
-		Driver:          gceDriver,
-		CloudProvider:   cloudProvider,
-		MetadataService: meta,
-		volumeLocks:     common.NewVolumeLocks(),
+		Driver:        gceDriver,
+		CloudProvider: cloudProvider,
+		volumeLocks:   common.NewVolumeLocks(),
 	}
 }