✨ support HA components in controller manager

HA runnable such as webhook server can be run in this mode

Author: Mengqi Yu

pkg/manager/internal.go

@@ -52,7 +52,11 @@ type controllerManager struct {
 	scheme *runtime.Scheme
 
 	// runnables is the set of Controllers that the controllerManager injects deps into and Starts.
+	// These Runnables are managed by lead election.
 	runnables []Runnable
+	// haRunnables is the set of webhook servers that the controllerManager injects deps into and Starts.
+	// These Runnables are in HA mode (not blocked by lead election)
+	haRunnables []Runnable
 
 	cache cache.Cache
 
@@ -114,8 +118,13 @@ func (cm *controllerManager) Add(r Runnable) error {
 		return err
 	}
 
-	// Add the runnable to the list
-	cm.runnables = append(cm.runnables, r)
+	// Add the runnable to the HA or the leader election list
+	if haRunable, ok := r.(HARunnable); ok && haRunable.IsHARunnable() {
+		cm.haRunnables = append(cm.haRunnables, r)
+	} else {
+		cm.runnables = append(cm.runnables, r)
+	}
+
 	if cm.started {
 		// If already started, start the controller
 		go func() {
@@ -237,6 +246,8 @@ func (cm *controllerManager) Start(stop <-chan struct{}) error {
 		go cm.serveMetrics(cm.internalStop)
 	}
 
+	go cm.startHA()
+
 	if cm.resourceLock != nil {
 		err := cm.startLeaderElection()
 		if err != nil {
@@ -256,7 +267,7 @@ func (cm *controllerManager) Start(stop <-chan struct{}) error {
 	}
 }
 
-func (cm *controllerManager) start() {
+func (cm *controllerManager) startHA() {
 	cm.mu.Lock()
 	defer cm.mu.Unlock()
 
@@ -274,6 +285,24 @@ func (cm *controllerManager) start() {
 	// TODO(community): Check the return value and write a test
 	cm.cache.WaitForCacheSync(cm.internalStop)
 
+	// Start the HA runnables after the cache has synced
+	for _, c := range cm.haRunnables {
+		// Controllers block, but we want to return an error if any have an error starting.
+		// Write any Start errors to a channel so we can return them
+		ctrl := c
+		go func() {
+			cm.errChan <- ctrl.Start(cm.internalStop)
+		}()
+	}
+
+	cm.started = true
+}
+
+func (cm *controllerManager) start() {
+	// Wait for the caches to sync.
+	// TODO(community): Check the return value and write a test
+	cm.cache.WaitForCacheSync(cm.internalStop)
+
 	// Start the runnables after the cache has synced
 	for _, c := range cm.runnables {
 		// Controllers block, but we want to return an error if any have an error starting.

pkg/manager/manager.go

@@ -42,9 +42,11 @@ import (
 // Manager initializes shared dependencies such as Caches and Clients, and provides them to Runnables.
 // A Manager is required to create Controllers.
 type Manager interface {
-	// Add will set reqeusted dependencies on the component, and cause the component to be
+	// Add will set requested dependencies on the component, and cause the component to be
 	// started when Start is called.  Add will inject any dependencies for which the argument
-	// implements the inject interface - e.g. inject.Client
+	// implements the inject interface - e.g. inject.Client.
+	// Depending on if a Runnable implements HARunnable interface, a Runnable can be run in either
+	// HA mode (always running) or non-HA mode (managed by leader election if enabled).
 	Add(Runnable) error
 
 	// SetFields will set any dependencies on an object for which the object has implemented the inject
@@ -172,6 +174,12 @@ func (r RunnableFunc) Start(s <-chan struct{}) error {
 	return r(s)
 }
 
+// HARunnable knows if a Runnable needs to be run in HA mode.
+type HARunnable interface {
+	// IsHARunnable returns true if the Runnable (i.e. webhook server) need to be run in HA mode.
+	IsHARunnable() bool
+}
+
 // New returns a new Manager for creating Controllers.
 func New(config *rest.Config, options Options) (Manager, error) {
 	// Initialize a rest.config if none was specified

pkg/manager/manager_test.go

@@ -284,6 +284,11 @@ var _ = Describe("manger.Manager", func() {
 				<-c2
 				<-c3
 			})
+
+			It("should return an error if any HA Components fail to Start", func() {
+				// TODO(mengqiy): implement this after resolving https://github.com/kubernetes-sigs/controller-runtime/issues/429
+				// Example: ListOptions
+			})
 		}
 
 		Context("with defaults", func() {