feat: Add the ServiceLoadbalancer Addon, with MetalLB as first provider (#592)

**What problem does this PR solve?**:
Adds the `ServiceLoadbalancer` addon, and support for MetalLB as the
first provider.

### TODO:
- [x] API type
- [x] Generic handler
- [x] MetalLB handler
- [x] Deploy MetalLB Helm chart 
- [x] Default configuration for MetalLB Helm chart
- [ ] Document the ServiceLoadbalancer Addon.

**Which issue(s) this PR fixes**:
Fixes https://jira.nutanix.com/browse/D2IQ-100444

**How Has This Been Tested?**:
<!--
Please describe the tests that you ran to verify your changes.
Provide output from the tests and any manual steps needed to replicate
the tests.
-->

**Special notes for your reviewer**:
<!--
Use this to provide any additional information to the reviewers.
This may include:
- Best way to review the PR.
- Where the author wants the most review attention on.
- etc.
-->

---------

Co-authored-by: Jimmi Dyson <[email protected]>

Author: dlipovetsky

api/v1alpha1/addon_types.go

@@ -24,6 +24,8 @@ const (
 
 	VirtualIPProviderKubeVIP = "KubeVIP"
 
+	ServiceLoadBalancerProviderMetalLB = "MetalLB"
+
 	AddonStrategyClusterResourceSet AddonStrategy = "ClusterResourceSet"
 	AddonStrategyHelmAddon          AddonStrategy = "HelmAddon"
 
@@ -69,6 +71,9 @@ type Addons struct {
 
 	// +optional
 	CSIProviders *CSI `json:"csi,omitempty"`
+
+	// +optional
+	ServiceLoadBalancer *ServiceLoadBalancer `json:"serviceLoadBalancer,omitempty"`
 }
 
 type AddonStrategy string
@@ -160,3 +165,10 @@ type CCM struct {
 	// +optional
 	Credentials *corev1.LocalObjectReference `json:"credentials,omitempty"`
 }
+
+type ServiceLoadBalancer struct {
+	// The LoadBalancer-type Service provider to deploy. Not required in infrastructures where
+	// the CCM acts as the provider.
+	// +kubebuilder:validation:Enum=MetalLB
+	Provider string `json:"provider"`
+}

api/v1alpha1/constants.go

@@ -14,4 +14,6 @@ const (
 	AWSVariableName = "aws"
 	// NutanixVariableName is the Nutanix config patch variable name.
 	NutanixVariableName = "nutanix"
+	// ServiceLoadBalancerName is the Service LoadBalancer config patch variable name.
+	ServiceLoadBalancerVariableName = "serviceLoadBalancer"
 )

api/v1alpha1/crds/caren.nutanix.com_awsclusterconfigs.yaml

@@ -201,6 +201,18 @@ spec:
                     required:
                     - strategy
                     type: object
+                  serviceLoadBalancer:
+                    properties:
+                      provider:
+                        description: |-
+                          The LoadBalancer-type Service provider to deploy. Not required in infrastructures where
+                          the CCM acts as the provider.
+                        enum:
+                        - MetalLB
+                        type: string
+                    required:
+                    - provider
+                    type: object
                 type: object
               aws:
                 description: AWS cluster configuration.

api/v1alpha1/crds/caren.nutanix.com_dockerclusterconfigs.yaml

@@ -202,6 +202,18 @@ spec:
                     required:
                     - strategy
                     type: object
+                  serviceLoadBalancer:
+                    properties:
+                      provider:
+                        description: |-
+                          The LoadBalancer-type Service provider to deploy. Not required in infrastructures where
+                          the CCM acts as the provider.
+                        enum:
+                        - MetalLB
+                        type: string
+                    required:
+                    - provider
+                    type: object
                 type: object
               controlPlane:
                 description: DockerNodeConfigSpec defines the desired state of DockerNodeSpec.

api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml

@@ -202,6 +202,18 @@ spec:
                     required:
                     - strategy
                     type: object
+                  serviceLoadBalancer:
+                    properties:
+                      provider:
+                        description: |-
+                          The LoadBalancer-type Service provider to deploy. Not required in infrastructures where
+                          the CCM acts as the provider.
+                        enum:
+                        - MetalLB
+                        type: string
+                    required:
+                    - provider
+                    type: object
                 type: object
               controlPlane:
                 description: NutanixNodeSpec defines the desired state of NutanixNodeSpec.

api/v1alpha1/zz_generated.deepcopy.go

@@ -62,6 +62,8 @@ A Helm chart for cluster-api-runtime-extensions-nutanix
 | hooks.nfd.crsStrategy.defaultInstallationConfigMap.name | string | `"node-feature-discovery"` |  |
 | hooks.nfd.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` |  |
 | hooks.nfd.helmAddonStrategy.defaultValueTemplateConfigMap.name | string | `"default-nfd-helm-values-template"` |  |
+| hooks.serviceLoadBalancer.metalLB.defaultValueTemplateConfigMap.create | bool | `true` |  |
+| hooks.serviceLoadBalancer.metalLB.defaultValueTemplateConfigMap.name | string | `"default-metallb-helm-values-template"` |  |
 | hooks.virtualIP.kubeVip.defaultTemplateConfigMap.create | bool | `true` |  |
 | hooks.virtualIP.kubeVip.defaultTemplateConfigMap.name | string | `"default-kube-vip-template"` |  |
 | image.pullPolicy | string | `"IfNotPresent"` |  |

charts/cluster-api-runtime-extensions-nutanix/README.md

@@ -15,6 +15,10 @@ data:
     ChartName: cluster-autoscaler
     ChartVersion: 9.35.0
     RepositoryURL: https://kubernetes.github.io/autoscaler
+  metallb: |
+    ChartName: metallb
+    ChartVersion: v0.14.5
+    RepositoryURL: https://metallb.github.io/metallb
   nfd: |
     ChartName: node-feature-discovery
     ChartVersion: 0.15.2

charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml

@@ -0,0 +1,31 @@
+# Copyright 2023 D2iQ, Inc. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+{{- if .Values.hooks.serviceLoadBalancer.metalLB.defaultValueTemplateConfigMap.create }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: '{{ .Values.hooks.serviceLoadBalancer.metalLB.defaultValueTemplateConfigMap.name }}'
+data:
+  values.yaml: |-
+    controller:
+      tolerations:
+        - key: node-role.kubernetes.io/control-plane
+          effect: NoSchedule
+          operator: Exists
+        - key: CriticalAddonsOnly
+          operator: Exists
+        - effect: NoExecute
+          operator: Exists
+          tolerationSeconds: 300
+    speaker:
+      tolerations:
+        - key: node-role.kubernetes.io/control-plane
+          effect: NoSchedule
+          operator: Exists
+        - key: CriticalAddonsOnly
+          operator: Exists
+        - effect: NoExecute
+          operator: Exists
+          tolerationSeconds: 300
+{{- end -}}

charts/cluster-api-runtime-extensions-nutanix/templates/serviceloadbalancer/metallb/helm-addon-installation.yaml

@@ -71,7 +71,11 @@ hooks:
       defaultValueTemplateConfigMap:
         create: true
         name: default-cluster-autoscaler-helm-values-template
-
+  serviceLoadBalancer:
+    metalLB:
+      defaultValueTemplateConfigMap:
+        create: true
+        name: default-metallb-helm-values-template
   virtualIP:
     kubeVip:
       defaultTemplateConfigMap:

charts/cluster-api-runtime-extensions-nutanix/values.yaml

@@ -0,0 +1,40 @@
++++
+title = "Service LoadBalancer"
++++
+
+When an application running in a cluster needs to be exposed outside of the cluster, one option is
+to use an [external load balancer], by creating a Kubernetes Service of the
+`LoadBalancer` type.
+
+The Service Load Balancer is the component that backs this Kubernetes Service, either by creating
+a Virtual IP, creating a machine that runs load balancer software, by delegating to APIs, such as
+the underlying infrastructure, or a hardware load balancer.
+
+CAREN currently supports the following Service Load Balancers:
+
+- [MetalLB]
+
+## Example
+
+To enable deployment of MetalLB on a cluster, specify the following values:
+
+```yaml
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  name: <NAME>
+spec:
+  topology:
+    variables:
+      - name: clusterConfig
+        value:
+          addons:
+            serviceLoadBalancer:
+              provider: MetalLB
+```
+
+See [MetalLB documentation] for details on configuration.
+
+[external load balancer]: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/
+[MetalLB]: https://metallb.org
+[MetalLB documentation]: https://metallb.org/configuration/

docs/content/addons/serviceloadbalancer.md

@@ -0,0 +1,27 @@
+# Copyright 2024 D2iQ, Inc. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# NOTE This file is used by the tool in `hack/tools/helm-cm` to add
+# metallb chart metadata to the "helm-addons" ConfigMap. The tool takes
+# a kustomization as input. We do not use this file with kustomize.
+
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+metadata:
+  name: metallb
+
+sortOptions:
+  order: fifo
+
+helmCharts:
+- name: metallb
+  repo: https://metallb.github.io/metallb
+  releaseName: metallb
+  version: ${METALLB_CHART_VERSION}
+  valuesFile: helm-values.yaml
+  includeCRDs: true
+  skipTests: true
+  namespace: metallb-system
+
+namespace: metallb-system

hack/addons/kustomize/metallb/kustomization.yaml.tmpl

@@ -21,6 +21,8 @@ export NUTANIX_CCM_CHART_VERSION := 0.3.3
 
 export KUBE_VIP_VERSION := v0.8.0
 
+export METALLB_CHART_VERSION := v0.14.5
+
 .PHONY: addons.sync
 addons.sync: $(addprefix update-addon.,calico cilium nfd cluster-autoscaler aws-ebs-csi aws-ccm.127 aws-ccm.128 aws-ccm.129 kube-vip)
 

make/addons.mk

@@ -34,7 +34,7 @@ var _ = Describe("Generate AWS ControlPlane LoadBalancer patches", func() {
 			Name: "unset variable",
 		},
 		{
-			Name: "ControlPlaneLoadbalancer scheme set to internet-facing",
+			Name: "ControlPlaneLoadBalancer scheme set to internet-facing",
 			Vars: []runtimehooksv1.Variable{
 				capitest.VariableWithValue(
 					clusterconfig.MetaVariableName,
@@ -55,7 +55,7 @@ var _ = Describe("Generate AWS ControlPlane LoadBalancer patches", func() {
 			}},
 		},
 		{
-			Name: "ControlPlaneLoadbalancer scheme set to internal",
+			Name: "ControlPlaneLoadBalancer scheme set to internal",
 			Vars: []runtimehooksv1.Variable{
 				capitest.VariableWithValue(
 					clusterconfig.MetaVariableName,

pkg/handlers/aws/mutation/controlplaneloadbalancer/inject_test.go

@@ -24,6 +24,7 @@ const (
 	NutanixStorageCSI  Component = "nutanix-storage-csi"
 	NutanixSnapshotCSI Component = "nutanix-snapshot-csi"
 	NutanixCCM         Component = "nutanix-ccm"
+	MetalLB            Component = "metallb"
 )
 
 type HelmChartGetter struct {

pkg/handlers/generic/lifecycle/config/cm.go

@@ -21,6 +21,8 @@ import (
 	nutanixcsi "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/csi/nutanix-csi"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/nfd"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/servicelbgc"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/serviceloadbalancer"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/serviceloadbalancer/metallb"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/options"
 )
 
@@ -34,6 +36,7 @@ type Handlers struct {
 	nutnaixCSIConfig        *nutanixcsi.NutanixCSIConfig
 	awsccmConfig            *awsccm.AWSCCMConfig
 	nutanixCCMConfig        *nutanixccm.Config
+	metalLBConfig           *metallb.Config
 }
 
 func New(
@@ -51,6 +54,7 @@ func New(
 		awsccmConfig:            &awsccm.AWSCCMConfig{GlobalOptions: globalOptions},
 		nutnaixCSIConfig:        &nutanixcsi.NutanixCSIConfig{GlobalOptions: globalOptions},
 		nutanixCCMConfig:        &nutanixccm.Config{GlobalOptions: globalOptions},
+		metalLBConfig:           &metallb.Config{GlobalOptions: globalOptions},
 	}
 }
 
@@ -76,6 +80,13 @@ func (h *Handlers) AllHandlers(mgr manager.Manager) []handlers.Named {
 			helmChartInfoGetter,
 		),
 	}
+	serviceLoadBalancerHandlers := map[string]serviceloadbalancer.ServiceLoadBalancerProvider{
+		v1alpha1.ServiceLoadBalancerProviderMetalLB: metallb.New(
+			mgr.GetClient(),
+			h.metalLBConfig,
+			helmChartInfoGetter,
+		),
+	}
 	return []handlers.Named{
 		calico.New(mgr.GetClient(), h.calicoCNIConfig, helmChartInfoGetter),
 		cilium.New(mgr.GetClient(), h.ciliumCNIConfig, helmChartInfoGetter),
@@ -84,6 +95,7 @@ func (h *Handlers) AllHandlers(mgr manager.Manager) []handlers.Named {
 		servicelbgc.New(mgr.GetClient()),
 		csi.New(mgr.GetClient(), csiHandlers),
 		ccm.New(mgr.GetClient(), ccmHandlers),
+		serviceloadbalancer.New(mgr.GetClient(), serviceLoadBalancerHandlers),
 	}
 }
 
@@ -96,4 +108,5 @@ func (h *Handlers) AddFlags(flagSet *pflag.FlagSet) {
 	h.awsccmConfig.AddFlags("awsccm", pflag.CommandLine)
 	h.nutnaixCSIConfig.AddFlags("nutanixcsi", flagSet)
 	h.nutanixCCMConfig.AddFlags("nutanixccm", flagSet)
+	h.metalLBConfig.AddFlags("metallb", flagSet)
 }