feat: Make addon provider configurable

Author: jimmidyson

README.md

@@ -25,7 +25,7 @@ clusterctl generate cluster capi-quickstart \
   --control-plane-machine-count=1 \
   --worker-machine-count=1 | \
   gojq --yaml-input --yaml-output --slurp \
-    '.[] | (select( .kind=="Cluster").metadata.labels += {"capi-runtime-extensions.d2iq-labs.com/cni": "calico"})' \
+    '.[] | (select( .kind=="Cluster").metadata.labels += {"capi-runtime-extensions.d2iq-labs.com/cni": "calico"})' | \
   kubectl apply -f -
 ```
 

charts/capi-runtime-extensions/README.md

@@ -25,13 +25,14 @@ A Helm chart for capi-runtime-extensions
 
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
+| addons.provider | string | `"ClusterResourceSet"` |  |
 | certificate.issuer.kind | string | `"Issuer"` |  |
-| certificate.issuer.name | string | `nil` |  |
+| certificate.issuer.name | string | `""` |  |
 | certificate.issuer.selfSigned | bool | `true` |  |
 | env | object | `{}` |  |
 | image.pullPolicy | string | `"IfNotPresent"` |  |
 | image.repository | string | `"ghcr.io/d2iq-labs/capi-runtime-extensions"` |  |
-| image.tag | string | `nil` |  |
+| image.tag | string | `""` |  |
 | imagePullSecrets | list | `[]` | Optional secrets used for pulling the container image |
 | nodeSelector | object | `{}` |  |
 | priorityClassName | string | `""` | Optional priority class to be used for the pod. |

charts/capi-runtime-extensions/values.schema.json

@@ -0,0 +1,115 @@
+{
+    "$schema": "http://json-schema.org/schema#",
+    "type": "object",
+    "properties": {
+        "addons": {
+            "type": "object",
+            "properties": {
+                "provider": {
+                    "type": "string",
+                    "enum": ["ClusterResourceSet", "FluxHelmRelease"]
+                }
+            }
+        },
+        "certificate": {
+            "type": "object",
+            "properties": {
+                "issuer": {
+                    "type": "object",
+                    "properties": {
+                        "kind": {
+                            "type": "string",
+                            "enum": ["Issuer", "ClusterIssuer"],
+                            "default": "Issuer"
+                        },
+                        "name": {
+                            "type": "string"
+                        },
+                        "selfSigned": {
+                            "type": "boolean"
+                        }
+                    }
+                }
+            }
+        },
+        "env": {
+            "type": "object"
+        },
+        "image": {
+            "type": "object",
+            "properties": {
+                "pullPolicy": {
+                    "type": "string",
+                    "default": "IfNotPresent"
+                },
+                "repository": {
+                    "type": "string"
+                },
+                "tag": {
+                    "type": "string"
+                }
+            }
+        },
+        "imagePullSecrets": {
+            "type": "array"
+        },
+        "nodeSelector": {
+            "type": "object"
+        },
+        "priorityClassName": {
+            "type": "string"
+        },
+        "resources": {
+            "type": "object",
+            "properties": {
+                "limits": {
+                    "type": "object",
+                    "properties": {
+                        "cpu": {
+                            "type": "string"
+                        },
+                        "memory": {
+                            "type": "string"
+                        }
+                    }
+                },
+                "requests": {
+                    "type": "object",
+                    "properties": {
+                        "cpu": {
+                            "type": "string"
+                        },
+                        "memory": {
+                            "type": "string"
+                        }
+                    }
+                }
+            }
+        },
+        "securityContext": {
+            "type": "object",
+            "properties": {
+                "runAsUser": {
+                    "type": "integer"
+                }
+            }
+        },
+        "service": {
+            "type": "object",
+            "properties": {
+                "annotations": {
+                    "type": "object"
+                },
+                "port": {
+                    "type": "integer"
+                },
+                "type": {
+                    "type": "string"
+                }
+            }
+        },
+        "tolerations": {
+            "type": "array"
+        }
+    }
+}

charts/capi-runtime-extensions/values.yaml

@@ -1,9 +1,12 @@
 # Copyright 2023 D2iQ, Inc. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
+addons:
+  provider: ClusterResourceSet
+
 image:
   repository: ghcr.io/d2iq-labs/capi-runtime-extensions
-  tag:
+  tag: ""
   pullPolicy: IfNotPresent
 
 # -- Optional secrets used for pulling the container image
@@ -13,7 +16,7 @@ imagePullSecrets: []
 certificate:
   issuer:
     selfSigned: true
-    name:
+    name: ""
     kind: Issuer
 
 env: {}

cmd/capi-runtime-extensions/flags.go

@@ -0,0 +1,47 @@
+// Copyright 2023 D2iQ, Inc. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+	"fmt"
+
+	"github.com/spf13/pflag"
+
+	"github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/lifecycle"
+)
+
+type addonProviderValue lifecycle.AddonProvider
+
+func (v addonProviderValue) String() string {
+	return string(v)
+}
+
+func (v *addonProviderValue) Set(value string) error {
+	switch lifecycle.AddonProvider(value) {
+	case lifecycle.ClusterResourceSetAddonProvider, lifecycle.FluxHelmReleaseAddonProvider:
+		break
+	default:
+		return fmt.Errorf(
+			"invalid addon provider: %q (must be one of %v)",
+			value,
+			[]string{
+				string(lifecycle.ClusterResourceSetAddonProvider),
+				string(lifecycle.FluxHelmReleaseAddonProvider),
+			},
+		)
+	}
+
+	*v = addonProviderValue(value)
+
+	return nil
+}
+
+func (*addonProviderValue) Type() string {
+	return "addonProvider"
+}
+
+func newAddonProviderValue(val lifecycle.AddonProvider, p *lifecycle.AddonProvider) pflag.Value {
+	*p = val
+	return (*addonProviderValue)(p)
+}

cmd/capi-runtime-extensions/main.go

@@ -5,6 +5,7 @@ package main
 
 import (
 	"flag"
+	"fmt"
 	"net/http"
 	"os"
 	"time"
@@ -31,6 +32,7 @@ var (
 	profilerAddress string
 	webhookPort     int
 	webhookCertDir  string
+	addonProvider   lifecycle.AddonProvider
 	logOptions      = logs.NewOptions()
 )
 
@@ -49,6 +51,18 @@ func InitFlags(fs *pflag.FlagSet) {
 
 	fs.StringVar(&webhookCertDir, "webhook-cert-dir", "/tmp/k8s-webhook-server/serving-certs/",
 		"Webhook cert dir, only used when webhook-port is specified.")
+
+	fs.Var(newAddonProviderValue(
+		lifecycle.ClusterResourceSetAddonProvider, &addonProvider),
+		"addon-provider",
+		fmt.Sprintf(
+			"addon provider (one of %v)",
+			[]string{
+				string(lifecycle.ClusterResourceSetAddonProvider),
+				string(lifecycle.FluxHelmReleaseAddonProvider),
+			},
+		),
+	)
 }
 
 func main() {
@@ -114,7 +128,7 @@ func main() {
 	}
 
 	// Create the ExtensionHandlers for the lifecycle hooks
-	lifecycleExtensionHandlers := lifecycle.NewExtensionHandlers(client)
+	lifecycleExtensionHandlers := lifecycle.NewExtensionHandlers(addonProvider, client)
 
 	// Register extension handlers.
 	if err := webhookServer.AddExtensionHandler(server.ExtensionHandler{

hack/addons/update-calico-manifests.sh

@@ -33,4 +33,4 @@ curl -fsSL "https://docs.projectcalico.org/archive/${CALICO_VERSION}/manifests/t
 
 kubectl create configmap tigera-operator --dry-run=client --output yaml \
   --from-file "${CALICO_CNI_ASSETS_DIR}/tigera-operator.json" \
-  >"${GIT_REPO_ROOT}/pkg/addons/templates/cni/tigera-operator-configmap.yaml"
+  >"${GIT_REPO_ROOT}/pkg/addons/clusterresourcesets/templates/cni/tigera-operator-configmap.yaml"

pkg/addons/cni.go renamed to pkg/addons/clusterresourcesets/cni.go

@@ -1,7 +1,7 @@
 // Copyright 2023 D2iQ, Inc. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-package addons
+package clusterresourcesets
 
 import (
 	_ "embed" // embedding as []byte does not import the package.

pkg/addons/crs.go renamed to pkg/addons/clusterresourcesets/crs.go

@@ -1,7 +1,7 @@
 // Copyright 2023 D2iQ, Inc. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-package addons
+package clusterresourcesets
 
 import (
 	"fmt"

pkg/addons/templates/cni/calico-cni-installation-crs.yaml renamed to pkg/addons/clusterresourcesets/templates/cni/calico-cni-installation-crs.yaml

@@ -1,3 +1,6 @@
+# Copyright 2023 D2iQ, Inc. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
 apiVersion: v1
 data:
   tigera-operator.json: |

pkg/addons/templates/cni/docker-calico-cni-installation-configmap.yaml renamed to pkg/addons/clusterresourcesets/templates/cni/docker-calico-cni-installation-configmap.yaml

@@ -5,24 +5,38 @@ package lifecycle
 
 import (
 	"context"
+	"fmt"
 
+	"sigs.k8s.io/cluster-api/api/v1beta1"
 	runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
 	ctrl "sigs.k8s.io/controller-runtime"
 	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
 
-	"github.com/d2iq-labs/capi-runtime-extensions/pkg/addons"
+	"github.com/d2iq-labs/capi-runtime-extensions/pkg/addons/clusterresourcesets"
 	k8sclient "github.com/d2iq-labs/capi-runtime-extensions/pkg/k8s/client"
 )
 
+type AddonProvider string
+
+const (
+	ClusterResourceSetAddonProvider AddonProvider = "ClusterResourceSet"
+	FluxHelmReleaseAddonProvider    AddonProvider = "FluxHelmRelease"
+)
+
 // ExtensionHandlers provides a common struct shared across the lifecycle hook handlers.
 type ExtensionHandlers struct {
-	client ctrlclient.Client
+	addonProvider AddonProvider
+	client        ctrlclient.Client
 }
 
 // NewExtensionHandlers returns a ExtensionHandlers for the lifecycle hooks handlers.
-func NewExtensionHandlers(client ctrlclient.Client) *ExtensionHandlers {
+func NewExtensionHandlers(
+	addonProvider AddonProvider,
+	client ctrlclient.Client,
+) *ExtensionHandlers {
 	return &ExtensionHandlers{
-		client: client,
+		addonProvider: addonProvider,
+		client:        client,
 	}
 }
 
@@ -45,18 +59,18 @@ func (m *ExtensionHandlers) DoAfterControlPlaneInitialized(
 
 	genericResourcesClient := k8sclient.NewGenericResourcesClient(m.client, log)
 
-	// Create CNI ClusterResourceSet and let the CAPI controller reconcile it
-	objs, err := addons.CNIForCluster(&request.Cluster)
-	if err != nil {
-		response.Status = runtimehooksv1.ResponseStatusFailure
-		response.Message = err.Error()
-		return
+	var err error
+	switch m.addonProvider {
+	case ClusterResourceSetAddonProvider:
+		err = applyCNICRS(ctx, &request.Cluster, genericResourcesClient)
+	case FluxHelmReleaseAddonProvider:
+		// TODO Apply flux helm releases
+	default:
+		err = fmt.Errorf("unsupported provider: %q", m.addonProvider)
 	}
-	err = genericResourcesClient.Apply(ctx, objs)
 	if err != nil {
 		response.Status = runtimehooksv1.ResponseStatusFailure
 		response.Message = err.Error()
-		return
 	}
 }
 
@@ -77,3 +91,16 @@ func (m *ExtensionHandlers) DoBeforeClusterDelete(
 	log := ctrl.LoggerFrom(ctx)
 	log.Info("BeforeClusterDelete is called")
 }
+
+func applyCNICRS(
+	ctx context.Context,
+	cluster *v1beta1.Cluster,
+	genericResourcesClient *k8sclient.GenericResourcesClient,
+) error {
+	// Create CNI ClusterResourceSet and let the CAPI controller reconcile it.
+	objs, err := clusterresourcesets.CNIForCluster(cluster)
+	if err != nil {
+		return err
+	}
+	return genericResourcesClient.Apply(ctx, objs)
+}