feat: Deploy calico CNI via CRS

Author: jimmidyson

.golangci.yml

@@ -78,3 +78,6 @@ issues:
     - source: "flags.Parse|response.WriteError"
       linters:
         - errcheck
+    - source: "^// \\+kubebuilder:"
+      linters:
+        - lll

.goreleaser.yml

@@ -23,6 +23,9 @@ release:
     ### Summary
     **Full Changelog**: https://github.com/d2iq-labs/{{.ProjectName}}/compare/{{ .PreviousTag }}...{{ .Tag }}
 
+gomod:
+  proxy: true
+
 builds:
   - id: capi-runtime-extensions
     dir: ./cmd/capi-runtime-extensions

.pre-commit-config.yaml

@@ -31,6 +31,7 @@ repos:
     stages: [commit]
   - id: check-added-large-files
     stages: [commit]
+    exclude: ^pkg/handlers/cni/calico/manifests/tigera-operator-configmap.yaml$
   - id: check-case-conflict
     stages: [commit]
   - id: check-merge-conflict

README.md

@@ -23,7 +23,8 @@ example), run:
 make SKIP_BUILD=true dev.run-on-kind
 ```
 
-To create a cluster with [clusterctl](https://cluster-api.sigs.k8s.io/user/quick-start.html), run:
+To create a cluster with [clusterctl](https://cluster-api.sigs.k8s.io/user/quick-start.html), and label it for Calico
+CNI at the same time, run:
 
 ```shell
 env POD_SECURITY_STANDARD_ENABLED=false \
@@ -32,6 +33,7 @@ env POD_SECURITY_STANDARD_ENABLED=false \
     --kubernetes-version v1.27.2 \
     --control-plane-machine-count=1 \
     --worker-machine-count=1 | \
+  kubectl label -f - --local --dry-run=client -oyaml capiext.labs.d2iq.io/cni=calico | \
   kubectl apply --server-side -f -
 ```
 
@@ -55,19 +57,6 @@ kubectl config set-cluster capi-quickstart \
   --server=https://$(docker port capi-quickstart-lb 6443/tcp)
 ```
 
-Deploy Calico to the workload cluster (TODO deploy via lifecycle hook):
-
-```shell
-helm repo add --force-update projectcalico https://docs.tigera.io/calico/charts
-helm upgrade --install calico projectcalico/tigera-operator \
-  --version v3.26.1 \
-  --namespace tigera-operator \
-  --create-namespace \
-  --wait \
-  --wait-for-jobs \
-  --kubeconfig capd-kubeconfig
-```
-
 Wait until all nodes are ready (this indicates that CNI has been deployed successfully):
 
 ```shell

charts/capi-runtime-extensions/templates/role.yaml

@@ -10,8 +10,6 @@ rules:
   - ""
   resources:
   - configmaps
-  - namespaces
-  - secrets
   verbs:
   - create
   - delete
@@ -21,10 +19,14 @@ rules:
   - update
   - watch
 - apiGroups:
-  - cluster.x-k8s.io
+  - addons.cluster.x-k8s.io
   resources:
-  - clusters
+  - clusterresourcesets
   verbs:
+  - create
+  - delete
   - get
   - list
+  - patch
+  - update
   - watch

hack/addons/kustomize/tigera-operator/ds-priorityClass.yaml

@@ -0,0 +1,11 @@
+# Copyright 2023 D2iQ, Inc. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: NA
+spec:
+  template:
+    spec:
+      priorityClassName: system-cluster-critical

hack/addons/kustomize/tigera-operator/kustomization.yaml

@@ -0,0 +1,17 @@
+# Copyright 2023 D2iQ, Inc. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+metadata:
+  name: tigera-operator
+sortOptions:
+  order: fifo
+patches:
+- path: ds-priorityClass.yaml
+  target:
+    kind: Deployment
+    name: tigera-operator
+    namespace: tigera-operator
+resources:
+- tigera-operator.yaml

hack/addons/update-calico-manifests.sh

@@ -1,5 +1,4 @@
 #!/usr/bin/env bash
-
 set -euo pipefail
 IFS=$'\n\t'
 
@@ -9,7 +8,7 @@ readonly SCRIPT_DIR
 # shellcheck source=hack/common.sh
 source "${SCRIPT_DIR}/../common.sh"
 
-if [ -z "${CALICO_VERSION-}" ]; then
+if [ -z "${CALICO_VERSION:-}" ]; then
   echo "Missing environment variable: CALICO_VERSION"
   exit 1
 fi
@@ -18,19 +17,32 @@ CALICO_CNI_ASSETS_DIR="$(mktemp -d -p "${TMPDIR:-/tmp}")"
 readonly CALICO_CNI_ASSETS_DIR
 trap 'rm -rf ${CALICO_CNI_ASSETS_DIR}' EXIT
 
-# The operator manifest in YAML format is 1226666 bytes. It turns out that much of that is whitespace. Converting the
-# manifest to JSON without indentation allows us to remove most of the whitespace, reducing the size by more than half,
-# to 527614 bytes.
+curl -fsSL "https://raw.githubusercontent.com/projectcalico/calico/${CALICO_VERSION}/manifests/tigera-operator.yaml" \
+  -o "${CALICO_CNI_ASSETS_DIR}/tigera-operator.yaml"
+
+readonly KUSTOMIZATION_DIR=${SCRIPT_DIR}/kustomize/tigera-operator
+cp -r "${KUSTOMIZATION_DIR}"/* "${CALICO_CNI_ASSETS_DIR}"
+kustomize --load-restrictor=LoadRestrictionsNone build "${CALICO_CNI_ASSETS_DIR}" -o "${CALICO_CNI_ASSETS_DIR}/kustomized.yaml"
+
+# The operator manifest in YAML format is pretty big. It turns out that much of that is whitespace. Converting the
+# manifest to JSON without indentation allows us to remove most of the whitespace, reducing the size by more than half.
 #
 # Some important notes:
 # 1. The YAML manifest includes many documents, and the documents must become elements in a JSON array in order for the ClusterResourceController to [parse them](https://github.com/mesosphere/cluster-api//blob/65586de0080a960d085031de87ec627b2d606a6b/exp/addons/internal/controllers/clusterresourceset_helpers.go#L59). We create a JSON array with the --slurp flag.
 # 2. The YAML manifest has some whitespace between YAML document markers (`---`), and these become `null` entries in the JSON array. This causes the ["SortForCreate" subroutine](https://github.com/mesosphere/cluster-api//blob/65586de0080a960d085031de87ec627b2d606a6b/exp/addons/internal/controllers/clusterresourceset_helpers.go#L84) of the ClusterResourceSet controller to misbehave. We remove these null entries using a filter expression.
 # 3. If we indent the JSON document, it is nearly as large as the YAML document, at 1099093 bytes. We remove indentation with the --indent=0 flag.
-curl -fsSL "https://docs.projectcalico.org/archive/${CALICO_VERSION}/manifests/tigera-operator.yaml" |
-  gojq --yaml-input --slurp --indent=0 \
-    '[ .[] | select( . != null ) | (select( .kind=="Namespace").metadata.labels += {"pod-security.kubernetes.io/enforce": "privileged", "pod-security.kubernetes.io/enforce-version": "latest"}) ]' \
-    >"${CALICO_CNI_ASSETS_DIR}/tigera-operator.json"
+gojq --yaml-input \
+  --slurp \
+  --indent=0 \
+  '[ .[] | select( . != null ) |
+    (select( .kind=="Namespace").metadata.labels += {
+      "pod-security.kubernetes.io/enforce": "privileged",
+      "pod-security.kubernetes.io/enforce-version": "latest"
+    })
+  ]' \
+  <"${CALICO_CNI_ASSETS_DIR}/kustomized.yaml" \
+  >"${CALICO_CNI_ASSETS_DIR}/tigera-operator.json"
 
 kubectl create configmap tigera-operator --dry-run=client --output yaml \
   --from-file "${CALICO_CNI_ASSETS_DIR}/tigera-operator.json" \
-  >"${GIT_REPO_ROOT}/pkg/addons/clusterresourcesets/templates/cni/tigera-operator-configmap.yaml"
+  >"${GIT_REPO_ROOT}/pkg/handlers/cni/calico/manifests/tigera-operator-configmap.yaml"

internal/controllermanager/controllermanager.go

@@ -17,9 +17,6 @@ import (
 	capiextv1alpha1 "github.com/d2iq-labs/capi-runtime-extensions/api/v1alpha1"
 )
 
-//+kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters,verbs=get;list;watch
-//+kubebuilder:rbac:groups="",resources=namespaces;configmaps;secrets,verbs=watch;list;get;create;patch;update;delete
-
 type Manager struct {
 	port                 uint16
 	webhookCertDir       string

internal/runtimehooks/webhooks/server.go

@@ -7,13 +7,19 @@ import (
 	"context"
 
 	"github.com/spf13/pflag"
+	"k8s.io/apimachinery/pkg/runtime"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	capiv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	crsv1 "sigs.k8s.io/cluster-api/exp/addons/api/v1beta1"
 	runtimecatalog "sigs.k8s.io/cluster-api/exp/runtime/catalog"
 	runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
 	"sigs.k8s.io/cluster-api/exp/runtime/server"
 	ctrl "sigs.k8s.io/controller-runtime"
 	ctrclient "sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers"
+	"github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/cni/calico"
 	"github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/servicelbgc"
 )
 
@@ -72,13 +78,21 @@ func (s *Server) Start(ctx context.Context) error {
 		return err
 	}
 
-	client, err := ctrclient.New(restConfig, ctrclient.Options{})
+	scheme := runtime.NewScheme()
+	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
+	utilruntime.Must(crsv1.AddToScheme(scheme))
+	utilruntime.Must(capiv1.AddToScheme(scheme))
+
+	client, err := ctrclient.New(restConfig, ctrclient.Options{Scheme: scheme})
 	if err != nil {
 		setupLog.Error(err, "error creating client to the cluster")
 		return err
 	}
 
-	allHandlers := []handlers.NamedHandler{servicelbgc.New(client)}
+	allHandlers := []handlers.NamedHandler{
+		servicelbgc.New(client),
+		calico.New(client),
+	}
 
 	for idx := range allHandlers {
 		h := allHandlers[idx]

make/addons.mk

@@ -1,7 +1,7 @@
 # Copyright 2023 D2iQ, Inc. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-export CALICO_VERSION := v3.25
+export CALICO_VERSION := v3.26.1
 
 .PHONY: update-addon.calico
 update-addon.calico: ; $(info $(M) updating calico manifests)

pkg/handlers/cni/calico/doc.go

@@ -0,0 +1,12 @@
+// Copyright 2023 D2iQ, Inc. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package calico provides a handler for managing Calico deployments on clusters, configurable via
+// labels and annotations.
+//
+// To enable Calico deployment, a cluster must be labelled with `capiext.labs.d2iq.io/cni=calico`.
+// This will ensure the Tigera Configmap and associated ClusterResourceSet.
+//
+// +kubebuilder:rbac:groups=addons.cluster.x-k8s.io,resources=clusterresourcesets,verbs=watch;list;get;create;patch;update;delete
+// +kubebuilder:rbac:groups="",resources=configmaps,verbs=watch;list;get;create;patch;update;delete
+package calico