feat: Deploy calico CNI via CRS

.gitignore

@@ -19,7 +19,7 @@ openapi_violations.report
 .idea/
 
 #ignore report files generated by unittest
-*/pkg/**/junit_node*.xml
+**/pkg/**/junit_node*.xml
 
 dist/
 .local/

.go-tools

@@ -1,2 +1,2 @@
 github.com/oligot/[email protected]
-sigs.k8s.io/controller-runtime/tools/[email protected]20230611165747-7edfc04cacbd
+sigs.k8s.io/controller-runtime/tools/[email protected]20230817155522-304027bcbe4b

.golangci.yml

@@ -78,3 +78,6 @@ issues:
     - source: "flags.Parse|response.WriteError"
       linters:
         - errcheck
+    - source: "^// \\+kubebuilder:"
+      linters:
+        - lll

.goreleaser.yml

@@ -23,6 +23,9 @@ release:
     ### Summary
     **Full Changelog**: https://github.com/d2iq-labs/{{.ProjectName}}/compare/{{ .PreviousTag }}...{{ .Tag }}
 
+gomod:
+  proxy: true
+
 builds:
   - id: capi-runtime-extensions
     dir: ./cmd/capi-runtime-extensions

.pre-commit-config.yaml

@@ -31,6 +31,7 @@ repos:
     stages: [commit]
   - id: check-added-large-files
     stages: [commit]
+    exclude: ^pkg/handlers/cni/calico/manifests/tigera-operator-configmap.yaml$
   - id: check-case-conflict
     stages: [commit]
   - id: check-merge-conflict

README.md

@@ -23,16 +23,18 @@ example), run:
 make SKIP_BUILD=true dev.run-on-kind
 ```
 
-To create a cluster with [clusterctl](https://cluster-api.sigs.k8s.io/user/quick-start.html), run:
+To create a cluster with [clusterctl](https://cluster-api.sigs.k8s.io/user/quick-start.html), and label it for Calico
+CNI at the same time, run:
 
 ```shell
-env POD_SECURITY_STANDARD_ENABLED=false \
-  clusterctl generate cluster capi-quickstart \
-    --flavor development \
-    --kubernetes-version v1.27.2 \
-    --control-plane-machine-count=1 \
-    --worker-machine-count=1 | \
-  kubectl apply --server-side -f -
+clusterctl generate cluster capi-quickstart \
+  --flavor development \
+  --kubernetes-version v1.27.2 \
+  --control-plane-machine-count=1 \
+  --worker-machine-count=1 | \
+gojq --yaml-input --yaml-output \
+  '. | (select(.kind=="Cluster").metadata.labels["capiext.labs.d2iq.io/cni"]|="calico")' | \
+kubectl apply --server-side -f -
 ```
 
 Wait until control plane is ready:
@@ -55,19 +57,6 @@ kubectl config set-cluster capi-quickstart \
   --server=https://$(docker port capi-quickstart-lb 6443/tcp)
 ```
 
-Deploy Calico to the workload cluster (TODO deploy via lifecycle hook):
-
-```shell
-helm repo add --force-update projectcalico https://docs.tigera.io/calico/charts
-helm upgrade --install calico projectcalico/tigera-operator \
-  --version v3.26.1 \
-  --namespace tigera-operator \
-  --create-namespace \
-  --wait \
-  --wait-for-jobs \
-  --kubeconfig capd-kubeconfig
-```
-
 Wait until all nodes are ready (this indicates that CNI has been deployed successfully):
 
 ```shell

charts/capi-runtime-extensions/README.md

@@ -31,6 +31,13 @@ A Helm chart for capi-runtime-extensions
 | controllers.enableLeaderElection | bool | `false` |  |
 | deployment.replicas | int | `1` |  |
 | env | object | `{}` |  |
+| handlers.CalicoCNI.defaultInstallationConfigMaps.DockerCluster.configMap.content | string | `""` |  |
+| handlers.CalicoCNI.defaultInstallationConfigMaps.DockerCluster.configMap.name | string | `"calico-cni-installation-dockercluster"` |  |
+| handlers.CalicoCNI.defaultInstallationConfigMaps.DockerCluster.create | bool | `true` |  |
+| handlers.CalicoCNI.defaultPodSubnet | string | `"192.168.0.0/16"` |  |
+| handlers.CalicoCNI.defaultTigeraOperatorConfigMap.name | string | `"tigera-operator"` |  |
+| handlers.CalicoCNI.enabled | bool | `true` |  |
+| handlers.ServiceLoadBalancerGC.enabled | bool | `true` |  |
 | image.pullPolicy | string | `"IfNotPresent"` |  |
 | image.repository | string | `"ghcr.io/d2iq-labs/capi-runtime-extensions"` |  |
 | image.tag | string | `""` |  |

charts/capi-runtime-extensions/templates/cni/calico/manifests/docker/installation.yaml

@@ -0,0 +1,35 @@
+# Copyright 2023 D2iQ, Inc. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+{{- if and .Values.handlers.CalicoCNI.enabled .Values.handlers.CalicoCNI.defaultInstallationConfigMaps.DockerCluster.create }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: '{{ .Values.handlers.CalicoCNI.defaultInstallationConfigMaps.DockerCluster.configMap.name }}'
+data:
+  calico-installation: |
+{{- if .Values.handlers.CalicoCNI.defaultInstallationConfigMaps.DockerCluster.configMap.content -}}
+    {{ .Values.handlers.CalicoCNI.defaultInstallationConfigMaps.DockerCluster.configMap.content | nindent 4}}
+{{- else -}}
+    # This section includes base Calico installation configuration.
+    # For more information, see: https://docs.projectcalico.org/reference/installation/api
+    apiVersion: operator.tigera.io/v1
+    kind: Installation
+    metadata:
+      name: default
+    spec:
+      cni:
+        type: Calico
+      # Configures Calico networking.
+      calicoNetwork:
+        # Note: The ipPools section cannot be modified post-install.
+        ipPools:
+        - blockSize: 26
+          cidr: {{ .Values.handlers.CalicoCNI.defaultPodSubnet }}
+          encapsulation: VXLANCrossSubnet
+          natOutgoing: Enabled
+          nodeSelector: all()
+      nodeMetricsPort: 9091
+      typhaMetricsPort: 9093
+{{- end -}}
+{{- end -}}

charts/capi-runtime-extensions/templates/deployment.yaml

@@ -27,20 +27,28 @@ spec:
       - name: webhook
         image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default $.Chart.AppVersion }}"
         imagePullPolicy: "{{ .Values.image.pullPolicy }}"
-{{- if .Values.env }}
-        env:
-{{- range $key, $value := .Values.env }}
-        - name: "{{ $key }}"
-          value: "{{ $value }}"
-{{- end }}
-{{- end }}
         args:
         - --controllermanager.cert-dir=/controllers-certs/
         - --controllermanager.leader-elect={{ if gt (.Values.deployment.replicas | int) 1 }}true{{ else }}{{ .Values.controllers.enableLeaderElection }}{{ end }}
         - --runtimehooks.cert-dir=/runtimehooks-certs/
+        {{- range $key, $value := .Values.handlers }}{{ if $value.enabled }}
+        - --runtimehooks.enabled-handlers={{ $key }}
+        {{ end }}{{- end }}
+        - --runtimehooks.calicocni.defaultsNamespace=$(POD_NAMESPACE)
         {{- range $key, $value := .Values.extraArgs }}
         - --{{ $key }}={{ $value }}
         {{- end }}
+        env:
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+{{- if .Values.env }}
+{{- range $key, $value := .Values.env }}
+        - name: "{{ $key }}"
+          value: "{{ $value }}"
+{{- end }}
+{{- end }}
         ports:
         - containerPort: 8443
           name: controllers

charts/capi-runtime-extensions/templates/role.yaml

@@ -10,8 +10,6 @@ rules:
   - ""
   resources:
   - configmaps
-  - namespaces
-  - secrets
   verbs:
   - create
   - delete
@@ -21,10 +19,14 @@ rules:
   - update
   - watch
 - apiGroups:
-  - cluster.x-k8s.io
+  - addons.cluster.x-k8s.io
   resources:
-  - clusters
+  - clusterresourcesets
   verbs:
+  - create
+  - delete
   - get
   - list
+  - patch
+  - update
   - watch

charts/capi-runtime-extensions/values.schema.json

@@ -45,6 +45,68 @@
       "type": "object",
       "additionalProperties": { "type": "string" }
     },
+    "handlers": {
+      "type": "object",
+      "properties": {
+        "CalicoCNI": {
+          "type": "object",
+          "properties": {
+            "defaultInstallationConfigMaps": {
+              "type": "object",
+              "properties": {
+                "DockerCluster": {
+                  "type": "object",
+                  "properties": {
+                    "configMap": {
+                      "type": "object",
+                      "properties": {
+                        "content": {
+                          "type": "string"
+                        },
+                        "name": {
+                          "type": "string"
+                        }
+                      },
+                      "required": ["name"]
+                    },
+                    "create": {
+                      "type": "boolean",
+                      "default": true
+                    }
+                  }
+                }
+              }
+            },
+            "defaultPodSubnet": {
+              "type": "string",
+              "default": "192.168.0.0/16"
+            },
+            "defaultTigeraOperatorConfigMap": {
+              "type": "object",
+              "properties": {
+                "name": {
+                  "type": "string"
+                }
+              },
+              "required": ["name"]
+            },
+            "enabled": {
+              "type": "boolean",
+              "default": true
+            }
+          }
+        },
+        "ServiceLoadBalancerGC": {
+          "type": "object",
+          "properties": {
+            "enabled": {
+              "type": "boolean",
+              "default": true
+            }
+          }
+        }
+      }
+    },
     "image": {
       "type": "object",
       "properties": {

charts/capi-runtime-extensions/values.yaml

@@ -1,6 +1,21 @@
 # Copyright 2023 D2iQ, Inc. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
+handlers:
+  CalicoCNI:
+    enabled: true
+    defaultPodSubnet: 192.168.0.0/16
+    defaultTigeraOperatorConfigMap:
+      name: tigera-operator
+    defaultInstallationConfigMaps:
+      DockerCluster:
+        create: true
+        configMap:
+          name: calico-cni-installation-dockercluster
+          content: ""
+  ServiceLoadBalancerGC:
+    enabled: true
+
 deployment:
   replicas: 1
 

devbox.lock

@@ -14,16 +14,16 @@
       "version": "3.8.0"
     },
     "clusterctl@latest": {
-      "last_modified": "2023-07-23T03:35:12Z",
-      "resolved": "github:NixOS/nixpkgs/af8cd5ded7735ca1df1a1174864daab75feeb64a#clusterctl",
+      "last_modified": "2023-08-08T03:07:33Z",
+      "resolved": "github:NixOS/nixpkgs/844ffa82bbe2a2779c86ab3a72ff1b4176cec467#clusterctl",
       "source": "devbox-search",
-      "version": "1.4.4"
+      "version": "1.5.0"
     },
     "crane@latest": {
-      "last_modified": "2023-06-30T04:44:22Z",
-      "resolved": "github:NixOS/nixpkgs/3c614fbc76fc152f3e1bc4b2263da6d90adf80fb#crane",
+      "last_modified": "2023-08-08T03:07:33Z",
+      "resolved": "github:NixOS/nixpkgs/844ffa82bbe2a2779c86ab3a72ff1b4176cec467#crane",
       "source": "devbox-search",
-      "version": "0.15.2"
+      "version": "0.16.1"
     },
     "envsubst@latest": {
       "last_modified": "2023-06-30T04:44:22Z",
@@ -140,10 +140,10 @@
       "version": "3.11.1"
     },
     "kubectl@latest": {
-      "last_modified": "2023-06-30T04:44:22Z",
-      "resolved": "github:NixOS/nixpkgs/3c614fbc76fc152f3e1bc4b2263da6d90adf80fb#kubectl",
+      "last_modified": "2023-08-08T03:07:33Z",
+      "resolved": "github:NixOS/nixpkgs/844ffa82bbe2a2779c86ab3a72ff1b4176cec467#kubectl",
       "source": "devbox-search",
-      "version": "1.27.3"
+      "version": "1.27.4"
     },
     "kubernetes-controller-tools@latest": {
       "last_modified": "2023-07-23T03:35:12Z",
@@ -170,8 +170,8 @@
       "version": "3.3.3"
     },
     "shfmt@latest": {
-      "last_modified": "2023-06-30T04:44:22Z",
-      "resolved": "github:NixOS/nixpkgs/3c614fbc76fc152f3e1bc4b2263da6d90adf80fb#shfmt",
+      "last_modified": "2023-08-08T03:07:33Z",
+      "resolved": "github:NixOS/nixpkgs/844ffa82bbe2a2779c86ab3a72ff1b4176cec467#shfmt",
       "source": "devbox-search",
       "version": "3.7.0"
     },

go.mod

@@ -15,6 +15,7 @@ require (
 	k8s.io/client-go v0.28.0
 	k8s.io/component-base v0.28.0
 	k8s.io/klog/v2 v2.100.1
+	k8s.io/utils v0.0.0-20230406110748-d93618cff8a2
 	sigs.k8s.io/cluster-api v1.5.0
 	sigs.k8s.io/controller-runtime v0.15.1
 )
@@ -72,7 +73,6 @@ require (
 	k8s.io/apiextensions-apiserver v0.27.2 // indirect
 	k8s.io/cluster-bootstrap v0.27.2 // indirect
 	k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect
-	k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect

hack/addons/kustomize/tigera-operator/ds-priorityClass.yaml

@@ -0,0 +1,11 @@
+# Copyright 2023 D2iQ, Inc. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: NA
+spec:
+  template:
+    spec:
+      priorityClassName: system-cluster-critical

hack/addons/kustomize/tigera-operator/kustomization.yaml

@@ -0,0 +1,17 @@
+# Copyright 2023 D2iQ, Inc. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+metadata:
+  name: tigera-operator
+sortOptions:
+  order: fifo
+patches:
+- path: ds-priorityClass.yaml
+  target:
+    kind: Deployment
+    name: tigera-operator
+    namespace: tigera-operator
+resources:
+- tigera-operator.yaml