From 0b54c5b1413018c0ed104673b1f0b3c485b1978b Mon Sep 17 00:00:00 2001 From: Jimmi Dyson Date: Thu, 11 Jan 2024 11:47:42 +0000 Subject: [PATCH] feat: Add ClusterResourceSet strategy for CNI installation Allows for further strategies to be added later, e.g. CAAPH. --- api/v1alpha1/addon_types.go | 139 ++++++++ api/v1alpha1/clusterconfig_types.go | 114 ------ charts/capi-runtime-extensions/README.md | 14 +- .../calico/manifests/aws/installation.yaml | 8 +- .../calico/manifests/docker/installation.yaml | 8 +- .../manifests/tigera-operator-configmap.yaml | 3 +- .../templates/deployment.yaml | 2 +- charts/capi-runtime-extensions/values.yaml | 27 +- examples/capi-quick-start/aws-cluster.yaml | 1 + examples/capi-quick-start/docker-cluster.yaml | 1 + hack/addons/update-calico-manifests.sh | 2 +- .../bases/aws/kustomization.yaml.tmpl | 1 + .../bases/docker/kustomization.yaml.tmpl | 1 + .../generic/lifecycle/cni/calico/handler.go | 313 +---------------- .../lifecycle/cni/calico/strategy_crs.go | 329 ++++++++++++++++++ .../generic/lifecycle/cni/variables_test.go | 14 + 16 files changed, 537 insertions(+), 440 deletions(-) create mode 100644 api/v1alpha1/addon_types.go create mode 100644 pkg/handlers/generic/lifecycle/cni/calico/strategy_crs.go diff --git a/api/v1alpha1/addon_types.go b/api/v1alpha1/addon_types.go new file mode 100644 index 000000000..4a9ecc155 --- /dev/null +++ b/api/v1alpha1/addon_types.go @@ -0,0 +1,139 @@ +// Copyright 2024 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1 + +import ( + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + + "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/capi/clustertopology/variables" +) + +type Addons struct { + // +optional + CNI *CNI `json:"cni,omitempty"` + + // +optional + NFD *NFD `json:"nfd,omitempty"` + + // +optional + CPI *CPI `json:"cpi,omitempty"` + + // +optional + CSIProviders *CSIProviders `json:"csi,omitempty"` +} + +func (Addons) VariableSchema() clusterv1.VariableSchema { + return clusterv1.VariableSchema{ + OpenAPIV3Schema: clusterv1.JSONSchemaProps{ + Description: "Cluster configuration", + Type: "object", + Properties: map[string]clusterv1.JSONSchemaProps{ + "cni": CNI{}.VariableSchema().OpenAPIV3Schema, + "nfd": NFD{}.VariableSchema().OpenAPIV3Schema, + "csi": CSIProviders{}.VariableSchema().OpenAPIV3Schema, + "cpi": CPI{}.VariableSchema().OpenAPIV3Schema, + }, + }, + } +} + +type AddonStrategy string + +const ( + AddonStrategyClusterResourceSet AddonStrategy = "ClusterResourceSet" +) + +// CNI required for providing CNI configuration. +type CNI struct { + // +optional + Provider string `json:"provider,omitempty"` + // +optional + Strategy AddonStrategy `json:"strategy,omitempty"` +} + +func (CNI) VariableSchema() clusterv1.VariableSchema { + supportedCNIProviders := []string{CNIProviderCalico} + + return clusterv1.VariableSchema{ + OpenAPIV3Schema: clusterv1.JSONSchemaProps{ + Type: "object", + Properties: map[string]clusterv1.JSONSchemaProps{ + "provider": { + Description: "CNI provider to deploy", + Type: "string", + Enum: variables.MustMarshalValuesToEnumJSON(supportedCNIProviders...), + }, + "strategy": { + Description: "Addon strategy used to deploy the CNI provider to the workload cluster", + Type: "string", + Enum: variables.MustMarshalValuesToEnumJSON( + AddonStrategyClusterResourceSet, + ), + }, + }, + Required: []string{"provider", "strategy"}, + }, + } +} + +// NFD tells us to enable or disable the node feature discovery addon. +type NFD struct{} + +func (NFD) VariableSchema() clusterv1.VariableSchema { + return clusterv1.VariableSchema{ + OpenAPIV3Schema: clusterv1.JSONSchemaProps{ + Type: "object", + }, + } +} + +type CSIProviders struct { + // +optional + Providers []CSIProvider `json:"providers,omitempty"` + // +optional + DefaultClassName string `json:"defaultClassName,omitempty"` +} + +type CSIProvider struct { + Name string `json:"name,omitempty"` +} + +func (CSIProviders) VariableSchema() clusterv1.VariableSchema { + supportedCSIProviders := []string{CSIProviderAWSEBS} + return clusterv1.VariableSchema{ + OpenAPIV3Schema: clusterv1.JSONSchemaProps{ + Type: "object", + Properties: map[string]clusterv1.JSONSchemaProps{ + "providers": { + Type: "array", + Items: &clusterv1.JSONSchemaProps{ + Type: "object", + Properties: map[string]clusterv1.JSONSchemaProps{ + "name": { + Type: "string", + Enum: variables.MustMarshalValuesToEnumJSON( + supportedCSIProviders...), + }, + }, + }, + }, + "defaultClassName": { + Type: "string", + Enum: variables.MustMarshalValuesToEnumJSON(supportedCSIProviders...), + }, + }, + }, + } +} + +// CPI tells us to enable or disable the cloud provider interface. +type CPI struct{} + +func (CPI) VariableSchema() clusterv1.VariableSchema { + return clusterv1.VariableSchema{ + OpenAPIV3Schema: clusterv1.JSONSchemaProps{ + Type: "object", + }, + } +} diff --git a/api/v1alpha1/clusterconfig_types.go b/api/v1alpha1/clusterconfig_types.go index 0640c56f3..c04ba04e1 100644 --- a/api/v1alpha1/clusterconfig_types.go +++ b/api/v1alpha1/clusterconfig_types.go @@ -10,7 +10,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/capi/clustertopology/variables" "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/openapi/patterns" ) @@ -316,119 +315,6 @@ func (ImageRegistryCredentialsResource) VariableSchema() clusterv1.VariableSchem } } -type Addons struct { - // +optional - CNI *CNI `json:"cni,omitempty"` - - // +optional - NFD *NFD `json:"nfd,omitempty"` - - // +optional - CPI *CPI `json:"cpi,omitempty"` - - // +optional - CSIProviders *CSIProviders `json:"csi,omitempty"` -} - -func (Addons) VariableSchema() clusterv1.VariableSchema { - return clusterv1.VariableSchema{ - OpenAPIV3Schema: clusterv1.JSONSchemaProps{ - Description: "Cluster configuration", - Type: "object", - Properties: map[string]clusterv1.JSONSchemaProps{ - "cni": CNI{}.VariableSchema().OpenAPIV3Schema, - "nfd": NFD{}.VariableSchema().OpenAPIV3Schema, - "csi": CSIProviders{}.VariableSchema().OpenAPIV3Schema, - "cpi": CPI{}.VariableSchema().OpenAPIV3Schema, - }, - }, - } -} - -// CNI required for providing CNI configuration. -type CNI struct { - Provider string `json:"provider,omitempty"` -} - -func (CNI) VariableSchema() clusterv1.VariableSchema { - supportedCNIProviders := []string{CNIProviderCalico} - - return clusterv1.VariableSchema{ - OpenAPIV3Schema: clusterv1.JSONSchemaProps{ - Type: "object", - Properties: map[string]clusterv1.JSONSchemaProps{ - "provider": { - Description: "CNI provider to deploy", - Type: "string", - Enum: variables.MustMarshalValuesToEnumJSON(supportedCNIProviders...), - }, - }, - Required: []string{"provider"}, - }, - } -} - -// NFD tells us to enable or disable the node feature discovery addon. -type NFD struct{} - -func (NFD) VariableSchema() clusterv1.VariableSchema { - return clusterv1.VariableSchema{ - OpenAPIV3Schema: clusterv1.JSONSchemaProps{ - Type: "object", - }, - } -} - -type CSIProviders struct { - // +optional - Providers []CSIProvider `json:"providers,omitempty"` - // +optional - DefaultClassName string `json:"defaultClassName,omitempty"` -} - -type CSIProvider struct { - Name string `json:"name,omitempty"` -} - -func (CSIProviders) VariableSchema() clusterv1.VariableSchema { - supportedCSIProviders := []string{CSIProviderAWSEBS} - return clusterv1.VariableSchema{ - OpenAPIV3Schema: clusterv1.JSONSchemaProps{ - Type: "object", - Properties: map[string]clusterv1.JSONSchemaProps{ - "providers": { - Type: "array", - Items: &clusterv1.JSONSchemaProps{ - Type: "object", - Properties: map[string]clusterv1.JSONSchemaProps{ - "name": { - Type: "string", - Enum: variables.MustMarshalValuesToEnumJSON( - supportedCSIProviders...), - }, - }, - }, - }, - "defaultClassName": { - Type: "string", - Enum: variables.MustMarshalValuesToEnumJSON(supportedCSIProviders...), - }, - }, - }, - } -} - -// CPI tells us to enable or disable the cloud provider interface. -type CPI struct{} - -func (CPI) VariableSchema() clusterv1.VariableSchema { - return clusterv1.VariableSchema{ - OpenAPIV3Schema: clusterv1.JSONSchemaProps{ - Type: "object", - }, - } -} - func init() { SchemeBuilder.Register(&ClusterConfig{}) } diff --git a/charts/capi-runtime-extensions/README.md b/charts/capi-runtime-extensions/README.md index 2e25025fe..dfd5a2cb0 100644 --- a/charts/capi-runtime-extensions/README.md +++ b/charts/capi-runtime-extensions/README.md @@ -31,14 +31,14 @@ A Helm chart for capi-runtime-extensions | deployDefaultClusterClasses | bool | `true` | | | deployment.replicas | int | `1` | | | env | object | `{}` | | -| hooks.CalicoCNI.defaultInstallationConfigMaps.AWSCluster.configMap.content | string | `""` | | -| hooks.CalicoCNI.defaultInstallationConfigMaps.AWSCluster.configMap.name | string | `"calico-cni-installation-awscluster"` | | -| hooks.CalicoCNI.defaultInstallationConfigMaps.AWSCluster.create | bool | `true` | | -| hooks.CalicoCNI.defaultInstallationConfigMaps.DockerCluster.configMap.content | string | `""` | | -| hooks.CalicoCNI.defaultInstallationConfigMaps.DockerCluster.configMap.name | string | `"calico-cni-installation-dockercluster"` | | -| hooks.CalicoCNI.defaultInstallationConfigMaps.DockerCluster.create | bool | `true` | | +| hooks.CalicoCNI.crsStrategy.defaultInstallationConfigMaps.AWSCluster.configMap.content | string | `""` | | +| hooks.CalicoCNI.crsStrategy.defaultInstallationConfigMaps.AWSCluster.configMap.name | string | `"calico-cni-installation-awscluster"` | | +| hooks.CalicoCNI.crsStrategy.defaultInstallationConfigMaps.AWSCluster.create | bool | `true` | | +| hooks.CalicoCNI.crsStrategy.defaultInstallationConfigMaps.DockerCluster.configMap.content | string | `""` | | +| hooks.CalicoCNI.crsStrategy.defaultInstallationConfigMaps.DockerCluster.configMap.name | string | `"calico-cni-installation-dockercluster"` | | +| hooks.CalicoCNI.crsStrategy.defaultInstallationConfigMaps.DockerCluster.create | bool | `true` | | +| hooks.CalicoCNI.crsStrategy.defaultTigeraOperatorConfigMap.name | string | `"tigera-operator"` | | | hooks.CalicoCNI.defaultPodSubnet | string | `"192.168.0.0/16"` | | -| hooks.CalicoCNI.defaultTigeraOperatorConfigMap.name | string | `"tigera-operator"` | | | image.pullPolicy | string | `"IfNotPresent"` | | | image.repository | string | `"ghcr.io/d2iq-labs/capi-runtime-extensions"` | | | image.tag | string | `""` | | diff --git a/charts/capi-runtime-extensions/templates/cni/calico/manifests/aws/installation.yaml b/charts/capi-runtime-extensions/templates/cni/calico/manifests/aws/installation.yaml index bb072239a..af6a67b5f 100644 --- a/charts/capi-runtime-extensions/templates/cni/calico/manifests/aws/installation.yaml +++ b/charts/capi-runtime-extensions/templates/cni/calico/manifests/aws/installation.yaml @@ -1,15 +1,15 @@ # Copyright 2023 D2iQ, Inc. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -{{- if .Values.hooks.CalicoCNI.defaultInstallationConfigMaps.AWSCluster.create }} +{{- if .Values.hooks.CalicoCNI.crsStrategy.defaultInstallationConfigMaps.AWSCluster.create }} apiVersion: v1 kind: ConfigMap metadata: - name: '{{ .Values.hooks.CalicoCNI.defaultInstallationConfigMaps.AWSCluster.configMap.name }}' + name: '{{ .Values.hooks.CalicoCNI.crsStrategy.defaultInstallationConfigMaps.AWSCluster.configMap.name }}' data: calico-installation: | -{{- if .Values.hooks.CalicoCNI.defaultInstallationConfigMaps.AWSCluster.configMap.content -}} - {{ .Values.hooks.CalicoCNI.defaultInstallationConfigMaps.AWSCluster.configMap.content | nindent 4}} +{{- if .Values.hooks.CalicoCNI.crsStrategy.defaultInstallationConfigMaps.AWSCluster.configMap.content -}} + {{ .Values.hooks.CalicoCNI.crsStrategy.defaultInstallationConfigMaps.AWSCluster.configMap.content | nindent 4 }} {{- else -}} # This section includes base Calico installation configuration. # For more information, see: https://docs.projectcalico.org/reference/installation/api diff --git a/charts/capi-runtime-extensions/templates/cni/calico/manifests/docker/installation.yaml b/charts/capi-runtime-extensions/templates/cni/calico/manifests/docker/installation.yaml index f7d2a8e2f..e4e562c0e 100644 --- a/charts/capi-runtime-extensions/templates/cni/calico/manifests/docker/installation.yaml +++ b/charts/capi-runtime-extensions/templates/cni/calico/manifests/docker/installation.yaml @@ -1,15 +1,15 @@ # Copyright 2023 D2iQ, Inc. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -{{- if .Values.hooks.CalicoCNI.defaultInstallationConfigMaps.DockerCluster.create }} +{{- if .Values.hooks.CalicoCNI.crsStrategy.defaultInstallationConfigMaps.DockerCluster.create }} apiVersion: v1 kind: ConfigMap metadata: - name: '{{ .Values.hooks.CalicoCNI.defaultInstallationConfigMaps.DockerCluster.configMap.name }}' + name: '{{ .Values.hooks.CalicoCNI.crsStrategy.defaultInstallationConfigMaps.DockerCluster.configMap.name }}' data: calico-installation: | -{{- if .Values.hooks.CalicoCNI.defaultInstallationConfigMaps.DockerCluster.configMap.content -}} - {{ .Values.hooks.CalicoCNI.defaultInstallationConfigMaps.DockerCluster.configMap.content | nindent 4}} +{{- if .Values.hooks.CalicoCNI.crsStrategy.defaultInstallationConfigMaps.DockerCluster.configMap.content -}} + {{ .Values.hooks.CalicoCNI.crsStrategy.defaultInstallationConfigMaps.DockerCluster.configMap.content | nindent 4 }} {{- else -}} # This section includes base Calico installation configuration. # For more information, see: https://docs.projectcalico.org/reference/installation/api diff --git a/charts/capi-runtime-extensions/templates/cni/calico/manifests/tigera-operator-configmap.yaml b/charts/capi-runtime-extensions/templates/cni/calico/manifests/tigera-operator-configmap.yaml index a357bff3a..48233d438 100644 --- a/charts/capi-runtime-extensions/templates/cni/calico/manifests/tigera-operator-configmap.yaml +++ b/charts/capi-runtime-extensions/templates/cni/calico/manifests/tigera-operator-configmap.yaml @@ -12,4 +12,5 @@ data: kind: ConfigMap metadata: creationTimestamp: null - name: '{{ .Values.hooks.CalicoCNI.defaultTigeraOperatorConfigMap.name }}' + name: '{{ .Values.hooks.CalicoCNI.crsStrategy.defaultTigeraOperatorConfigMap.name + }}' diff --git a/charts/capi-runtime-extensions/templates/deployment.yaml b/charts/capi-runtime-extensions/templates/deployment.yaml index 8378d5b45..4979da5a0 100644 --- a/charts/capi-runtime-extensions/templates/deployment.yaml +++ b/charts/capi-runtime-extensions/templates/deployment.yaml @@ -29,7 +29,7 @@ spec: imagePullPolicy: "{{ .Values.image.pullPolicy }}" args: - --webhook-cert-dir=/runtimehooks-certs/ - - --calicocni.defaultsNamespace={{ .Release.Namespace }} + - --calicocni.crs.defaultsNamespace={{ .Release.Namespace }} - --nfd.defaultsNamespace={{ .Release.Namespace }} {{- range $key, $value := .Values.extraArgs }} - --{{ $key }}={{ $value }} diff --git a/charts/capi-runtime-extensions/values.yaml b/charts/capi-runtime-extensions/values.yaml index 6ef1b1b1e..6ea0f06b2 100644 --- a/charts/capi-runtime-extensions/values.yaml +++ b/charts/capi-runtime-extensions/values.yaml @@ -4,19 +4,20 @@ hooks: CalicoCNI: defaultPodSubnet: 192.168.0.0/16 - defaultTigeraOperatorConfigMap: - name: tigera-operator - defaultInstallationConfigMaps: - DockerCluster: - create: true - configMap: - name: calico-cni-installation-dockercluster - content: "" - AWSCluster: - create: true - configMap: - name: calico-cni-installation-awscluster - content: "" + crsStrategy: + defaultTigeraOperatorConfigMap: + name: tigera-operator + defaultInstallationConfigMaps: + DockerCluster: + create: true + configMap: + name: calico-cni-installation-dockercluster + content: "" + AWSCluster: + create: true + configMap: + name: calico-cni-installation-awscluster + content: "" deployDefaultClusterClasses: true diff --git a/examples/capi-quick-start/aws-cluster.yaml b/examples/capi-quick-start/aws-cluster.yaml index 7755403a0..c033a7338 100644 --- a/examples/capi-quick-start/aws-cluster.yaml +++ b/examples/capi-quick-start/aws-cluster.yaml @@ -23,6 +23,7 @@ spec: addons: cni: provider: calico + strategy: ClusterResourceSet cpi: {} csi: providers: diff --git a/examples/capi-quick-start/docker-cluster.yaml b/examples/capi-quick-start/docker-cluster.yaml index 75f7f115b..15c595ffb 100644 --- a/examples/capi-quick-start/docker-cluster.yaml +++ b/examples/capi-quick-start/docker-cluster.yaml @@ -24,6 +24,7 @@ spec: addons: cni: provider: calico + strategy: ClusterResourceSet nfd: {} docker: {} version: v1.27.5 diff --git a/hack/addons/update-calico-manifests.sh b/hack/addons/update-calico-manifests.sh index ca0088102..36f548413 100755 --- a/hack/addons/update-calico-manifests.sh +++ b/hack/addons/update-calico-manifests.sh @@ -43,7 +43,7 @@ gojq --yaml-input \ <"${ASSETS_DIR}/${FILE_NAME}" \ >"${ASSETS_DIR}/tigera-operator.json" -kubectl create configmap "{{ .Values.hooks.CalicoCNI.defaultTigeraOperatorConfigMap.name }}" --dry-run=client --output yaml \ +kubectl create configmap "{{ .Values.hooks.CalicoCNI.crsStrategy.defaultTigeraOperatorConfigMap.name }}" --dry-run=client --output yaml \ --from-file "${ASSETS_DIR}/tigera-operator.json" \ >"${ASSETS_DIR}/tigera-operator-configmap.yaml" diff --git a/hack/examples/bases/aws/kustomization.yaml.tmpl b/hack/examples/bases/aws/kustomization.yaml.tmpl index 5176237ef..b716f2d09 100644 --- a/hack/examples/bases/aws/kustomization.yaml.tmpl +++ b/hack/examples/bases/aws/kustomization.yaml.tmpl @@ -38,6 +38,7 @@ patches: value: cni: provider: calico + strategy: ClusterResourceSet nfd: {} cpi: {} csi: diff --git a/hack/examples/bases/docker/kustomization.yaml.tmpl b/hack/examples/bases/docker/kustomization.yaml.tmpl index 9d1c7512f..e48ffaae1 100644 --- a/hack/examples/bases/docker/kustomization.yaml.tmpl +++ b/hack/examples/bases/docker/kustomization.yaml.tmpl @@ -38,6 +38,7 @@ patches: nfd: {} cni: provider: calico + strategy: ClusterResourceSet - op: "remove" path: "/spec/topology/workers/machinePools" - target: diff --git a/pkg/handlers/generic/lifecycle/cni/calico/handler.go b/pkg/handlers/generic/lifecycle/cni/calico/handler.go index 0a816fdb9..617228038 100644 --- a/pkg/handlers/generic/lifecycle/cni/calico/handler.go +++ b/pkg/handlers/generic/lifecycle/cni/calico/handler.go @@ -4,65 +4,28 @@ package calico import ( - "bytes" "context" "fmt" "github.com/spf13/pflag" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured/unstructuredscheme" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/runtime/serializer/json" - capiv1 "sigs.k8s.io/cluster-api/api/v1beta1" - crsv1 "sigs.k8s.io/cluster-api/exp/addons/api/v1beta1" runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" ctrl "sigs.k8s.io/controller-runtime" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "github.com/d2iq-labs/capi-runtime-extensions/api/v1alpha1" commonhandlers "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/capi/clustertopology/handlers" "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/capi/clustertopology/handlers/lifecycle" "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/capi/clustertopology/variables" - "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/k8s/client" - "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/k8s/parser" "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/clusterconfig" - "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/lifecycle/cni" ) type CalicoCNIConfig struct { - defaultsNamespace string - - defaultTigeraOperatorConfigMapName string - defaultProviderInstallationConfigMapNames map[string]string + crsConfig crsConfig } func (c *CalicoCNIConfig) AddFlags(prefix string, flags *pflag.FlagSet) { - flags.StringVar( - &c.defaultsNamespace, - prefix+".defaultsNamespace", - corev1.NamespaceDefault, - "namespace of the ConfigMap used to deploy Tigera Operator", - ) - - flags.StringVar( - &c.defaultTigeraOperatorConfigMapName, - prefix+".default-tigera-operator-configmap-name", - "tigera-operator", - "name of the ConfigMap used to deploy Tigera Operator", - ) - flags.StringToStringVar( - &c.defaultProviderInstallationConfigMapNames, - prefix+".default-provider-installation-configmap-names", - map[string]string{ - "DockerCluster": "calico-cni-installation-dockercluster", - "AWSCluster": "calico-cni-installation-awscluster", - }, - "map of provider cluster implementation type to default installation ConfigMap name", - ) + c.crsConfig.AddFlags(prefix+".crs", flags) } type CalicoCNI struct { @@ -124,279 +87,39 @@ func (s *CalicoCNI) AfterControlPlaneInitialized( ) return } - if !found || cniVar.Provider != v1alpha1.CNIProviderCalico { - log.V(4).Info( - fmt.Sprintf( - "Skipping Calico CNI handler, cluster does not specify %q as value of CNI provider variable", - v1alpha1.CNIProviderCalico, - ), - ) + if !found { + log.V(4). + Info("Skipping Calico CNI handler, cluster does not specify request CNI addon deployment") return } - - infraKind := req.Cluster.Spec.InfrastructureRef.Kind - defaultInstallationConfigMapName, ok := s.config.defaultProviderInstallationConfigMapNames[infraKind] - if !ok { + if cniVar.Provider != v1alpha1.CNIProviderCalico { log.V(4).Info( fmt.Sprintf( - "Skipping Calico CNI handler, no default installation ConfigMap configured for infrastructure provider %q", - req.Cluster.Spec.InfrastructureRef.Kind, + "Skipping Calico CNI handler, cluster does not specify %q as value of CNI provider variable", + v1alpha1.CNIProviderCalico, ), ) return } - log.Info("Ensuring Tigera manifests ConfigMap exist in cluster namespace") - if err := s.ensureTigeraOperatorConfigMap(ctx, &req.Cluster); err != nil { - log.Error( - err, - "failed to ensure Tigera Operator manifests ConfigMap exists in cluster namespace", - ) + switch cniVar.Strategy { + case v1alpha1.AddonStrategyClusterResourceSet: + s := crsStrategy{ + config: s.config.crsConfig, + client: s.client, + } + err = s.applyViaCRS(ctx, req, log) + default: resp.SetStatus(runtimehooksv1.ResponseStatusFailure) - resp.SetMessage( - fmt.Sprintf( - "failed to ensure Tigera Operator manifests ConfigMap exists in cluster namespace: %v", - err, - ), - ) + resp.SetMessage(fmt.Sprintf("unknown CNI addon deployment strategy %q", cniVar.Strategy)) return } - log.Info("Ensuring Calico installation CRS and ConfigMap exist for cluster") - err = s.ensureCNICRSForCluster(ctx, &req.Cluster, defaultInstallationConfigMapName) if err != nil { - log.Error( - err, - "failed to ensure Calico installation manifests ConfigMap and ClusterResourceSet exist in cluster namespace", - ) resp.SetStatus(runtimehooksv1.ResponseStatusFailure) - resp.SetMessage( - fmt.Sprintf( - "failed to ensure Tigera Operator manifests ConfigMap exists in cluster namespace: %v", - err, - ), - ) + resp.SetMessage(err.Error()) return } resp.SetStatus(runtimehooksv1.ResponseStatusSuccess) } - -func (s *CalicoCNI) ensureCNICRSForCluster( - ctx context.Context, - cluster *capiv1.Cluster, - defaultInstallationConfigMapName string, -) error { - defaultInstallationConfigMap := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: s.config.defaultsNamespace, - Name: defaultInstallationConfigMapName, - }, - } - defaultInstallationConfigMapObjName := ctrlclient.ObjectKeyFromObject( - defaultInstallationConfigMap, - ) - err := s.client.Get(ctx, defaultInstallationConfigMapObjName, defaultInstallationConfigMap) - if err != nil { - return fmt.Errorf( - "failed to retrieve default default installation ConfigMap %q: %w", - defaultInstallationConfigMapObjName, - err, - ) - } - - calicoCNIObjs, err := generateProviderCNICRS( - defaultInstallationConfigMap, - s.config.defaultTigeraOperatorConfigMapName, - cluster, - s.client.Scheme(), - ) - if err != nil { - return fmt.Errorf( - "failed to generate Calico provider CNI CRS: %w", - err, - ) - } - - if err := client.ServerSideApply(ctx, s.client, calicoCNIObjs...); err != nil { - return fmt.Errorf( - "failed to apply Calico CNI installation CRS: %w", - err, - ) - } - - return nil -} - -func (s *CalicoCNI) ensureTigeraOperatorConfigMap( - ctx context.Context, - cluster *capiv1.Cluster, -) error { - defaultTigeraOperatorConfigMap := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: s.config.defaultsNamespace, - Name: s.config.defaultTigeraOperatorConfigMapName, - }, - } - defaultTigeraOperatorConfigMapObjName := ctrlclient.ObjectKeyFromObject( - defaultTigeraOperatorConfigMap, - ) - err := s.client.Get(ctx, defaultTigeraOperatorConfigMapObjName, defaultTigeraOperatorConfigMap) - if err != nil { - return fmt.Errorf( - "failed to retrieve default Tigera Operator manifests ConfigMap %q: %w", - defaultTigeraOperatorConfigMapObjName, - err, - ) - } - - tigeraConfigMap := generateTigeraOperatorConfigMap(defaultTigeraOperatorConfigMap, cluster) - if err := client.ServerSideApply(ctx, s.client, tigeraConfigMap); err != nil { - return fmt.Errorf( - "failed to apply Tigera Operator manifests ConfigMap: %w", - err, - ) - } - - return nil -} - -func generateTigeraOperatorConfigMap( - defaultTigeraOperatorConfigMap *corev1.ConfigMap, cluster *capiv1.Cluster, -) ctrlclient.Object { - namespacedTigeraConfigMap := &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{ - APIVersion: corev1.SchemeGroupVersion.String(), - Kind: "ConfigMap", - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: cluster.Namespace, - Name: defaultTigeraOperatorConfigMap.Name, - }, - Data: defaultTigeraOperatorConfigMap.Data, - BinaryData: defaultTigeraOperatorConfigMap.BinaryData, - } - - return namespacedTigeraConfigMap -} - -func generateProviderCNICRS( - installationConfigMap *corev1.ConfigMap, - tigeraOperatorConfigMapName string, - cluster *capiv1.Cluster, - scheme *runtime.Scheme, -) ([]ctrlclient.Object, error) { - defaultManifestStrings := make([]string, 0, len(installationConfigMap.Data)) - for _, v := range installationConfigMap.Data { - defaultManifestStrings = append(defaultManifestStrings, v) - } - parsed, err := parser.StringsToUnstructured(defaultManifestStrings...) - if err != nil { - return nil, fmt.Errorf("failed to parse embedded manifests: %w", err) - } - - cm := &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{ - APIVersion: corev1.SchemeGroupVersion.String(), - Kind: "ConfigMap", - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: cluster.Namespace, - Name: "calico-cni-installation-" + cluster.Name, - }, - Data: make(map[string]string, 1), - } - - yamlSerializer := json.NewSerializerWithOptions( - json.DefaultMetaFactory, - unstructuredscheme.NewUnstructuredCreator(), - unstructuredscheme.NewUnstructuredObjectTyper(), - json.SerializerOptions{ - Yaml: true, - Strict: true, - }, - ) - - podSubnet, err := cni.PodCIDR(cluster) - if err != nil { - return nil, err - } - - var b bytes.Buffer - - for _, o := range parsed { - if podSubnet != "" && - o.GetObjectKind().GroupVersionKind().GroupKind() == calicoInstallationGK { - obj := o.(*unstructured.Unstructured).Object - - ipPoolsRef, exists, err := unstructured.NestedFieldNoCopy( - obj, - "spec", "calicoNetwork", "ipPools", - ) - if err != nil { - return nil, fmt.Errorf("failed to get ipPools from unstructured object: %w", err) - } - if !exists { - return nil, fmt.Errorf("missing ipPools in unstructured object") - } - - ipPools := ipPoolsRef.([]interface{}) - - err = unstructured.SetNestedField( - ipPools[0].(map[string]interface{}), - podSubnet, - "cidr", - ) - if err != nil { - return nil, fmt.Errorf("failed to set default pod subnet: %w", err) - } - - err = unstructured.SetNestedSlice(obj, ipPools, "spec", "calicoNetwork", "ipPools") - if err != nil { - return nil, fmt.Errorf("failed to update ipPools: %w", err) - } - } - - if err := yamlSerializer.Encode(o, &b); err != nil { - return nil, fmt.Errorf("failed to serialize manifests: %w", err) - } - - _, _ = b.WriteString("\n---\n") - } - - cm.Data["manifests"] = b.String() - - if err := controllerutil.SetOwnerReference(cluster, cm, scheme); err != nil { - return nil, fmt.Errorf("failed to set owner reference: %w", err) - } - - crs := &crsv1.ClusterResourceSet{ - TypeMeta: metav1.TypeMeta{ - APIVersion: crsv1.GroupVersion.String(), - Kind: "ClusterResourceSet", - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: cluster.Namespace, - Name: cm.Name, - }, - Spec: crsv1.ClusterResourceSetSpec{ - Resources: []crsv1.ResourceRef{{ - Kind: string(crsv1.ConfigMapClusterResourceSetResourceKind), - Name: tigeraOperatorConfigMapName, - }, { - Kind: string(crsv1.ConfigMapClusterResourceSetResourceKind), - Name: cm.Name, - }}, - Strategy: string(crsv1.ClusterResourceSetStrategyReconcile), - ClusterSelector: metav1.LabelSelector{ - MatchLabels: map[string]string{capiv1.ClusterNameLabel: cluster.Name}, - }, - }, - } - - if err := controllerutil.SetOwnerReference(cluster, crs, scheme); err != nil { - return nil, fmt.Errorf("failed to set owner reference: %w", err) - } - - return []ctrlclient.Object{cm, crs}, nil -} diff --git a/pkg/handlers/generic/lifecycle/cni/calico/strategy_crs.go b/pkg/handlers/generic/lifecycle/cni/calico/strategy_crs.go new file mode 100644 index 000000000..aa0a689ae --- /dev/null +++ b/pkg/handlers/generic/lifecycle/cni/calico/strategy_crs.go @@ -0,0 +1,329 @@ +// Copyright 2023 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package calico + +import ( + "bytes" + "context" + "fmt" + + "github.com/go-logr/logr" + "github.com/spf13/pflag" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured/unstructuredscheme" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer/json" + capiv1 "sigs.k8s.io/cluster-api/api/v1beta1" + crsv1 "sigs.k8s.io/cluster-api/exp/addons/api/v1beta1" + runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/k8s/client" + "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/k8s/parser" + "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/lifecycle/cni" +) + +type crsConfig struct { + defaultsNamespace string + + defaultTigeraOperatorConfigMapName string + defaultProviderInstallationConfigMapNames map[string]string +} + +func (c *crsConfig) AddFlags(prefix string, flags *pflag.FlagSet) { + flags.StringVar( + &c.defaultsNamespace, + prefix+".defaultsNamespace", + corev1.NamespaceDefault, + "namespace of the ConfigMap used to deploy Tigera Operator", + ) + + flags.StringVar( + &c.defaultTigeraOperatorConfigMapName, + prefix+".default-tigera-operator-configmap-name", + "tigera-operator", + "name of the ConfigMap used to deploy Tigera Operator", + ) + flags.StringToStringVar( + &c.defaultProviderInstallationConfigMapNames, + prefix+".default-provider-installation-configmap-names", + map[string]string{ + "DockerCluster": "calico-cni-installation-dockercluster", + "AWSCluster": "calico-cni-installation-awscluster", + }, + "map of provider cluster implementation type to default installation ConfigMap name", + ) +} + +type crsStrategy struct { + config crsConfig + + client ctrlclient.Client +} + +func (s crsStrategy) applyViaCRS( + ctx context.Context, + req *runtimehooksv1.AfterControlPlaneInitializedRequest, + log logr.Logger, +) error { + infraKind := req.Cluster.Spec.InfrastructureRef.Kind + defaultInstallationConfigMapName, ok := s.config.defaultProviderInstallationConfigMapNames[infraKind] + if !ok { + log.V(4).Info( + fmt.Sprintf( + "Skipping Calico CNI handler, no default installation ConfigMap configured for infrastructure provider %q", + req.Cluster.Spec.InfrastructureRef.Kind, + ), + ) + return nil + } + + log.Info("Ensuring Tigera manifests ConfigMap exist in cluster namespace") + if err := s.ensureTigeraOperatorConfigMap(ctx, &req.Cluster); err != nil { + log.Error( + err, + "failed to ensure Tigera Operator manifests ConfigMap exists in cluster namespace", + ) + return fmt.Errorf( + "failed to ensure Tigera Operator manifests ConfigMap exists in cluster namespace: %w", + err, + ) + } + + log.Info("Ensuring Calico installation CRS and ConfigMap exist for cluster") + if err := s.ensureCNICRSForCluster(ctx, &req.Cluster, defaultInstallationConfigMapName); err != nil { + log.Error( + err, + "failed to ensure Calico installation manifests ConfigMap and ClusterResourceSet exist in cluster namespace", + ) + return fmt.Errorf( + "failed to ensure Tigera Operator manifests ConfigMap exists in cluster namespace: %w", + err, + ) + } + + return nil +} + +func (s crsStrategy) ensureCNICRSForCluster( + ctx context.Context, + cluster *capiv1.Cluster, + defaultInstallationConfigMapName string, +) error { + defaultInstallationConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: s.config.defaultsNamespace, + Name: defaultInstallationConfigMapName, + }, + } + defaultInstallationConfigMapObjName := ctrlclient.ObjectKeyFromObject( + defaultInstallationConfigMap, + ) + err := s.client.Get(ctx, defaultInstallationConfigMapObjName, defaultInstallationConfigMap) + if err != nil { + return fmt.Errorf( + "failed to retrieve default default installation ConfigMap %q: %w", + defaultInstallationConfigMapObjName, + err, + ) + } + + calicoCNIObjs, err := generateProviderCNICRS( + defaultInstallationConfigMap, + s.config.defaultTigeraOperatorConfigMapName, + cluster, + s.client.Scheme(), + ) + if err != nil { + return fmt.Errorf( + "failed to generate Calico provider CNI CRS: %w", + err, + ) + } + + if err := client.ServerSideApply(ctx, s.client, calicoCNIObjs...); err != nil { + return fmt.Errorf( + "failed to apply Calico CNI installation CRS: %w", + err, + ) + } + + return nil +} + +func (s crsStrategy) ensureTigeraOperatorConfigMap( + ctx context.Context, + cluster *capiv1.Cluster, +) error { + defaultTigeraOperatorConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: s.config.defaultsNamespace, + Name: s.config.defaultTigeraOperatorConfigMapName, + }, + } + defaultTigeraOperatorConfigMapObjName := ctrlclient.ObjectKeyFromObject( + defaultTigeraOperatorConfigMap, + ) + err := s.client.Get(ctx, defaultTigeraOperatorConfigMapObjName, defaultTigeraOperatorConfigMap) + if err != nil { + return fmt.Errorf( + "failed to retrieve default Tigera Operator manifests ConfigMap %q: %w", + defaultTigeraOperatorConfigMapObjName, + err, + ) + } + + tigeraConfigMap := generateTigeraOperatorConfigMap(defaultTigeraOperatorConfigMap, cluster) + if err := client.ServerSideApply(ctx, s.client, tigeraConfigMap); err != nil { + return fmt.Errorf( + "failed to apply Tigera Operator manifests ConfigMap: %w", + err, + ) + } + + return nil +} + +func generateTigeraOperatorConfigMap( + defaultTigeraOperatorConfigMap *corev1.ConfigMap, cluster *capiv1.Cluster, +) ctrlclient.Object { + namespacedTigeraConfigMap := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: cluster.Namespace, + Name: defaultTigeraOperatorConfigMap.Name, + }, + Data: defaultTigeraOperatorConfigMap.Data, + BinaryData: defaultTigeraOperatorConfigMap.BinaryData, + } + + return namespacedTigeraConfigMap +} + +func generateProviderCNICRS( + installationConfigMap *corev1.ConfigMap, + tigeraOperatorConfigMapName string, + cluster *capiv1.Cluster, + scheme *runtime.Scheme, +) ([]ctrlclient.Object, error) { + defaultManifestStrings := make([]string, 0, len(installationConfigMap.Data)) + for _, v := range installationConfigMap.Data { + defaultManifestStrings = append(defaultManifestStrings, v) + } + parsed, err := parser.StringsToUnstructured(defaultManifestStrings...) + if err != nil { + return nil, fmt.Errorf("failed to parse embedded manifests: %w", err) + } + + cm := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: cluster.Namespace, + Name: "calico-cni-installation-" + cluster.Name, + }, + Data: make(map[string]string, 1), + } + + yamlSerializer := json.NewSerializerWithOptions( + json.DefaultMetaFactory, + unstructuredscheme.NewUnstructuredCreator(), + unstructuredscheme.NewUnstructuredObjectTyper(), + json.SerializerOptions{ + Yaml: true, + Strict: true, + }, + ) + + podSubnet, err := cni.PodCIDR(cluster) + if err != nil { + return nil, err + } + + var b bytes.Buffer + + for _, o := range parsed { + if podSubnet != "" && + o.GetObjectKind().GroupVersionKind().GroupKind() == calicoInstallationGK { + obj := o.(*unstructured.Unstructured).Object + + ipPoolsRef, exists, err := unstructured.NestedFieldNoCopy( + obj, + "spec", "calicoNetwork", "ipPools", + ) + if err != nil { + return nil, fmt.Errorf("failed to get ipPools from unstructured object: %w", err) + } + if !exists { + return nil, fmt.Errorf("missing ipPools in unstructured object") + } + + ipPools := ipPoolsRef.([]interface{}) + + err = unstructured.SetNestedField( + ipPools[0].(map[string]interface{}), + podSubnet, + "cidr", + ) + if err != nil { + return nil, fmt.Errorf("failed to set default pod subnet: %w", err) + } + + err = unstructured.SetNestedSlice(obj, ipPools, "spec", "calicoNetwork", "ipPools") + if err != nil { + return nil, fmt.Errorf("failed to update ipPools: %w", err) + } + } + + if err := yamlSerializer.Encode(o, &b); err != nil { + return nil, fmt.Errorf("failed to serialize manifests: %w", err) + } + + _, _ = b.WriteString("\n---\n") + } + + cm.Data["manifests"] = b.String() + + if err := controllerutil.SetOwnerReference(cluster, cm, scheme); err != nil { + return nil, fmt.Errorf("failed to set owner reference: %w", err) + } + + crs := &crsv1.ClusterResourceSet{ + TypeMeta: metav1.TypeMeta{ + APIVersion: crsv1.GroupVersion.String(), + Kind: "ClusterResourceSet", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: cluster.Namespace, + Name: cm.Name, + }, + Spec: crsv1.ClusterResourceSetSpec{ + Resources: []crsv1.ResourceRef{{ + Kind: string(crsv1.ConfigMapClusterResourceSetResourceKind), + Name: tigeraOperatorConfigMapName, + }, { + Kind: string(crsv1.ConfigMapClusterResourceSetResourceKind), + Name: cm.Name, + }}, + Strategy: string(crsv1.ClusterResourceSetStrategyReconcile), + ClusterSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{capiv1.ClusterNameLabel: cluster.Name}, + }, + }, + } + + if err := controllerutil.SetOwnerReference(cluster, crs, scheme); err != nil { + return nil, fmt.Errorf("failed to set owner reference: %w", err) + } + + return []ctrlclient.Object{cm, crs}, nil +} diff --git a/pkg/handlers/generic/lifecycle/cni/variables_test.go b/pkg/handlers/generic/lifecycle/cni/variables_test.go index a6104b80f..65484c050 100644 --- a/pkg/handlers/generic/lifecycle/cni/variables_test.go +++ b/pkg/handlers/generic/lifecycle/cni/variables_test.go @@ -26,6 +26,7 @@ func TestVariableValidation(t *testing.T) { Addons: &v1alpha1.Addons{ CNI: &v1alpha1.CNI{ Provider: v1alpha1.CNIProviderCalico, + Strategy: v1alpha1.AddonStrategyClusterResourceSet, }, }, }, @@ -36,6 +37,19 @@ func TestVariableValidation(t *testing.T) { Addons: &v1alpha1.Addons{ CNI: &v1alpha1.CNI{ Provider: "invalid-provider", + Strategy: v1alpha1.AddonStrategyClusterResourceSet, + }, + }, + }, + ExpectError: true, + }, + capitest.VariableTestDef{ + Name: "set with invalid strategy", + Vals: v1alpha1.GenericClusterConfig{ + Addons: &v1alpha1.Addons{ + CNI: &v1alpha1.CNI{ + Provider: v1alpha1.CNIProviderCalico, + Strategy: "invalid-strategy", }, }, },