diff --git a/api/v1alpha1/addon_types.go b/api/v1alpha1/addon_types.go index 55d3af60c..7bf204c10 100644 --- a/api/v1alpha1/addon_types.go +++ b/api/v1alpha1/addon_types.go @@ -70,6 +70,9 @@ type DockerAddons struct { // +kubebuilder:validation:Optional CSI *DockerCSI `json:"csi,omitempty"` + + // +kubebuilder:validation:Optional + COSI *DockerCOSI `json:"cosi,omitempty"` } type NutanixAddons struct { @@ -77,6 +80,9 @@ type NutanixAddons struct { // +kubebuilder:validation:Optional CSI *NutanixCSI `json:"csi,omitempty"` + + // +kubebuilder:validation:Optional + COSI *NutanixCOSI `json:"cosi,omitempty"` } type GenericAddons struct { @@ -96,8 +102,6 @@ type GenericAddons struct { ServiceLoadBalancer *ServiceLoadBalancer `json:"serviceLoadBalancer,omitempty"` } -// +kubebuilder:validation:Optional -// +kubebuilder:validation:Enum=ClusterResourceSet;HelmAddon type AddonStrategy string // CNI required for providing CNI configuration. @@ -109,6 +113,7 @@ type CNI struct { // Addon strategy used to deploy the CNI provider to the workload cluster. // +kubebuilder:default=HelmAddon + // +kubebuilder:validation:Enum=ClusterResourceSet;HelmAddon Strategy *AddonStrategy `json:"strategy,omitempty"` } @@ -116,6 +121,7 @@ type CNI struct { type NFD struct { // Addon strategy used to deploy Node Feature Discovery (NFD) to the workload cluster. // +kubebuilder:default=HelmAddon + // +kubebuilder:validation:Enum=ClusterResourceSet;HelmAddon Strategy *AddonStrategy `json:"strategy,omitempty"` } @@ -124,6 +130,7 @@ type ClusterAutoscaler struct { // Addon strategy used to deploy cluster-autoscaler to the management cluster // targeting the workload cluster. // +kubebuilder:default=HelmAddon + // +kubebuilder:validation:Enum=ClusterResourceSet;HelmAddon Strategy *AddonStrategy `json:"strategy,omitempty"` } @@ -136,9 +143,17 @@ type GenericCSI struct { SnapshotController *SnapshotController `json:"snapshotController,omitempty"` } +type GenericCOSI struct { + // Addon strategy used to deploy the COSI controller to the workload cluster. + // +kubebuilder:default=HelmAddon + // +kubebuilder:validation:Enum=HelmAddon + Strategy *AddonStrategy `json:"strategy,omitempty"` +} + type SnapshotController struct { // Addon strategy used to deploy the snapshot controller to the workload cluster. // +kubebuilder:default=HelmAddon + // +kubebuilder:validation:Enum=ClusterResourceSet;HelmAddon Strategy *AddonStrategy `json:"strategy,omitempty"` } @@ -197,6 +212,7 @@ type CSIProvider struct { // Addon strategy used to deploy the CSI provider to the workload cluster. // +kubebuilder:default=HelmAddon + // +kubebuilder:validation:Enum=ClusterResourceSet;HelmAddon Strategy *AddonStrategy `json:"strategy,omitempty"` // The reference to any secret used by the CSI Provider. @@ -231,6 +247,14 @@ type CSICredentials struct { SecretRef LocalObjectReference `json:"secretRef"` } +type DockerCOSI struct { + GenericCOSI `json:",inline"` +} + +type NutanixCOSI struct { + GenericCOSI `json:",inline"` +} + // CCM tells us to enable or disable the cloud provider interface. type CCM struct { // A reference to the Secret for credential information for the target Prism Central instance @@ -239,6 +263,7 @@ type CCM struct { // Addon strategy used to deploy the CCM to the workload cluster. // +kubebuilder:default=HelmAddon + // +kubebuilder:validation:Enum=ClusterResourceSet;HelmAddon Strategy *AddonStrategy `json:"strategy,omitempty"` } diff --git a/api/v1alpha1/constants.go b/api/v1alpha1/constants.go index 34c4f6461..613db6993 100644 --- a/api/v1alpha1/constants.go +++ b/api/v1alpha1/constants.go @@ -22,7 +22,8 @@ const ( CNIVariableName = "cni" // NFDVariableName is the NFD external patch variable name. NFDVariableName = "nfd" - + // COSIVariableName is the COSI external patch variable name. + COSIVariableName = "cosi" // ClusterAutoscalerVariableName is the cluster-autoscaler external patch variable name. ClusterAutoscalerVariableName = "clusterAutoscaler" // ServiceLoadBalancerVariableName is the Service LoadBalancer config patch variable name. diff --git a/api/v1alpha1/crds/caren.nutanix.com_dockerclusterconfigs.yaml b/api/v1alpha1/crds/caren.nutanix.com_dockerclusterconfigs.yaml index 29c676f17..dd8c3b4f6 100644 --- a/api/v1alpha1/crds/caren.nutanix.com_dockerclusterconfigs.yaml +++ b/api/v1alpha1/crds/caren.nutanix.com_dockerclusterconfigs.yaml @@ -104,6 +104,15 @@ spec: required: - provider type: object + cosi: + properties: + strategy: + default: HelmAddon + description: Addon strategy used to deploy the COSI controller to the workload cluster. + enum: + - HelmAddon + type: string + type: object csi: properties: defaultStorage: diff --git a/api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml b/api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml index 4a4e52ee7..76e4e086b 100644 --- a/api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml +++ b/api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml @@ -104,6 +104,15 @@ spec: required: - provider type: object + cosi: + properties: + strategy: + default: HelmAddon + description: Addon strategy used to deploy the COSI controller to the workload cluster. + enum: + - HelmAddon + type: string + type: object csi: properties: defaultStorage: diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index d530988d2..887cb6a44 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -681,6 +681,11 @@ func (in *DockerAddons) DeepCopyInto(out *DockerAddons) { *out = new(DockerCSI) (*in).DeepCopyInto(*out) } + if in.COSI != nil { + in, out := &in.COSI, &out.COSI + *out = new(DockerCOSI) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DockerAddons. @@ -693,6 +698,22 @@ func (in *DockerAddons) DeepCopy() *DockerAddons { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DockerCOSI) DeepCopyInto(out *DockerCOSI) { + *out = *in + in.GenericCOSI.DeepCopyInto(&out.GenericCOSI) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DockerCOSI. +func (in *DockerCOSI) DeepCopy() *DockerCOSI { + if in == nil { + return nil + } + out := new(DockerCOSI) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DockerCSI) DeepCopyInto(out *DockerCSI) { *out = *in @@ -999,6 +1020,26 @@ func (in *GenericAddons) DeepCopy() *GenericAddons { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GenericCOSI) DeepCopyInto(out *GenericCOSI) { + *out = *in + if in.Strategy != nil { + in, out := &in.Strategy, &out.Strategy + *out = new(AddonStrategy) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GenericCOSI. +func (in *GenericCOSI) DeepCopy() *GenericCOSI { + if in == nil { + return nil + } + out := new(GenericCOSI) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GenericCSI) DeepCopyInto(out *GenericCSI) { *out = *in @@ -1269,6 +1310,11 @@ func (in *NutanixAddons) DeepCopyInto(out *NutanixAddons) { *out = new(NutanixCSI) (*in).DeepCopyInto(*out) } + if in.COSI != nil { + in, out := &in.COSI, &out.COSI + *out = new(NutanixCOSI) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NutanixAddons. @@ -1281,6 +1327,22 @@ func (in *NutanixAddons) DeepCopy() *NutanixAddons { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NutanixCOSI) DeepCopyInto(out *NutanixCOSI) { + *out = *in + in.GenericCOSI.DeepCopyInto(&out.GenericCOSI) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NutanixCOSI. +func (in *NutanixCOSI) DeepCopy() *NutanixCOSI { + if in == nil { + return nil + } + out := new(NutanixCOSI) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NutanixCSI) DeepCopyInto(out *NutanixCSI) { *out = *in diff --git a/api/variables/aggregate_types.go b/api/variables/aggregate_types.go index b8cd4c0fe..9064d9c86 100644 --- a/api/variables/aggregate_types.go +++ b/api/variables/aggregate_types.go @@ -53,6 +53,8 @@ type Addons struct { carenv1.GenericAddons `json:",inline"` CSI *CSI `json:"csi,omitempty"` + + COSI *COSI `json:"cosi,omitempty"` } type CSI struct { @@ -60,3 +62,7 @@ type CSI struct { Providers map[string]carenv1.CSIProvider `json:"providers"` } + +type COSI struct { + carenv1.GenericCOSI `json:",inline"` +} diff --git a/charts/cluster-api-runtime-extensions-nutanix/README.md b/charts/cluster-api-runtime-extensions-nutanix/README.md index f9a7d3f09..987df6923 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/README.md +++ b/charts/cluster-api-runtime-extensions-nutanix/README.md @@ -74,6 +74,8 @@ A Helm chart for cluster-api-runtime-extensions-nutanix | hooks.cni.cilium.crsStrategy.defaultCiliumConfigMap.name | string | `"cilium"` | | | hooks.cni.cilium.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` | | | hooks.cni.cilium.helmAddonStrategy.defaultValueTemplateConfigMap.name | string | `"default-cilium-cni-helm-values-template"` | | +| hooks.cosi.controller.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` | | +| hooks.cosi.controller.helmAddonStrategy.defaultValueTemplateConfigMap.name | string | `"default-cosi-controller-helm-values-template"` | | | hooks.csi.aws-ebs.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` | | | hooks.csi.aws-ebs.helmAddonStrategy.defaultValueTemplateConfigMap.name | string | `"default-aws-ebs-csi-helm-values-template"` | | | hooks.csi.local-path.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` | | diff --git a/charts/cluster-api-runtime-extensions-nutanix/addons/cosi/controller/values-template.yaml b/charts/cluster-api-runtime-extensions-nutanix/addons/cosi/controller/values-template.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/cosi/controller/manifests/helm-addon-installation.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/cosi/controller/manifests/helm-addon-installation.yaml new file mode 100644 index 000000000..6a5eb3587 --- /dev/null +++ b/charts/cluster-api-runtime-extensions-nutanix/templates/cosi/controller/manifests/helm-addon-installation.yaml @@ -0,0 +1,12 @@ +# Copyright 2025 Nutanix. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{{- if (index .Values.hooks.cosi "controller").helmAddonStrategy.defaultValueTemplateConfigMap.create }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: '{{ (index .Values.hooks.cosi "controller").helmAddonStrategy.defaultValueTemplateConfigMap.name }}' +data: + values.yaml: |- + {{- .Files.Get "addons/cosi/controller/values-template.yaml" | nindent 4 }} +{{- end -}} diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/deployment.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/deployment.yaml index a9fc425d2..129913ef5 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/templates/deployment.yaml +++ b/charts/cluster-api-runtime-extensions-nutanix/templates/deployment.yaml @@ -41,6 +41,7 @@ spec: - --csi.local-path.helm-addon.default-values-template-configmap-name={{ (index .Values.hooks.csi "local-path").helmAddonStrategy.defaultValueTemplateConfigMap.name }} - --csi.snapshot-controller.helm-addon.default-values-template-configmap-name={{ (index .Values.hooks.csi "snapshot-controller").helmAddonStrategy.defaultValueTemplateConfigMap.name }} - --ccm.aws.helm-addon.default-values-template-configmap-name={{ .Values.hooks.ccm.aws.helmAddonStrategy.defaultValueTemplateConfigMap.name }} + - --cosi.controller.helm-addon.default-values-template-configmap-name={{ .Values.hooks.cosi.controller.helmAddonStrategy.defaultValueTemplateConfigMap.name }} {{- range $k, $v := .Values.hooks.ccm.aws.k8sMinorVersionToCCMVersion }} - --ccm.aws.aws-ccm-versions={{ $k }}={{ $v }} {{- end }} diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml index bcb804a43..e3a028e3c 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml +++ b/charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml @@ -23,6 +23,10 @@ data: ChartName: cluster-autoscaler ChartVersion: 9.43.2 RepositoryURL: '{{ if .Values.helmRepository.enabled }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://kubernetes.github.io/autoscaler{{ end }}' + cosi-controller: | + ChartName: cosi + ChartVersion: 0.0.1-alpha.1 + RepositoryURL: '{{ if .Values.helmRepository.enabled }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://mesosphere.github.io/charts/stable/{{ end }}' local-path-provisioner-csi: | ChartName: local-path-provisioner ChartVersion: 0.0.30 diff --git a/charts/cluster-api-runtime-extensions-nutanix/values.schema.json b/charts/cluster-api-runtime-extensions-nutanix/values.schema.json index 605bf9320..0be2e2734 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/values.schema.json +++ b/charts/cluster-api-runtime-extensions-nutanix/values.schema.json @@ -361,6 +361,32 @@ }, "type": "object" }, + "cosi": { + "properties": { + "controller": { + "properties": { + "helmAddonStrategy": { + "properties": { + "defaultValueTemplateConfigMap": { + "properties": { + "create": { + "type": "boolean" + }, + "name": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + } + }, + "type": "object" + } + }, + "type": "object" + }, "csi": { "properties": { "aws-ebs": { diff --git a/charts/cluster-api-runtime-extensions-nutanix/values.yaml b/charts/cluster-api-runtime-extensions-nutanix/values.yaml index cf929e6c6..b616d98cc 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/values.yaml +++ b/charts/cluster-api-runtime-extensions-nutanix/values.yaml @@ -107,6 +107,12 @@ hooks: defaultTemplateConfigMap: create: true name: default-kube-vip-template + cosi: + controller: + helmAddonStrategy: + defaultValueTemplateConfigMap: + create: true + name: default-cosi-controller-helm-values-template helmAddonsConfigMap: default-helm-addons-config diff --git a/docs/content/addons/cosi.md b/docs/content/addons/cosi.md new file mode 100644 index 000000000..57fc51e87 --- /dev/null +++ b/docs/content/addons/cosi.md @@ -0,0 +1,32 @@ ++++ +title = " Container Object Storage Interface (COSI)" +icon = "fa-solid fa-eye" ++++ + +By leveraging CAPI cluster lifecycle hooks, this handler deploys [Container Object Storage Interface] (COSI) +on the new cluster at the `AfterControlPlaneInitialized` phase. + +Deployment of COSI is opt-in via the [provider-specific cluster configuration]({{< ref ".." >}}). + +The hook uses the [Cluster API Add-on Provider for Helm] to deploy the COSI resources. + +## Example + +To enable deployment of COSI on a cluster, specify the following values: + +```yaml +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: +spec: + topology: + variables: + - name: clusterConfig + value: + addons: + cosi: {} +``` + +[Container Object Storage Interface]: https://kubernetes.io/blog/2022/09/02/cosi-kubernetes-object-storage-management/ +[Cluster API Add-on Provider for Helm]: https://github.com/kubernetes-sigs/cluster-api-addon-provider-helm diff --git a/examples/capi-quick-start/docker-cluster-calico-crs.yaml b/examples/capi-quick-start/docker-cluster-calico-crs.yaml index aeaf9aa93..87d178f12 100644 --- a/examples/capi-quick-start/docker-cluster-calico-crs.yaml +++ b/examples/capi-quick-start/docker-cluster-calico-crs.yaml @@ -27,6 +27,7 @@ spec: cni: provider: Calico strategy: ClusterResourceSet + cosi: {} csi: defaultStorage: provider: local-path diff --git a/examples/capi-quick-start/docker-cluster-calico-helm-addon.yaml b/examples/capi-quick-start/docker-cluster-calico-helm-addon.yaml index 984a3d64b..2b24fcfdd 100644 --- a/examples/capi-quick-start/docker-cluster-calico-helm-addon.yaml +++ b/examples/capi-quick-start/docker-cluster-calico-helm-addon.yaml @@ -25,6 +25,7 @@ spec: clusterAutoscaler: {} cni: provider: Calico + cosi: {} csi: defaultStorage: provider: local-path diff --git a/examples/capi-quick-start/docker-cluster-cilium-crs.yaml b/examples/capi-quick-start/docker-cluster-cilium-crs.yaml index cd503b75f..af0a7bb50 100644 --- a/examples/capi-quick-start/docker-cluster-cilium-crs.yaml +++ b/examples/capi-quick-start/docker-cluster-cilium-crs.yaml @@ -27,6 +27,7 @@ spec: cni: provider: Cilium strategy: ClusterResourceSet + cosi: {} csi: defaultStorage: provider: local-path diff --git a/examples/capi-quick-start/docker-cluster-cilium-helm-addon.yaml b/examples/capi-quick-start/docker-cluster-cilium-helm-addon.yaml index 94da0f6d9..c95472b59 100644 --- a/examples/capi-quick-start/docker-cluster-cilium-helm-addon.yaml +++ b/examples/capi-quick-start/docker-cluster-cilium-helm-addon.yaml @@ -25,6 +25,7 @@ spec: clusterAutoscaler: {} cni: provider: Cilium + cosi: {} csi: defaultStorage: provider: local-path diff --git a/examples/capi-quick-start/nutanix-cluster-calico-crs.yaml b/examples/capi-quick-start/nutanix-cluster-calico-crs.yaml index ceb489010..ef2798ac1 100644 --- a/examples/capi-quick-start/nutanix-cluster-calico-crs.yaml +++ b/examples/capi-quick-start/nutanix-cluster-calico-crs.yaml @@ -73,6 +73,7 @@ spec: cni: provider: Calico strategy: ClusterResourceSet + cosi: {} csi: defaultStorage: provider: nutanix diff --git a/examples/capi-quick-start/nutanix-cluster-calico-helm-addon.yaml b/examples/capi-quick-start/nutanix-cluster-calico-helm-addon.yaml index 747f64620..4211d068d 100644 --- a/examples/capi-quick-start/nutanix-cluster-calico-helm-addon.yaml +++ b/examples/capi-quick-start/nutanix-cluster-calico-helm-addon.yaml @@ -71,6 +71,7 @@ spec: clusterAutoscaler: {} cni: provider: Calico + cosi: {} csi: defaultStorage: provider: nutanix diff --git a/examples/capi-quick-start/nutanix-cluster-cilium-crs.yaml b/examples/capi-quick-start/nutanix-cluster-cilium-crs.yaml index be7f45db8..399fdca1d 100644 --- a/examples/capi-quick-start/nutanix-cluster-cilium-crs.yaml +++ b/examples/capi-quick-start/nutanix-cluster-cilium-crs.yaml @@ -73,6 +73,7 @@ spec: cni: provider: Cilium strategy: ClusterResourceSet + cosi: {} csi: defaultStorage: provider: nutanix diff --git a/examples/capi-quick-start/nutanix-cluster-cilium-helm-addon.yaml b/examples/capi-quick-start/nutanix-cluster-cilium-helm-addon.yaml index c7b986816..052cb6dec 100644 --- a/examples/capi-quick-start/nutanix-cluster-cilium-helm-addon.yaml +++ b/examples/capi-quick-start/nutanix-cluster-cilium-helm-addon.yaml @@ -71,6 +71,7 @@ spec: clusterAutoscaler: {} cni: provider: Cilium + cosi: {} csi: defaultStorage: provider: nutanix diff --git a/hack/addons/helm-chart-bundler/repos.yaml b/hack/addons/helm-chart-bundler/repos.yaml index 452b1eaa1..57fa2e208 100644 --- a/hack/addons/helm-chart-bundler/repos.yaml +++ b/hack/addons/helm-chart-bundler/repos.yaml @@ -26,6 +26,11 @@ repositories: charts: cluster-autoscaler: - 9.43.2 + cosi: + repoURL: https://mesosphere.github.io/charts/stable/ + charts: + cosi: + - 0.0.1-alpha.1 local-path-provisioner: repoURL: https://charts.containeroo.ch charts: diff --git a/hack/addons/kustomize/cosi-controller/kustomization.yaml.tmpl b/hack/addons/kustomize/cosi-controller/kustomization.yaml.tmpl new file mode 100644 index 000000000..27b938e82 --- /dev/null +++ b/hack/addons/kustomize/cosi-controller/kustomization.yaml.tmpl @@ -0,0 +1,17 @@ +# Copyright 2025 Nutanix. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +metadata: + name: cosi-controller-kustomize + +helmCharts: +- name: cosi + namespace: container-object-storage-system + repo: https://mesosphere.github.io/charts/stable/ + releaseName: cosi-controller + version: ${COSI_CONTROLLER_VERSION} + includeCRDs: true + skipTests: true diff --git a/hack/examples/bases/docker/cluster/kustomization.yaml.tmpl b/hack/examples/bases/docker/cluster/kustomization.yaml.tmpl index 824072a6a..92bc7ee96 100644 --- a/hack/examples/bases/docker/cluster/kustomization.yaml.tmpl +++ b/hack/examples/bases/docker/cluster/kustomization.yaml.tmpl @@ -38,6 +38,9 @@ patches: - target: kind: Cluster path: ../../../patches/docker/csi.yaml +- target: + kind: Cluster + path: ../../../patches/nutanix/cosi.yaml - target: kind: Cluster path: ../../../patches/encryption.yaml diff --git a/hack/examples/bases/nutanix/cluster/kustomization.yaml.tmpl b/hack/examples/bases/nutanix/cluster/kustomization.yaml.tmpl index c6a36bb1e..d5a88e961 100644 --- a/hack/examples/bases/nutanix/cluster/kustomization.yaml.tmpl +++ b/hack/examples/bases/nutanix/cluster/kustomization.yaml.tmpl @@ -30,6 +30,9 @@ patches: - target: kind: Cluster path: ../../../patches/nutanix/csi.yaml +- target: + kind: Cluster + path: ../../../patches/nutanix/cosi.yaml - target: kind: Cluster path: ../../../patches/nutanix/ccm.yaml diff --git a/hack/examples/patches/docker/cosi.yaml b/hack/examples/patches/docker/cosi.yaml new file mode 100644 index 000000000..2bce49bb4 --- /dev/null +++ b/hack/examples/patches/docker/cosi.yaml @@ -0,0 +1,6 @@ +# Copyright 2025 Nutanix. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +- op: "add" + path: "/spec/topology/variables/0/value/addons/cosi" + value: {} diff --git a/hack/examples/patches/nutanix/cosi.yaml b/hack/examples/patches/nutanix/cosi.yaml new file mode 100644 index 000000000..2bce49bb4 --- /dev/null +++ b/hack/examples/patches/nutanix/cosi.yaml @@ -0,0 +1,6 @@ +# Copyright 2025 Nutanix. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +- op: "add" + path: "/spec/topology/variables/0/value/addons/cosi" + value: {} diff --git a/hack/tools/fetch-images/main.go b/hack/tools/fetch-images/main.go index b6aa01d0b..50325a68c 100644 --- a/hack/tools/fetch-images/main.go +++ b/hack/tools/fetch-images/main.go @@ -314,6 +314,8 @@ func getValuesFileForChartIfNeeded(chartName, carenChartDirectory string) (strin } return tempFile.Name(), nil + case "cosi-controller": + return filepath.Join(carenChartDirectory, "addons", "cosi", "controller", defaultHelmAddonFilename), nil default: return "", nil } diff --git a/make/addons.mk b/make/addons.mk index afaf656dc..a5b3898ad 100644 --- a/make/addons.mk +++ b/make/addons.mk @@ -25,6 +25,8 @@ export KUBE_VIP_VERSION := v0.8.3 export METALLB_CHART_VERSION := 0.14.8 +export COSI_CONTROLLER_VERSION := 0.0.1-alpha.1 + .PHONY: addons.sync addons.sync: $(addprefix update-addon.,calico cilium nfd cluster-autoscaler snapshot-controller local-path-provisioner-csi aws-ebs-csi kube-vip) addons.sync: $(addprefix update-addon.aws-ccm.,127 128 129 130 131) diff --git a/pkg/handlers/generic/lifecycle/config/cm.go b/pkg/handlers/generic/lifecycle/config/cm.go index 69ffde068..e7be0c717 100644 --- a/pkg/handlers/generic/lifecycle/config/cm.go +++ b/pkg/handlers/generic/lifecycle/config/cm.go @@ -28,6 +28,7 @@ const ( LocalPathProvisionerCSI Component = "local-path-provisioner-csi" AWSEBSCSI Component = "aws-ebs-csi" AWSCCM Component = "aws-ccm" + COSIController Component = "cosi-controller" ) type HelmChartGetter struct { diff --git a/pkg/handlers/generic/lifecycle/cosi/doc.go b/pkg/handlers/generic/lifecycle/cosi/doc.go new file mode 100644 index 000000000..894f11ab6 --- /dev/null +++ b/pkg/handlers/generic/lifecycle/cosi/doc.go @@ -0,0 +1,8 @@ +// Copyright 2023 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Package COSI provides a handler for managing COSI deployments on clusters +// +// +kubebuilder:rbac:groups=addons.cluster.x-k8s.io,resources=clusterresourcesets,verbs=watch;list;get;create;patch;update;delete +// +kubebuilder:rbac:groups="",resources=configmaps,verbs=watch;list;get;create;patch;update;delete +package cosi diff --git a/pkg/handlers/generic/lifecycle/cosi/handler.go b/pkg/handlers/generic/lifecycle/cosi/handler.go new file mode 100644 index 000000000..e186dd3c4 --- /dev/null +++ b/pkg/handlers/generic/lifecycle/cosi/handler.go @@ -0,0 +1,192 @@ +// Copyright 2023 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package cosi + +import ( + "context" + "fmt" + + "github.com/spf13/pflag" + "k8s.io/utils/ptr" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" + ctrl "sigs.k8s.io/controller-runtime" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" + apivariables "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/variables" + commonhandlers "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/lifecycle" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/variables" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/addons" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/config" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/options" +) + +const ( + defaultHelmReleaseName = "cosi-controller" + defaultHelmReleaseNamespace = "container-object-storage-system" +) + +type ControllerConfig struct { + *options.GlobalOptions + + helmAddonConfig *addons.HelmAddonConfig +} + +func NewControllerConfig(globalOptions *options.GlobalOptions) *ControllerConfig { + return &ControllerConfig{ + GlobalOptions: globalOptions, + helmAddonConfig: addons.NewHelmAddonConfig( + "default-cosi-controller-helm-values-template", + defaultHelmReleaseNamespace, + defaultHelmReleaseName, + ), + } +} + +func (c *ControllerConfig) AddFlags(prefix string, flags *pflag.FlagSet) { + c.helmAddonConfig.AddFlags(prefix+".helm-addon", flags) +} + +type DefaultCOSIController struct { + client ctrlclient.Client + config *ControllerConfig + helmChartInfoGetter *config.HelmChartGetter + + variableName string // points to the global config variable + variablePath []string // path of this variable on the global config variable +} + +var ( + _ commonhandlers.Named = &DefaultCOSIController{} + _ lifecycle.AfterControlPlaneInitialized = &DefaultCOSIController{} + _ lifecycle.BeforeClusterUpgrade = &DefaultCOSIController{} +) + +func New( + c ctrlclient.Client, + cfg *ControllerConfig, + helmChartInfoGetter *config.HelmChartGetter, +) *DefaultCOSIController { + return &DefaultCOSIController{ + client: c, + config: cfg, + helmChartInfoGetter: helmChartInfoGetter, + variableName: v1alpha1.ClusterConfigVariableName, + variablePath: []string{"addons", v1alpha1.COSIVariableName}, + } +} + +func (n *DefaultCOSIController) Name() string { + return "COSIControllerHandler" +} + +func (n *DefaultCOSIController) AfterControlPlaneInitialized( + ctx context.Context, + req *runtimehooksv1.AfterControlPlaneInitializedRequest, + resp *runtimehooksv1.AfterControlPlaneInitializedResponse, +) { + commonResponse := &runtimehooksv1.CommonResponse{} + n.apply(ctx, &req.Cluster, commonResponse) + resp.Status = commonResponse.GetStatus() + resp.Message = commonResponse.GetMessage() +} + +func (n *DefaultCOSIController) BeforeClusterUpgrade( + ctx context.Context, + req *runtimehooksv1.BeforeClusterUpgradeRequest, + resp *runtimehooksv1.BeforeClusterUpgradeResponse, +) { + commonResponse := &runtimehooksv1.CommonResponse{} + n.apply(ctx, &req.Cluster, commonResponse) + resp.Status = commonResponse.GetStatus() + resp.Message = commonResponse.GetMessage() +} + +func (n *DefaultCOSIController) apply( + ctx context.Context, + cluster *clusterv1.Cluster, + resp *runtimehooksv1.CommonResponse, +) { + clusterKey := ctrlclient.ObjectKeyFromObject(cluster) + + log := ctrl.LoggerFrom(ctx).WithValues( + "cluster", + clusterKey, + ) + + varMap := variables.ClusterVariablesToVariablesMap(cluster.Spec.Topology.Variables) + + cosiVar, err := variables.Get[apivariables.COSI](varMap, n.variableName, n.variablePath...) + if err != nil { + if variables.IsNotFoundError(err) { + log. + Info( + "Skipping COSI handler, cluster does not specify request COSI Controller addon deployment", + ) + return + } + log.Error( + err, + "failed to read COSI variable from cluster definition", + ) + resp.SetStatus(runtimehooksv1.ResponseStatusFailure) + resp.SetMessage( + fmt.Sprintf("failed to read COSI variable from cluster definition: %v", + err, + ), + ) + return + } + + var strategy addons.Applier + switch ptr.Deref(cosiVar.Strategy, "") { + case v1alpha1.AddonStrategyHelmAddon: + helmChart, err := n.helmChartInfoGetter.For(ctx, log, config.COSIController) + if err != nil { + log.Error( + err, + "failed to get configmap with helm settings", + ) + resp.SetStatus(runtimehooksv1.ResponseStatusFailure) + resp.SetMessage( + fmt.Sprintf("failed to get configuration to create helm addon: %v", + err, + ), + ) + return + } + strategy = addons.NewHelmAddonApplier( + n.config.helmAddonConfig, + n.client, + helmChart, + ) + case v1alpha1.AddonStrategyClusterResourceSet: + resp.SetStatus(runtimehooksv1.ResponseStatusFailure) + resp.SetMessage( + fmt.Sprintf( + "strategy %q not provided for COSI", v1alpha1.AddonStrategyClusterResourceSet, + ), + ) + return + case "": + resp.SetStatus(runtimehooksv1.ResponseStatusFailure) + resp.SetMessage("strategy not provided for COSI") + return + default: + resp.SetStatus(runtimehooksv1.ResponseStatusFailure) + resp.SetMessage(fmt.Sprintf("unknown COSI addon deployment strategy %q", *cosiVar.Strategy)) + return + } + + if err := strategy.Apply(ctx, cluster, n.config.DefaultsNamespace(), log); err != nil { + err = fmt.Errorf("failed to apply COSI addon: %w", err) + resp.SetStatus(runtimehooksv1.ResponseStatusFailure) + resp.SetMessage(err.Error()) + return + } + + resp.SetStatus(runtimehooksv1.ResponseStatusSuccess) +} diff --git a/pkg/handlers/generic/lifecycle/cosi/variables_test.go b/pkg/handlers/generic/lifecycle/cosi/variables_test.go new file mode 100644 index 000000000..c88c3a2d7 --- /dev/null +++ b/pkg/handlers/generic/lifecycle/cosi/variables_test.go @@ -0,0 +1,75 @@ +// Copyright 2023 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package cosi + +import ( + "testing" + + "k8s.io/utils/ptr" + + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" + apivariables "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/variables" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest" + dockerclusterconfig "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/docker/clusterconfig" + nutanixclusterconfig "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/nutanix/clusterconfig" +) + +var testDefs = []capitest.VariableTestDef{{ + Name: "HelmAddon strategy", + Vals: apivariables.ClusterConfigSpec{ + Addons: &apivariables.Addons{ + COSI: &apivariables.COSI{ + GenericCOSI: v1alpha1.GenericCOSI{ + Strategy: ptr.To(v1alpha1.AddonStrategyHelmAddon), + }, + }, + }, + }, +}, { + Name: "ClusterResourceSet strategy", + Vals: apivariables.ClusterConfigSpec{ + Addons: &apivariables.Addons{ + COSI: &apivariables.COSI{ + GenericCOSI: v1alpha1.GenericCOSI{ + Strategy: ptr.To(v1alpha1.AddonStrategyClusterResourceSet), + }, + }, + }, + }, + ExpectError: true, +}, { + Name: "invalid strategy", + Vals: apivariables.ClusterConfigSpec{ + Addons: &apivariables.Addons{ + COSI: &apivariables.COSI{ + GenericCOSI: v1alpha1.GenericCOSI{ + Strategy: ptr.To[v1alpha1.AddonStrategy]("invalid-strategy"), + }, + }, + }, + }, + ExpectError: true, +}} + +func TestVariableValidation_Docker(t *testing.T) { + capitest.ValidateDiscoverVariables( + t, + v1alpha1.ClusterConfigVariableName, + ptr.To(v1alpha1.DockerClusterConfig{}.VariableSchema()), + true, + dockerclusterconfig.NewVariable, + testDefs..., + ) +} + +func TestVariableValidation_Nutanix(t *testing.T) { + capitest.ValidateDiscoverVariables( + t, + v1alpha1.ClusterConfigVariableName, + ptr.To(v1alpha1.NutanixClusterConfig{}.VariableSchema()), + true, + nutanixclusterconfig.NewVariable, + testDefs..., + ) +} diff --git a/pkg/handlers/generic/lifecycle/handlers.go b/pkg/handlers/generic/lifecycle/handlers.go index 5b4267321..462eb1ac0 100644 --- a/pkg/handlers/generic/lifecycle/handlers.go +++ b/pkg/handlers/generic/lifecycle/handlers.go @@ -18,6 +18,7 @@ import ( "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/cni/calico" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/cni/cilium" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/config" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/cosi" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/csi" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/csi/awsebs" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/csi/localpath" @@ -43,6 +44,7 @@ type Handlers struct { metalLBConfig *metallb.Config localPathCSIConfig *localpath.Config snapshotControllerConfig *snapshotcontroller.Config + cosiControllerConfig *cosi.ControllerConfig } func New( @@ -63,6 +65,7 @@ func New( metalLBConfig: &metallb.Config{GlobalOptions: globalOptions}, localPathCSIConfig: localpath.NewConfig(globalOptions), snapshotControllerConfig: snapshotcontroller.NewConfig(globalOptions), + cosiControllerConfig: cosi.NewControllerConfig(globalOptions), } } @@ -112,6 +115,7 @@ func (h *Handlers) AllHandlers(mgr manager.Manager) []handlers.Named { clusterautoscaler.New(mgr.GetClient(), h.clusterAutoscalerConfig, helmChartInfoGetter), csi.New(mgr.GetClient(), csiHandlers), snapshotcontroller.New(mgr.GetClient(), h.snapshotControllerConfig, helmChartInfoGetter), + cosi.New(mgr.GetClient(), h.cosiControllerConfig, helmChartInfoGetter), servicelbgc.New(mgr.GetClient()), // The order of the handlers in the list is important and are called consecutively. // The MetalLB provider may be configured to create a IPAddressPool on the remote cluster. @@ -213,4 +217,5 @@ func (h *Handlers) AddFlags(flagSet *pflag.FlagSet) { h.awsccmConfig.AddFlags("ccm.aws", pflag.CommandLine) h.nutanixCCMConfig.AddFlags("ccm.nutanix", flagSet) h.metalLBConfig.AddFlags("metallb", flagSet) + h.cosiControllerConfig.AddFlags("cosi.controller", flagSet) }