Skip to content

Commit 51def9d

Browse files
dkoshkindkoshkin
committed
feat: add COSI controller handler
1 parent d41549b commit 51def9d

File tree

17 files changed

+384
-1
lines changed

17 files changed

+384
-1
lines changed

‎api/v1alpha1/constants.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@ const (
2222
CNIVariableName = "cni"
2323
// NFDVariableName is the NFD external patch variable name.
2424
NFDVariableName = "nfd"
25-
25+
// COSIVariableName is the COSI external patch variable name.
26+
COSIVariableName = "cosi"
2627
// ClusterAutoscalerVariableName is the cluster-autoscaler external patch variable name.
2728
ClusterAutoscalerVariableName = "clusterAutoscaler"
2829
// ServiceLoadBalancerVariableName is the Service LoadBalancer config patch variable name.

‎charts/cluster-api-runtime-extensions-nutanix/README.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,8 @@ A Helm chart for cluster-api-runtime-extensions-nutanix
7474
| hooks.cni.cilium.crsStrategy.defaultCiliumConfigMap.name | string | `"cilium"` | |
7575
| hooks.cni.cilium.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` | |
7676
| hooks.cni.cilium.helmAddonStrategy.defaultValueTemplateConfigMap.name | string | `"default-cilium-cni-helm-values-template"` | |
77+
| hooks.cosi.controller.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` | |
78+
| hooks.cosi.controller.helmAddonStrategy.defaultValueTemplateConfigMap.name | string | `"default-cosi-controller-helm-values-template"` | |
7779
| hooks.csi.aws-ebs.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` | |
7880
| hooks.csi.aws-ebs.helmAddonStrategy.defaultValueTemplateConfigMap.name | string | `"default-aws-ebs-csi-helm-values-template"` | |
7981
| hooks.csi.local-path.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` | |

‎charts/cluster-api-runtime-extensions-nutanix/addons/cosi/controller/values-template.yaml

Whitespace-only changes.

‎charts/cluster-api-runtime-extensions-nutanix/templates/cosi/controller/manifests/helm-addon-installation.yaml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
# Copyright 2024 Nutanix. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
{{- if (index .Values.hooks.cosi "controller").helmAddonStrategy.defaultValueTemplateConfigMap.create }}
5+
apiVersion: v1
6+
kind: ConfigMap
7+
metadata:
8+
name: '{{ (index .Values.hooks.cosi "controller").helmAddonStrategy.defaultValueTemplateConfigMap.name }}'
9+
data:
10+
values.yaml: |-
11+
{{- .Files.Get "addons/cosi/controller/values-template.yaml" | nindent 4 }}
12+
{{- end -}}

‎charts/cluster-api-runtime-extensions-nutanix/templates/deployment.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ spec:
4141
- --csi.local-path.helm-addon.default-values-template-configmap-name={{ (index .Values.hooks.csi "local-path").helmAddonStrategy.defaultValueTemplateConfigMap.name }}
4242
- --csi.snapshot-controller.helm-addon.default-values-template-configmap-name={{ (index .Values.hooks.csi "snapshot-controller").helmAddonStrategy.defaultValueTemplateConfigMap.name }}
4343
- --ccm.aws.helm-addon.default-values-template-configmap-name={{ .Values.hooks.ccm.aws.helmAddonStrategy.defaultValueTemplateConfigMap.name }}
44+
- --cosi.controller.helm-addon.default-values-template-configmap-name={{ .Values.hooks.cosi.controller.helmAddonStrategy.defaultValueTemplateConfigMap.name }}
4445
{{- range $k, $v := .Values.hooks.ccm.aws.k8sMinorVersionToCCMVersion }}
4546
- --ccm.aws.aws-ccm-versions={{ $k }}={{ $v }}
4647
{{- end }}

‎charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,10 @@ data:
2323
ChartName: cluster-autoscaler
2424
ChartVersion: 9.43.2
2525
RepositoryURL: '{{ if .Values.helmRepository.enabled }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://kubernetes.github.io/autoscaler{{ end }}'
26+
cosi-controller: |
27+
ChartName: cosi
28+
ChartVersion: 0.0.1-alpha.1
29+
RepositoryURL: '{{ if .Values.helmRepository.enabled }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://mesosphere.github.io/charts/stable/{{ end }}'
2630
local-path-provisioner-csi: |
2731
ChartName: local-path-provisioner
2832
ChartVersion: 0.0.30

‎charts/cluster-api-runtime-extensions-nutanix/values.schema.json

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -361,6 +361,32 @@
361361
},
362362
"type": "object"
363363
},
364+
"cosi": {
365+
"properties": {
366+
"controller": {
367+
"properties": {
368+
"helmAddonStrategy": {
369+
"properties": {
370+
"defaultValueTemplateConfigMap": {
371+
"properties": {
372+
"create": {
373+
"type": "boolean"
374+
},
375+
"name": {
376+
"type": "string"
377+
}
378+
},
379+
"type": "object"
380+
}
381+
},
382+
"type": "object"
383+
}
384+
},
385+
"type": "object"
386+
}
387+
},
388+
"type": "object"
389+
},
364390
"csi": {
365391
"properties": {
366392
"aws-ebs": {

‎charts/cluster-api-runtime-extensions-nutanix/values.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,12 @@ hooks:
107107
defaultTemplateConfigMap:
108108
create: true
109109
name: default-kube-vip-template
110+
cosi:
111+
controller:
112+
helmAddonStrategy:
113+
defaultValueTemplateConfigMap:
114+
create: true
115+
name: default-cosi-controller-helm-values-template
110116

111117
helmAddonsConfigMap: default-helm-addons-config
112118

‎docs/content/addons/cosi.md

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
+++
2+
title = " Container Object Storage Interface (COSI)"
3+
icon = "fa-solid fa-eye"
4+
+++
5+
6+
By leveraging CAPI cluster lifecycle hooks, this handler deploys [Container Object Storage Interface] (COSI)
7+
on the new cluster at the `AfterControlPlaneInitialized` phase.
8+
9+
Deployment of COSI is opt-in via the [provider-specific cluster configuration]({{< ref ".." >}}).
10+
11+
The hook uses the [Cluster API Add-on Provider for Helm] to deploy the NFD resources.
12+
13+
## Example
14+
15+
To enable deployment of COSI on a cluster, specify the following values:
16+
17+
```yaml
18+
apiVersion: cluster.x-k8s.io/v1beta1
19+
kind: Cluster
20+
metadata:
21+
name: <NAME>
22+
spec:
23+
topology:
24+
variables:
25+
- name: clusterConfig
26+
value:
27+
addons:
28+
cosi: {}
29+
```
30+
31+
[Container Object Storage Interface]: https://kubernetes.io/blog/2022/09/02/cosi-kubernetes-object-storage-management/
32+
[Cluster API Add-on Provider for Helm]: https://github.com/kubernetes-sigs/cluster-api-addon-provider-helm

‎hack/addons/helm-chart-bundler/repos.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,11 @@ repositories:
2626
charts:
2727
cluster-autoscaler:
2828
- 9.43.2
29+
cosi:
30+
repoURL: https://mesosphere.github.io/charts/stable/
31+
charts:
32+
cosi:
33+
- 0.0.1-alpha.1
2934
local-path-provisioner:
3035
repoURL: https://charts.containeroo.ch
3136
charts:

‎hack/addons/kustomize/cosi-controller/kustomization.yaml.tmpl

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# Copyright 2024 Nutanix. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
apiVersion: kustomize.config.k8s.io/v1beta1
5+
kind: Kustomization
6+
7+
metadata:
8+
name: cosi-controller-kustomize
9+
10+
helmCharts:
11+
- name: cosi
12+
namespace: container-object-storage-system
13+
repo: https://mesosphere.github.io/charts/stable/
14+
releaseName: cosi-controller
15+
version: ${COSI_CONTROLLER_VERSION}
16+
includeCRDs: true
17+
skipTests: true

‎make/addons.mk

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ export KUBE_VIP_VERSION := v0.8.3
2525

2626
export METALLB_CHART_VERSION := 0.14.8
2727

28+
export COSI_CONTROLLER_VERSION := 0.0.1-alpha.1
29+
2830
.PHONY: addons.sync
2931
addons.sync: $(addprefix update-addon.,calico cilium nfd cluster-autoscaler snapshot-controller local-path-provisioner-csi aws-ebs-csi kube-vip)
3032
addons.sync: $(addprefix update-addon.aws-ccm.,127 128 129 130 131)

‎pkg/handlers/generic/lifecycle/config/cm.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ const (
2828
LocalPathProvisionerCSI Component = "local-path-provisioner-csi"
2929
AWSEBSCSI Component = "aws-ebs-csi"
3030
AWSCCM Component = "aws-ccm"
31+
COSIController Component = "cosi-controller"
3132
)
3233

3334
type HelmChartGetter struct {

‎pkg/handlers/generic/lifecycle/cosi/doc.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
// Copyright 2023 Nutanix. All rights reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
// Package COSI provides a handler for managing NFD deployments on clusters
5+
//
6+
// +kubebuilder:rbac:groups=addons.cluster.x-k8s.io,resources=clusterresourcesets,verbs=watch;list;get;create;patch;update;delete
7+
// +kubebuilder:rbac:groups="",resources=configmaps,verbs=watch;list;get;create;patch;update;delete
8+
package cosi

‎pkg/handlers/generic/lifecycle/cosi/handler.go

Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,186 @@
1+
// Copyright 2023 Nutanix. All rights reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
package cosi
5+
6+
import (
7+
"context"
8+
"fmt"
9+
10+
"github.com/spf13/pflag"
11+
"k8s.io/utils/ptr"
12+
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
13+
runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
14+
ctrl "sigs.k8s.io/controller-runtime"
15+
ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
16+
17+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1"
18+
apivariables "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/variables"
19+
commonhandlers "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers"
20+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/lifecycle"
21+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/variables"
22+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/addons"
23+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/config"
24+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/options"
25+
)
26+
27+
const (
28+
defaultHelmReleaseName = "cosi-controller"
29+
defaultHelmReleaseNamespace = "container-object-storage-system"
30+
)
31+
32+
type ControllerConfig struct {
33+
*options.GlobalOptions
34+
35+
helmAddonConfig *addons.HelmAddonConfig
36+
}
37+
38+
func NewControllerConfig(globalOptions *options.GlobalOptions) *ControllerConfig {
39+
return &ControllerConfig{
40+
GlobalOptions: globalOptions,
41+
helmAddonConfig: addons.NewHelmAddonConfig(
42+
"default-cosi-controller-helm-values-template",
43+
defaultHelmReleaseNamespace,
44+
defaultHelmReleaseName,
45+
),
46+
}
47+
}
48+
49+
func (c *ControllerConfig) AddFlags(prefix string, flags *pflag.FlagSet) {
50+
c.helmAddonConfig.AddFlags(prefix+".helm-addon", flags)
51+
}
52+
53+
type DefaultCOSIController struct {
54+
client ctrlclient.Client
55+
config *ControllerConfig
56+
helmChartInfoGetter *config.HelmChartGetter
57+
58+
variableName string // points to the global config variable
59+
variablePath []string // path of this variable on the global config variable
60+
}
61+
62+
var (
63+
_ commonhandlers.Named = &DefaultCOSIController{}
64+
_ lifecycle.AfterControlPlaneInitialized = &DefaultCOSIController{}
65+
_ lifecycle.BeforeClusterUpgrade = &DefaultCOSIController{}
66+
)
67+
68+
func New(
69+
c ctrlclient.Client,
70+
cfg *ControllerConfig,
71+
helmChartInfoGetter *config.HelmChartGetter,
72+
) *DefaultCOSIController {
73+
return &DefaultCOSIController{
74+
client: c,
75+
config: cfg,
76+
helmChartInfoGetter: helmChartInfoGetter,
77+
variableName: v1alpha1.ClusterConfigVariableName,
78+
variablePath: []string{"addons", v1alpha1.COSIVariableName},
79+
}
80+
}
81+
82+
func (n *DefaultCOSIController) Name() string {
83+
return "COSIControllerHandler"
84+
}
85+
86+
func (n *DefaultCOSIController) AfterControlPlaneInitialized(
87+
ctx context.Context,
88+
req *runtimehooksv1.AfterControlPlaneInitializedRequest,
89+
resp *runtimehooksv1.AfterControlPlaneInitializedResponse,
90+
) {
91+
commonResponse := &runtimehooksv1.CommonResponse{}
92+
n.apply(ctx, &req.Cluster, commonResponse)
93+
resp.Status = commonResponse.GetStatus()
94+
resp.Message = commonResponse.GetMessage()
95+
}
96+
97+
func (n *DefaultCOSIController) BeforeClusterUpgrade(
98+
ctx context.Context,
99+
req *runtimehooksv1.BeforeClusterUpgradeRequest,
100+
resp *runtimehooksv1.BeforeClusterUpgradeResponse,
101+
) {
102+
commonResponse := &runtimehooksv1.CommonResponse{}
103+
n.apply(ctx, &req.Cluster, commonResponse)
104+
resp.Status = commonResponse.GetStatus()
105+
resp.Message = commonResponse.GetMessage()
106+
}
107+
108+
func (n *DefaultCOSIController) apply(
109+
ctx context.Context,
110+
cluster *clusterv1.Cluster,
111+
resp *runtimehooksv1.CommonResponse,
112+
) {
113+
clusterKey := ctrlclient.ObjectKeyFromObject(cluster)
114+
115+
log := ctrl.LoggerFrom(ctx).WithValues(
116+
"cluster",
117+
clusterKey,
118+
)
119+
120+
varMap := variables.ClusterVariablesToVariablesMap(cluster.Spec.Topology.Variables)
121+
122+
cosiVar, err := variables.Get[apivariables.COSI](varMap, n.variableName, n.variablePath...)
123+
if err != nil {
124+
if variables.IsNotFoundError(err) {
125+
log.
126+
Info("Skipping COSI handler, cluster does not specify request COSI Controller addon deployment")
127+
return
128+
}
129+
log.Error(
130+
err,
131+
"failed to read COSI variable from cluster definition",
132+
)
133+
resp.SetStatus(runtimehooksv1.ResponseStatusFailure)
134+
resp.SetMessage(
135+
fmt.Sprintf("failed to read COSI variable from cluster definition: %v",
136+
err,
137+
),
138+
)
139+
return
140+
}
141+
142+
var strategy addons.Applier
143+
switch ptr.Deref(cosiVar.Strategy, "") {
144+
case v1alpha1.AddonStrategyHelmAddon:
145+
helmChart, err := n.helmChartInfoGetter.For(ctx, log, config.COSIController)
146+
if err != nil {
147+
log.Error(
148+
err,
149+
"failed to get configmap with helm settings",
150+
)
151+
resp.SetStatus(runtimehooksv1.ResponseStatusFailure)
152+
resp.SetMessage(
153+
fmt.Sprintf("failed to get configuration to create helm addon: %v",
154+
err,
155+
),
156+
)
157+
return
158+
}
159+
strategy = addons.NewHelmAddonApplier(
160+
n.config.helmAddonConfig,
161+
n.client,
162+
helmChart,
163+
)
164+
case v1alpha1.AddonStrategyClusterResourceSet:
165+
resp.SetStatus(runtimehooksv1.ResponseStatusFailure)
166+
resp.SetMessage(fmt.Sprintf("strategy %q not provided for COSI", v1alpha1.AddonStrategyClusterResourceSet))
167+
return
168+
case "":
169+
resp.SetStatus(runtimehooksv1.ResponseStatusFailure)
170+
resp.SetMessage("strategy not provided for COSI")
171+
return
172+
default:
173+
resp.SetStatus(runtimehooksv1.ResponseStatusFailure)
174+
resp.SetMessage(fmt.Sprintf("unknown COSI addon deployment strategy %q", *cosiVar.Strategy))
175+
return
176+
}
177+
178+
if err := strategy.Apply(ctx, cluster, n.config.DefaultsNamespace(), log); err != nil {
179+
err = fmt.Errorf("failed to apply COSI addon: %w", err)
180+
resp.SetStatus(runtimehooksv1.ResponseStatusFailure)
181+
resp.SetMessage(err.Error())
182+
return
183+
}
184+
185+
resp.SetStatus(runtimehooksv1.ResponseStatusSuccess)
186+
}

0 commit comments

Comments
 (0)