Skip to content

Commit fafd426

Browse files
dlipovetskydlipovetsky
committed
feat: Controller that copies ClusterClasses to namespaces
1 parent d0ad572 commit fafd426

File tree

7 files changed

+686
-0
lines changed

7 files changed

+686
-0
lines changed

‎pkg/controllers/namespacesync/controller.go

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
// Copyright 2024 Nutanix. All rights reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
package namespacesync
5+
6+
import (
7+
"context"
8+
"fmt"
9+
10+
corev1 "k8s.io/api/core/v1"
11+
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
12+
ctrl "sigs.k8s.io/controller-runtime"
13+
"sigs.k8s.io/controller-runtime/pkg/builder"
14+
"sigs.k8s.io/controller-runtime/pkg/client"
15+
"sigs.k8s.io/controller-runtime/pkg/controller"
16+
"sigs.k8s.io/controller-runtime/pkg/event"
17+
"sigs.k8s.io/controller-runtime/pkg/handler"
18+
"sigs.k8s.io/controller-runtime/pkg/predicate"
19+
)
20+
21+
type Reconciler struct {
22+
Client client.Client
23+
24+
// UnstructuredCachingClient provides a client that forces caching of unstructured objects,
25+
// optimizing reads of provider-specific resources.
26+
UnstructuredCachingClient client.Client
27+
28+
// SourceClusterClassNamespace is the namespace from which ClusterClasses are copied.
29+
SourceClusterClassNamespace string
30+
31+
// TargetNamespaceFilter determines whether ClusterClasses should be copied to a given namespace.
32+
TargetNamespaceFilter func(ns *corev1.Namespace) bool
33+
}
34+
35+
var NamespaceHasLabelKey = func(key string) func(ns *corev1.Namespace) bool {
36+
return func(ns *corev1.Namespace) bool {
37+
_, ok := ns.GetLabels()[key]
38+
return ok
39+
}
40+
}
41+
42+
func (r *Reconciler) SetupWithManager(
43+
ctx context.Context,
44+
mgr ctrl.Manager,
45+
options controller.Options,
46+
) error {
47+
if r.TargetNamespaceFilter == nil {
48+
return fmt.Errorf("target Namespace filter is nil")
49+
}
50+
51+
err := ctrl.NewControllerManagedBy(mgr).
52+
For(&corev1.Namespace{},
53+
builder.WithPredicates(
54+
predicate.Funcs{
55+
CreateFunc: func(e event.CreateEvent) bool {
56+
// Called when an object is first seen by the cache, i.e. when a new object is created,
57+
// or when the cache is populated on start.
58+
ns, ok := e.Object.(*corev1.Namespace)
59+
if !ok {
60+
return false
61+
}
62+
return r.TargetNamespaceFilter(ns)
63+
},
64+
UpdateFunc: func(e event.UpdateEvent) bool {
65+
// Called when an object is already in the cache, and it is either updated,
66+
// or fetched as part of a re-list (aka re-sync).
67+
return false
68+
},
69+
DeleteFunc: func(e event.DeleteEvent) bool {
70+
// Ignore deletes.
71+
return false
72+
},
73+
GenericFunc: func(e event.GenericEvent) bool {
74+
// Ignore generic events, i.e. events that don't come from the API server.
75+
return false
76+
},
77+
},
78+
)).
79+
Watches(&clusterv1.ClusterClass{},
80+
handler.EnqueueRequestsFromMapFunc(r.clusterClassToNamespaces),
81+
).
82+
Named("syncclusterclass").
83+
WithOptions(options).
84+
Complete(r)
85+
if err != nil {
86+
return fmt.Errorf("failed to set up with controller manager: %w", err)
87+
}
88+
89+
return nil
90+
}
91+
92+
func (r *Reconciler) clusterClassToNamespaces(ctx context.Context, o client.Object) []ctrl.Request {
93+
namespaceList := &corev1.NamespaceList{}
94+
err := r.Client.List(ctx, namespaceList)
95+
if err != nil {
96+
// TODO Log the error, and record an Event.
97+
return nil
98+
}
99+
100+
rs := []ctrl.Request{}
101+
for i := range namespaceList.Items {
102+
ns := &namespaceList.Items[i]
103+
if r.TargetNamespaceFilter(ns) {
104+
rs = append(rs,
105+
ctrl.Request{
106+
NamespacedName: client.ObjectKeyFromObject(ns),
107+
},
108+
)
109+
}
110+
}
111+
return rs
112+
}
113+
114+
func (r *Reconciler) Reconcile(
115+
ctx context.Context,
116+
req ctrl.Request,
117+
) (
118+
_ ctrl.Result,
119+
rerr error,
120+
) {
121+
namespace := req.Name
122+
123+
sccs, err := r.listSourceClusterClasses(ctx)
124+
if err != nil {
125+
// TODO Record an Event.
126+
return ctrl.Result{}, fmt.Errorf("failed to list source ClusterClasses: %w", err)
127+
}
128+
129+
// TODO Consider running in parallel.
130+
for i := range sccs {
131+
scc := &sccs[i]
132+
err := copyClusterClassAndTemplates(
133+
ctx,
134+
r.Client,
135+
r.UnstructuredCachingClient,
136+
scc,
137+
namespace,
138+
)
139+
if client.IgnoreAlreadyExists(err) != nil {
140+
// TODO Record an Event.
141+
return ctrl.Result{}, fmt.Errorf(
142+
"failed to copy source ClusterClass %s or its referenced Templates to namespace %s: %w",
143+
client.ObjectKeyFromObject(scc),
144+
namespace,
145+
err,
146+
)
147+
}
148+
}
149+
150+
// TODO Record an Event.
151+
return ctrl.Result{}, nil
152+
}
153+
154+
func (r *Reconciler) listSourceClusterClasses(
155+
ctx context.Context,
156+
) (
157+
[]clusterv1.ClusterClass,
158+
error,
159+
) {
160+
ccl := &clusterv1.ClusterClassList{}
161+
err := r.Client.List(ctx, ccl, client.InNamespace(r.SourceClusterClassNamespace))
162+
if err != nil {
163+
return nil, err
164+
}
165+
return ccl.Items, nil
166+
}

‎pkg/controllers/namespacesync/controller_test.go

Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
// Copyright 2024 Nutanix. All rights reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
package namespacesync
4+
5+
import (
6+
"context"
7+
"fmt"
8+
"testing"
9+
"time"
10+
11+
. "github.com/onsi/gomega"
12+
corev1 "k8s.io/api/core/v1"
13+
"k8s.io/apiserver/pkg/storage/names"
14+
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
15+
"sigs.k8s.io/controller-runtime/pkg/client"
16+
17+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/internal/test/builder"
18+
)
19+
20+
func TestReconcileNewNamespaces(t *testing.T) {
21+
g := NewWithT(t)
22+
timeout := 5 * time.Second
23+
24+
sourceClusterClassName, cleanup, err := createUniqueClusterClassAndTemplates(
25+
sourceClusterClassNamespace,
26+
)
27+
g.Expect(err).ToNot(HaveOccurred())
28+
defer func() {
29+
g.Expect(cleanup()).To(Succeed())
30+
}()
31+
32+
targetNamespaces, err := createTargetNamespaces(3)
33+
g.Expect(err).ToNot(HaveOccurred())
34+
35+
for _, targetNamespace := range targetNamespaces {
36+
g.Eventually(func() error {
37+
return verifyClusterClassAndTemplates(
38+
env.Client,
39+
sourceClusterClassName,
40+
targetNamespace.Name,
41+
)
42+
},
43+
timeout,
44+
).Should(Succeed())
45+
}
46+
}
47+
48+
func TestReconcileNewClusterClass(t *testing.T) {
49+
g := NewWithT(t)
50+
timeout := 5 * time.Second
51+
52+
targetNamespaces, err := createTargetNamespaces(3)
53+
g.Expect(err).ToNot(HaveOccurred())
54+
55+
sourceClusterClassName, cleanup, err := createUniqueClusterClassAndTemplates(
56+
sourceClusterClassNamespace,
57+
)
58+
g.Expect(err).ToNot(HaveOccurred())
59+
defer func() {
60+
g.Expect(cleanup()).To(Succeed())
61+
}()
62+
63+
for _, targetNamespace := range targetNamespaces {
64+
g.Eventually(func() error {
65+
return verifyClusterClassAndTemplates(
66+
env.Client,
67+
sourceClusterClassName,
68+
targetNamespace.Name,
69+
)
70+
},
71+
timeout,
72+
).Should(Succeed())
73+
}
74+
}
75+
76+
func verifyClusterClassAndTemplates(
77+
cli client.Reader,
78+
name,
79+
namespace string,
80+
) error {
81+
cc := &clusterv1.ClusterClass{}
82+
key := client.ObjectKey{
83+
Name: name,
84+
Namespace: namespace,
85+
}
86+
err := cli.Get(ctx, key, cc)
87+
if err != nil {
88+
return fmt.Errorf("failed to get ClusterClass %s: %w", key, err)
89+
}
90+
91+
return walkReferences(ctx, cc, func(ctx context.Context, ref *corev1.ObjectReference) error {
92+
_, err := getReference(ctx, cli, ref)
93+
return err
94+
})
95+
}
96+
97+
func createUniqueClusterClassAndTemplates(namespace string) (
98+
clusterClassName string,
99+
cleanup func() error,
100+
err error,
101+
) {
102+
return createClusterClassAndTemplates(
103+
names.SimpleNameGenerator.GenerateName("test-"),
104+
namespace,
105+
)
106+
}
107+
108+
func createClusterClassAndTemplates(
109+
prefix,
110+
namespace string,
111+
) (
112+
clusterClassName string,
113+
cleanup func() error,
114+
err error,
115+
) {
116+
// The below objects are created in order to feed the reconcile loop all the information it needs to create a
117+
// full tree of ClusterClass objects (the objects should have owner references to the ClusterClass).
118+
119+
// Bootstrap templates for the workers.
120+
bootstrapTemplate := builder.BootstrapTemplate(namespace, prefix).Build()
121+
122+
// InfraMachineTemplates for the workers and the control plane.
123+
infraMachineTemplateControlPlane := builder.InfrastructureMachineTemplate(
124+
namespace,
125+
fmt.Sprintf("%s-control-plane", prefix),
126+
).Build()
127+
infraMachineTemplateWorker := builder.InfrastructureMachineTemplate(
128+
namespace,
129+
fmt.Sprintf("%s-worker", prefix),
130+
).Build()
131+
132+
// Control plane template.
133+
controlPlaneTemplate := builder.ControlPlaneTemplate(namespace, prefix).Build()
134+
135+
// InfraClusterTemplate.
136+
infraClusterTemplate := builder.InfrastructureClusterTemplate(namespace, prefix).Build()
137+
138+
// MachineDeploymentClasses that will be part of the ClusterClass.
139+
machineDeploymentClass := builder.MachineDeploymentClass(fmt.Sprintf("%s-worker", prefix)).
140+
WithBootstrapTemplate(bootstrapTemplate).
141+
WithInfrastructureTemplate(infraMachineTemplateWorker).
142+
Build()
143+
144+
// ClusterClass.
145+
clusterClass := builder.ClusterClass(namespace, prefix).
146+
WithInfrastructureClusterTemplate(infraClusterTemplate).
147+
WithControlPlaneTemplate(controlPlaneTemplate).
148+
WithControlPlaneInfrastructureMachineTemplate(infraMachineTemplateControlPlane).
149+
WithWorkerMachineDeploymentClasses(*machineDeploymentClass).
150+
Build()
151+
152+
// Create the set of initObjects from the objects above to add to the API server when the test environment starts.
153+
154+
templates := []client.Object{
155+
bootstrapTemplate,
156+
infraMachineTemplateWorker,
157+
infraMachineTemplateControlPlane,
158+
controlPlaneTemplate,
159+
infraClusterTemplate,
160+
}
161+
162+
for _, obj := range templates {
163+
if err := env.CreateAndWait(ctx, obj); err != nil {
164+
return "", nil, err
165+
}
166+
}
167+
if err := env.CreateAndWait(ctx, clusterClass); err != nil {
168+
return "", nil, err
169+
}
170+
171+
cleanup = func() error {
172+
for _, obj := range templates {
173+
if err := env.CleanupAndWait(ctx, obj); err != nil {
174+
return err
175+
}
176+
}
177+
return env.CleanupAndWait(ctx, clusterClass)
178+
}
179+
180+
return clusterClass.Name, cleanup, nil
181+
}
182+
183+
func createTargetNamespaces(number int) ([]*corev1.Namespace, error) {
184+
targetNamespaces := []*corev1.Namespace{}
185+
for i := 0; i < number; i++ {
186+
targetNamespace, err := env.CreateNamespace(ctx, "target", map[string]string{
187+
targetNamespaceLabelKey: "",
188+
})
189+
if err != nil {
190+
return nil, err
191+
}
192+
targetNamespaces = append(targetNamespaces, targetNamespace)
193+
}
194+
return targetNamespaces, nil
195+
}

0 commit comments

Comments
 (0)