Skip to content

Commit a603e7f

Browse files
dlipovetskydlipovetsky
committed
test: Verify ServiceLoadBalancer in e2e Docker and Nutanix tests
- Add test helper functions. - Configure ServiceLoadBalancer in test cluster templates. Address ranges are within the TEST-2-NET range defined in RFC 5735. - Define kustomize patch to deploy MetalLB. - Apply kustomize patch to Docker and Nutanix example Clusters.
1 parent b5d15ca commit a603e7f

16 files changed

+248
-0
lines changed

‎examples/capi-quick-start/docker-cluster-calico-crs.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,14 @@ spec:
4040
strategy: ClusterResourceSet
4141
nfd:
4242
strategy: ClusterResourceSet
43+
serviceLoadBalancer:
44+
configuration:
45+
addressRanges:
46+
- end: 198.18.1.10
47+
start: 198.18.1.1
48+
- end: 198.18.1.30
49+
start: 198.18.1.21
50+
provider: MetalLB
4351
encryptionAtRest:
4452
providers:
4553
- aescbc: {}

‎examples/capi-quick-start/docker-cluster-calico-helm-addon.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,14 @@ spec:
3535
default: {}
3636
snapshotController: {}
3737
nfd: {}
38+
serviceLoadBalancer:
39+
configuration:
40+
addressRanges:
41+
- end: 198.18.1.10
42+
start: 198.18.1.1
43+
- end: 198.18.1.30
44+
start: 198.18.1.21
45+
provider: MetalLB
3846
encryptionAtRest:
3947
providers:
4048
- aescbc: {}

‎examples/capi-quick-start/docker-cluster-cilium-crs.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,14 @@ spec:
4040
strategy: ClusterResourceSet
4141
nfd:
4242
strategy: ClusterResourceSet
43+
serviceLoadBalancer:
44+
configuration:
45+
addressRanges:
46+
- end: 198.18.1.10
47+
start: 198.18.1.1
48+
- end: 198.18.1.30
49+
start: 198.18.1.21
50+
provider: MetalLB
4351
encryptionAtRest:
4452
providers:
4553
- aescbc: {}

‎examples/capi-quick-start/docker-cluster-cilium-helm-addon.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,14 @@ spec:
3535
default: {}
3636
snapshotController: {}
3737
nfd: {}
38+
serviceLoadBalancer:
39+
configuration:
40+
addressRanges:
41+
- end: 198.18.1.10
42+
start: 198.18.1.1
43+
- end: 198.18.1.30
44+
start: 198.18.1.21
45+
provider: MetalLB
3846
encryptionAtRest:
3947
providers:
4048
- aescbc: {}

‎examples/capi-quick-start/nutanix-cluster-calico-crs.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,14 @@ spec:
9191
strategy: HelmAddon
9292
nfd:
9393
strategy: ClusterResourceSet
94+
serviceLoadBalancer:
95+
configuration:
96+
addressRanges:
97+
- end: 198.18.1.10
98+
start: 198.18.1.1
99+
- end: 198.18.1.30
100+
start: 198.18.1.21
101+
provider: MetalLB
94102
controlPlane:
95103
nutanix:
96104
machineDetails:

‎examples/capi-quick-start/nutanix-cluster-calico-helm-addon.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,14 @@ spec:
8888
snapshotController:
8989
strategy: HelmAddon
9090
nfd: {}
91+
serviceLoadBalancer:
92+
configuration:
93+
addressRanges:
94+
- end: 198.18.1.10
95+
start: 198.18.1.1
96+
- end: 198.18.1.30
97+
start: 198.18.1.21
98+
provider: MetalLB
9199
controlPlane:
92100
nutanix:
93101
machineDetails:

‎examples/capi-quick-start/nutanix-cluster-cilium-crs.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,14 @@ spec:
9191
strategy: HelmAddon
9292
nfd:
9393
strategy: ClusterResourceSet
94+
serviceLoadBalancer:
95+
configuration:
96+
addressRanges:
97+
- end: 198.18.1.10
98+
start: 198.18.1.1
99+
- end: 198.18.1.30
100+
start: 198.18.1.21
101+
provider: MetalLB
94102
controlPlane:
95103
nutanix:
96104
machineDetails:

‎examples/capi-quick-start/nutanix-cluster-cilium-helm-addon.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,14 @@ spec:
8888
snapshotController:
8989
strategy: HelmAddon
9090
nfd: {}
91+
serviceLoadBalancer:
92+
configuration:
93+
addressRanges:
94+
- end: 198.18.1.10
95+
start: 198.18.1.1
96+
- end: 198.18.1.30
97+
start: 198.18.1.21
98+
provider: MetalLB
9199
controlPlane:
92100
nutanix:
93101
machineDetails:

‎hack/examples/bases/docker/cluster/kustomization.yaml.tmpl

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,3 +41,8 @@ patches:
4141
- target:
4242
kind: Cluster
4343
path: ../../../patches/encryption.yaml
44+
45+
# Deploy ServiceLoadBalancer MetalLB
46+
- target:
47+
kind: Cluster
48+
path: ../../../patches/metallb.yaml

‎hack/examples/bases/nutanix/cluster/kustomization.yaml.tmpl

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,3 +77,8 @@ patches:
7777
- op: "replace"
7878
path: "/spec/topology/workers/machineDeployments/0/class"
7979
value: default-worker
80+
81+
# Deploy ServiceLoadBalancer MetalLB
82+
- target:
83+
kind: Cluster
84+
path: ../../../patches/metallb.yaml

‎hack/examples/patches/metallb.yaml

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# Copyright 2024 Nutanix. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
- op: "add"
5+
path: "/spec/topology/variables/0/value/addons/serviceLoadBalancer"
6+
value:
7+
provider: MetalLB
8+
configuration:
9+
addressRanges:
10+
- start: 198.18.1.1
11+
end: 198.18.1.10
12+
- start: 198.18.1.21
13+
end: 198.18.1.30

‎test/e2e/addon_helpers.go

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ type WaitForAddonsToBeReadyInWorkloadClusterInput struct {
2323
DaemonSetIntervals []interface{}
2424
HelmReleaseIntervals []interface{}
2525
ClusterResourceSetIntervals []interface{}
26+
ResourceIntervals []interface{}
2627
}
2728

2829
func WaitForAddonsToBeReadyInWorkloadCluster(
@@ -94,4 +95,17 @@ func WaitForAddonsToBeReadyInWorkloadCluster(
9495
ClusterResourceSetIntervals: input.ClusterResourceSetIntervals,
9596
},
9697
)
98+
99+
WaitForServiceLoadBalancerToBeReadyInWorkloadCluster(
100+
ctx,
101+
WaitForServiceLoadBalancerToBeReadyInWorkloadClusterInput{
102+
ServiceLoadBalancer: input.AddonsConfig.ServiceLoadBalancer,
103+
WorkloadCluster: input.WorkloadCluster,
104+
ClusterProxy: input.ClusterProxy,
105+
DeploymentIntervals: input.DeploymentIntervals,
106+
DaemonSetIntervals: input.DaemonSetIntervals,
107+
HelmReleaseIntervals: input.HelmReleaseIntervals,
108+
ResourceIntervals: input.ResourceIntervals,
109+
},
110+
)
97111
}

‎test/e2e/config/caren.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -224,3 +224,4 @@ intervals:
224224
default/wait-daemonset: [ "5m", "10s" ]
225225
default/wait-clusterresourceset: [ "5m", "10s" ]
226226
default/wait-helmrelease: [ "5m", "10s" ]
227+
default/wait-resource: [ "5m", "10s" ]

‎test/e2e/quick_start_test.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -195,6 +195,10 @@ var _ = Describe("Quick start", Serial, func() {
195195
flavour,
196196
"wait-clusterresourceset",
197197
),
198+
ResourceIntervals: testE2EConfig.GetIntervals(
199+
flavour,
200+
"wait-resource",
201+
),
198202
},
199203
)
200204

‎test/e2e/self_hosted_test.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,10 @@ var _ = Describe("Self-hosted", Serial, func() {
9999
flavour,
100100
"wait-clusterresourceset",
101101
),
102+
ResourceIntervals: e2eConfig.GetIntervals(
103+
flavour,
104+
"wait-resource",
105+
),
102106
},
103107
)
104108

‎test/e2e/serviceloadbalancer_helpers.go

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
//go:build e2e
2+
3+
// Copyright 2024 Nutanix. All rights reserved.
4+
// SPDX-License-Identifier: Apache-2.0
5+
6+
package e2e
7+
8+
import (
9+
"context"
10+
"fmt"
11+
12+
. "github.com/onsi/ginkgo/v2"
13+
. "github.com/onsi/gomega"
14+
appsv1 "k8s.io/api/apps/v1"
15+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
16+
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
17+
"sigs.k8s.io/cluster-api/test/framework"
18+
"sigs.k8s.io/controller-runtime/pkg/client"
19+
20+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1"
21+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/serviceloadbalancer/metallb"
22+
)
23+
24+
type WaitForServiceLoadBalancerToBeReadyInWorkloadClusterInput struct {
25+
ServiceLoadBalancer *v1alpha1.ServiceLoadBalancer
26+
WorkloadCluster *clusterv1.Cluster
27+
ClusterProxy framework.ClusterProxy
28+
DeploymentIntervals []interface{}
29+
DaemonSetIntervals []interface{}
30+
HelmReleaseIntervals []interface{}
31+
ResourceIntervals []interface{}
32+
}
33+
34+
func WaitForServiceLoadBalancerToBeReadyInWorkloadCluster(
35+
ctx context.Context,
36+
input WaitForServiceLoadBalancerToBeReadyInWorkloadClusterInput, //nolint:gocritic // This hugeParam is OK in tests.
37+
) {
38+
if input.ServiceLoadBalancer == nil {
39+
return
40+
}
41+
42+
switch providerName := input.ServiceLoadBalancer.Provider; providerName {
43+
case v1alpha1.ServiceLoadBalancerProviderMetalLB:
44+
waitForMetalLBServiceLoadBalancerToBeReadyInWorkloadCluster(
45+
ctx,
46+
waitForMetalLBServiceLoadBalancerToBeReadyInWorkloadClusterInput{
47+
workloadCluster: input.WorkloadCluster,
48+
clusterProxy: input.ClusterProxy,
49+
deploymentIntervals: input.DeploymentIntervals,
50+
daemonSetIntervals: input.DaemonSetIntervals,
51+
helmReleaseIntervals: input.HelmReleaseIntervals,
52+
resourceIntervals: input.ResourceIntervals,
53+
},
54+
)
55+
default:
56+
Fail(
57+
fmt.Sprintf(
58+
"Do not know how to wait for ServiceLoadBalancer provider %s to be ready",
59+
providerName,
60+
),
61+
)
62+
}
63+
}
64+
65+
type waitForMetalLBServiceLoadBalancerToBeReadyInWorkloadClusterInput struct {
66+
workloadCluster *clusterv1.Cluster
67+
clusterProxy framework.ClusterProxy
68+
helmReleaseIntervals []interface{}
69+
deploymentIntervals []interface{}
70+
daemonSetIntervals []interface{}
71+
resourceIntervals []interface{}
72+
}
73+
74+
func waitForMetalLBServiceLoadBalancerToBeReadyInWorkloadCluster(
75+
ctx context.Context,
76+
input waitForMetalLBServiceLoadBalancerToBeReadyInWorkloadClusterInput, //nolint:gocritic // OK in tests.
77+
) {
78+
WaitForHelmReleaseProxyReadyForCluster(
79+
ctx,
80+
WaitForHelmReleaseProxyReadyForClusterInput{
81+
GetLister: input.clusterProxy.GetClient(),
82+
Cluster: input.workloadCluster,
83+
HelmChartProxyName: "metallb-" + input.workloadCluster.Name,
84+
},
85+
input.helmReleaseIntervals...,
86+
)
87+
88+
workloadClusterClient := input.clusterProxy.GetWorkloadCluster(
89+
ctx, input.workloadCluster.Namespace, input.workloadCluster.Name,
90+
).GetClient()
91+
92+
WaitForDeploymentsAvailable(ctx, framework.WaitForDeploymentsAvailableInput{
93+
Getter: workloadClusterClient,
94+
Deployment: &appsv1.Deployment{
95+
ObjectMeta: metav1.ObjectMeta{
96+
Name: "metallb-controller",
97+
Namespace: "metallb-system",
98+
},
99+
},
100+
}, input.deploymentIntervals...)
101+
102+
WaitForDaemonSetsAvailable(ctx, WaitForDaemonSetsAvailableInput{
103+
Getter: workloadClusterClient,
104+
DaemonSet: &appsv1.DaemonSet{
105+
ObjectMeta: metav1.ObjectMeta{
106+
Name: "metallb-speaker",
107+
Namespace: "metallb-system",
108+
},
109+
},
110+
}, input.daemonSetIntervals...)
111+
112+
// Generate the MetalLB configuration objects, so we can wait for them to be
113+
// created on the workload cluster.
114+
cos, err := metallb.ConfigurationObjects(&metallb.ConfigurationInput{
115+
Name: "metallb",
116+
Namespace: "metallb-system",
117+
// We need to populate AddressRanges to generate the configuration,
118+
// but the values are not important, because this test does not compare
119+
// them against the actual values.
120+
AddressRanges: []v1alpha1.AddressRange{
121+
{
122+
Start: "1.2.3.4",
123+
End: "1.2.3.5",
124+
},
125+
},
126+
})
127+
Expect(err).NotTo(HaveOccurred())
128+
129+
resources := make([]client.Object, len(cos))
130+
for i := range cos {
131+
resources[i] = cos[i]
132+
}
133+
134+
WaitForResources(ctx, WaitForResourcesInput{
135+
Getter: workloadClusterClient,
136+
Resources: resources,
137+
}, input.resourceIntervals...)
138+
}

0 commit comments

Comments
 (0)