Skip to content

Commit b160a03

Browse files
dlipovetskydlipovetsky
authored
feat: Apply MetalLB configuration to remote cluster (#783)
**What problem does this PR solve?**: - Generates MetalLB configuration from ServiceLoadBalancer Configuration API (added in #778) - Waits for MetalLB helm chart to be successfully deployed (using waiter added in #777) - Applies MetalLB configuration to remote cluster, also using a wait. To apply MetalLB configuration, the MetalLB CRD webhooks must be ready. This is not guaranteed to happen once the MetalLB helm chart is successfully deployed. Because the MetalLB configuration is applied in a non-blocking lifecycle hook, the topology controller retries the hook immediately after it returns; we cannot ask for a delay. Because the topology controller backs off its requests exponentially, this often results in the hook requests being delayed for 3+ minutes before succeeding. To mitigate this exponential backoff, the hook retries internally for to up to 20 seconds, within the 30 seconds recommended by the Cluster API. <details> <summary>Related log excerpt</summary> ``` I0628 19:39:31.051929 1 handler.go:132] "Deploying ServiceLoadBalancer provider MetalLB" cluster="default/dlipovetsky" I0628 19:39:31.051954 1 handler.go:82] "Applying MetalLB installation" cluster="default/dlipovetsky" I0628 19:39:31.100706 1 cm.go:82] "Fetching HelmChart info for \"metallb\" from configmap default/default-helm-addons-config" cluster="default/dlipovetsky" E0628 19:39:41.031497 1 handler.go:140] "failed to deploy ServiceLoadBalancer provider MetalLB" err="context canceled: last apply error: failed to apply MetalLB configuration IPAddressPool metallb-system/metallb: server-side apply failed: Patch \"https://172.18.0.2:6443/apis/metallb.io/v1beta1/namespaces/metallb-system/ipaddresspools/metallb?fieldManager=cluster-api-runtime-extensions-nutanix&fieldValidation=Strict&timeout=10s\": context canceled" cluster="default/dlipovetsky" ... E0628 19:43:49.496027 1 handler.go:140] "failed to deploy ServiceLoadBalancer provider MetalLB" err="context canceled: last apply error: failed to apply MetalLB configuration IPAddressPool metallb-system/metallb: server-side apply failed: Internal error occurred: failed calling webhook \"ipaddresspoolvalidationwebhook.metallb.io\": failed to call webhook: Post \"https://metallb-webhook-service.metallb-system.svc:443/validate-metallb-io-v1beta1-ipaddresspool?timeout=10s\": dial tcp 10.130.84.48:443: connect: connection refused" cluster="default/dlipovetsky" I0628 19:43:51.785907 1 handler.go:132] "Deploying ServiceLoadBalancer provider MetalLB" cluster="default/dlipovetsky" I0628 19:43:51.785923 1 handler.go:82] "Applying MetalLB installation" cluster="default/dlipovetsky" I0628 19:43:51.790521 1 cm.go:82] "Fetching HelmChart info for \"metallb\" from configmap default/default-helm-addons-config" cluster="default/dlipovetsky" ``` </details> **Which issue(s) this PR fixes**: Fixes # **How Has This Been Tested?**: <!-- Please describe the tests that you ran to verify your changes. Provide output from the tests and any manual steps needed to replicate the tests. --> **Special notes for your reviewer**: <!-- Use this to provide any additional information to the reviewers. This may include: - Best way to review the PR. - Where the author wants the most review attention on. - etc. -->
1 parent 9039258 commit b160a03

File tree

4 files changed

+148
-0
lines changed

4 files changed

+148
-0
lines changed

‎pkg/handlers/generic/lifecycle/serviceloadbalancer/handler.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
type ServiceLoadBalancerProvider interface {
2323
Apply(
2424
ctx context.Context,
25+
slb v1alpha1.ServiceLoadBalancer,
2526
cluster *clusterv1.Cluster,
2627
log logr.Logger,
2728
) error
@@ -131,6 +132,7 @@ func (s *ServiceLoadBalancerHandler) apply(
131132
log.Info(fmt.Sprintf("Deploying ServiceLoadBalancer provider %s", slb.Provider))
132133
err = handler.Apply(
133134
ctx,
135+
slb,
134136
cluster,
135137
log,
136138
)

‎pkg/handlers/generic/lifecycle/serviceloadbalancer/handler_test.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ type fakeServiceLoadBalancerProvider struct {
2525

2626
func (p *fakeServiceLoadBalancerProvider) Apply(
2727
ctx context.Context,
28+
slb v1alpha1.ServiceLoadBalancer,
2829
cluster *clusterv1.Cluster,
2930
log logr.Logger,
3031
) error {

‎pkg/handlers/generic/lifecycle/serviceloadbalancer/metallb/configuration.go

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
// Copyright 2024 Nutanix. All rights reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
package metallb
4+
5+
import (
6+
"fmt"
7+
8+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
9+
"k8s.io/apimachinery/pkg/runtime/schema"
10+
11+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1"
12+
)
13+
14+
func groupVersionKind(kind string) schema.GroupVersionKind {
15+
return schema.GroupVersionKind{
16+
Group: "metallb.io",
17+
Version: "v1beta1",
18+
Kind: kind,
19+
}
20+
}
21+
22+
type configurationInput struct {
23+
name string
24+
namespace string
25+
addressRanges []v1alpha1.AddressRange
26+
}
27+
28+
func configurationObjects(input *configurationInput) ([]unstructured.Unstructured, error) {
29+
ipAddressPool := unstructured.Unstructured{}
30+
ipAddressPool.SetGroupVersionKind(groupVersionKind("IPAddressPool"))
31+
ipAddressPool.SetName(input.name)
32+
ipAddressPool.SetNamespace(input.namespace)
33+
34+
addresses := []string{}
35+
for _, ar := range input.addressRanges {
36+
addresses = append(addresses, fmt.Sprintf("%s-%s", ar.Start, ar.End))
37+
}
38+
if err := unstructured.SetNestedStringSlice(
39+
ipAddressPool.Object,
40+
addresses,
41+
"spec",
42+
"addresses",
43+
); err != nil {
44+
return nil, fmt.Errorf("failed to set IPAddressPool .spec.addresses: %w", err)
45+
}
46+
47+
l2Advertisement := unstructured.Unstructured{}
48+
l2Advertisement.SetGroupVersionKind(groupVersionKind("L2Advertisement"))
49+
l2Advertisement.SetName(input.name)
50+
l2Advertisement.SetNamespace(input.namespace)
51+
52+
if err := unstructured.SetNestedStringSlice(
53+
l2Advertisement.Object,
54+
[]string{
55+
ipAddressPool.GetName(),
56+
},
57+
"spec",
58+
"ipAddressPools",
59+
); err != nil {
60+
return nil, fmt.Errorf("failed to set L2Advertisement .spec.ipAddressPools: %w", err)
61+
}
62+
63+
return []unstructured.Unstructured{
64+
ipAddressPool,
65+
l2Advertisement,
66+
}, nil
67+
}

‎pkg/handlers/generic/lifecycle/serviceloadbalancer/metallb/handler.go

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,20 +6,25 @@ package metallb
66
import (
77
"context"
88
"fmt"
9+
"time"
910

1011
"github.com/go-logr/logr"
1112
"github.com/spf13/pflag"
13+
corev1 "k8s.io/api/core/v1"
1214
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
15+
kwait "k8s.io/apimachinery/pkg/util/wait"
1316
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
1417
"sigs.k8s.io/cluster-api/controllers/remote"
1518
ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
1619
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
1720

1821
caaphv1 "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/external/sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1"
22+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1"
1923
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/k8s/client"
2024
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/config"
2125
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/options"
2226
handlersutils "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/utils"
27+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/wait"
2328
)
2429

2530
const (
@@ -70,6 +75,7 @@ func New(
7075

7176
func (n *MetalLB) Apply(
7277
ctx context.Context,
78+
slb v1alpha1.ServiceLoadBalancer,
7379
cluster *clusterv1.Cluster,
7480
log logr.Logger,
7581
) error {
@@ -152,5 +158,77 @@ func (n *MetalLB) Apply(
152158
return fmt.Errorf("failed to apply MetalLB installation HelmChartProxy: %w", err)
153159
}
154160

161+
if err := wait.ForObject(
162+
ctx,
163+
wait.ForObjectInput[*caaphv1.HelmChartProxy]{
164+
Reader: n.client,
165+
Target: hcp.DeepCopy(),
166+
Check: func(_ context.Context, obj *caaphv1.HelmChartProxy) (bool, error) {
167+
for _, c := range obj.GetConditions() {
168+
if c.Type == caaphv1.HelmReleaseProxiesReadyCondition && c.Status == corev1.ConditionTrue {
169+
return true, nil
170+
}
171+
}
172+
return false, nil
173+
},
174+
Interval: 5 * time.Second,
175+
Timeout: 30 * time.Second,
176+
},
177+
); err != nil {
178+
return fmt.Errorf("failed to wait for MetalLB to deploy: %w", err)
179+
}
180+
181+
log.Info(
182+
fmt.Sprintf("Applying MetalLB configuration to cluster %s",
183+
ctrlclient.ObjectKeyFromObject(cluster),
184+
),
185+
)
186+
187+
cos, err := configurationObjects(&configurationInput{
188+
name: defaultHelmReleaseName,
189+
namespace: defaultHelmReleaseNamespace,
190+
addressRanges: slb.Configuration.AddressRanges,
191+
})
192+
if err != nil {
193+
return fmt.Errorf("failed to generate MetalLB configuration: %w", err)
194+
}
195+
196+
var applyErr error
197+
if waitErr := kwait.PollUntilContextTimeout(
198+
ctx,
199+
5*time.Second,
200+
30*time.Second,
201+
true,
202+
func(ctx context.Context) (done bool, err error) {
203+
for i := range cos {
204+
o := &cos[i]
205+
if err = client.ServerSideApply(
206+
ctx,
207+
remoteClient,
208+
o,
209+
&ctrlclient.PatchOptions{
210+
Raw: &metav1.PatchOptions{
211+
FieldValidation: metav1.FieldValidationStrict,
212+
},
213+
},
214+
); err != nil {
215+
applyErr = fmt.Errorf(
216+
"failed to apply MetalLB configuration %s %s: %w",
217+
o.GetKind(),
218+
ctrlclient.ObjectKeyFromObject(o),
219+
err,
220+
)
221+
return false, nil
222+
}
223+
}
224+
return true, nil
225+
},
226+
); waitErr != nil {
227+
if applyErr != nil {
228+
return fmt.Errorf("%w: last apply error: %w", waitErr, applyErr)
229+
}
230+
return fmt.Errorf("%w: failed to apply MetalLB configuration", waitErr)
231+
}
232+
155233
return nil
156234
}

0 commit comments

Comments
 (0)