Skip to content

Commit 658f01e

Browse files
jimmidysonjimmidyson
authored
test(e2e): Add CNI e2e tests (#383)
Support Calico and Cilium using ClusterResourceSet and HelmAddon strategies.
1 parent c730e61 commit 658f01e

File tree

11 files changed

+660
-199
lines changed

11 files changed

+660
-199
lines changed

‎.github/workflows/checks.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ jobs:
8484
${{ runner.os }}-go-
8585
8686
- name: Run e2e tests
87-
run: devbox run -- make e2e-test E2E_FOCUS='\[${{ matrix.PROVIDER }}\]'
87+
run: devbox run -- make e2e-test E2E_LABEL='provider:${{ matrix.PROVIDER }}'
8888
env:
8989
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
9090

‎test/e2e/clusterresourceset_helpers.go

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
//go:build e2e
2+
3+
// Copyright 2024 D2iQ, Inc. All rights reserved.
4+
// SPDX-License-Identifier: Apache-2.0
5+
6+
package e2e
7+
8+
import (
9+
"context"
10+
11+
. "github.com/onsi/gomega"
12+
"k8s.io/apimachinery/pkg/types"
13+
capiv1 "sigs.k8s.io/cluster-api/api/v1beta1"
14+
addonsv1 "sigs.k8s.io/cluster-api/exp/addons/api/v1beta1"
15+
"sigs.k8s.io/cluster-api/test/framework"
16+
)
17+
18+
type waitForClusterResourceSetToApplyResourcesInClusterInput struct {
19+
name string
20+
clusterProxy framework.ClusterProxy
21+
cluster *capiv1.Cluster
22+
intervals []interface{}
23+
}
24+
25+
func waitForClusterResourceSetToApplyResourcesInCluster(
26+
ctx context.Context,
27+
input waitForClusterResourceSetToApplyResourcesInClusterInput,
28+
) {
29+
crs := &addonsv1.ClusterResourceSet{}
30+
Expect(input.clusterProxy.GetClient().Get(
31+
ctx, types.NamespacedName{Name: input.name, Namespace: input.cluster.Namespace}, crs,
32+
)).To(Succeed())
33+
34+
framework.WaitForClusterResourceSetToApplyResources(
35+
ctx,
36+
framework.WaitForClusterResourceSetToApplyResourcesInput{
37+
ClusterProxy: input.clusterProxy,
38+
Cluster: input.cluster,
39+
ClusterResourceSet: crs,
40+
},
41+
input.intervals...,
42+
)
43+
}

‎test/e2e/cni_helpers.go

Lines changed: 275 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,275 @@
1+
//go:build e2e
2+
3+
// Copyright 2024 D2iQ, Inc. All rights reserved.
4+
// SPDX-License-Identifier: Apache-2.0
5+
6+
package e2e
7+
8+
import (
9+
"context"
10+
"fmt"
11+
12+
. "github.com/onsi/ginkgo/v2"
13+
appsv1 "k8s.io/api/apps/v1"
14+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
15+
capiv1 "sigs.k8s.io/cluster-api/api/v1beta1"
16+
"sigs.k8s.io/cluster-api/test/framework"
17+
18+
"github.com/d2iq-labs/capi-runtime-extensions/api/v1alpha1"
19+
)
20+
21+
type WaitForAddonsToBeReadyInWorkloadClusterInput struct {
22+
AddonsConfig v1alpha1.Addons
23+
WorkloadCluster *capiv1.Cluster
24+
ClusterProxy framework.ClusterProxy
25+
DeploymentIntervals []interface{}
26+
DaemonSetIntervals []interface{}
27+
HelmReleaseIntervals []interface{}
28+
ClusterResourceSetIntervals []interface{}
29+
}
30+
31+
func WaitForAddonsToBeReadyInWorkloadCluster(
32+
ctx context.Context,
33+
input WaitForAddonsToBeReadyInWorkloadClusterInput, //nolint:gocritic // This hugeParam is OK in tests.
34+
) {
35+
WaitForCNIToBeReadyInWorkloadCluster(
36+
ctx,
37+
WaitForCNIToBeReadyInWorkloadClusterInput{
38+
CNI: input.AddonsConfig.CNI,
39+
WorkloadCluster: input.WorkloadCluster,
40+
ClusterProxy: input.ClusterProxy,
41+
DeploymentIntervals: input.DeploymentIntervals,
42+
DaemonSetIntervals: input.DaemonSetIntervals,
43+
HelmReleaseIntervals: input.HelmReleaseIntervals,
44+
ClusterResourceSetIntervals: input.ClusterResourceSetIntervals,
45+
},
46+
)
47+
}
48+
49+
type WaitForCNIToBeReadyInWorkloadClusterInput struct {
50+
CNI *v1alpha1.CNI
51+
WorkloadCluster *capiv1.Cluster
52+
ClusterProxy framework.ClusterProxy
53+
DeploymentIntervals []interface{}
54+
DaemonSetIntervals []interface{}
55+
HelmReleaseIntervals []interface{}
56+
ClusterResourceSetIntervals []interface{}
57+
}
58+
59+
func WaitForCNIToBeReadyInWorkloadCluster(
60+
ctx context.Context,
61+
input WaitForCNIToBeReadyInWorkloadClusterInput, //nolint:gocritic // This hugeParam is OK in tests.
62+
) {
63+
if input.CNI == nil {
64+
return
65+
}
66+
67+
switch input.CNI.Provider {
68+
case v1alpha1.CNIProviderCalico:
69+
waitForCalicoToBeReadyInWorkloadCluster(
70+
ctx,
71+
waitForCalicoToBeReadyInWorkloadClusterInput{
72+
strategy: input.CNI.Strategy,
73+
workloadCluster: input.WorkloadCluster,
74+
clusterProxy: input.ClusterProxy,
75+
deploymentIntervals: input.DeploymentIntervals,
76+
daemonSetIntervals: input.DaemonSetIntervals,
77+
helmReleaseIntervals: input.HelmReleaseIntervals,
78+
clusterResourceSetIntervals: input.ClusterResourceSetIntervals,
79+
},
80+
)
81+
case v1alpha1.CNIProviderCilium:
82+
waitForCiliumToBeReadyInWorkloadCluster(
83+
ctx,
84+
waitForCiliumToBeReadyInWorkloadClusterInput{
85+
strategy: input.CNI.Strategy,
86+
workloadCluster: input.WorkloadCluster,
87+
clusterProxy: input.ClusterProxy,
88+
deploymentIntervals: input.DeploymentIntervals,
89+
daemonSetIntervals: input.DaemonSetIntervals,
90+
helmReleaseIntervals: input.HelmReleaseIntervals,
91+
clusterResourceSetIntervals: input.ClusterResourceSetIntervals,
92+
},
93+
)
94+
default:
95+
Fail(
96+
fmt.Sprintf(
97+
"Do not know how to wait for CNI provider %s to be ready",
98+
input.CNI.Provider,
99+
),
100+
)
101+
}
102+
}
103+
104+
type waitForCalicoToBeReadyInWorkloadClusterInput struct {
105+
strategy v1alpha1.AddonStrategy
106+
workloadCluster *capiv1.Cluster
107+
clusterProxy framework.ClusterProxy
108+
deploymentIntervals []interface{}
109+
daemonSetIntervals []interface{}
110+
helmReleaseIntervals []interface{}
111+
clusterResourceSetIntervals []interface{}
112+
}
113+
114+
func waitForCalicoToBeReadyInWorkloadCluster(
115+
ctx context.Context,
116+
input waitForCalicoToBeReadyInWorkloadClusterInput, //nolint:gocritic // This hugeParam is OK in tests.
117+
) {
118+
switch input.strategy {
119+
case v1alpha1.AddonStrategyClusterResourceSet:
120+
waitForClusterResourceSetToApplyResourcesInCluster(
121+
ctx,
122+
waitForClusterResourceSetToApplyResourcesInClusterInput{
123+
name: "calico-cni-installation-" + input.workloadCluster.Name,
124+
clusterProxy: input.clusterProxy,
125+
cluster: input.workloadCluster,
126+
intervals: input.clusterResourceSetIntervals,
127+
},
128+
)
129+
case v1alpha1.AddonStrategyHelmAddon:
130+
WaitForHelmReleaseProxyReadyForCluster(
131+
ctx,
132+
WaitForHelmReleaseProxyReadyForClusterInput{
133+
GetLister: input.clusterProxy.GetClient(),
134+
Cluster: input.workloadCluster,
135+
HelmChartProxyName: "calico-cni-installation-" + input.workloadCluster.Name,
136+
},
137+
input.helmReleaseIntervals...,
138+
)
139+
default:
140+
Fail(
141+
fmt.Sprintf(
142+
"Do not know how to wait for Calico using strategy %s to be ready",
143+
input.strategy,
144+
),
145+
)
146+
}
147+
148+
workloadClusterClient := input.clusterProxy.GetWorkloadCluster(
149+
ctx, input.workloadCluster.Namespace, input.workloadCluster.Name,
150+
).GetClient()
151+
152+
WaitForDeploymentsAvailable(ctx, framework.WaitForDeploymentsAvailableInput{
153+
Getter: workloadClusterClient,
154+
Deployment: &appsv1.Deployment{
155+
ObjectMeta: metav1.ObjectMeta{
156+
Name: "tigera-operator",
157+
Namespace: "tigera-operator",
158+
},
159+
},
160+
}, input.deploymentIntervals...)
161+
WaitForDeploymentsAvailable(ctx, framework.WaitForDeploymentsAvailableInput{
162+
Getter: workloadClusterClient,
163+
Deployment: &appsv1.Deployment{
164+
ObjectMeta: metav1.ObjectMeta{
165+
Name: "calico-typha",
166+
Namespace: "calico-system",
167+
},
168+
},
169+
}, input.deploymentIntervals...)
170+
WaitForDeploymentsAvailable(ctx, framework.WaitForDeploymentsAvailableInput{
171+
Getter: workloadClusterClient,
172+
Deployment: &appsv1.Deployment{
173+
ObjectMeta: metav1.ObjectMeta{
174+
Name: "calico-kube-controllers",
175+
Namespace: "calico-system",
176+
},
177+
},
178+
}, input.deploymentIntervals...)
179+
WaitForDaemonSetsAvailable(ctx, WaitForDaemonSetsAvailableInput{
180+
Getter: workloadClusterClient,
181+
DaemonSet: &appsv1.DaemonSet{
182+
ObjectMeta: metav1.ObjectMeta{
183+
Name: "calico-node",
184+
Namespace: "calico-system",
185+
},
186+
},
187+
}, input.daemonSetIntervals...)
188+
WaitForDaemonSetsAvailable(ctx, WaitForDaemonSetsAvailableInput{
189+
Getter: workloadClusterClient,
190+
DaemonSet: &appsv1.DaemonSet{
191+
ObjectMeta: metav1.ObjectMeta{
192+
Name: "csi-node-driver",
193+
Namespace: "calico-system",
194+
},
195+
},
196+
}, input.daemonSetIntervals...)
197+
WaitForDeploymentsAvailable(ctx, framework.WaitForDeploymentsAvailableInput{
198+
Getter: workloadClusterClient,
199+
Deployment: &appsv1.Deployment{
200+
ObjectMeta: metav1.ObjectMeta{
201+
Name: "calico-apiserver",
202+
Namespace: "calico-apiserver",
203+
},
204+
},
205+
}, input.deploymentIntervals...)
206+
}
207+
208+
type waitForCiliumToBeReadyInWorkloadClusterInput struct {
209+
strategy v1alpha1.AddonStrategy
210+
workloadCluster *capiv1.Cluster
211+
clusterProxy framework.ClusterProxy
212+
deploymentIntervals []interface{}
213+
daemonSetIntervals []interface{}
214+
helmReleaseIntervals []interface{}
215+
clusterResourceSetIntervals []interface{}
216+
}
217+
218+
func waitForCiliumToBeReadyInWorkloadCluster(
219+
ctx context.Context,
220+
input waitForCiliumToBeReadyInWorkloadClusterInput, //nolint:gocritic // This hugeParam is OK in tests.
221+
) {
222+
switch input.strategy {
223+
case v1alpha1.AddonStrategyClusterResourceSet:
224+
waitForClusterResourceSetToApplyResourcesInCluster(
225+
ctx,
226+
waitForClusterResourceSetToApplyResourcesInClusterInput{
227+
name: "cilium-cni-installation-" + input.workloadCluster.Name,
228+
clusterProxy: input.clusterProxy,
229+
cluster: input.workloadCluster,
230+
intervals: input.clusterResourceSetIntervals,
231+
},
232+
)
233+
case v1alpha1.AddonStrategyHelmAddon:
234+
WaitForHelmReleaseProxyReadyForCluster(
235+
ctx,
236+
WaitForHelmReleaseProxyReadyForClusterInput{
237+
GetLister: input.clusterProxy.GetClient(),
238+
Cluster: input.workloadCluster,
239+
HelmChartProxyName: "cilium-cni-installation-" + input.workloadCluster.Name,
240+
},
241+
input.helmReleaseIntervals...,
242+
)
243+
default:
244+
Fail(
245+
fmt.Sprintf(
246+
"Do not know how to wait for Cilium using strategy %s to be ready",
247+
input.strategy,
248+
),
249+
)
250+
}
251+
252+
workloadClusterClient := input.clusterProxy.GetWorkloadCluster(
253+
ctx, input.workloadCluster.Namespace, input.workloadCluster.Name,
254+
).GetClient()
255+
256+
WaitForDeploymentsAvailable(ctx, framework.WaitForDeploymentsAvailableInput{
257+
Getter: workloadClusterClient,
258+
Deployment: &appsv1.Deployment{
259+
ObjectMeta: metav1.ObjectMeta{
260+
Name: "cilium-operator",
261+
Namespace: "kube-system",
262+
},
263+
},
264+
}, input.deploymentIntervals...)
265+
266+
WaitForDaemonSetsAvailable(ctx, WaitForDaemonSetsAvailableInput{
267+
Getter: workloadClusterClient,
268+
DaemonSet: &appsv1.DaemonSet{
269+
ObjectMeta: metav1.ObjectMeta{
270+
Name: "cilium",
271+
Namespace: "kube-system",
272+
},
273+
},
274+
}, input.daemonSetIntervals...)
275+
}

‎test/e2e/common.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66
package e2e
77

88
import (
9+
"fmt"
10+
911
. "github.com/onsi/ginkgo/v2"
1012
"github.com/onsi/ginkgo/v2/types"
1113
)
@@ -22,3 +24,7 @@ func CheckTestBeforeCleanup() {
2224
}
2325
Logf("Cleaning up after \"%s\" spec", CurrentSpecReport().FullText())
2426
}
27+
28+
func Byf(format string, a ...interface{}) {
29+
By(fmt.Sprintf(format, a...))
30+
}

‎test/e2e/config/cre.yaml

Lines changed: 6 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -66,9 +66,9 @@ providers:
6666
targetName: cluster-template-topology-calico-helm-addon.yaml
6767
- sourcePath: "../../../examples/capi-quick-start/docker-cluster-calico-crs.yaml"
6868
targetName: cluster-template-topology-calico-crs.yaml
69-
replacements:
70-
- old: --metrics-addr=127.0.0.1:8080
71-
new: --metrics-addr=:8080
69+
replacements:
70+
- old: --metrics-addr=127.0.0.1:8080
71+
new: --metrics-addr=:8080
7272

7373
- name: helm
7474
type: AddonProvider
@@ -108,12 +108,8 @@ providers:
108108
new: "--v=8"
109109
- old: --metrics-addr=127.0.0.1:8080
110110
new: --metrics-addr=:8080
111-
# Ensure no images are pulled but are instead must have been loaded into the bootstrap cluster (see `images`
112-
# value above for the images to load).
113-
- old: "imagePullPolicy: IfNotPresent"
114-
new: "imagePullPolicy: Never"
115111
- old: "imagePullPolicy: Always"
116-
new: "imagePullPolicy: Never"
112+
new: "imagePullPolicy: IfNotPresent"
117113

118114
variables:
119115
# Default variables for the e2e test; those values could be overridden via env variables, thus
@@ -146,10 +142,9 @@ intervals:
146142
default/wait-machine-pool-upgrade: ["5m", "10s"]
147143
default/wait-nodes-ready: ["10m", "10s"]
148144
default/wait-machine-remediation: ["5m", "10s"]
149-
default/wait-autoscaler: ["5m", "10s"]
150145
default/wait-deployment: ["15m", "10s"]
151-
default/wait-nsg-update: [ "20m", "10s" ]
152146
default/wait-daemonset: [ "15m", "10s" ]
153-
default/wait-deployment-available: [ "15m", "10s" ]
147+
default/wait-clusterresourceset: [ "15m", "10s" ]
148+
default/wait-helmrelease: [ "15m", "10s" ]
154149
default/wait-job: [ "5m", "10s" ]
155150
default/wait-service: [ "15m", "10s" ]

0 commit comments

Comments
 (0)