Skip to content

Commit c05b791

Browse files
alexander-demicevrichardcase
alexander-demicev
and
richardcase
committed
Add EKS clusterclass e2e suite
Signed-off-by: Alexandr Demicev <[email protected]> Co-authored-by: Richard Case <[email protected]>
1 parent 9a67dc3 commit c05b791

File tree

7 files changed

+244
-1
lines changed

7 files changed

+244
-1
lines changed

‎test/e2e/data/e2e_eks_conf.yaml

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ images:
1616
- name: gcr.io/k8s-staging-cluster-api/capa-manager:e2e
1717
loadBehavior: mustLoad
1818

19-
## PLEASE KEEP THESE UP TO DATE WITH THE COMPONENTS
19+
## PLEASE KEEP THESE UP TO DATE WITH THE COMPONENTS
2020
- name: quay.io/jetstack/cert-manager-cainjector:v1.16.1
2121
loadBehavior: tryLoad
2222
- name: quay.io/jetstack/cert-manager-webhook:v1.16.1
@@ -116,6 +116,8 @@ providers:
116116
targetName: "cluster-template-eks-control-plane-only-legacy.yaml"
117117
- sourcePath: "./eks/cluster-template-eks-control-plane-bare-eks.yaml"
118118
targetName: "cluster-template-eks-control-plane-bare-eks.yaml"
119+
- sourcePath: "./infrastructure-aws/withclusterclass/kustomize_sources/eks-clusterclass/clusterclass-eks-e2e.yaml"
120+
- sourcePath: "./infrastructure-aws/withclusterclass/generated/cluster-template-eks-clusterclass.yaml"
119121

120122
variables:
121123
KUBERNETES_VERSION: "v1.31.0"
@@ -124,6 +126,7 @@ variables:
124126
UPGRADE_FROM_VERSION: "v1.30.8"
125127
EXP_MACHINE_POOL: "true"
126128
EXP_CLUSTER_RESOURCE_SET: "true"
129+
CLUSTER_TOPOLOGY: "true"
127130
EVENT_BRIDGE_INSTANCE_STATE: "true"
128131
AWS_NODE_MACHINE_TYPE: t3.large
129132
AWS_MACHINE_TYPE_VCPU_USAGE: 2

‎test/e2e/data/infrastructure-aws/withclusterclass/kustomize_sources/eks-clusterclass/cluster-template.yaml

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
apiVersion: cluster.x-k8s.io/v1beta1
2+
kind: Cluster
3+
metadata:
4+
name: "${CLUSTER_NAME}"
5+
spec:
6+
clusterNetwork:
7+
pods:
8+
cidrBlocks:
9+
- 192.168.0.0/16
10+
topology:
11+
class: eks-e2e
12+
version: "${KUBERNETES_VERSION}"
13+
workers:
14+
machineDeployments:
15+
- class: default-worker
16+
name: md-0
17+
replicas: ${WORKER_MACHINE_COUNT}

‎test/e2e/data/infrastructure-aws/withclusterclass/kustomize_sources/eks-clusterclass/clusterclass-eks-e2e.yaml

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
apiVersion: cluster.x-k8s.io/v1beta1
2+
kind: ClusterClass
3+
metadata:
4+
name: eks-e2e
5+
spec:
6+
controlPlane:
7+
ref:
8+
apiVersion: controlplane.cluster.x-k8s.io/v1beta2
9+
kind: AWSManagedControlPlaneTemplate
10+
name: "${CLUSTER_NAME}-control-plane"
11+
infrastructure:
12+
ref:
13+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta2
14+
kind: AWSManagedClusterTemplate
15+
name: "${CLUSTER_NAME}"
16+
workers:
17+
machineDeployments:
18+
- class: default-worker
19+
template:
20+
bootstrap:
21+
ref:
22+
name: "${CLUSTER_NAME}-md-0"
23+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta2
24+
kind: EKSConfigTemplate
25+
infrastructure:
26+
ref:
27+
name: "${CLUSTER_NAME}-md-0"
28+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta2
29+
kind: AWSMachineTemplate
30+
---
31+
kind: AWSManagedClusterTemplate
32+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta2
33+
metadata:
34+
name: "${CLUSTER_NAME}"
35+
spec:
36+
template:
37+
spec: {}
38+
---
39+
kind: AWSManagedControlPlaneTemplate
40+
apiVersion: controlplane.cluster.x-k8s.io/v1beta2
41+
metadata:
42+
name: "${CLUSTER_NAME}-control-plane"
43+
spec:
44+
template:
45+
spec:
46+
region: "${AWS_REGION}"
47+
sshKeyName: "${AWS_SSH_KEY_NAME}"
48+
version: "${KUBERNETES_VERSION}"
49+
---
50+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta2
51+
kind: AWSMachineTemplate
52+
metadata:
53+
name: "${CLUSTER_NAME}-md-0"
54+
spec:
55+
template:
56+
spec:
57+
instanceType: "${AWS_NODE_MACHINE_TYPE}"
58+
iamInstanceProfile: "nodes.cluster-api-provider-aws.sigs.k8s.io"
59+
sshKeyName: "${AWS_SSH_KEY_NAME}"
60+
---
61+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta2
62+
kind: EKSConfigTemplate
63+
metadata:
64+
name: "${CLUSTER_NAME}-md-0"
65+
spec:
66+
template: {}

‎test/e2e/data/infrastructure-aws/withclusterclass/kustomize_sources/eks-clusterclass/kustomization.yaml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
resources:
2+
- cluster-template.yaml
3+
generatorOptions:
4+
disableNameSuffixHash: true
5+
labels:
6+
type: generated
7+
annotations:
8+
note: generated
9+

‎test/e2e/suites/managed/control_plane_helpers.go

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,14 @@ import (
2828
"github.com/aws/aws-sdk-go/service/eks"
2929
. "github.com/onsi/ginkgo/v2"
3030
. "github.com/onsi/gomega"
31+
corev1 "k8s.io/api/core/v1"
3132
"k8s.io/apimachinery/pkg/util/version"
3233
crclient "sigs.k8s.io/controller-runtime/pkg/client"
3334

3435
ekscontrolplanev1 "sigs.k8s.io/cluster-api-provider-aws/v2/controlplane/eks/api/v1beta2"
36+
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
3537
"sigs.k8s.io/cluster-api/test/framework"
38+
clusterctl "sigs.k8s.io/cluster-api/test/framework/clusterctl"
3639
)
3740

3841
type waitForControlPlaneToBeUpgradedInput struct {
@@ -93,3 +96,73 @@ func GetControlPlaneByName(ctx context.Context, input GetControlPlaneByNameInput
9396
Expect(input.Getter.Get(ctx, key, cp)).To(Succeed(), "Failed to get AWSManagedControlPlane object %s/%s", input.Namespace, input.Name)
9497
return cp
9598
}
99+
100+
func WaitForEKSControlPlaneInitialized(ctx context.Context, input clusterctl.ApplyCustomClusterTemplateAndWaitInput, result *clusterctl.ApplyCustomClusterTemplateAndWaitResult) {
101+
Expect(ctx).NotTo(BeNil(), "ctx is required for WaitForEKSControlPlaneInitialized")
102+
Expect(input.ClusterProxy).ToNot(BeNil(), "Invalid argument. input.ClusterProxy can't be nil")
103+
104+
var awsCP ekscontrolplanev1.AWSManagedControlPlane
105+
Eventually(func(g Gomega) {
106+
list, err := listAWSManagedControlPlanes(ctx, input.ClusterProxy.GetClient(), result.Cluster.Namespace, result.Cluster.Name)
107+
g.Expect(err).To(Succeed(), "failed to list AWSManagedControlPlane resource")
108+
109+
g.Expect(len(list.Items)).To(Equal(1),
110+
"expected exactly one AWSManagedControlPlane for %s/%s",
111+
result.Cluster.Namespace, result.Cluster.Name,
112+
)
113+
awsCP = list.Items[0]
114+
}, 10*time.Second, 1*time.Second).Should(Succeed())
115+
116+
key := crclient.ObjectKey{Namespace: awsCP.Namespace, Name: awsCP.Name}
117+
waitForControlPlaneReady(ctx, input.ClusterProxy.GetClient(), key, input.WaitForControlPlaneIntervals...)
118+
}
119+
120+
func WaitForEKSControlPlaneMachinesReady(ctx context.Context, input clusterctl.ApplyCustomClusterTemplateAndWaitInput, result *clusterctl.ApplyCustomClusterTemplateAndWaitResult) {
121+
Expect(ctx).NotTo(BeNil(), "ctx is required for WaitForEKSControlPlaneMachinesReady")
122+
Expect(input.ClusterProxy).ToNot(BeNil(), "input.ClusterProxy can't be nil")
123+
124+
var awsCP ekscontrolplanev1.AWSManagedControlPlane
125+
Eventually(func(g Gomega) {
126+
list, err := listAWSManagedControlPlanes(ctx, input.ClusterProxy.GetClient(), result.Cluster.Namespace, result.Cluster.Name)
127+
g.Expect(err).To(Succeed())
128+
awsCP = list.Items[0]
129+
130+
g.Expect(awsCP.Status.Ready).To(BeTrue(),
131+
"waiting for AWSManagedControlPlane %s/%s to become Ready",
132+
awsCP.Namespace, awsCP.Name,
133+
)
134+
}, input.WaitForControlPlaneIntervals...).Should(Succeed())
135+
136+
workloadClusterProxy := input.ClusterProxy.GetWorkloadCluster(ctx, result.Cluster.Namespace, input.ClusterName)
137+
waitForWorkloadClusterReachable(ctx, workloadClusterProxy.GetClient(), input.WaitForControlPlaneIntervals...)
138+
}
139+
140+
// listAWSManagedControlPlanes returns a list of AWSManagedControlPlanes for the given cluster.
141+
func listAWSManagedControlPlanes(ctx context.Context, client crclient.Client, namespace, clusterName string) (*ekscontrolplanev1.AWSManagedControlPlaneList, error) {
142+
list := &ekscontrolplanev1.AWSManagedControlPlaneList{}
143+
err := client.List(ctx, list,
144+
crclient.InNamespace(namespace),
145+
crclient.MatchingLabels{clusterv1.ClusterNameLabel: clusterName},
146+
)
147+
return list, err
148+
}
149+
150+
// waitForControlPlaneReady polls until the given AWSManagedControlPlane is marked Ready.
151+
func waitForControlPlaneReady(ctx context.Context, client crclient.Client, key crclient.ObjectKey, intervals ...interface{}) {
152+
Eventually(func(g Gomega) {
153+
var latest ekscontrolplanev1.AWSManagedControlPlane
154+
g.Expect(client.Get(ctx, key, &latest)).To(Succeed())
155+
g.Expect(latest.Status.Ready).To(BeTrue(),
156+
"AWSManagedControlPlane %s/%s is not Ready", key.Namespace, key.Name,
157+
)
158+
}, intervals...).Should(Succeed())
159+
}
160+
161+
// waitForWorkloadClusterReachable checks when the kube-system namespace is reachable in the workload cluster.
162+
func waitForWorkloadClusterReachable(ctx context.Context, client crclient.Client, intervals ...interface{}) {
163+
Eventually(func(g Gomega) {
164+
ns := &corev1.Namespace{}
165+
g.Expect(client.Get(ctx, crclient.ObjectKey{Name: "kube-system"}, ns)).
166+
To(Succeed(), "workload API server not yet reachable")
167+
}, intervals...).Should(Succeed())
168+
}

‎test/e2e/suites/managed/eks_clusterclass_test.go

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
//go:build e2e
2+
// +build e2e
3+
4+
/*
5+
Copyright 2025 The Kubernetes Authors.
6+
7+
Licensed under the Apache License, Version 2.0 (the "License");
8+
you may not use this file except in compliance with the License.
9+
You may obtain a copy of the License at
10+
11+
http://www.apache.org/licenses/LICENSE-2.0
12+
13+
Unless required by applicable law or agreed to in writing, software
14+
distributed under the License is distributed on an "AS IS" BASIS,
15+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
See the License for the specific language governing permissions and
17+
limitations under the License.
18+
*/
19+
20+
package managed
21+
22+
import (
23+
"context"
24+
"fmt"
25+
26+
"github.com/onsi/ginkgo/v2"
27+
. "github.com/onsi/gomega"
28+
"k8s.io/utils/ptr"
29+
30+
ekscontrolplanev1 "sigs.k8s.io/cluster-api-provider-aws/v2/controlplane/eks/api/v1beta2"
31+
"sigs.k8s.io/cluster-api-provider-aws/v2/test/e2e/shared"
32+
capi_e2e "sigs.k8s.io/cluster-api/test/e2e"
33+
"sigs.k8s.io/cluster-api/test/framework/clusterctl"
34+
"sigs.k8s.io/cluster-api/util"
35+
)
36+
37+
var _ = ginkgo.Describe("[managed] [general] EKS clusterclass tests", func() {
38+
const specName = "cluster"
39+
var clusterName string
40+
41+
ginkgo.BeforeEach(func() {
42+
if !runGeneralTests() {
43+
ginkgo.Skip("skipping due to unmet condition")
44+
}
45+
46+
ginkgo.By("should have a valid test configuration")
47+
Expect(e2eCtx.Environment.BootstrapClusterProxy).ToNot(BeNil(), "BootstrapClusterProxy can't be nil")
48+
Expect(e2eCtx.E2EConfig).ToNot(BeNil(), "E2EConfig can't be nil")
49+
Expect(e2eCtx.E2EConfig.Variables).To(HaveKey(shared.KubernetesVersion))
50+
Expect(e2eCtx.E2EConfig.Variables).To(HaveKey(shared.CNIAddonVersion))
51+
52+
clusterName = fmt.Sprintf("%s-%s", specName, util.RandomString(6))
53+
54+
ginkgo.By("default iam role should exist")
55+
VerifyRoleExistsAndOwned(ekscontrolplanev1.DefaultEKSControlPlaneRole, "", false, e2eCtx.BootstrapUserAWSSession)
56+
})
57+
58+
capi_e2e.QuickStartSpec(context.TODO(), func() capi_e2e.QuickStartSpecInput {
59+
return capi_e2e.QuickStartSpecInput{
60+
E2EConfig: e2eCtx.E2EConfig,
61+
ClusterctlConfigPath: e2eCtx.Environment.ClusterctlConfigPath,
62+
BootstrapClusterProxy: e2eCtx.Environment.BootstrapClusterProxy,
63+
ArtifactFolder: e2eCtx.Settings.ArtifactFolder,
64+
SkipCleanup: e2eCtx.Settings.SkipCleanup,
65+
Flavor: ptr.To(EKSClusterClassFlavor),
66+
ClusterName: ptr.To(clusterName),
67+
WorkerMachineCount: ptr.To(int64(3)),
68+
ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{
69+
WaitForControlPlaneInitialized: WaitForEKSControlPlaneInitialized,
70+
WaitForControlPlaneMachinesReady: WaitForEKSControlPlaneMachinesReady,
71+
},
72+
}
73+
})
74+
})

‎test/e2e/suites/managed/helpers.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ const (
5050
EKSMachinePoolOnlyFlavor = "eks-machinepool-only"
5151
EKSIPv6ClusterFlavor = "eks-ipv6-cluster"
5252
EKSControlPlaneOnlyLegacyFlavor = "eks-control-plane-only-legacy"
53+
EKSClusterClassFlavor = "eks-clusterclass"
5354
)
5455

5556
const (

0 commit comments

Comments
 (0)