Skip to content

Commit 98ce95c

Browse files
jimmidysonjimmidyson
committed
test(e2e): Add AWS e2e tests
These can be run via: ``` make e2e-test E2E_LABEL='provider:AWS && cni:Cilium && addonStrategy:ClusterResourceSet' ```
1 parent 0f522fc commit 98ce95c

File tree

15 files changed

+252
-110
lines changed

15 files changed

+252
-110
lines changed

‎charts/capi-runtime-extensions/defaultclusterclasses/aws-cluster-class.yaml

Lines changed: 11 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -21,29 +21,28 @@ spec:
2121
kind: AWSClusterTemplate
2222
name: aws-quick-start
2323
patches:
24-
- external:
25-
discoverVariablesExtension: awsclusterconfigvars.capi-runtime-extensions
26-
generateExtension: awsclusterconfigpatch.capi-runtime-extensions
27-
name: cluster-config
28-
- external:
29-
discoverVariablesExtension: awsworkerconfigvars.capi-runtime-extensions
30-
generateExtension: awsworkerconfigpatch.capi-runtime-extensions
31-
name: worker-config
3224
- definitions:
3325
- jsonPatches:
3426
- op: add
3527
path: /spec/template/spec/identityRef
36-
valueFrom:
37-
template: |
38-
kind: AWSClusterStaticIdentity
39-
name: {{ .builtin.cluster.name }}
28+
value:
29+
kind: AWSClusterControllerIdentity
30+
name: default
4031
selector:
4132
apiVersion: infrastructure.cluster.x-k8s.io/v1beta2
4233
kind: AWSClusterTemplate
4334
matchResources:
4435
infrastructureCluster: true
4536
description: AWSClusterStaticIdentity identityRef to use when creating the cluster
4637
name: identityRef
38+
- external:
39+
discoverVariablesExtension: awsclusterconfigvars.capi-runtime-extensions
40+
generateExtension: awsclusterconfigpatch.capi-runtime-extensions
41+
name: cluster-config
42+
- external:
43+
discoverVariablesExtension: awsworkerconfigvars.capi-runtime-extensions
44+
generateExtension: awsworkerconfigpatch.capi-runtime-extensions
45+
name: worker-config
4746
workers:
4847
machineDeployments:
4948
- class: default-worker

‎devbox.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
"shfmt@latest",
3232
"yamale@latest",
3333
"yamllint@latest",
34+
"path:./hack/flakes#clusterawsadm",
3435
"path:./hack/flakes#goprintconst",
3536
"path:./hack/flakes#go-mod-upgrade",
3637
"path:./hack/flakes#golangci-lint",

‎examples/capi-quick-start/aws-cluster-calico-crs.yaml

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,21 @@ spec:
3232
strategy: ClusterResourceSet
3333
aws:
3434
region: us-west-2
35+
controlPlane:
36+
aws:
37+
ami:
38+
lookup:
39+
baseOS: centos-7
40+
format: capa-ami-{{.BaseOS}}-?{{.K8sVersion}}-*
41+
org: "258751437250"
42+
- name: workerConfig
43+
value:
44+
aws:
45+
ami:
46+
lookup:
47+
baseOS: centos-7
48+
format: capa-ami-{{.BaseOS}}-?{{.K8sVersion}}-*
49+
org: "258751437250"
3550
version: ${KUBERNETES_VERSION}
3651
workers:
3752
machineDeployments:

‎examples/capi-quick-start/aws-cluster-calico-helm-addon.yaml

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,21 @@ spec:
3232
strategy: HelmAddon
3333
aws:
3434
region: us-west-2
35+
controlPlane:
36+
aws:
37+
ami:
38+
lookup:
39+
baseOS: centos-7
40+
format: capa-ami-{{.BaseOS}}-?{{.K8sVersion}}-*
41+
org: "258751437250"
42+
- name: workerConfig
43+
value:
44+
aws:
45+
ami:
46+
lookup:
47+
baseOS: centos-7
48+
format: capa-ami-{{.BaseOS}}-?{{.K8sVersion}}-*
49+
org: "258751437250"
3550
version: ${KUBERNETES_VERSION}
3651
workers:
3752
machineDeployments:

‎examples/capi-quick-start/aws-cluster-cilium-crs.yaml

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,21 @@ spec:
3232
strategy: ClusterResourceSet
3333
aws:
3434
region: us-west-2
35+
controlPlane:
36+
aws:
37+
ami:
38+
lookup:
39+
baseOS: centos-7
40+
format: capa-ami-{{.BaseOS}}-?{{.K8sVersion}}-*
41+
org: "258751437250"
42+
- name: workerConfig
43+
value:
44+
aws:
45+
ami:
46+
lookup:
47+
baseOS: centos-7
48+
format: capa-ami-{{.BaseOS}}-?{{.K8sVersion}}-*
49+
org: "258751437250"
3550
version: ${KUBERNETES_VERSION}
3651
workers:
3752
machineDeployments:

‎examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,21 @@ spec:
3232
strategy: HelmAddon
3333
aws:
3434
region: us-west-2
35+
controlPlane:
36+
aws:
37+
ami:
38+
lookup:
39+
baseOS: centos-7
40+
format: capa-ami-{{.BaseOS}}-?{{.K8sVersion}}-*
41+
org: "258751437250"
42+
- name: workerConfig
43+
value:
44+
aws:
45+
ami:
46+
lookup:
47+
baseOS: centos-7
48+
format: capa-ami-{{.BaseOS}}-?{{.K8sVersion}}-*
49+
org: "258751437250"
3550
version: ${KUBERNETES_VERSION}
3651
workers:
3752
machineDeployments:

‎hack/examples/bases/aws/kustomization.yaml.tmpl

Lines changed: 31 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,26 @@ patches:
3232
path: "/spec/topology/variables/0/value/aws"
3333
value:
3434
region: us-west-2
35+
- op: "add"
36+
path: "/spec/topology/variables/0/value/controlPlane"
37+
value:
38+
aws:
39+
ami:
40+
lookup:
41+
format: "capa-ami-{{.BaseOS}}-?{{.K8sVersion}}-*"
42+
org: "258751437250"
43+
baseOS: centos-7
44+
- op: "add"
45+
path: "/spec/topology/variables/-"
46+
value:
47+
name: "workerConfig"
48+
value:
49+
aws:
50+
ami:
51+
lookup:
52+
format: "capa-ami-{{.BaseOS}}-?{{.K8sVersion}}-*"
53+
org: "258751437250"
54+
baseOS: centos-7
3555
- op: "add"
3656
path: "/spec/topology/variables/0/value/addons/cpi"
3757
value: {}
@@ -67,30 +87,23 @@ patches:
6787
- jsonPatches:
6888
- op: add
6989
path: /spec/template/spec/identityRef
70-
valueFrom:
71-
template: |
72-
kind: AWSClusterStaticIdentity
73-
name: {{ .builtin.cluster.name }}
90+
value:
91+
kind: AWSClusterControllerIdentity
92+
name: default
7493
selector:
7594
apiVersion: infrastructure.cluster.x-k8s.io/v1beta2
7695
kind: AWSClusterTemplate
7796
matchResources:
7897
infrastructureCluster: true
7998
description: AWSClusterStaticIdentity identityRef to use when creating the cluster
80-
- op: "add"
81-
path: "/spec/patches/0"
82-
value:
83-
name: "cluster-config"
84-
external:
85-
generateExtension: "awsclusterconfigpatch.capi-runtime-extensions"
86-
discoverVariablesExtension: "awsclusterconfigvars.capi-runtime-extensions"
87-
- op: "add"
88-
path: "/spec/patches/1"
89-
value:
90-
name: "worker-config"
91-
external:
92-
generateExtension: "awsworkerconfigpatch.capi-runtime-extensions"
93-
discoverVariablesExtension: "awsworkerconfigvars.capi-runtime-extensions"
99+
- name: "cluster-config"
100+
external:
101+
generateExtension: "awsclusterconfigpatch.capi-runtime-extensions"
102+
discoverVariablesExtension: "awsclusterconfigvars.capi-runtime-extensions"
103+
- name: "worker-config"
104+
external:
105+
generateExtension: "awsworkerconfigpatch.capi-runtime-extensions"
106+
discoverVariablesExtension: "awsworkerconfigvars.capi-runtime-extensions"
94107
- op: "remove"
95108
path: "/spec/variables"
96109
- target:

‎hack/flakes/flake.nix

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,21 @@
5656
vendorHash = null;
5757
ldflags = [ "-s" "-w" ];
5858
};
59+
60+
clusterawsadm = buildGo122Module rec {
61+
name = "clusterawsadm";
62+
version = "2.3.1";
63+
src = fetchFromGitHub {
64+
owner = "kubernetes-sigs";
65+
repo = "cluster-api-provider-aws";
66+
rev = "v${version}";
67+
hash = "sha256-5SWi23h7Wvkbha2RTE8gMec7sKnHVTrbtTwF646Mml0=";
68+
};
69+
doCheck = false;
70+
subPackages = [ "cmd/clusterawsadm" ];
71+
vendorHash = "sha256-1osn61l9f6lGiDSnvFZJbFXD4dxWKB6qH2ETop9hQRU=";
72+
ldflags = [ "-s" "-w" ];
73+
};
5974
};
6075

6176
formatter = alejandra;

‎make/clusterctl.mk

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ clusterctl.init:
77
EXP_RUNTIME_SDK=true \
88
EXP_CLUSTER_RESOURCE_SET=true \
99
EXP_MACHINE_POOL=true \
10-
AWS_B64ENCODED_CREDENTIALS= \
10+
AWS_B64ENCODED_CREDENTIALS=$$(clusterawsadm bootstrap credentials encode-as-profile) \
1111
clusterctl init \
1212
--kubeconfig=$(KIND_KUBECONFIG) \
1313
--infrastructure docker,aws \

‎make/go.mk

Lines changed: 27 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -89,34 +89,34 @@ endif
8989
$(info $(M) running e2e tests$(if $(E2E_LABEL), labelled "$(E2E_LABEL)")$(if $(E2E_FOCUS), matching "$(E2E_FOCUS)"))
9090
env E2E_IMAGE_TAG="$$(gojq --raw-output '.version' $(REPO_ROOT)/dist/metadata.json)" \
9191
envsubst -no-unset -no-empty -i '$(E2E_CONF_FILE)' -o '$(E2E_CONF_FILE_ENVSUBST)'
92-
ginkgo run \
93-
--r \
94-
--show-node-events \
95-
--trace \
96-
--randomize-all \
97-
--randomize-suites \
98-
--fail-on-pending \
99-
--keep-going \
100-
$(if $(filter $(CI),true),-v) \
101-
--covermode=atomic \
102-
--coverprofile coverage-e2e.out \
103-
--procs=$(E2E_PARALLEL_NODES) \
104-
--compilers=$(E2E_PARALLEL_NODES) \
105-
--flake-attempts=$(E2E_FLAKE_ATTEMPTS) \
106-
$(if $(E2E_FOCUS),--focus="$(E2E_FOCUS)") \
107-
$(if $(E2E_SKIP),--skip="$(E2E_SKIP)") \
108-
$(if $(E2E_LABEL),--label-filter="$(E2E_LABEL)") \
109-
$(E2E_GINKGO_FLAGS) \
110-
--junit-report=junit-e2e.xml \
111-
--json-report=report-e2e.json \
112-
--tags e2e \
113-
test/e2e/... -- \
114-
-e2e.artifacts-folder="$(ARTIFACTS)" \
115-
-e2e.config="$(E2E_CONF_FILE_ENVSUBST)" \
116-
&& \
92+
env AWS_B64ENCODED_CREDENTIALS="$$(clusterawsadm bootstrap credentials encode-as-profile)" \
93+
ginkgo run \
94+
--r \
95+
--show-node-events \
96+
--trace \
97+
--randomize-all \
98+
--randomize-suites \
99+
--fail-on-pending \
100+
--keep-going \
101+
$(if $(filter $(CI),true),-v) \
102+
--covermode=atomic \
103+
--coverprofile coverage-e2e.out \
104+
--procs=$(E2E_PARALLEL_NODES) \
105+
--compilers=$(E2E_PARALLEL_NODES) \
106+
--flake-attempts=$(E2E_FLAKE_ATTEMPTS) \
107+
$(if $(E2E_FOCUS),--focus="$(E2E_FOCUS)") \
108+
$(if $(E2E_SKIP),--skip="$(E2E_SKIP)") \
109+
$(if $(E2E_LABEL),--label-filter="$(E2E_LABEL)") \
110+
$(E2E_GINKGO_FLAGS) \
111+
--junit-report=junit-e2e.xml \
112+
--json-report=report-e2e.json \
113+
--tags e2e \
114+
test/e2e/... -- \
115+
-e2e.artifacts-folder="$(ARTIFACTS)" \
116+
-e2e.config="$(E2E_CONF_FILE_ENVSUBST)" \
117117
go tool cover \
118-
-html=coverage-e2e.out \
119-
-o coverage-e2e.html
118+
-html=coverage-e2e.out \
119+
-o coverage-e2e.html
120120
endif
121121

122122
GOLANGCI_CONFIG_FILE ?= $(wildcard $(REPO_ROOT)/.golangci.y*ml)

‎test/e2e/config/cre.yaml

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,29 @@ providers:
4747
- old: --metrics-addr=127.0.0.1:8080
4848
new: --metrics-addr=:8080
4949

50+
- name: aws
51+
type: InfrastructureProvider
52+
versions:
53+
- name: "{go://sigs.k8s.io/cluster-api-provider-aws/[email protected]}"
54+
value: "https://github.com/kubernetes-sigs/cluster-api-provider-aws/releases/download/{go://sigs.k8s.io/cluster-api-provider-aws/[email protected]}/infrastructure-components.yaml"
55+
type: "url"
56+
contract: v1beta1
57+
files:
58+
- sourcePath: "../data/shared/v1beta1-capa/metadata.yaml"
59+
- sourcePath: "../../../charts/capi-runtime-extensions/defaultclusterclasses/aws-cluster-class.yaml"
60+
targetName: clusterclass-aws-quick-start.yaml
61+
- sourcePath: "../../../examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml"
62+
targetName: cluster-template-topology-cilium-helm-addon.yaml
63+
- sourcePath: "../../../examples/capi-quick-start/aws-cluster-cilium-crs.yaml"
64+
targetName: cluster-template-topology-cilium-crs.yaml
65+
- sourcePath: "../../../examples/capi-quick-start/aws-cluster-calico-helm-addon.yaml"
66+
targetName: cluster-template-topology-calico-helm-addon.yaml
67+
- sourcePath: "../../../examples/capi-quick-start/aws-cluster-calico-crs.yaml"
68+
targetName: cluster-template-topology-calico-crs.yaml
69+
replacements:
70+
- old: --metrics-addr=127.0.0.1:8080
71+
new: --metrics-addr=:8080
72+
5073
- name: docker
5174
type: InfrastructureProvider
5275
versions:
@@ -116,8 +139,8 @@ variables:
116139
# allowing the same e2e config file to be re-used in different Prow jobs e.g. each one with a K8s version permutation.
117140
# The following Kubernetes versions should be the latest versions with already published kindest/node images.
118141
# This avoids building node images in the default case which improves the test duration significantly.
119-
KUBERNETES_VERSION_MANAGEMENT: "v1.29.0"
120-
KUBERNETES_VERSION: "v1.29.0"
142+
KUBERNETES_VERSION_MANAGEMENT: "v1.29.2"
143+
KUBERNETES_VERSION: "v1.28.3"
121144
SERVICE_CIDR: "10.128.0.0/12"
122145
POD_CIDR: "192.168.0.0/16"
123146
NODE_DRAIN_TIMEOUT: "60s"

‎test/e2e/data/shared/v1beta1-capa/metadata.yaml

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# Copyright 2024 D2iQ, Inc. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
# maps release series of major.minor to cluster-api contract version
5+
# the contract version may change between minor or major versions, but *not*
6+
# between patch versions.
7+
#
8+
# update this file only when a new major or minor version is released
9+
apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3
10+
releaseSeries:
11+
- major: 2
12+
minor: 3
13+
contract: v1beta1

‎test/e2e/e2e_suite_test.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ import (
2121
"k8s.io/apimachinery/pkg/runtime"
2222
"k8s.io/klog/v2"
2323
addonsv1 "sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1"
24-
capi_e2e "sigs.k8s.io/cluster-api/test/e2e"
24+
capie2e "sigs.k8s.io/cluster-api/test/e2e"
2525
"sigs.k8s.io/cluster-api/test/framework"
2626
capibootstrap "sigs.k8s.io/cluster-api/test/framework/bootstrap"
2727
"sigs.k8s.io/cluster-api/test/framework/clusterctl"
@@ -141,9 +141,9 @@ var _ = SynchronizedBeforeSuite(func() []byte {
141141

142142
// we unset Kubernetes version variables to make sure we use the ones resolved from the first Ginkgo ParallelNode in
143143
// the e2e config.
144-
os.Unsetenv(capi_e2e.KubernetesVersion)
145-
os.Unsetenv(capi_e2e.KubernetesVersionUpgradeFrom)
146-
os.Unsetenv(capi_e2e.KubernetesVersionUpgradeTo)
144+
os.Unsetenv(capie2e.KubernetesVersion)
145+
os.Unsetenv(capie2e.KubernetesVersionUpgradeFrom)
146+
os.Unsetenv(capie2e.KubernetesVersionUpgradeTo)
147147

148148
kubeconfigPath := parts[3]
149149
bootstrapClusterProxy = framework.NewClusterProxy(

0 commit comments

Comments
 (0)