Skip to content

Commit 3caed23

Browse files
jimmidysonjimmidyson
authored
test(e2e): Consistent core/bootstrap/control-plane provider versions (#639)
In line with other provider types, this commit ensures a consistent version is used for core, bootstrap, and control plane providers to ensure a repeatable e2e test setup.
1 parent 6047bec commit 3caed23

File tree

3 files changed

+109
-38
lines changed

3 files changed

+109
-38
lines changed

‎test/e2e/e2e_suite_test.go

Lines changed: 21 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,13 @@ import (
2222
"k8s.io/apimachinery/pkg/runtime"
2323
"k8s.io/klog/v2"
2424
capie2e "sigs.k8s.io/cluster-api/test/e2e"
25-
"sigs.k8s.io/cluster-api/test/framework"
25+
capie2eframework "sigs.k8s.io/cluster-api/test/framework"
2626
capibootstrap "sigs.k8s.io/cluster-api/test/framework/bootstrap"
2727
"sigs.k8s.io/cluster-api/test/framework/clusterctl"
2828
ctrl "sigs.k8s.io/controller-runtime"
2929

3030
helmaddonsv1 "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/external/sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1"
31+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/test/e2e/framework"
3132
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/test/framework/bootstrap"
3233
clusterctltemp "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/test/framework/clusterctl"
3334
)
@@ -147,11 +148,11 @@ var _ = SynchronizedBeforeSuite(func() []byte {
147148
os.Unsetenv(capie2e.KubernetesVersionUpgradeTo)
148149

149150
kubeconfigPath := parts[3]
150-
bootstrapClusterProxy = framework.NewClusterProxy(
151+
bootstrapClusterProxy = capie2eframework.NewClusterProxy(
151152
"bootstrap",
152153
kubeconfigPath,
153154
initScheme(),
154-
framework.WithMachineLogCollector(framework.DockerLogCollector{}),
155+
capie2eframework.WithMachineLogCollector(capie2eframework.DockerLogCollector{}),
155156
)
156157
})
157158

@@ -204,7 +205,7 @@ func createClusterctlLocalRepository(config *clusterctl.E2EConfig, repositoryFol
204205

205206
func initScheme() *runtime.Scheme {
206207
scheme := runtime.NewScheme()
207-
framework.TryAddDefaultSchemes(scheme)
208+
capie2eframework.TryAddDefaultSchemes(scheme)
208209
Expect(helmaddonsv1.AddToScheme(scheme)).To(Succeed())
209210
return scheme
210211
}
@@ -213,7 +214,7 @@ func setupBootstrapCluster(
213214
config *clusterctl.E2EConfig,
214215
scheme *runtime.Scheme,
215216
useExistingCluster bool,
216-
) (capibootstrap.ClusterProvider, framework.ClusterProxy) {
217+
) (capibootstrap.ClusterProvider, capie2eframework.ClusterProxy) {
217218
var clusterProvider capibootstrap.ClusterProvider
218219
kubeconfigPath := ""
219220
if !useExistingCluster {
@@ -245,13 +246,13 @@ func setupBootstrapCluster(
245246
Expect(err).To(BeNil(), "Failed to load images to the bootstrap cluster: %s", err)
246247
}
247248

248-
clusterProxy := framework.NewClusterProxy("bootstrap", kubeconfigPath, scheme)
249+
clusterProxy := capie2eframework.NewClusterProxy("bootstrap", kubeconfigPath, scheme)
249250
Expect(clusterProxy).NotTo(BeNil(), "Failed to get a bootstrap cluster proxy")
250251
return clusterProvider, clusterProxy
251252
}
252253

253254
func initBootstrapCluster(
254-
bootstrapClusterProxy framework.ClusterProxy,
255+
bootstrapClusterProxy capie2eframework.ClusterProxy,
255256
config *clusterctl.E2EConfig,
256257
clusterctlConfig, artifactFolder string,
257258
) {
@@ -260,6 +261,18 @@ func initBootstrapCluster(
260261
clusterctl.InitManagementClusterAndWatchControllerLogsInput{
261262
ClusterProxy: bootstrapClusterProxy,
262263
ClusterctlConfigPath: clusterctlConfig,
264+
CoreProvider: config.GetProviderLatestVersionsByContract(
265+
"*",
266+
framework.CoreProvider(config),
267+
)[0],
268+
BootstrapProviders: config.GetProviderLatestVersionsByContract(
269+
"*",
270+
framework.BootstrapProviders(config)...,
271+
),
272+
ControlPlaneProviders: config.GetProviderLatestVersionsByContract(
273+
"*",
274+
framework.ControlPlaneProviders(config)...,
275+
),
263276
InfrastructureProviders: config.GetProviderLatestVersionsByContract(
264277
"*",
265278
config.InfrastructureProviders()...),
@@ -280,7 +293,7 @@ func initBootstrapCluster(
280293

281294
func tearDown(
282295
bootstrapClusterProvider capibootstrap.ClusterProvider,
283-
bootstrapClusterProxy framework.ClusterProxy,
296+
bootstrapClusterProxy capie2eframework.ClusterProxy,
284297
) {
285298
if bootstrapClusterProxy != nil {
286299
bootstrapClusterProxy.Dispose(context.TODO())

‎test/e2e/framework/provider_utils.go

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
//go:build e2e
2+
3+
// Copyright 2024 Nutanix. All rights reserved.
4+
// SPDX-License-Identifier: Apache-2.0
5+
6+
package framework
7+
8+
import (
9+
clusterctlv1 "sigs.k8s.io/cluster-api/cmd/clusterctl/api/v1alpha3"
10+
"sigs.k8s.io/cluster-api/test/framework/clusterctl"
11+
)
12+
13+
// The functions in this file are used to get the provider names from the e2e config
14+
// based on the provider type. They are adaptations of the functions that already exist
15+
// on the E2EConfig type for other provider types, but unfortunately the existing functions
16+
// do not include core, bootstrap, and control plane providers.
17+
18+
func CoreProvider(cfg *clusterctl.E2EConfig) string {
19+
return getProviders(cfg, clusterctlv1.CoreProviderType)[0]
20+
}
21+
22+
func BootstrapProviders(cfg *clusterctl.E2EConfig) []string {
23+
return getProviders(cfg, clusterctlv1.BootstrapProviderType)
24+
}
25+
26+
func ControlPlaneProviders(cfg *clusterctl.E2EConfig) []string {
27+
return getProviders(cfg, clusterctlv1.ControlPlaneProviderType)
28+
}
29+
30+
func getProviders(cfg *clusterctl.E2EConfig, t clusterctlv1.ProviderType) []string {
31+
providers := []string{}
32+
for _, provider := range cfg.Providers {
33+
if provider.Type == string(t) {
34+
providers = append(providers, provider.Name)
35+
}
36+
}
37+
return providers
38+
}

‎test/e2e/framework/self_hosted.go

Lines changed: 50 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ import (
2121
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
2222
clusterctlv1 "sigs.k8s.io/cluster-api/cmd/clusterctl/api/v1alpha3"
2323
capie2e "sigs.k8s.io/cluster-api/test/e2e"
24-
"sigs.k8s.io/cluster-api/test/framework"
24+
capie2eframework "sigs.k8s.io/cluster-api/test/framework"
2525
"sigs.k8s.io/cluster-api/test/framework/bootstrap"
2626
"sigs.k8s.io/cluster-api/test/framework/clusterctl"
2727
"sigs.k8s.io/cluster-api/util"
@@ -32,7 +32,7 @@ import (
3232
type SelfHostedSpecInput struct {
3333
E2EConfig *clusterctl.E2EConfig
3434
ClusterctlConfigPath string
35-
BootstrapClusterProxy framework.ClusterProxy
35+
BootstrapClusterProxy capie2eframework.ClusterProxy
3636
ArtifactFolder string
3737
SkipCleanup bool
3838
ControlPlaneWaiters clusterctl.ControlPlaneWaiters
@@ -67,7 +67,7 @@ type SelfHostedSpecInput struct {
6767
WorkerMachineCount *int64
6868

6969
// PostClusterMoved is a function that is called after the cluster is moved to self-hosted.
70-
PostClusterMoved func(proxy framework.ClusterProxy, cluster *clusterv1.Cluster)
70+
PostClusterMoved func(proxy capie2eframework.ClusterProxy, cluster *clusterv1.Cluster)
7171
}
7272

7373
// SelfHostedSpec implements a test that verifies Cluster API creating a cluster, pivoting to a self-hosted cluster.
@@ -79,7 +79,7 @@ func SelfHostedSpec(ctx context.Context, inputGetter func() SelfHostedSpecInput)
7979
cancelWatches context.CancelFunc
8080
clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult
8181

82-
selfHostedClusterProxy framework.ClusterProxy
82+
selfHostedClusterProxy capie2eframework.ClusterProxy
8383
selfHostedNamespace *corev1.Namespace
8484
selfHostedCancelWatches context.CancelFunc
8585
selfHostedCluster *clusterv1.Cluster
@@ -212,13 +212,13 @@ func SelfHostedSpec(ctx context.Context, inputGetter func() SelfHostedSpecInput)
212212
ctx,
213213
cluster.Namespace,
214214
cluster.Name,
215-
framework.WithMachineLogCollector(input.BootstrapClusterProxy.GetLogCollector()),
215+
capie2eframework.WithMachineLogCollector(input.BootstrapClusterProxy.GetLogCollector()),
216216
)
217217

218218
capie2e.Byf("Creating a namespace for hosting the %s test spec", specName)
219-
selfHostedNamespace, selfHostedCancelWatches = framework.CreateNamespaceAndWatchEvents(
219+
selfHostedNamespace, selfHostedCancelWatches = capie2eframework.CreateNamespaceAndWatchEvents(
220220
ctx,
221-
framework.CreateNamespaceAndWatchEventsInput{
221+
capie2eframework.CreateNamespaceAndWatchEventsInput{
222222
Creator: selfHostedClusterProxy.GetClient(),
223223
ClientSet: selfHostedClusterProxy.GetClientSet(),
224224
Name: namespace.Name,
@@ -234,12 +234,29 @@ func SelfHostedSpec(ctx context.Context, inputGetter func() SelfHostedSpecInput)
234234
clusterctl.InitManagementClusterAndWatchControllerLogs(
235235
watchesCtx,
236236
clusterctl.InitManagementClusterAndWatchControllerLogsInput{
237-
ClusterProxy: selfHostedClusterProxy,
238-
ClusterctlConfigPath: input.ClusterctlConfigPath,
239-
InfrastructureProviders: input.E2EConfig.InfrastructureProviders(),
240-
IPAMProviders: input.E2EConfig.IPAMProviders(),
241-
RuntimeExtensionProviders: input.E2EConfig.RuntimeExtensionProviders(),
242-
AddonProviders: input.E2EConfig.AddonProviders(),
237+
ClusterProxy: selfHostedClusterProxy,
238+
ClusterctlConfigPath: input.ClusterctlConfigPath,
239+
CoreProvider: input.E2EConfig.GetProviderLatestVersionsByContract(
240+
"*",
241+
CoreProvider(input.E2EConfig),
242+
)[0],
243+
BootstrapProviders: input.E2EConfig.GetProviderLatestVersionsByContract(
244+
"*",
245+
BootstrapProviders(input.E2EConfig)...,
246+
),
247+
ControlPlaneProviders: input.E2EConfig.GetProviderLatestVersionsByContract(
248+
"*",
249+
ControlPlaneProviders(input.E2EConfig)...,
250+
),
251+
InfrastructureProviders: input.E2EConfig.GetProviderLatestVersionsByContract(
252+
"*",
253+
input.E2EConfig.InfrastructureProviders()...),
254+
AddonProviders: input.E2EConfig.GetProviderLatestVersionsByContract(
255+
"*",
256+
input.E2EConfig.AddonProviders()...),
257+
RuntimeExtensionProviders: input.E2EConfig.GetProviderLatestVersionsByContract(
258+
"*",
259+
input.E2EConfig.RuntimeExtensionProviders()...),
243260
LogFolder: filepath.Join(
244261
input.ArtifactFolder,
245262
"clusters",
@@ -274,9 +291,9 @@ func SelfHostedSpec(ctx context.Context, inputGetter func() SelfHostedSpecInput)
274291
})
275292

276293
By("Waiting for the cluster to be reconciled after moving to self hosted")
277-
selfHostedCluster = framework.DiscoveryAndWaitForCluster(
294+
selfHostedCluster = capie2eframework.DiscoveryAndWaitForCluster(
278295
ctx,
279-
framework.DiscoveryAndWaitForClusterInput{
296+
capie2eframework.DiscoveryAndWaitForClusterInput{
280297
Getter: selfHostedClusterProxy.GetClient(),
281298
Namespace: selfHostedNamespace.Name,
282299
Name: cluster.Name,
@@ -335,9 +352,9 @@ func SelfHostedSpec(ctx context.Context, inputGetter func() SelfHostedSpecInput)
335352
})
336353

337354
By("Waiting for the cluster to be reconciled after moving back to bootstrap")
338-
clusterResources.Cluster = framework.DiscoveryAndWaitForCluster(
355+
clusterResources.Cluster = capie2eframework.DiscoveryAndWaitForCluster(
339356
ctx,
340-
framework.DiscoveryAndWaitForClusterInput{
357+
capie2eframework.DiscoveryAndWaitForClusterInput{
341358
Getter: input.BootstrapClusterProxy.GetClient(),
342359
Namespace: namespace.Name,
343360
Name: clusterResources.Cluster.Name,
@@ -377,13 +394,13 @@ func hasProvider(ctx context.Context, c client.Client, providerName string) bool
377394
func setupSpecNamespace(
378395
ctx context.Context,
379396
specName string,
380-
clusterProxy framework.ClusterProxy,
397+
clusterProxy capie2eframework.ClusterProxy,
381398
artifactFolder string,
382399
) (*corev1.Namespace, context.CancelFunc) {
383400
capie2e.Byf("Creating a namespace for hosting the %q test spec", specName)
384-
namespace, cancelWatches := framework.CreateNamespaceAndWatchEvents(
401+
namespace, cancelWatches := capie2eframework.CreateNamespaceAndWatchEvents(
385402
ctx,
386-
framework.CreateNamespaceAndWatchEventsInput{
403+
capie2eframework.CreateNamespaceAndWatchEventsInput{
387404
Creator: clusterProxy.GetClient(),
388405
ClientSet: clusterProxy.GetClientSet(),
389406
Name: fmt.Sprintf("%s-%s", specName, util.RandomString(6)),
@@ -397,7 +414,7 @@ func setupSpecNamespace(
397414
// dumpAllResources dumps all the resources in the spec namespace and the workload cluster.
398415
func dumpAllResources(
399416
ctx context.Context,
400-
clusterProxy framework.ClusterProxy,
417+
clusterProxy capie2eframework.ClusterProxy,
401418
artifactFolder string,
402419
namespace *corev1.Namespace,
403420
cluster *clusterv1.Cluster,
@@ -415,7 +432,7 @@ func dumpAllResources(
415432
capie2e.Byf("Dumping all the Cluster API resources in the %q namespace", namespace.Name)
416433

417434
// Dump all Cluster API related resources to artifacts.
418-
framework.DumpAllResources(ctx, framework.DumpAllResourcesInput{
435+
capie2eframework.DumpAllResources(ctx, capie2eframework.DumpAllResourcesInput{
419436
Lister: clusterProxy.GetClient(),
420437
Namespace: namespace.Name,
421438
LogPath: filepath.Join(artifactFolder, "clusters", clusterProxy.GetName(), "resources"),
@@ -424,12 +441,12 @@ func dumpAllResources(
424441
// If the cluster still exists, dump pods and nodes of the workload cluster.
425442
if err := clusterProxy.GetClient().Get(ctx, client.ObjectKeyFromObject(cluster), &clusterv1.Cluster{}); err == nil {
426443
capie2e.Byf("Dumping Pods and Nodes of Cluster %s", klog.KObj(cluster))
427-
framework.DumpResourcesForCluster(ctx, framework.DumpResourcesForClusterInput{
444+
capie2eframework.DumpResourcesForCluster(ctx, capie2eframework.DumpResourcesForClusterInput{
428445
Lister: clusterProxy.GetWorkloadCluster(ctx, cluster.Namespace, cluster.Name).
429446
GetClient(),
430447
Cluster: cluster,
431448
LogPath: filepath.Join(artifactFolder, "clusters", cluster.Name, "resources"),
432-
Resources: []framework.DumpNamespaceAndGVK{
449+
Resources: []capie2eframework.DumpNamespaceAndGVK{
433450
{
434451
GVK: schema.GroupVersionKind{
435452
Version: corev1.SchemeGroupVersion.Version,
@@ -451,7 +468,7 @@ func dumpAllResources(
451468
func dumpSpecResourcesAndCleanup(
452469
ctx context.Context,
453470
specName string,
454-
clusterProxy framework.ClusterProxy,
471+
clusterProxy capie2eframework.ClusterProxy,
455472
artifactFolder string,
456473
namespace *corev1.Namespace,
457474
cancelWatches context.CancelFunc,
@@ -467,13 +484,16 @@ func dumpSpecResourcesAndCleanup(
467484
// While https://github.com/kubernetes-sigs/cluster-api/issues/2955 is addressed in future iterations, there is a
468485
// chance that cluster variable is not set even if the cluster exists, so we are calling DeleteAllClustersAndWait
469486
// instead of DeleteClusterAndWait
470-
framework.DeleteAllClustersAndWait(ctx, framework.DeleteAllClustersAndWaitInput{
471-
Client: clusterProxy.GetClient(),
472-
Namespace: namespace.Name,
473-
}, intervalsGetter(specName, "wait-delete-cluster")...)
487+
capie2eframework.DeleteAllClustersAndWait(
488+
ctx,
489+
capie2eframework.DeleteAllClustersAndWaitInput{
490+
Client: clusterProxy.GetClient(),
491+
Namespace: namespace.Name,
492+
},
493+
intervalsGetter(specName, "wait-delete-cluster")...)
474494

475495
capie2e.Byf("Deleting namespace used for hosting the %q test spec", specName)
476-
framework.DeleteNamespace(ctx, framework.DeleteNamespaceInput{
496+
capie2eframework.DeleteNamespace(ctx, capie2eframework.DeleteNamespaceInput{
477497
Deleter: clusterProxy.GetClient(),
478498
Name: namespace.Name,
479499
})

0 commit comments

Comments
 (0)