fixup! test(e2e): Add ownership assertions for AWS resources

Author: jimmidyson

make/go.mk

@@ -114,6 +114,7 @@ endif
 	    test/e2e/... -- \
 	      -e2e.artifacts-folder="$(ARTIFACTS)" \
 	      -e2e.config="$(E2E_CONF_FILE_ENVSUBST)" \
+	      $(if $(filter $(E2E_SKIP_CLEANUP),true),-e2e.skip-resource-cleanup)
 	go tool cover \
 	  -html=coverage-e2e.out \
 	  -o coverage-e2e.html

test/e2e/common.go

@@ -159,18 +159,12 @@ variables:
 
 intervals:
   default/wait-controllers: ["3m", "10s"]
-  default/wait-cluster: ["5m", "10s"]
+  default/wait-cluster: ["10m", "10s"]
   default/wait-control-plane: ["10m", "10s"]
-  default/wait-worker-nodes: ["5m", "10s"]
-  default/wait-machine-pool-nodes: ["5m", "10s"]
-  default/wait-delete-cluster: ["3m", "10s"]
-  default/wait-machine-upgrade: ["20m", "10s"]
-  default/wait-machine-pool-upgrade: ["5m", "10s"]
+  default/wait-worker-nodes: ["10m", "10s"]
+  default/wait-delete-cluster: ["30m", "10s"]
   default/wait-nodes-ready: ["10m", "10s"]
-  default/wait-machine-remediation: ["5m", "10s"]
-  default/wait-deployment: ["15m", "10s"]
-  default/wait-daemonset: [ "15m", "10s" ]
-  default/wait-clusterresourceset: [ "15m", "10s" ]
-  default/wait-helmrelease: [ "15m", "10s" ]
-  default/wait-job: [ "5m", "10s" ]
-  default/wait-service: [ "15m", "10s" ]
+  default/wait-deployment: ["10m", "10s"]
+  default/wait-daemonset: [ "5m", "10s" ]
+  default/wait-clusterresourceset: [ "5m", "10s" ]
+  default/wait-helmrelease: [ "5m", "10s" ]

test/e2e/config/cre.yaml

@@ -11,6 +11,7 @@ import (
 
 	. "github.com/onsi/gomega"
 	appsv1 "k8s.io/api/apps/v1"
+	capie2e "sigs.k8s.io/cluster-api/test/e2e"
 	"sigs.k8s.io/cluster-api/test/framework"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -28,7 +29,7 @@ func WaitForDaemonSetsAvailable(
 ) {
 	start := time.Now()
 	namespace, name := input.DaemonSet.GetNamespace(), input.DaemonSet.GetName()
-	Byf("waiting for deployment %s/%s to be available", namespace, name)
+	capie2e.Byf("waiting for deployment %s/%s to be available", namespace, name)
 	Log("starting to wait for deployment to become available")
 	Eventually(func() bool {
 		key := client.ObjectKey{Namespace: namespace, Name: name}

test/e2e/daemonset_helpers.go

@@ -12,6 +12,7 @@ import (
 	. "github.com/onsi/gomega"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	capie2e "sigs.k8s.io/cluster-api/test/e2e"
 	"sigs.k8s.io/cluster-api/test/framework"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -23,7 +24,7 @@ func WaitForDeploymentsAvailable(
 ) {
 	start := time.Now()
 	key := client.ObjectKeyFromObject(input.Deployment)
-	Byf("waiting for deployment %s to be available", key)
+	capie2e.Byf("waiting for deployment %s to be available", key)
 	Log("starting to wait for deployment to become available")
 	Eventually(func() bool {
 		if err := input.Getter.Get(ctx, key, input.Deployment); err == nil {

test/e2e/deployment_helpers.go

@@ -91,10 +91,10 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 	By("Initializing a runtime.Scheme with all the GVK relevant for this test")
 	scheme := initScheme()
 
-	Byf("Loading the e2e test configuration from %q", configPath)
+	capie2e.Byf("Loading the e2e test configuration from %q", configPath)
 	e2eConfig = loadE2EConfig(configPath)
 
-	Byf("Creating a clusterctl local repository into %q", artifactFolder)
+	capie2e.Byf("Creating a clusterctl local repository into %q", artifactFolder)
 	clusterctlConfigPath = createClusterctlLocalRepository(
 		e2eConfig,
 		filepath.Join(artifactFolder, "repository"),

test/e2e/e2e_suite_test.go

@@ -13,6 +13,7 @@ import (
 	. "github.com/onsi/gomega"
 	addonsv1 "sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	capie2e "sigs.k8s.io/cluster-api/test/e2e"
 	"sigs.k8s.io/cluster-api/test/framework"
 	"sigs.k8s.io/cluster-api/util/conditions"
 	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
@@ -43,7 +44,7 @@ func WaitForHelmReleaseProxyReadyForCluster(
 	Expect(err).ToNot(HaveOccurred())
 	hrpKey := ctrlclient.ObjectKeyFromObject(hrp)
 
-	Byf("waiting for HelmReleaseProxy for %s to be ready", hrpKey)
+	capie2e.Byf("waiting for HelmReleaseProxy for %s to be ready", hrpKey)
 	Log("starting to wait for HelmReleaseProxy to become available")
 	Eventually(func() bool {
 		err := input.GetLister.Get(ctx, hrpKey, hrp)

test/e2e/helmreleaseproxy_helpers.go

@@ -6,25 +6,60 @@
 package e2e
 
 import (
+	"slices"
+
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/utils/ptr"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	addonsv1 "sigs.k8s.io/cluster-api/exp/addons/api/v1beta1"
 	"sigs.k8s.io/cluster-api/test/framework"
 
 	caaphv1 "github.com/d2iq-labs/capi-runtime-extensions/api/external/sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1"
 )
 
 const (
-	clusterKind            = "Cluster"
-	clusterResourceSetKind = "ClusterResourceSet"
-	helmChartProxyKind     = "HelmChartProxy"
-	helmReleaseProxyKind   = "HelmReleaseProxy"
+	clusterKind                   = "Cluster"
+	clusterClassKind              = "ClusterClass"
+	machineDeploymentKind         = "MachineDeployment"
+	machineSetKind                = "MachineSet"
+	machineKind                   = "Machine"
+	clusterResourceSetKind        = "ClusterResourceSet"
+	clusterResourceSetBindingKind = "ClusterResourceSetBinding"
+
+	awsMachineKind                   = "AWSMachine"
+	awsMachineTemplateKind           = "AWSMachineTemplate"
+	awsClusterKind                   = "AWSCluster"
+	awsClusterTemplateKind           = "AWSClusterTemplate"
+	awsClusterControllerIdentityKind = "AWSClusterControllerIdentity"
+
+	helmChartProxyKind   = "HelmChartProxy"
+	helmReleaseProxyKind = "HelmReleaseProxy"
 )
 
 var (
+	coreGroupVersion = clusterv1.GroupVersion.String()
+
 	clusterOwner = metav1.OwnerReference{
 		Kind:       clusterKind,
-		APIVersion: clusterv1.GroupVersion.String(),
+		APIVersion: coreGroupVersion,
+	}
+	clusterController = metav1.OwnerReference{
+		Kind:       clusterKind,
+		APIVersion: coreGroupVersion,
+		Controller: ptr.To(true),
+	}
+	clusterClassOwner = metav1.OwnerReference{
+		Kind:       clusterClassKind,
+		APIVersion: coreGroupVersion,
+	}
+	machineController = metav1.OwnerReference{
+		Kind:       machineKind,
+		APIVersion: coreGroupVersion,
+		Controller: ptr.To(true),
+	}
+	clusterResourceSetOwner = metav1.OwnerReference{
+		Kind:       clusterResourceSetKind,
+		APIVersion: addonsv1.GroupVersion.String(),
 	}
 	helmChartProxyOwner = metav1.OwnerReference{
 		Kind:       helmChartProxyKind,
@@ -43,6 +78,8 @@ var (
 			)
 		},
 
+		clusterResourceSetBindingKind: clusterResourceSetBindingIsOnlyOwnedByClusterResourceSets,
+
 		helmChartProxyKind: func(owners []metav1.OwnerReference) error {
 			// The HelmChartProxies that we create are cluster specific and so should be owned by the cluster.
 			return framework.HasExactOwners(
@@ -59,4 +96,51 @@ var (
 			)
 		},
 	}
+
+	// AWSInfraOwnerReferenceAssertions maps AWS Infrastructure types to functions which return an error if the passed
+	// OwnerReferences aren't as expected.
+	AWSInfraOwnerReferenceAssertions = map[string]func([]metav1.OwnerReference) error{
+		awsMachineKind: func(owners []metav1.OwnerReference) error {
+			// The AWSMachine must be owned and controlled by a Machine.
+			return framework.HasExactOwners(owners, machineController)
+		},
+		awsMachineTemplateKind: func(owners []metav1.OwnerReference) error {
+			// Base AWSrMachineTemplates referenced in a ClusterClass must be owned by the ClusterClass.
+			// AWSMachineTemplates created for specific Clusters in the Topology controller must be owned by a Cluster.
+			return framework.HasOneOfExactOwners(
+				owners,
+				[]metav1.OwnerReference{clusterOwner},
+				[]metav1.OwnerReference{clusterClassOwner},
+			)
+		},
+		awsClusterKind: func(owners []metav1.OwnerReference) error {
+			// AWSCluster must be owned and controlled by a Cluster.
+			return framework.HasExactOwners(owners, clusterController)
+		},
+		awsClusterTemplateKind: func(owners []metav1.OwnerReference) error {
+			// AWSClusterTemplate must be owned by a ClusterClass.
+			return framework.HasExactOwners(owners, clusterClassOwner)
+		},
+		awsClusterControllerIdentityKind: func(owners []metav1.OwnerReference) error {
+			// AWSClusterControllerIdentity should have no owners.
+			return framework.HasExactOwners(owners)
+		},
+	}
 )
+
+// dedupeOwners returns a a list of owners without duplicate owner types. Only the fields used in the
+// CAPI e2e owner comparison funcs are used in the comparison here to assert equality.
+func dedupeOwners(owners []metav1.OwnerReference) []metav1.OwnerReference {
+	return slices.CompactFunc(owners, func(a, b metav1.OwnerReference) bool {
+		return a.APIVersion == b.APIVersion && a.Kind == b.Kind &&
+			ptr.Equal(a.Controller, b.Controller)
+	})
+}
+
+// clusterResourceSetBindingIsOnlyOwnedByClusterResourceSets returns a function that checks that the passed
+// OwnerReferences are as expected, which means only owned by ClusterResourceSets and not by any other kinds.
+func clusterResourceSetBindingIsOnlyOwnedByClusterResourceSets(
+	gotOwners []metav1.OwnerReference,
+) error {
+	return framework.HasExactOwners(dedupeOwners(gotOwners), clusterResourceSetOwner)
+}

test/e2e/ownerreference_helpers.go

@@ -13,7 +13,7 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"k8s.io/utils/ptr"
-	"sigs.k8s.io/cluster-api/test/e2e"
+	capie2e "sigs.k8s.io/cluster-api/test/e2e"
 	"sigs.k8s.io/cluster-api/test/framework"
 
 	"github.com/d2iq-labs/capi-runtime-extensions/api/v1alpha1"
@@ -23,6 +23,7 @@ import (
 
 var _ = Describe("Quick start", Serial, func() {
 	for _, provider := range []string{"Docker", "AWS"} {
+		lowercaseProvider := strings.ToLower(provider)
 		for _, cniProvider := range []string{"Cilium", "Calico"} {
 			for _, addonStrategy := range []string{"HelmAddon", "ClusterResourceSet"} {
 				strategy := ""
@@ -47,20 +48,25 @@ var _ = Describe("Quick start", Serial, func() {
 					Label("cni:"+cniProvider),
 					Label("addonStrategy:"+addonStrategy),
 					func() {
-						e2e.QuickStartSpec(ctx, func() e2e.QuickStartSpecInput {
-							prov := strings.ToLower(provider)
-							if !slices.Contains(e2eConfig.InfrastructureProviders(), prov) {
-								Skip(fmt.Sprintf("provider %s is not enabled", prov))
+						capie2e.QuickStartSpec(ctx, func() capie2e.QuickStartSpecInput {
+							if !slices.Contains(
+								e2eConfig.InfrastructureProviders(),
+								lowercaseProvider,
+							) {
+								Fail(fmt.Sprintf(
+									"provider %s is not enabled - check environment setup for provider specific requirements",
+									lowercaseProvider,
+								))
 							}
 
-							return e2e.QuickStartSpecInput{
+							return capie2e.QuickStartSpecInput{
 								E2EConfig:              e2eConfig,
 								ClusterctlConfigPath:   clusterctlConfigPath,
 								BootstrapClusterProxy:  bootstrapClusterProxy,
 								ArtifactFolder:         artifactFolder,
 								SkipCleanup:            skipCleanup,
 								Flavor:                 ptr.To(flavour),
-								InfrastructureProvider: ptr.To(prov),
+								InfrastructureProvider: ptr.To(lowercaseProvider),
 								PostMachinesProvisioned: func(proxy framework.ClusterProxy, namespace, clusterName string) {
 									framework.AssertOwnerReferences(
 										namespace,
@@ -71,6 +77,7 @@ var _ = Describe("Quick start", Serial, func() {
 										framework.KubeadmBootstrapOwnerReferenceAssertions,
 										framework.KubeadmControlPlaneOwnerReferenceAssertions,
 										framework.KubernetesReferenceAssertions,
+										AWSInfraOwnerReferenceAssertions,
 										AddonReferenceAssertions,
 									)
 
@@ -86,7 +93,7 @@ var _ = Describe("Quick start", Serial, func() {
 										framework.WaitForNodesReadyInput{
 											Lister: workloadClient,
 											KubernetesVersion: e2eConfig.GetVariable(
-												KubernetesVersion,
+												capie2e.KubernetesVersion,
 											),
 											Count: 2,
 											WaitForNodesReady: e2eConfig.GetIntervals(