feat: Added e2e test for capx cluster (#523)

We need to set following vars in either env or caren.yaml
    
      # IP/FQDN of Prism Central.
      NUTANIX_ENDPOINT: ""
      # Port of Prism Central. Default: 9440
      NUTANIX_PORT: 9440
      # Disable Prism Central certificate checking. Default: false
      NUTANIX_INSECURE: false
      # Prism Central user
      NUTANIX_USER: ""
      # Prism Central password
      NUTANIX_PASSWORD: ""
      # Host IP to be assigned to the CAPX Kubernetes cluster.
      CONTROL_PLANE_ENDPOINT_IP: ""
      # Port of the CAPX Kubernetes cluster. Default: 6443
      CONTROL_PLANE_ENDPOINT_PORT: 6443
      # Name of the Prism Element cluster.
      NUTANIX_PRISM_ELEMENT_CLUSTER_NAME: ""
      # Name of the OS image pre-uploaded in PC.
      NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME: ""
      # Name of the subnet to be assigned to the VMs.
      NUTANIX_SUBNET_NAME: ""
      # Name of the storage container to CSI driver
      NUTANIX_STORAGE_CONTAINER_NAME: ""
# Username/Password of dockerhub account to avoid download rate limiting
      DOCKER_HUB_USERNAME: ""
      DOCKER_HUB_PASSWORD: ""

<!--
 Copyright 2023 D2iQ, Inc. All rights reserved.
 SPDX-License-Identifier: Apache-2.0
 -->
**What problem does this PR solve?**:
Adds Nutanix provider to quick start test

**Which issue(s) this PR fixes**:
Fixes #https://jira.nutanix.com/browse/D2IQ-100400 

**How Has This Been Tested?**:
<!--
Please describe the tests that you ran to verify your changes.
Provide output from the tests and any manual steps needed to replicate
the tests.
-->
<pre>
make e2e-test
</pre>

**Special notes for your reviewer**:
<!--
Use this to provide any additional information to the reviewers.
This may include:
- Best way to review the PR.
- Where the author wants the most review attention on.
- etc.
-->

Author: deepakm-ntnx

.github/workflows/e2e.yml

@@ -40,6 +40,9 @@ jobs:
         run: devbox run -- make e2e-test E2E_LABEL='provider:${{ inputs.provider }}'
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          NUTANIX_ENDPOINT: ${{ secrets.NUTANIX_ENDPOINT }}
+          NUTANIX_PASSWORD: ${{ secrets.NUTANIX_PASSWORD }}
+          NUTANIX_USER: ${{ secrets.NUTANIX_USER }}
 
       - if: success() || failure() # always run even if the previous step fails
         name: Publish e2e test report

pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go

@@ -215,7 +215,14 @@ func (n *NutanixCSI) handleHelmAddonApply(
 		},
 	}
 
-	if err = client.ServerSideApply(ctx, n.client, snapshotChart, client.ForceOwnership); err != nil {
+	if err = controllerutil.SetOwnerReference(&req.Cluster, snapshotChart, n.client.Scheme()); err != nil {
+		return fmt.Errorf(
+			"failed to set owner reference on nutanix-csi-snapshot installation HelmChartProxy: %w",
+			err,
+		)
+	}
+
+	if err = client.ServerSideApply(ctx, n.client, snapshotChart); err != nil {
 		return fmt.Errorf(
 			"failed to apply nutanix-csi-snapshot installation HelmChartProxy: %w",
 			err,

test/e2e/config/caren.yaml

@@ -93,6 +93,29 @@ providers:
     - old: --metrics-addr=127.0.0.1:8080
       new: --metrics-addr=:8080
 
+- name: nutanix
+  type: InfrastructureProvider
+  versions:
+  - name: "${CAPX_VERSION}"
+    value: "https://github.com/nutanix-cloud-native/cluster-api-provider-nutanix/releases/download/${CAPX_VERSION}/infrastructure-components.yaml"
+    type: "url"
+    contract: v1beta1
+    files:
+    - sourcePath: "../data/shared/v1beta1-capx/metadata.yaml"
+    - sourcePath: "../../../charts/cluster-api-runtime-extensions-nutanix/defaultclusterclasses/nutanix-cluster-class.yaml"
+      targetName: clusterclass-nutanix-quick-start.yaml
+    - sourcePath: "../../../examples/capi-quick-start/nutanix-cluster-cilium-helm-addon.yaml"
+      targetName: cluster-template-topology-cilium-helm-addon.yaml
+    - sourcePath: "../../../examples/capi-quick-start/nutanix-cluster-cilium-crs.yaml"
+      targetName: cluster-template-topology-cilium-crs.yaml
+    - sourcePath: "../../../examples/capi-quick-start/nutanix-cluster-calico-helm-addon.yaml"
+      targetName: cluster-template-topology-calico-helm-addon.yaml
+    - sourcePath: "../../../examples/capi-quick-start/nutanix-cluster-calico-crs.yaml"
+      targetName: cluster-template-topology-calico-crs.yaml
+    replacements:
+    - old: --metrics-addr=127.0.0.1:8080
+      new: --metrics-addr=:8080
+
 - name: helm
   type: AddonProvider
   versions:
@@ -161,6 +184,32 @@ variables:
   AMI_LOOKUP_FORMAT: "konvoy-ami-{{.BaseOS}}-release-?{{.K8sVersion}}-*"
   AMI_LOOKUP_BASEOS: "rocky-9.1"
   AMI_LOOKUP_ORG: "999867407951"
+  # To run Nutanix provider tests, set following variables here or as an env var
+  # # IP/FQDN of Prism Central.
+  # NUTANIX_ENDPOINT: ""
+  # # Port of Prism Central. Default: 9440
+  # NUTANIX_PORT: 9440
+  # # Disable Prism Central certificate checking. Default: false
+  # NUTANIX_INSECURE: false
+  # # Prism Central user
+  # NUTANIX_USER: ""
+  # # Prism Central password
+  # NUTANIX_PASSWORD: ""
+  # # Host IP to be assigned to the CAPX Kubernetes cluster.
+  # CONTROL_PLANE_ENDPOINT_IP: ""
+  # # Port of the CAPX Kubernetes cluster. Default: 6443
+  # CONTROL_PLANE_ENDPOINT_PORT: 6443
+  # # Name of the Prism Element cluster.
+  # NUTANIX_PRISM_ELEMENT_CLUSTER_NAME: ""
+  # # Name of the OS image pre-uploaded in PC.
+  # NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME: ""
+  # # Name of the subnet to be assigned to the VMs.
+  # NUTANIX_SUBNET_NAME: ""
+  # # Name of the storage container to CSI driver
+  # NUTANIX_STORAGE_CONTAINER_NAME: ""
+  # # Username/Password of dockerhub account to avoid download rate limiting
+  # DOCKER_HUB_USERNAME: ""
+  # DOCKER_HUB_PASSWORD: ""
 
 intervals:
   default/wait-controllers: ["3m", "10s"]

test/e2e/data/shared/v1beta1-capx/metadata.yaml

@@ -0,0 +1,15 @@
+# Copyright 2024 D2iQ, Inc. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# maps release series of major.minor to cluster-api contract version
+# the contract version may change between minor or major versions, but *not*
+# between patch versions.
+#
+# TODO(release-blocker): update this file only when a new major or minor version is released
+
+apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3
+kind: Metadata
+releaseSeries:
+  - major: 1
+    minor: 4
+    contract: v1beta1

test/e2e/ownerreference_helpers.go

@@ -32,8 +32,15 @@ const (
 	awsClusterTemplateKind           = "AWSClusterTemplate"
 	awsClusterControllerIdentityKind = "AWSClusterControllerIdentity"
 
+	nutanixMachineKind         = "NutanixMachine"
+	nutanixMachineTemplateKind = "NutanixMachineTemplate"
+	nutanixClusterKind         = "NutanixCluster"
+	nutanixClusterTemplateKind = "NutanixClusterTemplate"
+
 	helmChartProxyKind   = "HelmChartProxy"
 	helmReleaseProxyKind = "HelmReleaseProxy"
+
+	secretKind = "Secret"
 )
 
 var (
@@ -105,7 +112,7 @@ var (
 			return framework.HasExactOwners(owners, machineController)
 		},
 		awsMachineTemplateKind: func(owners []metav1.OwnerReference) error {
-			// Base AWSrMachineTemplates referenced in a ClusterClass must be owned by the ClusterClass.
+			// Base AWSMachineTemplates referenced in a ClusterClass must be owned by the ClusterClass.
 			// AWSMachineTemplates created for specific Clusters in the Topology controller must be owned by a Cluster.
 			return framework.HasOneOfExactOwners(
 				owners,
@@ -126,6 +133,37 @@ var (
 			return framework.HasExactOwners(owners)
 		},
 	}
+
+	// NutanixInfraOwnerReferenceAssertions maps Nutanix Infrastructure types to functions which return an error
+	// if the passed OwnerReferences aren't as expected.
+	NutanixInfraOwnerReferenceAssertions = map[string]func([]metav1.OwnerReference) error{
+		nutanixMachineKind: func(owners []metav1.OwnerReference) error {
+			// The NutanixMachine must be owned and controlled by a Machine.
+			return framework.HasExactOwners(owners, machineController)
+		},
+		nutanixMachineTemplateKind: func(owners []metav1.OwnerReference) error {
+			// Base NutanixMachineTemplates referenced in a ClusterClass must be owned by the ClusterClass.
+			// NutanixMachineTemplates created for specific Clusters in the Topology controller must be owned by a Cluster.
+			return framework.HasOneOfExactOwners(
+				owners,
+				[]metav1.OwnerReference{clusterOwner},
+				[]metav1.OwnerReference{clusterClassOwner},
+			)
+		},
+		nutanixClusterKind: func(owners []metav1.OwnerReference) error {
+			// NutanixCluster must be owned and controlled by a Cluster.
+			return framework.HasExactOwners(owners, clusterController)
+		},
+		nutanixClusterTemplateKind: func(owners []metav1.OwnerReference) error {
+			// NutanixClusterTemplate must be owned by a ClusterClass.
+			return framework.HasExactOwners(owners, clusterClassOwner)
+		},
+		secretKind: func(owners []metav1.OwnerReference) error {
+			// TODO:deepakm-ntnx Currently pc-creds, pc-creds-for-csi, dockerhub-credentials
+			// and registry-creds have unexpected owners which needs more investigation
+			return nil
+		},
+	}
 )
 
 // dedupeOwners returns a a list of owners without duplicate owner types. Only the fields used in the

test/e2e/quick_start_test.go

@@ -22,7 +22,7 @@ import (
 )
 
 var _ = Describe("Quick start", Serial, func() {
-	for _, provider := range []string{"Docker", "AWS"} {
+	for _, provider := range []string{"Docker", "AWS", "Nutanix"} {
 		lowercaseProvider := strings.ToLower(provider)
 		for _, cniProvider := range []string{"Cilium", "Calico"} {
 			for _, addonStrategy := range []string{"HelmAddon", "ClusterResourceSet"} {
@@ -94,6 +94,7 @@ var _ = Describe("Quick start", Serial, func() {
 										framework.KubeadmControlPlaneOwnerReferenceAssertions,
 										framework.KubernetesReferenceAssertions,
 										AWSInfraOwnerReferenceAssertions,
+										NutanixInfraOwnerReferenceAssertions,
 										AddonReferenceAssertions,
 									)