test(e2e): Various e2e tests fixes (#754)

Fixed up Nutanix CSI namespace in e2e tests.

Updated Kubernetes version for e2e to use to v1.29.5, which is the
latest available version of Nutanix and AWS images.

Tested locally by running AWS and Nutanix e2e tests - all passed! Docker
is tested in CI.

Depends on #751.

Author: jimmidyson

README.md

@@ -44,7 +44,7 @@ For example, the following command will create a Docker cluster with Cilium CNI
 ```shell
 export CLUSTER_NAME=docker-cluster-cilium-helm-addon
 export CLUSTER_FILE=examples/capi-quick-start/docker-cluster-cilium-helm-addon.yaml
-export KUBERNETES_VERSION=v1.29.4
+export KUBERNETES_VERSION=v1.29.5
 ```
 
 ```shell

go.mod

@@ -23,6 +23,7 @@ require (
 	github.com/onsi/ginkgo/v2 v2.19.0
 	github.com/onsi/gomega v1.33.1
 	github.com/pkg/errors v0.9.1
+	github.com/samber/lo v1.39.0
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.9.0
 	gopkg.in/yaml.v2 v2.4.0

go.sum

@@ -218,6 +218,8 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6ke
 github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4=
 github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
 github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
+github.com/samber/lo v1.39.0 h1:4gTz1wUhNYLhFSKl6O+8peW0v2F4BCY034GRpU9WnuA=
+github.com/samber/lo v1.39.0/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA=
 github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
 github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
 github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=

make/go.mk

@@ -80,7 +80,7 @@ E2E_PARALLEL_NODES ?= $(if $(filter $(E2E_DRYRUN),true),1,$(shell nproc --ignore
 E2E_FLAKE_ATTEMPTS ?= 1
 E2E_CONF_FILE ?= $(REPO_ROOT)/test/e2e/config/caren.yaml
 E2E_CONF_FILE_ENVSUBST ?= $(basename $(E2E_CONF_FILE))-envsubst.yaml
-export E2E_DEFAULT_KUBERNETES_VERSION ?= v1.29.4
+export E2E_DEFAULT_KUBERNETES_VERSION ?= v1.29.5
 ARTIFACTS ?= ${REPO_ROOT}/_artifacts
 
 .PHONY: e2e-test

make/kind.mk

@@ -7,10 +7,10 @@ KIND_CLUSTER_NAME ?= $(GITHUB_REPOSITORY)-dev
 KIND_KUBECONFIG ?= $(KIND_DIR)/$(KIND_CLUSTER_NAME)/kubeconfig
 
 KINDEST_NODE_IMAGE ?= ghcr.io/mesosphere/kind-node
-KINDEST_NODE_VERSION_v1.27 ?= v1.27.13
-KINDEST_NODE_VERSION_v1.28 ?= v1.28.9
-KINDEST_NODE_VERSION_v1.29 ?= v1.29.4
-KINDEST_NODE_VERSION_v1.30 ?= v1.30.0
+KINDEST_NODE_VERSION_v1.27 ?= v1.27.15
+KINDEST_NODE_VERSION_v1.28 ?= v1.28.11
+KINDEST_NODE_VERSION_v1.29 ?= v1.29.6
+KINDEST_NODE_VERSION_v1.30 ?= v1.30.2
 # Allow easy override of Kubernetes version to use via `make KIND_KUBERNETES_VERSION=v1.23` to use in CI
 KIND_KUBERNETES_VERSION ?= v1.29
 ifndef KINDEST_NODE_VERSION_$(KIND_KUBERNETES_VERSION)

test/e2e/config/caren.yaml

@@ -167,7 +167,7 @@ variables:
   KUBERNETES_VERSION: "${E2E_DEFAULT_KUBERNETES_VERSION}"
   # Override Kubernetes version for test workload clusters for specific providers by setting the env variables
   # `KUBERNETES_VERSION_<PROVIDER>`, where `<PROVIDER>` is the uppercase provider name, e.g.
-  # `KUBERNETES_VERSION_DOCKER: v1.29.4`.
+  # `KUBERNETES_VERSION_DOCKER: v1.29.5`.
   # KUBERNETES_VERSION_DOCKER: "${KINDEST_IMAGE_TAG}"
   SERVICE_CIDR: "10.128.0.0/12"
   POD_CIDR: "192.168.0.0/16"

test/e2e/csi_helpers.go

@@ -318,7 +318,7 @@ func waitForNutanixCSIToBeReadyInWorkloadCluster(
 		Deployment: &appsv1.Deployment{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "nutanix-csi-controller",
-				Namespace: metav1.NamespaceSystem,
+				Namespace: "ntnx-system",
 			},
 		},
 	}, input.deploymentIntervals...)
@@ -328,7 +328,7 @@ func waitForNutanixCSIToBeReadyInWorkloadCluster(
 		DaemonSet: &appsv1.DaemonSet{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "nutanix-csi-node",
-				Namespace: metav1.NamespaceSystem,
+				Namespace: "ntnx-system",
 			},
 		},
 	}, input.daemonSetIntervals...)
@@ -357,6 +357,8 @@ func waitForStorageClassesToExistInWorkloadCluster(
 		provisioner = string(v1alpha1.LocalPathProvisioner)
 	case v1alpha1.CSIProviderAWSEBS:
 		provisioner = string(v1alpha1.AWSEBSProvisioner)
+	case v1alpha1.CSIProviderNutanix:
+		provisioner = string(v1alpha1.NutanixProvisioner)
 	default:
 		Fail(
 			fmt.Sprintf(

test/e2e/e2e_suite_test.go

@@ -70,7 +70,7 @@ func init() {
 	flag.StringVar(
 		&bootstrapKubernetesVersion,
 		"e2e.bootstrap-kind-version",
-		"v1.29.4",
+		"v1.29.5",
 		"the version of the image used in bootstrap cluster",
 	)
 }

test/e2e/ownerreference_helpers.go

@@ -16,6 +16,7 @@ import (
 	addonsv1 "sigs.k8s.io/cluster-api/exp/addons/api/v1beta1"
 	"sigs.k8s.io/cluster-api/test/framework"
 
+	capxv1 "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/external/github.com/nutanix-cloud-native/cluster-api-provider-nutanix/api/v1beta1"
 	caaphv1 "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/external/sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1"
 )
 
@@ -91,6 +92,12 @@ var (
 		Controller: ptr.To(true),
 	}
 
+	capxGroupVersion    = capxv1.GroupVersion.String()
+	nutanixClusterOwner = metav1.OwnerReference{
+		Kind:       nutanixClusterKind,
+		APIVersion: capxGroupVersion,
+	}
+
 	// AddonReferenceAssertions maps addontypes to functions which return an error if the passed OwnerReferences
 	// aren't as expected.
 	AddonReferenceAssertions = map[string]func([]metav1.OwnerReference) error{
@@ -191,6 +198,7 @@ var (
 				[]metav1.OwnerReference{kubeadmControlPlaneController},
 				[]metav1.OwnerReference{kubeadmConfigController},
 				[]metav1.OwnerReference{clusterOwner},
+				[]metav1.OwnerReference{clusterOwner, nutanixClusterOwner},
 			)
 		},
 		configMapKind: func(owners []metav1.OwnerReference) error {

test/e2e/quick_start_test.go

@@ -8,11 +8,14 @@ package e2e
 import (
 	"fmt"
 	"slices"
+	"strconv"
 	"strings"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	"github.com/samber/lo"
 	"k8s.io/utils/ptr"
+	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	clusterctlcluster "sigs.k8s.io/cluster-api/cmd/clusterctl/client/cluster"
 	capie2e "sigs.k8s.io/cluster-api/test/e2e"
 	"sigs.k8s.io/cluster-api/test/framework"
@@ -63,7 +66,7 @@ var _ = Describe("Quick start", Serial, func() {
 							// available that are not available in other providers.
 							// This version can be specified in `test/e2e/config/caren.yaml` with a variable named
 							// `KUBERNETES_VERSION_<PROVIDER>`, where `<PROVIDER>` is the uppercase provider name, e.g.
-							// `KUBERNETES_VERSION_DOCKER: v1.29.4`.
+							// `KUBERNETES_VERSION_DOCKER: v1.29.5`.
 							testE2EConfig := e2eConfig.DeepCopy()
 							varName := capie2e.KubernetesVersion + "_" + strings.ToUpper(
 								lowercaseProvider,
@@ -99,21 +102,57 @@ var _ = Describe("Quick start", Serial, func() {
 										KubernetesReferenceAssertions,
 									)
 
+									workloadCluster := framework.GetClusterByName(
+										ctx,
+										framework.GetClusterByNameInput{
+											Namespace: namespace,
+											Name:      clusterName,
+											Getter:    proxy.GetClient(),
+										},
+									)
+									Expect(workloadCluster.Spec.Topology).ToNot(BeNil())
+
 									By("Waiting until nodes are ready")
 									workloadProxy := proxy.GetWorkloadCluster(
 										ctx,
 										namespace,
 										clusterName,
 									)
 									workloadClient := workloadProxy.GetClient()
+
+									nodeCount := int(
+										ptr.Deref(
+											workloadCluster.Spec.Topology.ControlPlane.Replicas,
+											0,
+										),
+									) +
+										lo.Reduce(
+											workloadCluster.Spec.Topology.Workers.MachineDeployments,
+											func(agg int, md clusterv1.MachineDeploymentTopology, _ int) int {
+												switch {
+												case md.Replicas != nil:
+													return agg + int(ptr.Deref(md.Replicas, 0))
+												case md.Metadata.Annotations["cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size"] != "":
+													minSize, err := strconv.Atoi(
+														md.Metadata.Annotations["cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size"],
+													)
+													Expect(err).ToNot(HaveOccurred())
+													return agg + minSize
+												default:
+													return agg
+												}
+											},
+											0,
+										)
+
 									framework.WaitForNodesReady(
 										ctx,
 										framework.WaitForNodesReadyInput{
 											Lister: workloadClient,
 											KubernetesVersion: testE2EConfig.GetVariable(
 												capie2e.KubernetesVersion,
 											),
-											Count: 2,
+											Count: nodeCount,
 											WaitForNodesReady: testE2EConfig.GetIntervals(
 												flavour,
 												"wait-nodes-ready",
@@ -124,15 +163,6 @@ var _ = Describe("Quick start", Serial, func() {
 									By(
 										"Waiting for all requested addons to be ready in workload cluster",
 									)
-									workloadCluster := framework.GetClusterByName(
-										ctx,
-										framework.GetClusterByNameInput{
-											Namespace: namespace,
-											Name:      clusterName,
-											Getter:    proxy.GetClient(),
-										},
-									)
-									Expect(workloadCluster.Spec.Topology).ToNot(BeNil())
 									clusterVars := variables.ClusterVariablesToVariablesMap(
 										workloadCluster.Spec.Topology.Variables,
 									)