test(e2e): Fix up e2e tests wrt Kubernetes versions

Updated Kubernetes version for e2e to use to v1.29.5,
which is the latest available version of Nutanix and AWS
images.

Author: jimmidyson

README.md

@@ -44,7 +44,7 @@ For example, the following command will create a Docker cluster with Cilium CNI
 ```shell
 export CLUSTER_NAME=docker-cluster-cilium-helm-addon
 export CLUSTER_FILE=examples/capi-quick-start/docker-cluster-cilium-helm-addon.yaml
-export KUBERNETES_VERSION=v1.29.4
+export KUBERNETES_VERSION=v1.29.5
 ```
 
 ```shell

go.mod

@@ -23,6 +23,7 @@ require (
 	github.com/onsi/ginkgo/v2 v2.19.0
 	github.com/onsi/gomega v1.33.1
 	github.com/pkg/errors v0.9.1
+	github.com/samber/lo v1.39.0
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.9.0
 	gopkg.in/yaml.v2 v2.4.0

go.sum

@@ -218,6 +218,8 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6ke
 github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4=
 github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
 github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
+github.com/samber/lo v1.39.0 h1:4gTz1wUhNYLhFSKl6O+8peW0v2F4BCY034GRpU9WnuA=
+github.com/samber/lo v1.39.0/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA=
 github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
 github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
 github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=

make/go.mk

@@ -80,7 +80,7 @@ E2E_PARALLEL_NODES ?= $(if $(filter $(E2E_DRYRUN),true),1,$(shell nproc --ignore
 E2E_FLAKE_ATTEMPTS ?= 1
 E2E_CONF_FILE ?= $(REPO_ROOT)/test/e2e/config/caren.yaml
 E2E_CONF_FILE_ENVSUBST ?= $(basename $(E2E_CONF_FILE))-envsubst.yaml
-export E2E_DEFAULT_KUBERNETES_VERSION ?= v1.29.4
+export E2E_DEFAULT_KUBERNETES_VERSION ?= v1.29.5
 ARTIFACTS ?= ${REPO_ROOT}/_artifacts
 
 .PHONY: e2e-test

make/kind.mk

@@ -7,10 +7,10 @@ KIND_CLUSTER_NAME ?= $(GITHUB_REPOSITORY)-dev
 KIND_KUBECONFIG ?= $(KIND_DIR)/$(KIND_CLUSTER_NAME)/kubeconfig
 
 KINDEST_NODE_IMAGE ?= ghcr.io/mesosphere/kind-node
-KINDEST_NODE_VERSION_v1.27 ?= v1.27.13
-KINDEST_NODE_VERSION_v1.28 ?= v1.28.9
-KINDEST_NODE_VERSION_v1.29 ?= v1.29.4
-KINDEST_NODE_VERSION_v1.30 ?= v1.30.0
+KINDEST_NODE_VERSION_v1.27 ?= v1.27.15
+KINDEST_NODE_VERSION_v1.28 ?= v1.28.11
+KINDEST_NODE_VERSION_v1.29 ?= v1.29.6
+KINDEST_NODE_VERSION_v1.30 ?= v1.30.2
 # Allow easy override of Kubernetes version to use via `make KIND_KUBERNETES_VERSION=v1.23` to use in CI
 KIND_KUBERNETES_VERSION ?= v1.29
 ifndef KINDEST_NODE_VERSION_$(KIND_KUBERNETES_VERSION)

test/e2e/config/caren.yaml

@@ -167,7 +167,7 @@ variables:
   KUBERNETES_VERSION: "${E2E_DEFAULT_KUBERNETES_VERSION}"
   # Override Kubernetes version for test workload clusters for specific providers by setting the env variables
   # `KUBERNETES_VERSION_<PROVIDER>`, where `<PROVIDER>` is the uppercase provider name, e.g.
-  # `KUBERNETES_VERSION_DOCKER: v1.29.4`.
+  # `KUBERNETES_VERSION_DOCKER: v1.29.5`.
   # KUBERNETES_VERSION_DOCKER: "${KINDEST_IMAGE_TAG}"
   SERVICE_CIDR: "10.128.0.0/12"
   POD_CIDR: "192.168.0.0/16"

test/e2e/csi_helpers.go

@@ -318,7 +318,7 @@ func waitForNutanixCSIToBeReadyInWorkloadCluster(
 		Deployment: &appsv1.Deployment{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "nutanix-csi-controller",
-				Namespace: metav1.NamespaceSystem,
+				Namespace: "ntnx-system",
 			},
 		},
 	}, input.deploymentIntervals...)
@@ -328,7 +328,7 @@ func waitForNutanixCSIToBeReadyInWorkloadCluster(
 		DaemonSet: &appsv1.DaemonSet{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "nutanix-csi-node",
-				Namespace: metav1.NamespaceSystem,
+				Namespace: "ntnx-system",
 			},
 		},
 	}, input.daemonSetIntervals...)
@@ -357,6 +357,8 @@ func waitForStorageClassesToExistInWorkloadCluster(
 		provisioner = string(v1alpha1.LocalPathProvisioner)
 	case v1alpha1.CSIProviderAWSEBS:
 		provisioner = string(v1alpha1.AWSEBSProvisioner)
+	case v1alpha1.CSIProviderNutanix:
+		provisioner = string(v1alpha1.NutanixProvisioner)
 	default:
 		Fail(
 			fmt.Sprintf(

test/e2e/e2e_suite_test.go

@@ -70,7 +70,7 @@ func init() {
 	flag.StringVar(
 		&bootstrapKubernetesVersion,
 		"e2e.bootstrap-kind-version",
-		"v1.29.4",
+		"v1.29.5",
 		"the version of the image used in bootstrap cluster",
 	)
 }

test/e2e/ownerreference_helpers.go

@@ -16,6 +16,7 @@ import (
 	addonsv1 "sigs.k8s.io/cluster-api/exp/addons/api/v1beta1"
 	"sigs.k8s.io/cluster-api/test/framework"
 
+	capxv1 "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/external/github.com/nutanix-cloud-native/cluster-api-provider-nutanix/api/v1beta1"
 	caaphv1 "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/external/sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1"
 )
 
@@ -91,6 +92,12 @@ var (
 		Controller: ptr.To(true),
 	}
 
+	capxGroupVersion    = capxv1.GroupVersion.String()
+	nutanixClusterOwner = metav1.OwnerReference{
+		Kind:       nutanixClusterKind,
+		APIVersion: capxGroupVersion,
+	}
+
 	// AddonReferenceAssertions maps addontypes to functions which return an error if the passed OwnerReferences
 	// aren't as expected.
 	AddonReferenceAssertions = map[string]func([]metav1.OwnerReference) error{
@@ -191,6 +198,7 @@ var (
 				[]metav1.OwnerReference{kubeadmControlPlaneController},
 				[]metav1.OwnerReference{kubeadmConfigController},
 				[]metav1.OwnerReference{clusterOwner},
+				[]metav1.OwnerReference{clusterOwner, nutanixClusterOwner},
 			)
 		},
 		configMapKind: func(owners []metav1.OwnerReference) error {

test/e2e/quick_start_test.go

@@ -8,11 +8,14 @@ package e2e
 import (
 	"fmt"
 	"slices"
+	"strconv"
 	"strings"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	"github.com/samber/lo"
 	"k8s.io/utils/ptr"
+	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	clusterctlcluster "sigs.k8s.io/cluster-api/cmd/clusterctl/client/cluster"
 	capie2e "sigs.k8s.io/cluster-api/test/e2e"
 	"sigs.k8s.io/cluster-api/test/framework"
@@ -63,7 +66,7 @@ var _ = Describe("Quick start", Serial, func() {
 							// available that are not available in other providers.
 							// This version can be specified in `test/e2e/config/caren.yaml` with a variable named
 							// `KUBERNETES_VERSION_<PROVIDER>`, where `<PROVIDER>` is the uppercase provider name, e.g.
-							// `KUBERNETES_VERSION_DOCKER: v1.29.4`.
+							// `KUBERNETES_VERSION_DOCKER: v1.29.5`.
 							testE2EConfig := e2eConfig.DeepCopy()
 							varName := capie2e.KubernetesVersion + "_" + strings.ToUpper(
 								lowercaseProvider,
@@ -99,21 +102,57 @@ var _ = Describe("Quick start", Serial, func() {
 										KubernetesReferenceAssertions,
 									)
 
+									workloadCluster := framework.GetClusterByName(
+										ctx,
+										framework.GetClusterByNameInput{
+											Namespace: namespace,
+											Name:      clusterName,
+											Getter:    proxy.GetClient(),
+										},
+									)
+									Expect(workloadCluster.Spec.Topology).ToNot(BeNil())
+
 									By("Waiting until nodes are ready")
 									workloadProxy := proxy.GetWorkloadCluster(
 										ctx,
 										namespace,
 										clusterName,
 									)
 									workloadClient := workloadProxy.GetClient()
+
+									nodeCount := int(
+										ptr.Deref(
+											workloadCluster.Spec.Topology.ControlPlane.Replicas,
+											0,
+										),
+									) +
+										lo.Reduce(
+											workloadCluster.Spec.Topology.Workers.MachineDeployments,
+											func(agg int, md clusterv1.MachineDeploymentTopology, _ int) int {
+												switch {
+												case md.Replicas != nil:
+													return agg + int(ptr.Deref(md.Replicas, 0))
+												case md.Metadata.Annotations["cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size"] != "":
+													minSize, err := strconv.Atoi(
+														md.Metadata.Annotations["cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size"],
+													)
+													Expect(err).ToNot(HaveOccurred())
+													return agg + minSize
+												default:
+													return agg
+												}
+											},
+											0,
+										)
+
 									framework.WaitForNodesReady(
 										ctx,
 										framework.WaitForNodesReadyInput{
 											Lister: workloadClient,
 											KubernetesVersion: testE2EConfig.GetVariable(
 												capie2e.KubernetesVersion,
 											),
-											Count: 2,
+											Count: nodeCount,
 											WaitForNodesReady: testE2EConfig.GetIntervals(
 												flavour,
 												"wait-nodes-ready",
@@ -124,15 +163,6 @@ var _ = Describe("Quick start", Serial, func() {
 									By(
 										"Waiting for all requested addons to be ready in workload cluster",
 									)
-									workloadCluster := framework.GetClusterByName(
-										ctx,
-										framework.GetClusterByNameInput{
-											Namespace: namespace,
-											Name:      clusterName,
-											Getter:    proxy.GetClient(),
-										},
-									)
-									Expect(workloadCluster.Spec.Topology).ToNot(BeNil())
 									clusterVars := variables.ClusterVariablesToVariablesMap(
 										workloadCluster.Spec.Topology.Variables,
 									)