✨ Add ControlPlane upgrade testCase

Author: m-messiah

virtualcluster/pkg/controller/controllers/clusterversion_controller_test.go

@@ -95,7 +95,7 @@ var defaultClusterVersion = &v1alpha1.ClusterVersionSpec{
 					},
 				},
 				UpdateStrategy: appsv1.StatefulSetUpdateStrategy{
-					Type: appsv1.OnDeleteStatefulSetStrategyType,
+					Type: appsv1.RollingUpdateStatefulSetStrategyType,
 				},
 				Template: corev1.PodTemplateSpec{
 					ObjectMeta: metav1.ObjectMeta{
@@ -247,7 +247,7 @@ var defaultClusterVersion = &v1alpha1.ClusterVersionSpec{
 					},
 				},
 				UpdateStrategy: appsv1.StatefulSetUpdateStrategy{
-					Type: appsv1.OnDeleteStatefulSetStrategyType,
+					Type: appsv1.RollingUpdateStatefulSetStrategyType,
 				},
 				Template: corev1.PodTemplateSpec{
 					ObjectMeta: metav1.ObjectMeta{
@@ -302,6 +302,7 @@ var defaultClusterVersion = &v1alpha1.ClusterVersionSpec{
 									{
 										ContainerPort: 6443,
 										Name:          "api",
+										Protocol:      corev1.ProtocolTCP, // Must be explicit in 1.19
 									},
 								},
 								LivenessProbe: &corev1.Probe{
@@ -453,7 +454,7 @@ var defaultClusterVersion = &v1alpha1.ClusterVersionSpec{
 					},
 				},
 				UpdateStrategy: appsv1.StatefulSetUpdateStrategy{
-					Type: appsv1.OnDeleteStatefulSetStrategyType,
+					Type: appsv1.RollingUpdateStatefulSetStrategyType,
 				},
 				Template: corev1.PodTemplateSpec{
 					ObjectMeta: metav1.ObjectMeta{

virtualcluster/pkg/controller/controllers/suite_test.go

@@ -34,6 +34,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
 	"sigs.k8s.io/cluster-api-provider-nested/virtualcluster/pkg/apis"
+	"sigs.k8s.io/cluster-api-provider-nested/virtualcluster/pkg/syncer/util/featuregate"
 	// +kubebuilder:scaffold:imports
 )
 
@@ -72,6 +73,8 @@ var _ = BeforeSuite(func() {
 	err = apis.AddToScheme(scheme.Scheme)
 	Expect(err).NotTo(HaveOccurred())
 
+	Expect(featuregate.DefaultFeatureGate.Set(featuregate.VirtualClusterApplyUpdate, true)).NotTo(HaveOccurred())
+
 	// +kubebuilder:scaffold:scheme
 
 	mgr, err := ctrl.NewManager(cfg, ctrl.Options{

virtualcluster/pkg/controller/controllers/virtualcluster_controller_test.go

@@ -18,10 +18,10 @@ package controllers
 
 import (
 	"context"
+	"time"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -32,6 +32,7 @@ import (
 
 	tenancyv1alpha1 "sigs.k8s.io/cluster-api-provider-nested/virtualcluster/pkg/apis/tenancy/v1alpha1"
 	"sigs.k8s.io/cluster-api-provider-nested/virtualcluster/pkg/controller/secret"
+	"sigs.k8s.io/cluster-api-provider-nested/virtualcluster/pkg/syncer/constants"
 )
 
 func getClusterObjectKey(instance *tenancyv1alpha1.VirtualCluster, name string) client.ObjectKey {
@@ -41,11 +42,14 @@ func getClusterObjectKey(instance *tenancyv1alpha1.VirtualCluster, name string)
 var _ = Describe("VirtualCluster Controller", func() {
 
 	Context("Reconcile VirtualCluster Cluster", func() {
+		var cvInstance *tenancyv1alpha1.ClusterVersion
+		var instance *tenancyv1alpha1.VirtualCluster
+
 		It("Should create resources successfully", func() {
 			ctx := context.TODO()
 			Expect(cli).ShouldNot(BeNil())
 
-			cvInstance := createClusterVersion()
+			cvInstance = createClusterVersion()
 			Expect(cli.Create(ctx, cvInstance)).Should(Succeed())
 
 			By("Fetching ClusterVersion")
@@ -55,7 +59,7 @@ var _ = Describe("VirtualCluster Controller", func() {
 				return !apierrors.IsNotFound(err)
 			}, timeout, interval).Should(BeTrue())
 
-			instance := &tenancyv1alpha1.VirtualCluster{
+			instance = &tenancyv1alpha1.VirtualCluster{
 				ObjectMeta: metav1.ObjectMeta{
 					GenerateName: "virtualcluster-sample",
 					Namespace:    "default",
@@ -161,9 +165,71 @@ var _ = Describe("VirtualCluster Controller", func() {
 			By("Faking controller-manager STS Status Updates")
 			err = cli.Status().Update(ctx, cmSts)
 			Expect(err).To(BeNil())
+		})
+		It("Should upgrade resources successfully", func() {
+			ctx := context.TODO()
+			Expect(cli).ShouldNot(BeNil())
+
+			objectKey := client.ObjectKeyFromObject(instance)
+
+			By("Checking cluster phase")
+			Eventually(func() bool {
+				err := cli.Get(ctx, objectKey, instance)
+				return err == nil && instance.Status.Phase == tenancyv1alpha1.ClusterRunning
+			}, time.Minute*5, interval).Should(BeTrue())
+
+			By("Updating ClusterVersion")
+			cvInstance.Spec.ETCD.StatefulSet.Spec.Template.Spec.Containers[0].Args = append([]string{"-debug"}, cvInstance.Spec.ETCD.StatefulSet.Spec.Template.Spec.Containers[0].Args...)
+			cvInstance.Spec.APIServer.StatefulSet.Spec.Template.Spec.Containers[0].Args = append([]string{"-v=7"}, cvInstance.Spec.APIServer.StatefulSet.Spec.Template.Spec.Containers[0].Args...)
+			if cvInstance.Spec.APIServer.Service.Labels == nil {
+				cvInstance.Spec.APIServer.Service.Labels = map[string]string{}
+			}
+			cvInstance.Spec.APIServer.Service.Labels["test-label"] = "test"
+			cvInstance.Spec.ControllerManager.StatefulSet.Spec.Template.Spec.Containers[0].Args = append([]string{"-v=7"}, cvInstance.Spec.ControllerManager.StatefulSet.Spec.Template.Spec.Containers[0].Args...)
+			cvInstance.ObjectMeta.ManagedFields = nil
+			forceTrue := true
+			Expect(cli.Patch(ctx, cvInstance, client.Apply, &client.PatchOptions{Force: &forceTrue, FieldManager: "test"})).Should(Succeed())
+
+			By("Enable upgrade for cluster")
+			instance.Labels[constants.LabelVCReadyForUpgrade] = "true"
+			instance.ObjectMeta.ManagedFields = nil
+			Expect(cli.Patch(ctx, instance, client.Apply, &client.PatchOptions{Force: &forceTrue, FieldManager: "test"})).Should(Succeed())
+
+			By("APIServer Service upgraded")
+			Eventually(func() bool {
+				svcObjectKey := getClusterObjectKey(instance, "apiserver-svc")
+				svc := &corev1.Service{}
+				err := cli.Get(ctx, svcObjectKey, svc)
+				return !apierrors.IsNotFound(err) && svc.Labels["test-label"] == "test"
+			}, time.Minute*2, interval).Should(BeTrue())
 
+			etcdSts := &appsv1.StatefulSet{}
+			By("Control Plane etcd StatefulSet upgraded")
+			Eventually(func() bool {
+				stsObjectKey := getClusterObjectKey(instance, "etcd")
+				err := cli.Get(ctx, stsObjectKey, etcdSts)
+				return !apierrors.IsNotFound(err) && etcdSts.Spec.Template.Spec.Containers[0].Args[0] == "-debug"
+			}, time.Minute*2, interval).Should(BeTrue())
+
+			apiserverSts := &appsv1.StatefulSet{}
+			By("Control Plane apiserver StatefulSet upgraded")
+			Eventually(func() bool {
+				stsObjectKey := getClusterObjectKey(instance, "apiserver")
+				err := cli.Get(ctx, stsObjectKey, apiserverSts)
+				return !apierrors.IsNotFound(err) && apiserverSts.Spec.Template.Spec.Containers[0].Args[0] == "-v=7"
+			}, time.Minute*2, interval).Should(BeTrue())
+
+			cmSts := &appsv1.StatefulSet{}
+			By("Control Plane controller-manager StatefulSet upgraded")
+			Eventually(func() bool {
+				stsObjectKey := getClusterObjectKey(instance, "controller-manager")
+				err := cli.Get(ctx, stsObjectKey, cmSts)
+				return !apierrors.IsNotFound(err) && cmSts.Spec.Template.Spec.Containers[0].Args[0] == "-v=7"
+			}, time.Minute*2, interval).Should(BeTrue())
+		})
+		It("Should delete cluster", func() {
 			By("Deleting VirtualCluster")
-			Expect(cli.Delete(ctx, instance)).To(BeNil())
+			Expect(cli.Delete(context.TODO(), instance)).To(BeNil())
 		})
 	})
 })