feat: adds cluster's ownerref on cilium helm values source object (#1034)

**What problem does this PR solve?**:
Adds cluster's ownerref on cilium helm values source object so users
won't have to explicitly need to add `clusterctl.cluster.x-k8s.io/move:
""` label

**Which issue(s) this PR fixes**:
Fixes #
NCN-105148

**How Has This Been Tested?**:
<!--
Please describe the tests that you ran to verify your changes.
Provide output from the tests and any manual steps needed to replicate
the tests.
-->

**Special notes for your reviewer**:
<!--
Use this to provide any additional information to the reviewers.
This may include:
- Best way to review the PR.
- Where the author wants the most review attention on.
- etc.
-->

```
apiVersion: v1
data:
  values.yaml: |-
    cni:
      chainingMode: portmap
      exclusive: false
    hubble:
      enabled: true
      tls:
        auto:
          enabled: true               # enable automatic TLS certificate generation
          method: cronJob             # auto generate certificates using cronJob method
          certValidityDuration: 60    # certificates validity duration in days (default 2 months)
          schedule: "0 0 5 * *"       # schedule on the 1st day regeneration of each month
      relay:
        enabled: true
        image:
          useDigest: false
    ipam:
      mode: kubernetes
    image:
      useDigest: false
    operator:
      image:
        useDigest: false
    certgen:
      image:
        useDigest: false
    socketLB:
      hostNamespaceOnly: true
    envoy:
      image:
        useDigest: false
kind: ConfigMap
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"v1","data":{"values.yaml":"cni:\n  chainingMode: portmap\n  exclusive: false\nhubble:\n  enabled: true\n  tls:\n    auto:\n      enabled: true               # enable automatic TLS certificate generation\n      method: cronJob             # auto generate certificates using cronJob method\n      certValidityDuration: 60    # certificates validity duration in days (default 2 months)\n      schedule: \"0 0 5 * *\"       # schedule on the 1st day regeneration of each month\n  relay:\n    enabled: true\n    image:\n      useDigest: false\nipam:\n  mode: kubernetes\nimage:\n  useDigest: false\noperator:\n  image:\n    useDigest: false\ncertgen:\n  image:\n    useDigest: false\nsocketLB:\n  hostNamespaceOnly: true\nenvoy:\n  image:\n    useDigest: false"},"kind":"ConfigMap","metadata":{"annotations":{},"labels":{"cluster.x-k8s.io/cluster-name":"nkp-mgmt-cluster"},"name":"custom-cilium-cni-helm-values-template","namespace":"default"}}
  creationTimestamp: "2025-02-05T12:47:38Z"
  labels:
    cluster.x-k8s.io/cluster-name: nkp-mgmt-cluster
  name: custom-cilium-cni-helm-values-template
  namespace: default
  ownerReferences:
  - apiVersion: cluster.x-k8s.io/v1beta1
    kind: Cluster
    name: nkp-mgmt-cluster
    uid: 6915b86b-91ae-4673-bb6e-3a94f96dc22f
  resourceVersion: "9260"
  uid: c541aec0-2b5d-4b0d-8cdb-ba63bef0ca06
```
```
➜  ~ kg cluster -oyaml
apiVersion: v1
items:
- apiVersion: cluster.x-k8s.io/v1beta1
  kind: Cluster
  metadata:
    annotations:
      caren.nutanix.com/cluster-uuid: 0194d059-494c-7879-a6bd-fe281ba362d9
      kubectl.kubernetes.io/last-applied-configuration: |
        {"apiVersion":"cluster.x-k8s.io/v1beta1","kind":"Cluster","metadata":{"annotations":{"caren.nutanix.com/cluster-uuid":"0194d059-494c-7879-a6bd-fe281ba362d9"},"labels":{"cluster.x-k8s.io/provider":"nutanix","konvoy.d2iq.io/cluster-name":"nkp-mgmt-cluster","konvoy.d2iq.io/provider":"nutanix"},"name":"nkp-mgmt-cluster","namespace":"default"},"spec":{"clusterNetwork":{"pods":{"cidrBlocks":["192.168.0.0/16"]},"services":{"cidrBlocks":["10.96.0.0/12"]}},"controlPlaneEndpoint":{"host":"","port":0},"topology":{"class":"nutanix-quick-start","controlPlane":{"metadata":{},"replicas":1},"variables":[{"name":"clusterConfig","value":{"addons":{"ccm":{"credentials":{"secretRef":{"name":"nkp-mgmt-cluster-pc-credentials"}},"strategy":"HelmAddon"},"clusterAutoscaler":{"strategy":"HelmAddon"},"cni":{"provider":"Cilium","strategy":"HelmAddon","values":{"sourceRef":{"kind":"ConfigMap","name":"custom-cilium-cni-helm-values-template"}}},"csi":{"defaultStorage":{"provider":"nutanix","storageClassConfig":"volume"},"providers":{"nutanix":{"credentials":{"secretRef":{"name":"nkp-mgmt-cluster-pc-credentials-for-csi"}},"storageClassConfigs":{"volume":{"allowExpansion":true,"parameters":{"csi.storage.k8s.io/fstype":"ext4","description":"CSI StorageClass nutanix-volume for nkp-mgmt-cluster","flashMode":"DISABLED","hypervisorAttached":"ENABLED","storageContainer":"default-container-32638919133770","storageType":"NutanixVolumes"},"reclaimPolicy":"Delete","volumeBindingMode":"WaitForFirstConsumer"}},"strategy":"HelmAddon"}},"snapshotController":{"strategy":"HelmAddon"}},"nfd":{"strategy":"HelmAddon"},"serviceLoadBalancer":{"configuration":{"addressRanges":[{"end":"10.47.10.82","start":"10.47.10.82"}]},"provider":"MetalLB"}},"controlPlane":{"nutanix":{"machineDetails":{"bootType":"uefi","cluster":{"name":"auto_cluster_prod_manoj_surudwad_1a2aac0a51c7","type":"name"},"image":{"name":"nkp-rocky-9.5-release-1.31.4-20250122010854.qcow2","type":"name"},"memorySize":"8Gi","subnets":[{"name":"vlan.155","type":"name"}],"systemDiskSize":"80Gi","vcpuSockets":4,"vcpusPerSocket":1}}},"dns":{"coreDNS":{}},"encryptionAtRest":{"providers":[{"secretbox":{}}]},"imageRegistries":[{"credentials":{"secretRef":{"name":"nkp-mgmt-cluster-image-registry-credentials"}},"url":"https://docker.io"}],"nutanix":{"controlPlaneEndpoint":{"host":"10.47.10.4","port":6443,"virtualIP":{"provider":"KubeVIP"}},"prismCentralEndpoint":{"credentials":{"secretRef":{"name":"nkp-mgmt-cluster-pc-credentials"}},"insecure":true,"url":"https://10.47.10.25:9440"}},"users":[{"name":"konvoy","sshAuthorizedKeys":["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCky5ata76YkcrZi2EdXcD4ESrHtGUU3+0aqtzG4PueOtjhcLSd5160+fKsyLt/CC7+0MbLW6nQYZH3HHgQuF0gb5gfXHuHH4kLJnmRmP3nM3KhZZ2f0lOtKpSJ5lPxvIlbTHacWkd4s4tlmUQvg4XRasQ1t8dgjQGmyQdCuMHE02jOsmc4JGseJ6u3Ne6dJtTwglwLbN+IlsJECuVubU7yqN0ZHL1G5mAy2IWzCRKmDL9ZyCXSM9QWsbTqnvuVGVUp0cW3YquTuoEuiupWYw6HRQSAvFHGUR+juHmUwIVFkgGjBlOAdMiRhRJHEUnE6YpTKZEfIhw9nlwXQfdXRXsoNcV5pDNr2Pd71DlHocAQRSlEa3sZ75W8O5TPzkzxF8W0WLLiiS2PlnbZFLiY2RyQcUBXKOeYVHZTvUpLNnNISPzF4IB8OwZDpJWWQ4kLZ7PAdL2NeN/uI3a5eDXcyBkqisnBewm6YHvDjfV/dTeJrxo9vPX++rPlAe7C6Edr7as=\n"],"sudo":"ALL=(ALL) NOPASSWD:ALL"}]}}],"version":"v1.31.4","workers":{"machineDeployments":[{"class":"default-worker","metadata":{"annotations":{"cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size":"3","cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size":"3"}},"name":"md-0","variables":{"overrides":[{"name":"workerConfig","value":{"nutanix":{"machineDetails":{"bootType":"uefi","cluster":{"name":"auto_cluster_prod_manoj_surudwad_1a2aac0a51c7","type":"name"},"image":{"name":"nkp-rocky-9.5-release-1.31.4-20250122010854.qcow2","type":"name"},"memorySize":"8Gi","subnets":[{"name":"vlan.155","type":"name"}],"systemDiskSize":"80Gi","vcpuSockets":8,"vcpusPerSocket":1}}}}]}}]}}}}
    creationTimestamp: "2025-02-05T12:47:16Z"
    finalizers:
    - cluster.cluster.x-k8s.io
    generation: 5
    labels:
      cluster.x-k8s.io/cluster-name: nkp-mgmt-cluster
      cluster.x-k8s.io/provider: nutanix
      konvoy.d2iq.io/cluster-name: nkp-mgmt-cluster
      konvoy.d2iq.io/provider: nutanix
      topology.cluster.x-k8s.io/owned: ""
    name: nkp-mgmt-cluster
    namespace: default
    resourceVersion: "9931"
    uid: 6915b86b-91ae-4673-bb6e-3a94f96dc22f
  spec:
...
...
...
  status:
    conditions:
    - lastTransitionTime: "2025-02-05T12:48:54Z"
      status: "True"
      type: Ready
    - lastTransitionTime: "2025-02-05T12:48:54Z"
      status: "True"
      type: ControlPlaneInitialized
    - lastTransitionTime: "2025-02-05T12:48:54Z"
      status: "True"
      type: ControlPlaneReady
    - lastTransitionTime: "2025-02-05T12:47:22Z"
      status: "True"
      type: InfrastructureReady
    - lastTransitionTime: "2025-02-05T12:50:34Z"
      status: "True"
      type: TopologyReconciled
    controlPlaneReady: true
    infrastructureReady: true
    observedGeneration: 5
    phase: Provisioned
```

---------

Co-authored-by: Dimitri Koshkin <[email protected]>

Author: manoj-nutanix

docs/content/addons/cni.md

@@ -75,8 +75,6 @@ data:
       mode: kubernetes
 kind: ConfigMap
 metadata:
-  labels:
-    clusterctl.cluster.x-k8s.io/move: ""
   name: <CLUSTER_NAME>-cilium-cni-helm-values-template
   namespace: <CLUSTER_NAMESPACE>
 ```

pkg/handlers/generic/lifecycle/ccm/nutanix/handler.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/go-logr/logr"
 	"github.com/spf13/pflag"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
@@ -84,10 +85,13 @@ func (p *provider) Apply(
 	// However, that would leave the credentials visible in the HelmChartProxy.
 	// Instead, we'll create the Secret on the remote cluster and reference it in the Helm values.
 	if clusterConfig.Addons.CCM.Credentials != nil {
-		err := handlersutils.EnsureOwnerReferenceForSecret(
+		err := handlersutils.EnsureClusterOwnerReferenceForObject(
 			ctx,
 			p.client,
-			clusterConfig.Addons.CCM.Credentials.SecretRef.Name,
+			corev1.TypedLocalObjectReference{
+				Kind: "Secret",
+				Name: clusterConfig.Addons.CCM.Credentials.SecretRef.Name,
+			},
 			cluster,
 		)
 		if err != nil {

pkg/handlers/generic/lifecycle/cni/cilium/handler.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 
 	"github.com/spf13/pflag"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/utils/ptr"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
@@ -22,6 +23,7 @@ import (
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/addons"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/config"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/options"
+	handlersutils "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/utils"
 )
 
 type CNIConfig struct {
@@ -183,6 +185,33 @@ func (c *CiliumCNI) apply(
 			helmValuesSourceRefName = cniVar.Values.SourceRef.Name
 			// Use cluster's namespace since Values.SourceRef is always a LocalObjectReference
 			targetNamespace = cluster.Namespace
+
+			err := handlersutils.EnsureClusterOwnerReferenceForObject(
+				ctx,
+				c.client,
+				corev1.TypedLocalObjectReference{
+					Kind: cniVar.Values.SourceRef.Kind,
+					Name: cniVar.Values.SourceRef.Name,
+				},
+				cluster,
+			)
+			if err != nil {
+				log.Error(
+					err,
+					"error updating Cluster's owner reference on Cilium helm values source object",
+					"name",
+					cniVar.Values.SourceRef.Name,
+					"kind",
+					cniVar.Values.SourceRef.Kind,
+				)
+				resp.SetStatus(runtimehooksv1.ResponseStatusFailure)
+				resp.SetMessage(
+					fmt.Sprintf(
+						"failed to set Cluster's owner reference on Cilium helm values source object: %v",
+						err,
+					),
+				)
+			}
 		}
 
 		strategy = addons.NewHelmAddonApplier(

pkg/handlers/generic/lifecycle/csi/nutanix/handler.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/go-logr/logr"
 	"github.com/spf13/pflag"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/utils/ptr"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
@@ -108,10 +109,13 @@ func (n *NutanixCSI) Apply(
 	}
 
 	if provider.Credentials != nil {
-		err := handlersutils.EnsureOwnerReferenceForSecret(
+		err := handlersutils.EnsureClusterOwnerReferenceForObject(
 			ctx,
 			n.client,
-			provider.Credentials.SecretRef.Name,
+			corev1.TypedLocalObjectReference{
+				Kind: "Secret",
+				Name: provider.Credentials.SecretRef.Name,
+			},
 			cluster,
 		)
 		if err != nil {

pkg/handlers/generic/mutation/imageregistries/credentials/inject.go

@@ -8,6 +8,7 @@ import (
 	"errors"
 	"fmt"
 
+	corev1 "k8s.io/api/core/v1"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -287,10 +288,13 @@ func ensureOwnerReferenceOnCredentialsSecrets(
 		if secretName := handlersutils.SecretNameForImageRegistryCredentials(credential); secretName != "" {
 			// Ensure the Secret is owned by the Cluster so it is correctly moved and deleted with the Cluster.
 			// This code assumes that Secret exists and that was validated before calling this function.
-			err := handlersutils.EnsureOwnerReferenceForSecret(
+			err := handlersutils.EnsureClusterOwnerReferenceForObject(
 				ctx,
 				c,
-				secretName,
+				corev1.TypedLocalObjectReference{
+					Kind: "Secret",
+					Name: secretName,
+				},
 				cluster,
 			)
 			if err != nil {

pkg/handlers/utils/secrets.go

@@ -9,7 +9,9 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	"sigs.k8s.io/cluster-api/controllers/external"
 	"sigs.k8s.io/cluster-api/controllers/remote"
 	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
@@ -64,30 +66,62 @@ func CopySecretToRemoteCluster(
 	return nil
 }
 
-// EnsureOwnerReferenceForSecret will ensure that the secretName Secret has an OwnerReference of the cluster.
-func EnsureOwnerReferenceForSecret(
+// EnsureClusterOwnerReferenceForObject ensures that OwnerReference of the cluster is added on provided object.
+func EnsureClusterOwnerReferenceForObject(
 	ctx context.Context,
 	cl ctrlclient.Client,
-	secretName string,
+	objectRef corev1.TypedLocalObjectReference,
 	cluster *clusterv1.Cluster,
 ) error {
-	secret, err := getSecretForCluster(ctx, cl, secretName, cluster)
+	targetObj, err := GetResourceFromTypedLocalObjectReference(
+		ctx,
+		cl,
+		objectRef,
+		cluster.Namespace,
+	)
 	if err != nil {
 		return err
 	}
 
-	err = controllerutil.SetOwnerReference(cluster, secret, cl.Scheme())
+	err = controllerutil.SetOwnerReference(cluster, targetObj, cl.Scheme())
 	if err != nil {
-		return fmt.Errorf("failed to set owner reference on Secret: %w", err)
+		return fmt.Errorf("failed to set cluster's owner reference on object: %w", err)
 	}
 
-	err = cl.Update(ctx, secret)
+	err = cl.Update(ctx, targetObj)
 	if err != nil {
-		return fmt.Errorf("failed to update Secret with owner references: %w", err)
+		return fmt.Errorf("failed to update object with cluster's owner reference: %w", err)
 	}
 	return nil
 }
 
+// GetResourceFromTypedLocalObjectReference gets the resource from the provided TypedLocalObjectReference.
+func GetResourceFromTypedLocalObjectReference(
+	ctx context.Context,
+	cl ctrlclient.Client,
+	typedLocalObjectRef corev1.TypedLocalObjectReference,
+	ns string,
+) (*unstructured.Unstructured, error) {
+	apiVersion := corev1.SchemeGroupVersion.String()
+	if typedLocalObjectRef.APIGroup != nil {
+		apiVersion = *typedLocalObjectRef.APIGroup
+	}
+
+	objectRef := &corev1.ObjectReference{
+		APIVersion: apiVersion,
+		Kind:       typedLocalObjectRef.Kind,
+		Name:       typedLocalObjectRef.Name,
+		Namespace:  ns,
+	}
+
+	targetObj, err := external.Get(ctx, cl, objectRef)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get resource from object reference: %w", err)
+	}
+
+	return targetObj, nil
+}
+
 func getSecretForCluster(
 	ctx context.Context,
 	c ctrlclient.Client,

pkg/handlers/utils/secrets_test.go

@@ -5,12 +5,14 @@ package utils
 
 import (
 	"context"
+	"fmt"
 	"testing"
 
+	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/api/errors"
+	apiErrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -97,23 +99,38 @@ func Test_EnsureOwnerReferenceForSecret(t *testing.T) {
 			client:     buildFakeClient(t, testSecret, testCluster),
 			secretName: "missing-secret",
 			cluster:    testCluster,
-			wantErr:    errors.NewNotFound(corev1.Resource("secrets"), "missing-secret"),
+			wantErr: fmt.Errorf(
+				"failed to get resource from object reference: %w",
+				errors.Wrapf(
+					apiErrors.NewNotFound(corev1.Resource("secrets"), "missing-secret"),
+					"failed to retrieve %s %s",
+					"Secret",
+					"missing-secret",
+				),
+			),
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			t.Parallel()
 
-			err := EnsureOwnerReferenceForSecret(
+			err := EnsureClusterOwnerReferenceForObject(
 				context.Background(),
 				tt.client,
-				tt.secretName,
+				corev1.TypedLocalObjectReference{
+					Kind: "Secret",
+					Name: tt.secretName,
+				},
 				tt.cluster,
 			)
-			require.Equal(t, tt.wantErr, err)
+
 			if tt.wantErr != nil {
+				assert.Equal(t, tt.wantErr.Error(), err.Error())
 				return
+			} else {
+				require.NoError(t, err)
 			}
+
 			// verify that the owner reference was added
 			secret := &corev1.Secret{}
 			err = tt.client.Get(