refactor: use a string type for Nutanix's AdditionalTrustBundle (#28)

* refactor: use a string type for Nutanix's AdditionalTrustBundle

Use a string instead of a ConfigMap reference
to make it easier for both the handlers to use, and the clients to set in the API.

* fix: force insecure: false with additionalTrustBundle

Author: dkoshkin

api/v1alpha1/nutanix_clusterconfig_types.go

@@ -49,11 +49,11 @@ type NutanixPrismCentralEndpointSpec struct {
 	// +optional
 	Insecure bool `json:"insecure"`
 
-	// A reference to the ConfigMap containing a PEM encoded x509 cert for the RootCA that was used to create
+	// A base64 PEM encoded x509 cert for the RootCA that was used to create
 	// the certificate for a Prism Central that uses certificates that were issued by a non-publicly trusted RootCA.
 	// The trust bundle is added to the cert pool used to authenticate the TLS connection to the Prism Central.
 	// +optional
-	AdditionalTrustBundle *corev1.LocalObjectReference `json:"additionalTrustBundle,omitempty"`
+	AdditionalTrustBundle *string `json:"additionalTrustBundle,omitempty"`
 
 	// A reference to the Secret for credential information for the target Prism Central instance
 	Credentials corev1.LocalObjectReference `json:"credentials"`
@@ -82,19 +82,13 @@ func (NutanixPrismCentralEndpointSpec) VariableSchema() clusterv1.VariableSchema
 					Type:        "boolean",
 				},
 				"additionalTrustBundle": {
-					Description: "A reference to the ConfigMap containing a PEM encoded x509 cert for the RootCA " +
+					Description: "A base64 PEM encoded x509 cert for the RootCA " +
 						"that was used to create the certificate for a Prism Central that uses certificates " +
 						"that were issued by a non-publicly trusted RootCA." +
 						"The trust bundle is added to the cert pool used to authenticate the TLS connection " +
 						"to the Prism Central.",
-					Type: "object",
-					Properties: map[string]clusterv1.JSONSchemaProps{
-						"name": {
-							Description: "The name of the ConfigMap",
-							Type:        "string",
-						},
-					},
-					Required: []string{"name"},
+					Type:   "string",
+					Format: "byte",
 				},
 				"credentials": {
 					Description: "A reference to the Secret for credential information" +

api/v1alpha1/zz_generated.deepcopy.go

@@ -29,7 +29,7 @@ spec:
 
 Applying this configuration will result in the following value being set:
 
-- control-plane NutanixClusterTemplate:
+- `NutanixClusterTemplate`:
 
 ```yaml
 spec:
@@ -43,3 +43,47 @@ spec:
           kind: Secret
           name: secret-name
 ```
+
+### Provide an Optional Trusted CA Bundle
+
+If the Prism Central endpoint uses a self-signed certificate, you can provide an additional trust bundle
+to be used by the Nutanix provider.
+This is a base64 PEM encoded x509 cert for the RootCA that was used to create the certificate for a Prism Central
+
+See [Nutanix Security Guide] for more information.
+
+```yaml
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  name: <NAME>
+spec:
+  topology:
+    variables:
+      - name: clusterConfig
+        value:
+          nutanix:
+            prismCentralEndpoint:
+              # ...
+              additionalTrustBundle: "LS0...="
+```
+
+Applying this configuration will result in the following value being set:
+
+- `NutanixClusterTemplate`:
+
+```yaml
+spec:
+  template:
+    spec:
+      prismCentral:
+        # ...
+        additionalTrustBundle:
+          kind: String
+          data: |-
+            -----BEGIN CERTIFICATE-----
+            ...
+            -----END CERTIFICATE-----
+```
+
+[Nutanix Security Guide]: https://portal.nutanix.com/page/documents/details?targetId=Nutanix-Security-Guide-v6_5:mul-security-ssl-certificate-pc-t.html

docs/content/customization/nutanix/prism-central-endpoint.md

@@ -1,13 +1,4 @@
 apiVersion: v1
-binaryData:
-  ca.crt: ${NUTANIX_ADDITIONAL_TRUST_BUNDLE=""}
-kind: ConfigMap
-metadata:
-  labels:
-    cluster.x-k8s.io/provider: nutanix
-  name: ${CLUSTER_NAME}-pc-trusted-ca-bundle
----
-apiVersion: v1
 data:
   nutanix-ccm.yaml: |
     ---

examples/capi-quick-start/nutanix-cluster-calico-crs.yaml

@@ -1,13 +1,4 @@
 apiVersion: v1
-binaryData:
-  ca.crt: ${NUTANIX_ADDITIONAL_TRUST_BUNDLE=""}
-kind: ConfigMap
-metadata:
-  labels:
-    cluster.x-k8s.io/provider: nutanix
-  name: ${CLUSTER_NAME}-pc-trusted-ca-bundle
----
-apiVersion: v1
 data:
   nutanix-ccm.yaml: |
     ---

examples/capi-quick-start/nutanix-cluster-calico-helm-addon.yaml

@@ -1,13 +1,4 @@
 apiVersion: v1
-binaryData:
-  ca.crt: ${NUTANIX_ADDITIONAL_TRUST_BUNDLE=""}
-kind: ConfigMap
-metadata:
-  labels:
-    cluster.x-k8s.io/provider: nutanix
-  name: ${CLUSTER_NAME}-pc-trusted-ca-bundle
----
-apiVersion: v1
 data:
   nutanix-ccm.yaml: |
     ---

examples/capi-quick-start/nutanix-cluster-cilium-crs.yaml

@@ -1,13 +1,4 @@
 apiVersion: v1
-binaryData:
-  ca.crt: ${NUTANIX_ADDITIONAL_TRUST_BUNDLE=""}
-kind: ConfigMap
-metadata:
-  labels:
-    cluster.x-k8s.io/provider: nutanix
-  name: ${CLUSTER_NAME}-pc-trusted-ca-bundle
----
-apiVersion: v1
 data:
   nutanix-ccm.yaml: |
     ---

examples/capi-quick-start/nutanix-cluster-cilium-helm-addon.yaml

@@ -28,3 +28,9 @@ patches:
 - target:
     kind: Cluster
   path: ../../../patches/nutanix/initialize-variables.yaml
+
+# Remove Additional Trust Bundle ConfigMap
+- target:
+    kind: ConfigMap
+    name: ".*-pc-trusted-ca-bundle"
+  path: ../../../patches/nutanix/remove-additional-trust-bundle/cm.yaml

hack/examples/bases/nutanix/cluster/kustomization.yaml.tmpl

@@ -0,0 +1,8 @@
+# Copyright 2024 D2iQ, Inc. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+$patch: delete
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ${CLUSTER_NAME}-pc-trusted-ca-bundle

hack/examples/patches/nutanix/remove-additional-trust-bundle/cm.yaml

@@ -5,10 +5,13 @@ package prismcentralendpoint
 
 import (
 	"context"
+	"encoding/base64"
+	"fmt"
 
 	"github.com/nutanix-cloud-native/prism-go-client/environment/credentials"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/utils/ptr"
 	runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -105,15 +108,27 @@ func (h *nutanixPrismCentralEndpoint) Mutate(
 					Namespace: clusterKey.Namespace,
 				},
 			}
-			if prismCentralEndpointVar.AdditionalTrustBundle != nil {
+			additionalTrustBundle := ptr.Deref(prismCentralEndpointVar.AdditionalTrustBundle, "")
+			if additionalTrustBundle != "" {
+				var decoded []byte
+				decoded, err = base64.StdEncoding.DecodeString(additionalTrustBundle)
+				if err != nil {
+					log.Error(err, "error decoding additional trust bundle")
+					return fmt.Errorf("error decoding additional trust bundle: %w", err)
+				}
 				prismCentral.AdditionalTrustBundle = &credentials.NutanixTrustBundleReference{
-					Kind: credentials.NutanixTrustBundleKindConfigMap,
-					Name: prismCentralEndpointVar.AdditionalTrustBundle.Name,
-					// Assume the ConfigMap is in the same namespace as Cluster
-					Namespace: clusterKey.Namespace,
+					Kind: credentials.NutanixTrustBundleKindString,
+					Data: string(decoded),
 				}
 			}
 
+			// Always force insecure to false if additional trust bundle is provided.
+			// This ensures that the trust bundle is actually used to validate the connection.
+			if additionalTrustBundle != "" && prismCentral.Insecure {
+				log.Info("AdditionalTrustBundle is provided, setting insecure to false")
+				prismCentral.Insecure = false
+			}
+
 			obj.Spec.Template.Spec.PrismCentral = prismCentral
 
 			return nil

pkg/handlers/nutanix/mutation/prismcentralendpoint/inject.go

@@ -17,6 +17,9 @@ import (
 	"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest/request"
 )
 
+//nolint:lll // just a long string
+const testCertBundle = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVjekNDQTF1Z0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRUUZBRC4uQWtHQTFVRUJoTUNSMEl4CkV6QVJCZ05WQkFnVENsTnZiV1V0VTNSaGRHVXhGREFTQmdOVkJBb1RDMC4uMEVnVEhSa01UY3dOUVlEClZRUUxFeTVEYkdGemN5QXhJRkIxWW14cFl5QlFjbWx0WVhKNUlFTmxjbi4uWFJwYjI0Z1FYVjBhRzl5CmFYUjVNUlF3RWdZRFZRUURFd3RDWlhOMElFTkJJRXgwWkRBZUZ3MHdNRC4uVFV3TVRaYUZ3MHdNVEF5Ck1EUXhPVFV3TVRaYU1JR0hNUXN3Q1FZRFZRUUdFd0pIUWpFVE1CRUdBMS4uMjl0WlMxVGRHRjBaVEVVCk1CSUdBMVVFQ2hNTFFtVnpkQ0JEUVNCTWRHUXhOekExQmdOVkJBc1RMay4uREVnVUhWaWJHbGpJRkJ5CmFXMWhjbmtnUTJWeWRHbG1hV05oZEdsdmJpQkJkWFJvYjNKcGRIa3hGRC4uQU1UQzBKbGMzUWdRMEVnClRIUmtNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZy4uVHoybXI3U1ppQU1mUXl1CnZCak05T2lKalJhelhCWjFCalA1Q0UvV20vUnI1MDBQUksrTGg5eDVlSi4uL0FOQkUwc1RLMFpzREdNCmFrMm0xZzdvcnVJM2RZM1ZIcUl4RlR6MFRhMWQrTkFqd25MZTRuT2I3Ly4uazA1U2hoQnJKR0JLS3hiCjhuMTA0by81cDhIQXNaUGR6YkZNSXlOakp6Qk0ybzV5NUExM3dpTGl0RS4uZnlZa1F6YXhDdzBBd3psCmtWSGlJeUN1YUY0d2o1NzFwU3prdjZzdis0SURNYlQvWHBDbzhMNndUYS4uc2grZXRMRDZGdFRqWWJiCnJ2WjhSUU0xdGxLZG9NSGcycXhyYUFWKytITkJZbU5XczBkdUVkalViSi4uWEk5VHRuUzRvMUNrajdQCk9mbGppUUlEQVFBQm80SG5NSUhrTUIwR0ExVWREZ1FXQkJROHVyTUNSTC4uNUFrSXA5TkpISnc1VENCCnRBWURWUjBqQklHc01JR3BnQlE4dXJNQ1JMWVlNSFVLVTVBa0lwOU5KSC4uYVNCaWpDQmh6RUxNQWtHCkExVUVCaE1DUjBJeEV6QVJCZ05WQkFnVENsTnZiV1V0VTNSaGRHVXhGRC4uQW9UQzBKbGMzUWdRMEVnClRIUmtNVGN3TlFZRFZRUUxFeTVEYkdGemN5QXhJRkIxWW14cFl5QlFjbS4uRU5sY25ScFptbGpZWFJwCmIyNGdRWFYwYUc5eWFYUjVNUlF3RWdZRFZRUURFd3RDWlhOMElFTkJJRS4uREFNQmdOVkhSTUVCVEFECkFRSC9NQTBHQ1NxR1NJYjNEUUVCQkFVQUE0SUJBUUMxdVlCY3NTbmN3QS4uRENzUWVyNzcyQzJ1Y3BYCnhRVUUvQzBwV1dtNmdEa3dkNUQwRFNNREpScVYvd2VvWjR3QzZCNzNmNS4uYkxoR1lIYVhKZVNENktyClhjb093TGRTYUdtSllzbExLWkIzWklERXAwd1lUR2hndGViNkpGaVR0bi4uc2YyeGRyWWZQQ2lJQjdnCkJNQVY3R3pkYzRWc3BTNmxqckFoYmlpYXdkQmlRbFFtc0JlRno5SmtGNC4uYjNsOEJvR04rcU1hNTZZCkl0OHVuYTJnWTRsMk8vL29uODhyNUlXSmxtMUwwb0E4ZTRmUjJ5ckJIWC4uYWRzR2VGS2t5TnJ3R2kvCjd2UU1mWGRHc1JyWE5HUkduWCt2V0RaMy96V0kwam9EdENrTm5xRXBWbi4uSG9YCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0="
+
 func TestGeneratePatches(
 	t *testing.T,
 	generatorFunc func() mutation.GeneratePatches,
@@ -31,20 +34,52 @@ func TestGeneratePatches(
 			Name: "unset variable",
 		},
 		capitest.PatchTestDef{
-			Name: "all fields set",
+			Name: "all required fields set",
+			Vars: []runtimehooksv1.Variable{
+				capitest.VariableWithValue(
+					variableName,
+					v1alpha1.NutanixPrismCentralEndpointSpec{
+						Host:     "prism-central.nutanix.com",
+						Port:     9441,
+						Insecure: true,
+						Credentials: corev1.LocalObjectReference{
+							Name: "credentials",
+						},
+					},
+					variablePath...,
+				),
+			},
+			RequestItem: request.NewNutanixClusterTemplateRequestItem(""),
+			ExpectedPatchMatchers: []capitest.JSONPatchMatcher{
+				{
+					Operation: "replace",
+					Path:      "/spec/template/spec/prismCentral",
+					ValueMatcher: gomega.SatisfyAll(
+						gomega.HaveKeyWithValue(
+							"address",
+							gomega.BeEquivalentTo("prism-central.nutanix.com"),
+						),
+						gomega.HaveKeyWithValue("port", gomega.BeEquivalentTo(9441)),
+						gomega.HaveKeyWithValue("insecure", true),
+						gomega.HaveKey("credentialRef"),
+						gomega.Not(gomega.HaveKey("additionalTrustBundle")),
+					),
+				},
+			},
+		},
+		capitest.PatchTestDef{
+			Name: "additional trust bundle is set",
 			Vars: []runtimehooksv1.Variable{
 				capitest.VariableWithValue(
 					variableName,
 					v1alpha1.NutanixPrismCentralEndpointSpec{
 						Host:     "prism-central.nutanix.com",
 						Port:     9441,
-						Insecure: false,
+						Insecure: true,
 						Credentials: corev1.LocalObjectReference{
 							Name: "credentials",
 						},
-						AdditionalTrustBundle: ptr.To(corev1.LocalObjectReference{
-							Name: "bundle",
-						}),
+						AdditionalTrustBundle: ptr.To(testCertBundle),
 					},
 					variablePath...,
 				),
@@ -60,6 +95,7 @@ func TestGeneratePatches(
 							gomega.BeEquivalentTo("prism-central.nutanix.com"),
 						),
 						gomega.HaveKeyWithValue("port", gomega.BeEquivalentTo(9441)),
+						// Assert the insecure field was set to false as the additional trust bundle is set
 						gomega.HaveKeyWithValue("insecure", false),
 						gomega.HaveKey("credentialRef"),
 						gomega.HaveKey("additionalTrustBundle"),