feat: deploy Nutanix CCM addon

Aligns the method of deploying the CCM with all other addons.

Author: dkoshkin

api/v1alpha1/addon_types.go

@@ -308,12 +308,34 @@ func (CSI) VariableSchema() clusterv1.VariableSchema {
 }
 
 // CCM tells us to enable or disable the cloud provider interface.
-type CCM struct{}
+type CCM struct {
+	// A reference to the Secret for credential information for the target Prism Central instance
+	// +optional
+	Credentials *corev1.LocalObjectReference `json:"credentials"`
+}
 
 func (CCM) VariableSchema() clusterv1.VariableSchema {
+	// TODO Validate credentials is set.
+	// This CCM is shared across all providers.
+	// Some of these providers may require credentials to be set, but we don't want to require it for all providers.
+	// The Nutanix CCM handler will fail in at runtime if credentials are not set.
 	return clusterv1.VariableSchema{
 		OpenAPIV3Schema: clusterv1.JSONSchemaProps{
 			Type: "object",
+			Properties: map[string]clusterv1.JSONSchemaProps{
+				"credentials": {
+					Description: "A reference to the Secret for credential information" +
+						"for the target Prism Central instance",
+					Type: "object",
+					Properties: map[string]clusterv1.JSONSchemaProps{
+						"name": {
+							Description: "The name of the Secret",
+							Type:        "string",
+						},
+					},
+					Required: []string{"name"},
+				},
+			},
 		},
 	}
 }

api/v1alpha1/clusterconfig_types.go

@@ -25,7 +25,8 @@ const (
 	CSIProviderAWSEBS  = "aws-ebs"
 	CSIProviderNutanix = "nutanix"
 
-	CCMProviderAWS = "aws"
+	CCMProviderAWS     = "aws"
+	CCMProviderNutanix = "nutanix"
 )
 
 // +kubebuilder:object:root=true

api/v1alpha1/nutanix_clusterconfig_types.go

@@ -4,6 +4,10 @@
 package v1alpha1
 
 import (
+	"fmt"
+	"net/url"
+	"strconv"
+
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/utils/ptr"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
@@ -99,3 +103,26 @@ func (NutanixPrismCentralEndpointSpec) VariableSchema() clusterv1.VariableSchema
 		},
 	}
 }
+
+//nolint:gocritic // no need for named return values
+func (s NutanixPrismCentralEndpointSpec) ParseURL() (string, int32, error) {
+	var prismCentralURL *url.URL
+	prismCentralURL, err := url.Parse(s.URL)
+	if err != nil {
+		return "", -1, fmt.Errorf("error parsing Prism Central URL: %w", err)
+	}
+
+	hostname := prismCentralURL.Hostname()
+
+	// return early with the default port if no port is specified
+	if prismCentralURL.Port() == "" {
+		return hostname, DefaultPrismCentralPort, nil
+	}
+
+	port, err := strconv.ParseInt(prismCentralURL.Port(), 10, 32)
+	if err != nil {
+		return "", -1, fmt.Errorf("error converting port to int: %w", err)
+	}
+
+	return hostname, int32(port), nil
+}

api/v1alpha1/zz_generated.deepcopy.go

@@ -15,6 +15,7 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
 
+	"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/v1alpha1"
 	"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/k8s/client"
 	lifecycleutils "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/utils"
 	"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/options"
@@ -56,6 +57,7 @@ func New(
 func (a *AWSCCM) Apply(
 	ctx context.Context,
 	cluster *clusterv1.Cluster,
+	_ *v1alpha1.ClusterConfigSpec,
 ) error {
 	log := ctrl.LoggerFrom(ctx).WithValues(
 		"cluster",

pkg/handlers/generic/lifecycle/ccm/aws/handler.go

@@ -25,7 +25,7 @@ const (
 )
 
 type CCMProvider interface {
-	Apply(context.Context, *clusterv1.Cluster) error
+	Apply(context.Context, *clusterv1.Cluster, *v1alpha1.ClusterConfigSpec) error
 }
 
 type CCMHandler struct {
@@ -78,7 +78,7 @@ func (c *CCMHandler) AfterControlPlaneInitialized(
 		)
 		resp.SetStatus(runtimehooksv1.ResponseStatusFailure)
 		resp.SetMessage(
-			fmt.Sprintf("failed to read CCM provider from cluster definition: %v",
+			fmt.Sprintf("failed to read CCM from cluster definition: %v",
 				err,
 			),
 		)
@@ -88,17 +88,39 @@ func (c *CCMHandler) AfterControlPlaneInitialized(
 		log.V(4).Info("Skipping CCM handler.")
 		return
 	}
+
+	clusterConfigVar, _, err := variables.Get[v1alpha1.ClusterConfigSpec](
+		varMap,
+		clusterconfig.MetaVariableName,
+	)
+	if err != nil {
+		log.Error(
+			err,
+			"failed to read clusterConfig variable from cluster definition",
+		)
+		resp.SetStatus(runtimehooksv1.ResponseStatusFailure)
+		resp.SetMessage(
+			fmt.Sprintf("ffailed to read clusterConfig variable from cluster definition: %v",
+				err,
+			),
+		)
+		return
+	}
+
 	infraKind := req.Cluster.Spec.InfrastructureRef.Kind
 	log.Info(fmt.Sprintf("finding CCM handler for %s", infraKind))
 	var handler CCMProvider
 	switch {
 	case strings.Contains(strings.ToLower(infraKind), v1alpha1.CCMProviderAWS):
 		handler = c.ProviderHandler[v1alpha1.CCMProviderAWS]
+	case strings.Contains(strings.ToLower(infraKind), v1alpha1.CCMProviderNutanix):
+		handler = c.ProviderHandler[v1alpha1.CCMProviderNutanix]
 	default:
 		log.Info(fmt.Sprintf("No CCM handler provided for infra kind %s", infraKind))
 		return
 	}
-	err = handler.Apply(ctx, &req.Cluster)
+
+	err = handler.Apply(ctx, &req.Cluster, &clusterConfigVar)
 	if err != nil {
 		log.Error(
 			err,

pkg/handlers/generic/lifecycle/ccm/handler.go

@@ -0,0 +1,201 @@
+// Copyright 2023 D2iQ, Inc. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package nutanix
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"text/template"
+
+	"github.com/spf13/pflag"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	ctrl "sigs.k8s.io/controller-runtime"
+	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+
+	caaphv1 "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/external/sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1"
+	"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/v1alpha1"
+	"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/k8s/client"
+	"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/config"
+	lifecycleutils "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/utils"
+	"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/options"
+)
+
+const (
+	defaultHelmReleaseName      = "nutanix-ccm"
+	defaultHelmReleaseNamespace = "kube-system"
+
+	// This is the name of the Secret on the remote cluster that should match what is defined in Helm values.
+	//nolint:gosec // Does not contain hard coded credentials.
+	defaultCredentialsSecretName = "nutanix-ccm-credentials"
+)
+
+var ErrMissingCredentials = errors.New("name of the Secret containing PC credentials must be set")
+
+type Config struct {
+	*options.GlobalOptions
+
+	defaultValuesTemplateConfigMapName string
+}
+
+func (c *Config) AddFlags(prefix string, flags *pflag.FlagSet) {
+	flags.StringVar(
+		&c.defaultValuesTemplateConfigMapName,
+		prefix+".default-values-template-configmap-name",
+		"default-nutanix-ccm-helm-values-template",
+		"default values ConfigMap name",
+	)
+}
+
+type provider struct {
+	client              ctrlclient.Client
+	config              *Config
+	helmChartInfoGetter *config.HelmChartGetter
+}
+
+func New(
+	c ctrlclient.Client,
+	cfg *Config,
+	helmChartInfoGetter *config.HelmChartGetter,
+) *provider {
+	return &provider{
+		client:              c,
+		config:              cfg,
+		helmChartInfoGetter: helmChartInfoGetter,
+	}
+}
+
+func (p *provider) Apply(
+	ctx context.Context,
+	cluster *clusterv1.Cluster,
+	clusterConfig *v1alpha1.ClusterConfigSpec,
+) error {
+	// No need to check for nil values in the struct, this function will only be called if CCM is not nil
+	if clusterConfig.Addons.CCM.Credentials == nil {
+		return ErrMissingCredentials
+	}
+
+	valuesTemplateConfigMap, err := lifecycleutils.RetrieveValuesTemplateConfigMap(
+		ctx,
+		p.client,
+		p.config.defaultValuesTemplateConfigMapName,
+		p.config.DefaultsNamespace(),
+	)
+	if err != nil {
+		return fmt.Errorf(
+			"failed to retrieve Nutanix CCM installation values template ConfigMap for cluster: %w",
+			err,
+		)
+	}
+
+	// It's possible to have the credentials Secret be created by the Helm chart.
+	// However, that would leave the credentials visible in the HelmChartProxy.
+	// Instead, we'll create the Secret on the remote cluster and reference it in the Helm values.
+	if clusterConfig.Addons.CCM.Credentials != nil {
+		key := ctrlclient.ObjectKey{
+			Name:      defaultCredentialsSecretName,
+			Namespace: defaultHelmReleaseNamespace,
+		}
+		err = lifecycleutils.CopySecretToRemoteCluster(
+			ctx,
+			p.client,
+			clusterConfig.Addons.CCM.Credentials.Name,
+			key,
+			cluster,
+		)
+		if err != nil {
+			return fmt.Errorf(
+				"error creating Nutanix CCM Credentials Secret on the remote cluster: %w",
+				err,
+			)
+		}
+	}
+
+	log := ctrl.LoggerFrom(ctx).WithValues(
+		"cluster",
+		ctrlclient.ObjectKeyFromObject(cluster),
+	)
+	helmChart, err := p.helmChartInfoGetter.For(ctx, log, config.NutanixCCM)
+	if err != nil {
+		return fmt.Errorf("failed to get values for nutanix-ccm-config %w", err)
+	}
+
+	values := valuesTemplateConfigMap.Data["values.yaml"]
+	// The configMap will contain the Helm values, but templated with fields that need to be filled in.
+	values, err = templateValues(clusterConfig, values)
+	if err != nil {
+		return fmt.Errorf("failed to template Helm values read from ConfigMap: %w", err)
+	}
+
+	hcp := &caaphv1.HelmChartProxy{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: caaphv1.GroupVersion.String(),
+			Kind:       "HelmChartProxy",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: cluster.Namespace,
+			Name:      "nutanix-ccm-" + cluster.Name,
+		},
+		Spec: caaphv1.HelmChartProxySpec{
+			RepoURL:   helmChart.Repository,
+			ChartName: helmChart.Name,
+			ClusterSelector: metav1.LabelSelector{
+				MatchLabels: map[string]string{clusterv1.ClusterNameLabel: cluster.Name},
+			},
+			ReleaseNamespace: defaultHelmReleaseNamespace,
+			ReleaseName:      defaultHelmReleaseName,
+			Version:          helmChart.Version,
+			ValuesTemplate:   values,
+		},
+	}
+
+	if err = controllerutil.SetOwnerReference(cluster, hcp, p.client.Scheme()); err != nil {
+		return fmt.Errorf(
+			"failed to set owner reference on nutanix-ccm installation HelmChartProxy: %w",
+			err,
+		)
+	}
+
+	if err = client.ServerSideApply(ctx, p.client, hcp); err != nil {
+		return fmt.Errorf("failed to apply nutanix-ccm installation HelmChartProxy: %w", err)
+	}
+
+	return nil
+}
+
+func templateValues(clusterConfig *v1alpha1.ClusterConfigSpec, text string) (string, error) {
+	helmValuesTemplate, err := template.New("").Parse(text)
+	if err != nil {
+		return "", fmt.Errorf("failed to parse Helm values template: %w", err)
+	}
+
+	type input struct {
+		PrismCentralHost                  string
+		PrismCentralPort                  int32
+		PrismCentralInsecure              bool
+		PrismCentralAdditionalTrustBundle *string
+	}
+
+	address, port, err := clusterConfig.Nutanix.PrismCentralEndpoint.ParseURL()
+	if err != nil {
+		return "", err
+	}
+	templateInput := input{
+		PrismCentralHost:                  address,
+		PrismCentralPort:                  port,
+		PrismCentralInsecure:              clusterConfig.Nutanix.PrismCentralEndpoint.Insecure,
+		PrismCentralAdditionalTrustBundle: clusterConfig.Nutanix.PrismCentralEndpoint.AdditionalTrustBundle,
+	}
+
+	var b bytes.Buffer
+	err = helmValuesTemplate.Execute(&b, templateInput)
+	if err != nil {
+		return "", fmt.Errorf("failed setting PrismCentral configuration in template: %w", err)
+	}
+
+	return b.String(), nil
+}