Create external managed kubernetes cluster on CloudStack

Author: vishesh92

api/v1beta3/cloudstackcluster_types.go

@@ -43,6 +43,10 @@ type CloudStackClusterStatus struct {
 	// +optional
 	FailureDomains clusterv1.FailureDomains `json:"failureDomains,omitempty"`
 
+	// Id of CAPC managed kubernetes cluster created in CloudStack
+	// +optional
+	CloudStackClusterID string `json:"cloudStackClusterId"`
+
 	// Reflects the readiness of the CS cluster.
 	Ready bool `json:"ready"`
 }

config/crd/bases/infrastructure.cluster.x-k8s.io_cloudstackclusters.yaml

@@ -417,6 +417,9 @@ spec:
           status:
             description: The actual cluster state reported by CloudStack.
             properties:
+              cloudStackClusterId:
+                description: Id of CAPC managed kubernetes cluster created in CloudStack
+                type: string
               failureDomains:
                 additionalProperties:
                   description: FailureDomainSpec is the Schema for Cluster API failure

controllers/cloudstackcluster_controller.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 	"reflect"
+	"strings"
 
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
@@ -93,9 +94,25 @@ func (r *CloudStackClusterReconciliationRunner) Reconcile() (res ctrl.Result, re
 		r.GetFailureDomains(r.FailureDomains),
 		r.RemoveExtraneousFailureDomains(r.FailureDomains),
 		r.VerifyFailureDomainCRDs,
+		r.GetOrCreateCluster,
 		r.SetReady)
 }
 
+// GetOrCreateCluster checks if an unmanaged cluster is present in Cloudstack else creates one.
+func (r *CloudStackClusterReconciliationRunner) GetOrCreateCluster() (ctrl.Result, error) {
+	r.AsFailureDomainUser(&r.FailureDomains.Items[0].Spec)()
+	err := r.CSUser.GetOrCreateCluster(r.CAPICluster, r.ReconciliationSubject, &r.FailureDomains.Items[0].Spec)
+	if err != nil {
+		if strings.Contains(err.Error(), "Kubernetes Service plugin is disabled") {
+			r.Log.Info("Kubernetes Service plugin is disabled on CloudStack. Skipping creating unmanaged kubernets cluster")
+			return ctrl.Result{}, nil
+		} else {
+			return r.RequeueWithMessage(fmt.Sprintf("Creating unmanaged kubernetes cluster failed. Error: %s", err.Error()))
+		}
+	}
+	return ctrl.Result{}, nil
+}
+
 // SetReady adds a finalizer and sets the cluster status to ready.
 func (r *CloudStackClusterReconciliationRunner) SetReady() (ctrl.Result, error) {
 	controllerutil.AddFinalizer(r.ReconciliationSubject, infrav1.ClusterFinalizer)
@@ -150,10 +167,27 @@ func (r *CloudStackClusterReconciliationRunner) ReconcileDelete() (ctrl.Result,
 		}
 		return r.RequeueWithMessage("Child FailureDomains still present, requeueing.")
 	}
+	if res, err := r.DeleteCluster(); r.ShouldReturn(res, err) {
+		return res, err
+	}
 	controllerutil.RemoveFinalizer(r.ReconciliationSubject, infrav1.ClusterFinalizer)
 	return ctrl.Result{}, nil
 }
 
+// DeleteCluster checks if an unmanaged cluster is present in Cloudstack and then deletes it.
+func (r *CloudStackClusterReconciliationRunner) DeleteCluster() (ctrl.Result, error) {
+	// If field is present and delete fails, then requeue
+	r.AsFailureDomainUser(&r.CSCluster.Spec.FailureDomains[0])()
+	err := r.CSUser.DeleteCluster(r.ReconciliationSubject)
+	if err != nil {
+		if strings.Contains(err.Error(), " not found") {
+			return ctrl.Result{}, nil
+		}
+		return r.RequeueWithMessage(fmt.Sprintf("Deleting unmanaged kubernetes cluster on CloudStack failed. error: %s", err.Error()))
+	}
+	return ctrl.Result{}, nil
+}
+
 // Called in main, this registers the cluster reconciler to the CAPI controller manager.
 func (reconciler *CloudStackClusterReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, opts controller.Options) error {
 	controller, err := ctrl.NewControllerManagedBy(mgr).

controllers/cloudstackfailuredomain_controller.go

@@ -18,14 +18,15 @@ package controllers
 
 import (
 	"context"
+	"sort"
+
 	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
-	"sort"
 
 	infrav1 "sigs.k8s.io/cluster-api-provider-cloudstack/api/v1beta3"
 	csCtrlrUtils "sigs.k8s.io/cluster-api-provider-cloudstack/controllers/utils"

controllers/cloudstackmachine_controller.go

@@ -127,6 +127,7 @@ func (r *CloudStackMachineReconciliationRunner) Reconcile() (retRes ctrl.Result,
 		r.RequeueIfInstanceNotRunning,
 		r.AddToLBIfNeeded,
 		r.GetOrCreateMachineStateChecker,
+		r.AttachVM,
 	)
 }
 
@@ -213,6 +214,20 @@ func (r *CloudStackMachineReconciliationRunner) DeleteMachineIfFailuredomainNotE
 	return ctrl.Result{}, nil
 }
 
+// AttachVM adds the VM to CloudStack Unmanaged kubernetes.
+// No action taken if it fails
+func (r *CloudStackMachineReconciliationRunner) AttachVM() (retRes ctrl.Result, reterr error) {
+	_ = r.CSUser.AddVMToCluster(r.CSCluster, r.ReconciliationSubject)
+	return ctrl.Result{}, nil
+}
+
+// RemoveVM removes the VM from CloudStack Unmanaged kubernetes.
+// No action taken if it fails
+func (r *CloudStackMachineReconciliationRunner) RemoveVM() (retRes ctrl.Result, reterr error) {
+	_ = r.CSUser.RemoveVMFromCluster(r.CSCluster, r.ReconciliationSubject)
+	return ctrl.Result{}, nil
+}
+
 // GetOrCreateVMInstance gets or creates a VM instance.
 // Implicitly it also fetches its bootstrap secret in order to create said instance.
 func (r *CloudStackMachineReconciliationRunner) GetOrCreateVMInstance() (retRes ctrl.Result, reterr error) {
@@ -340,6 +355,7 @@ func (r *CloudStackMachineReconciliationRunner) ReconcileDelete() (retRes ctrl.R
 		return ctrl.Result{}, err
 	}
 
+	r.RemoveVM()
 	controllerutil.RemoveFinalizer(r.ReconciliationSubject, infrav1.MachineFinalizer)
 	r.Log.Info("VM Deleted", "instanceID", r.ReconciliationSubject.Spec.InstanceID)
 	return ctrl.Result{}, nil

controllers/controllers_suite_test.go

@@ -21,16 +21,17 @@ import (
 	"flag"
 	"fmt"
 	"go/build"
-	"k8s.io/client-go/tools/record"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"regexp"
-	"sigs.k8s.io/cluster-api-provider-cloudstack/test/fakes"
 	"strings"
 	"testing"
 	"time"
 
+	"k8s.io/client-go/tools/record"
+	"sigs.k8s.io/cluster-api-provider-cloudstack/test/fakes"
+
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/klog/v2"
 	"k8s.io/klog/v2/klogr"

go.mod

@@ -105,3 +105,5 @@ replace github.com/dgrijalva/jwt-go => github.com/golang-jwt/jwt/v4 v4.0.0 // In
 replace sigs.k8s.io/cluster-api/test => sigs.k8s.io/cluster-api/test v1.2.12
 
 replace sigs.k8s.io/cluster-api => sigs.k8s.io/cluster-api v1.2.12
+
+replace github.com/apache/cloudstack-go/v2 => github.com/shapeblue/cloudstack-go/v2 support-for-unmanaged-k8s

pkg/cloud/client.go

@@ -37,6 +37,7 @@ import (
 //go:generate ../../hack/tools/bin/mockgen -destination=../mocks/mock_client.go -package=mocks sigs.k8s.io/cluster-api-provider-cloudstack/pkg/cloud Client
 
 type Client interface {
+	ClusterIface
 	VMIface
 	NetworkIface
 	AffinityGroupIface

pkg/cloud/cluster.go

@@ -0,0 +1,145 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cloud
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/apache/cloudstack-go/v2/cloudstack"
+	"github.com/pkg/errors"
+	infrav1 "sigs.k8s.io/cluster-api-provider-cloudstack/api/v1beta3"
+	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+)
+
+type ClusterIface interface {
+	GetOrCreateCluster(*clusterv1.Cluster, *infrav1.CloudStackCluster, *infrav1.CloudStackFailureDomainSpec) error
+	DeleteCluster(*infrav1.CloudStackCluster) error
+	AddVMToCluster(*infrav1.CloudStackCluster, *infrav1.CloudStackMachine) error
+	RemoveVMFromCluster(*infrav1.CloudStackCluster, *infrav1.CloudStackMachine) error
+}
+
+type ClustertypeSetter interface {
+	SetClustertype(string)
+}
+
+func withExternalManaged() cloudstack.OptionFunc {
+	return func(cs *cloudstack.CloudStackClient, p interface{}) error {
+		ps, ok := p.(ClustertypeSetter)
+		if !ok {
+			return errors.New("invalid params type")
+		}
+		ps.SetClustertype("ExternalManaged")
+		return nil
+	}
+}
+
+func (c *client) GetOrCreateCluster(cluster *clusterv1.Cluster, csCluster *infrav1.CloudStackCluster, fd *infrav1.CloudStackFailureDomainSpec) error {
+	// Get cluster
+	if csCluster.Status.CloudStackClusterID != "" {
+		_, count, err := c.cs.Kubernetes.GetKubernetesClusterByID(csCluster.Status.CloudStackClusterID, withExternalManaged())
+		if err != nil {
+			return err
+		}
+		if count == 1 {
+			return nil
+		}
+	}
+
+	// Check if a cluster exists with the same name
+	clusterName := fmt.Sprintf("%s - %s", cluster.GetName(), csCluster.GetName())
+	csUnmanagedCluster, count, err := c.cs.Kubernetes.GetKubernetesClusterByName(clusterName, withExternalManaged())
+	if err != nil && !strings.Contains(err.Error(), "No match found for ") {
+		return err
+	}
+	if count <= 0 {
+		// Create cluster
+		domain := Domain{Path: rootDomain}
+		if csCluster.Spec.FailureDomains[0].Domain != "" {
+			domain.Path = fd.Domain
+		}
+		_ = c.ResolveDomain(&domain)
+
+		accountName := csCluster.Spec.FailureDomains[0].Account
+		if accountName == "" {
+			userParams := c.cs.User.NewGetUserParams(c.config.APIKey)
+			user, err := c.cs.User.GetUser(userParams)
+			if err != nil {
+				return err
+			}
+			accountName = user.Account
+		}
+		params := c.cs.Kubernetes.NewCreateKubernetesClusterParams(fmt.Sprintf("%s managed by CAPC", clusterName), clusterName, fd.Zone.ID)
+
+		setIfNotEmpty(accountName, params.SetAccount)
+		setIfNotEmpty(domain.ID, params.SetDomainid)
+		setIfNotEmpty(fd.Zone.Network.ID, params.SetNetworkid)
+		setIfNotEmpty(csCluster.Spec.ControlPlaneEndpoint.Host, params.SetExternalloadbalanceripaddress)
+		params.SetClustertype("ExternalManaged")
+
+		r, err := c.cs.Kubernetes.CreateKubernetesCluster(params)
+		if err != nil {
+			return err
+		}
+		csUnmanagedCluster, count, err = c.cs.Kubernetes.GetKubernetesClusterByID(r.Id)
+		if err != nil {
+			return err
+		}
+		if count == 0 {
+			return errors.New("cluster not found")
+		}
+	}
+	csCluster.Status.CloudStackClusterID = csUnmanagedCluster.Id
+	return nil
+}
+
+func (c *client) DeleteCluster(csCluster *infrav1.CloudStackCluster) error {
+	if csCluster.Status.CloudStackClusterID != "" {
+		csUnmanagedCluster, count, err := c.cs.Kubernetes.GetKubernetesClusterByID(csCluster.Status.CloudStackClusterID, withExternalManaged())
+		if err != nil {
+			return err
+		}
+		if count != 0 {
+			params := c.cs.Kubernetes.NewDeleteKubernetesClusterParams(csUnmanagedCluster.Id)
+			_, err = c.cs.Kubernetes.DeleteKubernetesCluster(params)
+			if err != nil {
+				return err
+			}
+		}
+		csCluster.Status.CloudStackClusterID = ""
+		return nil
+	}
+	return nil
+}
+
+func (c *client) AddVMToCluster(csCluster *infrav1.CloudStackCluster, csMachine *infrav1.CloudStackMachine) error {
+	if csCluster.Status.CloudStackClusterID != "" {
+		params := c.cs.Kubernetes.NewAddVirtualMachinesToKubernetesClusterParams(csCluster.Status.CloudStackClusterID, []string{*csMachine.Spec.InstanceID})
+		_, err := c.cs.Kubernetes.AddVirtualMachinesToKubernetesCluster(params)
+		return err
+	}
+	return nil
+}
+
+func (c *client) RemoveVMFromCluster(csCluster *infrav1.CloudStackCluster, csMachine *infrav1.CloudStackMachine) error {
+	if csCluster.Status.CloudStackClusterID != "" {
+		params := c.cs.Kubernetes.NewRemoveVirtualMachinesFromKubernetesClusterParams(csCluster.Status.CloudStackClusterID, []string{*csMachine.Spec.InstanceID})
+		_, err := c.cs.Kubernetes.RemoveVirtualMachinesFromKubernetesCluster(params)
+		return err
+	}
+	return nil
+}

pkg/cloud/isolated_network.go

@@ -100,7 +100,7 @@ func (c *client) CreateIsolatedNetwork(fd *infrav1.CloudStackFailureDomain, isoN
 	}
 
 	// Do isolated network creation.
-	p := c.cs.Network.NewCreateNetworkParams(isoNet.Spec.Name, isoNet.Spec.Name, offeringID, fd.Spec.Zone.ID)
+	p := c.cs.Network.NewCreateNetworkParams(isoNet.Spec.Name, offeringID, fd.Spec.Zone.ID)
 	resp, err := c.cs.Network.CreateNetwork(p)
 	if err != nil {
 		c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err)

pkg/cloud/isolated_network_test.go

@@ -72,7 +72,7 @@ var _ = Describe("Network", func() {
 			dummies.Zone1.Network.ID = ""
 
 			nos.EXPECT().GetNetworkOfferingID(gomock.Any()).Return("someOfferingID", 1, nil)
-			ns.EXPECT().NewCreateNetworkParams(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
+			ns.EXPECT().NewCreateNetworkParams(gomock.Any(), gomock.Any(), gomock.Any()).
 				Return(&csapi.CreateNetworkParams{})
 			ns.EXPECT().GetNetworkByName(dummies.ISONet1.Name).Return(nil, 0, nil)
 			ns.EXPECT().GetNetworkByID(dummies.ISONet1.ID).Return(nil, 0, nil)