From 6264c5321cb218d09872774cb968a338cda42d2a Mon Sep 17 00:00:00 2001 From: Vishesh Date: Thu, 27 Jul 2023 17:52:08 +0530 Subject: [PATCH 1/6] Create external managed kubernetes cluster on CloudStack --- api/v1beta3/cloudstackcluster_types.go | 4 + ...e.cluster.x-k8s.io_cloudstackclusters.yaml | 3 + controllers/cloudstackcluster_controller.go | 34 ++++ .../cloudstackfailuredomain_controller.go | 3 +- controllers/cloudstackmachine_controller.go | 16 ++ controllers/controllers_suite_test.go | 5 +- go.mod | 2 + pkg/cloud/client.go | 1 + pkg/cloud/cluster.go | 145 ++++++++++++++++++ pkg/cloud/isolated_network.go | 2 +- pkg/cloud/isolated_network_test.go | 2 +- 11 files changed, 212 insertions(+), 5 deletions(-) create mode 100644 pkg/cloud/cluster.go diff --git a/api/v1beta3/cloudstackcluster_types.go b/api/v1beta3/cloudstackcluster_types.go index 1b47ff89..9ffafbe0 100644 --- a/api/v1beta3/cloudstackcluster_types.go +++ b/api/v1beta3/cloudstackcluster_types.go @@ -43,6 +43,10 @@ type CloudStackClusterStatus struct { // +optional FailureDomains clusterv1.FailureDomains `json:"failureDomains,omitempty"` + // Id of CAPC managed kubernetes cluster created in CloudStack + // +optional + CloudStackClusterID string `json:"cloudStackClusterId"` + // Reflects the readiness of the CS cluster. Ready bool `json:"ready"` } diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudstackclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudstackclusters.yaml index a9a15239..7b8e7804 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudstackclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudstackclusters.yaml @@ -417,6 +417,9 @@ spec: status: description: The actual cluster state reported by CloudStack. properties: + cloudStackClusterId: + description: Id of CAPC managed kubernetes cluster created in CloudStack + type: string failureDomains: additionalProperties: description: FailureDomainSpec is the Schema for Cluster API failure diff --git a/controllers/cloudstackcluster_controller.go b/controllers/cloudstackcluster_controller.go index 6874ee77..57f6dcde 100644 --- a/controllers/cloudstackcluster_controller.go +++ b/controllers/cloudstackcluster_controller.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "reflect" + "strings" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/controller" @@ -93,9 +94,25 @@ func (r *CloudStackClusterReconciliationRunner) Reconcile() (res ctrl.Result, re r.GetFailureDomains(r.FailureDomains), r.RemoveExtraneousFailureDomains(r.FailureDomains), r.VerifyFailureDomainCRDs, + r.GetOrCreateCluster, r.SetReady) } +// GetOrCreateCluster checks if an unmanaged cluster is present in Cloudstack else creates one. +func (r *CloudStackClusterReconciliationRunner) GetOrCreateCluster() (ctrl.Result, error) { + r.AsFailureDomainUser(&r.FailureDomains.Items[0].Spec)() + err := r.CSUser.GetOrCreateCluster(r.CAPICluster, r.ReconciliationSubject, &r.FailureDomains.Items[0].Spec) + if err != nil { + if strings.Contains(err.Error(), "Kubernetes Service plugin is disabled") { + r.Log.Info("Kubernetes Service plugin is disabled on CloudStack. Skipping creating unmanaged kubernets cluster") + return ctrl.Result{}, nil + } else { + return r.RequeueWithMessage(fmt.Sprintf("Creating unmanaged kubernetes cluster failed. Error: %s", err.Error())) + } + } + return ctrl.Result{}, nil +} + // SetReady adds a finalizer and sets the cluster status to ready. func (r *CloudStackClusterReconciliationRunner) SetReady() (ctrl.Result, error) { controllerutil.AddFinalizer(r.ReconciliationSubject, infrav1.ClusterFinalizer) @@ -150,10 +167,27 @@ func (r *CloudStackClusterReconciliationRunner) ReconcileDelete() (ctrl.Result, } return r.RequeueWithMessage("Child FailureDomains still present, requeueing.") } + if res, err := r.DeleteCluster(); r.ShouldReturn(res, err) { + return res, err + } controllerutil.RemoveFinalizer(r.ReconciliationSubject, infrav1.ClusterFinalizer) return ctrl.Result{}, nil } +// DeleteCluster checks if an unmanaged cluster is present in Cloudstack and then deletes it. +func (r *CloudStackClusterReconciliationRunner) DeleteCluster() (ctrl.Result, error) { + // If field is present and delete fails, then requeue + r.AsFailureDomainUser(&r.CSCluster.Spec.FailureDomains[0])() + err := r.CSUser.DeleteCluster(r.ReconciliationSubject) + if err != nil { + if strings.Contains(err.Error(), " not found") { + return ctrl.Result{}, nil + } + return r.RequeueWithMessage(fmt.Sprintf("Deleting unmanaged kubernetes cluster on CloudStack failed. error: %s", err.Error())) + } + return ctrl.Result{}, nil +} + // Called in main, this registers the cluster reconciler to the CAPI controller manager. func (reconciler *CloudStackClusterReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, opts controller.Options) error { controller, err := ctrl.NewControllerManagedBy(mgr). diff --git a/controllers/cloudstackfailuredomain_controller.go b/controllers/cloudstackfailuredomain_controller.go index 9995796b..380d9223 100644 --- a/controllers/cloudstackfailuredomain_controller.go +++ b/controllers/cloudstackfailuredomain_controller.go @@ -18,6 +18,8 @@ package controllers import ( "context" + "sort" + "github.com/pkg/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" @@ -25,7 +27,6 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sort" infrav1 "sigs.k8s.io/cluster-api-provider-cloudstack/api/v1beta3" csCtrlrUtils "sigs.k8s.io/cluster-api-provider-cloudstack/controllers/utils" diff --git a/controllers/cloudstackmachine_controller.go b/controllers/cloudstackmachine_controller.go index cfcb7ac0..e9dc33a2 100644 --- a/controllers/cloudstackmachine_controller.go +++ b/controllers/cloudstackmachine_controller.go @@ -127,6 +127,7 @@ func (r *CloudStackMachineReconciliationRunner) Reconcile() (retRes ctrl.Result, r.RequeueIfInstanceNotRunning, r.AddToLBIfNeeded, r.GetOrCreateMachineStateChecker, + r.AttachVM, ) } @@ -213,6 +214,20 @@ func (r *CloudStackMachineReconciliationRunner) DeleteMachineIfFailuredomainNotE return ctrl.Result{}, nil } +// AttachVM adds the VM to CloudStack Unmanaged kubernetes. +// No action taken if it fails +func (r *CloudStackMachineReconciliationRunner) AttachVM() (retRes ctrl.Result, reterr error) { + _ = r.CSUser.AddVMToCluster(r.CSCluster, r.ReconciliationSubject) + return ctrl.Result{}, nil +} + +// RemoveVM removes the VM from CloudStack Unmanaged kubernetes. +// No action taken if it fails +func (r *CloudStackMachineReconciliationRunner) RemoveVM() (retRes ctrl.Result, reterr error) { + _ = r.CSUser.RemoveVMFromCluster(r.CSCluster, r.ReconciliationSubject) + return ctrl.Result{}, nil +} + // GetOrCreateVMInstance gets or creates a VM instance. // Implicitly it also fetches its bootstrap secret in order to create said instance. func (r *CloudStackMachineReconciliationRunner) GetOrCreateVMInstance() (retRes ctrl.Result, reterr error) { @@ -340,6 +355,7 @@ func (r *CloudStackMachineReconciliationRunner) ReconcileDelete() (retRes ctrl.R return ctrl.Result{}, err } + r.RemoveVM() controllerutil.RemoveFinalizer(r.ReconciliationSubject, infrav1.MachineFinalizer) r.Log.Info("VM Deleted", "instanceID", r.ReconciliationSubject.Spec.InstanceID) return ctrl.Result{}, nil diff --git a/controllers/controllers_suite_test.go b/controllers/controllers_suite_test.go index a9776f64..05c9a88d 100644 --- a/controllers/controllers_suite_test.go +++ b/controllers/controllers_suite_test.go @@ -21,16 +21,17 @@ import ( "flag" "fmt" "go/build" - "k8s.io/client-go/tools/record" "os" "os/exec" "path/filepath" "regexp" - "sigs.k8s.io/cluster-api-provider-cloudstack/test/fakes" "strings" "testing" "time" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/cluster-api-provider-cloudstack/test/fakes" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog/v2" "k8s.io/klog/v2/klogr" diff --git a/go.mod b/go.mod index 075d13b1..485f9ed0 100644 --- a/go.mod +++ b/go.mod @@ -105,3 +105,5 @@ replace github.com/dgrijalva/jwt-go => github.com/golang-jwt/jwt/v4 v4.0.0 // In replace sigs.k8s.io/cluster-api/test => sigs.k8s.io/cluster-api/test v1.2.12 replace sigs.k8s.io/cluster-api => sigs.k8s.io/cluster-api v1.2.12 + +replace github.com/apache/cloudstack-go/v2 => github.com/shapeblue/cloudstack-go/v2 support-for-unmanaged-k8s \ No newline at end of file diff --git a/pkg/cloud/client.go b/pkg/cloud/client.go index fa72b5a5..7ce6dca2 100644 --- a/pkg/cloud/client.go +++ b/pkg/cloud/client.go @@ -37,6 +37,7 @@ import ( //go:generate ../../hack/tools/bin/mockgen -destination=../mocks/mock_client.go -package=mocks sigs.k8s.io/cluster-api-provider-cloudstack/pkg/cloud Client type Client interface { + ClusterIface VMIface NetworkIface AffinityGroupIface diff --git a/pkg/cloud/cluster.go b/pkg/cloud/cluster.go new file mode 100644 index 00000000..b5e1d2d6 --- /dev/null +++ b/pkg/cloud/cluster.go @@ -0,0 +1,145 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloud + +import ( + "fmt" + "strings" + + "github.com/apache/cloudstack-go/v2/cloudstack" + "github.com/pkg/errors" + infrav1 "sigs.k8s.io/cluster-api-provider-cloudstack/api/v1beta3" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" +) + +type ClusterIface interface { + GetOrCreateCluster(*clusterv1.Cluster, *infrav1.CloudStackCluster, *infrav1.CloudStackFailureDomainSpec) error + DeleteCluster(*infrav1.CloudStackCluster) error + AddVMToCluster(*infrav1.CloudStackCluster, *infrav1.CloudStackMachine) error + RemoveVMFromCluster(*infrav1.CloudStackCluster, *infrav1.CloudStackMachine) error +} + +type ClustertypeSetter interface { + SetClustertype(string) +} + +func withExternalManaged() cloudstack.OptionFunc { + return func(cs *cloudstack.CloudStackClient, p interface{}) error { + ps, ok := p.(ClustertypeSetter) + if !ok { + return errors.New("invalid params type") + } + ps.SetClustertype("ExternalManaged") + return nil + } +} + +func (c *client) GetOrCreateCluster(cluster *clusterv1.Cluster, csCluster *infrav1.CloudStackCluster, fd *infrav1.CloudStackFailureDomainSpec) error { + // Get cluster + if csCluster.Status.CloudStackClusterID != "" { + _, count, err := c.cs.Kubernetes.GetKubernetesClusterByID(csCluster.Status.CloudStackClusterID, withExternalManaged()) + if err != nil { + return err + } + if count == 1 { + return nil + } + } + + // Check if a cluster exists with the same name + clusterName := fmt.Sprintf("%s - %s", cluster.GetName(), csCluster.GetName()) + csUnmanagedCluster, count, err := c.cs.Kubernetes.GetKubernetesClusterByName(clusterName, withExternalManaged()) + if err != nil && !strings.Contains(err.Error(), "No match found for ") { + return err + } + if count <= 0 { + // Create cluster + domain := Domain{Path: rootDomain} + if csCluster.Spec.FailureDomains[0].Domain != "" { + domain.Path = fd.Domain + } + _ = c.ResolveDomain(&domain) + + accountName := csCluster.Spec.FailureDomains[0].Account + if accountName == "" { + userParams := c.cs.User.NewGetUserParams(c.config.APIKey) + user, err := c.cs.User.GetUser(userParams) + if err != nil { + return err + } + accountName = user.Account + } + params := c.cs.Kubernetes.NewCreateKubernetesClusterParams(fmt.Sprintf("%s managed by CAPC", clusterName), clusterName, fd.Zone.ID) + + setIfNotEmpty(accountName, params.SetAccount) + setIfNotEmpty(domain.ID, params.SetDomainid) + setIfNotEmpty(fd.Zone.Network.ID, params.SetNetworkid) + setIfNotEmpty(csCluster.Spec.ControlPlaneEndpoint.Host, params.SetExternalloadbalanceripaddress) + params.SetClustertype("ExternalManaged") + + r, err := c.cs.Kubernetes.CreateKubernetesCluster(params) + if err != nil { + return err + } + csUnmanagedCluster, count, err = c.cs.Kubernetes.GetKubernetesClusterByID(r.Id) + if err != nil { + return err + } + if count == 0 { + return errors.New("cluster not found") + } + } + csCluster.Status.CloudStackClusterID = csUnmanagedCluster.Id + return nil +} + +func (c *client) DeleteCluster(csCluster *infrav1.CloudStackCluster) error { + if csCluster.Status.CloudStackClusterID != "" { + csUnmanagedCluster, count, err := c.cs.Kubernetes.GetKubernetesClusterByID(csCluster.Status.CloudStackClusterID, withExternalManaged()) + if err != nil { + return err + } + if count != 0 { + params := c.cs.Kubernetes.NewDeleteKubernetesClusterParams(csUnmanagedCluster.Id) + _, err = c.cs.Kubernetes.DeleteKubernetesCluster(params) + if err != nil { + return err + } + } + csCluster.Status.CloudStackClusterID = "" + return nil + } + return nil +} + +func (c *client) AddVMToCluster(csCluster *infrav1.CloudStackCluster, csMachine *infrav1.CloudStackMachine) error { + if csCluster.Status.CloudStackClusterID != "" { + params := c.cs.Kubernetes.NewAddVirtualMachinesToKubernetesClusterParams(csCluster.Status.CloudStackClusterID, []string{*csMachine.Spec.InstanceID}) + _, err := c.cs.Kubernetes.AddVirtualMachinesToKubernetesCluster(params) + return err + } + return nil +} + +func (c *client) RemoveVMFromCluster(csCluster *infrav1.CloudStackCluster, csMachine *infrav1.CloudStackMachine) error { + if csCluster.Status.CloudStackClusterID != "" { + params := c.cs.Kubernetes.NewRemoveVirtualMachinesFromKubernetesClusterParams(csCluster.Status.CloudStackClusterID, []string{*csMachine.Spec.InstanceID}) + _, err := c.cs.Kubernetes.RemoveVirtualMachinesFromKubernetesCluster(params) + return err + } + return nil +} diff --git a/pkg/cloud/isolated_network.go b/pkg/cloud/isolated_network.go index d6ace207..f440d70d 100644 --- a/pkg/cloud/isolated_network.go +++ b/pkg/cloud/isolated_network.go @@ -100,7 +100,7 @@ func (c *client) CreateIsolatedNetwork(fd *infrav1.CloudStackFailureDomain, isoN } // Do isolated network creation. - p := c.cs.Network.NewCreateNetworkParams(isoNet.Spec.Name, isoNet.Spec.Name, offeringID, fd.Spec.Zone.ID) + p := c.cs.Network.NewCreateNetworkParams(isoNet.Spec.Name, offeringID, fd.Spec.Zone.ID) resp, err := c.cs.Network.CreateNetwork(p) if err != nil { c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err) diff --git a/pkg/cloud/isolated_network_test.go b/pkg/cloud/isolated_network_test.go index e4c998e8..358c0768 100644 --- a/pkg/cloud/isolated_network_test.go +++ b/pkg/cloud/isolated_network_test.go @@ -72,7 +72,7 @@ var _ = Describe("Network", func() { dummies.Zone1.Network.ID = "" nos.EXPECT().GetNetworkOfferingID(gomock.Any()).Return("someOfferingID", 1, nil) - ns.EXPECT().NewCreateNetworkParams(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + ns.EXPECT().NewCreateNetworkParams(gomock.Any(), gomock.Any(), gomock.Any()). Return(&csapi.CreateNetworkParams{}) ns.EXPECT().GetNetworkByName(dummies.ISONet1.Name).Return(nil, 0, nil) ns.EXPECT().GetNetworkByID(dummies.ISONet1.ID).Return(nil, 0, nil) From 0e3abb64f27293b007ee235f2e81c122c8b08cce Mon Sep 17 00:00:00 2001 From: Vishesh Date: Fri, 28 Jul 2023 13:22:25 +0530 Subject: [PATCH 2/6] fix tests --- controllers/cloudstackcluster_controller.go | 17 ++++++++++------ controllers/cloudstackmachine_controller.go | 5 ++++- .../cloudstackmachine_controller_test.go | 15 +++++++++++++- go.mod | 11 ++++++---- go.sum | 20 ++++++++++++------- 5 files changed, 49 insertions(+), 19 deletions(-) diff --git a/controllers/cloudstackcluster_controller.go b/controllers/cloudstackcluster_controller.go index 57f6dcde..7d1468cc 100644 --- a/controllers/cloudstackcluster_controller.go +++ b/controllers/cloudstackcluster_controller.go @@ -100,15 +100,17 @@ func (r *CloudStackClusterReconciliationRunner) Reconcile() (res ctrl.Result, re // GetOrCreateCluster checks if an unmanaged cluster is present in Cloudstack else creates one. func (r *CloudStackClusterReconciliationRunner) GetOrCreateCluster() (ctrl.Result, error) { - r.AsFailureDomainUser(&r.FailureDomains.Items[0].Spec)() - err := r.CSUser.GetOrCreateCluster(r.CAPICluster, r.ReconciliationSubject, &r.FailureDomains.Items[0].Spec) + res, err := r.AsFailureDomainUser(&r.CSCluster.Spec.FailureDomains[0])() + if r.ShouldReturn(res, err) { + return res, err + } + err = r.CSUser.GetOrCreateCluster(r.CAPICluster, r.ReconciliationSubject, &r.FailureDomains.Items[0].Spec) if err != nil { if strings.Contains(err.Error(), "Kubernetes Service plugin is disabled") { r.Log.Info("Kubernetes Service plugin is disabled on CloudStack. Skipping creating unmanaged kubernets cluster") return ctrl.Result{}, nil - } else { - return r.RequeueWithMessage(fmt.Sprintf("Creating unmanaged kubernetes cluster failed. Error: %s", err.Error())) } + return r.RequeueWithMessage(fmt.Sprintf("Creating unmanaged kubernetes cluster failed. Error: %s", err.Error())) } return ctrl.Result{}, nil } @@ -177,8 +179,11 @@ func (r *CloudStackClusterReconciliationRunner) ReconcileDelete() (ctrl.Result, // DeleteCluster checks if an unmanaged cluster is present in Cloudstack and then deletes it. func (r *CloudStackClusterReconciliationRunner) DeleteCluster() (ctrl.Result, error) { // If field is present and delete fails, then requeue - r.AsFailureDomainUser(&r.CSCluster.Spec.FailureDomains[0])() - err := r.CSUser.DeleteCluster(r.ReconciliationSubject) + res, err := r.AsFailureDomainUser(&r.CSCluster.Spec.FailureDomains[0])() + if r.ShouldReturn(res, err) { + return res, err + } + err = r.CSUser.DeleteCluster(r.ReconciliationSubject) if err != nil { if strings.Contains(err.Error(), " not found") { return ctrl.Result{}, nil diff --git a/controllers/cloudstackmachine_controller.go b/controllers/cloudstackmachine_controller.go index e9dc33a2..5541ffdf 100644 --- a/controllers/cloudstackmachine_controller.go +++ b/controllers/cloudstackmachine_controller.go @@ -355,7 +355,10 @@ func (r *CloudStackMachineReconciliationRunner) ReconcileDelete() (retRes ctrl.R return ctrl.Result{}, err } - r.RemoveVM() + res, err := r.RemoveVM() + if r.ShouldReturn(res, err) { + return res, err + } controllerutil.RemoveFinalizer(r.ReconciliationSubject, infrav1.MachineFinalizer) r.Log.Info("VM Deleted", "instanceID", r.ReconciliationSubject.Spec.InstanceID) return ctrl.Result{}, nil diff --git a/controllers/cloudstackmachine_controller_test.go b/controllers/cloudstackmachine_controller_test.go index 6d44b7de..b3cb44dc 100644 --- a/controllers/cloudstackmachine_controller_test.go +++ b/controllers/cloudstackmachine_controller_test.go @@ -44,7 +44,7 @@ var _ = Describe("CloudStackMachineReconciler", func() { dummies.CSCluster.Spec.FailureDomains = dummies.CSCluster.Spec.FailureDomains[:1] dummies.CSCluster.Spec.FailureDomains[0].Name = dummies.CSFailureDomain1.Spec.Name - SetupTestEnvironment() // Must happen before setting up managers/reconcilers. + SetupTestEnvironment() // Must happen before setting up managers/reconcilers. Ω(MachineReconciler.SetupWithManager(k8sManager, controller.Options{})).Should(Succeed()) // Register the CloudStack MachineReconciler. // Point CAPI machine Bootstrap secret ref to dummy bootstrap secret. @@ -54,6 +54,17 @@ var _ = Describe("CloudStackMachineReconciler", func() { // Setup a failure domain for the machine reconciler to find. Ω(k8sClient.Create(ctx, dummies.CSFailureDomain1)).Should(Succeed()) setClusterReady(k8sClient) + + mockCloudClient.EXPECT().GetOrCreateCluster(gomock.Any(), gomock.Any(), gomock.Any()).Do( + func(arg1, _, _ interface{}) { + arg1.(*infrav1.CloudStackCluster).Status.CloudStackClusterID = "cluster-id-123" + }).AnyTimes().Return(nil) + + mockCloudClient.EXPECT().AddVMToCluster( + gomock.Any(), gomock.Any()).AnyTimes().Return(nil) + + mockCloudClient.EXPECT().RemoveVMFromCluster( + gomock.Any(), gomock.Any()).AnyTimes().Return(nil) }) It("Should call GetOrCreateVMInstance and set Status.Ready to true", func() { @@ -240,6 +251,8 @@ var _ = Describe("CloudStackMachineReconciler", func() { func(arg1, _, _, _, _, _ interface{}) { arg1.(*infrav1.CloudStackMachine).Status.InstanceState = "Running" }).AnyTimes() + mockCloudClient.EXPECT().AddVMToCluster( + gomock.Any(), gomock.Any()).AnyTimes().Return(nil) Ω(fakeCtrlClient.Get(ctx, key, dummies.CSCluster)).Should(Succeed()) Ω(fakeCtrlClient.Create(ctx, dummies.CAPIMachine)).Should(Succeed()) Ω(fakeCtrlClient.Create(ctx, dummies.CSMachine1)).Should(Succeed()) diff --git a/go.mod b/go.mod index 485f9ed0..fe3a0825 100644 --- a/go.mod +++ b/go.mod @@ -8,8 +8,8 @@ require ( github.com/go-logr/logr v1.2.3 github.com/golang/mock v1.6.0 github.com/hashicorp/go-multierror v1.1.1 - github.com/onsi/ginkgo/v2 v2.4.0 - github.com/onsi/gomega v1.24.0 + github.com/onsi/ginkgo/v2 v2.9.1 + github.com/onsi/gomega v1.27.4 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.14.0 github.com/smallfish/simpleyaml v0.1.0 @@ -49,14 +49,16 @@ require ( github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.20.0 // indirect github.com/go-openapi/swag v0.22.3 // indirect + github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect github.com/gobuffalo/flect v0.3.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.2.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/google/gnostic v0.6.9 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect + github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect github.com/google/uuid v1.2.0 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/imdario/mergo v0.3.13 // indirect @@ -84,6 +86,7 @@ require ( golang.org/x/sys v0.7.0 // indirect golang.org/x/term v0.7.0 // indirect golang.org/x/time v0.2.0 // indirect + golang.org/x/tools v0.7.0 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20221107162902-2d387536bcdd // indirect @@ -106,4 +109,4 @@ replace sigs.k8s.io/cluster-api/test => sigs.k8s.io/cluster-api/test v1.2.12 replace sigs.k8s.io/cluster-api => sigs.k8s.io/cluster-api v1.2.12 -replace github.com/apache/cloudstack-go/v2 => github.com/shapeblue/cloudstack-go/v2 support-for-unmanaged-k8s \ No newline at end of file +replace github.com/apache/cloudstack-go/v2 => github.com/shapeblue/cloudstack-go/v2 v2.9.1-0.20230717062313-73e4efc8a510 diff --git a/go.sum b/go.sum index e54396df..ef9f8cb0 100644 --- a/go.sum +++ b/go.sum @@ -89,8 +89,6 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 h1:yL7+Jz0jTC6yykIK/Wh74gnTJnrGr5AyrNMXuA0gves= github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= -github.com/apache/cloudstack-go/v2 v2.13.0 h1:t0uj7QxQpnzD/LSTP6a4w2NTuZXisxIM/mIDNkF44lc= -github.com/apache/cloudstack-go/v2 v2.13.0/go.mod h1:aosD8Svfu5nhH5Sp4zcsVV1hT5UGt3mTgRXM8YqTKe0= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= @@ -192,6 +190,8 @@ github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/gobuffalo/flect v0.3.0 h1:erfPWM+K1rFNIQeRPdeEXxo8yFr/PO17lhRnS8FUrtk= github.com/gobuffalo/flect v0.3.0/go.mod h1:5pf3aGnsvqvCj50AVni7mJJF8ICxGZ8HomberC3pXLE= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -234,8 +234,9 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -278,6 +279,7 @@ github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -394,10 +396,10 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRW github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/ginkgo/v2 v2.4.0 h1:+Ig9nvqgS5OBSACXNk15PLdp0U9XPYROt9CFzVdFGIs= -github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= -github.com/onsi/gomega v1.24.0 h1:+0glovB9Jd6z3VR+ScSwQqXVTIfJcGA9UBM8yzQxhqg= -github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= +github.com/onsi/ginkgo/v2 v2.9.1 h1:zie5Ly042PD3bsCvsSOPvRnFwyo3rKe64TJlD6nu0mk= +github.com/onsi/ginkgo/v2 v2.9.1/go.mod h1:FEcmzVcCHl+4o9bQZVab+4dC9+j+91t2FHSzmGAPfuo= +github.com/onsi/gomega v1.27.4 h1:Z2AnStgsdSayCMDiCU42qIz+HLqEPcgiOCXjAU/w+8E= +github.com/onsi/gomega v1.27.4/go.mod h1:riYq/GJKh8hhoM01HN6Vmuy93AarCXCBGpvFDK3q3fQ= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= @@ -448,6 +450,8 @@ github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shapeblue/cloudstack-go/v2 v2.9.1-0.20230717062313-73e4efc8a510 h1:FPRBv784robz6sZSqDGfZDZMse31lj96i+enH02Xzds= +github.com/shapeblue/cloudstack-go/v2 v2.9.1-0.20230717062313-73e4efc8a510/go.mod h1:Mc+tXpujtslBuZFk5atoGT2LanVxOrXS2GGgidAoz1A= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= @@ -804,6 +808,8 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= +golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From 038fe6451d8424e74f6eb8d1f05e24deec6532e3 Mon Sep 17 00:00:00 2001 From: Vishesh Date: Mon, 31 Jul 2023 20:03:07 +0530 Subject: [PATCH 3/6] fix support for older cs versions --- controllers/cloudstackcluster_controller.go | 5 ++-- pkg/cloud/cluster.go | 30 +++++++++------------ pkg/cloud/isolated_network.go | 1 + 3 files changed, 16 insertions(+), 20 deletions(-) diff --git a/controllers/cloudstackcluster_controller.go b/controllers/cloudstackcluster_controller.go index 7d1468cc..8ec3883a 100644 --- a/controllers/cloudstackcluster_controller.go +++ b/controllers/cloudstackcluster_controller.go @@ -107,10 +107,11 @@ func (r *CloudStackClusterReconciliationRunner) GetOrCreateCluster() (ctrl.Resul err = r.CSUser.GetOrCreateCluster(r.CAPICluster, r.ReconciliationSubject, &r.FailureDomains.Items[0].Spec) if err != nil { if strings.Contains(err.Error(), "Kubernetes Service plugin is disabled") { - r.Log.Info("Kubernetes Service plugin is disabled on CloudStack. Skipping creating unmanaged kubernets cluster") + r.Log.Info("Kubernetes Service plugin is disabled on CloudStack. Skipping ExternalManaged kubernetes cluster creation") return ctrl.Result{}, nil } - return r.RequeueWithMessage(fmt.Sprintf("Creating unmanaged kubernetes cluster failed. Error: %s", err.Error())) + // Not requeing the failure to support CloudStack v4.18 and before + r.Log.Info(fmt.Sprintf("Failed creating ExternalManaged kubernetes cluster on CloudStack. Error: %s", err.Error())) } return ctrl.Result{}, nil } diff --git a/pkg/cloud/cluster.go b/pkg/cloud/cluster.go index b5e1d2d6..7313445f 100644 --- a/pkg/cloud/cluster.go +++ b/pkg/cloud/cluster.go @@ -51,22 +51,24 @@ func withExternalManaged() cloudstack.OptionFunc { func (c *client) GetOrCreateCluster(cluster *clusterv1.Cluster, csCluster *infrav1.CloudStackCluster, fd *infrav1.CloudStackFailureDomainSpec) error { // Get cluster if csCluster.Status.CloudStackClusterID != "" { - _, count, err := c.cs.Kubernetes.GetKubernetesClusterByID(csCluster.Status.CloudStackClusterID, withExternalManaged()) + externalManagedCluster, count, err := c.cs.Kubernetes.GetKubernetesClusterByID(csCluster.Status.CloudStackClusterID, withExternalManaged()) if err != nil { return err - } - if count == 1 { + } else if count > 0 { + csCluster.Status.CloudStackClusterID = externalManagedCluster.Id return nil } } // Check if a cluster exists with the same name - clusterName := fmt.Sprintf("%s - %s", cluster.GetName(), csCluster.GetName()) - csUnmanagedCluster, count, err := c.cs.Kubernetes.GetKubernetesClusterByName(clusterName, withExternalManaged()) + clusterName := fmt.Sprintf("%s - %s - %s", cluster.GetName(), csCluster.GetName(), csCluster.GetUID()) + externalManagedCluster, count, err := c.cs.Kubernetes.GetKubernetesClusterByName(clusterName, withExternalManaged()) if err != nil && !strings.Contains(err.Error(), "No match found for ") { return err } - if count <= 0 { + if count > 0 { + csCluster.Status.CloudStackClusterID = externalManagedCluster.Id + } else if err == nil || (err != nil && strings.Contains(err.Error(), "No match found for ")) { // Create cluster domain := Domain{Path: rootDomain} if csCluster.Spec.FailureDomains[0].Domain != "" { @@ -91,27 +93,20 @@ func (c *client) GetOrCreateCluster(cluster *clusterv1.Cluster, csCluster *infra setIfNotEmpty(csCluster.Spec.ControlPlaneEndpoint.Host, params.SetExternalloadbalanceripaddress) params.SetClustertype("ExternalManaged") - r, err := c.cs.Kubernetes.CreateKubernetesCluster(params) + cloudStackCKSCluster, err := c.cs.Kubernetes.CreateKubernetesCluster(params) if err != nil { return err } - csUnmanagedCluster, count, err = c.cs.Kubernetes.GetKubernetesClusterByID(r.Id) - if err != nil { - return err - } - if count == 0 { - return errors.New("cluster not found") - } + csCluster.Status.CloudStackClusterID = cloudStackCKSCluster.Id } - csCluster.Status.CloudStackClusterID = csUnmanagedCluster.Id return nil } func (c *client) DeleteCluster(csCluster *infrav1.CloudStackCluster) error { if csCluster.Status.CloudStackClusterID != "" { csUnmanagedCluster, count, err := c.cs.Kubernetes.GetKubernetesClusterByID(csCluster.Status.CloudStackClusterID, withExternalManaged()) - if err != nil { - return err + if err != nil && strings.Contains(err.Error(), " not found") { + return nil } if count != 0 { params := c.cs.Kubernetes.NewDeleteKubernetesClusterParams(csUnmanagedCluster.Id) @@ -121,7 +116,6 @@ func (c *client) DeleteCluster(csCluster *infrav1.CloudStackCluster) error { } } csCluster.Status.CloudStackClusterID = "" - return nil } return nil } diff --git a/pkg/cloud/isolated_network.go b/pkg/cloud/isolated_network.go index f440d70d..d1458637 100644 --- a/pkg/cloud/isolated_network.go +++ b/pkg/cloud/isolated_network.go @@ -101,6 +101,7 @@ func (c *client) CreateIsolatedNetwork(fd *infrav1.CloudStackFailureDomain, isoN // Do isolated network creation. p := c.cs.Network.NewCreateNetworkParams(isoNet.Spec.Name, offeringID, fd.Spec.Zone.ID) + p.SetDisplaytext(isoNet.Spec.Name) resp, err := c.cs.Network.CreateNetwork(p) if err != nil { c.customMetrics.EvaluateErrorAndIncrementAcsReconciliationErrorCounter(err) From dbadbcdd31624c068f9dd6c9c30443ac6822613e Mon Sep 17 00:00:00 2001 From: Vishesh Date: Fri, 4 Aug 2023 13:46:19 +0530 Subject: [PATCH 4/6] Update cloudstack-go version to v2.16.0-rc.1 --- controllers/cloudstackcluster_controller.go | 2 +- go.mod | 4 +--- go.sum | 4 ++-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/controllers/cloudstackcluster_controller.go b/controllers/cloudstackcluster_controller.go index 8ec3883a..39017173 100644 --- a/controllers/cloudstackcluster_controller.go +++ b/controllers/cloudstackcluster_controller.go @@ -110,7 +110,7 @@ func (r *CloudStackClusterReconciliationRunner) GetOrCreateCluster() (ctrl.Resul r.Log.Info("Kubernetes Service plugin is disabled on CloudStack. Skipping ExternalManaged kubernetes cluster creation") return ctrl.Result{}, nil } - // Not requeing the failure to support CloudStack v4.18 and before + // Not requeueing the failure to support CloudStack v4.18 and before r.Log.Info(fmt.Sprintf("Failed creating ExternalManaged kubernetes cluster on CloudStack. Error: %s", err.Error())) } return ctrl.Result{}, nil diff --git a/go.mod b/go.mod index fe3a0825..1df29418 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.19 require ( github.com/ReneKroon/ttlcache v1.7.0 - github.com/apache/cloudstack-go/v2 v2.13.0 + github.com/apache/cloudstack-go/v2 v2.16.0-rc.1 github.com/go-logr/logr v1.2.3 github.com/golang/mock v1.6.0 github.com/hashicorp/go-multierror v1.1.1 @@ -108,5 +108,3 @@ replace github.com/dgrijalva/jwt-go => github.com/golang-jwt/jwt/v4 v4.0.0 // In replace sigs.k8s.io/cluster-api/test => sigs.k8s.io/cluster-api/test v1.2.12 replace sigs.k8s.io/cluster-api => sigs.k8s.io/cluster-api v1.2.12 - -replace github.com/apache/cloudstack-go/v2 => github.com/shapeblue/cloudstack-go/v2 v2.9.1-0.20230717062313-73e4efc8a510 diff --git a/go.sum b/go.sum index ef9f8cb0..f659ec74 100644 --- a/go.sum +++ b/go.sum @@ -89,6 +89,8 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 h1:yL7+Jz0jTC6yykIK/Wh74gnTJnrGr5AyrNMXuA0gves= github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= +github.com/apache/cloudstack-go/v2 v2.16.0-rc.1 h1:Jzp5XG9dxnYJMU9F7+7jXR5fuD2KlcIXaxe3JzkcYdo= +github.com/apache/cloudstack-go/v2 v2.16.0-rc.1/go.mod h1:Mc+tXpujtslBuZFk5atoGT2LanVxOrXS2GGgidAoz1A= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= @@ -450,8 +452,6 @@ github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/shapeblue/cloudstack-go/v2 v2.9.1-0.20230717062313-73e4efc8a510 h1:FPRBv784robz6sZSqDGfZDZMse31lj96i+enH02Xzds= -github.com/shapeblue/cloudstack-go/v2 v2.9.1-0.20230717062313-73e4efc8a510/go.mod h1:Mc+tXpujtslBuZFk5atoGT2LanVxOrXS2GGgidAoz1A= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= From 924ef9702d87bba95f7e900e799b704ca0ff2f6f Mon Sep 17 00:00:00 2001 From: Vishesh Date: Wed, 16 Aug 2023 13:58:46 +0530 Subject: [PATCH 5/6] Rename cluster to unmanaged cluster --- controllers/cloudstackcluster_controller.go | 16 ++++++++-------- controllers/cloudstackmachine_controller.go | 4 ++-- controllers/cloudstackmachine_controller_test.go | 8 ++++---- pkg/cloud/cluster.go | 16 ++++++++-------- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/controllers/cloudstackcluster_controller.go b/controllers/cloudstackcluster_controller.go index 39017173..a1d8091d 100644 --- a/controllers/cloudstackcluster_controller.go +++ b/controllers/cloudstackcluster_controller.go @@ -94,17 +94,17 @@ func (r *CloudStackClusterReconciliationRunner) Reconcile() (res ctrl.Result, re r.GetFailureDomains(r.FailureDomains), r.RemoveExtraneousFailureDomains(r.FailureDomains), r.VerifyFailureDomainCRDs, - r.GetOrCreateCluster, + r.GetOrCreateUnmanagedCluster, r.SetReady) } -// GetOrCreateCluster checks if an unmanaged cluster is present in Cloudstack else creates one. -func (r *CloudStackClusterReconciliationRunner) GetOrCreateCluster() (ctrl.Result, error) { +// GetOrCreateUnmanagedCluster checks if an unmanaged cluster is present in Cloudstack else creates one. +func (r *CloudStackClusterReconciliationRunner) GetOrCreateUnmanagedCluster() (ctrl.Result, error) { res, err := r.AsFailureDomainUser(&r.CSCluster.Spec.FailureDomains[0])() if r.ShouldReturn(res, err) { return res, err } - err = r.CSUser.GetOrCreateCluster(r.CAPICluster, r.ReconciliationSubject, &r.FailureDomains.Items[0].Spec) + err = r.CSUser.GetOrCreateUnmanagedCluster(r.CAPICluster, r.ReconciliationSubject, &r.FailureDomains.Items[0].Spec) if err != nil { if strings.Contains(err.Error(), "Kubernetes Service plugin is disabled") { r.Log.Info("Kubernetes Service plugin is disabled on CloudStack. Skipping ExternalManaged kubernetes cluster creation") @@ -170,21 +170,21 @@ func (r *CloudStackClusterReconciliationRunner) ReconcileDelete() (ctrl.Result, } return r.RequeueWithMessage("Child FailureDomains still present, requeueing.") } - if res, err := r.DeleteCluster(); r.ShouldReturn(res, err) { + if res, err := r.DeleteUnmanagedCluster(); r.ShouldReturn(res, err) { return res, err } controllerutil.RemoveFinalizer(r.ReconciliationSubject, infrav1.ClusterFinalizer) return ctrl.Result{}, nil } -// DeleteCluster checks if an unmanaged cluster is present in Cloudstack and then deletes it. -func (r *CloudStackClusterReconciliationRunner) DeleteCluster() (ctrl.Result, error) { +// DeleteUnmanagedCluster checks if an unmanaged cluster is present in Cloudstack and then deletes it. +func (r *CloudStackClusterReconciliationRunner) DeleteUnmanagedCluster() (ctrl.Result, error) { // If field is present and delete fails, then requeue res, err := r.AsFailureDomainUser(&r.CSCluster.Spec.FailureDomains[0])() if r.ShouldReturn(res, err) { return res, err } - err = r.CSUser.DeleteCluster(r.ReconciliationSubject) + err = r.CSUser.DeleteUnmanagedCluster(r.ReconciliationSubject) if err != nil { if strings.Contains(err.Error(), " not found") { return ctrl.Result{}, nil diff --git a/controllers/cloudstackmachine_controller.go b/controllers/cloudstackmachine_controller.go index 5541ffdf..5d0cb23c 100644 --- a/controllers/cloudstackmachine_controller.go +++ b/controllers/cloudstackmachine_controller.go @@ -217,14 +217,14 @@ func (r *CloudStackMachineReconciliationRunner) DeleteMachineIfFailuredomainNotE // AttachVM adds the VM to CloudStack Unmanaged kubernetes. // No action taken if it fails func (r *CloudStackMachineReconciliationRunner) AttachVM() (retRes ctrl.Result, reterr error) { - _ = r.CSUser.AddVMToCluster(r.CSCluster, r.ReconciliationSubject) + _ = r.CSUser.AddVMToUnmanagedCluster(r.CSCluster, r.ReconciliationSubject) return ctrl.Result{}, nil } // RemoveVM removes the VM from CloudStack Unmanaged kubernetes. // No action taken if it fails func (r *CloudStackMachineReconciliationRunner) RemoveVM() (retRes ctrl.Result, reterr error) { - _ = r.CSUser.RemoveVMFromCluster(r.CSCluster, r.ReconciliationSubject) + _ = r.CSUser.RemoveVMFromUnmanagedCluster(r.CSCluster, r.ReconciliationSubject) return ctrl.Result{}, nil } diff --git a/controllers/cloudstackmachine_controller_test.go b/controllers/cloudstackmachine_controller_test.go index b3cb44dc..0d7676db 100644 --- a/controllers/cloudstackmachine_controller_test.go +++ b/controllers/cloudstackmachine_controller_test.go @@ -55,15 +55,15 @@ var _ = Describe("CloudStackMachineReconciler", func() { Ω(k8sClient.Create(ctx, dummies.CSFailureDomain1)).Should(Succeed()) setClusterReady(k8sClient) - mockCloudClient.EXPECT().GetOrCreateCluster(gomock.Any(), gomock.Any(), gomock.Any()).Do( + mockCloudClient.EXPECT().GetOrCreateUnmanagedCluster(gomock.Any(), gomock.Any(), gomock.Any()).Do( func(arg1, _, _ interface{}) { arg1.(*infrav1.CloudStackCluster).Status.CloudStackClusterID = "cluster-id-123" }).AnyTimes().Return(nil) - mockCloudClient.EXPECT().AddVMToCluster( + mockCloudClient.EXPECT().AddVMToUnmanagedCluster( gomock.Any(), gomock.Any()).AnyTimes().Return(nil) - mockCloudClient.EXPECT().RemoveVMFromCluster( + mockCloudClient.EXPECT().RemoveVMFromUnmanagedCluster( gomock.Any(), gomock.Any()).AnyTimes().Return(nil) }) @@ -251,7 +251,7 @@ var _ = Describe("CloudStackMachineReconciler", func() { func(arg1, _, _, _, _, _ interface{}) { arg1.(*infrav1.CloudStackMachine).Status.InstanceState = "Running" }).AnyTimes() - mockCloudClient.EXPECT().AddVMToCluster( + mockCloudClient.EXPECT().AddVMToUnmanagedCluster( gomock.Any(), gomock.Any()).AnyTimes().Return(nil) Ω(fakeCtrlClient.Get(ctx, key, dummies.CSCluster)).Should(Succeed()) Ω(fakeCtrlClient.Create(ctx, dummies.CAPIMachine)).Should(Succeed()) diff --git a/pkg/cloud/cluster.go b/pkg/cloud/cluster.go index 7313445f..43442a1d 100644 --- a/pkg/cloud/cluster.go +++ b/pkg/cloud/cluster.go @@ -27,10 +27,10 @@ import ( ) type ClusterIface interface { - GetOrCreateCluster(*clusterv1.Cluster, *infrav1.CloudStackCluster, *infrav1.CloudStackFailureDomainSpec) error - DeleteCluster(*infrav1.CloudStackCluster) error - AddVMToCluster(*infrav1.CloudStackCluster, *infrav1.CloudStackMachine) error - RemoveVMFromCluster(*infrav1.CloudStackCluster, *infrav1.CloudStackMachine) error + GetOrCreateUnmanagedCluster(*clusterv1.Cluster, *infrav1.CloudStackCluster, *infrav1.CloudStackFailureDomainSpec) error + DeleteUnmanagedCluster(*infrav1.CloudStackCluster) error + AddVMToUnmanagedCluster(*infrav1.CloudStackCluster, *infrav1.CloudStackMachine) error + RemoveVMFromUnmanagedCluster(*infrav1.CloudStackCluster, *infrav1.CloudStackMachine) error } type ClustertypeSetter interface { @@ -48,7 +48,7 @@ func withExternalManaged() cloudstack.OptionFunc { } } -func (c *client) GetOrCreateCluster(cluster *clusterv1.Cluster, csCluster *infrav1.CloudStackCluster, fd *infrav1.CloudStackFailureDomainSpec) error { +func (c *client) GetOrCreateUnmanagedCluster(cluster *clusterv1.Cluster, csCluster *infrav1.CloudStackCluster, fd *infrav1.CloudStackFailureDomainSpec) error { // Get cluster if csCluster.Status.CloudStackClusterID != "" { externalManagedCluster, count, err := c.cs.Kubernetes.GetKubernetesClusterByID(csCluster.Status.CloudStackClusterID, withExternalManaged()) @@ -102,7 +102,7 @@ func (c *client) GetOrCreateCluster(cluster *clusterv1.Cluster, csCluster *infra return nil } -func (c *client) DeleteCluster(csCluster *infrav1.CloudStackCluster) error { +func (c *client) DeleteUnmanagedCluster(csCluster *infrav1.CloudStackCluster) error { if csCluster.Status.CloudStackClusterID != "" { csUnmanagedCluster, count, err := c.cs.Kubernetes.GetKubernetesClusterByID(csCluster.Status.CloudStackClusterID, withExternalManaged()) if err != nil && strings.Contains(err.Error(), " not found") { @@ -120,7 +120,7 @@ func (c *client) DeleteCluster(csCluster *infrav1.CloudStackCluster) error { return nil } -func (c *client) AddVMToCluster(csCluster *infrav1.CloudStackCluster, csMachine *infrav1.CloudStackMachine) error { +func (c *client) AddVMToUnmanagedCluster(csCluster *infrav1.CloudStackCluster, csMachine *infrav1.CloudStackMachine) error { if csCluster.Status.CloudStackClusterID != "" { params := c.cs.Kubernetes.NewAddVirtualMachinesToKubernetesClusterParams(csCluster.Status.CloudStackClusterID, []string{*csMachine.Spec.InstanceID}) _, err := c.cs.Kubernetes.AddVirtualMachinesToKubernetesCluster(params) @@ -129,7 +129,7 @@ func (c *client) AddVMToCluster(csCluster *infrav1.CloudStackCluster, csMachine return nil } -func (c *client) RemoveVMFromCluster(csCluster *infrav1.CloudStackCluster, csMachine *infrav1.CloudStackMachine) error { +func (c *client) RemoveVMFromUnmanagedCluster(csCluster *infrav1.CloudStackCluster, csMachine *infrav1.CloudStackMachine) error { if csCluster.Status.CloudStackClusterID != "" { params := c.cs.Kubernetes.NewRemoveVirtualMachinesFromKubernetesClusterParams(csCluster.Status.CloudStackClusterID, []string{*csMachine.Spec.InstanceID}) _, err := c.cs.Kubernetes.RemoveVirtualMachinesFromKubernetesCluster(params) From 254d93b48d85236239cd4017132546efc2738073 Mon Sep 17 00:00:00 2001 From: Vishesh Date: Mon, 21 Aug 2023 15:51:51 +0530 Subject: [PATCH 6/6] Add test for unmanaged cluster on ACS --- go.mod | 2 +- go.sum | 2 + test/e2e/common.go | 13 +++- test/e2e/go.mod | 28 +++++-- test/e2e/go.sum | 50 ++++++++++++ test/e2e/unmanaged_k8s.go | 136 +++++++++++++++++++++++++++++++++ test/e2e/unmanaged_k8s_test.go | 36 +++++++++ 7 files changed, 257 insertions(+), 10 deletions(-) create mode 100644 test/e2e/unmanaged_k8s.go create mode 100644 test/e2e/unmanaged_k8s_test.go diff --git a/go.mod b/go.mod index 1df29418..85d793f3 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.19 require ( github.com/ReneKroon/ttlcache v1.7.0 - github.com/apache/cloudstack-go/v2 v2.16.0-rc.1 + github.com/apache/cloudstack-go/v2 v2.16.0-rc.2 github.com/go-logr/logr v1.2.3 github.com/golang/mock v1.6.0 github.com/hashicorp/go-multierror v1.1.1 diff --git a/go.sum b/go.sum index f659ec74..c6f6cb6e 100644 --- a/go.sum +++ b/go.sum @@ -91,6 +91,8 @@ github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 h1:yL7+Jz0jTC6yykIK/Wh74gnTJnrG github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= github.com/apache/cloudstack-go/v2 v2.16.0-rc.1 h1:Jzp5XG9dxnYJMU9F7+7jXR5fuD2KlcIXaxe3JzkcYdo= github.com/apache/cloudstack-go/v2 v2.16.0-rc.1/go.mod h1:Mc+tXpujtslBuZFk5atoGT2LanVxOrXS2GGgidAoz1A= +github.com/apache/cloudstack-go/v2 v2.16.0-rc.2 h1:Vare0MkgdEbAEY1dZtcz/m6HRdK+NKkuoiF6WwC1t2g= +github.com/apache/cloudstack-go/v2 v2.16.0-rc.2/go.mod h1:Mc+tXpujtslBuZFk5atoGT2LanVxOrXS2GGgidAoz1A= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= diff --git a/test/e2e/common.go b/test/e2e/common.go index 0496a2fd..308d1cac 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -22,9 +22,9 @@ import ( "errors" "fmt" "path/filepath" + "strconv" "strings" "time" - "strconv" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -250,6 +250,17 @@ func DownloadMetricsFromCAPCManager(ctx context.Context, bootstrapKubeconfigPath return result, nil } +func GetACSVersion(client *cloudstack.CloudStackClient) (string, error) { + msServersResp, err := client.InfrastructureUsage.ListManagementServersMetrics(client.InfrastructureUsage.NewListManagementServersMetricsParams()) + if err != nil { + return "", err + } + if msServersResp.Count == 0 { + return "", errors.New("no management servers found") + } + return msServersResp.ManagementServersMetrics[0].Version, nil +} + func DestroyOneMachine(client *cloudstack.CloudStackClient, clusterName string, machineType string) { matcher := clusterName + "-" + machineType diff --git a/test/e2e/go.mod b/test/e2e/go.mod index c6bc0384..1754f9c0 100644 --- a/test/e2e/go.mod +++ b/test/e2e/go.mod @@ -4,10 +4,10 @@ go 1.19 require ( github.com/Shopify/toxiproxy/v2 v2.5.0 - github.com/apache/cloudstack-go/v2 v2.13.0 + github.com/apache/cloudstack-go/v2 v2.16.0-rc.2 github.com/blang/semver v3.5.1+incompatible github.com/onsi/ginkgo v1.16.5 - github.com/onsi/gomega v1.20.1 + github.com/onsi/gomega v1.27.4 gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.25.3 k8s.io/apimachinery v0.25.3 @@ -44,6 +44,7 @@ require ( github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/fsnotify/fsnotify v1.5.4 // indirect + github.com/go-errors/errors v1.0.1 // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.19.5 // indirect @@ -52,13 +53,14 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/mock v1.6.0 // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/google/cel-go v0.12.4 // indirect github.com/google/gnostic v0.5.7-v3refs // indirect - github.com/google/go-cmp v0.5.8 // indirect + github.com/google/go-cmp v0.5.9 // indirect github.com/google/go-github/v45 v45.2.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/uuid v1.2.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/huandu/xstrings v1.3.3 // indirect @@ -69,14 +71,17 @@ require ( github.com/magiconair/properties v1.8.6 // indirect github.com/mailru/easyjson v0.7.6 // indirect github.com/mattn/go-isatty v0.0.16 // indirect + github.com/mattn/go-runewidth v0.0.13 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/nxadm/tail v1.4.8 // indirect + github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.2 // indirect github.com/pelletier/go-toml v1.9.5 // indirect @@ -86,6 +91,7 @@ require ( github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect + github.com/rivo/uniseg v0.2.0 // indirect github.com/shopspring/decimal v1.2.0 // indirect github.com/spf13/afero v1.8.2 // indirect github.com/spf13/cast v1.5.0 // indirect @@ -96,12 +102,14 @@ require ( github.com/stoewer/go-strcase v1.2.0 // indirect github.com/subosito/gotenv v1.3.0 // indirect github.com/valyala/fastjson v1.6.3 // indirect + github.com/xlab/treeprint v1.1.0 // indirect + go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect golang.org/x/crypto v0.3.0 // indirect - golang.org/x/net v0.7.0 // indirect + golang.org/x/net v0.8.0 // indirect golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb // indirect - golang.org/x/sys v0.5.0 // indirect - golang.org/x/term v0.5.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/sys v0.6.0 // indirect + golang.org/x/term v0.6.0 // indirect + golang.org/x/text v0.8.0 // indirect golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect @@ -120,6 +128,10 @@ require ( k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect sigs.k8s.io/kind v0.14.0 // indirect + sigs.k8s.io/kustomize/api v0.12.1 // indirect + sigs.k8s.io/kustomize/cmd/config v0.10.9 // indirect + sigs.k8s.io/kustomize/kustomize/v4 v4.5.7 // indirect + sigs.k8s.io/kustomize/kyaml v0.13.9 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/test/e2e/go.sum b/test/e2e/go.sum index 67f0ebb3..0ca4b03f 100644 --- a/test/e2e/go.sum +++ b/test/e2e/go.sum @@ -72,6 +72,10 @@ github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220418222510-f25a4f6275ed h1:u github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220418222510-f25a4f6275ed/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= github.com/apache/cloudstack-go/v2 v2.13.0 h1:t0uj7QxQpnzD/LSTP6a4w2NTuZXisxIM/mIDNkF44lc= github.com/apache/cloudstack-go/v2 v2.13.0/go.mod h1:aosD8Svfu5nhH5Sp4zcsVV1hT5UGt3mTgRXM8YqTKe0= +github.com/apache/cloudstack-go/v2 v2.16.0-rc.1 h1:Jzp5XG9dxnYJMU9F7+7jXR5fuD2KlcIXaxe3JzkcYdo= +github.com/apache/cloudstack-go/v2 v2.16.0-rc.1/go.mod h1:Mc+tXpujtslBuZFk5atoGT2LanVxOrXS2GGgidAoz1A= +github.com/apache/cloudstack-go/v2 v2.16.0-rc.2 h1:Vare0MkgdEbAEY1dZtcz/m6HRdK+NKkuoiF6WwC1t2g= +github.com/apache/cloudstack-go/v2 v2.16.0-rc.2/go.mod h1:Mc+tXpujtslBuZFk5atoGT2LanVxOrXS2GGgidAoz1A= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= @@ -158,6 +162,8 @@ github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwV github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= +github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -225,6 +231,8 @@ github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/cel-go v0.12.4 h1:YINKfuHZ8n72tPOqSPZBwGiDpew2CJS48mdM5W8LZQU= @@ -244,6 +252,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github/v45 v45.2.0 h1:5oRLszbrkvxDDqBCNj2hjDZMKmvexaZ1xw/FCD+K3FI= github.com/google/go-github/v45 v45.2.0/go.mod h1:FObaZJEDSTa/WGCzZ2Z3eoCDXWJKMenWWTrd8jrta28= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= @@ -266,6 +276,8 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= @@ -353,6 +365,9 @@ github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= +github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= @@ -382,6 +397,8 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= @@ -395,6 +412,8 @@ github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= @@ -407,6 +426,8 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.20.1 h1:PA/3qinGoukvymdIDV8pii6tiZgC8kbmJO6Z5+b002Q= github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= +github.com/onsi/gomega v1.27.4 h1:Z2AnStgsdSayCMDiCU42qIz+HLqEPcgiOCXjAU/w+8E= +github.com/onsi/gomega v1.27.4/go.mod h1:riYq/GJKh8hhoM01HN6Vmuy93AarCXCBGpvFDK3q3fQ= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= @@ -456,6 +477,8 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -464,6 +487,12 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shapeblue/cloudstack-go/v2 v2.9.1-0.20230821174931-850c8f836fba h1:xQURtr9H7Ik+WJaEo4qYuN5Vo+aagAd7bVA5qAWeQj8= +github.com/shapeblue/cloudstack-go/v2 v2.9.1-0.20230821174931-850c8f836fba/go.mod h1:Mc+tXpujtslBuZFk5atoGT2LanVxOrXS2GGgidAoz1A= +github.com/shapeblue/cloudstack-go/v2 v2.9.1-0.20230821182853-7e3eacb8c381 h1:eMPPCGnfpj8hKQBpZvp4Bn85gV0gawZRLolD++jU86k= +github.com/shapeblue/cloudstack-go/v2 v2.9.1-0.20230821182853-7e3eacb8c381/go.mod h1:Mc+tXpujtslBuZFk5atoGT2LanVxOrXS2GGgidAoz1A= +github.com/shapeblue/cloudstack-go/v2 v2.9.1-0.20230823130230-d079d38eca10 h1:J5AU6Z0yKvjaNZ67z8YMY0e3JtIloG3aRmfcoZ2pggo= +github.com/shapeblue/cloudstack-go/v2 v2.9.1-0.20230823130230-d079d38eca10/go.mod h1:Mc+tXpujtslBuZFk5atoGT2LanVxOrXS2GGgidAoz1A= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= @@ -515,6 +544,8 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1 github.com/valyala/fastjson v1.6.3 h1:tAKFnnwmeMGPbwJ7IwxcTPCNr3uIzoIj3/Fh90ra4xc= github.com/valyala/fastjson v1.6.3/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xlab/treeprint v1.1.0 h1:G/1DjNkPpfZCFt9CSh6b5/nY4VimlbHF3Rh4obvtzDk= +github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -529,6 +560,8 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 h1:+FNtrFTmVw0YZGpBGX56XDee331t6JAXeK2bcyhLOOc= +go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= @@ -635,6 +668,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -678,6 +713,7 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -726,11 +762,15 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -743,6 +783,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -997,6 +1039,14 @@ sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/kind v0.14.0 h1:cNmI3jGBvp7UegEGbC5we8plDtCUmaNRL+bod7JoSCE= sigs.k8s.io/kind v0.14.0/go.mod h1:UrFRPHG+2a5j0Q7qiR4gtJ4rEyn8TuMQwuOPf+m4oHg= +sigs.k8s.io/kustomize/api v0.12.1 h1:7YM7gW3kYBwtKvoY216ZzY+8hM+lV53LUayghNRJ0vM= +sigs.k8s.io/kustomize/api v0.12.1/go.mod h1:y3JUhimkZkR6sbLNwfJHxvo1TCLwuwm14sCYnkH6S1s= +sigs.k8s.io/kustomize/cmd/config v0.10.9 h1:LV8AUwZPuvqhGfia50uNwsPwNg1xOy9koEf5hyBnYs4= +sigs.k8s.io/kustomize/cmd/config v0.10.9/go.mod h1:T0s850zPV3wKfBALA0dyeP/K74jlJcoP8Pr9ZWwE3MQ= +sigs.k8s.io/kustomize/kustomize/v4 v4.5.7 h1:cDW6AVMl6t/SLuQaezMET8hgnadZGIAr8tUrxFVOrpg= +sigs.k8s.io/kustomize/kustomize/v4 v4.5.7/go.mod h1:VSNKEH9D9d9bLiWEGbS6Xbg/Ih0tgQalmPvntzRxZ/Q= +sigs.k8s.io/kustomize/kyaml v0.13.9 h1:Qz53EAaFFANyNgyOEJbT/yoIHygK40/ZcvU3rgry2Tk= +sigs.k8s.io/kustomize/kyaml v0.13.9/go.mod h1:QsRbD0/KcU+wdk0/L0fIp2KLnohkVzs6fQ85/nOXac4= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= diff --git a/test/e2e/unmanaged_k8s.go b/test/e2e/unmanaged_k8s.go new file mode 100644 index 00000000..b8298c63 --- /dev/null +++ b/test/e2e/unmanaged_k8s.go @@ -0,0 +1,136 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/apache/cloudstack-go/v2/cloudstack" + "github.com/blang/semver" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/pointer" + + "sigs.k8s.io/cluster-api/test/framework/clusterctl" + "sigs.k8s.io/cluster-api/util" +) + +// UnmanagedK8SSpec implements a spec that creates a cluster and checks whether an entry is created in ACS. +func UnmanagedK8SSpec(ctx context.Context, inputGetter func() CommonSpecInput) { + var ( + specName = "k8s-unmanaged" + input CommonSpecInput + namespace *corev1.Namespace + cancelWatches context.CancelFunc + clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult + ) + + BeforeEach(func() { + Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName) + input = inputGetter() + + csClient := CreateCloudStackClient(ctx, input.BootstrapClusterProxy.GetKubeconfigPath()) + version, err := GetACSVersion(csClient) + + if err != nil || version == "" { + Skip("Failed to get CloudStack's version") + } + + v, err := semver.ParseTolerant(strings.Join(strings.Split(version, ".")[0:3], ".")) + + if err != nil { + Skip("Failed to parse CloudStack version " + version) + } + + expectedRange, _ := semver.ParseRange(">=4.19.0") + + if !expectedRange(v) { + Skip("Cloudstack version " + version + " is less than 4.19.") + } + + Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName) + Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling %s spec", specName) + Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName) + Expect(os.MkdirAll(input.ArtifactFolder, 0750)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName) + + Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion)) + + // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. + namespace, cancelWatches = setupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder) + clusterResources = new(clusterctl.ApplyClusterTemplateAndWaitResult) + }) + + It("Should create a workload cluster", func() { + By("Creating a workload cluster") + + clusterName := fmt.Sprintf("%s-%s", specName, util.RandomString(6)) + + clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: input.BootstrapClusterProxy, + CNIManifestPath: input.E2EConfig.GetVariable(CNIPath), + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()), + ClusterctlConfigPath: input.ClusterctlConfigPath, + KubeconfigPath: input.BootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + Flavor: clusterctl.DefaultFlavor, + Namespace: namespace.Name, + ClusterName: clusterName, + KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersion), + ControlPlaneMachineCount: pointer.Int64Ptr(1), + WorkerMachineCount: pointer.Int64Ptr(1), + }, + WaitForClusterIntervals: input.E2EConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"), + }, clusterResources) + + By("checking unmanaged k8s resource is created on ACS") + // Get details from ACS + csClient := CreateCloudStackClient(ctx, input.BootstrapClusterProxy.GetKubeconfigPath()) + lkcp := csClient.Kubernetes.NewListKubernetesClustersParams() + lkcp.SetListall(true) + + clusters, err := csClient.Kubernetes.ListKubernetesClusters(lkcp) + + if err != nil { + Fail("Failed to get Kubernetes clusters from ACS") + } + + var cluster *cloudstack.KubernetesCluster + + for _, d := range clusters.KubernetesClusters { + if strings.HasPrefix(d.Name, fmt.Sprintf("%s - %s", clusterName, clusterName)) { + cluster = d + } + } + + Expect(cluster).ShouldNot(BeNil(), "Couldn't find the external managed kubernetes in ACS") + Expect(len(cluster.Virtualmachines)).Should(Equal(2), "Expected 2 VMs in the cluster, found %d", len(cluster.Virtualmachines)) + By("PASSED!") + }) + + AfterEach(func() { + // Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself. + dumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, namespace, cancelWatches, clusterResources.Cluster, input.E2EConfig.GetIntervals, input.SkipCleanup) + }) +} diff --git a/test/e2e/unmanaged_k8s_test.go b/test/e2e/unmanaged_k8s_test.go new file mode 100644 index 00000000..ee8bd487 --- /dev/null +++ b/test/e2e/unmanaged_k8s_test.go @@ -0,0 +1,36 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + . "github.com/onsi/ginkgo" +) + +var _ = Describe("When testing creation of unmanaged CKS cluster in ACS", func() { + UnmanagedK8SSpec(ctx, func() CommonSpecInput { + return CommonSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: clusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + } + }) +})