Merge pull request #2005 from Nordix/mquhuy/ensure-cleanup-e2e-tests

🌱 Ensure E2E cleanup

Author: k8s-ci-robot

test/e2e/shared/openstack.go

@@ -39,6 +39,7 @@ import (
 	"github.com/gophercloud/gophercloud/openstack/compute/v2/flavors"
 	"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
 	"github.com/gophercloud/gophercloud/openstack/imageservice/v2/images"
+	"github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/loadbalancers"
 	"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/routers"
 	"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/security/groups"
 	"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/security/rules"
@@ -871,3 +872,28 @@ func GetFlavorFromName(e2eCtx *E2EContext, name string) (*flavors.Flavor, error)
 
 	return flavors.Get(computeClient, flavorID).Extract()
 }
+
+func DumpOpenStackLoadBalancers(e2eCtx *E2EContext, filter loadbalancers.ListOpts) ([]loadbalancers.LoadBalancer, error) {
+	providerClient, clientOpts, _, err := GetTenantProviderClient(e2eCtx)
+	if err != nil {
+		_, _ = fmt.Fprintf(GinkgoWriter, "error creating provider client: %s\n", err)
+		return nil, err
+	}
+
+	loadBalancerClient, err := openstack.NewLoadBalancerV2(providerClient, gophercloud.EndpointOpts{
+		Region: clientOpts.RegionName,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("error creating network client: %s", err)
+	}
+
+	allPages, err := loadbalancers.List(loadBalancerClient, filter).AllPages()
+	if err != nil {
+		return nil, fmt.Errorf("error getting load balancers: %s", err)
+	}
+	loadBalancersList, err := loadbalancers.ExtractLoadBalancers(allPages)
+	if err != nil {
+		return nil, fmt.Errorf("error getting load balancers: %s", err)
+	}
+	return loadBalancersList, nil
+}

test/e2e/suites/e2e/e2e_suite_test.go

@@ -23,15 +23,29 @@ import (
 	"os"
 	"testing"
 
+	"github.com/gophercloud/gophercloud/openstack/blockstorage/v3/volumes"
+	"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
+	"github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/loadbalancers"
+	"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/security/groups"
+	"github.com/gophercloud/gophercloud/openstack/networking/v2/networks"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"k8s.io/klog/v2"
+	"k8s.io/utils/ptr"
 	ctrl "sigs.k8s.io/controller-runtime"
 
 	"sigs.k8s.io/cluster-api-provider-openstack/test/e2e/shared"
 )
 
-var e2eCtx *shared.E2EContext
+var (
+	e2eCtx                *shared.E2EContext
+	initialServers        []servers.Server
+	initialNetworks       []networks.Network
+	initialSecurityGroups []groups.SecGroup
+	initialLoadBalancers  []loadbalancers.LoadBalancer
+	initialVolumes        []volumes.Volume
+	err                   error
+)
 
 func init() {
 	e2eCtx = shared.NewE2EContext()
@@ -55,10 +69,60 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 	return data
 }, func(data []byte) {
 	shared.AllNodesBeforeSuite(e2eCtx, data)
+	initialServers, err = shared.DumpOpenStackServers(e2eCtx, servers.ListOpts{})
+	Expect(err).NotTo(HaveOccurred())
+	initialNetworks, err = shared.DumpOpenStackNetworks(e2eCtx, networks.ListOpts{})
+	Expect(err).NotTo(HaveOccurred())
+	initialSecurityGroups, err = shared.DumpOpenStackSecurityGroups(e2eCtx, groups.ListOpts{})
+	Expect(err).NotTo(HaveOccurred())
+	initialLoadBalancers, err = shared.DumpOpenStackLoadBalancers(e2eCtx, loadbalancers.ListOpts{})
+	Expect(err).NotTo(HaveOccurred())
+	initialVolumes, err = shared.DumpOpenStackVolumes(e2eCtx, volumes.ListOpts{})
+	Expect(err).NotTo(HaveOccurred())
 })
 
+// CheckResourceCleanup checks if all resources created during the test are cleaned up by comparing the resources
+// before and after the test.
+// The function f is used to list the resources of type T, whose list opts is of type L.
+// The list of resources is then compared to the initialResources, using the ConsistOfIDs custom matcher.
+func CheckResourceCleanup[T any, L any](f func(*shared.E2EContext, L) ([]T, error), l L, initialResources []T) *string {
+	endResources, err := f(e2eCtx, l)
+	if err != nil {
+		return ptr.To(err.Error())
+	}
+
+	matcher := ConsistOfIDs(initialResources)
+	success, err := matcher.Match(endResources)
+	if err != nil {
+		return ptr.To(err.Error())
+	}
+	if !success {
+		return ptr.To(matcher.FailureMessage(endResources))
+	}
+
+	return nil
+}
+
 var _ = SynchronizedAfterSuite(func() {
 	shared.AllNodesAfterSuite(e2eCtx)
 }, func() {
+	failed := false
+	for _, error := range []*string{
+		CheckResourceCleanup(shared.DumpOpenStackServers, servers.ListOpts{}, initialServers),
+		CheckResourceCleanup(shared.DumpOpenStackNetworks, networks.ListOpts{}, initialNetworks),
+		CheckResourceCleanup(shared.DumpOpenStackSecurityGroups, groups.ListOpts{}, initialSecurityGroups),
+		CheckResourceCleanup(shared.DumpOpenStackLoadBalancers, loadbalancers.ListOpts{}, initialLoadBalancers),
+		CheckResourceCleanup(shared.DumpOpenStackVolumes, volumes.ListOpts{}, initialVolumes),
+	} {
+		if error != nil {
+			GinkgoWriter.Println(*error)
+			failed = true
+		}
+	}
+
 	shared.Node1AfterSuite(e2eCtx)
+
+	if failed {
+		Fail("Not all resources were cleaned up")
+	}
 })

test/e2e/suites/e2e/idmatcher_test.go

@@ -0,0 +1,75 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/onsi/gomega"
+	"github.com/onsi/gomega/format"
+	"github.com/onsi/gomega/gstruct"
+	"github.com/onsi/gomega/types"
+)
+
+type idMatcher struct {
+	expected interface{}
+}
+
+var (
+	_ types.GomegaMatcher   = &idMatcher{}
+	_ format.GomegaStringer = &idMatcher{}
+)
+
+func (m *idMatcher) Match(actual interface{}) (bool, error) {
+	v := reflect.ValueOf(m.expected)
+	id := v.FieldByName("ID").String()
+
+	matcher := &gstruct.FieldsMatcher{
+		Fields: gstruct.Fields{
+			"ID": gomega.Equal(id),
+		},
+		IgnoreExtras: true,
+	}
+
+	return matcher.Match(actual)
+}
+
+func (m *idMatcher) FailureMessage(actual interface{}) string {
+	return fmt.Sprintf("Expected:\n%s\nto have the same ID as:\n%s", format.Object(actual, 1), format.Object(m.expected, 1))
+}
+
+func (m *idMatcher) NegatedFailureMessage(actual interface{}) string {
+	return fmt.Sprintf("Expected:\n%s\nnot to have the same ID as:\n%s", format.Object(actual, 1), format.Object(m.expected, 1))
+}
+
+func (m *idMatcher) GomegaString() string {
+	return fmt.Sprintf("ID match for:\n%s", format.Object(m.expected, 1))
+}
+
+func IDOf(expected interface{}) types.GomegaMatcher {
+	return &idMatcher{expected: expected}
+}
+
+func ConsistOfIDs[T any](expected []T) types.GomegaMatcher {
+	matchers := make([]types.GomegaMatcher, len(expected))
+	for i := range expected {
+		matchers[i] = IDOf(expected[i])
+	}
+
+	return gomega.ConsistOf(matchers)
+}