Merge pull request #4205 from zac-nixon/znixon/gw-integ

[gw api] Add Gateway API integ framework

Author: k8s-ci-robot

controllers/gateway/utils.go

@@ -35,6 +35,10 @@ func updateGatewayClassLastProcessedConfig(ctx context.Context, k8sClient client
 		return nil
 	}
 
+	if gwClass.Annotations == nil {
+		gwClass.Annotations = make(map[string]string)
+	}
+
 	gwClassOld := gwClass.DeepCopy()
 	gwClass.Annotations[gatewayClassAnnotationLastProcessedConfig] = calculatedVersion
 	gwClass.Annotations[gatewayClassAnnotationLastProcessedConfigTimestamp] = strconv.FormatInt(time.Now().Unix(), 10)

pkg/gateway/routeutils/tcp.go

@@ -73,7 +73,6 @@ func (tcpRoute *tcpRouteDescription) loadAttachedRules(ctx context.Context, k8sC
 
 		convertedRules = append(convertedRules, convertTCPRouteRule(&rule, convertedBackends))
 	}
-
 	tcpRoute.rules = convertedRules
 	return tcpRoute, nil
 }

test/e2e/gateway/alb_instance_target.go

@@ -0,0 +1,68 @@
+package gateway
+
+import (
+	"context"
+	corev1 "k8s.io/api/core/v1"
+	elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1"
+	"sigs.k8s.io/aws-load-balancer-controller/pkg/k8s"
+	"sigs.k8s.io/aws-load-balancer-controller/test/framework"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	gwv1 "sigs.k8s.io/gateway-api/apis/v1"
+)
+
+type ALBInstanceTestStack struct {
+	albResourceStack *albResourceStack
+}
+
+func (s *ALBInstanceTestStack) Deploy(ctx context.Context, f *framework.Framework, spec elbv2gw.LoadBalancerConfigurationSpec) error {
+	dp := buildDeploymentSpec(f.Options.TestImageRegistry)
+	svc := buildServiceSpec()
+	gwc := buildGatewayClassSpec("gateway.k8s.aws/alb")
+	gw := buildBasicGatewaySpec(gwc, gwv1.HTTPProtocolType)
+	lbc := buildLoadBalancerConfig(spec)
+	httpr := buildHTTPRoute()
+	s.albResourceStack = newALBResourceStack(dp, svc, gwc, gw, lbc, httpr, "service-instance-e2e", false)
+
+	return s.albResourceStack.Deploy(ctx, f)
+}
+
+func (s *ALBInstanceTestStack) ScaleDeployment(ctx context.Context, f *framework.Framework, numReplicas int32) error {
+	return s.albResourceStack.ScaleDeployment(ctx, f, numReplicas)
+}
+
+func (s *ALBInstanceTestStack) Cleanup(ctx context.Context, f *framework.Framework) {
+	_ = f.K8sClient.Delete(ctx, s.albResourceStack.httpr)
+	s.albResourceStack.Cleanup(ctx, f)
+}
+
+func (s *ALBInstanceTestStack) GetLoadBalancerIngressHostName() string {
+	return s.albResourceStack.GetLoadBalancerIngressHostname()
+}
+
+func (s *ALBInstanceTestStack) GetWorkerNodes(ctx context.Context, f *framework.Framework) ([]corev1.Node, error) {
+	allNodes := &corev1.NodeList{}
+	err := f.K8sClient.List(ctx, allNodes)
+	if err != nil {
+		return nil, err
+	}
+	nodeList := []corev1.Node{}
+	for _, node := range allNodes.Items {
+		if _, notarget := node.Labels["node.kubernetes.io/exclude-from-external-load-balancers"]; !notarget {
+			nodeList = append(nodeList, node)
+		}
+	}
+	return nodeList, nil
+}
+
+func (s *ALBInstanceTestStack) ApplyNodeLabels(ctx context.Context, f *framework.Framework, node *corev1.Node, labels map[string]string) error {
+	f.Logger.Info("applying node labels", "node", k8s.NamespacedName(node))
+	oldNode := node.DeepCopy()
+	for key, value := range labels {
+		node.Labels[key] = value
+	}
+	if err := f.K8sClient.Patch(ctx, node, client.MergeFrom(oldNode)); err != nil {
+		f.Logger.Info("failed to update node", "node", k8s.NamespacedName(node))
+		return err
+	}
+	return nil
+}

test/e2e/gateway/alb_instance_target_test.go

@@ -0,0 +1,94 @@
+package gateway
+
+import (
+	"context"
+	"fmt"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1"
+	"sigs.k8s.io/aws-load-balancer-controller/test/framework/http"
+	"sigs.k8s.io/aws-load-balancer-controller/test/framework/utils"
+	"sigs.k8s.io/aws-load-balancer-controller/test/framework/verifier"
+)
+
+var _ = Describe("test k8s alb gateway reconciled by the aws load balancer controller", func() {
+	var (
+		ctx     context.Context
+		stack   ALBInstanceTestStack
+		dnsName string
+		lbARN   string
+	)
+	BeforeEach(func() {
+		if !tf.Options.EnableGatewayTests {
+			Skip("Skipping gateway tests")
+		}
+		ctx = context.Background()
+		stack = ALBInstanceTestStack{}
+	})
+	AfterEach(func() {
+		stack.Cleanup(ctx, tf)
+	})
+	Context("with ALB instance target configuration", func() {
+		BeforeEach(func() {})
+		It("should provision internet-facing load balancer resources", func() {
+			interf := elbv2gw.LoadBalancerSchemeInternetFacing
+			lbcSpec := elbv2gw.LoadBalancerConfigurationSpec{
+				Scheme: &interf,
+			}
+			By("deploying stack", func() {
+				err := stack.Deploy(ctx, tf, lbcSpec)
+				Expect(err).NotTo(HaveOccurred())
+			})
+
+			By("checking service status for lb dns name", func() {
+				dnsName = stack.GetLoadBalancerIngressHostName()
+				Expect(dnsName).ToNot(BeEmpty())
+			})
+
+			By("querying AWS loadbalancer from the dns name", func() {
+				var err error
+				lbARN, err = tf.LBManager.FindLoadBalancerByDNSName(ctx, dnsName)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(lbARN).ToNot(BeEmpty())
+			})
+
+			By("verifying AWS loadbalancer resources", func() {
+				nodeList, err := stack.GetWorkerNodes(ctx, tf)
+				Expect(err).ToNot(HaveOccurred())
+				err = verifier.VerifyAWSLoadBalancerResources(ctx, tf, lbARN, verifier.LoadBalancerExpectation{
+					Type:         "application",
+					Scheme:       "internet-facing",
+					TargetType:   "instance",
+					Listeners:    stack.albResourceStack.getListenersPortMap(),
+					TargetGroups: stack.albResourceStack.getTargetGroupNodePortMap(),
+					NumTargets:   len(nodeList),
+					TargetGroupHC: &verifier.TargetGroupHC{
+						Protocol:           "HTTP",
+						Port:               "traffic-port",
+						Path:               "/",
+						Interval:           15,
+						Timeout:            5,
+						HealthyThreshold:   3,
+						UnhealthyThreshold: 3,
+					},
+				})
+				Expect(err).NotTo(HaveOccurred())
+			})
+			By("waiting for target group targets to be healthy", func() {
+				nodeList, err := stack.GetWorkerNodes(ctx, tf)
+				Expect(err).ToNot(HaveOccurred())
+				err = verifier.WaitUntilTargetsAreHealthy(ctx, tf, lbARN, len(nodeList))
+				Expect(err).NotTo(HaveOccurred())
+			})
+			By("waiting until DNS name is available", func() {
+				err := utils.WaitUntilDNSNameAvailable(ctx, dnsName)
+				Expect(err).NotTo(HaveOccurred())
+			})
+			By("sending http request to the lb", func() {
+				url := fmt.Sprintf("http://%v/any-path", dnsName)
+				err := tf.HTTPVerifier.VerifyURL(url, http.ResponseCodeMatches(200))
+				Expect(err).NotTo(HaveOccurred())
+			})
+		})
+	})
+})

test/e2e/gateway/alb_resource_stack.go

@@ -0,0 +1,83 @@
+package gateway
+
+import (
+	"context"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1"
+	"sigs.k8s.io/aws-load-balancer-controller/pkg/k8s"
+	"sigs.k8s.io/aws-load-balancer-controller/test/framework"
+	gwv1 "sigs.k8s.io/gateway-api/apis/v1"
+)
+
+func newALBResourceStack(dp *appsv1.Deployment, svc *corev1.Service, gwc *gwv1.GatewayClass, gw *gwv1.Gateway, lbc *elbv2gw.LoadBalancerConfiguration, httpr *gwv1.HTTPRoute, baseName string, enablePodReadinessGate bool) *albResourceStack {
+
+	commonStack := newCommonResourceStack(dp, svc, gwc, gw, lbc, baseName, enablePodReadinessGate)
+	return &albResourceStack{
+		httpr:       httpr,
+		commonStack: commonStack,
+	}
+}
+
+// resourceStack containing the deployment and service resources
+type albResourceStack struct {
+	commonStack *commonResourceStack
+	httpr       *gwv1.HTTPRoute
+}
+
+func (s *albResourceStack) Deploy(ctx context.Context, f *framework.Framework) error {
+	return s.commonStack.Deploy(ctx, f, func(ctx context.Context, f *framework.Framework, namespace string) error {
+		s.httpr.Namespace = namespace
+		return s.createHTTPRoute(ctx, f)
+	})
+}
+
+func (s *albResourceStack) ScaleDeployment(ctx context.Context, f *framework.Framework, numReplicas int32) error {
+	return s.commonStack.ScaleDeployment(ctx, f, numReplicas)
+}
+
+func (s *albResourceStack) Cleanup(ctx context.Context, f *framework.Framework) {
+	s.commonStack.Cleanup(ctx, f)
+}
+
+func (s *albResourceStack) GetLoadBalancerIngressHostname() string {
+	return s.commonStack.GetLoadBalancerIngressHostname()
+}
+
+func (s *albResourceStack) GetStackName() string {
+	return s.commonStack.GetStackName()
+}
+
+func (s *albResourceStack) getListenersPortMap() map[string]string {
+	return s.commonStack.getListenersPortMap()
+}
+
+func (s *albResourceStack) getTargetGroupNodePortMap() map[string]string {
+	res := s.commonStack.getTargetGroupNodePortMap()
+
+	for p := range res {
+		// TODO - kinda a hack to get HTTP to work.
+		if res[p] == string(corev1.ProtocolTCP) {
+			res[p] = "HTTP"
+		}
+	}
+
+	return res
+}
+
+func (s *albResourceStack) getHealthCheckNodePort() string {
+	return s.commonStack.getHealthCheckNodePort()
+}
+
+func (s *albResourceStack) waitUntilDeploymentReady(ctx context.Context, f *framework.Framework) error {
+	return s.commonStack.waitUntilDeploymentReady(ctx, f)
+}
+
+func (s *albResourceStack) createHTTPRoute(ctx context.Context, f *framework.Framework) error {
+	f.Logger.Info("creating http route", "httpr", k8s.NamespacedName(s.httpr))
+	return f.K8sClient.Create(ctx, s.httpr)
+}
+
+func (s *albResourceStack) deleteHTTPRoute(ctx context.Context, f *framework.Framework) error {
+	return f.K8sClient.Delete(ctx, s.httpr)
+}