✨ edge subnets/gateway: carrier gateway routing for Wavelength subnets

✨ edge subnets/cagw: carrier gateway for public subnets in Wavelength

Introduce Carrier Gateway resource reconciliator in the network service.

Carrier Gateway is the gateway responsible to route ingress and egress
traffic **in/out the Wavelength Zone**, located in the Carrier
Infrastructure - communications service providers’ (CSP) 5G networks.

Carrier Gateway is similar Internet Gatewat resource, responsible for
the network border groups in the Region and Local Zones for public
subnets.

✨ edge subnets/routes: supporting custom routes for Wavelength

For private and public subnets in edge zones, the following changes is
introduced according to each rule:

General:

- IPv6 subnets is not be supported in AWS Local Zones and Wavelength
  zone, consequently no ip6 routes will be created
- nat gateways is not supported, default gateway's route for private
  subnets will use nat gateways from the zones in the Region
(availability-zone's zone type)
- one route table by zone's role by zone (standard flow)

Private tables for Local Zones and Wavelength:
- default route's gateways is assigned using nat gateway created in
  the region (availability-zones).

Public tables for Wavelength zones:
- default route's gateways is assigned using Carrier Gateway, resource
  introduced in the edge zone's feature.

The changes in the standard flow (without edge subnets' support) was
isolated in the PR #4900

Author: mtulio

api/v1beta2/conditions_consts.go

@@ -69,6 +69,14 @@ const (
 	EgressOnlyInternetGatewayFailedReason = "EgressOnlyInternetGatewayFailed"
 )
 
+const (
+	// CarrierGatewayReadyCondition reports on the successful reconciliation of carrier gateways.
+	// Only applicable to managed clusters.
+	CarrierGatewayReadyCondition clusterv1.ConditionType = "CarrierGatewayReady"
+	// CarrierGatewayFailedReason used when errors occur during carrier gateway reconciliation.
+	CarrierGatewayFailedReason = "CarrierGatewayFailed"
+)
+
 const (
 	// NatGatewaysReadyCondition reports successful reconciliation of NAT gateways.
 	// Only applicable to managed clusters.

pkg/cloud/awserrors/errors.go

@@ -33,6 +33,7 @@ const (
 	GatewayNotFound                   = "InvalidGatewayID.NotFound"
 	GroupNotFound                     = "InvalidGroup.NotFound"
 	InternetGatewayNotFound           = "InvalidInternetGatewayID.NotFound"
+	InvalidCarrierGatewayNotFound     = "InvalidCarrierGatewayID.NotFound"
 	EgressOnlyInternetGatewayNotFound = "InvalidEgressOnlyInternetGatewayID.NotFound"
 	InUseIPAddress                    = "InvalidIPAddress.InUse"
 	InvalidAccessKeyID                = "InvalidAccessKeyId"

pkg/cloud/services/network/carriergateways.go

@@ -0,0 +1,145 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package network
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/pkg/errors"
+
+	infrav1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2"
+	"sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/awserrors"
+	"sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/converters"
+	"sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/filter"
+	"sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services"
+	"sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/wait"
+	"sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/tags"
+	"sigs.k8s.io/cluster-api-provider-aws/v2/pkg/record"
+	"sigs.k8s.io/cluster-api/util/conditions"
+)
+
+func (s *Service) reconcileCarrierGateway() error {
+	if s.scope.VPC().IsUnmanaged(s.scope.Name()) {
+		s.scope.Trace("Skipping carrier gateway reconcile in unmanaged mode")
+		return nil
+	}
+
+	if !s.scope.Subnets().HasPublicSubnetWavelength() {
+		s.scope.Trace("Skipping carrier gateway reconcile in VPC without subnets in zone type wavelength-zone")
+		return nil
+	}
+
+	s.scope.Debug("Reconciling carrier gateway")
+
+	cagw, err := s.describeVpcCarrierGateway()
+	if awserrors.IsNotFound(err) {
+		if s.scope.VPC().IsUnmanaged(s.scope.Name()) {
+			return errors.Errorf("failed to validate network: no carrier gateway found in VPC %q", s.scope.VPC().ID)
+		}
+
+		cg, err := s.createCarrierGateway()
+		if err != nil {
+			return err
+		}
+		cagw = cg
+	} else if err != nil {
+		return err
+	}
+
+	s.scope.VPC().CarrierGatewayID = cagw.CarrierGatewayId
+
+	// Make sure tags are up-to-date.
+	if err := wait.WaitForWithRetryable(wait.NewBackoff(), func() (bool, error) {
+		buildParams := s.getGatewayTagParams(*cagw.CarrierGatewayId)
+		tagsBuilder := tags.New(&buildParams, tags.WithEC2(s.EC2Client))
+		if err := tagsBuilder.Ensure(converters.TagsToMap(cagw.Tags)); err != nil {
+			return false, err
+		}
+		return true, nil
+	}, awserrors.InvalidCarrierGatewayNotFound); err != nil {
+		record.Warnf(s.scope.InfraCluster(), "FailedTagCarrierGateway", "Failed to tag managed Carrier Gateway %q: %v", cagw.CarrierGatewayId, err)
+		return errors.Wrapf(err, "failed to tag carrier gateway %q", *cagw.CarrierGatewayId)
+	}
+	conditions.MarkTrue(s.scope.InfraCluster(), infrav1.CarrierGatewayReadyCondition)
+	return nil
+}
+
+func (s *Service) deleteCarrierGateway() error {
+	if s.scope.VPC().IsUnmanaged(s.scope.Name()) {
+		s.scope.Trace("Skipping carrier gateway deletion in unmanaged mode")
+		return nil
+	}
+
+	cagw, err := s.describeVpcCarrierGateway()
+	if awserrors.IsNotFound(err) {
+		return nil
+	} else if err != nil {
+		return err
+	}
+
+	deleteReq := &ec2.DeleteCarrierGatewayInput{
+		CarrierGatewayId: cagw.CarrierGatewayId,
+	}
+
+	if _, err = s.EC2Client.DeleteCarrierGatewayWithContext(context.TODO(), deleteReq); err != nil {
+		record.Warnf(s.scope.InfraCluster(), "FailedDeleteCarrierGateway", "Failed to delete Carrier Gateway %q previously attached to VPC %q: %v", *cagw.CarrierGatewayId, s.scope.VPC().ID, err)
+		return errors.Wrapf(err, "failed to delete carrier gateway %q", *cagw.CarrierGatewayId)
+	}
+
+	record.Eventf(s.scope.InfraCluster(), "SuccessfulDeleteCarrierGateway", "Deleted Carrier Gateway %q previously attached to VPC %q", *cagw.CarrierGatewayId, s.scope.VPC().ID)
+	s.scope.Info("Deleted Carrier Gateway in VPC", "carrier-gateway-id", *cagw.CarrierGatewayId, "vpc-id", s.scope.VPC().ID)
+
+	return nil
+}
+
+func (s *Service) createCarrierGateway() (*ec2.CarrierGateway, error) {
+	ig, err := s.EC2Client.CreateCarrierGatewayWithContext(context.TODO(), &ec2.CreateCarrierGatewayInput{
+		VpcId: aws.String(s.scope.VPC().ID),
+		TagSpecifications: []*ec2.TagSpecification{
+			tags.BuildParamsToTagSpecification(ec2.ResourceTypeCarrierGateway, s.getGatewayTagParams(services.TemporaryResourceID)),
+		},
+	})
+	if err != nil {
+		record.Warnf(s.scope.InfraCluster(), "FailedCreateCarrierGateway", "Failed to create new managed Internet Gateway: %v", err)
+		return nil, errors.Wrap(err, "failed to create carrier gateway")
+	}
+	record.Eventf(s.scope.InfraCluster(), "SuccessfulCreateCarrierGateway", "Created new managed Internet Gateway %q", *ig.CarrierGateway.CarrierGatewayId)
+	s.scope.Info("Created Internet gateway for VPC", "internet-gateway-id", *ig.CarrierGateway.CarrierGatewayId, "vpc-id", s.scope.VPC().ID)
+
+	return ig.CarrierGateway, nil
+}
+
+func (s *Service) describeVpcCarrierGateway() (*ec2.CarrierGateway, error) {
+	out, err := s.EC2Client.DescribeCarrierGatewaysWithContext(context.TODO(), &ec2.DescribeCarrierGatewaysInput{
+		Filters: []*ec2.Filter{
+			filter.EC2.VPC(s.scope.VPC().ID),
+		},
+	})
+	if err != nil {
+		record.Eventf(s.scope.InfraCluster(), "FailedDescribeCarrierGateway", "Failed to describe carrier gateways in vpc %q: %v", s.scope.VPC().ID, err)
+		return nil, errors.Wrapf(err, "failed to describe carrier gateways in vpc %q", s.scope.VPC().ID)
+	}
+
+	if len(out.CarrierGateways) == 0 {
+		return nil, awserrors.NewNotFound(fmt.Sprintf("no carrier gateways found in vpc %q", s.scope.VPC().ID))
+	}
+
+	return out.CarrierGateways[0], nil
+}