Skip to content

Commit 3abd2fe

Browse files
mtuliofad3t
mtulio
authored and
fad3t
committed
✨ edge subnets: support Wavelength Zone networks
 Subnets in AWS Wavelength Zone is a classified as a type of edge subnets, not used to create regular control plane resources, like nodes, NAT Gateways or API Load Balancers. The ZoneType is used to group the zones from regular and the edge zones. Regular zones are with type 'availability-zone', and the edge zones are types 'local-zone' and 'wavelength-zone'. The following statements are valid for edge subnets: - private subnets supports egress traffic only using NAT Gateway in the region. - public subnets in Wavelength must be attached to a route table with valid Carrier Gateway as a default route. - public subnets in Wavelength zones does not support map public ip on launch flag, instead, the runInstance must set the network interface flag to assign public ip from carrier gateway - IPv6 subnets is not supported in edge zones - subnet tags for load balancer are not set in edge subnets. Edge subnets should not be elected by CCM to create service load balancers. Use ALB ingress instead
1 parent a75c689 commit 3abd2fe

File tree

2 files changed

+206
-5
lines changed

2 files changed

+206
-5
lines changed

‎pkg/cloud/services/network/subnets.go

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -410,6 +410,9 @@ func (s *Service) describeVpcSubnets() (infrav1.Subnets, error) {
410410
if route.GatewayId != nil && strings.HasPrefix(*route.GatewayId, "igw") {
411411
spec.IsPublic = true
412412
}
413+
if route.CarrierGatewayId != nil && strings.HasPrefix(*route.CarrierGatewayId, "cagw-") {
414+
spec.IsPublic = true
415+
}
413416
}
414417
}
415418

@@ -468,6 +471,8 @@ func (s *Service) createSubnet(sn *infrav1.SubnetSpec) (*infrav1.SubnetSpec, err
468471
// IPv6 subnets are not generally supported by AWS Local Zones and Wavelength Zones.
469472
// Local Zones have limited zone support for IPv6 subnets:
470473
// https://docs.aws.amazon.com/local-zones/latest/ug/how-local-zones-work.html#considerations
474+
// Wavelength Zones is currently not supporting IPv6 subnets.
475+
// https://docs.aws.amazon.com/wavelength/latest/developerguide/wavelength-quotas.html#vpc-considerations
471476
if sn.IsIPv6 && sn.IsEdge() {
472477
err := fmt.Errorf("failed to create subnet: IPv6 is not supported with zone type %q", sn.ZoneType)
473478
record.Warnf(s.scope.InfraCluster(), "FailedCreateSubnet", "Failed creating managed Subnet for edge zones: %v", err)
@@ -526,7 +531,12 @@ func (s *Service) createSubnet(sn *infrav1.SubnetSpec) (*infrav1.SubnetSpec, err
526531
record.Eventf(s.scope.InfraCluster(), "SuccessfulModifySubnetAttributes", "Modified managed Subnet %q attributes", *out.Subnet.SubnetId)
527532
}
528533

529-
if sn.IsPublic {
534+
// AWS Wavelength Zone's public subnets does not support to map Carrier IP address on launch, and
535+
// MapPublicIpOnLaunch option[1] set to the subnet will fail, instead set the EC2 instance's network
536+
// interface to associate Carrier IP Address on launch[2].
537+
// [1] https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_ModifySubnetAttribute.html
538+
// [2] https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_InstanceNetworkInterfaceSpecification.html
539+
if sn.IsPublic && !sn.IsEdgeWavelength() {
530540
if err := wait.WaitForWithRetryable(wait.NewBackoff(), func() (bool, error) {
531541
if _, err := s.EC2Client.ModifySubnetAttributeWithContext(context.TODO(), &ec2.ModifySubnetAttributeInput{
532542
SubnetId: out.Subnet.SubnetId,

‎pkg/cloud/services/network/subnets_test.go

Lines changed: 195 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -59,9 +59,14 @@ func TestReconcileSubnets(t *testing.T) {
5959
{ID: "subnet-private-us-east-1-nyc-1a", AvailabilityZone: "us-east-1-nyc-1a", CidrBlock: "10.0.5.0/24", IsPublic: false},
6060
{ID: "subnet-public-us-east-1-nyc-1a", AvailabilityZone: "us-east-1-nyc-1a", CidrBlock: "10.0.6.0/24", IsPublic: true},
6161
}
62+
stubSubnetsWavelengthZone := []infrav1.SubnetSpec{
63+
{ID: "subnet-private-us-east-1-wl1-nyc-wlz-1", AvailabilityZone: "us-east-1-wl1-nyc-wlz-1", CidrBlock: "10.0.7.0/24", IsPublic: false},
64+
{ID: "subnet-public-us-east-1-wl1-nyc-wlz-1", AvailabilityZone: "us-east-1-wl1-nyc-wlz-1", CidrBlock: "10.0.8.0/24", IsPublic: true},
65+
}
6266
// TODO(mtulio): replace by slices.Concat(...) on go 1.22+
6367
stubSubnetsAllZones := stubSubnetsAvailabilityZone
6468
stubSubnetsAllZones = append(stubSubnetsAllZones, stubSubnetsLocalZone...)
69+
stubSubnetsAllZones = append(stubSubnetsAllZones, stubSubnetsWavelengthZone...)
6570

6671
// NetworkSpec with subnets in zone type availability-zone
6772
stubNetworkSpecWithSubnets := &infrav1.NetworkSpec{
@@ -655,7 +660,7 @@ func TestReconcileSubnets(t *testing.T) {
655660
tagUnmanagedNetworkResources: true,
656661
},
657662
{
658-
name: "Unmanaged VPC, 2 existing matching subnets, subnet tagging fails with subnet update, should succeed",
663+
name: "Unmanaged VPC, one existing matching subnets, subnet tagging fails with subnet update, should succeed",
659664
input: NewClusterScope().WithNetwork(&infrav1.NetworkSpec{
660665
VPC: infrav1.VPCSpec{
661666
ID: subnetsVPCID,
@@ -767,6 +772,9 @@ func TestReconcileSubnets(t *testing.T) {
767772
{
768773
ID: "subnet-1",
769774
},
775+
{
776+
ID: "subnet-2",
777+
},
770778
},
771779
}).WithTagUnmanagedNetworkResources(true),
772780
optionalExpectSubnets: infrav1.Subnets{
@@ -778,6 +786,14 @@ func TestReconcileSubnets(t *testing.T) {
778786
IsPublic: true,
779787
Tags: infrav1.Tags{},
780788
},
789+
{
790+
ID: "subnet-2",
791+
ResourceID: "subnet-2",
792+
AvailabilityZone: "us-east-1b",
793+
CidrBlock: "10.0.11.0/24",
794+
IsPublic: true,
795+
Tags: infrav1.Tags{},
796+
},
781797
},
782798
expect: func(m *mocks.MockEC2APIMockRecorder) {
783799
m.DescribeSubnetsWithContext(context.TODO(), gomock.Eq(&ec2.DescribeSubnetsInput{
@@ -799,6 +815,13 @@ func TestReconcileSubnets(t *testing.T) {
799815
SubnetId: aws.String("subnet-1"),
800816
AvailabilityZone: aws.String("us-east-1a"),
801817
CidrBlock: aws.String("10.0.10.0/24"),
818+
MapPublicIpOnLaunch: aws.Bool(true),
819+
},
820+
{
821+
VpcId: aws.String(subnetsVPCID),
822+
SubnetId: aws.String("subnet-2"),
823+
AvailabilityZone: aws.String("us-east-1b"),
824+
CidrBlock: aws.String("10.0.11.0/24"),
802825
MapPublicIpOnLaunch: aws.Bool(false),
803826
},
804827
},
@@ -821,6 +844,20 @@ func TestReconcileSubnets(t *testing.T) {
821844
},
822845
},
823846
},
847+
{
848+
VpcId: aws.String(subnetsVPCID),
849+
Associations: []*ec2.RouteTableAssociation{
850+
{
851+
SubnetId: aws.String("subnet-2"),
852+
RouteTableId: aws.String("rt-00000"),
853+
},
854+
},
855+
Routes: []*ec2.Route{
856+
{
857+
GatewayId: aws.String("igw-12345"),
858+
},
859+
},
860+
},
824861
},
825862
}, nil)
826863

@@ -840,10 +877,10 @@ func TestReconcileSubnets(t *testing.T) {
840877
gomock.Any()).Return(nil)
841878

842879
stubMockDescribeAvailabilityZonesWithContextCustomZones(m, []*ec2.AvailabilityZone{
843-
{ZoneName: aws.String("us-east-1a")},
880+
{ZoneName: aws.String("us-east-1a")}, {ZoneName: aws.String("us-east-1b")},
844881
}).AnyTimes()
845882

846-
m.CreateTagsWithContext(context.TODO(), gomock.Eq(&ec2.CreateTagsInput{
883+
subnet1tag := m.CreateTagsWithContext(context.TODO(), gomock.Eq(&ec2.CreateTagsInput{
847884
Resources: aws.StringSlice([]string{"subnet-1"}),
848885
Tags: []*ec2.Tag{
849886
{
@@ -857,6 +894,21 @@ func TestReconcileSubnets(t *testing.T) {
857894
},
858895
})).
859896
Return(&ec2.CreateTagsOutput{}, fmt.Errorf("tagging failed"))
897+
898+
m.CreateTagsWithContext(context.TODO(), gomock.Eq(&ec2.CreateTagsInput{
899+
Resources: aws.StringSlice([]string{"subnet-2"}),
900+
Tags: []*ec2.Tag{
901+
{
902+
Key: aws.String("kubernetes.io/cluster/test-cluster"),
903+
Value: aws.String("shared"),
904+
},
905+
{
906+
Key: aws.String("kubernetes.io/role/elb"),
907+
Value: aws.String("1"),
908+
},
909+
},
910+
})).
911+
Return(&ec2.CreateTagsOutput{}, fmt.Errorf("tagging failed")).After(subnet1tag)
860912
},
861913
tagUnmanagedNetworkResources: true,
862914
},
@@ -975,6 +1027,11 @@ func TestReconcileSubnets(t *testing.T) {
9751027
})).
9761028
Return(&ec2.CreateTagsOutput{}, nil)
9771029

1030+
stubMockDescribeAvailabilityZonesWithContextCustomZones(m, []*ec2.AvailabilityZone{
1031+
{ZoneName: aws.String("us-east-1a"), ZoneType: aws.String("availability-zone")},
1032+
{ZoneName: aws.String("us-east-1b"), ZoneType: aws.String("availability-zone")},
1033+
}).AnyTimes()
1034+
9781035
m.CreateTagsWithContext(context.TODO(), gomock.Eq(&ec2.CreateTagsInput{
9791036
Resources: aws.StringSlice([]string{"subnet-2"}),
9801037
Tags: []*ec2.Tag{
@@ -2873,6 +2930,7 @@ func TestReconcileSubnets(t *testing.T) {
28732930
{ZoneName: aws.String("us-east-1a"), ZoneType: aws.String("availability-zone")},
28742931
{ZoneName: aws.String("us-east-1b"), ZoneType: aws.String("availability-zone")},
28752932
{ZoneName: aws.String("us-east-1-nyc-1a"), ZoneType: aws.String("local-zone"), ParentZoneName: aws.String("us-east-1a")},
2933+
{ZoneName: aws.String("us-east-1-wl1-nyc-wlz-1"), ZoneType: aws.String("wavelength-zone"), ParentZoneName: aws.String("us-east-1a")},
28762934
}).AnyTimes()
28772935

28782936
m.WaitUntilSubnetAvailableWithContext(context.TODO(), gomock.Any()).AnyTimes()
@@ -2928,6 +2986,12 @@ func TestReconcileSubnets(t *testing.T) {
29282986

29292987
lz1Public := stubGenMockCreateSubnetWithContext(m, "test-cluster", "us-east-1-nyc-1a", "public", "10.0.6.0/24", true).After(lz1Private)
29302988
stubMockModifySubnetAttributeWithContext(m, "subnet-public-us-east-1-nyc-1a").After(lz1Public)
2989+
2990+
// Wavelength zone nyc-1.
2991+
wz1Private := stubGenMockCreateSubnetWithContext(m, "test-cluster", "us-east-1-wl1-nyc-wlz-1", "private", "10.0.7.0/24", true).
2992+
After(describeCall)
2993+
2994+
stubGenMockCreateSubnetWithContext(m, "test-cluster", "us-east-1-wl1-nyc-wlz-1", "public", "10.0.8.0/24", true).After(wz1Private)
29312995
},
29322996
},
29332997
{
@@ -2990,14 +3054,21 @@ func TestReconcileSubnets(t *testing.T) {
29903054
{ResourceID: "subnet-az-1a-public"},
29913055
{ResourceID: "subnet-lz-1a-private"},
29923056
{ResourceID: "subnet-lz-1a-public"},
3057+
{ResourceID: "subnet-wl-1a-private"},
3058+
{ResourceID: "subnet-wl-1a-public"},
29933059
}
29943060
return NewClusterScope().WithNetwork(net)
29953061
}(),
29963062
expect: func(m *mocks.MockEC2APIMockRecorder) {
29973063
stubMockDescribeSubnetsWithContextUnmanaged(m)
29983064
stubMockDescribeAvailabilityZonesWithContextAllZones(m)
2999-
stubMockDescribeRouteTablesWithContext(m)
3065+
stubMockDescribeRouteTablesWithContextWithWavelength(m,
3066+
[]string{"subnet-az-1a-private", "subnet-lz-1a-private", "subnet-wl-1a-private"},
3067+
[]string{"subnet-az-1a-public", "subnet-lz-1a-public"},
3068+
[]string{"subnet-wl-1a-public"})
3069+
30003070
stubMockDescribeNatGatewaysPagesWithContext(m)
3071+
stubMockCreateTagsWithContext(m, "test-cluster", "subnet-az-1a-private", "us-east-1a", "private", false).AnyTimes()
30013072
},
30023073
},
30033074
}
@@ -3601,6 +3672,41 @@ func TestService_retrieveZoneInfo(t *testing.T) {
36013672
},
36023673
},
36033674
},
3675+
{
3676+
name: "get type wavelength zones",
3677+
inputZoneNames: []string{"us-east-1-wl1-nyc-wlz-1", "us-east-1-wl1-bos-wlz-1"},
3678+
expect: func(m *mocks.MockEC2APIMockRecorder) {
3679+
m.DescribeAvailabilityZonesWithContext(context.TODO(), &ec2.DescribeAvailabilityZonesInput{
3680+
ZoneNames: aws.StringSlice([]string{"us-east-1-wl1-nyc-wlz-1", "us-east-1-wl1-bos-wlz-1"}),
3681+
}).
3682+
Return(&ec2.DescribeAvailabilityZonesOutput{
3683+
AvailabilityZones: []*ec2.AvailabilityZone{
3684+
{
3685+
ZoneName: aws.String("us-east-1-wl1-nyc-wlz-1"),
3686+
ZoneType: aws.String("wavelength-zone"),
3687+
ParentZoneName: aws.String("us-east-1a"),
3688+
},
3689+
{
3690+
ZoneName: aws.String("us-east-1-wl1-bos-wlz-1"),
3691+
ZoneType: aws.String("wavelength-zone"),
3692+
ParentZoneName: aws.String("us-east-1b"),
3693+
},
3694+
},
3695+
}, nil)
3696+
},
3697+
want: []*ec2.AvailabilityZone{
3698+
{
3699+
ZoneName: aws.String("us-east-1-wl1-nyc-wlz-1"),
3700+
ZoneType: aws.String("wavelength-zone"),
3701+
ParentZoneName: aws.String("us-east-1a"),
3702+
},
3703+
{
3704+
ZoneName: aws.String("us-east-1-wl1-bos-wlz-1"),
3705+
ZoneType: aws.String("wavelength-zone"),
3706+
ParentZoneName: aws.String("us-east-1b"),
3707+
},
3708+
},
3709+
},
36043710
{
36053711
name: "get all zone types",
36063712
inputZoneNames: []string{"us-east-1a", "us-east-1-nyc-1a", "us-east-1-wl1-nyc-wlz-1"},
@@ -3620,6 +3726,11 @@ func TestService_retrieveZoneInfo(t *testing.T) {
36203726
ZoneType: aws.String("local-zone"),
36213727
ParentZoneName: aws.String("us-east-1a"),
36223728
},
3729+
{
3730+
ZoneName: aws.String("us-east-1-wl1-nyc-wlz-1"),
3731+
ZoneType: aws.String("wavelength-zone"),
3732+
ParentZoneName: aws.String("us-east-1a"),
3733+
},
36233734
},
36243735
}, nil)
36253736
},
@@ -3634,6 +3745,11 @@ func TestService_retrieveZoneInfo(t *testing.T) {
36343745
ZoneType: aws.String("local-zone"),
36353746
ParentZoneName: aws.String("us-east-1a"),
36363747
},
3748+
{
3749+
ZoneName: aws.String("us-east-1-wl1-nyc-wlz-1"),
3750+
ZoneType: aws.String("wavelength-zone"),
3751+
ParentZoneName: aws.String("us-east-1a"),
3752+
},
36373753
},
36383754
},
36393755
}
@@ -3732,11 +3848,79 @@ func stubGenMockCreateSubnetWithContext(m *mocks.MockEC2APIMockRecorder, prefix,
37323848
}, nil)
37333849
}
37343850

3851+
func stubMockCreateTagsWithContext(m *mocks.MockEC2APIMockRecorder, prefix, name, zone, role string, isEdge bool) *gomock.Call {
3852+
return m.CreateTagsWithContext(context.TODO(), gomock.Eq(&ec2.CreateTagsInput{
3853+
Resources: aws.StringSlice([]string{name}),
3854+
Tags: stubGetTags(prefix, role, zone, isEdge),
3855+
})).
3856+
Return(&ec2.CreateTagsOutput{}, nil)
3857+
}
3858+
37353859
func stubMockDescribeRouteTablesWithContext(m *mocks.MockEC2APIMockRecorder) {
37363860
m.DescribeRouteTablesWithContext(context.TODO(), gomock.AssignableToTypeOf(&ec2.DescribeRouteTablesInput{})).
37373861
Return(&ec2.DescribeRouteTablesOutput{}, nil)
37383862
}
37393863

3864+
func stubMockDescribeRouteTablesWithContextWithWavelength(m *mocks.MockEC2APIMockRecorder, privSubnets, pubSubnetsIGW, pubSubnetsCarrier []string) *gomock.Call {
3865+
routes := []*ec2.RouteTable{}
3866+
3867+
// create public route table
3868+
pubTable := &ec2.RouteTable{
3869+
Routes: []*ec2.Route{
3870+
{
3871+
DestinationCidrBlock: aws.String("0.0.0.0/0"),
3872+
GatewayId: aws.String("igw-0"),
3873+
},
3874+
},
3875+
RouteTableId: aws.String("rtb-public"),
3876+
}
3877+
for _, sub := range pubSubnetsIGW {
3878+
pubTable.Associations = append(pubTable.Associations, &ec2.RouteTableAssociation{
3879+
SubnetId: aws.String(sub),
3880+
})
3881+
}
3882+
routes = append(routes, pubTable)
3883+
3884+
// create public carrier route table
3885+
pubCarrierTable := &ec2.RouteTable{
3886+
Routes: []*ec2.Route{
3887+
{
3888+
DestinationCidrBlock: aws.String("0.0.0.0/0"),
3889+
CarrierGatewayId: aws.String("cagw-0"),
3890+
},
3891+
},
3892+
RouteTableId: aws.String("rtb-carrier"),
3893+
}
3894+
for _, sub := range pubSubnetsCarrier {
3895+
pubCarrierTable.Associations = append(pubCarrierTable.Associations, &ec2.RouteTableAssociation{
3896+
SubnetId: aws.String(sub),
3897+
})
3898+
}
3899+
routes = append(routes, pubCarrierTable)
3900+
3901+
// create private route table
3902+
privTable := &ec2.RouteTable{
3903+
Routes: []*ec2.Route{
3904+
{
3905+
DestinationCidrBlock: aws.String("10.0.11.0/24"),
3906+
GatewayId: aws.String("vpc-natgw-1a"),
3907+
},
3908+
},
3909+
RouteTableId: aws.String("rtb-private"),
3910+
}
3911+
for _, sub := range privSubnets {
3912+
privTable.Associations = append(privTable.Associations, &ec2.RouteTableAssociation{
3913+
SubnetId: aws.String(sub),
3914+
})
3915+
}
3916+
routes = append(routes, privTable)
3917+
3918+
return m.DescribeRouteTablesWithContext(context.TODO(), gomock.AssignableToTypeOf(&ec2.DescribeRouteTablesInput{})).
3919+
Return(&ec2.DescribeRouteTablesOutput{
3920+
RouteTables: routes,
3921+
}, nil)
3922+
}
3923+
37403924
func stubMockDescribeSubnetsWithContext(m *mocks.MockEC2APIMockRecorder, out *ec2.DescribeSubnetsOutput, filterKey, filterValue string) *gomock.Call {
37413925
return m.DescribeSubnetsWithContext(context.TODO(), gomock.Eq(&ec2.DescribeSubnetsInput{
37423926
Filters: []*ec2.Filter{
@@ -3760,6 +3944,8 @@ func stubMockDescribeSubnetsWithContextUnmanaged(m *mocks.MockEC2APIMockRecorder
37603944
{SubnetId: aws.String("subnet-az-1a-public"), AvailabilityZone: aws.String("us-east-1a")},
37613945
{SubnetId: aws.String("subnet-lz-1a-private"), AvailabilityZone: aws.String("us-east-1-nyc-1a")},
37623946
{SubnetId: aws.String("subnet-lz-1a-public"), AvailabilityZone: aws.String("us-east-1-nyc-1a")},
3947+
{SubnetId: aws.String("subnet-wl-1a-private"), AvailabilityZone: aws.String("us-east-1-wl1-nyc-wlz-1")},
3948+
{SubnetId: aws.String("subnet-wl-1a-public"), AvailabilityZone: aws.String("us-east-1-wl1-nyc-wlz-1")},
37633949
},
37643950
}, "vpc-id", subnetsVPCID)
37653951
}
@@ -3801,6 +3987,11 @@ func stubMockDescribeAvailabilityZonesWithContextAllZones(m *mocks.MockEC2APIMoc
38013987
ZoneType: aws.String("local-zone"),
38023988
ParentZoneName: aws.String("us-east-1a"),
38033989
},
3990+
{
3991+
ZoneName: aws.String("us-east-1-wl1-nyc-wlz-1"),
3992+
ZoneType: aws.String("wavelength-zone"),
3993+
ParentZoneName: aws.String("us-east-1a"),
3994+
},
38043995
},
38053996
}, nil).AnyTimes()
38063997
}

0 commit comments

Comments
 (0)