Skip to content

Commit a61c695

Browse files
l1b0kl1b0k
authored
Merge pull request #693 from BSWANG/erdma_19
2 parents 325d07e + 7689000 commit a61c695

File tree

8 files changed

+153
-10
lines changed

8 files changed

+153
-10
lines changed

Diff for: ‎daemon/builder.go

+5-5
Original file line numberDiff line numberDiff line change
@@ -275,7 +275,7 @@ func (b *NetworkServiceBuilder) setupENIManager() error {
275275
return err
276276
}
277277
realRdmaCount := b.limit.ERDMARes()
278-
if b.config.EnableERDMA && len(attached) >= b.limit.Adapters-1-b.limit.ERdmaAdapters {
278+
if b.config.EnableERDMA && len(attached) >= b.limit.Adapters-1-b.limit.ERDMARes() {
279279
attachedERdma := lo.Filter(attached, func(ni *daemon.ENI, idx int) bool { return ni.ERdma })
280280
if len(attachedERdma) <= 0 {
281281
// turn off only when no one use it
@@ -359,8 +359,8 @@ func (b *NetworkServiceBuilder) setupENIManager() error {
359359
}
360360
normalENINeeded := poolConfig.MaxENI - normalENICount
361361
if b.config.EnableERDMA {
362-
normalENINeeded = poolConfig.MaxENI - b.limit.ERdmaAdapters - normalENICount
363-
for i := 0; i < b.limit.ERdmaAdapters-erdmaENICount; i++ {
362+
normalENINeeded = poolConfig.MaxENI - b.limit.ERDMARes() - normalENICount
363+
for i := 0; i < b.limit.ERDMARes()-erdmaENICount; i++ {
364364
eniList = append(eniList, eni.NewLocal(nil, "erdma", factory, poolConfig))
365365
}
366366
}
@@ -389,8 +389,8 @@ func (b *NetworkServiceBuilder) setupENIManager() error {
389389
}
390390
normalENINeeded := poolConfig.MaxENI - normalENICount
391391
if b.config.EnableERDMA {
392-
normalENINeeded = poolConfig.MaxENI - b.limit.ERdmaAdapters - normalENICount
393-
for i := 0; i < b.limit.ERdmaAdapters-erdmaENICount; i++ {
392+
normalENINeeded = poolConfig.MaxENI - b.limit.ERDMARes() - normalENICount
393+
for i := 0; i < b.limit.ERDMARes()-erdmaENICount; i++ {
394394
eniList = append(eniList, eni.NewLocal(nil, "erdma", factory, poolConfig))
395395
}
396396
}

Diff for: ‎daemon/config.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,7 @@ func getPoolConfig(cfg *daemon.Config, daemonMode string, limit *client.Limits)
123123

124124
poolConfig.MaxIPPerENI = 1
125125
if cfg.EnableERDMA {
126-
poolConfig.ERdmaCapacity = limit.ERdmaAdapters
126+
poolConfig.ERdmaCapacity = limit.ERDMARes()
127127
}
128128
case daemon.ModeENIMultiIP:
129129
maxENI = limit.Adapters
@@ -161,7 +161,7 @@ func getPoolConfig(cfg *daemon.Config, daemonMode string, limit *client.Limits)
161161
poolConfig.MaxIPPerENI = ipPerENI
162162

163163
if cfg.EnableERDMA {
164-
poolConfig.ERdmaCapacity = limit.ERdmaAdapters * limit.IPv4PerAdapter
164+
poolConfig.ERdmaCapacity = limit.ERDMARes() * limit.IPv4PerAdapter
165165
}
166166
}
167167

Diff for: ‎pkg/aliyun/client/limit.go

+10-1
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,16 @@ func (l *Limits) MultiIPPod() int {
6666
}
6767

6868
func (l *Limits) ERDMARes() int {
69-
return l.ERdmaAdapters
69+
if l.ERdmaAdapters <= 0 || l.Adapters <= 2 {
70+
return 0
71+
}
72+
// limit adapters
73+
if l.Adapters >= 8 {
74+
// for multi physical network card instance
75+
return min(2, l.ERdmaAdapters)
76+
}
77+
// limit normal ecs eri to 1, to avoid too many normal multiip pod quota consume
78+
return min(1, l.ERdmaAdapters)
7079
}
7180

7281
func (l *Limits) ExclusiveENIPod() int {

Diff for: ‎pkg/aliyun/client/limit_test.go

+86
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,92 @@ func TestGetInstanceType(t *testing.T) {
7272
}
7373
}
7474

75+
func TestGetERIRes(t *testing.T) {
76+
tests := []struct {
77+
name string
78+
input *ecs.InstanceType
79+
expected int
80+
}{
81+
{
82+
name: "not support instance type",
83+
input: &ecs.InstanceType{
84+
EniQuantity: 2,
85+
EniPrivateIpAddressQuantity: 5,
86+
EniIpv6AddressQuantity: 10,
87+
EniTotalQuantity: 6,
88+
EriQuantity: 0,
89+
InstanceBandwidthRx: 1000,
90+
InstanceBandwidthTx: 500,
91+
EniTrunkSupported: true,
92+
},
93+
expected: 0,
94+
},
95+
{
96+
name: "Small instance type",
97+
input: &ecs.InstanceType{
98+
EniQuantity: 2,
99+
EniPrivateIpAddressQuantity: 5,
100+
EniIpv6AddressQuantity: 10,
101+
EniTotalQuantity: 6,
102+
EriQuantity: 2,
103+
InstanceBandwidthRx: 1000,
104+
InstanceBandwidthTx: 500,
105+
EniTrunkSupported: true,
106+
},
107+
expected: 0,
108+
},
109+
{
110+
name: "Basic instance type",
111+
input: &ecs.InstanceType{
112+
EniQuantity: 4,
113+
EniPrivateIpAddressQuantity: 5,
114+
EniIpv6AddressQuantity: 10,
115+
EniTotalQuantity: 6,
116+
EriQuantity: 2,
117+
InstanceBandwidthRx: 1000,
118+
InstanceBandwidthTx: 500,
119+
EniTrunkSupported: true,
120+
},
121+
expected: 1,
122+
},
123+
{
124+
name: "giant instance type only one eri",
125+
input: &ecs.InstanceType{
126+
EniQuantity: 8,
127+
EniPrivateIpAddressQuantity: 5,
128+
EniIpv6AddressQuantity: 10,
129+
EniTotalQuantity: 10,
130+
EriQuantity: 1,
131+
InstanceBandwidthRx: 1000,
132+
InstanceBandwidthTx: 500,
133+
EniTrunkSupported: true,
134+
},
135+
expected: 1,
136+
},
137+
{
138+
name: "giant instance type",
139+
input: &ecs.InstanceType{
140+
EniQuantity: 8,
141+
EniPrivateIpAddressQuantity: 5,
142+
EniIpv6AddressQuantity: 10,
143+
EniTotalQuantity: 10,
144+
EriQuantity: 4,
145+
InstanceBandwidthRx: 1000,
146+
InstanceBandwidthTx: 500,
147+
EniTrunkSupported: true,
148+
},
149+
expected: 2,
150+
},
151+
}
152+
153+
for _, tt := range tests {
154+
t.Run(tt.name, func(t *testing.T) {
155+
actual := getInstanceType(tt.input)
156+
assert.Equal(t, tt.expected, actual.ERDMARes())
157+
})
158+
}
159+
}
160+
75161
func TestECSLimitProvider_GetLimitFromAnno(t *testing.T) {
76162

77163
type args struct {

Diff for: ‎pkg/controller/node/node.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,7 @@ func (r *ReconcileNode) createOrUpdate(ctx context.Context, k8sNode *corev1.Node
144144
InstanceBandwidthTx: limit.InstanceBandwidthTx,
145145
InstanceBandwidthRx: limit.InstanceBandwidthRx,
146146
Adapters: limit.Adapters,
147-
EriQuantity: limit.ERdmaAdapters,
147+
EriQuantity: limit.ERDMARes(),
148148
TotalAdapters: limit.TotalAdapters,
149149
IPv6PerAdapter: limit.IPv6PerAdapter,
150150
MemberAdapterLimit: limit.MemberAdapterLimit,

Diff for: ‎plugin/datapath/policy_router_linux.go

+5-1
Original file line numberDiff line numberDiff line change
@@ -351,7 +351,11 @@ func (d *PolicyRoute) Setup(cfg *types.SetupConfig, netNS ns.NetNS) error {
351351
}
352352

353353
if cfg.ERDMA {
354-
err = smc.ConfigSMCForDevice("erdma_0", cfg.ContainerIfName, netNS)
354+
rdmaDev, err := utils.GetERdmaFromLink(eni)
355+
if err != nil {
356+
return fmt.Errorf("error get erdma device: %w", err)
357+
}
358+
err = smc.ConfigSMCForDevice(rdmaDev.Attrs.Name, cfg.ContainerIfName, netNS)
355359
if err != nil {
356360
return fmt.Errorf("error setup pnet config for pod: %w", err)
357361
}

Diff for: ‎plugin/driver/utils/utils_linux.go

+38
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ import (
55
"fmt"
66
"net"
77
"os"
8+
"strconv"
9+
"strings"
810

911
terwayIP "github.com/AliyunContainerService/terway/pkg/ip"
1012
terwaySysctl "github.com/AliyunContainerService/terway/pkg/sysctl"
@@ -844,3 +846,39 @@ func CleanIPRules() (err error) {
844846

845847
return nil
846848
}
849+
850+
func GetERdmaFromLink(link netlink.Link) (*netlink.RdmaLink, error) {
851+
rdmaLinks, err := netlink.RdmaLinkList()
852+
if err != nil {
853+
return nil, fmt.Errorf("error list rdma links, %v", err)
854+
}
855+
for _, rl := range rdmaLinks {
856+
rdmaHwAddr, err := parseERdmaLinkHwAddr(rl.Attrs.NodeGuid)
857+
if err != nil {
858+
return nil, err
859+
}
860+
linkHwAddr := link.Attrs().HardwareAddr
861+
// erdma guid first byte is ^= 0x2
862+
linkHwAddr[0] ^= 0x2
863+
if rdmaHwAddr.String() == linkHwAddr.String() {
864+
return rl, nil
865+
}
866+
}
867+
return nil, fmt.Errorf("cannot found rdma link for %s", link.Attrs().Name)
868+
}
869+
870+
func parseERdmaLinkHwAddr(guid string) (net.HardwareAddr, error) {
871+
hwAddrSlice := make([]byte, 8)
872+
guidSlice := strings.Split(guid, ":")
873+
if len(guidSlice) != 8 {
874+
return nil, fmt.Errorf("invalid rdma guid: %s", guid)
875+
}
876+
for i, s := range guidSlice {
877+
sint, err := strconv.ParseUint(s, 16, 8)
878+
if err != nil {
879+
return nil, fmt.Errorf("invalid rdma guid: %s, err: %v", guid, err)
880+
}
881+
hwAddrSlice[7-i] = uint8(sint)
882+
}
883+
return append(hwAddrSlice[0:3], hwAddrSlice[5:8]...), nil
884+
}

Diff for: ‎plugin/driver/utils/utils_linux_test.go

+6
Original file line numberDiff line numberDiff line change
@@ -293,3 +293,9 @@ var _ = Describe("Test TC filter", func() {
293293
Expect(err).NotTo(HaveOccurred())
294294
})
295295
})
296+
297+
func TestParseERdmaLinkHwAddress(t *testing.T) {
298+
hwaddr, err := parseERdmaLinkHwAddr("0d:d3:04:fe:ff:3e:16:02")
299+
assert.NoError(t, err)
300+
assert.Equal(t, "02:16:3e:04:d3:0d", hwaddr.String())
301+
}

0 commit comments

Comments
 (0)