Skip to content

Commit b408c2f

Browse files
BSWANGBSWANG
authored
Merge pull request #706 from l1b0k/feat/update_gc
enable gc for crd ipam
2 parents 542a887 + 06b3c20 commit b408c2f

File tree

16 files changed

+143
-50
lines changed

16 files changed

+143
-50
lines changed

‎daemon/builder.go

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -400,7 +400,10 @@ func (b *NetworkServiceBuilder) PostInitForCRDV2() *NetworkServiceBuilder {
400400
crdv2 := eni.NewCRDV2(b.service.k8s.NodeName())
401401
mgr := eni.NewManager(0, 0, 0, 0, []eni.NetworkInterface{crdv2}, types.EniSelectionPolicy(b.config.EniSelectionPolicy), nil)
402402

403-
return b.RunENIMgr(b.ctx, mgr)
403+
svc := b.RunENIMgr(b.ctx, mgr)
404+
go b.service.startGarbageCollectionLoop(b.ctx)
405+
406+
return svc
404407
}
405408

406409
func (b *NetworkServiceBuilder) InitResourceDB() *NetworkServiceBuilder {

‎daemon/daemon.go

Lines changed: 44 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
"sync"
1212
"time"
1313

14+
"github.com/go-logr/logr"
1415
"k8s.io/apimachinery/pkg/util/sets"
1516
logf "sigs.k8s.io/controller-runtime/pkg/log"
1617

@@ -135,8 +136,8 @@ func (n *networkService) AllocIP(ctx context.Context, r *rpc.AllocIPRequest) (*r
135136
}
136137
}()
137138

138-
// 0. Get pod Info
139-
pod, err := n.k8s.GetPod(ctx, r.K8SPodNamespace, r.K8SPodName, true)
139+
// 0. Get pod Info, change the req to no cache , we want to get the exact pod uid
140+
pod, err := n.k8s.GetPod(ctx, r.K8SPodNamespace, r.K8SPodName, false)
140141
if err != nil {
141142
return nil, &types.Error{
142143
Code: types.ErrInvalidArgsErrCode,
@@ -352,7 +353,11 @@ func (n *networkService) ReleaseIP(ctx context.Context, r *rpc.ReleaseIPRequest)
352353
return reply, nil
353354
}
354355
}
355-
if pod.IPStickTime == 0 {
356+
if oldRes.PodInfo != nil && oldRes.PodInfo.PodUID != "" {
357+
cni.PodUID = oldRes.PodInfo.PodUID
358+
}
359+
360+
if n.ipamType == types.IPAMTypeCRD || pod.IPStickTime == 0 {
356361
for _, resource := range oldRes.Resources {
357362
res := parseNetworkResource(resource)
358363
if res == nil {
@@ -510,6 +515,8 @@ func (n *networkService) gcPods(ctx context.Context) error {
510515
n.Lock()
511516
defer n.Unlock()
512517

518+
serviceLog.V(4).WithName("gc").Info("gcPods")
519+
513520
pods, err := n.k8s.GetLocalPods()
514521
if err != nil {
515522
return err
@@ -540,18 +547,23 @@ func (n *networkService) gcPods(ctx context.Context) error {
540547
}
541548
}
542549

550+
if serviceLog.V(4).Enabled() {
551+
serviceLog.Info("pod res", "pod", podRes)
552+
}
553+
543554
podID := utils.PodInfoKey(podRes.PodInfo.Namespace, podRes.PodInfo.Name)
544555
if _, ok := exist[podID]; ok {
545556
continue
546557
}
558+
547559
// check kube-api again
548560
ok, err := n.k8s.PodExist(podRes.PodInfo.Namespace, podRes.PodInfo.Name)
549561
if err != nil || ok {
550562
continue
551563
}
552564

553565
// that is old logic ... keep it
554-
if podRes.PodInfo.IPStickTime != 0 {
566+
if n.ipamType != types.IPAMTypeCRD && podRes.PodInfo.IPStickTime != 0 {
555567
podRes.PodInfo.IPStickTime = 0
556568

557569
err = n.resourceDB.Put(podID, podRes)
@@ -566,6 +578,34 @@ func (n *networkService) gcPods(ctx context.Context) error {
566578
if res == nil {
567579
continue
568580
}
581+
// clean up rules
582+
switch res.ResourceType() {
583+
case eni.ResourceTypeLocalIP, eni.ResourceTypeRemoteIP, eni.ResourceTypeRDMA:
584+
585+
for _, v := range res.ToRPC() {
586+
if v.ENIInfo == nil ||
587+
v.BasicInfo == nil ||
588+
v.BasicInfo.PodIP == nil {
589+
continue
590+
}
591+
592+
containerIP := &types.IPNetSet{}
593+
if v.BasicInfo.PodIP.IPv4 != "" {
594+
containerIP.SetIPNet(v.BasicInfo.PodIP.IPv4 + "/32")
595+
}
596+
597+
if v.BasicInfo.PodIP.IPv6 != "" {
598+
containerIP.SetIPNet(v.BasicInfo.PodIP.IPv6 + "/128")
599+
}
600+
601+
ctx = logr.NewContext(ctx, serviceLog)
602+
err = gcPolicyRoutes(ctx, v.ENIInfo.MAC, containerIP, podRes.PodInfo.Namespace, podRes.PodInfo.Name)
603+
if err != nil {
604+
return err
605+
}
606+
}
607+
}
608+
569609
err = n.eniMgr.Release(ctx, &daemon.CNI{
570610
PodName: podRes.PodInfo.Name,
571611
PodNamespace: podRes.PodInfo.Namespace,

‎daemon/daemon_linux.go

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,38 @@
11
package daemon
22

33
import (
4+
"context"
45
"encoding/binary"
56
"net"
67

78
"github.com/samber/lo"
89
"github.com/vishvananda/netlink"
910
"k8s.io/apimachinery/pkg/util/sets"
11+
12+
"github.com/AliyunContainerService/terway/pkg/link"
13+
"github.com/AliyunContainerService/terway/plugin/datapath"
14+
cnitypes "github.com/AliyunContainerService/terway/plugin/driver/types"
15+
"github.com/AliyunContainerService/terway/types"
1016
)
1117

18+
func gcPolicyRoutes(ctx context.Context, mac string, containerIPNet *types.IPNetSet, namespace, name string) error {
19+
index, err := link.GetDeviceNumber(mac)
20+
if err != nil {
21+
if _, ok := err.(netlink.LinkNotFoundError); ok {
22+
return nil
23+
}
24+
return err
25+
}
26+
vethName, _ := link.VethNameForPod(name, namespace, "", "cali")
27+
p := &datapath.PolicyRoute{}
28+
err = p.Teardown(ctx, &cnitypes.TeardownCfg{
29+
HostVETHName: vethName,
30+
ContainerIPNet: containerIPNet,
31+
ENIIndex: int(index),
32+
}, nil)
33+
return err
34+
}
35+
1236
func gcLeakedRules(existIP sets.Set[string]) {
1337
links, err := netlink.LinkList()
1438
if err != nil {

‎daemon/daemon_unsupported.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,15 @@
33
package daemon
44

55
import (
6+
"context"
7+
68
"k8s.io/apimachinery/pkg/util/sets"
9+
10+
"github.com/AliyunContainerService/terway/types"
711
)
812

913
func gcLeakedRules(existIP sets.Set[string]) {}
14+
15+
func gcPolicyRoutes(ctx context.Context, mac string, containerIPNet *types.IPNetSet, namespace, name string) error {
16+
return nil
17+
}

‎pkg/eni/crdv2.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -116,8 +116,8 @@ func (r *CRDV2) Allocate(ctx context.Context, cni *daemon.CNI, request ResourceR
116116
}
117117
}
118118

119-
func (r *CRDV2) Release(ctx context.Context, cni *daemon.CNI, request NetworkResource) bool {
120-
return false
119+
func (r *CRDV2) Release(ctx context.Context, cni *daemon.CNI, request NetworkResource) (bool, error) {
120+
return false, nil
121121
}
122122

123123
func (r *CRDV2) Dispose(n int) int {

‎pkg/eni/local.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -447,9 +447,9 @@ func (l *Local) Allocate(ctx context.Context, cni *daemon.CNI, request ResourceR
447447
}
448448

449449
// Release take the cni Del request and release resource to pool
450-
func (l *Local) Release(ctx context.Context, cni *daemon.CNI, request NetworkResource) bool {
450+
func (l *Local) Release(ctx context.Context, cni *daemon.CNI, request NetworkResource) (bool, error) {
451451
if request.ResourceType() != ResourceTypeLocalIP {
452-
return false
452+
return false, nil
453453
}
454454

455455
l.cond.L.Lock()
@@ -458,7 +458,7 @@ func (l *Local) Release(ctx context.Context, cni *daemon.CNI, request NetworkRes
458458
res := request.(*LocalIPResource)
459459

460460
if l.eni == nil || l.eni.ID != res.ENI.ID {
461-
return false
461+
return false, nil
462462
}
463463

464464
log := logf.FromContext(ctx)
@@ -478,7 +478,7 @@ func (l *Local) Release(ctx context.Context, cni *daemon.CNI, request NetworkRes
478478
log.Info("release ipv6", "ipv6", res.IP.IPv6)
479479
}
480480

481-
return true
481+
return true, nil
482482
}
483483

484484
// Priority for local resource only

‎pkg/eni/local_test.go

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,8 @@ func TestLocal_Release_ValidIPv4(t *testing.T) {
5050
local.ipv4.Add(NewValidIP(netip.MustParseAddr("192.0.2.1"), false))
5151
local.ipv4[netip.MustParseAddr("192.0.2.1")].Allocate("pod-1")
5252

53-
assert.True(t, local.Release(context.Background(), cni, request))
53+
ok, _ := local.Release(context.Background(), cni, request)
54+
assert.True(t, ok)
5455
}
5556

5657
func TestLocal_Release_ValidIPv6(t *testing.T) {
@@ -64,7 +65,8 @@ func TestLocal_Release_ValidIPv6(t *testing.T) {
6465
local.ipv6.Add(NewValidIP(netip.MustParseAddr("fd00:46dd:e::1"), false))
6566
local.ipv6[netip.MustParseAddr("fd00:46dd:e::1")].Allocate("pod-1")
6667

67-
assert.True(t, local.Release(context.Background(), cni, request))
68+
ok, _ := local.Release(context.Background(), cni, request)
69+
assert.True(t, ok)
6870
}
6971

7072
func TestLocal_Release_NilENI(t *testing.T) {
@@ -75,7 +77,8 @@ func TestLocal_Release_NilENI(t *testing.T) {
7577
}
7678
cni := &daemon.CNI{PodID: "pod-1"}
7779

78-
assert.False(t, local.Release(context.Background(), cni, request))
80+
ok, _ := local.Release(context.Background(), cni, request)
81+
assert.False(t, ok)
7982
}
8083

8184
func TestLocal_Release_DifferentENIID(t *testing.T) {
@@ -86,7 +89,8 @@ func TestLocal_Release_DifferentENIID(t *testing.T) {
8689
}
8790
cni := &daemon.CNI{PodID: "pod-1"}
8891

89-
assert.False(t, local.Release(context.Background(), cni, request))
92+
ok, _ := local.Release(context.Background(), cni, request)
93+
assert.False(t, ok)
9094
}
9195

9296
func TestLocal_Release_ValidIPv4_ReleaseIPv6(t *testing.T) {
@@ -103,7 +107,8 @@ func TestLocal_Release_ValidIPv4_ReleaseIPv6(t *testing.T) {
103107
local.ipv6.Add(NewValidIP(netip.MustParseAddr("fd00:46dd:e::1"), false))
104108
local.ipv6[netip.MustParseAddr("fd00:46dd:e::1")].Allocate("pod-1")
105109

106-
assert.True(t, local.Release(context.Background(), cni, request))
110+
ok, _ := local.Release(context.Background(), cni, request)
111+
assert.True(t, ok)
107112

108113
assert.Equal(t, ipStatusValid, local.ipv4[netip.MustParseAddr("192.0.2.1")].status)
109114
assert.Equal(t, "", local.ipv4[netip.MustParseAddr("192.0.2.1")].podID)

‎pkg/eni/manager.go

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ type ReportStatus interface {
7575
}
7676
type NetworkInterface interface {
7777
Allocate(ctx context.Context, cni *daemon.CNI, request ResourceRequest) (chan *AllocResp, []Trace)
78-
Release(ctx context.Context, cni *daemon.CNI, request NetworkResource) bool
78+
Release(ctx context.Context, cni *daemon.CNI, request NetworkResource) (bool, error)
7979
Priority() int
8080
Dispose(n int) int
8181
Run(ctx context.Context, podResources []daemon.PodResources, wg *sync.WaitGroup) error
@@ -237,7 +237,11 @@ func (m *Manager) Release(ctx context.Context, cni *daemon.CNI, req *ReleaseRequ
237237

238238
for _, networkResource := range req.NetworkResources {
239239
for _, ni := range m.networkInterfaces {
240-
ok := ni.Release(ctx, cni, networkResource)
240+
ok, err := ni.Release(ctx, cni, networkResource)
241+
if err != nil {
242+
return err
243+
}
244+
241245
if ok {
242246
if networkResource.ResourceType() == ResourceTypeLocalIP {
243247
m.node.UnsetIPExhaustive()

‎pkg/eni/manager_test.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,8 @@ func (o *timeOut) Allocate(ctx context.Context, cni *daemon.CNI, request Resourc
2525
return make(chan *AllocResp), nil
2626
}
2727

28-
func (o *timeOut) Release(ctx context.Context, cni *daemon.CNI, request NetworkResource) bool {
29-
return true
28+
func (o *timeOut) Release(ctx context.Context, cni *daemon.CNI, request NetworkResource) (bool, error) {
29+
return true, nil
3030
}
3131

3232
func (o *timeOut) Priority() int {
@@ -66,8 +66,8 @@ func (s *success) Allocate(ctx context.Context, cni *daemon.CNI, request Resourc
6666
return ch, nil
6767
}
6868

69-
func (s *success) Release(ctx context.Context, cni *daemon.CNI, request NetworkResource) bool {
70-
return true
69+
func (s *success) Release(ctx context.Context, cni *daemon.CNI, request NetworkResource) (bool, error) {
70+
return true, nil
7171
}
7272

7373
func (s *success) Priority() int {

‎pkg/eni/remote.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -206,8 +206,8 @@ func (r *Remote) Allocate(ctx context.Context, cni *daemon.CNI, request Resource
206206
return resp, nil
207207
}
208208

209-
func (r *Remote) Release(ctx context.Context, cni *daemon.CNI, request NetworkResource) bool {
210-
return false
209+
func (r *Remote) Release(ctx context.Context, cni *daemon.CNI, request NetworkResource) (bool, error) {
210+
return false, nil
211211
}
212212

213213
func (r *Remote) Dispose(n int) int {

‎pkg/eni/trunk.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,14 +47,14 @@ func (r *Trunk) Allocate(ctx context.Context, cni *daemon.CNI, request ResourceR
4747
}
4848
}
4949

50-
func (r *Trunk) Release(ctx context.Context, cni *daemon.CNI, request NetworkResource) bool {
50+
func (r *Trunk) Release(ctx context.Context, cni *daemon.CNI, request NetworkResource) (bool, error) {
5151
switch request.ResourceType() {
5252
case ResourceTypeLocalIP:
5353
return r.local.Release(ctx, cni, request)
5454
case ResourceTypeRemoteIP:
5555
return r.remote.Release(ctx, cni, request)
5656
default:
57-
return false
57+
return false, nil
5858
}
5959
}
6060

‎plugin/datapath/ipvlan_linux.go

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -330,7 +330,7 @@ func (d *IPvlanDriver) Teardown(ctx context.Context, cfg *types.TeardownCfg, net
330330
return err
331331
}
332332
} else {
333-
err = utils.DelEgressPriority(link, cfg.ContainerIPNet)
333+
err = utils.DelEgressPriority(ctx, link, cfg.ContainerIPNet)
334334
if err != nil {
335335
return err
336336
}
@@ -345,7 +345,7 @@ func (d *IPvlanDriver) Teardown(ctx context.Context, cfg *types.TeardownCfg, net
345345
}
346346
return nil
347347
}
348-
return utils.DelFilter(link, netlink.HANDLE_MIN_EGRESS, cfg.ContainerIPNet)
348+
return utils.DelFilter(ctx, link, netlink.HANDLE_MIN_EGRESS, cfg.ContainerIPNet)
349349
}()
350350
if err != nil {
351351
return err
@@ -447,7 +447,7 @@ func (d *IPvlanDriver) createSlaveIfNotExist(ctx context.Context, parentLink net
447447
return link, nil
448448
}
449449

450-
func (d *IPvlanDriver) setupFilters(link netlink.Link, cidrs []*net.IPNet, dstIndex int) error {
450+
func (d *IPvlanDriver) setupFilters(ctx context.Context, link netlink.Link, cidrs []*net.IPNet, dstIndex int) error {
451451
parent := uint32(netlink.HANDLE_CLSACT&0xffff0000 | netlink.HANDLE_MIN_EGRESS&0x0000ffff)
452452
filters, err := netlink.FilterList(link, parent)
453453
if err != nil {
@@ -478,7 +478,7 @@ func (d *IPvlanDriver) setupFilters(link netlink.Link, cidrs []*net.IPNet, dstIn
478478
if filter.Attrs() != nil && filter.Attrs().Priority != 4000 {
479479
continue
480480
}
481-
if err := utils.FilterDel(filter); err != nil {
481+
if err := utils.FilterDel(ctx, filter); err != nil {
482482
return fmt.Errorf("delete filter of %s error, %w", link.Attrs().Name, err)
483483
}
484484
}
@@ -487,7 +487,7 @@ func (d *IPvlanDriver) setupFilters(link netlink.Link, cidrs []*net.IPNet, dstIn
487487
if !in {
488488
u32 := rule.toU32Filter()
489489
u32.Parent = parent
490-
if err := utils.FilterAdd(u32); err != nil {
490+
if err := utils.FilterAdd(ctx, u32); err != nil {
491491
return fmt.Errorf("add filter for %s error, %w", link.Attrs().Name, err)
492492
}
493493
}
@@ -521,7 +521,7 @@ func (d *IPvlanDriver) setupInitNamespace(ctx context.Context, parentLink netlin
521521
}
522522

523523
redirectCIDRs := append(cfg.HostStackCIDRs, cfg.ServiceCIDR.IPv4)
524-
err = d.setupFilters(parentLink, redirectCIDRs, slaveLink.Attrs().Index)
524+
err = d.setupFilters(ctx, parentLink, redirectCIDRs, slaveLink.Attrs().Index)
525525
if err != nil {
526526
return err
527527
}

0 commit comments

Comments
 (0)