Skip to content

Commit cef1874

Browse files
BSWANGBSWANG
authored
Merge pull request #707 from l1b0k/feat/cni
enhance ipam
2 parents b408c2f + 5d4befe commit cef1874

File tree

22 files changed

+1382
-278
lines changed

22 files changed

+1382
-278
lines changed

‎Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ vet: ## Run go vet against code.
5151

5252
.PHONY: test
5353
test: manifests generate fmt vet envtest ## Run tests.
54-
KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test -race --tags "$(GO_BUILD_TAGS)" $$(go list ./... | grep -Ev '/e2e|/mocks|/generated|/apis|/examples|/tests') -coverprofile coverage.txt
54+
KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test -race --tags "$(GO_BUILD_TAGS)" $$(go list ./... | grep -Ev '/e2e|/mocks|/generated|/apis|/examples|/tests|/rpc') -coverprofile coverage.txt
5555

5656
.PHONY: lint
5757
lint: golangci-lint ## Run golangci-lint linter & yamllint

‎cmd/terway-controlplane/terway-controlplane.go

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,7 @@ import (
3939
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
4040
"k8s.io/apimachinery/pkg/util/wait"
4141
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
42-
"k8s.io/klog/v2"
43-
"k8s.io/klog/v2/klogr"
42+
"k8s.io/klog/v2/textlogger"
4443
ctrl "sigs.k8s.io/controller-runtime"
4544
"sigs.k8s.io/controller-runtime/pkg/cache"
4645
"sigs.k8s.io/controller-runtime/pkg/client"
@@ -92,12 +91,13 @@ func main() {
9291
)
9392
flag.StringVar(&configFilePath, "config", "/etc/config/ctrl-config.yaml", "config file for controlplane")
9493
flag.StringVar(&credentialFilePath, "credential", "/etc/credential/ctrl-secret.yaml", "secret file for controlplane")
95-
klog.InitFlags(nil)
96-
defer klog.Flush()
94+
95+
logCfg := textlogger.NewConfig()
96+
logCfg.AddFlags(flag.CommandLine)
9797

9898
flag.Parse()
9999

100-
ctrl.SetLogger(klogr.New())
100+
ctrl.SetLogger(textlogger.NewLogger(textlogger.NewConfig()))
101101
log.Info(version.Version)
102102

103103
ctx := ctrl.SetupSignalHandler()
@@ -123,7 +123,7 @@ func main() {
123123
os.Exit(1)
124124
}
125125

126-
lo.ForEach([]string{crds.CRDPodENI, crds.CRDPodNetworking, crds.CRDNode}, func(item string, index int) {
126+
lo.ForEach([]string{crds.CRDPodENI, crds.CRDPodNetworking, crds.CRDNode, crds.CRDNodeRuntime}, func(item string, index int) {
127127
err = crds.CreateOrUpdateCRD(ctx, directClient, item)
128128
if err != nil {
129129
log.Error(err, "unable sync crd")

‎cmd/terway/main.go

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,13 @@ import (
77
"strings"
88
"time"
99

10+
"k8s.io/klog/v2/textlogger"
11+
1012
"github.com/AliyunContainerService/terway/daemon"
1113
"github.com/AliyunContainerService/terway/pkg/utils"
1214
"github.com/AliyunContainerService/terway/pkg/version"
1315

1416
"k8s.io/klog/v2"
15-
"k8s.io/klog/v2/klogr"
1617
ctrl "sigs.k8s.io/controller-runtime"
1718
)
1819

@@ -32,27 +33,25 @@ var (
3233

3334
func main() {
3435
rand.New(rand.NewSource(time.Now().UnixNano()))
35-
klog.InitFlags(nil)
36-
defer klog.Flush()
37-
38-
ctrl.SetLogger(klogr.New())
39-
log.Info(version.Version)
4036

4137
fs := flag.NewFlagSet("terway", flag.ExitOnError)
42-
4338
fs.StringVar(&daemonMode, "daemon-mode", "VPC", "terway network mode")
4439
fs.StringVar(&logLevel, "log-level", "info", "terway log level")
4540
fs.StringVar(&readonlyListen, "readonly-listen", utils.NormalizePath(debugSocketPath), "terway readonly listen")
4641
ctrl.RegisterFlags(fs)
47-
klog.InitFlags(fs)
4842

4943
err := fs.Parse(os.Args[1:])
5044
if err != nil {
5145
panic(err)
5246
}
47+
var opts []textlogger.ConfigOption
48+
5349
if strings.ToLower(logLevel) == "debug" {
54-
_ = fs.Set("v", "5")
50+
opts = append(opts, textlogger.Verbosity(4))
5551
}
52+
ctrl.SetLogger(textlogger.NewLogger(textlogger.NewConfig(opts...)))
53+
54+
log.Info(version.Version)
5655

5756
ctx := ctrl.SetupSignalHandler()
5857
err = daemon.Run(ctx, utils.NormalizePath(defaultSocketPath), readonlyListen, utils.NormalizePath(defaultConfigPath), daemonMode)

‎daemon/daemon.go

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,17 @@ import (
1212
"time"
1313

1414
"github.com/go-logr/logr"
15+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1516
"k8s.io/apimachinery/pkg/util/sets"
17+
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
1618
logf "sigs.k8s.io/controller-runtime/pkg/log"
1719

20+
ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
21+
1822
"github.com/AliyunContainerService/terway/deviceplugin"
1923
"github.com/AliyunContainerService/terway/pkg/aliyun/client"
24+
25+
networkv1beta1 "github.com/AliyunContainerService/terway/pkg/apis/network.alibabacloud.com/v1beta1"
2026
"github.com/AliyunContainerService/terway/pkg/eni"
2127
"github.com/AliyunContainerService/terway/pkg/factory"
2228
"github.com/AliyunContainerService/terway/pkg/k8s"
@@ -540,8 +546,12 @@ func (n *networkService) gcPods(ctx context.Context) error {
540546
}
541547
podResources := getPodResources(objList)
542548

549+
uidInLocal := sets.New[string]()
543550
for _, podRes := range podResources {
544551
if podRes.PodInfo != nil {
552+
if podRes.PodInfo.PodUID != "" {
553+
uidInLocal.Insert(podRes.PodInfo.PodUID)
554+
}
545555
if podRes.PodInfo.PodIPs.IPv4 != nil {
546556
existIPs.Insert(podRes.PodInfo.PodIPs.IPv4.String())
547557
}
@@ -623,6 +633,7 @@ func (n *networkService) gcPods(ctx context.Context) error {
623633
if err != nil {
624634
return err
625635
}
636+
uidInLocal.Delete(podRes.PodInfo.PodUID)
626637
serviceLog.Info("removed pod", "pod", podID)
627638
}
628639

@@ -632,6 +643,68 @@ func (n *networkService) gcPods(ctx context.Context) error {
632643
})
633644
}
634645

646+
// clean runtime node records
647+
err = n.cleanRuntimeNode(ctx, uidInLocal)
648+
if err != nil {
649+
serviceLog.Error(err, "error cleaning runtime node")
650+
}
651+
return nil
652+
}
653+
654+
// cleanRuntimeNode localUIDs is the pod uid stored in db, so those pods should not be release in ipam
655+
func (n *networkService) cleanRuntimeNode(ctx context.Context, localUIDs sets.Set[string]) error {
656+
if n.ipamType != types.IPAMTypeCRD || n.daemonMode != daemon.ModeENIMultiIP {
657+
return nil
658+
}
659+
nodeRuntime := &networkv1beta1.NodeRuntime{}
660+
err := n.k8s.GetClient().Get(ctx, ctrlclient.ObjectKey{Name: n.k8s.NodeName()}, nodeRuntime)
661+
if err != nil {
662+
return err
663+
}
664+
l := logf.FromContext(ctx)
665+
666+
if l.V(4).Enabled() {
667+
l.Info("clean runtime node", "localUIDs", localUIDs, nodeRuntime)
668+
}
669+
670+
for uid, status := range nodeRuntime.Status.Pods {
671+
if localUIDs.Has(uid) {
672+
continue
673+
}
674+
// pod don't have record in local, need to delete it
675+
676+
s, last, ok := utils.RuntimeFinalStatus(status.Status)
677+
if ok && s == networkv1beta1.CNIStatusInitial {
678+
if time.Now().Before(last.LastUpdateTime.Add(30 * time.Second)) {
679+
continue
680+
}
681+
682+
list := strings.Split(status.PodID, "/")
683+
if len(list) != 2 {
684+
l.Info("invalid pod id format", "podID", status.PodID)
685+
continue
686+
}
687+
ok, err := n.k8s.PodExist(list[0], list[1])
688+
if err != nil || ok {
689+
continue
690+
}
691+
l.Info("clean runtime pod", "pod", s, "uid", uid)
692+
status.Status[networkv1beta1.CNIStatusDeleted] = &networkv1beta1.CNIStatusInfo{
693+
LastUpdateTime: metav1.NewTime(time.Now()),
694+
}
695+
}
696+
}
697+
update := nodeRuntime.DeepCopy()
698+
_, err = controllerutil.CreateOrPatch(ctx, n.k8s.GetClient(), update, func() error {
699+
update.Status = nodeRuntime.Status
700+
update.Spec = nodeRuntime.Spec
701+
update.Labels = nodeRuntime.Labels
702+
update.Name = nodeRuntime.Name
703+
return nil
704+
})
705+
if err != nil {
706+
return fmt.Errorf("failed to save node runtime status %w", err)
707+
}
635708
return nil
636709
}
637710

‎go.mod

Lines changed: 36 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -18,21 +18,21 @@ require (
1818
github.com/go-playground/mold/v4 v4.2.0
1919
github.com/go-playground/validator/v10 v10.11.1
2020
github.com/google/uuid v1.6.0
21-
github.com/onsi/ginkgo/v2 v2.9.5
22-
github.com/onsi/gomega v1.27.7
21+
github.com/onsi/ginkgo/v2 v2.13.0
22+
github.com/onsi/gomega v1.29.0
2323
github.com/pkg/errors v0.9.1
24-
github.com/prometheus/client_golang v1.15.1
24+
github.com/prometheus/client_golang v1.16.0
2525
github.com/pterm/pterm v0.12.62
2626
github.com/samber/lo v1.39.0
27-
github.com/spf13/cobra v1.6.1
27+
github.com/spf13/cobra v1.7.0
2828
github.com/stretchr/testify v1.9.0
2929
github.com/vishvananda/netlink v1.2.1-beta.2
30-
go.opentelemetry.io/otel v1.10.0
31-
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0
32-
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0
33-
go.opentelemetry.io/otel/sdk v1.10.0
34-
go.opentelemetry.io/otel/trace v1.10.0
35-
go.uber.org/atomic v1.9.0
30+
go.opentelemetry.io/otel v1.19.0
31+
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0
32+
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0
33+
go.opentelemetry.io/otel/sdk v1.19.0
34+
go.opentelemetry.io/otel/trace v1.19.0
35+
go.uber.org/atomic v1.10.0
3636
golang.org/x/mod v0.17.0
3737
golang.org/x/net v0.26.0
3838
golang.org/x/sync v0.7.0
@@ -43,16 +43,16 @@ require (
4343
google.golang.org/protobuf v1.33.0
4444
gopkg.in/ini.v1 v1.67.0
4545
gopkg.in/yaml.v2 v2.4.0
46-
k8s.io/api v0.27.9
47-
k8s.io/apiextensions-apiserver v0.27.9
48-
k8s.io/apimachinery v0.27.9
49-
k8s.io/apiserver v0.27.9
50-
k8s.io/client-go v0.27.9
51-
k8s.io/code-generator v0.27.9
52-
k8s.io/component-base v0.27.9
53-
k8s.io/klog/v2 v2.100.1
54-
k8s.io/kubelet v0.27.9
55-
k8s.io/utils v0.0.0-20230406110748-d93618cff8a2
46+
k8s.io/api v0.29.9
47+
k8s.io/apiextensions-apiserver v0.29.9
48+
k8s.io/apimachinery v0.29.9
49+
k8s.io/apiserver v0.29.9
50+
k8s.io/client-go v0.29.9
51+
k8s.io/code-generator v0.29.9
52+
k8s.io/component-base v0.29.9
53+
k8s.io/klog/v2 v2.110.1
54+
k8s.io/kubelet v0.29.9
55+
k8s.io/utils v0.0.0-20230726121419-3b25d923346b
5656
sigs.k8s.io/controller-runtime v0.15.3
5757
sigs.k8s.io/e2e-framework v0.3.0
5858
sigs.k8s.io/yaml v1.3.0
@@ -64,15 +64,15 @@ require (
6464
atomicgo.dev/schedule v0.0.2 // indirect
6565
github.com/beorn7/perks v1.0.1 // indirect
6666
github.com/blang/semver/v4 v4.0.0 // indirect
67-
github.com/cenkalti/backoff/v4 v4.1.3 // indirect
67+
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
6868
github.com/cespare/xxhash/v2 v2.2.0 // indirect
6969
github.com/containerd/cgroups v1.1.0 // indirect
7070
github.com/containerd/console v1.0.3 // indirect
7171
github.com/coreos/go-iptables v0.6.0 // indirect
7272
github.com/davecgh/go-spew v1.1.1 // indirect
73-
github.com/emicklei/go-restful/v3 v3.10.2 // indirect
73+
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
7474
github.com/evanphx/json-patch/v5 v5.6.0 // indirect
75-
github.com/fsnotify/fsnotify v1.6.0 // indirect
75+
github.com/fsnotify/fsnotify v1.7.0 // indirect
7676
github.com/go-logr/stdr v1.2.2 // indirect
7777
github.com/go-logr/zapr v1.2.4 // indirect
7878
github.com/go-openapi/jsonpointer v0.19.6 // indirect
@@ -84,14 +84,15 @@ require (
8484
github.com/gogo/protobuf v1.3.2 // indirect
8585
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
8686
github.com/golang/protobuf v1.5.4 // indirect
87-
github.com/google/gnostic v0.6.9 // indirect
87+
github.com/google/gnostic-models v0.6.8 // indirect
8888
github.com/google/go-cmp v0.6.0 // indirect
8989
github.com/google/gofuzz v1.2.0 // indirect
9090
github.com/google/pprof v0.0.0-20230323073829-e72429f035bd // indirect
9191
github.com/gookit/color v1.5.3 // indirect
92-
github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
92+
github.com/gorilla/websocket v1.5.0 // indirect
93+
github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect
9394
github.com/imdario/mergo v0.3.15 // indirect
94-
github.com/inconshreveable/mousetrap v1.0.1 // indirect
95+
github.com/inconshreveable/mousetrap v1.1.0 // indirect
9596
github.com/jmespath/go-jmespath v0.4.0 // indirect
9697
github.com/josharian/intern v1.0.0 // indirect
9798
github.com/json-iterator/go v1.1.12 // indirect
@@ -105,11 +106,12 @@ require (
105106
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
106107
github.com/modern-go/reflect2 v1.0.2 // indirect
107108
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
109+
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
108110
github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect
109111
github.com/pmezard/go-difflib v1.0.0 // indirect
110112
github.com/prometheus/client_model v0.4.0 // indirect
111-
github.com/prometheus/common v0.42.0 // indirect
112-
github.com/prometheus/procfs v0.9.0 // indirect
113+
github.com/prometheus/common v0.44.0 // indirect
114+
github.com/prometheus/procfs v0.10.1 // indirect
113115
github.com/rivo/uniseg v0.4.4 // indirect
114116
github.com/safchain/ethtool v0.3.0 // indirect
115117
github.com/segmentio/go-camelcase v0.0.0-20160726192923-7085f1e3c734 // indirect
@@ -121,9 +123,9 @@ require (
121123
github.com/vladimirvivien/gexe v0.2.0 // indirect
122124
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
123125
go.opencensus.io v0.24.0 // indirect
124-
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0 // indirect
125-
go.opentelemetry.io/proto/otlp v0.19.0 // indirect
126-
go.uber.org/multierr v1.6.0 // indirect
126+
go.opentelemetry.io/otel/metric v1.19.0 // indirect
127+
go.opentelemetry.io/proto/otlp v1.0.0 // indirect
128+
go.uber.org/multierr v1.11.0 // indirect
127129
go.uber.org/zap v1.24.0 // indirect
128130
golang.org/x/crypto v0.24.0 // indirect
129131
golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561 // indirect
@@ -132,15 +134,14 @@ require (
132134
golang.org/x/text v0.16.0 // indirect
133135
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
134136
google.golang.org/appengine v1.6.8 // indirect
135-
google.golang.org/genproto v0.0.0-20230525234025-438c736192d0 // indirect
136137
google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 // indirect
137138
google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect
138139
gopkg.in/inf.v0 v0.9.1 // indirect
139140
gopkg.in/yaml.v3 v3.0.1 // indirect
140-
k8s.io/gengo v0.0.0-20220902162205-c0856e24416d // indirect
141-
k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect
141+
k8s.io/gengo v0.0.0-20230829151522-9cce18d56c01 // indirect
142+
k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
142143
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
143-
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
144+
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
144145
)
145146

146147
replace github.com/vishvananda/netlink => github.com/BSWANG/netlink v1.0.1-0.20220803105814-1f63f9d61229

0 commit comments

Comments
 (0)