Skip to content

Commit 04cc3fb

Browse files
l1b0kl1b0k
committed
fix per-package lb
Signed-off-by: l1b0k <[email protected]>
1 parent 5a310b0 commit 04cc3fb

File tree

2 files changed

+214
-104
lines changed

2 files changed

+214
-104
lines changed

Diff for: ‎policy/cilium/0032-fix-disable-per-package-lb.patch

+101
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
2+
From: l1b0k <[email protected]>
3+
Date: Sat, 12 Oct 2024 09:33:24 +0800
4+
Subject: fix disable per-package lb
5+
6+
Signed-off-by: l1b0k <[email protected]>
7+
---
8+
bpf/bpf_lxc.c | 5 ++---
9+
daemon/cmd/daemon_main.go | 3 +++
10+
pkg/datapath/linux/config/config.go | 5 +++++
11+
pkg/option/config.go | 6 ++++++
12+
4 files changed, 16 insertions(+), 3 deletions(-)
13+
14+
diff --git a/bpf/bpf_lxc.c b/bpf/bpf_lxc.c
15+
index 2773e2c276..7e424204e2 100644
16+
--- a/bpf/bpf_lxc.c
17+
+++ b/bpf/bpf_lxc.c
18+
@@ -58,9 +58,9 @@
19+
* Most services with L7 LB flag can not be redirected to their proxy port
20+
* in bpf_sock, so we must check for those via per packet LB as well.
21+
*/
22+
-#if !defined(ENABLE_SOCKET_LB_FULL) || \
23+
+#if (!defined(ENABLE_SOCKET_LB_FULL) || \
24+
defined(ENABLE_SOCKET_LB_HOST_ONLY) || \
25+
- defined(ENABLE_L7_LB)
26+
+ defined(ENABLE_L7_LB) ) && !defined(DISABLE_PER_PACKET_LB)
27+
# define ENABLE_PER_PACKET_LB 1
28+
#endif
29+
30+
@@ -1247,7 +1247,6 @@ static __always_inline int __tail_handle_ipv4(struct __ctx_buff *ctx)
31+
32+
if (unlikely(!is_valid_lxc_src_ipv4(ip4)))
33+
return DROP_INVALID_SIP;
34+
-
35+
#ifdef ENABLE_PER_PACKET_LB
36+
{
37+
struct ipv4_ct_tuple tuple = {};
38+
diff --git a/daemon/cmd/daemon_main.go b/daemon/cmd/daemon_main.go
39+
index de2ad4900b..8b495d972f 100644
40+
--- a/daemon/cmd/daemon_main.go
41+
+++ b/daemon/cmd/daemon_main.go
42+
@@ -1147,6 +1147,9 @@ func initializeFlags() {
43+
flags.MarkHidden(option.EnableStaleCiliumEndpointCleanup)
44+
option.BindEnv(option.EnableStaleCiliumEndpointCleanup)
45+
46+
+ flags.Bool(option.DisablePerPacketLB, false, "Disable svc func.")
47+
+ option.BindEnv(option.DisablePerPacketLB)
48+
+
49+
viper.BindPFlags(flags)
50+
}
51+
52+
diff --git a/pkg/datapath/linux/config/config.go b/pkg/datapath/linux/config/config.go
53+
index 060328adf9..4201e878fb 100644
54+
--- a/pkg/datapath/linux/config/config.go
55+
+++ b/pkg/datapath/linux/config/config.go
56+
@@ -98,6 +98,11 @@ func (h *HeaderfileWriter) WriteNodeConfig(w io.Writer, cfg *datapath.LocalNodeC
57+
58+
cDefinesMap["KERNEL_HZ"] = fmt.Sprintf("%d", option.Config.KernelHz)
59+
60+
+ if option.Config.DisablePerPacketLB {
61+
+ log.Infof("Disabling per-packet LB policy")
62+
+ cDefinesMap["DISABLE_PER_PACKET_LB"] = "1"
63+
+ }
64+
+
65+
if option.Config.EnableIPv6 {
66+
extraMacrosMap["ROUTER_IP"] = routerIP.String()
67+
fw.WriteString(defineIPv6("ROUTER_IP", routerIP))
68+
diff --git a/pkg/option/config.go b/pkg/option/config.go
69+
index d4d9643f4e..80b5eb2c7c 100644
70+
--- a/pkg/option/config.go
71+
+++ b/pkg/option/config.go
72+
@@ -1104,6 +1104,8 @@ const (
73+
// EnableStaleCiliumEndpointCleanup sets whether Cilium should perform cleanup of
74+
// stale CiliumEndpoints during init.
75+
EnableStaleCiliumEndpointCleanup = "enable-stale-cilium-endpoint-cleanup"
76+
+
77+
+ DisablePerPacketLB = "disable-per-package-lb"
78+
)
79+
80+
// Default string arguments
81+
@@ -2262,6 +2264,8 @@ type DaemonConfig struct {
82+
// This will attempt to remove local CiliumEndpoints that are not managed by Cilium
83+
// following Endpoint restoration.
84+
EnableStaleCiliumEndpointCleanup bool
85+
+
86+
+ DisablePerPacketLB bool
87+
}
88+
89+
var (
90+
@@ -3257,6 +3261,8 @@ func (c *DaemonConfig) Populate() {
91+
c.EnableK8sTerminatingEndpoint = viper.GetBool(EnableK8sTerminatingEndpoint)
92+
c.EnableStaleCiliumEndpointCleanup = viper.GetBool(EnableStaleCiliumEndpointCleanup)
93+
94+
+ c.DisablePerPacketLB = viper.GetBool(DisablePerPacketLB)
95+
+
96+
// Disable Envoy version check if L7 proxy is disabled.
97+
c.DisableEnvoyVersionCheck = viper.GetBool(DisableEnvoyVersionCheck)
98+
if !c.EnableL7Proxy {
99+
--
100+
2.46.0
101+

Diff for: ‎policy/policyinit.sh

+113-104
Original file line numberDiff line numberDiff line change
@@ -5,19 +5,19 @@ mount -o remount rw /proc/sys
55
export DATASTORE_TYPE=kubernetes
66

77
masq_eni_only() {
8-
if ! "$1" -t nat -L terway-masq; then
9-
# Create a new chain in nat table.
10-
"$1" -t nat -N terway-masq
11-
fi
8+
if ! "$1" -t nat -L terway-masq; then
9+
# Create a new chain in nat table.
10+
"$1" -t nat -N terway-masq
11+
fi
1212

13-
if ! "$1" -t nat -L POSTROUTING | grep -q terway-masq; then
14-
# Append that chain to POSTROUTING table.
15-
"$1" -t nat -A POSTROUTING -m comment --comment "terway:masq-outgoing" ! -o lo -j terway-masq
16-
fi
13+
if ! "$1" -t nat -L POSTROUTING | grep -q terway-masq; then
14+
# Append that chain to POSTROUTING table.
15+
"$1" -t nat -A POSTROUTING -m comment --comment "terway:masq-outgoing" ! -o lo -j terway-masq
16+
fi
1717

18-
if ! "$1" -t nat -L terway-masq | grep -q MASQUERADE; then
19-
"$1" -t nat -A terway-masq -j MASQUERADE
20-
fi
18+
if ! "$1" -t nat -L terway-masq | grep -q MASQUERADE; then
19+
"$1" -t nat -A terway-masq -j MASQUERADE
20+
fi
2121
}
2222

2323
terway_config_val() {
@@ -51,16 +51,16 @@ if grep -q "cni_exclusive_eni *= *eniOnly" "$node_capabilities"; then
5151

5252
# for health check
5353
if [ "$FELIX_HEALTHPORT" != "" ]; then
54-
# shellcheck disable=SC2016
55-
exec socat TCP-LISTEN:"$FELIX_HEALTHPORT",bind=127.0.0.1,fork,reuseaddr system:'sleep 2;kill -9 $SOCAT_PID 2>/dev/null'
54+
# shellcheck disable=SC2016
55+
exec socat TCP-LISTEN:"$FELIX_HEALTHPORT",bind=127.0.0.1,fork,reuseaddr system:'sleep 2;kill -9 $SOCAT_PID 2>/dev/null'
5656
else
57-
# shellcheck disable=SC2016
58-
exec socat TCP-LISTEN:9099,bind=127.0.0.1,fork,reuseaddr system:'sleep 2;kill -9 $SOCAT_PID 2>/dev/null'
57+
# shellcheck disable=SC2016
58+
exec socat TCP-LISTEN:9099,bind=127.0.0.1,fork,reuseaddr system:'sleep 2;kill -9 $SOCAT_PID 2>/dev/null'
5959
fi
6060
fi
6161

6262
if grep -q "datapath *= *datapathv2" "$node_capabilities"; then
63-
datapath_mode=veth
63+
datapath_mode=veth
6464
fi
6565

6666
# kernel version has already checked in initContainer, so just determine whether plugin chaining exists
@@ -69,7 +69,7 @@ if [ "$virtyal_type" = "ipvlan" ] || [ "$virtyal_type" = "datapathv2" ]; then
6969

7070
# kernel version equal and above 4.19
7171
if { [ "$KERNEL_MAJOR_VERSION" -eq 4 ] && [ "$KERNEL_MINOR_VERSION" -ge 19 ]; } ||
72-
[ "$KERNEL_MAJOR_VERSION" -gt 4 ]; then
72+
[ "$KERNEL_MAJOR_VERSION" -gt 4 ]; then
7373

7474
extra_args=$(terway_config_val 'cilium_args')
7575
if [ -z "$DISABLE_POLICY" ] || [ "$DISABLE_POLICY" = "false" ] || [ "$DISABLE_POLICY" = "0" ]; then
@@ -95,106 +95,115 @@ if [ "$virtyal_type" = "ipvlan" ] || [ "$virtyal_type" = "datapathv2" ]; then
9595
echo "turning up hubble, passing args \"${extra_args}\""
9696
fi
9797

98-
if [ "$IN_CLUSTER_LOADBALANCE" = "true" ]; then
99-
extra_args="${extra_args} --enable-in-cluster-loadbalance=true "
100-
echo "turning up in cluster loadbalance, passing args \"${extra_args}\""
101-
fi
98+
if [ "$IN_CLUSTER_LOADBALANCE" = "true" ]; then
99+
extra_args="${extra_args} --enable-in-cluster-loadbalance=true "
100+
echo "turning up in cluster loadbalance, passing args \"${extra_args}\""
101+
fi
102102

103-
if bpftool -j feature probe | grep bpf_skb_ecn_set_ce ; then
104-
extra_args="${extra_args} --enable-bandwidth-manager=true "
105-
fi
103+
if bpftool -j feature probe | grep bpf_skb_ecn_set_ce; then
104+
extra_args="${extra_args} --enable-bandwidth-manager=true "
105+
fi
106106

107107
echo "using cilium as network routing & policy"
108108

109109
# shellcheck disable=SC2086
110110
exec cilium-agent --tunnel=disabled --enable-ipv4-masquerade=false --enable-ipv6-masquerade=false \
111-
--enable-policy=$ENABLE_POLICY \
112-
--agent-health-port=9099 --disable-envoy-version-check=true \
113-
--enable-local-node-route=false --ipv4-range=169.254.10.0/30 --ipv6-range=fe80:2400:3200:baba::/30 --enable-endpoint-health-checking=false \
114-
--enable-health-checking=false --enable-service-topology=true --disable-cnp-status-updates=true --k8s-heartbeat-timeout=0 --enable-session-affinity=true \
115-
--install-iptables-rules=false --enable-l7-proxy=false \
116-
--ipam=cluster-pool --datapath-mode=${datapath_mode} --enable-runtime-device-detection=true ${extra_args}
111+
--enable-policy=$ENABLE_POLICY \
112+
--agent-health-port=9099 --disable-envoy-version-check=true \
113+
--enable-local-node-route=false --ipv4-range=169.254.10.0/30 --ipv6-range=fe80:2400:3200:baba::/30 --enable-endpoint-health-checking=false \
114+
--enable-health-checking=false --enable-service-topology=true --disable-cnp-status-updates=true --k8s-heartbeat-timeout=0 --enable-session-affinity=true \
115+
--install-iptables-rules=false --enable-l7-proxy=false \
116+
--ipam=cluster-pool --datapath-mode=${datapath_mode} --enable-runtime-device-detection=true ${extra_args}
117117
fi
118118
fi
119-
# shellcheck disable=SC1091
120-
source uninstall_policy.sh
121-
122-
# check kernel version
119+
# shellcheck disable=SC1091
120+
source uninstall_policy.sh
123121

124-
export FELIX_IPTABLESBACKEND=Auto
125-
if ( uname -r | grep -E "el7|an7" && [ "${KERNEL_MAJOR_VERSION}" -eq 3 ] ) || ( uname -r | grep -E "al7" && [ "${KERNEL_MAJOR_VERSION}" -eq 4 ] ); then
126-
export FELIX_IPTABLESBACKEND=Legacy
127-
elif ( uname -r | grep -E "el8|an8" && [ "${KERNEL_MAJOR_VERSION}" -ge 4 ] ) || ( uname -r | grep -E "al8|lifsea8" && [ "${KERNEL_MAJOR_VERSION}" -ge 5 ] ); then
128-
export FELIX_IPTABLESBACKEND=NFT
122+
# check kernel version
129123

130-
# clean legacy rules if exist
131-
cleanup_legacy
132-
fi
124+
export FELIX_IPTABLESBACKEND=Auto
125+
if (uname -r | grep -E "el7|an7" && [ "${KERNEL_MAJOR_VERSION}" -eq 3 ]) || (uname -r | grep -E "al7" && [ "${KERNEL_MAJOR_VERSION}" -eq 4 ]); then
126+
export FELIX_IPTABLESBACKEND=Legacy
127+
elif (uname -r | grep -E "el8|an8" && [ "${KERNEL_MAJOR_VERSION}" -ge 4 ]) || (uname -r | grep -E "al8|lifsea8" && [ "${KERNEL_MAJOR_VERSION}" -ge 5 ]); then
128+
export FELIX_IPTABLESBACKEND=NFT
133129

134-
# default for veth
135-
export FELIX_LOGSEVERITYSYS=none
136-
export FELIX_LOGSEVERITYSCREEN=info
137-
export CALICO_NETWORKING_BACKEND=none
138-
export CLUSTER_TYPE=k8s,aliyun
139-
export CALICO_DISABLE_FILE_LOGGING=true
140-
# shellcheck disable=SC2154
141-
export CALICO_IPV4POOL_CIDR="${Network}"
142-
export FELIX_IPTABLESREFRESHINTERVAL="${IPTABLESREFRESHINTERVAL:-60}"
143-
export FELIX_IPV6SUPPORT=true
144-
export WAIT_FOR_DATASTORE=true
145-
export IP=""
146-
export NO_DEFAULT_POOLS=true
147-
export FELIX_DEFAULTENDPOINTTOHOSTACTION=ACCEPT
148-
export FELIX_HEALTHENABLED=true
149-
export FELIX_LOGFILEPATH=/dev/null
150-
export FELIX_BPFENABLED=false
151-
export FELIX_XDPENABLED=false
152-
export FELIX_BPFCONNECTTIMELOADBALANCINGENABLED=false
153-
export FELIX_BPFKUBEPROXYIPTABLESCLEANUPENABLED=false
154-
exec 2>&1
155-
if [ -n "$NODENAME" ]; then
156-
export FELIX_FELIXHOSTNAME="$NODENAME"
157-
fi
158-
if [ -n "$DATASTORE_TYPE" ]; then
159-
export FELIX_DATASTORETYPE="$DATASTORE_TYPE"
160-
fi
130+
# clean legacy rules if exist
131+
cleanup_legacy
132+
fi
161133

162-
if [ "$network_policy_provider" = "ebpf" ]; then
163-
cleanup_felix
164-
# kernel version equal and above 4.19
165-
if { [ "$KERNEL_MAJOR_VERSION" -eq 4 ] && [ "$KERNEL_MINOR_VERSION" -ge 19 ]; } ||
166-
[ "$KERNEL_MAJOR_VERSION" -gt 4 ]; then
167-
168-
if [ -z "$DISABLE_POLICY" ] || [ "$DISABLE_POLICY" = "false" ] || [ "$DISABLE_POLICY" = "0" ]; then
169-
ENABLE_POLICY="default"
170-
else
171-
ENABLE_POLICY="never"
172-
fi
173-
# shellcheck disable=SC2086
174-
exec cilium-agent --kube-proxy-replacement=disabled --tunnel=disabled --enable-ipv4-masquerade=false --enable-ipv6-masquerade=false \
175-
--enable-policy=$ENABLE_POLICY \
176-
--agent-health-port=9099 --disable-envoy-version-check=true \
177-
--enable-local-node-route=false --ipv4-range=169.254.10.0/30 --ipv6-range=fe80:2400:3200:baba::/30 --enable-endpoint-health-checking=false \
178-
--enable-health-checking=false --enable-service-topology=true --disable-cnp-status-updates=true --k8s-heartbeat-timeout=0 --enable-session-affinity=true \
179-
--install-iptables-rules=false --enable-l7-proxy=false \
180-
--ipam=cluster-pool
181-
else
182-
echo "unsupported kernel version"
183-
exit 1
184-
fi
185-
else
186-
if [ -z "$DISABLE_POLICY" ] || [ "$DISABLE_POLICY" = "false" ] || [ "$DISABLE_POLICY" = "0" ]; then
187-
exec calico-felix
188-
fi
189-
fi
134+
# default for veth
135+
export FELIX_LOGSEVERITYSYS=none
136+
export FELIX_LOGSEVERITYSCREEN=info
137+
export CALICO_NETWORKING_BACKEND=none
138+
export CLUSTER_TYPE=k8s,aliyun
139+
export CALICO_DISABLE_FILE_LOGGING=true
140+
# shellcheck disable=SC2154
141+
export CALICO_IPV4POOL_CIDR="${Network}"
142+
export FELIX_IPTABLESREFRESHINTERVAL="${IPTABLESREFRESHINTERVAL:-60}"
143+
export FELIX_IPV6SUPPORT=true
144+
export WAIT_FOR_DATASTORE=true
145+
export IP=""
146+
export NO_DEFAULT_POOLS=true
147+
export FELIX_DEFAULTENDPOINTTOHOSTACTION=ACCEPT
148+
export FELIX_HEALTHENABLED=true
149+
export FELIX_LOGFILEPATH=/dev/null
150+
export FELIX_BPFENABLED=false
151+
export FELIX_XDPENABLED=false
152+
export FELIX_BPFCONNECTTIMELOADBALANCINGENABLED=false
153+
export FELIX_BPFKUBEPROXYIPTABLESCLEANUPENABLED=false
154+
exec 2>&1
155+
if [ -n "$NODENAME" ]; then
156+
export FELIX_FELIXHOSTNAME="$NODENAME"
157+
fi
158+
if [ -n "$DATASTORE_TYPE" ]; then
159+
export FELIX_DATASTORETYPE="$DATASTORE_TYPE"
160+
fi
190161

191-
config_masquerade
162+
if [ "$network_policy_provider" = "ebpf" ]; then
192163
cleanup_felix
193-
# for health check
194-
if [ "$FELIX_HEALTHPORT" != "" ]; then
195-
# shellcheck disable=SC2016
196-
exec socat TCP-LISTEN:"$FELIX_HEALTHPORT",bind=127.0.0.1,fork,reuseaddr system:'sleep 2;kill -9 $SOCAT_PID 2>/dev/null'
164+
# kernel version equal and above 4.19
165+
if { [ "$KERNEL_MAJOR_VERSION" -eq 4 ] && [ "$KERNEL_MINOR_VERSION" -ge 19 ]; } ||
166+
[ "$KERNEL_MAJOR_VERSION" -gt 4 ]; then
167+
168+
extra_args=$(terway_config_val 'cilium_args')
169+
170+
if [ -z "$DISABLE_POLICY" ] || [ "$DISABLE_POLICY" = "false" ] || [ "$DISABLE_POLICY" = "0" ]; then
171+
ENABLE_POLICY="default"
172+
else
173+
ENABLE_POLICY="never"
174+
extra_args="${extra_args} --labels=k8s:io\\.kubernetes\\.pod\\.namespace "
175+
fi
176+
177+
if [ "$IN_CLUSTER_LOADBALANCE" = "true" ]; then
178+
extra_args="${extra_args} --enable-in-cluster-loadbalance=true "
179+
echo "turning up in cluster loadbalance, passing args \"${extra_args}\""
180+
fi
181+
182+
# shellcheck disable=SC2086
183+
exec cilium-agent --kube-proxy-replacement=disabled --tunnel=disabled --enable-ipv4-masquerade=false --enable-ipv6-masquerade=false \
184+
--enable-policy=$ENABLE_POLICY \
185+
--agent-health-port=9099 --disable-envoy-version-check=true \
186+
--enable-local-node-route=false --ipv4-range=169.254.10.0/30 --ipv6-range=fe80:2400:3200:baba::/30 --enable-endpoint-health-checking=false \
187+
--enable-health-checking=false --enable-service-topology=true --disable-cnp-status-updates=true --k8s-heartbeat-timeout=0 --enable-session-affinity=true \
188+
--install-iptables-rules=false --enable-l7-proxy=false \
189+
--ipam=cluster-pool ${extra_args}
197190
else
198-
# shellcheck disable=SC2016
199-
exec socat TCP-LISTEN:9099,bind=127.0.0.1,fork,reuseaddr system:'sleep 2;kill -9 $SOCAT_PID 2>/dev/null'
191+
echo "unsupported kernel version"
192+
exit 1
193+
fi
194+
else
195+
if [ -z "$DISABLE_POLICY" ] || [ "$DISABLE_POLICY" = "false" ] || [ "$DISABLE_POLICY" = "0" ]; then
196+
exec calico-felix
200197
fi
198+
fi
199+
200+
config_masquerade
201+
cleanup_felix
202+
# for health check
203+
if [ "$FELIX_HEALTHPORT" != "" ]; then
204+
# shellcheck disable=SC2016
205+
exec socat TCP-LISTEN:"$FELIX_HEALTHPORT",bind=127.0.0.1,fork,reuseaddr system:'sleep 2;kill -9 $SOCAT_PID 2>/dev/null'
206+
else
207+
# shellcheck disable=SC2016
208+
exec socat TCP-LISTEN:9099,bind=127.0.0.1,fork,reuseaddr system:'sleep 2;kill -9 $SOCAT_PID 2>/dev/null'
209+
fi

0 commit comments

Comments
 (0)