Skip to content
This repository was archived by the owner on Oct 28, 2024. It is now read-only.

Commit 6053e25

Browse files
LuBingtanLuBingtan
committed
Add externalIPs in service uws, support external dns
Signed-off-by: Bingtan Lu <[email protected]>
1 parent c5935f8 commit 6053e25

File tree

4 files changed

+214
-0
lines changed

4 files changed

+214
-0
lines changed

‎virtualcluster/config/sampleswithspec/coredns_external.yaml

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,196 @@
1+
apiVersion: v1
2+
kind: ServiceAccount
3+
metadata:
4+
name: coredns
5+
namespace: kube-system
6+
---
7+
apiVersion: rbac.authorization.k8s.io/v1
8+
kind: ClusterRole
9+
metadata:
10+
labels:
11+
kubernetes.io/bootstrapping: rbac-defaults
12+
name: system:coredns
13+
rules:
14+
- apiGroups:
15+
- ""
16+
resources:
17+
- endpoints
18+
- services
19+
- pods
20+
- namespaces
21+
verbs:
22+
- list
23+
- watch
24+
- apiGroups:
25+
- ""
26+
resources:
27+
- nodes
28+
verbs:
29+
- get
30+
---
31+
apiVersion: rbac.authorization.k8s.io/v1
32+
kind: ClusterRoleBinding
33+
metadata:
34+
annotations:
35+
rbac.authorization.kubernetes.io/autoupdate: "true"
36+
labels:
37+
kubernetes.io/bootstrapping: rbac-defaults
38+
name: system:coredns
39+
roleRef:
40+
apiGroup: rbac.authorization.k8s.io
41+
kind: ClusterRole
42+
name: system:coredns
43+
subjects:
44+
- kind: ServiceAccount
45+
name: coredns
46+
namespace: kube-system
47+
---
48+
apiVersion: v1
49+
kind: ConfigMap
50+
metadata:
51+
name: coredns
52+
namespace: kube-system
53+
data:
54+
Corefile: |
55+
.:53 {
56+
log
57+
errors
58+
health {
59+
lameduck 5s
60+
}
61+
ready
62+
kubernetes cluster.virtual in-addr.arpa ip6.arpa {
63+
fallthrough in-addr.arpa ip6.arpa
64+
}
65+
k8s_external svc.cluster.local
66+
prometheus :9153
67+
forward . /etc/resolv.conf
68+
cache 30
69+
loop
70+
reload
71+
loadbalance
72+
}
73+
---
74+
apiVersion: v1
75+
kind: Service
76+
metadata:
77+
name: kube-dns
78+
namespace: kube-system
79+
annotations:
80+
prometheus.io/port: "9153"
81+
prometheus.io/scrape: "true"
82+
labels:
83+
k8s-app: kube-dns
84+
kubernetes.io/cluster-service: "true"
85+
kubernetes.io/name: "CoreDNS"
86+
spec:
87+
selector:
88+
k8s-app: kube-dns
89+
ports:
90+
- name: dns
91+
port: 53
92+
protocol: UDP
93+
- name: dns-tcp
94+
port: 53
95+
protocol: TCP
96+
- name: metrics
97+
port: 9153
98+
protocol: TCP
99+
---
100+
apiVersion: apps/v1
101+
kind: Deployment
102+
metadata:
103+
name: coredns
104+
namespace: kube-system
105+
labels:
106+
k8s-app: kube-dns
107+
kubernetes.io/name: "CoreDNS"
108+
spec:
109+
# replicas: not specified here:
110+
# 1. Default is 1.
111+
# 2. Will be tuned in real time if DNS horizontal auto-scaling is turned on.
112+
strategy:
113+
type: RollingUpdate
114+
rollingUpdate:
115+
maxUnavailable: 1
116+
selector:
117+
matchLabels:
118+
k8s-app: kube-dns
119+
template:
120+
metadata:
121+
labels:
122+
k8s-app: kube-dns
123+
spec:
124+
serviceAccountName: coredns
125+
tolerations:
126+
- key: "CriticalAddonsOnly"
127+
operator: "Exists"
128+
nodeSelector:
129+
kubernetes.io/os: linux
130+
affinity:
131+
podAntiAffinity:
132+
requiredDuringSchedulingIgnoredDuringExecution:
133+
- labelSelector:
134+
matchExpressions:
135+
- key: k8s-app
136+
operator: In
137+
values: ["kube-dns"]
138+
topologyKey: kubernetes.io/hostname
139+
containers:
140+
- name: coredns
141+
image: registry.k8s.io/coredns/coredns:v1.9.3
142+
imagePullPolicy: IfNotPresent
143+
resources:
144+
limits:
145+
memory: 170Mi
146+
requests:
147+
cpu: 100m
148+
memory: 70Mi
149+
args: [ "-conf", "/etc/coredns/Corefile" ]
150+
volumeMounts:
151+
- name: config-volume
152+
mountPath: /etc/coredns
153+
readOnly: true
154+
ports:
155+
- containerPort: 53
156+
name: dns
157+
protocol: UDP
158+
- containerPort: 53
159+
name: dns-tcp
160+
protocol: TCP
161+
- containerPort: 9153
162+
name: metrics
163+
protocol: TCP
164+
securityContext:
165+
allowPrivilegeEscalation: false
166+
capabilities:
167+
add:
168+
- NET_BIND_SERVICE
169+
drop:
170+
- all
171+
readOnlyRootFilesystem: true
172+
livenessProbe:
173+
httpGet:
174+
path: /health
175+
port: 8080
176+
scheme: HTTP
177+
initialDelaySeconds: 60
178+
timeoutSeconds: 5
179+
successThreshold: 1
180+
failureThreshold: 5
181+
readinessProbe:
182+
httpGet:
183+
path: /ready
184+
port: 8181
185+
scheme: HTTP
186+
env:
187+
- name: KUBERNETES_SERVICE_HOST
188+
value: kubernetes
189+
dnsPolicy: Default
190+
volumes:
191+
- name: config-volume
192+
configMap:
193+
name: coredns
194+
items:
195+
- key: Corefile
196+
path: Corefile

‎virtualcluster/pkg/syncer/conversion/equality.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -692,6 +692,11 @@ func (e vcEquality) CheckServiceEquality(pObj, vObj *v1.Service) *v1.Service {
692692
vSpec.IPFamilies = pSpec.IPFamilies
693693
vSpec.IPFamilyPolicy = pSpec.IPFamilyPolicy
694694

695+
if featuregate.DefaultFeatureGate.Enabled(featuregate.VServiceExternalIP) {
696+
// Ignore ExternalIPs
697+
vSpec.ExternalIPs = pSpec.ExternalIPs
698+
}
699+
695700
if !equality.Semantic.DeepEqual(vSpec, pSpec) {
696701
if updated == nil {
697702
updated = pObj.DeepCopy()

‎virtualcluster/pkg/syncer/resources/service/uws.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import (
3232
"sigs.k8s.io/cluster-api-provider-nested/virtualcluster/pkg/syncer/constants"
3333
"sigs.k8s.io/cluster-api-provider-nested/virtualcluster/pkg/syncer/conversion"
3434
"sigs.k8s.io/cluster-api-provider-nested/virtualcluster/pkg/syncer/util"
35+
"sigs.k8s.io/cluster-api-provider-nested/virtualcluster/pkg/syncer/util/featuregate"
3536
)
3637

3738
// StartUWS starts the upward syncer
@@ -105,6 +106,12 @@ func (c *controller) BackPopulate(key string) error {
105106
if updatedMeta != nil {
106107
newService = vService.DeepCopy()
107108
newService.ObjectMeta = *updatedMeta
109+
if featuregate.DefaultFeatureGate.Enabled(featuregate.VServiceExternalIP) &&
110+
updatedMeta.Annotations[constants.LabelSuperClusterIP] != "" &&
111+
len(newService.Spec.ExternalIPs) == 0 {
112+
// Add clusterIP to ExternalIPs if it hasn't been set on purpose
113+
newService.Spec.ExternalIPs = []string{updatedMeta.Annotations[constants.LabelSuperClusterIP]}
114+
}
108115
if _, err = tenantClient.CoreV1().Services(vService.Namespace).Update(context.TODO(), newService, metav1.UpdateOptions{}); err != nil {
109116
return fmt.Errorf("failed to back populate service %s/%s meta update for cluster %s: %v", vService.Namespace, vService.Name, clusterName, err)
110117
}

‎virtualcluster/pkg/syncer/util/featuregate/gate.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,11 @@ const (
8080
// RootCACertConfigMapSupport is an experimental feature that allows clusters +1.21 to support
8181
// the kube-root-ca.crt dropped into each Namespace
8282
RootCACertConfigMapSupport = "RootCACertConfigMapSupport"
83+
84+
// VServiceExternalIP is an experimental feature that allows the syncer to
85+
// add clusterIP of pService to vService's externalIPs.
86+
// So that vService can be resolved by using the k8s_external plugin in coredns.
87+
VServiceExternalIP = "VServiceExternalIP"
8388
)
8489

8590
var defaultFeatures = FeatureList{
@@ -94,6 +99,7 @@ var defaultFeatures = FeatureList{
9499
TenantAllowResourceNoSync: {Default: false},
95100
DisableCRDPreserveUnknownFields: {Default: false},
96101
RootCACertConfigMapSupport: {Default: false},
102+
VServiceExternalIP: {Default: false},
97103
}
98104

99105
type Feature string

0 commit comments

Comments
 (0)