Skip to content

Commit 8c7f77b

Browse files
jingxu97jingxu97
committed
WIP New kustomization for pd driver
This can work for both linux and windows
1 parent 85088c4 commit 8c7f77b

File tree

15 files changed

+1316
-2
lines changed

15 files changed

+1316
-2
lines changed

‎deploy/kubernetes/deploy-driver.sh

+33-2
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ set -x
1818

1919
readonly NAMESPACE="${GCE_PD_DRIVER_NAMESPACE:-gce-pd-csi-driver}"
2020
readonly DEPLOY_VERSION="${GCE_PD_DRIVER_VERSION:-stable}"
21+
readonly DEPLOY_OS="${GCE_PD_DRIVER_OS:-linux}"
2122
readonly PKGDIR="${GOPATH}/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver"
2223
source "${PKGDIR}/deploy/common.sh"
2324

@@ -95,6 +96,36 @@ fi
9596
${KUBECTL} version
9697

9798
readonly tmp_spec=/tmp/gcp-compute-persistent-disk-csi-driver-specs-generated.yaml
98-
${KUSTOMIZE_PATH} build ${PKGDIR}/deploy/kubernetes/overlays/${DEPLOY_VERSION} | tee $tmp_spec
99-
${KUBECTL} apply -v="${VERBOSITY}" -f $tmp_spec
10099

100+
if [[ ${DEPLOY_OS} = "mixed" ]]; then
101+
FIRST_OS=linux
102+
SECOND_OS=windows
103+
else
104+
FIRST_OS=${DEPLOY_OS}
105+
fi
106+
107+
os_dir=$(mktemp -d -p ./ -t os-XXXXXXXXXX)
108+
cat <<EOF >${os_dir}/kustomization.yaml
109+
apiVersion: kustomize.config.k8s.io/v1beta1
110+
kind: Kustomization
111+
namespace:
112+
gce-pd-csi-driver
113+
bases:
114+
- ${PKGDIR}/deploy/kubernetes/kustomization/base_setup
115+
- ${PKGDIR}/deploy/kubernetes/kustomization/node_setup/${FIRST_OS}
116+
EOF
117+
118+
if [[ -n ${SECOND_OS} ]]; then
119+
echo "- ../node_setup/${SECOND_OS}" >> ${os_dir}/kustomization.yaml
120+
fi
121+
122+
image_dir=$(mktemp -d -p ./ -t image-XXXXXXXXXX)
123+
cp ${PKGDIR}/deploy/kubernetes/kustomization/image_setup/${DEPLOY_VERSION}/kustomization.yaml ${image_dir}
124+
cat <<EOF >>${image_dir}/kustomization.yaml
125+
bases:
126+
- ../${os_dir}
127+
EOF
128+
129+
${KUSTOMIZE_PATH} build ${image_dir} | tee $tmp_spec; \
130+
${KUBECTL} apply -v="${VERBOSITY}" -f $tmp_spec; \
131+
rm -rf ${os_dir} ${image_dir}

‎deploy/kubernetes/kustomization/base_setup/controller.yaml

+93
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
kind: StatefulSet
2+
apiVersion: apps/v1
3+
metadata:
4+
name: csi-gce-pd-controller
5+
spec:
6+
serviceName: "csi-gce-pd"
7+
replicas: 1
8+
selector:
9+
matchLabels:
10+
app: gcp-compute-persistent-disk-csi-driver
11+
template:
12+
metadata:
13+
labels:
14+
app: gcp-compute-persistent-disk-csi-driver
15+
spec:
16+
# Host network must be used for interaction with Workload Identity in GKE
17+
# since it replaces GCE Metadata Server with GKE Metadata Server. Remove
18+
# this requirement when issue is resolved and before any exposure of
19+
# metrics ports
20+
nodeSelector:
21+
kubernetes.io/os: linux
22+
hostNetwork: true
23+
serviceAccountName: csi-gce-pd-controller-sa
24+
priorityClassName: csi-gce-pd-controller
25+
containers:
26+
- name: csi-provisioner
27+
image: gke.gcr.io/csi-provisioner
28+
args:
29+
- "--v=5"
30+
- "--csi-address=/csi/csi.sock"
31+
- "--feature-gates=Topology=true"
32+
- "--metrics-address=:22011"
33+
# - "--run-controller-service=false" # disable the controller service of the CSI driver
34+
# - "--run-node-service=false" # disable the node service of the CSI driver
35+
volumeMounts:
36+
- name: socket-dir
37+
mountPath: /csi
38+
- name: csi-attacher
39+
image: gke.gcr.io/csi-attacher
40+
args:
41+
- "--v=5"
42+
- "--csi-address=/csi/csi.sock"
43+
- "--metrics-address=:22012"
44+
volumeMounts:
45+
- name: socket-dir
46+
mountPath: /csi
47+
- name: csi-resizer
48+
image: gke.gcr.io/csi-resizer
49+
args:
50+
- "--v=5"
51+
- "--csi-address=/csi/csi.sock"
52+
- "--metrics-address=:22013"
53+
volumeMounts:
54+
- name: socket-dir
55+
mountPath: /csi
56+
- name: csi-snapshotter
57+
image: gke.gcr.io/csi-snapshotter
58+
args:
59+
- "--v=5"
60+
- "--csi-address=/csi/csi.sock"
61+
- "--metrics-address=:22014"
62+
volumeMounts:
63+
- name: socket-dir
64+
mountPath: /csi
65+
- name: gce-pd-driver
66+
# Don't change base image without changing pdImagePlaceholder in
67+
# test/k8s-integration/main.go
68+
image: gke.gcr.io/gcp-compute-persistent-disk-csi-driver
69+
args:
70+
- "--v=5"
71+
- "--endpoint=unix:/csi/csi.sock"
72+
env:
73+
- name: GOOGLE_APPLICATION_CREDENTIALS
74+
value: "/etc/cloud-sa/cloud-sa.json"
75+
volumeMounts:
76+
- name: socket-dir
77+
mountPath: /csi
78+
- name: cloud-sa-volume
79+
readOnly: true
80+
mountPath: "/etc/cloud-sa"
81+
volumes:
82+
- name: socket-dir
83+
emptyDir: {}
84+
- name: cloud-sa-volume
85+
secret:
86+
secretName: cloud-sa
87+
# https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
88+
# See "special case". This will tolerate everything. Node component should
89+
# be scheduled on all nodes.
90+
tolerations:
91+
- operator: Exists
92+
# This is needed due to https://github.com/kubernetes-sigs/kustomize/issues/504
93+
volumeClaimTemplates: []

‎deploy/kubernetes/kustomization/base_setup/kustomization.yaml

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
commonLabels:
2+
k8s-app: gcp-compute-persistent-disk-csi-driver
3+
namespace:
4+
gce-pd-csi-driver
5+
resources:
6+
- controller.yaml
7+
- setup-cluster.yaml

‎deploy/kubernetes/kustomization/base_setup/setup-cluster.yaml

+246
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,246 @@
1+
##### Node Service Account, Roles, RoleBindings
2+
apiVersion: v1
3+
kind: ServiceAccount
4+
metadata:
5+
name: csi-gce-pd-node-sa
6+
7+
---
8+
##### Controller Service Account, Roles, Rolebindings
9+
apiVersion: v1
10+
kind: ServiceAccount
11+
metadata:
12+
name: csi-gce-pd-controller-sa
13+
14+
---
15+
# xref: https://github.com/kubernetes-csi/external-provisioner/blob/master/deploy/kubernetes/rbac.yaml
16+
kind: ClusterRole
17+
apiVersion: rbac.authorization.k8s.io/v1
18+
metadata:
19+
name: csi-gce-pd-provisioner-role
20+
rules:
21+
- apiGroups: [""]
22+
resources: ["persistentvolumes"]
23+
verbs: ["get", "list", "watch", "create", "delete"]
24+
- apiGroups: [""]
25+
resources: ["persistentvolumeclaims"]
26+
verbs: ["get", "list", "watch", "update"]
27+
- apiGroups: ["storage.k8s.io"]
28+
resources: ["storageclasses"]
29+
verbs: ["get", "list", "watch"]
30+
- apiGroups: [""]
31+
resources: ["events"]
32+
verbs: ["list", "watch", "create", "update", "patch"]
33+
- apiGroups: ["storage.k8s.io"]
34+
resources: ["csinodes"]
35+
verbs: ["get", "list", "watch"]
36+
- apiGroups: [""]
37+
resources: ["nodes"]
38+
verbs: ["get", "list", "watch"]
39+
- apiGroups: ["snapshot.storage.k8s.io"]
40+
resources: ["volumesnapshots"]
41+
verbs: ["get", "list"]
42+
- apiGroups: ["snapshot.storage.k8s.io"]
43+
resources: ["volumesnapshotcontents"]
44+
verbs: ["get", "list"]
45+
---
46+
47+
kind: ClusterRoleBinding
48+
apiVersion: rbac.authorization.k8s.io/v1
49+
metadata:
50+
name: csi-gce-pd-controller-provisioner-binding
51+
subjects:
52+
- kind: ServiceAccount
53+
name: csi-gce-pd-controller-sa
54+
roleRef:
55+
kind: ClusterRole
56+
name: csi-gce-pd-provisioner-role
57+
apiGroup: rbac.authorization.k8s.io
58+
59+
---
60+
# xref: https://github.com/kubernetes-csi/external-attacher/blob/master/deploy/kubernetes/rbac.yaml
61+
kind: ClusterRole
62+
apiVersion: rbac.authorization.k8s.io/v1
63+
metadata:
64+
name: csi-gce-pd-attacher-role
65+
rules:
66+
- apiGroups: [""]
67+
resources: ["persistentvolumes"]
68+
verbs: ["get", "list", "watch", "update", "patch"]
69+
- apiGroups: [""]
70+
resources: ["nodes"]
71+
verbs: ["get", "list", "watch"]
72+
- apiGroups: ["storage.k8s.io"]
73+
resources: ["csinodes"]
74+
verbs: ["get", "list", "watch"]
75+
- apiGroups: ["storage.k8s.io"]
76+
resources: ["volumeattachments"]
77+
verbs: ["get", "list", "watch", "update", "patch"]
78+
- apiGroups: ["storage.k8s.io"]
79+
resources: ["volumeattachments/status"]
80+
verbs: ["patch"]
81+
---
82+
83+
kind: ClusterRoleBinding
84+
apiVersion: rbac.authorization.k8s.io/v1
85+
metadata:
86+
name: csi-gce-pd-controller-attacher-binding
87+
subjects:
88+
- kind: ServiceAccount
89+
name: csi-gce-pd-controller-sa
90+
roleRef:
91+
kind: ClusterRole
92+
name: csi-gce-pd-attacher-role
93+
apiGroup: rbac.authorization.k8s.io
94+
95+
---
96+
97+
apiVersion: scheduling.k8s.io/v1
98+
kind: PriorityClass
99+
metadata:
100+
name: csi-gce-pd-controller
101+
value: 900000000
102+
globalDefault: false
103+
description: "This priority class should be used for the GCE PD CSI driver controller deployment only."
104+
105+
---
106+
107+
apiVersion: scheduling.k8s.io/v1
108+
kind: PriorityClass
109+
metadata:
110+
name: csi-gce-pd-node
111+
value: 900001000
112+
globalDefault: false
113+
description: "This priority class should be used for the GCE PD CSI driver node deployment only."
114+
115+
---
116+
117+
# Resizer must be able to work with PVCs, PVs, SCs.
118+
kind: ClusterRole
119+
apiVersion: rbac.authorization.k8s.io/v1
120+
metadata:
121+
name: csi-gce-pd-resizer-role
122+
rules:
123+
- apiGroups: [""]
124+
resources: ["persistentvolumes"]
125+
verbs: ["get", "list", "watch", "update", "patch"]
126+
- apiGroups: [""]
127+
resources: ["persistentvolumeclaims"]
128+
verbs: ["get", "list", "watch"]
129+
- apiGroups: [""]
130+
resources: ["persistentvolumeclaims/status"]
131+
verbs: ["update", "patch"]
132+
- apiGroups: [""]
133+
resources: ["events"]
134+
verbs: ["list", "watch", "create", "update", "patch"]
135+
136+
---
137+
kind: ClusterRoleBinding
138+
apiVersion: rbac.authorization.k8s.io/v1
139+
metadata:
140+
name: csi-gce-pd-resizer-binding
141+
subjects:
142+
- kind: ServiceAccount
143+
name: csi-gce-pd-controller-sa
144+
roleRef:
145+
kind: ClusterRole
146+
name: csi-gce-pd-resizer-role
147+
apiGroup: rbac.authorization.k8s.io
148+
149+
---
150+
apiVersion: policy/v1beta1
151+
kind: PodSecurityPolicy
152+
metadata:
153+
name: csi-gce-pd-node-psp
154+
spec:
155+
seLinux:
156+
rule: RunAsAny
157+
supplementalGroups:
158+
rule: RunAsAny
159+
runAsUser:
160+
rule: RunAsAny
161+
fsGroup:
162+
rule: RunAsAny
163+
privileged: true
164+
volumes:
165+
- '*'
166+
hostNetwork: true
167+
allowedHostPaths:
168+
- pathPrefix: "/var/lib/kubelet/plugins_registry/"
169+
- pathPrefix: "/var/lib/kubelet"
170+
- pathPrefix: "/var/lib/kubelet/plugins/pd.csi.storage.gke.io/"
171+
- pathPrefix: "/dev"
172+
- pathPrefix: "/etc/udev"
173+
- pathPrefix: "/lib/udev"
174+
- pathPrefix: "/run/udev"
175+
- pathPrefix: "/sys"
176+
---
177+
178+
kind: ClusterRole
179+
apiVersion: rbac.authorization.k8s.io/v1
180+
metadata:
181+
name: csi-gce-pd-node-deploy
182+
rules:
183+
- apiGroups: ['policy']
184+
resources: ['podsecuritypolicies']
185+
verbs: ['use']
186+
resourceNames:
187+
- csi-gce-pd-node-psp
188+
---
189+
190+
apiVersion: rbac.authorization.k8s.io/v1
191+
kind: ClusterRoleBinding
192+
metadata:
193+
name: csi-gce-pd-node
194+
roleRef:
195+
apiGroup: rbac.authorization.k8s.io
196+
kind: ClusterRole
197+
name: csi-gce-pd-node-deploy
198+
subjects:
199+
- kind: ServiceAccount
200+
name: csi-gce-pd-node-sa
201+
202+
---
203+
apiVersion: rbac.authorization.k8s.io/v1
204+
kind: ClusterRoleBinding
205+
metadata:
206+
name: csi-gce-pd-controller
207+
roleRef:
208+
apiGroup: rbac.authorization.k8s.io
209+
kind: ClusterRole
210+
name: csi-gce-pd-node-deploy
211+
subjects:
212+
- kind: ServiceAccount
213+
name: csi-gce-pd-controller-sa
214+
215+
---
216+
apiVersion: rbac.authorization.k8s.io/v1
217+
kind: ClusterRole
218+
metadata:
219+
name: csi-gce-pd-snapshotter-role
220+
rules:
221+
- apiGroups: [""]
222+
resources: ["events"]
223+
verbs: ["list", "watch", "create", "update", "patch"]
224+
# Secrets resource omitted since GCE PD snapshots does not require them
225+
- apiGroups: ["snapshot.storage.k8s.io"]
226+
resources: ["volumesnapshotclasses"]
227+
verbs: ["get", "list", "watch"]
228+
- apiGroups: ["snapshot.storage.k8s.io"]
229+
resources: ["volumesnapshotcontents"]
230+
verbs: ["create", "get", "list", "watch", "update", "delete"]
231+
- apiGroups: ["snapshot.storage.k8s.io"]
232+
resources: ["volumesnapshotcontents/status"]
233+
verbs: ["update"]
234+
---
235+
236+
kind: ClusterRoleBinding
237+
apiVersion: rbac.authorization.k8s.io/v1
238+
metadata:
239+
name: csi-gce-pd-controller-snapshotter-binding
240+
subjects:
241+
- kind: ServiceAccount
242+
name: csi-gce-pd-controller-sa
243+
roleRef:
244+
kind: ClusterRole
245+
name: csi-gce-pd-snapshotter-role
246+
apiGroup: rbac.authorization.k8s.io

0 commit comments

Comments
 (0)