Skip to content

Commit eb31712

Browse files
jingxu97jingxu97
committed
New kustomization for pd driver
This can work for both linux and windows Also update to the latest version of kustomize bases is deprecated, use resources instead.
1 parent 85088c4 commit eb31712

File tree

15 files changed

+202
-43
lines changed

15 files changed

+202
-43
lines changed

‎deploy/kubernetes/base/setup-cluster.yaml renamed to ‎deploy/kubernetes/base/controller_setup/cluster_setup.yaml

+38
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,18 @@ rules:
187187
- csi-gce-pd-node-psp
188188
---
189189

190+
kind: ClusterRole
191+
apiVersion: rbac.authorization.k8s.io/v1
192+
metadata:
193+
name: csi-gce-pd-node-deploy-win
194+
rules:
195+
- apiGroups: ['policy']
196+
resources: ['podsecuritypolicies']
197+
verbs: ['use']
198+
resourceNames:
199+
- csi-gce-pd-node-psp-win
200+
---
201+
190202
apiVersion: rbac.authorization.k8s.io/v1
191203
kind: ClusterRoleBinding
192204
metadata:
@@ -198,6 +210,32 @@ roleRef:
198210
subjects:
199211
- kind: ServiceAccount
200212
name: csi-gce-pd-node-sa
213+
---
214+
215+
apiVersion: rbac.authorization.k8s.io/v1
216+
kind: ClusterRoleBinding
217+
metadata:
218+
name: csi-gce-pd-node-win
219+
roleRef:
220+
apiGroup: rbac.authorization.k8s.io
221+
kind: ClusterRole
222+
name: csi-gce-pd-node-deploy-win
223+
subjects:
224+
- kind: ServiceAccount
225+
name: csi-gce-pd-node-sa
226+
227+
---
228+
apiVersion: rbac.authorization.k8s.io/v1
229+
kind: ClusterRoleBinding
230+
metadata:
231+
name: csi-gce-pd-controller
232+
roleRef:
233+
apiGroup: rbac.authorization.k8s.io
234+
kind: ClusterRole
235+
name: csi-gce-pd-node-deploy
236+
subjects:
237+
- kind: ServiceAccount
238+
name: csi-gce-pd-controller-sa
201239

202240
---
203241
apiVersion: rbac.authorization.k8s.io/v1

‎deploy/kubernetes/base/controller.yaml renamed to ‎deploy/kubernetes/base/controller_setup/controller.yaml

+7
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ spec:
1717
# since it replaces GCE Metadata Server with GKE Metadata Server. Remove
1818
# this requirement when issue is resolved and before any exposure of
1919
# metrics ports
20+
nodeSelector:
21+
kubernetes.io/os: linux
2022
hostNetwork: true
2123
serviceAccountName: csi-gce-pd-controller-sa
2224
priorityClassName: csi-gce-pd-controller
@@ -82,5 +84,10 @@ spec:
8284
- name: cloud-sa-volume
8385
secret:
8486
secretName: cloud-sa
87+
# https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
88+
# See "special case". This will tolerate everything. Node component should
89+
# be scheduled on all nodes.
90+
tolerations:
91+
- operator: Exists
8592
# This is needed due to https://github.com/kubernetes-sigs/kustomize/issues/504
8693
volumeClaimTemplates: []

‎deploy/kubernetes/base/controller_setup/kustomization.yaml

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
commonLabels:
2+
k8s-app: gcp-compute-persistent-disk-csi-driver
3+
namespace:
4+
gce-pd-csi-driver
5+
resources:
6+
- controller.yaml
7+
- cluster_setup.yaml

‎deploy/kubernetes/base/kustomization.yaml

+4-5
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
1-
commonLabels:
2-
k8s-app: gcp-compute-persistent-disk-csi-driver
1+
apiVersion: kustomize.config.k8s.io/v1beta1
2+
kind: Kustomization
33
namespace:
44
gce-pd-csi-driver
55
resources:
6-
- node.yaml
7-
- controller.yaml
8-
- setup-cluster.yaml
6+
- controller_setup
7+
- node_setup

‎deploy/kubernetes/base/node_setup/kustomization.yaml

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
apiVersion: kustomize.config.k8s.io/v1beta1
2+
kind: Kustomization
3+
namespace:
4+
gce-pd-csi-driver
5+
resources:
6+
- linux/node.yaml
7+
- windows/node.yaml
8+
- windows/psp.yaml

‎deploy/kubernetes/base/node.yaml renamed to ‎deploy/kubernetes/base/node_setup/linux/node.yaml

+6-9
Original file line numberDiff line numberDiff line change
@@ -15,21 +15,20 @@ spec:
1515
# Host network must be used for interaction with Workload Identity in GKE
1616
# since it replaces GCE Metadata Server with GKE Metadata Server. Remove
1717
# this requirement when issue is resolved and before any exposure of
18-
# metrics ports.
18+
# metrics ports. But hostNetwork is not working for Windows, might be an issue
19+
# when deploying on GKE Windows node.
1920
hostNetwork: true
2021
priorityClassName: csi-gce-pd-node
2122
serviceAccountName: csi-gce-pd-node-sa
23+
nodeSelector:
24+
kubernetes.io/os: linux
2225
containers:
2326
- name: csi-driver-registrar
2427
image: gke.gcr.io/csi-node-driver-registrar
2528
args:
2629
- "--v=5"
2730
- "--csi-address=/csi/csi.sock"
2831
- "--kubelet-registration-path=/var/lib/kubelet/plugins/pd.csi.storage.gke.io/csi.sock"
29-
lifecycle:
30-
preStop:
31-
exec:
32-
command: ["/bin/sh", "-c", "rm -rf /registration/pd.csi.storage.gke.io /registration/pd.csi.storage.gke.io-reg.sock"]
3332
env:
3433
- name: KUBE_NODE_NAME
3534
valueFrom:
@@ -41,14 +40,14 @@ spec:
4140
- name: registration-dir
4241
mountPath: /registration
4342
- name: gce-pd-driver
44-
securityContext:
45-
privileged: true
4643
# Don't change base image without changing pdImagePlaceholder in
4744
# test/k8s-integration/main.go
4845
image: gke.gcr.io/gcp-compute-persistent-disk-csi-driver
4946
args:
5047
- "--v=5"
5148
- "--endpoint=unix:/csi/csi.sock"
49+
securityContext:
50+
privileged: true
5251
volumeMounts:
5352
- name: kubelet-dir
5453
mountPath: /var/lib/kubelet
@@ -67,8 +66,6 @@ spec:
6766
mountPath: /run/udev
6867
- name: sys
6968
mountPath: /sys
70-
nodeSelector:
71-
kubernetes.io/os: linux
7269
volumes:
7370
- name: registration-dir
7471
hostPath:

‎deploy/kubernetes/base/node_setup/windows/node.yaml

+82
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
#TODO(#40): Force DaemonSet to not run on master.
2+
kind: DaemonSet
3+
apiVersion: apps/v1
4+
metadata:
5+
name: csi-gce-pd-node-win
6+
spec:
7+
selector:
8+
matchLabels:
9+
app: gcp-compute-persistent-disk-csi-driver
10+
template:
11+
metadata:
12+
labels:
13+
app: gcp-compute-persistent-disk-csi-driver
14+
spec:
15+
# Host network must be used for interaction with Workload Identity in GKE
16+
# since it replaces GCE Metadata Server with GKE Metadata Server. Remove
17+
# this requirement when issue is resolved and before any exposure of
18+
# metrics ports. But hostNetwork is not working for Windows, might be an issue
19+
# when deploying on GKE Windows node.
20+
# hostNetwork: true
21+
priorityClassName: csi-gce-pd-node
22+
serviceAccountName: csi-gce-pd-node-sa
23+
nodeSelector:
24+
kubernetes.io/os: windows
25+
containers:
26+
- name: csi-driver-registrar
27+
image: gcr.io/k8s-staging-csi/csi-node-driver-registrar:amd64-windows-v20200428-v1.3.0-26-g510710d5
28+
args:
29+
- --v=5
30+
- --csi-address=unix://C:\\csi\\csi.sock
31+
- --kubelet-registration-path=C:\\var\\lib\\kubelet\\plugins\\pd.csi.storage.gke.io\\csi.sock
32+
env:
33+
- name: KUBE_NODE_NAME
34+
valueFrom:
35+
fieldRef:
36+
fieldPath: spec.nodeName
37+
volumeMounts:
38+
- name: plugin-dir
39+
mountPath: /csi
40+
- name: registration-dir
41+
mountPath: /registration
42+
- name: gce-pd-driver
43+
# Don't change base image without changing pdImagePlaceholder in
44+
# test/k8s-integration/main.go
45+
image: gcr.io/jing-k8s-dev/gce-pd-windows-2019:0.2.0
46+
args:
47+
- "--v=5"
48+
- "--endpoint=unix:/csi/csi.sock"
49+
volumeMounts:
50+
- name: kubelet-dir
51+
mountPath: C:\var\lib\kubelet
52+
mountPropagation: "None"
53+
- name: plugin-dir
54+
mountPath: C:\csi
55+
- name: csi-proxy-disk-pipe
56+
mountPath: \\.\pipe\csi-proxy-disk-v1alpha1
57+
- name: csi-proxy-volume-pipe
58+
mountPath: \\.\pipe\csi-proxy-volume-v1alpha1
59+
- name: csi-proxy-filesystem-pipe
60+
mountPath: \\.\pipe\csi-proxy-filesystem-v1alpha1
61+
volumes:
62+
- name: csi-proxy-disk-pipe
63+
hostPath:
64+
path: \\.\pipe\csi-proxy-disk-v1alpha1
65+
type: ""
66+
- name: csi-proxy-volume-pipe
67+
hostPath:
68+
path: \\.\pipe\csi-proxy-volume-v1alpha1
69+
type: ""
70+
- name: csi-proxy-filesystem-pipe
71+
hostPath:
72+
path: \\.\pipe\csi-proxy-filesystem-v1alpha1
73+
type: ""
74+
- name: registration-dir
75+
hostPath:
76+
path: \var\lib\kubelet\plugins_registry
77+
- name: kubelet-dir
78+
hostPath:
79+
path: \var\lib\kubelet
80+
- name: plugin-dir
81+
hostPath:
82+
path: \var\lib\kubelet\plugins\pd.csi.storage.gke.io

‎deploy/kubernetes/base/node_setup/windows/psp.yaml

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
apiVersion: policy/v1beta1
2+
kind: PodSecurityPolicy
3+
metadata:
4+
name: csi-gce-pd-node-psp-win
5+
spec:
6+
supplementalGroups:
7+
rule: RunAsAny
8+
runAsUser:
9+
rule: RunAsAny
10+
fsGroup:
11+
rule: RunAsAny
12+
seLinux:
13+
rule: RunAsAny
14+
volumes:
15+
- '*'
16+
hostNetwork: true
17+
allowedHostPaths:
18+
- pathPrefix: \var\lib\kubelet
19+
- pathPrefix: \var\lib\kubelet\plugins_registry
20+
- pathPrefix: \var\lib\kubelet\plugins\pd.csi.storage.gke.io
21+
- pathPrefix: \\.\pipe\csi-proxy-disk-v1alpha1
22+
- pathPrefix: \\.\pipe\csi-proxy-volume-v1alpha1
23+
- pathPrefix: \\.\pipe\csi-proxy-filesystem-v1alpha1

‎deploy/kubernetes/deploy-driver.sh

-1
Original file line numberDiff line numberDiff line change
@@ -97,4 +97,3 @@ ${KUBECTL} version
9797
readonly tmp_spec=/tmp/gcp-compute-persistent-disk-csi-driver-specs-generated.yaml
9898
${KUSTOMIZE_PATH} build ${PKGDIR}/deploy/kubernetes/overlays/${DEPLOY_VERSION} | tee $tmp_spec
9999
${KUBECTL} apply -v="${VERBOSITY}" -f $tmp_spec
100-

‎deploy/kubernetes/install-kustomize.sh

+6-11
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ set -o errexit
99

1010
readonly INSTALL_DIR="${GOPATH}/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/bin"
1111
readonly KUSTOMIZE_PATH="${INSTALL_DIR}/kustomize"
12-
readonly KUSTOMIZE_VERSION="2.0.3"
12+
readonly KUSTOMIZE_VERSION="3.6.1"
1313
readonly VERSION_REGEX="KustomizeVersion:([0-9]\.[0-9]\.[0-9])"
1414

1515
if [ -f "${KUSTOMIZE_PATH}" ]; then
@@ -25,14 +25,9 @@ if [ ! -f "${KUSTOMIZE_PATH}" ]; then
2525
if [ ! -f "${INSTALL_DIR}" ]; then
2626
mkdir -p ${INSTALL_DIR}
2727
fi
28-
29-
echo "Installing Kustomize v${KUSTOMIZE_VERSION} in ${KUSTOMIZE_PATH}"
30-
opsys=linux # or darwin, or windows
31-
curl -s https://api.github.com/repos/kubernetes-sigs/kustomize/releases/tags/v${KUSTOMIZE_VERSION} |\
32-
grep browser_download |\
33-
grep $opsys |\
34-
cut -d '"' -f 4 |\
35-
xargs curl -O -L
36-
mv kustomize_*_${opsys}_amd64 ${KUSTOMIZE_PATH}
37-
chmod u+x ${KUSTOMIZE_PATH}
28+
if [ -f "kustomize" ]; then
29+
rm kustomize
30+
fi
31+
curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash
32+
mv kustomize ${INSTALL_DIR}
3833
fi

‎deploy/kubernetes/overlays/alpha/kustomization.yaml

+3-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
apiVersion: kustomize.config.k8s.io/v1beta1
22
kind: Kustomization
3-
bases:
3+
namespace:
4+
gce-pd-csi-driver
5+
resources:
46
- ../stable
5-

‎deploy/kubernetes/overlays/dev/kustomization.yaml

+4-3
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,14 @@
11
apiVersion: kustomize.config.k8s.io/v1beta1
22
kind: Kustomization
3-
bases:
4-
- ../alpha
3+
resources:
4+
- ../stable
55
patches:
66
- controller_always_pull.yaml
77
- node_always_pull.yaml
8+
namespace:
9+
gce-pd-csi-driver
810
images:
911
# Replace this with your private image names and tags
1012
- name: gke.gcr.io/gcp-compute-persistent-disk-csi-driver
1113
newName: gcr.io/dyzz-csi-staging/csi/gce-pd-driver
1214
newTag: "latest"
13-

‎deploy/kubernetes/overlays/prow-gke-release-staging-head/kustomization.yaml

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
apiVersion: kustomize.config.k8s.io/v1beta1
22
kind: Kustomization
3-
bases:
4-
- ../../base
3+
resources:
4+
- ../stable
55
images:
66
- name: gke.gcr.io/gcp-compute-persistent-disk-csi-driver
77
newName: gcr.io/gke-release-staging/gcp-compute-persistent-disk-csi-driver

‎deploy/kubernetes/overlays/prow-gke-release-staging-rc/kustomization.yaml

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
apiVersion: kustomize.config.k8s.io/v1beta1
22
kind: Kustomization
3-
bases:
4-
- ../../base
3+
resources:
4+
- ../stable
55
images:
66
- name: gke.gcr.io/gcp-compute-persistent-disk-csi-driver
77
newName: gcr.io/gke-release-staging/gcp-compute-persistent-disk-csi-driver

‎deploy/kubernetes/overlays/stable/kustomization.yaml

+10-8
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,27 @@
11
apiVersion: kustomize.config.k8s.io/v1beta1
22
kind: Kustomization
3-
bases:
4-
- ../../base
3+
namespace:
4+
gce-pd-csi-driver
55
images:
6-
- name: gke.gcr.io/gcp-compute-persistent-disk-csi-driver
76
# Don't change stable image without changing pdImagePlaceholder in
87
# test/k8s-integration/main.go
9-
newName: gke.gcr.io/gcp-compute-persistent-disk-csi-driver
10-
newTag: "v0.7.0-gke.0"
118
- name: gke.gcr.io/csi-provisioner
129
newName: gke.gcr.io/csi-provisioner
1310
newTag: "v1.6.0-gke.0"
1411
- name: gke.gcr.io/csi-attacher
1512
newName: gke.gcr.io/csi-attacher
1613
newTag: "v2.2.0-gke.0"
17-
- name: gke.gcr.io/csi-node-driver-registrar
18-
newName: gke.gcr.io/csi-node-driver-registrar
19-
newTag: "v1.3.0-gke.0"
2014
- name: gke.gcr.io/csi-resizer
2115
newName: gke.gcr.io/csi-resizer
2216
newTag: "v0.5.0-gke.0"
2317
- name: gke.gcr.io/csi-snapshotter
2418
newName: gke.gcr.io/csi-snapshotter
2519
newTag: "v2.1.1-gke.0"
20+
- name: gke.gcr.io/csi-node-driver-registrar
21+
newName: gke.gcr.io/csi-node-driver-registrar
22+
newTag: "v1.3.0-gke.0"
23+
- name: gke.gcr.io/gcp-compute-persistent-disk-csi-driver
24+
newName: gke.gcr.io/gcp-compute-persistent-disk-csi-driver
25+
newTag: "v0.7.0-gke.0"
26+
resources:
27+
- ../../base/

0 commit comments

Comments
 (0)