Skip to content

Commit bed7829

Browse files
jingxu97jingxu97
committed
WIP New kustomization for pd driver
This can work for both linux and windows
1 parent 85088c4 commit bed7829

16 files changed

+1447
-2
lines changed

‎deploy/kubernetes/deploy-driver-new.sh

+131
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
#!/bin/bash
2+
3+
# This script will deploy the Google Compute Engine Persistent Disk CSI Driver
4+
# to the currently available Kubernetes cluster
5+
6+
# Note: setup-cluster.yaml depends on the existence of cluster-roles
7+
# system:csi-external-attacher and system:csi-external-provisioner
8+
# which are in Kubernetes version 1.10.5+
9+
10+
# Args:
11+
# GCE_PD_SA_DIR: Directory the service account key has been saved in (generated by setup-project.sh)
12+
# GCE_PD_DRIVER_VERSION: The kustomize overlay (located in
13+
# deploy/kubernetes/overlays) to deploy. Can be one of {stable, dev}
14+
15+
set -o nounset
16+
set -o errexit
17+
set -x
18+
19+
readonly NAMESPACE="${GCE_PD_DRIVER_NAMESPACE:-gce-pd-csi-driver}"
20+
readonly DEPLOY_VERSION="${GCE_PD_DRIVER_VERSION:-stable}"
21+
readonly DEPLOY_OS="${GCE_PD_DRIVER_OS:-linux}"
22+
readonly PKGDIR="${GOPATH}/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver"
23+
source "${PKGDIR}/deploy/common.sh"
24+
25+
print_usage()
26+
{
27+
echo "deploy-driver.sh [--skip-sa-check]\n"
28+
echo "\t--skip-sa-check: don't check the service account for required roles"
29+
echo
30+
}
31+
32+
skip_sa_check=
33+
while [ ! -z "${1-}" ]; do
34+
case $1 in
35+
--skip-sa-check ) shift
36+
skip_sa_check=true
37+
;;
38+
-h | --help ) print_usage
39+
exit 1
40+
;;
41+
* ) print_usage
42+
exit 1
43+
;;
44+
esac
45+
done
46+
47+
ensure_var GCE_PD_SA_DIR
48+
49+
function check_service_account()
50+
{
51+
# Using bash magic to parse JSON for IAM
52+
# Grepping for a line with client email returning anything quoted after the colon
53+
readonly IAM_NAME=$(grep -Po '"client_email": *\K"[^"]*"' ${GCE_PD_SA_DIR}/cloud-sa.json | tr -d '"')
54+
readonly PROJECT=$(grep -Po '"project_id": *\K"[^"]*"' ${GCE_PD_SA_DIR}/cloud-sa.json | tr -d '"')
55+
readonly GOTTEN_BIND_ROLES=$(gcloud projects get-iam-policy ${PROJECT} --flatten="bindings[].members" --format='table(bindings.role)' --filter="bindings.members:${IAM_NAME}")
56+
readonly BIND_ROLES=$(get_needed_roles)
57+
MISSING_ROLES=false
58+
for role in ${BIND_ROLES}
59+
do
60+
if ! grep -q $role <<<${GOTTEN_BIND_ROLES} ;
61+
then
62+
echo "Missing role: $role"
63+
MISSING_ROLES=true
64+
fi
65+
done
66+
if [ "${MISSING_ROLES}" = true ];
67+
then
68+
echo "Cannot deploy with missing roles in service account, please run setup-project.sh to setup Service Account"
69+
exit 1
70+
fi
71+
}
72+
73+
ensure_kustomize
74+
75+
if [ "$skip_sa_check" != true ]; then
76+
check_service_account
77+
fi
78+
79+
if ! ${KUBECTL} get namespace ${NAMESPACE} -v="${VERBOSITY}";
80+
then
81+
${KUBECTL} create namespace ${NAMESPACE} -v="${VERBOSITY}"
82+
fi
83+
84+
if ! ${KUBECTL} get secret cloud-sa -v="${VERBOSITY}" -n ${NAMESPACE};
85+
then
86+
${KUBECTL} create secret generic cloud-sa -v="${VERBOSITY}" --from-file="${GCE_PD_SA_DIR}/cloud-sa.json" -n ${NAMESPACE}
87+
fi
88+
89+
# GKE Required Setup
90+
if ! ${KUBECTL} get clusterrolebinding -v="${VERBOSITY}" cluster-admin-binding;
91+
then
92+
${KUBECTL} create clusterrolebinding cluster-admin-binding -v="${VERBOSITY}" --clusterrole cluster-admin --user $(gcloud config get-value account)
93+
fi
94+
95+
# Debug log: print ${KUBECTL} version
96+
${KUBECTL} version
97+
98+
readonly tmp_spec=/tmp/gcp-compute-persistent-disk-csi-driver-specs-generated.yaml
99+
100+
if [[ ${DEPLOY_OS} = "mixed" ]]; then
101+
FIRST_OS=linux
102+
SECOND_OS=windows
103+
else
104+
FIRST_OS=${DEPLOY_OS}
105+
fi
106+
107+
os_dir=$(mktemp -d -p ./ -t os-XXXXXXXXXX)
108+
cat <<EOF >${os_dir}/kustomization.yaml
109+
apiVersion: kustomize.config.k8s.io/v1beta1
110+
kind: Kustomization
111+
namespace:
112+
gce-pd-csi-driver
113+
bases:
114+
- ${PKGDIR}/deploy/kubernetes/kustomization/base_setup
115+
- ${PKGDIR}/deploy/kubernetes/kustomization/node_setup/${FIRST_OS}
116+
EOF
117+
118+
if [[ -n ${SECOND_OS} ]]; then
119+
echo "- ../node_setup/${SECOND_OS}" >> ${os_dir}/kustomization.yaml
120+
fi
121+
122+
image_dir=$(mktemp -d -p ./ -t image-XXXXXXXXXX)
123+
cp ${PKGDIR}/deploy/kubernetes/kustomization/image_setup/${DEPLOY_VERSION}/kustomization.yaml ${image_dir}
124+
cat <<EOF >>${image_dir}/kustomization.yaml
125+
bases:
126+
- ../${os_dir}
127+
EOF
128+
129+
${KUSTOMIZE_PATH} build ${image_dir} | tee $tmp_spec; \
130+
${KUBECTL} apply -v="${VERBOSITY}" -f $tmp_spec; \
131+
rm -rf ${os_dir} ${image_dir}

‎deploy/kubernetes/deploy-driver.sh

+33-2
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ set -x
1818

1919
readonly NAMESPACE="${GCE_PD_DRIVER_NAMESPACE:-gce-pd-csi-driver}"
2020
readonly DEPLOY_VERSION="${GCE_PD_DRIVER_VERSION:-stable}"
21+
readonly DEPLOY_OS="${GCE_PD_DRIVER_OS:-linux}"
2122
readonly PKGDIR="${GOPATH}/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver"
2223
source "${PKGDIR}/deploy/common.sh"
2324

@@ -95,6 +96,36 @@ fi
9596
${KUBECTL} version
9697

9798
readonly tmp_spec=/tmp/gcp-compute-persistent-disk-csi-driver-specs-generated.yaml
98-
${KUSTOMIZE_PATH} build ${PKGDIR}/deploy/kubernetes/overlays/${DEPLOY_VERSION} | tee $tmp_spec
99-
${KUBECTL} apply -v="${VERBOSITY}" -f $tmp_spec
10099

100+
if [[ ${DEPLOY_OS} = "mixed" ]]; then
101+
FIRST_OS=linux
102+
SECOND_OS=windows
103+
else
104+
FIRST_OS=${DEPLOY_OS}
105+
fi
106+
107+
os_dir=$(mktemp -d -p ./ -t os-XXXXXXXXXX)
108+
cat <<EOF >${os_dir}/kustomization.yaml
109+
apiVersion: kustomize.config.k8s.io/v1beta1
110+
kind: Kustomization
111+
namespace:
112+
gce-pd-csi-driver
113+
bases:
114+
- ${PKGDIR}/deploy/kubernetes/kustomization/base_setup
115+
- ${PKGDIR}/deploy/kubernetes/kustomization/node_setup/${FIRST_OS}
116+
EOF
117+
118+
if [[ -n ${SECOND_OS} ]]; then
119+
echo "- ../node_setup/${SECOND_OS}" >> ${os_dir}/kustomization.yaml
120+
fi
121+
122+
image_dir=$(mktemp -d -p ./ -t image-XXXXXXXXXX)
123+
cp ${PKGDIR}/deploy/kubernetes/kustomization/image_setup/${DEPLOY_VERSION}/kustomization.yaml ${image_dir}
124+
cat <<EOF >>${image_dir}/kustomization.yaml
125+
bases:
126+
- ../${os_dir}
127+
EOF
128+
129+
${KUSTOMIZE_PATH} build ${image_dir} | tee $tmp_spec; \
130+
${KUBECTL} apply -v="${VERBOSITY}" -f $tmp_spec; \
131+
rm -rf ${os_dir} ${image_dir}

‎deploy/kubernetes/kustomization/base_setup/controller.yaml

+93
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
kind: StatefulSet
2+
apiVersion: apps/v1
3+
metadata:
4+
name: csi-gce-pd-controller
5+
spec:
6+
serviceName: "csi-gce-pd"
7+
replicas: 1
8+
selector:
9+
matchLabels:
10+
app: gcp-compute-persistent-disk-csi-driver
11+
template:
12+
metadata:
13+
labels:
14+
app: gcp-compute-persistent-disk-csi-driver
15+
spec:
16+
# Host network must be used for interaction with Workload Identity in GKE
17+
# since it replaces GCE Metadata Server with GKE Metadata Server. Remove
18+
# this requirement when issue is resolved and before any exposure of
19+
# metrics ports
20+
nodeSelector:
21+
kubernetes.io/os: linux
22+
hostNetwork: true
23+
serviceAccountName: csi-gce-pd-controller-sa
24+
priorityClassName: csi-gce-pd-controller
25+
containers:
26+
- name: csi-provisioner
27+
image: gke.gcr.io/csi-provisioner
28+
args:
29+
- "--v=5"
30+
- "--csi-address=/csi/csi.sock"
31+
- "--feature-gates=Topology=true"
32+
- "--metrics-address=:22011"
33+
# - "--run-controller-service=false" # disable the controller service of the CSI driver
34+
# - "--run-node-service=false" # disable the node service of the CSI driver
35+
volumeMounts:
36+
- name: socket-dir
37+
mountPath: /csi
38+
- name: csi-attacher
39+
image: gke.gcr.io/csi-attacher
40+
args:
41+
- "--v=5"
42+
- "--csi-address=/csi/csi.sock"
43+
- "--metrics-address=:22012"
44+
volumeMounts:
45+
- name: socket-dir
46+
mountPath: /csi
47+
- name: csi-resizer
48+
image: gke.gcr.io/csi-resizer
49+
args:
50+
- "--v=5"
51+
- "--csi-address=/csi/csi.sock"
52+
- "--metrics-address=:22013"
53+
volumeMounts:
54+
- name: socket-dir
55+
mountPath: /csi
56+
- name: csi-snapshotter
57+
image: gke.gcr.io/csi-snapshotter
58+
args:
59+
- "--v=5"
60+
- "--csi-address=/csi/csi.sock"
61+
- "--metrics-address=:22014"
62+
volumeMounts:
63+
- name: socket-dir
64+
mountPath: /csi
65+
- name: gce-pd-driver
66+
# Don't change base image without changing pdImagePlaceholder in
67+
# test/k8s-integration/main.go
68+
image: gke.gcr.io/gcp-compute-persistent-disk-csi-driver
69+
args:
70+
- "--v=5"
71+
- "--endpoint=unix:/csi/csi.sock"
72+
env:
73+
- name: GOOGLE_APPLICATION_CREDENTIALS
74+
value: "/etc/cloud-sa/cloud-sa.json"
75+
volumeMounts:
76+
- name: socket-dir
77+
mountPath: /csi
78+
- name: cloud-sa-volume
79+
readOnly: true
80+
mountPath: "/etc/cloud-sa"
81+
volumes:
82+
- name: socket-dir
83+
emptyDir: {}
84+
- name: cloud-sa-volume
85+
secret:
86+
secretName: cloud-sa
87+
# https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
88+
# See "special case". This will tolerate everything. Node component should
89+
# be scheduled on all nodes.
90+
tolerations:
91+
- operator: Exists
92+
# This is needed due to https://github.com/kubernetes-sigs/kustomize/issues/504
93+
volumeClaimTemplates: []

‎deploy/kubernetes/kustomization/base_setup/kustomization.yaml

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
commonLabels:
2+
k8s-app: gcp-compute-persistent-disk-csi-driver
3+
namespace:
4+
gce-pd-csi-driver
5+
resources:
6+
- controller.yaml
7+
- setup-cluster.yaml

0 commit comments

Comments
 (0)