diff --git a/deploy/kubernetes/base/controller.yaml b/deploy/kubernetes/base/controller.yaml index 1dcb10c46..e357b3166 100644 --- a/deploy/kubernetes/base/controller.yaml +++ b/deploy/kubernetes/base/controller.yaml @@ -48,6 +48,14 @@ spec: volumeMounts: - name: socket-dir mountPath: /csi + - name: csi-snapshotter + image: gke.gcr.io/csi-snapshotter + args: + - "--v=5" + - "--csi-address=/csi/csi.sock" + volumeMounts: + - name: socket-dir + mountPath: /csi - name: gce-pd-driver # Don't change base image without changing pdImagePlaceholder in # test/k8s-integration/main.go diff --git a/deploy/kubernetes/base/setup-cluster.yaml b/deploy/kubernetes/base/setup-cluster.yaml index fc0a896d0..0a70f781e 100644 --- a/deploy/kubernetes/base/setup-cluster.yaml +++ b/deploy/kubernetes/base/setup-cluster.yaml @@ -36,7 +36,12 @@ rules: - apiGroups: [""] resources: ["nodes"] verbs: ["get", "list", "watch"] - + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list"] --- kind: ClusterRoleBinding @@ -193,3 +198,36 @@ roleRef: subjects: - kind: ServiceAccount name: csi-gce-pd-node-sa + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: csi-gce-pd-snapshotter-role +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + # Secrets resource omitted since GCE PD snapshots does not require them + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update"] +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-gce-pd-controller-snapshotter-binding +subjects: + - kind: ServiceAccount + name: csi-gce-pd-controller-sa +roleRef: + kind: ClusterRole + name: csi-gce-pd-snapshotter-role + apiGroup: rbac.authorization.k8s.io diff --git a/deploy/kubernetes/overlays/alpha/controller_add_snapshotter.yaml b/deploy/kubernetes/overlays/alpha/controller_add_snapshotter.yaml deleted file mode 100644 index 76c8039fa..000000000 --- a/deploy/kubernetes/overlays/alpha/controller_add_snapshotter.yaml +++ /dev/null @@ -1,17 +0,0 @@ -kind: StatefulSet -apiVersion: apps/v1 -metadata: - name: csi-gce-pd-controller -spec: - template: - spec: - containers: - - name: csi-snapshotter - imagePullPolicy: Always - image: gke.gcr.io/csi-snapshotter:v1.2.2-gke.0 - args: - - "--v=5" - - "--csi-address=/csi/csi.sock" - volumeMounts: - - name: socket-dir - mountPath: /csi diff --git a/deploy/kubernetes/overlays/alpha/kustomization.yaml b/deploy/kubernetes/overlays/alpha/kustomization.yaml index dbfbc9ec3..06cb34b25 100644 --- a/deploy/kubernetes/overlays/alpha/kustomization.yaml +++ b/deploy/kubernetes/overlays/alpha/kustomization.yaml @@ -2,17 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization bases: - ../stable -patches: -- controller_add_snapshotter.yaml -patchesJson6902: -- target: - group: rbac.authorization.k8s.io - version: v1 - kind: ClusterRole - name: csi-gce-pd-provisioner-role - path: rbac_add_snapshots_to_provisioner.yaml -resources: -- rbac_add_snapshotter.yaml -# Reapplying namespace transformer to include newly added RBAC rules. -namespace: - gce-pd-csi-driver + diff --git a/deploy/kubernetes/overlays/alpha/rbac_add_snapshots_to_provisioner.yaml b/deploy/kubernetes/overlays/alpha/rbac_add_snapshots_to_provisioner.yaml deleted file mode 100644 index 746ea82fe..000000000 --- a/deploy/kubernetes/overlays/alpha/rbac_add_snapshots_to_provisioner.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# arrays without strategic patch merge defined need to be appended -# using jsonpatch -# https://github.com/kubernetes-sigs/kustomize/blob/master/examples/jsonpatch.md -- op: add - path: /rules/- - value: - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list"] - -- op: add - path: /rules/- - value: - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["get", "list"] diff --git a/deploy/kubernetes/overlays/alpha/rbac_add_snapshotter.yaml b/deploy/kubernetes/overlays/alpha/rbac_add_snapshotter.yaml deleted file mode 100644 index 3902b6ae6..000000000 --- a/deploy/kubernetes/overlays/alpha/rbac_add_snapshotter.yaml +++ /dev/null @@ -1,48 +0,0 @@ -# xref: https://github.com/kubernetes-csi/external-snapshotter/blob/master/deploy/kubernetes/rbac.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: csi-gce-pd-snapshotter-role -rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - # Secrets resource omitted since GCE PD snapshots does not require them - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "get", "list", "watch", "update", "delete"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots/status"] - verbs: ["update"] - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["create", "list", "watch", "delete"] - ---- - -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-gce-pd-controller-snapshotter-binding -subjects: - - kind: ServiceAccount - name: csi-gce-pd-controller-sa -roleRef: - kind: ClusterRole - name: csi-gce-pd-snapshotter-role - apiGroup: rbac.authorization.k8s.io diff --git a/deploy/kubernetes/overlays/prow-gke-release-staging-head/controller_add_snapshotter.yaml b/deploy/kubernetes/overlays/prow-gke-release-staging-head/controller_add_snapshotter.yaml deleted file mode 100644 index fdc225977..000000000 --- a/deploy/kubernetes/overlays/prow-gke-release-staging-head/controller_add_snapshotter.yaml +++ /dev/null @@ -1,17 +0,0 @@ -kind: StatefulSet -apiVersion: apps/v1 -metadata: - name: csi-gce-pd-controller -spec: - template: - spec: - containers: - - name: csi-snapshotter - imagePullPolicy: Always - image: quay.io/k8scsi/csi-snapshotter:canary - args: - - "--v=5" - - "--csi-address=/csi/csi.sock" - volumeMounts: - - name: socket-dir - mountPath: /csi diff --git a/deploy/kubernetes/overlays/prow-gke-release-staging-head/kustomization.yaml b/deploy/kubernetes/overlays/prow-gke-release-staging-head/kustomization.yaml index 9a91fb988..a882ae3d8 100644 --- a/deploy/kubernetes/overlays/prow-gke-release-staging-head/kustomization.yaml +++ b/deploy/kubernetes/overlays/prow-gke-release-staging-head/kustomization.yaml @@ -18,17 +18,6 @@ images: - name: gke.gcr.io/csi-resizer newName: quay.io/k8scsi/csi-resizer newTag: "canary" -patches: -- controller_add_snapshotter.yaml -patchesJson6902: -- target: - group: rbac.authorization.k8s.io - version: v1 - kind: ClusterRole - name: csi-gce-pd-provisioner-role - path: rbac_add_snapshots_to_provisioner.yaml -resources: -- rbac_add_snapshotter.yaml -# Reapplying namespace transformer to include newly added RBAC rules. -namespace: - gce-pd-csi-driver +- name: gke.gcr.io/csi-snapshotter + newName: quay.io/k8scsi/csi-snapshotter + newTag: "canary" diff --git a/deploy/kubernetes/overlays/prow-gke-release-staging-head/rbac_add_snapshots_to_provisioner.yaml b/deploy/kubernetes/overlays/prow-gke-release-staging-head/rbac_add_snapshots_to_provisioner.yaml deleted file mode 100644 index 746ea82fe..000000000 --- a/deploy/kubernetes/overlays/prow-gke-release-staging-head/rbac_add_snapshots_to_provisioner.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# arrays without strategic patch merge defined need to be appended -# using jsonpatch -# https://github.com/kubernetes-sigs/kustomize/blob/master/examples/jsonpatch.md -- op: add - path: /rules/- - value: - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list"] - -- op: add - path: /rules/- - value: - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["get", "list"] diff --git a/deploy/kubernetes/overlays/prow-gke-release-staging-head/rbac_add_snapshotter.yaml b/deploy/kubernetes/overlays/prow-gke-release-staging-head/rbac_add_snapshotter.yaml deleted file mode 100644 index dcc7af90a..000000000 --- a/deploy/kubernetes/overlays/prow-gke-release-staging-head/rbac_add_snapshotter.yaml +++ /dev/null @@ -1,32 +0,0 @@ -# xref: https://github.com/kubernetes-csi/external-snapshotter/blob/master/deploy/kubernetes/rbac.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: csi-gce-pd-snapshotter-role -rules: - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - # Secrets resource omitted since GCE PD snapshots does not require them - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "get", "list", "watch", "update", "delete"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents/status"] - verbs: ["update"] ---- - -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-gce-pd-controller-snapshotter-binding -subjects: - - kind: ServiceAccount - name: csi-gce-pd-controller-sa -roleRef: - kind: ClusterRole - name: csi-gce-pd-snapshotter-role - apiGroup: rbac.authorization.k8s.io diff --git a/deploy/kubernetes/overlays/prow-gke-release-staging-rc/controller_add_snapshotter.yaml b/deploy/kubernetes/overlays/prow-gke-release-staging-rc/controller_add_snapshotter.yaml deleted file mode 100644 index 27f1dd727..000000000 --- a/deploy/kubernetes/overlays/prow-gke-release-staging-rc/controller_add_snapshotter.yaml +++ /dev/null @@ -1,17 +0,0 @@ -kind: StatefulSet -apiVersion: apps/v1 -metadata: - name: csi-gce-pd-controller -spec: - template: - spec: - containers: - - name: csi-snapshotter - imagePullPolicy: Always - image: gcr.io/gke-release-staging/csi-snapshotter:v2.1.1-gke.0 - args: - - "--v=5" - - "--csi-address=/csi/csi.sock" - volumeMounts: - - name: socket-dir - mountPath: /csi diff --git a/deploy/kubernetes/overlays/prow-gke-release-staging-rc/enable_sidecar_metrics.yaml b/deploy/kubernetes/overlays/prow-gke-release-staging-rc/enable_sidecar_metrics.yaml index b4dc7d28f..1bf04c7be 100644 --- a/deploy/kubernetes/overlays/prow-gke-release-staging-rc/enable_sidecar_metrics.yaml +++ b/deploy/kubernetes/overlays/prow-gke-release-staging-rc/enable_sidecar_metrics.yaml @@ -1,20 +1,19 @@ -# for external-snapshotter -- op: add - path: /spec/template/spec/containers/0/args/- - value: "--metrics-address=:22014" - # for external-provisioner - op: add - path: /spec/template/spec/containers/1/args/- + path: /spec/template/spec/containers/0/args/- value: "--metrics-address=:22011" # for external-attacher - op: add - path: /spec/template/spec/containers/2/args/- + path: /spec/template/spec/containers/1/args/- value: "--metrics-address=:22012" # for external-resizer - op: add - path: /spec/template/spec/containers/3/args/- + path: /spec/template/spec/containers/2/args/- value: "--metrics-address=:22013" +# for external-snapshotter +- op: add + path: /spec/template/spec/containers/3/args/- + value: "--metrics-address=:22014" diff --git a/deploy/kubernetes/overlays/prow-gke-release-staging-rc/kustomization.yaml b/deploy/kubernetes/overlays/prow-gke-release-staging-rc/kustomization.yaml index 95150bcbb..ae61405a7 100644 --- a/deploy/kubernetes/overlays/prow-gke-release-staging-rc/kustomization.yaml +++ b/deploy/kubernetes/overlays/prow-gke-release-staging-rc/kustomization.yaml @@ -18,23 +18,13 @@ images: - name: gke.gcr.io/csi-resizer newName: gcr.io/gke-release-staging/csi-resizer newTag: "v0.5.0-gke.0" -patches: -- controller_add_snapshotter.yaml +- name: gke.gcr.io/csi-snapshotter + newName: gcr.io/gke-release-staging/csi-snapshotter + newTag: "v2.1.1-gke.0" patchesJson6902: -- target: - group: rbac.authorization.k8s.io - version: v1 - kind: ClusterRole - name: csi-gce-pd-provisioner-role - path: rbac_add_snapshots_to_provisioner.yaml - target: group: apps version: v1 kind: StatefulSet name: csi-gce-pd-controller path: enable_sidecar_metrics.yaml -resources: -- rbac_add_snapshotter.yaml -# Reapplying namespace transformer to include newly added RBAC rules. -namespace: - gce-pd-csi-driver diff --git a/deploy/kubernetes/overlays/prow-gke-release-staging-rc/rbac_add_snapshots_to_provisioner.yaml b/deploy/kubernetes/overlays/prow-gke-release-staging-rc/rbac_add_snapshots_to_provisioner.yaml deleted file mode 100644 index 746ea82fe..000000000 --- a/deploy/kubernetes/overlays/prow-gke-release-staging-rc/rbac_add_snapshots_to_provisioner.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# arrays without strategic patch merge defined need to be appended -# using jsonpatch -# https://github.com/kubernetes-sigs/kustomize/blob/master/examples/jsonpatch.md -- op: add - path: /rules/- - value: - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list"] - -- op: add - path: /rules/- - value: - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["get", "list"] diff --git a/deploy/kubernetes/overlays/prow-gke-release-staging-rc/rbac_add_snapshotter.yaml b/deploy/kubernetes/overlays/prow-gke-release-staging-rc/rbac_add_snapshotter.yaml deleted file mode 100644 index dcc7af90a..000000000 --- a/deploy/kubernetes/overlays/prow-gke-release-staging-rc/rbac_add_snapshotter.yaml +++ /dev/null @@ -1,32 +0,0 @@ -# xref: https://github.com/kubernetes-csi/external-snapshotter/blob/master/deploy/kubernetes/rbac.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: csi-gce-pd-snapshotter-role -rules: - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - # Secrets resource omitted since GCE PD snapshots does not require them - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "get", "list", "watch", "update", "delete"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents/status"] - verbs: ["update"] ---- - -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-gce-pd-controller-snapshotter-binding -subjects: - - kind: ServiceAccount - name: csi-gce-pd-controller-sa -roleRef: - kind: ClusterRole - name: csi-gce-pd-snapshotter-role - apiGroup: rbac.authorization.k8s.io diff --git a/deploy/kubernetes/overlays/stable/kustomization.yaml b/deploy/kubernetes/overlays/stable/kustomization.yaml index 8115c994d..9aac76765 100644 --- a/deploy/kubernetes/overlays/stable/kustomization.yaml +++ b/deploy/kubernetes/overlays/stable/kustomization.yaml @@ -20,3 +20,6 @@ images: - name: gke.gcr.io/csi-resizer newName: gke.gcr.io/csi-resizer newTag: "v0.4.0-gke.0" +- name: gke.gcr.io/csi-snapshotter + newName: gke.gcr.io/csi-snapshotter + newTag: "v2.1.1-gke.0" diff --git a/test/run-k8s-integration.sh b/test/run-k8s-integration.sh index 07abc9a49..625d0ecc6 100755 --- a/test/run-k8s-integration.sh +++ b/test/run-k8s-integration.sh @@ -28,7 +28,7 @@ make -C ${PKGDIR} test-k8s-integration base_cmd="${PKGDIR}/bin/k8s-integration-test \ --run-in-prow=true --deploy-overlay-name=${overlay_name} --service-account-file=${E2E_GOOGLE_APPLICATION_CREDENTIALS} \ --do-driver-build=${do_driver_build} --boskos-resource-type=${boskos_resource_type} \ - --storageclass-file=sc-standard.yaml --test-focus="External.Storage" \ + --storageclass-file=sc-standard.yaml --snapshotclass-file=pd-volumesnapshotclass.yaml --test-focus="External.Storage" \ --deployment-strategy=${deployment_strategy} --test-version=${test_version} --num-nodes=3 \ --image-type=${image_type}" @@ -44,8 +44,4 @@ else base_cmd="${base_cmd} --gce-region=${gce_region}" fi -if [[ "$overlay_name" =~ .*"gke-release-staging".* ]]; then - base_cmd="${base_cmd} --snapshotclass-file=pd-volumesnapshotclass.yaml" -fi - eval $base_cmd