From 977a57b26738f8cb4e890546f6c451ed7a69f468 Mon Sep 17 00:00:00 2001
From: Mauricio Poppe <mauriciopoppe@google.com>
Date: Tue, 13 Apr 2021 23:06:07 +0000
Subject: [PATCH 1/7] enable remote debugging with delve

---
 Dockerfile.debug                              | 38 +++++++++++++++++++
 Makefile                                      | 18 ++++++++-
 deploy/kubernetes/deploy-driver.sh            |  8 ++--
 deploy/kubernetes/install-kustomize.sh        |  2 +-
 .../noauth-dev/controller-overlay.yaml        | 34 +++++++++++++++++
 .../overlays/noauth-dev/kustomization.yaml    | 16 ++++++++
 .../overlays/noauth-dev/noauth.yaml           | 25 ++++++++++++
 7 files changed, 135 insertions(+), 6 deletions(-)
 create mode 100644 Dockerfile.debug
 create mode 100644 deploy/kubernetes/overlays/noauth-dev/controller-overlay.yaml
 create mode 100644 deploy/kubernetes/overlays/noauth-dev/kustomization.yaml
 create mode 100644 deploy/kubernetes/overlays/noauth-dev/noauth.yaml

diff --git a/Dockerfile.debug b/Dockerfile.debug
new file mode 100644
index 000000000..8c465362c
--- /dev/null
+++ b/Dockerfile.debug
@@ -0,0 +1,38 @@
+# Copyright 2018 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM golang:1.13.15 as builder
+WORKDIR /go/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver
+ADD . .
+
+RUN CGO_ENABLED=0 go get -ldflags "-s -w -extldflags '-static'" github.com/go-delve/delve/cmd/dlv
+RUN GCE_PD_CSI_DEBUG=1 make
+
+# MAD HACKS: Build a version first so we can take the scsi_id bin and put it somewhere else in our real build
+FROM k8s.gcr.io/build-image/debian-base-amd64:v2.1.3 as base
+RUN clean-install udev
+
+# Start from Kubernetes Debian base
+FROM k8s.gcr.io/build-image/debian-base-amd64:v2.1.3
+# Copy source code too?
+WORKDIR /go/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver
+ADD . .
+
+COPY --from=builder /go/bin/dlv /go/bin/dlv
+
+# Install necessary dependencies
+RUN clean-install util-linux e2fsprogs mount ca-certificates udev xfsprogs
+COPY --from=base /lib/udev/scsi_id /lib/udev_containerized/scsi_id
+
+ENTRYPOINT ["/go/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/bin/gce-pd-csi-driver"]
diff --git a/Makefile b/Makefile
index fd6b632d9..bfe62cdb0 100644
--- a/Makefile
+++ b/Makefile
@@ -20,6 +20,12 @@ ifdef GCE_PD_CSI_STAGING_VERSION
 else
 	STAGINGVERSION=${REV}
 endif
+
+GCFLAGS=""
+ifdef GCE_PD_CSI_DEBUG
+	GCFLAGS=-gcflags="all=-N -L"
+endif
+
 STAGINGIMAGE=${GCE_PD_CSI_STAGING_IMAGE}
 DRIVERBINARY=gce-pd-csi-driver
 DRIVERWINDOWSBINARY=${DRIVERBINARY}.exe
@@ -38,7 +44,7 @@ WINDOWS_BASE_IMAGES=$(BASE_IMAGE_LTSC2019) $(BASE_IMAGE_1909) $(BASE_IMAGE_2004)
 all: gce-pd-driver gce-pd-driver-windows
 gce-pd-driver:
 	mkdir -p bin
-	go build -mod=vendor -ldflags "-X main.version=$(STAGINGVERSION)" -o bin/${DRIVERBINARY} ./cmd/gce-pd-csi-driver/
+	go build -mod=vendor -gcflags="all=-N -l" -ldflags "-X main.version=$(STAGINGVERSION)" -o bin/${DRIVERBINARY} ./cmd/gce-pd-csi-driver/
 
 gce-pd-driver-windows:
 	mkdir -p bin
@@ -76,6 +82,11 @@ build-and-push-multi-arch: build-and-push-container-linux build-and-push-windows
 	STAGINGIMAGE="$(STAGINGIMAGE)" STAGINGVERSION="$(STAGINGVERSION)" WINDOWS_IMAGE_TAGS="$(WINDOWS_IMAGE_TAGS)" WINDOWS_BASE_IMAGES="$(WINDOWS_BASE_IMAGES)" ./manifest_osversion.sh
 	$(DOCKER) manifest push -p $(STAGINGIMAGE):$(STAGINGVERSION)
 
+build-and-push-multi-arch-dev: build-and-push-container-linux-debug build-and-push-windows-container-ltsc2019
+	$(DOCKER) manifest create --amend $(STAGINGIMAGE):$(STAGINGVERSION) $(STAGINGIMAGE):$(STAGINGVERSION)_linux $(STAGINGIMAGE):$(STAGINGVERSION)_ltsc2019
+	STAGINGIMAGE="$(STAGINGIMAGE)" STAGINGVERSION="$(STAGINGVERSION)" WINDOWS_IMAGE_TAGS="$(WINDOWS_IMAGE_TAGS_DEV)" WINDOWS_BASE_IMAGES="$(WINDOWS_BASE_IMAGES_DEV)" ./manifest_osversion.sh
+	$(DOCKER) manifest push -p $(STAGINGIMAGE):$(STAGINGVERSION)
+
 push-container: build-container
 	gcloud docker -- push $(STAGINGIMAGE):$(STAGINGVERSION)
 
@@ -84,6 +95,11 @@ build-and-push-container-linux: require-GCE_PD_CSI_STAGING_IMAGE init-buildx
 		-t $(STAGINGIMAGE):$(STAGINGVERSION)_linux \
 		--build-arg TAG=$(STAGINGVERSION) --push .
 
+build-and-push-container-linux-debug: require-GCE_PD_CSI_STAGING_IMAGE init-buildx
+	$(DOCKER) buildx build --file=Dockerfile.debug --platform=linux \
+		-t $(STAGINGIMAGE):$(STAGINGVERSION)_linux \
+		--build-arg TAG=$(STAGINGVERSION) --push .
+
 test-sanity: gce-pd-driver
 	go test -mod=vendor --v -timeout 30s sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/sanity -run ^TestSanity$
 
diff --git a/deploy/kubernetes/deploy-driver.sh b/deploy/kubernetes/deploy-driver.sh
index dc0a37eff..2617a14bd 100755
--- a/deploy/kubernetes/deploy-driver.sh
+++ b/deploy/kubernetes/deploy-driver.sh
@@ -8,7 +8,7 @@
 # which are in Kubernetes version 1.10.5+
 
 # Args:
-# GCE_PD_SA_DIR: Directory the service account key has been saved in (generated 
+# GCE_PD_SA_DIR: Directory the service account key has been saved in (generated
 #   by setup-project.sh). Ignored if GCE_PD_DRIVER_VERSION == noauth.
 # GCE_PD_DRIVER_VERSION: The kustomize overlay (located in
 #   deploy/kubernetes/overlays) to deploy. Can be one of {stable, dev}
@@ -44,7 +44,7 @@ while [ -n "${1-}" ]; do
   esac
 done
 
-if [ "${DEPLOY_VERSION}" != noauth ]; then
+if [ "${DEPLOY_VERSION}" =~ '^(?!noauth).*' ]; then
   ensure_var GCE_PD_SA_DIR
 fi
 
@@ -74,7 +74,7 @@ function check_service_account()
 
 ensure_kustomize
 
-if [ "$skip_sa_check" != true -a "${DEPLOY_VERSION}" != noauth ]; then
+if [ "$skip_sa_check" != true -a "${DEPLOY_VERSION}" =~ '^(?!noauth).*' ]; then
   check_service_account
 fi
 
@@ -83,7 +83,7 @@ then
   ${KUBECTL} create namespace "${NAMESPACE}" -v="${VERBOSITY}"
 fi
 
-if [ "${DEPLOY_VERSION}" != noauth ]; then
+if [ "${DEPLOY_VERSION}" =~ '^(?!noauth).*' ]; then
   if ! ${KUBECTL} get secret cloud-sa -v="${VERBOSITY}" -n "${NAMESPACE}";
   then
     ${KUBECTL} create secret generic cloud-sa -v="${VERBOSITY}" --from-file="${GCE_PD_SA_DIR}/cloud-sa.json" -n "${NAMESPACE}"
diff --git a/deploy/kubernetes/install-kustomize.sh b/deploy/kubernetes/install-kustomize.sh
index 88c41913b..eaeb8ebbe 100755
--- a/deploy/kubernetes/install-kustomize.sh
+++ b/deploy/kubernetes/install-kustomize.sh
@@ -48,7 +48,7 @@ elif [[ "$OSTYPE" == darwin* ]]; then
 fi
 
 # As github has a limit on what stored in releases/, and kustomize has many different package
-# versions, we just point directly at the version we want. See 
+# versions, we just point directly at the version we want. See
 # github.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh.
 
 version=v3.9.4
diff --git a/deploy/kubernetes/overlays/noauth-dev/controller-overlay.yaml b/deploy/kubernetes/overlays/noauth-dev/controller-overlay.yaml
new file mode 100644
index 000000000..adc68a679
--- /dev/null
+++ b/deploy/kubernetes/overlays/noauth-dev/controller-overlay.yaml
@@ -0,0 +1,34 @@
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: csi-gce-pd-controller
+  annotations:
+    # https://kubernetes.io/docs/tutorials/clusters/apparmor/
+    container.apparmor.security.beta.kubernetes.io/gce-pd-driver: unconfined
+spec:
+  template:
+    spec:
+      containers:
+        - name: gce-pd-driver
+          imagePullPolicy: Always
+          command: ["/go/bin/dlv"]
+          args:
+            - "--listen=:2345"
+            - "--headless=true"
+            - "--api-version=2"
+            # https://github.com/go-delve/delve/blob/master/Documentation/usage/dlv_exec.md#options
+            - "--accept-multiclient"
+            - "--continue"
+            - "--log"
+            - "exec"
+            - "/go/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/bin/gce-pd-csi-driver"
+            - "--"
+            - "--v=5"
+            - "--endpoint=unix:/csi/csi.sock"
+          ports:
+            - containerPort: 2345
+          securityContext:
+            capabilities:
+              add:
+              - SYS_PTRACE
+
diff --git a/deploy/kubernetes/overlays/noauth-dev/kustomization.yaml b/deploy/kubernetes/overlays/noauth-dev/kustomization.yaml
new file mode 100644
index 000000000..fcf56920e
--- /dev/null
+++ b/deploy/kubernetes/overlays/noauth-dev/kustomization.yaml
@@ -0,0 +1,16 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ../../base/
+# Here noauth overlay is using the same image as alpha
+transformers:
+- ../../images/alpha
+patchesStrategicMerge:
+- noauth.yaml
+- controller-overlay.yaml
+namespace: gce-pd-csi-driver
+# To change the dev image, add something like the following.
+images:
+- name: gke.gcr.io/gcp-compute-persistent-disk-csi-driver
+  newName: gcr.io/mauriciopoppe-gke-dev/gcp-compute-persistent-disk-csi-driver
+  newTag: latest
diff --git a/deploy/kubernetes/overlays/noauth-dev/noauth.yaml b/deploy/kubernetes/overlays/noauth-dev/noauth.yaml
new file mode 100644
index 000000000..3c06103c0
--- /dev/null
+++ b/deploy/kubernetes/overlays/noauth-dev/noauth.yaml
@@ -0,0 +1,25 @@
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: csi-gce-pd-controller
+spec:
+  template:
+    spec:
+      containers:
+        - name: gce-pd-driver
+          env:
+            - $patch: delete
+              name: GOOGLE_APPLICATION_CREDENTIALS
+              value: "/etc/cloud-sa/cloud-sa.json"
+          volumeMounts:
+            - $patch: delete
+              name: cloud-sa-volume
+              readOnly: true
+              mountPath: "/etc/cloud-sa"
+      volumes:
+        - $patch: delete
+          name: cloud-sa-volume
+          secret:
+            secretName: cloud-sa
+
+

From 5c2696b61b4e96228ce358acbc4f9ae8dcea8fe2 Mon Sep 17 00:00:00 2001
From: Mauricio Poppe <mauriciopoppe@google.com>
Date: Tue, 13 Apr 2021 23:46:32 +0000
Subject: [PATCH 2/7] use GCFLAGS with envvar

---
 Dockerfile.debug                                         | 8 ++++++--
 Makefile                                                 | 2 +-
 deploy/kubernetes/overlays/noauth-dev/kustomization.yaml | 8 ++++----
 3 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/Dockerfile.debug b/Dockerfile.debug
index 8c465362c..063a68d44 100644
--- a/Dockerfile.debug
+++ b/Dockerfile.debug
@@ -1,4 +1,4 @@
-# Copyright 2018 The Kubernetes Authors.
+# Copyright 2021 The Kubernetes Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -25,7 +25,8 @@ RUN clean-install udev
 
 # Start from Kubernetes Debian base
 FROM k8s.gcr.io/build-image/debian-base-amd64:v2.1.3
-# Copy source code too?
+
+# Copy source code too to correlate the binary and the breakpoints
 WORKDIR /go/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver
 ADD . .
 
@@ -35,4 +36,7 @@ COPY --from=builder /go/bin/dlv /go/bin/dlv
 RUN clean-install util-linux e2fsprogs mount ca-certificates udev xfsprogs
 COPY --from=base /lib/udev/scsi_id /lib/udev_containerized/scsi_id
 
+# PDCSI driver isn't copied to / because of delve not being able to correlate
+# the binary and the source code, instead just run the binary where it was
+# compiled, the overlay noauth-dev calls this binary
 ENTRYPOINT ["/go/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/bin/gce-pd-csi-driver"]
diff --git a/Makefile b/Makefile
index bfe62cdb0..2d17cb8fd 100644
--- a/Makefile
+++ b/Makefile
@@ -44,7 +44,7 @@ WINDOWS_BASE_IMAGES=$(BASE_IMAGE_LTSC2019) $(BASE_IMAGE_1909) $(BASE_IMAGE_2004)
 all: gce-pd-driver gce-pd-driver-windows
 gce-pd-driver:
 	mkdir -p bin
-	go build -mod=vendor -gcflags="all=-N -l" -ldflags "-X main.version=$(STAGINGVERSION)" -o bin/${DRIVERBINARY} ./cmd/gce-pd-csi-driver/
+	go build -mod=vendor $(GCFLAGS) -ldflags "-X main.version=$(STAGINGVERSION)" -o bin/${DRIVERBINARY} ./cmd/gce-pd-csi-driver/
 
 gce-pd-driver-windows:
 	mkdir -p bin
diff --git a/deploy/kubernetes/overlays/noauth-dev/kustomization.yaml b/deploy/kubernetes/overlays/noauth-dev/kustomization.yaml
index fcf56920e..d920895f0 100644
--- a/deploy/kubernetes/overlays/noauth-dev/kustomization.yaml
+++ b/deploy/kubernetes/overlays/noauth-dev/kustomization.yaml
@@ -10,7 +10,7 @@ patchesStrategicMerge:
 - controller-overlay.yaml
 namespace: gce-pd-csi-driver
 # To change the dev image, add something like the following.
-images:
-- name: gke.gcr.io/gcp-compute-persistent-disk-csi-driver
-  newName: gcr.io/mauriciopoppe-gke-dev/gcp-compute-persistent-disk-csi-driver
-  newTag: latest
+# images:
+# - name: gke.gcr.io/gcp-compute-persistent-disk-csi-driver
+#   newName: gcr.io/mauriciopoppe-gke-dev/gcp-compute-persistent-disk-csi-driver
+#   newTag: latest

From 765dc4154a9c9a591ee0870e211090124b726dfc Mon Sep 17 00:00:00 2001
From: Mauricio Poppe <mauriciopoppe@google.com>
Date: Thu, 15 Apr 2021 17:44:22 +0000
Subject: [PATCH 3/7] always set gcflags in Makefile

---
 Dockerfile.debug | 2 +-
 Makefile         | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Dockerfile.debug b/Dockerfile.debug
index 063a68d44..4690404ad 100644
--- a/Dockerfile.debug
+++ b/Dockerfile.debug
@@ -17,7 +17,7 @@ WORKDIR /go/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver
 ADD . .
 
 RUN CGO_ENABLED=0 go get -ldflags "-s -w -extldflags '-static'" github.com/go-delve/delve/cmd/dlv
-RUN GCE_PD_CSI_DEBUG=1 make
+RUN GCE_PD_CSI_DEBUG=1 make gce-pd-driver
 
 # MAD HACKS: Build a version first so we can take the scsi_id bin and put it somewhere else in our real build
 FROM k8s.gcr.io/build-image/debian-base-amd64:v2.1.3 as base
diff --git a/Makefile b/Makefile
index 2d17cb8fd..8a67b82c3 100644
--- a/Makefile
+++ b/Makefile
@@ -23,7 +23,7 @@ endif
 
 GCFLAGS=""
 ifdef GCE_PD_CSI_DEBUG
-	GCFLAGS=-gcflags="all=-N -L"
+	GCFLAGS="all=-N -l"
 endif
 
 STAGINGIMAGE=${GCE_PD_CSI_STAGING_IMAGE}
@@ -44,7 +44,7 @@ WINDOWS_BASE_IMAGES=$(BASE_IMAGE_LTSC2019) $(BASE_IMAGE_1909) $(BASE_IMAGE_2004)
 all: gce-pd-driver gce-pd-driver-windows
 gce-pd-driver:
 	mkdir -p bin
-	go build -mod=vendor $(GCFLAGS) -ldflags "-X main.version=$(STAGINGVERSION)" -o bin/${DRIVERBINARY} ./cmd/gce-pd-csi-driver/
+	go build -mod=vendor -gcflags=$(GCFLAGS) -ldflags "-X main.version=$(STAGINGVERSION)" -o bin/${DRIVERBINARY} ./cmd/gce-pd-csi-driver/
 
 gce-pd-driver-windows:
 	mkdir -p bin

From 93b9ec907b6b7e7be88f9f1d1128901ee0bd970b Mon Sep 17 00:00:00 2001
From: Mauricio Poppe <mauriciopoppe@google.com>
Date: Thu, 15 Apr 2021 18:34:45 +0000
Subject: [PATCH 4/7] Add docs for remote debugging

---
 docs/kubernetes/development.md | 87 +++++++++++++++++++++++++++++++++-
 1 file changed, 86 insertions(+), 1 deletion(-)

diff --git a/docs/kubernetes/development.md b/docs/kubernetes/development.md
index fb4bf93ab..f9e29d642 100644
--- a/docs/kubernetes/development.md
+++ b/docs/kubernetes/development.md
@@ -3,6 +3,7 @@
 ## Manual
 
 To build and install a development version of the driver:
+
 ```
 $ GCE_PD_CSI_STAGING_IMAGE=gcr.io/path/to/driver/image:dev   # Location to push dev image to
 $ make push-container
@@ -13,9 +14,93 @@ $ ./deploy/kubernetes/deploy-driver.sh
 ```
 
 To bring down driver:
+
 ```
 $ ./deploy/kubernetes/delete-driver.sh
 ```
 
-## TODO Testing
+## Debugging
+
+We use https://github.com/go-delve/delve and its remote debugging feature for debugging, this feature
+is only available in the PD CSI Controller (which runs in a linux node)
+
+Requirements:
+
+- https://github.com/go-delve/delve
+
+Steps:
+
+- Build the PD CSI driver with additional compiler flags
+
+```
+export GCE_PD_CSI_STAGING_VERSION=latest
+export GCE_PD_CSI_STAGING_IMAGE=image/repo/gcp-compute-persistent-disk-csi-driver
+make build-and-push-multi-arch-dev
+```
+
+- Update `deploy/kubernetes/overlays/noauth-dev/kustomization.yaml` to match the repo you wrote above e.g.
+
+```yaml
+images:
+- name: gke.gcr.io/gcp-compute-persistent-disk-csi-driver
+  newName: image/repo/gcp-compute-persistent-disk-csi-driver
+  newTag: latest
+```
+
+- Delete and deploy the driver with this overlay
+
+```sh
+./deploy/kubernetes/delete-driver.sh && \
+  GCE_PD_DRIVER_VERSION=noauth-dev ./deploy/kubernetes/deploy-driver.sh
+```
+
+At this point you could verify that delve is running in the controller logs:
+
+```text
+API server listening at: [::]:2345
+ 2021-04-15T18:28:51Z info layer=debugger launching process with args: [/go/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/bin/gce-pd-csi-driver --v=5 --endpoint=unix:/csi/csi.sock]
+ 2021-04-15T18:28:53Z debug layer=debugger continuing
+```
+
+- Enable port forwading of the PD CSI controller of port 2345
+
+```sh
+kubectl -n gce-pd-csi-driver get pods | grep controller | awk '{print $1}' | xargs -I % kubectl -n gce-pd-csi-driver port-forward % 2345:2345
+```
+
+- Connect to the headless server and issue commands
+
+```sh
+dlv connect localhost:2345
+Type 'help' for list of commands.
+(dlv) clearall
+(dlv) break pkg/gce-pd-csi-driver/controller.go:509
+Breakpoint 1 set at 0x159ba32 for sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-pd-csi-driver.(*GCEControllerServer).ListVolumes() ./pkg/gce-pd-csi-driver/controller.go:509
+(dlv) c
+> sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-pd-csi-driver.(*GCEControllerServer).ListVolumes() ./pkg/gce-pd-csi-driver/controller.go:509 (hits goroutine(69):1 total:1) (PC: 0x159ba32)
+Warning: debugging optimized function
+   504:         }
+   505: }
+   506:
+   507: func (gceCS *GCEControllerServer) ListVolumes(ctx context.Context, req *csi.ListVolumesRequest) (*csi.ListVolumesResponse, error) {
+   508:         // https//cloud.google.com/compute/docs/reference/beta/disks/list
+=> 509:         if req.MaxEntries < 0 {
+   510:                 return nil, status.Error(codes.InvalidArgument, fmt.Sprintf(
+   511:                         "ListVolumes got max entries request %v. GCE only supports values between 0-500", req.MaxEntries))
+   512:         }
+   513:         var maxEntries int64 = int64(req.MaxEntries)
+   514:         if maxEntries > 500 {
+(dlv) req
+Command failed: command not available
+(dlv) p req
+*github.com/container-storage-interface/spec/lib/go/csi.ListVolumesRequest {
+        MaxEntries: 0,
+        StartingToken: "",
+        XXX_NoUnkeyedLiteral: struct {} {},
+        XXX_unrecognized: []uint8 len: 0, cap: 0, nil,
+        XXX_sizecache: 0,}
+(dlv)
+```
+
+See https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/pull/742 for the implementation details
 

From 649538ea55cb6e136fab00ac54a6b24d8fc73cf6 Mon Sep 17 00:00:00 2001
From: Mauricio Poppe <mauriciopoppe@google.com>
Date: Thu, 15 Apr 2021 19:10:33 +0000
Subject: [PATCH 5/7] Use negation of == in comparison for the overlay name

---
 deploy/kubernetes/deploy-driver.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/deploy/kubernetes/deploy-driver.sh b/deploy/kubernetes/deploy-driver.sh
index 2617a14bd..4f6443eed 100755
--- a/deploy/kubernetes/deploy-driver.sh
+++ b/deploy/kubernetes/deploy-driver.sh
@@ -44,7 +44,7 @@ while [ -n "${1-}" ]; do
   esac
 done
 
-if [ "${DEPLOY_VERSION}" =~ '^(?!noauth).*' ]; then
+if [[ ! "${DEPLOY_VERSION}" == *noauth* ]]; then
   ensure_var GCE_PD_SA_DIR
 fi
 
@@ -74,7 +74,7 @@ function check_service_account()
 
 ensure_kustomize
 
-if [ "$skip_sa_check" != true -a "${DEPLOY_VERSION}" =~ '^(?!noauth).*' ]; then
+if [[ "$skip_sa_check" != true ]] && [[ ! "${DEPLOY_VERSION}" == *noauth* ]]; then
   check_service_account
 fi
 
@@ -83,7 +83,7 @@ then
   ${KUBECTL} create namespace "${NAMESPACE}" -v="${VERBOSITY}"
 fi
 
-if [ "${DEPLOY_VERSION}" =~ '^(?!noauth).*' ]; then
+if [[ ! "${DEPLOY_VERSION}" == *noauth* ]]; then
   if ! ${KUBECTL} get secret cloud-sa -v="${VERBOSITY}" -n "${NAMESPACE}";
   then
     ${KUBECTL} create secret generic cloud-sa -v="${VERBOSITY}" --from-file="${GCE_PD_SA_DIR}/cloud-sa.json" -n "${NAMESPACE}"

From 2c3a835b449a7106f0596f0d531258b20bb0e8b8 Mon Sep 17 00:00:00 2001
From: Mauricio Poppe <mauriciopoppe@google.com>
Date: Thu, 15 Apr 2021 19:15:08 +0000
Subject: [PATCH 6/7] Add missing _DEV env vars

---
 Makefile | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/Makefile b/Makefile
index 8a67b82c3..44f231e95 100644
--- a/Makefile
+++ b/Makefile
@@ -40,6 +40,9 @@ BASE_IMAGE_20H2=mcr.microsoft.com/windows/servercore:20H2
 # Both arrays MUST be index aligned.
 WINDOWS_IMAGE_TAGS=ltsc2019 1909 2004 20H2
 WINDOWS_BASE_IMAGES=$(BASE_IMAGE_LTSC2019) $(BASE_IMAGE_1909) $(BASE_IMAGE_2004) $(BASE_IMAGE_20H2)
+# Development only
+WINDOWS_IMAGE_TAGS_DEV=ltsc2019
+WINDOWS_BASE_IMAGES_DEV=$(BASE_IMAGE_LTSC2019)
 
 all: gce-pd-driver gce-pd-driver-windows
 gce-pd-driver:

From ca3e20f26775996111e1c9d3d691999a52ee8de0 Mon Sep 17 00:00:00 2001
From: Mauricio Poppe <mauriciopoppe@google.com>
Date: Mon, 19 Apr 2021 20:27:58 +0000
Subject: [PATCH 7/7] Renamed noauth-dev to noauth-debug, update image versions

---
 Dockerfile.debug                                     |  6 +++---
 Makefile                                             |  7 ++-----
 deploy/kubernetes/README.md                          |  3 ++-
 .../controller-overlay.yaml                          |  0
 .../{noauth-dev => noauth-debug}/kustomization.yaml  |  0
 .../{noauth-dev => noauth-debug}/noauth.yaml         |  0
 docs/kubernetes/development.md                       | 12 ++++++------
 7 files changed, 13 insertions(+), 15 deletions(-)
 rename deploy/kubernetes/overlays/{noauth-dev => noauth-debug}/controller-overlay.yaml (100%)
 rename deploy/kubernetes/overlays/{noauth-dev => noauth-debug}/kustomization.yaml (100%)
 rename deploy/kubernetes/overlays/{noauth-dev => noauth-debug}/noauth.yaml (100%)

diff --git a/Dockerfile.debug b/Dockerfile.debug
index 4690404ad..0336eec32 100644
--- a/Dockerfile.debug
+++ b/Dockerfile.debug
@@ -20,11 +20,11 @@ RUN CGO_ENABLED=0 go get -ldflags "-s -w -extldflags '-static'" github.com/go-de
 RUN GCE_PD_CSI_DEBUG=1 make gce-pd-driver
 
 # MAD HACKS: Build a version first so we can take the scsi_id bin and put it somewhere else in our real build
-FROM k8s.gcr.io/build-image/debian-base-amd64:v2.1.3 as base
+FROM k8s.gcr.io/build-image/debian-base-amd64:buster-v1.5.0 as mad-hack
 RUN clean-install udev
 
 # Start from Kubernetes Debian base
-FROM k8s.gcr.io/build-image/debian-base-amd64:v2.1.3
+FROM k8s.gcr.io/build-image/debian-base-amd64:buster-v1.5.0
 
 # Copy source code too to correlate the binary and the breakpoints
 WORKDIR /go/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver
@@ -34,7 +34,7 @@ COPY --from=builder /go/bin/dlv /go/bin/dlv
 
 # Install necessary dependencies
 RUN clean-install util-linux e2fsprogs mount ca-certificates udev xfsprogs
-COPY --from=base /lib/udev/scsi_id /lib/udev_containerized/scsi_id
+COPY --from=mad-hack /lib/udev/scsi_id /lib/udev_containerized/scsi_id
 
 # PDCSI driver isn't copied to / because of delve not being able to correlate
 # the binary and the source code, instead just run the binary where it was
diff --git a/Makefile b/Makefile
index 44f231e95..6ec2c345c 100644
--- a/Makefile
+++ b/Makefile
@@ -40,9 +40,6 @@ BASE_IMAGE_20H2=mcr.microsoft.com/windows/servercore:20H2
 # Both arrays MUST be index aligned.
 WINDOWS_IMAGE_TAGS=ltsc2019 1909 2004 20H2
 WINDOWS_BASE_IMAGES=$(BASE_IMAGE_LTSC2019) $(BASE_IMAGE_1909) $(BASE_IMAGE_2004) $(BASE_IMAGE_20H2)
-# Development only
-WINDOWS_IMAGE_TAGS_DEV=ltsc2019
-WINDOWS_BASE_IMAGES_DEV=$(BASE_IMAGE_LTSC2019)
 
 all: gce-pd-driver gce-pd-driver-windows
 gce-pd-driver:
@@ -85,9 +82,9 @@ build-and-push-multi-arch: build-and-push-container-linux build-and-push-windows
 	STAGINGIMAGE="$(STAGINGIMAGE)" STAGINGVERSION="$(STAGINGVERSION)" WINDOWS_IMAGE_TAGS="$(WINDOWS_IMAGE_TAGS)" WINDOWS_BASE_IMAGES="$(WINDOWS_BASE_IMAGES)" ./manifest_osversion.sh
 	$(DOCKER) manifest push -p $(STAGINGIMAGE):$(STAGINGVERSION)
 
-build-and-push-multi-arch-dev: build-and-push-container-linux-debug build-and-push-windows-container-ltsc2019
+build-and-push-multi-arch-debug: build-and-push-container-linux-debug build-and-push-windows-container-ltsc2019
 	$(DOCKER) manifest create --amend $(STAGINGIMAGE):$(STAGINGVERSION) $(STAGINGIMAGE):$(STAGINGVERSION)_linux $(STAGINGIMAGE):$(STAGINGVERSION)_ltsc2019
-	STAGINGIMAGE="$(STAGINGIMAGE)" STAGINGVERSION="$(STAGINGVERSION)" WINDOWS_IMAGE_TAGS="$(WINDOWS_IMAGE_TAGS_DEV)" WINDOWS_BASE_IMAGES="$(WINDOWS_BASE_IMAGES_DEV)" ./manifest_osversion.sh
+	STAGINGIMAGE="$(STAGINGIMAGE)" STAGINGVERSION="$(STAGINGVERSION)" WINDOWS_IMAGE_TAGS="ltsc2019" WINDOWS_BASE_IMAGES="$(BASE_IMAGE_LTSC2019)" ./manifest_osversion.sh
 	$(DOCKER) manifest push -p $(STAGINGIMAGE):$(STAGINGVERSION)
 
 push-container: build-container
diff --git a/deploy/kubernetes/README.md b/deploy/kubernetes/README.md
index 92949c196..c21de286d 100644
--- a/deploy/kubernetes/README.md
+++ b/deploy/kubernetes/README.md
@@ -16,8 +16,9 @@ The current structure for kustomization is as follows. Note that Windows support
 * `overlays`: It has the k8s minor version-specific driver manifest bundle.
   * `stable-master`: Contains deployment specs of a stable driver for k8s master.
   * `stable-{k8s-minor}`: Contains deployment specs of a stable driver for given k8s minor version release.
-  * `alpha`: Contains deployment specs for features in development. Both Linux and Windows are supported. 
+  * `alpha`: Contains deployment specs for features in development. Both Linux and Windows are supported.
   * `dev`: Based on alpha, and also contains the developer's specs for use in driver development.
+  * `noauth-debug`: Based on alpha, used for debugging purposes only, see docs/kubernetes/development.md.
   * `prow-gke-release-staging-rc-master`: Used for prow tests. Contains deployment specs of a driver for latest k8s master.
   * `prow-gke-release-staging-rc-{k8s-minor}`: Used for prow tests. Contains deployment specs of a driver for given k8s    minor version release.
   * `prow-gke-release-staging-rc-head`: Used for prow tests. Contains deployment specs of a driver with latest sidecar images, for latest k8s master.
diff --git a/deploy/kubernetes/overlays/noauth-dev/controller-overlay.yaml b/deploy/kubernetes/overlays/noauth-debug/controller-overlay.yaml
similarity index 100%
rename from deploy/kubernetes/overlays/noauth-dev/controller-overlay.yaml
rename to deploy/kubernetes/overlays/noauth-debug/controller-overlay.yaml
diff --git a/deploy/kubernetes/overlays/noauth-dev/kustomization.yaml b/deploy/kubernetes/overlays/noauth-debug/kustomization.yaml
similarity index 100%
rename from deploy/kubernetes/overlays/noauth-dev/kustomization.yaml
rename to deploy/kubernetes/overlays/noauth-debug/kustomization.yaml
diff --git a/deploy/kubernetes/overlays/noauth-dev/noauth.yaml b/deploy/kubernetes/overlays/noauth-debug/noauth.yaml
similarity index 100%
rename from deploy/kubernetes/overlays/noauth-dev/noauth.yaml
rename to deploy/kubernetes/overlays/noauth-debug/noauth.yaml
diff --git a/docs/kubernetes/development.md b/docs/kubernetes/development.md
index f9e29d642..6cdc19fcb 100644
--- a/docs/kubernetes/development.md
+++ b/docs/kubernetes/development.md
@@ -21,8 +21,8 @@ $ ./deploy/kubernetes/delete-driver.sh
 
 ## Debugging
 
-We use https://github.com/go-delve/delve and its remote debugging feature for debugging, this feature
-is only available in the PD CSI Controller (which runs in a linux node)
+We use https://github.com/go-delve/delve and its feature for remote debugging. This feature
+is only available in the PD CSI Controller (which runs in a linux node).
 
 Requirements:
 
@@ -30,15 +30,15 @@ Requirements:
 
 Steps:
 
-- Build the PD CSI driver with additional compiler flags
+- Build the PD CSI driver with additional compiler flags.
 
 ```
 export GCE_PD_CSI_STAGING_VERSION=latest
 export GCE_PD_CSI_STAGING_IMAGE=image/repo/gcp-compute-persistent-disk-csi-driver
-make build-and-push-multi-arch-dev
+make build-and-push-multi-arch-debug
 ```
 
-- Update `deploy/kubernetes/overlays/noauth-dev/kustomization.yaml` to match the repo you wrote above e.g.
+- Update `deploy/kubernetes/overlays/noauth-debug/kustomization.yaml` to match the repo you wrote above e.g.
 
 ```yaml
 images:
@@ -51,7 +51,7 @@ images:
 
 ```sh
 ./deploy/kubernetes/delete-driver.sh && \
-  GCE_PD_DRIVER_VERSION=noauth-dev ./deploy/kubernetes/deploy-driver.sh
+  GCE_PD_DRIVER_VERSION=noauth-debug ./deploy/kubernetes/deploy-driver.sh
 ```
 
 At this point you could verify that delve is running in the controller logs: