Merge pull request #610 from mattcary/creds

k8s-ci-robot · web-flow · commit 254f9253afd1 · 2020-09-28T17:09:25.000-07:00
Enable usages of application default credentials
diff --git a/deploy/kubernetes/deploy-driver.sh b/deploy/kubernetes/deploy-driver.sh
@@ -8,7 +8,8 @@
 # which are in Kubernetes version 1.10.5+
 
 # Args:
-# GCE_PD_SA_DIR: Directory the service account key has been saved in (generated by setup-project.sh)
+# GCE_PD_SA_DIR: Directory the service account key has been saved in (generated 
+#   by setup-project.sh). Ignored if GCE_PD_DRIVER_VERSION == noauth.
 # GCE_PD_DRIVER_VERSION: The kustomize overlay (located in
 #   deploy/kubernetes/overlays) to deploy. Can be one of {stable, dev}
 
@@ -43,7 +44,9 @@ while [ -n "${1-}" ]; do
   esac
 done
 
-ensure_var GCE_PD_SA_DIR
+if [ "${DEPLOY_VERSION}" != noauth ]; then
+  ensure_var GCE_PD_SA_DIR
+fi
 
 function check_service_account()
 {
@@ -71,7 +74,7 @@ function check_service_account()
 
 ensure_kustomize
 
-if [ "$skip_sa_check" != true ]; then
+if [ "$skip_sa_check" != true -a "${DEPLOY_VERSION}" != noauth ]; then
   check_service_account
 fi
 
@@ -80,9 +83,11 @@ then
   ${KUBECTL} create namespace "${NAMESPACE}" -v="${VERBOSITY}"
 fi
 
-if ! ${KUBECTL} get secret cloud-sa -v="${VERBOSITY}" -n "${NAMESPACE}";
-then
-  ${KUBECTL} create secret generic cloud-sa -v="${VERBOSITY}" --from-file="${GCE_PD_SA_DIR}/cloud-sa.json" -n "${NAMESPACE}"
+if [ "${DEPLOY_VERSION}" != noauth ]; then
+  if ! ${KUBECTL} get secret cloud-sa -v="${VERBOSITY}" -n "${NAMESPACE}";
+  then
+    ${KUBECTL} create secret generic cloud-sa -v="${VERBOSITY}" --from-file="${GCE_PD_SA_DIR}/cloud-sa.json" -n "${NAMESPACE}"
+  fi
 fi
 
 # GKE Required Setup
diff --git a/deploy/kubernetes/images/dev/image.yaml b/deploy/kubernetes/images/dev/image.yaml
diff --git a/deploy/kubernetes/images/dev/kustomization.yaml b/deploy/kubernetes/images/dev/kustomization.yaml
diff --git a/deploy/kubernetes/overlays/dev/kustomization.yaml b/deploy/kubernetes/overlays/dev/kustomization.yaml
@@ -2,11 +2,12 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
 - ../alpha
-patches:
+patchesStrategicMerge:
 - controller_always_pull.yaml
 - node_always_pull.yaml
-namespace:
-  gce-pd-csi-driver
-transformers:
-- ../../images/dev
-
+namespace: gce-pd-csi-driver
+# To change the dev image, add something like the following.
+#images:
+#- name: gke.gcr.io/gcp-compute-persistent-disk-csi-driver
+#  newName: gcr.io/mattcary-gke-dev-owned/csi/gce-pd-driver
+#  newTag: latest.
diff --git a/deploy/kubernetes/overlays/noauth/kustomization.yaml b/deploy/kubernetes/overlays/noauth/kustomization.yaml
@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ../dev
+patchesStrategicMerge:
+- noauth.yaml
+namespace: gce-pd-csi-driver
diff --git a/deploy/kubernetes/overlays/noauth/noauth.yaml b/deploy/kubernetes/overlays/noauth/noauth.yaml
@@ -0,0 +1,25 @@
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: csi-gce-pd-controller
+spec:
+  template:
+    spec:
+      containers:
+        - name: gce-pd-driver
+          env:
+            - $patch: delete
+              name: GOOGLE_APPLICATION_CREDENTIALS
+              value: "/etc/cloud-sa/cloud-sa.json"
+          volumeMounts:
+            - $patch: delete
+              name: cloud-sa-volume
+              readOnly: true
+              mountPath: "/etc/cloud-sa"
+      volumes:
+        - $patch: delete
+          name: cloud-sa-volume
+          secret:
+            secretName: cloud-sa
+
+
diff --git a/test/k8s-integration/driver.go b/test/k8s-integration/driver.go
@@ -56,25 +56,29 @@ func installDriver(platform, goPath, pkgDir, stagingImage, stagingVersion, deplo
 		}
 	}
 
-	// setup service account file for secret creation
-	tmpSaFile := filepath.Join(generateUniqueTmpDir(), "cloud-sa.json")
-	defer removeDir(filepath.Dir(tmpSaFile))
+	var deployEnv []string
+	if deployOverlayName != "noauth" {
+		// setup service account file for secret creation
+		tmpSaFile := filepath.Join(generateUniqueTmpDir(), "cloud-sa.json")
+		defer removeDir(filepath.Dir(tmpSaFile))
+
+		// Need to copy it to name the file "cloud-sa.json"
+		out, err := exec.Command("cp", *saFile, tmpSaFile).CombinedOutput()
+		if err != nil {
+			return fmt.Errorf("error copying service account key: %s, err: %v", out, err)
+		}
+		defer shredFile(tmpSaFile)
 
-	// Need to copy it to name the file "cloud-sa.json"
-	out, err := exec.Command("cp", *saFile, tmpSaFile).CombinedOutput()
-	if err != nil {
-		return fmt.Errorf("error copying service account key: %s, err: %v", out, err)
+		deployEnv = append(deployEnv, fmt.Sprintf("GCE_PD_SA_DIR=%s", filepath.Dir(tmpSaFile)))
 	}
-	defer shredFile(tmpSaFile)
 
 	// deploy driver
 	deployCmd := exec.Command(filepath.Join(pkgDir, "deploy", "kubernetes", "deploy-driver.sh"), "--skip-sa-check")
-	deployCmd.Env = append(os.Environ(),
+	deployEnv = append(deployEnv,
 		fmt.Sprintf("GOPATH=%s", goPath),
-		fmt.Sprintf("GCE_PD_SA_DIR=%s", filepath.Dir(tmpSaFile)),
-		fmt.Sprintf("GCE_PD_DRIVER_VERSION=%s", deployOverlayName),
-	)
-	err = runCommand("Deploying driver", deployCmd)
+		fmt.Sprintf("GCE_PD_DRIVER_VERSION=%s", deployOverlayName))
+	deployCmd.Env = append(os.Environ(), deployEnv...)
+	err := runCommand("Deploying driver", deployCmd)
 	if err != nil {
 		return fmt.Errorf("failed to deploy driver: %v", err)
 	}
@@ -87,7 +91,7 @@ func installDriver(platform, goPath, pkgDir, stagingImage, stagingVersion, deplo
 		klog.Infof("Waiting 5 minutes for the driver to start on Linux")
 		time.Sleep(5 * time.Minute)
 	}
-	out, err = exec.Command("kubectl", "describe", "pods", "-n", driverNamespace).CombinedOutput()
+	out, err := exec.Command("kubectl", "describe", "pods", "-n", driverNamespace).CombinedOutput()
 	klog.Infof("describe pods \n %s", string(out))
 
 	if err != nil {
diff --git a/test/k8s-integration/main.go b/test/k8s-integration/main.go
@@ -96,7 +96,9 @@ func main() {
 		ensureVariable(deployOverlayName, false, "'deploy-overlay-name' must not be set when using GKE managed driver")
 	}
 
-	ensureVariable(saFile, true, "service-account-file is a required flag")
+	if *deployOverlayName != "noauth" {
+		ensureVariable(saFile, true, "service-account-file is a required flag")
+	}
 	if !*useGKEManagedDriver {
 		ensureVariable(deployOverlayName, true, "deploy-overlay-name is a required flag")
 	}
diff --git a/test/run-k8s-integration-local.sh b/test/run-k8s-integration-local.sh
@@ -10,8 +10,6 @@ readonly test_version=${TEST_VERSION:-master}
 
 source "${PKGDIR}/deploy/common.sh"
 
-ensure_var GCE_PD_SA_DIR
-
 make -C "${PKGDIR}" test-k8s-integration
 
 # This version of the command creates a GKE cluster. It also downloads and builds a k8s release
@@ -73,11 +71,29 @@ make -C "${PKGDIR}" test-k8s-integration
 # --gce-zone="us-central1-c" --num-nodes=${NUM_NODES:-3} --gke-release-channel="rapid" --deployment-strategy="gke" \
 # --use-gke-managed-driver=true --teardown-cluster=true
 
-# This version of the command does not build the driver or K8s, points to a
-# local K8s repo to get the e2e.test binary, and does not bring up or down the cluster
-
+# This version of the command does not build the driver or K8s, points to a local K8s repo to get 
+# the e2e.test binary, does not bring up or down the cluster, and uses application default
+# credentials instead of requiring a service account key.
+# 
+# Cluster nodes must have the proper GCP scopes set. This is done with kubetest by
+# NODE_SCOPES=https://www.googleapis.com/auth/cloud-platform \
+# KUBE_GCE_NODE_SERVICE_ACCOUNT=$SERVICE_ACCOUNT_NAME@$PROJECT.iam.gserviceaccount.com \
+# kubetest --up
+#
+# GCE_PD_SA_DIR is not used.
+#
+# As with all other methods local credentials must be set by running 
+#  gcloud auth application-default login
 "${PKGDIR}/bin/k8s-integration-test" --run-in-prow=false \
---staging-image="${GCE_PD_CSI_STAGING_IMAGE}" --service-account-file="${GCE_PD_SA_DIR}/cloud-sa.json" \
---deploy-overlay-name=dev --bringup-cluster=false --teardown-cluster=false --local-k8s-dir="$KTOP" \
---storageclass-files=sc-standard.yaml,sc-balanced.yaml,sc-ssd.yaml --do-driver-build=false --test-focus='External.Storage' \
+--deploy-overlay-name=noauth --bringup-cluster=false --teardown-cluster=false --local-k8s-dir="$KTOP" \
+--storageclass-files=sc-standard.yaml --do-driver-build=false --test-focus='External.Storage' \
 --gce-zone="us-central1-b" --num-nodes="${NUM_NODES:-3}"
+
+
+# This version of the command does not build the driver or K8s, points to a
+# local K8s repo to get the e2e.test binary, and does not bring up or down the cluster
+# "${PKGDIR}/bin/k8s-integration-test" --run-in-prow=false \
+# --staging-image="${GCE_PD_CSI_STAGING_IMAGE}" --service-account-file="${GCE_PD_SA_DIR}/cloud-sa.json" \
+# --deploy-overlay-name=dev --bringup-cluster=false --teardown-cluster=false --local-k8s-dir="$KTOP" \
+# --storageclass-files=sc-standard.yaml --do-driver-build=false --test-focus='External.Storage' \
+# --gce-zone="us-central1-b" --num-nodes="${NUM_NODES:-3}"

Original file line number	Diff line number	Diff line change
`@@ -96,7 +96,9 @@ func main() {`
`96`	`96`	`ensureVariable(deployOverlayName, false, "'deploy-overlay-name' must not be set when using GKE managed driver")`
`97`	`97`	`}`
`98`	`98`
`99`		`- ensureVariable(saFile, true, "service-account-file is a required flag")`
	`99`	`+ if *deployOverlayName != "noauth" {`
	`100`	`+ ensureVariable(saFile, true, "service-account-file is a required flag")`
	`101`	`+ }`
`100`	`102`	`if !*useGKEManagedDriver {`
`101`	`103`	`ensureVariable(deployOverlayName, true, "deploy-overlay-name is a required flag")`
`102`	`104`	`}`