Update gce operation timeout

jingxu97 · jingxu97 · commit aa1e558dcf56 · 2020-11-10T19:54:26.000-08:00
Update gce operation timeout to 2 mins instead of 5 mins. The 90% of
attach/detach disk is less than 15 second, and create volume is less
than 1 min.

Reduce the timeout of volume operation so that it can recover from
previous operation quicker in case operation is dropped.
diff --git a/deploy/kubernetes/base/controller/controller.yaml b/deploy/kubernetes/base/controller/controller.yaml
@@ -32,7 +32,7 @@ spec:
             - "--enable-leader-election"
             - "--leader-election-type=leases"
             - "--leader-election-namespace=$(PDCSI_NAMESPACE)"
-            - "--timeout=250s"
+            - "--timeout=180s"
             - "--extra-create-metadata"
           # - "--run-controller-service=false" # disable the controller service of the CSI driver
           # - "--run-node-service=false"       # disable the node service of the CSI driver
@@ -52,7 +52,7 @@ spec:
             - "--metrics-address=:22012"
             - "--leader-election"
             - "--leader-election-namespace=$(PDCSI_NAMESPACE)"
-            - "--timeout=250s"
+            - "--timeout=60s"
           env:
             - name: PDCSI_NAMESPACE
               valueFrom:
diff --git a/deploy/kubernetes/images/alpha/image.yaml b/deploy/kubernetes/images/alpha/image.yaml
@@ -4,8 +4,10 @@ metadata:
   name: imagetag-gcepd-driver-alpha-win
 imageTag:
   name: gke.gcr.io/gcp-compute-persistent-disk-csi-driver
-  newName: gcr.io/gke-release-staging/gcp-compute-persistent-disk-csi-driver
-  newTag: "v1.0.1-gke.9"
+  #newName: gcr.io/gke-release-staging/gcp-compute-persistent-disk-csi-driver
+  #newTag: "v1.0.1-gke.9"
+  newName: gcr.io/jinxu-gke-multi-cloud-dev/gce-pd-windows
+  newTag: v5
 ---
 
 apiVersion: builtin
diff --git a/go.mod b/go.mod
@@ -9,7 +9,7 @@ require (
 	github.com/golang/protobuf v1.4.2
 	github.com/google/uuid v1.1.1
 	github.com/hashicorp/go-multierror v1.0.0 // indirect
-	github.com/kubernetes-csi/csi-proxy/client v0.2.1
+	github.com/kubernetes-csi/csi-proxy/client v0.2.2
 	github.com/kubernetes-csi/csi-test/v3 v3.0.0
 	github.com/onsi/ginkgo v1.11.0
 	github.com/onsi/gomega v1.7.1
diff --git a/go.sum b/go.sum
@@ -476,8 +476,11 @@ github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kubernetes-csi/csi-proxy v0.2.2 h1:LqablYFEGw7FYBjwoh5TeXFzlcx8C+YQjKfGy6fFWJs=
 github.com/kubernetes-csi/csi-proxy/client v0.2.1 h1:n21d2U9HvgQ6jfJayafRv8kXXtLvnRNEqoD0mQNucKc=
 github.com/kubernetes-csi/csi-proxy/client v0.2.1/go.mod h1:6ptQQmti5QHwBxSsh8Cy00oGdogj0JXewFnu8FFjgOs=
+github.com/kubernetes-csi/csi-proxy/client v0.2.2 h1:VpMddHnbYA1oBeU5nrisdyrpOAAT0HqME7fsTi6BG2w=
+github.com/kubernetes-csi/csi-proxy/client v0.2.2/go.mod h1:6ptQQmti5QHwBxSsh8Cy00oGdogj0JXewFnu8FFjgOs=
 github.com/kubernetes-csi/csi-test/v3 v3.0.0 h1:mVsfA4J67uNm8fdF/Pr84oMqL92qjIhjWbEUH8zv1fU=
 github.com/kubernetes-csi/csi-test/v3 v3.0.0/go.mod h1:VdIKGnDZHOjg4M5yd0OZICtsoEzdn64d0K33N6dm35Q=
 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
diff --git a/pkg/gce-cloud-provider/compute/gce-compute.go b/pkg/gce-cloud-provider/compute/gce-compute.go
@@ -46,6 +46,10 @@ const (
 	GCEAPIVersionV1 GCEAPIVersion = "v1"
 	// Alpha key type
 	GCEAPIVersionBeta GCEAPIVersion = "beta"
+	// Timeout for checking on an operation
+	OpTimeout = 2 * time.Minute
+	// Time interval for checking on an operation
+	OpInterval = 3 * time.Second
 )
 
 type GCECompute interface {
@@ -611,22 +615,24 @@ func (cloud *CloudProvider) AttachDisk(ctx context.Context, volKey *meta.Key, re
 	if err != nil {
 		return fmt.Errorf("failed cloud service attach disk call: %v", err)
 	}
+	klog.V(5).Infof("Attaching disk %s operation id is %s", volKey, op.Name)
 	err = cloud.waitForZonalOp(ctx, op.Name, instanceZone)
 	if err != nil {
-		return fmt.Errorf("failed when waiting for zonal op: %v", err)
+		return fmt.Errorf("failed when waiting for zonal op %s on attaching volume %v: %v", op.Name, volKey, err)
 	}
 	return nil
 }
 
 func (cloud *CloudProvider) DetachDisk(ctx context.Context, deviceName, instanceZone, instanceName string) error {
-	klog.V(5).Infof("Detaching disk %v from %v", deviceName, instanceName)
+	klog.V(5).Infof("Detaching disk %s from %v", deviceName, instanceName)
 	op, err := cloud.service.Instances.DetachDisk(cloud.project, instanceZone, instanceName, deviceName).Context(ctx).Do()
 	if err != nil {
 		return err
 	}
+	klog.V(5).Infof("Detaching disk %s operation id is %s", deviceName, op.Name)
 	err = cloud.waitForZonalOp(ctx, op.Name, instanceZone)
 	if err != nil {
-		return err
+		return fmt.Errorf("failed when waiting for zonal op %s on detaching volume %s: %v", op.Name, deviceName, err)
 	}
 	return nil
 }
@@ -681,7 +687,11 @@ func (cloud *CloudProvider) waitForZonalOp(ctx context.Context, opName string, z
 	// The v1 API can query for v1, alpha, or beta operations.
 	svc := cloud.service
 	project := cloud.project
-	return wait.Poll(3*time.Second, 5*time.Minute, func() (bool, error) {
+	timeout := OpTimeout
+	if deadline, ok := ctx.Deadline(); ok {
+		timeout = time.Until(deadline)
+	}
+	return wait.Poll(OpInterval, timeout, func() (bool, error) {
 		pollOp, err := svc.ZoneOperations.Get(project, zone, opName).Context(ctx).Do()
 		if err != nil {
 			klog.Errorf("WaitForOp(op: %s, zone: %#v) failed to poll the operation", opName, zone)
@@ -743,7 +753,10 @@ func (cloud *CloudProvider) WaitForAttach(ctx context.Context, volKey *meta.Key,
 }
 
 func opIsDone(op *computev1.Operation) (bool, error) {
-	if op == nil || op.Status != operationStatusDone {
+	if op == nil {
+		return true, fmt.Errorf("operation is nil")
+	}
+	if op.Status != operationStatusDone {
 		return false, nil
 	}
 	if op.Error != nil && len(op.Error.Errors) > 0 && op.Error.Errors[0] != nil {
diff --git a/pkg/gce-pd-csi-driver/node.go b/pkg/gce-pd-csi-driver/node.go
@@ -99,10 +99,63 @@ func (ns *GCENodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePub
 		return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("VolumeCapability is invalid: %v", err))
 	}
 
-	notMnt, err := ns.Mounter.Interface.IsLikelyNotMountPoint(targetPath)
+	/*notMnt, err := ns.Mounter.Interface.IsLikelyNotMountPoint(targetPath)
 	if err != nil && !os.IsNotExist(err) {
 		return nil, status.Error(codes.Internal, fmt.Sprintf("cannot validate mount point: %s %v", targetPath, err))
+	}*/
+
+	_, lserr := os.Lstat(targetPath)
+	klog.Infof("lstat %s: %v", targetPath, lserr)
+	if lserr != nil {
+		if os.IsNotExist(lserr) {
+			klog.Infof("lstat not exist error")
+		}
+	}
+
+	_, lserr = os.Stat(targetPath)
+	klog.Infof("lstat %s: %v", targetPath, lserr)
+	if lserr != nil {
+		if os.IsNotExist(lserr) {
+			klog.Infof("Stat not exist error")
+		}
 	}
+	merr := os.MkdirAll(targetPath, 0750)
+	klog.Infof("mkdir %s, %v", targetPath, merr)
+
+	_, lserr = os.Lstat(targetPath)
+	klog.Infof("lstat %s: %v", targetPath, lserr)
+	if lserr != nil {
+		if os.IsNotExist(lserr) {
+			klog.Infof("lstat not exist error")
+		}
+	}
+
+	_, lserr = os.Stat(targetPath)
+	klog.Infof("lstat %s: %v", targetPath, lserr)
+	if lserr != nil {
+		if os.IsNotExist(lserr) {
+			klog.Infof("Stat not exist error")
+		}
+	}
+
+	rerr := os.RemoveAll(targetPath)
+	klog.Infof("remove it %v", rerr)
+
+	_, lserr = os.Lstat(targetPath)
+	klog.Infof("lstat %s: %v", targetPath, lserr)
+	if lserr != nil {
+		if os.IsNotExist(lserr) {
+			klog.Infof("lstat not exist error")
+		}
+	}
+
+	notMnt, err := ns.Mounter.Interface.IsLikelyNotMountPoint(targetPath)
+	klog.Infof("ismountpoint %s: %v", targetPath, err)
+	if err != nil && !os.IsNotExist(err) {
+		klog.Errorf("ismountpoint not exist err")
+		//return nil, status.Error(codes.Internal, fmt.Sprintf("cannot validate mount point: %s %v", targetPath, err))
+	}
+
 	if !notMnt {
 		// TODO(#95): check if mount is compatible. Return OK if it is, or appropriate error.
 		/*

-Original file line number
+Diff line change
 	github.com/golang/protobuf v1.4.2
 	github.com/google/uuid v1.1.1
 	github.com/hashicorp/go-multierror v1.0.0 // indirect
 -	github.com/kubernetes-csi/csi-proxy/client v0.2.1
 +	github.com/kubernetes-csi/csi-proxy/client v0.2.2
 	github.com/kubernetes-csi/csi-test/v3 v3.0.0
 	github.com/onsi/ginkgo v1.11.0
 	github.com/onsi/gomega v1.7.1