Update gce operation timeout

Update gce operation timeout to 2 mins instead of 5 mins. The 90% of
attach/detach disk is less than 15 second, and create volume is less
than 1 min.

Reduce the timeout of volume operation so that it can recover from
previous operation quicker in case operation is dropped.

Author: jingxu97

deploy/kubernetes/base/controller/controller.yaml

@@ -52,7 +52,7 @@ spec:
             - "--metrics-address=:22012"
             - "--leader-election"
             - "--leader-election-namespace=$(PDCSI_NAMESPACE)"
-            - "--timeout=250s"
+            - "--timeout=60s"
           env:
             - name: PDCSI_NAMESPACE
               valueFrom:

pkg/gce-cloud-provider/compute/gce-compute.go

@@ -46,6 +46,10 @@ const (
 	GCEAPIVersionV1 GCEAPIVersion = "v1"
 	// Alpha key type
 	GCEAPIVersionBeta GCEAPIVersion = "beta"
+	// Timeout for checking on an operation
+	OpTimeout = 2 * time.Minute
+	// Time interval for checking on an operation
+	OpInterval = 3 * time.Second
 )
 
 type GCECompute interface {
@@ -611,22 +615,24 @@ func (cloud *CloudProvider) AttachDisk(ctx context.Context, volKey *meta.Key, re
 	if err != nil {
 		return fmt.Errorf("failed cloud service attach disk call: %v", err)
 	}
+	klog.V(5).Infof("Attaching disk %s operation id is %s", volKey, op.Name)
 	err = cloud.waitForZonalOp(ctx, op.Name, instanceZone)
 	if err != nil {
-		return fmt.Errorf("failed when waiting for zonal op: %v", err)
+		return fmt.Errorf("failed when waiting for zonal op %s on attaching volume %v: %v", op.Name, volKey, err)
 	}
 	return nil
 }
 
 func (cloud *CloudProvider) DetachDisk(ctx context.Context, deviceName, instanceZone, instanceName string) error {
-	klog.V(5).Infof("Detaching disk %v from %v", deviceName, instanceName)
+	klog.V(5).Infof("Detaching disk %s from %v", deviceName, instanceName)
 	op, err := cloud.service.Instances.DetachDisk(cloud.project, instanceZone, instanceName, deviceName).Context(ctx).Do()
 	if err != nil {
 		return err
 	}
+	klog.V(5).Infof("Detaching disk %s operation id is %s", deviceName, op.Name)
 	err = cloud.waitForZonalOp(ctx, op.Name, instanceZone)
 	if err != nil {
-		return err
+		return fmt.Errorf("failed when waiting for zonal op %s on detaching volume %s: %v", op.Name, deviceName, err)
 	}
 	return nil
 }
@@ -681,7 +687,7 @@ func (cloud *CloudProvider) waitForZonalOp(ctx context.Context, opName string, z
 	// The v1 API can query for v1, alpha, or beta operations.
 	svc := cloud.service
 	project := cloud.project
-	return wait.Poll(3*time.Second, 5*time.Minute, func() (bool, error) {
+	return wait.Poll(OpInterval, OpTimeout, func() (bool, error) {
 		pollOp, err := svc.ZoneOperations.Get(project, zone, opName).Context(ctx).Do()
 		if err != nil {
 			klog.Errorf("WaitForOp(op: %s, zone: %#v) failed to poll the operation", opName, zone)
@@ -743,7 +749,10 @@ func (cloud *CloudProvider) WaitForAttach(ctx context.Context, volKey *meta.Key,
 }
 
 func opIsDone(op *computev1.Operation) (bool, error) {
-	if op == nil || op.Status != operationStatusDone {
+	if op == nil {
+		return true, fmt.Errorf("operation is nil")
+	}
+	if op.Status != operationStatusDone {
 		return false, nil
 	}
 	if op.Error != nil && len(op.Error.Errors) > 0 && op.Error.Errors[0] != nil {