return Unavailable for connection reset by peer errors

Author: amacaskill

pkg/common/utils.go

@@ -346,6 +346,9 @@ func CodeForError(sourceError error) codes.Code {
 	if code, err := isContextError(sourceError); err == nil {
 		return code
 	}
+	if code, err := isConnectionResetError(sourceError); err == nil {
+		return code
+	}
 
 	var apiErr *googleapi.Error
 	if !errors.As(sourceError, &apiErr) {
@@ -377,6 +380,20 @@ func isContextError(err error) (codes.Code, error) {
 	return codes.Unknown, fmt.Errorf("Not a context error: %w", err)
 }
 
+// isConnectionResetError returns the grpc error code Unavailable if the
+// passed in error contains the "connection reset by peer" string.
+func isConnectionResetError(err error) (codes.Code, error) {
+	if err == nil {
+		return codes.Unknown, fmt.Errorf("null error")
+	}
+
+	errStr := err.Error()
+	if strings.Contains(errStr, "connection reset by peer") {
+		return codes.Unavailable, nil
+	}
+	return codes.Unknown, fmt.Errorf("Not a connection reset error: %w", err)
+}
+
 // isUserMultiAttachError returns an InvalidArgument if the error is
 // multi-attach detected from the API server. If we get this error from the API
 // server, it means that the kubelet doesn't know about the multiattch so it is

pkg/common/utils_test.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"net/http"
 	"reflect"
+	"syscall"
 	"testing"
 
 	"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta"
@@ -1006,6 +1007,16 @@ func TestCodeForError(t *testing.T) {
 			inputErr: context.DeadlineExceeded,
 			expCode:  codes.DeadlineExceeded,
 		},
+		{
+			name:     "connection reset error",
+			inputErr: fmt.Errorf("failed to getDisk: connection reset by peer"),
+			expCode:  codes.Unavailable,
+		},
+		{
+			name:     "wrapped connection reset error",
+			inputErr: fmt.Errorf("received error: %v", syscall.ECONNRESET),
+			expCode:  codes.Unavailable,
+		},
 		{
 			name:     "status error with Aborted error code",
 			inputErr: status.Error(codes.Aborted, "aborted error"),