SA1026, SA5008: don't loop forever when marshaling cyclic pointer type

Closes gh-1202

(cherry picked from commit 4a64968)

Author: dominikh

staticcheck/fakexml/marshal.go

@@ -113,6 +113,15 @@ var textMarshalerType = types.NewInterfaceType([]*types.Func{
 
 var N = 0
 
+type CyclicTypeError struct {
+	Type types.Type
+	Path string
+}
+
+func (err *CyclicTypeError) Error() string {
+	return "cyclic type"
+}
+
 // marshalValue writes one or more XML elements representing val.
 // If val was obtained from a struct field, finfo must have its details.
 func (e *Encoder) marshalValue(val fakereflect.TypeAndCanAddr, finfo *fieldInfo, startTemplate *StartElement, stack string) error {
@@ -122,11 +131,17 @@ func (e *Encoder) marshalValue(val fakereflect.TypeAndCanAddr, finfo *fieldInfo,
 	e.seen[val] = struct{}{}
 
 	// Drill into interfaces and pointers.
+	seen := map[fakereflect.TypeAndCanAddr]struct{}{}
 	for val.IsInterface() || val.IsPtr() {
 		if val.IsInterface() {
 			return nil
 		}
 		val = val.Elem()
+		if _, ok := seen[val]; ok {
+			// Loop in type graph, e.g. 'type P *P'
+			return &CyclicTypeError{val.Type, stack}
+		}
+		seen[val] = struct{}{}
 	}
 
 	// Check for marshaler.

staticcheck/fakexml/typeinfo.go

@@ -249,8 +249,14 @@ func StructFieldInfo(f fakereflect.StructField) (*fieldInfo, error) {
 // in case it exists and has a valid xml field tag, otherwise
 // it returns nil.
 func lookupXMLName(typ fakereflect.TypeAndCanAddr) (xmlname *fieldInfo) {
+	seen := map[fakereflect.TypeAndCanAddr]struct{}{}
 	for typ.IsPtr() {
 		typ = typ.Elem()
+		if _, ok := seen[typ]; ok {
+			// Loop in type graph, e.g. 'type P *P'
+			return nil
+		}
+		seen[typ] = struct{}{}
 	}
 	if !typ.IsStruct() {
 		return nil

staticcheck/lint.go

@@ -897,6 +897,13 @@ func checkUnsupportedMarshalXML(call *Call) {
 			} else {
 				arg.Invalid(fmt.Sprintf("trying to marshal unsupported type %s, via %s", typ, err.Path))
 			}
+		case *fakexml.CyclicTypeError:
+			typ := types.TypeString(err.Type, types.RelativeTo(arg.Value.Value.Parent().Pkg.Pkg))
+			if err.Path == "x" {
+				arg.Invalid(fmt.Sprintf("trying to marshal cyclic type %s", typ))
+			} else {
+				arg.Invalid(fmt.Sprintf("trying to marshal cyclic type %s, via %s", typ, err.Path))
+			}
 		case *fakexml.TagPathError:
 			// Vet does a better job at reporting this error, because it can flag the actual struct tags, not just the call to Marshal
 		default:

staticcheck/testdata/src/CheckUnsupportedMarshal/CheckUnsupportedMarshal.go

@@ -303,3 +303,15 @@ func toplevelPointer() {
 	xml.Marshal(ToplevelPointerMarshalerXML{})  // want `unsupported type`
 	xml.Marshal(ToplevelPointerMarshalerText{}) // want `unsupported type`
 }
+
+func cyclicPointer() {
+	type P *P
+	type S2 struct {
+		Bar P
+	}
+	type S1 struct {
+		Foo S2
+	}
+	var s S1
+	xml.Marshal(s) // want `cyclic type P, via x\.Foo\.Bar`
+}