unused: don't get stuck chasing mutually recursive type instantiations

Closes gh-1247

(cherry picked from commit 6dbb73d)

Author: dominikh

unused/testdata/src/typeparams/typeparams.go

@@ -70,3 +70,17 @@ func Fn9[T struct { // used
 type s2 struct{} // used
 
 func fn10[E any](x []E) {} // unused
+
+type Tree[T any] struct { // used
+	Root *Node[T] // used
+}
+
+type Node[T any] struct { // used
+	Tree *Tree[T] // used
+}
+
+type foo struct{} // used
+
+type Bar *Node[foo] // used
+
+func (n Node[T]) anyMethod() {} // unused

unused/unused.go

@@ -832,6 +832,9 @@ func (g *graph) see(obj interface{}) *node {
 	if fn, ok := obj.(*types.Func); ok {
 		obj = typeparams.OriginMethod(fn)
 	}
+	if t, ok := obj.(*types.Named); ok {
+		obj = typeparams.NamedTypeOrigin(t)
+	}
 
 	// add new node to graph
 	node, _ := g.node(obj)
@@ -871,6 +874,13 @@ func (g *graph) use(used, by interface{}, kind edgeKind) {
 		by = typeparams.OriginMethod(fn)
 	}
 
+	if t, ok := used.(*types.Named); ok {
+		used = typeparams.NamedTypeOrigin(t)
+	}
+	if t, ok := by.(*types.Named); ok {
+		by = typeparams.NamedTypeOrigin(t)
+	}
+
 	usedNode, new := g.node(used)
 	assert(!new)
 	if by == nil {
@@ -1427,13 +1437,14 @@ func (g *graph) typ(t types.Type, parent types.Type) {
 		// Nothing to do
 	case *types.Named:
 		// (9.3) types use their underlying and element types
-		g.seeAndUse(t.Underlying(), t, edgeUnderlyingType)
+		origin := typeparams.NamedTypeOrigin(t)
+		g.seeAndUse(origin.Underlying(), t, edgeUnderlyingType)
 		g.seeAndUse(t.Obj(), t, edgeTypeName)
 		g.seeAndUse(t, t.Obj(), edgeNamedType)
 
 		// (2.4) named types use the pointer type
 		if _, ok := t.Underlying().(*types.Interface); !ok && t.NumMethods() > 0 {
-			g.seeAndUse(g.newPointer(t), t, edgePointerType)
+			g.seeAndUse(g.newPointer(origin), t, edgePointerType)
 		}
 
 		// (2.5) named types use their type parameters
@@ -1462,7 +1473,7 @@ func (g *graph) typ(t types.Type, parent types.Type) {
 			g.function(g.pkg.IR.Prog.FuncValue(t.Method(i)))
 		}
 
-		g.typ(t.Underlying(), t)
+		g.typ(origin.Underlying(), t)
 	case *types.Slice:
 		// (9.3) types use their underlying and element types
 		g.seeAndUse(t.Elem(), t, edgeElementType)