Handle bidirectional typebounds in AvoidMap

An example of a bidirectional TypeBounds is i15523: `[A, B >: A]`, as an
ordering: `A <: B`, as types and their bounds: `A >: Nothing <: B` and
`B >: A <: Any`.  It's not an fatal cycle, such as `A <: B <: A`.  It's
more like "My father is DJ, DJ's son is me" - if you then go ask who
DJ's son's father is... you might never get an answer back.

Author: dwijnand

compiler/src/dotty/tools/dotc/core/TypeOps.scala

@@ -429,6 +429,8 @@ object TypeOps:
         sym.is(Package) || sym.isStatic && isStaticPrefix(pre.prefix)
       case _ => true
 
+    private var alreadyExpanding: List[TypeRef] = Nil
+
     override def apply(tp: Type): Type =
       try
         tp match
@@ -442,8 +444,12 @@ object TypeOps:
             tp.info match {
               case info: AliasingBounds =>
                 apply(info.alias)
-              case TypeBounds(lo, hi) =>
-                range(atVariance(-variance)(apply(lo)), apply(hi))
+              case bounds: TypeBounds if !alreadyExpanding.contains(tp) =>
+                val saved = alreadyExpanding
+                try
+                  alreadyExpanding ::= tp
+                  expandBounds(bounds)
+                finally alreadyExpanding = saved
               case info: ClassInfo =>
                 range(defn.NothingType, apply(classBound(info)))
               case _ =>

compiler/src/dotty/tools/dotc/typer/Inferencing.scala

@@ -417,7 +417,7 @@ object Inferencing {
         if safeToInstantiate then tvar.instantiate(fromBelow = v == -1)
         else {
           val bounds = TypeComparer.fullBounds(tvar.origin)
-          if bounds.hi <:< bounds.lo || bounds.hi.classSymbol.is(Final) then
+          if (bounds.hi frozen_<:< bounds.lo) || bounds.hi.classSymbol.is(Final) then
             tvar.instantiate(fromBelow = false)
           else {
             // We do not add the created symbols to GADT constraint immediately, since they may have inter-dependencies.

tests/pos/i14287.min.scala

@@ -0,0 +1,7 @@
+enum Foo[+F[_]]:
+  case Bar[B[_]](value: Foo[B]) extends Foo[B]
+
+class Test:
+  def test[X[_]](foo: Foo[X]): Foo[X] = foo match
+    case Foo.Bar(Foo.Bar(x)) => Foo.Bar(x)
+    case _                   => foo

tests/pos/i14287.scala

@@ -0,0 +1,16 @@
+// scalac: -Yno-deep-subtypes
+enum Free[+F[_], A]:
+  case Return(a: A)
+  case Suspend(s: F[A])
+  case FlatMap[F[_], A, B](
+    s: Free[F, A],
+    f: A => Free[F, B]) extends Free[F, B]
+
+  def flatMap[F2[x] >: F[x], B](f: A => Free[F2,B]): Free[F2,B] =
+    FlatMap(this, f)
+
+  @scala.annotation.tailrec
+  final def step: Free[F, A] = this match
+    case FlatMap(FlatMap(fx, f), g) => fx.flatMap(x => f(x).flatMap(y => g(y))).step
+    case FlatMap(Return(x), f) => f(x).step
+    case _ => this

tests/pos/i15523.avoid.scala

@@ -0,0 +1,8 @@
+// scalac: -Werror
+// like the original, but with a case body `a`
+// which caused type avoidance to infinitely recurse
+sealed trait Parent
+final case class Leaf[A, B >: A](a: A, b: B) extends Parent
+
+def run(x: Parent) = x match
+  case Leaf(a, _) => a

tests/pos/i15523.scala

@@ -0,0 +1,7 @@
+// scalac: -Werror
+sealed trait Parent
+final case class Leaf[A, B >: A](a: A, b: B) extends Parent
+
+def run(x: Parent): Unit = x match {
+  case Leaf(a, b) =>
+}