Fix scala#11234: avoid cycles in unifying F-bounded type parameters

**Note**: this is a trial as `Constraints` should be immutable.

In tests/pos/11234.scala, we have the following F-bounded constraints:

    bounds =
	A0 <: Foo[LazyRef(A0)]
	A <: Foo[LazyRef(A)]
    ordering =
	A <: A0

As `Foo[T]` is non-variant, at some point we will add `A0 <: A`, it
will trigger unification of `A0` and `A`. The unification will call
`Foo[A0].&(Foo[A])`, which in turn calls `TypeComparer.glb(Foo[A0],
Foo[A])`.  The call `glb(Foo[A0], Foo[A])` in a fresh TypeComparer
will in turn add `A0 <: A` thus trigger the unification again.

We break the loop by remembering the unification pair and use the
knowledge in subtyping.

Author: liufengyun

compiler/src/dotty/tools/dotc/core/Constraint.scala

@@ -101,6 +101,9 @@ abstract class Constraint extends Showable {
    */
   def unify(p1: TypeParamRef, p2: TypeParamRef)(using Context): This
 
+  /** Is it currently unifying p1 and p2? */
+  def isUnifying(p1: TypeParamRef, p2: TypeParamRef)(using Context): Boolean
+
   /** A new constraint which is derived from this constraint by removing
    *  the type parameter `param` from the domain and replacing all top-level occurrences
    *  of the parameter elsewhere in the constraint by type `tp`, or a conservative

compiler/src/dotty/tools/dotc/core/OrderingConstraint.scala

@@ -393,9 +393,16 @@ class OrderingConstraint(private val boundsMap: ParamBounds,
     order(this, param1, param2).checkNonCyclic()
 
   def unify(p1: TypeParamRef, p2: TypeParamRef)(using Context): This =
+    unifyingPair = (p2, p1)
     val p1Bounds = (nonParamBounds(p1) & nonParamBounds(p2)).substParam(p2, p1)
+    unifyingPair = (NoType, NoType)
     updateEntry(p1, p1Bounds).replace(p2, p1)
 
+  private var unifyingPair: (Type, Type) = (NoType, NoType)
+
+  def isUnifying(p1: TypeParamRef, p2: TypeParamRef)(using Context): Boolean =
+    unifyingPair._1 == p1 && unifyingPair._2 == p2
+
 // ---------- Replacements and Removals -------------------------------------
 
   /** A new constraint which is derived from this constraint by removing

compiler/src/dotty/tools/dotc/core/TypeComparer.scala

@@ -390,6 +390,13 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
         }
         def compareTypeParamRef =
           assumedTrue(tp1) ||
+          tp2.match {
+            case tp2: TypeParamRef =>
+              constraint.isUnifying(tp1, tp2) ||
+              inFrozenConstraint(constraint.isLess(tp1, tp2))
+            case _ =>
+              false
+          } ||
           isSubTypeWhenFrozen(bounds(tp1).hi, tp2) || {
             if (canConstrain(tp1) && !approx.high)
               addConstraint(tp1, tp2, fromBelow = false) && flagNothingBound
@@ -540,11 +547,7 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
           // widening in `fourthTry` before adding to the constraint.
           if (frozenConstraint) recur(tp1, bounds(tp2).lo)
           else isSubTypeWhenFrozen(tp1, tp2)
-        alwaysTrue ||
-        frozenConstraint && (tp1 match {
-          case tp1: TypeParamRef => constraint.isLess(tp1, tp2)
-          case _ => false
-        }) || {
+        alwaysTrue || {
           if (canConstrain(tp2) && !approx.low)
             addConstraint(tp2, tp1.widenExpr, fromBelow = true)
           else fourthTry

compiler/src/dotty/tools/dotc/core/TypeOps.scala

@@ -3,7 +3,7 @@ package dotc
 package core
 
 import Contexts._, Types._, Symbols._, Names._, Flags._
-import Denotations._, SymDenotations._
+import SymDenotations._
 import util.Spans._
 import util.Stats
 import NameKinds.DepParamName

tests/pos/i11234.scala

@@ -0,0 +1,19 @@
+trait Foo[A <: Foo[A]]
+trait FooCreator[A <: Foo[A]] {
+  def createFoo(): A
+}
+
+trait FooWrapper {
+  type A <: Foo[A]
+  def foo: A
+}
+
+object FooWrapper {
+  def apply[A0 <: Foo[A0]](toWrap: A0): FooWrapper { type A = A0 } = new FooWrapper {
+    type A = A0
+    def foo: A0 = toWrap
+  }
+}
+
+def error(fooWrapper: FooWrapper, processor: [A <: Foo[A]] => A => A): FooWrapper =
+  FooWrapper(processor(fooWrapper.foo))