Fix #11078: avoid widening F-bounds in type avoidance

compiler/src/dotty/tools/dotc/core/TypeOps.scala

@@ -3,7 +3,7 @@ package dotc
 package core
 
 import Contexts._, Types._, Symbols._, Names._, Flags._
-import SymDenotations._
+import Denotations._, SymDenotations._
 import util.Spans._
 import util.Stats
 import NameKinds.DepParamName
@@ -462,6 +462,34 @@ object TypeOps:
           else
             range(defn.NothingType, defn.AnyType)
         }
+
+      /** Deviation from standard tryWiden:
+       *  - Don't widen F-bounds
+       */
+      override def tryWiden(tp: NamedType, pre: Type): Type = pre.member(tp.name) match {
+        case d: SingleDenotation =>
+          val tp1 = d.info.dealiasKeepAnnots
+          tp1.stripAnnots match {
+            case TypeAlias(alias) =>
+              // if H#T = U, then for any x in L..H, x.T =:= U,
+              // hence we can replace with U under all variances
+              reapply(alias.rewrapAnnots(tp1))
+
+            case tb: TypeBounds =>
+              // Don't widen F-bounds
+              val isFBounds = tb.existsPart(p => p.isInstanceOf[LazyRef], forceLazy = false)
+              if isFBounds then NoType
+              else expandBounds(tb)
+            case info: SingletonType =>
+              // if H#x: y.type, then for any x in L..H, x.type =:= y.type,
+              // hence we can replace with y.type under all variances
+              reapply(info)
+            case _ =>
+              NoType
+          }
+        case _ => NoType
+      }
+
     }
 
     widenMap(tp)

tests/pos/i11078.scala

@@ -0,0 +1,29 @@
+trait Foo[A <: Foo[A]]
+trait FooCreator[A <: Foo[A]] {
+  def createFoo(): A
+}
+
+trait FooWrapper {
+  type A <: Foo[A]
+  def foo: A
+}
+object FooWrapper {
+  def apply[A0 <: Foo[A0]](toWrap: A0): FooWrapper { type A = A0 } = new FooWrapper {
+    type A = A0
+    def foo: A0 = toWrap
+  }
+}
+
+trait FooCreatorWrapper {
+  type A <: Foo[A]
+  def fooCreator: FooCreator[A]
+}
+
+sealed trait Bar
+object Bar {
+  case class Baz(wrapper: FooCreatorWrapper) extends Bar
+}
+
+def process(bar: Bar): FooWrapper = bar match {
+  case Bar.Baz(wrapper) => FooWrapper(wrapper.fooCreator.createFoo())
+}

tests/pos/i11078b.scala

@@ -0,0 +1,11 @@
+class Test {
+  trait Foo[A <: Foo[A]]
+
+  trait FooWrapper with self =>
+    type A <: Foo[A]
+    def doThing(foo: FooWrapper): FooWrapper { type A = self.A } = ???
+  end FooWrapper
+
+  val foos: scala.Seq[FooWrapper] = ???
+  val newFoo = foos.foldLeft(??? : FooWrapper)((topFoo, foo) => topFoo.doThing(foo))
+}