Fix #8976: Do not instantiate type variables outside bounds

compiler/src/dotty/tools/dotc/core/TypeErrors.scala

@@ -192,3 +192,11 @@ class MergeError(val sym1: Symbol, val sym2: Symbol, val tp1: Type, val tp2: Typ
        """
   }
 }
+
+class NoInstance(param: TypeParamRef, val inst: Type, bounds: TypeBounds) extends TypeError:
+  override def produceMessage(using Context): Message =
+    i"""Type variable $param cannot be instantiated.
+       |Its attempted instantiation type $inst
+       |does not conform to its bounds $bounds}"""
+
+

compiler/src/dotty/tools/dotc/core/Types.scala

@@ -4029,6 +4029,13 @@ object Types {
       case bound: OrType   => occursIn(bound.tp1, fromBelow) || occursIn(bound.tp2, fromBelow)
       case _ => false
     }
+
+    /** Is `inst` a legal instance type for this type parameter?
+     *  TODO: Check whether we can always check both bounds even if
+     *        one check would be redundant. But maybe that's cheap enough.
+     */
+    def canInstantiateWith(inst: Type, fromBelow: Boolean)(using Context): Boolean =
+      if fromBelow then inst frozen_<:< this else this frozen_<:< inst
   }
 
   private final class TypeParamRefImpl(binder: TypeLambda, paramNum: Int) extends TypeParamRef(binder, paramNum)
@@ -4176,7 +4183,7 @@ object Types {
         def msg = i"Inaccessible variables captured in instantation of type variable $this.\n$tp was fixed to $atp"
         typr.println(msg)
         val bound = ctx.typeComparer.fullUpperBound(origin)
-        if !(atp <:< bound) then
+        if !(atp <:< bound) then // TODO: compare with param instead
           throw new TypeError(s"$msg,\nbut the latter type does not conform to the upper bound $bound")
         atp
       // AVOIDANCE TODO: This really works well only if variables are instantiated from below
@@ -4187,24 +4194,48 @@ object Types {
       // To do this, we need first test cases for that situation.
 
     /** Instantiate variable with given type */
-    def instantiateWith(tp: Type)(implicit ctx: Context): Type = {
+    def instantiateWith(tp: Type)(implicit ctx: Context): Unit =
       assert(tp ne this, s"self instantiation of ${tp.show}, constraint = ${ctx.typerState.constraint.show}")
       typr.println(s"instantiating ${this.show} with ${tp.show}")
-      if ((ctx.typerState eq owningState.get) && !ctx.typeComparer.subtypeCheckInProgress)
+      if (ctx.typerState eq owningState.get) && !ctx.typeComparer.subtypeCheckInProgress then
         inst = tp
       ctx.typerState.constraint = ctx.typerState.constraint.replace(origin, tp)
-      tp
-    }
+
+    /** The type this type variable should be instantiated with.
+     *  This is the type-comparer's instance type with the added tweak that
+     *  captured for more deeply nested symbols are avoided.
+     */
+    def instanceType(fromBelow: Boolean)(using Context): Type =
+      avoidCaptures(ctx.typeComparer.instanceType(origin, fromBelow))
 
     /** Instantiate variable from the constraints over its `origin`.
-     *  If `fromBelow` is true, the variable is instantiated to the lub
-     *  of its lower bounds in the current constraint; otherwise it is
-     *  instantiated to the glb of its upper bounds. However, a lower bound
-     *  instantiation can be a singleton type only if the upper bound
-     *  is also a singleton type.
+     *  If `fromBelow` is true, the variable's instance type is computed from
+     *  the lub of its lower bounds in the current constraint; otherwise it is
+     *  computed from the glb of its upper bounds.
+     *  After that, there are some further transformations and checks:
+     *
+     *   - Singletons and union types are sometimes widened according to
+     *     `ConstraintHandling.widenInferred`.
+     *   - References to more neeply nested symbols are avoided
+     *     according to `avoidCaptures`
+     *   - It is checked that the resulting type still fits within
+     *     bounds. If not, a `NoInstance` exception is thrown,
+     *     but only after instantiating the variable anyway because
+     *     this avoids follow-on errors.
      */
     def instantiate(fromBelow: Boolean)(implicit ctx: Context): Type =
-      instantiateWith(avoidCaptures(ctx.typeComparer.instanceType(origin, fromBelow)))
+      val inst = instanceType(fromBelow)
+      val instOK = origin.canInstantiateWith(inst, fromBelow)
+      if instOK then
+        instantiateWith(inst)
+      else
+        typr.println(i"failure to instantiate $this with $inst")
+        val bounds = ctx.typeComparer.fullBounds(origin)
+        instantiateWith(inst) // instantiate anyway to prevent subsequent instantiation attempts and errors
+        val noInstance = NoInstance(origin, inst, bounds)
+        if ctx.settings.Ydebug.value then noInstance.printStackTrace()
+        throw noInstance
+      inst
 
     /** For uninstantiated type variables: Is the lower bound different from Nothing? */
     def hasLowerBound(implicit ctx: Context): Boolean =

compiler/src/dotty/tools/dotc/typer/Inferencing.scala

@@ -112,11 +112,17 @@ object Inferencing {
   private class IsFullyDefinedAccumulator(force: ForceDegree.Value, minimizeSelected: Boolean = false)
     (using Context) extends TypeAccumulator[Boolean] {
 
-    private def instantiate(tvar: TypeVar, fromBelow: Boolean): Type = {
-      val inst = tvar.instantiate(fromBelow)
-      typr.println(i"forced instantiation of ${tvar.origin} = $inst")
+    private var instancesOK = true
+
+    private def instantiate(tvar: TypeVar, fromBelow: Boolean): Type =
+      val inst = tvar.instanceType(fromBelow)
+      if tvar.origin.canInstantiateWith(inst, fromBelow) then
+        tvar.instantiateWith(inst)
+        typr.println(i"forced instantiation of ${tvar.origin} = $inst")
+      else
+        typr.println(i"failed forced instantiation of ${tvar.origin} = $inst")
+        instancesOK = false
       inst
-    }
 
     private var toMaximize: List[TypeVar] = Nil
 
@@ -147,6 +153,7 @@ object Inferencing {
     }
 
     def process(tp: Type): Boolean =
+      instancesOK = true
       // Maximize type vars in the order they were visited before */
       def maximize(tvars: List[TypeVar]): Unit = tvars match
         case tvar :: tvars1 =>
@@ -158,10 +165,11 @@ object Inferencing {
       && (
         toMaximize.isEmpty
         || { maximize(toMaximize)
-             toMaximize = Nil       // Do another round since the maximixing instances
-             process(tp)            // might have type uninstantiated variables themselves.
-           }
+            toMaximize = Nil       // Do another round since the maximixing instances
+            process(tp)            // might have type uninstantiated variables themselves.
+          }
       )
+      && instancesOK
   }
 
   def approximateGADT(tp: Type)(implicit ctx: Context): Type = {
@@ -233,8 +241,11 @@ object Inferencing {
               case TypeBounds(lo, hi)
               if (hi frozen_<:< lo) =>
                 val inst = accCtx.typeComparer.approximation(param, fromBelow = true)
-                typr.println(i"replace singleton $param := $inst")
-                accCtx.typerState.constraint = constraint.replace(param, inst)
+                if param.canInstantiateWith(inst, fromBelow = true) then
+                  typr.println(i"replace singleton $param := $inst")
+                  accCtx.typerState.constraint = constraint.replace(param, inst)
+                else
+                  typr.println(i"failed to replace singleton $param $inst")
               case _ =>
             }
           case _ =>

tests/neg/i8976.scala

@@ -0,0 +1,5 @@
+trait Cons[X, Y]
+
+def solve[X, Y](using Cons[X, Y] =:= Cons[1, Cons[2, Y]]) = ()
+
+@main def main = solve // error: Type variable Y cannot be instantiated.