infer: Compare variances in tvar instantiate check

Author: dwijnand

compiler/src/dotty/tools/dotc/typer/Inferencing.scala

@@ -409,15 +409,17 @@ object Inferencing {
     Stats.record("maximizeType")
     val vs = variances(tp)
     val patternBindings = new mutable.ListBuffer[(Symbol, TypeParamRef)]
-    val gadtBounds = ctx.gadt.symbols.map(ctx.gadt.bounds(_).nn)
+    val gadtVariances = ctx.gadt.symbols.map(sym => variances(ctx.gadt.bounds(sym).nn))
     vs foreachBinding { (tvar, v) =>
       if !tvar.isInstantiated then
         // if the tvar is covariant/contravariant (v == 1/-1, respectively) in the input type tp
-        // then check the tvar doesn't occur in the opposite GADT bound (lower/upper) within any of the GADT bounds
-        // if it doesn't occur then it's safe to instantiate the tvar
-        // Eg neg/i14983 the C in Node[+C] is in the GADT lower bound X >: List[C] so maximising to Node[Any] is unsound
-        // Eg pos/precise-pattern-type the T in Tree[-T] is in no GADT upper bound so can maximise to Tree[Type]
-        val safeToInstantiate = v != 0 && gadtBounds.forall(tb => !tvar.occursIn(if v == 1 then tb.lo else tb.hi))
+        // then it is safe to instantiate if it has the same variance within the GADT bounds.
+        // Eg neg/i14983 the C in Node[+C] differs in variance to the GADT bound X >: List[C] so maximising to Node[Any] is unsound
+        // Eg pos/precise-pattern-type the T in Tree[-T] is in no GADT bound so can maximise to Tree[Type]
+        val safeToInstantiate = v != 0 && gadtVariances.forall { vmap =>
+          val v2 = vmap(tvar)
+          v2 == null || v2 == v
+        }
         if safeToInstantiate then tvar.instantiate(fromBelow = v == -1)
         else {
           val bounds = TypeComparer.fullBounds(tvar.origin)

tests/neg/i14983.co-contra.scala

@@ -0,0 +1,23 @@
+case class Showing[-C](show: C => String)
+
+sealed trait Tree[+A]
+final case class Leaf[+B](b: B)       extends Tree[B]
+final case class Node[-C](l: Showing[C]) extends Tree[Showing[C]]
+
+object Test:
+  def meth[X](tree: Tree[X]): X = tree match
+    case Leaf(v) => v
+    case Node(x) =>
+      // tree: Tree[X] vs Node[C] aka Tree[Showing[C]]
+      // PTC: X >: Showing[C]
+      // max: Node[C] to Node[Nothing], instantiating C := Nothing, which makes X >: Showing[Nothing]
+      // adapt: Showing[String] <: X = OKwithGADTUsed; insert GADT cast asInstanceOf[X]
+      Showing[String](_ + " boom!") // error: Found: Showing[String] Required: X where: X is a type in method meth with bounds >: Showing[C$1]
+      // after fix:
+      // max: Node[C] => Node[C$1], instantiating C := C$1, a new symbol, so X >: Showing[C$1]
+      // adapt: Showing[String] <: X = Fail, because String !<: C$1
+
+  def main(args: Array[String]): Unit =
+    val tree = Node(Showing[Int](_.toString))
+    val res = meth(tree)
+    println(res.show(42)) // was: ClassCastException: class java.lang.Integer cannot be cast to class java.lang.String