Fix i11545

compiler/src/dotty/tools/dotc/core/TypeComparer.scala

@@ -736,14 +736,21 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
         fourthTry
     }
 
-    def tryBaseType(cls2: Symbol) = {
-      val base = nonExprBaseType(tp1, cls2)
-      if (base.exists && (base `ne` tp1))
-        isSubType(base, tp2, if (tp1.isRef(cls2)) approx else approx.addLow) ||
-        base.isInstanceOf[OrType] && fourthTry
-          // if base is a disjunction, this might have come from a tp1 type that
-          // expands to a match type. In this case, we should try to reduce the type
-          // and compare the redux. This is done in fourthTry
+    def tryBaseType(cls2: Symbol): Boolean = {
+      // nonExprBaseType approximates its result in ways that are unsound for GADTs
+      // when it handles intersections. See neg/i11545a.scala for a specific example.
+      // Note that we can't simply freeze GADTs at this point. PatternTypeConstrainer
+      // passes a Skolem here and we _need_ to calculate its base type. Only intersection
+      // types are a problem.
+      if !(ctx.mode.is(Mode.GadtConstraintInference) && tp1.isInstanceOf[AndType]) then
+        val base = nonExprBaseType(tp1, cls2)
+        if (base.exists && (base `ne` tp1))
+          isSubType(base, tp2, if (tp1.isRef(cls2)) approx else approx.addLow) ||
+          base.isInstanceOf[OrType] && fourthTry
+            // if base is a disjunction, this might have come from a tp1 type that
+            // expands to a match type. In this case, we should try to reduce the type
+            // and compare the redux. This is done in fourthTry
+        else fourthTry
       else fourthTry
     }
 

tests/neg/i11545.scala

@@ -0,0 +1,13 @@
+@main def test: Unit = {
+  trait S[A]
+  trait Inv[A]
+
+  class P[X] extends S[Inv[X] & Inv[String]]
+
+  def patmat[A, Y](s: S[Inv[A] & Y]): A = s match {
+    case p: P[x] =>
+      "Hello" // error
+  }
+
+  val got: Int = patmat[Int, Inv[String]](new P) // ClassCastException: String cannot be cast to Integer
+}

tests/neg/i11545a.scala

@@ -0,0 +1,20 @@
+@main def test: Unit = {
+  trait Inv[A]
+
+  trait S[+A]
+  final class P[+X] extends S[Inv[String] & X]
+
+
+  def patmat[A](s: S[Inv[A]]): A = s match {
+    // When inferring the GADT cstr here, we need to infer cstr following from:
+    //   Inv[String] & a <: Inv[A]
+    // We end up invoking nonExprBaseType(`Inv[String] & a`, `Inv`),
+    // which returns just `Inv[String]`. After that we approximate with:
+    //   Inv[String] <: Inv[A]
+    // Which is simply wrong.
+    case p: P[a] => "Hello" // error
+  }
+
+  val i: Int = patmat[Int](P[Inv[Int]]())
+  i
+}