Fix #5826: specialize union types to avoid constraint cut

compiler/src/dotty/tools/dotc/transform/TypeTestsCasts.scala

@@ -96,17 +96,17 @@ object TypeTestsCasts {
       val tvars = constrained(typeLambda, untpd.EmptyTree, alwaysAddTypeVars = true)._2.map(_.tpe)
       val P1 = tycon.appliedTo(tvars)
 
-      debug.println("P : " + P)
-      debug.println("P1 : " + P1)
-      debug.println("X : " + X)
+      debug.println("P : " + P.show)
+      debug.println("P1 : " + P1.show)
+      debug.println("X : " + X.show)
 
       P1 <:< X       // constraint P1
 
       // use fromScala2x to avoid generating pattern bound symbols
       maximizeType(P1, span, fromScala2x = true)
 
       val res = P1 <:< P
-      debug.println("P1 : " + P1)
+      debug.println("P1 : " + P1.show)
       debug.println("P1 <:< P = " + res)
 
       res
@@ -122,9 +122,19 @@ object TypeTestsCasts {
           case _                   => recur(defn.AnyType, tpT)
         }
       case tpe: AppliedType     =>
-        // first try withou striping type parameters for performance
-        isClassDetermined(X, tpe)(ctx.fresh.setNewTyperState()) ||
-        isClassDetermined(stripTypeParam(X), tpe)(ctx.fresh.setNewTyperState())
+        X.widen match {
+          case OrType(tp1, tp2) =>
+            // This case is required to retrofit type inference,
+            // which cut constraints in the following two cases:
+            //   - T1 <:< T2 | T3
+            //   - T1 & T2 <:< T3
+            // See TypeComparer#either
+            recur(tp1, P) && recur(tp2, P)
+          case _ =>
+            // first try withou striping type parameters for performance
+            isClassDetermined(X, tpe)(ctx.fresh.setNewTyperState()) ||
+            isClassDetermined(stripTypeParam(X), tpe)(ctx.fresh.setNewTyperState())
+        }
       case AndType(tp1, tp2)    => recur(X, tp1) && recur(X, tp2)
       case OrType(tp1, tp2)     => recur(X, tp1) && recur(X, tp2)
       case AnnotatedType(t, _)  => recur(X, t)

tests/neg-custom-args/isInstanceOf/3324f.scala

@@ -3,6 +3,7 @@ class D[T]
 
 class Test {
   def foo[T](x: C[T]) = x match {
-    case _: D[T] =>  // error
+    case _: D[T] =>    // error
+    case _: C[Int] =>  // error
   }
 }

tests/neg-custom-args/isInstanceOf/i5826.scala

@@ -0,0 +1,20 @@
+class Foo {
+  def test[A]: List[Int] | A => Int = {
+    case ls: List[Int] => ls.head       // error
+    case _ => 0
+  }
+
+  def test2: List[Int] | List[String] => Int = {
+    case ls: List[Int] => ls.head       // error
+    case _ => 0
+  }
+
+  trait A[T]
+  trait B[T]
+
+  // suppose: class C extends A[Int] with B[String]
+  def test3[X]: A[X] | B[X] => Int = {
+    case ls: A[X] => 4                 // error
+    case _ => 0
+  }
+}

tests/neg-custom-args/isInstanceOf/i5826b.scala

@@ -0,0 +1,11 @@
+class Foo {
+  def test1[A]: List[Int] | A => Int = {
+    case ls: List[_] => ls.head       // error
+    case _ => 0
+  }
+
+  def test2[A]: List[Int] | A => Int = {
+    case ls: List[_] => ls.size
+    case _ => 0
+  }
+}