Handle singleton type unions when comparing types

Author: abgruszecki

compiler/src/dotty/tools/dotc/core/TypeComparer.scala

@@ -545,11 +545,18 @@ class TypeComparer(initctx: Context) extends ConstraintHandling[AbsentContext] {
         }
         compareTypeLambda
       case OrType(tp21, tp22) =>
-        val tp1a = tp1.widenDealiasKeepRefiningAnnots
-        if (tp1a ne tp1)
-          // Follow the alias; this might avoid truncating the search space in the either below
-          // Note that it's safe to widen here because singleton types cannot be part of `|`.
-          return recur(tp1a, tp2)
+        def isSingletonlessTypeUnion(tp: Type): Boolean = tp match {
+          case OrType(a, b) => isSingletonlessTypeUnion(a) && isSingletonlessTypeUnion(b)
+          case _ => !tp.isSingleton
+        }
+        if (isSingletonlessTypeUnion(tp21) && isSingletonlessTypeUnion(tp22)) {
+          val tp1a = tp1.widenDealiasKeepRefiningAnnots
+          if (tp1a ne tp1)
+            // Follow the alias; this might avoid truncating the search space in the either below
+            // Note that it's safe to widen here because we ensured that
+            // singleton types are not top-level members of tp2
+            return recur(tp1a, tp2)
+        }
 
         // Rewrite T1 <: (T211 & T212) | T22 to T1 <: (T211 | T22) and T1 <: (T212 | T22)
         // and analogously for T1 <: T21 | (T221 & T222)

tests/neg/gadt-union-subtyping.scala

@@ -0,0 +1,34 @@
+object `gadt-union-subtyping` {
+  enum SUB[A, +B] { case Refl[T]() extends SUB[T, T] }
+
+  def foo[T](t: T, sub1: 5 SUB T, sub2: 6 SUB T): T = {
+    (sub1, sub2) match {
+      case (SUB.Refl(), SUB.Refl()) =>
+        val a: 5 = t // error
+        val _t: T = (5 : 5)
+        ???
+    }
+  }
+
+  def bar[T](t: T, sub1: 5 SUB T, sub2: 6 SUB T, sub3: String SUB T): T = {
+    (sub1, sub2, sub3) match {
+      case (SUB.Refl(), SUB.Refl(), SUB.Refl()) =>
+        val a: 5 = t // error
+        val b: 6 = t // error
+        val c: String = t // error
+        val _t: T = (5 : 5)
+        ???
+    }
+  }
+
+  def baz[T](t: T, sub1: String SUB T, sub2: 5 SUB T, sub3: 6 SUB T): T = {
+    (sub1, sub2, sub3) match {
+      case (SUB.Refl(), SUB.Refl(), SUB.Refl()) =>
+        val a: 5 = t // error
+        val b: 6 = t // error
+        val c: String = t // error
+        val _t: T = (5 : 5)
+        ???
+    }
+  }
+}