Local classes are uncheckable (type tests)

Author: dwijnand

compiler/src/dotty/tools/dotc/transform/TypeTestsCasts.scala

@@ -152,6 +152,20 @@ object TypeTestsCasts {
       case AnnotatedType(t, _)  => recur(X, t)
       case tp2: RefinedType     => recur(X, tp2.parent) && TypeComparer.hasMatchingMember(tp2.refinedName, X, tp2)
       case tp2: RecType         => recur(X, tp2.parent)
+      case tp2
+      if tp2.typeSymbol.isLocal =>
+        val sym = tp2.typeSymbol
+        val methodSymbol = sym.owner
+        val tpSyms = typer.ErrorReporting.substitutableTypeSymbolsInScope(sym).toSet
+        def isAccessible(sym: Symbol): Boolean = sym == methodSymbol || sym.isType && isAccessible(sym.owner)
+        def hasPoison(tp: Type): Boolean =
+          tp.baseClasses.filter(isAccessible).exists { sym =>
+            sym.info.decls.exists { sym =>
+              sym.info.existsPart(tp => tpSyms.contains(tp.typeSymbol))
+                || isAccessible(sym.info.typeSymbol.maybeOwner) && hasPoison(sym.info)
+            }
+          }
+        !hasPoison(tp2)
       case _                    => true
     })
 

tests/neg/i4812.check

@@ -0,0 +1,20 @@
+-- Error: tests/neg/i4812.scala:8:11 -----------------------------------------------------------------------------------
+8 |      case prev: A => // error: the type test for A cannot be checked at runtime
+  |           ^
+  |           the type test for A cannot be checked at runtime
+-- Error: tests/neg/i4812.scala:18:11 ----------------------------------------------------------------------------------
+18 |      case prev: A => // error: the type test for A cannot be checked at runtime
+   |           ^
+   |           the type test for A cannot be checked at runtime
+-- Error: tests/neg/i4812.scala:28:11 ----------------------------------------------------------------------------------
+28 |      case prev: A => // error: the type test for A cannot be checked at runtime
+   |           ^
+   |           the type test for A cannot be checked at runtime
+-- Error: tests/neg/i4812.scala:38:11 ----------------------------------------------------------------------------------
+38 |      case prev: A => // error: the type test for A cannot be checked at runtime
+   |           ^
+   |           the type test for A cannot be checked at runtime
+-- Error: tests/neg/i4812.scala:50:13 ----------------------------------------------------------------------------------
+50 |        case prev: A => // error: the type test for A cannot be checked at runtime
+   |             ^
+   |             the type test for A cannot be checked at runtime

tests/neg/i4812.scala

@@ -0,0 +1,58 @@
+// scalac: -Werror
+object Test:
+  var prev: Any = _
+
+  def test[T](x: T): T =
+    class A(val elem: (T, Boolean))
+    prev match
+      case prev: A => // error: the type test for A cannot be checked at runtime
+        prev.elem._1
+      case _ =>
+        prev = new A((x, true))
+        x
+
+  def test2[T](x: T): T =
+    abstract class Parent(_elem: T) { def elem: T = _elem }
+    class A extends Parent(x)
+    prev match
+      case prev: A => // error: the type test for A cannot be checked at runtime
+        prev.elem
+      case _ =>
+        prev = new A
+        x
+
+  def test3[T](x: T): T =
+    class Holder(val elem: T)
+    class A(val holder: Holder)
+    prev match
+      case prev: A => // error: the type test for A cannot be checked at runtime
+        prev.holder.elem
+      case _ =>
+        prev = new A(new Holder(x))
+        x
+
+  def test4[T](x: T): T =
+    class Holder(val elem: (Int, (Unit, (T, Boolean))))
+    class A { var holder: Holder = null }
+    prev match
+      case prev: A => // error: the type test for A cannot be checked at runtime
+        prev.holder.elem._2._2._1
+      case _ =>
+        val a = new A
+        a.holder = new Holder((42, ((), (x, true))))
+        prev = a
+        x
+
+  class Foo[U]:
+    def test5(x: U): U =
+      class A(val elem: U)
+      prev match
+        case prev: A => // error: the type test for A cannot be checked at runtime
+          prev.elem
+        case _ =>
+          prev = new A(x)
+          x
+
+  def main(args: Array[String]): Unit =
+    test(1)
+    val x: String = test("") // was: ClassCastException: java.lang.Integer cannot be cast to java.lang.String