Fix #5970: suppress spurious warning in isInstanceOf check

compiler/src/dotty/tools/dotc/core/SymDenotations.scala

@@ -1575,6 +1575,17 @@ object SymDenotations {
           (symbol eq defn.NothingClass) ||
             (symbol eq defn.NullClass) && (base ne defn.NothingClass))
 
+    /** Is it possible that a class inherits both `this` and `that`?
+     *
+     *  @note The test is based on single-class inheritance and the closed
+     *        hierarchy of final classes.
+     *
+     *  @return The result may contain false positives, but never false negatives.
+     */
+    final def mayHaveCommonChild(that: ClassSymbol)(implicit ctx: Context): Boolean =
+      !this.is(Final) && !that.is(Final) && (this.is(Trait) || that.is(Trait)) ||
+        this.derivesFrom(that) || that.derivesFrom(this.symbol)
+
     final override def typeParamCreationFlags: FlagSet = ClassTypeParamCreationFlags
 
     /** Hook to do a pre-enter test. Overridden in PackageDenotation */

compiler/src/dotty/tools/dotc/transform/TypeTestsCasts.scala

@@ -132,6 +132,7 @@ object TypeTestsCasts {
             recur(tp1, P) && recur(tp2, P)
           case _ =>
             // first try withou striping type parameters for performance
+            X.classSymbol.exists && P.classSymbol.exists && !X.classSymbol.asClass.mayHaveCommonChild(P.classSymbol.asClass) ||
             isClassDetermined(X, tpe)(ctx.fresh.setNewTyperState()) ||
             isClassDetermined(stripTypeParam(X), tpe)(ctx.fresh.setNewTyperState())
         }

tests/pos-special/fatal-warnings/i5970.scala

@@ -0,0 +1,12 @@
+object Test extends App {
+  case class Foo[T](t: T)
+
+  def foo[T](ft: Unit|Foo[T]): T = {
+    ft match {
+      case Foo(t) => t
+      case () => ???
+    }
+  }
+
+  foo(Foo(23))
+}