Fix #4297: handle type param reference

Author: liufengyun

compiler/src/dotty/tools/dotc/transform/IsInstanceOfChecker.scala

@@ -88,19 +88,35 @@ object Checkable {
       }
     }
 
+    def maximize(implicit ctx: Context) = new ApproximatingTypeMap {
+      def apply(tp: Type): Type = tp match {
+        case tp: TypeRef if tp.underlying.isInstanceOf[TypeBounds] =>
+          val lo = this(tp.info.loBound)
+          val hi = this(tp.info.hiBound)
+          if (variance == 0) range(lo, hi)
+          else if (variance == 1) hi
+          else lo
+        case _ =>
+          mapOver(tp)
+      }
+    }
+
     def isClassDetermined(X: Type, P: AppliedType)(implicit ctx: Context) = {
       val AppliedType(tycon, _) = P
       val typeLambda = tycon.ensureLambdaSub.asInstanceOf[TypeLambda]
       val tvars = constrained(typeLambda, untpd.EmptyTree, alwaysAddTypeVars = true)._2.map(_.tpe)
       val P1 = tycon.appliedTo(tvars)
 
-      debug.println("P : " + P.show)
-      debug.println("P1 : " + P1.show)
-      debug.println("X : " + X.show)
+      debug.println("P : " + P)
+      debug.println("P1 : " + P1)
+      debug.println("X : " + X)
+
+      P1 <:< X       // constraint P1
 
-      P1 <:< X  // may fail, ignore
+      // use fromScala2x to avoid generating pattern bound symbols
+      maximizeType(P1, pos, fromScala2x = true)
 
-      val res = isFullyDefined(P1, ForceDegree.noBottom) &&  P1 <:< P
+      val res = P1 <:< P
       debug.println("P1 : " + P1)
       debug.println("P1 <:< P = " + res)
 
@@ -116,7 +132,9 @@ object Checkable {
           case defn.ArrayOf(tpE)   => recur(tpE, tpT)
           case _                   => recur(defn.AnyType, tpT)
         }
-      case tpe: AppliedType     => isClassDetermined(X, tpe)(ctx.fresh.setNewTyperState())
+      case tpe: AppliedType     =>
+        isClassDetermined(X, tpe)(ctx.fresh.setNewTyperState()) ||
+        isClassDetermined(maximize.apply(X), tpe)(ctx.fresh.setNewTyperState())
       case AndType(tp1, tp2)    => recur(X, tp1) && recur(X, tp2)
       case OrType(tp1, tp2)     => recur(X, tp1) && recur(X, tp2)
       case AnnotatedType(t, _)  => recur(X, t)

tests/neg-custom-args/isInstanceOf/i4297.scala

@@ -0,0 +1,11 @@
+class Test {
+  def test[X <: Option[Int]](x: X) = x.isInstanceOf[Some[Int]]
+  def test1[Y <: Int, X <: Option[Y]](x: X) = x.isInstanceOf[Some[Int]]
+  def test2(x: Any) = x.isInstanceOf[Function1[Nothing, _]]
+  def test3a(x: Any) = x.isInstanceOf[Function1[Any, _]] // error
+  def test3b(x: Any) = x.isInstanceOf[Function1[Int, _]] // error
+  def test4[Y <: Int, X <: Function1[Y, Unit]](x: X) = x.isInstanceOf[Function1[Int, _]] // error
+  def test5[Y <: Int, X <: Function1[Y, Unit]](x: X) = x.isInstanceOf[Function1[Int, Unit]]  // error
+  def test6[Y <: Int, X <: Function1[Y, Unit]](x: X) = x.isInstanceOf[Function1[Int, Any]] // error
+  def test7[Y <: Int, X <: Function1[Y, Unit]](x: X) = x.isInstanceOf[Function1[_, Unit]]
+}