Synthesize reflective proxies at link time.

Instead of relying on reflective proxies stored in sjsir files
by the compiler, we synthesize them at link time, on demand.
During deserialization of sjsir files, we throw away all the
reflective proxies. The Analyzer identifies the reflective
proxies needed to satisfy calls of reflective methods, and the
Linker synthesizes the bridges.

In this commit, the compiler is not modified and still emits
the reflective proxies, so that we can test the backward
binary compatibility hack.

Author: sjrd

ir/src/main/scala/org/scalajs/core/ir/InfoSerializers.scala

@@ -101,6 +101,9 @@ object InfoSerializers {
 
       import input._
 
+      val useHacks065 =
+        Set("0.6.0", "0.6.3", "0.6.4", "0.6.5").contains(version)
+
       val encodedName = readUTF()
       val isExported = readBoolean()
       val kind = ClassKind.fromByte(readByte())
@@ -126,7 +129,12 @@ object InfoSerializers {
             accessedClassData)
       }
 
-      val methods = readList(readMethod())
+      val methods0 = readList(readMethod())
+      val methods = if (true) { // useHacks065
+        methods0.filter(m => !Definitions.isReflProxyName(m.encodedName))
+      } else {
+        methods0
+      }
 
       val info = ClassInfo(encodedName, isExported, kind,
           superClass, interfaces, methods)

ir/src/main/scala/org/scalajs/core/ir/Serializers.scala

@@ -712,7 +712,17 @@ object Serializers {
           val superClass = readOptIdent()
           val parents = readIdents()
           val jsName = Some(readString()).filter(_ != "")
-          val defs = readTrees()
+          val defs0 = readTrees()
+          val defs = if (true) { // useHacks065
+            defs0.filter {
+              case MethodDef(_, Ident(name, _), _, _, _) =>
+                !Definitions.isReflProxyName(name)
+              case _ =>
+                true
+            }
+          } else {
+            defs0
+          }
           val optimizerHints = new OptimizerHints(readInt())
           ClassDef(name, kind, superClass, parents, jsName, defs)(optimizerHints)
 

test-suite/js/src/test/scala/org/scalajs/testsuite/compiler/ReflectiveCallTest.scala

@@ -221,6 +221,24 @@ object ReflectiveCallTest extends JasmineTest {
       expect(call(new C)).toEqual(1)
     }
 
+    it("should be bug-compatible with Scala/JVM for inherited overloads") {
+      class Base {
+        def foo(x: Option[Int]): String = "a"
+      }
+
+      class Sub extends Base {
+        def foo(x: Option[String]): Int = 1
+      }
+
+      val sub = new Sub
+
+      val x: { def foo(x: Option[Int]): Any } = sub
+      expect(x.foo(Some(1)).asInstanceOf[js.Any]).toEqual(1) // here is the "bug"
+
+      val y: { def foo(x: Option[String]): Any } = sub
+      expect(y.foo(Some("hello")).asInstanceOf[js.Any]).toEqual(1)
+    }
+
     it("should work on java.lang.Object.{ notify, notifyAll } - #303") {
       type ObjNotifyLike = Any {
         def notify(): Unit

tools/shared/src/main/scala/org/scalajs/core/tools/optimizer/Analysis.scala

@@ -75,6 +75,7 @@ object Analysis {
     final case object None extends MethodSyntheticKind
     // TODO Get rid of InheritedConstructor when we can break binary compat
     final case object InheritedConstructor extends MethodSyntheticKind
+    final case class ReflectiveProxy(target: String) extends MethodSyntheticKind
   }
 
   sealed trait Error {

tools/shared/src/main/scala/org/scalajs/core/tools/optimizer/Analyzer.scala

@@ -15,22 +15,24 @@ import scala.collection.mutable
 
 import org.scalajs.core.ir
 import ir.{ClassKind, Definitions, Infos}
-import Definitions.{isConstructorName, isReflProxyName}
+import Definitions._
 
 import org.scalajs.core.tools.sem._
 import org.scalajs.core.tools.javascript.{LongImpl, OutputMode}
 
 import ScalaJSOptimizer._
 
 final class Analyzer(semantics: Semantics, outputMode: OutputMode,
-    reachOptimizerSymbols: Boolean, initialLink: Boolean) extends Analysis {
+    reachOptimizerSymbols: Boolean,
+    allowAddingSyntheticMethods: Boolean) extends Analysis {
   import Analyzer._
   import Analysis._
 
-  @deprecated("Use the overload with an explicit `initialLink` flag", "0.6.6")
+  @deprecated("Use the overload with an explicit `allowAddingSyntheticMethods` flag", "0.6.6")
   def this(semantics: Semantics, outputMode: OutputMode,
       reachOptimizerSymbols: Boolean) = {
-    this(semantics, outputMode, reachOptimizerSymbols, initialLink = true)
+    this(semantics, outputMode, reachOptimizerSymbols,
+        allowAddingSyntheticMethods = true)
   }
 
   @deprecated("Use the overload with an explicit output mode", "0.6.3")
@@ -297,7 +299,7 @@ final class Analyzer(semantics: Semantics, outputMode: OutputMode,
        * during the initial link. In a refiner, this must not happen anymore.
        */
       methodInfos.get(ctorName).getOrElse {
-        if (!initialLink) {
+        if (!allowAddingSyntheticMethods) {
           createNonExistentMethod(ctorName)
         } else {
           val inherited = lookupMethod(ctorName)
@@ -350,6 +352,139 @@ final class Analyzer(semantics: Semantics, outputMode: OutputMode,
       loop(this)
     }
 
+    def tryLookupReflProxyMethod(proxyName: String): Option[MethodInfo] = {
+      if (!allowAddingSyntheticMethods) {
+        tryLookupMethod(proxyName)
+      } else {
+        /* The lookup for a target method in this code implements the
+         * algorithm defining `java.lang.Class.getMethod`. This mimics how
+         * reflective calls are implemented on the JVM, at link time.
+         *
+         * Caveat: protected methods are not ignored. This can only make an
+         * otherwise invalid reflective call suddenly able to call a protected
+         * method. It never breaks valid reflective calls. This could be fixed
+         * if the IR retained the information that a method is protected.
+         */
+
+        @tailrec
+        def loop(ancestorInfo: ClassInfo): Option[MethodInfo] = {
+          if (ancestorInfo ne null) {
+            ancestorInfo.methodInfos.get(proxyName) match {
+              case Some(m) =>
+                assert(m.isReflProxy && !m.isAbstract)
+                Some(m)
+
+              case _ =>
+                ancestorInfo.findProxyMatch(proxyName) match {
+                  case Some(target) =>
+                    val targetName = target.encodedName
+                    Some(ancestorInfo.createReflProxy(proxyName, targetName))
+
+                  case None =>
+                    loop(ancestorInfo.superClass)
+                }
+            }
+          } else {
+            None
+          }
+        }
+
+        loop(this)
+      }
+    }
+
+    private def findProxyMatch(proxyName: String): Option[MethodInfo] = {
+      val candidates = methodInfos.valuesIterator.filter { m =>
+        // TODO In theory we should filter out protected methods
+        !m.isReflProxy && !m.isExported && !m.isAbstract &&
+        reflProxyMatches(m.encodedName, proxyName)
+      }.toSeq
+
+      /* From the JavaDoc of java.lang.Class.getMethod:
+       *
+       *   If more than one [candidate] method is found in C, and one of these
+       *   methods has a return type that is more specific than any of the
+       *   others, that method is reflected; otherwise one of the methods is
+       *   chosen arbitrarily.
+       */
+
+      val targets = candidates.filterNot { c =>
+        val resultType = methodResultType(c.encodedName)
+        candidates.exists { other =>
+          (other ne c) &&
+          isMoreSpecific(methodResultType(other.encodedName), resultType)
+        }
+      }
+
+      /* This last step (chosen arbitrarily) causes some soundness issues of
+       * the implementation of reflective calls. This is bug-compatible with
+       * Scala/JVM.
+       */
+      targets.headOption
+    }
+
+    private def reflProxyMatches(methodName: String, proxyName: String): Boolean = {
+      val sepPos = methodName.lastIndexOf("__")
+      sepPos >= 0 && methodName.substring(0, sepPos + 2) == proxyName
+    }
+
+    private def methodResultType(methodName: String): ir.Types.ReferenceType = {
+      val typeName = methodName.substring(methodName.lastIndexOf("__") + 2)
+      val arrayDepth = typeName.indexWhere(_ != 'A')
+      if (arrayDepth == 0)
+        ir.Types.ClassType(typeName)
+      else
+        ir.Types.ArrayType(typeName.substring(arrayDepth), arrayDepth)
+    }
+
+    private def isMoreSpecific(left: ir.Types.ReferenceType,
+        right: ir.Types.ReferenceType): Boolean = {
+      import ir.Types._
+
+      def classIsMoreSpecific(leftCls: String, rightCls: String): Boolean = {
+        leftCls != rightCls && {
+          val leftInfo = _classInfos.get(leftCls)
+          val rightInfo = _classInfos.get(rightCls)
+          leftInfo.zip(rightInfo).exists { case (l, r) =>
+            l.ancestors.contains(r)
+          }
+        }
+      }
+
+      (left, right) match {
+        case (ClassType(leftCls), ClassType(rightCls)) =>
+          classIsMoreSpecific(leftCls, rightCls)
+        case (ArrayType(leftBase, leftDepth), ArrayType(rightBase, rightDepth)) =>
+          leftDepth == rightDepth && classIsMoreSpecific(leftBase, rightBase)
+        case (ArrayType(_, _), ClassType(ObjectClass)) =>
+          true
+        case _ =>
+          false
+      }
+    }
+
+    private def createReflProxy(proxyName: String,
+        targetName: String): MethodInfo = {
+      assert(this.isScalaClass,
+          s"Cannot create reflective proxy in non-Scala class $this")
+
+      val returnsChar = targetName.endsWith("__C")
+      val syntheticInfo = Infos.MethodInfo(
+          encodedName = proxyName,
+          methodsCalled = Map(
+              this.encodedName -> List(targetName)),
+          methodsCalledStatically = (
+              if (returnsChar) Map(BoxedCharacterClass -> List("init___C"))
+              else Map.empty),
+          instantiatedClasses = (
+              if (returnsChar) List(BoxedCharacterClass)
+              else Nil))
+      val m = new MethodInfo(this, syntheticInfo)
+      m.syntheticKind = MethodSyntheticKind.ReflectiveProxy(targetName)
+      methodInfos += proxyName -> m
+      m
+    }
+
     def lookupStaticMethod(methodName: String): MethodInfo = {
       tryLookupStaticMethod(methodName).getOrElse {
         val syntheticData = createMissingMethodInfo(methodName, isStatic = true)
@@ -473,7 +608,7 @@ final class Analyzer(semantics: Semantics, outputMode: OutputMode,
 
     private def delayedCallMethod(methodName: String)(implicit from: From): Unit = {
       if (isReflProxyName(methodName)) {
-        tryLookupMethod(methodName).foreach(_.reach(this))
+        tryLookupReflProxyMethod(methodName).foreach(_.reach(this))
       } else {
         lookupMethod(methodName).reach(this)
       }

tools/shared/src/main/scala/org/scalajs/core/tools/optimizer/Linker.scala

@@ -95,7 +95,7 @@ final class Linker(semantics: Semantics, outputMode: OutputMode,
 
     val analysis = logTime(logger, "Linker: Compute reachability") {
       val analyzer = new Analyzer(semantics, outputMode, reachOptimizerSymbols,
-          initialLink = true)
+          allowAddingSyntheticMethods = true)
       analyzer.computeReachability(infoInput)
     }
 
@@ -236,6 +236,11 @@ final class Linker(semantics: Semantics, outputMode: OutputMode,
           val syntheticMDef = synthesizeInheritedConstructor(
               analyzerInfo, m, getTree, analysis)(classDef.pos)
           memberMethods += linkedSyntheticMethod(syntheticMDef)
+
+        case MethodSyntheticKind.ReflectiveProxy(targetName) =>
+          val syntheticMDef = synthesizeReflectiveProxy(
+              analyzerInfo, m, targetName, getTree, analysis)
+          memberMethods += linkedSyntheticMethod(syntheticMDef)
       }
     }
 
@@ -276,23 +281,8 @@ final class Linker(semantics: Semantics, outputMode: OutputMode,
       implicit pos: Position): MethodDef = {
     val encodedName = methodInfo.encodedName
 
-    @tailrec
-    def findInheritedMethodDef(ancestorInfo: Analysis.ClassInfo): MethodDef = {
-      val inherited = ancestorInfo.methodInfos(methodInfo.encodedName)
-      if (inherited.syntheticKind == MethodSyntheticKind.None) {
-        val (classDef, _) = getTree(ancestorInfo.encodedName)
-        classDef.defs.collectFirst {
-          case mDef: MethodDef if mDef.name.name == encodedName => mDef
-        }.getOrElse {
-          throw new AssertionError(
-              s"Cannot find $encodedName in ${ancestorInfo.encodedName}")
-        }
-      } else {
-        findInheritedMethodDef(ancestorInfo.superClass)
-      }
-    }
-
-    val inheritedMDef = findInheritedMethodDef(classInfo.superClass)
+    val inheritedMDef = findInheritedMethodDef(classInfo.superClass,
+        encodedName, getTree, _.syntheticKind == MethodSyntheticKind.None)
 
     val origName = inheritedMDef.name.asInstanceOf[Ident].originalName
     val ctorIdent = Ident(encodedName, origName)
@@ -307,6 +297,64 @@ final class Linker(semantics: Semantics, outputMode: OutputMode,
         inheritedMDef.hash) // over-approximation
   }
 
+  private def synthesizeReflectiveProxy(
+      classInfo: Analysis.ClassInfo, methodInfo: Analysis.MethodInfo,
+      targetName: String, getTree: TreeProvider,
+      analysis: Analysis): MethodDef = {
+    val encodedName = methodInfo.encodedName
+
+    val targetMDef = findInheritedMethodDef(classInfo, targetName, getTree)
+
+    implicit val pos = targetMDef.pos
+
+    val targetIdent = targetMDef.name.asInstanceOf[Ident].copy() // for the new pos
+    val proxyIdent = Ident(encodedName, None)
+    val params = targetMDef.args.map(_.copy()) // for the new pos
+    val currentClassType = ClassType(classInfo.encodedName)
+
+    val call = Apply(This()(currentClassType),
+        targetIdent, params.map(_.ref))(targetMDef.resultType)
+
+    val body = if (targetName.endsWith("__C")) {
+      // A Char needs to be boxed
+      New(ClassType(Definitions.BoxedCharacterClass),
+          Ident("init___C"), List(call))
+    } else if (targetName.endsWith("__V")) {
+      // Materialize an `undefined` result for void methods
+      Block(call, Undefined())
+    } else {
+      call
+    }
+
+    MethodDef(static = false, proxyIdent, params, AnyType, body)(
+        OptimizerHints.empty, targetMDef.hash)
+  }
+
+  private def findInheritedMethodDef(classInfo: Analysis.ClassInfo,
+      methodName: String, getTree: TreeProvider,
+      p: Analysis.MethodInfo => Boolean = _ => true): MethodDef = {
+    @tailrec
+    def loop(ancestorInfo: Analysis.ClassInfo): MethodDef = {
+      assert(ancestorInfo != null,
+          s"Could not find $methodName anywhere in ${classInfo.encodedName}")
+
+      val inherited = ancestorInfo.methodInfos.get(methodName)
+      if (inherited.exists(p)) {
+        val (classDef, _) = getTree(ancestorInfo.encodedName)
+        classDef.defs.collectFirst {
+          case mDef: MethodDef if mDef.name.name == methodName => mDef
+        }.getOrElse {
+          throw new AssertionError(
+              s"Cannot find $methodName in ${ancestorInfo.encodedName}")
+        }
+      } else {
+        loop(ancestorInfo.superClass)
+      }
+    }
+
+    loop(classInfo)
+  }
+
   private def startRun(): Unit = {
     statsReused = 0
     statsInvalidated = 0

tools/shared/src/main/scala/org/scalajs/core/tools/optimizer/Refiner.scala

@@ -16,7 +16,7 @@ final class Refiner(semantics: Semantics, outputMode: OutputMode) {
   def refine(unit: LinkingUnit, logger: Logger): LinkingUnit = {
     val analysis = logTime(logger, "Refiner: Compute reachability") {
       val analyzer = new Analyzer(semantics, outputMode,
-          reachOptimizerSymbols = false, initialLink = false)
+          reachOptimizerSymbols = false, allowAddingSyntheticMethods = false)
       analyzer.computeReachability(unit.infos.values.toList)
     }