Fix scala-js#3661: Load linker reflectively

This gives us the following advantages:

- Allows us to give resources to the linker that the build itself
  compiles first (unblocks scala-js#3537).
- Hides linker dependencies away from other sbt plugins. Notably the
  Google Closure Compiler and Guava (fixes scala-js#3193).

Author: gzm0

Jenkinsfile

@@ -298,7 +298,7 @@ def Tasks = [
   "bootstrap": '''
     setJavaVersion $java
     npm install &&
-    sbt ++$scala linker/test &&
+    sbt ++$scala! linker/test &&
     sbt ++$scala irJS/test linkerJS/test &&
     sbt 'set scalaJSStage in Global := FullOptStage' \
         'set scalaJSStage in testSuite := FastOptStage' \
@@ -315,14 +315,14 @@ def Tasks = [
   "tools": '''
     setJavaVersion $java
     npm install &&
-    sbt ++$scala ir/test logging/compile linkerInterface/compile \
+    sbt ++$scala! ir/test logging/compile linkerInterface/compile \
         linker/compile jsEnvs/test nodeJSEnv/test testAdapter/test \
         ir/mimaReportBinaryIssues logging/mimaReportBinaryIssues \
         linkerInterface/mimaReportBinaryIssues linker/mimaReportBinaryIssues \
         jsEnvs/mimaReportBinaryIssues jsEnvsTestKit/mimaReportBinaryIssues \
         nodeJSEnv/mimaReportBinaryIssues \
         testAdapter/mimaReportBinaryIssues &&
-    sbt ++$scala ir/compile:doc logging/compile:doc \
+    sbt ++$scala! ir/compile:doc logging/compile:doc \
         linkerInterface/compile:doc \
         linker/compile:doc jsEnvs/compile:doc \
         jsEnvsTestKit/compile:doc nodeJSEnv/compile:doc \
@@ -332,7 +332,7 @@ def Tasks = [
   "tools-sbtplugin": '''
     setJavaVersion $java
     npm install &&
-    sbt ++$scala ir/test logging/compile linkerInterface/compile \
+    sbt ++$scala! ir/test logging/compile linkerInterface/compile \
         linker/compile jsEnvs/test nodeJSEnv/test testAdapter/test \
         sbtPlugin/package \
         ir/mimaReportBinaryIssues logging/mimaReportBinaryIssues \
@@ -341,7 +341,7 @@ def Tasks = [
         nodeJSEnv/mimaReportBinaryIssues \
         testAdapter/mimaReportBinaryIssues \
         sbtPlugin/mimaReportBinaryIssues &&
-    sbt ++$scala library/scalastyle javalanglib/scalastyle javalib/scalastyle \
+    sbt ++$scala! library/scalastyle javalanglib/scalastyle javalib/scalastyle \
         ir/scalastyle compiler/scalastyle \
         compiler/test:scalastyle \
         logging/scalastyle logging/test:scalastyle \
@@ -359,7 +359,7 @@ def Tasks = [
         jUnitPlugin/scalastyle jUnitRuntime/scalastyle \
         jUnitTestOutputsJVM/scalastyle jUnitTestOutputsJVM/test:scalastyle \
         jUnitTestOutputsJS/scalastyle jUnitTestOutputsJS/test:scalastyle &&
-    sbt ++$scala ir/compile:doc logging/compile:doc \
+    sbt ++$scala! ir/compile:doc logging/compile:doc \
         linkerInterface/compile:doc \
         linker/compile:doc jsEnvs/compile:doc \
         jsEnvsTestKit/compile:doc nodeJSEnv/compile:doc \
@@ -382,8 +382,7 @@ def Tasks = [
     sbt ++2.11.12 compiler/publishLocal library/publishLocal \
                   testInterface/publishLocal testBridge/publishLocal \
                   jUnitPlugin/publishLocal jUnitRuntime/publishLocal &&
-    sbt ++$toolsscala \
-        ir/publishLocal logging/publishLocal \
+    sbt ir/publishLocal logging/publishLocal \
         linkerInterface/publishLocal \
         linker/publishLocal jsEnvs/publishLocal \
         nodeJSEnv/publishLocal testAdapter/publishLocal \
@@ -403,19 +402,19 @@ def Tasks = [
   "partest-noopt": '''
     setJavaVersion $java
     npm install &&
-    sbt ++$scala package "partestSuite/testOnly -- --showDiff"
+    sbt ++$scala! package "partestSuite/testOnly -- --showDiff"
   ''',
 
   "partest-fastopt": '''
     setJavaVersion $java
     npm install &&
-    sbt ++$scala package "partestSuite/testOnly -- --fastOpt --showDiff"
+    sbt ++$scala! package "partestSuite/testOnly -- --fastOpt --showDiff"
   ''',
 
   "partest-fullopt": '''
     setJavaVersion $java
     npm install &&
-    sbt ++$scala package "partestSuite/testOnly -- --fullOpt --showDiff"
+    sbt ++$scala! package "partestSuite/testOnly -- --fullOpt --showDiff"
   '''
 ]
 
@@ -456,7 +455,7 @@ allJavaVersions.each { javaVersion ->
   quickMatrix.add([task: "tools", scala: "2.11.12", java: javaVersion])
 }
 quickMatrix.add([task: "partestc", scala: "2.12.1", java: mainJavaVersion])
-quickMatrix.add([task: "sbtplugin-test", toolsscala: "2.12.8", java: mainJavaVersion])
+quickMatrix.add([task: "sbtplugin-test", java: mainJavaVersion])
 
 // The 'full' matrix
 def fullMatrix = quickMatrix.clone()

project/Build.scala

@@ -105,6 +105,13 @@ object MyScalaJSPlugin extends AutoPlugin {
       jsEnv := new NodeJSEnv(
           NodeJSEnv.Config().withSourceMap(wantSourceMaps.value)),
 
+      scalaJSLinkerImpl := {
+        val v = (scalaVersion in Build.linker).value
+        assert(v.startsWith("2.12."), "The linker is configured with a Scala " +
+            s"version ($v) that is not compatible with this build (2.12.x). ")
+        LinkerImpl.default(Attributed.data((fullClasspath in (Build.linker, Runtime)).value))
+      },
+
       // Link source maps to GitHub sources
       addScalaJSCompilerOption(Def.setting {
         "mapSourceURI:" +
@@ -413,6 +420,10 @@ object Build {
       scalacOptions += "-Xfatal-warnings"
   )
 
+  val freezeVersionForToolsSettings = Def.settings(
+      crossScalaVersions := Seq("2.12.8")
+  )
+
   private def publishToBintraySettings = Def.settings(
       publishTo := {
         val proj = bintrayProjectName.value
@@ -555,6 +566,7 @@ object Build {
 
   lazy val irProject: Project = Project(id = "ir", base = file("ir")).settings(
       commonIrProjectSettings,
+      freezeVersionForToolsSettings,
       libraryDependencies +=
         "com.novocode" % "junit-interface" % "0.9" % "test"
   )
@@ -639,7 +651,8 @@ object Build {
   )
 
   lazy val logging: Project = (project in file("logging/jvm")).settings(
-      commonLoggingSettings
+      commonLoggingSettings,
+      freezeVersionForToolsSettings,
   )
 
   lazy val loggingJS: Project = (project in file("logging/js")).enablePlugins(
@@ -671,6 +684,7 @@ object Build {
 
   lazy val linkerInterface: Project = (project in file("linker-interface/jvm")).settings(
       commonLinkerInterfaceSettings,
+      freezeVersionForToolsSettings,
   ).dependsOn(irProject, logging)
 
   lazy val linkerInterfaceJS: Project = (project in file("linker-interface/js")).settings(
@@ -712,6 +726,7 @@ object Build {
 
   lazy val linker: Project = (project in file("linker/jvm")).settings(
       commonLinkerSettings,
+      freezeVersionForToolsSettings,
       libraryDependencies ++= Seq(
           "com.google.javascript" % "closure-compiler" % "v20190513",
           "com.novocode" % "junit-interface" % "0.9" % "test"
@@ -733,6 +748,7 @@ object Build {
 
   lazy val jsEnvs: Project = (project in file("js-envs")).settings(
       commonSettings,
+      freezeVersionForToolsSettings,
       publishSettings,
       fatalWarningsSettings,
       name := "Scala.js JS Envs",
@@ -743,6 +759,7 @@ object Build {
 
   lazy val jsEnvsTestKit: Project = (project in file("js-envs-test-kit")).settings(
       commonSettings,
+      freezeVersionForToolsSettings,
       publishSettings,
       fatalWarningsSettings,
       name := "Scala.js JS Envs Test Kit",
@@ -757,6 +774,7 @@ object Build {
 
   lazy val nodeJSEnv: Project = (project in file("nodejs-env")).settings(
       commonSettings,
+      freezeVersionForToolsSettings,
       publishSettings,
       fatalWarningsSettings,
       name := "Scala.js Node.js env",
@@ -771,6 +789,7 @@ object Build {
 
   lazy val testAdapter = (project in file("test-adapter")).settings(
       commonSettings,
+      freezeVersionForToolsSettings,
       publishSettings,
       fatalWarningsSettings,
       name := "Scala.js sbt test adapter",
@@ -848,7 +867,7 @@ object Build {
 
         sbtJars.map(_.data -> docUrl).toMap
       }
-  ).dependsOn(linker, jsEnvs, nodeJSEnv, testAdapter)
+  ).dependsOn(linkerInterface, jsEnvs, nodeJSEnv, testAdapter)
 
   lazy val delambdafySetting = {
     scalacOptions ++= (
@@ -1307,6 +1326,7 @@ object Build {
 
   lazy val jUnitTestOutputsJVM = (project in file("junit-test/output-jvm")).settings(
       commonJUnitTestOutputsSettings,
+      freezeVersionForToolsSettings,
       name := "Tests for Scala.js JUnit output in JVM.",
       libraryDependencies ++= Seq(
           "org.scala-sbt" % "test-interface" % "1.0" % "test",
@@ -1336,6 +1356,7 @@ object Build {
 
   lazy val jUnitAsyncJVM = (project in file("junit-async/jvm")).settings(
       commonSettings,
+      freezeVersionForToolsSettings,
       name := "Scala.js internal JUnit async JVM support",
       publishArtifact in Compile := false
   )

project/build.sbt

@@ -6,8 +6,6 @@ addSbtPlugin("org.scalastyle" % "scalastyle-sbt-plugin" % "1.0.0")
 
 addSbtPlugin("org.portable-scala" % "sbt-platform-deps" % "1.0.0")
 
-libraryDependencies += "com.google.javascript" % "closure-compiler" % "v20190513"
-
 libraryDependencies += "com.google.jimfs" % "jimfs" % "1.1"
 
 libraryDependencies += "org.eclipse.jgit" % "org.eclipse.jgit.pgm" % "3.2.0.201312181205-r"
@@ -20,8 +18,6 @@ unmanagedSourceDirectories in Compile ++= {
     root / "logging/jvm/src/main/scala",
     root / "linker-interface/shared/src/main/scala",
     root / "linker-interface/jvm/src/main/scala",
-    root / "linker/shared/src/main/scala",
-    root / "linker/jvm/src/main/scala",
     root / "js-envs/src/main/scala",
     root / "nodejs-env/src/main/scala",
     root / "test-adapter/src/main/scala",

sbt-plugin/src/main/scala/org/scalajs/sbtplugin/CacheBox.scala

@@ -0,0 +1,35 @@
+/*
+ * Scala.js (https://www.scala-js.org/)
+ *
+ * Copyright EPFL.
+ *
+ * Licensed under Apache License 2.0
+ * (https://www.apache.org/licenses/LICENSE-2.0).
+ *
+ * See the NOTICE file distributed with this work for
+ * additional information regarding copyright ownership.
+ */
+
+package org.scalajs.sbtplugin
+
+/** A CacheBox is a support class to abuse an sbt setting as a cache.
+ *
+ *  A CacheBox is needed, once the cached result needs to depend on a task,
+ *  since then it cannot simply be made a setting anymore.
+ */
+private[sbtplugin] final class CacheBox[T] {
+  private[this] var value: T = _
+
+  def ensure(f: => T): T = synchronized {
+    if (value == null) {
+      value = f
+    }
+    value
+  }
+
+  def foreach(f: T => Unit): Unit = synchronized {
+    if (value != null) {
+      f(value)
+    }
+  }
+}

sbt-plugin/src/main/scala/org/scalajs/sbtplugin/LinkerImpl.scala

@@ -0,0 +1,133 @@
+/*
+ * Scala.js (https://www.scala-js.org/)
+ *
+ * Copyright EPFL.
+ *
+ * Licensed under Apache License 2.0
+ * (https://www.apache.org/licenses/LICENSE-2.0).
+ *
+ * See the NOTICE file distributed with this work for
+ * additional information regarding copyright ownership.
+ */
+
+package org.scalajs.sbtplugin
+
+import scala.concurrent._
+
+import java.lang.reflect.{Method, Modifier}
+import java.io.File
+import java.net.URLClassLoader
+import java.nio.file.Path
+
+import org.scalajs.linker.interface._
+
+/** Abstract implementation of a linker as needed by the sbt plugin.
+ *
+ *  @note This trait does not guarantee full compatibility: Methods may be added
+ *      / removed in the future. Use [[LinkerImpl.Forwarding]] to override things
+ *      selectively in a compatible manner.
+ */
+trait LinkerImpl {
+  def clearableLinker(cfg: StandardConfig): ClearableLinker
+
+  def irFileCache(): IRFileCache
+
+  def irContainers(classpath: Seq[Path])(
+      implicit ec: ExecutionContext): Future[(Seq[IRContainer], Seq[Path])]
+
+  def outputFile(path: Path): LinkerOutput.File
+}
+
+object LinkerImpl {
+  /** Returns the default implementation.
+   *
+   *  It loads a StandardLinker via reflection.
+   */
+  def default(files: Seq[File]): LinkerImpl = {
+    val urls = files.map(_.toURI.toURL).toArray
+    val loader = new URLClassLoader(urls, new FilteringClassLoader(getClass.getClassLoader))
+    new Reflect(loader)
+  }
+
+  /** A [[LinkerImpl]] that forwards everything to `parent`.
+   *
+   *  This is useful if only parts of the linker implementation need to be
+   *  replaced. A subclass only overriding these can be created, ensuring easier
+   *  transition when methods get added.
+   */
+  class Forwarding(parent: LinkerImpl) extends LinkerImpl {
+    def clearableLinker(cfg: StandardConfig): ClearableLinker =
+      parent.clearableLinker(cfg)
+
+    def irFileCache(): IRFileCache =
+      parent.irFileCache()
+
+    def irContainers(classpath: Seq[Path])(
+        implicit ec: ExecutionContext): Future[(Seq[IRContainer], Seq[Path])] =
+      parent.irContainers(classpath)
+
+    def outputFile(path: Path): LinkerOutput.File =
+      parent.outputFile(path)
+  }
+
+  private final class FilteringClassLoader(parent: ClassLoader)
+      extends ClassLoader(parent) {
+    private val parentPrefixes = List(
+      "java.",
+      "scala.",
+      "org.scalajs.linker.interface.",
+      "org.scalajs.logging.",
+      "org.scalajs.ir."
+    )
+
+    override def loadClass(name: String, resolve: Boolean): Class[_] = {
+      if (parentPrefixes.exists(name.startsWith _))
+        super.loadClass(name, resolve)
+      else
+        null
+    }
+  }
+
+  private final class Reflect(loader: ClassLoader) extends LinkerImpl {
+    private def loadMethod(clazz: String, method: String, result: Class[_], params: Class[_]*): Method = {
+      val m = Class.forName("org.scalajs.linker." + clazz, true, loader).getMethod(method, params: _*)
+      require(Modifier.isStatic(m.getModifiers()))
+      require(result.isAssignableFrom(m.getReturnType()))
+      m
+    }
+
+    private def invoke[T](method: Method, args: AnyRef*): T =
+      method.invoke(null, args: _*).asInstanceOf[T]
+
+    /* We load everything eagerly to fail immediately, not only when the methods
+     * are invoked.
+     */
+    private val clearableLinkerMethod =
+      loadMethod("StandardImpl", "clearableLinker", classOf[ClearableLinker], classOf[StandardConfig])
+
+    private val irFileCacheMethod =
+      loadMethod("StandardImpl", "irFileCache", classOf[IRFileCache])
+
+    private val irContainersMethod = {
+      loadMethod("PathIRContainer", "fromClasspath", classOf[Future[_]],
+          classOf[Seq[Path]], classOf[ExecutionContext])
+    }
+
+    private val outputFileMethod =
+      loadMethod("PathOutputFile", "atomic", classOf[LinkerOutput.File], classOf[Path])
+
+    def clearableLinker(cfg: StandardConfig): ClearableLinker =
+      invoke(clearableLinkerMethod, cfg)
+
+    def irFileCache(): IRFileCache =
+      invoke(irFileCacheMethod)
+
+    def irContainers(classpath: Seq[Path])(
+        implicit ec: ExecutionContext): Future[(Seq[IRContainer], Seq[Path])] = {
+      invoke(irContainersMethod, classpath, ec)
+    }
+
+    def outputFile(path: Path): LinkerOutput.File =
+      invoke(outputFileMethod, path)
+  }
+}