Fix scala-js#3661: Load linker reflectively

gzm0 · gzm0 · commit 8e66c5546048 · 2019-10-13T16:57:00.000+02:00
This gives us the following advantages: - Allows us to give resources to the linker that the build itself compiles first (unblocks scala-js#3537). - Hides linker dependencies away from other sbt plugins. Notably the Google Closure Compiler and Guava (fixes scala-js#3193).
diff --git a/Jenkinsfile b/Jenkinsfile
diff --git a/project/Build.scala b/project/Build.scala
@@ -46,12 +46,17 @@ object ExposedValues extends AutoPlugin {
       }
     }
 
+    val jvmScalaVersion = SettingKey[String]("jvmScalaVersion")
+    val jsScalaVersion = SettingKey[String]("jsScalaVersion")
+
     val CheckedBehavior = org.scalajs.linker.interface.CheckedBehavior
 
     type NodeJSEnvForcePolyfills = build.NodeJSEnvForcePolyfills
   }
 }
 
+import ExposedValues.autoImport.{jvmScalaVersion, jsScalaVersion}
+
 object MyScalaJSPlugin extends AutoPlugin {
   override def requires: Plugins = ScalaJSPlugin
 
@@ -97,13 +102,22 @@ object MyScalaJSPlugin extends AutoPlugin {
        */
       crossVersion := CrossVersion.binary,
 
+      scalaVersion := jsScalaVersion.value,
+
       scalaJSLinkerConfig ~= (_.withCheckIR(true)),
 
       wantSourceMaps := true,
 
       jsEnv := new NodeJSEnv(
           NodeJSEnv.Config().withSourceMap(wantSourceMaps.value)),
 
+      scalaJSLinkerImpl := {
+        val v = (scalaVersion in Build.linker).value
+        assert(v.startsWith("2.12."), "The linker is configured with a Scala " +
+            s"version ($v) that is not compatible with this build (2.12.x). ")
+        LinkerImpl.default(Attributed.data((fullClasspath in (Build.linker, Runtime)).value))
+      },
+
       // Link source maps to GitHub sources
       addScalaJSCompilerOption(Def.setting {
         "mapSourceURI:" +
@@ -201,7 +215,7 @@ object Build {
   }
 
   val commonSettings = Seq(
-      scalaVersion := "2.12.8",
+      scalaVersion := jvmScalaVersion.value,
       organization := "org.scala-js",
       version := scalaJSVersion,
 
@@ -495,7 +509,10 @@ object Build {
         if (v < 8)
           throw new MessageOnlyException("This build requires JDK 8 or later. Aborting.")
         v
-      }
+      },
+
+      jvmScalaVersion in Global := "2.12.8",
+      jsScalaVersion in Global := "2.12.8",
   )
 
   lazy val root: Project = Project(id = "scalajs", base = file(".")).settings(
@@ -815,7 +832,7 @@ object Build {
 
         sbtJars.map(_.data -> docUrl).toMap
       }
-  ).dependsOn(linker, jsEnvs, nodeJSEnv, testAdapter)
+  ).dependsOn(linkerInterface, jsEnvs, nodeJSEnv, testAdapter)
 
   lazy val delambdafySetting = {
     scalacOptions ++= (
diff --git a/project/build.sbt b/project/build.sbt
@@ -6,8 +6,6 @@ addSbtPlugin("org.scalastyle" % "scalastyle-sbt-plugin" % "1.0.0")
 
 addSbtPlugin("org.portable-scala" % "sbt-platform-deps" % "1.0.0")
 
-libraryDependencies += "com.google.javascript" % "closure-compiler" % "v20190513"
-
 libraryDependencies += "com.google.jimfs" % "jimfs" % "1.1"
 
 libraryDependencies += "org.eclipse.jgit" % "org.eclipse.jgit.pgm" % "3.2.0.201312181205-r"
@@ -20,8 +18,6 @@ unmanagedSourceDirectories in Compile ++= {
     root / "logging/jvm/src/main/scala",
     root / "linker-interface/shared/src/main/scala",
     root / "linker-interface/jvm/src/main/scala",
-    root / "linker/shared/src/main/scala",
-    root / "linker/jvm/src/main/scala",
     root / "js-envs/src/main/scala",
     root / "nodejs-env/src/main/scala",
     root / "test-adapter/src/main/scala",
diff --git a/sbt-plugin/src/main/scala/org/scalajs/sbtplugin/CacheBox.scala b/sbt-plugin/src/main/scala/org/scalajs/sbtplugin/CacheBox.scala
@@ -0,0 +1,35 @@
+/*
+ * Scala.js (https://www.scala-js.org/)
+ *
+ * Copyright EPFL.
+ *
+ * Licensed under Apache License 2.0
+ * (https://www.apache.org/licenses/LICENSE-2.0).
+ *
+ * See the NOTICE file distributed with this work for
+ * additional information regarding copyright ownership.
+ */
+
+package org.scalajs.sbtplugin
+
+/** A CacheBox is support class to abuse an sbt setting as a cache.
+ *
+ *  A CacheBox is needed, once the cached result needs to depend on a task,
+ *  since then it cannot simply be made a setting anymore.
+ */
+private[sbtplugin] final class CacheBox[T] {
+  private[this] var value: T = _
+
+  def ensure(f: => T): T = synchronized {
+    if (value == null) {
+      value = f
+    }
+    value
+  }
+
+  def foreach(f: T => Unit): Unit = synchronized {
+    if (value != null) {
+      f(value)
+    }
+  }
+}
diff --git a/sbt-plugin/src/main/scala/org/scalajs/sbtplugin/LinkerImpl.scala b/sbt-plugin/src/main/scala/org/scalajs/sbtplugin/LinkerImpl.scala
@@ -0,0 +1,132 @@
+/*
+ * Scala.js (https://www.scala-js.org/)
+ *
+ * Copyright EPFL.
+ *
+ * Licensed under Apache License 2.0
+ * (https://www.apache.org/licenses/LICENSE-2.0).
+ *
+ * See the NOTICE file distributed with this work for
+ * additional information regarding copyright ownership.
+ */
+
+package org.scalajs.sbtplugin
+
+import scala.concurrent._
+
+import java.lang.reflect.{Method, Modifier}
+import java.io.File
+import java.net.URLClassLoader
+import java.nio.file.Path
+
+import org.scalajs.linker.interface._
+
+/** Abstract implementation of a linker as needed by the sbt plugin.
+ *
+ *  @note This trait does not guarantee full compatibility: Methods may be added
+ *      / removed in the future. Use [[LinkerImpl.Forwarding]] to override things
+ *      selectively in a compatible manner.
+ */
+trait LinkerImpl {
+  def clearableLinker(cfg: StandardConfig): ClearableLinker
+
+  def irFileCache(): IRFileCache
+
+  def irContainers(classpath: Seq[Path])(
+      implicit ec: ExecutionContext): Future[(Seq[IRContainer], Seq[Path])]
+
+  def outputFile(path: Path): LinkerOutput.File
+}
+
+object LinkerImpl {
+  /** Returns the default implementation.
+   *
+   *  It loads a StandardLinker via reflection.
+   */
+  def default(files: Seq[File]): LinkerImpl = {
+    val urls = files.map(_.toURI.toURL).toArray
+    val loader = new URLClassLoader(urls, new FilteringClassLoader(getClass.getClassLoader))
+    new Reflect(loader)
+  }
+
+  /** A [[LinkerImpl]] that forwards everything to `parent`.
+   *
+   *  This is useful if only parts of the linker implementation need to be
+   *  replaced. A subclass only overriding these can be created, ensuring easier
+   *  transition when methods get added.
+   */
+  class Forwarding(parent: LinkerImpl) extends LinkerImpl {
+    def clearableLinker(cfg: StandardConfig): ClearableLinker =
+      parent.clearableLinker(cfg)
+
+    def irFileCache(): IRFileCache =
+      parent.irFileCache()
+
+    def irContainers(classpath: Seq[Path])(
+        implicit ec: ExecutionContext): Future[(Seq[IRContainer], Seq[Path])] =
+      parent.irContainers(classpath)
+
+    def outputFile(path: Path): LinkerOutput.File =
+      parent.outputFile(path)
+  }
+
+  private final class FilteringClassLoader(parent: ClassLoader)
+      extends ClassLoader(parent) {
+    private val parentPrefixes = List(
+      "java.",
+      "scala.",
+      "org.scalajs.linker.interface.",
+      "org.scalajs.logging.",
+    )
+
+    override def loadClass(name: String, resolve: Boolean): Class[_] = {
+      if (parentPrefixes.exists(name.startsWith _))
+        super.loadClass(name, resolve)
+      else
+        null
+    }
+  }
+
+  private final class Reflect(loader: ClassLoader) extends LinkerImpl {
+    private def loadMethod(clazz: String, method: String, result: Class[_], params: Class[_]*): Method = {
+      val m = Class.forName("org.scalajs.linker." + clazz, true, loader).getMethod(method, params: _*)
+      require(Modifier.isStatic(m.getModifiers()))
+      require(result.isAssignableFrom(m.getReturnType()))
+      m
+    }
+
+    private def invoke[T](method: Method, args: AnyRef*): T =
+      method.invoke(null, args: _*).asInstanceOf[T]
+
+    /* We load everything eagery to fail immediately, not only when the methods
+     * are invoked.
+     */
+    private val clearableLinkerMethod =
+      loadMethod("StandardImpl", "clearableLinker", classOf[ClearableLinker], classOf[StandardConfig])
+
+    private val irFileCacheMethod =
+      loadMethod("StandardImpl", "irFileCache", classOf[IRFileCache])
+
+    private val irContainersMethod = {
+      loadMethod("PathIRContainer", "fromClasspath", classOf[Future[_]],
+        classOf[Seq[Path]], classOf[ExecutionContext])
+    }
+
+    private val outputFileMethod =
+      loadMethod("PathOutputFile", "atomic", classOf[LinkerOutput.File], classOf[Path])
+
+    def clearableLinker(cfg: StandardConfig): ClearableLinker =
+      invoke(clearableLinkerMethod, cfg)
+
+    def irFileCache(): IRFileCache =
+      invoke(irFileCacheMethod)
+
+    def irContainers(classpath: Seq[Path])(
+        implicit ec: ExecutionContext): Future[(Seq[IRContainer], Seq[Path])] = {
+      invoke(irContainersMethod, classpath, ec)
+    }
+
+    def outputFile(path: Path): LinkerOutput.File =
+      invoke(outputFileMethod, path)
+  }
+}
diff --git a/sbt-plugin/src/main/scala/org/scalajs/sbtplugin/ScalaJSPlugin.scala b/sbt-plugin/src/main/scala/org/scalajs/sbtplugin/ScalaJSPlugin.scala
@@ -57,13 +57,17 @@ object ScalaJSPlugin extends AutoPlugin {
 
     // All our public-facing keys
 
-    val scalaJSIRCache = SettingKey[IRFileCache.Cache](
-        "scalaJSIRCache",
-        "Scala.js internal: Task to access a cache.", KeyRanks.Invisible)
+    val scalaJSIRCacheBox = SettingKey[CacheBox[IRFileCache.Cache]](
+        "scalaJSIRCacheBox",
+        "Scala.js internal: CacheBox for a cache.", KeyRanks.Invisible)
 
-    /** Persisted instance of the Scala.js linker.
+    val scalaJSGlobalIRCacheBox = SettingKey[CacheBox[IRFileCache]](
+        "scalaJSGlobalIRCacheBox",
+        "Scala.js internal: CacheBox for the global cache.", KeyRanks.Invisible)
+
+    /** Instance of the Scala.js linker.
      *
-     *  This setting must be scoped per project, configuration, and stage task
+     *  This task must be scoped per project, configuration, and stage task
      *  (`fastOptJS` or `fullOptJS`).
      *
      *  If a task uses the `link` method of the `ClearableLinker`, it must be
@@ -82,9 +86,19 @@ object ScalaJSPlugin extends AutoPlugin {
      *    }.tag(usesLinkerTag)
      *  }.value,
      *  }}}
+     *
+     *  Do not set this value. Instead, set [[scalaJSLinkerImpl]]. This will
+     *  automatically set up the correct caching behavior.
      */
-    val scalaJSLinker = SettingKey[ClearableLinker]("scalaJSLinker",
-        "Persisted instance of the Scala.js linker", KeyRanks.Invisible)
+    val scalaJSLinker = TaskKey[ClearableLinker]("scalaJSLinker",
+        "Access task for a Scala.js linker. Use this if you want to use the linker.",
+        KeyRanks.Invisible)
+
+    val scalaJSLinkerImpl = TaskKey[LinkerImpl]("scalaJSLinkerImpl",
+        "Factory for a Scala.js linker", KeyRanks.Invisible)
+
+    val scalaJSLinkerBox = SettingKey[CacheBox[ClearableLinker]]("scalaJSLinkerBox",
+        "CacheBox for a Scala.js linker", KeyRanks.Invisible)
 
     /** A tag to indicate that a task is using the value of [[scalaJSLinker]]
      *  and its `link` method.
@@ -172,9 +186,14 @@ object ScalaJSPlugin extends AutoPlugin {
   import autoImport._
 
   /** Logs the current statistics about the global IR cache. */
-  def logIRCacheStats(logger: Logger): Unit = {
-    import ScalaJSPluginInternal.globalIRCache
-    logger.debug("Global IR cache stats: " + globalIRCache.stats.logLine)
+  val irCacheStatsLogger: Def.Initialize[Logger => Unit] = Def.setting {
+    val globalIRCacheBox = scalaJSGlobalIRCacheBox.value
+
+    logger: Logger => {
+      globalIRCacheBox.foreach { globalIRCache =>
+        logger.debug("Global IR cache stats: " + globalIRCache.stats.logLine)
+      }
+    }
   }
 
   override def globalSettings: Seq[Setting[_]] = {
@@ -183,16 +202,34 @@ object ScalaJSPlugin extends AutoPlugin {
 
         scalaJSLinkerConfig := StandardConfig(),
 
+        scalaJSLinkerImpl := {
+          val s = streams.value
+          val log = s.log
+          val retrieveDir = s.cacheDirectory / "scalajs-linker" / scalaJSVersion
+          val lm = {
+            import sbt.librarymanagement.ivy._
+            val ivyConfig = InlineIvyConfiguration().withLog(log)
+            IvyDependencyResolution(ivyConfig)
+          }
+          lm.retrieve(
+              "org.scala-js" % "scalajs-linker_2.12" % scalaJSVersion,
+              scalaModuleInfo = None, retrieveDir, log)
+            .fold(w => throw w.resolveException, LinkerImpl.default _)
+        },
+
+        scalaJSGlobalIRCacheBox := new CacheBox,
+
         jsEnv := new NodeJSEnv(),
 
         // Clear the IR cache stats every time a sequence of tasks ends
         onComplete := {
           val prev = onComplete.value
+          val globalIRCacheBox = scalaJSGlobalIRCacheBox.value
 
           { () =>
             prev()
             ScalaJSPluginInternal.closeAllTestAdapters()
-            ScalaJSPluginInternal.globalIRCache.clearStats()
+            globalIRCacheBox.foreach(_.clearStats())
           }
         },
 
diff --git a/sbt-plugin/src/main/scala/org/scalajs/sbtplugin/ScalaJSPluginInternal.scala b/sbt-plugin/src/main/scala/org/scalajs/sbtplugin/ScalaJSPluginInternal.scala
