Compile benchmark with Dotty, fixes scala#29.

Note, the benchmarks currently don't actually run with Dotty because of
scala/scala3#2704.

- When `scalaVersion := "0.x"`, then "src/main/dotc" is  added to
  unmanagedSourceDirectories, otherwise "src/main/scalac" is added.
- compilation/test checks the setup is OK without the need to remember cli args.
- `fork in run` is necessary for -usejavacp to work with compilation/run
- `FileVisitOption.FOLLOW_LINKS` produces duplicate filenames, which
  causes "X is already compiled in this run" errors in Dotty.

Author: olafurpg

build.sbt

@@ -36,8 +36,17 @@ lazy val compilation = addJmh(project).settings(
   // We should be able to switch this project to a broad range of Scala versions for comparative
   // benchmarking. As such, this project should only depend on the high level `MainClass` compiler API.
   description := "Black box benchmark of the compiler",
-  libraryDependencies += scalaOrganization.value % "scala-compiler" % scalaVersion.value,
-  mainClass in (Jmh, run) := Some("scala.bench.ScalacBenchmarkRunner")
+  libraryDependencies += {
+    if (isDotty.value) "ch.epfl.lamp" %% "dotty-compiler" % scalaVersion.value
+    else scalaOrganization.value % "scala-compiler" % scalaVersion.value
+  },
+  unmanagedSourceDirectories.in(Compile) +=
+    sourceDirectory.in(Compile).value / (if (isDotty.value) "dotc" else "scalac"),
+  mainClass in (Jmh, run) := Some("scala.bench.ScalacBenchmarkRunner"),
+  libraryDependencies += "com.novocode" % "junit-interface" % "0.11" % Test,
+  testOptions in Test += Tests.Argument(TestFrameworks.JUnit),
+  fork in (Test, test) := true,
+  fork in run := true
 ).settings(addJavaOptions).dependsOn(infrastructure)
 
 lazy val micro = addJmh(project).settings(

compilation/src/main/dotc/scala/tools/benchmark/BenchmarkDriver.scala

@@ -0,0 +1,22 @@
+package scala.tools.benchmark
+
+import java.io.File
+import scala.tools.nsc.BaseBenchmarkDriver
+import dotty.tools.dotc.core.Contexts.ContextBase
+
+trait BenchmarkDriver extends BaseBenchmarkDriver {
+  def compileImpl(): Unit = {
+    implicit val ctx = (new ContextBase).initialCtx.fresh
+    ctx.setSetting(ctx.settings.usejavacp, true)
+    if (depsClasspath != null) {
+      ctx.setSetting(ctx.settings.classpath,
+                     depsClasspath.mkString(File.pathSeparator))
+    }
+    ctx.setSetting(ctx.settings.d, tempDir.getAbsolutePath)
+    ctx.setSetting(ctx.settings.language, List("Scala2"))
+    val compiler = new dotty.tools.dotc.Compiler
+    val reporter =
+      dotty.tools.dotc.Bench.doCompile(compiler, compilerArgs.toList)
+    assert(!reporter.hasErrors)
+  }
+}

compilation/src/main/scala/scala/tools/nsc/ScalacBenchmark.scala

@@ -12,9 +12,20 @@ import org.openjdk.jmh.annotations.Mode._
 import org.openjdk.jmh.annotations._
 
 import scala.collection.JavaConverters._
+import scala.tools.benchmark.BenchmarkDriver
+
+trait BaseBenchmarkDriver {
+  def source: String
+  def extraArgs: String
+  def corpusVersion: String
+  def depsClasspath: String
+  def tempDir: File
+  def corpusSourcePath: Path
+  def compilerArgs: Array[String]
+}
 
 @State(Scope.Benchmark)
-class ScalacBenchmark {
+class ScalacBenchmark extends BenchmarkDriver {
   @Param(value = Array())
   var source: String = _
 
@@ -26,22 +37,9 @@ class ScalacBenchmark {
   @Param(value = Array("latest"))
   var corpusVersion: String = _
 
-  var driver: Driver = _
-
   var depsClasspath: String = _
 
   def compileImpl(): Unit = {
-    val (compilerArgs, sourceFiles) =
-      if (source.startsWith("@")) (List(source), List[String]())
-      else {
-        import scala.collection.JavaConverters._
-        val allFiles = Files.walk(findSourceDir, FileVisitOption.FOLLOW_LINKS).collect(Collectors.toList[Path]).asScala.toList
-        val files = allFiles.filter(f => {
-          val name = f.getFileName.toString
-          name.endsWith(".scala") || name.endsWith(".java")
-        }).map(_.toAbsolutePath.normalize.toString).toList
-        (List[String](), files)
-      }
 
     // MainClass is copy-pasted from compiler for source compatibility with 2.10.x - 2.13.x
     class MainClass extends Driver with EvalLoop {
@@ -73,7 +71,21 @@ class ScalacBenchmark {
     assert(!driver.reporter.hasErrors)
   }
 
-  private var tempDir: File = null
+  def compilerArgs: List[String] = if (source.startsWith("@")) List(source) else Nil
+
+  def sourceFiles: List[String] =
+    if (source.startsWith("@")) Nil
+    else {
+      import scala.collection.JavaConverters._
+      val allFiles = Files.walk(findSourceDir, FileVisitOption.FOLLOW_LINKS).collect(Collectors.toList[Path]).asScala.toList
+      val files = allFiles.filter(f => {
+        val name = f.getFileName.toString
+        name.endsWith(".scala") || name.endsWith(".java")
+      }).map(_.toAbsolutePath.normalize.toString).toList
+      files
+    }
+
+  var tempDir: File = null
 
   // Executed once per fork
   @Setup(Level.Trial) def initTemp(): Unit = {
@@ -86,7 +98,7 @@ class ScalacBenchmark {
     BenchmarkUtils.deleteRecursive(tempDir.toPath)
   }
 
-  private def corpusSourcePath = Paths.get(s"../corpus/$source/$corpusVersion")
+  def corpusSourcePath: Path = Paths.get(s"../corpus/$source/$corpusVersion")
 
   @Setup(Level.Trial) def initDepsClasspath(): Unit = {
     val classPath = BenchmarkUtils.initDeps(corpusSourcePath)
@@ -138,6 +150,8 @@ class ColdScalacBenchmark extends ScalacBenchmark {
 @OutputTimeUnit(TimeUnit.MILLISECONDS)
 @Warmup(iterations = 0)
 @Measurement(iterations = 1, time = 30, timeUnit = TimeUnit.SECONDS)
+// @Fork triggers match error in dotty, see https://github.com/lampepfl/dotty/issues/2704
+// Comment out `@Fork` to run compilation/test with Dotty or wait for that issue to be fixed.
 @Fork(value = 3, jvmArgs = Array("-Xms2G", "-Xmx2G"))
 class WarmScalacBenchmark extends ScalacBenchmark {
   @Benchmark

compilation/src/main/scalac/scala/tools/benchmark/BenchmarkDriver.scala

@@ -0,0 +1,36 @@
+package scala.tools.benchmark
+
+import java.nio.file._
+import scala.tools.nsc._
+
+trait BenchmarkDriver extends BaseBenchmarkDriver {
+  def compileImpl(): Unit = {
+    // MainClass is copy-pasted from compiler for source compatibility with 2.10.x - 2.13.x
+    class MainClass extends Driver with EvalLoop {
+      def resident(compiler: Global): Unit = loop { line =>
+        val command = new CompilerCommand(line split "\\s+" toList, new Settings(scalacError))
+        compiler.reporter.reset()
+        new compiler.Run() compile command.files
+      }
+
+      override def newCompiler(): Global = Global(settings, reporter)
+
+      override protected def processSettingsHook(): Boolean = {
+        if (source == "scala")
+          settings.sourcepath.value = Paths.get(s"../corpus/$source/$corpusVersion/library").toAbsolutePath.normalize.toString
+        else
+          settings.usejavacp.value = true
+        settings.outdir.value = tempDir.getAbsolutePath
+        settings.nowarn.value = true
+        if (depsClasspath != null)
+          settings.processArgumentString(s"-cp $depsClasspath")
+        if (extraArgs != null && extraArgs != "")
+          settings.processArgumentString(extraArgs)
+        true
+      }
+    }
+    val driver = new MainClass
+    driver.process(compilerArgs)
+    assert(!driver.reporter.hasErrors)
+  }
+}

compilation/src/test/scala/scala/tools/benchmark/BenchmarkTest.scala

@@ -0,0 +1,9 @@
+package scala.tools.benchmark
+
+import scala.tools.nsc.ScalacBenchmarkStandalone
+import org.junit.Test
+
+class BenchmarkTest {
+  @Test def compilesOK =
+    ScalacBenchmarkStandalone.main(Array("../corpus/vector", "1"))
+}

project/plugins.sbt

@@ -3,3 +3,6 @@ logLevel := Level.Warn
 
 // sbt-jmh plugin - pulls in JMH dependencies too
 addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.2.25")
+
+// sbt-dotty plugin - to support `scalaVersion := "0.x"`
+addSbtPlugin("ch.epfl.lamp" % "sbt-dotty" % "0.1.2")