Support for benchmarking compilation under SBT

build.sbt

@@ -61,7 +61,8 @@ lazy val addJavaOptions = javaOptions ++= {
   }
   List(
     "-DscalaVersion=" + scalaVersion.value,
-    "-DscalaRef=" + refOf(scalaVersion.value)
+    "-DscalaRef=" + refOf(scalaVersion.value),
+    "-Dsbt.launcher=" + (sys.props("java.class.path").split(java.io.File.pathSeparatorChar).find(_.contains("sbt-launch")).getOrElse(""))
   )
 }
 

compilation/src/main/scala/scala/tools/nsc/BenchmarkUtils.scala

@@ -0,0 +1,48 @@
+package scala.tools.nsc
+
+import java.io.{File, IOException}
+import java.net.URL
+import java.nio.file._
+import java.nio.file.attribute.BasicFileAttributes
+
+import com.typesafe.config.ConfigFactory
+
+import scala.collection.JavaConverters._
+
+object BenchmarkUtils {
+  def deleteRecursive(directory: Path): Unit = {
+    Files.walkFileTree(directory, new SimpleFileVisitor[Path]() {
+      override def visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult = {
+        Files.delete(file)
+        FileVisitResult.CONTINUE
+      }
+      override def postVisitDirectory(dir: Path, exc: IOException): FileVisitResult = {
+        Files.delete(dir)
+        FileVisitResult.CONTINUE
+      }
+    })
+  }
+  def initDeps(corpusSourcePath: Path): Seq[Path] = {
+    val depsDir = Paths.get(ConfigFactory.load.getString("deps.localdir"))
+    val depsFile = corpusSourcePath.resolve("deps.txt")
+    val depsClasspath = Seq.newBuilder[Path]
+    if (Files.exists(depsFile)) {
+      val res = new StringBuilder()
+      for (depUrlString <- Files.lines(depsFile).iterator().asScala) {
+        val depUrl = new URL(depUrlString)
+        val filename = Paths.get(depUrl.getPath).getFileName.toString
+        val depFile = depsDir.resolve(filename)
+        // TODO: check hash if file exists, or after downloading
+        if (!Files.exists(depFile)) {
+          if (!Files.exists(depsDir)) Files.createDirectories(depsDir)
+          val in = depUrl.openStream
+          Files.copy(in, depFile, StandardCopyOption.REPLACE_EXISTING)
+          in.close()
+        }
+        if (res.nonEmpty) res.append(File.pathSeparator)
+        depsClasspath += depFile
+      }
+      depsClasspath.result()
+    } else Nil
+  }
+}

compilation/src/main/scala/scala/tools/nsc/HotSbtBenchmark.scala

@@ -0,0 +1,144 @@
+package scala.tools.nsc
+
+import java.io._
+import java.net.URL
+import java.nio.file._
+import java.nio.file.attribute.BasicFileAttributes
+import java.util.concurrent.TimeUnit
+
+import com.typesafe.config.ConfigFactory
+import org.openjdk.jmh.annotations.Mode.SampleTime
+import org.openjdk.jmh.annotations._
+
+import scala.collection.JavaConverters._
+
+@State(Scope.Benchmark)
+@BenchmarkMode(Array(SampleTime))
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@Warmup(iterations = 10, time = 10, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 10, time = 10, timeUnit = TimeUnit.SECONDS)
+@Fork(value = 3)
+class HotSbtBenchmark {
+  @Param(value = Array())
+  var source: String = _
+
+  @Param(value = Array(""))
+  var extraArgs: String = _
+
+  @Param(value = Array("0.13.15"))
+  var sbtVersion: String = _
+
+  // This parameter is set by ScalacBenchmarkRunner / UploadingRunner based on the Scala version.
+  // When running the benchmark directly the "latest" symlink is used.
+  @Param(value = Array("latest"))
+  var corpusVersion: String = _
+
+  var sbtProcess: Process = _
+  var inputRedirect: ProcessBuilder.Redirect = _
+  var outputRedirect: ProcessBuilder.Redirect = _
+  var tempDir: Path = _
+  var scalaHome: Path = _
+  var processOutputReader: BufferedReader = _
+  var processInputReader: BufferedWriter = _
+  var output= new java.lang.StringBuilder()
+
+  def buildDef =
+    s"""
+       |scalaHome := Some(file("${scalaHome.toAbsolutePath.toString}"))
+       |
+       |val cleanClasses = taskKey[Unit]("clean the classes directory")
+       |
+       |cleanClasses := IO.delete((classDirectory in Compile).value)
+       |
+       |scalaSource in Compile := file("${corpusSourcePath.toAbsolutePath.toString}")
+       |
+       |libraryDependencies += "org.scala-lang" % "scala-compiler" % scalaVersion.value
+       |libraryDependencies += "org.scala-lang" % "scala-reflect" % scalaVersion.value
+       |
+       |// TODO support .java sources
+    """.stripMargin
+
+  @Setup(Level.Trial) def spawn(): Unit = {
+    tempDir = Files.createTempDirectory("sbt-")
+    scalaHome = Files.createTempDirectory("scalaHome-")
+    initDepsClasspath()
+    Files.createDirectory(tempDir.resolve("project"))
+    Files.write(tempDir.resolve("project/build.properties"), java.util.Arrays.asList("sbt.version=" + sbtVersion))
+    Files.write(tempDir.resolve("build.sbt"), buildDef.getBytes("UTF-8"))
+    val sbtLaucherPath = System.getProperty("sbt.launcher")
+    if (sbtLaucherPath == null) sys.error("System property -Dsbt.launcher absent")
+    val builder = new ProcessBuilder(sys.props("java.home") + "/bin/java", "-Xms2G", "-Xmx2G", "-Dsbt.log.format=false", "-jar", sbtLaucherPath)
+    builder.directory(tempDir.toFile)
+    inputRedirect = builder.redirectInput()
+    outputRedirect = builder.redirectOutput()
+    sbtProcess = builder.start()
+    processOutputReader = new BufferedReader(new InputStreamReader(sbtProcess.getInputStream))
+    processInputReader = new BufferedWriter(new OutputStreamWriter(sbtProcess.getOutputStream))
+    awaitPrompt()
+  }
+
+  @Benchmark
+  def compile(): Unit = {
+    issue(";cleanClasses;compile")
+    awaitPrompt()
+  }
+
+  def issue(str: String) = {
+    processInputReader.write(str + "\n")
+    processInputReader.flush()
+  }
+
+  def awaitPrompt(): Unit = {
+    output.setLength(0)
+    var line = ""
+    val buffer = new Array[Char](128)
+    var read : Int = -1
+    while (true) {
+      read = processOutputReader.read(buffer)
+      if (read == -1) sys.error("EOF: " + output.toString)
+      else {
+        output.append(buffer, 0, read)
+        if (output.toString.contains("\n> ")) {
+          if (output.toString.contains("[error")) sys.error(output.toString)
+          return
+        }
+      }
+    }
+
+  }
+
+  private def corpusSourcePath = Paths.get(s"../corpus/$source/$corpusVersion")
+
+  def initDepsClasspath(): Unit = {
+    val libDir = tempDir.resolve("lib")
+    Files.createDirectories(libDir)
+    for (depFile <- BenchmarkUtils.initDeps(corpusSourcePath)) {
+      val libDirFile = libDir.resolve(depFile.getFileName)
+      Files.copy(depFile, libDir)
+    }
+
+    val scalaHomeLibDir = scalaHome.resolve("lib")
+    Files.createDirectories(scalaHomeLibDir)
+    for (elem <- sys.props("java.class.path").split(File.pathSeparatorChar)) {
+      val jarFile = Paths.get(elem)
+      var name = jarFile.getFileName.toString
+      if (name.startsWith("scala") && name.endsWith(".jar")) {
+        if (name.startsWith("scala-library"))
+          name = "scala-library.jar"
+        else if (name.startsWith("scala-reflect"))
+          name = "scala-reflect.jar"
+        else if (name.startsWith("scala-compiler"))
+          name = "scala-compiler.jar"
+        Files.copy(jarFile, scalaHomeLibDir.resolve(name))
+      }
+    }
+
+  }
+
+  @TearDown(Level.Trial) def terminate(): Unit = {
+    processOutputReader.close()
+    sbtProcess.destroyForcibly()
+    BenchmarkUtils.deleteRecursive(tempDir)
+    BenchmarkUtils.deleteRecursive(scalaHome)
+  }
+}

compilation/src/main/scala/scala/tools/nsc/ScalacBenchmark.scala

@@ -82,37 +82,16 @@ class ScalacBenchmark {
     tempDir = tempFile
   }
   @TearDown(Level.Trial) def clearTemp(): Unit = {
-    val directory = tempDir.toPath
-    Files.walkFileTree(directory, new SimpleFileVisitor[Path]() {
-      override def visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult = {
-        Files.delete(file)
-        FileVisitResult.CONTINUE
-      }
-      override def postVisitDirectory(dir: Path, exc: IOException): FileVisitResult = {
-        Files.delete(dir)
-        FileVisitResult.CONTINUE
-      }
-    })
+    BenchmarkUtils.deleteRecursive(tempDir.toPath)
   }
 
   private def corpusSourcePath = Paths.get(s"../corpus/$source/$corpusVersion")
 
   @Setup(Level.Trial) def initDepsClasspath(): Unit = {
-    val depsDir = Paths.get(ConfigFactory.load.getString("deps.localdir"))
-    val depsFile = corpusSourcePath.resolve("deps.txt")
-    if (Files.exists(depsFile)) {
+    val classPath = BenchmarkUtils.initDeps(corpusSourcePath)
+    if (classPath.nonEmpty) {
       val res = new StringBuilder()
-      for (depUrlString <- Files.lines(depsFile).iterator().asScala) {
-        val depUrl = new URL(depUrlString)
-        val filename = Paths.get(depUrl.getPath).getFileName.toString
-        val depFile = depsDir.resolve(filename)
-        // TODO: check hash if file exists, or after downloading
-        if (!Files.exists(depFile)) {
-          if (!Files.exists(depsDir)) Files.createDirectories(depsDir)
-          val in = depUrl.openStream
-          Files.copy(in, depFile, StandardCopyOption.REPLACE_EXISTING)
-          in.close()
-        }
+      for (depFile <- classPath) {
         if (res.nonEmpty) res.append(File.pathSeparator)
         res.append(depFile.toAbsolutePath.normalize.toString)
       }