From 51352a091c482d7479dc30d36aa7e663abfec4e3 Mon Sep 17 00:00:00 2001 From: Jason Zaugg Date: Tue, 13 Jun 2017 22:00:07 +1000 Subject: [PATCH 1/3] Experimental support for benchmarking compilation under SBT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Works by driving a child SBT process via stdin/stdout scraping. ``` > compilation/jmh:run HotSbtBenchmark -f 1 -psource=scalap [info] Compiling 1 Scala source to /Users/jz/code/compiler-benchmark/compilation/target/scala-2.11/classes... Processing 46 classes from /Users/jz/code/compiler-benchmark/compilation/target/scala-2.11/classes with "reflection" generator Writing out Java source to /Users/jz/code/compiler-benchmark/compilation/target/scala-2.11/src_managed/jmh and resources to /Users/jz/code/compiler-benchmark/compilation/target/scala-2.11/resource_managed/jmh [info] Compiling 5 Java sources to /Users/jz/code/compiler-benchmark/compilation/target/scala-2.11/classes... [info] Running scala.bench.ScalacBenchmarkRunner HotSbtBenchmark -f 1 -psource=scalap [info] # JMH version: 1.19 [info] # VM version: JDK 1.8.0_112, VM 25.112-b16 [info] # VM invoker: /Library/Java/JavaVirtualMachines/jdk1.8.0_112.jdk/Contents/Home/jre/bin/java [info] # VM options: -Dsbt.launcher=/usr/local/Cellar/sbt/0.13.15/libexec/bin/sbt-launch.jar [info] # Warmup: 10 iterations, 10 s each [info] # Measurement: 10 iterations, 10 s each [info] # Timeout: 10 min per iteration [info] # Threads: 1 thread, will synchronize iterations [info] # Benchmark mode: Sampling time [info] # Benchmark: scala.tools.nsc.HotSbtBenchmark.compile [info] # Parameters: (corpusVersion = a8c43dc, extraArgs = , source = scalap) [info] [info] # Run progress: 0.00% complete, ETA 00:03:20 [info] # Fork: 1 of 1 [info] # Warmup Iteration 1: 6683.623 ms/op [info] # Warmup Iteration 2: 2724.200 ±(99.9%) 2851.803 ms/op [info] # Warmup Iteration 3: 1867.863 ±(99.9%) 314.998 ms/op [info] # Warmup Iteration 4: 2167.197 ±(99.9%) 3228.941 ms/op [info] # Warmup Iteration 5: 2198.235 ±(99.9%) 713.756 ms/op [info] # Warmup Iteration 6: 1831.862 ±(99.9%) 452.490 ms/op [info] # Warmup Iteration 7: 1726.306 ±(99.9%) 115.085 ms/op [info] # Warmup Iteration 8: 1856.679 ±(99.9%) 1093.828 ms/op [info] # Warmup Iteration 9: 1615.406 ±(99.9%) 796.722 ms/op [info] # Warmup Iteration 10: 1492.573 ±(99.9%) 62.763 ms/op ... ⚡ sbt [info] Loading global plugins from /Users/jz/.sbt/0.13/plugins [info] Loading project definition from /Users/jz/code/compiler-benchmark/project [info] Set current project to compiler-benchmark (in build file:/Users/jz/code/compiler-benchmark/) > compilation/jmh:run HotScalacBenchmark -f 1 -psource=scalap [info] Running scala.bench.ScalacBenchmarkRunner HotScalacBenchmark -f 1 -psource=scalap [info] # JMH version: 1.19 [info] # VM version: JDK 1.8.0_112, VM 25.112-b16 [info] # VM invoker: /Library/Java/JavaVirtualMachines/jdk1.8.0_112.jdk/Contents/Home/jre/bin/java [info] # VM options: -Dsbt.launcher=/usr/local/Cellar/sbt/0.13.15/libexec/bin/sbt-launch.jar [info] # Warmup: 10 iterations, 10 s each [info] # Measurement: 10 iterations, 10 s each [info] # Timeout: 10 min per iteration [info] # Threads: 1 thread, will synchronize iterations [info] # Benchmark mode: Sampling time [info] # Benchmark: scala.tools.nsc.HotScalacBenchmark.compile [info] # Parameters: (corpusVersion = a8c43dc, extraArgs = , source = scalap) [info] [info] # Run progress: 0.00% complete, ETA 00:03:20 [info] # Fork: 1 of 1 [info] # Warmup Iteration 1: 5312.086 ms/op [info] # Warmup Iteration 2: 2054.790 ±(99.9%) 1420.469 ms/op [info] # Warmup Iteration 3: 1456.622 ±(99.9%) 141.405 ms/op [info] # Warmup Iteration 4: 1297.613 ±(99.9%) 100.157 ms/op [info] # Warmup Iteration 5: 1248.038 ±(99.9%) 38.623 ms/op [info] # Warmup Iteration 6: 1221.941 ±(99.9%) 31.619 ms/op [info] # Warmup Iteration 7: 1200.969 ±(99.9%) 33.985 ms/op [info] # Warmup Iteration 8: 1210.756 ±(99.9%) 44.681 ms/op ``` --- build.sbt | 3 +- .../scala/tools/nsc/BenchmarkUtils.scala | 48 ++++++ .../scala/tools/nsc/HotSbtBenchmark.scala | 141 ++++++++++++++++++ .../scala/tools/nsc/ScalacBenchmark.scala | 29 +--- 4 files changed, 195 insertions(+), 26 deletions(-) create mode 100644 compilation/src/main/scala/scala/tools/nsc/BenchmarkUtils.scala create mode 100644 compilation/src/main/scala/scala/tools/nsc/HotSbtBenchmark.scala diff --git a/build.sbt b/build.sbt index 3b7a311..6dd12d0 100644 --- a/build.sbt +++ b/build.sbt @@ -61,7 +61,8 @@ lazy val addJavaOptions = javaOptions ++= { } List( "-DscalaVersion=" + scalaVersion.value, - "-DscalaRef=" + refOf(scalaVersion.value) + "-DscalaRef=" + refOf(scalaVersion.value), + "-Dsbt.launcher=" + (sys.props("java.class.path").split(java.io.File.pathSeparatorChar).find(_.contains("sbt-launch")).getOrElse("")) ) } diff --git a/compilation/src/main/scala/scala/tools/nsc/BenchmarkUtils.scala b/compilation/src/main/scala/scala/tools/nsc/BenchmarkUtils.scala new file mode 100644 index 0000000..2df0252 --- /dev/null +++ b/compilation/src/main/scala/scala/tools/nsc/BenchmarkUtils.scala @@ -0,0 +1,48 @@ +package scala.tools.nsc + +import java.io.{File, IOException} +import java.net.URL +import java.nio.file._ +import java.nio.file.attribute.BasicFileAttributes + +import com.typesafe.config.ConfigFactory + +import scala.collection.JavaConverters._ + +object BenchmarkUtils { + def deleteRecursive(directory: Path): Unit = { + Files.walkFileTree(directory, new SimpleFileVisitor[Path]() { + override def visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult = { + Files.delete(file) + FileVisitResult.CONTINUE + } + override def postVisitDirectory(dir: Path, exc: IOException): FileVisitResult = { + Files.delete(dir) + FileVisitResult.CONTINUE + } + }) + } + def initDeps(corpusSourcePath: Path): Seq[Path] = { + val depsDir = Paths.get(ConfigFactory.load.getString("deps.localdir")) + val depsFile = corpusSourcePath.resolve("deps.txt") + val depsClasspath = Seq.newBuilder[Path] + if (Files.exists(depsFile)) { + val res = new StringBuilder() + for (depUrlString <- Files.lines(depsFile).iterator().asScala) { + val depUrl = new URL(depUrlString) + val filename = Paths.get(depUrl.getPath).getFileName.toString + val depFile = depsDir.resolve(filename) + // TODO: check hash if file exists, or after downloading + if (!Files.exists(depFile)) { + if (!Files.exists(depsDir)) Files.createDirectories(depsDir) + val in = depUrl.openStream + Files.copy(in, depFile, StandardCopyOption.REPLACE_EXISTING) + in.close() + } + if (res.nonEmpty) res.append(File.pathSeparator) + depsClasspath += depFile + } + depsClasspath.result() + } else Nil + } +} diff --git a/compilation/src/main/scala/scala/tools/nsc/HotSbtBenchmark.scala b/compilation/src/main/scala/scala/tools/nsc/HotSbtBenchmark.scala new file mode 100644 index 0000000..fd3f104 --- /dev/null +++ b/compilation/src/main/scala/scala/tools/nsc/HotSbtBenchmark.scala @@ -0,0 +1,141 @@ +package scala.tools.nsc + +import java.io._ +import java.net.URL +import java.nio.file._ +import java.nio.file.attribute.BasicFileAttributes +import java.util.concurrent.TimeUnit + +import com.typesafe.config.ConfigFactory +import org.openjdk.jmh.annotations.Mode.SampleTime +import org.openjdk.jmh.annotations._ + +import scala.collection.JavaConverters._ + +@State(Scope.Benchmark) +@BenchmarkMode(Array(SampleTime)) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@Warmup(iterations = 10, time = 10, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 10, time = 10, timeUnit = TimeUnit.SECONDS) +@Fork(value = 3) +class HotSbtBenchmark { + @Param(value = Array()) + var source: String = _ + + @Param(value = Array("")) + var extraArgs: String = _ + + // This parameter is set by ScalacBenchmarkRunner / UploadingRunner based on the Scala version. + // When running the benchmark directly the "latest" symlink is used. + @Param(value = Array("latest")) + var corpusVersion: String = _ + + var sbtProcess: Process = _ + var inputRedirect: ProcessBuilder.Redirect = _ + var outputRedirect: ProcessBuilder.Redirect = _ + var tempDir: Path = _ + var scalaHome: Path = _ + var processOutputReader: BufferedReader = _ + var processInputReader: BufferedWriter = _ + var output= new java.lang.StringBuilder() + + def buildDef = + s""" + |scalaHome := Some(file("${scalaHome.toAbsolutePath.toString}")) + | + |val cleanClasses = taskKey[Unit]("clean the classes directory") + | + |cleanClasses := IO.delete((classDirectory in Compile).value) + | + |scalaSource in Compile := file("${corpusSourcePath.toAbsolutePath.toString}") + | + |libraryDependencies += "org.scala-lang" % "scala-compiler" % scalaVersion.value + |libraryDependencies += "org.scala-lang" % "scala-reflect" % scalaVersion.value + | + |// TODO support .java sources + """.stripMargin + + @Setup(Level.Trial) def spawn(): Unit = { + tempDir = Files.createTempDirectory("sbt-") + scalaHome = Files.createTempDirectory("scalaHome-") + initDepsClasspath() + Files.createDirectory(tempDir.resolve("project")) + Files.write(tempDir.resolve("project/build.properties"), java.util.Arrays.asList("sbt.version=0.13.15")) + Files.write(tempDir.resolve("build.sbt"), buildDef.getBytes("UTF-8")) + val sbtLaucherPath = System.getProperty("sbt.launcher") + if (sbtLaucherPath == null) sys.error("System property -Dsbt.launcher absent") + val builder = new ProcessBuilder(sys.props("java.home") + "/bin/java", "-Xms2G", "-Xmx2G", "-Dsbt.log.format=false", "-jar", sbtLaucherPath) + builder.directory(tempDir.toFile) + inputRedirect = builder.redirectInput() + outputRedirect = builder.redirectOutput() + sbtProcess = builder.start() + processOutputReader = new BufferedReader(new InputStreamReader(sbtProcess.getInputStream)) + processInputReader = new BufferedWriter(new OutputStreamWriter(sbtProcess.getOutputStream)) + awaitPrompt() + } + + @Benchmark + def compile(): Unit = { + issue(";cleanClasses;compile") + awaitPrompt() + } + + def issue(str: String) = { + processInputReader.write(str + "\n") + processInputReader.flush() + } + + def awaitPrompt(): Unit = { + output.setLength(0) + var line = "" + val buffer = new Array[Char](128) + var read : Int = -1 + while (true) { + read = processOutputReader.read(buffer) + if (read == -1) sys.error("EOF") + else { + output.append(buffer, 0, read) + if (output.toString.contains("\n> ")) { + if (output.toString.contains("[error")) sys.error(output.toString) + return + } + } + } + + } + + private def corpusSourcePath = Paths.get(s"../corpus/$source/$corpusVersion") + + def initDepsClasspath(): Unit = { + val libDir = tempDir.resolve("lib") + Files.createDirectories(libDir) + for (depFile <- BenchmarkUtils.initDeps(corpusSourcePath)) { + val libDirFile = libDir.resolve(depFile.getFileName) + Files.copy(depFile, libDir) + } + + val scalaHomeLibDir = scalaHome.resolve("lib") + Files.createDirectories(scalaHomeLibDir) + for (elem <- sys.props("java.class.path").split(File.pathSeparatorChar)) { + val jarFile = Paths.get(elem) + var name = jarFile.getFileName.toString + if (name.startsWith("scala") && name.endsWith(".jar")) { + if (name.startsWith("scala-library")) + name = "scala-library.jar" + else if (name.startsWith("scala-reflect")) + name = "scala-reflect.jar" + else if (name.startsWith("scala-compiler")) + name = "scala-compiler.jar" + Files.copy(jarFile, scalaHomeLibDir.resolve(name)) + } + } + + } + + @TearDown(Level.Trial) def terminate(): Unit = { + processOutputReader.close() + sbtProcess.destroyForcibly() + BenchmarkUtils.deleteRecursive(tempDir) + BenchmarkUtils.deleteRecursive(scalaHome) + } +} \ No newline at end of file diff --git a/compilation/src/main/scala/scala/tools/nsc/ScalacBenchmark.scala b/compilation/src/main/scala/scala/tools/nsc/ScalacBenchmark.scala index 2b394bd..d4838e5 100644 --- a/compilation/src/main/scala/scala/tools/nsc/ScalacBenchmark.scala +++ b/compilation/src/main/scala/scala/tools/nsc/ScalacBenchmark.scala @@ -82,37 +82,16 @@ class ScalacBenchmark { tempDir = tempFile } @TearDown(Level.Trial) def clearTemp(): Unit = { - val directory = tempDir.toPath - Files.walkFileTree(directory, new SimpleFileVisitor[Path]() { - override def visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult = { - Files.delete(file) - FileVisitResult.CONTINUE - } - override def postVisitDirectory(dir: Path, exc: IOException): FileVisitResult = { - Files.delete(dir) - FileVisitResult.CONTINUE - } - }) + BenchmarkUtils.deleteRecursive(tempDir.toPath) } private def corpusSourcePath = Paths.get(s"../corpus/$source/$corpusVersion") @Setup(Level.Trial) def initDepsClasspath(): Unit = { - val depsDir = Paths.get(ConfigFactory.load.getString("deps.localdir")) - val depsFile = corpusSourcePath.resolve("deps.txt") - if (Files.exists(depsFile)) { + val classPath = BenchmarkUtils.initDeps(corpusSourcePath) + if (classPath.nonEmpty) { val res = new StringBuilder() - for (depUrlString <- Files.lines(depsFile).iterator().asScala) { - val depUrl = new URL(depUrlString) - val filename = Paths.get(depUrl.getPath).getFileName.toString - val depFile = depsDir.resolve(filename) - // TODO: check hash if file exists, or after downloading - if (!Files.exists(depFile)) { - if (!Files.exists(depsDir)) Files.createDirectories(depsDir) - val in = depUrl.openStream - Files.copy(in, depFile, StandardCopyOption.REPLACE_EXISTING) - in.close() - } + for (depFile <- classPath) { if (res.nonEmpty) res.append(File.pathSeparator) res.append(depFile.toAbsolutePath.normalize.toString) } From d6e3a63a0c5e58e30d8f2894596837d1266dc232 Mon Sep 17 00:00:00 2001 From: Jason Zaugg Date: Mon, 19 Jun 2017 18:47:24 +1000 Subject: [PATCH 2/3] Make sbt version configurable This enables benchmarking of, for example, SBT 1.0.0-M6. --- .../src/main/scala/scala/tools/nsc/HotSbtBenchmark.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/compilation/src/main/scala/scala/tools/nsc/HotSbtBenchmark.scala b/compilation/src/main/scala/scala/tools/nsc/HotSbtBenchmark.scala index fd3f104..a98c6c6 100644 --- a/compilation/src/main/scala/scala/tools/nsc/HotSbtBenchmark.scala +++ b/compilation/src/main/scala/scala/tools/nsc/HotSbtBenchmark.scala @@ -25,6 +25,9 @@ class HotSbtBenchmark { @Param(value = Array("")) var extraArgs: String = _ + @Param(value = Array("0.13.15")) + var sbtVersion: String = _ + // This parameter is set by ScalacBenchmarkRunner / UploadingRunner based on the Scala version. // When running the benchmark directly the "latest" symlink is used. @Param(value = Array("latest")) @@ -60,7 +63,7 @@ class HotSbtBenchmark { scalaHome = Files.createTempDirectory("scalaHome-") initDepsClasspath() Files.createDirectory(tempDir.resolve("project")) - Files.write(tempDir.resolve("project/build.properties"), java.util.Arrays.asList("sbt.version=0.13.15")) + Files.write(tempDir.resolve("project/build.properties"), java.util.Arrays.asList("sbt.version=" + sbtVersion)) Files.write(tempDir.resolve("build.sbt"), buildDef.getBytes("UTF-8")) val sbtLaucherPath = System.getProperty("sbt.launcher") if (sbtLaucherPath == null) sys.error("System property -Dsbt.launcher absent") From b495678e2d9a47276d478854627ba5819139fd1d Mon Sep 17 00:00:00 2001 From: Jason Zaugg Date: Mon, 19 Jun 2017 18:48:39 +1000 Subject: [PATCH 3/3] Include console output in error when unable screen scrape a prompt --- .../src/main/scala/scala/tools/nsc/HotSbtBenchmark.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compilation/src/main/scala/scala/tools/nsc/HotSbtBenchmark.scala b/compilation/src/main/scala/scala/tools/nsc/HotSbtBenchmark.scala index a98c6c6..96f5ffb 100644 --- a/compilation/src/main/scala/scala/tools/nsc/HotSbtBenchmark.scala +++ b/compilation/src/main/scala/scala/tools/nsc/HotSbtBenchmark.scala @@ -95,7 +95,7 @@ class HotSbtBenchmark { var read : Int = -1 while (true) { read = processOutputReader.read(buffer) - if (read == -1) sys.error("EOF") + if (read == -1) sys.error("EOF: " + output.toString) else { output.append(buffer, 0, read) if (output.toString.contains("\n> ")) {