diff --git a/bench/src/main/scala/Benchmarks.scala b/bench/src/main/scala/Benchmarks.scala
index b3224e6ee97b..b4c1a369aa33 100644
--- a/bench/src/main/scala/Benchmarks.scala
+++ b/bench/src/main/scala/Benchmarks.scala
@@ -2,6 +2,8 @@ package dotty.tools.benchmarks
 
 import dotty.tools.dotc._
 import core.Contexts.Context
+import dotty.tools.FatalError
+import reporting._
 
 import org.openjdk.jmh.results.RunResult
 import org.openjdk.jmh.runner.Runner
@@ -34,10 +36,9 @@ object Bench {
     }
     storeCompileOptions(args2)
 
-    val libs = System.getProperty("BENCH_CLASS_PATH")
-
     val opts = new OptionsBuilder()
-               .jvmArgsPrepend(s"-classpath $libs", "-Xms2G", "-Xmx2G")
+               .shouldFailOnError(true)
+               .jvmArgs("-Xms2G", "-Xmx2G")
                .mode(Mode.AverageTime)
                .timeUnit(TimeUnit.MILLISECONDS)
                .warmupIterations(warmup)
@@ -54,9 +55,20 @@ object Bench {
   def removeCompileOptions: Unit = new File(COMPILE_OPTS_FILE).delete()
 
   def storeCompileOptions(args: Array[String]): Unit = {
+    val standard_libs = System.getProperty("BENCH_CLASS_PATH")
+    val compiler_libs = System.getProperty("BENCH_COMPILER_CLASS_PATH")
+
+    val libs = if (args.contains("-with-compiler")) compiler_libs else standard_libs
+    var argsNorm = args.filter(_ != "-with-compiler")
+
+    var cpIndex = argsNorm.indexOf("-classpath")
+    if (cpIndex == -1) cpIndex = argsNorm.indexOf("-cp")
+    if (cpIndex != -1) argsNorm(cpIndex + 1) = argsNorm(cpIndex + 1) + ":" + libs
+    else argsNorm = argsNorm :+ "-classpath" :+ libs
+
     val file = new File(COMPILE_OPTS_FILE)
     val bw = new BufferedWriter(new FileWriter(file))
-    bw.write(args.mkString("\n"))
+    bw.write(argsNorm.mkString("", "\n", "\n"))
     bw.close()
   }
 
@@ -75,6 +87,21 @@ class CompilerOptions {
 }
 
 class Worker extends Driver {
+  // override to avoid printing summary information
+  override  def doCompile(compiler: Compiler, fileNames: List[String])(implicit ctx: Context): Reporter =
+    if (fileNames.nonEmpty)
+      try {
+        val run = compiler.newRun
+        run.compile(fileNames)
+        ctx.reporter
+      }
+      catch {
+        case ex: FatalError  =>
+          ctx.error(ex.getMessage) // signals that we should fail compilation.
+          ctx.reporter
+      }
+    else ctx.reporter
+
   @Benchmark
   def compile(state: CompilerOptions): Unit = {
     val res = process(state.opts)
diff --git a/project/Build.scala b/project/Build.scala
index e50bfa73362a..00f5fe84cc1a 100644
--- a/project/Build.scala
+++ b/project/Build.scala
@@ -302,7 +302,8 @@ object Build {
   lazy val commonBenchmarkSettings = Seq(
     outputStrategy := Some(StdoutOutput),
     mainClass in (Jmh, run) := Some("dotty.tools.benchmarks.Bench"), // custom main for jmh:run
-    javaOptions += "-DBENCH_CLASS_PATH=" + Attributed.data((fullClasspath in Compile).value).mkString("", ":", "")
+    javaOptions += "-DBENCH_COMPILER_CLASS_PATH=" + Attributed.data((fullClasspath in (`dotty-bootstrapped`, Compile)).value).mkString("", ":", ""),
+    javaOptions += "-DBENCH_CLASS_PATH=" + Attributed.data((fullClasspath in (`dotty-library-bootstrapped`, Compile)).value).mkString("", ":", "")
   )
 
   // sbt >= 0.13.12 will automatically rewrite transitive dependencies on
diff --git a/project/scripts/cmdTests b/project/scripts/cmdTests
index 21473b2c3c7b..27d57480f6e2 100755
--- a/project/scripts/cmdTests
+++ b/project/scripts/cmdTests
@@ -12,6 +12,7 @@ EXPECTED_OUTPUT="hello world"
 # check that benchmarks can run
 "$SBT" "dotty-bench/jmh:run 1 1 tests/pos/alias.scala"
 "$SBT" "dotty-bench-bootstrapped/jmh:run 1 1 tests/pos/alias.scala"
+"$SBT" "dotty-bench-bootstrapped/jmh:run 1 1 -with-compiler compiler/src/dotty/tools/dotc/core/Types.scala"
 
 OUT=$(mktemp -d)
 OUT1=$(mktemp -d)