Add test filtering via filterTest <regex>

Author: felixmulder

.drone.yml

@@ -6,7 +6,7 @@ pipeline:
       - ln -s /var/cache/drone/scala-scala scala-scala
       - ln -s /var/cache/drone/ivy2 "$HOME/.ivy2"
       - ./scripts/update-scala-library
-      - sbt -J-Xmx4096m -J-XX:ReservedCodeCacheSize=512m -J-XX:MaxMetaspaceSize=1024m -Ddotty.drone.mem=4096m "${TEST}"
+      - sbt -J-Xmx24G -J-XX:ReservedCodeCacheSize=512m -J-XX:MaxMetaspaceSize=4G -Ddotty.drone.mem=4096m "${TEST}"
     when:
       branch:
         exclude: gh-pages
@@ -34,9 +34,9 @@ pipeline:
 
 matrix:
   TEST:
-    - ;test;dotty-bin-tests/test
-    - ;publishLocal;dotty-bootstrapped/test
-    - partest-only-no-bootstrap --show-diff --verbose
-    - partest-only --show-diff --verbose
-    - ;set testOptions in LocalProject("dotty-compiler") := Seq() ;dotty-compiler/testOnly dotty.tools.dotc.CompilationTests
-    - ;publishLocal ;set testOptions in LocalProject("dotty-compiler-bootstrapped") := Seq() ;dotty-bootstrapped/testOnly dotty.tools.dotc.CompilationTests
+    - ;set testOptions in LocalProject("dotty-compiler") += Tests.Argument(TestFrameworks.JUnit, "--exclude-categories=dotty.tools.dotc.ParallelTesting") ;test ;dotty-bin-tests/test
+    - ;set testOptions in LocalProject("dotty-compiler-bootstrapped") += Tests.Argument(TestFrameworks.JUnit, "--exclude-categories=dotty.tools.dotc.ParallelTesting") ;publishLocal ;dotty-bootstrapped/test
+    - ;set testOptions in LocalProject("dotty-compiler") += Tests.Argument(TestFrameworks.JUnit, "--exclude-categories=dotty.tools.dotc.ParallelTesting") ;partest-only-no-bootstrap --show-diff --verbose
+    - ;set testOptions in LocalProject("dotty-compiler-bootstrapped") += Tests.Argument(TestFrameworks.JUnit, "--exclude-categories=dotty.tools.dotc.ParallelTesting") ;partest-only --show-diff --verbose
+    - ;dotty-compiler/testOnly dotty.tools.dotc.CompilationTests
+    - ;publishLocal ;dotty-bootstrapped/testOnly dotty.tools.dotc.CompilationTests

.drone.yml.sig

@@ -1 +1 @@
-eyJhbGciOiJIUzI1NiJ9.cGlwZWxpbmU6CiAgdGVzdDoKICAgIGltYWdlOiBsYW1wZXBmbC9kb3R0eTpsYXRlc3QKICAgIHB1bGw6IHRydWUKICAgIGNvbW1hbmRzOgogICAgICAtIGxuIC1zIC92YXIvY2FjaGUvZHJvbmUvc2NhbGEtc2NhbGEgc2NhbGEtc2NhbGEKICAgICAgLSBsbiAtcyAvdmFyL2NhY2hlL2Ryb25lL2l2eTIgIiRIT01FLy5pdnkyIgogICAgICAtIC4vc2NyaXB0cy91cGRhdGUtc2NhbGEtbGlicmFyeQogICAgICAtIHNidCAtSi1YbXg0MDk2bSAtSi1YWDpSZXNlcnZlZENvZGVDYWNoZVNpemU9NTEybSAtSi1YWDpNYXhNZXRhc3BhY2VTaXplPTEwMjRtIC1EZG90dHkuZHJvbmUubWVtPTQwOTZtICIke1RFU1R9IgogICAgd2hlbjoKICAgICAgYnJhbmNoOgogICAgICAgIGV4Y2x1ZGU6IGdoLXBhZ2VzCgogIGRvY3VtZW50YXRpb246CiAgICBpbWFnZTogbGFtcGVwZmwvZG90dHk6bGF0ZXN0CiAgICBwdWxsOiB0cnVlCiAgICBjb21tYW5kczoKICAgICAgLSAuL3Byb2plY3Qvc2NyaXB0cy9nZW5Eb2NzICIke1RFU1R9IiAkQk9UX1BBU1MKICAgIHdoZW46CiAgICAgIGJyYW5jaDogbWFzdGVyCgogIGdpdHRlcjoKICAgIGltYWdlOiBwbHVnaW5zL2dpdHRlcgogICAgd2hlbjoKICAgICAgYnJhbmNoOiBtYXN0ZXIKICAgICAgc3RhdHVzOiBjaGFuZ2VkCgogIHNsYWNrOgogICAgaW1hZ2U6IHBsdWdpbnMvc2xhY2sKICAgIGNoYW5uZWw6IGRvdHR5CiAgICB3aGVuOgogICAgICBicmFuY2g6IG1hc3RlcgogICAgICBzdGF0dXM6IGNoYW5nZWQKCm1hdHJpeDoKICBURVNUOgogICAgLSA7dGVzdDtkb3R0eS1iaW4tdGVzdHMvdGVzdAogICAgLSA7cHVibGlzaExvY2FsO2RvdHR5LWJvb3RzdHJhcHBlZC90ZXN0CiAgICAtIHBhcnRlc3Qtb25seS1uby1ib290c3RyYXAgLS1zaG93LWRpZmYgLS12ZXJib3NlCiAgICAtIHBhcnRlc3Qtb25seSAtLXNob3ctZGlmZiAtLXZlcmJvc2UKICAgIC0gO3NldCB0ZXN0T3B0aW9ucyBpbiBMb2NhbFByb2plY3QoImRvdHR5LWNvbXBpbGVyIikgOj0gU2VxKCkgO2RvdHR5LWNvbXBpbGVyL3Rlc3RPbmx5IGRvdHR5LnRvb2xzLmRvdGMuQ29tcGlsYXRpb25UZXN0cwogICAgLSA7cHVibGlzaExvY2FsIDtzZXQgdGVzdE9wdGlvbnMgaW4gTG9jYWxQcm9qZWN0KCJkb3R0eS1jb21waWxlci1ib290c3RyYXBwZWQiKSA6PSBTZXEoKSA7ZG90dHktYm9vdHN0cmFwcGVkL3Rlc3RPbmx5IGRvdHR5LnRvb2xzLmRvdGMuQ29tcGlsYXRpb25UZXN0cwo.qsDrUBsZtyXeEeRXf9CnC0Rh5FF0lZpKCgf2iZvPckE
+eyJhbGciOiJIUzI1NiJ9.cGlwZWxpbmU6CiAgdGVzdDoKICAgIGltYWdlOiBsYW1wZXBmbC9kb3R0eTpsYXRlc3QKICAgIHB1bGw6IHRydWUKICAgIGNvbW1hbmRzOgogICAgICAtIGxuIC1zIC92YXIvY2FjaGUvZHJvbmUvc2NhbGEtc2NhbGEgc2NhbGEtc2NhbGEKICAgICAgLSBsbiAtcyAvdmFyL2NhY2hlL2Ryb25lL2l2eTIgIiRIT01FLy5pdnkyIgogICAgICAtIC4vc2NyaXB0cy91cGRhdGUtc2NhbGEtbGlicmFyeQogICAgICAtIHNidCAtSi1YbXgyNEcgLUotWFg6UmVzZXJ2ZWRDb2RlQ2FjaGVTaXplPTUxMm0gLUotWFg6TWF4TWV0YXNwYWNlU2l6ZT00RyAtRGRvdHR5LmRyb25lLm1lbT00MDk2bSAiJHtURVNUfSIKICAgIHdoZW46CiAgICAgIGJyYW5jaDoKICAgICAgICBleGNsdWRlOiBnaC1wYWdlcwoKICBkb2N1bWVudGF0aW9uOgogICAgaW1hZ2U6IGxhbXBlcGZsL2RvdHR5OmxhdGVzdAogICAgcHVsbDogdHJ1ZQogICAgY29tbWFuZHM6CiAgICAgIC0gLi9wcm9qZWN0L3NjcmlwdHMvZ2VuRG9jcyAiJHtURVNUfSIgJEJPVF9QQVNTCiAgICB3aGVuOgogICAgICBicmFuY2g6IG1hc3RlcgoKICBnaXR0ZXI6CiAgICBpbWFnZTogcGx1Z2lucy9naXR0ZXIKICAgIHdoZW46CiAgICAgIGJyYW5jaDogbWFzdGVyCiAgICAgIHN0YXR1czogY2hhbmdlZAoKICBzbGFjazoKICAgIGltYWdlOiBwbHVnaW5zL3NsYWNrCiAgICBjaGFubmVsOiBkb3R0eQogICAgd2hlbjoKICAgICAgYnJhbmNoOiBtYXN0ZXIKICAgICAgc3RhdHVzOiBjaGFuZ2VkCgptYXRyaXg6CiAgVEVTVDoKICAgIC0gO3NldCB0ZXN0T3B0aW9ucyBpbiBMb2NhbFByb2plY3QoImRvdHR5LWNvbXBpbGVyIikgKz0gVGVzdHMuQXJndW1lbnQoVGVzdEZyYW1ld29ya3MuSlVuaXQsICItLWV4Y2x1ZGUtY2F0ZWdvcmllcz1kb3R0eS50b29scy5kb3RjLlBhcmFsbGVsVGVzdGluZyIpIDt0ZXN0IDtkb3R0eS1iaW4tdGVzdHMvdGVzdAogICAgLSA7c2V0IHRlc3RPcHRpb25zIGluIExvY2FsUHJvamVjdCgiZG90dHktY29tcGlsZXItYm9vdHN0cmFwcGVkIikgKz0gVGVzdHMuQXJndW1lbnQoVGVzdEZyYW1ld29ya3MuSlVuaXQsICItLWV4Y2x1ZGUtY2F0ZWdvcmllcz1kb3R0eS50b29scy5kb3RjLlBhcmFsbGVsVGVzdGluZyIpIDtwdWJsaXNoTG9jYWwgO2RvdHR5LWJvb3RzdHJhcHBlZC90ZXN0CiAgICAtIDtzZXQgdGVzdE9wdGlvbnMgaW4gTG9jYWxQcm9qZWN0KCJkb3R0eS1jb21waWxlciIpICs9IFRlc3RzLkFyZ3VtZW50KFRlc3RGcmFtZXdvcmtzLkpVbml0LCAiLS1leGNsdWRlLWNhdGVnb3JpZXM9ZG90dHkudG9vbHMuZG90Yy5QYXJhbGxlbFRlc3RpbmciKSA7cGFydGVzdC1vbmx5LW5vLWJvb3RzdHJhcCAtLXNob3ctZGlmZiAtLXZlcmJvc2UKICAgIC0gO3NldCB0ZXN0T3B0aW9ucyBpbiBMb2NhbFByb2plY3QoImRvdHR5LWNvbXBpbGVyLWJvb3RzdHJhcHBlZCIpICs9IFRlc3RzLkFyZ3VtZW50KFRlc3RGcmFtZXdvcmtzLkpVbml0LCAiLS1leGNsdWRlLWNhdGVnb3JpZXM9ZG90dHkudG9vbHMuZG90Yy5QYXJhbGxlbFRlc3RpbmciKSA7cGFydGVzdC1vbmx5IC0tc2hvdy1kaWZmIC0tdmVyYm9zZQogICAgLSA7ZG90dHktY29tcGlsZXIvdGVzdE9ubHkgZG90dHkudG9vbHMuZG90Yy5Db21waWxhdGlvblRlc3RzCiAgICAtIDtwdWJsaXNoTG9jYWwgO2RvdHR5LWJvb3RzdHJhcHBlZC90ZXN0T25seSBkb3R0eS50b29scy5kb3RjLkNvbXBpbGF0aW9uVGVzdHMK.IAe3X-XmMyf3-GnA1ENr-D3t_YM_28CAhHX6lP0EB6E

compiler/test/dotc/comptest.scala

@@ -5,6 +5,7 @@ import dotty.tools.dotc.ParallelTesting
 object comptest extends ParallelTesting {
 
   def interactive: Boolean = true
+  def regex: Option[String] = None
 
   implicit val defaultOutputDir: String = "."
 

compiler/test/dotty/tools/dotc/CompilationTests.scala

@@ -7,13 +7,14 @@ import java.io.{ File => JFile }
 
 import org.junit.experimental.categories.Category
 
-
 @Category(Array(classOf[ParallelTesting]))
 class CompilationTests extends ParallelTesting {
   import CompilationTests._
 
   def interactive: Boolean = !sys.env.contains("DRONE")
 
+  def regex: Option[String] = sys.props.get("dotty.partest.filter")
+
   // Positive tests ------------------------------------------------------------
 
   @Test def compilePos: Unit = {

compiler/test/dotty/tools/dotc/ParallelTestTests.scala

@@ -12,6 +12,8 @@ class ParallelTestTests extends ParallelTesting {
 
   def interactive: Boolean = !sys.env.contains("DRONE")
 
+  def regex: Option[String] = None
+
   @Test def missingFile: Unit =
     try {
       compileFile("../tests/partest-test/i-dont-exist.scala", defaultOptions).expectFailure.neg()

compiler/test/dotty/tools/dotc/ParallelTesting.scala

@@ -14,6 +14,7 @@ import scala.io.Source
 import scala.util.control.NonFatal
 import scala.util.Try
 import scala.collection.mutable
+import scala.util.matching.Regex
 
 import core.Contexts._
 import reporting.{ Reporter, TestReporter }
@@ -25,6 +26,10 @@ trait ParallelTesting {
 
   def interactive: Boolean
 
+  def regex: Option[String]
+
+  private lazy val filter: Option[Regex] = regex.map(str => new Regex(str))
+
   private sealed trait Target { self =>
     def outDir: JFile
     def flags: Array[String]
@@ -121,7 +126,16 @@ trait ParallelTesting {
     /** Actual compilation run logic, the test behaviour is defined here */
     protected def compilationRunnable(target: Target): Runnable
 
-    val totalTargets = targets.length
+    private val allTargets =
+      if (!filter.isDefined) targets
+      else targets.filter {
+        case ConcurrentCompilationTarget(files, _, _) =>
+          files.exists(file => filter.get.findFirstIn(file.getAbsolutePath).isDefined)
+        case SeparateCompilationTarget(dir, _, _) =>
+          filter.get.findFirstIn(dir.getAbsolutePath).isDefined
+      }
+
+    val totalTargets = allTargets.length
 
     private[this] var _errors =  0
     def errors: Int = synchronized { _errors }
@@ -271,31 +285,39 @@ trait ParallelTesting {
 
     private[ParallelTesting] def execute(): this.type = {
       assert(_targetsCompiled == 0, "not allowed to re-use a `CompileRun`")
-      val pool = threadLimit match {
-        case Some(i) => JExecutors.newWorkStealingPool(i)
-        case None => JExecutors.newWorkStealingPool()
-      }
 
-      if (interactive && !suppressAllOutput) pool.submit(statusRunner)
+      if (allTargets.nonEmpty) {
+        val pool = threadLimit match {
+          case Some(i) => JExecutors.newWorkStealingPool(i)
+          case None => JExecutors.newWorkStealingPool()
+        }
 
-      targets.foreach { target =>
-        pool.submit(compilationRunnable(target))
-      }
+        if (interactive && !suppressAllOutput) pool.submit(statusRunner)
 
-      pool.shutdown()
-      if (!pool.awaitTermination(10, TimeUnit.MINUTES))
-        throw new TimeoutException("Compiling targets timed out")
-
-      if (didFail) {
-        echo {
-          """|
-             |================================================================================
-             |Test Report
-             |================================================================================
-             |Failing tests:""".stripMargin
+        allTargets.foreach { target =>
+          pool.submit(compilationRunnable(target))
         }
-        failedCompilationTargets.toArray.sorted.foreach(echo)
-        failureInstructions.iterator.foreach(echo)
+
+        pool.shutdown()
+        if (!pool.awaitTermination(10, TimeUnit.MINUTES))
+          throw new TimeoutException("Compiling targets timed out")
+
+        if (didFail) {
+          echo {
+            """|
+               |================================================================================
+               |Test Report
+               |================================================================================
+               |Failing tests:""".stripMargin
+          }
+          failedCompilationTargets.toArray.sorted.foreach(echo)
+          failureInstructions.iterator.foreach(echo)
+        }
+      }
+      else echo {
+        regex
+          .map(r => s"""No files matched regex "$r" in test""")
+          .getOrElse("No tests available under target - erroneous test?")
       }
 
       this

project/Build.scala

@@ -45,6 +45,9 @@ object Build {
   // Spawns a repl with the correct classpath
   lazy val repl = inputKey[Unit]("run the REPL with correct classpath")
 
+  // Run tests with filter
+  lazy val filterTest = inputKey[Unit]("runs integration test with the supplied filter")
+
   // Used to compile files similar to ./bin/dotc script
   lazy val dotc =
     inputKey[Unit]("run the compiler using the correct classpath, or the user supplied classpath")
@@ -286,7 +289,6 @@ object Build {
       libraryDependencies += "org.scala-lang.modules" %% "scala-partest" % "1.0.11" % "test",
       testOptions in Test += Tests.Cleanup({ () => partestLockFile.delete }),
       // this option is needed so that partest doesn't run
-      testOptions += Tests.Argument(TestFrameworks.JUnit, "--exclude-categories=dotty.tools.dotc.ParallelTesting"),
       partestDeps := Seq(
         scalaCompiler,
         "org.scala-lang" % "scala-reflect" % scalacVersion,
@@ -336,6 +338,14 @@ object Build {
         )
       }.evaluated,
 
+      filterTest := Def.inputTaskDyn {
+        val args: Seq[String] = spaceDelimited("<arg>").parsed
+        testOptions := Seq()
+        (testOnly in Test).toTask(
+          " dotty.tools.dotc.CompilationTests -- -Ddotty.partest.filter=" + args.head
+        )
+      }.evaluated,
+
       // Override run to be able to run compiled classfiles
       dotr := {
         val args: Seq[String] = spaceDelimited("<arg>").parsed