Add an option to only rerun tests that have failed (#16263)

prolativ · web-flow · commit bf808b380859 · 2022-11-03T09:57:03.000+01:00
- testCompilation supports running only failed tests with --failed argument fixes #3798
diff --git a/compiler/test/dotc/comptest.scala b/compiler/test/dotc/comptest.scala
@@ -12,6 +12,7 @@ object comptest extends ParallelTesting {
   def isInteractive = true
   def testFilter = Nil
   def updateCheckFiles: Boolean = false
+  def failedTests = None
 
   val posDir = "./tests/pos/"
   val negDir = "./tests/neg/"
diff --git a/compiler/test/dotty/Properties.scala b/compiler/test/dotty/Properties.scala
@@ -13,6 +13,10 @@ object Properties {
     prop == null || prop == "TRUE"
   }
 
+  /** If property is unset or FALSE we consider it `false` */
+  private def propIsTrue(name: String): Boolean =
+    sys.props.getOrElse(name, "FALSE") == "TRUE"
+
   /** Are we running on the CI? */
   val isRunByCI: Boolean = sys.env.isDefinedAt("DOTTY_CI_RUN")
   || sys.env.isDefinedAt("DRONE")  // TODO remove this when we drop Drone
@@ -30,9 +34,11 @@ object Properties {
    */
   val testsFilter: List[String] = sys.props.get("dotty.tests.filter").fold(Nil)(_.split(',').toList)
 
+  /** Run only failed tests */
+  val rerunFailed: Boolean = propIsTrue("dotty.tests.rerunFailed")
+
   /** Tests should override the checkfiles with the current output */
-  val testsUpdateCheckfile: Boolean =
-    sys.props.getOrElse("dotty.tests.updateCheckfiles", "FALSE") == "TRUE"
+  val testsUpdateCheckfile: Boolean = propIsTrue("dotty.tests.updateCheckfiles")
 
   /** When set, the run tests are only compiled - not run, a warning will be
    *  issued
diff --git a/compiler/test/dotty/tools/dotc/BootstrappedOnlyCompilationTests.scala b/compiler/test/dotty/tools/dotc/BootstrappedOnlyCompilationTests.scala
@@ -10,6 +10,7 @@ import org.junit.Assume._
 import org.junit.experimental.categories.Category
 
 import scala.concurrent.duration._
+import reporting.TestReporter
 import vulpix._
 
 import java.nio.file._
@@ -214,6 +215,7 @@ object BootstrappedOnlyCompilationTests extends ParallelTesting {
   def isInteractive = SummaryReport.isInteractive
   def testFilter = Properties.testsFilter
   def updateCheckFiles: Boolean = Properties.testsUpdateCheckfile
+  def failedTests = TestReporter.lastRunFailedTests
 
   implicit val summaryReport: SummaryReporting = new SummaryReport
   @AfterClass def tearDown(): Unit = {
diff --git a/compiler/test/dotty/tools/dotc/CompilationTests.scala b/compiler/test/dotty/tools/dotc/CompilationTests.scala
@@ -16,6 +16,7 @@ import scala.jdk.CollectionConverters._
 import scala.util.matching.Regex
 import scala.concurrent.duration._
 import TestSources.sources
+import reporting.TestReporter
 import vulpix._
 
 class CompilationTests {
@@ -317,6 +318,7 @@ object CompilationTests extends ParallelTesting {
   def isInteractive = SummaryReport.isInteractive
   def testFilter = Properties.testsFilter
   def updateCheckFiles: Boolean = Properties.testsUpdateCheckfile
+  def failedTests = TestReporter.lastRunFailedTests
 
   implicit val summaryReport: SummaryReporting = new SummaryReport
   @AfterClass def tearDown(): Unit = {
diff --git a/compiler/test/dotty/tools/dotc/FromTastyTests.scala b/compiler/test/dotty/tools/dotc/FromTastyTests.scala
@@ -5,6 +5,7 @@ package dotc
 import scala.language.unsafeNulls
 
 import org.junit.{AfterClass, Test}
+import reporting.TestReporter
 import vulpix._
 
 import java.io.{File => JFile}
@@ -48,6 +49,7 @@ object FromTastyTests extends ParallelTesting {
   def isInteractive = SummaryReport.isInteractive
   def testFilter = Properties.testsFilter
   def updateCheckFiles: Boolean = Properties.testsUpdateCheckfile
+  def failedTests = TestReporter.lastRunFailedTests
 
   implicit val summaryReport: SummaryReporting = new SummaryReport
   @AfterClass def tearDown(): Unit = {
diff --git a/compiler/test/dotty/tools/dotc/IdempotencyTests.scala b/compiler/test/dotty/tools/dotc/IdempotencyTests.scala
@@ -12,6 +12,7 @@ import org.junit.{AfterClass, Test}
 import org.junit.experimental.categories.Category
 
 import scala.concurrent.duration._
+import reporting.TestReporter
 import vulpix._
 
 
@@ -76,6 +77,7 @@ object IdempotencyTests extends ParallelTesting {
   def isInteractive = SummaryReport.isInteractive
   def testFilter = Properties.testsFilter
   def updateCheckFiles: Boolean = Properties.testsUpdateCheckfile
+  def failedTests = TestReporter.lastRunFailedTests
 
   implicit val summaryReport: SummaryReporting = new SummaryReport
   @AfterClass def tearDown(): Unit = {
diff --git a/compiler/test/dotty/tools/dotc/TastyBootstrapTests.scala b/compiler/test/dotty/tools/dotc/TastyBootstrapTests.scala
@@ -17,6 +17,7 @@ import scala.util.matching.Regex
 import scala.concurrent.duration._
 import TestSources.sources
 import vulpix._
+import reporting.TestReporter
 
 class TastyBootstrapTests {
   import ParallelTesting._
@@ -114,6 +115,7 @@ object TastyBootstrapTests extends ParallelTesting {
   def isInteractive = SummaryReport.isInteractive
   def testFilter = Properties.testsFilter
   def updateCheckFiles: Boolean = Properties.testsUpdateCheckfile
+  def failedTests = TestReporter.lastRunFailedTests
 
   implicit val summaryReport: SummaryReporting = new SummaryReport
   @AfterClass def tearDown(): Unit = {
diff --git a/compiler/test/dotty/tools/dotc/coverage/CoverageTests.scala b/compiler/test/dotty/tools/dotc/coverage/CoverageTests.scala
@@ -4,13 +4,13 @@ import org.junit.Test
 import org.junit.AfterClass
 import org.junit.Assert.*
 import org.junit.experimental.categories.Category
-
 import dotty.{BootstrappedOnlyTests, Properties}
 import dotty.tools.vulpix.*
 import dotty.tools.vulpix.TestConfiguration.*
 import dotty.tools.dotc.Main
+import dotty.tools.dotc.reporting.TestReporter
 
-import java.nio.file.{Files, FileSystems, Path, Paths, StandardCopyOption}
+import java.nio.file.{FileSystems, Files, Path, Paths, StandardCopyOption}
 import scala.jdk.CollectionConverters.*
 import scala.util.Properties.userDir
 import scala.language.unsafeNulls
@@ -85,6 +85,7 @@ object CoverageTests extends ParallelTesting:
   def testFilter = Properties.testsFilter
   def isInteractive = SummaryReport.isInteractive
   def updateCheckFiles = Properties.testsUpdateCheckfile
+  def failedTests = TestReporter.lastRunFailedTests
 
   given summaryReport: SummaryReporting = SummaryReport()
   @AfterClass def tearDown(): Unit =
diff --git a/compiler/test/dotty/tools/dotc/reporting/TestReporter.scala b/compiler/test/dotty/tools/dotc/reporting/TestReporter.scala
@@ -3,18 +3,20 @@ package dotc
 package reporting
 
 import scala.language.unsafeNulls
-
-import java.io.{ PrintStream, PrintWriter, File => JFile, FileOutputStream, StringWriter }
+import java.io.{BufferedReader, FileInputStream, FileOutputStream, FileReader, PrintStream, PrintWriter, StringReader, StringWriter, File as JFile}
 import java.text.SimpleDateFormat
 import java.util.Date
-import core.Decorators._
+import core.Decorators.*
 
 import scala.collection.mutable
-
+import scala.jdk.CollectionConverters.*
 import util.SourcePosition
-import core.Contexts._
-import Diagnostic._
-import interfaces.Diagnostic.{ ERROR, WARNING }
+import core.Contexts.*
+import Diagnostic.*
+import dotty.Properties
+import interfaces.Diagnostic.{ERROR, WARNING}
+
+import scala.io.Codec
 
 class TestReporter protected (outWriter: PrintWriter, filePrintln: String => Unit, logLevel: Int)
 extends Reporter with UniqueMessagePositions with HideNonSensicalMessages with MessageRendering {
@@ -84,17 +86,23 @@ extends Reporter with UniqueMessagePositions with HideNonSensicalMessages with M
 }
 
 object TestReporter {
+  private val testLogsDirName: String = "testlogs"
+  private val failedTestsFileName: String = "last-failed.log"
+  private val failedTestsFile: JFile = new JFile(s"$testLogsDirName/$failedTestsFileName")
+
   private var outFile: JFile = _
   private var logWriter: PrintWriter = _
+  private var failedTestsWriter: PrintWriter = _
 
   private def initLog() = if (logWriter eq null) {
     val date = new Date
     val df0 = new SimpleDateFormat("yyyy-MM-dd")
     val df1 = new SimpleDateFormat("yyyy-MM-dd-'T'HH-mm-ss")
-    val folder = s"testlogs/tests-${df0.format(date)}"
+    val folder = s"$testLogsDirName/tests-${df0.format(date)}"
     new JFile(folder).mkdirs()
     outFile = new JFile(s"$folder/tests-${df1.format(date)}.log")
     logWriter = new PrintWriter(new FileOutputStream(outFile, true))
+    failedTestsWriter = new PrintWriter(new FileOutputStream(failedTestsFile, false))
   }
 
   def logPrintln(str: String) = {
@@ -144,4 +152,16 @@ object TestReporter {
     }
     rep
   }
+
+  def lastRunFailedTests: Option[List[String]] =
+    Option.when(
+      Properties.rerunFailed &&
+        failedTestsFile.exists() &&
+        failedTestsFile.isFile
+    )(java.nio.file.Files.readAllLines(failedTestsFile.toPath).asScala.toList)
+
+  def writeFailedTests(tests: List[String]): Unit =
+    initLog()
+    tests.foreach(failed => failedTestsWriter.println(failed))
+    failedTestsWriter.flush()
 }
diff --git a/compiler/test/dotty/tools/vulpix/FailedTestInfo.scala b/compiler/test/dotty/tools/vulpix/FailedTestInfo.scala
@@ -0,0 +1,3 @@
+package dotty.tools.vulpix
+
+case class FailedTestInfo(title: String, extra: String)
diff --git a/compiler/test/dotty/tools/vulpix/ParallelTesting.scala b/compiler/test/dotty/tools/vulpix/ParallelTesting.scala
@@ -57,6 +57,9 @@ trait ParallelTesting extends RunnerOrchestration { self =>
   /** Tests should override the checkfiles with the current output */
   def updateCheckFiles: Boolean
 
+  /** Contains a list of failed tests to run, if list is empty no tests will run */
+  def failedTests: Option[List[String]]
+
   /** A test source whose files or directory of files is to be compiled
    *  in a specific way defined by the `Test`
    */
@@ -204,6 +207,14 @@ trait ParallelTesting extends RunnerOrchestration { self =>
 
   protected def shouldSkipTestSource(testSource: TestSource): Boolean = false
 
+  protected def shouldReRun(testSource: TestSource): Boolean =
+    failedTests.forall(rerun => testSource match {
+      case JointCompilationSource(_, files, _, _, _, _) =>
+        rerun.exists(filter => files.exists(file => file.getPath.contains(filter)))
+      case SeparateCompilationSource(_, dir, _, _) =>
+        rerun.exists(dir.getPath.contains)
+    })
+
   private trait CompilationLogic { this: Test =>
     def suppressErrors = false
 
@@ -359,7 +370,7 @@ trait ParallelTesting extends RunnerOrchestration { self =>
           case SeparateCompilationSource(_, dir, _, _) =>
             testFilter.exists(dir.getPath.contains)
         }
-      filteredByName.filterNot(shouldSkipTestSource(_))
+      filteredByName.filterNot(shouldSkipTestSource(_)).filter(shouldReRun(_))
 
     /** Total amount of test sources being compiled by this test */
     val sourceCount = filteredSources.length
@@ -409,14 +420,14 @@ trait ParallelTesting extends RunnerOrchestration { self =>
       synchronized { reproduceInstructions.append(ins) }
 
     /** The test sources that failed according to the implementing subclass */
-    private val failedTestSources = mutable.ArrayBuffer.empty[String]
+    private val failedTestSources = mutable.ArrayBuffer.empty[FailedTestInfo]
     protected final def failTestSource(testSource: TestSource, reason: Failure = Generic) = synchronized {
       val extra = reason match {
         case TimeoutFailure(title) => s", test '$title' timed out"
         case JavaCompilationFailure(msg) => s", java test sources failed to compile with: \n$msg"
         case Generic => ""
       }
-      failedTestSources.append(testSource.title + s" failed" + extra)
+      failedTestSources.append(FailedTestInfo(testSource.title, s" failed" + extra))
       fail(reason)
     }
 
diff --git a/compiler/test/dotty/tools/vulpix/SummaryReport.scala b/compiler/test/dotty/tools/vulpix/SummaryReport.scala
@@ -3,7 +3,6 @@ package tools
 package vulpix
 
 import scala.language.unsafeNulls
-
 import scala.collection.mutable
 import dotc.reporting.TestReporter
 
@@ -23,7 +22,7 @@ trait SummaryReporting {
   def reportPassed(): Unit
 
   /** Add the name of the failed test */
-  def addFailedTest(msg: String): Unit
+  def addFailedTest(msg: FailedTestInfo): Unit
 
   /** Add instructions to reproduce the error */
   def addReproduceInstruction(instr: String): Unit
@@ -49,7 +48,7 @@ trait SummaryReporting {
 final class NoSummaryReport extends SummaryReporting {
   def reportFailed(): Unit = ()
   def reportPassed(): Unit = ()
-  def addFailedTest(msg: String): Unit = ()
+  def addFailedTest(msg: FailedTestInfo): Unit = ()
   def addReproduceInstruction(instr: String): Unit = ()
   def addStartingMessage(msg: String): Unit = ()
   def addCleanup(f: () => Unit): Unit = ()
@@ -66,7 +65,7 @@ final class SummaryReport extends SummaryReporting {
   import scala.jdk.CollectionConverters._
 
   private val startingMessages = new java.util.concurrent.ConcurrentLinkedDeque[String]
-  private val failedTests = new java.util.concurrent.ConcurrentLinkedDeque[String]
+  private val failedTests = new java.util.concurrent.ConcurrentLinkedDeque[FailedTestInfo]
   private val reproduceInstructions = new java.util.concurrent.ConcurrentLinkedDeque[String]
   private val cleanUps = new java.util.concurrent.ConcurrentLinkedDeque[() => Unit]
 
@@ -79,7 +78,7 @@ final class SummaryReport extends SummaryReporting {
   def reportPassed(): Unit =
     passed += 1
 
-  def addFailedTest(msg: String): Unit =
+  def addFailedTest(msg: FailedTestInfo): Unit =
     failedTests.add(msg)
 
   def addReproduceInstruction(instr: String): Unit =
@@ -108,7 +107,8 @@ final class SummaryReport extends SummaryReporting {
 
     startingMessages.asScala.foreach(rep.append)
 
-    failedTests.asScala.map(x => s"    $x\n").foreach(rep.append)
+    failedTests.asScala.map(x => s"    ${x.title}${x.extra}\n").foreach(rep.append)
+    TestReporter.writeFailedTests(failedTests.asScala.toList.map(_.title))
 
     // If we're compiling locally, we don't need instructions on how to
     // reproduce failures
diff --git a/compiler/test/dotty/tools/vulpix/VulpixMetaTests.scala b/compiler/test/dotty/tools/vulpix/VulpixMetaTests.scala
@@ -30,6 +30,7 @@ object VulpixMetaTests extends ParallelTesting {
   def isInteractive = false // Don't beautify output for interactive use.
   def testFilter = Nil // Run all the tests.
   def updateCheckFiles: Boolean = false
+  def failedTests = None
 
   @AfterClass
   def tearDown() = this.cleanup()
diff --git a/compiler/test/dotty/tools/vulpix/VulpixUnitTests.scala b/compiler/test/dotty/tools/vulpix/VulpixUnitTests.scala
@@ -108,6 +108,7 @@ object VulpixUnitTests extends ParallelTesting {
   def isInteractive = !sys.env.contains("DRONE")
   def testFilter = Nil
   def updateCheckFiles: Boolean = false
+  def failedTests = None
 
   @AfterClass
   def tearDown() = this.cleanup()
diff --git a/project/Build.scala b/project/Build.scala
@@ -607,14 +607,15 @@ object Build {
         if (args.contains("--help")) {
           println(
             s"""
-               |usage: testCompilation [--help] [--from-tasty] [--update-checkfiles] [<filter>]
+               |usage: testCompilation [--help] [--from-tasty] [--update-checkfiles] [--failed] [<filter>]
                |
                |By default runs tests in dotty.tools.dotc.*CompilationTests and dotty.tools.dotc.coverage.*,
                |excluding tests tagged with dotty.SlowTests.
                |
                |  --help                show this message
                |  --from-tasty          runs tests in dotty.tools.dotc.FromTastyTests
                |  --update-checkfiles   override the checkfiles that did not match with the current output
+               |  --failed              re-run only failed tests
                |  <filter>              substring of the path of the tests file
                |
              """.stripMargin
@@ -623,11 +624,13 @@ object Build {
         }
         else {
           val updateCheckfile = args.contains("--update-checkfiles")
+          val rerunFailed = args.contains("--failed")
           val fromTasty = args.contains("--from-tasty")
-          val args1 = if (updateCheckfile | fromTasty) args.filter(x => x != "--update-checkfiles" && x != "--from-tasty") else args
+          val args1 = if (updateCheckfile | fromTasty | rerunFailed) args.filter(x => x != "--update-checkfiles" && x != "--from-tasty" && x != "--failed") else args
           val test = if (fromTasty) "dotty.tools.dotc.FromTastyTests" else "dotty.tools.dotc.*CompilationTests dotty.tools.dotc.coverage.*"
           val cmd = s" $test -- --exclude-categories=dotty.SlowTests" +
             (if (updateCheckfile) " -Ddotty.tests.updateCheckfiles=TRUE" else "") +
+            (if (rerunFailed) " -Ddotty.tests.rerunFailed=TRUE" else "") +
             (if (args1.nonEmpty) " -Ddotty.tests.filter=" + args1.mkString(" ") else "")
           (Test / testOnly).toTask(cmd)
         }
diff --git a/sjs-compiler-tests/test/scala/dotty/tools/dotc/ScalaJSCompilationTests.scala b/sjs-compiler-tests/test/scala/dotty/tools/dotc/ScalaJSCompilationTests.scala
@@ -6,6 +6,7 @@ import org.junit.{ Test, BeforeClass, AfterClass }
 import org.junit.experimental.categories.Category
 
 import scala.concurrent.duration._
+import reporting.TestReporter
 import vulpix._
 
 @Category(Array(classOf[ScalaJSCompilationTests]))
@@ -23,6 +24,7 @@ class ScalaJSCompilationTests extends ParallelTesting {
   def isInteractive = SummaryReport.isInteractive
   def testFilter = Properties.testsFilter
   def updateCheckFiles: Boolean = Properties.testsUpdateCheckfile
+  def failedTests = TestReporter.lastRunFailedTests
 
   // Negative tests ------------------------------------------------------------
 

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+package dotty.tools.vulpix`
	`2`	`+`
	`3`	`+case class FailedTestInfo(title: String, extra: String)`