diff --git a/compiler/test/dotc/comptest.scala b/compiler/test/dotc/comptest.scala index bd0d800e641c..fb53f561a94d 100644 --- a/compiler/test/dotc/comptest.scala +++ b/compiler/test/dotc/comptest.scala @@ -12,6 +12,7 @@ object comptest extends ParallelTesting { def isInteractive = true def testFilter = Nil def updateCheckFiles: Boolean = false + def failedTests = None val posDir = "./tests/pos/" val negDir = "./tests/neg/" diff --git a/compiler/test/dotty/Properties.scala b/compiler/test/dotty/Properties.scala index f4e0ed5f615f..71569f2f0e08 100644 --- a/compiler/test/dotty/Properties.scala +++ b/compiler/test/dotty/Properties.scala @@ -13,6 +13,10 @@ object Properties { prop == null || prop == "TRUE" } + /** If property is unset or FALSE we consider it `false` */ + private def propIsTrue(name: String): Boolean = + sys.props.getOrElse(name, "FALSE") == "TRUE" + /** Are we running on the CI? */ val isRunByCI: Boolean = sys.env.isDefinedAt("DOTTY_CI_RUN") || sys.env.isDefinedAt("DRONE") // TODO remove this when we drop Drone @@ -30,9 +34,11 @@ object Properties { */ val testsFilter: List[String] = sys.props.get("dotty.tests.filter").fold(Nil)(_.split(',').toList) + /** Run only failed tests */ + val rerunFailed: Boolean = propIsTrue("dotty.tests.rerunFailed") + /** Tests should override the checkfiles with the current output */ - val testsUpdateCheckfile: Boolean = - sys.props.getOrElse("dotty.tests.updateCheckfiles", "FALSE") == "TRUE" + val testsUpdateCheckfile: Boolean = propIsTrue("dotty.tests.updateCheckfiles") /** When set, the run tests are only compiled - not run, a warning will be * issued diff --git a/compiler/test/dotty/tools/dotc/BootstrappedOnlyCompilationTests.scala b/compiler/test/dotty/tools/dotc/BootstrappedOnlyCompilationTests.scala index cce23cb5c9a6..7ea05c615e0d 100644 --- a/compiler/test/dotty/tools/dotc/BootstrappedOnlyCompilationTests.scala +++ b/compiler/test/dotty/tools/dotc/BootstrappedOnlyCompilationTests.scala @@ -10,6 +10,7 @@ import org.junit.Assume._ import org.junit.experimental.categories.Category import scala.concurrent.duration._ +import reporting.TestReporter import vulpix._ import java.nio.file._ @@ -214,6 +215,7 @@ object BootstrappedOnlyCompilationTests extends ParallelTesting { def isInteractive = SummaryReport.isInteractive def testFilter = Properties.testsFilter def updateCheckFiles: Boolean = Properties.testsUpdateCheckfile + def failedTests = TestReporter.lastRunFailedTests implicit val summaryReport: SummaryReporting = new SummaryReport @AfterClass def tearDown(): Unit = { diff --git a/compiler/test/dotty/tools/dotc/CompilationTests.scala b/compiler/test/dotty/tools/dotc/CompilationTests.scala index 915e4e5f2e50..0ce2e514922c 100644 --- a/compiler/test/dotty/tools/dotc/CompilationTests.scala +++ b/compiler/test/dotty/tools/dotc/CompilationTests.scala @@ -16,6 +16,7 @@ import scala.jdk.CollectionConverters._ import scala.util.matching.Regex import scala.concurrent.duration._ import TestSources.sources +import reporting.TestReporter import vulpix._ class CompilationTests { @@ -313,6 +314,7 @@ object CompilationTests extends ParallelTesting { def isInteractive = SummaryReport.isInteractive def testFilter = Properties.testsFilter def updateCheckFiles: Boolean = Properties.testsUpdateCheckfile + def failedTests = TestReporter.lastRunFailedTests implicit val summaryReport: SummaryReporting = new SummaryReport @AfterClass def tearDown(): Unit = { diff --git a/compiler/test/dotty/tools/dotc/FromTastyTests.scala b/compiler/test/dotty/tools/dotc/FromTastyTests.scala index 2684a47b870c..1d46cbbce95c 100644 --- a/compiler/test/dotty/tools/dotc/FromTastyTests.scala +++ b/compiler/test/dotty/tools/dotc/FromTastyTests.scala @@ -5,6 +5,7 @@ package dotc import scala.language.unsafeNulls import org.junit.{AfterClass, Test} +import reporting.TestReporter import vulpix._ import java.io.{File => JFile} @@ -48,6 +49,7 @@ object FromTastyTests extends ParallelTesting { def isInteractive = SummaryReport.isInteractive def testFilter = Properties.testsFilter def updateCheckFiles: Boolean = Properties.testsUpdateCheckfile + def failedTests = TestReporter.lastRunFailedTests implicit val summaryReport: SummaryReporting = new SummaryReport @AfterClass def tearDown(): Unit = { diff --git a/compiler/test/dotty/tools/dotc/IdempotencyTests.scala b/compiler/test/dotty/tools/dotc/IdempotencyTests.scala index 84b3f1f8a48f..b515ebb05f96 100644 --- a/compiler/test/dotty/tools/dotc/IdempotencyTests.scala +++ b/compiler/test/dotty/tools/dotc/IdempotencyTests.scala @@ -12,6 +12,7 @@ import org.junit.{AfterClass, Test} import org.junit.experimental.categories.Category import scala.concurrent.duration._ +import reporting.TestReporter import vulpix._ @@ -76,6 +77,7 @@ object IdempotencyTests extends ParallelTesting { def isInteractive = SummaryReport.isInteractive def testFilter = Properties.testsFilter def updateCheckFiles: Boolean = Properties.testsUpdateCheckfile + def failedTests = TestReporter.lastRunFailedTests implicit val summaryReport: SummaryReporting = new SummaryReport @AfterClass def tearDown(): Unit = { diff --git a/compiler/test/dotty/tools/dotc/TastyBootstrapTests.scala b/compiler/test/dotty/tools/dotc/TastyBootstrapTests.scala index 9e71b10b206d..50e07f388dc4 100644 --- a/compiler/test/dotty/tools/dotc/TastyBootstrapTests.scala +++ b/compiler/test/dotty/tools/dotc/TastyBootstrapTests.scala @@ -17,6 +17,7 @@ import scala.util.matching.Regex import scala.concurrent.duration._ import TestSources.sources import vulpix._ +import reporting.TestReporter class TastyBootstrapTests { import ParallelTesting._ @@ -114,6 +115,7 @@ object TastyBootstrapTests extends ParallelTesting { def isInteractive = SummaryReport.isInteractive def testFilter = Properties.testsFilter def updateCheckFiles: Boolean = Properties.testsUpdateCheckfile + def failedTests = TestReporter.lastRunFailedTests implicit val summaryReport: SummaryReporting = new SummaryReport @AfterClass def tearDown(): Unit = { diff --git a/compiler/test/dotty/tools/dotc/coverage/CoverageTests.scala b/compiler/test/dotty/tools/dotc/coverage/CoverageTests.scala index 5d9458fe95c9..77e172f61167 100644 --- a/compiler/test/dotty/tools/dotc/coverage/CoverageTests.scala +++ b/compiler/test/dotty/tools/dotc/coverage/CoverageTests.scala @@ -4,13 +4,13 @@ import org.junit.Test import org.junit.AfterClass import org.junit.Assert.* import org.junit.experimental.categories.Category - import dotty.{BootstrappedOnlyTests, Properties} import dotty.tools.vulpix.* import dotty.tools.vulpix.TestConfiguration.* import dotty.tools.dotc.Main +import dotty.tools.dotc.reporting.TestReporter -import java.nio.file.{Files, FileSystems, Path, Paths, StandardCopyOption} +import java.nio.file.{FileSystems, Files, Path, Paths, StandardCopyOption} import scala.jdk.CollectionConverters.* import scala.util.Properties.userDir import scala.language.unsafeNulls @@ -85,6 +85,7 @@ object CoverageTests extends ParallelTesting: def testFilter = Properties.testsFilter def isInteractive = SummaryReport.isInteractive def updateCheckFiles = Properties.testsUpdateCheckfile + def failedTests = TestReporter.lastRunFailedTests given summaryReport: SummaryReporting = SummaryReport() @AfterClass def tearDown(): Unit = diff --git a/compiler/test/dotty/tools/dotc/reporting/TestReporter.scala b/compiler/test/dotty/tools/dotc/reporting/TestReporter.scala index 475cd1160296..940fc875a021 100644 --- a/compiler/test/dotty/tools/dotc/reporting/TestReporter.scala +++ b/compiler/test/dotty/tools/dotc/reporting/TestReporter.scala @@ -3,18 +3,20 @@ package dotc package reporting import scala.language.unsafeNulls - -import java.io.{ PrintStream, PrintWriter, File => JFile, FileOutputStream, StringWriter } +import java.io.{BufferedReader, FileInputStream, FileOutputStream, FileReader, PrintStream, PrintWriter, StringReader, StringWriter, File as JFile} import java.text.SimpleDateFormat import java.util.Date -import core.Decorators._ +import core.Decorators.* import scala.collection.mutable - +import scala.jdk.CollectionConverters.* import util.SourcePosition -import core.Contexts._ -import Diagnostic._ -import interfaces.Diagnostic.{ ERROR, WARNING } +import core.Contexts.* +import Diagnostic.* +import dotty.Properties +import interfaces.Diagnostic.{ERROR, WARNING} + +import scala.io.Codec class TestReporter protected (outWriter: PrintWriter, filePrintln: String => Unit, logLevel: Int) extends Reporter with UniqueMessagePositions with HideNonSensicalMessages with MessageRendering { @@ -84,17 +86,23 @@ extends Reporter with UniqueMessagePositions with HideNonSensicalMessages with M } object TestReporter { + private val testLogsDirName: String = "testlogs" + private val failedTestsFileName: String = "last-failed.log" + private val failedTestsFile: JFile = new JFile(s"$testLogsDirName/$failedTestsFileName") + private var outFile: JFile = _ private var logWriter: PrintWriter = _ + private var failedTestsWriter: PrintWriter = _ private def initLog() = if (logWriter eq null) { val date = new Date val df0 = new SimpleDateFormat("yyyy-MM-dd") val df1 = new SimpleDateFormat("yyyy-MM-dd-'T'HH-mm-ss") - val folder = s"testlogs/tests-${df0.format(date)}" + val folder = s"$testLogsDirName/tests-${df0.format(date)}" new JFile(folder).mkdirs() outFile = new JFile(s"$folder/tests-${df1.format(date)}.log") logWriter = new PrintWriter(new FileOutputStream(outFile, true)) + failedTestsWriter = new PrintWriter(new FileOutputStream(failedTestsFile, false)) } def logPrintln(str: String) = { @@ -144,4 +152,16 @@ object TestReporter { } rep } + + def lastRunFailedTests: Option[List[String]] = + Option.when( + Properties.rerunFailed && + failedTestsFile.exists() && + failedTestsFile.isFile + )(java.nio.file.Files.readAllLines(failedTestsFile.toPath).asScala.toList) + + def writeFailedTests(tests: List[String]): Unit = + initLog() + tests.foreach(failed => failedTestsWriter.println(failed)) + failedTestsWriter.flush() } diff --git a/compiler/test/dotty/tools/vulpix/FailedTestInfo.scala b/compiler/test/dotty/tools/vulpix/FailedTestInfo.scala new file mode 100644 index 000000000000..c7172f54aadc --- /dev/null +++ b/compiler/test/dotty/tools/vulpix/FailedTestInfo.scala @@ -0,0 +1,3 @@ +package dotty.tools.vulpix + +case class FailedTestInfo(title: String, extra: String) diff --git a/compiler/test/dotty/tools/vulpix/ParallelTesting.scala b/compiler/test/dotty/tools/vulpix/ParallelTesting.scala index 44565c44b681..b64142c0021f 100644 --- a/compiler/test/dotty/tools/vulpix/ParallelTesting.scala +++ b/compiler/test/dotty/tools/vulpix/ParallelTesting.scala @@ -57,6 +57,9 @@ trait ParallelTesting extends RunnerOrchestration { self => /** Tests should override the checkfiles with the current output */ def updateCheckFiles: Boolean + /** Contains a list of failed tests to run, if list is empty no tests will run */ + def failedTests: Option[List[String]] + /** A test source whose files or directory of files is to be compiled * in a specific way defined by the `Test` */ @@ -204,6 +207,14 @@ trait ParallelTesting extends RunnerOrchestration { self => protected def shouldSkipTestSource(testSource: TestSource): Boolean = false + protected def shouldReRun(testSource: TestSource): Boolean = + failedTests.forall(rerun => testSource match { + case JointCompilationSource(_, files, _, _, _, _) => + rerun.exists(filter => files.exists(file => file.getPath.contains(filter))) + case SeparateCompilationSource(_, dir, _, _) => + rerun.exists(dir.getPath.contains) + }) + private trait CompilationLogic { this: Test => def suppressErrors = false @@ -359,7 +370,7 @@ trait ParallelTesting extends RunnerOrchestration { self => case SeparateCompilationSource(_, dir, _, _) => testFilter.exists(dir.getPath.contains) } - filteredByName.filterNot(shouldSkipTestSource(_)) + filteredByName.filterNot(shouldSkipTestSource(_)).filter(shouldReRun(_)) /** Total amount of test sources being compiled by this test */ val sourceCount = filteredSources.length @@ -409,14 +420,14 @@ trait ParallelTesting extends RunnerOrchestration { self => synchronized { reproduceInstructions.append(ins) } /** The test sources that failed according to the implementing subclass */ - private val failedTestSources = mutable.ArrayBuffer.empty[String] + private val failedTestSources = mutable.ArrayBuffer.empty[FailedTestInfo] protected final def failTestSource(testSource: TestSource, reason: Failure = Generic) = synchronized { val extra = reason match { case TimeoutFailure(title) => s", test '$title' timed out" case JavaCompilationFailure(msg) => s", java test sources failed to compile with: \n$msg" case Generic => "" } - failedTestSources.append(testSource.title + s" failed" + extra) + failedTestSources.append(FailedTestInfo(testSource.title, s" failed" + extra)) fail(reason) } diff --git a/compiler/test/dotty/tools/vulpix/SummaryReport.scala b/compiler/test/dotty/tools/vulpix/SummaryReport.scala index e216ac1c5d4f..74612387015f 100644 --- a/compiler/test/dotty/tools/vulpix/SummaryReport.scala +++ b/compiler/test/dotty/tools/vulpix/SummaryReport.scala @@ -3,7 +3,6 @@ package tools package vulpix import scala.language.unsafeNulls - import scala.collection.mutable import dotc.reporting.TestReporter @@ -23,7 +22,7 @@ trait SummaryReporting { def reportPassed(): Unit /** Add the name of the failed test */ - def addFailedTest(msg: String): Unit + def addFailedTest(msg: FailedTestInfo): Unit /** Add instructions to reproduce the error */ def addReproduceInstruction(instr: String): Unit @@ -49,7 +48,7 @@ trait SummaryReporting { final class NoSummaryReport extends SummaryReporting { def reportFailed(): Unit = () def reportPassed(): Unit = () - def addFailedTest(msg: String): Unit = () + def addFailedTest(msg: FailedTestInfo): Unit = () def addReproduceInstruction(instr: String): Unit = () def addStartingMessage(msg: String): Unit = () def addCleanup(f: () => Unit): Unit = () @@ -66,7 +65,7 @@ final class SummaryReport extends SummaryReporting { import scala.jdk.CollectionConverters._ private val startingMessages = new java.util.concurrent.ConcurrentLinkedDeque[String] - private val failedTests = new java.util.concurrent.ConcurrentLinkedDeque[String] + private val failedTests = new java.util.concurrent.ConcurrentLinkedDeque[FailedTestInfo] private val reproduceInstructions = new java.util.concurrent.ConcurrentLinkedDeque[String] private val cleanUps = new java.util.concurrent.ConcurrentLinkedDeque[() => Unit] @@ -79,7 +78,7 @@ final class SummaryReport extends SummaryReporting { def reportPassed(): Unit = passed += 1 - def addFailedTest(msg: String): Unit = + def addFailedTest(msg: FailedTestInfo): Unit = failedTests.add(msg) def addReproduceInstruction(instr: String): Unit = @@ -108,7 +107,8 @@ final class SummaryReport extends SummaryReporting { startingMessages.asScala.foreach(rep.append) - failedTests.asScala.map(x => s" $x\n").foreach(rep.append) + failedTests.asScala.map(x => s" ${x.title}${x.extra}\n").foreach(rep.append) + TestReporter.writeFailedTests(failedTests.asScala.toList.map(_.title)) // If we're compiling locally, we don't need instructions on how to // reproduce failures diff --git a/compiler/test/dotty/tools/vulpix/VulpixMetaTests.scala b/compiler/test/dotty/tools/vulpix/VulpixMetaTests.scala index 75af0aa94893..0044ab8a94e5 100644 --- a/compiler/test/dotty/tools/vulpix/VulpixMetaTests.scala +++ b/compiler/test/dotty/tools/vulpix/VulpixMetaTests.scala @@ -30,6 +30,7 @@ object VulpixMetaTests extends ParallelTesting { def isInteractive = false // Don't beautify output for interactive use. def testFilter = Nil // Run all the tests. def updateCheckFiles: Boolean = false + def failedTests = None @AfterClass def tearDown() = this.cleanup() diff --git a/compiler/test/dotty/tools/vulpix/VulpixUnitTests.scala b/compiler/test/dotty/tools/vulpix/VulpixUnitTests.scala index 8a32fd636e76..baf61c845d96 100644 --- a/compiler/test/dotty/tools/vulpix/VulpixUnitTests.scala +++ b/compiler/test/dotty/tools/vulpix/VulpixUnitTests.scala @@ -108,6 +108,7 @@ object VulpixUnitTests extends ParallelTesting { def isInteractive = !sys.env.contains("DRONE") def testFilter = Nil def updateCheckFiles: Boolean = false + def failedTests = None @AfterClass def tearDown() = this.cleanup() diff --git a/project/Build.scala b/project/Build.scala index a8b9c1f2d749..c6fd9ea8f139 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -607,7 +607,7 @@ object Build { if (args.contains("--help")) { println( s""" - |usage: testCompilation [--help] [--from-tasty] [--update-checkfiles] [] + |usage: testCompilation [--help] [--from-tasty] [--update-checkfiles] [--failed] [] | |By default runs tests in dotty.tools.dotc.*CompilationTests and dotty.tools.dotc.coverage.*, |excluding tests tagged with dotty.SlowTests. @@ -615,6 +615,7 @@ object Build { | --help show this message | --from-tasty runs tests in dotty.tools.dotc.FromTastyTests | --update-checkfiles override the checkfiles that did not match with the current output + | --failed re-run only failed tests | substring of the path of the tests file | """.stripMargin @@ -623,11 +624,13 @@ object Build { } else { val updateCheckfile = args.contains("--update-checkfiles") + val rerunFailed = args.contains("--failed") val fromTasty = args.contains("--from-tasty") - val args1 = if (updateCheckfile | fromTasty) args.filter(x => x != "--update-checkfiles" && x != "--from-tasty") else args + val args1 = if (updateCheckfile | fromTasty | rerunFailed) args.filter(x => x != "--update-checkfiles" && x != "--from-tasty" && x != "--failed") else args val test = if (fromTasty) "dotty.tools.dotc.FromTastyTests" else "dotty.tools.dotc.*CompilationTests dotty.tools.dotc.coverage.*" val cmd = s" $test -- --exclude-categories=dotty.SlowTests" + (if (updateCheckfile) " -Ddotty.tests.updateCheckfiles=TRUE" else "") + + (if (rerunFailed) " -Ddotty.tests.rerunFailed=TRUE" else "") + (if (args1.nonEmpty) " -Ddotty.tests.filter=" + args1.mkString(" ") else "") (Test / testOnly).toTask(cmd) } diff --git a/sjs-compiler-tests/test/scala/dotty/tools/dotc/ScalaJSCompilationTests.scala b/sjs-compiler-tests/test/scala/dotty/tools/dotc/ScalaJSCompilationTests.scala index ca4f292568bb..0f4eb633b770 100644 --- a/sjs-compiler-tests/test/scala/dotty/tools/dotc/ScalaJSCompilationTests.scala +++ b/sjs-compiler-tests/test/scala/dotty/tools/dotc/ScalaJSCompilationTests.scala @@ -6,6 +6,7 @@ import org.junit.{ Test, BeforeClass, AfterClass } import org.junit.experimental.categories.Category import scala.concurrent.duration._ +import reporting.TestReporter import vulpix._ @Category(Array(classOf[ScalaJSCompilationTests])) @@ -23,6 +24,7 @@ class ScalaJSCompilationTests extends ParallelTesting { def isInteractive = SummaryReport.isInteractive def testFilter = Properties.testsFilter def updateCheckFiles: Boolean = Properties.testsUpdateCheckfile + def failedTests = TestReporter.lastRunFailedTests // Negative tests ------------------------------------------------------------