From f9db9097d0cf314d675669d80f8d5eb8d20f0710 Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Thu, 30 Mar 2017 17:33:06 +0200 Subject: [PATCH 1/4] Fix #2151: don't die for wrong number of typeargs applied --- compiler/src/dotty/tools/dotc/core/Decorators.scala | 2 +- tests/neg/i2151.scala | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 tests/neg/i2151.scala diff --git a/compiler/src/dotty/tools/dotc/core/Decorators.scala b/compiler/src/dotty/tools/dotc/core/Decorators.scala index f8267072e46d..0e8ae196a8a2 100644 --- a/compiler/src/dotty/tools/dotc/core/Decorators.scala +++ b/compiler/src/dotty/tools/dotc/core/Decorators.scala @@ -103,7 +103,7 @@ object Decorators { * as long as `xs`. */ def zipWithConserve[U](ys: List[U])(f: (T, U) => T): List[T] = - if (xs.isEmpty) xs + if (xs.isEmpty || ys.isEmpty) Nil else { val x1 = f(xs.head, ys.head) val xs1 = xs.tail.zipWithConserve(ys.tail)(f) diff --git a/tests/neg/i2151.scala b/tests/neg/i2151.scala new file mode 100644 index 000000000000..1ae034c02fab --- /dev/null +++ b/tests/neg/i2151.scala @@ -0,0 +1,6 @@ +trait Test { + type Nil = [K] => K + type StrangeCons[H, Tail <: [H, A] => H] = Tail[H, H] + + def list: StrangeCons[Int, Nil] // error +} From 6d7c23016ec866de0b29e9bdde00b60c99df90c2 Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Fri, 31 Mar 2017 11:47:47 +0200 Subject: [PATCH 2/4] Make DiffUtil's rendering readable in logs --- .../dotc/reporting/MessageRendering.scala | 2 +- .../src/dotty/tools/dotc/util/DiffUtil.scala | 19 ++++++++++++++++++- .../dotty/tools/dotc/ParallelTesting.scala | 2 +- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/reporting/MessageRendering.scala b/compiler/src/dotty/tools/dotc/reporting/MessageRendering.scala index 17eb8d39b416..91e65ab66d30 100644 --- a/compiler/src/dotty/tools/dotc/reporting/MessageRendering.scala +++ b/compiler/src/dotty/tools/dotc/reporting/MessageRendering.scala @@ -21,7 +21,7 @@ trait MessageRendering { * @return string stripped of ANSI escape codes */ def stripColor(str: String): String = - str.replaceAll("\u001B\\[[;\\d]*m", "") + str.replaceAll("\u001b\\[.*?m", "") /** When inlining a method call, if there's an error we'd like to get the * outer context and the `pos` at which the call was inlined. diff --git a/compiler/src/dotty/tools/dotc/util/DiffUtil.scala b/compiler/src/dotty/tools/dotc/util/DiffUtil.scala index b55aee719045..6f7df13a6f75 100644 --- a/compiler/src/dotty/tools/dotc/util/DiffUtil.scala +++ b/compiler/src/dotty/tools/dotc/util/DiffUtil.scala @@ -58,8 +58,25 @@ object DiffUtil { (fnd, exp, totalChange.toDouble / (expected.length + found.length)) } - def mkColoredCodeDiff(code: String, lastCode: String, printDiffDel: Boolean): String = { + def mkColoredLineDiff(expected: String, actual: String): String = { + val tokens = splitTokens(expected, Nil).toArray + val lastTokens = splitTokens(actual, Nil).toArray + + val diff = hirschberg(lastTokens, tokens) + " |SOF\n" + diff.collect { + case Unmodified(str) => + " |" + str + case Inserted(str) => + ADDITION_COLOR + "e |" + str + ANSI_DEFAULT + case Modified(old, str) => + DELETION_COLOR + "a |" + old + "\ne |" + ADDITION_COLOR + str + ANSI_DEFAULT + case Deleted(str) => + DELETION_COLOR + "\na |" + str + ANSI_DEFAULT + }.mkString + "\n |EOF" + } + + def mkColoredCodeDiff(code: String, lastCode: String, printDiffDel: Boolean): String = { val tokens = splitTokens(code, Nil).toArray val lastTokens = splitTokens(lastCode, Nil).toArray diff --git a/compiler/test/dotty/tools/dotc/ParallelTesting.scala b/compiler/test/dotty/tools/dotc/ParallelTesting.scala index 289351d81e63..8e87c117003b 100644 --- a/compiler/test/dotty/tools/dotc/ParallelTesting.scala +++ b/compiler/test/dotty/tools/dotc/ParallelTesting.scala @@ -456,7 +456,7 @@ trait ParallelTesting { self => if (outputLines.length != checkLines.length || !linesMatch) { // Print diff to files and summary: val diff = outputLines.zip(checkLines).map { case (act, exp) => - DiffUtil.mkColoredCodeDiff(exp, act, true) + DiffUtil.mkColoredLineDiff(exp, act) }.mkString("\n") val msg = s"\nOutput from run test '$checkFile' did not match expected, output:\n$diff\n" echo(msg) From 57a11776480ab5259281dd29777c45af9ea3ca78 Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Fri, 31 Mar 2017 11:49:43 +0200 Subject: [PATCH 3/4] Improve summary report by dumping all to stdout on CI --- .gitignore | 1 + .../dotty/tools/dotc/CompilationTests.scala | 2 +- .../tools/dotc/ParallelSummaryReport.java | 8 ++++++- .../dotty/tools/dotc/ParallelTesting.scala | 23 +++++++++++++++++-- .../tools/dotc/reporting/TestReporter.scala | 15 ++++++++---- 5 files changed, 41 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 9842e0c6bbaa..170cf4823dd9 100644 --- a/.gitignore +++ b/.gitignore @@ -44,6 +44,7 @@ scala-scala out/ build/ !out/.keep +testlogs/ # Ignore build-file .packages diff --git a/compiler/test/dotty/tools/dotc/CompilationTests.scala b/compiler/test/dotty/tools/dotc/CompilationTests.scala index 788e30aa38a9..742b93fae71a 100644 --- a/compiler/test/dotty/tools/dotc/CompilationTests.scala +++ b/compiler/test/dotty/tools/dotc/CompilationTests.scala @@ -12,7 +12,7 @@ import scala.util.matching.Regex class CompilationTests extends ParallelSummaryReport with ParallelTesting { import CompilationTests._ - def isInteractive: Boolean = !sys.env.contains("DRONE") + def isInteractive: Boolean = ParallelSummaryReport.isInteractive def testFilter: Option[Regex] = sys.props.get("dotty.partest.filter").map(r => new Regex(r)) diff --git a/compiler/test/dotty/tools/dotc/ParallelSummaryReport.java b/compiler/test/dotty/tools/dotc/ParallelSummaryReport.java index 9214e7d25f7e..d32cda974d17 100644 --- a/compiler/test/dotty/tools/dotc/ParallelSummaryReport.java +++ b/compiler/test/dotty/tools/dotc/ParallelSummaryReport.java @@ -11,6 +11,8 @@ * this class */ public class ParallelSummaryReport { + public final static boolean isInteractive = !System.getenv().containsKey("DRONE"); + private static TestReporter rep = TestReporter.reporter(-1); private static ArrayDeque failedTests = new ArrayDeque<>(); private static ArrayDeque reproduceInstructions = new ArrayDeque<>(); @@ -54,7 +56,8 @@ public final static void addReproduceInstruction(String msg) { .map(x -> " " + x) .forEach(rep::echo); - rep.flushToStdErr(); + // If we're compiling locally, we don't need reproduce instructions + if (isInteractive) rep.flushToStdErr(); rep.echo(""); @@ -62,6 +65,9 @@ public final static void addReproduceInstruction(String msg) { .stream() .forEach(rep::echo); + // If we're on the CI, we want everything + if (!isInteractive) rep.flushToStdErr(); + if (failed > 0) rep.flushToFile(); } } diff --git a/compiler/test/dotty/tools/dotc/ParallelTesting.scala b/compiler/test/dotty/tools/dotc/ParallelTesting.scala index 8e87c117003b..990a856caedc 100644 --- a/compiler/test/dotty/tools/dotc/ParallelTesting.scala +++ b/compiler/test/dotty/tools/dotc/ParallelTesting.scala @@ -51,6 +51,16 @@ trait ParallelTesting { self => def outDir: JFile def flags: Array[String] + + def title: String = self match { + case self: JointCompilationSource => + if (self.files.length > 1) name + else self.files.head.getPath + + case self: SeparateCompilationSource => + self.dir.getPath + } + /** Adds the flags specified in `newFlags0` if they do not already exist */ def withFlags(newFlags0: String*) = { val newFlags = newFlags0.toArray @@ -69,7 +79,11 @@ trait ParallelTesting { self => val maxLen = 80 var lineLen = 0 - sb.append(s"\n\nTest compiled with $errors error(s) and $warnings warning(s), the test can be reproduced by running:") + sb.append( + s"""| + |Test '$title' compiled with $errors error(s) and $warnings warning(s), + |the test can be reproduced by running:""".stripMargin + ) sb.append("\n\n./bin/dotc ") flags.foreach { arg => if (lineLen > maxLen) { @@ -447,6 +461,7 @@ trait ParallelTesting { self => private def verifyOutput(checkFile: JFile, dir: JFile, testSource: TestSource, warnings: Int) = { val outputLines = runMain(dir, testSource) val checkLines = Source.fromFile(checkFile).getLines.toArray + val sourceTitle = testSource.title def linesMatch = outputLines @@ -458,7 +473,11 @@ trait ParallelTesting { self => val diff = outputLines.zip(checkLines).map { case (act, exp) => DiffUtil.mkColoredLineDiff(exp, act) }.mkString("\n") - val msg = s"\nOutput from run test '$checkFile' did not match expected, output:\n$diff\n" + + val msg = + s"""|Output from '$sourceTitle' did not match check file. + |Diff ('e' is expected, 'a' is actual): + |""".stripMargin + diff + "\n" echo(msg) addFailureInstruction(msg) diff --git a/compiler/test/dotty/tools/dotc/reporting/TestReporter.scala b/compiler/test/dotty/tools/dotc/reporting/TestReporter.scala index 521cf95762f4..2d7e6c70a998 100644 --- a/compiler/test/dotty/tools/dotc/reporting/TestReporter.scala +++ b/compiler/test/dotty/tools/dotc/reporting/TestReporter.scala @@ -25,10 +25,16 @@ extends Reporter with UniqueMessagePositions with HideNonSensicalMessages with M protected final val _messageBuf = mutable.ArrayBuffer.empty[String] final def flushToFile(): Unit = - _messageBuf.iterator.foreach(filePrintln) + _messageBuf + .iterator + .map(_.replaceAll("\u001b\\[.*?m", "")) + .foreach(filePrintln) final def flushToStdErr(): Unit = - _messageBuf.iterator.foreach(System.err.println) + _messageBuf + .iterator + .map(_.replaceAll("\u001b\\[.*?m", "")) + .foreach(System.err.println) final def inlineInfo(pos: SourcePosition): String = if (pos.exists) { @@ -75,10 +81,11 @@ extends Reporter with UniqueMessagePositions with HideNonSensicalMessages with M } object TestReporter { - private[this] val logWriter = { + private[this] lazy val logWriter = { val df = new SimpleDateFormat("yyyy-MM-dd-HH:mm") val timestamp = df.format(new Date) - new PrintWriter(new FileOutputStream(new JFile(s"../tests-$timestamp.log"), true)) + new JFile("../testlogs").mkdirs() + new PrintWriter(new FileOutputStream(new JFile(s"../testlogs/tests-$timestamp.log"), true)) } def writeToLog(str: String) = { From ed10ef0208f794060b5351cc601a915177e0a1c5 Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Fri, 31 Mar 2017 11:50:07 +0200 Subject: [PATCH 4/4] Fix deadlock in `runAll` --- .../tools/dotc/ParallelSummaryReport.java | 4 +- .../dotty/tools/dotc/ParallelTesting.scala | 45 ++++++++++------ .../tools/dotc/reporting/TestReporter.scala | 51 +++++++------------ 3 files changed, 51 insertions(+), 49 deletions(-) diff --git a/compiler/test/dotty/tools/dotc/ParallelSummaryReport.java b/compiler/test/dotty/tools/dotc/ParallelSummaryReport.java index d32cda974d17..5608b36566ae 100644 --- a/compiler/test/dotty/tools/dotc/ParallelSummaryReport.java +++ b/compiler/test/dotty/tools/dotc/ParallelSummaryReport.java @@ -13,7 +13,7 @@ public class ParallelSummaryReport { public final static boolean isInteractive = !System.getenv().containsKey("DRONE"); - private static TestReporter rep = TestReporter.reporter(-1); + private static TestReporter rep = TestReporter.reporter(System.out, -1); private static ArrayDeque failedTests = new ArrayDeque<>(); private static ArrayDeque reproduceInstructions = new ArrayDeque<>(); private static int passed; @@ -36,7 +36,7 @@ public final static void addReproduceInstruction(String msg) { } @BeforeClass public final static void setup() { - rep = TestReporter.reporter(-1); + rep = TestReporter.reporter(System.out, -1); failedTests = new ArrayDeque<>(); reproduceInstructions = new ArrayDeque<>(); } diff --git a/compiler/test/dotty/tools/dotc/ParallelTesting.scala b/compiler/test/dotty/tools/dotc/ParallelTesting.scala index 990a856caedc..c1da7807b244 100644 --- a/compiler/test/dotty/tools/dotc/ParallelTesting.scala +++ b/compiler/test/dotty/tools/dotc/ParallelTesting.scala @@ -174,6 +174,8 @@ trait ParallelTesting { self => * according to the implementing class "neg", "run" or "pos". */ private abstract class Test(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean) { + protected final val realStdout = System.out + protected final val realStderr = System.err /** Actual compilation run logic, the test behaviour is defined here */ protected def compilationRunnable(testSource: TestSource): Runnable @@ -192,10 +194,10 @@ trait ParallelTesting { self => val sourceCount = filteredSources.length private[this] var _errorCount = 0 - def errorCount: Int = synchronized { _errorCount } + def errorCount: Int = _errorCount private[this] var _testSourcesCompiled = 0 - private def testSourcesCompiled : Int = synchronized { _testSourcesCompiled } + private def testSourcesCompiled: Int = _testSourcesCompiled /** Complete the current compilation with the amount of errors encountered */ protected final def registerCompilation(errors: Int) = synchronized { @@ -228,7 +230,7 @@ trait ParallelTesting { self => /** Prints to `System.err` if we're not suppressing all output */ protected def echo(msg: String): Unit = - if (!suppressAllOutput) System.err.println(msg) + if (!suppressAllOutput) realStderr.println(msg) /** A single `Runnable` that prints a progress bar for the curent `Test` */ private def createProgressMonitor: Runnable = new Runnable { @@ -238,17 +240,19 @@ trait ParallelTesting { self => while (tCompiled < sourceCount) { val timestamp = (System.currentTimeMillis - start) / 1000 val progress = (tCompiled.toDouble / sourceCount * 40).toInt - print( + + realStdout.print( "[" + ("=" * (math.max(progress - 1, 0))) + (if (progress > 0) ">" else "") + (" " * (39 - progress)) + s"] compiling ($tCompiled/$sourceCount, ${timestamp}s)\r" ) + Thread.sleep(100) tCompiled = testSourcesCompiled } // println, otherwise no newline and cursor at start of line - println( + realStdout.println( s"[=======================================] compiled ($sourceCount/$sourceCount, " + s"${(System.currentTimeMillis - start) / 1000}s) " ) @@ -259,7 +263,10 @@ trait ParallelTesting { self => * if it did, the test should automatically fail. */ protected def tryCompile(testSource: TestSource)(op: => Unit): Unit = - try op catch { + try { + if (!isInteractive) realStdout.println(s"Testing ${testSource.title}") + op + } catch { case NonFatal(e) => { // if an exception is thrown during compilation, the complete test // run should fail @@ -309,8 +316,10 @@ trait ParallelTesting { self => Runtime.getRuntime.exec(fullArgs).waitFor() == 0 } else true - val reporter = TestReporter.parallelReporter(this, logLevel = - if (suppressErrors || suppressAllOutput) ERROR + 1 else ERROR) + val reporter = + TestReporter.reporter(realStdout, logLevel = + if (suppressErrors || suppressAllOutput) ERROR + 1 else ERROR) + val driver = if (times == 1) new Driver { def newCompiler(implicit ctx: Context) = new Compiler } else new Driver { @@ -353,8 +362,12 @@ trait ParallelTesting { self => } pool.shutdown() - if (!pool.awaitTermination(10, TimeUnit.MINUTES)) + if (!pool.awaitTermination(20, TimeUnit.MINUTES)) { + pool.shutdownNow() + System.setOut(realStdout) + System.setErr(realStderr) throw new TimeoutException("Compiling targets timed out") + } if (didFail) { reportFailed() @@ -417,8 +430,6 @@ trait ParallelTesting { self => import java.net.{ URL, URLClassLoader } val printStream = new ByteArrayOutputStream - val oldOut = System.out - val oldErr = System.err try { // Do classloading magic and running here: @@ -426,7 +437,7 @@ trait ParallelTesting { self => val cls = ucl.loadClass("Test") val meth = cls.getMethod("main", classOf[Array[String]]) - self.synchronized { + synchronized { try { val ps = new PrintStream(printStream) System.setOut(ps) @@ -436,9 +447,13 @@ trait ParallelTesting { self => meth.invoke(null, Array("jvm")) // partest passes at least "jvm" as an arg } } - } finally { - System.setOut(oldOut) - System.setErr(oldErr) + System.setOut(realStdout) + System.setErr(realStderr) + } catch { + case t: Throwable => + System.setOut(realStdout) + System.setErr(realStderr) + throw t } } } diff --git a/compiler/test/dotty/tools/dotc/reporting/TestReporter.scala b/compiler/test/dotty/tools/dotc/reporting/TestReporter.scala index 2d7e6c70a998..5641240a722d 100644 --- a/compiler/test/dotty/tools/dotc/reporting/TestReporter.scala +++ b/compiler/test/dotty/tools/dotc/reporting/TestReporter.scala @@ -2,7 +2,7 @@ package dotty.tools package dotc package reporting -import java.io.{ PrintWriter, File => JFile, FileOutputStream } +import java.io.{ PrintStream, PrintWriter, File => JFile, FileOutputStream } import java.text.SimpleDateFormat import java.util.Date @@ -93,38 +93,25 @@ object TestReporter { logWriter.flush() } - def parallelReporter(lock: AnyRef, logLevel: Int): TestReporter = new TestReporter( - new PrintWriter(Console.err, true), - str => lock.synchronized { - logWriter.println(str) - logWriter.flush() - }, - logLevel - ) - - def reporter(logLevel: Int): TestReporter = new TestReporter( - new PrintWriter(Console.err, true), - writeToLog, - logLevel - ) - - def simplifiedReporter(writer: PrintWriter): TestReporter = new TestReporter( - writer, - writeToLog, - WARNING - ) { - /** Prints the message with the given position indication in a simplified manner */ - override def printMessageAndPos(m: MessageContainer, extra: String)(implicit ctx: Context): Unit = { - val msg = s"${m.pos.line + 1}: " + m.contained.kind + extra - val extraInfo = inlineInfo(m.pos) - - writer.println(msg) - _messageBuf.append(msg) - - if (extraInfo.nonEmpty) { - writer.println(extraInfo) - _messageBuf.append(extraInfo) + def reporter(ps: PrintStream, logLevel: Int): TestReporter = + new TestReporter(new PrintWriter(ps, true), writeToLog, logLevel) + + def simplifiedReporter(writer: PrintWriter): TestReporter = { + val rep = new TestReporter(writer, writeToLog, WARNING) { + /** Prints the message with the given position indication in a simplified manner */ + override def printMessageAndPos(m: MessageContainer, extra: String)(implicit ctx: Context): Unit = { + val msg = s"${m.pos.line + 1}: " + m.contained.kind + extra + val extraInfo = inlineInfo(m.pos) + + writer.println(msg) + _messageBuf.append(msg) + + if (extraInfo.nonEmpty) { + writer.println(extraInfo) + _messageBuf.append(extraInfo) + } } } + rep } }