Neg tests check files for // error markers

src/dotty/tools/dotc/reporting/Reporter.scala

@@ -210,6 +210,8 @@ abstract class Reporter {
   var warningCount = 0
   def hasErrors = errorCount > 0
   def hasWarnings = warningCount > 0
+  private var errors: List[Error] = Nil
+  def allErrors = errors
 
   val unreportedWarnings = new mutable.HashMap[String, Int] {
     override def default(key: String) = 0
@@ -220,7 +222,9 @@ abstract class Reporter {
     d match {
       case d: ConditionalWarning if !d.enablingOption.value => unreportedWarnings(d.enablingOption.name) += 1
       case d: Warning => warningCount += 1
-      case d: Error => errorCount += 1
+      case d: Error =>
+        errors = d :: errors
+        errorCount += 1
       case d: Info => // nothing to do here
       // match error if d is something else
     }

test/test/CompilerTest.scala

@@ -3,9 +3,12 @@ package test
 import dotty.partest.DPConfig
 import dotty.tools.dotc.{Main, Bench, Driver}
 import dotty.tools.dotc.reporting.Reporter
+import dotty.tools.dotc.util.SourcePosition
+import dotty.tools.dotc.config.CompilerCommand
 import scala.collection.mutable.ListBuffer
-import scala.reflect.io.{ Path, Directory, File => SFile }
+import scala.reflect.io.{ Path, Directory, File => SFile, AbstractFile }
 import scala.tools.partest.nest.{ FileManager, NestUI }
+import scala.annotation.tailrec
 import java.io.{ RandomAccessFile, File => JFile }
 
 import org.junit.Test
@@ -178,13 +181,138 @@ abstract class CompilerTest extends DottyTest {
 
   // ========== HELPERS =============
 
-  private def compileArgs(args: Array[String], xerrors: Int = 0)(implicit defaultOptions: List[String]): Unit = {
+  private def compileArgs(args: Array[String], xerrors: Int = 0)
+      (implicit defaultOptions: List[String]): Unit = {
     val allArgs = args ++ defaultOptions
     val processor = if (allArgs.exists(_.startsWith("#"))) Bench else Main
-    val nerrors = processor.process(allArgs, ctx).errorCount
+    val reporter = processor.process(allArgs, ctx)
+
+    val nerrors = reporter.errorCount
     assert(nerrors == xerrors, s"Wrong # of errors. Expected: $xerrors, found: $nerrors")
+
+    // NEG TEST
+    if (xerrors > 0) {
+      val errorLines = reporter.allErrors.map(_.pos)
+      // reporter didn't record as many errors as its errorCount says
+      assert(errorLines.length == nerrors, s"Not enough errors recorded.")
+
+      val allFiles = CompilerCommand.distill(allArgs)(ctx).arguments
+      val expectedErrorsPerFile = allFiles.map(getErrors(_))
+
+      // Some compiler errors have an associated source position. Each error
+      // needs to correspond to a "// error" marker on that line in the source
+      // file and vice versa.
+      // Other compiler errors don't have an associated source position. Their
+      // number should correspond to the total count of "// nopos-error"
+      // markers in all files
+      val (errorsByFile, errorsWithoutPos) = errorLines.groupBy(_.source.file).toList.partition(_._1.toString != "<no source>")
+
+      // check errors with source position
+      val foundErrorsPerFile = errorsByFile.map({ case (fileName, errorList) =>
+        val posErrorLinesToNr = errorList.groupBy(_.line).toList.map({ case (line, list) => (line, list.length) }).sortBy(_._1)
+        ErrorsInFile(fileName.toString, 0, posErrorLinesToNr)
+      })
+      val expectedErrorsPerFileZeroed = expectedErrorsPerFile.map({
+        case ErrorsInFile(fileName, _, posErrorLinesToNr) =>
+          ErrorsInFile(fileName.toString, 0, posErrorLinesToNr)
+      })
+      checkErrorsWithPosition(expectedErrorsPerFileZeroed, foundErrorsPerFile)
+
+      // check errors without source position
+      val expectedNoPos = expectedErrorsPerFile.map(_.noposErrorNr).sum
+      val foundNoPos = errorsWithoutPos.map(_._2.length).sum
+      assert(foundNoPos == expectedNoPos,
+        s"Wrong # of errors without source position. Expected (all files): $expectedNoPos, found (compiler): $foundNoPos")
+    }
   }
 
+  // ========== NEG TEST HELPERS =============
+
+  /** Captures the number of nopos-errors in the given file and the number of
+    * errors with a position, represented as a tuple of source line and number
+    * of errors on that line. */
+  case class ErrorsInFile(fileName: String, noposErrorNr: Int, posErrorLinesToNr: List[(Int, Int)])
+
+  /** Extracts the errors expected for the given neg test file. */
+  def getErrors(fileName: String): ErrorsInFile = {
+    val content = SFile(fileName).slurp
+    val (line, rest) = content.span(_ != '\n')
+
+    @tailrec
+    def checkLine(line: String, rest: String, index: Int, noposAcc: Int, posAcc: List[(Int, Int)]): ErrorsInFile = {
+      val posErrors = "// ?error".r.findAllIn(line).length
+      val newPosAcc = if (posErrors > 0) (index, posErrors) :: posAcc else posAcc
+      val newNoPosAcc = noposAcc + "// ?nopos-error".r.findAllIn(line).length
+      val (newLine, newRest) = rest.span(_ != '\n')
+      if (newRest.isEmpty)
+        ErrorsInFile(fileName.toString, newNoPosAcc, newPosAcc.reverse)
+      else
+        checkLine(newLine, newRest.tail, index + 1, newNoPosAcc, newPosAcc) // skip leading '\n'
+    }
+
+    checkLine(line, rest.tail, 0, 0, Nil) // skip leading '\n'
+  }
+
+  /** Asserts that the expected and found number of errors correspond, and
+    * otherwise throws an error with the filename, plus optionally a line
+    * number if available. */
+  def errorMsg(fileName: String, lineNumber: Option[Int], exp: Int, found: Int) = {
+    val i = lineNumber.map({ i => ":" + (i + 1) }).getOrElse("")
+    assert(found == exp, s"Wrong # of errors for $fileName$i. Expected (file): $exp, found (compiler): $found")
+  }
+
+  /** Compares the expected with the found errors and creates a nice error
+    * message if they don't agree. */
+  def checkErrorsWithPosition(expected: List[ErrorsInFile], found: List[ErrorsInFile]): Unit = {
+      // create nice error messages
+      expected.diff(found) match {
+        case Nil => // nothing missing
+        case ErrorsInFile(fileName, _, expectedLines) :: xs =>
+          found.find(_.fileName == fileName) match {
+            case None =>
+              // expected some errors, but none found for this file
+              errorMsg(fileName, None, expectedLines.map(_._2).sum, 0)
+            case Some(ErrorsInFile(_,_,foundLines)) =>
+              // found wrong number/location of markers for this file
+              compareLines(fileName, expectedLines, foundLines)
+          }
+      }
+
+      found.diff(expected) match {
+        case Nil => // nothing missing
+        case ErrorsInFile(fileName, _, foundLines) :: xs =>
+          expected.find(_.fileName == fileName) match {
+            case None =>
+              // found some errors, but none expected for this file
+              errorMsg(fileName, None, 0, foundLines.map(_._2).sum)
+            case Some(ErrorsInFile(_,_,expectedLines)) =>
+              // found wrong number/location of markers for this file
+              compareLines(fileName, expectedLines, foundLines)
+          }
+      }
+  }
+
+  /** Gives an error message for one line where the expected number of errors and
+    * the number of compiler errors differ. */
+  def compareLines(fileName: String, expectedLines: List[(Int, Int)], foundLines: List[(Int, Int)]) = {
+    expectedLines.foreach({ case (line, expNr) =>
+      foundLines.find(_._1 == line) match {
+        case Some((_, `expNr`)) => // this line is ok
+        case Some((_, foundNr)) => errorMsg(fileName, Some(line), expNr, foundNr)
+        case None               => errorMsg(fileName, Some(line), expNr, 0)
+      }
+    })
+    foundLines.foreach({ case (line, foundNr) =>
+      expectedLines.find(_._1 == line) match {
+        case Some((_, `foundNr`)) => // this line is ok
+        case Some((_, expNr))     => errorMsg(fileName, Some(line), expNr, foundNr)
+        case None                 => errorMsg(fileName, Some(line), 0,     foundNr)
+      }
+    })
+  }
+
+  // ========== PARTEST HELPERS =============
+
   // In particular, don't copy flags from scalac tests
   private val extensionsToCopy = scala.collection.immutable.HashSet("scala", "java")
 

tests/neg/arrayclone-new.scala

@@ -7,7 +7,7 @@ object Test extends dotty.runtime.LegacyApp{
 }
 
 object ObjectArrayClone{
-  val it : Array[String] = Array("1", "0");
+  val it : Array[String] = Array("1", "0");           // error
   val cloned = it.clone();
   assert(cloned.sameElements(it));
   cloned(0) = "0";
@@ -22,7 +22,7 @@ object PolymorphicArrayClone{
     assert(it(0) == one)
   }
 
-  testIt(Array("one", "two"), "one", "two");
+  testIt(Array("one", "two"), "one", "two");          // error
 
   class Mangler[T: ClassTag](ts : T*){
     // this will always be a BoxedAnyArray even after we've unboxed its contents.

tests/neg/assignments.scala

@@ -13,11 +13,11 @@ object assignments {
     x = x + 1
     x *= 2
 
-    x_= = 2  // should give missing arguments + reassignment to val
+    x_= = 2  // error should give missing arguments + // error reassignment to val
   }
 
   var c = new C
-  import c._ // should give: prefix is not stable
+  import c._ // error should give: prefix is not stable
   x = x + 1
   x *= 2
 }

tests/neg/autoTuplingTest.scala

@@ -2,10 +2,10 @@ import dotty.language.noAutoTupling
 
 object autoTuplingNeg {
 
-  val x = Some(1, 2)
+  val x = Some(1, 2)                                  // error
 
   x match {
-    case Some(a, b) => a + b
+    case Some(a, b) => a + b                          // error // error // error
     case None =>
   }
 }

tests/neg/blockescapesNeg.scala

@@ -1,6 +1,6 @@
 object blockescapesNeg {
   def m0 = { object Foo { class Bar { val field = 2 }} ; new Foo.Bar }
-  m0.field
+  m0.field                                            // error
   class A[T]
   def m1 = { val x = 1; new A[x.type]}
 }

tests/neg/boundspropagation.scala

@@ -13,15 +13,15 @@ object test2 {
 
 
   def f(x: Any): Tree[Null] = x match {
-    case y: Tree[_] => y
+    case y: Tree[_] => y                              // error
   }
 }
 object test3 {
   class Tree[+T >: Null]
 
 
   def f(x: Any): Tree[Null] = x match {
-    case y: Tree[_] => y
+    case y: Tree[_] => y                              // error
   }
 }
 
@@ -34,11 +34,11 @@ object test4 {
     class Tree[-S, -T >: Option[S]]
 
     def g(x: Any): Tree[_, _ <: Option[N]] = x match {
-      case y: Tree[_, _] => y
+      case y: Tree[_, _] => y                         // error
     }
   }
 }
 
 class Test5 {
-"": ({ type U = this.type })#U
+"": ({ type U = this.type })#U                        // error // error
 }

tests/neg/companions.scala

@@ -8,7 +8,7 @@ object companionsNeg {
 
     { object C {
       private val p = 1
-      println(new C().q)
+      println(new C().q)   // error
     }}
   }
 

tests/neg/cycles.scala

@@ -1,39 +1,39 @@
-class Foo[T <: U, U <: T]
+class Foo[T <: U, U <: T]                             // error
 
-class Bar[T >: T]
+class Bar[T >: T]                                     // error
 
 class A {
   val x: T = ???
-  type T <: x.type
+  type T <: x.type                                    // error
 }
 
 class B {
-  type T <: x.type
+  type T <: x.type                                    // error
   val x: T = ???
 }
 
 class C {
   val x: D#T = ???
   class D {
-    type T <: x.type
+    type T <: x.type                                  // error
     val z: x.type = ???
   }
 }
 
 class E {
   class F {
     type T <: x.type
-    val z: x.type = ???
+    val z: x.type = ???                               // error
   }
   val x: F#T = ???
 }
 
 class T1 {
-  type X = (U, U) // cycle
+  type X = (U, U) // cycle                            // error
   type U = X & Int
 }
 class T2 {
-  type X = (U, U) // cycle
+  type X = (U, U) // cycle                            // error
   type U = X | Int
 }
 object T12 {

tests/neg/escapingRefs.scala

@@ -3,8 +3,8 @@ object O {
   class B
   def f[T](x: T, y: T): T = y
 
-  val x: A = f(new A { }, new B { })
+  val x: A = f(new A { }, new B { })                  // error
 
   val y = f({ class C { def member: Int = 1 }; new C }, { class C { def member: Int = 1 }; new C })
-  val z = y.member
+  val z = y.member                                    // error
 }

tests/neg/i0091-infpaths.scala

@@ -2,15 +2,15 @@ object infpaths {
 
   object a {
     trait T { t =>
-      type M <: t.b.M
+      type M <: t.b.M                                 // error
       type T <: a.T
       val b: t.T
     }
     val x: a.T = ???
   }
 
   val m1: a.x.M = ???
-  val m2: a.x.b.M = m1
-  val m3: a.x.b.b.M = m2
+  val m2: a.x.b.M = m1                                // error
+  val m3: a.x.b.b.M = m2                              // error
 
 }

tests/neg/i0248-inherit-refined.scala

@@ -1,10 +1,10 @@
 object test {
   class A { type T }
   type X = A { type T = Int }
-  class B extends X
+  class B extends X                                   // error
   type Y = A & B
-  class C extends Y
+  class C extends Y                                   // error
   type Z = A | B
-  class D extends Z
-  abstract class E extends ({ val x: Int })
+  class D extends Z                                   // error
+  abstract class E extends ({ val x: Int })           // error
 }

tests/neg/i0281-null-primitive-conforms.scala

@@ -1,6 +1,6 @@
 object test {
-  val b: scala.Boolean = null
+  val b: scala.Boolean = null                         // error
   val c: Unit = null
-  val d: Float = null
-  val e: AnyVal = null
+  val d: Float = null                                 // error
+  val e: AnyVal = null                                // error
 }

tests/neg/i0583-skolemize.scala

@@ -12,15 +12,15 @@ object Test1 {
 
   val xs: List[C[_]] = List(c, d)
 
-  xs(0).x = xs(1).x
+  xs(0).x = xs(1).x                                   // error
 
 }
 object Test {
   def main(args: Array[String]): Unit = {
     val f: ListBuffer[Int] = ListBuffer(1,2)
     val g: ListBuffer[Double] = ListBuffer(3.0,4.0)
     val lb: ListBuffer[ListBuffer[_]] = ListBuffer(f, g)
-    lb(0)(0) = lb(1)(0)
+    lb(0)(0) = lb(1)(0)                               // error
     val x: Int = f(0)
   }
 }