Add meta tests for vulpix

This new test checks that the framework rejects incorrect pos/neg and
run test. I think addition meta tests should rather be added as part of
the unit test infrastructure, given than this is rather heavy to maintain.
(Note that sbt-output.check is manually annotated with "SKIP" lines)

Author: OlivierBlanvillain

compiler/test/dotty/tools/vulpix/VulpixMetaTests.scala

@@ -0,0 +1,27 @@
+package dotty.tools
+package vulpix
+
+import org.junit.Test
+import org.junit.experimental.categories.Category
+import scala.concurrent.duration._
+import dotty.Properties
+import TestConfiguration._
+
+/** Meta tests for the Vulpix test suite. This test follows the structure of
+ *  CompilationTests.scala. It is ment to be called from bash to diff with
+ *  output againts an expected result.
+ */
+@Category(Array(classOf[dotty.VulpixMetaTests]))
+class VulpixMetaTests extends ParallelTesting {
+  def maxDuration = 1.seconds
+  def numberOfSlaves = 5
+  def safeMode = Properties.testsSafeMode
+  def isInteractive = SummaryReport.isInteractive
+  def testFilter = Properties.testsFilter
+  implicit val summaryReport: SummaryReporting = new SummaryReport
+  implicit def testGroup: TestGroup = TestGroup("VulpixMetaTests")
+
+  @Test def compilePos: Unit = compileFilesInDir("../tests/meta-tests/pos", defaultOptions).checkCompile()
+  @Test def compileNeg: Unit = compileFilesInDir("../tests/meta-tests/neg", defaultOptions).checkExpectedErrors()
+  @Test def runAll: Unit     = compileFilesInDir("../tests/meta-tests/run", defaultOptions).checkRuns()
+}

compiler/test/dotty/tools/vulpix/VulpixTests.scala

@@ -7,7 +7,7 @@ import org.junit.Test
 import scala.concurrent.duration._
 import scala.util.control.NonFatal
 
-/** Meta tests for the Vulpix test suite */
+/** Unit tests for the Vulpix test suite */
 class VulpixTests extends ParallelTesting {
   import TestConfiguration._
 

project/scripts/cmdTests

@@ -92,3 +92,22 @@ mkdir -p out/scriptedtest2
 
 # echo ":quit" | ./dist-bootstrapped/target/pack/bin/dotr  # not supported by CI
 mkdir -p _site && ./bin/dotd -project Hello -siteroot _site tests/run/hello.scala
+
+echo "running Vulpix meta test"
+tmp=$(mktemp)
+if sbt "dotty-compiler/testOnly dotty.tools.vulpix.VulpixMetaTests" > "$tmp" 2>&1; then
+  echo "failed: sbt exited without error on VulpixMetaTests, these tests are expected to fail"
+  exit -1
+fi
+tmp1=$(mktemp)
+cat "$tmp" | sed '/Test run started/,$!d' > "$tmp1"
+while read actual <&3 && read expected <&4; do
+  if [[ "$expected" != *"SKIP" ]]; then
+    if [ "$actual" != "$expected" ]; then
+      echo "failed Vulpix meta: the output of sbt differs from the expected output"
+      echo "expected : $expected"
+      echo "actual   : $actual"
+      exit -1
+    fi
+  fi
+done 3<"$tmp1" 4<"./tests/meta-tests/sbt-output.check"

tests/meta-tests/neg/missing-error-annotation.scala

@@ -0,0 +1,7 @@
+class MissingErrorAnnotation {
+  // Missing annotation for the following unbound identifier:
+  a
+
+  // Another one (to check that both of them get reported)
+  b
+}

tests/meta-tests/pos/does-not-compile.scala

@@ -0,0 +1,3 @@
+class NegTest {
+  a // Does not compile
+}

tests/meta-tests/run/wrong-check-file.check

@@ -0,0 +1,2 @@
+hello
+world

tests/meta-tests/run/wrong-check-file.scala

@@ -0,0 +1,6 @@
+object Test {
+  def main(args: Array[String]): Unit = {
+    println("world")
+    println("hello") // Out of order
+  }
+}

tests/meta-tests/sbt-output.check

@@ -0,0 +1,43 @@
+[info] Test run started
+[info] Test dotty.tools.vulpix.VulpixMetaTests.runAll started
+Testing ../tests/meta-tests/run/wrong-check-file.scala
+Output from '../tests/meta-tests/run/wrong-check-file.scala' did not match check file.
+Diff (expected on the left, actual right):
+hello  |  world
+world  |  hello
+EOF    |  EOF
+
+[error] Test dotty.tools.vulpix.VulpixMetaTests.runAll failed: java.lang.AssertionError: Run test failed, but should not, reasons:
+[error] , took 1.673 sec SKIP
+[error]     at dotty.tools.vulpix.ParallelTesting$CompilationTest.checkRuns(ParallelTesting.scala:1000)
+[error]     at dotty.tools.vulpix.VulpixMetaTests.runAll(VulpixMetaTests.scala:26)
+[error]     ...
+[info] Test dotty.tools.vulpix.VulpixMetaTests.compileNeg started
+Testing ../tests/meta-tests/neg/missing-error-annotation.scala
+
+Wrong number of errors encountered when compiling ../out/VulpixMetaTests/neg/missing-error-annotation, expected: 0, actual: 2
+
+[error] Test dotty.tools.vulpix.VulpixMetaTests.compileNeg failed: java.lang.AssertionError: Neg test shouldn't have failed, but did. Reasons:
+[error] 
+[error]   - encountered 2 error(s), took 0.072 sec SKIP
+[error]     at dotty.tools.vulpix.ParallelTesting$CompilationTest.checkExpectedErrors(ParallelTesting.scala:982)
+[error]     at dotty.tools.vulpix.VulpixMetaTests.compileNeg(VulpixMetaTests.scala:25)
+[error]     ...
+[info] Test dotty.tools.vulpix.VulpixMetaTests.compilePos started
+Testing ../tests/meta-tests/pos/does-not-compile.scala
+-- [E006] Unbound Identifier Error: /home/olivier/workspace/dotty/compiler/../tests/meta-tests/pos/does-not-compile.scala:2:2 SKIP
+2 |  a // Does not compile
+  |  ^
+  |  not found: a
+[error] Test dotty.tools.vulpix.VulpixMetaTests.compilePos failed: java.lang.AssertionError: Expected no errors when compiling, failed for the following reason(s):
+[error] 
+[error]   - encountered 1 error(s), took 0.069 sec SKIP
+[error]     at dotty.tools.vulpix.ParallelTesting$CompilationTest.checkCompile(ParallelTesting.scala:965)
+[error]     at dotty.tools.vulpix.VulpixMetaTests.compilePos(VulpixMetaTests.scala:24)
+[error]     ...
+[info] Test run finished: 3 failed, 0 ignored, 3 total, 1.964s SKIP
+[error] Failed: Total 3, Failed 3, Errors 0, Passed 0
+[error] Failed tests:
+[error] 	dotty.tools.vulpix.VulpixMetaTests
+[error] (dotty-compiler/test:testOnly) sbt.TestsFailedException: Tests unsuccessful
+[error] Total time: 3 s, completed Feb 5, 2018 10:08:08 AM SKIP