Remove legacy tests

.drone.yml

@@ -14,13 +14,6 @@ pipeline:
 
   # TESTS:
   # We run tests in parallel. Tests run in a copy of the working directory to avoid conflict
-  test_legacy:
-    group: test
-    image: lampepfl/dotty:2018-01-17
-    commands:
-      - cp -R . /tmp/0/ && cd /tmp/0/
-      - ./project/scripts/sbt legacyTests
-
   test:
     group: test
     image: lampepfl/dotty:2018-01-17

compiler/test/dotty/TestCategories.scala

@@ -3,5 +3,5 @@ package dotty
 /** SlowTest category for JUnit */
 trait SlowTests
 
-/** Legacy tests category for JUnit */
-trait LegacyTests
+/** Meta tests category for JUnit */
+trait VulpixMetaTests

compiler/test/dotty/tools/vulpix/VulpixMetaTests.scala

@@ -0,0 +1,27 @@
+package dotty.tools
+package vulpix
+
+import org.junit.Test
+import org.junit.experimental.categories.Category
+import scala.concurrent.duration._
+import dotty.Properties
+import TestConfiguration._
+
+/** Meta tests for the Vulpix test suite. This test follows the structure of
+ *  CompilationTests.scala. It is ment to be called from bash to diff with
+ *  output againts an expected result.
+ */
+@Category(Array(classOf[dotty.VulpixMetaTests]))
+class VulpixMetaTests extends ParallelTesting {
+  def maxDuration = 1.seconds
+  def numberOfSlaves = 5
+  def safeMode = Properties.testsSafeMode
+  def isInteractive = SummaryReport.isInteractive
+  def testFilter = Properties.testsFilter
+  implicit val summaryReport: SummaryReporting = new SummaryReport
+  implicit def testGroup: TestGroup = TestGroup("VulpixMetaTests")
+
+  @Test def compilePos: Unit = compileFilesInDir("tests/vulpix-tests/meta/pos", defaultOptions).checkCompile()
+  @Test def compileNeg: Unit = compileFilesInDir("tests/vulpix-tests/meta/neg", defaultOptions).checkExpectedErrors()
+  @Test def runAll: Unit     = compileFilesInDir("tests/vulpix-tests/meta/run", defaultOptions).checkRuns()
+}

compiler/test/dotty/tools/vulpix/VulpixUnitTests.scala

@@ -0,0 +1,94 @@
+package dotty.tools
+package vulpix
+
+import org.junit.Assert._
+import org.junit.Test
+
+import scala.concurrent.duration._
+import scala.util.control.NonFatal
+
+/** Unit tests for the Vulpix test suite */
+class VulpixUnitTests extends ParallelTesting {
+  import TestConfiguration._
+
+  implicit val _: SummaryReporting = new NoSummaryReport
+
+  implicit def testGroup: TestGroup = TestGroup("VulpixTests")
+
+  def maxDuration = 3.seconds
+  def numberOfSlaves = 5
+  def safeMode = sys.env.get("SAFEMODE").isDefined
+  def isInteractive = !sys.env.contains("DRONE")
+  def testFilter = None
+
+  @Test def missingFile: Unit =
+    try {
+      compileFile("tests/vulpix-tests/unit/i-dont-exist.scala", defaultOptions).expectFailure.checkExpectedErrors()
+      fail("didn't fail properly")
+    }
+    catch {
+      case _: IllegalArgumentException => // pass!
+      case NonFatal(_) => fail("wrong exception thrown")
+    }
+
+  @Test def pos1Error: Unit =
+    compileFile("tests/vulpix-tests/unit/posFail1Error.scala", defaultOptions).expectFailure.checkCompile()
+
+  @Test def negMissingAnnot: Unit =
+    compileFile("tests/vulpix-tests/unit/negMissingAnnot.scala", defaultOptions).expectFailure.checkExpectedErrors()
+
+  @Test def negAnnotWrongLine: Unit =
+    compileFile("tests/vulpix-tests/unit/negAnnotWrongLine.scala", defaultOptions).expectFailure.checkExpectedErrors()
+
+  @Test def negTooManyAnnots: Unit =
+    compileFile("tests/vulpix-tests/unit/negTooManyAnnots.scala", defaultOptions).expectFailure.checkExpectedErrors()
+
+  @Test def negNoPositionAnnot: Unit =
+    compileFile("tests/vulpix-tests/unit/negNoPositionAnnots.scala", defaultOptions).expectFailure.checkExpectedErrors()
+
+  @Test def runCompileFail: Unit =
+    compileFile("tests/vulpix-tests/unit/posFail1Error.scala", defaultOptions).expectFailure.checkRuns()
+
+  @Test def runWrongOutput1: Unit =
+    compileFile("tests/vulpix-tests/unit/runWrongOutput1.scala", defaultOptions).expectFailure.checkRuns()
+
+  @Test def runWrongOutput2: Unit =
+    compileFile("tests/vulpix-tests/unit/runWrongOutput2.scala", defaultOptions).expectFailure.checkRuns()
+
+  @Test def runDiffOutput1: Unit =
+    compileFile("tests/vulpix-tests/unit/runDiffOutput1.scala", defaultOptions).expectFailure.checkRuns()
+
+  @Test def runStackOverflow: Unit =
+    compileFile("tests/vulpix-tests/unit/stackOverflow.scala", defaultOptions).expectFailure.checkRuns()
+
+  @Test def runOutRedirects: Unit =
+    compileFile("tests/vulpix-tests/unit/i2147.scala", defaultOptions).expectFailure.checkRuns()
+
+  @Test def infiteNonRec: Unit =
+    compileFile("tests/vulpix-tests/unit/infinite.scala", defaultOptions).expectFailure.checkRuns()
+
+  @Test def infiteTailRec: Unit =
+    compileFile("tests/vulpix-tests/unit/infiniteTail.scala", defaultOptions).expectFailure.checkRuns()
+
+  @Test def infiniteAlloc: Unit =
+    compileFile("tests/vulpix-tests/unit/infiniteAlloc.scala", defaultOptions).expectFailure.checkRuns()
+
+  @Test def deadlock: Unit =
+    compileFile("tests/vulpix-tests/unit/deadlock.scala", defaultOptions).expectFailure.checkRuns()
+
+  @Test def badJava: Unit =
+    try compileFile("tests/vulpix-tests/unit/BadJava.java", defaultOptions).suppressAllOutput.checkCompile()
+    catch {
+      case ae: AssertionError => assert(ae.getMessage.contains("java compilation failed"))
+    }
+
+  @Test def runTimeout: Unit = {
+    try {
+      compileFile("tests/vulpix-tests/unit/timeout.scala", defaultOptions).checkRuns()
+      assert(false, "unreachable")
+    } catch {
+      case ae: AssertionError =>
+        assert(ae.getMessage == "Run test failed, but should not, reasons:\n  - test 'tests/vulpix-tests/unit/timeout.scala' timed out")
+    }
+  }
+}

project/Build.scala

@@ -501,8 +501,8 @@ object Build {
       baseDirectory in Test := baseDirectory.value / "..",
 
       test in Test := {
-        // Exclude legacy tests by default
-        (testOnly in Test).toTask(" -- --exclude-categories=dotty.LegacyTests").value
+        // Exclude VulpixMetaTests
+        (testOnly in Test).toTask(" -- --exclude-categories=dotty.VulpixMetaTests").value
       },
 
       testOptions in Test += Tests.Argument(
@@ -1148,8 +1148,7 @@ object Build {
       dependsOn(dottyCompiler).
       dependsOn(dottyLibrary).
       nonBootstrappedSettings(
-        addCommandAlias("run", "dotty-compiler/run") ++
-        addCommandAlias("legacyTests", "dotty-compiler/testOnly dotc.tests")
+        addCommandAlias("run", "dotty-compiler/run")
       )
 
     def asDottyCompiler(implicit mode: Mode): Project = project.withCommonSettings.

project/scripts/cmdTests

@@ -92,3 +92,23 @@ mkdir -p out/scriptedtest2
 
 # echo ":quit" | ./dist-bootstrapped/target/pack/bin/dotr  # not supported by CI
 mkdir -p _site && ./bin/dotd -project Hello -siteroot _site tests/run/hello.scala
+
+echo "running Vulpix meta test"
+tmp=$(mktemp)
+if ./project/scripts/sbt "dotty-compiler/testOnly dotty.tools.vulpix.VulpixMetaTests" > "$tmp" 2>&1; then
+  cat "$tmp"
+  echo "failed: sbt exited without error on VulpixMetaTests, these tests are expected to fail"
+  exit -1
+fi
+tmp1=$(mktemp)
+cat "$tmp" | sed '/Test run started/,$!d' > "$tmp1"
+while read expected <&4 && read actual <&3; do
+  if [[ "$expected" != *"SKIP" ]]; then
+    if [ "$actual" != "$expected" ]; then
+      echo "failed Vulpix meta: the output of sbt differs from the expected output"
+      echo "expected : $expected"
+      echo "actual   : $actual"
+      exit -1
+    fi
+  fi
+done 3<"$tmp1" 4<"./tests/vulpix-tests/meta/sbt-output.check"

project/scripts/sbt

@@ -12,4 +12,5 @@ sbt -J-Xmx4096m \
     -J-XX:MaxMetaspaceSize=1024m \
     -Ddotty.drone.mem=4096m \
     -Dsbt.ivy.home=/var/cache/drone/ivy2 \
+    -no-colors \
     "$CMD"

tests/vulpix-tests/meta/neg/missing-error-annotation.scala

@@ -0,0 +1,7 @@
+class MissingErrorAnnotation {
+  // Missing annotation for the following unbound identifier:
+  a
+
+  // Another one (to check that both of them get reported)
+  b
+}

tests/vulpix-tests/meta/pos/does-not-compile.scala

@@ -0,0 +1,3 @@
+class NegTest {
+  a // Does not compile
+}

tests/vulpix-tests/meta/run/wrong-check-file.check

@@ -0,0 +1,2 @@
+hello
+world

tests/vulpix-tests/meta/run/wrong-check-file.scala

@@ -0,0 +1,6 @@
+object Test {
+  def main(args: Array[String]): Unit = {
+    println("world")
+    println("hello") // Out of order
+  }
+}

tests/vulpix-tests/meta/sbt-output.check

@@ -0,0 +1,42 @@
+[info] Test run started
+[info] Test dotty.tools.vulpix.VulpixMetaTests.runAll started
+Testing tests/vulpix-tests/meta/run/wrong-check-file.scala
+Output from 'tests/vulpix-tests/meta/run/wrong-check-file.scala' did not match check file.
+Diff (expected on the left, actual right):
+hello  |  world
+world  |  hello
+EOF    |  EOF
+
+[error] Test dotty.tools.vulpix.VulpixMetaTests.runAll failed: java.lang.AssertionError: Run test failed, but should not, reasons:
+[error] , took 1.682 sec SKIP
+[error]     at dotty.tools.vulpix.ParallelTesting$CompilationTest.checkRuns(ParallelTesting.scala:993) SKIP
+[error]     at dotty.tools.vulpix.VulpixMetaTests.runAll(VulpixMetaTests.scala:26) SKIP
+[error]     ...
+[info] Test dotty.tools.vulpix.VulpixMetaTests.compileNeg started
+Testing tests/vulpix-tests/meta/neg/missing-error-annotation.scala
+
+Wrong number of errors encountered when compiling ../out/VulpixMetaTests/neg/missing-error-annotation, expected: 0, actual: 2
+[error] Test dotty.tools.vulpix.VulpixMetaTests.compileNeg failed: java.lang.AssertionError: Neg test shouldn't have failed, but did. Reasons:
+[error]
+[error]   - encountered 2 error(s), took 0.093 sec SKIP
+[error]     at dotty.tools.vulpix.ParallelTesting$CompilationTest.checkExpectedErrors(ParallelTesting.scala:975) SKIP
+[error]     at dotty.tools.vulpix.VulpixMetaTests.compileNeg(VulpixMetaTests.scala:25) SKIP
+[error]     ...
+[info] Test dotty.tools.vulpix.VulpixMetaTests.compilePos started
+Testing tests/vulpix-tests/meta/pos/does-not-compile.scala
+-- [E006] Unbound Identifier Error: /home/olivier/workspace/dotty/tests/vulpix-tests/meta/pos/does-not-compile.scala:2:2 SKIP
+2 |  a // Does not compile
+  |  ^
+  |  not found: a
+[error] Test dotty.tools.vulpix.VulpixMetaTests.compilePos failed: java.lang.AssertionError: Expected no errors when compiling, failed for the following reason(s):
+[error]
+[error]   - encountered 1 error(s), took 0.069 sec SKIP
+[error]     at dotty.tools.vulpix.ParallelTesting$CompilationTest.checkCompile(ParallelTesting.scala:958) SKIP
+[error]     at dotty.tools.vulpix.VulpixMetaTests.compilePos(VulpixMetaTests.scala:24) SKIP
+[error]     ...
+[info] Test run finished: 3 failed, 0 ignored, 3 total, 1.982s SKIP
+[error] Failed: Total 3, Failed 3, Errors 0, Passed 0
+[error] Failed tests:
+[error] 	dotty.tools.vulpix.VulpixMetaTests
+[error] (dotty-compiler/test:testOnly) sbt.TestsFailedException: Tests unsuccessful
+[error] Total time: 3 s, completed Feb 5, 2018 5:10:12 PM SKIP

tests/vulpix-tests/unit/timeout.scala

@@ -0,0 +1,5 @@
+object Test {
+  def main(args: Array[String]): Unit = {
+    Thread.sleep(10 * 1000)
+  }
+}