Adapt the tests to the use of -Yscala2-unpickler

The tests that import things defined in the dotty-compiler jar should
only be run with a bootstrapped compiler.

Author: smarter

compiler/test/dotty/TestCategories.scala

@@ -5,3 +5,6 @@ trait SlowTests
 
 /** Meta tests category for JUnit */
 trait VulpixMetaTests
+
+/** Tests that should only be run with a bootstrapped compiler */
+trait BootstrappedOnlyTests

compiler/test/dotty/tools/dotc/BootstrappedOnlyCompilationTests.scala

@@ -0,0 +1,115 @@
+package dotty
+package tools
+package dotc
+
+import org.junit.{ Test, BeforeClass, AfterClass }
+import org.junit.Assert._
+import org.junit.Assume._
+import org.junit.experimental.categories.Category
+
+import java.nio.file._
+import java.util.stream.{ Stream => JStream }
+import scala.collection.JavaConverters._
+import scala.util.matching.Regex
+import scala.concurrent.duration._
+import vulpix._
+import dotty.tools.io.JFile
+
+@Category(Array(classOf[BootstrappedOnlyTests]))
+class BootstrappedOnlyCompilationTests extends ParallelTesting {
+  import ParallelTesting._
+  import TestConfiguration._
+  import CompilationTests._
+
+  // Test suite configuration --------------------------------------------------
+
+  def maxDuration = 30.seconds
+  def numberOfSlaves = 5
+  def safeMode = Properties.testsSafeMode
+  def isInteractive = SummaryReport.isInteractive
+  def testFilter = Properties.testsFilter
+
+  // Positive tests ------------------------------------------------------------
+
+  @Test def posWithCompiler: Unit = {
+    implicit val testGroup: TestGroup = TestGroup("compilePosWithCompiler")
+    compileFilesInDir("tests/pos-with-compiler", defaultOptions) +
+    compileDir("compiler/src/dotty/tools/dotc/ast", defaultOptions) +
+    compileDir("compiler/src/dotty/tools/dotc/config", defaultOptions) +
+    compileDir("compiler/src/dotty/tools/dotc/core", defaultOptions) +
+    compileDir("compiler/src/dotty/tools/dotc/transform", defaultOptions) +
+    compileDir("compiler/src/dotty/tools/dotc/parsing", defaultOptions) +
+    compileDir("compiler/src/dotty/tools/dotc/printing", defaultOptions) +
+    compileDir("compiler/src/dotty/tools/dotc/reporting", defaultOptions) +
+    compileDir("compiler/src/dotty/tools/dotc/typer", defaultOptions) +
+    compileDir("compiler/src/dotty/tools/dotc/util", defaultOptions) +
+    compileDir("compiler/src/dotty/tools/io", defaultOptions) +
+    compileDir("compiler/src/dotty/tools/dotc/core", TestFlags(classPath, noCheckOptions))
+  }
+
+  @Test def posTwiceWithCompiler: Unit = {
+    implicit val testGroup: TestGroup = TestGroup("posTwiceWithCompiler")
+    compileFile("tests/pos-with-compiler/Labels.scala", defaultOptions) +
+    compileFile("tests/pos-with-compiler/Patterns.scala", defaultOptions) +
+    compileList(
+      "testNonCyclic",
+      List(
+        "compiler/src/dotty/tools/dotc/CompilationUnit.scala",
+        "compiler/src/dotty/tools/dotc/core/Types.scala",
+        "compiler/src/dotty/tools/dotc/ast/Trees.scala"
+      ),
+      defaultOptions.and("-Xprompt")
+    ) +
+    compileList(
+      "testIssue34",
+      List(
+        "compiler/src/dotty/tools/dotc/config/Properties.scala",
+        "compiler/src/dotty/tools/dotc/config/PathResolver.scala"
+      ),
+      defaultOptions.and("-Xprompt")
+    )
+  }.times(2).checkCompile()
+
+  // Negative tests ------------------------------------------------------------
+
+  @Test def negAll: Unit = {
+    implicit val testGroup: TestGroup = TestGroup("compileNegWithCompiler")
+    compileFilesInDir("tests/neg-with-compiler", defaultOptions)
+  }.checkExpectedErrors()
+
+  // Run tests -----------------------------------------------------------------
+
+  @Test def runWithCompiler: Unit = {
+    implicit val testGroup: TestGroup = TestGroup("runWithCompiler")
+    compileFilesInDir("tests/run-with-compiler", defaultRunWithCompilerOptions) +
+    compileFile("tests/run-with-compiler-custom-args/staged-streams_1.scala", defaultRunWithCompilerOptions without "-Yno-deep-subtypes")
+  }.checkRuns()
+
+  // Pickling Tests ------------------------------------------------------------
+  //
+  // Pickling tests are very memory intensive and as such need to be run with a
+  // lower level of concurrency as to not kill their running VMs
+
+  @Test def picklingWithCompiler: Unit = {
+    implicit val testGroup: TestGroup = TestGroup("testPicklingWithCompiler")
+    compileDir("compiler/src/dotty/tools", picklingOptions, recursive = false) +
+    compileDir("compiler/src/dotty/tools/dotc", picklingOptions, recursive = false) +
+    compileDir("library/src/dotty/runtime", picklingOptions) +
+    compileDir("compiler/src/dotty/tools/backend/jvm", picklingOptions) +
+    compileDir("compiler/src/dotty/tools/dotc/ast", picklingOptions) +
+    compileDir("compiler/src/dotty/tools/dotc/core", picklingOptions, recursive = false) +
+    compileDir("compiler/src/dotty/tools/dotc/config", picklingOptions) +
+    compileDir("compiler/src/dotty/tools/dotc/parsing", picklingOptions) +
+    compileDir("compiler/src/dotty/tools/dotc/printing", picklingOptions) +
+    compileDir("compiler/src/dotty/tools/repl", picklingOptions) +
+    compileDir("compiler/src/dotty/tools/dotc/rewrite", picklingOptions) +
+    compileDir("compiler/src/dotty/tools/dotc/transform", picklingOptions) +
+    compileDir("compiler/src/dotty/tools/dotc/typer", picklingOptions) +
+    compileDir("compiler/src/dotty/tools/dotc/util", picklingOptions) +
+    compileDir("compiler/src/dotty/tools/io", picklingOptions) +
+    compileFile("tests/pos/pickleinf.scala", picklingOptions) +
+    compileDir("compiler/src/dotty/tools/dotc/core/classfile", picklingOptions) +
+    compileDir("compiler/src/dotty/tools/dotc/core/tasty", picklingOptions) +
+    compileDir("compiler/src/dotty/tools/dotc/core/unpickleScala2", picklingOptions)
+  }.limitThreads(4).checkCompile()
+}

compiler/test/dotty/tools/dotc/CompilationTests.scala

@@ -37,20 +37,9 @@ class CompilationTests extends ParallelTesting {
     compileList("compileStdLib", TestSources.stdLibWhitelisted, scala2Mode.and("-migration", "-Yno-inline", "-Ydetailed-stats"))
   }.checkCompile()
 
-  @Test def compilePos: Unit = {
+  @Test def pos: Unit = {
     implicit val testGroup: TestGroup = TestGroup("compilePos")
     compileList("compileStdLib", TestSources.stdLibWhitelisted, scala2Mode.and("-migration", "-Yno-inline")) +
-    compileDir("compiler/src/dotty/tools/dotc/ast", defaultOptions) +
-    compileDir("compiler/src/dotty/tools/dotc/config", defaultOptions) +
-    compileDir("compiler/src/dotty/tools/dotc/core", defaultOptions) +
-    compileDir("compiler/src/dotty/tools/dotc/transform", defaultOptions) +
-    compileDir("compiler/src/dotty/tools/dotc/parsing", defaultOptions) +
-    compileDir("compiler/src/dotty/tools/dotc/printing", defaultOptions) +
-    compileDir("compiler/src/dotty/tools/dotc/reporting", defaultOptions) +
-    compileDir("compiler/src/dotty/tools/dotc/typer", defaultOptions) +
-    compileDir("compiler/src/dotty/tools/dotc/util", defaultOptions) +
-    compileDir("compiler/src/dotty/tools/io", defaultOptions) +
-    compileDir("compiler/src/dotty/tools/dotc/core", TestFlags(classPath, noCheckOptions)) +
     compileFile("tests/pos/nullarify.scala", defaultOptions.and("-Ycheck:nullarify")) +
     compileFile("tests/pos-scala2/rewrites.scala", scala2Mode.and("-rewrite")).copyToTarget() +
     compileFile("tests/pos-special/utf8encoded.scala", explicitUTF8) +
@@ -111,7 +100,6 @@ class CompilationTests extends ParallelTesting {
 
   @Test def posTwice: Unit = {
     implicit val testGroup: TestGroup = TestGroup("posTwice")
-    compileFile("tests/pos/Labels.scala", defaultOptions) +
     compileFilesInDir("tests/pos-java-interop", defaultOptions) +
     compileFilesInDir("tests/pos-java-interop-separate", defaultOptions) +
     compileFile("tests/pos/t2168.scala", defaultOptions) +
@@ -122,7 +110,6 @@ class CompilationTests extends ParallelTesting {
     compileFile("tests/pos/functions1.scala", defaultOptions) +
     compileFile("tests/pos/implicits1.scala", defaultOptions) +
     compileFile("tests/pos/inferred.scala", defaultOptions) +
-    compileFile("tests/pos/Patterns.scala", defaultOptions) +
     compileFile("tests/pos/selftypes.scala", defaultOptions) +
     compileFile("tests/pos/varargs.scala", defaultOptions) +
     compileFile("tests/pos/vararg-pattern.scala", defaultOptions) +
@@ -147,29 +134,12 @@ class CompilationTests extends ParallelTesting {
     compileFile("tests/pos/i0239.scala", defaultOptions) +
     compileFile("tests/pos/anonClassSubtyping.scala", defaultOptions) +
     compileFile("tests/pos/extmethods.scala", defaultOptions) +
-    compileFile("tests/pos/companions.scala", defaultOptions) +
-    compileList(
-      "testNonCyclic",
-      List(
-        "compiler/src/dotty/tools/dotc/CompilationUnit.scala",
-        "compiler/src/dotty/tools/dotc/core/Types.scala",
-        "compiler/src/dotty/tools/dotc/ast/Trees.scala"
-      ),
-      defaultOptions.and("-Xprompt")
-    ) +
-    compileList(
-      "testIssue34",
-      List(
-        "compiler/src/dotty/tools/dotc/config/Properties.scala",
-        "compiler/src/dotty/tools/dotc/config/PathResolver.scala"
-      ),
-      defaultOptions.and("-Xprompt")
-    )
+    compileFile("tests/pos/companions.scala", defaultOptions)
   }.times(2).checkCompile()
 
   // Negative tests ------------------------------------------------------------
 
-  @Test def compileNeg: Unit = {
+  @Test def negAll: Unit = {
     implicit val testGroup: TestGroup = TestGroup("compileNeg")
     compileFilesInDir("tests/neg", defaultOptions) +
     compileFilesInDir("tests/neg-tailcall", defaultOptions) +
@@ -195,9 +165,7 @@ class CompilationTests extends ParallelTesting {
 
   @Test def runAll: Unit = {
     implicit val testGroup: TestGroup = TestGroup("runAll")
-    compileFilesInDir("tests/run", defaultOptions) +
-    compileFilesInDir("tests/run-with-compiler", defaultRunWithCompilerOptions) +
-    compileFile("tests/run-with-compiler-custom-args/staged-streams_1.scala", defaultRunWithCompilerOptions without "-Yno-deep-subtypes")
+    compileFilesInDir("tests/run", defaultOptions)
   }.checkRuns()
 
   // Generic java signatures tests ---------------------------------------------
@@ -208,34 +176,12 @@ class CompilationTests extends ParallelTesting {
   }.checkRuns()
 
   // Pickling Tests ------------------------------------------------------------
-  //
-  // Pickling tests are very memory intensive and as such need to be run with a
-  // lower level of concurrency as to not kill their running VMs
 
-  @Test def testPickling: Unit = {
+  @Test def pickling: Unit = {
     implicit val testGroup: TestGroup = TestGroup("testPickling")
-    compileDir("compiler/src/dotty/tools", picklingOptions, recursive = false) +
-    compileDir("compiler/src/dotty/tools/dotc", picklingOptions, recursive = false) +
     compileFilesInDir("tests/new", picklingOptions) +
-    compileFilesInDir("tests/pickling", picklingOptions) +
-    compileDir("library/src/dotty/runtime", picklingOptions) +
-    compileDir("compiler/src/dotty/tools/backend/jvm", picklingOptions) +
-    compileDir("compiler/src/dotty/tools/dotc/ast", picklingOptions) +
-    compileDir("compiler/src/dotty/tools/dotc/core", picklingOptions, recursive = false) +
-    compileDir("compiler/src/dotty/tools/dotc/config", picklingOptions) +
-    compileDir("compiler/src/dotty/tools/dotc/parsing", picklingOptions) +
-    compileDir("compiler/src/dotty/tools/dotc/printing", picklingOptions) +
-    compileDir("compiler/src/dotty/tools/repl", picklingOptions) +
-    compileDir("compiler/src/dotty/tools/dotc/rewrite", picklingOptions) +
-    compileDir("compiler/src/dotty/tools/dotc/transform", picklingOptions) +
-    compileDir("compiler/src/dotty/tools/dotc/typer", picklingOptions) +
-    compileDir("compiler/src/dotty/tools/dotc/util", picklingOptions) +
-    compileDir("compiler/src/dotty/tools/io", picklingOptions) +
-    compileFile("tests/pos/pickleinf.scala", picklingOptions) +
-    compileDir("compiler/src/dotty/tools/dotc/core/classfile", picklingOptions) +
-    compileDir("compiler/src/dotty/tools/dotc/core/tasty", picklingOptions) +
-    compileDir("compiler/src/dotty/tools/dotc/core/unpickleScala2", picklingOptions)
-  }.limitThreads(4).checkCompile()
+    compileFilesInDir("tests/pickling", picklingOptions)
+  }.checkCompile()
 
   /** The purpose of this test is two-fold, being able to compile dotty
    *  bootstrapped, and making sure that TASTY can link against a compiled

project/Build.scala

@@ -530,7 +530,8 @@ object Build {
       },
 
       testOptions in Test += Tests.Argument(
-        TestFrameworks.JUnit, "--run-listener=dotty.tools.ContextEscapeDetector"
+        TestFrameworks.JUnit,
+        "--run-listener=dotty.tools.ContextEscapeDetector",
       ),
 
       // Spawn new JVM in run and test
@@ -606,6 +607,9 @@ object Build {
         } else if (scalaLib == "") {
           println("Couldn't find scala-library on classpath, please run using script in bin dir instead")
         } else if (args.contains("-with-compiler")) {
+          if (!isDotty.value) {
+            throw new MessageOnlyException("-with-compiler can only be used with a bootstrapped compiler")
+          }
           val args1 = args.filter(_ != "-with-compiler")
           val asm = findLib(attList, "scala-asm")
           val dottyCompiler = jars("dotty-compiler")
@@ -671,7 +675,12 @@ object Build {
 
     var extraClasspath = dottyLib
     if ((decompile || printTasty) && !args.contains("-classpath")) extraClasspath += ":."
-    if (args0.contains("-with-compiler")) extraClasspath += s":$dottyCompiler"
+    if (args0.contains("-with-compiler")) {
+      if (!isDotty.value) {
+        throw new MessageOnlyException("-with-compiler can only be used with a bootstrapped compiler")
+      }
+      extraClasspath += s":$dottyCompiler"
+    }
 
     val fullArgs = main :: insertClasspathInArgs(args, extraClasspath)
 
@@ -699,7 +708,12 @@ object Build {
           "dotty-library"       -> packageBin.in(`dotty-library-bootstrapped`, Compile).value
         ).mapValues(_.getAbsolutePath)
       }
-    }.value
+    }.value,
+
+    testOptions in Test += Tests.Argument(
+      TestFrameworks.JUnit,
+      "--exclude-categories=dotty.BootstrappedOnlyTests",
+    ),
   )
 
   lazy val bootstrapedDottyCompilerSettings = commonDottyCompilerSettings ++ Seq(

project/scripts/cmdTests

@@ -8,7 +8,6 @@ SBT="./project/scripts/sbt" # if run on CI
 SOURCE="tests/pos/HelloWorld.scala"
 MAIN="HelloWorld"
 EXPECTED_OUTPUT="hello world"
-COMPILER_CP="compiler/target/scala-2.12/classes" # FIXME: This is not very reliable
 
 # check that benchmarks can run
 "$SBT" "dotty-bench/jmh:run 1 1 tests/pos/alias.scala"
@@ -51,7 +50,7 @@ clear_out "$OUT"
 grep -qe "def main(args: scala.Array\[scala.Predef.String\]): scala.Unit =" "$tmp"
 
 echo "testing scala.quoted.Expr.run from sbt dotr"
-"$SBT" ";dotty-compiler/compile ;dotc -classpath $COMPILER_CP tests/run-with-compiler/quote-run.scala; dotr -with-compiler Test" > "$tmp"
+"$SBT" ";dotty-compiler-bootstrapped/dotc -with-compiler tests/run-with-compiler/quote-run.scala; dotty-compiler-bootstrapped/dotr -with-compiler Test" > "$tmp"
 grep -qe "val a: scala.Int = 3" "$tmp"
 
 

tests/neg/Main.scala renamed to tests/neg-with-compiler/Main.scala

@@ -1,8 +1,5 @@
-package dotty.tools.dotc
-package util
+package lst
 
-import printing.{Printer, Texts}
-import Texts.Text
 import collection.mutable.{ListBuffer, StringBuilder}
 import collection.immutable.Map
 import reflect.ClassTag

tests/neg/quote-run-in-macro-1/quoted_1.scala renamed to tests/neg-with-compiler/quote-run-in-macro-1/quoted_1.scala

@@ -1,5 +1,5 @@
 object Test extends App {
-  import dotty.tools.dotc.util.Lst
+  import lst.Lst
 
   val xs0: Lst[String] = Lst.Empty
   val xs1 = Lst("a")