Always run the compiler with a bootstrapped dotty-library

The non-bootstrapped dotty-library may not be binary compatible with the
compiler, this hasn't affected us so far because the non-bootstrapped
dotty-library has always been compiled with Scala 2, but in the future
it will be compiled by an old version of Dotty itself.

To enforce this, I added a new flag -Yscala2-unpickler and set it so
that scala-library is the only jar from which we're allowed to unpickle
Scala 2 symbols in both the build and the tests.

Author: smarter

compiler/src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -94,6 +94,8 @@ class ScalaSettings extends Settings.SettingGroup {
   val YlogClasspath = BooleanSetting("-Ylog-classpath", "Output information about what classpath is being applied.")
   val YdisableFlatCpCaching  = BooleanSetting("-YdisableFlatCpCaching", "Do not cache flat classpath representation of classpath elements from jars across compiler instances.")
 
+  val Yscala2Unpickler = StringSetting("-Yscala2-unpickler", "", "Control where we may get Scala 2 symbols from. This is either \"always\", \"never\", or a classpath.", "always")
+
   val YnoImports = BooleanSetting("-Yno-imports", "Compile without importing scala.*, java.lang.*, or Predef.")
   val YnoInline = BooleanSetting("-Yno-inline", "Suppress inlining.")
   val YnoGenericSig = BooleanSetting("-Yno-generic-signatures", "Suppress generation of generic signatures for Java.")

compiler/src/dotty/tools/dotc/core/classfile/ClassfileParser.scala

@@ -755,6 +755,20 @@ class ClassfileParser(
       }
 
       def unpickleScala(bytes: Array[Byte]): Some[Embedded] = {
+        val allowed = ctx.settings.Yscala2Unpickler.value
+
+        def failUnless(cond: Boolean) =
+          assert(cond,
+            s"Unpickling ${classRoot.symbol.showLocated} from ${classRoot.symbol.associatedFile} is not allowed with -Yscala2-unpickler $allowed")
+
+        if (allowed != "always") {
+          failUnless(allowed != "never")
+          val allowedList = allowed.split(":").toList
+          val file = classRoot.symbol.associatedFile
+          // Using `.toString.contains` isn't great, but it's good enough for a debug flag.
+          failUnless(file == null || allowedList.exists(path => file.toString.contains(path)))
+        }
+
         val unpickler = new unpickleScala2.Scala2Unpickler(bytes, classRoot, moduleRoot)(ctx)
         unpickler.run()(ctx.addMode(Scala2UnpicklingMode))
         Some(unpickler)

compiler/test/dotty/Jars.scala

@@ -18,6 +18,10 @@ object Jars {
   lazy val scalaAsm: String =
     findJarFromRuntime("scala-asm-6.0.0-scala-1")
 
+  /** scala-xml jar */
+  lazy val scalaXml: String =
+    findJarFromRuntime("scala-xml")
+
   /** JLine Jar */
   lazy val jline: String =
     findJarFromRuntime("jline-3.7.0")

compiler/test/dotty/tools/vulpix/TestConfiguration.scala

@@ -10,6 +10,7 @@ object TestConfiguration {
   )
 
   val checkOptions = Array(
+    "-Yscala2-unpickler", s"${Jars.scalaLibrary}:${Jars.scalaXml}",
     "-Yno-deep-subtypes",
     "-Yno-double-bindings",
     "-Yforce-sbt-phases",

project/Build.scala

@@ -227,6 +227,13 @@ object Build {
     // otherwise sbt 0.13 incremental compilation breaks (https://github.com/sbt/sbt/issues/3142)
     scalacOptions ++= Seq("-bootclasspath", sys.props("sun.boot.class.path")),
 
+    // Enforce that the only Scala 2 classfiles we unpickle come from scala-library
+    scalacOptions ++= {
+      val attList = (dependencyClasspath in `dotty-library` in Compile).value
+      val scalaLib = findLib(attList, "scala-library")
+      Seq("-Yscala2-unpickler", scalaLib)
+    },
+
     // sbt gets very unhappy if two projects use the same target
     target := baseDirectory.value / ".." / "out" / "bootstrap" / name.value,
 
@@ -427,6 +434,11 @@ object Build {
     (testOnly in Test).toTask(cmd)
   }
 
+  def findLib(attList: Seq[Attributed[File]], name: String) = attList
+    .map(_.data.getAbsolutePath)
+    .find(_.contains(name))
+    .toList.mkString(":")
+
   // Settings shared between dotty-compiler and dotty-compiler-bootstrapped
   lazy val commonDottyCompilerSettings = Seq(
 
@@ -573,20 +585,15 @@ object Build {
         jarOpts ::: tuning ::: agentOptions ::: ci_build ::: path.toList
       },
 
-      testCompilation := testOnlyFiltered("dotty.tools.dotc.CompilationTests", "--exclude-categories=dotty.SlowTests").evaluated,
+      testCompilation := testOnlyFiltered("dotty.tools.dotc.*CompilationTests", "--exclude-categories=dotty.SlowTests").evaluated,
       testFromTasty := testOnlyFiltered("dotty.tools.dotc.FromTastyTests", "").evaluated,
 
       dotr := {
         val args: List[String] = spaceDelimited("<arg>").parsed.toList
         val attList = (dependencyClasspath in Runtime).value
         val jars = packageAll.value
 
-        def findLib(name: String) = attList
-          .map(_.data.getAbsolutePath)
-          .find(_.contains(name))
-          .toList.mkString(":")
-
-        val scalaLib = findLib("scala-library")
+        val scalaLib = findLib(attList, "scala-library")
         val dottyLib = jars("dotty-library")
 
         def run(args: List[String]): Unit = {
@@ -600,7 +607,7 @@ object Build {
           println("Couldn't find scala-library on classpath, please run using script in bin dir instead")
         } else if (args.contains("-with-compiler")) {
           val args1 = args.filter(_ != "-with-compiler")
-          val asm = findLib("scala-asm")
+          val asm = findLib(attList, "scala-asm")
           val dottyCompiler = jars("dotty-compiler")
           val dottyInterfaces = jars("dotty-interfaces")
           run(insertClasspathInArgs(args1, s"$dottyCompiler:$dottyInterfaces:$asm"))
@@ -679,20 +686,26 @@ object Build {
 
   lazy val nonBootstrapedDottyCompilerSettings = commonDottyCompilerSettings ++ Seq(
     // packageAll packages all and then returns a map with the abs location
-    packageAll := {
-      Map(
-        "dotty-interfaces"    -> packageBin.in(`dotty-interfaces`, Compile).value,
-        "dotty-compiler"      -> packageBin.in(Compile).value,
-        "dotty-library"       -> packageBin.in(`dotty-library`, Compile).value
-      ).mapValues(_.getAbsolutePath)
-    }
+    packageAll := Def.taskDyn { // Use a dynamic task to avoid loops when loading the settings
+      Def.task {
+        Map(
+          "dotty-interfaces"    -> packageBin.in(`dotty-interfaces`, Compile).value,
+          "dotty-compiler"      -> packageBin.in(Compile).value,
+
+          // NOTE: Using dotty-library-bootstrapped here is intentional: when
+          // running the compiler, we should always have the bootstrapped
+          // library on the compiler classpath since the non-bootstrapped one
+          // may not be binary-compatible.
+          "dotty-library"       -> packageBin.in(`dotty-library-bootstrapped`, Compile).value
+        ).mapValues(_.getAbsolutePath)
+      }
+    }.value
   )
 
   lazy val bootstrapedDottyCompilerSettings = commonDottyCompilerSettings ++ Seq(
     packageAll := {
       packageAll.in(`dotty-compiler`).value ++ Seq(
-        "dotty-compiler" -> packageBin.in(Compile).value.getAbsolutePath,
-        "dotty-library"  -> packageBin.in(`dotty-library-bootstrapped`, Compile).value.getAbsolutePath
+        "dotty-compiler" -> packageBin.in(Compile).value.getAbsolutePath
       )
     }
   )