Merge pull request #6194 from dotty-staging/prepare-bootstrap

Preliminary fix needed for the full bootstrap

Author: smarter

build.sbt

@@ -8,6 +8,7 @@ val `dotty-compiler-bootstrapped` = Build.`dotty-compiler-bootstrapped`
 val `dotty-library` = Build.`dotty-library`
 val `dotty-library-bootstrapped` = Build.`dotty-library-bootstrapped`
 val `dotty-sbt-bridge` = Build.`dotty-sbt-bridge`
+val `dotty-sbt-bridge-tests` = Build.`dotty-sbt-bridge-tests`
 val `dotty-language-server` = Build.`dotty-language-server`
 val `dotty-bench` = Build.`dotty-bench`
 val `dotty-bench-bootstrapped` = Build.`dotty-bench-bootstrapped`

compiler/src/dotty/tools/dotc/core/Definitions.scala

@@ -1391,13 +1391,6 @@ class Definitions {
       for (m <- ScalaShadowingPackageClass.info.decls)
         ScalaPackageClass.enter(m)
 
-      // Temporary measure, as long as we do not read these classes from Tasty.
-      // Scala-2 classes don't have NoInits set even if they are pure. We override this
-      // for Product and Serializable so that case classes can be pure. A full solution
-      // requires that we read all Scala code from Tasty.
-      ProductClass.setFlag(NoInits)
-      SerializableClass.setFlag(NoInits)
-
       // force initialization of every symbol that is synthesized or hijacked by the compiler
       val forced = syntheticCoreClasses ++ syntheticCoreMethods ++ ScalaValueClasses()
 

compiler/src/dotty/tools/dotc/core/StdNames.scala

@@ -197,6 +197,7 @@ object StdNames {
     final val Nothing: N             = "Nothing"
     final val Null: N                = "Null"
     final val Object: N              = "Object"
+    final val Product: N             = "Product"
     final val PartialFunction: N     = "PartialFunction"
     final val PrefixType: N          = "PrefixType"
     final val S: N                   = "S"

compiler/src/dotty/tools/dotc/core/classfile/ClassfileParser.scala

@@ -804,7 +804,7 @@ class ClassfileParser(
               ctx.error("Could not load TASTY from .tasty for virtual file " + classfile)
               Array.empty
             case Some(jar: ZipArchive) => // We are in a jar
-              val cl = new URLClassLoader(Array(jar.jpath.toUri.toURL))
+              val cl = new URLClassLoader(Array(jar.jpath.toUri.toURL), /*parent =*/ null)
               val path = classfile.path.stripSuffix(".class") + ".tasty"
               val stream = cl.getResourceAsStream(path)
               if (stream != null) {

compiler/src/dotty/tools/dotc/core/unpickleScala2/Scala2Unpickler.scala

@@ -467,6 +467,14 @@ class Scala2Unpickler(bytes: Array[Byte], classRoot: ClassDenotation, moduleClas
     def completeRoot(denot: ClassDenotation, completer: LazyType): Symbol = {
       denot.setFlag(flags)
       denot.resetFlag(Touched) // allow one more completion
+
+      // Temporary measure, as long as we do not read these classes from Tasty.
+      // Scala-2 classes don't have NoInits set even if they are pure. We override this
+      // for Product and Serializable so that case classes can be pure. A full solution
+      // requires that we read all Scala code from Tasty.
+      if (owner == defn.ScalaPackageClass && ((name eq tpnme.Serializable) || (name eq tpnme.Product)))
+        denot.setFlag(NoInits)
+
       denot.info = completer
       denot.symbol
     }

compiler/test/dotty/Properties.scala

@@ -1,5 +1,7 @@
 package dotty
 
+import java.nio.file._
+
 /** Runtime properties from defines or environmnent */
 object Properties {
 
@@ -36,6 +38,9 @@ object Properties {
    */
   val testsSafeMode: Boolean = sys.props.isDefinedAt("dotty.tests.safemode")
 
+  /** Extra directory containing sources for the compiler */
+  def dottyCompilerManagedSources: Path = Paths.get(sys.props("dotty.tests.dottyCompilerManagedSources"))
+
   /** dotty-interfaces jar */
   def dottyInterfaces: String = sys.props("dotty.tests.classes.dottyInterfaces")
 

compiler/test/dotty/tools/TestSources.scala

@@ -62,7 +62,7 @@ object TestSources {
       (fileName.endsWith(".scala") || fileName.endsWith(".java")) && !excludedFiles.contains(fileName)
     }
 
-    assert(Files.isDirectory(path))
+    assert(Files.isDirectory(path), s"Not a directory: $path")
     val files = if (shallow) Files.list(path) else Files.walk(path)
     try {
       val sources = files

compiler/test/dotty/tools/dotc/BootstrappedOnlyCompilationTests.scala

@@ -10,6 +10,8 @@ import org.junit.experimental.categories.Category
 import scala.concurrent.duration._
 import vulpix._
 
+import java.nio.file._
+
 @Category(Array(classOf[BootstrappedOnlyTests]))
 class BootstrappedOnlyCompilationTests extends ParallelTesting {
   import ParallelTesting._
@@ -109,4 +111,27 @@ class BootstrappedOnlyCompilationTests extends ParallelTesting {
     compileDir("compiler/src/dotty/tools/dotc/core/tasty", picklingWithCompilerOptions) +
     compileDir("compiler/src/dotty/tools/dotc/core/unpickleScala2", picklingWithCompilerOptions)
   }.limitThreads(4).checkCompile()
+
+  @Test def testPlugins: Unit = {
+    val pluginFile = "plugin.properties"
+
+    // 1. hack with absolute path for -Xplugin
+    // 2. copy `pluginFile` to destination
+    def compileFilesInDir(dir: String): CompilationTest = {
+      val outDir = defaultOutputDir + "testPlugins/"
+      val sourceDir = new java.io.File(dir)
+
+      val dirs = sourceDir.listFiles.toList.filter(_.isDirectory)
+      val targets = dirs.map { dir =>
+        val compileDir = createOutputDirsForDir(dir, sourceDir, outDir)
+        Files.copy(dir.toPath.resolve(pluginFile), compileDir.toPath.resolve(pluginFile), StandardCopyOption.REPLACE_EXISTING)
+        val flags = TestFlags(withCompilerClasspath, noCheckOptions).and("-Xplugin:" + compileDir.getAbsolutePath)
+        SeparateCompilationSource("testPlugins", dir, flags, compileDir)
+      }
+
+      new CompilationTest(targets)
+    }
+
+    compileFilesInDir("tests/plugins/neg").checkExpectedErrors()
+  }
 }

compiler/test/dotty/tools/dotc/CompilationTests.scala

@@ -239,12 +239,10 @@ class CompilationTests extends ParallelTesting {
         defaultOptions.and("-Ycheck-reentrant", "-strict", "-priorityclasspath", defaultOutputDir))(libGroup)
 
     val compilerSources = sources(Paths.get("compiler/src"))
+    val compilerManagedSources = sources(Properties.dottyCompilerManagedSources)
 
-    val scalaJSIRDir = Paths.get("compiler/target/scala-2.12/src_managed/main/scalajs-ir-src/org/scalajs/ir")
-    val scalaJSIRSources = sources(scalaJSIRDir, shallow = true)
-
-    val dotty1 = compileList("dotty", compilerSources ++ scalaJSIRSources, opt)(dotty1Group)
-    val dotty2 = compileList("dotty", compilerSources ++ scalaJSIRSources, opt)(dotty2Group)
+    val dotty1 = compileList("dotty", compilerSources ++ compilerManagedSources, opt)(dotty1Group)
+    val dotty2 = compileList("dotty", compilerSources ++ compilerManagedSources, opt)(dotty2Group)
 
     val tests = {
       lib.keepOutput :: dotty1.keepOutput :: {
@@ -262,7 +260,7 @@ class CompilationTests extends ParallelTesting {
         compileShallowFilesInDir("compiler/src/dotty/tools/dotc/util", opt) +
         compileShallowFilesInDir("compiler/src/dotty/tools/backend", opt) +
         compileShallowFilesInDir("compiler/src/dotty/tools/backend/jvm", opt) +
-        compileList("shallow-scalajs-ir", scalaJSIRSources, opt)
+        compileList("managed-sources", compilerManagedSources, opt)
       }.keepOutput :: Nil
     }.map(_.checkCompile())
 
@@ -274,29 +272,6 @@ class CompilationTests extends ParallelTesting {
 
     tests.foreach(_.delete())
   }
-
-  @Test def testPlugins: Unit = {
-    val pluginFile = "plugin.properties"
-
-    // 1. hack with absolute path for -Xplugin
-    // 2. copy `pluginFile` to destination
-    def compileFilesInDir(dir: String): CompilationTest = {
-      val outDir = defaultOutputDir + "testPlugins/"
-      val sourceDir = new java.io.File(dir)
-
-      val dirs = sourceDir.listFiles.toList.filter(_.isDirectory)
-      val targets = dirs.map { dir =>
-        val compileDir = createOutputDirsForDir(dir, sourceDir, outDir)
-        Files.copy(dir.toPath.resolve(pluginFile), compileDir.toPath.resolve(pluginFile), StandardCopyOption.REPLACE_EXISTING)
-        val flags = TestFlags(withCompilerClasspath, noCheckOptions).and("-Xplugin:" + compileDir.getAbsolutePath)
-        SeparateCompilationSource("testPlugins", dir, flags, compileDir)
-      }
-
-      new CompilationTest(targets)
-    }
-
-    compileFilesInDir("tests/plugins/neg").checkExpectedErrors()
-  }
 }
 
 object CompilationTests {

compiler/test/dotty/tools/dotc/InterfaceEntryPointTest.scala

@@ -3,6 +3,7 @@ package tools.dotc
 
 import org.junit.Test
 import org.junit.Assert._
+import org.junit.experimental.categories.Category
 import interfaces._
 import scala.collection.mutable.ListBuffer
 import java.nio.file._
@@ -18,6 +19,7 @@ import java.nio.file._
  *
  *  @see [[OtherEntryPointsTest]]
  */
+@Category(Array(classOf[BootstrappedOnlyTests]))
 class InterfaceEntryPointTest {
   @Test def runCompilerFromInterface = {
     val sources =

project/Build.scala

@@ -461,6 +461,7 @@ object Build {
           else List()
 
         val jarOpts = List(
+          "-Ddotty.tests.dottyCompilerManagedSources=" + (sourceManaged in Compile).value,
           "-Ddotty.tests.classes.dottyInterfaces=" + jars("dotty-interfaces"),
           "-Ddotty.tests.classes.dottyLibrary=" + jars("dotty-library"),
           "-Ddotty.tests.classes.dottyCompiler=" + jars("dotty-compiler"),
@@ -679,19 +680,39 @@ object Build {
     case Bootstrapped => `dotty-library-bootstrapped`
   }
 
-  lazy val `dotty-sbt-bridge` = project.in(file("sbt-bridge")).
-    dependsOn(dottyCompiler(NonBootstrapped) % Provided).
+  lazy val `dotty-sbt-bridge` = project.in(file("sbt-bridge/src")).
+    // We cannot depend on any bootstrapped project to compile the bridge, since the
+    // bridge is needed to compile these projects.
     dependsOn(dottyDoc(NonBootstrapped) % Provided).
     settings(commonJavaSettings).
     settings(
       description := "sbt compiler bridge for Dotty",
-      libraryDependencies ++= Seq(
-        Dependencies.`compiler-interface` % Provided,
-        (Dependencies.`zinc-api-info` % Test).withDottyCompat(scalaVersion.value)
-      ),
+
+      sources in Test := Seq(),
+      scalaSource in Compile := baseDirectory.value,
+      javaSource  in Compile := baseDirectory.value,
+
+      // Referring to the other project using a string avoids an infinite loop
+      // when sbt reads the settings.
+      test in Test := (test in (LocalProject("dotty-sbt-bridge-tests"), Test)).value,
+
+      libraryDependencies += Dependencies.`compiler-interface` % Provided
+    )
+
+  // We use a separate project for the bridge tests since they can only be run
+  // with the bootstrapped library on the classpath.
+  lazy val `dotty-sbt-bridge-tests` = project.in(file("sbt-bridge/test")).
+    dependsOn(dottyCompiler(Bootstrapped) % Test).
+    settings(commonBootstrappedSettings).
+    settings(
+      sources in Compile := Seq(),
+      scalaSource in Test := baseDirectory.value,
+      javaSource  in Test := baseDirectory.value,
 
       fork in Test := true,
-      parallelExecution in Test := false
+      parallelExecution in Test := false,
+
+      libraryDependencies += (Dependencies.`zinc-api-info` % Test).withDottyCompat(scalaVersion.value)
     )
 
   lazy val `dotty-language-server` = project.in(file("language-server")).
@@ -765,6 +786,7 @@ object Build {
    */
   lazy val sjsSandbox = project.in(file("sandbox/scalajs")).
     enablePlugins(ScalaJSPlugin).
+    dependsOn(dottyLibrary(Bootstrapped)).
     settings(commonBootstrappedSettings).
     settings(
       /* Remove the Scala.js compiler plugin for scalac, and enable the
@@ -787,8 +809,7 @@ object Build {
       scalaJSLinkerConfig ~= {
         _.withCheckIR(true).withParallel(false)
       }
-    ).
-    settings(compileWithDottySettings)
+    )
 
   lazy val `dotty-bench` = project.in(file("bench")).asDottyBench(NonBootstrapped)
   lazy val `dotty-bench-bootstrapped` = project.in(file("bench")).asDottyBench(Bootstrapped)
@@ -1035,102 +1056,6 @@ object Build {
     )
   )
 
-  // Compile with dotty
-  lazy val compileWithDottySettings = {
-    inConfig(Compile)(inTask(compile)(Defaults.runnerTask) ++ Seq(
-      // Compile with dotty
-      fork in compile := true,
-
-      compile := {
-        val inputs = (compileInputs in compile).value
-        val inputOptions = inputs.options()
-        import inputOptions._
-
-        val s = streams.value
-        val logger = s.log
-        val cacheDir = s.cacheDirectory
-
-        // Discover classpaths
-
-        def cpToString(cp: Seq[File]) =
-          cp.map(_.getAbsolutePath).mkString(File.pathSeparator)
-
-        val compilerCp = Attributed.data((fullClasspath in (`dotty-compiler`, Compile)).value)
-        val cpStr = cpToString(classpath ++ compilerCp)
-
-        // List all my dependencies (recompile if any of these changes)
-
-        val allMyDependencies = classpath filterNot (_ == classesDirectory) flatMap { cpFile =>
-          if (cpFile.isDirectory) (cpFile ** "*.class").get
-          else Seq(cpFile)
-        }
-
-        // Compile
-
-        val run = (runner in compile).value
-        val cachedCompile = FileFunction.cached(cacheDir / "compile",
-            FilesInfo.lastModified, FilesInfo.exists) { dependencies =>
-
-          logger.info(
-              "Compiling %d Scala sources to %s..." format (
-              sources.size, classesDirectory))
-
-          if (classesDirectory.exists)
-            IO.delete(classesDirectory)
-          IO.createDirectory(classesDirectory)
-
-          val sourcesArgs = sources.map(_.getAbsolutePath()).toList
-
-          /* run.run() below in doCompile() will emit a call to its
-           * logger.info("Running dotty.tools.dotc.Main [...]")
-           * which we do not want to see. We use this patched logger to
-           * filter out that particular message.
-           */
-          val patchedLogger = new Logger {
-            def log(level: Level.Value, message: => String) = {
-              val msg = message
-              if (level != Level.Info ||
-                  !msg.startsWith("Running dotty.tools.dotc.Main"))
-                logger.log(level, msg)
-            }
-            def success(message: => String) = logger.success(message)
-            def trace(t: => Throwable) = logger.trace(t)
-          }
-
-          def doCompile(sourcesArgs: List[String]): Unit = {
-            run.run("dotty.tools.dotc.Main", compilerCp,
-                "-classpath" :: cpStr ::
-                "-d" :: classesDirectory.getAbsolutePath ::
-                scalacOptions ++:
-                sourcesArgs,
-                patchedLogger)
-          }
-
-          // Work around the Windows limitation on command line length.
-          val isWindows =
-            System.getProperty("os.name").toLowerCase().indexOf("win") >= 0
-          if ((fork in compile).value && isWindows &&
-              (sourcesArgs.map(_.length).sum > 1536)) {
-            IO.withTemporaryFile("sourcesargs", ".txt") { sourceListFile =>
-              IO.writeLines(sourceListFile, sourcesArgs)
-              doCompile(List("@"+sourceListFile.getAbsolutePath))
-            }
-          } else {
-            doCompile(sourcesArgs)
-          }
-
-          // Output is all files in classesDirectory
-          (classesDirectory ** AllPassFilter).get.toSet
-        }
-
-        cachedCompile((sources ++ allMyDependencies).toSet)
-
-        // We do not have dependency analysis when compiling externally
-        sbt.internal.inc.Analysis.Empty
-      }
-    ))
-  }
-
   lazy val commonDistSettings = Seq(
     packMain := Map(),
     publishArtifact := false,