Merge pull request #2260 from dotty-staging/backend-submodule

Fix #2184: Hyper Bootstrap! Integrate the backend as a git submodule

Author: smarter

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "scala-backend"]
+	path = scala-backend
+	url = https://github.com/lampepfl/scala.git
+	branch = sharing-backend

bin/common

@@ -10,7 +10,7 @@ function getLastStringOnLineWith {
 # Configuration
 SCALA_VERSION=$(getLastStringOnLineWith "val scalacVersion")
 SCALA_BINARY_VERSION=2.11
-SCALA_COMPILER_VERSION=$(getLastStringOnLineWith "scala-compiler")
+SCALA_ASM_VERSION=$(getLastStringOnLineWith "% \"scala-asm\" %")
 SBT_VERSION=$(grep "sbt.version=" "$DOTTY_ROOT/project/build.properties" | sed 's/sbt.version=//')
 bootcp=true
 bootstrapped=false
@@ -146,8 +146,8 @@ if [ "$SCALA_REFLECT_JAR" == "" ]; then
     SCALA_REFLECT_JAR=$(find_jar "$HOME/.ivy2/cache/org.scala-lang/scala-reflect/jars" "scala-reflect-$SCALA_VERSION.jar")
 fi
 
-if [ "$SCALA_COMPILER_JAR" == "" ]; then
-    SCALA_COMPILER_JAR=$(find_jar "$HOME/.ivy2/cache/me.d-d/scala-compiler/jars" "scala-compiler-$SCALA_COMPILER_VERSION.jar")
+if [ "$SCALA_ASM_JAR" == "" ]; then
+    SCALA_ASM_JAR=$(find_jar "$HOME/.ivy2/cache/org.scala-lang.modules/scala-asm/bundles" "scala-asm-$SCALA_ASM_VERSION.jar")
 fi
 
 if [ "$SBT_INTERFACE_JAR" == "" ]; then

bin/dotc

@@ -20,12 +20,12 @@ CompilerMain=dotty.tools.dotc.Main
 FromTasty=dotty.tools.dotc.FromTasty
 ReplMain=dotty.tools.dotc.repl.Main
 
-if [ ! -f "$SCALA_LIBRARY_JAR" -o ! -f "$SCALA_REFLECT_JAR"  -o ! -f "$SCALA_COMPILER_JAR" -o ! -f "$SBT_INTERFACE_JAR" ]
+if [ ! -f "$SCALA_LIBRARY_JAR" -o ! -f "$SCALA_REFLECT_JAR"  -o ! -f "$SCALA_ASM_JAR" -o ! -f "$SBT_INTERFACE_JAR" ]
 then
   echo To use this script please set
   echo SCALA_LIBRARY_JAR to point to scala-library-$SCALA_VERSION.jar "(currently $SCALA_LIBRARY_JAR)"
   echo SCALA_REFLECT_JAR to point to scala-reflect-$SCALA_VERSION.jar "(currently $SCALA_REFLECT_JAR)"
-  echo SCALA_COMPILER_JAR to point to scala-compiler-$SCALA_VERSION.jar with bcode patches "(currently $SCALA_COMPILER_JAR)"
+  echo SCALA_ASM_JAR to point to scala-asm-$SCALA_ASM_VERSION.jar "(currently $SCALA_ASM_JAR)"
   echo SBT_INTERFACE_JAR to point to interface-$SBT_VERSION.jar "(currently $SBT_INTERFACE_JAR)"
 fi
 
@@ -116,9 +116,9 @@ trap onExit INT
 classpathArgs () {
   if [[ "true" == "$bootstrapped" ]]; then
     check_jar "dotty-bootstrapped" "$DOTTY_JAR" "target" 'build_jar "test:runMain dotc.build" target' &> /dev/null
-    toolchain="$DOTTY_JAR:$DOTTY_LIB_JAR:$SCALA_LIBRARY_JAR:$SCALA_REFLECT_JAR:$SCALA_COMPILER_JAR:$SBT_INTERFACE_JAR"
+    toolchain="$DOTTY_JAR:$DOTTY_LIB_JAR:$SCALA_LIBRARY_JAR:$SCALA_REFLECT_JAR:$SCALA_ASM_JAR:$SBT_INTERFACE_JAR"
   else
-    toolchain="$SCALA_LIBRARY_JAR:$DOTTY_LIB_JAR:$SCALA_REFLECT_JAR:$SCALA_COMPILER_JAR:$SBT_INTERFACE_JAR"
+    toolchain="$SCALA_LIBRARY_JAR:$DOTTY_LIB_JAR:$SCALA_REFLECT_JAR:$SCALA_ASM_JAR:$SBT_INTERFACE_JAR"
   fi
   bcpJars="$INTERFACES_JAR:$MAIN_JAR:$DOTTY_LIB_JAR"
   cpJars="$INTERFACES_JAR:$MAIN_JAR:$DOTTY_LIB_JAR:$TEST_JAR"

compiler/src/dotty/tools/backend/jvm/GenBCode.scala

@@ -8,7 +8,6 @@ import dotty.tools.dotc.core.Names.TypeName
 
 import scala.collection.mutable
 import scala.tools.asm.{ClassVisitor, CustomAttr, FieldVisitor, MethodVisitor}
-import scala.tools.nsc.Settings
 import scala.tools.nsc.backend.jvm._
 import dotty.tools.dotc
 import dotty.tools.dotc.backend.jvm.DottyPrimitives
@@ -35,7 +34,6 @@ import tpd._
 import StdNames._
 
 import scala.reflect.io.{AbstractFile, Directory, PlainDirectory}
-import scala.tools.nsc.backend.jvm.opt.LocalOpt
 
 class GenBCode extends Phase {
   def phaseName: String = "genBCode"
@@ -246,10 +244,10 @@ class GenBCodePipeline(val entryPoints: List[Symbol], val int: DottyBackendInter
      *          - converting the plain ClassNode to byte array and placing it on queue-3
      */
     class Worker2 {
-      lazy val localOpt = new LocalOpt(new Settings())
+      // lazy val localOpt = new LocalOpt(new Settings())
 
       def localOptimizations(classNode: ClassNode): Unit = {
-        /*BackendStats.timed(BackendStats.methodOptTimer)*/(localOpt.methodOptimizations(classNode))
+        // BackendStats.timed(BackendStats.methodOptTimer)(localOpt.methodOptimizations(classNode))
       }
 
       def run(): Unit = {

compiler/src/dotty/tools/backend/jvm/scalaPrimitives.scala

@@ -37,7 +37,7 @@ import core.Symbols.{Symbol, NoSymbol}
  * Inspired from the `scalac` compiler.
  */
 class DottyPrimitives(ctx: Context) {
-  import scala.tools.nsc.backend.ScalaPrimitives._
+  import scala.tools.nsc.backend.ScalaPrimitivesOps._
 
   private lazy val primitives: immutable.Map[Symbol, Int] = init
 

compiler/src/scala/tools/nsc/io/package.scala

@@ -0,0 +1,27 @@
+package scala.tools.nsc
+
+/**
+ * Compatibility layer needed for the backend.
+ *
+ * Our backend is based on the Scala 2.11 GenBCode backend and modified so that
+ * it compiles both with dotty and scalac, since the backend uses
+ * scala.tools.nsc.io.*, we need to also provide it.
+ *
+ * See http://dotty.epfl.ch/docs/contributing/backend.html for more information.
+ */
+package object io {
+  type AbstractFile = scala.reflect.io.AbstractFile
+  val AbstractFile = scala.reflect.io.AbstractFile
+
+  type Directory = scala.reflect.io.Directory
+  val Directory = scala.reflect.io.Directory
+
+  type Path = scala.reflect.io.Path
+  val Path = scala.reflect.io.Path
+
+  type File = scala.reflect.io.File
+  val File = scala.reflect.io.File
+
+  type Jar = dotty.tools.io.Jar
+  val Jar = dotty.tools.io.Jar
+}

compiler/test/dotc/tests.scala

@@ -358,7 +358,8 @@ class tests extends CompilerTest {
   @Test def tasty_dotc_util = compileDir(dotcDir, "util", testPickling)
   @Test def tasty_tools_io = compileDir(toolsDir, "io", testPickling)
 
-  @Test def tasty_bootstrap = {
+  // Disabled, not worth porting since we're getting rid of the old JUnit tests soon.
+  /*@Test*/ def tasty_bootstrap = {
     val logging = if (false) List("-Ylog-classpath", "-verbose") else Nil
     val opt = List("-priorityclasspath", defaultOutputDir) ++ logging
     // first compile dotty

compiler/test/dotty/tools/TypeStealer.scala

@@ -11,7 +11,7 @@ import asm._
 import asm.tree._
 import scala.collection.JavaConverters._
 
-import scala.tools.nsc.util.JavaClassPath
+import io.JavaClassPath
 import scala.collection.JavaConverters._
 import scala.tools.asm.{ClassWriter, ClassReader}
 import scala.tools.asm.tree._

compiler/test/dotty/tools/backend/jvm/DottyBytecodeTest.scala

@@ -4,11 +4,15 @@ package dotc
 
 import org.junit.{ Test, BeforeClass, AfterClass }
 
+import java.nio.file._
+import java.util.stream.{ Stream => JStream }
+import scala.collection.JavaConverters._
 import scala.util.matching.Regex
 import scala.concurrent.duration._
 
 import vulpix.{ ParallelTesting, SummaryReport, SummaryReporting, TestConfiguration }
 
+
 class CompilationTests extends ParallelTesting {
   import TestConfiguration._
   import CompilationTests._
@@ -219,16 +223,44 @@ class CompilationTests extends ParallelTesting {
       // compile with bootstrapped library on cp:
       defaultOutputDir + "lib/src/:" +
       // as well as bootstrapped compiler:
-      defaultOutputDir + "dotty1/dotty/:" +
+      defaultOutputDir + "dotty1/dotty1/:" +
       Jars.dottyInterfaces
     )
 
     def lib =
       compileDir("../library/src",
         allowDeepSubtypes.and("-Ycheck-reentrant", "-strict", "-priorityclasspath", defaultOutputDir))
 
-    def dotty1 =
-      compileDir("../compiler/src/dotty", opt)
+    def sources(paths: JStream[Path], excludedFiles: List[String] = Nil): List[String] =
+      paths.iterator().asScala
+        .filter(path =>
+          (path.toString.endsWith(".scala") || path.toString.endsWith(".java"))
+            && !excludedFiles.contains(path.getFileName.toString))
+        .map(_.toString).toList
+
+    val compilerDir = Paths.get("../compiler/src")
+    val compilerSources = sources(Files.walk(compilerDir))
+
+    val backendDir = Paths.get("../scala-backend/src/compiler/scala/tools/nsc/backend")
+    val backendJvmDir = Paths.get("../scala-backend/src/compiler/scala/tools/nsc/backend/jvm")
+
+    // NOTE: Keep these exclusions synchronized with the ones in the sbt build (Build.scala)
+    val backendExcluded =
+      List("JavaPlatform.scala", "Platform.scala", "ScalaPrimitives.scala")
+    val backendJvmExcluded =
+      List("BCodeICodeCommon.scala", "GenASM.scala", "GenBCode.scala", "ScalacBackendInterface.scala")
+
+    val backendSources =
+      sources(Files.list(backendDir), excludedFiles = backendExcluded)
+    val backendJvmSources =
+      sources(Files.list(backendJvmDir), excludedFiles = backendJvmExcluded)
+
+    def dotty1 = {
+      compileList(
+        "dotty1",
+        compilerSources ++ backendSources ++ backendJvmSources,
+        opt)
+    }
 
     def dotty2 =
       compileShallowFilesInDir("../compiler/src/dotty", opt)
@@ -247,7 +279,9 @@ class CompilationTests extends ParallelTesting {
         compileShallowFilesInDir("../compiler/src/dotty/tools/dotc/rewrite", opt) +
         compileShallowFilesInDir("../compiler/src/dotty/tools/dotc/transform", opt) +
         compileShallowFilesInDir("../compiler/src/dotty/tools/dotc/typer", opt) +
-        compileShallowFilesInDir("../compiler/src/dotty/tools/dotc/util", opt)
+        compileShallowFilesInDir("../compiler/src/dotty/tools/dotc/util", opt) +
+        compileList("shallow-backend", backendSources, opt) +
+        compileList("shallow-backend-jvm", backendJvmSources, opt)
       } :: Nil
     }.map(_.checkCompile()).foreach(_.delete())
   }

compiler/test/dotty/tools/dotc/CompilationTests.scala

@@ -0,0 +1,65 @@
+---
+layout: doc-page
+title: Working with the backend
+---
+
+The compiler backend is based on a fork of the Scala 2.11 `GenBCode` backend and
+lives at https://github.com/lampepfl/scala/tree/sharing-backend. The dotty
+source tree contains a git submodule in the directory
+[scala-backend](https://github.com/lampepfl/dotty/tree/master/scala-backend)
+that points to this fork. We do not compile every file in this submodule,
+instead we add the subset of files we need to the dotty-compiler project in the
+sbt build.
+
+The most important thing to know when working with git submodules is that
+their content is not automatically updated when you do `git checkout` or `git
+pull`, instead everytime you switch branch, you need to do:
+
+``` shell
+git submodule update --init
+```
+
+## Environment setup
+
+1. Set the following git configuration options to make working with submodules
+easier, see the [Git Book](https://git-scm.com/book/en/v2/Git-Tools-Submodules)
+for more information:
+``` shell
+git config --global diff.submodule log
+git config --global status.submodulesummary 1
+git config --global push.recurseSubmodules check
+```
+
+2. Fork https://github.com/lampepfl/scala (in the following commands,
+   `dotty-staging/scala` is used as a placeholder for your
+   own fork).
+3. ```shell
+   cd scala-backend
+   git remote add scala-staging [email protected]:dotty-staging/scala.git
+   cd ..
+   ```
+
+## Workflow when changing to the backend
+
+```shell
+cd scala-backend
+git checkout -b my-feature-branch
+# Make some changes ...
+
+git push -u scala-backend
+# Open a PR against https://github.com/lampepfl/scala/tree/sharing-backend
+cd ..
+```
+
+Once your PR has been merged into the backend, you'll need to make another PR
+against dotty itself to update the backend, the following commands should be run
+in the root dotty repository, not in the submodule:
+
+``` shell
+# The --remote option will update the submodule to the latest commit in the
+# https://github.com/lampepfl/dotty/tree/master/scala-backend branch
+git submodule update --init --remote
+
+git commit -am "Update backend to include ..."
+# Then push and make a PR against dotty as usual
+```

docs/docs/contributing/backend.md

@@ -5,31 +5,19 @@ title: Building Dotty with Eclipse
 
 Build setup
 -----------
-You may need to redo these steps when the build changes.
 
 1. Run `sbt eclipse`
-2. In dotty, go to `Properties > java build path > Libraries`.
-   Remove the Scala Compiler container (currently 2.11.4) and add as an
-   external jar the latest compiler version in the Ivy cache. This is
-   currently:
 
-   ```
-   .ivy2/cache/me.d-d/scala-compiler/jars/scala-compiler-2.11.5-20160322-171045-e19b30b3cd.jar
-   ```
-
-   But that might change in the future. Or, copy the latest scala compiler from
-   the cache to a stable name and use that as external jar.
-
-3. It is recommended to change the default output folder (in `Properties > java
+2. It is recommended to change the default output folder (in `Properties > java
    build path > Source`) to `dotty/classes` instead of `dotty/bin` because
    `dotty/bin` is reserved for shell scripts.
 
 If you have `CLASSPATH` defined:
 
-4. Update your classpath to contain any new required external libraries to run
+3. Update your classpath to contain any new required external libraries to run
    `./bin/dotc`, `./bin/dotr` outside of Eclipse.
 
-5. Open the `Run Configurations` tab, and edit the `tests` configuration so
+4. Open the `Run Configurations` tab, and edit the `tests` configuration so
    that it contains a `CLASSPATH` variable which reflects the current
    `CLASSPATH`.
 

docs/docs/contributing/eclipse.md

@@ -24,7 +24,7 @@ Compiling and Running
 Start by cloning the repository:
 
 ```bash
-$ git clone https://github.com/lampepfl/dotty.git
+$ git clone --recursive https://github.com/lampepfl/dotty.git
 $ cd dotty
 ```
 

docs/docs/contributing/getting-started.md

@@ -9,7 +9,7 @@ This document details common workflow patterns when working with Dotty.
 
 ```bash
 # Start by cloning the repository:
-git clone https://github.com/lampepfl/dotty.git
+git clone --recursive https://github.com/lampepfl/dotty.git
 cd dotty
 # Clone dotty-compatible stdlib. Needed for running the test suite.
 git clone -b dotty-library https://github.com/DarkDimius/scala.git scala-scala