Fix #2184: Hyper Bootstrap! Integrate the backend as a git submodule

The backend lives in the scalac fork at
https://github.com/lampepfl/scala/tree/sharing-backend. Before this
commit, the scala-compiler built from this fork was a dependency of
Scala just to get this backend. This made it much more cumbersome to
test changes to the backend and also forced us to depend on Scala 2.11.5
since that is the version of scalac that the forked backend is based on.

This commit changes this by adding a git submodule in the
`scala-backend` directory that points to the scalac fork. We do not
compile the whole submodule, instead we add the subset of files we need
to the dotty-compiler project in the sbt build. See backend.md for more
information (online at
http://dotty.epfl.ch/docs/contributing/backend.html) once this commit is
merged.

The most important thing to note is that whenever you clone dotty you
should do `git clone --recursive` to also clone the submodule. If you
already have a cloned dotty you'll need to do:

git submodule update --init

You will need to repeat this command anytime you update dotty and the
backend has been updated, otherwise the content of `scala-backend` will
be stale.

Author: smarter

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "scala-backend"]
+	path = scala-backend
+	url = https://github.com/lampepfl/scala.git
+	branch = sharing-backend

bin/common

@@ -10,7 +10,7 @@ function getLastStringOnLineWith {
 # Configuration
 SCALA_VERSION=$(getLastStringOnLineWith "val scalacVersion")
 SCALA_BINARY_VERSION=2.11
-SCALA_COMPILER_VERSION=$(getLastStringOnLineWith "scala-compiler")
+SCALA_ASM_VERSION=$(getLastStringOnLineWith "% \"scala-asm\" %")
 SBT_VERSION=$(grep "sbt.version=" "$DOTTY_ROOT/project/build.properties" | sed 's/sbt.version=//')
 bootcp=true
 bootstrapped=false
@@ -146,8 +146,8 @@ if [ "$SCALA_REFLECT_JAR" == "" ]; then
     SCALA_REFLECT_JAR=$(find_jar "$HOME/.ivy2/cache/org.scala-lang/scala-reflect/jars" "scala-reflect-$SCALA_VERSION.jar")
 fi
 
-if [ "$SCALA_COMPILER_JAR" == "" ]; then
-    SCALA_COMPILER_JAR=$(find_jar "$HOME/.ivy2/cache/me.d-d/scala-compiler/jars" "scala-compiler-$SCALA_COMPILER_VERSION.jar")
+if [ "$SCALA_ASM_JAR" == "" ]; then
+    SCALA_ASM_JAR=$(find_jar "$HOME/.ivy2/cache/org.scala-lang.modules/scala-asm/bundles" "scala-asm-$SCALA_ASM_VERSION.jar")
 fi
 
 if [ "$SBT_INTERFACE_JAR" == "" ]; then

bin/dotc

@@ -20,12 +20,12 @@ CompilerMain=dotty.tools.dotc.Main
 FromTasty=dotty.tools.dotc.FromTasty
 ReplMain=dotty.tools.dotc.repl.Main
 
-if [ ! -f "$SCALA_LIBRARY_JAR" -o ! -f "$SCALA_REFLECT_JAR"  -o ! -f "$SCALA_COMPILER_JAR" -o ! -f "$SBT_INTERFACE_JAR" ]
+if [ ! -f "$SCALA_LIBRARY_JAR" -o ! -f "$SCALA_REFLECT_JAR"  -o ! -f "$SCALA_ASM_JAR" -o ! -f "$SBT_INTERFACE_JAR" ]
 then
   echo To use this script please set
   echo SCALA_LIBRARY_JAR to point to scala-library-$SCALA_VERSION.jar "(currently $SCALA_LIBRARY_JAR)"
   echo SCALA_REFLECT_JAR to point to scala-reflect-$SCALA_VERSION.jar "(currently $SCALA_REFLECT_JAR)"
-  echo SCALA_COMPILER_JAR to point to scala-compiler-$SCALA_VERSION.jar with bcode patches "(currently $SCALA_COMPILER_JAR)"
+  echo SCALA_ASM_JAR to point to scala-asm-$SCALA_ASM_VERSION.jar "(currently $SCALA_ASM_JAR)"
   echo SBT_INTERFACE_JAR to point to interface-$SBT_VERSION.jar "(currently $SBT_INTERFACE_JAR)"
 fi
 
@@ -116,9 +116,9 @@ trap onExit INT
 classpathArgs () {
   if [[ "true" == "$bootstrapped" ]]; then
     check_jar "dotty-bootstrapped" "$DOTTY_JAR" "target" 'build_jar "test:runMain dotc.build" target' &> /dev/null
-    toolchain="$DOTTY_JAR:$DOTTY_LIB_JAR:$SCALA_LIBRARY_JAR:$SCALA_REFLECT_JAR:$SCALA_COMPILER_JAR:$SBT_INTERFACE_JAR"
+    toolchain="$DOTTY_JAR:$DOTTY_LIB_JAR:$SCALA_LIBRARY_JAR:$SCALA_REFLECT_JAR:$SCALA_ASM_JAR:$SBT_INTERFACE_JAR"
   else
-    toolchain="$SCALA_LIBRARY_JAR:$DOTTY_LIB_JAR:$SCALA_REFLECT_JAR:$SCALA_COMPILER_JAR:$SBT_INTERFACE_JAR"
+    toolchain="$SCALA_LIBRARY_JAR:$DOTTY_LIB_JAR:$SCALA_REFLECT_JAR:$SCALA_ASM_JAR:$SBT_INTERFACE_JAR"
   fi
   bcpJars="$INTERFACES_JAR:$MAIN_JAR:$DOTTY_LIB_JAR"
   cpJars="$INTERFACES_JAR:$MAIN_JAR:$DOTTY_LIB_JAR:$TEST_JAR"

compiler/src/dotty/tools/backend/jvm/GenBCode.scala

@@ -8,7 +8,6 @@ import dotty.tools.dotc.core.Names.TypeName
 
 import scala.collection.mutable
 import scala.tools.asm.{ClassVisitor, CustomAttr, FieldVisitor, MethodVisitor}
-import scala.tools.nsc.Settings
 import scala.tools.nsc.backend.jvm._
 import dotty.tools.dotc
 import dotty.tools.dotc.backend.jvm.DottyPrimitives
@@ -35,7 +34,6 @@ import tpd._
 import StdNames._
 
 import scala.reflect.io.{AbstractFile, Directory, PlainDirectory}
-import scala.tools.nsc.backend.jvm.opt.LocalOpt
 
 class GenBCode extends Phase {
   def phaseName: String = "genBCode"
@@ -246,10 +244,10 @@ class GenBCodePipeline(val entryPoints: List[Symbol], val int: DottyBackendInter
      *          - converting the plain ClassNode to byte array and placing it on queue-3
      */
     class Worker2 {
-      lazy val localOpt = new LocalOpt(new Settings())
+      // lazy val localOpt = new LocalOpt(new Settings())
 
       def localOptimizations(classNode: ClassNode): Unit = {
-        /*BackendStats.timed(BackendStats.methodOptTimer)*/(localOpt.methodOptimizations(classNode))
+        // BackendStats.timed(BackendStats.methodOptTimer)(localOpt.methodOptimizations(classNode))
       }
 
       def run(): Unit = {

compiler/src/dotty/tools/backend/jvm/scalaPrimitives.scala

@@ -37,7 +37,7 @@ import core.Symbols.{Symbol, NoSymbol}
  * Inspired from the `scalac` compiler.
  */
 class DottyPrimitives(ctx: Context) {
-  import scala.tools.nsc.backend.ScalaPrimitives._
+  import scala.tools.nsc.backend.ScalaPrimitivesOps._
 
   private lazy val primitives: immutable.Map[Symbol, Int] = init
 

compiler/src/scala/tools/nsc/io/package.scala

@@ -0,0 +1,27 @@
+package scala.tools.nsc
+
+/**
+ * Compatibility layer needed for the backend.
+ *
+ * Our backend is based on the Scala 2.11 GenBCode backend and modified so that
+ * it compiles both with dotty and scalac, since the backend uses
+ * scala.tools.nsc.io.*, we need to also provide it.
+ *
+ * See http://dotty.epfl.ch/docs/contributing/backend.html for more information.
+ */
+package object io {
+  type AbstractFile = scala.reflect.io.AbstractFile
+  val AbstractFile = scala.reflect.io.AbstractFile
+
+  type Directory = scala.reflect.io.Directory
+  val Directory = scala.reflect.io.Directory
+
+  type Path = scala.reflect.io.Path
+  val Path = scala.reflect.io.Path
+
+  type File = scala.reflect.io.File
+  val File = scala.reflect.io.File
+
+  type Jar = dotty.tools.io.Jar
+  val Jar = dotty.tools.io.Jar
+}

compiler/test/dotc/tests.scala

@@ -358,7 +358,8 @@ class tests extends CompilerTest {
   @Test def tasty_dotc_util = compileDir(dotcDir, "util", testPickling)
   @Test def tasty_tools_io = compileDir(toolsDir, "io", testPickling)
 
-  @Test def tasty_bootstrap = {
+  // Disabled, not worth porting since we're getting rid of the old JUnit tests soon.
+  /*@Test*/ def tasty_bootstrap = {
     val logging = if (false) List("-Ylog-classpath", "-verbose") else Nil
     val opt = List("-priorityclasspath", defaultOutputDir) ++ logging
     // first compile dotty

compiler/test/dotty/tools/TypeStealer.scala

@@ -11,7 +11,7 @@ import asm._
 import asm.tree._
 import scala.collection.JavaConverters._
 
-import scala.tools.nsc.util.JavaClassPath
+import io.JavaClassPath
 import scala.collection.JavaConverters._
 import scala.tools.asm.{ClassWriter, ClassReader}
 import scala.tools.asm.tree._

compiler/test/dotty/tools/backend/jvm/DottyBytecodeTest.scala

@@ -4,11 +4,15 @@ package dotc
 
 import org.junit.{ Test, BeforeClass, AfterClass }
 
+import java.nio.file._
+import java.util.stream.{ Stream => JStream }
+import scala.collection.JavaConverters._
 import scala.util.matching.Regex
 import scala.concurrent.duration._
 
 import vulpix.{ ParallelTesting, SummaryReport, SummaryReporting, TestConfiguration }
 
+
 class CompilationTests extends ParallelTesting {
   import TestConfiguration._
   import CompilationTests._
@@ -219,16 +223,44 @@ class CompilationTests extends ParallelTesting {
       // compile with bootstrapped library on cp:
       defaultOutputDir + "lib/src/:" +
       // as well as bootstrapped compiler:
-      defaultOutputDir + "dotty1/dotty/:" +
+      defaultOutputDir + "dotty1/dotty1/:" +
       Jars.dottyInterfaces
     )
 
     def lib =
       compileDir("../library/src",
         allowDeepSubtypes.and("-Ycheck-reentrant", "-strict", "-priorityclasspath", defaultOutputDir))
 
-    def dotty1 =
-      compileDir("../compiler/src/dotty", opt)
+    def sources(paths: JStream[Path], excludedFiles: List[String] = Nil): List[String] =
+      paths.iterator().asScala
+        .filter(path =>
+          (path.toString.endsWith(".scala") || path.toString.endsWith(".java"))
+            && !excludedFiles.contains(path.getFileName.toString))
+        .map(_.toString).toList
+
+    val compilerDir = Paths.get("../compiler/src")
+    val compilerSources = sources(Files.walk(compilerDir))
+
+    val backendDir = Paths.get("../scala-backend/src/compiler/scala/tools/nsc/backend")
+    val backendJvmDir = Paths.get("../scala-backend/src/compiler/scala/tools/nsc/backend/jvm")
+
+    // NOTE: Keep these exclusions synchronized with the ones in the sbt build (Build.scala)
+    val backendExcluded =
+      List("JavaPlatform.scala", "Platform.scala", "ScalaPrimitives.scala")
+    val backendJvmExcluded =
+      List("BCodeICodeCommon.scala", "GenASM.scala", "GenBCode.scala", "ScalacBackendInterface.scala")
+
+    val backendSources =
+      sources(Files.list(backendDir), excludedFiles = backendExcluded)
+    val backendJvmSources =
+      sources(Files.list(backendJvmDir), excludedFiles = backendJvmExcluded)
+
+    def dotty1 = {
+      compileList(
+        "dotty1",
+        compilerSources ++ backendSources ++ backendJvmSources,
+        opt)
+    }
 
     def dotty2 =
       compileShallowFilesInDir("../compiler/src/dotty", opt)
@@ -247,7 +279,9 @@ class CompilationTests extends ParallelTesting {
         compileShallowFilesInDir("../compiler/src/dotty/tools/dotc/rewrite", opt) +
         compileShallowFilesInDir("../compiler/src/dotty/tools/dotc/transform", opt) +
         compileShallowFilesInDir("../compiler/src/dotty/tools/dotc/typer", opt) +
-        compileShallowFilesInDir("../compiler/src/dotty/tools/dotc/util", opt)
+        compileShallowFilesInDir("../compiler/src/dotty/tools/dotc/util", opt) +
+        compileList("shallow-backend", backendSources, opt) +
+        compileList("shallow-backend-jvm", backendJvmSources, opt)
       } :: Nil
     }.map(_.checkCompile()).foreach(_.delete())
   }

compiler/test/dotty/tools/dotc/CompilationTests.scala

@@ -0,0 +1,65 @@
+---
+layout: doc-page
+title: Working with the backend
+---
+
+The compiler backend is based on a fork of the Scala 2.11 `GenBCode` backend and
+lives at https://github.com/lampepfl/scala/tree/sharing-backend. The dotty
+source tree contains a git submodule in the directory
+[scala-backend](https://github.com/lampepfl/dotty/tree/master/scala-backend)
+that points to this fork. We do not compile every file in this submodule,
+instead we add the subset of files we need to the dotty-compiler project in the
+sbt build.
+
+The most important thing to know when working with git submodules is that
+their content is not automatically updated when you do `git checkout` or `git
+pull`, instead everytime you switch branch, you need to do:
+
+``` shell
+git submodule update --init
+```
+
+## Environment setup
+
+1. Set the following git configuration options to make working with submodules
+easier, see the [Git Book](https://git-scm.com/book/en/v2/Git-Tools-Submodules)
+for more information:
+``` shell
+git config --global diff.submodule log
+git config --global status.submodulesummary 1
+git config --global push.recurseSubmodules check
+```
+
+2. Fork https://github.com/lampepfl/scala (in the following commands,
+   `dotty-staging/scala` is used as a placeholder for your
+   own fork).
+3. ```shell
+   cd scala-backend
+   git remote add scala-staging [email protected]:dotty-staging/scala.git
+   cd ..
+   ```
+
+## Workflow when changing to the backend
+
+```shell
+cd scala-backend
+git checkout -b my-feature-branch
+# Make some changes ...
+
+git push -u scala-backend
+# Open a PR against https://github.com/lampepfl/scala/tree/sharing-backend
+cd ..
+```
+
+Once your PR has been merged into the backend, you'll need to make another PR
+against dotty itself to update the backend, the following commands should be run
+in the root dotty repository, not in the submodule:
+
+``` shell
+# The --remote option will update the submodule to the latest commit in the
+# https://github.com/lampepfl/dotty/tree/master/scala-backend branch
+git submodule update --init --remote
+
+git commit -am "Update backend to include ..."
+# Then push and make a PR against dotty as usual
+```

docs/docs/contributing/backend.md

@@ -5,31 +5,19 @@ title: Building Dotty with Eclipse
 
 Build setup
 -----------
-You may need to redo these steps when the build changes.
 
 1. Run `sbt eclipse`
-2. In dotty, go to `Properties > java build path > Libraries`.
-   Remove the Scala Compiler container (currently 2.11.4) and add as an
-   external jar the latest compiler version in the Ivy cache. This is
-   currently:
 
-   ```
-   .ivy2/cache/me.d-d/scala-compiler/jars/scala-compiler-2.11.5-20160322-171045-e19b30b3cd.jar
-   ```
-
-   But that might change in the future. Or, copy the latest scala compiler from
-   the cache to a stable name and use that as external jar.
-
-3. It is recommended to change the default output folder (in `Properties > java
+2. It is recommended to change the default output folder (in `Properties > java
    build path > Source`) to `dotty/classes` instead of `dotty/bin` because
    `dotty/bin` is reserved for shell scripts.
 
 If you have `CLASSPATH` defined:
 
-4. Update your classpath to contain any new required external libraries to run
+3. Update your classpath to contain any new required external libraries to run
    `./bin/dotc`, `./bin/dotr` outside of Eclipse.
 
-5. Open the `Run Configurations` tab, and edit the `tests` configuration so
+4. Open the `Run Configurations` tab, and edit the `tests` configuration so
    that it contains a `CLASSPATH` variable which reflects the current
    `CLASSPATH`.
 

docs/docs/contributing/eclipse.md

@@ -24,7 +24,7 @@ Compiling and Running
 Start by cloning the repository:
 
 ```bash
-$ git clone https://github.com/lampepfl/dotty.git
+$ git clone --recursive https://github.com/lampepfl/dotty.git
 $ cd dotty
 ```
 

docs/docs/contributing/getting-started.md

@@ -9,7 +9,7 @@ This document details common workflow patterns when working with Dotty.
 
 ```bash
 # Start by cloning the repository:
-git clone https://github.com/lampepfl/dotty.git
+git clone --recursive https://github.com/lampepfl/dotty.git
 cd dotty
 # Clone dotty-compatible stdlib. Needed for running the test suite.
 git clone -b dotty-library https://github.com/DarkDimius/scala.git scala-scala