diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000000..37937bd8e575 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "scala-backend"] + path = scala-backend + url = https://github.com/lampepfl/scala.git + branch = sharing-backend diff --git a/bin/common b/bin/common index 0a084c8a68a6..d1a9db482333 100755 --- a/bin/common +++ b/bin/common @@ -10,7 +10,7 @@ function getLastStringOnLineWith { # Configuration SCALA_VERSION=$(getLastStringOnLineWith "val scalacVersion") SCALA_BINARY_VERSION=2.11 -SCALA_COMPILER_VERSION=$(getLastStringOnLineWith "scala-compiler") +SCALA_ASM_VERSION=$(getLastStringOnLineWith "% \"scala-asm\" %") SBT_VERSION=$(grep "sbt.version=" "$DOTTY_ROOT/project/build.properties" | sed 's/sbt.version=//') bootcp=true bootstrapped=false @@ -146,8 +146,8 @@ if [ "$SCALA_REFLECT_JAR" == "" ]; then SCALA_REFLECT_JAR=$(find_jar "$HOME/.ivy2/cache/org.scala-lang/scala-reflect/jars" "scala-reflect-$SCALA_VERSION.jar") fi -if [ "$SCALA_COMPILER_JAR" == "" ]; then - SCALA_COMPILER_JAR=$(find_jar "$HOME/.ivy2/cache/me.d-d/scala-compiler/jars" "scala-compiler-$SCALA_COMPILER_VERSION.jar") +if [ "$SCALA_ASM_JAR" == "" ]; then + SCALA_ASM_JAR=$(find_jar "$HOME/.ivy2/cache/org.scala-lang.modules/scala-asm/bundles" "scala-asm-$SCALA_ASM_VERSION.jar") fi if [ "$SBT_INTERFACE_JAR" == "" ]; then diff --git a/bin/dotc b/bin/dotc index ec65903d9ae7..39e3f8074b53 100755 --- a/bin/dotc +++ b/bin/dotc @@ -20,12 +20,12 @@ CompilerMain=dotty.tools.dotc.Main FromTasty=dotty.tools.dotc.FromTasty ReplMain=dotty.tools.dotc.repl.Main -if [ ! -f "$SCALA_LIBRARY_JAR" -o ! -f "$SCALA_REFLECT_JAR" -o ! -f "$SCALA_COMPILER_JAR" -o ! -f "$SBT_INTERFACE_JAR" ] +if [ ! -f "$SCALA_LIBRARY_JAR" -o ! -f "$SCALA_REFLECT_JAR" -o ! -f "$SCALA_ASM_JAR" -o ! -f "$SBT_INTERFACE_JAR" ] then echo To use this script please set echo SCALA_LIBRARY_JAR to point to scala-library-$SCALA_VERSION.jar "(currently $SCALA_LIBRARY_JAR)" echo SCALA_REFLECT_JAR to point to scala-reflect-$SCALA_VERSION.jar "(currently $SCALA_REFLECT_JAR)" - echo SCALA_COMPILER_JAR to point to scala-compiler-$SCALA_VERSION.jar with bcode patches "(currently $SCALA_COMPILER_JAR)" + echo SCALA_ASM_JAR to point to scala-asm-$SCALA_ASM_VERSION.jar "(currently $SCALA_ASM_JAR)" echo SBT_INTERFACE_JAR to point to interface-$SBT_VERSION.jar "(currently $SBT_INTERFACE_JAR)" fi @@ -116,9 +116,9 @@ trap onExit INT classpathArgs () { if [[ "true" == "$bootstrapped" ]]; then check_jar "dotty-bootstrapped" "$DOTTY_JAR" "target" 'build_jar "test:runMain dotc.build" target' &> /dev/null - toolchain="$DOTTY_JAR:$DOTTY_LIB_JAR:$SCALA_LIBRARY_JAR:$SCALA_REFLECT_JAR:$SCALA_COMPILER_JAR:$SBT_INTERFACE_JAR" + toolchain="$DOTTY_JAR:$DOTTY_LIB_JAR:$SCALA_LIBRARY_JAR:$SCALA_REFLECT_JAR:$SCALA_ASM_JAR:$SBT_INTERFACE_JAR" else - toolchain="$SCALA_LIBRARY_JAR:$DOTTY_LIB_JAR:$SCALA_REFLECT_JAR:$SCALA_COMPILER_JAR:$SBT_INTERFACE_JAR" + toolchain="$SCALA_LIBRARY_JAR:$DOTTY_LIB_JAR:$SCALA_REFLECT_JAR:$SCALA_ASM_JAR:$SBT_INTERFACE_JAR" fi bcpJars="$INTERFACES_JAR:$MAIN_JAR:$DOTTY_LIB_JAR" cpJars="$INTERFACES_JAR:$MAIN_JAR:$DOTTY_LIB_JAR:$TEST_JAR" diff --git a/compiler/src/dotty/tools/backend/jvm/GenBCode.scala b/compiler/src/dotty/tools/backend/jvm/GenBCode.scala index 8fd6d1bc0139..25a8061ac05f 100644 --- a/compiler/src/dotty/tools/backend/jvm/GenBCode.scala +++ b/compiler/src/dotty/tools/backend/jvm/GenBCode.scala @@ -8,7 +8,6 @@ import dotty.tools.dotc.core.Names.TypeName import scala.collection.mutable import scala.tools.asm.{ClassVisitor, CustomAttr, FieldVisitor, MethodVisitor} -import scala.tools.nsc.Settings import scala.tools.nsc.backend.jvm._ import dotty.tools.dotc import dotty.tools.dotc.backend.jvm.DottyPrimitives @@ -35,7 +34,6 @@ import tpd._ import StdNames._ import scala.reflect.io.{AbstractFile, Directory, PlainDirectory} -import scala.tools.nsc.backend.jvm.opt.LocalOpt class GenBCode extends Phase { def phaseName: String = "genBCode" @@ -246,10 +244,10 @@ class GenBCodePipeline(val entryPoints: List[Symbol], val int: DottyBackendInter * - converting the plain ClassNode to byte array and placing it on queue-3 */ class Worker2 { - lazy val localOpt = new LocalOpt(new Settings()) + // lazy val localOpt = new LocalOpt(new Settings()) def localOptimizations(classNode: ClassNode): Unit = { - /*BackendStats.timed(BackendStats.methodOptTimer)*/(localOpt.methodOptimizations(classNode)) + // BackendStats.timed(BackendStats.methodOptTimer)(localOpt.methodOptimizations(classNode)) } def run(): Unit = { diff --git a/compiler/src/dotty/tools/backend/jvm/scalaPrimitives.scala b/compiler/src/dotty/tools/backend/jvm/scalaPrimitives.scala index 7beae90c4ca6..851cacaa2275 100644 --- a/compiler/src/dotty/tools/backend/jvm/scalaPrimitives.scala +++ b/compiler/src/dotty/tools/backend/jvm/scalaPrimitives.scala @@ -37,7 +37,7 @@ import core.Symbols.{Symbol, NoSymbol} * Inspired from the `scalac` compiler. */ class DottyPrimitives(ctx: Context) { - import scala.tools.nsc.backend.ScalaPrimitives._ + import scala.tools.nsc.backend.ScalaPrimitivesOps._ private lazy val primitives: immutable.Map[Symbol, Int] = init diff --git a/compiler/src/scala/tools/nsc/io/package.scala b/compiler/src/scala/tools/nsc/io/package.scala new file mode 100644 index 000000000000..6b30e54413ed --- /dev/null +++ b/compiler/src/scala/tools/nsc/io/package.scala @@ -0,0 +1,27 @@ +package scala.tools.nsc + +/** + * Compatibility layer needed for the backend. + * + * Our backend is based on the Scala 2.11 GenBCode backend and modified so that + * it compiles both with dotty and scalac, since the backend uses + * scala.tools.nsc.io.*, we need to also provide it. + * + * See http://dotty.epfl.ch/docs/contributing/backend.html for more information. + */ +package object io { + type AbstractFile = scala.reflect.io.AbstractFile + val AbstractFile = scala.reflect.io.AbstractFile + + type Directory = scala.reflect.io.Directory + val Directory = scala.reflect.io.Directory + + type Path = scala.reflect.io.Path + val Path = scala.reflect.io.Path + + type File = scala.reflect.io.File + val File = scala.reflect.io.File + + type Jar = dotty.tools.io.Jar + val Jar = dotty.tools.io.Jar +} diff --git a/compiler/test/dotc/tests.scala b/compiler/test/dotc/tests.scala index efecc1df3d17..0f1732e2a71c 100644 --- a/compiler/test/dotc/tests.scala +++ b/compiler/test/dotc/tests.scala @@ -358,7 +358,8 @@ class tests extends CompilerTest { @Test def tasty_dotc_util = compileDir(dotcDir, "util", testPickling) @Test def tasty_tools_io = compileDir(toolsDir, "io", testPickling) - @Test def tasty_bootstrap = { + // Disabled, not worth porting since we're getting rid of the old JUnit tests soon. + /*@Test*/ def tasty_bootstrap = { val logging = if (false) List("-Ylog-classpath", "-verbose") else Nil val opt = List("-priorityclasspath", defaultOutputDir) ++ logging // first compile dotty diff --git a/compiler/test/dotty/tools/TypeStealer.scala b/compiler/test/dotty/tools/TypeStealer.scala deleted file mode 100644 index ccaf2d41ef47..000000000000 --- a/compiler/test/dotty/tools/TypeStealer.scala +++ /dev/null @@ -1,21 +0,0 @@ -package dotty.tools - -import scala.tools.nsc.interpreter._ -import scala.tools.nsc.Settings - -object TypeStealer { - def main(args: Array[String]): Unit = { - def repl = new ILoop {} - - val settings = new Settings - settings.Yreplsync.value = true - - //use when launching normally outside SBT - settings.usejavacp.value = true - - //an alternative to 'usejavacp' setting, when launching from within SBT - //settings.embeddedDefaults[Repl.type] - - repl.process(settings) - } -} diff --git a/compiler/test/dotty/tools/backend/jvm/DottyBytecodeTest.scala b/compiler/test/dotty/tools/backend/jvm/DottyBytecodeTest.scala index fc98536911c8..c423089d07a4 100644 --- a/compiler/test/dotty/tools/backend/jvm/DottyBytecodeTest.scala +++ b/compiler/test/dotty/tools/backend/jvm/DottyBytecodeTest.scala @@ -11,7 +11,7 @@ import asm._ import asm.tree._ import scala.collection.JavaConverters._ -import scala.tools.nsc.util.JavaClassPath +import io.JavaClassPath import scala.collection.JavaConverters._ import scala.tools.asm.{ClassWriter, ClassReader} import scala.tools.asm.tree._ diff --git a/compiler/test/dotty/tools/dotc/CompilationTests.scala b/compiler/test/dotty/tools/dotc/CompilationTests.scala index cf9b200d4631..504bdd2e0bce 100644 --- a/compiler/test/dotty/tools/dotc/CompilationTests.scala +++ b/compiler/test/dotty/tools/dotc/CompilationTests.scala @@ -4,11 +4,15 @@ package dotc import org.junit.{ Test, BeforeClass, AfterClass } +import java.nio.file._ +import java.util.stream.{ Stream => JStream } +import scala.collection.JavaConverters._ import scala.util.matching.Regex import scala.concurrent.duration._ import vulpix.{ ParallelTesting, SummaryReport, SummaryReporting, TestConfiguration } + class CompilationTests extends ParallelTesting { import TestConfiguration._ import CompilationTests._ @@ -219,7 +223,7 @@ class CompilationTests extends ParallelTesting { // compile with bootstrapped library on cp: defaultOutputDir + "lib/src/:" + // as well as bootstrapped compiler: - defaultOutputDir + "dotty1/dotty/:" + + defaultOutputDir + "dotty1/dotty1/:" + Jars.dottyInterfaces ) @@ -227,8 +231,36 @@ class CompilationTests extends ParallelTesting { compileDir("../library/src", allowDeepSubtypes.and("-Ycheck-reentrant", "-strict", "-priorityclasspath", defaultOutputDir)) - def dotty1 = - compileDir("../compiler/src/dotty", opt) + def sources(paths: JStream[Path], excludedFiles: List[String] = Nil): List[String] = + paths.iterator().asScala + .filter(path => + (path.toString.endsWith(".scala") || path.toString.endsWith(".java")) + && !excludedFiles.contains(path.getFileName.toString)) + .map(_.toString).toList + + val compilerDir = Paths.get("../compiler/src") + val compilerSources = sources(Files.walk(compilerDir)) + + val backendDir = Paths.get("../scala-backend/src/compiler/scala/tools/nsc/backend") + val backendJvmDir = Paths.get("../scala-backend/src/compiler/scala/tools/nsc/backend/jvm") + + // NOTE: Keep these exclusions synchronized with the ones in the sbt build (Build.scala) + val backendExcluded = + List("JavaPlatform.scala", "Platform.scala", "ScalaPrimitives.scala") + val backendJvmExcluded = + List("BCodeICodeCommon.scala", "GenASM.scala", "GenBCode.scala", "ScalacBackendInterface.scala") + + val backendSources = + sources(Files.list(backendDir), excludedFiles = backendExcluded) + val backendJvmSources = + sources(Files.list(backendJvmDir), excludedFiles = backendJvmExcluded) + + def dotty1 = { + compileList( + "dotty1", + compilerSources ++ backendSources ++ backendJvmSources, + opt) + } def dotty2 = compileShallowFilesInDir("../compiler/src/dotty", opt) @@ -247,7 +279,9 @@ class CompilationTests extends ParallelTesting { compileShallowFilesInDir("../compiler/src/dotty/tools/dotc/rewrite", opt) + compileShallowFilesInDir("../compiler/src/dotty/tools/dotc/transform", opt) + compileShallowFilesInDir("../compiler/src/dotty/tools/dotc/typer", opt) + - compileShallowFilesInDir("../compiler/src/dotty/tools/dotc/util", opt) + compileShallowFilesInDir("../compiler/src/dotty/tools/dotc/util", opt) + + compileList("shallow-backend", backendSources, opt) + + compileList("shallow-backend-jvm", backendJvmSources, opt) } :: Nil }.map(_.checkCompile()).foreach(_.delete()) } diff --git a/docs/docs/contributing/backend.md b/docs/docs/contributing/backend.md new file mode 100644 index 000000000000..e7d98bb7f371 --- /dev/null +++ b/docs/docs/contributing/backend.md @@ -0,0 +1,65 @@ +--- +layout: doc-page +title: Working with the backend +--- + +The compiler backend is based on a fork of the Scala 2.11 `GenBCode` backend and +lives at https://github.com/lampepfl/scala/tree/sharing-backend. The dotty +source tree contains a git submodule in the directory +[scala-backend](https://github.com/lampepfl/dotty/tree/master/scala-backend) +that points to this fork. We do not compile every file in this submodule, +instead we add the subset of files we need to the dotty-compiler project in the +sbt build. + +The most important thing to know when working with git submodules is that +their content is not automatically updated when you do `git checkout` or `git +pull`, instead everytime you switch branch, you need to do: + +``` shell +git submodule update --init +``` + +## Environment setup + +1. Set the following git configuration options to make working with submodules +easier, see the [Git Book](https://git-scm.com/book/en/v2/Git-Tools-Submodules) +for more information: +``` shell +git config --global diff.submodule log +git config --global status.submodulesummary 1 +git config --global push.recurseSubmodules check +``` + +2. Fork https://github.com/lampepfl/scala (in the following commands, + `dotty-staging/scala` is used as a placeholder for your + own fork). +3. ```shell + cd scala-backend + git remote add scala-staging git@github.com:dotty-staging/scala.git + cd .. + ``` + +## Workflow when changing to the backend + +```shell +cd scala-backend +git checkout -b my-feature-branch +# Make some changes ... + +git push -u scala-backend +# Open a PR against https://github.com/lampepfl/scala/tree/sharing-backend +cd .. +``` + +Once your PR has been merged into the backend, you'll need to make another PR +against dotty itself to update the backend, the following commands should be run +in the root dotty repository, not in the submodule: + +``` shell +# The --remote option will update the submodule to the latest commit in the +# https://github.com/lampepfl/dotty/tree/master/scala-backend branch +git submodule update --init --remote + +git commit -am "Update backend to include ..." +# Then push and make a PR against dotty as usual +``` diff --git a/docs/docs/contributing/eclipse.md b/docs/docs/contributing/eclipse.md index a7deab8046c2..fa59a5dc6b0a 100644 --- a/docs/docs/contributing/eclipse.md +++ b/docs/docs/contributing/eclipse.md @@ -5,31 +5,19 @@ title: Building Dotty with Eclipse Build setup ----------- -You may need to redo these steps when the build changes. 1. Run `sbt eclipse` -2. In dotty, go to `Properties > java build path > Libraries`. - Remove the Scala Compiler container (currently 2.11.4) and add as an - external jar the latest compiler version in the Ivy cache. This is - currently: - ``` - .ivy2/cache/me.d-d/scala-compiler/jars/scala-compiler-2.11.5-20160322-171045-e19b30b3cd.jar - ``` - - But that might change in the future. Or, copy the latest scala compiler from - the cache to a stable name and use that as external jar. - -3. It is recommended to change the default output folder (in `Properties > java +2. It is recommended to change the default output folder (in `Properties > java build path > Source`) to `dotty/classes` instead of `dotty/bin` because `dotty/bin` is reserved for shell scripts. If you have `CLASSPATH` defined: -4. Update your classpath to contain any new required external libraries to run +3. Update your classpath to contain any new required external libraries to run `./bin/dotc`, `./bin/dotr` outside of Eclipse. -5. Open the `Run Configurations` tab, and edit the `tests` configuration so +4. Open the `Run Configurations` tab, and edit the `tests` configuration so that it contains a `CLASSPATH` variable which reflects the current `CLASSPATH`. diff --git a/docs/docs/contributing/getting-started.md b/docs/docs/contributing/getting-started.md index 26862e790bf8..6f14c82992e8 100644 --- a/docs/docs/contributing/getting-started.md +++ b/docs/docs/contributing/getting-started.md @@ -24,7 +24,7 @@ Compiling and Running Start by cloning the repository: ```bash -$ git clone https://github.com/lampepfl/dotty.git +$ git clone --recursive https://github.com/lampepfl/dotty.git $ cd dotty ``` diff --git a/docs/docs/contributing/workflow.md b/docs/docs/contributing/workflow.md index b277cc243480..a8001366ae41 100644 --- a/docs/docs/contributing/workflow.md +++ b/docs/docs/contributing/workflow.md @@ -9,7 +9,7 @@ This document details common workflow patterns when working with Dotty. ```bash # Start by cloning the repository: -git clone https://github.com/lampepfl/dotty.git +git clone --recursive https://github.com/lampepfl/dotty.git cd dotty # Clone dotty-compatible stdlib. Needed for running the test suite. git clone -b dotty-library https://github.com/DarkDimius/scala.git scala-scala diff --git a/project/Build.scala b/project/Build.scala index 8b1c0e31e9af..cc6b5a6c3735 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -31,8 +31,6 @@ object Build { val JENKINS_BUILD = "dotty.jenkins.build" val DRONE_MEM = "dotty.drone.mem" - val scalaCompiler = "me.d-d" % "scala-compiler" % "2.11.5-20170111-125332-40bdc7b65a" - val agentOptions = List( // "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005" // "-agentpath:/home/dark/opt/yjp-2013-build-13072/bin/linux-x86-64/libyjpagent.so" @@ -243,6 +241,26 @@ object Build { // Settings shared between dotty-compiler and dotty-compiler-bootstrapped lazy val dottyCompilerSettings = Seq( + // The scala-backend folder is a git submodule that contains a fork of the Scala 2.11 + // compiler developed at https://github.com/lampepfl/scala/tree/sharing-backend. + // We do not compile the whole submodule, only the part of the Scala 2.11 GenBCode backend + // that we reuse for dotty. + // See http://dotty.epfl.ch/docs/contributing/backend.html for more information. + unmanagedSourceDirectories in Compile ++= { + val backendDir = baseDirectory.value / ".." / "scala-backend" / "src" / "compiler" / "scala" / "tools" / "nsc" / "backend" + val allScalaFiles = GlobFilter("*.scala") + + // NOTE: Keep these exclusions synchronized with the ones in the tests (CompilationTests.scala) + ((backendDir * + (allScalaFiles - "JavaPlatform.scala" - "Platform.scala" - "ScalaPrimitives.scala")) +++ + (backendDir / "jvm") * + (allScalaFiles - "BCodeICodeCommon.scala" - "GenASM.scala" - "GenBCode.scala" - "ScalacBackendInterface.scala") + ).get + }, + + // Used by the backend + libraryDependencies += "org.scala-lang.modules" % "scala-asm" % "5.1.0-scala-2", + // set system in/out for repl connectInput in run := true, outputStrategy := Some(StdoutOutput), @@ -261,8 +279,7 @@ object Build { // get libraries onboard resolvers += Resolver.typesafeIvyRepo("releases"), // For org.scala-sbt:interface - libraryDependencies ++= Seq(scalaCompiler, - "org.scala-sbt" % "interface" % sbtVersion.value, + libraryDependencies ++= Seq("org.scala-sbt" % "interface" % sbtVersion.value, "org.scala-lang.modules" %% "scala-xml" % "1.0.1", "com.novocode" % "junit-interface" % "0.11" % "test", "org.scala-lang" % "scala-reflect" % scalacVersion, @@ -394,10 +411,10 @@ object Build { if path.contains("scala-library") || // FIXME: currently needed for tests referencing scalac internals path.contains("scala-reflect") || - // FIXME: currently needed for tests referencing scalac internals - path.contains("scala-compile") || // FIXME: should go away when xml literal parsing is removed path.contains("scala-xml") || + // used for tests that compile dotty + path.contains("scala-asm") || // needed for the xsbti interface path.contains("org.scala-sbt/interface/") } yield "-Xbootclasspath/p:" + path @@ -631,10 +648,7 @@ object DottyInjectedPlugin extends AutoPlugin { baseDirectory in (Test,run) := (baseDirectory in `dotty-compiler`).value, - libraryDependencies ++= Seq( - scalaCompiler % Test, - "com.storm-enroute" %% "scalameter" % "0.6" % Test - ), + libraryDependencies += "com.storm-enroute" %% "scalameter" % "0.6" % Test, fork in Test := true, parallelExecution in Test := false, @@ -686,8 +700,7 @@ object DottyInjectedPlugin extends AutoPlugin { // depend on one of these projects. lazy val `scala-compiler` = project. settings( - crossPaths := false, - libraryDependencies := Seq(scalaCompiler) + crossPaths := false ). settings(publishing) lazy val `scala-reflect` = project. diff --git a/scala-backend b/scala-backend new file mode 160000 index 000000000000..7246c2bd1e98 --- /dev/null +++ b/scala-backend @@ -0,0 +1 @@ +Subproject commit 7246c2bd1e98fadd3a2de1bbb70403fdb8896458