Merge pull request #1244 from dotty-staging/add/sbt-phase

Add sbt incremental compilation support

Author: smarter

AUTHORS.md

@@ -42,7 +42,7 @@ The majority of the dotty codebase is new code, with the exception of the compon
 > The utilities package is a mix of new and adapted components. The files in  [scala/scala](https://github.com/scala/scala) were originally authored by many people,
 > including Paul Phillips, Martin Odersky, Sean McDirmid, and others.
   
-`dotty.tools.io`   
+`dotty.tools.io`
 
 > The I/O support library was adapted from current Scala compiler. Original authors were Paul Phillips and others.
 
@@ -52,3 +52,12 @@ The majority of the dotty codebase is new code, with the exception of the compon
 > [scala/scala](https://github.com/scala/scala). It has been reworked to fit
 > the needs of dotty. Original authors include: Adrian Moors, Lukas Rytz,
 > Grzegorz Kossakowski, Paul Phillips
+
+`dotty.tools.dotc.sbt`
+
+> The sbt compiler phases are based on
+> https://github.com/adriaanm/scala/tree/sbt-api-consolidate/src/compiler/scala/tools/sbt
+> which attempts to integrate the sbt phases into scalac and is itself based on
+> the [compiler bridge in sbt 0.13](https://github.com/sbt/sbt/tree/0.13/compile/interface/src/main/scala/xsbt),
+> but has been heavily adapted and refactored.
+> Original authors were Mark Harrah, Grzegorz Kossakowski, Martin Duhemm, Adriaan Moors and others.

bin/dotc

@@ -23,6 +23,7 @@ SCALA_BINARY_VERSION=2.11
 SCALA_COMPILER_VERSION=$(getLastStringOnLineWith "scala-compiler")
 DOTTY_VERSION=$(getLastStringOnLineWith "version in")
 JLINE_VERSION=$(getLastStringOnLineWith "jline")
+SBT_VERSION=$(grep "sbt.version=" "$DOTTY_ROOT/project/build.properties" | sed 's/sbt.version=//')
 bootcp=true
 bootstrapped=false
 default_java_opts="-Xmx768m -Xms768m"
@@ -100,13 +101,19 @@ then
   JLINE_JAR=$HOME/.ivy2/cache/jline/jline/jars/jline-$JLINE_VERSION.jar
 fi
 
-if [ ! -f "$SCALA_LIBRARY_JAR" -o ! -f "$SCALA_REFLECT_JAR"  -o ! -f "$SCALA_COMPILER_JAR"  -o ! -f "$JLINE_JAR" ]
+if [ "$SBT_INTERFACE_JAR" == "" ]
+then
+  SBT_INTERFACE_JAR=$HOME/.ivy2/cache/org.scala-sbt/interface/jars/interface-$SBT_VERSION.jar
+fi
+
+if [ ! -f "$SCALA_LIBRARY_JAR" -o ! -f "$SCALA_REFLECT_JAR"  -o ! -f "$SCALA_COMPILER_JAR"  -o ! -f "$JLINE_JAR" -o ! -f "$SBT_INTERFACE_JAR" ]
 then
   echo To use this script please set
   echo SCALA_LIBRARY_JAR to point to scala-library-$SCALA_VERSION.jar "(currently $SCALA_LIBRARY_JAR)"
   echo SCALA_REFLECT_JAR to point to scala-reflect-$SCALA_VERSION.jar "(currently $SCALA_REFLECT_JAR)"
   echo SCALA_COMPILER_JAR to point to scala-compiler-$SCALA_VERSION.jar with bcode patches "(currently $SCALA_COMPILER_JAR)"
   echo JLINE_JAR to point to jline-$JLINE_VERSION.jar "(currently $JLINE_JAR)"
+  echo SBT_INTERFACE_JAR to point to interface-$SBT_VERSION.jar "(currently $SBT_INTERFACE_JAR)"
 fi
 
 ifdebug () {
@@ -196,9 +203,9 @@ trap onExit INT
 classpathArgs () {
   if [[ "true" == $bootstrapped ]]; then 
     checkjar $DOTTY_JAR "test:runMain dotc.build" src
-    toolchain="$DOTTY_JAR:$SCALA_LIBRARY_JAR:$SCALA_REFLECT_JAR:$SCALA_COMPILER_JAR:$JLINE_JAR"
+    toolchain="$DOTTY_JAR:$SCALA_LIBRARY_JAR:$SCALA_REFLECT_JAR:$SCALA_COMPILER_JAR:$JLINE_JAR:$SBT_INTERFACE_JAR"
   else
-    toolchain="$SCALA_LIBRARY_JAR:$SCALA_REFLECT_JAR:$SCALA_COMPILER_JAR:$JLINE_JAR"
+    toolchain="$SCALA_LIBRARY_JAR:$SCALA_REFLECT_JAR:$SCALA_COMPILER_JAR:$JLINE_JAR:$SBT_INTERFACE_JAR"
   fi
   bcpJars="$INTERFACES_JAR:$MAIN_JAR"
   cpJars="$INTERFACES_JAR:$MAIN_JAR:$TEST_JAR"

project/Build.scala

@@ -88,7 +88,8 @@ object DottyBuild extends Build {
       libraryDependencies ++= Seq("org.scala-lang.modules" %% "scala-xml" % "1.0.1",
                                   "org.scala-lang.modules" %% "scala-partest" % "1.0.11" % "test",
                                   "com.novocode" % "junit-interface" % "0.11" % "test",
-                                  "jline" % "jline" % "2.12"),
+                                  "jline" % "jline" % "2.12",
+                                  "org.scala-sbt" % "interface" % sbtVersion.value),
 
       // enable improved incremental compilation algorithm
       incOptions := incOptions.value.withNameHashing(true),

src/dotty/tools/backend/jvm/GenBCode.scala

@@ -389,6 +389,8 @@ class GenBCodePipeline(val entryPoints: List[Symbol], val int: DottyBackendInter
             val className = jclassName.replace('/', '.')
             if (ctx.compilerCallback != null)
               ctx.compilerCallback.onClassGenerated(sourceFile, convertAbstractFile(outFile), className)
+            if (ctx.sbtCallback != null)
+              ctx.sbtCallback.generatedClass(sourceFile.jfile.orElse(null), outFile.file, className)
           }
           catch {
             case e: FileConflictException =>

src/dotty/tools/dotc/Compiler.scala

@@ -42,7 +42,9 @@ class Compiler {
   def phases: List[List[Phase]] =
     List(
       List(new FrontEnd),           // Compiler frontend: scanner, parser, namer, typer
+      List(new sbt.ExtractDependencies), // Sends information on classes' dependencies to sbt via callbacks
       List(new PostTyper),          // Additional checks and cleanups after type checking
+      List(new sbt.ExtractAPI),     // Sends a representation of the API of classes to sbt via callbacks
       List(new Pickler),            // Generate TASTY info
       List(new FirstTransform,      // Some transformations to put trees into a canonical form
            new CheckReentrant),     // Internal use only: Check that compiled program has no data races involving global vars

src/dotty/tools/dotc/ast/Desugar.scala

@@ -958,7 +958,7 @@ object desugar {
    *  Example: Given
    *
    *      class C
-   *      type T1 extends C { type T <: A }
+   *      type T1 = C { type T <: A }
    *
    *  the refined type
    *

src/dotty/tools/dotc/ast/Trees.scala

@@ -559,7 +559,10 @@ object Trees {
     type ThisTree[-T >: Untyped] = SingletonTypeTree[T]
   }
 
-  /** qualifier # name */
+  /** qualifier # name
+   *  In Scala, this always refers to a type, but in a Java
+   *  compilation unit this might refer to a term.
+   */
   case class SelectFromTypeTree[-T >: Untyped] private[ast] (qualifier: Tree[T], name: Name)
     extends RefTree[T] {
     type ThisTree[-T >: Untyped] = SelectFromTypeTree[T]

src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -160,6 +160,8 @@ class ScalaSettings extends Settings.SettingGroup {
   val YtestPickler = BooleanSetting("-Ytest-pickler", "self-test for pickling functionality; should be used with -Ystop-after:pickler")
   val YcheckReentrant = BooleanSetting("-Ycheck-reentrant", "check that compiled program does not contain vars that can be accessed from a global root.")
   val YkeepComments = BooleanSetting("-Ykeep-comments", "Keep comments when scanning source files.")
+  val YforceSbtPhases = BooleanSetting("-Yforce-sbt-phases", "Run the phases used by sbt for incremental compilation (ExtractDependencies and ExtractAPI) even if the compiler is ran outside of sbt, for debugging.")
+  val YdumpSbtInc = BooleanSetting("-Ydump-sbt-inc", "For every compiled foo.scala, output the API representation and dependencies used for sbt incremental compilation in foo.inc, implies -Yforce-sbt-phases.")
   def stop = YstopAfter
 
   /** Area-specific debug output.

src/dotty/tools/dotc/core/Contexts.scala

@@ -29,6 +29,7 @@ import printing._
 import config.{Settings, ScalaSettings, Platform, JavaPlatform, SJSPlatform}
 import language.implicitConversions
 import DenotTransformers.DenotTransformer
+import xsbti.AnalysisCallback
 
 object Contexts {
 
@@ -84,6 +85,12 @@ object Contexts {
       _compilerCallback = callback
     def compilerCallback: CompilerCallback = _compilerCallback
 
+    /** The sbt callback implementation if we are run from sbt, null otherwise */
+    private[this] var _sbtCallback: AnalysisCallback = _
+    protected def sbtCallback_=(callback: AnalysisCallback) =
+      _sbtCallback = callback
+    def sbtCallback: AnalysisCallback = _sbtCallback
+
     /** The current context */
     private[this] var _period: Period = _
     protected def period_=(period: Period) = {
@@ -426,6 +433,7 @@ object Contexts {
     def setPeriod(period: Period): this.type = { this.period = period; this }
     def setMode(mode: Mode): this.type = { this.mode = mode; this }
     def setCompilerCallback(callback: CompilerCallback): this.type = { this.compilerCallback = callback; this }
+    def setSbtCallback(callback: AnalysisCallback): this.type = { this.sbtCallback = callback; this }
     def setTyperState(typerState: TyperState): this.type = { this.typerState = typerState; this }
     def setReporter(reporter: Reporter): this.type = setTyperState(typerState.withReporter(reporter))
     def setNewTyperState: this.type = setTyperState(typerState.fresh(isCommittable = true))

src/dotty/tools/dotc/core/SymDenotations.scala

@@ -868,7 +868,7 @@ object SymDenotations {
      */
     final def topLevelClass(implicit ctx: Context): Symbol = {
       def topLevel(d: SymDenotation): Symbol = {
-        if ((d is PackageClass) || (d.owner is PackageClass)) d.symbol
+        if (d.isEffectiveRoot || (d is PackageClass) || (d.owner is PackageClass)) d.symbol
         else topLevel(d.owner)
       }
       val sym = topLevel(this)

src/dotty/tools/dotc/core/Symbols.scala

@@ -519,7 +519,7 @@ object Symbols {
 
     /** The source or class file from which this class was generated, null if not applicable. */
     override def associatedFile(implicit ctx: Context): AbstractFile =
-      if (assocFile != null || (this.owner is PackageClass)) assocFile
+      if (assocFile != null || (this.owner is PackageClass) || this.isEffectiveRoot) assocFile
       else super.associatedFile
 
     final def classDenot(implicit ctx: Context): ClassDenotation =

src/dotty/tools/dotc/core/Types.scala

@@ -415,8 +415,16 @@ object Types {
       memberExcluding(name, Flags.Private)
     }
 
-    final def memberExcluding(name: Name, excluding: FlagSet)(implicit ctx: Context): Denotation =
-      findMember(name, widenIfUnstable, excluding)
+    final def memberExcluding(name: Name, excluding: FlagSet)(implicit ctx: Context): Denotation = {
+      // We need a valid prefix for `asSeenFrom`
+      val pre = this match {
+        case tp: ClassInfo =>
+          tp.typeRef
+        case _ =>
+          widenIfUnstable
+      }
+      findMember(name, pre, excluding)
+    }
 
     /** Find member of this type with given name and
      *  produce a denotation that contains the type of the member
@@ -2476,10 +2484,14 @@ object Types {
 
   abstract class ParamType extends BoundType {
     def paramNum: Int
+    def paramName: Name
   }
 
   abstract case class MethodParam(binder: MethodType, paramNum: Int) extends ParamType with SingletonType {
     type BT = MethodType
+
+    def paramName = binder.paramNames(paramNum)
+
     override def underlying(implicit ctx: Context): Type = binder.paramTypes(paramNum)
     def copyBoundType(bt: BT) = new MethodParamImpl(bt, paramNum)
 
@@ -2492,7 +2504,7 @@ object Types {
         false
     }
 
-    override def toString = s"MethodParam(${binder.paramNames(paramNum)})"
+    override def toString = s"MethodParam($paramName)"
   }
 
   class MethodParamImpl(binder: MethodType, paramNum: Int) extends MethodParam(binder, paramNum)
@@ -2522,9 +2534,11 @@ object Types {
       case _ => false
     }
 
+    def paramName = binder.paramNames(paramNum)
+
     override def underlying(implicit ctx: Context): Type = binder.paramBounds(paramNum)
     // no customized hashCode/equals needed because cycle is broken in PolyType
-    override def toString = s"PolyParam(${binder.paramNames(paramNum)})"
+    override def toString = s"PolyParam($paramName)"
 
     override def computeHash = doHash(paramNum, binder.identityHash)
 
@@ -2776,9 +2790,9 @@ object Types {
       if ((prefix eq cls.owner.thisType) || !cls.owner.isClass || ctx.erasedTypes) tp
       else tp.substThis(cls.owner.asClass, prefix)
 
-    private var typeRefCache: Type = null
+    private var typeRefCache: TypeRef = null
 
-    def typeRef(implicit ctx: Context): Type = {
+    def typeRef(implicit ctx: Context): TypeRef = {
       def clsDenot = if (prefix eq cls.owner.thisType) cls.denot else cls.denot.copySymDenotation(info = this)
       if (typeRefCache == null)
         typeRefCache =
@@ -2787,7 +2801,7 @@ object Types {
       typeRefCache
     }
 
-    def symbolicTypeRef(implicit ctx: Context): Type = TypeRef(prefix, cls)
+    def symbolicTypeRef(implicit ctx: Context): TypeRef = TypeRef(prefix, cls)
 
     // cached because baseType needs parents
     private var parentsCache: List[TypeRef] = null
@@ -3400,6 +3414,12 @@ object Types {
     }
   }
 
+  abstract class TypeTraverser(implicit ctx: Context) extends TypeAccumulator[Unit] {
+    def traverse(tp: Type): Unit
+    def apply(x: Unit, tp: Type): Unit = traverse(tp)
+    protected def traverseChildren(tp: Type) = foldOver((), tp)
+  }
+
   class ExistsAccumulator(p: Type => Boolean)(implicit ctx: Context) extends TypeAccumulator[Boolean] {
     override def stopAtStatic = false
     def apply(x: Boolean, tp: Type) = x || p(tp) || foldOver(x, tp)