Merge pull request #1151 from felixmulder/topic/wip-docstrings

Add support for raw docstrings in ASTs

Author: odersky

src/dotty/tools/dotc/ast/Trees.scala

@@ -29,6 +29,9 @@ object Trees {
   /** The total number of created tree nodes, maintained if Stats.enabled */
   @sharable var ntrees = 0
 
+  /** Attachment key for trees with documentation strings attached */
+  val DocComment = new Attachment.Key[String]
+
   /** Modifiers and annotations for definitions
    *  @param flags          The set flags
    *  @param privateWithin  If a private or protected has is followed by a
@@ -321,6 +324,8 @@ object Trees {
     private[ast] def rawMods: Modifiers[T] =
       if (myMods == null) genericEmptyModifiers else myMods
 
+    def rawComment: Option[String] = getAttachment(DocComment)
+
     def withMods(mods: Modifiers[Untyped]): ThisTree[Untyped] = {
       val tree = if (myMods == null || (myMods == mods)) this else clone.asInstanceOf[MemberDef[Untyped]]
       tree.setMods(mods)
@@ -329,6 +334,11 @@ object Trees {
 
     def withFlags(flags: FlagSet): ThisTree[Untyped] = withMods(Modifiers(flags))
 
+    def setComment(comment: Option[String]): ThisTree[Untyped] = {
+      comment.map(putAttachment(DocComment, _))
+      asInstanceOf[ThisTree[Untyped]]
+    }
+
     protected def setMods(mods: Modifiers[T @uncheckedVariance]) = myMods = mods
 
     override def envelope: Position = rawMods.pos.union(pos).union(initialPos)

src/dotty/tools/dotc/config/Printers.scala

@@ -13,6 +13,7 @@ object Printers {
   }
 
   val default: Printer = new Printer
+  val dottydoc: Printer = noPrinter
   val core: Printer = noPrinter
   val typr: Printer = noPrinter
   val constr: Printer = noPrinter

src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -159,6 +159,7 @@ class ScalaSettings extends Settings.SettingGroup {
   val YprintSyms = BooleanSetting("-Yprint-syms", "when printing trees print info in symbols instead of corresponding info in trees.")
   val YtestPickler = BooleanSetting("-Ytest-pickler", "self-test for pickling functionality; should be used with -Ystop-after:pickler")
   val YcheckReentrant = BooleanSetting("-Ycheck-reentrant", "check that compiled program does not contain vars that can be accessed from a global root.")
+  val YkeepComments = BooleanSetting("-Ykeep-comments", "Keep comments when scanning source files.")
   def stop = YstopAfter
 
   /** Area-specific debug output.

src/dotty/tools/dotc/core/Contexts.scala

@@ -550,6 +550,14 @@ object Contexts {
     def squashed(p: Phase): Phase = {
       allPhases.find(_.period.containsPhaseId(p.id)).getOrElse(NoPhase)
     }
+
+    val _docstrings: mutable.Map[Symbol, String] =
+      mutable.Map.empty
+
+    def docstring(sym: Symbol): Option[String] = _docstrings.get(sym)
+
+    def addDocstring(sym: Symbol, doc: Option[String]): Unit =
+      doc.map(d => _docstrings += (sym -> d))
   }
 
   /** The essential mutable state of a context base, collected into a common class */

src/dotty/tools/dotc/parsing/Parsers.scala

@@ -1752,13 +1752,13 @@ object Parsers {
      */
     def defOrDcl(start: Int, mods: Modifiers): Tree = in.token match {
       case VAL =>
-        patDefOrDcl(posMods(start, mods))
+        patDefOrDcl(posMods(start, mods), in.getDocString(start))
       case VAR =>
-        patDefOrDcl(posMods(start, addFlag(mods, Mutable)))
+        patDefOrDcl(posMods(start, addFlag(mods, Mutable)), in.getDocString(start))
       case DEF =>
-        defDefOrDcl(posMods(start, mods))
+        defDefOrDcl(posMods(start, mods), in.getDocString(start))
       case TYPE =>
-        typeDefOrDcl(posMods(start, mods))
+        typeDefOrDcl(posMods(start, mods), in.getDocString(start))
       case _ =>
         tmplDef(start, mods)
     }
@@ -1768,7 +1768,7 @@ object Parsers {
      *  ValDcl ::= Id {`,' Id} `:' Type
      *  VarDcl ::= Id {`,' Id} `:' Type
      */
-    def patDefOrDcl(mods: Modifiers): Tree = {
+    def patDefOrDcl(mods: Modifiers, docstring: Option[String] = None): Tree = {
       val lhs = commaSeparated(pattern2)
       val tpt = typedOpt()
       val rhs =
@@ -1782,8 +1782,10 @@ object Parsers {
           }
         } else EmptyTree
       lhs match {
-        case (id @ Ident(name: TermName)) :: Nil => cpy.ValDef(id)(name, tpt, rhs).withMods(mods)
-        case _ => PatDef(mods, lhs, tpt, rhs)
+        case (id @ Ident(name: TermName)) :: Nil => {
+          cpy.ValDef(id)(name, tpt, rhs).withMods(mods).setComment(docstring)
+        } case _ =>
+          PatDef(mods, lhs, tpt, rhs)
       }
     }
 
@@ -1792,7 +1794,7 @@ object Parsers {
      *  DefDcl ::= DefSig `:' Type
      *  DefSig ::= id [DefTypeParamClause] ParamClauses
      */
-    def defDefOrDcl(mods: Modifiers): Tree = atPos(tokenRange) {
+    def defDefOrDcl(mods: Modifiers, docstring: Option[String] = None): Tree = atPos(tokenRange) {
       def scala2ProcedureSyntax(resultTypeStr: String) = {
         val toInsert =
           if (in.token == LBRACE) s"$resultTypeStr ="
@@ -1833,7 +1835,7 @@ object Parsers {
             accept(EQUALS)
             expr()
           }
-        DefDef(name, tparams, vparamss, tpt, rhs).withMods(mods1)
+        DefDef(name, tparams, vparamss, tpt, rhs).withMods(mods1).setComment(docstring)
       }
     }
 
@@ -1867,15 +1869,15 @@ object Parsers {
     /** TypeDef ::= type Id [TypeParamClause] `=' Type
      *  TypeDcl ::= type Id [TypeParamClause] TypeBounds
      */
-    def typeDefOrDcl(mods: Modifiers): Tree = {
+    def typeDefOrDcl(mods: Modifiers, docstring: Option[String] = None): Tree = {
       newLinesOpt()
       atPos(tokenRange) {
         val name = ident().toTypeName
         val tparams = typeParamClauseOpt(ParamOwner.Type)
         in.token match {
           case EQUALS =>
             in.nextToken()
-            TypeDef(name, tparams, typ()).withMods(mods)
+            TypeDef(name, tparams, typ()).withMods(mods).setComment(docstring)
           case SUPERTYPE | SUBTYPE | SEMI | NEWLINE | NEWLINES | COMMA | RBRACE | EOF =>
             TypeDef(name, tparams, typeBounds()).withMods(mods)
           case _ =>
@@ -1888,35 +1890,40 @@ object Parsers {
     /** TmplDef ::= ([`case'] `class' | `trait') ClassDef
      *            |  [`case'] `object' ObjectDef
      */
-    def tmplDef(start: Int, mods: Modifiers): Tree = in.token match {
-      case TRAIT =>
-        classDef(posMods(start, addFlag(mods, Trait)))
-      case CLASS =>
-        classDef(posMods(start, mods))
-      case CASECLASS =>
-        classDef(posMods(start, mods | Case))
-      case OBJECT =>
-        objectDef(posMods(start, mods | Module))
-      case CASEOBJECT =>
-        objectDef(posMods(start, mods | Case | Module))
-      case _ =>
-        syntaxErrorOrIncomplete("expected start of definition")
-        EmptyTree
+    def tmplDef(start: Int, mods: Modifiers): Tree = {
+      val docstring = in.getDocString(start)
+      in.token match {
+        case TRAIT =>
+          classDef(posMods(start, addFlag(mods, Trait)), docstring)
+        case CLASS =>
+          classDef(posMods(start, mods), docstring)
+        case CASECLASS =>
+          classDef(posMods(start, mods | Case), docstring)
+        case OBJECT =>
+          objectDef(posMods(start, mods | Module), docstring)
+        case CASEOBJECT =>
+          objectDef(posMods(start, mods | Case | Module), docstring)
+        case _ =>
+          syntaxErrorOrIncomplete("expected start of definition")
+          EmptyTree
+      }
     }
 
     /** ClassDef ::= Id [ClsTypeParamClause]
      *               [ConstrMods] ClsParamClauses TemplateOpt
      */
-    def classDef(mods: Modifiers): TypeDef = atPos(tokenRange) {
+    def classDef(mods: Modifiers, docstring: Option[String]): TypeDef = atPos(tokenRange) {
       val name = ident().toTypeName
       val constr = atPos(in.offset) {
         val tparams = typeParamClauseOpt(ParamOwner.Class)
         val cmods = constrModsOpt()
         val vparamss = paramClauses(name, mods is Case)
+
         makeConstructor(tparams, vparamss).withMods(cmods)
       }
       val templ = templateOpt(constr)
-      TypeDef(name, templ).withMods(mods)
+
+      TypeDef(name, templ).withMods(mods).setComment(docstring)
     }
 
     /** ConstrMods        ::=  AccessModifier
@@ -1932,10 +1939,11 @@ object Parsers {
 
     /** ObjectDef       ::= Id TemplateOpt
      */
-    def objectDef(mods: Modifiers): ModuleDef = {
+    def objectDef(mods: Modifiers, docstring: Option[String] = None): ModuleDef = {
       val name = ident()
       val template = templateOpt(emptyConstructor())
-      ModuleDef(name, template).withMods(mods)
+
+      ModuleDef(name, template).withMods(mods).setComment(docstring)
     }
 
 /* -------- TEMPLATES ------------------------------------------- */
@@ -2160,7 +2168,8 @@ object Parsers {
         if (in.token == PACKAGE) {
           in.nextToken()
           if (in.token == OBJECT) {
-            ts += objectDef(atPos(start, in.skipToken()) { Modifiers(Package) })
+            val docstring = in.getDocString(start)
+            ts += objectDef(atPos(start, in.skipToken()) { Modifiers(Package) }, docstring)
             if (in.token != EOF) {
               acceptStatSep()
               ts ++= topStatSeq()

src/dotty/tools/dotc/parsing/Scanners.scala

@@ -175,12 +175,39 @@ object Scanners {
   }
 
   class Scanner(source: SourceFile, override val startFrom: Offset = 0)(implicit ctx: Context) extends ScannerCommon(source)(ctx) {
-    var keepComments = false
+    val keepComments = ctx.settings.YkeepComments.value
 
-    /** All comments in the reverse order of their position in the source.
-     *  set only when `keepComments` is true.
+    /** All doc comments as encountered, each list contains doc comments from
+     *  the same block level. Starting with the deepest level and going upward
      */
-    var revComments: List[Comment] = Nil
+    private[this] var docsPerBlockStack: List[List[Comment]] = List(List())
+
+    /** Adds level of nesting to docstrings */
+    def enterBlock(): Unit =
+      docsPerBlockStack = Nil ::: docsPerBlockStack
+
+    /** Removes level of nesting for docstrings */
+    def exitBlock(): Unit = docsPerBlockStack = docsPerBlockStack match {
+      case x :: xs => List(List())
+      case _ => docsPerBlockStack.tail
+    }
+
+    /** Returns the closest docstring preceding the position supplied */
+    def getDocString(pos: Int): Option[String] = {
+      def closest(c: Comment, docstrings: List[Comment]): Comment = docstrings match {
+        case x :: xs if (c.pos.end < x.pos.end && x.pos.end <= pos) => closest(x, xs)
+        case Nil => c
+      }
+
+      docsPerBlockStack match {
+        case (list @ (x :: xs)) :: _ => {
+          val c = closest(x, xs)
+          docsPerBlockStack = list.dropWhile(_ != c).tail :: docsPerBlockStack.tail
+          Some(c.chrs)
+        }
+        case _ => None
+      }
+    }
 
     /** A buffer for comments */
     val commentBuf = new StringBuilder
@@ -487,13 +514,13 @@ object Scanners {
         case ',' =>
           nextChar(); token = COMMA
         case '(' =>
-          nextChar(); token = LPAREN
+          enterBlock(); nextChar(); token = LPAREN
         case '{' =>
-          nextChar(); token = LBRACE
+          enterBlock(); nextChar(); token = LBRACE
         case ')' =>
-          nextChar(); token = RPAREN
+          exitBlock(); nextChar(); token = RPAREN
         case '}' =>
-          nextChar(); token = RBRACE
+          exitBlock(); nextChar(); token = RBRACE
         case '[' =>
           nextChar(); token = LBRACKET
         case ']' =>
@@ -558,9 +585,12 @@ object Scanners {
       def finishComment(): Boolean = {
         if (keepComments) {
           val pos = Position(start, charOffset, start)
-          nextChar()
-          revComments = Comment(pos, flushBuf(commentBuf)) :: revComments
+          val comment = Comment(pos, flushBuf(commentBuf))
+
+          if (comment.isDocComment)
+            docsPerBlockStack = (docsPerBlockStack.head :+ comment) :: docsPerBlockStack.tail
         }
+
         true
       }
       nextChar()

src/dotty/tools/dotc/typer/Namer.scala

@@ -401,19 +401,29 @@ class Namer { typer: Typer =>
       val pkg = createPackageSymbol(pcl.pid)
       index(pcl.stats)(ctx.fresh.setOwner(pkg.moduleClass))
       invalidateCompanions(pkg, Trees.flatten(pcl.stats map expanded))
+      setDocstring(pkg, stat)
       ctx
     case imp: Import =>
       importContext(createSymbol(imp), imp.selectors)
     case mdef: DefTree =>
-      enterSymbol(createSymbol(mdef))
+      val sym = enterSymbol(createSymbol(mdef))
+      setDocstring(sym, stat)
       ctx
     case stats: Thicket =>
-      for (tree <- stats.toList) enterSymbol(createSymbol(tree))
+      for (tree <- stats.toList) {
+        val sym = enterSymbol(createSymbol(tree))
+        setDocstring(sym, stat)
+      }
       ctx
     case _ =>
       ctx
   }
 
+  def setDocstring(sym: Symbol, tree: Tree)(implicit ctx: Context) = tree match {
+    case t: MemberDef => ctx.base.addDocstring(sym, t.rawComment)
+    case _ => ()
+  }
+
   /** Create top-level symbols for statements and enter them into symbol table */
   def index(stats: List[Tree])(implicit ctx: Context): Context = {
 
@@ -859,7 +869,7 @@ class Namer { typer: Typer =>
         WildcardType
     }
     paramFn(typedAheadType(mdef.tpt, tptProto).tpe)
- }
+  }
 
   /** The type signature of a DefDef with given symbol */
   def defDefSig(ddef: DefDef, sym: Symbol)(implicit ctx: Context) = {

src/dotty/tools/dotc/typer/Typer.scala

@@ -586,7 +586,7 @@ class Typer extends Namer with TypeAssigner with Applications with Implicits wit
         case _ => false
       }
 
-      /** The funcion body to be returned in the closure. Can become a TypedSplice
+      /** The function body to be returned in the closure. Can become a TypedSplice
        *  of a typed expression if this is necessary to infer a parameter type.
        */
       var fnBody = tree.body