Merge pull request #4789 from dotty-staging/topic/dottydoc-from-tasty

Support `-from-tasty` in Dottydoc

Author: Duhemm

compiler/src/dotty/tools/dotc/Compiler.scala

@@ -59,7 +59,8 @@ class Compiler {
   protected def transformPhases: List[List[Phase]] =
     List(new FirstTransform,         // Some transformations to put trees into a canonical form
          new CheckReentrant,         // Internal use only: Check that compiled program has no data races involving global vars
-         new ElimPackagePrefixes) :: // Eliminate references to package prefixes in Select nodes
+         new ElimPackagePrefixes,    // Eliminate references to package prefixes in Select nodes
+         new CookComments) ::        // Cook the comments: expand variables, doc, etc.
     List(new CheckStatic,            // Check restrictions that apply to @static members
          new ElimRepeated,           // Rewrite vararg parameters and arguments
          new ExpandSAMs,             // Expand single abstract method closures to anonymous classes

compiler/src/dotty/tools/dotc/core/Comments.scala

@@ -3,7 +3,7 @@ package dotc
 package core
 
 import ast.{ untpd, tpd }
-import Decorators._, Symbols._, Contexts._, Flags.EmptyFlags
+import Decorators._, Symbols._, Contexts._
 import util.SourceFile
 import util.Positions._
 import util.CommentParsing._
@@ -33,41 +33,58 @@ object Comments {
     def docstring(sym: Symbol): Option[Comment] = _docstrings.get(sym)
 
     def addDocstring(sym: Symbol, doc: Option[Comment]): Unit =
-      doc.map(d => _docstrings.update(sym, d))
+      doc.foreach(d => _docstrings.update(sym, d))
   }
 
-  /** A `Comment` contains the unformatted docstring as well as a position
-    *
-    * The `Comment` contains functionality to create versions of itself without
-    * `@usecase` sections as well as functionality to map the `raw` docstring
-    */
-  abstract case class Comment(pos: Position, raw: String) { self =>
-    def isExpanded: Boolean
+  /**
+   * A `Comment` contains the unformatted docstring, it's position and potentially more
+   * information that is populated when the comment is "cooked".
+   *
+   * @param pos      The position of this `Comment`.
+   * @param raw      The raw comment, as seen in the source code, without any expansion.
+   * @param expanded If this comment has been expanded, it's expansion, otherwise `None`.
+   * @param usecases The usecases for this comment.
+   */
+  final case class Comment(pos: Position, raw: String, expanded: Option[String], usecases: List[UseCase]) {
 
-    def usecases: List[UseCase]
+    /** Has this comment been cooked or expanded? */
+    def isExpanded: Boolean = expanded.isDefined
 
-    val isDocComment = raw.startsWith("/**")
+    /** The body of this comment, without the `@usecase` and `@define` sections, after expansion. */
+    lazy val expandedBody: Option[String] =
+      expanded.map(removeSections(_, "@usecase", "@define"))
 
-    def expand(f: String => String): Comment = new Comment(pos, f(raw)) {
-      val isExpanded = true
-      val usecases = self.usecases
-    }
+    val isDocComment = Comment.isDocComment(raw)
 
-    def withUsecases(implicit ctx: Context): Comment = new Comment(pos, stripUsecases) {
-      val isExpanded = self.isExpanded
-      val usecases = parseUsecases
+    /**
+     * Expands this comment by giving its content to `f`, and then parsing the `@usecase` sections.
+     * Typically, `f` will take care of expanding the variables.
+     *
+     * @param f The expansion function.
+     * @return The expanded comment, with the `usecases` populated.
+     */
+    def expand(f: String => String)(implicit ctx: Context): Comment = {
+      val expandedComment = f(raw)
+      val useCases = Comment.parseUsecases(expandedComment, pos)
+      Comment(pos, raw, Some(expandedComment), useCases)
     }
+  }
+
+  object Comment {
 
-    private[this] lazy val stripUsecases: String =
-      removeSections(raw, "@usecase", "@define")
+    def isDocComment(comment: String): Boolean = comment.startsWith("/**")
 
-    private[this] def parseUsecases(implicit ctx: Context): List[UseCase] =
-      if (!raw.startsWith("/**"))
-        List.empty[UseCase]
-      else
-        tagIndex(raw)
-        .filter { startsWithTag(raw, _, "@usecase") }
-        .map { case (start, end) => decomposeUseCase(start, end) }
+    def apply(pos: Position, raw: String): Comment =
+      Comment(pos, raw, None, Nil)
+
+    private def parseUsecases(expandedComment: String, pos: Position)(implicit ctx: Context): List[UseCase] =
+      if (!isDocComment(expandedComment)) {
+        Nil
+      } else {
+        tagIndex(expandedComment)
+          .filter { startsWithTag(expandedComment, _, "@usecase") }
+          .map { case (start, end) => decomposeUseCase(expandedComment, pos, start, end) }
+      }
 
     /** Turns a usecase section into a UseCase, with code changed to:
      *  {{{
@@ -77,7 +94,7 @@ object Comments {
      *  def foo: A = ???
      *  }}}
      */
-    private[this] def decomposeUseCase(start: Int, end: Int)(implicit ctx: Context): UseCase = {
+    private[this] def decomposeUseCase(body: String, pos: Position, start: Int, end: Int)(implicit ctx: Context): UseCase = {
       def subPos(start: Int, end: Int) =
         if (pos == NoPosition) NoPosition
         else {
@@ -86,49 +103,34 @@ object Comments {
           pos withStart start1 withPoint start1 withEnd end1
         }
 
-      val codeStart    = skipWhitespace(raw, start + "@usecase".length)
-      val codeEnd      = skipToEol(raw, codeStart)
-      val code         = raw.substring(codeStart, codeEnd) + " = ???"
-      val codePos      = subPos(codeStart, codeEnd)
-      val commentStart = skipLineLead(raw, codeEnd + 1) min end
-      val commentStr   = "/** " + raw.substring(commentStart, end) + "*/"
-      val commentPos   = subPos(commentStart, end)
+      val codeStart = skipWhitespace(body, start + "@usecase".length)
+      val codeEnd   = skipToEol(body, codeStart)
+      val code      = body.substring(codeStart, codeEnd) + " = ???"
+      val codePos   = subPos(codeStart, codeEnd)
 
-      UseCase(Comment(commentPos, commentStr), code, codePos)
+      UseCase(code, codePos)
     }
   }
 
-  object Comment {
-    def apply(pos: Position, raw: String, expanded: Boolean = false, usc: List[UseCase] = Nil): Comment =
-      new Comment(pos, raw) {
-        val isExpanded = expanded
-        val usecases = usc
-      }
-  }
-
-  abstract case class UseCase(comment: Comment, code: String, codePos: Position) {
-    /** Set by typer */
-    var tpdCode: tpd.DefDef = _
-
-    def untpdCode: untpd.Tree
+  final case class UseCase(code: String, codePos: Position, untpdCode: untpd.Tree, tpdCode: Option[tpd.DefDef]) {
+    def typed(tpdCode: tpd.DefDef): UseCase = copy(tpdCode = Some(tpdCode))
   }
 
   object UseCase {
-    def apply(comment: Comment, code: String, codePos: Position)(implicit ctx: Context) =
-      new UseCase(comment, code, codePos) {
-        val untpdCode = {
-          val tree = new Parser(new SourceFile("<usecase>", code)).localDef(codePos.start)
-
-          tree match {
-            case tree: untpd.DefDef =>
-              val newName = ctx.freshNames.newName(tree.name, NameKinds.DocArtifactName)
-              untpd.DefDef(newName, tree.tparams, tree.vparamss, tree.tpt, tree.rhs)
-            case _ =>
-              ctx.error(ProperDefinitionNotFound(), codePos)
-              tree
-          }
+    def apply(code: String, codePos: Position)(implicit ctx: Context): UseCase = {
+      val tree = {
+        val tree = new Parser(new SourceFile("<usecase>", code)).localDef(codePos.start)
+        tree match {
+          case tree: untpd.DefDef =>
+            val newName = ctx.freshNames.newName(tree.name, NameKinds.DocArtifactName)
+            tree.copy(name = newName)
+          case _ =>
+            ctx.error(ProperDefinitionNotFound(), codePos)
+            tree
         }
       }
+      UseCase(code, codePos, tree, None)
+    }
   }
 
   /**

compiler/src/dotty/tools/dotc/core/tasty/CommentPickler.scala

@@ -23,7 +23,7 @@ class CommentPickler(pickler: TastyPickler, addrOfTree: tpd.Tree => Option[Addr]
       buf.writeAddr(addr)
       buf.writeNat(length)
       buf.writeBytes(bytes, length)
-      buf.writeByte(if (cmt.isExpanded) 1 else 0)
+      buf.writeLongInt(cmt.pos.coords)
     case other =>
       ()
   }

compiler/src/dotty/tools/dotc/core/tasty/CommentUnpickler.scala

@@ -3,7 +3,7 @@ package dotty.tools.dotc.core.tasty
 import dotty.tools.dotc.core.Comments.Comment
 import dotty.tools.dotc.core.Symbols.Symbol
 import dotty.tools.dotc.core.tasty.TastyBuffer.Addr
-import dotty.tools.dotc.util.Positions
+import dotty.tools.dotc.util.Positions.Position
 
 import scala.collection.mutable.HashMap
 
@@ -19,9 +19,9 @@ class CommentUnpickler(reader: TastyReader) {
       val length = readNat()
       if (length > 0) {
         val bytes = readBytes(length)
-        val expanded = readByte() == 1
+        val position = new Position(readLongInt())
         val rawComment = new String(bytes, Charset.forName("UTF-8"))
-        comments(addr) = Comment(Positions.NoPosition, rawComment, expanded = expanded)
+        comments(addr) = Comment(position, rawComment)
       }
     }
     comments.toMap

compiler/src/dotty/tools/dotc/core/tasty/TastyFormat.scala

@@ -235,8 +235,7 @@ Standard Section: "Positions" Assoc*
 
 Standard Section: "Comments" Comment*
 
-  Comment       = Length Bytes Byte         // Raw comment's bytes encoded as UTF-8, plus a byte indicating
-                                            // whether the comment is expanded (1) or not expanded (0)
+  Comment       = Length Bytes LongInt      // Raw comment's bytes encoded as UTF-8, followed by the comment's coordinates.
 
 
 **************************************************************************************/

compiler/src/dotty/tools/dotc/fromtasty/ReadTastyTreesFromClasses.scala

@@ -33,7 +33,7 @@ class ReadTastyTreesFromClasses extends FrontEnd {
       }
 
       def alreadyLoaded(): None.type = {
-        ctx.warning(s"sclass $className cannot be unpickled because it is already loaded")
+        ctx.warning(s"class $className cannot be unpickled because it is already loaded")
         None
       }
 
@@ -72,5 +72,7 @@ class ReadTastyTreesFromClasses extends FrontEnd {
         case _ =>
           cannotUnpickle(s"no class file was found")
       }
+    case unit =>
+     Some(unit)
   }
 }

compiler/src/dotty/tools/dotc/interactive/InteractiveCompiler.scala

@@ -13,6 +13,7 @@ class InteractiveCompiler extends Compiler {
   // This could be improved by reporting errors back to the IDE
   // after each phase group instead of waiting for the pipeline to finish.
   override def phases: List[List[Phase]] = List(
-    List(new FrontEnd)
+    List(new FrontEnd),
+    List(new transform.CookComments)
   )
 }

compiler/src/dotty/tools/dotc/interactive/InteractiveDriver.scala

@@ -33,6 +33,7 @@ class InteractiveDriver(val settings: List[String]) extends Driver {
   private val myInitCtx: Context = {
     val rootCtx = initCtx.fresh.addMode(Mode.ReadPositions).addMode(Mode.Interactive).addMode(Mode.ReadComments)
     rootCtx.setSetting(rootCtx.settings.YretainTrees, true)
+    rootCtx.setSetting(rootCtx.settings.YcookComments, true)
     val ctx = setup(settings.toArray, rootCtx)._2
     ctx.initialize()(ctx)
     ctx

compiler/src/dotty/tools/dotc/transform/CookComments.scala

@@ -0,0 +1,28 @@
+package dotty.tools.dotc.transform
+
+import dotty.tools.dotc.ast.tpd
+import dotty.tools.dotc.core.Contexts.Context
+import dotty.tools.dotc.typer.Docstrings
+
+class CookComments extends MegaPhase.MiniPhase {
+  override def phaseName = "cookComments"
+
+  override def transformTypeDef(tree: tpd.TypeDef)(implicit ctx: Context): tpd.Tree = {
+    if (ctx.settings.YcookComments.value && tree.isClassDef) {
+      val cls = tree.symbol
+      val cookingCtx = ctx.localContext(tree, cls).setNewScope
+      val template = tree.rhs.asInstanceOf[tpd.Template]
+      val owner = template.self.symbol.orElse(cls)
+
+      template.body.foreach { stat =>
+        Docstrings.cookComment(stat.symbol, owner)(cookingCtx)
+      }
+
+      Docstrings.cookComment(cls, cls)(cookingCtx)
+    }
+
+    tree
+
+  }
+
+}

compiler/src/dotty/tools/dotc/typer/Docstrings.scala

@@ -4,53 +4,67 @@ package typer
 
 import core._
 import Contexts._, Symbols._, Decorators._, Comments._
-import util.Positions._
 import ast.tpd
 
-trait Docstrings { self: Typer =>
+object Docstrings {
 
-  /** The Docstrings typer will handle the expansion of `@define` and
-    * `@inheritdoc` if there is a `DocContext` present as a property in the
-    * supplied `ctx`.
-    *
-    * It will also type any `@usecase` available in function definitions.
-    */
-  def cookComments(syms: List[Symbol], owner: Symbol)(implicit ctx: Context): Unit =
-    ctx.docCtx.foreach { docbase =>
-      val relevantSyms = syms.filter(docbase.docstring(_).exists(!_.isExpanded))
-      relevantSyms.foreach { sym =>
-        expandParentDocs(sym)
-        val usecases = docbase.docstring(sym).map(_.usecases).getOrElse(Nil)
-
-        usecases.foreach { usecase =>
-          enterSymbol(createSymbol(usecase.untpdCode))
+  /**
+   * Expands or cooks the documentation for `sym` in class `owner`.
+   * The expanded comment will directly replace the original comment in the doc context.
+   *
+   * The expansion registers `@define` sections, and will replace `@inheritdoc` and variable
+   * occurrences in the comments.
+   *
+   * If the doc comments contain `@usecase` sections, they will be typed.
+   *
+   * @param sym   The symbol for which the comment is being cooked.
+   * @param owner The class for which comments are being cooked.
+   */
+  def cookComment(sym: Symbol, owner: Symbol)(implicit ctx: Context): Option[Comment] = {
+    ctx.docCtx.flatMap { docCtx =>
+      expand(sym, owner)(ctx, docCtx)
+    }
+  }
 
-          typedStats(usecase.untpdCode :: Nil, owner) match {
-            case List(df: tpd.DefDef) => usecase.tpdCode = df
-            case _ => ctx.error("`@usecase` was not a valid definition", usecase.codePos)
+  private def expand(sym: Symbol, owner: Symbol)(implicit ctx: Context, docCtx: ContextDocstrings): Option[Comment] = {
+    docCtx.docstring(sym).flatMap {
+      case cmt if cmt.isExpanded =>
+        Some(cmt)
+      case _ =>
+        expandComment(sym).map { expanded =>
+          val typedUsecases = expanded.usecases.map { usecase =>
+            ctx.typer.enterSymbol(ctx.typer.createSymbol(usecase.untpdCode))
+            ctx.typer.typedStats(usecase.untpdCode :: Nil, owner) match {
+              case List(df: tpd.DefDef) =>
+                usecase.typed(df)
+              case _ =>
+                ctx.error("`@usecase` was not a valid definition", usecase.codePos)
+                usecase
+            }
           }
+
+          val commentWithUsecases = expanded.copy(usecases = typedUsecases)
+          docCtx.addDocstring(sym, Some(commentWithUsecases))
+          commentWithUsecases
         }
-      }
     }
+  }
 
-  private def expandParentDocs(sym: Symbol)(implicit ctx: Context): Unit =
-    ctx.docCtx.foreach { docCtx =>
-      docCtx.docstring(sym).foreach { cmt =>
-        def expandDoc(owner: Symbol): Unit = if (!cmt.isExpanded) {
-          val tplExp = docCtx.templateExpander
-          tplExp.defineVariables(sym)
-
-          val newCmt = cmt
-            .expand(tplExp.expandedDocComment(sym, owner, _))
-            .withUsecases
+  private def expandComment(sym: Symbol, owner: Symbol, comment: Comment)(implicit ctx: Context, docCtx: ContextDocstrings): Comment = {
+    val tplExp = docCtx.templateExpander
+    tplExp.defineVariables(sym)
+    val newComment = comment.expand(tplExp.expandedDocComment(sym, owner, _))
+    docCtx.addDocstring(sym, Some(newComment))
+    newComment
+  }
 
-          docCtx.addDocstring(sym, Some(newCmt))
-        }
-
-        if (sym ne NoSymbol) {
-          expandParentDocs(sym.owner)
-          expandDoc(sym.owner)
-        }
-      }
+  private def expandComment(sym: Symbol)(implicit ctx: Context, docCtx: ContextDocstrings): Option[Comment] = {
+    if (sym eq NoSymbol) None
+    else {
+      for {
+        cmt <- docCtx.docstring(sym) if !cmt.isExpanded
+        _ = expandComment(sym.owner)
+      } yield expandComment(sym, sym.owner, cmt)
     }
+  }
 }