Skip to content

Commit a48ee6d

Browse files
DuhemmDuhemm
committed
Always pickle uncooked comments
The comments are now always pickled as raw comments, as seen in the source file. It's the job of Dotty doc to trigger expansion of the comments after they are unpickled. This approach simplifies incremental generation of the documentation, as we don't need to teach Zinc how to extract dependencies in the documentation.
1 parent 12e4605 commit a48ee6d

16 files changed

+135
-97
lines changed

‎compiler/src/dotty/tools/dotc/core/Comments.scala

Lines changed: 27 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -40,14 +40,21 @@ object Comments {
4040
*
4141
* The `Comment` contains functionality to create versions of itself without
4242
* `@usecase` sections as well as functionality to map the `raw` docstring
43+
*
44+
* @param pos The position of this `Comment`.
45+
* @param raw The raw comment, as seen in the source code, without any expansion.
4346
*/
4447
abstract case class Comment(pos: Position, raw: String) { self =>
48+
49+
/** The expansion of this comment */
50+
def expanded: Option[String]
51+
4552
/** Has this comment been cooked or expanded? */
46-
def isExpanded: Boolean
53+
final def isExpanded: Boolean = expanded.isDefined
4754

48-
/** The body of this comment, without the `@usecase` and `@define` sections. */
49-
lazy val body: String =
50-
removeSections(raw, "@usecase", "@define")
55+
/** The body of this comment, without the `@usecase` and `@define` sections, after expansion. */
56+
lazy val expandedBody: Option[String] =
57+
expanded.map(removeSections(_, "@usecase", "@define"))
5158

5259
/**
5360
* The `@usecase` sections of this comment.
@@ -57,23 +64,26 @@ object Comments {
5764

5865
val isDocComment = raw.startsWith("/**")
5966

60-
def expand(f: String => String): Comment = new Comment(pos, f(raw)) {
61-
val isExpanded = true
67+
def expand(f: String => String): Comment = new Comment(pos, raw) {
68+
val expanded = Some(f(raw))
6269
val usecases = self.usecases
6370
}
6471

6572
def withUsecases(implicit ctx: Context): Comment = new Comment(pos, raw) {
66-
val isExpanded = self.isExpanded
73+
assert(self.isExpanded)
74+
val expanded = self.expanded
6775
val usecases = parseUsecases
6876
}
6977

7078
private[this] def parseUsecases(implicit ctx: Context): List[UseCase] =
7179
if (!isDocComment) {
7280
Nil
7381
} else {
74-
tagIndex(raw)
75-
.filter { startsWithTag(raw, _, "@usecase") }
76-
.map { case (start, end) => decomposeUseCase(start, end) }
82+
expanded.map { body =>
83+
tagIndex(body)
84+
.filter { startsWithTag(body, _, "@usecase") }
85+
.map { case (start, end) => decomposeUseCase(body, start, end) }
86+
}.getOrElse(Nil)
7787
}
7888

7989
/** Turns a usecase section into a UseCase, with code changed to:
@@ -84,7 +94,7 @@ object Comments {
8494
* def foo: A = ???
8595
* }}}
8696
*/
87-
private[this] def decomposeUseCase(start: Int, end: Int)(implicit ctx: Context): UseCase = {
97+
private[this] def decomposeUseCase(body: String, start: Int, end: Int)(implicit ctx: Context): UseCase = {
8898
def subPos(start: Int, end: Int) =
8999
if (pos == NoPosition) NoPosition
90100
else {
@@ -93,19 +103,19 @@ object Comments {
93103
pos withStart start1 withPoint start1 withEnd end1
94104
}
95105

96-
val codeStart = skipWhitespace(raw, start + "@usecase".length)
97-
val codeEnd = skipToEol(raw, codeStart)
98-
val code = raw.substring(codeStart, codeEnd) + " = ???"
99-
val codePos = subPos(codeStart, codeEnd)
106+
val codeStart = skipWhitespace(body, start + "@usecase".length)
107+
val codeEnd = skipToEol(body, codeStart)
108+
val code = body.substring(codeStart, codeEnd) + " = ???"
109+
val codePos = subPos(codeStart, codeEnd)
100110

101111
UseCase(code, codePos)
102112
}
103113
}
104114

105115
object Comment {
106-
def apply(pos: Position, raw: String, expanded: Boolean = false, usc: List[UseCase] = Nil): Comment =
116+
def apply(pos: Position, raw: String, expandedComment: Option[String] = None, usc: List[UseCase] = Nil): Comment =
107117
new Comment(pos, raw) {
108-
val isExpanded = expanded
118+
val expanded = expandedComment
109119
val usecases = usc
110120
}
111121
}

‎compiler/src/dotty/tools/dotc/core/tasty/CommentPickler.scala

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@ class CommentPickler(pickler: TastyPickler, addrOfTree: tpd.Tree => Option[Addr]
2424
buf.writeNat(length)
2525
buf.writeBytes(bytes, length)
2626
buf.writeLongInt(cmt.pos.coords)
27-
buf.writeByte(if (cmt.isExpanded) 1 else 0)
2827
case other =>
2928
()
3029
}

‎compiler/src/dotty/tools/dotc/core/tasty/CommentUnpickler.scala

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,8 @@ class CommentUnpickler(reader: TastyReader) {
2020
if (length > 0) {
2121
val bytes = readBytes(length)
2222
val position = new Position(readLongInt())
23-
val expanded = readByte() == 1
2423
val rawComment = new String(bytes, Charset.forName("UTF-8"))
25-
comments(addr) = Comment(position, rawComment, expanded = expanded)
24+
comments(addr) = Comment(position, rawComment)
2625
}
2726
}
2827
comments.toMap

‎compiler/src/dotty/tools/dotc/core/tasty/TastyFormat.scala

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -235,8 +235,7 @@ Standard Section: "Positions" Assoc*
235235
236236
Standard Section: "Comments" Comment*
237237
238-
Comment = Length Bytes LongInt Byte // Raw comment's bytes encoded as UTF-8, followed by the comment's coordinates,
239-
// plus a byte indicating whether the comment is expanded (1) or not expanded (0)
238+
Comment = Length Bytes LongInt // Raw comment's bytes encoded as UTF-8, followed by the comment's coordinates.
240239
241240
242241
**************************************************************************************/

‎compiler/src/dotty/tools/dotc/typer/Docstrings.scala

Lines changed: 24 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -9,29 +9,33 @@ import ast.tpd
99

1010
trait Docstrings { self: Typer =>
1111

12-
/** The Docstrings typer will handle the expansion of `@define` and
13-
* `@inheritdoc` if there is a `DocContext` present as a property in the
14-
* supplied `ctx`.
15-
*
16-
* It will also type any `@usecase` available in function definitions.
17-
*/
18-
def cookComments(syms: List[Symbol], owner: Symbol)(implicit ctx: Context): Unit =
19-
ctx.docCtx.foreach { docbase =>
20-
val relevantSyms = syms.filter(docbase.docstring(_).exists(!_.isExpanded))
21-
relevantSyms.foreach { sym =>
22-
expandParentDocs(sym)
23-
val usecases = docbase.docstring(sym).map(_.usecases).getOrElse(Nil)
24-
25-
usecases.foreach { usecase =>
26-
enterSymbol(createSymbol(usecase.untpdCode))
27-
28-
typedStats(usecase.untpdCode :: Nil, owner) match {
29-
case List(df: tpd.DefDef) => usecase.tpdCode = df
30-
case _ => ctx.error("`@usecase` was not a valid definition", usecase.codePos)
31-
}
12+
/**
13+
* Expands or cooks the documentation for `sym` in class `owner`.
14+
* The expanded comment will directly replace the original comment in the doc context.
15+
*
16+
* The expansion registers `@define` sections, and will replace `@inheritdoc` and variable
17+
* occurrences in the comments.
18+
*
19+
* If the doc comments contain `@usecase` sections, they will be typed.
20+
*
21+
* @param sym The symbol for which the comment is being cooked.
22+
* @param owner The class for which comments are being cooked.
23+
*/
24+
def cookComment(sym: Symbol, owner: Symbol)(implicit ctx: Context): Unit = {
25+
for {
26+
docbase <- ctx.docCtx
27+
comment <- docbase.docstring(sym).filter(!_.isExpanded)
28+
} {
29+
expandParentDocs(sym)
30+
docbase.docstring(sym).get.usecases.foreach { usecase =>
31+
enterSymbol(createSymbol(usecase.untpdCode))
32+
typedStats(usecase.untpdCode :: Nil, owner) match {
33+
case List(df: tpd.DefDef) => usecase.tpdCode = df
34+
case _ => ctx.error("`@usecase` was not a valid definition", usecase.codePos)
3235
}
3336
}
3437
}
38+
}
3539

3640
private def expandParentDocs(sym: Symbol)(implicit ctx: Context): Unit =
3741
ctx.docCtx.foreach { docCtx =>

‎compiler/src/dotty/tools/dotc/typer/Typer.scala

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1570,7 +1570,11 @@ class Typer extends Namer
15701570

15711571
// Expand comments and type usecases if `-Ycook-comments` is set.
15721572
if (ctx.settings.YcookComments.value) {
1573-
cookComments(cls :: body1.map(_.symbol), self1.symbol)(ctx.localContext(cdef, cls).setNewScope)
1573+
val cookingCtx = ctx.localContext(cdef, cls).setNewScope
1574+
body1.foreach { stat =>
1575+
cookComment(stat.symbol, self1.symbol)(cookingCtx)
1576+
}
1577+
cookComment(cls, cls)(cookingCtx)
15741578
}
15751579

15761580
checkNoDoubleDeclaration(cls)

‎doc-tool/src/dotty/tools/dottydoc/DocFrontEnd.scala

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,18 @@ package dotty.tools
22
package dottydoc
33

44
import dotc.fromtasty.ReadTastyTreesFromClasses
5-
import dotc.typer.FrontEnd
5+
import dotc.typer.{FrontEnd, Typer}
66
import dotc.core.Contexts.Context
77
import dotc.CompilationUnit
88

9+
import util.syntax.ContextWithContextDottydoc
10+
911
/** `DocFrontEnd` uses the Dotty `FrontEnd` without discarding the AnyVal
1012
* interfaces for Boolean, Int, Char, Long, Byte etc.
1113
*
14+
* If `-from-tasty` is set, then the trees and documentation will be loaded
15+
* from TASTY. The comments will be cooked after being unpickled.
16+
*
1217
* It currently still throws away Java sources by overriding
1318
* `discardAfterTyper`.
1419
*/
@@ -17,7 +22,18 @@ class DocFrontEnd extends FrontEnd {
1722
override def runOn(units: List[CompilationUnit])(implicit ctx: Context): List[CompilationUnit] = {
1823
if (ctx.settings.fromTasty.value) {
1924
val fromTastyFrontend = new ReadTastyTreesFromClasses
20-
fromTastyFrontend.runOn(units)
25+
val unpickledUnits = fromTastyFrontend.runOn(units)
26+
27+
val typer = new Typer()
28+
if (ctx.settings.YcookComments.value) {
29+
ctx.docbase.docstrings.keys.foreach { sym =>
30+
val owner = sym.owner
31+
val cookingCtx = ctx.withOwner(owner)
32+
typer.cookComment(sym, owner)(cookingCtx)
33+
}
34+
}
35+
36+
unpickledUnits
2137
} else {
2238
super.runOn(units)
2339
}

‎doc-tool/src/dotty/tools/dottydoc/core/DocstringPhase.scala

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ import util.syntax._
1515
/** Phase to add docstrings to the Dottydoc AST */
1616
class DocstringPhase extends DocMiniPhase with CommentParser with CommentCleaner {
1717

18-
private def getComment(sym: Symbol)(implicit ctx: Context): Option[CompilerComment] =
18+
private def getComment(sym: Symbol)(implicit ctx: Context): Option[CompilerComment] = {
1919
ctx.docbase.docstring(sym)
2020
.orElse {
2121
// If the symbol doesn't have a docstring, look for an overridden
@@ -26,17 +26,20 @@ class DocstringPhase extends DocMiniPhase with CommentParser with CommentCleaner
2626
}
2727
.flatMap(ctx.docbase.docstring)
2828
}
29+
}
2930

30-
private def parsedComment(ent: Entity)(implicit ctx: Context): Option[Comment] =
31-
getComment(ent.symbol).map { cmt =>
32-
val text = cmt.body
33-
val parsed = parse(ent, ctx.docbase.packages, clean(text), text, cmt.pos)
34-
31+
private def parsedComment(ent: Entity)(implicit ctx: Context): Option[Comment] = {
32+
for {
33+
comment <- getComment(ent.symbol)
34+
text <- comment.expandedBody
35+
} yield {
36+
val parsed = parse(ent, ctx.docbase.packages, clean(text), text, comment.pos)
3537
if (ctx.settings.wikiSyntax.value)
36-
WikiComment(ent, parsed, cmt.pos).comment
38+
WikiComment(ent, parsed, comment.pos).comment
3739
else
38-
MarkdownComment(ent, parsed, cmt.pos).comment
40+
MarkdownComment(ent, parsed, comment.pos).comment
3941
}
42+
}
4043

4144
override def transformPackage(implicit ctx: Context) = { case ent: PackageImpl =>
4245
ent.copy(comment = parsedComment(ent))

‎doc-tool/src/dotty/tools/dottydoc/core/UsecasePhase.scala

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ import util.syntax._
1313

1414
class UsecasePhase extends DocMiniPhase {
1515
private def defdefToDef(d: tpd.DefDef, sym: Symbol)(implicit ctx: Context) = {
16-
val name = d.name.show.split("\\$").head // UseCase defs get $pos appended to their names
16+
val name = d.name.show.split("\\$").head
1717
DefImpl(
1818
sym,
1919
annotations(sym),
@@ -27,6 +27,10 @@ class UsecasePhase extends DocMiniPhase {
2727
}
2828

2929
override def transformDef(implicit ctx: Context) = { case df: DefImpl =>
30-
ctx.docbase.docstring(df.symbol).flatMap(_.usecases.headOption.map(_.tpdCode)).map(defdefToDef(_, df.symbol)).getOrElse(df)
30+
ctx.docbase
31+
.docstring(df.symbol)
32+
.flatMap(_.usecases.headOption.map(_.tpdCode))
33+
.map(defdefToDef(_, df.symbol))
34+
.getOrElse(df)
3135
}
3236
}

‎doc-tool/test/ConstructorTest.scala

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ abstract class ConstructorsBase extends DottyDocTest {
2525

2626
val className = "scala.Class"
2727

28-
check(className :: Nil, source :: Nil) { packages =>
28+
check(className :: Nil, source :: Nil) { (ctx, packages) =>
2929
packages("scala") match {
3030
case PackageImpl(_, _, _, List(cls: Class), _, _, _, _) =>
3131
cls.constructors.headOption match {
@@ -49,7 +49,7 @@ abstract class ConstructorsBase extends DottyDocTest {
4949

5050
val className = "scala.Class"
5151

52-
check(className :: Nil, source :: Nil) { packages =>
52+
check(className :: Nil, source :: Nil) { (ctx, packages) =>
5353
packages("scala") match {
5454
case PackageImpl(_, _, _, List(cls: Class), _, _, _, _) =>
5555
cls.constructors match {
@@ -76,7 +76,7 @@ abstract class ConstructorsBase extends DottyDocTest {
7676

7777
val className = "scala.Class"
7878

79-
check(className :: Nil, source :: Nil) { packages =>
79+
check(className :: Nil, source :: Nil) { (ctx, packages) =>
8080
packages("scala") match {
8181
case PackageImpl(_, _, _, List(cls: Class), _, _, _, _) =>
8282
cls.constructors match {
@@ -110,7 +110,7 @@ abstract class ConstructorsBase extends DottyDocTest {
110110

111111
val className = "scala.Class"
112112

113-
check(className :: Nil, source :: Nil) { packages =>
113+
check(className :: Nil, source :: Nil) { (ctx, packages) =>
114114
packages("scala") match {
115115
case PackageImpl(_, _, _, List(cls: Class), _, _, _, _) =>
116116
cls.constructors match {
@@ -150,7 +150,7 @@ abstract class ConstructorsBase extends DottyDocTest {
150150

151151
val className = "scala.Class"
152152

153-
check(className :: Nil, source :: Nil) { packages =>
153+
check(className :: Nil, source :: Nil) { (ctx, packages) =>
154154
packages("scala") match {
155155
case PackageImpl(_, _, _, List(cls: CaseClass, obj: Object), _, _, _, _) =>
156156
cls.constructors match {
@@ -185,7 +185,7 @@ abstract class ConstructorsBase extends DottyDocTest {
185185

186186
val className = "scala.Trait"
187187

188-
check(className :: Nil, source :: Nil) { packages =>
188+
check(className :: Nil, source :: Nil) { (ctx, packages) =>
189189
packages("scala") match {
190190
case PackageImpl(_, _, _, List(trt: Trait), _, _, _, _) =>
191191
trt.traitParams match {

0 commit comments

Comments
 (0)