Skip to content

Commit 378b9f5

Browse files
committed
Add Comments section in TASTY
The comments are pickled when `-Ykeep-comments` is set.
1 parent 6f6816d commit 378b9f5

File tree

11 files changed

+315
-17
lines changed

11 files changed

+315
-17
lines changed

compiler/src/dotty/tools/dotc/Driver.scala

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,9 @@ package dotty.tools.dotc
22

33
import dotty.tools.FatalError
44
import config.CompilerCommand
5+
import core.Comments.{ContextDoc, ContextDocstrings}
56
import core.Contexts.{Context, ContextBase}
7+
import core.Mode
68
import util.DotClass
79
import reporting._
810
import scala.util.control.NonFatal
@@ -40,10 +42,24 @@ class Driver extends DotClass {
4042

4143
protected def sourcesRequired = true
4244

45+
/**
46+
* Should the `ContextDocstrings` be set for this context? The `ContextDocstrings` is used
47+
* to store doc comments when `-Ykeep-comments` is set, or when TASTY is configured to
48+
* unpickle the doc comments.
49+
*/
50+
protected def shouldAddDocContext(implicit ctx: Context): Boolean = {
51+
ctx.settings.YkeepComments.value || ctx.mode.is(Mode.ReadComments)
52+
}
53+
4354
def setup(args: Array[String], rootCtx: Context): (List[String], Context) = {
44-
val ctx = rootCtx.fresh
45-
val summary = CompilerCommand.distill(args)(ctx)
46-
ctx.setSettings(summary.sstate)
55+
val ctx0 = rootCtx.fresh
56+
val summary = CompilerCommand.distill(args)(ctx0)
57+
ctx0.setSettings(summary.sstate)
58+
59+
val ctx =
60+
if (shouldAddDocContext(ctx0)) ctx0.setProperty(ContextDoc, new ContextDocstrings)
61+
else ctx0
62+
4763
val fileNames = CompilerCommand.checkUsage(summary, sourcesRequired)(ctx)
4864
(fileNames, ctx)
4965
}

compiler/src/dotty/tools/dotc/core/Comments.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ object Comments {
9999
}
100100

101101
object Comment {
102-
def apply(pos: Position, raw: String, expanded: Boolean = false, usc: List[UseCase] = Nil)(implicit ctx: Context): Comment =
102+
def apply(pos: Position, raw: String, expanded: Boolean = false, usc: List[UseCase] = Nil): Comment =
103103
new Comment(pos, raw) {
104104
val isExpanded = expanded
105105
val usecases = usc

compiler/src/dotty/tools/dotc/core/Mode.scala

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,4 +94,7 @@ object Mode {
9494
/** We are in the IDE */
9595
val Interactive = newMode(20, "Interactive")
9696

97+
/** Read comments from definitions when unpickling from TASTY */
98+
val ReadComments = newMode(21, "ReadComments")
99+
97100
}

compiler/src/dotty/tools/dotc/core/quoted/TastyUnpickler.scala

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,10 @@ import dotty.tools.dotc.core.tasty._
44
import dotty.tools.dotc.core.tasty.TastyUnpickler.NameTable
55

66
object TastyUnpickler {
7-
class QuotedTreeSectionUnpickler(posUnpickler: Option[PositionUnpickler], splices: Seq[Any])
8-
extends DottyUnpickler.TreeSectionUnpickler(posUnpickler) {
7+
class QuotedTreeSectionUnpickler(posUnpickler: Option[PositionUnpickler], commentUnpickler: Option[CommentUnpickler], splices: Seq[Any])
8+
extends DottyUnpickler.TreeSectionUnpickler(posUnpickler, commentUnpickler) {
99
override def unpickle(reader: TastyReader, nameAtRef: NameTable) =
10-
new TreeUnpickler(reader, nameAtRef, posUnpickler, splices)
10+
new TreeUnpickler(reader, nameAtRef, posUnpickler, commentUnpickler, splices)
1111
}
1212
}
1313

@@ -19,6 +19,6 @@ class TastyUnpickler(bytes: Array[Byte], splices: Seq[Any]) extends DottyUnpickl
1919
import DottyUnpickler._
2020
import TastyUnpickler._
2121

22-
protected override def treeSectionUnpickler(posUnpicklerOpt: Option[PositionUnpickler]): TreeSectionUnpickler =
23-
new QuotedTreeSectionUnpickler(posUnpicklerOpt, splices)
22+
protected override def treeSectionUnpickler(posUnpicklerOpt: Option[PositionUnpickler], commentUnpicklerOpt: Option[CommentUnpickler]): TreeSectionUnpickler =
23+
new QuotedTreeSectionUnpickler(posUnpicklerOpt, commentUnpicklerOpt, splices)
2424
}
Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
package dotty.tools.dotc.core.tasty
2+
3+
import dotty.tools.dotc.ast.tpd
4+
import dotty.tools.dotc.core.Comments.{Comment, CommentsContext}
5+
import dotty.tools.dotc.core.Contexts.Context
6+
import dotty.tools.dotc.core.tasty.TastyBuffer.Addr
7+
8+
import java.nio.charset.Charset
9+
10+
class CommentPickler(pickler: TastyPickler, addrOfTree: tpd.Tree => Option[Addr])(implicit ctx: Context) {
11+
val buf = new TastyBuffer(5000)
12+
pickler.newSection("Comments", buf)
13+
14+
def pickleComment(root: tpd.Tree): Unit =
15+
new Traverser().traverse(root)
16+
17+
def pickleComment(addrOfTree: Option[Addr], comment: Option[Comment]): Unit = (addrOfTree, comment) match {
18+
case (Some(addr), Some(cmt)) =>
19+
val bytes = cmt.raw.getBytes(Charset.forName("UTF-8"))
20+
val length = bytes.length
21+
buf.writeAddr(addr)
22+
buf.writeNat(length)
23+
buf.writeBytes(bytes, length)
24+
case other =>
25+
()
26+
}
27+
28+
private class Traverser extends tpd.TreeTraverser {
29+
override def traverse(tree: tpd.Tree)(implicit ctx: Context): Unit =
30+
tree match {
31+
case md: tpd.MemberDef =>
32+
ctx.docCtx.foreach { docCtx =>
33+
val comment = docCtx.docstring(md.symbol)
34+
pickleComment(addrOfTree(md), comment)
35+
}
36+
traverseChildren(md)
37+
case _ =>
38+
traverseChildren(tree)
39+
}
40+
}
41+
42+
}
Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
package dotty.tools.dotc.core.tasty
2+
3+
import dotty.tools.dotc.core.Comments.Comment
4+
import dotty.tools.dotc.core.Symbols.Symbol
5+
import dotty.tools.dotc.core.tasty.TastyBuffer.Addr
6+
import dotty.tools.dotc.util.Positions
7+
8+
import scala.collection.mutable.HashMap
9+
10+
import java.nio.charset.Charset
11+
12+
class CommentUnpickler(reader: TastyReader) {
13+
import reader._
14+
15+
private[tasty] lazy val comments = {
16+
val comments = new HashMap[Addr, Comment]
17+
while (!isAtEnd) {
18+
val addr = readAddr()
19+
val length = readNat()
20+
if (length > 0) {
21+
val bytes = readBytes(length)
22+
val rawComment = new String(bytes, Charset.forName("UTF-8"))
23+
comments(addr) = Comment(Positions.NoPosition, rawComment)
24+
}
25+
}
26+
comments.toMap
27+
}
28+
29+
def commentAt(addr: Addr): Option[Comment] =
30+
comments.get(addr)
31+
32+
}

compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,16 +17,21 @@ object DottyUnpickler {
1717
/** Exception thrown if classfile is corrupted */
1818
class BadSignature(msg: String) extends RuntimeException(msg)
1919

20-
class TreeSectionUnpickler(posUnpickler: Option[PositionUnpickler])
20+
class TreeSectionUnpickler(posUnpickler: Option[PositionUnpickler], commentUnpickler: Option[CommentUnpickler])
2121
extends SectionUnpickler[TreeUnpickler]("ASTs") {
2222
def unpickle(reader: TastyReader, nameAtRef: NameTable) =
23-
new TreeUnpickler(reader, nameAtRef, posUnpickler, Seq.empty)
23+
new TreeUnpickler(reader, nameAtRef, posUnpickler, commentUnpickler, Seq.empty)
2424
}
2525

2626
class PositionsSectionUnpickler extends SectionUnpickler[PositionUnpickler]("Positions") {
2727
def unpickle(reader: TastyReader, nameAtRef: NameTable) =
2828
new PositionUnpickler(reader)
2929
}
30+
31+
class CommentsSectionUnpickler extends SectionUnpickler[CommentUnpickler]("Comments") {
32+
def unpickle(reader: TastyReader, nameAtRef: NameTable): CommentUnpickler =
33+
new CommentUnpickler(reader)
34+
}
3035
}
3136

3237
/** A class for unpickling Tasty trees and symbols.
@@ -38,16 +43,17 @@ class DottyUnpickler(bytes: Array[Byte]) extends ClassfileParser.Embedded with t
3843

3944
val unpickler = new TastyUnpickler(bytes)
4045
private val posUnpicklerOpt = unpickler.unpickle(new PositionsSectionUnpickler)
41-
private val treeUnpickler = unpickler.unpickle(treeSectionUnpickler(posUnpicklerOpt)).get
46+
private val commentUnpicklerOpt = unpickler.unpickle(new CommentsSectionUnpickler)
47+
private val treeUnpickler = unpickler.unpickle(treeSectionUnpickler(posUnpicklerOpt, commentUnpicklerOpt)).get
4248

4349
/** Enter all toplevel classes and objects into their scopes
4450
* @param roots a set of SymDenotations that should be overwritten by unpickling
4551
*/
4652
def enter(roots: Set[SymDenotation])(implicit ctx: Context): Unit =
4753
treeUnpickler.enterTopLevel(roots)
4854

49-
protected def treeSectionUnpickler(posUnpicklerOpt: Option[PositionUnpickler]): TreeSectionUnpickler = {
50-
new TreeSectionUnpickler(posUnpicklerOpt)
55+
protected def treeSectionUnpickler(posUnpicklerOpt: Option[PositionUnpickler], commentUnpicklerOpt: Option[CommentUnpickler]): TreeSectionUnpickler = {
56+
new TreeSectionUnpickler(posUnpicklerOpt, commentUnpicklerOpt)
5157
}
5258

5359
protected def computeTrees(implicit ctx: Context) = treeUnpickler.unpickle()

compiler/src/dotty/tools/dotc/core/tasty/TastyFormat.scala

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -221,6 +221,10 @@ Standard Section: "Positions" Assoc*
221221
// same offset in the previously recorded node (or 0 for the first recorded node)
222222
Delta = Int // Difference between consecutive offsets,
223223
224+
Standard Section: "Comments" Comment*
225+
226+
Comment = Length Bytes // Raw comment's bytes encoded as UTF-8
227+
224228
**************************************************************************************/
225229

226230
object TastyFormat {

compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package dotc
33
package core
44
package tasty
55

6+
import Comments.CommentsContext
67
import Contexts._, Symbols._, Types._, Scopes._, SymDenotations._, Names._, NameOps._
78
import StdNames._, Denotations._, Flags._, Constants._, Annotations._
89
import NameKinds._
@@ -25,13 +26,15 @@ import scala.quoted.Types.TreeType
2526
import scala.quoted.Exprs.TreeExpr
2627

2728
/** Unpickler for typed trees
28-
* @param reader the reader from which to unpickle
29-
* @param tastyName the nametable
30-
* @param posUNpicklerOpt the unpickler for positions, if it exists
29+
* @param reader the reader from which to unpickle
30+
* @param posUnpicklerOpt the unpickler for positions, if it exists
31+
* @param commentUnpicklerOpt the unpickler for comments, if it exists
32+
* @param splices
3133
*/
3234
class TreeUnpickler(reader: TastyReader,
3335
nameAtRef: NameRef => TermName,
3436
posUnpicklerOpt: Option[PositionUnpickler],
37+
commentUnpicklerOpt: Option[CommentUnpickler],
3538
splices: Seq[Any]) {
3639
import TastyFormat._
3740
import TreeUnpickler._
@@ -797,6 +800,16 @@ class TreeUnpickler(reader: TastyReader,
797800
// Child annotations for local classes and enum values are not pickled, so
798801
// need to be re-established here.
799802
sym.registerIfChild(late = true)
803+
804+
if (ctx.mode.is(Mode.ReadComments)) {
805+
for { docCtx <- ctx.docCtx
806+
commentUnpickler <- commentUnpicklerOpt } {
807+
val comment = commentUnpickler.commentAt(start)
808+
docCtx.addDocstring(tree.symbol, comment)
809+
tree.setComment(comment)
810+
}
811+
}
812+
800813
tree
801814
}
802815

compiler/src/dotty/tools/dotc/transform/Pickler.scala

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,9 @@ class Pickler extends Phase {
6060
if (tree.pos.exists)
6161
new PositionPickler(pickler, treePkl.buf.addrOfTree).picklePositions(tree :: Nil)
6262

63+
if (ctx.settings.YkeepComments.value)
64+
new CommentPickler(pickler, treePkl.buf.addrOfTree).pickleComment(tree)
65+
6366
// other pickle sections go here.
6467
val pickled = pickler.assembleParts()
6568
unit.pickled += (cls -> pickled)
@@ -84,6 +87,7 @@ class Pickler extends Phase {
8487
.setPeriod(Period(ctx.runId + 1, FirstPhaseId))
8588
.setReporter(new ThrowingReporter(ctx.reporter))
8689
.addMode(Mode.ReadPositions)
90+
.addMode(Mode.ReadComments)
8791
.addMode(Mode.PrintShowExceptions))
8892
result
8993
}

0 commit comments

Comments
 (0)