Skip to content

Commit 834c989

Browse files
authored
Merge pull request #9650 from dotty-staging/parallel-pickling
Re-enable parallel pickling
2 parents e6c099d + 4ecb501 commit 834c989

File tree

6 files changed

+131
-91
lines changed

6 files changed

+131
-91
lines changed
Lines changed: 25 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
package dotty.tools.dotc.core.tasty
22

3-
import dotty.tools.dotc.ast.tpd
3+
import dotty.tools.dotc.ast.{tpd, untpd}
44
import dotty.tools.dotc.core.Comments.{Comment, CommentsContext, ContextDocstrings}
55
import dotty.tools.dotc.core.Contexts._
66

@@ -9,36 +9,36 @@ import TastyBuffer.{Addr, NoAddr}
99

1010
import java.nio.charset.Charset
1111

12-
class CommentPickler(pickler: TastyPickler, addrOfTree: tpd.Tree => Addr)(using Context) {
12+
class CommentPickler(pickler: TastyPickler, addrOfTree: tpd.Tree => Addr, docString: untpd.MemberDef => Option[Comment]):
1313
private val buf = new TastyBuffer(5000)
1414
pickler.newSection("Comments", buf)
1515

16-
def pickleComment(root: tpd.Tree): Unit = {
17-
assert(ctx.docCtx.isDefined, "Trying to pickle comments, but there's no `docCtx`.")
18-
new Traverser(ctx.docCtx.get).traverse(root)
19-
}
16+
def pickleComment(root: tpd.Tree): Unit = traverse(root)
2017

21-
def pickleComment(addr: Addr, comment: Option[Comment]): Unit = comment match {
22-
case Some(cmt) if addr != NoAddr =>
23-
val bytes = cmt.raw.getBytes(Charset.forName("UTF-8"))
18+
private def pickleComment(addr: Addr, comment: Comment): Unit =
19+
if addr != NoAddr then
20+
val bytes = comment.raw.getBytes(Charset.forName("UTF-8"))
2421
val length = bytes.length
2522
buf.writeAddr(addr)
2623
buf.writeNat(length)
2724
buf.writeBytes(bytes, length)
28-
buf.writeLongInt(cmt.span.coords)
29-
case other =>
30-
()
31-
}
32-
33-
private class Traverser(docCtx: ContextDocstrings) extends tpd.TreeTraverser {
34-
override def traverse(tree: tpd.Tree)(using Context): Unit =
35-
tree match {
36-
case md: tpd.MemberDef =>
37-
val comment = docCtx.docstring(md.symbol)
38-
pickleComment(addrOfTree(md), comment)
39-
traverseChildren(md)
25+
buf.writeLongInt(comment.span.coords)
26+
27+
private def traverse(x: Any): Unit = x match
28+
case x: untpd.Tree @unchecked =>
29+
x match
30+
case x: tpd.MemberDef @unchecked => // at this point all MembderDefs are t(y)p(e)d.
31+
for comment <- docString(x) do pickleComment(addrOfTree(x), comment)
4032
case _ =>
41-
traverseChildren(tree)
42-
}
43-
}
44-
}
33+
val limit = x.productArity
34+
var n = 0
35+
while n < limit do
36+
traverse(x.productElement(n))
37+
n += 1
38+
case y :: ys =>
39+
traverse(y)
40+
traverse(ys)
41+
case _ =>
42+
43+
end CommentPickler
44+

compiler/src/dotty/tools/dotc/core/tasty/PositionPickler.scala

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,14 @@ import Contexts._, Symbols._, Annotations._, Decorators._
1616
import collection.mutable
1717
import util.Spans._
1818

19-
class PositionPickler(pickler: TastyPickler, addrOfTree: PositionPickler.TreeToAddr) {
19+
class PositionPickler(
20+
pickler: TastyPickler,
21+
addrOfTree: PositionPickler.TreeToAddr,
22+
treeAnnots: untpd.MemberDef => List[tpd.Tree]) {
23+
24+
import ast.tpd._
2025
val buf: TastyBuffer = new TastyBuffer(5000)
2126
pickler.newSection("Positions", buf)
22-
import ast.tpd._
2327

2428
private val pickledIndices = new mutable.BitSet
2529

@@ -28,7 +32,7 @@ class PositionPickler(pickler: TastyPickler, addrOfTree: PositionPickler.TreeToA
2832
(addrDelta << 3) | (toInt(hasStartDelta) << 2) | (toInt(hasEndDelta) << 1) | toInt(hasPoint)
2933
}
3034

31-
def picklePositions(roots: List[Tree])(using Context): Unit = {
35+
def picklePositions(roots: List[Tree], warnings: mutable.ListBuffer[String]): Unit = {
3236
var lastIndex = 0
3337
var lastSpan = Span(0, 0)
3438
def pickleDeltas(index: Int, span: Span) = {
@@ -55,7 +59,7 @@ class PositionPickler(pickler: TastyPickler, addrOfTree: PositionPickler.TreeToA
5559
val cwd = java.nio.file.Paths.get("").toAbsolutePath().normalize()
5660
try cwd.relativize(path)
5761
catch case _: IllegalArgumentException =>
58-
report.warning("Could not relativize path for pickling: " + originalPath)
62+
warnings += "Could not relativize path for pickling: " + originalPath
5963
originalPath
6064
else
6165
originalPath
@@ -100,10 +104,9 @@ class PositionPickler(pickler: TastyPickler, addrOfTree: PositionPickler.TreeToA
100104
pickleDeltas(addr.index, x.span)
101105
}
102106
}
103-
x match {
104-
case x: untpd.MemberDef @unchecked => traverse(x.symbol.annotations, x.source)
107+
x match
108+
case x: untpd.MemberDef => traverse(treeAnnots(x), x.source)
105109
case _ =>
106-
}
107110
val limit = x.productArity
108111
var n = 0
109112
while (n < limit) {
@@ -113,8 +116,6 @@ class PositionPickler(pickler: TastyPickler, addrOfTree: PositionPickler.TreeToA
113116
case y :: ys =>
114117
traverse(y, current)
115118
traverse(ys, current)
116-
case x: Annotation =>
117-
traverse(x.tree, current)
118119
case _ =>
119120
}
120121
for (root <- roots)

compiler/src/dotty/tools/dotc/core/tasty/TreeBuffer.scala

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -177,12 +177,11 @@ class TreeBuffer extends TastyBuffer(50000) {
177177
//println(s"offsets: ${offsets.take(numOffsets).deep}")
178178
//println(s"deltas: ${delta.take(numOffsets).deep}")
179179
var saved = 0
180-
while ({
180+
while
181181
saved = adjustDeltas()
182182
pickling.println(s"adjusting deltas, saved = $saved")
183183
saved > 0 && length / saved < 100
184-
})
185-
()
184+
do ()
186185
adjustOffsets()
187186
adjustTreeAddrs()
188187
val wasted = compress()

compiler/src/dotty/tools/dotc/core/tasty/TreePickler.scala

Lines changed: 44 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import ast.Trees._
1010
import ast.{untpd, tpd}
1111
import Contexts._, Symbols._, Types._, Names._, Constants._, Decorators._, Annotations._, Flags._
1212
import Denotations.MultiDenotation
13+
import Comments.{Comment, CommentsContext}
1314
import typer.Inliner
1415
import NameKinds._
1516
import StdNames.nme
@@ -18,6 +19,7 @@ import printing.Printer
1819
import printing.Texts._
1920
import util.SourceFile
2021
import annotation.constructorOnly
22+
import collection.mutable
2123

2224
object TreePickler {
2325

@@ -44,6 +46,25 @@ class TreePickler(pickler: TastyPickler) {
4446
private val forwardSymRefs = Symbols.newMutableSymbolMap[List[Addr]]
4547
private val pickledTypes = new java.util.IdentityHashMap[Type, Any] // Value type is really Addr, but that's not compatible with null
4648

49+
/** A list of annotation trees for every member definition, so that later
50+
* parallel position pickling does not need to access and force symbols.
51+
*/
52+
private val annotTrees = util.HashTable[untpd.MemberDef, mutable.ListBuffer[Tree]]()
53+
54+
/** A map from member definitions to their doc comments, so that later
55+
* parallel comment pickling does not need to access symbols of trees (which
56+
* would involve accessing symbols of named types and possibly changing phases
57+
* in doing so).
58+
*/
59+
private val docStrings = util.HashTable[untpd.MemberDef, Comment]()
60+
61+
def treeAnnots(tree: untpd.MemberDef): List[Tree] =
62+
val ts = annotTrees.lookup(tree)
63+
if ts == null then Nil else ts.toList
64+
65+
def docString(tree: untpd.MemberDef): Option[Comment] =
66+
Option(docStrings.lookup(tree))
67+
4768
private def withLength(op: => Unit) = {
4869
val lengthAddr = reserveRef(relative = true)
4970
op
@@ -309,7 +330,8 @@ class TreePickler(pickler: TastyPickler) {
309330
if (!tree.isEmpty) pickleTree(tree)
310331
}
311332

312-
def pickleDef(tag: Int, sym: Symbol, tpt: Tree, rhs: Tree = EmptyTree, pickleParams: => Unit = ())(using Context): Unit = {
333+
def pickleDef(tag: Int, mdef: MemberDef, tpt: Tree, rhs: Tree = EmptyTree, pickleParams: => Unit = ())(using Context): Unit = {
334+
val sym = mdef.symbol
313335
assert(symRefs(sym) == NoAddr, sym)
314336
registerDef(sym)
315337
writeByte(tag)
@@ -321,16 +343,21 @@ class TreePickler(pickler: TastyPickler) {
321343
case _ if tpt.isType => pickleTpt(tpt)
322344
}
323345
pickleTreeUnlessEmpty(rhs)
324-
pickleModifiers(sym)
346+
pickleModifiers(sym, mdef)
325347
}
348+
for
349+
docCtx <- ctx.docCtx
350+
comment <- docCtx.docstring(sym)
351+
do
352+
docStrings.enter(mdef, comment)
326353
}
327354

328355
def pickleParam(tree: Tree)(using Context): Unit = {
329356
registerTreeAddr(tree)
330357
tree match {
331-
case tree: ValDef => pickleDef(PARAM, tree.symbol, tree.tpt)
332-
case tree: DefDef => pickleDef(PARAM, tree.symbol, tree.tpt, tree.rhs)
333-
case tree: TypeDef => pickleDef(TYPEPARAM, tree.symbol, tree.rhs)
358+
case tree: ValDef => pickleDef(PARAM, tree, tree.tpt)
359+
case tree: DefDef => pickleDef(PARAM, tree, tree.tpt, tree.rhs)
360+
case tree: TypeDef => pickleDef(TYPEPARAM, tree, tree.rhs)
334361
}
335362
}
336363

@@ -520,7 +547,7 @@ class TreePickler(pickler: TastyPickler) {
520547
patterns.foreach(pickleTree)
521548
}
522549
case tree: ValDef =>
523-
pickleDef(VALDEF, tree.symbol, tree.tpt, tree.rhs)
550+
pickleDef(VALDEF, tree, tree.tpt, tree.rhs)
524551
case tree: DefDef =>
525552
def pickleParamss(paramss: List[List[ValDef]]): Unit = paramss match
526553
case Nil =>
@@ -531,9 +558,9 @@ class TreePickler(pickler: TastyPickler) {
531558
def pickleAllParams =
532559
pickleParams(tree.tparams)
533560
pickleParamss(tree.vparamss)
534-
pickleDef(DEFDEF, tree.symbol, tree.tpt, tree.rhs, pickleAllParams)
561+
pickleDef(DEFDEF, tree, tree.tpt, tree.rhs, pickleAllParams)
535562
case tree: TypeDef =>
536-
pickleDef(TYPEDEF, tree.symbol, tree.rhs)
563+
pickleDef(TYPEDEF, tree, tree.rhs)
537564
case tree: Template =>
538565
registerDef(tree.symbol)
539566
writeByte(TEMPLATE)
@@ -648,7 +675,7 @@ class TreePickler(pickler: TastyPickler) {
648675
pickleName(id.name)
649676
}
650677

651-
def pickleModifiers(sym: Symbol)(using Context): Unit = {
678+
def pickleModifiers(sym: Symbol, mdef: MemberDef)(using Context): Unit = {
652679
import Flags._
653680
var flags = sym.flags
654681
val privateWithin = sym.privateWithin
@@ -660,7 +687,7 @@ class TreePickler(pickler: TastyPickler) {
660687
if (flags.is(ParamAccessor) && sym.isTerm && !sym.isSetter)
661688
flags = flags &~ ParamAccessor // we only generate a tag for parameter setters
662689
pickleFlags(flags, sym.isTerm)
663-
sym.annotations.foreach(pickleAnnotation(sym, _))
690+
val annots = sym.annotations.foreach(pickleAnnotation(sym, mdef, _))
664691
}
665692

666693
def pickleFlags(flags: FlagSet, isTerm: Boolean)(using Context): Unit = {
@@ -723,12 +750,15 @@ class TreePickler(pickler: TastyPickler) {
723750
ann.symbol == defn.BodyAnnot // inline bodies are reconstituted automatically when unpickling
724751
}
725752

726-
def pickleAnnotation(owner: Symbol, ann: Annotation)(using Context): Unit = {
727-
if (!isUnpicklable(owner, ann)) {
753+
def pickleAnnotation(owner: Symbol, mdef: MemberDef, ann: Annotation)(using Context): Unit =
754+
if !isUnpicklable(owner, ann) then
728755
writeByte(ANNOTATION)
729756
withLength { pickleType(ann.symbol.typeRef); pickleTree(ann.tree) }
730-
}
731-
}
757+
var treeBuf = annotTrees.lookup(mdef)
758+
if treeBuf == null then
759+
treeBuf = new mutable.ListBuffer[Tree]
760+
annotTrees.enter(mdef, treeBuf)
761+
treeBuf += ann.tree
732762

733763
// ---- main entry points ---------------------------------------
734764

compiler/src/dotty/tools/dotc/quoted/PickledQuotes.scala

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,15 @@ import dotty.tools.dotc.core.tasty.TreePickler.Hole
1515
import dotty.tools.dotc.core.tasty.{ PositionPickler, TastyPickler, TastyPrinter }
1616
import dotty.tools.dotc.core.tasty.DottyUnpickler
1717
import dotty.tools.dotc.core.tasty.TreeUnpickler.UnpickleMode
18+
import dotty.tools.dotc.report
1819

1920
import dotty.tools.tasty.TastyString
2021

2122
import scala.reflect.ClassTag
2223

2324
import scala.internal.quoted.Unpickler._
2425
import scala.quoted.QuoteContext
26+
import scala.collection.mutable
2527

2628
object PickledQuotes {
2729
import tpd._
@@ -167,8 +169,11 @@ object PickledQuotes {
167169
val treePkl = pickler.treePkl
168170
treePkl.pickle(tree :: Nil)
169171
treePkl.compactify()
170-
if (tree.span.exists)
171-
new PositionPickler(pickler, treePkl.buf.addrOfTree).picklePositions(tree :: Nil)
172+
if tree.span.exists then
173+
val positionWarnings = new mutable.ListBuffer[String]()
174+
new PositionPickler(pickler, treePkl.buf.addrOfTree, treePkl.treeAnnots)
175+
.picklePositions(tree :: Nil, positionWarnings)
176+
positionWarnings.foreach(report.warning(_))
172177

173178
val pickled = pickler.assembleParts()
174179
quotePickling.println(s"**** pickled quote\n${new TastyPrinter(pickled).printContents()}")

0 commit comments

Comments
 (0)