Skip to content

Commit 6a8eaff

Browse files
nicolasstuckinicolasstucki
committed
Store source file in TASTY attributes
1 parent cb48537 commit 6a8eaff

16 files changed

+106
-36
lines changed

‎compiler/src/dotty/tools/dotc/core/CompilationUnitInfo.scala

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,8 @@ import dotty.tools.tasty.TastyVersion
1414
class CompilationUnitInfo(
1515
val associatedFile: AbstractFile,
1616
val tastyVersion: Option[TastyVersion],
17-
val explicitNulls: Boolean
17+
val explicitNulls: Boolean,
18+
val sourceFile: Option[String],
1819
) {
1920

2021
override def toString(): String =
@@ -28,4 +29,5 @@ object CompilationUnitInfo:
2829
assocFile,
2930
tastyVersion = None,
3031
explicitNulls = explicitNulls,
32+
sourceFile = if assocFile.path.endsWith(".scala") then Some(assocFile.path) else None,
3133
)

‎compiler/src/dotty/tools/dotc/core/SymbolLoaders.scala

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -434,6 +434,7 @@ class TastyLoader(val tastyFile: AbstractFile) extends SymbolLoader {
434434
tastyFile,
435435
tastyVersion = Some(tastyVersion),
436436
explicitNulls = attributes.explicitNulls,
437+
sourceFile = attributes.sourceFile,
437438
)
438439

439440
def description(using Context): String = "TASTy file " + tastyFile.toString

‎compiler/src/dotty/tools/dotc/core/Symbols.scala

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -293,6 +293,22 @@ object Symbols {
293293
val compUnitInfo = compilationUnitInfo
294294
compUnitInfo != null && compUnitInfo.explicitNulls
295295

296+
/** The version of TASTy from which the symbol was loaded, None if not applicable. */
297+
def sourceFile(using Context): Option[String] =
298+
val compUnitInfo = compilationUnitInfo
299+
if compUnitInfo == null then
300+
// Try to get it form the 3.0-3.3 @SourceFile annotation
301+
atPhaseNoLater(flattenPhase) {
302+
denot.topLevelClass.unforcedAnnotation(defn.SourceFileAnnot) match
303+
case Some(sourceAnnot) => sourceAnnot.argumentConstant(0) match
304+
case Some(Constant(path: String)) => Some(path)
305+
case none => None
306+
case none => None
307+
}
308+
else
309+
// Loaded from TASTy attributes for 3.4+
310+
compUnitInfo.sourceFile
311+
296312
/** The class file from which this class was generated, null if not applicable. */
297313
final def binaryFile(using Context): AbstractFile | Null = {
298314
val file = associatedFile
@@ -493,13 +509,7 @@ object Symbols {
493509
else
494510
mySource = defn.patchSource(this)
495511
if !mySource.exists then
496-
mySource = atPhaseNoLater(flattenPhase) {
497-
denot.topLevelClass.unforcedAnnotation(defn.SourceFileAnnot) match
498-
case Some(sourceAnnot) => sourceAnnot.argumentConstant(0) match
499-
case Some(Constant(path: String)) => ctx.getSource(path)
500-
case none => NoSource
501-
case none => NoSource
502-
}
512+
mySource = sourceFile.map(ctx.getSource).getOrElse(NoSource)
503513
mySource
504514
}
505515

‎compiler/src/dotty/tools/dotc/core/tasty/AttributePickler.scala

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,14 @@ object AttributePickler:
1212
pickler: TastyPickler,
1313
buf: TastyBuffer
1414
): Unit =
15-
if attributes.scala2StandardLibrary || attributes.explicitNulls then // or any other attribute is set
16-
pickler.newSection(AttributesSection, buf)
15+
pickler.newSection(AttributesSection, buf)
1716

18-
for tag <- attributes.booleanTags do
19-
buf.writeByte(tag)
17+
for tag <- attributes.booleanTags do
18+
buf.writeByte(tag)
19+
20+
for (tag, value) <- attributes.stringTagValues do
21+
buf.writeByte(tag)
22+
buf.writeUTF8(value)
2023

2124
end pickleAttributes
2225

‎compiler/src/dotty/tools/dotc/core/tasty/AttributeUnpickler.scala

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,16 @@ class AttributeUnpickler(reader: TastyReader):
1010

1111
lazy val attributes: Attributes = {
1212
val booleanTags = List.newBuilder[Int]
13+
val stringTagValue = List.newBuilder[(Int, String)]
1314

1415
while !isAtEnd do
15-
booleanTags += readByte()
16+
val tag = readByte()
17+
if tag < TastyFormat.firstStringAttrTag then
18+
booleanTags += tag
19+
else
20+
stringTagValue += ((tag, readUTF8()))
1621

17-
new Attributes(booleanTags.result())
22+
new Attributes(booleanTags.result(), stringTagValue.result())
1823
}
1924

2025
end AttributeUnpickler

‎compiler/src/dotty/tools/dotc/core/tasty/Attributes.scala

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,20 +4,29 @@ import dotty.tools.tasty.TastyFormat
44

55
class Attributes(
66
val booleanTags: List[Int],
7+
val stringTagValues: List[(Int, String)],
78
) {
89
def scala2StandardLibrary: Boolean =
910
booleanTags.contains(TastyFormat.SCALA2STANDARDLIBRARYattr)
1011
def explicitNulls: Boolean =
1112
booleanTags.contains(TastyFormat.EXPLICITNULLSattr)
13+
def sourceFile: Option[String] =
14+
stringTagValues.find(_._1 == TastyFormat.SOURCEFILEattr).map(_._2)
1215
}
1316

1417
object Attributes:
1518
def apply(
19+
sourceFile: String,
1620
scala2StandardLibrary: Boolean,
1721
explicitNulls: Boolean,
1822
): Attributes =
1923
val booleanTags = List.newBuilder[Int]
2024
if scala2StandardLibrary then booleanTags += TastyFormat.SCALA2STANDARDLIBRARYattr
2125
if explicitNulls then booleanTags += TastyFormat.EXPLICITNULLSattr
22-
new Attributes(booleanTags.result())
26+
27+
val stringTagValues = List(
28+
(TastyFormat.SOURCEFILEattr, sourceFile)
29+
)
30+
31+
new Attributes(booleanTags.result(), stringTagValues)
2332
end apply

‎compiler/src/dotty/tools/dotc/core/tasty/CommentPickler.scala

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,11 +22,8 @@ object CommentPickler:
2222

2323
def pickleComment(addr: Addr, comment: Comment): Unit =
2424
if addr != NoAddr then
25-
val bytes = comment.raw.getBytes(StandardCharsets.UTF_8).nn
26-
val length = bytes.length
2725
buf.writeAddr(addr)
28-
buf.writeNat(length)
29-
buf.writeBytes(bytes, length)
26+
buf.writeUTF8(comment.raw)
3027
buf.writeLongInt(comment.span.coords)
3128

3229
def traverse(x: Any): Unit = x match

‎compiler/src/dotty/tools/dotc/core/tasty/CommentUnpickler.scala

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,9 @@ class CommentUnpickler(reader: TastyReader) {
1919
val comments = new HashMap[Addr, Comment]
2020
while (!isAtEnd) {
2121
val addr = readAddr()
22-
val length = readNat()
23-
if (length > 0) {
24-
val bytes = readBytes(length)
22+
val rawComment = readUTF8()
23+
if (rawComment != "") {
2524
val position = new Span(readLongInt())
26-
val rawComment = new String(bytes, StandardCharsets.UTF_8)
2725
comments(addr) = Comment(position, rawComment)
2826
}
2927
}

‎compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ class DottyUnpickler(bytes: Array[Byte], mode: UnpickleMode = UnpickleMode.TopLe
5757
private val treeUnpickler = unpickler.unpickle(treeSectionUnpickler(posUnpicklerOpt, commentUnpicklerOpt, attributeUnpicklerOpt)).get
5858

5959
def tastyAttributes: Attributes =
60-
attributeUnpicklerOpt.map(_.attributes).getOrElse(new Attributes(booleanTags = Nil))
60+
attributeUnpicklerOpt.map(_.attributes).getOrElse(new Attributes(booleanTags = Nil, stringTagValues = Nil))
6161

6262
/** Enter all toplevel classes and objects into their scopes
6363
* @param roots a set of SymDenotations that should be overwritten by unpickling

‎compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import util.Spans.offsetToInt
1212
import dotty.tools.tasty.TastyFormat.{ASTsSection, PositionsSection, CommentsSection, AttributesSection}
1313
import java.nio.file.{Files, Paths}
1414
import dotty.tools.io.{JarArchive, Path}
15+
import java.nio.charset.StandardCharsets
1516

1617
object TastyPrinter:
1718

@@ -237,6 +238,9 @@ class TastyPrinter(bytes: Array[Byte]) {
237238

238239
for tag <- attributes.booleanTags do
239240
sb.append(" ").append(attributeTagToString(tag)).append("\n")
241+
for (tag, value) <- attributes.stringTagValues do
242+
sb.append(" ").append(attributeTagToString(tag))
243+
.append(" ").append(value).append("\n")
240244

241245
sb.result
242246
}

‎compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,10 @@ class TreeUnpickler(reader: TastyReader,
107107
private val explicitNulls =
108108
attributeUnpicklerOpt.exists(_.attributes.explicitNulls)
109109

110+
/** Source file in the TASTy attributes */
111+
private val sourceFile: Option[String] =
112+
attributeUnpicklerOpt.flatMap(_.attributes.sourceFile)
113+
110114
private def registerSym(addr: Addr, sym: Symbol) =
111115
symAtAddr(addr) = sym
112116

‎compiler/src/dotty/tools/dotc/transform/Pickler.scala

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,11 @@ class Pickler extends Phase {
108108
pickler, treePkl.buf.addrOfTree, treePkl.docString, tree,
109109
scratch.commentBuffer)
110110

111+
val sourceRelativePath =
112+
val reference = ctx.settings.sourceroot.value
113+
util.SourceFile.relativePath(unit.source, reference)
111114
val attributes = Attributes(
115+
sourceFile = sourceRelativePath,
112116
scala2StandardLibrary = ctx.settings.YcompileScala2Library.value,
113117
explicitNulls = ctx.settings.YexplicitNulls.value,
114118
)

‎compiler/src/dotty/tools/dotc/transform/PostTyper.scala

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -419,6 +419,7 @@ class PostTyper extends MacroTransform with InfoTransformer { thisPhase =>
419419
em"The type of a class parent cannot refer to constructor parameters, but ${parent.tpe} refers to ${illegalRefs.map(_.name.show).mkString(",")}", parent.srcPos)
420420
// Add SourceFile annotation to top-level classes
421421
if sym.owner.is(Package) then
422+
// TODO do not add SourceFile annotation
422423
if ctx.compilationUnit.source.exists && sym != defn.SourceFileAnnot then
423424
val reference = ctx.settings.sourceroot.value
424425
val relativePath = util.SourceFile.relativePath(ctx.compilationUnit.source, reference)

‎tasty/src/dotty/tools/tasty/TastyBuffer.scala

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package dotty.tools.tasty
22

33
import util.Util.dble
4+
import java.nio.charset.StandardCharsets
45

56
object TastyBuffer {
67

@@ -115,6 +116,14 @@ class TastyBuffer(initialSize: Int) {
115116
writeBytes(bytes, 8)
116117
}
117118

119+
/** Write a UTF8 string encoded as `Length UTF8-CodePoint*` */
120+
def writeUTF8(x: String): Unit = {
121+
val bytes = x.getBytes(StandardCharsets.UTF_8)
122+
val length = bytes.length
123+
writeNat(length)
124+
writeBytes(bytes, length)
125+
}
126+
118127
// -- Address handling --------------------------------------------
119128

120129
/** Write natural number `x` right-adjusted in a field of `width` bytes

‎tasty/src/dotty/tools/tasty/TastyFormat.scala

Lines changed: 27 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -234,11 +234,11 @@ Note: The signature of a SELECTin or TERMREFin node is the signature of the sele
234234
235235
Note: Tree tags are grouped into 5 categories that determine what follows, and thus allow to compute the size of the tagged tree in a generic way.
236236
```none
237-
Category 1 (tags 1-59) : tag
238-
Category 2 (tags 60-89) : tag Nat
239-
Category 3 (tags 90-109) : tag AST
240-
Category 4 (tags 110-127): tag Nat AST
241-
Category 5 (tags 128-255): tag Length <payload>
237+
Tree Category 1 (tags 1-59) : tag
238+
Tree Category 2 (tags 60-89) : tag Nat
239+
Tree Category 3 (tags 90-109) : tag AST
240+
Tree Category 4 (tags 110-127): tag Nat AST
241+
Tree Category 5 (tags 128-255): tag Length <payload>
242242
```
243243
244244
Standard-Section: "Positions" LinesSizes Assoc*
@@ -272,6 +272,13 @@ Standard Section: "Attributes" Attribute*
272272
```none
273273
Attribute = SCALA2STANDARDLIBRARYattr
274274
EXPLICITNULLSattr
275+
SOURCEFILEattr UTF8
276+
```
277+
278+
Note: Attribute tags are grouped into 2 categories that determine what follows, and thus allow to compute the size of the tagged tree in a generic way.
279+
```none
280+
Attribute Category 1 (tags 1-127) : tag
281+
Attribute Category 2 (tags 128-255): tag UTF8
275282
```
276283
277284
**************************************************************************************/
@@ -436,9 +443,8 @@ object TastyFormat {
436443
final val SOURCE = 4
437444

438445
// AST tags
439-
// Cat. 1: tag
446+
// Tree Cat. 1: tag
440447

441-
final val firstSimpleTreeTag = UNITconst
442448
// final val ??? = 1
443449
final val UNITconst = 2
444450
final val FALSEconst = 3
@@ -486,7 +492,7 @@ object TastyFormat {
486492
final val EMPTYCLAUSE = 45
487493
final val SPLITCLAUSE = 46
488494

489-
// Cat. 2: tag Nat
495+
// Tree Cat. 2: tag Nat
490496

491497
final val SHAREDterm = 60
492498
final val SHAREDtype = 61
@@ -506,7 +512,7 @@ object TastyFormat {
506512
final val IMPORTED = 75
507513
final val RENAMED = 76
508514

509-
// Cat. 3: tag AST
515+
// Tree Cat. 3: tag AST
510516

511517
final val THIS = 90
512518
final val QUALTHIS = 91
@@ -524,7 +530,7 @@ object TastyFormat {
524530
final val EXPLICITtpt = 103
525531

526532

527-
// Cat. 4: tag Nat AST
533+
// Tree Cat. 4: tag Nat AST
528534

529535
final val IDENT = 110
530536
final val IDENTtpt = 111
@@ -537,7 +543,7 @@ object TastyFormat {
537543
final val SELFDEF = 118
538544
final val NAMEDARG = 119
539545

540-
// Cat. 5: tag Length ...
546+
// Tree Cat. 5: tag Length ...
541547

542548
final val PACKAGE = 128
543549
final val VALDEF = 129
@@ -600,17 +606,25 @@ object TastyFormat {
600606

601607
final val HOLE = 255
602608

609+
final val firstSimpleTreeTag = UNITconst
603610
final val firstNatTreeTag = SHAREDterm
604611
final val firstASTTreeTag = THIS
605612
final val firstNatASTTreeTag = IDENT
606613
final val firstLengthTreeTag = PACKAGE
607614

608615

609-
// Attributes tags
616+
// Attributes tags
610617

618+
// Attr Cat. 1: tag
611619
final val SCALA2STANDARDLIBRARYattr = 1
612620
final val EXPLICITNULLSattr = 2
613621

622+
// Attr Cat. 2: tag UTF8
623+
final val SOURCEFILEattr = 128
624+
625+
final val firstBooleanAttrTag = SCALA2STANDARDLIBRARYattr
626+
final val firstStringAttrTag = SOURCEFILEattr
627+
614628
/** Useful for debugging */
615629
def isLegalTag(tag: Int): Boolean =
616630
firstSimpleTreeTag <= tag && tag <= SPLITCLAUSE ||
@@ -829,6 +843,7 @@ object TastyFormat {
829843
def attributeTagToString(tag: Int): String = tag match {
830844
case SCALA2STANDARDLIBRARYattr => "SCALA2STANDARDLIBRARYattr"
831845
case EXPLICITNULLSattr => "EXPLICITNULLSattr"
846+
case SOURCEFILEattr => "SOURCEFILEattr"
832847
}
833848

834849
/** @return If non-negative, the number of leading references (represented as nats) of a length/trees entry.

‎tasty/src/dotty/tools/tasty/TastyReader.scala

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package dotty.tools.tasty
33
import collection.mutable
44

55
import TastyBuffer._
6+
import java.nio.charset.StandardCharsets
67

78
/** A byte array buffer that can be filled with bytes or natural numbers in TASTY format,
89
* and that supports reading and patching addresses represented as natural numbers.
@@ -104,6 +105,13 @@ class TastyReader(val bytes: Array[Byte], start: Int, end: Int, val base: Int =
104105
x
105106
}
106107

108+
/** Read a UTF8 string encoded as `Length UTF8-CodePoint*` */
109+
def readUTF8(): String = {
110+
val length = readNat()
111+
if (length == 0) ""
112+
else new String(readBytes(length), StandardCharsets.UTF_8)
113+
}
114+
107115
/** Read a natural number and return as a NameRef */
108116
def readNameRef(): NameRef = NameRef(readNat())
109117

0 commit comments

Comments
 (0)