Skip to content

Commit d46d7a3

Browse files
nicolasstuckinicolasstucki
authored
Merge pull request #2910 from dotty-staging/link-from-tasty
Link from tasty
2 parents 7141ded + 4f0b070 commit d46d7a3

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

49 files changed

+488
-107
lines changed

‎compiler/src/dotty/tools/dotc/CompilationUnit.scala

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,9 @@ import dotty.tools.dotc.core.Types.Type
55
import dotty.tools.dotc.core.tasty.{TastyUnpickler, TastyBuffer, TastyPickler}
66
import util.SourceFile
77
import ast.{tpd, untpd}
8+
import dotty.tools.dotc.ast.tpd.{ Tree, TreeTraverser }
9+
import dotty.tools.dotc.core.Contexts.Context
10+
import dotty.tools.dotc.core.SymDenotations.ClassDenotation
811
import dotty.tools.dotc.core.Symbols._
912

1013
class CompilationUnit(val source: SourceFile) {
@@ -20,3 +23,20 @@ class CompilationUnit(val source: SourceFile) {
2023
/** Pickled TASTY binaries, indexed by class. */
2124
var pickled: Map[ClassSymbol, Array[Byte]] = Map()
2225
}
26+
27+
object CompilationUnit {
28+
29+
/** Make a compilation unit for top class `clsd` with the contends of the `unpickled` */
30+
def mkCompilationUnit(clsd: ClassDenotation, unpickled: Tree, forceTrees: Boolean)(implicit ctx: Context): CompilationUnit = {
31+
val unit1 = new CompilationUnit(new SourceFile(clsd.symbol.sourceFile, Seq()))
32+
unit1.tpdTree = unpickled
33+
if (forceTrees)
34+
force.traverse(unit1.tpdTree)
35+
unit1
36+
}
37+
38+
/** Force the tree to be loaded */
39+
private object force extends TreeTraverser {
40+
def traverse(tree: Tree)(implicit ctx: Context): Unit = traverseChildren(tree)
41+
}
42+
}

‎compiler/src/dotty/tools/dotc/Compiler.scala

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ class Compiler {
4747
List(new PostTyper), // Additional checks and cleanups after type checking
4848
List(new sbt.ExtractAPI), // Sends a representation of the API of classes to sbt via callbacks
4949
List(new Pickler), // Generate TASTY info
50+
List(new LinkAll), // Reload compilation units from TASTY for library code (if needed)
5051
List(new FirstTransform, // Some transformations to put trees into a canonical form
5152
new CheckReentrant, // Internal use only: Check that compiled program has no data races involving global vars
5253
new ElimJavaPackages), // Eliminate syntactic references to Java packages

‎compiler/src/dotty/tools/dotc/FromTasty.scala

Lines changed: 2 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -60,10 +60,6 @@ object FromTasty extends Driver {
6060
override def toString = s"class file $className"
6161
}
6262

63-
object force extends TreeTraverser {
64-
def traverse(tree: Tree)(implicit ctx: Context): Unit = traverseChildren(tree)
65-
}
66-
6763
class ReadTastyTreesFromClasses extends FrontEnd {
6864

6965
override def isTyper = false
@@ -85,13 +81,8 @@ object FromTasty extends Driver {
8581
case info: ClassfileLoader =>
8682
info.load(clsd)
8783
val unpickled = clsd.symbol.asClass.tree
88-
if (unpickled != null) {
89-
val unit1 = new CompilationUnit(new SourceFile(clsd.symbol.sourceFile, Seq()))
90-
unit1.tpdTree = unpickled
91-
force.traverse(unit1.tpdTree)
92-
unit1
93-
} else
94-
cannotUnpickle(s"its class file ${info.classfile} does not have a TASTY attribute")
84+
if (unpickled != null) CompilationUnit.mkCompilationUnit(clsd, unpickled, forceTrees = true)
85+
else cannotUnpickle(s"its class file ${info.classfile} does not have a TASTY attribute")
9586
case info =>
9687
cannotUnpickle(s"its info of type ${info.getClass} is not a ClassfileLoader")
9788
}

‎compiler/src/dotty/tools/dotc/config/ScalaSettings.scala

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,7 @@ class ScalaSettings extends Settings.SettingGroup {
113113
val YoptPhases = PhasesSetting("-Yopt-phases", "Restrict the optimisation phases to execute under -optimise.")
114114
val YoptFuel = IntSetting("-Yopt-fuel", "Maximum number of optimisations performed under -optimise.", -1)
115115
val optimise = BooleanSetting("-optimise", "Generates faster bytecode by applying local optimisations to the .program") withAbbreviation "-optimize"
116+
val XlinkOptimise = BooleanSetting("-Xlink-optimise", "Recompile library code with the application.").withAbbreviation("-Xlink-optimize")
116117

117118
/** Dottydoc specific settings */
118119
val siteRoot = StringSetting(

‎compiler/src/dotty/tools/dotc/core/SymbolLoaders.scala

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -323,11 +323,15 @@ class ClassfileLoader(val classfile: AbstractFile) extends SymbolLoader {
323323

324324
def load(root: SymDenotation)(implicit ctx: Context): Unit = {
325325
val (classRoot, moduleRoot) = rootDenots(root.asClass)
326-
(new ClassfileParser(classfile, classRoot, moduleRoot)(ctx)).run() match {
327-
case Some(unpickler: tasty.DottyUnpickler) if ctx.settings.YretainTrees.value =>
328-
classRoot.symbol.asClass.unpickler = unpickler
329-
moduleRoot.symbol.asClass.unpickler = unpickler
330-
case _ =>
326+
val classfileParser = new ClassfileParser(classfile, classRoot, moduleRoot)(ctx)
327+
val result = classfileParser.run()
328+
if (ctx.settings.YretainTrees.value || ctx.settings.XlinkOptimise.value) {
329+
result match {
330+
case Some(unpickler: tasty.DottyUnpickler) =>
331+
classRoot.symbol.asClass.unpickler = unpickler
332+
moduleRoot.symbol.asClass.unpickler = unpickler
333+
case _ =>
334+
}
331335
}
332336
}
333337
}

‎compiler/src/dotty/tools/dotc/core/Symbols.scala

Lines changed: 15 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -555,35 +555,31 @@ object Symbols {
555555

556556
type ThisName = TypeName
557557

558-
/** If this is a top-level class, and if `-Yretain-trees` is set, return the TypeDef tree
559-
* for this class, otherwise EmptyTree. This will force the info of the class.
560-
*/
561-
def tree(implicit ctx: Context): tpd.Tree /* tpd.TypeDef | tpd.EmptyTree */ = {
558+
/** If this is either:
559+
* - a top-level class and `-Yretain-trees` is set
560+
* - a top-level class loaded from TASTY and `-Xlink-optimise` is set
561+
* then return the TypeDef tree (possibly wrapped inside PackageDefs) for this class, otherwise EmptyTree.
562+
* This will force the info of the class.
563+
*/
564+
def tree(implicit ctx: Context): tpd.Tree /* tpd.PackageDef | tpd.TypeDef | tpd.EmptyTree */ = {
562565
denot.info
563566
// TODO: Consider storing this tree like we store lazy trees for inline functions
564567
if (unpickler != null && !denot.isAbsent) {
565568
assert(myTree.isEmpty)
566-
567-
import ast.Trees._
568-
569-
def findTree(tree: tpd.Tree): Option[tpd.TypeDef] = tree match {
570-
case PackageDef(_, stats) =>
571-
stats.flatMap(findTree).headOption
572-
case tree: tpd.TypeDef if tree.symbol == this =>
573-
Some(tree)
574-
case _ =>
575-
None
576-
}
577-
val List(unpickledTree) = unpickler.body(ctx.addMode(Mode.ReadPositions))
569+
val body = unpickler.body(ctx.addMode(Mode.ReadPositions))
570+
myTree = body.headOption.getOrElse(tpd.EmptyTree)
578571
unpickler = null
579-
580-
myTree = findTree(unpickledTree).get
581572
}
582573
myTree
583574
}
584-
private[dotc] var myTree: tpd.Tree = tpd.EmptyTree
575+
private[this] var myTree: tpd.Tree /* tpd.PackageDef | tpd.TypeDef | tpd.EmptyTree */ = tpd.EmptyTree
585576
private[dotc] var unpickler: tasty.DottyUnpickler = _
586577

578+
private[dotc] def registerTree(tree: tpd.TypeDef)(implicit ctx: Context): Unit = {
579+
if (ctx.settings.YretainTrees.value)
580+
myTree = tree
581+
}
582+
587583
/** The source or class file from which this class was generated, null if not applicable. */
588584
override def associatedFile(implicit ctx: Context): AbstractFile =
589585
if (assocFile != null || (this.owner is PackageClass) || this.isEffectiveRoot) assocFile

‎compiler/src/dotty/tools/dotc/interactive/SourceTree.scala

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,13 +43,15 @@ object SourceTree {
4343
sym.sourceFile == null) // FIXME: We cannot deal with external projects yet
4444
None
4545
else {
46-
sym.tree match {
47-
case tree: tpd.TypeDef =>
46+
import ast.Trees._
47+
def sourceTreeOfClass(tree: tpd.Tree): Option[SourceTree] = tree match {
48+
case PackageDef(_, stats) => stats.flatMap(sourceTreeOfClass).headOption
49+
case tree: tpd.TypeDef if tree.symbol == sym =>
4850
val sourceFile = new SourceFile(sym.sourceFile, Codec.UTF8)
4951
Some(SourceTree(tree, sourceFile))
50-
case _ =>
51-
None
52+
case _ => None
5253
}
54+
sourceTreeOfClass(sym.tree)
5355
}
5456
}
5557
}

‎compiler/src/dotty/tools/dotc/transform/FirstTransform.scala

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,10 @@ class FirstTransform extends MiniPhaseTransform with InfoTransformer with Annota
161161
cpy.Template(impl)(self = EmptyValDef)
162162
}
163163

164+
/** Eliminate empty package definitions that may have been stored in the TASTY trees */
165+
override def transformPackageDef(tree: PackageDef)(implicit ctx: Context, info: TransformerInfo): Tree =
166+
if (tree.stats.isEmpty) EmptyTree else tree
167+
164168
override def transformDefDef(ddef: DefDef)(implicit ctx: Context, info: TransformerInfo) = {
165169
if (ddef.symbol.hasAnnotation(defn.NativeAnnot)) {
166170
ddef.symbol.resetFlag(Deferred)

‎compiler/src/dotty/tools/dotc/transform/LinkAll.scala

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
package dotty.tools.dotc.transform
2+
3+
import dotty.tools.dotc.CompilationUnit
4+
import dotty.tools.dotc.ast.Trees._
5+
import dotty.tools.dotc.ast.tpd
6+
import dotty.tools.dotc.core.Contexts._
7+
import dotty.tools.dotc.core.SymDenotations.ClassDenotation
8+
import dotty.tools.dotc.core.Symbols._
9+
import dotty.tools.dotc.core.Flags._
10+
import dotty.tools.dotc.transform.TreeTransforms._
11+
12+
/** Loads all potentially reachable trees from tasty. ▲
13+
* Only performed on whole world optimization mode. ▲ ▲
14+
*
15+
* TODO: Next step is to only load compilation units reachable in the call graph
16+
*/
17+
class LinkAll extends MiniPhaseTransform {
18+
import tpd._
19+
import LinkAll._
20+
21+
override def phaseName = "linkAll"
22+
23+
/** Do not transform the any tree, runOn will traverse the trees and reload compilation units if needed */
24+
override def prepareForUnit(tree: tpd.Tree)(implicit ctx: Context): TreeTransform = NoTransform
25+
26+
override def runOn(units: List[CompilationUnit])(implicit ctx: Context): List[CompilationUnit] = {
27+
/** Loads and processes new compilation units, possibly loading more units. */
28+
def allUnits(processed: Set[CompilationUnit], unprocessed: Set[CompilationUnit], loadedClasses: Set[ClassDenotation])(implicit ctx: Context): List[CompilationUnit] = {
29+
if (unprocessed.isEmpty) processed.toList
30+
else {
31+
val accum = new ClassesToLoadAccumulator
32+
val classesToLoad = unprocessed.foldLeft(Set.empty[ClassDenotation])((acc, unit) => accum.apply(acc, unit.tpdTree)) -- loadedClasses
33+
val loadedUnits = classesToLoad.flatMap(cls => loadCompilationUnit(cls))
34+
allUnits(processed ++ unprocessed, loadedUnits, loadedClasses ++ classesToLoad)
35+
}
36+
}
37+
38+
if (ctx.settings.XlinkOptimise.value) super.runOn(allUnits(Set.empty, units.toSet, Set.empty))
39+
else super.runOn(units)
40+
}
41+
42+
/** Collects all class denotations that may need to be loaded. */
43+
private class ClassesToLoadAccumulator extends TreeAccumulator[Set[ClassDenotation]] {
44+
private var inParents = false
45+
override def apply(acc: Set[ClassDenotation], tree: tpd.Tree)(implicit ctx: Context): Set[ClassDenotation] = tree match {
46+
case New(tpt) => accum(acc, tpt.tpe.classSymbol)
47+
case AppliedTypeTree(tpt, _) if inParents => accum(acc, tpt.symbol)
48+
case tree: RefTree if inParents || tree.symbol.is(Module) =>
49+
foldOver(accum(acc, tree.symbol), tree)
50+
case tree @ Template(constr, parents, self, _) =>
51+
val acc1 = this(acc, constr)
52+
inParents = true
53+
val acc2 = this(acc1, parents)
54+
inParents = false
55+
this(this(acc2, self), tree.body)
56+
case _ => foldOver(acc, tree)
57+
}
58+
59+
/** Accumulate class denotation for `sym` if needed */
60+
private def accum(acc: Set[ClassDenotation], sym: Symbol)(implicit ctx: Context): Set[ClassDenotation] = {
61+
val topClass = sym.topLevelClass.denot.asClass
62+
if (topClass.is(JavaDefined) || topClass.is(Scala2x) || topClass.symbol == defn.ObjectClass) acc
63+
else acc + topClass
64+
}
65+
}
66+
}
67+
68+
object LinkAll {
69+
70+
private[LinkAll] def loadCompilationUnit(clsd: ClassDenotation)(implicit ctx: Context): Option[CompilationUnit] = {
71+
assert(ctx.settings.XlinkOptimise.value)
72+
val tree = clsd.symbol.asClass.tree
73+
if (tree.isEmpty) None
74+
else {
75+
ctx.log("Loading compilation unit for: " + clsd)
76+
Some(CompilationUnit.mkCompilationUnit(clsd, tree, forceTrees = false))
77+
}
78+
}
79+
80+
}

‎compiler/src/dotty/tools/dotc/typer/Typer.scala

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1427,9 +1427,8 @@ class Typer extends Namer with TypeAssigner with Applications with Implicits wit
14271427
// check value class constraints
14281428
checkDerivedValueClass(cls, body1)
14291429

1430-
if (ctx.settings.YretainTrees.value) {
1431-
cls.myTree = cdef1
1432-
}
1430+
cls.registerTree(cdef1)
1431+
14331432
cdef1
14341433

14351434
// todo later: check that

‎compiler/test/dotc/comptest.scala

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
package dotc
22

3-
import dotty.tools.vulpix.ParallelTesting
3+
import dotty.tools.vulpix.{ParallelTesting, TestFlags}
44

55
import scala.concurrent.duration._
66

@@ -26,9 +26,6 @@ object comptest extends ParallelTesting {
2626
dotcDir + "tools/dotc/core/Types.scala",
2727
dotcDir + "tools/dotc/ast/Trees.scala"
2828
),
29-
Array(
30-
"-Ylog:frontend",
31-
"-Xprompt"
32-
)
29+
TestFlags("", Array("-Ylog:frontend", "-Xprompt"))
3330
)
3431
}

‎compiler/test/dotty/tools/dotc/CompilationTests.scala

Lines changed: 27 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,15 @@ package tools
33
package dotc
44

55
import org.junit.{ Test, BeforeClass, AfterClass }
6+
import org.junit.Assert._
67

78
import java.nio.file._
89
import java.util.stream.{ Stream => JStream }
910
import scala.collection.JavaConverters._
1011
import scala.util.matching.Regex
1112
import scala.concurrent.duration._
12-
13-
import vulpix.{ ParallelTesting, SummaryReport, SummaryReporting, TestConfiguration }
13+
import vulpix._
14+
import dotty.tools.io.JFile
1415

1516

1617
class CompilationTests extends ParallelTesting {
@@ -29,7 +30,6 @@ class CompilationTests extends ParallelTesting {
2930

3031
@Test def compilePos: Unit = {
3132
compileList("compileStdLib", StdLibSources.whitelisted, scala2Mode.and("-migration", "-Yno-inline")) +
32-
compileDir("../collection-strawman/src/main", defaultOptions) +
3333
compileDir("../compiler/src/dotty/tools/dotc/ast", defaultOptions) +
3434
compileDir("../compiler/src/dotty/tools/dotc/config", defaultOptions) +
3535
compileDir("../compiler/src/dotty/tools/dotc/core", allowDeepSubtypes) +
@@ -40,7 +40,7 @@ class CompilationTests extends ParallelTesting {
4040
compileDir("../compiler/src/dotty/tools/dotc/typer", defaultOptions) +
4141
compileDir("../compiler/src/dotty/tools/dotc/util", defaultOptions) +
4242
compileDir("../compiler/src/dotty/tools/io", defaultOptions) +
43-
compileDir("../compiler/src/dotty/tools/dotc/core", noCheckOptions ++ classPath) +
43+
compileDir("../compiler/src/dotty/tools/dotc/core", TestFlags(classPath, noCheckOptions)) +
4444
compileFile("../tests/pos/nullarify.scala", defaultOptions.and("-Ycheck:nullarify")) +
4545
compileFile("../tests/pos-scala2/rewrites.scala", scala2Mode.and("-rewrite")).copyToTarget() +
4646
compileFile("../tests/pos-special/t8146a.scala", allowDeepSubtypes) +
@@ -222,14 +222,13 @@ class CompilationTests extends ParallelTesting {
222222
* version of Dotty
223223
*/
224224
@Test def tastyBootstrap: Unit = {
225-
val opt = Array(
226-
"-classpath",
225+
val opt = TestFlags(
227226
// compile with bootstrapped library on cp:
228227
defaultOutputDir + "lib/src/:" +
229228
// as well as bootstrapped compiler:
230229
defaultOutputDir + "dotty1/dotty/:" +
231230
Jars.dottyInterfaces,
232-
"-Ycheck-reentrant"
231+
Array("-Ycheck-reentrant")
233232
)
234233

235234
def lib =
@@ -292,6 +291,27 @@ class CompilationTests extends ParallelTesting {
292291

293292
tests.foreach(_.delete())
294293
}
294+
295+
private val (compilerSources, backendSources, backendJvmSources) = {
296+
val compilerDir = Paths.get("../compiler/src")
297+
val compilerSources0 = sources(Files.walk(compilerDir))
298+
299+
val backendDir = Paths.get("../scala-backend/src/compiler/scala/tools/nsc/backend")
300+
val backendJvmDir = Paths.get("../scala-backend/src/compiler/scala/tools/nsc/backend/jvm")
301+
302+
// NOTE: Keep these exclusions synchronized with the ones in the sbt build (Build.scala)
303+
val backendExcluded =
304+
List("JavaPlatform.scala", "Platform.scala", "ScalaPrimitives.scala")
305+
val backendJvmExcluded =
306+
List("BCodeICodeCommon.scala", "GenASM.scala", "GenBCode.scala", "ScalacBackendInterface.scala", "BackendStats.scala")
307+
308+
val backendSources0 =
309+
sources(Files.list(backendDir), excludedFiles = backendExcluded)
310+
val backendJvmSources0 =
311+
sources(Files.list(backendJvmDir), excludedFiles = backendJvmExcluded)
312+
313+
(compilerSources0, backendSources0, backendJvmSources0)
314+
}
295315
}
296316

297317
object CompilationTests {

0 commit comments

Comments
 (0)