Make ReifyQuotes a macro phase

The previous attempt to make it a miniphase in the FirstTransform group
turned out to be too complicated. The implementation so far did not handle
type splices in Expr trees correctly, and doing so would not have been easy.

We now run ReifyQuotes as a macro phase immediately after pickler. This
means trees are reified exactly as they are pickled. We address the problem
of slowdowns due to extra tree traversal by keeping a flag in CompilationUnit
which allows us to skip the traversal altogether for compilation units that
do not contain quotes.

Author: odersky

compiler/src/dotty/tools/dotc/CompilationUnit.scala

@@ -21,6 +21,11 @@ class CompilationUnit(val source: SourceFile) {
 
   /** Pickled TASTY binaries, indexed by class. */
   var pickled: Map[ClassSymbol, Array[Byte]] = Map()
+
+  /** Will be reset to `true` if `untpdTree` contains `Quote` trees. The information
+   *  is used in phase ReifyQuotes in order to avoid traversing a quote-less tree.
+   */
+  var containsQuotes: Boolean = false
 }
 
 object CompilationUnit {

compiler/src/dotty/tools/dotc/Compiler.scala

@@ -47,10 +47,10 @@ class Compiler {
       List(new sbt.ExtractAPI),     // Sends a representation of the API of classes to sbt via callbacks
       List(new Pickler),            // Generate TASTY info
       List(new LinkAll),            // Reload compilation units from TASTY for library code (if needed)
+      List(new ReifyQuotes),        // Turn quoted trees into explicit run-time data structures
       List(new FirstTransform,      // Some transformations to put trees into a canonical form
            new CheckReentrant,      // Internal use only: Check that compiled program has no data races involving global vars
-           new ElimPackagePrefixes, // Eliminate syntactic references to Java packages
-           new ReifyQuotes),        // Eliminate syntactic references to Java packages
+           new ElimPackagePrefixes), // Eliminate references to package prefixes in Select nodes
       List(new CheckStatic,         // Check restrictions that apply to @static members
            new ElimRepeated,        // Rewrite vararg parameters and arguments
            new NormalizeFlags,      // Rewrite some definition flags

compiler/src/dotty/tools/dotc/core/Definitions.scala

@@ -591,6 +591,9 @@ class Definitions {
   def ClassTagClass(implicit ctx: Context) = ClassTagType.symbol.asClass
   def ClassTagModule(implicit ctx: Context) = ClassTagClass.companionModule
 
+  lazy val MetaQuotedType = ctx.requiredClassRef("scala.meta.Quoted")
+  def MetaQuotedClass(implicit ctx: Context) = MetaQuotedType.symbol.asClass
+
   lazy val MetaExprType = ctx.requiredClassRef("scala.meta.Expr")
   def MetaExprClass(implicit ctx: Context) = MetaExprType.symbol.asClass
 

compiler/src/dotty/tools/dotc/core/Mode.scala

@@ -93,7 +93,4 @@ object Mode {
 
   /** We are in the IDE */
   val Interactive = newMode(20, "Interactive")
-
-  /** We are in a quoted type during the ReifyQuotes phase */
-  val InQuotedType = newMode(21, "InQuotedType")
 }

compiler/src/dotty/tools/dotc/transform/FirstTransform.scala

@@ -189,8 +189,8 @@ class FirstTransform extends MiniPhase with InfoTransformer { thisPhase =>
     ast.Trees.flatten(reorderAndComplete(trees)(ctx.withPhase(thisPhase.next)))
 
   private def toTypeTree(tree: Tree)(implicit ctx: Context) =
-    if (ctx.mode.is(Mode.InQuotedType)) tree else TypeTree(tree.tpe).withPos(tree.pos)
-    
+    TypeTree(tree.tpe).withPos(tree.pos)
+
   override def transformOther(tree: Tree)(implicit ctx: Context) = tree match {
     case tree: Import => EmptyTree
     case tree: NamedArg => transformAllDeep(tree.arg)

compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala

@@ -12,60 +12,64 @@ import scala.collection.mutable
  *  The mini phase needs to run in the same group as FirstTransform. So far we lack
  *  the machinery to express this constraint in code.
  */
-class ReifyQuotes extends MiniPhase {
+class ReifyQuotes extends MacroTransform {
   import ast.tpd._
 
   override def phaseName: String = "reifyQuotes"
 
+  override def run(implicit ctx: Context): Unit =
+    if (ctx.compilationUnit.containsQuotes) super.run
+
+  protected def newTransformer(implicit ctx: Context): Transformer = new Reifier
+
   /** Serialize `tree`. Embedded splices are represented as nodes of the form
    *
    *      Select(qual, sym)
    *
    *  where `sym` is either `defn.MetaExpr_~` or `defn.MetaType_~`. For any splice,
-   *  the `qual` part will be of the form `Typed(EmptyTree, TypeTree(<underlying type>))`,
-   *  but that knowledge is not needed to uniquely identify a splice node.
+   *  the `qual` part should not be pickled, since it will be added separately later
+   *  as a splice.
    */
   def pickleTree(tree: Tree, isType: Boolean)(implicit ctx: Context): String =
     tree.show // TODO: replace with TASTY
 
-  private def reifyCall(body: Tree, isType: Boolean)(implicit ctx: Context) = {
+  private class Reifier extends Transformer {
+
+    /** Turn `body` of quote into a call of `scala.meta.Unpickler.unpickleType` or
+     *  `scala.meta.Unpickler.unpickleExpr` depending onwhether `isType` is true or not.
+     *  The arguments to the method are:
+     *
+     *    - the serialized `body`, as returned from `pickleTree`
+     *    - all splices found in `body`
+     */
+    private def reifyCall(body: Tree, isType: Boolean)(implicit ctx: Context) = {
 
-    object liftSplices extends TreeMap {
-      val splices = new mutable.ListBuffer[Tree]
-      override def transform(tree: Tree)(implicit ctx: Context) = tree match {
-        case tree @ Select(qual, name)
-        if tree.symbol == defn.MetaExpr_~ || tree.symbol == defn.MetaType_~ =>
-          splices += {
-            if (isType) // transform splice again because embedded type trees were not rewritten before
-              transformAllDeep(qual)(ctx.retractMode(Mode.InQuotedType))
-            else qual
-          }
-          val placeHolder = Typed(EmptyTree, TypeTree(qual.tpe.widen))
-          cpy.Select(tree)(placeHolder, name)
-        case _ =>
-          super.transform(tree)
+      object collectSplices extends TreeAccumulator[mutable.ListBuffer[Tree]] {
+        override def apply(splices: mutable.ListBuffer[Tree], tree: Tree)(implicit ctx: Context) = tree match {
+          case tree @ Select(qual, _)
+          if tree.symbol == defn.MetaExpr_~ || tree.symbol == defn.MetaType_~ =>
+            splices += transform(qual)
+          case _ =>
+            foldOver(splices, tree)
+        }
       }
-    }
+      val splices = collectSplices(new mutable.ListBuffer[Tree], body).toList
+      val reified = pickleTree(body, isType)
 
-    val reified = pickleTree(liftSplices.transform(body), isType)
-    val splices = liftSplices.splices.toList
-    val spliceType = if (isType) defn.MetaTypeType else defn.MetaExprType
+      ref(if (isType) defn.Unpickler_unpickleType else defn.Unpickler_unpickleExpr)
+        .appliedToType(if (isType) body.tpe else body.tpe.widen)
+        .appliedTo(
+          Literal(Constant(reified)),
+          SeqLiteral(splices, TypeTree(defn.MetaQuotedType)))
+    }
 
-    ref(if (isType) defn.Unpickler_unpickleType else defn.Unpickler_unpickleExpr)
-      .appliedToType(if (isType) body.tpe else body.tpe.widen)
-      .appliedTo(
-        Literal(Constant(reified)),
-        SeqLiteral(splices, TypeTree(spliceType.appliedTo(TypeBounds.empty))))
+    override def transform(tree: Tree)(implicit ctx: Context): Tree = tree match {
+      case Apply(fn, arg :: Nil) if fn.symbol == defn.quoteMethod =>
+        reifyCall(arg, isType = false)
+      case TypeApply(fn, arg :: Nil) if fn.symbol == defn.typeQuoteMethod =>
+        reifyCall(arg, isType = true)
+      case _ =>
+        super.transform(tree)
+    }
   }
-
-  override def transformApply(tree: Apply)(implicit ctx: Context): Tree =
-    if (tree.fun.symbol == defn.quoteMethod) reifyCall(tree.args.head, isType = false)
-    else tree
-
-  override def prepareForTypeApply(tree: TypeApply)(implicit ctx: Context): Context =
-    if (tree.symbol == defn.typeQuoteMethod) ctx.addMode(Mode.InQuotedType) else ctx
-
-  override def transformTypeApply(tree: TypeApply)(implicit ctx: Context): Tree =
-    if (tree.fun.symbol == defn.typeQuoteMethod) reifyCall(tree.args.head, isType = true)
-    else tree
 }

compiler/src/dotty/tools/dotc/typer/Typer.scala

@@ -1073,6 +1073,7 @@ class Typer extends Namer with TypeAssigner with Applications with Implicits wit
   }
 
   def typedQuote(tree: untpd.Quote, pt: Type)(implicit ctx: Context): Tree = track("typedQuote") {
+    ctx.compilationUnit.containsQuotes = true
     val untpd.Quote(body) = tree
     val isType = body.isType
     val resultClass = if (isType) defn.MetaTypeClass else defn.MetaExprClass

library/src/scala/meta/Expr.scala

@@ -1,5 +1,5 @@
 package scala.meta
 
-class Expr[T] {
+class Expr[T] extends Quoted {
   def unary_~ : T = ???
 }

library/src/scala/meta/Quoted.scala

@@ -0,0 +1,6 @@
+package scala.meta
+
+/** Common superclass of Expr and Type */
+class Quoted
+
+

library/src/scala/meta/Type.scala

@@ -1,5 +1,5 @@
 package scala.meta
 
-class Type[T] {
+class Type[T] extends Quoted {
   type unary_~ = T
 }

library/src/scala/meta/Unpickler.scala

@@ -11,10 +11,10 @@ object Unpickler {
   /** Unpickle `repr` which represents a pickled `Expr` tree,
    *  replacing splice nodes with `args`
    */
-  def unpickleExpr[T](repr: Pickled, args: Seq[Expr[_]]): Expr[T] = ???
+  def unpickleExpr[T](repr: Pickled, args: Seq[Quoted]): Expr[T] = ???
 
   /** Unpickle `repr` which represents a pickled `Type` tree,
    *  replacing splice nodes with `args`
    */
-  def unpickleType[T](repr: Pickled, args: Seq[Type[_]]): Type[T] = ???
+  def unpickleType[T](repr: Pickled, args: Seq[Quoted]): Type[T] = ???
 }