Re-architecture quote pickling

Split cross quote reference handling from pickling

Fixes scala#8100
Fixes scala#12440
Fixes scala#13563

Author: nicolasstucki

compiler/src/dotty/tools/dotc/Compiler.scala

@@ -54,6 +54,7 @@ class Compiler {
     List(new Inlining) ::           // Inline and execute macros
     List(new PostInlining) ::       // Add mirror support for inlined code
     List(new Staging) ::            // Check staging levels and heal staged types
+    List(new Splicing) ::           // TODO
     List(new PickleQuotes) ::       // Turn quoted trees into explicit run-time data structures
     Nil
 

compiler/src/dotty/tools/dotc/ast/TreeTypeMap.scala

@@ -133,8 +133,8 @@ class TreeTypeMap(
           val bind1 = tmap.transformSub(bind)
           val expr1 = tmap.transform(expr)
           cpy.Labeled(labeled)(bind1, expr1)
-        case Hole(isTermHole, n, args) =>
-          Hole(isTermHole, n, args.mapConserve(transform)).withSpan(tree.span).withType(mapType(tree.tpe))
+        case Hole(isTerm, n, args, content, tpt) =>
+          Hole(isTerm, n, args.mapConserve(transform), transform(content), transform(tpt)).withSpan(tree.span).withType(mapType(tree.tpe))
         case lit @ Literal(Constant(tpe: Type)) =>
           cpy.Literal(lit)(Constant(mapType(tpe)))
         case tree1 =>

compiler/src/dotty/tools/dotc/ast/Trees.scala

@@ -525,7 +525,10 @@ object Trees {
       attachmentOrElse(untpd.KindOfApply, ApplyKind.Regular)
   }
 
-
+  object GenericApply:
+    def unapply[T >: Untyped](tree: Tree[T]): Option[(Tree[T], List[Tree[T]])] = tree match
+      case tree: GenericApply[T] => Some((tree.fun, tree.args))
+      case _ => None
 
   /** fun[args] */
   case class TypeApply[-T >: Untyped] private[ast] (fun: Tree[T], args: List[Tree[T]])(implicit @constructorOnly src: SourceFile)
@@ -975,7 +978,7 @@ object Trees {
   /** Tree that replaces a splice in pickled quotes.
    *  It is only used when picking quotes (Will never be in a TASTy file).
    */
-  case class Hole[-T >: Untyped](isTermHole: Boolean, idx: Int, args: List[Tree[T]])(implicit @constructorOnly src: SourceFile) extends Tree[T] {
+  case class Hole[-T >: Untyped](isTermHole: Boolean, idx: Int, args: List[Tree[T]], content: Tree[T], tpt: Tree[T])(implicit @constructorOnly src: SourceFile) extends Tree[T] {
     type ThisTree[-T >: Untyped] <: Hole[T]
     override def isTerm: Boolean = isTermHole
     override def isType: Boolean = !isTermHole
@@ -1331,6 +1334,10 @@ object Trees {
         case tree: Thicket if (trees eq tree.trees) => tree
         case _ => finalize(tree, untpd.Thicket(trees)(sourceFile(tree)))
       }
+      def Hole(tree: Tree)(isTerm: Boolean, idx: Int, args: List[Tree], content: Tree, tpt: Tree)(using Context): Hole = tree match {
+        case tree: Hole if isTerm == tree.isTerm && idx == tree.idx && args.eq(tree.args) && content.eq(tree.content) && content.eq(tree.content) => tree
+        case _ => finalize(tree, untpd.Hole(isTerm, idx, args, content, tpt)(sourceFile(tree)))
+      }
 
       // Copier methods with default arguments; these demand that the original tree
       // is of the same class as the copy. We only include trees with more than 2 elements here.
@@ -1352,6 +1359,9 @@ object Trees {
         TypeDef(tree: Tree)(name, rhs)
       def Template(tree: Template)(constr: DefDef = tree.constr, parents: List[Tree] = tree.parents, derived: List[untpd.Tree] = tree.derived, self: ValDef = tree.self, body: LazyTreeList = tree.unforcedBody)(using Context): Template =
         Template(tree: Tree)(constr, parents, derived, self, body)
+      def Hole(tree: Hole)(isTerm: Boolean = tree.isTerm, idx: Int = tree.idx, args: List[Tree] = tree.args, content: Tree = tree.content, tpt: Tree = tree.tpt)(using Context): Hole =
+        Hole(tree: Tree)(isTerm, idx, args, content, tpt)
+
     }
 
     /** Hook to indicate that a transform of some subtree should be skipped */
@@ -1481,6 +1491,8 @@ object Trees {
             case Thicket(trees) =>
               val trees1 = transform(trees)
               if (trees1 eq trees) tree else Thicket(trees1)
+            case Hole(isTerm, idx, args, content, tpt) =>
+              cpy.Hole(tree)(isTerm, idx, transform(args), transform(content), transform(tpt))
             case _ =>
               transformMoreCases(tree)
           }
@@ -1618,8 +1630,8 @@ object Trees {
               this(this(x, arg), annot)
             case Thicket(ts) =>
               this(x, ts)
-            case Hole(_, _, args) =>
-              this(x, args)
+            case Hole(_, _, args, content, tpt) =>
+              this(this(this(x, args), content), tpt)
             case _ =>
               foldMoreCases(x, tree)
           }

compiler/src/dotty/tools/dotc/ast/tpd.scala

@@ -373,6 +373,9 @@ object tpd extends Trees.Instance[Type] with TypedTreeInfo {
   def Throw(expr: Tree)(using Context): Tree =
     ref(defn.throwMethod).appliedTo(expr)
 
+  def Hole(isTerm: Boolean, idx: Int, args: List[Tree], content: Tree, tpt: Tree)(using Context): Hole =
+    ta.assignType(untpd.Hole(isTerm, idx, args, content, tpt), tpt)
+
   // ------ Making references ------------------------------------------------------
 
   def prefixIsElidable(tp: NamedType)(using Context): Boolean = {

compiler/src/dotty/tools/dotc/ast/untpd.scala

@@ -411,6 +411,7 @@ object untpd extends Trees.Instance[Untyped] with UntypedTreeInfo {
   def Export(expr: Tree, selectors: List[ImportSelector])(implicit src: SourceFile): Export = new Export(expr, selectors)
   def PackageDef(pid: RefTree, stats: List[Tree])(implicit src: SourceFile): PackageDef = new PackageDef(pid, stats)
   def Annotated(arg: Tree, annot: Tree)(implicit src: SourceFile): Annotated = new Annotated(arg, annot)
+  def Hole(isTerm: Boolean, idx: Int, args: List[Tree], content: Tree, tpt: Tree)(implicit src: SourceFile): Hole = new Hole(isTerm, idx, args, content, tpt)
 
   // ------ Additional creation methods for untyped only -----------------
 

compiler/src/dotty/tools/dotc/core/Phases.scala

@@ -201,7 +201,7 @@ object Phases {
     private var mySbtExtractDependenciesPhase: Phase = _
     private var myPicklerPhase: Phase = _
     private var myInliningPhase: Phase = _
-    private var myPickleQuotesPhase: Phase = _
+    private var mySplicingPhase: Phase = _
     private var myFirstTransformPhase: Phase = _
     private var myCollectNullableFieldsPhase: Phase = _
     private var myRefChecksPhase: Phase = _
@@ -223,7 +223,7 @@ object Phases {
     final def sbtExtractDependenciesPhase: Phase = mySbtExtractDependenciesPhase
     final def picklerPhase: Phase = myPicklerPhase
     final def inliningPhase: Phase = myInliningPhase
-    final def pickleQuotesPhase: Phase = myPickleQuotesPhase
+    final def splicingPhase: Phase = mySplicingPhase
     final def firstTransformPhase: Phase = myFirstTransformPhase
     final def collectNullableFieldsPhase: Phase = myCollectNullableFieldsPhase
     final def refchecksPhase: Phase = myRefChecksPhase
@@ -248,7 +248,7 @@ object Phases {
       mySbtExtractDependenciesPhase = phaseOfClass(classOf[sbt.ExtractDependencies])
       myPicklerPhase = phaseOfClass(classOf[Pickler])
       myInliningPhase = phaseOfClass(classOf[Inlining])
-      myPickleQuotesPhase = phaseOfClass(classOf[PickleQuotes])
+      mySplicingPhase = phaseOfClass(classOf[Splicing])
       myFirstTransformPhase = phaseOfClass(classOf[FirstTransform])
       myCollectNullableFieldsPhase = phaseOfClass(classOf[CollectNullableFields])
       myRefChecksPhase = phaseOfClass(classOf[RefChecks])
@@ -423,7 +423,7 @@ object Phases {
   def sbtExtractDependenciesPhase(using Context): Phase = ctx.base.sbtExtractDependenciesPhase
   def picklerPhase(using Context): Phase                = ctx.base.picklerPhase
   def inliningPhase(using Context): Phase               = ctx.base.inliningPhase
-  def pickleQuotesPhase(using Context): Phase           = ctx.base.pickleQuotesPhase
+  def splicingPhase(using Context): Phase               = ctx.base.splicingPhase
   def firstTransformPhase(using Context): Phase         = ctx.base.firstTransformPhase
   def refchecksPhase(using Context): Phase              = ctx.base.refchecksPhase
   def elimRepeatedPhase(using Context): Phase           = ctx.base.elimRepeatedPhase

compiler/src/dotty/tools/dotc/core/StagingContext.scala

@@ -13,11 +13,11 @@ import scala.collection.mutable
 
 object StagingContext {
 
-  /** A key to be used in a context property that tracks the quoteation level */
+  /** A key to be used in a context property that tracks the quotation level */
   private val QuotationLevel = new Property.Key[Int]
 
-  /** A key to be used in a context property that tracks the quoteation stack.
-   *  Stack containing the Quotes references recieved by the surrounding quotes.
+  /** A key to be used in a context property that tracks the quotation stack.
+   *  Stack containing the Quotes references received by the surrounding quotes.
    */
   private val QuotesStack = new Property.Key[List[tpd.Tree]]
 
@@ -31,7 +31,7 @@ object StagingContext {
   def quoteContext(using Context): Context =
     ctx.fresh.setProperty(QuotationLevel, level + 1)
 
-  /** Context with an incremented quotation level and pushes a refecence to a Quotes on the quote context stack */
+  /** Context with an incremented quotation level and pushes a reference to a Quotes on the quote context stack */
   def pushQuotes(qctxRef: tpd.Tree)(using Context): Context =
     val old = ctx.property(QuotesStack).getOrElse(List.empty)
     ctx.fresh.setProperty(QuotationLevel, level + 1)
@@ -48,7 +48,7 @@ object StagingContext {
     ctx.property(TaggedTypes).get
 
   /** Context with a decremented quotation level and pops the Some of top of the quote context stack or None if the stack is empty.
-   *  The quotation stack could be empty if we are in a top level splice or an eroneous splice directly witin a top level splice.
+   *  The quotation stack could be empty if we are in a top level splice or an erroneous splice directly within a top level splice.
    */
   def popQuotes()(using Context): (Option[tpd.Tree], Context) =
     val ctx1 = ctx.fresh.setProperty(QuotationLevel, level - 1)

compiler/src/dotty/tools/dotc/core/tasty/TreePickler.scala

@@ -644,11 +644,11 @@ class TreePickler(pickler: TastyPickler) {
               pickleTree(hi)
               pickleTree(alias)
           }
-        case Hole(_, idx, args) =>
+        case Hole(_, idx, args, _, tpt) =>
           writeByte(HOLE)
           withLength {
             writeNat(idx)
-            pickleType(tree.tpe, richTypes = true)
+            pickleType(tpt.tpe, richTypes = true)
             args.foreach(pickleTree)
           }
       }

compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala

@@ -1288,7 +1288,7 @@ class TreeUnpickler(reader: TastyReader,
               val idx = readNat()
               val tpe = readType()
               val args = until(end)(readTerm())
-              Hole(true, idx, args).withType(tpe)
+              Hole(true, idx, args, EmptyTree, TypeTree(tpe)).withType(tpe)
             case _ =>
               readPathTerm()
           }
@@ -1324,7 +1324,7 @@ class TreeUnpickler(reader: TastyReader,
           val idx = readNat()
           val tpe = readType()
           val args = until(end)(readTerm())
-          Hole(false, idx, args).withType(tpe)
+          Hole(false, idx, args, EmptyTree, TypeTree(tpe)).withType(tpe)
         case _ =>
           if (isTypeTreeTag(nextByte)) readTerm()
           else {

compiler/src/dotty/tools/dotc/printing/RefinedPrinter.scala

@@ -699,10 +699,12 @@ class RefinedPrinter(_ctx: Context) extends PlainPrinter(_ctx) {
         "Thicket {" ~~ toTextGlobal(trees, "\n") ~~ "}"
       case MacroTree(call) =>
         keywordStr("macro ") ~ toTextGlobal(call)
-      case Hole(isTermHole, idx, args) =>
-        val (prefix, postfix) = if isTermHole then ("{{{ ", " }}}") else ("[[[ ", " ]]]")
+      case Hole(isTerm, idx, args, content, tpt) =>
+        val (prefix, postfix) = if isTerm then ("{{{", "}}}") else ("[[[", "]]]")
         val argsText = toTextGlobal(args, ", ")
-        prefix ~~ idx.toString ~~ "|" ~~ argsText ~~ postfix
+        val contentText = toTextGlobal(content)
+        val tptText = toTextGlobal(tpt)
+        prefix ~~ idx.toString ~~ "|" ~~ tptText ~~ "|" ~~ argsText ~~ "|" ~~ contentText ~~ postfix
       case _ =>
         tree.fallbackToText(this)
     }

compiler/src/dotty/tools/dotc/quoted/PickledQuotes.scala

@@ -54,9 +54,9 @@ object PickledQuotes {
   /** Unpickle the tree contained in the TastyExpr */
   def unpickleTerm(pickled: String | List[String], typeHole: (Int, Seq[Any]) => scala.quoted.Type[?], termHole: (Int, Seq[Any], scala.quoted.Quotes) => scala.quoted.Expr[?])(using Context): Tree = {
     val unpickled = withMode(Mode.ReadPositions)(unpickle(pickled, isType = false))
-    val Inlined(call, Nil, expnasion) = unpickled
+    val Inlined(call, Nil, expansion0) = unpickled
     val inlineCtx = inlineContext(call)
-    val expansion1 = spliceTypes(expnasion, typeHole, termHole)(using inlineCtx)
+    val expansion1 = spliceTypes(expansion0, typeHole, termHole)(using inlineCtx)
     val expansion2 = spliceTerms(expansion1, typeHole, termHole)(using inlineCtx)
     cpy.Inlined(unpickled)(call, Nil, expansion2)
   }
@@ -71,10 +71,10 @@ object PickledQuotes {
   private def spliceTerms(tree: Tree, typeHole: (Int, Seq[Any]) => scala.quoted.Type[?], termHole: (Int, Seq[Any], scala.quoted.Quotes) => scala.quoted.Expr[?])(using Context): Tree = {
     val evaluateHoles = new TreeMap {
       override def transform(tree: tpd.Tree)(using Context): tpd.Tree = tree match {
-        case Hole(isTerm, idx, args) =>
+        case Hole(isTerm, idx, args, _, _) =>
           inContext(SpliceScope.contextWithNewSpliceScope(tree.sourcePos)) {
             val reifiedArgs = args.map { arg =>
-              if (arg.isTerm) (q: Quotes) ?=> new ExprImpl(arg, SpliceScope.getCurrent)
+              if (arg.isTerm) new ExprImpl(arg, SpliceScope.getCurrent)
               else new TypeImpl(arg, SpliceScope.getCurrent)
             }
             if isTerm then
@@ -131,11 +131,14 @@ object PickledQuotes {
           case tdef: TypeDef =>
             assert(tdef.symbol.hasAnnotation(defn.QuotedRuntime_SplicedTypeAnnot))
             val tree = tdef.rhs match
-              case TypeBoundsTree(_, Hole(_, idx, args), _) =>
+              case TypeBoundsTree(_, Hole(_, idx, args, _, _), _) => // keep for backwards compat
                 val quotedType = typeHole(idx, args)
                 PickledQuotes.quotedTypeToTree(quotedType)
-              case TypeBoundsTree(_, tpt, _) =>
+              case TypeBoundsTree(_, tpt, _) => // keep for backwards compat
                 tpt
+              case Hole(_, idx, args, _, _) =>
+                val quotedType = typeHole(idx, args)
+                PickledQuotes.quotedTypeToTree(quotedType)
             (tdef.symbol, tree.tpe)
         }.toMap
         class ReplaceSplicedTyped extends TypeMap() {