Re-architecture quote pickling

Separate the logic that creates holes in quotes from the logic that
pickles the quotes. Holes are created in the `Splicer` phase and the
result of the transformation can be `-Ycheck`ed. Now, the `PickleQuotes`
phase only needs to extract the contents of the holes, pickle the quote
and put them into a call to `unpickleExprV2`/`unpickleType`.

We add `unpickleExprV2` to support some optimization in the encoding of
the pickled quote. Namely we removed an unnecessary lambda from the
arguments of the hole passed into the contents of the hole. By not
changing `unpickleExpr` the current compiler will be able to handle the
old encoding in binaries compiled with older compilers.

Fixes scala#8100
Fixes scala#12440
Fixes scala#13563

Author: nicolasstucki

compiler/src/dotty/tools/dotc/Compiler.scala

@@ -54,6 +54,7 @@ class Compiler {
     List(new Inlining) ::           // Inline and execute macros
     List(new PostInlining) ::       // Add mirror support for inlined code
     List(new Staging) ::            // Check staging levels and heal staged types
+    List(new Splicing) ::           // Replace level 1 splices with holes
     List(new PickleQuotes) ::       // Turn quoted trees into explicit run-time data structures
     Nil
 

compiler/src/dotty/tools/dotc/ast/TreeTypeMap.scala

@@ -133,8 +133,11 @@ class TreeTypeMap(
           val bind1 = tmap.transformSub(bind)
           val expr1 = tmap.transform(expr)
           cpy.Labeled(labeled)(bind1, expr1)
-        case Hole(isTermHole, n, args) =>
-          Hole(isTermHole, n, args.mapConserve(transform)).withSpan(tree.span).withType(mapType(tree.tpe))
+        case tree @ Hole(_, _, args, content, tpt) =>
+          val args1 = args.mapConserve(transform)
+          val content1 = transform(content)
+          val tpt1 = transform(tpt)
+          cpy.Hole(tree)(args = args1, content = content1, tpt = tpt1)
         case lit @ Literal(Constant(tpe: Type)) =>
           cpy.Literal(lit)(Constant(mapType(tpe)))
         case tree1 =>

compiler/src/dotty/tools/dotc/ast/Trees.scala

@@ -503,6 +503,11 @@ object Trees {
     def forwardTo: Tree[T] = fun
   }
 
+  object GenericApply:
+    def unapply[T >: Untyped](tree: Tree[T]): Option[(Tree[T], List[Tree[T]])] = tree match
+      case tree: GenericApply[T] => Some((tree.fun, tree.args))
+      case _ => None
+
   /** The kind of application */
   enum ApplyKind:
     case Regular      // r.f(x)
@@ -525,8 +530,6 @@ object Trees {
       attachmentOrElse(untpd.KindOfApply, ApplyKind.Regular)
   }
 
-
-
   /** fun[args] */
   case class TypeApply[-T >: Untyped] private[ast] (fun: Tree[T], args: List[Tree[T]])(implicit @constructorOnly src: SourceFile)
     extends GenericApply[T] {
@@ -972,10 +975,16 @@ object Trees {
   def genericEmptyValDef[T >: Untyped]: ValDef[T]       = theEmptyValDef.asInstanceOf[ValDef[T]]
   def genericEmptyTree[T >: Untyped]: Thicket[T]        = theEmptyTree.asInstanceOf[Thicket[T]]
 
-  /** Tree that replaces a splice in pickled quotes.
-   *  It is only used when picking quotes (Will never be in a TASTy file).
+  /** Tree that replaces a level 1 splices in pickled (level 0) quotes.
+   *  It is only used when picking quotes (will never be in a TASTy file).
+   *
+   *  @param isTermHole If this hole is a term, otherwise it is a type hole.
+   *  @param idx The index of the hole in it's enclosing level 0 quote.
+   *  @param args The arguments of the splice to compute its content
+   *  @param content Lambda that computes the content of the hole. This tree is empty when in a quote pickle.
+   *  @param tpt Type of the hole
    */
-  case class Hole[-T >: Untyped](isTermHole: Boolean, idx: Int, args: List[Tree[T]])(implicit @constructorOnly src: SourceFile) extends Tree[T] {
+  case class Hole[-T >: Untyped](isTermHole: Boolean, idx: Int, args: List[Tree[T]], content: Tree[T], tpt: Tree[T])(implicit @constructorOnly src: SourceFile) extends Tree[T] {
     type ThisTree[-T >: Untyped] <: Hole[T]
     override def isTerm: Boolean = isTermHole
     override def isType: Boolean = !isTermHole
@@ -1331,6 +1340,10 @@ object Trees {
         case tree: Thicket if (trees eq tree.trees) => tree
         case _ => finalize(tree, untpd.Thicket(trees)(sourceFile(tree)))
       }
+      def Hole(tree: Tree)(isTerm: Boolean, idx: Int, args: List[Tree], content: Tree, tpt: Tree)(using Context): Hole = tree match {
+        case tree: Hole if isTerm == tree.isTerm && idx == tree.idx && args.eq(tree.args) && content.eq(tree.content) && content.eq(tree.content) => tree
+        case _ => finalize(tree, untpd.Hole(isTerm, idx, args, content, tpt)(sourceFile(tree)))
+      }
 
       // Copier methods with default arguments; these demand that the original tree
       // is of the same class as the copy. We only include trees with more than 2 elements here.
@@ -1352,6 +1365,9 @@ object Trees {
         TypeDef(tree: Tree)(name, rhs)
       def Template(tree: Template)(constr: DefDef = tree.constr, parents: List[Tree] = tree.parents, derived: List[untpd.Tree] = tree.derived, self: ValDef = tree.self, body: LazyTreeList = tree.unforcedBody)(using Context): Template =
         Template(tree: Tree)(constr, parents, derived, self, body)
+      def Hole(tree: Hole)(isTerm: Boolean = tree.isTerm, idx: Int = tree.idx, args: List[Tree] = tree.args, content: Tree = tree.content, tpt: Tree = tree.tpt)(using Context): Hole =
+        Hole(tree: Tree)(isTerm, idx, args, content, tpt)
+
     }
 
     /** Hook to indicate that a transform of some subtree should be skipped */
@@ -1481,6 +1497,8 @@ object Trees {
             case Thicket(trees) =>
               val trees1 = transform(trees)
               if (trees1 eq trees) tree else Thicket(trees1)
+            case tree @ Hole(_, _, args, content, tpt) =>
+              cpy.Hole(tree)(args = transform(args), content = transform(content), tpt = transform(tpt))
             case _ =>
               transformMoreCases(tree)
           }
@@ -1618,8 +1636,8 @@ object Trees {
               this(this(x, arg), annot)
             case Thicket(ts) =>
               this(x, ts)
-            case Hole(_, _, args) =>
-              this(x, args)
+            case Hole(_, _, args, content, tpt) =>
+              this(this(this(x, args), content), tpt)
             case _ =>
               foldMoreCases(x, tree)
           }

compiler/src/dotty/tools/dotc/ast/tpd.scala

@@ -373,6 +373,9 @@ object tpd extends Trees.Instance[Type] with TypedTreeInfo {
   def Throw(expr: Tree)(using Context): Tree =
     ref(defn.throwMethod).appliedTo(expr)
 
+  def Hole(isTermHole: Boolean, idx: Int, args: List[Tree], content: Tree, tpt: Tree)(using Context): Hole =
+    ta.assignType(untpd.Hole(isTermHole, idx, args, content, tpt), tpt)
+
   // ------ Making references ------------------------------------------------------
 
   def prefixIsElidable(tp: NamedType)(using Context): Boolean = {

compiler/src/dotty/tools/dotc/ast/untpd.scala

@@ -411,6 +411,7 @@ object untpd extends Trees.Instance[Untyped] with UntypedTreeInfo {
   def Export(expr: Tree, selectors: List[ImportSelector])(implicit src: SourceFile): Export = new Export(expr, selectors)
   def PackageDef(pid: RefTree, stats: List[Tree])(implicit src: SourceFile): PackageDef = new PackageDef(pid, stats)
   def Annotated(arg: Tree, annot: Tree)(implicit src: SourceFile): Annotated = new Annotated(arg, annot)
+  def Hole(isTermHole: Boolean, idx: Int, args: List[Tree], content: Tree, tpt: Tree)(implicit src: SourceFile): Hole = new Hole(isTermHole, idx, args, content, tpt)
 
   // ------ Additional creation methods for untyped only -----------------
 

compiler/src/dotty/tools/dotc/core/Definitions.scala

@@ -789,7 +789,7 @@ class Definitions {
   @tu lazy val QuotesClass: ClassSymbol = requiredClass("scala.quoted.Quotes")
 
   @tu lazy val QuoteUnpicklerClass: ClassSymbol = requiredClass("scala.quoted.runtime.QuoteUnpickler")
-    @tu lazy val QuoteUnpickler_unpickleExpr: Symbol = QuoteUnpicklerClass.requiredMethod("unpickleExpr")
+    @tu lazy val QuoteUnpickler_unpickleExprV2: Symbol = QuoteUnpicklerClass.requiredMethod("unpickleExprV2")
     @tu lazy val QuoteUnpickler_unpickleType: Symbol = QuoteUnpicklerClass.requiredMethod("unpickleType")
 
   @tu lazy val QuoteMatchingClass: ClassSymbol = requiredClass("scala.quoted.runtime.QuoteMatching")

compiler/src/dotty/tools/dotc/core/Phases.scala

@@ -201,7 +201,7 @@ object Phases {
     private var mySbtExtractDependenciesPhase: Phase = _
     private var myPicklerPhase: Phase = _
     private var myInliningPhase: Phase = _
-    private var myPickleQuotesPhase: Phase = _
+    private var mySplicingPhase: Phase = _
     private var myFirstTransformPhase: Phase = _
     private var myCollectNullableFieldsPhase: Phase = _
     private var myRefChecksPhase: Phase = _
@@ -223,7 +223,7 @@ object Phases {
     final def sbtExtractDependenciesPhase: Phase = mySbtExtractDependenciesPhase
     final def picklerPhase: Phase = myPicklerPhase
     final def inliningPhase: Phase = myInliningPhase
-    final def pickleQuotesPhase: Phase = myPickleQuotesPhase
+    final def splicingPhase: Phase = mySplicingPhase
     final def firstTransformPhase: Phase = myFirstTransformPhase
     final def collectNullableFieldsPhase: Phase = myCollectNullableFieldsPhase
     final def refchecksPhase: Phase = myRefChecksPhase
@@ -248,7 +248,7 @@ object Phases {
       mySbtExtractDependenciesPhase = phaseOfClass(classOf[sbt.ExtractDependencies])
       myPicklerPhase = phaseOfClass(classOf[Pickler])
       myInliningPhase = phaseOfClass(classOf[Inlining])
-      myPickleQuotesPhase = phaseOfClass(classOf[PickleQuotes])
+      mySplicingPhase = phaseOfClass(classOf[Splicing])
       myFirstTransformPhase = phaseOfClass(classOf[FirstTransform])
       myCollectNullableFieldsPhase = phaseOfClass(classOf[CollectNullableFields])
       myRefChecksPhase = phaseOfClass(classOf[RefChecks])
@@ -423,7 +423,7 @@ object Phases {
   def sbtExtractDependenciesPhase(using Context): Phase = ctx.base.sbtExtractDependenciesPhase
   def picklerPhase(using Context): Phase                = ctx.base.picklerPhase
   def inliningPhase(using Context): Phase               = ctx.base.inliningPhase
-  def pickleQuotesPhase(using Context): Phase           = ctx.base.pickleQuotesPhase
+  def splicingPhase(using Context): Phase               = ctx.base.splicingPhase
   def firstTransformPhase(using Context): Phase         = ctx.base.firstTransformPhase
   def refchecksPhase(using Context): Phase              = ctx.base.refchecksPhase
   def elimRepeatedPhase(using Context): Phase           = ctx.base.elimRepeatedPhase

compiler/src/dotty/tools/dotc/core/StagingContext.scala

@@ -13,11 +13,11 @@ import scala.collection.mutable
 
 object StagingContext {
 
-  /** A key to be used in a context property that tracks the quoteation level */
+  /** A key to be used in a context property that tracks the quotation level */
   private val QuotationLevel = new Property.Key[Int]
 
-  /** A key to be used in a context property that tracks the quoteation stack.
-   *  Stack containing the Quotes references recieved by the surrounding quotes.
+  /** A key to be used in a context property that tracks the quotation stack.
+   *  Stack containing the Quotes references received by the surrounding quotes.
    */
   private val QuotesStack = new Property.Key[List[tpd.Tree]]
 
@@ -31,7 +31,7 @@ object StagingContext {
   def quoteContext(using Context): Context =
     ctx.fresh.setProperty(QuotationLevel, level + 1)
 
-  /** Context with an incremented quotation level and pushes a refecence to a Quotes on the quote context stack */
+  /** Context with an incremented quotation level and pushes a reference to a Quotes on the quote context stack */
   def pushQuotes(qctxRef: tpd.Tree)(using Context): Context =
     val old = ctx.property(QuotesStack).getOrElse(List.empty)
     ctx.fresh.setProperty(QuotationLevel, level + 1)
@@ -48,7 +48,7 @@ object StagingContext {
     ctx.property(TaggedTypes).get
 
   /** Context with a decremented quotation level and pops the Some of top of the quote context stack or None if the stack is empty.
-   *  The quotation stack could be empty if we are in a top level splice or an eroneous splice directly witin a top level splice.
+   *  The quotation stack could be empty if we are in a top level splice or an erroneous splice directly within a top level splice.
    */
   def popQuotes()(using Context): (Option[tpd.Tree], Context) =
     val ctx1 = ctx.fresh.setProperty(QuotationLevel, level - 1)

compiler/src/dotty/tools/dotc/core/tasty/TreePickler.scala

@@ -644,11 +644,11 @@ class TreePickler(pickler: TastyPickler) {
               pickleTree(hi)
               pickleTree(alias)
           }
-        case Hole(_, idx, args) =>
+        case Hole(_, idx, args, _, tpt) =>
           writeByte(HOLE)
           withLength {
             writeNat(idx)
-            pickleType(tree.tpe, richTypes = true)
+            pickleType(tpt.tpe, richTypes = true)
             args.foreach(pickleTree)
           }
       }

compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala

@@ -1288,7 +1288,7 @@ class TreeUnpickler(reader: TastyReader,
               val idx = readNat()
               val tpe = readType()
               val args = until(end)(readTerm())
-              Hole(true, idx, args).withType(tpe)
+              Hole(true, idx, args, EmptyTree, TypeTree(tpe)).withType(tpe)
             case _ =>
               readPathTerm()
           }
@@ -1324,7 +1324,7 @@ class TreeUnpickler(reader: TastyReader,
           val idx = readNat()
           val tpe = readType()
           val args = until(end)(readTerm())
-          Hole(false, idx, args).withType(tpe)
+          Hole(false, idx, args, EmptyTree, TypeTree(tpe)).withType(tpe)
         case _ =>
           if (isTypeTreeTag(nextByte)) readTerm()
           else {

compiler/src/dotty/tools/dotc/printing/RefinedPrinter.scala

@@ -699,10 +699,12 @@ class RefinedPrinter(_ctx: Context) extends PlainPrinter(_ctx) {
         "Thicket {" ~~ toTextGlobal(trees, "\n") ~~ "}"
       case MacroTree(call) =>
         keywordStr("macro ") ~ toTextGlobal(call)
-      case Hole(isTermHole, idx, args) =>
-        val (prefix, postfix) = if isTermHole then ("{{{ ", " }}}") else ("[[[ ", " ]]]")
+      case Hole(isTermHole, idx, args, content, tpt) =>
+        val (prefix, postfix) = if isTermHole then ("{{{", "}}}") else ("[[[", "]]]")
         val argsText = toTextGlobal(args, ", ")
-        prefix ~~ idx.toString ~~ "|" ~~ argsText ~~ postfix
+        val contentText = toTextGlobal(content)
+        val tptText = toTextGlobal(tpt)
+        prefix ~~ idx.toString ~~ "|" ~~ tptText ~~ "|" ~~ argsText ~~ "|" ~~ contentText ~~ postfix
       case _ =>
         tree.fallbackToText(this)
     }

compiler/src/dotty/tools/dotc/quoted/PickledQuotes.scala

@@ -52,12 +52,12 @@ object PickledQuotes {
   }
 
   /** Unpickle the tree contained in the TastyExpr */
-  def unpickleTerm(pickled: String | List[String], typeHole: (Int, Seq[Any]) => scala.quoted.Type[?], termHole: (Int, Seq[Any], scala.quoted.Quotes) => scala.quoted.Expr[?])(using Context): Tree = {
+  def unpickleTerm(pickled: String | List[String], typeHole: (Int, Seq[Any]) => scala.quoted.Type[?], termHole: (Int, Seq[Any], scala.quoted.Quotes) => scala.quoted.Expr[?], version: Int)(using Context): Tree = {
     val unpickled = withMode(Mode.ReadPositions)(unpickle(pickled, isType = false))
-    val Inlined(call, Nil, expnasion) = unpickled
+    val Inlined(call, Nil, expansion) = unpickled
     val inlineCtx = inlineContext(call)
-    val expansion1 = spliceTypes(expnasion, typeHole, termHole)(using inlineCtx)
-    val expansion2 = spliceTerms(expansion1, typeHole, termHole)(using inlineCtx)
+    val expansion1 = spliceTypes(expansion, typeHole, termHole)(using inlineCtx)
+    val expansion2 = spliceTerms(expansion1, typeHole, termHole, version)(using inlineCtx)
     cpy.Inlined(unpickled)(call, Nil, expansion2)
   }
 
@@ -68,16 +68,19 @@ object PickledQuotes {
   }
 
   /** Replace all term holes with the spliced terms */
-  private def spliceTerms(tree: Tree, typeHole: (Int, Seq[Any]) => scala.quoted.Type[?], termHole: (Int, Seq[Any], scala.quoted.Quotes) => scala.quoted.Expr[?])(using Context): Tree = {
+  private def spliceTerms(tree: Tree, typeHole: (Int, Seq[Any]) => scala.quoted.Type[?], termHole: (Int, Seq[Any], scala.quoted.Quotes) => scala.quoted.Expr[?], version: Int)(using Context): Tree = {
     val evaluateHoles = new TreeMap {
       override def transform(tree: tpd.Tree)(using Context): tpd.Tree = tree match {
-        case Hole(isTerm, idx, args) =>
+        case Hole(isTermHole, idx, args, _, _) =>
           inContext(SpliceScope.contextWithNewSpliceScope(tree.sourcePos)) {
             val reifiedArgs = args.map { arg =>
-              if (arg.isTerm) (q: Quotes) ?=> new ExprImpl(arg, SpliceScope.getCurrent)
+              if (arg.isTerm)
+                version match
+                  case 1 => (q: Quotes) ?=> new ExprImpl(arg, SpliceScope.getCurrent)
+                  case 2 => new ExprImpl(arg, SpliceScope.getCurrent)
               else new TypeImpl(arg, SpliceScope.getCurrent)
             }
-            if isTerm then
+            if isTermHole then
               val quotedExpr = termHole(idx, reifiedArgs, QuotesImpl())
               val filled = PickledQuotes.quotedExprToTree(quotedExpr)
 
@@ -131,11 +134,14 @@ object PickledQuotes {
           case tdef: TypeDef =>
             assert(tdef.symbol.hasAnnotation(defn.QuotedRuntime_SplicedTypeAnnot))
             val tree = tdef.rhs match
-              case TypeBoundsTree(_, Hole(_, idx, args), _) =>
+              case TypeBoundsTree(_, Hole(_, idx, args, _, _), _) => // to keep for backwards compat
                 val quotedType = typeHole(idx, args)
                 PickledQuotes.quotedTypeToTree(quotedType)
-              case TypeBoundsTree(_, tpt, _) =>
+              case TypeBoundsTree(_, tpt, _) => // to keep for backwards compat
                 tpt
+              case Hole(_, idx, args, _, _) =>
+                val quotedType = typeHole(idx, args)
+                PickledQuotes.quotedTypeToTree(quotedType)
             (tdef.symbol, tree.tpe)
         }.toMap
         class ReplaceSplicedTyped extends TypeMap() {