Merge pull request #12540 from dotty-staging/rearchitecture-quote-splitting

Re-architecture quote pickling

Author: smarter

compiler/src/dotty/tools/dotc/CompilationUnit.scala

@@ -47,15 +47,10 @@ class CompilationUnit protected (val source: SourceFile) {
   var needsMirrorSupport: Boolean = false
 
   /** Will be set to `true` if contains `Quote`.
-   *  The information is used in phase `Staging` in order to avoid traversing trees that need no transformations.
+   *  The information is used in phase `Staging`/`Splicing`/`PickleQuotes` in order to avoid traversing trees that need no transformations.
    */
   var needsStaging: Boolean = false
 
-  /** Will be set to `true` if contains `Quote` that needs to be pickled
-   *  The information is used in phase `PickleQuotes` in order to avoid traversing trees that need no transformations.
-   */
-  var needsQuotePickling: Boolean = false
-
   var suspended: Boolean = false
   var suspendedAtInliningPhase: Boolean = false
 
@@ -115,7 +110,6 @@ object CompilationUnit {
       val force = new Force
       force.traverse(unit1.tpdTree)
       unit1.needsStaging = force.containsQuote
-      unit1.needsQuotePickling = force.containsQuote
       unit1.needsInlining = force.containsInline
     }
     unit1

compiler/src/dotty/tools/dotc/Compiler.scala

@@ -54,6 +54,7 @@ class Compiler {
     List(new Inlining) ::           // Inline and execute macros
     List(new PostInlining) ::       // Add mirror support for inlined code
     List(new Staging) ::            // Check staging levels and heal staged types
+    List(new Splicing) ::           // Replace level 1 splices with holes
     List(new PickleQuotes) ::       // Turn quoted trees into explicit run-time data structures
     Nil
 

compiler/src/dotty/tools/dotc/ast/TreeTypeMap.scala

@@ -134,8 +134,11 @@ class TreeTypeMap(
           val bind1 = tmap.transformSub(bind)
           val expr1 = tmap.transform(expr)
           cpy.Labeled(labeled)(bind1, expr1)
-        case Hole(isTermHole, n, args) =>
-          Hole(isTermHole, n, args.mapConserve(transform)).withSpan(tree.span).withType(mapType(tree.tpe))
+        case tree @ Hole(_, _, args, content, tpt) =>
+          val args1 = args.mapConserve(transform)
+          val content1 = transform(content)
+          val tpt1 = transform(tpt)
+          cpy.Hole(tree)(args = args1, content = content1, tpt = tpt1)
         case lit @ Literal(Constant(tpe: Type)) =>
           cpy.Literal(lit)(Constant(mapType(tpe)))
         case tree1 =>

compiler/src/dotty/tools/dotc/ast/Trees.scala

@@ -503,6 +503,11 @@ object Trees {
     def forwardTo: Tree[T] = fun
   }
 
+  object GenericApply:
+    def unapply[T >: Untyped](tree: Tree[T]): Option[(Tree[T], List[Tree[T]])] = tree match
+      case tree: GenericApply[T] => Some((tree.fun, tree.args))
+      case _ => None
+
   /** The kind of application */
   enum ApplyKind:
     case Regular      // r.f(x)
@@ -525,8 +530,6 @@ object Trees {
       attachmentOrElse(untpd.KindOfApply, ApplyKind.Regular)
   }
 
-
-
   /** fun[args] */
   case class TypeApply[-T >: Untyped] private[ast] (fun: Tree[T], args: List[Tree[T]])(implicit @constructorOnly src: SourceFile)
     extends GenericApply[T] {
@@ -972,10 +975,16 @@ object Trees {
   def genericEmptyValDef[T >: Untyped]: ValDef[T]       = theEmptyValDef.asInstanceOf[ValDef[T]]
   def genericEmptyTree[T >: Untyped]: Thicket[T]        = theEmptyTree.asInstanceOf[Thicket[T]]
 
-  /** Tree that replaces a splice in pickled quotes.
-   *  It is only used when picking quotes (Will never be in a TASTy file).
+  /** Tree that replaces a level 1 splices in pickled (level 0) quotes.
+   *  It is only used when picking quotes (will never be in a TASTy file).
+   *
+   *  @param isTermHole If this hole is a term, otherwise it is a type hole.
+   *  @param idx The index of the hole in it's enclosing level 0 quote.
+   *  @param args The arguments of the splice to compute its content
+   *  @param content Lambda that computes the content of the hole. This tree is empty when in a quote pickle.
+   *  @param tpt Type of the hole
    */
-  case class Hole[-T >: Untyped](isTermHole: Boolean, idx: Int, args: List[Tree[T]])(implicit @constructorOnly src: SourceFile) extends Tree[T] {
+  case class Hole[-T >: Untyped](isTermHole: Boolean, idx: Int, args: List[Tree[T]], content: Tree[T], tpt: Tree[T])(implicit @constructorOnly src: SourceFile) extends Tree[T] {
     type ThisTree[-T >: Untyped] <: Hole[T]
     override def isTerm: Boolean = isTermHole
     override def isType: Boolean = !isTermHole
@@ -1331,6 +1340,10 @@ object Trees {
         case tree: Thicket if (trees eq tree.trees) => tree
         case _ => finalize(tree, untpd.Thicket(trees)(sourceFile(tree)))
       }
+      def Hole(tree: Tree)(isTerm: Boolean, idx: Int, args: List[Tree], content: Tree, tpt: Tree)(using Context): Hole = tree match {
+        case tree: Hole if isTerm == tree.isTerm && idx == tree.idx && args.eq(tree.args) && content.eq(tree.content) && content.eq(tree.content) => tree
+        case _ => finalize(tree, untpd.Hole(isTerm, idx, args, content, tpt)(sourceFile(tree)))
+      }
 
       // Copier methods with default arguments; these demand that the original tree
       // is of the same class as the copy. We only include trees with more than 2 elements here.
@@ -1352,6 +1365,9 @@ object Trees {
         TypeDef(tree: Tree)(name, rhs)
       def Template(tree: Template)(constr: DefDef = tree.constr, parents: List[Tree] = tree.parents, derived: List[untpd.Tree] = tree.derived, self: ValDef = tree.self, body: LazyTreeList = tree.unforcedBody)(using Context): Template =
         Template(tree: Tree)(constr, parents, derived, self, body)
+      def Hole(tree: Hole)(isTerm: Boolean = tree.isTerm, idx: Int = tree.idx, args: List[Tree] = tree.args, content: Tree = tree.content, tpt: Tree = tree.tpt)(using Context): Hole =
+        Hole(tree: Tree)(isTerm, idx, args, content, tpt)
+
     }
 
     /** Hook to indicate that a transform of some subtree should be skipped */
@@ -1481,6 +1497,8 @@ object Trees {
             case Thicket(trees) =>
               val trees1 = transform(trees)
               if (trees1 eq trees) tree else Thicket(trees1)
+            case tree @ Hole(_, _, args, content, tpt) =>
+              cpy.Hole(tree)(args = transform(args), content = transform(content), tpt = transform(tpt))
             case _ =>
               transformMoreCases(tree)
           }
@@ -1620,8 +1638,8 @@ object Trees {
               this(this(x, arg), annot)
             case Thicket(ts) =>
               this(x, ts)
-            case Hole(_, _, args) =>
-              this(x, args)
+            case Hole(_, _, args, content, tpt) =>
+              this(this(this(x, args), content), tpt)
             case _ =>
               foldMoreCases(x, tree)
           }

compiler/src/dotty/tools/dotc/ast/tpd.scala

@@ -377,6 +377,9 @@ object tpd extends Trees.Instance[Type] with TypedTreeInfo {
   def Throw(expr: Tree)(using Context): Tree =
     ref(defn.throwMethod).appliedTo(expr)
 
+  def Hole(isTermHole: Boolean, idx: Int, args: List[Tree], content: Tree, tpt: Tree)(using Context): Hole =
+    ta.assignType(untpd.Hole(isTermHole, idx, args, content, tpt), tpt)
+
   // ------ Making references ------------------------------------------------------
 
   def prefixIsElidable(tp: NamedType)(using Context): Boolean = {
@@ -1518,10 +1521,10 @@ object tpd extends Trees.Instance[Type] with TypedTreeInfo {
    *  @param tpe    the type of the elements of the resulting list.
    *
    */
-  def mkList(trees: List[Tree], tpe: Tree)(using Context): Tree =
+  def mkList(trees: List[Tree], tpt: Tree)(using Context): Tree =
     ref(defn.ListModule).select(nme.apply)
-      .appliedToTypeTree(tpe)
-      .appliedToVarargs(trees, tpe)
+      .appliedToTypeTree(tpt)
+      .appliedToVarargs(trees, tpt)
 
 
   protected def FunProto(args: List[Tree], resType: Type)(using Context) =

compiler/src/dotty/tools/dotc/ast/untpd.scala

@@ -411,6 +411,7 @@ object untpd extends Trees.Instance[Untyped] with UntypedTreeInfo {
   def Export(expr: Tree, selectors: List[ImportSelector])(implicit src: SourceFile): Export = new Export(expr, selectors)
   def PackageDef(pid: RefTree, stats: List[Tree])(implicit src: SourceFile): PackageDef = new PackageDef(pid, stats)
   def Annotated(arg: Tree, annot: Tree)(implicit src: SourceFile): Annotated = new Annotated(arg, annot)
+  def Hole(isTermHole: Boolean, idx: Int, args: List[Tree], content: Tree, tpt: Tree)(implicit src: SourceFile): Hole = new Hole(isTermHole, idx, args, content, tpt)
 
   // ------ Additional creation methods for untyped only -----------------
 

compiler/src/dotty/tools/dotc/core/Definitions.scala

@@ -791,10 +791,44 @@ class Definitions {
   @tu lazy val QuotedExprClass: ClassSymbol = requiredClass("scala.quoted.Expr")
 
   @tu lazy val QuotesClass: ClassSymbol = requiredClass("scala.quoted.Quotes")
+    @tu lazy val Quotes_reflect: Symbol = QuotesClass.requiredValue("reflect")
+      @tu lazy val Quotes_reflect_asTerm: Symbol = Quotes_reflect.requiredMethod("asTerm")
+      @tu lazy val Quotes_reflect_Apply: Symbol = Quotes_reflect.requiredValue("Apply")
+        @tu lazy val Quotes_reflect_Apply_apply: Symbol = Quotes_reflect_Apply.requiredMethod(nme.apply)
+      @tu lazy val Quotes_reflect_TypeApply: Symbol = Quotes_reflect.requiredValue("TypeApply")
+        @tu lazy val Quotes_reflect_TypeApply_apply: Symbol = Quotes_reflect_TypeApply.requiredMethod(nme.apply)
+      @tu lazy val Quotes_reflect_Assign: Symbol = Quotes_reflect.requiredValue("Assign")
+        @tu lazy val Quotes_reflect_Assign_apply: Symbol = Quotes_reflect_Assign.requiredMethod(nme.apply)
+      @tu lazy val Quotes_reflect_Inferred: Symbol = Quotes_reflect.requiredValue("Inferred")
+        @tu lazy val Quotes_reflect_Inferred_apply: Symbol = Quotes_reflect_Inferred.requiredMethod(nme.apply)
+      @tu lazy val Quotes_reflect_Literal: Symbol = Quotes_reflect.requiredValue("Literal")
+      @tu lazy val Quotes_reflect_Literal_apply: Symbol = Quotes_reflect_Literal.requiredMethod(nme.apply)
+      @tu lazy val Quotes_reflect_TreeMethods: Symbol = Quotes_reflect.requiredMethod("TreeMethods")
+        @tu lazy val Quotes_reflect_TreeMethods_asExpr: Symbol = Quotes_reflect_TreeMethods.requiredMethod("asExpr")
+      @tu lazy val Quotes_reflect_TypeRepr: Symbol = Quotes_reflect.requiredValue("TypeRepr")
+        @tu lazy val Quotes_reflect_TypeRepr_of: Symbol = Quotes_reflect_TypeRepr.requiredMethod("of")
+        @tu lazy val Quotes_reflect_TypeRepr_typeConstructorOf: Symbol = Quotes_reflect_TypeRepr.requiredMethod("typeConstructorOf")
+      @tu lazy val Quotes_reflect_TypeReprMethods: Symbol = Quotes_reflect.requiredValue("TypeReprMethods")
+        @tu lazy val Quotes_reflect_TypeReprMethods_asType: Symbol = Quotes_reflect_TypeReprMethods.requiredMethod("asType")
+      @tu lazy val Quotes_reflect_TypeTreeType: Symbol = Quotes_reflect.requiredType("TypeTree")
+      @tu lazy val Quotes_reflect_TermType: Symbol = Quotes_reflect.requiredType("Term")
+      @tu lazy val Quotes_reflect_BooleanConstant: Symbol = Quotes_reflect.requiredValue("BooleanConstant")
+      @tu lazy val Quotes_reflect_ByteConstant: Symbol = Quotes_reflect.requiredValue("ByteConstant")
+      @tu lazy val Quotes_reflect_ShortConstant: Symbol = Quotes_reflect.requiredValue("ShortConstant")
+      @tu lazy val Quotes_reflect_IntConstant: Symbol = Quotes_reflect.requiredValue("IntConstant")
+      @tu lazy val Quotes_reflect_LongConstant: Symbol = Quotes_reflect.requiredValue("LongConstant")
+      @tu lazy val Quotes_reflect_FloatConstant: Symbol = Quotes_reflect.requiredValue("FloatConstant")
+      @tu lazy val Quotes_reflect_DoubleConstant: Symbol = Quotes_reflect.requiredValue("DoubleConstant")
+      @tu lazy val Quotes_reflect_CharConstant: Symbol = Quotes_reflect.requiredValue("CharConstant")
+      @tu lazy val Quotes_reflect_StringConstant: Symbol = Quotes_reflect.requiredValue("StringConstant")
+      @tu lazy val Quotes_reflect_UnitConstant: Symbol = Quotes_reflect.requiredValue("UnitConstant")
+      @tu lazy val Quotes_reflect_NullConstant: Symbol = Quotes_reflect.requiredValue("NullConstant")
+      @tu lazy val Quotes_reflect_ClassOfConstant: Symbol = Quotes_reflect.requiredValue("ClassOfConstant")
+
 
   @tu lazy val QuoteUnpicklerClass: ClassSymbol = requiredClass("scala.quoted.runtime.QuoteUnpickler")
-    @tu lazy val QuoteUnpickler_unpickleExpr: Symbol = QuoteUnpicklerClass.requiredMethod("unpickleExpr")
-    @tu lazy val QuoteUnpickler_unpickleType: Symbol = QuoteUnpicklerClass.requiredMethod("unpickleType")
+    @tu lazy val QuoteUnpickler_unpickleExprV2: Symbol = QuoteUnpicklerClass.requiredMethod("unpickleExprV2")
+    @tu lazy val QuoteUnpickler_unpickleTypeV2: Symbol = QuoteUnpicklerClass.requiredMethod("unpickleTypeV2")
 
   @tu lazy val QuoteMatchingClass: ClassSymbol = requiredClass("scala.quoted.runtime.QuoteMatching")
     @tu lazy val QuoteMatching_ExprMatch: Symbol = QuoteMatchingClass.requiredMethod("ExprMatch")

compiler/src/dotty/tools/dotc/core/Phases.scala

@@ -201,7 +201,7 @@ object Phases {
     private var mySbtExtractDependenciesPhase: Phase = _
     private var myPicklerPhase: Phase = _
     private var myInliningPhase: Phase = _
-    private var myPickleQuotesPhase: Phase = _
+    private var mySplicingPhase: Phase = _
     private var myFirstTransformPhase: Phase = _
     private var myCollectNullableFieldsPhase: Phase = _
     private var myRefChecksPhase: Phase = _
@@ -224,7 +224,7 @@ object Phases {
     final def sbtExtractDependenciesPhase: Phase = mySbtExtractDependenciesPhase
     final def picklerPhase: Phase = myPicklerPhase
     final def inliningPhase: Phase = myInliningPhase
-    final def pickleQuotesPhase: Phase = myPickleQuotesPhase
+    final def splicingPhase: Phase = mySplicingPhase
     final def firstTransformPhase: Phase = myFirstTransformPhase
     final def collectNullableFieldsPhase: Phase = myCollectNullableFieldsPhase
     final def refchecksPhase: Phase = myRefChecksPhase
@@ -250,7 +250,7 @@ object Phases {
       mySbtExtractDependenciesPhase = phaseOfClass(classOf[sbt.ExtractDependencies])
       myPicklerPhase = phaseOfClass(classOf[Pickler])
       myInliningPhase = phaseOfClass(classOf[Inlining])
-      myPickleQuotesPhase = phaseOfClass(classOf[PickleQuotes])
+      mySplicingPhase = phaseOfClass(classOf[Splicing])
       myFirstTransformPhase = phaseOfClass(classOf[FirstTransform])
       myCollectNullableFieldsPhase = phaseOfClass(classOf[CollectNullableFields])
       myRefChecksPhase = phaseOfClass(classOf[RefChecks])
@@ -426,7 +426,7 @@ object Phases {
   def sbtExtractDependenciesPhase(using Context): Phase = ctx.base.sbtExtractDependenciesPhase
   def picklerPhase(using Context): Phase                = ctx.base.picklerPhase
   def inliningPhase(using Context): Phase               = ctx.base.inliningPhase
-  def pickleQuotesPhase(using Context): Phase           = ctx.base.pickleQuotesPhase
+  def splicingPhase(using Context): Phase               = ctx.base.splicingPhase
   def firstTransformPhase(using Context): Phase         = ctx.base.firstTransformPhase
   def refchecksPhase(using Context): Phase              = ctx.base.refchecksPhase
   def elimRepeatedPhase(using Context): Phase           = ctx.base.elimRepeatedPhase

compiler/src/dotty/tools/dotc/core/StagingContext.scala

@@ -8,11 +8,11 @@ import dotty.tools.dotc.transform.PCPCheckAndHeal
 
 object StagingContext {
 
-  /** A key to be used in a context property that tracks the quoteation level */
+  /** A key to be used in a context property that tracks the quotation level */
   private val QuotationLevel = new Property.Key[Int]
 
-  /** A key to be used in a context property that tracks the quoteation stack.
-   *  Stack containing the Quotes references recieved by the surrounding quotes.
+  /** A key to be used in a context property that tracks the quotation stack.
+   *  Stack containing the Quotes references received by the surrounding quotes.
    */
   private val QuotesStack = new Property.Key[List[tpd.Tree]]
 
@@ -26,7 +26,7 @@ object StagingContext {
   def quoteContext(using Context): Context =
     ctx.fresh.setProperty(QuotationLevel, level + 1)
 
-  /** Context with an incremented quotation level and pushes a refecence to a Quotes on the quote context stack */
+  /** Context with an incremented quotation level and pushes a reference to a Quotes on the quote context stack */
   def pushQuotes(qctxRef: tpd.Tree)(using Context): Context =
     val old = ctx.property(QuotesStack).getOrElse(List.empty)
     ctx.fresh.setProperty(QuotationLevel, level + 1)
@@ -43,7 +43,7 @@ object StagingContext {
     ctx.property(TaggedTypes).get
 
   /** Context with a decremented quotation level and pops the Some of top of the quote context stack or None if the stack is empty.
-   *  The quotation stack could be empty if we are in a top level splice or an eroneous splice directly witin a top level splice.
+   *  The quotation stack could be empty if we are in a top level splice or an erroneous splice directly within a top level splice.
    */
   def popQuotes()(using Context): (Option[tpd.Tree], Context) =
     val ctx1 = ctx.fresh.setProperty(QuotationLevel, level - 1)

compiler/src/dotty/tools/dotc/core/StdNames.scala

@@ -433,6 +433,7 @@ object StdNames {
     val common: N               = "common"
     val compiletime : N         = "compiletime"
     val conforms_ : N           = "$conforms"
+    val contents: N             = "contents"
     val copy: N                 = "copy"
     val currentMirror: N        = "currentMirror"
     val create: N               = "create"
@@ -486,6 +487,7 @@ object StdNames {
     val hash_ : N               = "hash"
     val head: N                 = "head"
     val higherKinds: N          = "higherKinds"
+    val idx: N                  = "idx"
     val identity: N             = "identity"
     val implicitConversions: N  = "implicitConversions"
     val implicitly: N           = "implicitly"
@@ -553,6 +555,7 @@ object StdNames {
     val productElementName: N   = "productElementName"
     val productIterator: N      = "productIterator"
     val productPrefix: N        = "productPrefix"
+    val quotes : N              = "quotes"
     val raw_ : N                = "raw"
     val refl: N                 = "refl"
     val reflect: N              = "reflect"

compiler/src/dotty/tools/dotc/core/tasty/TreePickler.scala

@@ -637,11 +637,11 @@ class TreePickler(pickler: TastyPickler) {
               pickleTree(hi)
               pickleTree(alias)
           }
-        case Hole(_, idx, args) =>
+        case Hole(_, idx, args, _, tpt) =>
           writeByte(HOLE)
           withLength {
             writeNat(idx)
-            pickleType(tree.tpe, richTypes = true)
+            pickleType(tpt.tpe, richTypes = true)
             args.foreach(pickleTree)
           }
       }

compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala

@@ -1323,7 +1323,7 @@ class TreeUnpickler(reader: TastyReader,
               val idx = readNat()
               val tpe = readType()
               val args = until(end)(readTerm())
-              Hole(true, idx, args).withType(tpe)
+              Hole(true, idx, args, EmptyTree, TypeTree(tpe)).withType(tpe)
             case _ =>
               readPathTerm()
           }
@@ -1357,7 +1357,7 @@ class TreeUnpickler(reader: TastyReader,
           val idx = readNat()
           val tpe = readType()
           val args = until(end)(readTerm())
-          Hole(false, idx, args).withType(tpe)
+          Hole(false, idx, args, EmptyTree, TypeTree(tpe)).withType(tpe)
         case _ =>
           if (isTypeTreeTag(nextByte)) readTerm()
           else {

compiler/src/dotty/tools/dotc/printing/RefinedPrinter.scala

@@ -697,10 +697,12 @@ class RefinedPrinter(_ctx: Context) extends PlainPrinter(_ctx) {
         "Thicket {" ~~ toTextGlobal(trees, "\n") ~~ "}"
       case MacroTree(call) =>
         keywordStr("macro ") ~ toTextGlobal(call)
-      case Hole(isTermHole, idx, args) =>
-        val (prefix, postfix) = if isTermHole then ("{{{ ", " }}}") else ("[[[ ", " ]]]")
+      case Hole(isTermHole, idx, args, content, tpt) =>
+        val (prefix, postfix) = if isTermHole then ("{{{", "}}}") else ("[[[", "]]]")
         val argsText = toTextGlobal(args, ", ")
-        prefix ~~ idx.toString ~~ "|" ~~ argsText ~~ postfix
+        val contentText = toTextGlobal(content)
+        val tptText = toTextGlobal(tpt)
+        prefix ~~ idx.toString ~~ "|" ~~ tptText ~~ "|" ~~ argsText ~~ "|" ~~ contentText ~~ postfix
       case _ =>
         tree.fallbackToText(this)
     }