Re-architecture quote pickling

Separate the logic that creates holes in quotes from the logic that
pickles the quotes. Holes are created in the `Splicer` phase and the
result of the transformation can be `-Ycheck`ed. Now, the `PickleQuotes`
phase only needs to extract the contents of the holes, pickle the quote
and put them into a call to `unpickleExprV2`/`unpickleTypeV2`.

We add `unpickleExprV2` to support some optimization in the encoding of
the pickled quote. Namely we removed an unnecessary lambda from the
arguments of the hole passed into the contents of the hole. By not
changing `unpickleExpr` the current compiler will be able to handle the
old encoding in binaries compiled with older compilers.
The `unpickleTypeV2` is just a version of `unpickleType` that does not
take the `termHole` parameter which is always `null`.
With `-Yscala-relese` 3.0 or 3.1, the compiler will generate calls to
the old `unpickleExpr`/`unpickleType`.

Fixes scala#8100
Fixes scala#12440
Fixes scala#13563

Support -Yscala-relese macros

Disable tests/pos-macros/backwardCompat-3.0

Fix issue with captured types at level 0

Author: nicolasstucki

compiler/src/dotty/tools/dotc/CompilationUnit.scala

@@ -47,15 +47,10 @@ class CompilationUnit protected (val source: SourceFile) {
   var needsMirrorSupport: Boolean = false
 
   /** Will be set to `true` if contains `Quote`.
-   *  The information is used in phase `Staging` in order to avoid traversing trees that need no transformations.
+   *  The information is used in phase `Staging`/`Splicing`/`PickleQuotes` in order to avoid traversing trees that need no transformations.
    */
   var needsStaging: Boolean = false
 
-  /** Will be set to `true` if contains `Quote` that needs to be pickled
-   *  The information is used in phase `PickleQuotes` in order to avoid traversing trees that need no transformations.
-   */
-  var needsQuotePickling: Boolean = false
-
   var suspended: Boolean = false
   var suspendedAtInliningPhase: Boolean = false
 
@@ -104,7 +99,6 @@ object CompilationUnit {
       val force = new Force
       force.traverse(unit1.tpdTree)
       unit1.needsStaging = force.containsQuote
-      unit1.needsQuotePickling = force.containsQuote
       unit1.needsInlining = force.containsInline
     }
     unit1

compiler/src/dotty/tools/dotc/Compiler.scala

@@ -54,6 +54,7 @@ class Compiler {
     List(new Inlining) ::           // Inline and execute macros
     List(new PostInlining) ::       // Add mirror support for inlined code
     List(new Staging) ::            // Check staging levels and heal staged types
+    List(new Splicing) ::           // Replace level 1 splices with holes
     List(new PickleQuotes) ::       // Turn quoted trees into explicit run-time data structures
     Nil
 

compiler/src/dotty/tools/dotc/ast/TreeTypeMap.scala

@@ -133,8 +133,11 @@ class TreeTypeMap(
           val bind1 = tmap.transformSub(bind)
           val expr1 = tmap.transform(expr)
           cpy.Labeled(labeled)(bind1, expr1)
-        case Hole(isTermHole, n, args) =>
-          Hole(isTermHole, n, args.mapConserve(transform)).withSpan(tree.span).withType(mapType(tree.tpe))
+        case tree @ Hole(_, _, args, content, tpt) =>
+          val args1 = args.mapConserve(transform)
+          val content1 = transform(content)
+          val tpt1 = transform(tpt)
+          cpy.Hole(tree)(args = args1, content = content1, tpt = tpt1)
         case lit @ Literal(Constant(tpe: Type)) =>
           cpy.Literal(lit)(Constant(mapType(tpe)))
         case tree1 =>

compiler/src/dotty/tools/dotc/ast/Trees.scala

@@ -503,6 +503,11 @@ object Trees {
     def forwardTo: Tree[T] = fun
   }
 
+  object GenericApply:
+    def unapply[T >: Untyped](tree: Tree[T]): Option[(Tree[T], List[Tree[T]])] = tree match
+      case tree: GenericApply[T] => Some((tree.fun, tree.args))
+      case _ => None
+
   /** The kind of application */
   enum ApplyKind:
     case Regular      // r.f(x)
@@ -525,8 +530,6 @@ object Trees {
       attachmentOrElse(untpd.KindOfApply, ApplyKind.Regular)
   }
 
-
-
   /** fun[args] */
   case class TypeApply[-T >: Untyped] private[ast] (fun: Tree[T], args: List[Tree[T]])(implicit @constructorOnly src: SourceFile)
     extends GenericApply[T] {
@@ -972,10 +975,16 @@ object Trees {
   def genericEmptyValDef[T >: Untyped]: ValDef[T]       = theEmptyValDef.asInstanceOf[ValDef[T]]
   def genericEmptyTree[T >: Untyped]: Thicket[T]        = theEmptyTree.asInstanceOf[Thicket[T]]
 
-  /** Tree that replaces a splice in pickled quotes.
-   *  It is only used when picking quotes (Will never be in a TASTy file).
+  /** Tree that replaces a level 1 splices in pickled (level 0) quotes.
+   *  It is only used when picking quotes (will never be in a TASTy file).
+   *
+   *  @param isTermHole If this hole is a term, otherwise it is a type hole.
+   *  @param idx The index of the hole in it's enclosing level 0 quote.
+   *  @param args The arguments of the splice to compute its content
+   *  @param content Lambda that computes the content of the hole. This tree is empty when in a quote pickle.
+   *  @param tpt Type of the hole
    */
-  case class Hole[-T >: Untyped](isTermHole: Boolean, idx: Int, args: List[Tree[T]])(implicit @constructorOnly src: SourceFile) extends Tree[T] {
+  case class Hole[-T >: Untyped](isTermHole: Boolean, idx: Int, args: List[Tree[T]], content: Tree[T], tpt: Tree[T])(implicit @constructorOnly src: SourceFile) extends Tree[T] {
     type ThisTree[-T >: Untyped] <: Hole[T]
     override def isTerm: Boolean = isTermHole
     override def isType: Boolean = !isTermHole
@@ -1331,6 +1340,10 @@ object Trees {
         case tree: Thicket if (trees eq tree.trees) => tree
         case _ => finalize(tree, untpd.Thicket(trees)(sourceFile(tree)))
       }
+      def Hole(tree: Tree)(isTerm: Boolean, idx: Int, args: List[Tree], content: Tree, tpt: Tree)(using Context): Hole = tree match {
+        case tree: Hole if isTerm == tree.isTerm && idx == tree.idx && args.eq(tree.args) && content.eq(tree.content) && content.eq(tree.content) => tree
+        case _ => finalize(tree, untpd.Hole(isTerm, idx, args, content, tpt)(sourceFile(tree)))
+      }
 
       // Copier methods with default arguments; these demand that the original tree
       // is of the same class as the copy. We only include trees with more than 2 elements here.
@@ -1352,6 +1365,9 @@ object Trees {
         TypeDef(tree: Tree)(name, rhs)
       def Template(tree: Template)(constr: DefDef = tree.constr, parents: List[Tree] = tree.parents, derived: List[untpd.Tree] = tree.derived, self: ValDef = tree.self, body: LazyTreeList = tree.unforcedBody)(using Context): Template =
         Template(tree: Tree)(constr, parents, derived, self, body)
+      def Hole(tree: Hole)(isTerm: Boolean = tree.isTerm, idx: Int = tree.idx, args: List[Tree] = tree.args, content: Tree = tree.content, tpt: Tree = tree.tpt)(using Context): Hole =
+        Hole(tree: Tree)(isTerm, idx, args, content, tpt)
+
     }
 
     /** Hook to indicate that a transform of some subtree should be skipped */
@@ -1481,6 +1497,8 @@ object Trees {
             case Thicket(trees) =>
               val trees1 = transform(trees)
               if (trees1 eq trees) tree else Thicket(trees1)
+            case tree @ Hole(_, _, args, content, tpt) =>
+              cpy.Hole(tree)(args = transform(args), content = transform(content), tpt = transform(tpt))
             case _ =>
               transformMoreCases(tree)
           }
@@ -1618,8 +1636,8 @@ object Trees {
               this(this(x, arg), annot)
             case Thicket(ts) =>
               this(x, ts)
-            case Hole(_, _, args) =>
-              this(x, args)
+            case Hole(_, _, args, content, tpt) =>
+              this(this(this(x, args), content), tpt)
             case _ =>
               foldMoreCases(x, tree)
           }

compiler/src/dotty/tools/dotc/ast/tpd.scala

@@ -374,6 +374,9 @@ object tpd extends Trees.Instance[Type] with TypedTreeInfo {
   def Throw(expr: Tree)(using Context): Tree =
     ref(defn.throwMethod).appliedTo(expr)
 
+  def Hole(isTermHole: Boolean, idx: Int, args: List[Tree], content: Tree, tpt: Tree)(using Context): Hole =
+    ta.assignType(untpd.Hole(isTermHole, idx, args, content, tpt), tpt)
+
   // ------ Making references ------------------------------------------------------
 
   def prefixIsElidable(tp: NamedType)(using Context): Boolean = {
@@ -1493,10 +1496,10 @@ object tpd extends Trees.Instance[Type] with TypedTreeInfo {
    *  @param tpe    the type of the elements of the resulting list.
    *
    */
-  def mkList(trees: List[Tree], tpe: Tree)(using Context): Tree =
+  def mkList(trees: List[Tree], tpt: Tree)(using Context): Tree =
     ref(defn.ListModule).select(nme.apply)
-      .appliedToTypeTree(tpe)
-      .appliedToVarargs(trees, tpe)
+      .appliedToTypeTree(tpt)
+      .appliedToVarargs(trees, tpt)
 
 
   protected def FunProto(args: List[Tree], resType: Type)(using Context) =

compiler/src/dotty/tools/dotc/ast/untpd.scala

@@ -411,6 +411,7 @@ object untpd extends Trees.Instance[Untyped] with UntypedTreeInfo {
   def Export(expr: Tree, selectors: List[ImportSelector])(implicit src: SourceFile): Export = new Export(expr, selectors)
   def PackageDef(pid: RefTree, stats: List[Tree])(implicit src: SourceFile): PackageDef = new PackageDef(pid, stats)
   def Annotated(arg: Tree, annot: Tree)(implicit src: SourceFile): Annotated = new Annotated(arg, annot)
+  def Hole(isTermHole: Boolean, idx: Int, args: List[Tree], content: Tree, tpt: Tree)(implicit src: SourceFile): Hole = new Hole(isTermHole, idx, args, content, tpt)
 
   // ------ Additional creation methods for untyped only -----------------
 

compiler/src/dotty/tools/dotc/core/Definitions.scala

@@ -787,10 +787,46 @@ class Definitions {
   @tu lazy val QuotedExprModule: Symbol = QuotedExprClass.companionModule
 
   @tu lazy val QuotesClass: ClassSymbol = requiredClass("scala.quoted.Quotes")
+    @tu lazy val Quotes_reflect: Symbol = QuotesClass.requiredValue("reflect")
+      @tu lazy val Quotes_reflect_asTerm: Symbol = Quotes_reflect.requiredMethod("asTerm")
+      @tu lazy val Quotes_reflect_Apply: Symbol = Quotes_reflect.requiredValue("Apply")
+        @tu lazy val Quotes_reflect_Apply_apply: Symbol = Quotes_reflect_Apply.requiredMethod(nme.apply)
+      @tu lazy val Quotes_reflect_TypeApply: Symbol = Quotes_reflect.requiredValue("TypeApply")
+        @tu lazy val Quotes_reflect_TypeApply_apply: Symbol = Quotes_reflect_TypeApply.requiredMethod(nme.apply)
+      @tu lazy val Quotes_reflect_Assign: Symbol = Quotes_reflect.requiredValue("Assign")
+        @tu lazy val Quotes_reflect_Assign_apply: Symbol = Quotes_reflect_Assign.requiredMethod(nme.apply)
+      @tu lazy val Quotes_reflect_Inferred: Symbol = Quotes_reflect.requiredValue("Inferred")
+        @tu lazy val Quotes_reflect_Inferred_apply: Symbol = Quotes_reflect_Inferred.requiredMethod(nme.apply)
+      @tu lazy val Quotes_reflect_Literal: Symbol = Quotes_reflect.requiredValue("Literal")
+      @tu lazy val Quotes_reflect_Literal_apply: Symbol = Quotes_reflect_Literal.requiredMethod(nme.apply)
+      @tu lazy val Quotes_reflect_TreeMethods: Symbol = Quotes_reflect.requiredMethod("TreeMethods")
+        @tu lazy val Quotes_reflect_TreeMethods_asExpr: Symbol = Quotes_reflect_TreeMethods.requiredMethod("asExpr")
+      @tu lazy val Quotes_reflect_TypeRepr: Symbol = Quotes_reflect.requiredValue("TypeRepr")
+        @tu lazy val Quotes_reflect_TypeRepr_of: Symbol = Quotes_reflect_TypeRepr.requiredMethod("of")
+        @tu lazy val Quotes_reflect_TypeRepr_typeConstructorOf: Symbol = Quotes_reflect_TypeRepr.requiredMethod("typeConstructorOf")
+      @tu lazy val Quotes_reflect_TypeReprMethods: Symbol = Quotes_reflect.requiredValue("TypeReprMethods")
+        @tu lazy val Quotes_reflect_TypeReprMethods_asType: Symbol = Quotes_reflect_TypeReprMethods.requiredMethod("asType")
+      @tu lazy val Quotes_reflect_TypeTreeType: Symbol = Quotes_reflect.requiredType("TypeTree")
+      @tu lazy val Quotes_reflect_TermType: Symbol = Quotes_reflect.requiredType("Term")
+      @tu lazy val Quotes_reflect_BooleanConstant: Symbol = Quotes_reflect.requiredValue("BooleanConstant")
+      @tu lazy val Quotes_reflect_ByteConstant: Symbol = Quotes_reflect.requiredValue("ByteConstant")
+      @tu lazy val Quotes_reflect_ShortConstant: Symbol = Quotes_reflect.requiredValue("ShortConstant")
+      @tu lazy val Quotes_reflect_IntConstant: Symbol = Quotes_reflect.requiredValue("IntConstant")
+      @tu lazy val Quotes_reflect_LongConstant: Symbol = Quotes_reflect.requiredValue("LongConstant")
+      @tu lazy val Quotes_reflect_FloatConstant: Symbol = Quotes_reflect.requiredValue("FloatConstant")
+      @tu lazy val Quotes_reflect_DoubleConstant: Symbol = Quotes_reflect.requiredValue("DoubleConstant")
+      @tu lazy val Quotes_reflect_CharConstant: Symbol = Quotes_reflect.requiredValue("CharConstant")
+      @tu lazy val Quotes_reflect_StringConstant: Symbol = Quotes_reflect.requiredValue("StringConstant")
+      @tu lazy val Quotes_reflect_UnitConstant: Symbol = Quotes_reflect.requiredValue("UnitConstant")
+      @tu lazy val Quotes_reflect_NullConstant: Symbol = Quotes_reflect.requiredValue("NullConstant")
+      @tu lazy val Quotes_reflect_ClassOfConstant: Symbol = Quotes_reflect.requiredValue("ClassOfConstant")
+
 
   @tu lazy val QuoteUnpicklerClass: ClassSymbol = requiredClass("scala.quoted.runtime.QuoteUnpickler")
     @tu lazy val QuoteUnpickler_unpickleExpr: Symbol = QuoteUnpicklerClass.requiredMethod("unpickleExpr")
+    @tu lazy val QuoteUnpickler_unpickleExprV2: Symbol = QuoteUnpicklerClass.requiredMethod("unpickleExprV2")
     @tu lazy val QuoteUnpickler_unpickleType: Symbol = QuoteUnpicklerClass.requiredMethod("unpickleType")
+    @tu lazy val QuoteUnpickler_unpickleTypeV2: Symbol = QuoteUnpicklerClass.requiredMethod("unpickleTypeV2")
 
   @tu lazy val QuoteMatchingClass: ClassSymbol = requiredClass("scala.quoted.runtime.QuoteMatching")
     @tu lazy val QuoteMatching_ExprMatch: Symbol = QuoteMatchingClass.requiredMethod("ExprMatch")

compiler/src/dotty/tools/dotc/core/Phases.scala

@@ -201,7 +201,7 @@ object Phases {
     private var mySbtExtractDependenciesPhase: Phase = _
     private var myPicklerPhase: Phase = _
     private var myInliningPhase: Phase = _
-    private var myPickleQuotesPhase: Phase = _
+    private var mySplicingPhase: Phase = _
     private var myFirstTransformPhase: Phase = _
     private var myCollectNullableFieldsPhase: Phase = _
     private var myRefChecksPhase: Phase = _
@@ -224,7 +224,7 @@ object Phases {
     final def sbtExtractDependenciesPhase: Phase = mySbtExtractDependenciesPhase
     final def picklerPhase: Phase = myPicklerPhase
     final def inliningPhase: Phase = myInliningPhase
-    final def pickleQuotesPhase: Phase = myPickleQuotesPhase
+    final def splicingPhase: Phase = mySplicingPhase
     final def firstTransformPhase: Phase = myFirstTransformPhase
     final def collectNullableFieldsPhase: Phase = myCollectNullableFieldsPhase
     final def refchecksPhase: Phase = myRefChecksPhase
@@ -250,7 +250,7 @@ object Phases {
       mySbtExtractDependenciesPhase = phaseOfClass(classOf[sbt.ExtractDependencies])
       myPicklerPhase = phaseOfClass(classOf[Pickler])
       myInliningPhase = phaseOfClass(classOf[Inlining])
-      myPickleQuotesPhase = phaseOfClass(classOf[PickleQuotes])
+      mySplicingPhase = phaseOfClass(classOf[Splicing])
       myFirstTransformPhase = phaseOfClass(classOf[FirstTransform])
       myCollectNullableFieldsPhase = phaseOfClass(classOf[CollectNullableFields])
       myRefChecksPhase = phaseOfClass(classOf[RefChecks])
@@ -426,7 +426,7 @@ object Phases {
   def sbtExtractDependenciesPhase(using Context): Phase = ctx.base.sbtExtractDependenciesPhase
   def picklerPhase(using Context): Phase                = ctx.base.picklerPhase
   def inliningPhase(using Context): Phase               = ctx.base.inliningPhase
-  def pickleQuotesPhase(using Context): Phase           = ctx.base.pickleQuotesPhase
+  def splicingPhase(using Context): Phase               = ctx.base.splicingPhase
   def firstTransformPhase(using Context): Phase         = ctx.base.firstTransformPhase
   def refchecksPhase(using Context): Phase              = ctx.base.refchecksPhase
   def elimRepeatedPhase(using Context): Phase           = ctx.base.elimRepeatedPhase

compiler/src/dotty/tools/dotc/core/StagingContext.scala

@@ -13,11 +13,11 @@ import scala.collection.mutable
 
 object StagingContext {
 
-  /** A key to be used in a context property that tracks the quoteation level */
+  /** A key to be used in a context property that tracks the quotation level */
   private val QuotationLevel = new Property.Key[Int]
 
-  /** A key to be used in a context property that tracks the quoteation stack.
-   *  Stack containing the Quotes references recieved by the surrounding quotes.
+  /** A key to be used in a context property that tracks the quotation stack.
+   *  Stack containing the Quotes references received by the surrounding quotes.
    */
   private val QuotesStack = new Property.Key[List[tpd.Tree]]
 
@@ -31,7 +31,7 @@ object StagingContext {
   def quoteContext(using Context): Context =
     ctx.fresh.setProperty(QuotationLevel, level + 1)
 
-  /** Context with an incremented quotation level and pushes a refecence to a Quotes on the quote context stack */
+  /** Context with an incremented quotation level and pushes a reference to a Quotes on the quote context stack */
   def pushQuotes(qctxRef: tpd.Tree)(using Context): Context =
     val old = ctx.property(QuotesStack).getOrElse(List.empty)
     ctx.fresh.setProperty(QuotationLevel, level + 1)
@@ -48,7 +48,7 @@ object StagingContext {
     ctx.property(TaggedTypes).get
 
   /** Context with a decremented quotation level and pops the Some of top of the quote context stack or None if the stack is empty.
-   *  The quotation stack could be empty if we are in a top level splice or an eroneous splice directly witin a top level splice.
+   *  The quotation stack could be empty if we are in a top level splice or an erroneous splice directly within a top level splice.
    */
   def popQuotes()(using Context): (Option[tpd.Tree], Context) =
     val ctx1 = ctx.fresh.setProperty(QuotationLevel, level - 1)

compiler/src/dotty/tools/dotc/core/StdNames.scala

@@ -453,6 +453,7 @@ object StdNames {
     val common: N               = "common"
     val compiletime : N         = "compiletime"
     val conforms_ : N           = "$conforms"
+    val contents: N             = "contents"
     val copy: N                 = "copy"
     val currentMirror: N        = "currentMirror"
     val create: N               = "create"
@@ -506,6 +507,7 @@ object StdNames {
     val hash_ : N               = "hash"
     val head: N                 = "head"
     val higherKinds: N          = "higherKinds"
+    val idx: N                  = "idx"
     val identity: N             = "identity"
     val implicitConversions: N  = "implicitConversions"
     val implicitly: N           = "implicitly"
@@ -572,6 +574,7 @@ object StdNames {
     val productElementName: N   = "productElementName"
     val productIterator: N      = "productIterator"
     val productPrefix: N        = "productPrefix"
+    val quotes : N              = "quotes"
     val raw_ : N                = "raw"
     val refl: N                 = "refl"
     val reflect: N              = "reflect"

compiler/src/dotty/tools/dotc/core/tasty/TreePickler.scala

@@ -641,11 +641,11 @@ class TreePickler(pickler: TastyPickler) {
               pickleTree(hi)
               pickleTree(alias)
           }
-        case Hole(_, idx, args) =>
+        case Hole(_, idx, args, _, tpt) =>
           writeByte(HOLE)
           withLength {
             writeNat(idx)
-            pickleType(tree.tpe, richTypes = true)
+            pickleType(tpt.tpe, richTypes = true)
             args.foreach(pickleTree)
           }
       }

compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala

@@ -1288,7 +1288,7 @@ class TreeUnpickler(reader: TastyReader,
               val idx = readNat()
               val tpe = readType()
               val args = until(end)(readTerm())
-              Hole(true, idx, args).withType(tpe)
+              Hole(true, idx, args, EmptyTree, TypeTree(tpe)).withType(tpe)
             case _ =>
               readPathTerm()
           }
@@ -1324,7 +1324,7 @@ class TreeUnpickler(reader: TastyReader,
           val idx = readNat()
           val tpe = readType()
           val args = until(end)(readTerm())
-          Hole(false, idx, args).withType(tpe)
+          Hole(false, idx, args, EmptyTree, TypeTree(tpe)).withType(tpe)
         case _ =>
           if (isTypeTreeTag(nextByte)) readTerm()
           else {

compiler/src/dotty/tools/dotc/printing/RefinedPrinter.scala

@@ -699,10 +699,12 @@ class RefinedPrinter(_ctx: Context) extends PlainPrinter(_ctx) {
         "Thicket {" ~~ toTextGlobal(trees, "\n") ~~ "}"
       case MacroTree(call) =>
         keywordStr("macro ") ~ toTextGlobal(call)
-      case Hole(isTermHole, idx, args) =>
-        val (prefix, postfix) = if isTermHole then ("{{{ ", " }}}") else ("[[[ ", " ]]]")
+      case Hole(isTermHole, idx, args, content, tpt) =>
+        val (prefix, postfix) = if isTermHole then ("{{{", "}}}") else ("[[[", "]]]")
         val argsText = toTextGlobal(args, ", ")
-        prefix ~~ idx.toString ~~ "|" ~~ argsText ~~ postfix
+        val contentText = toTextGlobal(content)
+        val tptText = toTextGlobal(tpt)
+        prefix ~~ idx.toString ~~ "|" ~~ tptText ~~ "|" ~~ argsText ~~ "|" ~~ contentText ~~ postfix
       case _ =>
         tree.fallbackToText(this)
     }