Fix #7110: Pickle types of quote holes

To avoid wrong member selection when unpickling we must have qualifiers that have the same type as when it was pickled. This implies that we cannot expand the holes during unpickling as
the expansion of a qualifier could have a more precise type. To be able to unpickle a hole
without expanding it we must know its type, hence it must be pickled.

Once the pickled quote has been unpickled, we first replace all the type splices. After the
types are properly set we expand all the term holes.

Changes
* Types of holes are pickled in TASTy
* Pickled quotes are unpickled and then the holes are expanded
* Improved debugging info for unpickled quotes

Author: nicolasstucki

compiler/src/dotty/tools/dotc/ast/tpd.scala

@@ -42,7 +42,7 @@ object tpd extends Trees.Instance[Type] with TypedTreeInfo {
     Super(qual, if (mixName.isEmpty) untpd.EmptyTypeIdent else untpd.Ident(mixName), mixinClass)
 
   def Apply(fn: Tree, args: List[Tree])(implicit ctx: Context): Apply = {
-    assert(fn.isInstanceOf[RefTree] || fn.isInstanceOf[GenericApply[_]] || fn.isInstanceOf[Inlined])
+    assert(fn.isInstanceOf[RefTree] || fn.isInstanceOf[GenericApply[_]] || fn.isInstanceOf[Inlined] || fn.isInstanceOf[tasty.TreePickler.Hole])
     ta.assignType(untpd.Apply(fn, args), fn, args)
   }
 

compiler/src/dotty/tools/dotc/core/quoted/PickledQuotes.scala

@@ -61,44 +61,117 @@ object PickledQuotes {
   }.apply(tp)
 
   /** Unpickle the tree contained in the TastyExpr */
-  def unpickleExpr(tasty: PickledQuote, args: PickledArgs)(implicit ctx: Context): Tree = {
+  def unpickleExpr(tasty: PickledQuote, splices: PickledArgs)(implicit ctx: Context): Tree = {
     val tastyBytes = TastyString.unpickle(tasty)
-    val unpickled = unpickle(tastyBytes, args, isType = false)(ctx.addMode(Mode.ReadPositions))
-    /** Force unpickling of the tree, removes the spliced type `@quotedTypeTag type` definitions and dealiases references to `@quotedTypeTag type` */
-    val forceAndCleanArtefacts = new TreeMap {
-      override def transform(tree: tpd.Tree)(implicit ctx: Context): tpd.Tree = tree match {
-        case Block(stat :: rest, expr1) if stat.symbol.hasAnnotation(defn.InternalQuoted_QuoteTypeTagAnnot) =>
-          assert(rest.forall { case tdef: TypeDef => tdef.symbol.hasAnnotation(defn.InternalQuoted_QuoteTypeTagAnnot) })
-          transform(expr1)
-        case tree => super.transform(tree).withType(dealiasTypeTags(tree.tpe))
-      }
-    }
-    forceAndCleanArtefacts.transform(unpickled)
+    val unpickled = unpickle(tastyBytes, splices, isType = false)(ctx.addMode(Mode.ReadPositions))
+    val Inlined(call, Nil, expnasion) = unpickled
+    val inlineCtx = inlineContext(call)
+    val expansion1 = spliceTypes(expnasion, splices)(using inlineCtx)
+    val expansion2 = spliceTerms(expansion1, splices)(using inlineCtx)
+    cpy.Inlined(unpickled)(call, Nil, expansion2)
   }
 
   /** Unpickle the tree contained in the TastyType */
   def unpickleType(tasty: PickledQuote, args: PickledArgs)(implicit ctx: Context): Tree = {
     val tastyBytes = TastyString.unpickle(tasty)
     val unpickled = unpickle(tastyBytes, args, isType = true)(ctx.addMode(Mode.ReadPositions))
-    val tpt = unpickled match {
-      case Block(aliases, tpt) =>
-        // `@quoteTypeTag type` aliases are not required after unpickling.
-        // Type definitions are placeholders for type holes in the pickled quote, at this point
-        // those holes have been filled. As we already dealias al references to them in `dealiasTypeTags`
-        // there is no need to keep their definitions in the tree. As artifacts of quote reification
-        // they also do not have a meaningful position in the source.
-        val aliases1 = aliases.filter(!_.symbol.hasAnnotation(defn.InternalQuoted_QuoteTypeTagAnnot))
-        seq(aliases1, tpt)
-      case tpt => tpt
+    spliceTypes(unpickled, args)
+  }
+
+  /** Replace all term holes with the spliced terms */
+  private def spliceTerms(tree: Tree, splices: PickledArgs)(using Context): Tree = {
+    val evaluateHoles = new TreeMap {
+      override def transform(tree: tpd.Tree)(implicit ctx: Context): tpd.Tree = tree match {
+        case Hole(isTerm, idx, args) =>
+          val reifiedArgs = args.map { arg =>
+            if (arg.isTerm) (qctx: scala.quoted.QuoteContext) ?=> new TastyTreeExpr(arg, QuoteContext.scopeId)
+            else new TreeType(arg, QuoteContext.scopeId)
+          }
+          if isTerm then
+            val splice1 = splices(idx).asInstanceOf[Seq[Any] => scala.quoted.QuoteContext ?=> quoted.Expr[?]]
+            val quotedExpr = splice1(reifiedArgs)(using dotty.tools.dotc.quoted.QuoteContext())
+            val filled = PickledQuotes.quotedExprToTree(quotedExpr)
+
+            // We need to make sure a hole is created with the source file of the surrounding context, even if
+            // it filled with contents a different source file.
+            if filled.source == ctx.source then filled
+            else filled.cloneIn(ctx.source).withSpan(tree.span)
+          else
+            // Replaces type holes generated by ReifyQuotes (non-spliced types).
+            // These are types defined in a quote and used at the same level in a nested quote.
+            val quotedType = splices(idx).asInstanceOf[Seq[Any] => quoted.Type[?]](reifiedArgs)
+            PickledQuotes.quotedTypeToTree(quotedType)
+        case tree: Select =>
+          // Retain selected members
+          val qual = transform(tree.qualifier)
+          qual.select(tree.symbol).withSpan(tree.span)
+
+        case tree =>
+          if tree.isDef then
+            tree.symbol.annotations = tree.symbol.annotations.map {
+              annot => annot.derivedAnnotation(transform(annot.tree))
+            }
+          end if
+
+         val tree1 = super.transform(tree)
+         tree1.withType(mapAnnots(tree1.tpe))
+      }
+
+      // Evaluate holes in type annotations
+      private val mapAnnots = new TypeMap {
+        override def apply(tp: Type): Type = {
+            tp match
+              case tp @ AnnotatedType(underlying, annot) =>
+                val underlying1 = this(underlying)
+                derivedAnnotatedType(tp, underlying1, annot.derivedAnnotation(transform(annot.tree)))
+              case _ => mapOver(tp)
+        }
+      }
     }
-    tpt.withType(dealiasTypeTags(tpt.tpe))
+    val tree1 = evaluateHoles.transform(tree)
+    quotePickling.println(i"**** evaluated quote\n$tree1")
+    tree1
+  }
+
+  /** Replace all type holes generated with the spliced types */
+  private def spliceTypes(tree: Tree, splices: PickledArgs)(using Context): Tree = {
+    tree match
+      case Block(stat :: rest, expr1) if stat.symbol.hasAnnotation(defn.InternalQuoted_QuoteTypeTagAnnot) =>
+        val typeSpliceMap = (stat :: rest).iterator.map {
+          case tdef: TypeDef =>
+            assert(tdef.symbol.hasAnnotation(defn.InternalQuoted_QuoteTypeTagAnnot))
+            val tree = tdef.rhs match
+              case TypeBoundsTree(_, Hole(_, idx, args), _) =>
+                val quotedType = splices(idx).asInstanceOf[Seq[Any] => quoted.Type[?]](args)
+                PickledQuotes.quotedTypeToTree(quotedType)
+              case TypeBoundsTree(_, tpt, _) =>
+                tpt
+            (tdef.symbol, tree.tpe)
+        }.toMap
+        class ReplaceSplicedTyped extends TypeMap() {
+          override def apply(tp: Type): Type = {
+            val tp1 = tp match {
+              case tp: TypeRef =>
+                typeSpliceMap.get(tp.symbol) match
+                  case Some(t) if tp.typeSymbol.hasAnnotation(defn.InternalQuoted_QuoteTypeTagAnnot) => t
+                  case None => tp
+              case _ => tp
+            }
+            mapOver(tp1)
+          }
+        }
+        val expansion2 = new TreeTypeMap(new ReplaceSplicedTyped).transform(expr1)
+        quotePickling.println(i"**** typed quote\n${expansion2.show}")
+        expansion2
+      case _ =>
+        tree
   }
 
   // TASTY picklingtests/pos/quoteTest.scala
 
   /** Pickle tree into it's TASTY bytes s*/
   private def pickle(tree: Tree)(implicit ctx: Context): Array[Byte] = {
-    quotePickling.println(i"**** pickling quote of \n${tree.show}")
+    quotePickling.println(i"**** pickling quote of\n$tree")
     val pickler = new TastyPickler(defn.RootClass)
     val treePkl = pickler.treePkl
     treePkl.pickle(tree :: Nil)
@@ -122,7 +195,13 @@ object PickledQuotes {
     unpickler.enter(Set.empty)
 
     val tree = unpickler.tree
-    quotePickling.println(i"**** unpickle quote ${tree.show}")
+
+    // Make sure trees and positions are fully loaded
+    new TreeTraverser {
+      def traverse(tree: Tree)(implicit ctx: Context): Unit = traverseChildren(tree)
+    }.traverse(tree)
+
+    quotePickling.println(i"**** unpickled quote\n$tree")
     tree
   }
 

compiler/src/dotty/tools/dotc/core/tasty/TreePickler.scala

@@ -26,7 +26,8 @@ object TreePickler {
     override def isTerm: Boolean = isTermHole
     override def isType: Boolean = !isTermHole
     override def fallbackToText(printer: Printer): Text =
-      s"[[$idx|" ~~ printer.toTextGlobal(args, ", ") ~~ "]]"
+      if isTermHole then s"{{{ $idx |" ~~ printer.toTextGlobal(tpe) ~~ "|" ~~ printer.toTextGlobal(args, ", ") ~~ "}}}"
+      else s"[[[ $idx |" ~~ printer.toTextGlobal(tpe) ~~ "|" ~~ printer.toTextGlobal(args, ", ") ~~ "]]]"
   }
 }
 
@@ -603,6 +604,7 @@ class TreePickler(pickler: TastyPickler) {
           writeByte(HOLE)
           withLength {
             writeNat(idx)
+            pickleType(tree.tpe, richTypes = true)
             args.foreach(pickleTree)
           }
       }

compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala

@@ -1224,7 +1224,10 @@ class TreeUnpickler(reader: TastyReader,
               val alias = if currentAddr == end then EmptyTree else readTpt()
               TypeBoundsTree(lo, hi, alias)
             case HOLE =>
-              readHole(end, isType = false)
+              val idx = readNat()
+              val tpe = readType()
+              val args = until(end)(readTerm())
+              TreePickler.Hole(true, idx, args).withType(tpe)
             case _ =>
               readPathTerm()
           }
@@ -1257,7 +1260,10 @@ class TreeUnpickler(reader: TastyReader,
         case HOLE =>
           readByte()
           val end = readEnd()
-          readHole(end, isType = true)
+          val idx = readNat()
+          val tpe = readType()
+          val args = until(end)(readTerm())
+          TreePickler.Hole(false, idx, args).withType(tpe)
         case _ =>
           if (isTypeTreeTag(nextByte)) readTerm()
           else {
@@ -1299,35 +1305,6 @@ class TreeUnpickler(reader: TastyReader,
       owner => new LazyReader(localReader, owner, ctx.mode, ctx.source, op)
     }
 
-    def readHole(end: Addr, isType: Boolean)(implicit ctx: Context): Tree = {
-      val idx = readNat()
-      val args = until(end)(readTerm())
-      val splice = splices(idx)
-      def wrap(arg: Tree) =
-        if (arg.isTerm) (qctx: scala.quoted.QuoteContext) ?=> new TastyTreeExpr(arg, QuoteContext.scopeId)
-        else new TreeType(arg, QuoteContext.scopeId)
-      val reifiedArgs = args.map(wrap)
-      val filled = if (isType) {
-        val quotedType = splice.asInstanceOf[Seq[Any] => quoted.Type[?]](reifiedArgs)
-        PickledQuotes.quotedTypeToTree(quotedType)
-      }
-      else {
-        val splice1 = splice.asInstanceOf[Seq[Any] => scala.quoted.QuoteContext ?=> quoted.Expr[?]]
-        val quotedExpr = splice1(reifiedArgs)(using dotty.tools.dotc.quoted.QuoteContext())
-        PickledQuotes.quotedExprToTree(quotedExpr)
-      }
-      // We need to make sure a hole is created with the source file of the surrounding context, even if
-      // it filled with contents a different source file. Otherwise nodes containing holes might end
-      // up without a position. PositionPickler makes sure that holes always get spans assigned,
-      // so we can just return the filler tree with the new source and no span here.
-      if (filled.source == ctx.source) filled
-      else {
-        val filled1 = filled.cloneIn(ctx.source)
-        filled1.span = NoSpan
-        filled1
-      }
-    }
-
 // ------ Setting positions ------------------------------------------------
 
     /** Pickled span for `addr`. */

compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala

@@ -331,7 +331,37 @@ class ReifyQuotes extends MacroTransform {
      */
     private def makeHole(isTermHole: Boolean, body: Tree, splices: List[Tree], tpe: Type)(implicit ctx: Context): Hole = {
       val idx = embedded.addTree(body, NoSymbol)
-      Hole(isTermHole, idx, splices).withType(tpe).asInstanceOf[Hole]
+
+      /** Remove references to local types that will not be defined in this quote */
+      def getTypeHoleType(using ctx: Context) = new TypeMap() {
+        override def apply(tp: Type): Type = tp match
+          case tp: TypeRef if tp.typeSymbol.isSplice =>
+            apply(tp.dealias)
+          case tp @ TypeRef(pre, _) if pre == NoPrefix || pre.termSymbol.isLocal =>
+            val hiBound = tp.typeSymbol.info match
+              case info @ ClassInfo(_, _, classParents, _, _) => classParents.reduce(_ & _)
+              case info => info.hiBound
+            apply(hiBound)
+          case tp =>
+            mapOver(tp)
+      }
+
+      /** Remove references to local types that will not be defined in this quote */
+      def getTermHoleType(using ctx: Context) = new TypeMap() {
+        override def apply(tp: Type): Type = tp match
+          case tp @ TypeRef(NoPrefix, _) if capturers.contains(tp.symbol) =>
+            // reference to term with a type defined in outer quote
+            getTypeHoleType(tp)
+          case tp @ TermRef(NoPrefix, _) if capturers.contains(tp.symbol) =>
+            // widen term refs to terms defined in outer quote
+            apply(tp.widenTermRefExpr)
+          case tp =>
+            mapOver(tp)
+      }
+
+      val holeType = if isTermHole then getTermHoleType(tpe) else getTypeHoleType(tpe)
+
+      Hole(isTermHole, idx, splices).withType(holeType).asInstanceOf[Hole]
     }
 
     override def transform(tree: Tree)(implicit ctx: Context): Tree =

tests/pos-macros/i7853/JsonEncoder_1.scala renamed to tests/disabled/pos-macros/i7853/JsonEncoder_1.scala

@@ -0,0 +1,15 @@
+import scala.quoted._
+
+object Macros {
+
+  inline def m[R](sym: Symantics[R]) : R = ${  mImpl[R]('{sym}) }
+
+  def mImpl[R: Type](using qctx: QuoteContext)(sym: Expr[Symantics[R]]): Expr[R] =  '{
+    $sym.Meth(42)
+  }
+}
+
+trait Symantics[R]  {
+  def Meth(exp: Int): R
+  def Meth(): R
+}

tests/pos-macros/i7853/SummonJsonEncoderTest_2.scala renamed to tests/disabled/pos-macros/i7853/SummonJsonEncoderTest_2.scala

@@ -0,0 +1,14 @@
+import scala.quoted._
+import Macros._
+
+object Test {
+  def main(args: Array[String]): Unit = {
+
+    val sym = new Symantics[Int] {
+      def Meth(exp: Int): Int = exp
+      def Meth(): Int = 42
+    }
+
+    val test = m[Int](sym)
+  }
+}

tests/pos-macros/i7853/Test_3.scala renamed to tests/disabled/pos-macros/i7853/Test_3.scala

@@ -0,0 +1,16 @@
+import scala.quoted._
+
+object Macros {
+
+  inline def m[T](sym: Symantics {type R = T}) : T = ${  mImpl[T]('{sym}) }
+
+  def mImpl[T: Type](using qctx: QuoteContext)(sym: Expr[Symantics { type R = T }]): Expr[T] =  '{
+    $sym.Meth(42)
+  }
+}
+
+trait Symantics {
+  type R
+  def Meth(exp: Int): R
+  def Meth(): R
+}

tests/pos-macros/i7110a/Macro_1.scala

@@ -0,0 +1,15 @@
+import scala.quoted._
+import Macros._
+
+object Test {
+  def main(args: Array[String]): Unit = {
+
+    val sym = new Symantics {
+      type R = Int
+      def Meth(exp: Int): Int = exp
+      def Meth(): Int = 42
+    }
+
+    val test = m(sym)
+  }
+}

tests/pos-macros/i7110a/Test_2.scala

@@ -0,0 +1,14 @@
+import scala.quoted._
+
+object Macros {
+
+  inline def m[R](sym: Symantics[R]) : R = ${  mImpl[R]('{sym}) }
+
+  def mImpl[R: Type](using qctx: QuoteContext)(sym: Expr[Symantics[R]]): Expr[R] =  '{
+    $sym.Meth(42)
+  }
+}
+
+trait Symantics[R]  {
+  def Meth(exp: Int): R
+}

tests/pos-macros/i7110b/Macro_1.scala

@@ -0,0 +1,16 @@
+import scala.quoted._
+import Macros._
+
+object Test {
+  def main(args: Array[String]): Unit = {
+
+    val sym = new Symantics2
+
+    val test = m[Int](sym)
+  }
+}
+
+class Symantics2 extends Symantics[Int] {
+  def Meth(exp: Int): Int = exp
+  def Meth(): Int = 42
+}

tests/pos-macros/i7110b/Test_2.scala

@@ -0,0 +1,14 @@
+import scala.quoted._
+
+object Macros {
+
+  inline def m(sym: Symantics) : Int = ${  mImpl('sym) }
+
+  def mImpl(using qctx: QuoteContext)(sym: Expr[Symantics]): Expr[Int] = '{
+    $sym.Meth(42)
+  }
+}
+
+trait Symantics  {
+  def Meth(exp: Int): Int
+}