Make sure captured types are unpickled as types

compiler/src/dotty/tools/dotc/ast/TreeTypeMap.scala

@@ -146,11 +146,12 @@ class TreeTypeMap(
           val bind1 = tmap.transformSub(bind)
           val expr1 = tmap.transform(expr)
           cpy.Labeled(labeled)(bind1, expr1)
-        case tree @ Hole(_, _, args, content, tpt) =>
+        case tree @ Hole(_, _, targs, args, content, tpt) =>
+          val targs1 = targs.mapConserve(transform)
           val args1 = args.mapConserve(transform)
           val content1 = transform(content)
           val tpt1 = transform(tpt)
-          cpy.Hole(tree)(args = args1, content = content1, tpt = tpt1)
+          cpy.Hole(tree)(targs = targs1, args = args1, content = content1, tpt = tpt1)
         case tree1 =>
           super.transform(tree1)
       }

compiler/src/dotty/tools/dotc/ast/Trees.scala

@@ -976,20 +976,42 @@ object Trees {
   def genericEmptyTree[T <: Untyped]: Thicket[T]        = theEmptyTree.asInstanceOf[Thicket[T]]
 
   /** Tree that replaces a level 1 splices in pickled (level 0) quotes.
-   *  It is only used when picking quotes (will never be in a TASTy file).
+   *  It is only used when encoding pickled quotes. These will be encoded
+   *  as PickledHole when pickled.
    *
    *  @param isTermHole If this hole is a term, otherwise it is a type hole.
    *  @param idx The index of the hole in it's enclosing level 0 quote.
-   *  @param args The arguments of the splice to compute its content
-   *  @param content Lambda that computes the content of the hole. This tree is empty when in a quote pickle.
+   *  @param targs The type arguments of the splice to compute its content
+   *  @param args The term (or type) arguments of the splice to compute its content
    *  @param tpt Type of the hole
+   *  @param content Lambda that computes the content of the hole. This tree is empty when in a quote pickle.
    */
-  case class Hole[+T <: Untyped](isTermHole: Boolean, idx: Int, args: List[Tree[T]], content: Tree[T], tpt: Tree[T])(implicit @constructorOnly src: SourceFile) extends Tree[T] {
+  case class Hole[+T <: Untyped](isTermHole: Boolean, idx: Int, targs: List[Tree[T]], args: List[Tree[T]], content: Tree[T], tpt: Tree[T])(implicit @constructorOnly src: SourceFile) extends Tree[T] {
     type ThisTree[+T <: Untyped] <: Hole[T]
     override def isTerm: Boolean = isTermHole
     override def isType: Boolean = !isTermHole
   }
 
+  /** Tree that replaces a level 1 splices in pickled (level 0) quotes.
+   *  It is only used when picking quotes (will never be in a TASTy file).
+   *
+   *  Hole created by this compile separate the targs from the args. Holes
+   *  generated with 3.0-3.3 contain all type args and targs in any order in
+   *  a single list. For backwards compatibility we read holes from tasty as
+   *  if they had no targs and have only args. Therefore the args may contain
+   *  type trees.
+   *
+   *  @param isTermHole If this hole is a term, otherwise it is a type hole.
+   *  @param idx The index of the hole in it's enclosing level 0 quote.
+   *  @param args The term (or type) arguments of the splice to compute its content
+   *  @param tpt Type of the hole
+   */
+  case class PickledHole[+T <: Untyped](isTermHole: Boolean, idx: Int, args: List[Tree[T]], tpt: Tree[T])(implicit @constructorOnly src: SourceFile) extends Tree[T] {
+    type ThisTree[+T <: Untyped] <: PickledHole[T]
+    override def isTerm: Boolean = isTermHole
+    override def isType: Boolean = !isTermHole
+  }
+
   def flatten[T <: Untyped](trees: List[Tree[T]]): List[Tree[T]] = {
     def recur(buf: ListBuffer[Tree[T]] | Null, remaining: List[Tree[T]]): ListBuffer[Tree[T]] | Null =
       remaining match {
@@ -1112,6 +1134,7 @@ object Trees {
     type Thicket = Trees.Thicket[T]
 
     type Hole = Trees.Hole[T]
+    type PickledHole = Trees.PickledHole[T]
 
     @sharable val EmptyTree: Thicket = genericEmptyTree
     @sharable val EmptyValDef: ValDef = genericEmptyValDef
@@ -1337,9 +1360,13 @@ object Trees {
         case tree: Thicket if (trees eq tree.trees) => tree
         case _ => finalize(tree, untpd.Thicket(trees)(sourceFile(tree)))
       }
-      def Hole(tree: Tree)(isTerm: Boolean, idx: Int, args: List[Tree], content: Tree, tpt: Tree)(using Context): Hole = tree match {
-        case tree: Hole if isTerm == tree.isTerm && idx == tree.idx && args.eq(tree.args) && content.eq(tree.content) && content.eq(tree.content) => tree
-        case _ => finalize(tree, untpd.Hole(isTerm, idx, args, content, tpt)(sourceFile(tree)))
+      def Hole(tree: Tree)(isTerm: Boolean, idx: Int, targs: List[Tree], args: List[Tree], content: Tree, tpt: Tree)(using Context): Hole = tree match {
+        case tree: Hole if isTerm == tree.isTerm && idx == tree.idx && targs.eq(tree.targs) && args.eq(tree.args) && content.eq(tree.content) && tpt.eq(tree.tpt) => tree
+        case _ => finalize(tree, untpd.Hole(isTerm, idx, targs, args, content, tpt)(sourceFile(tree)))
+      }
+      def PickledHole(tree: Tree)(isTerm: Boolean, idx: Int, args: List[Tree], tpt: Tree)(using Context): PickledHole = tree match {
+        case tree: PickledHole if isTerm == tree.isTerm && idx == tree.idx && args.eq(tree.args) && tpt.eq(tree.tpt) => tree
+        case _ => finalize(tree, untpd.PickledHole(isTerm, idx, args, tpt)(sourceFile(tree)))
       }
 
       // Copier methods with default arguments; these demand that the original tree
@@ -1362,8 +1389,10 @@ object Trees {
         TypeDef(tree: Tree)(name, rhs)
       def Template(tree: Template)(using Context)(constr: DefDef = tree.constr, parents: List[Tree] = tree.parents, derived: List[untpd.Tree] = tree.derived, self: ValDef = tree.self, body: LazyTreeList = tree.unforcedBody): Template =
         Template(tree: Tree)(constr, parents, derived, self, body)
-      def Hole(tree: Hole)(isTerm: Boolean = tree.isTerm, idx: Int = tree.idx, args: List[Tree] = tree.args, content: Tree = tree.content, tpt: Tree = tree.tpt)(using Context): Hole =
-        Hole(tree: Tree)(isTerm, idx, args, content, tpt)
+      def Hole(tree: Hole)(isTerm: Boolean = tree.isTerm, idx: Int = tree.idx, targs: List[Tree] = tree.targs, args: List[Tree] = tree.args, content: Tree = tree.content, tpt: Tree = tree.tpt)(using Context): Hole =
+        Hole(tree: Tree)(isTerm, idx, targs, args, content, tpt)
+      def PickledHole(tree: PickledHole)(isTerm: Boolean = tree.isTerm, idx: Int = tree.idx, args: List[Tree] = tree.args, tpt: Tree = tree.tpt)(using Context): PickledHole =
+        PickledHole(tree: Tree)(isTerm, idx, args, tpt)
 
     }
 
@@ -1494,8 +1523,10 @@ object Trees {
             case Thicket(trees) =>
               val trees1 = transform(trees)
               if (trees1 eq trees) tree else Thicket(trees1)
-            case tree @ Hole(_, _, args, content, tpt) =>
-              cpy.Hole(tree)(args = transform(args), content = transform(content), tpt = transform(tpt))
+            case tree @ Hole(_, _, targs, args, content, tpt) =>
+              cpy.Hole(tree)(targs = transform(targs), args = transform(args), content = transform(content), tpt = transform(tpt))
+            case tree @ PickledHole(_, _, args, tpt) =>
+              cpy.PickledHole(tree)(args = transform(args), tpt = transform(tpt))
             case _ =>
               transformMoreCases(tree)
           }
@@ -1635,8 +1666,10 @@ object Trees {
               this(this(x, arg), annot)
             case Thicket(ts) =>
               this(x, ts)
-            case Hole(_, _, args, content, tpt) =>
-              this(this(this(x, args), content), tpt)
+            case Hole(_, _, targs, args, content, tpt) =>
+              this(this(this(this(x, targs), args), content), tpt)
+            case PickledHole(_, _, args, tpt) =>
+              this(this(x, args), tpt)
             case _ =>
               foldMoreCases(x, tree)
           }

compiler/src/dotty/tools/dotc/ast/tpd.scala

@@ -391,8 +391,11 @@ object tpd extends Trees.Instance[Type] with TypedTreeInfo {
   def Throw(expr: Tree)(using Context): Tree =
     ref(defn.throwMethod).appliedTo(expr)
 
-  def Hole(isTermHole: Boolean, idx: Int, args: List[Tree], content: Tree, tpt: Tree)(using Context): Hole =
-    ta.assignType(untpd.Hole(isTermHole, idx, args, content, tpt), tpt)
+  def Hole(isTermHole: Boolean, idx: Int, targs: List[Tree], args: List[Tree], content: Tree, tpt: Tree)(using Context): Hole =
+    ta.assignType(untpd.Hole(isTermHole, idx, targs, args, content, tpt), tpt)
+
+  // def PickledHole(isTermHole: Boolean, idx: Int, args: List[Tree], tpt: Tree)(using Context): PickledHole =
+  //   ta.assignType(untpd.PickledHole(isTermHole, idx, args, tpt), tpt)
 
   // ------ Making references ------------------------------------------------------
 

compiler/src/dotty/tools/dotc/ast/untpd.scala

@@ -426,7 +426,8 @@ object untpd extends Trees.Instance[Untyped] with UntypedTreeInfo {
   def Export(expr: Tree, selectors: List[ImportSelector])(implicit src: SourceFile): Export = new Export(expr, selectors)
   def PackageDef(pid: RefTree, stats: List[Tree])(implicit src: SourceFile): PackageDef = new PackageDef(pid, stats)
   def Annotated(arg: Tree, annot: Tree)(implicit src: SourceFile): Annotated = new Annotated(arg, annot)
-  def Hole(isTermHole: Boolean, idx: Int, args: List[Tree], content: Tree, tpt: Tree)(implicit src: SourceFile): Hole = new Hole(isTermHole, idx, args, content, tpt)
+  def Hole(isTermHole: Boolean, idx: Int, targs: List[Tree], args: List[Tree], content: Tree, tpt: Tree)(implicit src: SourceFile): Hole = new Hole(isTermHole, idx, targs, args, content, tpt)
+  def PickledHole(isTermHole: Boolean, idx: Int, args: List[Tree], tpt: Tree)(implicit src: SourceFile): PickledHole = new PickledHole(isTermHole, idx, args, tpt)
 
   // ------ Additional creation methods for untyped only -----------------
 

compiler/src/dotty/tools/dotc/core/tasty/TreePickler.scala

@@ -665,14 +665,14 @@ class TreePickler(pickler: TastyPickler) {
               pickleTree(hi)
               pickleTree(alias)
           }
-        case Hole(_, idx, args, _, tpt) =>
+        case Hole(_, idx, targs, args, _, tpt) =>
           writeByte(HOLE)
           withLength {
-            writeNat(idx)
-            pickleType(tpt.tpe, richTypes = true)
+            writeNat(0x0100_0000 | idx)
+            pickleTree(untpd.AppliedTypeTree(tpt, targs))
             args.foreach(pickleTree)
           }
-      }
+        }
       catch {
         case ex: TypeError =>
           report.error(ex.toMessage, tree.srcPos.focus)

compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala

@@ -1436,10 +1436,7 @@ class TreeUnpickler(reader: TastyReader,
               val alias = if currentAddr == end then EmptyTree else readTpt()
               createNullableTypeBoundsTree(lo, hi, alias)
             case HOLE =>
-              val idx = readNat()
-              val tpe = readType()
-              val args = until(end)(readTerm())
-              Hole(true, idx, args, EmptyTree, TypeTree(tpe)).withType(tpe)
+              readHole(end, isTerm = true)
             case _ =>
               readPathTerm()
           }
@@ -1469,11 +1466,7 @@ class TreeUnpickler(reader: TastyReader,
           Block(aliases, tpt)
         case HOLE =>
           readByte()
-          val end = readEnd()
-          val idx = readNat()
-          val tpe = readType()
-          val args = until(end)(readTerm())
-          Hole(false, idx, args, EmptyTree, TypeTree(tpe)).withType(tpe)
+          readHole(readEnd(), isTerm = false)
         case _ =>
           if (isTypeTreeTag(nextByte)) readTerm()
           else {
@@ -1517,6 +1510,18 @@ class TreeUnpickler(reader: TastyReader,
       owner => new LazyReader(localReader, owner, mode, source, op)
     }
 
+    def readHole(end: Addr, isTerm: Boolean)(using Context): Tree =
+      val idx = readNat()
+      idx >> 24 match
+        case 0 =>
+          val tpe = readType()
+          val args = until(end)(readTerm())
+          PickledHole(isTerm, idx, args, TypeTree(tpe)).withType(tpe)
+        case 1 =>
+          val AppliedTypeTree(tpt, targs) = readTerm(): @unchecked
+          val args = until(end)(readTerm())
+          PickledHole(isTerm, idx & 0x00FF_FFFF, targs ::: args, tpt).withType(tpt.tpe)
+
 // ------ Setting positions ------------------------------------------------
 
     /** Pickled span for `addr`. */

compiler/src/dotty/tools/dotc/printing/RefinedPrinter.scala

@@ -724,12 +724,13 @@ class RefinedPrinter(_ctx: Context) extends PlainPrinter(_ctx) {
         "Thicket {" ~~ toTextGlobal(trees, "\n") ~~ "}"
       case MacroTree(call) =>
         keywordStr("macro ") ~ toTextGlobal(call)
-      case Hole(isTermHole, idx, args, content, tpt) =>
+      case Hole(isTermHole, idx, targs, args, content, tpt) =>
         val (prefix, postfix) = if isTermHole then ("{{{", "}}}") else ("[[[", "]]]")
-        val argsText = toTextGlobal(args, ", ")
-        val contentText = toTextGlobal(content)
+        val targsText = ("[" ~ toTextGlobal(targs, ", ") ~ "]").provided(targs.nonEmpty)
+        val argsText =  ("(" ~ toTextGlobal(args, ", ") ~ ")").provided(args.nonEmpty)
         val tptText = toTextGlobal(tpt)
-        prefix ~~ idx.toString ~~ "|" ~~ tptText ~~ "|" ~~ argsText ~~ "|" ~~ contentText ~~ postfix
+        val contentText = ("|" ~~ toTextGlobal(content)).provided(content ne EmptyTree)
+        prefix ~~ idx.toString ~ targsText ~ argsText ~ ":" ~~ tptText ~~ contentText ~~ postfix
       case CapturingTypeTree(refs, parent) =>
         parent match
           case ImpureByNameTypeTree(bntpt) =>

compiler/src/dotty/tools/dotc/quoted/PickledQuotes.scala

@@ -100,7 +100,7 @@ object PickledQuotes {
   private def spliceTerms(tree: Tree, typeHole: TypeHole, termHole: ExprHole)(using Context): Tree = {
     def evaluateHoles = new TreeMap {
       override def transform(tree: tpd.Tree)(using Context): tpd.Tree = tree match {
-        case Hole(isTermHole, idx, args, _, _) =>
+        case PickledHole(isTermHole, idx, args, _) =>
           inContext(SpliceScope.contextWithNewSpliceScope(tree.sourcePos)) {
             if isTermHole then
               val quotedExpr = termHole match
@@ -165,15 +165,15 @@ object PickledQuotes {
             val tree = typeHole match
               case TypeHole.V1(evalHole) =>
                 tdef.rhs match
-                  case TypeBoundsTree(_, Hole(_, idx, args, _, _), _) =>
+                  case TypeBoundsTree(_, PickledHole(_, idx, args, _), _) =>
                     // To keep for backwards compatibility. In some older version holes where created in the bounds.
                     val quotedType = evalHole.nn.apply(idx, reifyTypeHoleArgs(args))
                     PickledQuotes.quotedTypeToTree(quotedType)
                   case TypeBoundsTree(_, tpt, _) =>
                     // To keep for backwards compatibility. In some older version we missed the creation of some holes.
                     tpt
               case TypeHole.V2(types) =>
-                val Hole(_, idx, _, _, _) = tdef.rhs: @unchecked
+                val PickledHole(_, idx, _, _) = tdef.rhs: @unchecked
                 PickledQuotes.quotedTypeToTree(types.nn.apply(idx))
             (tdef.symbol, tree.tpe)
         }.toMap

compiler/src/dotty/tools/dotc/transform/PickleQuotes.scala

@@ -126,7 +126,7 @@ class PickleQuotes extends MacroTransform {
       private val contents = List.newBuilder[Tree]
       override def transform(tree: tpd.Tree)(using Context): tpd.Tree =
         tree match
-          case tree @ Hole(isTerm, _, _, content, _) =>
+          case tree @ Hole(isTerm, _, _, _, content, _) =>
             if !content.isEmpty then
               contents += content
             val holeType =

compiler/src/dotty/tools/dotc/transform/Splicing.scala

@@ -132,7 +132,7 @@ class Splicing extends MacroTransform:
             case None =>
               val holeIdx = numHoles
               numHoles += 1
-              val hole = tpd.Hole(false, holeIdx, Nil, ref(qual), TypeTree(tp))
+              val hole = tpd.Hole(false, holeIdx, Nil, Nil, ref(qual), TypeTree(tp))
               typeHoles.put(qual.symbol, hole)
               hole
           cpy.TypeDef(tree)(rhs = hole)
@@ -188,22 +188,25 @@ class Splicing extends MacroTransform:
    *  ```
    */
   private class SpliceTransformer(spliceOwner: Symbol, isCaptured: Symbol => Boolean) extends Transformer:
-    private var refBindingMap = mutable.Map.empty[Symbol, (Tree, Symbol)]
+    private val typeBindingMap = mutable.Map.empty[Symbol, (Tree, Symbol)]
+    private val termBindingMap = mutable.Map.empty[Symbol, (Tree, Symbol)]
     /** Reference to the `Quotes` instance of the current level 1 splice */
     private var quotes: Tree | Null = null // TODO: add to the context
     private var healedTypes: QuoteTypeTags | Null = null // TODO: add to the context
 
     def transformSplice(tree: tpd.Tree, tpe: Type, holeIdx: Int)(using Context): tpd.Tree =
       assert(level == 0)
       val newTree = transform(tree)
-      val (refs, bindings) = refBindingMap.values.toList.unzip
+      val (typeRefs, typeBindings) = typeBindingMap.values.toList.unzip
+      val (termRefs, termBindings) = termBindingMap.values.toList.unzip
+      val bindings = typeBindings ::: termBindings
       val bindingsTypes = bindings.map(_.termRef.widenTermRefExpr)
       val methType = MethodType(bindingsTypes, newTree.tpe)
       val meth = newSymbol(spliceOwner, nme.ANON_FUN, Synthetic | Method, methType)
       val ddef = DefDef(meth, List(bindings), newTree.tpe, newTree.changeOwner(ctx.owner, meth))
       val fnType = defn.FunctionType(bindings.size, isContextual = false).appliedTo(bindingsTypes :+ newTree.tpe)
       val closure = Block(ddef :: Nil, Closure(Nil, ref(meth), TypeTree(fnType)))
-      tpd.Hole(true, holeIdx, refs, closure, TypeTree(tpe))
+      tpd.Hole(true, holeIdx, typeRefs, termRefs, closure, TypeTree(tpe))
 
     override def transform(tree: tpd.Tree)(using Context): tpd.Tree =
       tree match
@@ -348,7 +351,7 @@ class Splicing extends MacroTransform:
         Param,
         defn.QuotedExprClass.typeRef.appliedTo(tpe),
       )
-      val bindingSym = refBindingMap.getOrElseUpdate(tree.symbol, (tree, newBinding))._2
+      val bindingSym = termBindingMap.getOrElseUpdate(tree.symbol, (tree, newBinding))._2
       ref(bindingSym)
 
     private def newQuotedTypeClassBinding(tpe: Type)(using Context) =
@@ -361,7 +364,7 @@ class Splicing extends MacroTransform:
 
     private def capturedType(tree: Tree)(using Context): Symbol =
       val tpe = tree.tpe.widenTermRefExpr
-      val bindingSym = refBindingMap
+      val bindingSym = typeBindingMap
         .getOrElseUpdate(tree.symbol, (TypeTree(tree.tpe), newQuotedTypeClassBinding(tpe)))._2
       bindingSym
 
@@ -371,7 +374,7 @@ class Splicing extends MacroTransform:
       val capturePartTypes = new TypeMap {
         def apply(tp: Type) = tp match {
           case typeRef: TypeRef if containsCapturedType(typeRef) =>
-            val termRef = refBindingMap
+            val termRef = typeBindingMap
               .getOrElseUpdate(typeRef.symbol, (TypeTree(typeRef), newQuotedTypeClassBinding(typeRef)))._2.termRef
             val tagRef = healedTypes.nn.getTagRef(termRef)
             tagRef

compiler/src/dotty/tools/dotc/transform/TreeChecker.scala

@@ -653,19 +653,19 @@ object TreeChecker {
         super.typedPackageDef(tree)
 
     override def typedHole(tree: untpd.Hole, pt: Type)(using Context): Tree = {
-      val tree1 @ Hole(isTermHole, _, args, content, tpt) = super.typedHole(tree, pt): @unchecked
+      val tree1 @ Hole(isTermHole, _, targs, args, content, tpt) = super.typedHole(tree, pt): @unchecked
 
       // Check that we only add the captured type `T` instead of a more complex type like `List[T]`.
       // If we have `F[T]` with captured `F` and `T`, we should list `F` and `T` separately in the args.
-      for arg <- args do
+      for arg <- (targs ::: args) do // TODO check targs and terms separately
         assert(arg.isTerm || arg.tpe.isInstanceOf[TypeRef], "Expected TypeRef in Hole type args but got: " + arg.tpe)
 
       // Check result type of the hole
       if isTermHole then assert(tpt.typeOpt <:< pt)
       else assert(tpt.typeOpt =:= pt)
 
       // Check that the types of the args conform to the types of the contents of the hole
-      val argQuotedTypes = args.map { arg =>
+      val argQuotedTypes = (targs ::: args).map { arg =>
         if arg.isTerm then
           val tpe = arg.typeOpt.widenTermRefExpr match
             case _: MethodicType =>