Fix management of compiler instances of quoted.Expr run and show

Changes:
* Make quotes eager: fixes #5376 (requires context)
* Fix bugs shown in #5429: Tree from the macro parameters is passed to another compiler to show it.
* Protect against scope extrusion with early (meaningful) runtime failure (rather than in the compiler)
* Add an implicit function type to provide context.
* Fixes #5161

Author: nicolasstucki

compiler/src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -134,7 +134,7 @@ class ScalaSettings extends Settings.SettingGroup {
   val YprintDebug: Setting[Boolean] = BooleanSetting("-Yprint-debug", "when printing trees, print some extra information useful for debugging.")
   val YprintDebugOwners: Setting[Boolean] = BooleanSetting("-Yprint-debug-owners", "when printing trees, print owners of definitions.")
   val YshowPrintErrors: Setting[Boolean] = BooleanSetting("-Yshow-print-errors", "don't suppress exceptions thrown during tree printing.")
-  val YshowRawQuoteTrees: Setting[Boolean] = BooleanSetting("-Yshow-raw-tree", "don't remove quote artifacts")
+  val YshowRawQuoteTrees: Setting[Boolean] = BooleanSetting("-Yshow-raw-quote-tree", "don't remove quote artifacts")
   val YtestPickler: Setting[Boolean] = BooleanSetting("-Ytest-pickler", "self-test for pickling functionality; should be used with -Ystop-after:pickler")
   val YcheckReentrant: Setting[Boolean] = BooleanSetting("-Ycheck-reentrant", "check that compiled program does not contain vars that can be accessed from a global root.")
   val YdropComments: Setting[Boolean] = BooleanSetting("-Ydrop-comments", "Drop comments when scanning source files.")

compiler/src/dotty/tools/dotc/core/Definitions.scala

@@ -745,15 +745,26 @@ class Definitions {
   lazy val QuotedMatchingBindingType: TypeRef = ctx.requiredClassRef("scala.quoted.matching.Bind")
   def QuotedMatchingBindingClass(implicit ctx: Context): ClassSymbol = QuotedMatchingBindingType.symbol.asClass
 
-  def Unpickler_unpickleExpr: TermSymbol = ctx.requiredMethod("scala.runtime.quoted.Unpickler.unpickleExpr")
-  def Unpickler_liftedExpr: TermSymbol = ctx.requiredMethod("scala.runtime.quoted.Unpickler.liftedExpr")
-  def Unpickler_unpickleType: TermSymbol = ctx.requiredMethod("scala.runtime.quoted.Unpickler.unpickleType")
+  lazy val UnpicklerModuleRef: TermRef = ctx.requiredModuleRef("scala.runtime.quoted.Unpickler")
+  def UnpicklerModule(implicit ctx: Context): Symbol = UnpicklerModuleRef.symbol
+  lazy val UnpicklerType: TypeRef = ctx.requiredClassRef("scala.runtime.quoted.Unpickler")
+  def UnpicklerClass(implicit ctx: Context): ClassSymbol = UnpicklerType.symbol.asClass
+
+  lazy val Unpickler_unpickleExprR: TermRef = UnpicklerModule.requiredMethodRef("unpickleExpr")
+  def Unpickler_unpickleExpr: Symbol = Unpickler_unpickleExprR.symbol
+  lazy val Unpickler_liftedExprR: TermRef = UnpicklerModule.requiredMethodRef("liftedExpr")
+  def Unpickler_liftedExpr: Symbol = Unpickler_liftedExprR.symbol
+  lazy val Unpickler_unpickleTypeR: TermRef = UnpicklerModule.requiredMethodRef("unpickleType")
+  def Unpickler_unpickleType: Symbol = Unpickler_unpickleTypeR.symbol
 
   lazy val TastyReflectionType: TypeRef = ctx.requiredClassRef("scala.tasty.Reflection")
   def TastyReflectionClass(implicit ctx: Context): ClassSymbol = TastyReflectionType.symbol.asClass
 
-  lazy val TastyReflectionModule: TermSymbol = ctx.requiredModule("scala.tasty.Reflection")
-    lazy val TastyReflection_macroContext: TermSymbol = TastyReflectionModule.requiredMethod("macroContext")
+  lazy val StagingContextType: TypeRef = ctx.requiredClassRef("scala.quoted.StagingContext")
+  def StagingContextClass(implicit ctx: Context): ClassSymbol = StagingContextType.symbol.asClass
+
+  lazy val StagingContextModule: TermSymbol = ctx.requiredModule("scala.quoted.StagingContext")
+  lazy val StagingContext_macroContext: TermSymbol = StagingContextModule.requiredMethod("macroContext")
 
   lazy val EqlType: TypeRef = ctx.requiredClassRef("scala.Eql")
   def EqlClass(implicit ctx: Context): ClassSymbol = EqlType.symbol.asClass

compiler/src/dotty/tools/dotc/core/quoted/PickledQuotes.scala

@@ -34,40 +34,46 @@ object PickledQuotes {
 
   /** Transform the expression into its fully spliced Tree */
   def quotedExprToTree[T](expr: quoted.Expr[T])(implicit ctx: Context): Tree = expr match {
-    case expr: TastyExpr[_] =>
-      val unpickled = unpickleExpr(expr)
-      val force = new TreeTraverser {
-        def traverse(tree: tpd.Tree)(implicit ctx: Context): Unit = traverseChildren(tree)
-      }
-      force.traverse(unpickled)
-      unpickled
     case expr: LiftedExpr[T] =>
       expr.value match {
         case value: Class[_] => ref(defn.Predef_classOf).appliedToType(classToType(value))
         case value => Literal(Constant(value))
       }
-    case expr: TastyTreeExpr[Tree] @unchecked => healOwner(expr.tree)
+    case expr: TastyTreeExpr[Tree] @unchecked =>
+      if (contextId != expr.ctxId)
+        throw new scala.quoted.QuoteError(
+          """Quote used outside of the `Staging` context where it was defined.
+            |This usualy indicated that a `run` was called within another `run` or a macro.
+          """.stripMargin)
+      healOwner(expr.tree)
     case expr: FunctionAppliedTo[_] =>
       functionAppliedTo(quotedExprToTree(expr.f), expr.args.map(arg => quotedExprToTree(arg)).toList)
   }
 
   /** Transform the expression into its fully spliced TypeTree */
   def quotedTypeToTree(expr: quoted.Type[_])(implicit ctx: Context): Tree = expr match {
-    case expr: TastyType[_] => unpickleType(expr)
     case expr: TaggedType[_] => classTagToTypeTree(expr.ct)
     case expr: TreeType[Tree] @unchecked => healOwner(expr.typeTree)
   }
 
   /** Unpickle the tree contained in the TastyExpr */
-  private def unpickleExpr(expr: TastyExpr[_])(implicit ctx: Context): Tree = {
-    val tastyBytes = TastyString.unpickle(expr.tasty)
-    unpickle(tastyBytes, expr.args, isType = false)(ctx.addMode(Mode.ReadPositions))
+  def unpickleExpr(tasty: scala.runtime.quoted.Unpickler.Pickled, args: Seq[Any])(implicit ctx: Context): Tree = {
+    val tastyBytes = TastyString.unpickle(tasty)
+    val unpickled = unpickle(tastyBytes, args, isType = false)(ctx.addMode(Mode.ReadPositions))
+    force.traverse(unpickled)
+    unpickled
   }
 
   /** Unpickle the tree contained in the TastyType */
-  private def unpickleType(ttpe: TastyType[_])(implicit ctx: Context): Tree = {
-    val tastyBytes = TastyString.unpickle(ttpe.tasty)
-    unpickle(tastyBytes, ttpe.args, isType = true)(ctx.addMode(Mode.ReadPositions))
+  def unpickleType(tasty: scala.runtime.quoted.Unpickler.Pickled, args: Seq[Any])(implicit ctx: Context): Tree = {
+    val tastyBytes = TastyString.unpickle(tasty)
+    val unpickled = unpickle(tastyBytes, args, isType = true)(ctx.addMode(Mode.ReadPositions))
+    force.traverse(unpickled)
+    unpickled
+  }
+
+  private def force = new TreeTraverser {
+    def traverse(tree: tpd.Tree)(implicit ctx: Context): Unit = traverseChildren(tree)
   }
 
   // TASTY picklingtests/pos/quoteTest.scala
@@ -193,4 +199,12 @@ object PickledQuotes {
       case _ => tree
     }
   }
+
+  /** Id of the current compilation (macros) or Expr[_] run */
+  def contextId(implicit ctx: Context): Int = {
+    def root(ctx: Context): Context =
+      if (ctx.outer != NoContext) root(ctx.outer) else ctx
+    root(ctx).hashCode
+  }
+
 }

compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala

@@ -31,6 +31,7 @@ import scala.collection.mutable.ListBuffer
 import scala.collection.mutable
 import config.Printers.pickling
 import core.quoted.PickledQuotes
+import core.quoted.Quoted
 
 import scala.quoted
 import scala.quoted.Types.TreeType
@@ -1272,7 +1273,7 @@ class TreeUnpickler(reader: TastyReader,
       val args = until(end)(readTerm())
       val splice = splices(idx)
       def wrap(arg: Tree) =
-        if (arg.isTerm) new TastyTreeExpr(arg)
+        if (arg.isTerm) new TastyTreeExpr(arg, PickledQuotes.contextId)
         else new TreeType(arg)
       val reifiedArgs = args.map(wrap)
       val filled = if (isType) {

compiler/src/dotty/tools/dotc/quoted/ExprCompilationUnit.scala

@@ -3,9 +3,7 @@ package dotty.tools.dotc.quoted
 import dotty.tools.dotc.CompilationUnit
 import dotty.tools.dotc.util.NoSource
 
-import scala.quoted.Expr
+import scala.quoted.{Expr, StagingContext}
 
 /* Compilation unit containing the contents of a quoted expression */
-class ExprCompilationUnit(val expr: Expr[_]) extends CompilationUnit(NoSource) {
-  override def toString: String = s"Expr($expr)"
-}
+class ExprCompilationUnit(val expr: StagingContext => Expr[Any]) extends CompilationUnit(NoSource)

compiler/src/dotty/tools/dotc/quoted/QuoteCompiler.scala

@@ -17,9 +17,9 @@ import dotty.tools.dotc.core.quoted.PickledQuotes
 import dotty.tools.dotc.transform.Staging
 import dotty.tools.dotc.util.Spans.Span
 import dotty.tools.dotc.util.SourceFile
-import dotty.tools.io.{Path, VirtualFile}
+import dotty.tools.io.{VirtualFile}
 
-import scala.quoted.{Expr, Type}
+import scala.quoted.{Expr, StagingContext, Type}
 
 /** Compiler that takes the contents of a quoted expression `expr` and produces
  *  a class file with `class ' { def apply: Object = expr }`.
@@ -50,7 +50,7 @@ class QuoteCompiler extends Compiler {
         case exprUnit: ExprCompilationUnit =>
           val tree =
             if (putInClass) inClass(exprUnit.expr)
-            else PickledQuotes.quotedExprToTree(exprUnit.expr)
+            else unpickleExpr(exprUnit.expr)
           val source = SourceFile.virtual("<quoted.Expr>", "")
           CompilationUnit(source, tree, forceTrees = true)
         case typeUnit: TypeCompilationUnit =>
@@ -61,11 +61,16 @@ class QuoteCompiler extends Compiler {
       }
     }
 
+    private def unpickleExpr(code: StagingContext => Expr[Any])(implicit ctx: Context) = {
+      val expr = code(new StagingImpl)
+      PickledQuotes.quotedExprToTree(expr)
+    }
+
     /** Places the contents of expr in a compilable tree for a class
       *  with the following format.
       *  `package __root__ { class ' { def apply: Any = <expr> } }`
       */
-    private def inClass(expr: Expr[_])(implicit ctx: Context): Tree = {
+    private def inClass(code: StagingContext => Expr[_])(implicit ctx: Context): Tree = {
       val pos = Span(0)
       val assocFile = new VirtualFile("<quote>")
 
@@ -74,7 +79,7 @@ class QuoteCompiler extends Compiler {
       cls.enter(ctx.newDefaultConstructor(cls), EmptyScope)
       val meth = ctx.newSymbol(cls, nme.apply, Method, ExprType(defn.AnyType), coord = pos).entered
 
-      val quoted = PickledQuotes.quotedExprToTree(expr)(ctx.withOwner(meth))
+      val quoted = unpickleExpr(code)(ctx.withOwner(meth))
 
       val run = DefDef(meth, quoted)
       val classTree = ClassDef(cls, DefDef(cls.primaryConstructor.asTerm), run :: Nil)
@@ -85,7 +90,7 @@ class QuoteCompiler extends Compiler {
   }
 
   class ExprRun(comp: Compiler, ictx: Context) extends Run(comp, ictx) {
-    def compileExpr(expr: Expr[_]): Unit = {
+    def compileExpr(expr: StagingContext => Expr[_]): Unit = {
       val units = new ExprCompilationUnit(expr) :: Nil
       compileUnits(units)
     }

compiler/src/dotty/tools/dotc/quoted/QuoteContextImpl.scala

@@ -0,0 +1,35 @@
+package dotty.tools.dotc.quoted
+
+import dotty.tools.dotc.core.Contexts._
+import dotty.tools.dotc.core.quoted.PickledQuotes
+import dotty.tools.dotc.tastyreflect.ReflectionImpl
+
+import scala.quoted.{Expr, Type}
+import scala.runtime.quoted.Unpickler.Pickled
+
+class StagingImpl(implicit ctx: Context) extends scala.quoted.StagingContext {
+
+  def unpickleExpr[T](repr: Pickled, args: Seq[Any]): Expr[T] =
+    new scala.quoted.Exprs.TastyTreeExpr(PickledQuotes.unpickleExpr(repr, args), PickledQuotes.contextId).asInstanceOf[Expr[T]]
+
+  def unpickleType[T](repr: Pickled, args: Seq[Any]): Type[T] =
+    new scala.quoted.Types.TreeType(PickledQuotes.unpickleType(repr, args)).asInstanceOf[Type[T]]
+
+  def show[T](expr: Expr[T]): String = {
+    val tree = PickledQuotes.quotedExprToTree(expr)
+    // TODO freshen names
+    val tree1 =
+     if (ctx.settings.YshowRawQuoteTrees.value) tree else (new TreeCleaner).transform(tree)
+    new ReflectionImpl(ctx).showSourceCode.showTree(tree1)
+  }
+
+  def show[T](tpe: Type[T]): String = {
+    val tree = PickledQuotes.quotedTypeToTree(tpe)
+    // TODO freshen names
+    val tree1 = if (ctx.settings.YshowRawQuoteTrees.value) tree else (new TreeCleaner).transform(tree)
+    new ReflectionImpl(ctx).showSourceCode.showTypeOrBoundsTree(tree1)
+  }
+
+  val reflection: scala.tasty.Reflection = new ReflectionImpl(ctx)
+
+}

compiler/src/dotty/tools/dotc/quoted/QuoteDecompiler.scala

@@ -1,16 +1,16 @@
 package dotty.tools.dotc.quoted
 
-import dotty.tools.dotc.ast.tpd
 import dotty.tools.dotc.Driver
 import dotty.tools.dotc.core.Contexts.{Context, ContextBase, FreshContext}
-import dotty.tools.dotc.tastyreflect.ReflectionImpl
 import dotty.tools.io.{AbstractFile, Directory, PlainDirectory, VirtualDirectory}
 import dotty.tools.repl.AbstractFileClassLoader
 import dotty.tools.dotc.reporting._
-import scala.quoted.{Expr, Type}
+import scala.quoted.{Expr, StagingContext}
 import scala.quoted.Toolbox
 import java.net.URLClassLoader
 
+import dotty.tools.dotc.core.quoted.PickledQuotes
+
 /** Driver to compile quoted code
   *
   * @param appClassloader classloader of the application that generated the quotes
@@ -20,7 +20,7 @@ class QuoteDriver(appClassloader: ClassLoader) extends Driver {
 
   private[this] val contextBase: ContextBase = new ContextBase
 
-  def run[T](expr: Expr[T], settings: Toolbox.Settings): T = {
+  def run[T](code: StagingContext => Expr[T], settings: Toolbox.Settings): T = {
     val outDir: AbstractFile = settings.outDir match {
       case Some(out) =>
         val dir = Directory(out)
@@ -34,7 +34,7 @@ class QuoteDriver(appClassloader: ClassLoader) extends Driver {
     val ctx = setToolboxSettings(ctx0.fresh.setSetting(ctx0.settings.outputDir, outDir), settings)
 
     val driver = new QuoteCompiler
-    driver.newRun(ctx).compileExpr(expr)
+    driver.newRun(ctx).compileExpr(code)
 
     assert(!ctx.reporter.hasErrors)
 
@@ -47,44 +47,6 @@ class QuoteDriver(appClassloader: ClassLoader) extends Driver {
     method.invoke(inst).asInstanceOf[T]
   }
 
-  private def doShow(tree: Tree, ctx: Context): String = {
-    implicit val c: Context = ctx
-    val tree1 =
-      if (ctx.settings.YshowRawQuoteTrees.value) tree
-      else (new TreeCleaner).transform(tree)
-    ReflectionImpl.showTree(tree1)
-  }
-
-  def show(expr: Expr[_], settings: Toolbox.Settings): String =
-    withTree(expr, doShow, settings)
-
-  def show(tpe: Type[_], settings: Toolbox.Settings): String =
-    withTypeTree(tpe, doShow, settings)
-
-  def withTree[T](expr: Expr[_], f: (Tree, Context) => T, settings: Toolbox.Settings): T = {
-    val ctx = setToolboxSettings(setup(settings.compilerArgs.toArray :+ "dummy.scala", initCtx.fresh)._2.fresh, settings)
-
-    var output: Option[T] = None
-    def registerTree(tree: tpd.Tree)(ctx: Context): Unit = {
-      assert(output.isEmpty)
-      output = Some(f(tree, ctx))
-    }
-    new QuoteDecompiler(registerTree).newRun(ctx).compileExpr(expr)
-    output.getOrElse(throw new Exception("Could not extract " + expr))
-  }
-
-  def withTypeTree[T](tpe: Type[_], f: (TypTree, Context) => T, settings: Toolbox.Settings): T = {
-    val ctx = setToolboxSettings(setup(settings.compilerArgs.toArray :+ "dummy.scala", initCtx.fresh)._2.fresh, settings)
-
-    var output: Option[T] = None
-    def registerTree(tree: tpd.Tree)(ctx: Context): Unit = {
-      assert(output.isEmpty)
-      output = Some(f(tree.asInstanceOf[TypTree], ctx))
-    }
-    new QuoteDecompiler(registerTree).newRun(ctx).compileType(tpe)
-    output.getOrElse(throw new Exception("Could not extract " + tpe))
-  }
-
   override def initCtx: Context = {
     val ictx = contextBase.initialCtx
     ictx.settings.classpath.update(QuoteDriver.currentClasspath(appClassloader))(ictx)