WIP Try to fix management of compiler instances of quoted.Expr run and show

Also fix some issues along the way.

Author: nicolasstucki

compiler/src/dotty/tools/dotc/Compiler.scala

@@ -44,7 +44,7 @@ class Compiler {
   /** Phases dealing with TASTY tree pickling and unpickling */
   protected def picklerPhases: List[List[Phase]] =
     List(new Pickler) ::            // Generate TASTY info
-    List(new Staging) ::            // Inline calls, expand macros and turn quoted trees into explicit run-time data structures
+    List(new StagingPhase) ::       // Inline calls, expand macros and turn quoted trees into explicit run-time data structures
     Nil
 
   /** Phases dealing with the transformation from pickled trees to backend trees */

compiler/src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -116,7 +116,7 @@ class ScalaSettings extends Settings.SettingGroup {
   val YprintDebug: Setting[Boolean] = BooleanSetting("-Yprint-debug", "when printing trees, print some extra information useful for debugging.")
   val YprintDebugOwners: Setting[Boolean] = BooleanSetting("-Yprint-debug-owners", "when printing trees, print owners of definitions.")
   val YshowPrintErrors: Setting[Boolean] = BooleanSetting("-Yshow-print-errors", "don't suppress exceptions thrown during tree printing.")
-  val YshowRawQuoteTrees: Setting[Boolean] = BooleanSetting("-Yshow-raw-tree", "don't remove quote artifacts")
+  val YshowRawQuoteTrees: Setting[Boolean] = BooleanSetting("-Yshow-raw-quote-tree", "don't remove quote artifacts")
   val YtestPickler: Setting[Boolean] = BooleanSetting("-Ytest-pickler", "self-test for pickling functionality; should be used with -Ystop-after:pickler")
   val YcheckReentrant: Setting[Boolean] = BooleanSetting("-Ycheck-reentrant", "check that compiled program does not contain vars that can be accessed from a global root.")
   val YdropComments: Setting[Boolean] = BooleanSetting("-Ydrop-comments", "Drop comments when scanning source files.")

compiler/src/dotty/tools/dotc/core/Definitions.scala

@@ -690,16 +690,30 @@ class Definitions {
   lazy val QuotedLiftableType: TypeRef = ctx.requiredClassRef("scala.quoted.Liftable")
   def QuotedLiftableClass(implicit ctx: Context): ClassSymbol = QuotedLiftableType.symbol.asClass
 
-  def Unpickler_unpickleExpr: TermSymbol = ctx.requiredMethod("scala.runtime.quoted.Unpickler.unpickleExpr")
-  def Unpickler_liftedExpr: TermSymbol = ctx.requiredMethod("scala.runtime.quoted.Unpickler.liftedExpr")
-  def Unpickler_unpickleType: TermSymbol = ctx.requiredMethod("scala.runtime.quoted.Unpickler.unpickleType")
+  lazy val UnpicklerModuleRef: TermRef = ctx.requiredModuleRef("scala.runtime.quoted.Unpickler")
+  def UnpicklerModule(implicit ctx: Context): Symbol = UnpicklerModuleRef.symbol
+  lazy val UnpicklerType: TypeRef = ctx.requiredClassRef("scala.runtime.quoted.Unpickler")
+  def UnpicklerClass(implicit ctx: Context): ClassSymbol = UnpicklerType.symbol.asClass
+
+  lazy val Unpickler_unpickleExprR: TermRef = UnpicklerModule.requiredMethodRef("unpickleExpr")
+  def Unpickler_unpickleExpr: Symbol = Unpickler_unpickleExprR.symbol
+  lazy val Unpickler_liftedExprR: TermRef = UnpicklerModule.requiredMethodRef("liftedExpr")
+  def Unpickler_liftedExpr: Symbol = Unpickler_liftedExprR.symbol
+  lazy val Unpickler_unpickleTypeR: TermRef = UnpicklerModule.requiredMethodRef("unpickleType")
+  def Unpickler_unpickleType: Symbol = Unpickler_unpickleTypeR.symbol
 
   lazy val TastyTastyType: TypeRef = ctx.requiredClassRef("scala.tasty.Tasty")
   def TastyTastyClass(implicit ctx: Context): ClassSymbol = TastyTastyType.symbol.asClass
 
   lazy val TastyTastyModule: TermSymbol = ctx.requiredModule("scala.tasty.Tasty")
     lazy val TastyTasty_macroContext: TermSymbol = TastyTastyModule.requiredMethod("macroContext")
 
+  lazy val StagingType: TypeRef = ctx.requiredClassRef("scala.quoted.Staging")
+  def StagingClass(implicit ctx: Context): ClassSymbol = StagingType.symbol.asClass
+
+  lazy val StagingModule: TermSymbol = ctx.requiredModule("scala.quoted.Staging")
+  lazy val Staging_macroContext: TermSymbol = StagingModule.requiredMethod("macroContext")
+
   lazy val EqType: TypeRef = ctx.requiredClassRef("scala.Eq")
   def EqClass(implicit ctx: Context): ClassSymbol = EqType.symbol.asClass
   def EqModule(implicit ctx: Context): Symbol = EqClass.companionModule

compiler/src/dotty/tools/dotc/core/quoted/PickledQuotes.scala

@@ -34,40 +34,38 @@ object PickledQuotes {
 
   /** Transform the expression into its fully spliced Tree */
   def quotedExprToTree[T](expr: quoted.Expr[T])(implicit ctx: Context): Tree = expr match {
-    case expr: TastyExpr[_] =>
-      val unpickled = unpickleExpr(expr)
-      val force = new TreeTraverser {
-        def traverse(tree: tpd.Tree)(implicit ctx: Context): Unit = traverseChildren(tree)
-      }
-      force.traverse(unpickled)
-      unpickled
     case expr: LiftedExpr[T] =>
       expr.value match {
         case value: Class[_] => ref(defn.Predef_classOf).appliedToType(classToType(value))
         case value => Literal(Constant(value))
       }
-    case expr: TastyTreeExpr[Tree] @unchecked => expr.tree
+    case expr: TastyTreeExpr[Tree] @unchecked => healOwner(expr.tree)
     case expr: FunctionAppliedTo[_, _] =>
       functionAppliedTo(quotedExprToTree(expr.f), quotedExprToTree(expr.x))
   }
 
   /** Transform the expression into its fully spliced TypeTree */
   def quotedTypeToTree(expr: quoted.Type[_])(implicit ctx: Context): Tree = expr match {
-    case expr: TastyType[_] => unpickleType(expr)
     case expr: TaggedType[_] => classTagToTypeTree(expr.ct)
-    case expr: TreeType[Tree] @unchecked => expr.typeTree
+    case expr: TreeType[Tree] @unchecked => healOwner(expr.typeTree)
   }
 
   /** Unpickle the tree contained in the TastyExpr */
-  private def unpickleExpr(expr: TastyExpr[_])(implicit ctx: Context): Tree = {
-    val tastyBytes = TastyString.unpickle(expr.tasty)
-    unpickle(tastyBytes, expr.args, isType = false)(ctx.addMode(Mode.ReadPositions))
+  def unpickleExpr(tasty: scala.runtime.quoted.Unpickler.Pickled, args: Seq[Any])(implicit ctx: Context): Tree = {
+    val tastyBytes = TastyString.unpickle(tasty)
+    val unpickled = unpickle(tastyBytes, args, isType = false)(ctx.addMode(Mode.ReadPositions))
+    // TODO force should not be done here it should be in run and show
+    val force = new TreeTraverser {
+      def traverse(tree: tpd.Tree)(implicit ctx: Context): Unit = traverseChildren(tree)
+    }
+    force.traverse(unpickled)
+    unpickled
   }
 
   /** Unpickle the tree contained in the TastyType */
-  private def unpickleType(ttpe: TastyType[_])(implicit ctx: Context): Tree = {
-    val tastyBytes = TastyString.unpickle(ttpe.tasty)
-    unpickle(tastyBytes, ttpe.args, isType = true)(ctx.addMode(Mode.ReadPositions))
+  def unpickleType(tasty: scala.runtime.quoted.Unpickler.Pickled, args: Seq[Any])(implicit ctx: Context): Tree = {
+    val tastyBytes = TastyString.unpickle(tasty)
+    unpickle(tastyBytes, args, isType = true)(ctx.addMode(Mode.ReadPositions))
   }
 
   // TASTY picklingtests/pos/quoteTest.scala
@@ -170,4 +168,21 @@ object PickledQuotes {
       }
     } else ctx.getClassIfDefined(clazz.getCanonicalName).typeRef
   }
+
+  /** Make sure that the owner of this tree is `ctx.owner` */
+  private def healOwner(tree: Tree)(implicit ctx: Context): Tree = {
+    val getCurrentOwner = new TreeAccumulator[Option[Symbol]] {
+      def apply(x: Option[Symbol], tree: tpd.Tree)(implicit ctx: Context): Option[Symbol] = {
+        if (x.isDefined) x
+        else tree match {
+          case tree: DefTree => Some(tree.symbol.owner)
+          case _ => foldOver(x, tree)
+        }
+      }
+    }
+    getCurrentOwner(None, tree) match {
+      case Some(owner) if owner != ctx.owner => tree.changeOwner(owner, ctx.owner)
+      case _ => tree
+    }
+  }
 }

compiler/src/dotty/tools/dotc/quoted/ExprCompilationUnit.scala

@@ -3,9 +3,7 @@ package dotty.tools.dotc.quoted
 import dotty.tools.dotc.CompilationUnit
 import dotty.tools.dotc.util.NoSource
 
-import scala.quoted.Expr
+import scala.quoted.{Expr, Staging}
 
 /* Compilation unit containing the contents of a quoted expression */
-class ExprCompilationUnit(val expr: Expr[_]) extends CompilationUnit(NoSource) {
-  override def toString: String = s"Expr($expr)"
-}
+class ExprCompilationUnit(val expr: Staging => Expr[Any]) extends CompilationUnit(NoSource)

compiler/src/dotty/tools/dotc/quoted/QuoteCompiler.scala

@@ -13,13 +13,15 @@ import dotty.tools.dotc.core.StdNames.nme
 import dotty.tools.dotc.core.Symbols.defn
 import dotty.tools.dotc.core.Types.ExprType
 import dotty.tools.dotc.core.quoted.PickledQuotes
-import dotty.tools.dotc.transform.Staging
+import dotty.tools.dotc.transform.StagingPhase
+import dotty.tools.dotc.tastyreflect.TastyImpl
 import dotty.tools.dotc.typer.FrontEnd
 import dotty.tools.dotc.util.Positions.Position
 import dotty.tools.dotc.util.SourceFile
 import dotty.tools.io.{Path, PlainFile}
 
-import scala.quoted.{Expr, Type}
+import scala.quoted.{Expr, Staging, Type}
+import scala.runtime.quoted.Unpickler.Pickled
 
 /** Compiler that takes the contents of a quoted expression `expr` and produces
  *  a class file with `class ' { def apply: Object = expr }`.
@@ -30,7 +32,7 @@ class QuoteCompiler extends Compiler {
     List(List(new QuotedFrontend(putInClass = true)))
 
   override protected def picklerPhases: List[List[Phase]] =
-    List(List(new Staging))
+    List(List(new StagingPhase))
 
   override def newRun(implicit ctx: Context): ExprRun = {
     reset()
@@ -50,7 +52,7 @@ class QuoteCompiler extends Compiler {
         case exprUnit: ExprCompilationUnit =>
           val tree =
             if (putInClass) inClass(exprUnit.expr)
-            else PickledQuotes.quotedExprToTree(exprUnit.expr)
+            else unpickleExpr(exprUnit.expr)
           val source = new SourceFile("", "")
           CompilationUnit.mkCompilationUnit(source, tree, forceTrees = true)
         case typeUnit: TypeCompilationUnit =>
@@ -61,11 +63,16 @@ class QuoteCompiler extends Compiler {
       }
     }
 
+    private def unpickleExpr(code: Staging => Expr[Any])(implicit ctx: Context) = {
+      val expr = code(new StagingImpl)
+      PickledQuotes.quotedExprToTree(expr)
+    }
+
     /** Places the contents of expr in a compilable tree for a class
       *  with the following format.
       *  `package __root__ { class ' { def apply: Any = <expr> } }`
       */
-    private def inClass(expr: Expr[_])(implicit ctx: Context): Tree = {
+    private def inClass(code: Staging => Expr[_])(implicit ctx: Context): Tree = {
       val pos = Position(0)
       val assocFile = new PlainFile(Path("<quote>"))
 
@@ -74,7 +81,7 @@ class QuoteCompiler extends Compiler {
       cls.enter(ctx.newDefaultConstructor(cls), EmptyScope)
       val meth = ctx.newSymbol(cls, nme.apply, Method, ExprType(defn.AnyType), coord = pos).entered
 
-      val quoted = PickledQuotes.quotedExprToTree(expr)(ctx.withOwner(meth))
+      val quoted = unpickleExpr(code)(ctx.withOwner(meth))
 
       val run = DefDef(meth, quoted)
       val classTree = ClassDef(cls, DefDef(cls.primaryConstructor.asTerm), run :: Nil)
@@ -83,7 +90,7 @@ class QuoteCompiler extends Compiler {
   }
 
   class ExprRun(comp: Compiler, ictx: Context) extends Run(comp, ictx) {
-    def compileExpr(expr: Expr[_]): Unit = {
+    def compileExpr(expr: Staging => Expr[_]): Unit = {
       val units = new ExprCompilationUnit(expr) :: Nil
       compileUnits(units)
     }

compiler/src/dotty/tools/dotc/quoted/QuoteContextImpl.scala

@@ -0,0 +1,33 @@
+package dotty.tools.dotc.quoted
+
+import dotty.tools.dotc.core.Contexts._
+import dotty.tools.dotc.core.quoted.PickledQuotes
+import dotty.tools.dotc.tastyreflect.TastyImpl
+
+import scala.quoted.{Expr, Type}
+import scala.runtime.quoted.Unpickler.Pickled
+
+class StagingImpl(implicit ctx: Context) extends scala.quoted.Staging {
+
+  def unpickleExpr[T](repr: Pickled, args: Seq[Any]): Expr[T] =
+    new scala.quoted.Exprs.TastyTreeExpr(PickledQuotes.unpickleExpr(repr, args)).asInstanceOf[Expr[T]]
+
+  def unpickleType[T](repr: Pickled, args: Seq[Any]): Type[T] =
+    new scala.quoted.Types.TreeType(PickledQuotes.unpickleType(repr, args)).asInstanceOf[Type[T]]
+
+  def show[T](expr: Expr[T]): String = {
+    val tree = PickledQuotes.quotedExprToTree(expr)
+    // TODO freshen names
+    val tree1 =
+     if (ctx.settings.YshowRawQuoteTrees.value) tree else (new TreeCleaner).transform(tree)
+    new TastyImpl(ctx).showSourceCode.showTree(tree1)
+  }
+
+  def show[T](tpe: Type[T]): String = {
+    val tree = PickledQuotes.quotedTypeToTree(tpe)
+    // TODO freshen names
+    val tree1 = if (ctx.settings.YshowRawQuoteTrees.value) tree else (new TreeCleaner).transform(tree)
+    new TastyImpl(ctx).showSourceCode.showTypeOrBoundsTree(tree1)
+  }
+
+}

compiler/src/dotty/tools/dotc/quoted/QuoteDecompiler.scala

@@ -7,16 +7,17 @@ import dotty.tools.dotc.tastyreflect.TastyImpl
 import dotty.tools.io.{AbstractFile, Directory, PlainDirectory, VirtualDirectory}
 import dotty.tools.repl.AbstractFileClassLoader
 
-import scala.quoted.{Expr, Type}
+import scala.quoted.{Expr, Staging, Type}
 import scala.quoted.Toolbox
 import java.net.URLClassLoader
 
+import dotty.tools.dotc.core.quoted.PickledQuotes
+
 class QuoteDriver extends Driver {
-  import tpd._
 
   private[this] val contextBase: ContextBase = new ContextBase
 
-  def run[T](expr: Expr[T], settings: Toolbox.Settings): T = {
+  def run[T](code: Staging => Expr[T], settings: Toolbox.Settings): T = {
     val outDir: AbstractFile = settings.outDir match {
       case Some(out) =>
         val dir = Directory(out)
@@ -30,7 +31,7 @@ class QuoteDriver extends Driver {
     val ctx = setToolboxSettings(ctx0.fresh.setSetting(ctx0.settings.outputDir, outDir), settings)
 
     val driver = new QuoteCompiler
-    driver.newRun(ctx).compileExpr(expr)
+    driver.newRun(ctx).compileExpr(code)
 
     val classLoader = new AbstractFileClassLoader(outDir, this.getClass.getClassLoader)
 
@@ -41,41 +42,6 @@ class QuoteDriver extends Driver {
     method.invoke(instance).asInstanceOf[T]
   }
 
-  def show(expr: Expr[_], settings: Toolbox.Settings): String = {
-    def show(tree: Tree, ctx: Context): String = {
-      implicit val c: Context = ctx
-      val tree1 =
-        if (ctx.settings.YshowRawQuoteTrees.value) tree
-        else (new TreeCleaner).transform(tree)
-      new TastyImpl(ctx).showSourceCode.showTree(tree1)
-    }
-    withTree(expr, show, settings)
-  }
-
-  def withTree[T](expr: Expr[_], f: (Tree, Context) => T, settings: Toolbox.Settings): T = {
-    val ctx = setToolboxSettings(setup(settings.compilerArgs.toArray :+ "dummy.scala", initCtx.fresh)._2.fresh, settings)
-
-    var output: Option[T] = None
-    def registerTree(tree: tpd.Tree)(ctx: Context): Unit = {
-      assert(output.isEmpty)
-      output = Some(f(tree, ctx))
-    }
-    new QuoteDecompiler(registerTree).newRun(ctx).compileExpr(expr)
-    output.getOrElse(throw new Exception("Could not extract " + expr))
-  }
-
-  def withTypeTree[T](tpe: Type[_], f: (TypTree, Context) => T, settings: Toolbox.Settings): T = {
-    val (_, ctx: Context) = setup(settings.compilerArgs.toArray :+ "dummy.scala", initCtx.fresh)
-
-    var output: Option[T] = None
-    def registerTree(tree: tpd.Tree)(ctx: Context): Unit = {
-      assert(output.isEmpty)
-      output = Some(f(tree.asInstanceOf[TypTree], ctx))
-    }
-    new QuoteDecompiler(registerTree).newRun(ctx).compileType(tpe)
-    output.getOrElse(throw new Exception("Could not extract " + tpe))
-  }
-
   override def initCtx: Context = {
     val ictx = contextBase.initialCtx
     ictx.settings.classpath.update(QuoteDriver.currentClasspath)(ictx)

compiler/src/dotty/tools/dotc/quoted/QuoteDriver.scala

@@ -8,6 +8,7 @@ import dotty.tools.dotc.core.NameKinds.{NumberedInfo, UniqueName}
 import dotty.tools.dotc.core.SymDenotations.SymDenotation
 import dotty.tools.dotc.transform.MegaPhase.MiniPhase
 
+// TODO use this in some form again
 /** Refreshes local names starting from the second use of the name. Intended for readability of the pretty printed code. */
 class RefreshNames extends MiniPhase with SymTransformer {
 

compiler/src/dotty/tools/dotc/quoted/RefreshNames.scala

@@ -1,28 +1,18 @@
 package dotty.tools.dotc.quoted
 
-import dotty.tools.dotc.ast.tpd
-
-import scala.quoted.Expr
-import scala.quoted.Exprs.{LiftedExpr, TastyTreeExpr}
+import scala.quoted.{Expr, Staging, Type}
 
 /** Default runners for quoted expressions */
 object ToolboxImpl {
-  import tpd._
 
   def make(settings: scala.quoted.Toolbox.Settings): scala.quoted.Toolbox = new scala.quoted.Toolbox {
 
     private[this] val driver: QuoteDriver = new QuoteDriver()
 
-    def run[T](expr: Expr[T]): T = expr match {
-      case expr: LiftedExpr[T] =>
-        expr.value
-      case expr: TastyTreeExpr[Tree] @unchecked =>
-        throw new Exception("Cannot call `Expr.run` on an `Expr` that comes from a macro argument.")
-      case _ =>
-        synchronized(driver.run(expr, settings))
+    protected def runImpl[T](code: Staging => Expr[T]): T = {
+      // TODO check for recursion and throw if possible (i.e. run inside a run)
+      synchronized(driver.run(code, settings))
     }
 
-    def show[T](expr: Expr[T]): String = synchronized(driver.show(expr, settings))
-
   }
 }