WIP Try to fix management of compiler instances of quoted.Expr run and show

Also fix some issues along the way.

Author: nicolasstucki

compiler/src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -116,6 +116,7 @@ class ScalaSettings extends Settings.SettingGroup {
   val YprintDebug: Setting[Boolean] = BooleanSetting("-Yprint-debug", "when printing trees, print some extra information useful for debugging.")
   val YprintDebugOwners: Setting[Boolean] = BooleanSetting("-Yprint-debug-owners", "when printing trees, print owners of definitions.")
   val YshowPrintErrors: Setting[Boolean] = BooleanSetting("-Yshow-print-errors", "don't suppress exceptions thrown during tree printing.")
+  val YshowRawTree: Setting[Boolean] = BooleanSetting("-Yshow-raw-tree", "don't remove quote artifacts")
   val YtestPickler: Setting[Boolean] = BooleanSetting("-Ytest-pickler", "self-test for pickling functionality; should be used with -Ystop-after:pickler")
   val YcheckReentrant: Setting[Boolean] = BooleanSetting("-Ycheck-reentrant", "check that compiled program does not contain vars that can be accessed from a global root.")
   val YdropComments: Setting[Boolean] = BooleanSetting("-Ydrop-comments", "Drop comments when scanning source files.")

compiler/src/dotty/tools/dotc/core/Definitions.scala

@@ -690,16 +690,30 @@ class Definitions {
   lazy val QuotedLiftableType: TypeRef = ctx.requiredClassRef("scala.quoted.Liftable")
   def QuotedLiftableClass(implicit ctx: Context): ClassSymbol = QuotedLiftableType.symbol.asClass
 
-  def Unpickler_unpickleExpr: TermSymbol = ctx.requiredMethod("scala.runtime.quoted.Unpickler.unpickleExpr")
-  def Unpickler_liftedExpr: TermSymbol = ctx.requiredMethod("scala.runtime.quoted.Unpickler.liftedExpr")
-  def Unpickler_unpickleType: TermSymbol = ctx.requiredMethod("scala.runtime.quoted.Unpickler.unpickleType")
+  lazy val UnpicklerModuleRef: TermRef = ctx.requiredModuleRef("scala.runtime.quoted.Unpickler")
+  def UnpicklerModule(implicit ctx: Context): Symbol = UnpicklerModuleRef.symbol
+  lazy val UnpicklerType: TypeRef = ctx.requiredClassRef("scala.runtime.quoted.Unpickler")
+  def UnpicklerClass(implicit ctx: Context): ClassSymbol = UnpicklerType.symbol.asClass
+
+  lazy val Unpickler_unpickleExprR: TermRef = UnpicklerModule.requiredMethodRef("unpickleExpr")
+  def Unpickler_unpickleExpr: Symbol = Unpickler_unpickleExprR.symbol
+  lazy val Unpickler_liftedExprR: TermRef = UnpicklerModule.requiredMethodRef("liftedExpr")
+  def Unpickler_liftedExpr: Symbol = Unpickler_liftedExprR.symbol
+  lazy val Unpickler_unpickleTypeR: TermRef = UnpicklerModule.requiredMethodRef("unpickleType")
+  def Unpickler_unpickleType: Symbol = Unpickler_unpickleTypeR.symbol
 
   lazy val TastyTastyType: TypeRef = ctx.requiredClassRef("scala.tasty.Tasty")
   def TastyTastyClass(implicit ctx: Context): ClassSymbol = TastyTastyType.symbol.asClass
 
   lazy val TastyTastyModule: TermSymbol = ctx.requiredModule("scala.tasty.Tasty")
     lazy val TastyTasty_macroContext: TermSymbol = TastyTastyModule.requiredMethod("macroContext")
 
+  lazy val QuoteContextType: TypeRef = ctx.requiredClassRef("scala.quoted.QuoteContext")
+  def QuoteContextClass(implicit ctx: Context): ClassSymbol = QuoteContextType.symbol.asClass
+
+  lazy val QuoteContextModule: TermSymbol = ctx.requiredModule("scala.quoted.QuoteContext")
+  lazy val QuoteContext_macroContext: TermSymbol = QuoteContextModule.requiredMethod("macroContext")
+
   lazy val EqType: TypeRef = ctx.requiredClassRef("scala.Eq")
   def EqClass(implicit ctx: Context): ClassSymbol = EqType.symbol.asClass
   def EqModule(implicit ctx: Context): Symbol = EqClass.companionModule

compiler/src/dotty/tools/dotc/core/quoted/PickledQuotes.scala

@@ -22,12 +22,6 @@ import scala.reflect.ClassTag
 object PickledQuotes {
   import tpd._
 
-  /** Pickle the tree of the quoted.Expr */
-  def pickleExpr(tree: Tree)(implicit ctx: Context): scala.quoted.Expr[Any] = {
-    val pickled = pickleQuote(tree)
-    scala.runtime.quoted.Unpickler.unpickleExpr(pickled, Nil)
-  }
-
   /** Pickle the tree of the quote into strings */
   def pickleQuote(tree: Tree)(implicit ctx: Context): scala.runtime.quoted.Unpickler.Pickled = {
     if (ctx.reporter.hasErrors) Nil
@@ -40,13 +34,6 @@ object PickledQuotes {
 
   /** Transform the expression into its fully spliced Tree */
   def quotedExprToTree[T](expr: quoted.Expr[T])(implicit ctx: Context): Tree = expr match {
-    case expr: TastyExpr[_] =>
-      val unpickled = unpickleExpr(expr)
-      val force = new TreeTraverser {
-        def traverse(tree: tpd.Tree)(implicit ctx: Context): Unit = traverseChildren(tree)
-      }
-      force.traverse(unpickled)
-      unpickled
     case expr: LiftedExpr[T] =>
       expr.value match {
         case value: Class[_] => ref(defn.Predef_classOf).appliedToType(classToType(value))
@@ -59,21 +46,26 @@ object PickledQuotes {
 
   /** Transform the expression into its fully spliced TypeTree */
   def quotedTypeToTree(expr: quoted.Type[_])(implicit ctx: Context): Tree = expr match {
-    case expr: TastyType[_] => unpickleType(expr)
     case expr: TaggedType[_] => classTagToTypeTree(expr.ct)
     case expr: TreeType[Tree] @unchecked => expr.typeTree
   }
 
   /** Unpickle the tree contained in the TastyExpr */
-  private def unpickleExpr(expr: TastyExpr[_])(implicit ctx: Context): Tree = {
-    val tastyBytes = TastyString.unpickle(expr.tasty)
-    unpickle(tastyBytes, expr.args, isType = false)(ctx.addMode(Mode.ReadPositions))
+  def unpickleExpr(tasty: scala.runtime.quoted.Unpickler.Pickled, args: Seq[Any])(implicit ctx: Context): Tree = {
+    val tastyBytes = TastyString.unpickle(tasty)
+    val unpickled = unpickle(tastyBytes, args, isType = false)(ctx.addMode(Mode.ReadPositions))
+    // TODO force should not be done here it should be in run and show
+    val force = new TreeTraverser {
+      def traverse(tree: tpd.Tree)(implicit ctx: Context): Unit = traverseChildren(tree)
+    }
+    force.traverse(unpickled)
+    unpickled
   }
 
   /** Unpickle the tree contained in the TastyType */
-  private def unpickleType(ttpe: TastyType[_])(implicit ctx: Context): Tree = {
-    val tastyBytes = TastyString.unpickle(ttpe.tasty)
-    unpickle(tastyBytes, ttpe.args, isType = true)(ctx.addMode(Mode.ReadPositions))
+  def unpickleType(tasty: scala.runtime.quoted.Unpickler.Pickled, args: Seq[Any])(implicit ctx: Context): Tree = {
+    val tastyBytes = TastyString.unpickle(tasty)
+    unpickle(tastyBytes, args, isType = true)(ctx.addMode(Mode.ReadPositions))
   }
 
   // TASTY picklingtests/pos/quoteTest.scala

compiler/src/dotty/tools/dotc/quoted/ExprCompilationUnit.scala

@@ -3,9 +3,7 @@ package dotty.tools.dotc.quoted
 import dotty.tools.dotc.CompilationUnit
 import dotty.tools.dotc.util.NoSource
 
-import scala.quoted.Expr
+import scala.quoted.{Expr, QuoteContext}
 
 /* Compilation unit containing the contents of a quoted expression */
-class ExprCompilationUnit(val expr: Expr[_]) extends CompilationUnit(NoSource) {
-  override def toString: String = s"Expr($expr)"
-}
+class ExprCompilationUnit(val expr: QuoteContext => Expr[_]) extends CompilationUnit(NoSource)

compiler/src/dotty/tools/dotc/quoted/QuoteCompiler.scala

@@ -13,13 +13,15 @@ import dotty.tools.dotc.core.StdNames.nme
 import dotty.tools.dotc.core.Symbols.defn
 import dotty.tools.dotc.core.Types.ExprType
 import dotty.tools.dotc.core.quoted.PickledQuotes
+import dotty.tools.dotc.tastyreflect.TastyImpl
 import dotty.tools.dotc.transform.ReifyQuotes
 import dotty.tools.dotc.typer.FrontEnd
 import dotty.tools.dotc.util.Positions.Position
 import dotty.tools.dotc.util.SourceFile
 import dotty.tools.io.{Path, PlainFile}
 
-import scala.quoted.{Expr, Type}
+import scala.quoted.{Expr, QuoteContext, Type}
+import scala.runtime.quoted.Unpickler.Pickled
 
 /** Compiler that takes the contents of a quoted expression `expr` and produces
  *  a class file with `class ' { def apply: Object = expr }`.
@@ -50,7 +52,7 @@ class QuoteCompiler extends Compiler {
         case exprUnit: ExprCompilationUnit =>
           val tree =
             if (putInClass) inClass(exprUnit.expr)
-            else PickledQuotes.quotedExprToTree(exprUnit.expr)
+            else unpickleExpr(exprUnit.expr)
           val source = new SourceFile("", "")
           CompilationUnit.mkCompilationUnit(source, tree, forceTrees = true)
         case typeUnit: TypeCompilationUnit =>
@@ -61,11 +63,16 @@ class QuoteCompiler extends Compiler {
       }
     }
 
+    private def unpickleExpr(code: QuoteContext => Expr[_])(implicit ctx: Context) = {
+      val expr = code(new QuoteContextImpl)
+      PickledQuotes.quotedExprToTree(expr)
+    }
+
     /** Places the contents of expr in a compilable tree for a class
       *  with the following format.
       *  `package __root__ { class ' { def apply: Any = <expr> } }`
       */
-    private def inClass(expr: Expr[_])(implicit ctx: Context): Tree = {
+    private def inClass(code: QuoteContext => Expr[_])(implicit ctx: Context): Tree = {
       val pos = Position(0)
       val assocFile = new PlainFile(Path("<quote>"))
 
@@ -74,7 +81,7 @@ class QuoteCompiler extends Compiler {
       cls.enter(ctx.newDefaultConstructor(cls), EmptyScope)
       val meth = ctx.newSymbol(cls, nme.apply, Method, ExprType(defn.AnyType), coord = pos).entered
 
-      val quoted = PickledQuotes.quotedExprToTree(expr)(ctx.withOwner(meth))
+      val quoted = unpickleExpr(code)(ctx.withOwner(meth))
 
       val run = DefDef(meth, quoted)
       val classTree = ClassDef(cls, DefDef(cls.primaryConstructor.asTerm), run :: Nil)
@@ -83,7 +90,7 @@ class QuoteCompiler extends Compiler {
   }
 
   class ExprRun(comp: Compiler, ictx: Context) extends Run(comp, ictx) {
-    def compileExpr(expr: Expr[_]): Unit = {
+    def compileExpr(expr: QuoteContext => Expr[_]): Unit = {
       val units = new ExprCompilationUnit(expr) :: Nil
       compileUnits(units)
     }

compiler/src/dotty/tools/dotc/quoted/QuoteContextImpl.scala

@@ -0,0 +1,33 @@
+package dotty.tools.dotc.quoted
+
+import dotty.tools.dotc.core.Contexts._
+import dotty.tools.dotc.core.quoted.PickledQuotes
+import dotty.tools.dotc.tastyreflect.TastyImpl
+
+import scala.quoted.{Expr, Type}
+import scala.runtime.quoted.Unpickler.Pickled
+
+class QuoteContextImpl(implicit ctx: Context) extends scala.quoted.QuoteContext {
+
+  def unpickleExpr[T](repr: Pickled, args: Seq[Any]): Expr[T] =
+    new scala.quoted.Exprs.TastyTreeExpr(PickledQuotes.unpickleExpr(repr, args)).asInstanceOf[Expr[T]]
+
+  def unpickleType[T](repr: Pickled, args: Seq[Any]): Type[T] =
+    new scala.quoted.Types.TreeType(PickledQuotes.unpickleType(repr, args)).asInstanceOf[Type[T]]
+
+  def show[T](expr: Expr[T]): String = {
+    val tree = PickledQuotes.quotedExprToTree(expr)
+    // TODO freshen names
+    val tree1 =
+     if (ctx.settings.YshowRawTree.value) tree else (new TreeCleaner).transform(tree)
+    new TastyImpl(ctx).showSourceCode.showTree(tree1)
+  }
+
+  def show[T](tpe: Type[T]): String = {
+    val tree = PickledQuotes.quotedTypeToTree(tpe)
+    // TODO freshen names
+    val tree1 = if (ctx.settings.YshowRawTree.value) tree else (new TreeCleaner).transform(tree)
+    new TastyImpl(ctx).showSourceCode.showTypeOrBoundsTree(tree1)
+  }
+
+}

compiler/src/dotty/tools/dotc/quoted/QuoteDecompiler.scala

@@ -7,16 +7,17 @@ import dotty.tools.dotc.tastyreflect.TastyImpl
 import dotty.tools.io.{AbstractFile, Directory, PlainDirectory, VirtualDirectory}
 import dotty.tools.repl.AbstractFileClassLoader
 
-import scala.quoted.{Expr, Type}
+import scala.quoted.{Expr, QuoteContext, Type}
 import scala.quoted.Toolbox
 import java.net.URLClassLoader
 
+import dotty.tools.dotc.core.quoted.PickledQuotes
+
 class QuoteDriver extends Driver {
-  import tpd._
 
   private[this] val contextBase: ContextBase = new ContextBase
 
-  def run[T](expr: Expr[T], settings: Toolbox.Settings): T = {
+  def run[T](code: QuoteContext => Expr[T], settings: Toolbox.Settings): T = {
     val outDir: AbstractFile = settings.outDir match {
       case Some(out) =>
         val dir = Directory(out)
@@ -27,10 +28,10 @@ class QuoteDriver extends Driver {
     }
 
     val (_, ctx0: Context) = setup(settings.compilerArgs.toArray :+ "dummy.scala", initCtx.fresh)
-    val ctx = setColor(ctx0.fresh.setSetting(ctx0.settings.outputDir, outDir), settings)
+    val ctx = setQuoteSettings(ctx0.fresh.setSetting(ctx0.settings.outputDir, outDir), settings)
 
     val driver = new QuoteCompiler
-    driver.newRun(ctx).compileExpr(expr)
+    driver.newRun(ctx).compileExpr(code)
 
     val classLoader = new AbstractFileClassLoader(outDir, this.getClass.getClassLoader)
 
@@ -42,35 +43,17 @@ class QuoteDriver extends Driver {
   }
 
   def show(expr: Expr[_], settings: Toolbox.Settings): String = {
-    def show(tree: Tree, ctx: Context): String = {
-      val tree1 = if (settings.showRawTree) tree else (new TreeCleaner).transform(tree)(ctx)
-      new TastyImpl(ctx).showSourceCode.showTree(tree1)(ctx)
-    }
-    withTree(expr, show, settings)
-  }
-
-  def withTree[T](expr: Expr[_], f: (Tree, Context) => T, settings: Toolbox.Settings): T = {
-    val ctx = setColor(setup(settings.compilerArgs.toArray :+ "dummy.scala", initCtx.fresh)._2.fresh, settings)
-
-    var output: Option[T] = None
-    def registerTree(tree: tpd.Tree)(ctx: Context): Unit = {
-      assert(output.isEmpty)
-      output = Some(f(tree, ctx))
-    }
-    new QuoteDecompiler(registerTree).newRun(ctx).compileExpr(expr)
-    output.getOrElse(throw new Exception("Could not extract " + expr))
+    val (_, ctx: Context) = setup(settings.compilerArgs.toArray :+ "dummy.scala", initCtx.fresh)
+    val tree = PickledQuotes.quotedExprToTree(expr)(ctx)
+    val tree1 = if (settings.showRawTree) tree else (new TreeCleaner).transform(tree)(ctx)
+    new TastyImpl(ctx).showSourceCode.showTree(tree1)(ctx)
   }
 
-  def withTypeTree[T](tpe: Type[_], f: (TypTree, Context) => T, settings: Toolbox.Settings): T = {
+  def show(tpe: Type[_], settings: Toolbox.Settings): String = {
     val (_, ctx: Context) = setup(settings.compilerArgs.toArray :+ "dummy.scala", initCtx.fresh)
-
-    var output: Option[T] = None
-    def registerTree(tree: tpd.Tree)(ctx: Context): Unit = {
-      assert(output.isEmpty)
-      output = Some(f(tree.asInstanceOf[TypTree], ctx))
-    }
-    new QuoteDecompiler(registerTree).newRun(ctx).compileType(tpe)
-    output.getOrElse(throw new Exception("Could not extract " + tpe))
+    val tree = PickledQuotes.quotedTypeToTree(tpe)(ctx)
+    val tree1 = if (settings.showRawTree) tree else (new TreeCleaner).transform(tree)(ctx)
+    new TastyImpl(ctx).showSourceCode.showTypeOrBoundsTree(tree1)(ctx)
   }
 
   override def initCtx: Context = {
@@ -79,8 +62,10 @@ class QuoteDriver extends Driver {
     ictx
   }
 
-  private def setColor(ctx: FreshContext, settings: Toolbox.Settings): FreshContext =
+  private def setQuoteSettings(ctx: FreshContext, settings: Toolbox.Settings): ctx.type = {
     ctx.setSetting(ctx.settings.color, if (settings.color) "always" else "never")
+    ctx.setSetting(ctx.settings.YshowRawTree, settings.showRawTree)
+  }
 
 }
 

compiler/src/dotty/tools/dotc/quoted/QuoteDriver.scala

@@ -8,6 +8,7 @@ import dotty.tools.dotc.core.NameKinds.{NumberedInfo, UniqueName}
 import dotty.tools.dotc.core.SymDenotations.SymDenotation
 import dotty.tools.dotc.transform.MegaPhase.MiniPhase
 
+// TODO use this in some form again
 /** Refreshes local names starting from the second use of the name. Intended for readability of the pretty printed code. */
 class RefreshNames extends MiniPhase with SymTransformer {
 

compiler/src/dotty/tools/dotc/quoted/RefreshNames.scala

@@ -1,28 +1,18 @@
 package dotty.tools.dotc.quoted
 
-import dotty.tools.dotc.ast.tpd
-
-import scala.quoted.Expr
-import scala.quoted.Exprs.{LiftedExpr, TastyTreeExpr}
+import scala.quoted.{Expr, QuoteContext, Type}
 
 /** Default runners for quoted expressions */
 object ToolboxImpl {
-  import tpd._
 
   def make(settings: scala.quoted.Toolbox.Settings): scala.quoted.Toolbox = new scala.quoted.Toolbox {
 
     private[this] val driver: QuoteDriver = new QuoteDriver()
 
-    def run[T](expr: Expr[T]): T = expr match {
-      case expr: LiftedExpr[T] =>
-        expr.value
-      case expr: TastyTreeExpr[Tree] @unchecked =>
-        throw new Exception("Cannot call `Expr.run` on an `Expr` that comes from a macro argument.")
-      case _ =>
-        synchronized(driver.run(expr, settings))
+    protected def runImpl[T](code: QuoteContext => Expr[T]): T = {
+      // TODO check for recursion and throw if possible (i.e. run inside a run)
+      synchronized(driver.run(code, settings))
     }
 
-    def show[T](expr: Expr[T]): String = synchronized(driver.show(expr, settings))
-
   }
 }

compiler/src/dotty/tools/dotc/quoted/ToolboxImpl.scala

@@ -396,9 +396,18 @@ class ReifyQuotes extends MacroTransformWithImplicits {
         val meth =
           if (isType) ref(defn.Unpickler_unpickleType).appliedToType(originalTp)
           else ref(defn.Unpickler_unpickleExpr).appliedToType(originalTp.widen)
-        meth.appliedTo(
-          liftList(PickledQuotes.pickleQuote(body).map(x => Literal(Constant(x))), defn.StringType),
-          liftList(splices, defn.AnyType))
+        val unpickler = ctx.typer.inferImplicitArg(defn.UnpicklerType, body.pos)
+        unpickler.tpe match {
+          case fail: SearchFailureType =>
+            ctx.error("Missing an implicit to unpickle the quote. Use Staged[...] as return type", body.pos) // FIXME change text
+            EmptyTree
+          case _ =>
+            meth.appliedTo(
+              liftList(PickledQuotes.pickleQuote(body).map(x => Literal(Constant(x))), defn.StringType),
+              liftList(splices, defn.AnyType),
+              unpickler
+            )
+        }
       }
       if (splices.nonEmpty) pickleAsTasty()
       else if (isType) {