Extract parser into its own phase

Author: allanrenucci

compiler/src/dotty/tools/dotc/Compiler.scala

@@ -9,6 +9,7 @@ import transform._
 import dotty.tools.backend.jvm.{CollectSuperCalls, GenBCode}
 import dotty.tools.backend.sjs
 import dotty.tools.dotc.transform.localopt.StringInterpolatorOpt
+import dotty.tools.dotc.parsing.Parsing
 
 /** The central class of the dotc compiler. The job of a compiler is to create
  *  runs, which process given `phases` in a given `rootContext`.
@@ -36,6 +37,7 @@ class Compiler {
 
   /** Phases dealing with the frontend up to trees ready for TASTY pickling */
   protected def frontendPhases: List[List[Phase]] =
+    List(new Parsing) ::
     List(new FrontEnd) ::           // Compiler frontend: scanner, parser, namer, typer
     List(new YCheckPositions) ::    // YCheck positions
     List(new Staging) ::            // Check PCP, heal quoted types and expand macros

compiler/src/dotty/tools/dotc/Run.scala

@@ -155,7 +155,7 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint
           Stats.trackTime(s"$phase ms ") {
             val start = System.currentTimeMillis
             val profileBefore = profiler.beforePhase(phase)
-            units = phase.runOn(units)
+            units = phase.runOn(units)(ctx.fresh.setPhase(phase.start))
             profiler.afterPhase(phase, profileBefore)
             if (ctx.settings.Xprint.value.containsPhase(phase)) {
               for (unit <- units) {
@@ -217,7 +217,10 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint
     val unit = ctx.compilationUnit
     val prevPhase = ctx.phase.prev // can be a mini-phase
     val squashedPhase = ctx.base.squashed(prevPhase)
-    val treeString = unit.tpdTree.show(ctx.withProperty(XprintMode, Some(())))
+    val tree =
+      if (ctx.isAfterTyper) unit.tpdTree
+      else unit.untpdTree
+    val treeString = tree.show(ctx.withProperty(XprintMode, Some(())))
 
     ctx.echo(s"result of $unit after $squashedPhase:")
 

compiler/src/dotty/tools/dotc/core/Phases.scala

@@ -10,6 +10,7 @@ import Denotations._
 import Decorators._
 import config.Printers.config
 import scala.collection.mutable.ListBuffer
+import scala.util.control.NonFatal
 import dotty.tools.dotc.transform.MegaPhase._
 import dotty.tools.dotc.transform._
 import Periods._
@@ -312,7 +313,7 @@ object Phases {
     /** @pre `isRunnable` returns true */
     def runOn(units: List[CompilationUnit])(implicit ctx: Context): List[CompilationUnit] =
       units.map { unit =>
-        val unitCtx = ctx.fresh.setPhase(this.start).setCompilationUnit(unit)
+        val unitCtx = ctx.fresh.setCompilationUnit(unit)
         run(unitCtx)
         unitCtx.compilationUnit
       }
@@ -327,7 +328,7 @@ object Phases {
     def checkPostCondition(tree: tpd.Tree)(implicit ctx: Context): Unit = ()
 
     /** Is this phase the standard typerphase? True for FrontEnd, but
-     *  not for other first phases (such as FromTasty). The predicate
+     *  not for other first phases (such as ReadTasty). The predicate
      *  is tested in some places that perform checks and corrections. It's
      *  different from isAfterTyper (and cheaper to test).
      */
@@ -412,6 +413,14 @@ object Phases {
     final def iterator: Iterator[Phase] =
       Iterator.iterate(this)(_.next) takeWhile (_.hasNext)
 
+    final def monitor(doing: String)(body: => Unit)(implicit ctx: Context): Unit =
+      try body
+      catch {
+        case NonFatal(ex) =>
+          ctx.echo(s"exception occurred while $doing ${ctx.compilationUnit}")
+          throw ex
+      }
+
     override def toString: String = phaseName
   }
 

compiler/src/dotty/tools/dotc/interactive/InteractiveCompiler.scala

@@ -5,13 +5,15 @@ package interactive
 import core._
 import Phases._
 import typer._
+import parsing.Parsing
 
 class InteractiveCompiler extends Compiler {
   // TODO: Figure out what phases should be run in IDEs
   // More phases increase latency but allow us to report more errors.
   // This could be improved by reporting errors back to the IDE
   // after each phase group instead of waiting for the pipeline to finish.
   override def phases: List[List[Phase]] = List(
+    List(new Parsing),
     List(new FrontEnd),
     List(new transform.SetRootTree),
     List(new transform.CookComments)

compiler/src/dotty/tools/dotc/parsing/Parsing.scala

@@ -0,0 +1,50 @@
+package dotty.tools.dotc.parsing
+
+import dotty.tools.dotc.ast.Trees
+import dotty.tools.dotc.config.Config
+import dotty.tools.dotc.core.Contexts.Context
+import dotty.tools.dotc.core.Phases.Phase
+import dotty.tools.dotc.core.Symbols.defn
+import dotty.tools.dotc.parsing.JavaParsers.JavaParser
+import dotty.tools.dotc.parsing.Parsers.Parser
+import dotty.tools.dotc.util.{SourcePosition, NoSourcePosition}
+import dotty.tools.dotc.util.Stats.record
+
+class Parsing extends Phase {
+
+  override def phaseName: String = "parsing"
+
+  // We run TreeChecker only after type checking
+  override def isCheckable: Boolean = false
+
+  override def isRunnable(implicit ctx: Context): Boolean =
+    !ctx.settings.fromTasty.value
+
+  /** The position of the first XML literal encountered while parsing,
+   *  NoSourcePosition if there were no XML literals.
+   */
+  private[this] var firstXmlPos: SourcePosition = NoSourcePosition
+
+  override def run(implicit ctx: Context): Unit = monitor("parsing") {
+    val unit = ctx.compilationUnit
+    unit.untpdTree =
+      if (unit.isJava) new JavaParser(unit.source).parse()
+      else {
+        val p = new Parser(unit.source)
+        val tree = p.parse()
+        if (p.firstXmlPos.exists && !firstXmlPos.exists)
+          firstXmlPos = p.firstXmlPos
+        tree
+      }
+
+    record("parsedTrees", Trees.ntrees)
+
+    if (firstXmlPos.exists && !defn.ScalaXmlPackageClass.exists)
+      ctx.error("""To support XML literals, your project must depend on scala-xml.
+                  |See https://github.com/scala/scala-xml for more information.""".stripMargin,
+        firstXmlPos)
+
+    if (Config.checkPositions)
+      unit.untpdTree.checkPos(nonOverlapping = !unit.isJava && !ctx.reporter.hasErrors)
+  }
+}

compiler/src/dotty/tools/dotc/typer/FrontEnd.scala

@@ -6,79 +6,48 @@ import core._
 import Phases._
 import Contexts._
 import Symbols._
-import dotty.tools.dotc.parsing.JavaParsers.JavaParser
-import parsing.Parsers.Parser
 import config.Config
 import config.Printers.{typr, default}
 import util.Stats._
-import util.{ SourcePosition, NoSourcePosition }
 import scala.util.control.NonFatal
-import ast.Trees._
+import ast.{tpd, Trees}
+import Trees._
 
 class FrontEnd extends Phase {
+  import tpd._
 
   override def phaseName: String = FrontEnd.name
   override def isTyper: Boolean = true
-  import ast.tpd
+
+  // Run regardless of parsing errors
+  override def isRunnable(implicit ctx: Context): Boolean = true
 
   override def allowsImplicitSearch: Boolean = true
 
   /** The contexts for compilation units that are parsed but not yet entered */
   private[this] var remaining: List[Context] = Nil
 
-  /** The position of the first XML literal encountered while parsing,
-   *  NoSourcePosition if there were no XML literals.
-   */
-  private[this] var firstXmlPos: SourcePosition = NoSourcePosition
-
   /** Does a source file ending with `<name>.scala` belong to a compilation unit
    *  that is parsed but not yet entered?
    */
   def stillToBeEntered(name: String): Boolean =
     remaining.exists(_.compilationUnit.toString.endsWith(name + ".scala"))
 
-  def monitor(doing: String)(body: => Unit)(implicit ctx: Context): Unit =
-    try body
-    catch {
-      case NonFatal(ex) =>
-        ctx.echo(s"exception occurred while $doing ${ctx.compilationUnit}")
-        throw ex
-    }
-
-  def parse(implicit ctx: Context): Unit = monitor("parsing") {
-    val unit = ctx.compilationUnit
-
-    unit.untpdTree =
-      if (unit.isJava) new JavaParser(unit.source).parse()
-      else {
-        val p = new Parser(unit.source)
-        val tree = p.parse()
-        if (p.firstXmlPos.exists && !firstXmlPos.exists)
-          firstXmlPos = p.firstXmlPos
-        tree
-      }
-
-    val printer = if (ctx.settings.Xprint.value.contains("parser")) default else typr
-    printer.println("parsed:\n" + unit.untpdTree.show)
-    if (Config.checkPositions)
-      unit.untpdTree.checkPos(nonOverlapping = !unit.isJava && !ctx.reporter.hasErrors)
-  }
-
-  def enterSyms(implicit ctx: Context): Unit = monitor("indexing") {
+  private def enterSyms(implicit ctx: Context): Unit = monitor("indexing") {
     val unit = ctx.compilationUnit
     ctx.typer.index(unit.untpdTree)
     typr.println("entered: " + unit.source)
   }
 
-  def typeCheck(implicit ctx: Context): Unit = monitor("typechecking") {
+  private def typeCheck(implicit ctx: Context): Unit = monitor("typechecking") {
     val unit = ctx.compilationUnit
     unit.tpdTree = ctx.typer.typedExpr(unit.untpdTree)
     typr.println("typed: " + unit.source)
     record("retained untyped trees", unit.untpdTree.treeSize)
     record("retained typed trees after typer", unit.tpdTree.treeSize)
   }
 
-  private def firstTopLevelDef(trees: List[tpd.Tree])(implicit ctx: Context): Symbol = trees match {
+  private def firstTopLevelDef(trees: List[Tree])(implicit ctx: Context): Symbol = trees match {
     case PackageDef(_, defs) :: _    => firstTopLevelDef(defs)
     case Import(_, _, _) :: defs     => firstTopLevelDef(defs)
     case (tree @ TypeDef(_, _)) :: _ => tree.symbol
@@ -93,21 +62,15 @@ class FrontEnd extends Phase {
       ctx.inform(s"compiling ${unit.source}")
       ctx.fresh.setCompilationUnit(unit)
     }
-    unitContexts foreach (parse(_))
-    record("parsedTrees", ast.Trees.ntrees)
+
     remaining = unitContexts
     while (remaining.nonEmpty) {
       enterSyms(remaining.head)
       remaining = remaining.tail
     }
 
-    if (firstXmlPos.exists && !defn.ScalaXmlPackageClass.exists)
-      ctx.error("""To support XML literals, your project must depend on scala-xml.
-                  |See https://github.com/scala/scala-xml for more information.""".stripMargin,
-        firstXmlPos)
-
     unitContexts.foreach(typeCheck(_))
-    record("total trees after typer", ast.Trees.ntrees)
+    record("total trees after typer", Trees.ntrees)
     unitContexts.map(_.compilationUnit).filterNot(discardAfterTyper)
   }
 

compiler/src/dotty/tools/repl/ReplCompiler.scala

@@ -13,7 +13,7 @@ import dotty.tools.dotc.core.StdNames._
 import dotty.tools.dotc.core.Symbols._
 import dotty.tools.dotc.reporting.diagnostic.messages
 import dotty.tools.dotc.transform.{PostTyper, Staging}
-import dotty.tools.dotc.typer.ImportInfo
+import dotty.tools.dotc.typer.{FrontEnd, ImportInfo}
 import dotty.tools.dotc.util.Spans._
 import dotty.tools.dotc.util.{ParsedComment, SourceFile}
 import dotty.tools.dotc.{CompilationUnit, Compiler, Run}
@@ -32,7 +32,7 @@ import scala.collection.mutable
 class ReplCompiler extends Compiler {
 
   override protected def frontendPhases: List[List[Phase]] = List(
-    List(new REPLFrontEnd),
+    List(new FrontEnd),
     List(new CollectTopLevelImports),
     List(new Staging),
     List(new PostTyper)

compiler/src/dotty/tools/repl/ReplFrontEnd.scala

@@ -11,6 +11,7 @@ import dotc.typer.FrontEnd
 
 import dotty.tools.dotc.fromtasty.{ReadTasty, TASTYRun}
 import dotty.tools.dotc.transform.CookComments
+import dotty.tools.dotc.parsing.Parsing
 
 /** Custom Compiler with phases for the documentation tool
  *
@@ -53,6 +54,7 @@ class DocCompiler extends Compiler {
   }
 
   override def phases: List[List[Phase]] = List(
+    List(new Parsing),
     List(new DocFrontEnd),
     List(new ReadTasty),
     List(new CookComments),