Merge pull request #10792 from dotty-staging/rework-tasty-inspector-api

Rework TastyInspector API to allow inspection of all files

Author: nicolasstucki

compiler/src/scala/quoted/runtime/impl/QuotesImpl.scala

@@ -2723,7 +2723,11 @@ class QuotesImpl private (using val ctx: Context) extends Quotes, QuoteUnpickler
     type SourceFile = dotc.util.SourceFile
 
     object SourceFile extends SourceFileModule {
-      def current: SourceFile = ctx.compilationUnit.source
+      def current: SourceFile =
+        if ctx.compilationUnit == null then
+          throw new java.lang.UnsupportedOperationException(
+            "`reflect.SourceFile.current` cannot be called within the TASTy ispector")
+        ctx.compilationUnit.source
     }
 
     given SourceFileMethods: SourceFileMethods with

docs/docs/reference/metaprogramming/tasty-inspect.md

@@ -10,7 +10,7 @@ libraryDependencies += "org.scala-lang" %% "scala3-tasty-inspector" % scalaVersi
 TASTy files contain the full typed tree of a class including source positions
 and documentation. This is ideal for tools that analyze or extract semantic
 information from the code. To avoid the hassle of working directly with the TASTy
-file we provide the `TastyInspector` which loads the contents and exposes it
+file we provide the `Inspector` which loads the contents and exposes it
 through the TASTy reflect API.
 
 ## Inspecting TASTy files
@@ -21,18 +21,22 @@ To inspect the trees of a TASTy file a consumer can be defined in the following
 import scala.quoted._
 import scala.tasty.inspector._
 
-class MyInspector extends TastyInspector:
-   protected def processCompilationUnit(using Quotes)(tree: quotes.reflect.Tree): Unit =
+class MyInspector extends Inspector:
+    def inspect(using Quotes)(tastys: List[Tasty[quotes.type]]): Unit =
       import quotes.reflect._
-      // Do something with the tree
+      for tasty <- tastys do
+        val tree = tasty.ast
+        // Do something with the tree
 ```
 
 Then the consumer can be instantiated with the following code to get the tree of the `foo/Bar.tasty` file.
 
 ```scala
 object Test:
    def main(args: Array[String]): Unit =
-      new MyInspector().inspectTastyFiles("foo/Bar.tasty")
+      val tastyFiles = List("foo/Bar.tasty")
+      TastyInspector.inspectTastyFiles(tastyFiles)(new MyInspector)
+
 ```
 
 Note that if we need to run the main (in the example below defined in an object called `Test`) after compilation we need to make the compiler available to the runtime:

sbt-dotty/sbt-test/sbt-dotty/tasty-inspector-example-project/app/Main.scala

@@ -1,18 +1,20 @@
 package hello
 
 import scala.quoted._
-import scala.tasty.inspector.TastyInspector
+import scala.tasty.inspector._
 
 import scala.jdk.StreamConverters._
 
 import java.nio.file.{Path, Files, Paths, FileSystems}
 
 object Main extends App {
 
-  val inspector = new TastyInspector {
-    protected def processCompilationUnit(using Quotes)(root: quotes.reflect.Tree): Unit = {
-      val tastyStr = root.show
-      println(tastyStr)
+  val inspector = new Inspector {
+    def inspect(using Quotes)(tastys: List[Tasty[quotes.type]]): Unit = {
+      for tasty <- tastys do
+        val tastyStr = tasty.ast.show
+        println(tastyStr)
+
     }
   }
 
@@ -25,7 +27,7 @@ object Main extends App {
 
     val tastyFiles = for p <- walk(pwd) if `lib/Foo.tasty`.matches(p) yield p.toString
 
-    inspector.inspectTastyFiles(List(tastyFiles.head))
+    TastyInspector.inspectTastyFiles(List(tastyFiles.head))(inspector)
 
   }
 

scala3doc/src/scala/tasty/inspector/DocTastyInspector.scala

@@ -2,7 +2,7 @@ package scala.tasty.inspector
 
 import dotty.tools.dotc.core.Contexts.Context
 
-abstract class DocTastyInspector extends TastyInspector:
+abstract class DocTastyInspector extends OldTastyInspector:
   def inspectFilesInDocContext(
     classpath: List[String],
     filePaths: List[String])(

scala3doc/src/scala/tasty/inspector/OldTastyInspector.scala

@@ -0,0 +1,133 @@
+package scala.tasty.inspector
+
+import scala.quoted._
+import scala.quoted.runtime.impl.QuotesImpl
+
+import dotty.tools.dotc.Compiler
+import dotty.tools.dotc.Driver
+import dotty.tools.dotc.Run
+import dotty.tools.dotc.core.Contexts.Context
+import dotty.tools.dotc.core.Mode
+import dotty.tools.dotc.core.Phases.Phase
+import dotty.tools.dotc.fromtasty._
+import dotty.tools.dotc.util.ClasspathFromClassloader
+import dotty.tools.dotc.CompilationUnit
+import dotty.tools.unsupported
+import dotty.tools.dotc.report
+
+import java.io.File.pathSeparator
+
+// COPY OF OLD IMPLEMENTATION
+// TODO: update to new implementation
+trait OldTastyInspector:
+  self =>
+
+  /** Process a TASTy file using TASTy reflect */
+  protected def processCompilationUnit(using Quotes)(root: quotes.reflect.Tree): Unit
+
+  /** Called after all compilation units are processed */
+  protected def postProcess(using Quotes): Unit = ()
+
+  /** Load and process TASTy files using TASTy reflect
+   *
+   *  @param tastyFiles List of paths of `.tasty` files
+   */
+  def inspectTastyFiles(tastyFiles: List[String]): Boolean =
+    inspectAllTastyFiles(tastyFiles, Nil, Nil)
+
+  /** Load and process TASTy files in a `jar` file using TASTy reflect
+   *
+   *  @param jars Path of `.jar` file
+   */
+  def inspectTastyFilesInJar(jar: String): Boolean =
+    inspectAllTastyFiles(Nil, List(jar), Nil)
+
+  /** Load and process TASTy files using TASTy reflect
+   *
+   *  @param tastyFiles List of paths of `.tasty` files
+   *  @param jars List of path of `.jar` files
+   *  @param dependenciesClasspath Classpath with extra dependencies needed to load class in the `.tasty` files
+   */
+  def inspectAllTastyFiles(tastyFiles: List[String], jars: List[String], dependenciesClasspath: List[String]): Boolean =
+    def checkFile(fileName: String, ext: String): Unit =
+      val file = dotty.tools.io.Path(fileName)
+      if file.extension != ext then
+        throw new IllegalArgumentException(s"File extension is not `.$ext`: $file")
+      else if !file.exists then
+        throw new IllegalArgumentException(s"File not found: ${file.toAbsolute}")
+    tastyFiles.foreach(checkFile(_, "tasty"))
+    jars.foreach(checkFile(_, "jar"))
+    val files = tastyFiles ::: jars
+    files.nonEmpty && inspectFiles(dependenciesClasspath, files)
+
+  /** Load and process TASTy files using TASTy reflect and provided context
+   *
+   *  Used in doctool to reuse reporter and setup provided by sbt
+   *
+   *  @param classes List of paths of `.tasty` and `.jar` files (no validation is performed)
+   *  @param classpath Classpath with extra dependencies needed to load class in the `.tasty` files
+   */
+  protected[inspector] def inspectFilesInContext(classpath: List[String], classes: List[String])(using Context): Unit =
+    if (classes.isEmpty) report.error("Parameter classes should no be empty")
+    inspectorDriver().process(inspectorArgs(classpath, classes), summon[Context])
+
+
+  private def inspectorDriver() =
+    class InspectorDriver extends Driver:
+      override protected def newCompiler(implicit ctx: Context): Compiler = new TastyFromClass
+
+    class TastyInspectorPhase extends Phase:
+      override def phaseName: String = "tastyInspector"
+
+      override def run(implicit ctx: Context): Unit =
+        val qctx = QuotesImpl()
+        self.processCompilationUnit(using qctx)(ctx.compilationUnit.tpdTree.asInstanceOf[qctx.reflect.Tree])
+
+    class TastyInspectorFinishPhase extends Phase:
+      override def phaseName: String = "tastyInspectorFinish"
+
+      override def runOn(units: List[CompilationUnit])(using Context): List[CompilationUnit] =
+        val qctx = QuotesImpl()
+        self.postProcess(using qctx)
+        units
+
+      override def run(implicit ctx: Context): Unit = unsupported("run")
+
+    class TastyFromClass extends TASTYCompiler:
+
+      override protected def frontendPhases: List[List[Phase]] =
+        List(new ReadTasty) :: // Load classes from tasty
+        Nil
+
+      override protected def picklerPhases: List[List[Phase]] = Nil
+
+      override protected def transformPhases: List[List[Phase]] = Nil
+
+      override protected def backendPhases: List[List[Phase]] =
+        List(new TastyInspectorPhase) ::  // Perform a callback for each compilation unit
+        List(new TastyInspectorFinishPhase) :: // Perform a final callback
+        Nil
+
+      override def newRun(implicit ctx: Context): Run =
+        reset()
+        new TASTYRun(this, ctx.fresh.addMode(Mode.ReadPositions).addMode(Mode.ReadComments))
+
+    new InspectorDriver
+
+  private def inspectorArgs(classpath: List[String], classes: List[String]): Array[String] =
+    val currentClasspath = ClasspathFromClassloader(getClass.getClassLoader)
+    val fullClasspath = (classpath :+ currentClasspath).mkString(pathSeparator)
+    ("-from-tasty" :: "-Yretain-trees" :: "-classpath" :: fullClasspath :: classes).toArray
+
+
+  private def inspectFiles(classpath: List[String], classes: List[String]): Boolean =
+    if (classes.isEmpty)
+      throw new IllegalArgumentException("Parameter classes should no be empty")
+
+    val reporter = inspectorDriver().process(inspectorArgs(classpath, classes))
+    reporter.hasErrors
+
+  end inspectFiles
+
+
+end OldTastyInspector

scala3doc/test/dotty/dokka/tasty/comments/CommentExpanderTests.scala

@@ -37,8 +37,8 @@ class CommentExpanderTests {
 
   @Test
   def test(): Unit = {
-    import scala.tasty.inspector.TastyInspector
-    class Inspector extends TastyInspector:
+    import scala.tasty.inspector.OldTastyInspector
+    class Inspector extends OldTastyInspector:
 
       def processCompilationUnit(using quoted.Quotes)(root: quotes.reflect.Tree): Unit = ()
 

scala3doc/test/dotty/dokka/tasty/comments/MemberLookupTests.scala

@@ -101,8 +101,8 @@ class MemberLookupTests {
 
   @Test
   def test(): Unit = {
-    import scala.tasty.inspector.TastyInspector
-    class Inspector extends TastyInspector:
+    import scala.tasty.inspector.OldTastyInspector
+    class Inspector extends OldTastyInspector:
       var alreadyRan: Boolean = false
 
       override def processCompilationUnit(using ctx: quoted.Quotes)(root: ctx.reflect.Tree): Unit =

stdlib-bootstrapped-tasty-tests/test/BootstrappedStdLibTASYyTest.scala

@@ -8,6 +8,7 @@ import dotty.tools.io._
 import dotty.tools.dotc.util.ClasspathFromClassloader
 
 import scala.quoted._
+import scala.tasty.inspector._
 
 import java.io.File.pathSeparator
 import java.io.File.separator
@@ -101,13 +102,14 @@ object BootstrappedStdLibTASYyTest:
       .toList
 
   def loadWithTastyInspector(blacklisted: Set[String]): Unit =
-    val inspector = new scala.tasty.inspector.TastyInspector {
-      def processCompilationUnit(using Quotes)(root: quotes.reflect.Tree): Unit =
-        root.show(using quotes.reflect.Printer.TreeStructure) // Check that we can traverse the full tree
+    val inspector = new scala.tasty.inspector.Inspector {
+      def inspect(using Quotes)(tastys: List[Tasty[quotes.type]]): Unit =
+        for tasty <- tastys do
+          tasty.ast.show(using quotes.reflect.Printer.TreeStructure) // Check that we can traverse the full tree
         ()
     }
     val tastyFiles = scalaLibTastyPaths.filterNot(blacklisted)
-    val hasErrors = inspector.inspectTastyFiles(tastyFiles.map(x => scalaLibClassesPath.resolve(x).toString))
+    val hasErrors = TastyInspector.inspectTastyFiles(tastyFiles.map(x => scalaLibClassesPath.resolve(x).toString))(inspector)
     assert(!hasErrors, "Errors reported while loading from TASTy")
 
   def compileFromTastyInJar(blacklisted: Set[String]): Unit = {

tasty-inspector/src/scala/tasty/inspector/Inspector.scala

@@ -0,0 +1,30 @@
+package scala.tasty.inspector
+
+import scala.quoted._
+import scala.quoted.runtime.impl.QuotesImpl
+
+import dotty.tools.dotc.Compiler
+import dotty.tools.dotc.Driver
+import dotty.tools.dotc.Run
+import dotty.tools.dotc.core.Contexts.Context
+import dotty.tools.dotc.core.Mode
+import dotty.tools.dotc.core.Phases.Phase
+import dotty.tools.dotc.fromtasty._
+import dotty.tools.dotc.util.ClasspathFromClassloader
+import dotty.tools.dotc.CompilationUnit
+import dotty.tools.unsupported
+import dotty.tools.dotc.report
+
+import java.io.File.pathSeparator
+
+trait Inspector:
+
+  /** Inspect all TASTy files using `Quotes` reflect API.
+   *
+   *  Note: Within this method `quotes.reflect.SourceFile.current` will not work, hence the explicit source paths.
+   *
+   *  @param tastys List of `Tasty` containing `.tasty`file path and AST
+   */
+  def inspect(using Quotes)(tastys: List[Tasty[quotes.type]]): Unit
+
+end Inspector

tasty-inspector/src/scala/tasty/inspector/Tasty.scala

@@ -0,0 +1,17 @@
+package scala.tasty.inspector
+
+import scala.quoted._
+
+/** `.tasty` file representation containing file path and the AST */
+trait Tasty[Q <: Quotes & Singleton]:
+
+  /** Instance of `Quotes` used to load the AST */
+  val quotes: Q
+
+  /** Path to the `.tasty` file */
+  def path: String
+
+  /** Abstract Syntax Tree contained in the `.tasty` file */
+  def ast: quotes.reflect.Tree
+
+end Tasty