Fix scala#8020: Create dotty-tasty-consumer library

Author: nicolasstucki

.drone.yml

@@ -40,7 +40,7 @@ steps:
   depends_on: [ clone ]
   commands:
   - cp -R . /tmp/2/ && cd /tmp/2/
-  - ./project/scripts/sbt ";dotty-bootstrapped/compile ;dotty-bootstrapped/test ;dotty-staging/test ;sjsSandbox/run;sjsSandbox/test;sjsJUnitTests/test ;configureIDE"
+  - ./project/scripts/sbt ";dotty-bootstrapped/compile ;dotty-bootstrapped/test ;dotty-staging/test ;dotty-tasty-consumer/test  ;sjsSandbox/run;sjsSandbox/test;sjsJUnitTests/test ;configureIDE"
   - ./project/scripts/bootstrapCmdTests
 
 - name: community_build

build.sbt

@@ -11,6 +11,7 @@ val `dotty-library-bootstrappedJS` = Build.`dotty-library-bootstrappedJS`
 val `dotty-sbt-bridge` = Build.`dotty-sbt-bridge`
 val `dotty-sbt-bridge-tests` = Build.`dotty-sbt-bridge-tests`
 val `dotty-staging` = Build.`dotty-staging`
+val `dotty-tasty-consumer` = Build.`dotty-tasty-consumer`
 val `dotty-language-server` = Build.`dotty-language-server`
 val `dotty-bench` = Build.`dotty-bench`
 val `dotty-bench-bootstrapped` = Build.`dotty-bench-bootstrapped`

compiler/test/dotty/Properties.scala

@@ -53,6 +53,9 @@ object Properties {
   /** dotty-staging jar */
   def dottyStaging: String = sys.props("dotty.tests.classes.dottyStaging")
 
+  /** dotty-tasty-consumer jar */
+  def dottyTastyConsumer: String = sys.props("dotty.tests.classes.dottyTastyConsumer")
+
   /** tasty-core jar */
   def tastyCore: String = sys.props("dotty.tests.classes.tastyCore")
 

compiler/test/dotty/tools/dotc/BootstrappedOnlyCompilationTests.scala

@@ -127,7 +127,8 @@ class BootstrappedOnlyCompilationTests extends ParallelTesting {
     aggregateTests(
       compileFilesInDir("tests/run-with-compiler", withCompilerOptions),
       compileFilesInDir("tests/run-staging", withStagingOptions),
-      compileDir("tests/run-with-compiler-custom-args/tasty-interpreter", withCompilerOptions)
+      compileFilesInDir("tests/run-custom-args/tasty-consumer", withTastyConsumerOptions),
+      compileDir("tests/run-custom-args/tasty-interpreter", withTastyConsumerOptions),
     ).checkRuns()
   }
 

compiler/test/dotty/tools/vulpix/TestConfiguration.scala

@@ -39,6 +39,9 @@ object TestConfiguration {
   lazy val withStagingClasspath =
     withCompilerClasspath + File.pathSeparator + mkClasspath(List(Properties.dottyStaging))
 
+  lazy val withTastyConsumerClasspath =
+    withCompilerClasspath + File.pathSeparator + mkClasspath(List(Properties.dottyTastyConsumer))
+
   def mkClasspath(classpaths: List[String]): String =
     classpaths.map({ p =>
       val file = new java.io.File(p)
@@ -54,6 +57,8 @@ object TestConfiguration {
     defaultOptions.withClasspath(withCompilerClasspath).withRunClasspath(withCompilerClasspath)
   lazy val withStagingOptions =
     defaultOptions.withClasspath(withStagingClasspath).withRunClasspath(withStagingClasspath)
+  lazy val withTastyConsumerOptions =
+    defaultOptions.withClasspath(withTastyConsumerClasspath).withRunClasspath(withTastyConsumerClasspath)
   val allowDeepSubtypes = defaultOptions without "-Yno-deep-subtypes"
   val allowDoubleBindings = defaultOptions without "-Yno-double-bindings"
   val picklingOptions = defaultOptions and (

dist/bin/common

@@ -108,6 +108,7 @@ DOTTY_COMP=$(find_lib "*dotty-compiler*")
 DOTTY_INTF=$(find_lib "*dotty-interfaces*")
 DOTTY_LIB=$(find_lib "*dotty-library*")
 DOTTY_STAGING=$(find_lib "*dotty-staging*")
+DOTTY_CONSUME_TASTY=$(find_lib "*dotty-consume-tasty*")
 TASTY_CORE=$(find_lib "*tasty-core*")
 SCALA_ASM=$(find_lib "*scala-asm*")
 SCALA_LIB=$(find_lib "*scala-library*")

dist/bin/dotr

@@ -102,7 +102,7 @@ elif [ $execute_repl == true ] || [ ${#residual_args[@]} -ne 0 ]; then
     echo "warning: multiple classpaths are found, dotr only use the last one."
   fi
   if [ $with_compiler == true ]; then
-    cp_arg+="$PSEP$DOTTY_COMP$PSEP$TASTY_CORE$PSEP$DOTTY_INTF$PSEP$SCALA_ASM$PSEP$DOTTY_STAGING"
+    cp_arg+="$PSEP$DOTTY_COMP$PSEP$TASTY_CORE$PSEP$DOTTY_INTF$PSEP$SCALA_ASM$PSEP$DOTTY_STAGING$PSEP$DOTTY_CONSUME_TASTY"
   fi
   eval exec "\"$JAVACMD\"" "$DEBUG"  "-classpath \"$cp_arg\"" "${jvm_options[@]}" "${residual_args[@]}"
 else

docs/docs/reference/metaprogramming/tasty-inspect.md

@@ -33,4 +33,18 @@ object Test {
     ConsumeTasty("", List("foo.Bar"), new Consumer)
   }
 }
-```
+```
+
+Note that if we need to run the main (in an object called `Test`) after
+compilation we need make available the compiler to the runtime:
+
+```shell
+dotc -with-compiler -d out Test.scala
+dotr -with-compiler -classpath out Test
+```
+
+Or, from SBT:
+
+```scala
+libraryDependencies += "ch.epfl.lamp" %% "dotty-tasty-consumer" % scalaVersion.value
+```

library/src/scala/tasty/file/ConsumeTasty.scala

@@ -595,9 +595,10 @@ object Build {
           val asm = findArtifactPath(externalDeps, "scala-asm")
           val dottyCompiler = jars("dotty-compiler")
           val dottyStaging = jars("dotty-staging")
+          val dottyTastyConsumer = jars("dotty-tasty-consumer")
           val dottyInterfaces = jars("dotty-interfaces")
           val tastyCore = jars("tasty-core")
-          run(insertClasspathInArgs(args1, List(dottyCompiler, dottyInterfaces, asm, dottyStaging, tastyCore).mkString(File.pathSeparator)))
+          run(insertClasspathInArgs(args1, List(dottyCompiler, dottyInterfaces, asm, dottyStaging, dottyTastyConsumer, tastyCore).mkString(File.pathSeparator)))
         } else run(args)
       },
 
@@ -671,9 +672,10 @@ object Build {
       }
       val dottyInterfaces = jars("dotty-interfaces")
       val dottyStaging = jars("dotty-staging")
+      val dottyTastyConsumer = jars("dotty-tasty-consumer")
       val tastyCore = jars("tasty-core")
       val asm = findArtifactPath(externalDeps, "scala-asm")
-      extraClasspath ++= Seq(dottyCompiler, dottyInterfaces, asm, dottyStaging, tastyCore)
+      extraClasspath ++= Seq(dottyCompiler, dottyInterfaces, asm, dottyStaging, dottyTastyConsumer, tastyCore)
     }
 
     val fullArgs = main :: insertClasspathInArgs(args, extraClasspath.mkString(File.pathSeparator))
@@ -715,14 +717,18 @@ object Build {
   )
 
   lazy val bootstrapedDottyCompilerSettings = commonDottyCompilerSettings ++ Seq(
-    javaOptions += {
+    javaOptions ++= {
       val jars = packageAll.value
-      "-Ddotty.tests.classes.dottyStaging=" + jars("dotty-staging")
+      Seq(
+        "-Ddotty.tests.classes.dottyStaging=" + jars("dotty-staging"),
+        "-Ddotty.tests.classes.dottyTastyConsumer=" + jars("dotty-tasty-consumer"),
+      )
     },
     packageAll := {
       packageAll.in(`dotty-compiler`).value ++ Seq(
         "dotty-compiler" -> packageBin.in(Compile).value.getAbsolutePath,
         "dotty-staging"  -> packageBin.in(LocalProject("dotty-staging"), Compile).value.getAbsolutePath,
+        "dotty-tasty-consumer"  -> packageBin.in(LocalProject("dotty-tasty-consumer"), Compile).value.getAbsolutePath,
         "tasty-core"     -> packageBin.in(LocalProject("tasty-core-bootstrapped"), Compile).value.getAbsolutePath,
       )
     }
@@ -803,6 +809,20 @@ object Build {
       javaOptions := (javaOptions in `dotty-compiler-bootstrapped`).value
     )
 
+  lazy val `dotty-tasty-consumer` = project.in(file("tasty-consumer")).
+    withCommonSettings(Bootstrapped).
+    // FIXME: update comment
+    // We want the compiler to be present in the compiler classpath when compiling this project but not
+    // when compiling a project that depends on dotty-staging (see sbt-dotty/sbt-test/sbt-dotty/quoted-example-project),
+    // but we always need it to be present on the JVM classpath at runtime.
+    dependsOn(dottyCompiler(Bootstrapped) % "provided").
+    dependsOn(dottyCompiler(Bootstrapped) % "compile->runtime").
+    dependsOn(dottyCompiler(Bootstrapped) % "test->test").
+    settings(commonBootstrappedSettings).
+    settings(
+      javaOptions := (javaOptions in `dotty-compiler-bootstrapped`).value
+    )
+
   lazy val `dotty-sbt-bridge` = project.in(file("sbt-bridge/src")).
     // We cannot depend on any bootstrapped project to compile the bridge, since the
     // bridge is needed to compile these projects.
@@ -1106,6 +1126,7 @@ object Build {
         publishLocal in `dotty-library-bootstrapped`,
         publishLocal in `tasty-core-bootstrapped`,
         publishLocal in `dotty-staging`,
+        publishLocal in `dotty-tasty-consumer`,
         publishLocal in `scala-library`,
         publishLocal in `scala-reflect`,
         publishLocal in `dotty-doc-bootstrapped`,
@@ -1315,7 +1336,7 @@ object Build {
     // FIXME: we do not aggregate `bin` because its tests delete jars, thus breaking other tests
     def asDottyRoot(implicit mode: Mode): Project = project.withCommonSettings.
       aggregate(`dotty-interfaces`, dottyLibrary, dottyCompiler, tastyCore, dottyDoc, `dotty-sbt-bridge`).
-      bootstrappedAggregate(`scala-library`, `scala-compiler`, `scala-reflect`, scalap, `dotty-language-server`, `dotty-staging`).
+      bootstrappedAggregate(`scala-library`, `scala-compiler`, `scala-reflect`, scalap, `dotty-language-server`, `dotty-staging`, `dotty-tasty-consumer`).
       dependsOn(tastyCore).
       dependsOn(dottyCompiler).
       dependsOn(dottyLibrary).

project/Build.scala

@@ -1,4 +1,4 @@
-package dotty.tools.dotc.consumetasty
+package scala.tasty.file
 
 import java.net.URLClassLoader
 
@@ -14,6 +14,12 @@ import scala.tasty.file.TastyConsumer
 
 object ConsumeTasty {
 
+  /** Load and process TASTy files using TASTy reflect
+   *
+   * @param classpath Classpath where the classes are located
+   * @param classes classes to be consumed
+   * @param tastyConsumer consumer that will process the tasty trees
+   */
   def apply(classpath: String, classes: List[String], tastyConsumer: TastyConsumer): Unit = {
     if (classes.isEmpty)
       throw new IllegalArgumentException("Parameter classes should no be empty")
@@ -33,6 +39,7 @@ object ConsumeTasty {
    *  BEWARE: with exotic enough classloaders, this may not work at all or do
    *  the wrong thing.
    */
+   // FIXME factor out
    private def classpathFromClassloader(cl: ClassLoader): String = {
     @tailrec
     def loop(cl: ClassLoader, suffixClasspath: String): String =

compiler/src/dotty/tools/dotc/consumetasty/ConsumeTasty.scala renamed to tasty-consumer/src/scala/tasty/file/ConsumeTasty.scala

@@ -1,12 +1,12 @@
-package dotty.tools.dotc.consumetasty
+package scala.tasty.file
 
 import dotty.tools.dotc.core.Contexts._
 import dotty.tools.dotc.core.Phases.Phase
 import dotty.tools.dotc.tastyreflect.ReflectionImpl
 
 import scala.tasty.file.TastyConsumer
 
-class TastyConsumerPhase(consumer: TastyConsumer) extends Phase {
+private class TastyConsumerPhase(consumer: TastyConsumer) extends Phase {
 
   override def phaseName: String = "tastyConsumer"
 

library/src/scala/tasty/file/TastyConsumer.scala renamed to tasty-consumer/src/scala/tasty/file/TastyConsumer.scala

@@ -1,14 +1,12 @@
-package dotty.tools.dotc.consumetasty
+package scala.tasty.file
 
 import dotty.tools.dotc.Run
 import dotty.tools.dotc.core.Mode
 import dotty.tools.dotc.core.Contexts.Context
 import dotty.tools.dotc.core.Phases.Phase
 import dotty.tools.dotc.fromtasty._
 
-import scala.tasty.file.TastyConsumer
-
-class TastyFromClass(consumer: TastyConsumer) extends TASTYCompiler {
+private class TastyFromClass(consumer: TastyConsumer) extends TASTYCompiler {
 
   override protected def frontendPhases: List[List[Phase]] =
     List(new ReadTasty) :: // Load classes from tasty

compiler/src/dotty/tools/dotc/consumetasty/TastyConsumerPhase.scala renamed to tasty-consumer/src/scala/tasty/file/TastyConsumerPhase.scala

@@ -0,0 +1,10 @@
+scala> import scala.quoted._
+scala> import scala.quoted.staging._
+scala> implicit def toolbox: Toolbox = Toolbox.make(getClass.getClassLoader)
+def toolbox: quoted.staging.Toolbox
+scala> def v(given QuoteContext) = '{ (if true then Some(1) else None).map(v => v+1) }
+def v(given x$1: quoted.QuoteContext): quoted.Expr[Option[Int]]
+scala> scala.quoted.staging.withQuoteContext(v.show)
+val res0: String = (if (true) scala.Some.apply[scala.Int](1) else scala.None).map[scala.Int](((v: scala.Int) => v.+(1)))
+scala> scala.quoted.staging.run(v)
+val res1: Option[Int] = Some(2)

compiler/src/dotty/tools/dotc/consumetasty/TastyFromClass.scala renamed to tasty-consumer/src/scala/tasty/file/TastyFromClass.scala

@@ -0,0 +1,15 @@
+package scala.quoted.file.repl
+
+import dotty.BootstrappedOnlyTests
+import dotty.tools.repl.ReplTest
+import dotty.tools.vulpix.TestConfiguration
+import org.junit.Test
+import org.junit.experimental.categories.Category
+
+/** Runs all tests contained in `staging/test-resources/repl-tasty-consumer` */
+class TastyConsumerScriptedReplTests extends ReplTest(withStaging = false) { // FIXME: withStaging?
+
+  @Category(Array(classOf[BootstrappedOnlyTests]))
+  @Test def replTastyConsumerTests = scripts("/repl-tasty-consumer").foreach(testFile)
+
+}

tasty-consumer/test-resources/repl-consume-tasty/i6007

@@ -93,7 +93,7 @@ object Main {
       println("Running Dotty Tastydoc on: " + classes.mkString(" "))
       val mutablePackagesMap: scala.collection.mutable.HashMap[String, EmulatedPackageRepresentation] = new scala.collection.mutable.HashMap[String, EmulatedPackageRepresentation]()
       val tc = new TastydocConsumer(mutablePackagesMap)
-      ConsumeTasty(extraClasspath, classes, tc)
+      TastyConsumer(extraClasspath, classes, tc)
 
       representations.setSubClasses(mutablePackagesMap)