Fix scala#8020: Create dotty-tasty-consumer library

Author: nicolasstucki

.drone.yml

@@ -40,7 +40,7 @@ steps:
   depends_on: [ clone ]
   commands:
   - cp -R . /tmp/2/ && cd /tmp/2/
-  - ./project/scripts/sbt ";dotty-bootstrapped/compile ;dotty-bootstrapped/test ;dotty-staging/test ;sjsSandbox/run;sjsSandbox/test;sjsJUnitTests/test ;configureIDE"
+  - ./project/scripts/sbt ";dotty-bootstrapped/compile ;dotty-bootstrapped/test ;dotty-staging/test ;dotty-tasty-consumer/test ;dotty-tastydoc/test ;sjsSandbox/run;sjsSandbox/test;sjsJUnitTests/test ;configureIDE"
   - ./project/scripts/bootstrapCmdTests
 
 - name: community_build

build.sbt

@@ -11,6 +11,7 @@ val `dotty-library-bootstrappedJS` = Build.`dotty-library-bootstrappedJS`
 val `dotty-sbt-bridge` = Build.`dotty-sbt-bridge`
 val `dotty-sbt-bridge-tests` = Build.`dotty-sbt-bridge-tests`
 val `dotty-staging` = Build.`dotty-staging`
+val `dotty-tasty-consumer` = Build.`dotty-tasty-consumer`
 val `dotty-language-server` = Build.`dotty-language-server`
 val `dotty-bench` = Build.`dotty-bench`
 val `dotty-bench-bootstrapped` = Build.`dotty-bench-bootstrapped`

community-build/community-projects/semanticdb

@@ -0,0 +1,47 @@
+package dotty.tools.dotc.util
+
+import java.net.URLClassLoader
+import java.nio.file.Paths
+
+import dotty.tools.repl.AbstractFileClassLoader
+
+object ClasspathFromClassloader {
+
+  /** Attempt to recreate a classpath from a classloader.
+   *
+   *  BEWARE: with exotic enough classloaders, this may not work at all or do
+   *  the wrong thing.
+   */
+  def apply(cl: ClassLoader): String = {
+    val classpathBuff = List.newBuilder[String]
+    def collectClassLoaderPaths(cl: ClassLoader): Unit = {
+      if (cl != null) {
+        cl match {
+          case cl: URLClassLoader =>
+            // This is wrong if we're in a subclass of URLClassLoader
+            // that filters loading classes from its parent ¯\_(ツ)_/¯
+            collectClassLoaderPaths(cl.getParent)
+            // Parent classloaders are searched before their child, so the part of
+            // the classpath coming from the child is added at the _end_ of the
+            // classpath.
+            classpathBuff ++=
+              cl.getURLs.iterator.map(url => Paths.get(url.toURI).toAbsolutePath.toString)
+          case _ =>
+            // HACK: We can't just collect the classpath from arbitrary parent
+            // classloaders since the current classloader might intentionally
+            // filter loading classes from its parent (for example
+            // BootFilteredLoader in the sbt launcher does this and we really
+            // don't want to include the scala-library that sbt depends on
+            // here), but we do need to look at the parent of the REPL
+            // classloader, so we special case it. We can't do this using a type
+            // test since the REPL classloader class itself is normally loaded
+            // with a different classloader.
+            if (cl.getClass.getName == classOf[AbstractFileClassLoader].getName)
+              collectClassLoaderPaths(cl.getParent)
+        }
+      }
+    }
+    collectClassLoaderPaths(cl)
+    classpathBuff.result().mkString(java.io.File.pathSeparator)
+  }
+}

compiler/src/dotty/tools/dotc/consumetasty/ConsumeTasty.scala

@@ -53,6 +53,9 @@ object Properties {
   /** dotty-staging jar */
   def dottyStaging: String = sys.props("dotty.tests.classes.dottyStaging")
 
+  /** dotty-tasty-consumer jar */
+  def dottyTastyConsumer: String = sys.props("dotty.tests.classes.dottyTastyConsumer")
+
   /** tasty-core jar */
   def tastyCore: String = sys.props("dotty.tests.classes.tastyCore")
 

compiler/src/dotty/tools/dotc/util/ClasspathFromClassloader.scala

@@ -127,7 +127,8 @@ class BootstrappedOnlyCompilationTests extends ParallelTesting {
     aggregateTests(
       compileFilesInDir("tests/run-with-compiler", withCompilerOptions),
       compileFilesInDir("tests/run-staging", withStagingOptions),
-      compileDir("tests/run-with-compiler-custom-args/tasty-interpreter", withCompilerOptions)
+      compileFilesInDir("tests/run-custom-args/tasty-consumer", withTastyConsumerOptions),
+      compileDir("tests/run-custom-args/tasty-interpreter", withTastyConsumerOptions),
     ).checkRuns()
   }
 

compiler/test/dotty/Properties.scala

@@ -39,6 +39,9 @@ object TestConfiguration {
   lazy val withStagingClasspath =
     withCompilerClasspath + File.pathSeparator + mkClasspath(List(Properties.dottyStaging))
 
+  lazy val withTastyConsumerClasspath =
+    withCompilerClasspath + File.pathSeparator + mkClasspath(List(Properties.dottyTastyConsumer))
+
   def mkClasspath(classpaths: List[String]): String =
     classpaths.map({ p =>
       val file = new java.io.File(p)
@@ -54,6 +57,8 @@ object TestConfiguration {
     defaultOptions.withClasspath(withCompilerClasspath).withRunClasspath(withCompilerClasspath)
   lazy val withStagingOptions =
     defaultOptions.withClasspath(withStagingClasspath).withRunClasspath(withStagingClasspath)
+  lazy val withTastyConsumerOptions =
+    defaultOptions.withClasspath(withTastyConsumerClasspath).withRunClasspath(withTastyConsumerClasspath)
   val allowDeepSubtypes = defaultOptions without "-Yno-deep-subtypes"
   val allowDoubleBindings = defaultOptions without "-Yno-double-bindings"
   val picklingOptions = defaultOptions and (

compiler/test/dotty/tools/dotc/BootstrappedOnlyCompilationTests.scala

@@ -108,6 +108,7 @@ DOTTY_COMP=$(find_lib "*dotty-compiler*")
 DOTTY_INTF=$(find_lib "*dotty-interfaces*")
 DOTTY_LIB=$(find_lib "*dotty-library*")
 DOTTY_STAGING=$(find_lib "*dotty-staging*")
+DOTTY_CONSUME_TASTY=$(find_lib "*dotty-consume-tasty*")
 TASTY_CORE=$(find_lib "*tasty-core*")
 SCALA_ASM=$(find_lib "*scala-asm*")
 SCALA_LIB=$(find_lib "*scala-library*")

compiler/test/dotty/tools/vulpix/TestConfiguration.scala

@@ -102,7 +102,7 @@ elif [ $execute_repl == true ] || [ ${#residual_args[@]} -ne 0 ]; then
     echo "warning: multiple classpaths are found, dotr only use the last one."
   fi
   if [ $with_compiler == true ]; then
-    cp_arg+="$PSEP$DOTTY_COMP$PSEP$TASTY_CORE$PSEP$DOTTY_INTF$PSEP$SCALA_ASM$PSEP$DOTTY_STAGING"
+    cp_arg+="$PSEP$DOTTY_COMP$PSEP$TASTY_CORE$PSEP$DOTTY_INTF$PSEP$SCALA_ASM$PSEP$DOTTY_STAGING$PSEP$DOTTY_CONSUME_TASTY"
   fi
   eval exec "\"$JAVACMD\"" "$DEBUG"  "-classpath \"$cp_arg\"" "${jvm_options[@]}" "${residual_args[@]}"
 else

dist/bin/common

@@ -16,6 +16,9 @@ To inspect the TASTy Reflect trees of a TASTy file a consumer can be defined in
 the following way.
 
 ```scala
+import scala.tasty.Reflection
+import scala.tasty.file._
+
 class Consumer extends TastyConsumer {
   final def apply(reflect: Reflection)(root: reflect.Tree): Unit = {
     import reflect._
@@ -33,4 +36,18 @@ object Test {
     ConsumeTasty("", List("foo.Bar"), new Consumer)
   }
 }
-```
+```
+
+Note that if we need to run the main (in an object called `Test`) after
+compilation we need make available the compiler to the runtime:
+
+```shell
+dotc -with-compiler -d out Test.scala
+dotr -with-compiler -classpath out Test
+```
+
+Or, from SBT:
+
+```scala
+libraryDependencies += "ch.epfl.lamp" %% "dotty-tasty-consumer" % scalaVersion.value
+```

dist/bin/dotr

@@ -595,9 +595,10 @@ object Build {
           val asm = findArtifactPath(externalDeps, "scala-asm")
           val dottyCompiler = jars("dotty-compiler")
           val dottyStaging = jars("dotty-staging")
+          val dottyTastyConsumer = jars("dotty-tasty-consumer")
           val dottyInterfaces = jars("dotty-interfaces")
           val tastyCore = jars("tasty-core")
-          run(insertClasspathInArgs(args1, List(dottyCompiler, dottyInterfaces, asm, dottyStaging, tastyCore).mkString(File.pathSeparator)))
+          run(insertClasspathInArgs(args1, List(dottyCompiler, dottyInterfaces, asm, dottyStaging, dottyTastyConsumer, tastyCore).mkString(File.pathSeparator)))
         } else run(args)
       },
 
@@ -671,9 +672,10 @@ object Build {
       }
       val dottyInterfaces = jars("dotty-interfaces")
       val dottyStaging = jars("dotty-staging")
+      val dottyTastyConsumer = jars("dotty-tasty-consumer")
       val tastyCore = jars("tasty-core")
       val asm = findArtifactPath(externalDeps, "scala-asm")
-      extraClasspath ++= Seq(dottyCompiler, dottyInterfaces, asm, dottyStaging, tastyCore)
+      extraClasspath ++= Seq(dottyCompiler, dottyInterfaces, asm, dottyStaging, dottyTastyConsumer, tastyCore)
     }
 
     val fullArgs = main :: insertClasspathInArgs(args, extraClasspath.mkString(File.pathSeparator))
@@ -715,14 +717,18 @@ object Build {
   )
 
   lazy val bootstrapedDottyCompilerSettings = commonDottyCompilerSettings ++ Seq(
-    javaOptions += {
+    javaOptions ++= {
       val jars = packageAll.value
-      "-Ddotty.tests.classes.dottyStaging=" + jars("dotty-staging")
+      Seq(
+        "-Ddotty.tests.classes.dottyStaging=" + jars("dotty-staging"),
+        "-Ddotty.tests.classes.dottyTastyConsumer=" + jars("dotty-tasty-consumer"),
+      )
     },
     packageAll := {
       packageAll.in(`dotty-compiler`).value ++ Seq(
         "dotty-compiler" -> packageBin.in(Compile).value.getAbsolutePath,
         "dotty-staging"  -> packageBin.in(LocalProject("dotty-staging"), Compile).value.getAbsolutePath,
+        "dotty-tasty-consumer"  -> packageBin.in(LocalProject("dotty-tasty-consumer"), Compile).value.getAbsolutePath,
         "tasty-core"     -> packageBin.in(LocalProject("tasty-core-bootstrapped"), Compile).value.getAbsolutePath,
       )
     }
@@ -803,6 +809,19 @@ object Build {
       javaOptions := (javaOptions in `dotty-compiler-bootstrapped`).value
     )
 
+  lazy val `dotty-tasty-consumer` = project.in(file("tasty-consumer")).
+    withCommonSettings(Bootstrapped).
+    // We want the compiler to be present in the compiler classpath when compiling this project but not
+    // when compiling a project that depends on dotty-staging (see sbt-dotty/sbt-test/sbt-dotty/tasty-consumer-example-project),
+    // but we always need it to be present on the JVM classpath at runtime.
+    dependsOn(dottyCompiler(Bootstrapped) % "provided").
+    dependsOn(dottyCompiler(Bootstrapped) % "compile->runtime").
+    dependsOn(dottyCompiler(Bootstrapped) % "test->test").
+    settings(commonBootstrappedSettings).
+    settings(
+      javaOptions := (javaOptions in `dotty-compiler-bootstrapped`).value
+    )
+
   lazy val `dotty-sbt-bridge` = project.in(file("sbt-bridge/src")).
     // We cannot depend on any bootstrapped project to compile the bridge, since the
     // bridge is needed to compile these projects.
@@ -1106,6 +1125,7 @@ object Build {
         publishLocal in `dotty-library-bootstrapped`,
         publishLocal in `tasty-core-bootstrapped`,
         publishLocal in `dotty-staging`,
+        publishLocal in `dotty-tasty-consumer`,
         publishLocal in `scala-library`,
         publishLocal in `scala-reflect`,
         publishLocal in `dotty-doc-bootstrapped`,
@@ -1315,7 +1335,7 @@ object Build {
     // FIXME: we do not aggregate `bin` because its tests delete jars, thus breaking other tests
     def asDottyRoot(implicit mode: Mode): Project = project.withCommonSettings.
       aggregate(`dotty-interfaces`, dottyLibrary, dottyCompiler, tastyCore, dottyDoc, `dotty-sbt-bridge`).
-      bootstrappedAggregate(`scala-library`, `scala-compiler`, `scala-reflect`, scalap, `dotty-language-server`, `dotty-staging`).
+      bootstrappedAggregate(`scala-library`, `scala-compiler`, `scala-reflect`, scalap, `dotty-language-server`, `dotty-staging`, `dotty-tasty-consumer`).
       dependsOn(tastyCore).
       dependsOn(dottyCompiler).
       dependsOn(dottyLibrary).
@@ -1354,6 +1374,7 @@ object Build {
     def asDottyTastydoc(implicit mode: Mode): Project = project.withCommonSettings.
       aggregate(`dotty-tastydoc-input`).
       dependsOn(dottyCompiler).
+      dependsOn(`dotty-tasty-consumer`).
       settings(commonDocSettings)
 
     def asDottyTastydocInput(implicit mode: Mode): Project = project.withCommonSettings.

docs/docs/reference/metaprogramming/tasty-inspect.md

@@ -0,0 +1,19 @@
+package hello
+
+import scala.tasty.Reflection
+import scala.tasty.file._
+
+object Main extends App {
+
+
+  class Consumer extends TastyConsumer {
+    final def apply(reflect: Reflection)(root: reflect.Tree): Unit = {
+      import reflect._
+      val tastyStr = root.show
+      println(tastyStr)
+    }
+  }
+
+  ConsumeTasty("", List("lib.Foo"), new Consumer)
+
+}

library/src/scala/tasty/file/ConsumeTasty.scala

@@ -0,0 +1,15 @@
+lazy val dottyVersion = sys.props("plugin.scalaVersion")
+
+lazy val lib = project
+  .in(file("lib"))
+  .settings(
+    scalaVersion := dottyVersion
+  )
+
+lazy val app = project
+  .in(file("app"))
+  .settings(
+    scalaVersion := dottyVersion,
+    libraryDependencies += "ch.epfl.lamp" %% "dotty-tasty-consumer" % scalaVersion.value,
+  )
+  .dependsOn(lib)

project/Build.scala

@@ -0,0 +1,5 @@
+package lib
+
+class Foo {
+  def foo(x: Int) = x
+}

sbt-dotty/sbt-test/sbt-dotty/tasty-consumer-example-project/app/Main.scala

@@ -0,0 +1 @@
+addSbtPlugin("ch.epfl.lamp" % "sbt-dotty" % sys.props("plugin.version"))

sbt-dotty/sbt-test/sbt-dotty/tasty-consumer-example-project/build.sbt

@@ -0,0 +1 @@
+> app/run