add support for pipeline build of scala files

- add '-Ypickle-java' and '-Ypickle-write' flags expected by Zinc
- move ExtractAPI phase to after pickler, this way we can do it in
  parallel with generating TASTy bytes. At the end of this phase we
  write the TASTy to the early-output destination.
- test the pipelining with sbt scripted tests, including for inline
  methods and macros with pipelining
- describe semantics with respect to suspensions,
  introduce -Yno-suspended-units flag for greater control by the user.

Author: bishabosha

compiler/src/dotty/tools/dotc/CompilationUnit.scala

@@ -88,12 +88,15 @@ class CompilationUnit protected (val source: SourceFile) {
     // when this unit is unsuspended.
     depRecorder.clear()
     if !suspended then
-      if (ctx.settings.XprintSuspension.value)
-        report.echo(i"suspended: $this")
-      suspended = true
-      ctx.run.nn.suspendedUnits += this
-      if ctx.phase == Phases.inliningPhase then
-        suspendedAtInliningPhase = true
+      if ctx.settings.YnoSuspendedUnits.value then
+        report.error(i"Compilation unit suspended $this (-Yno-suspended-units is set)")
+      else
+        if (ctx.settings.XprintSuspension.value)
+          report.echo(i"suspended: $this")
+        suspended = true
+        ctx.run.nn.suspendedUnits += this
+        if ctx.phase == Phases.inliningPhase then
+          suspendedAtInliningPhase = true
     throw CompilationUnit.SuspendException()
 
   private var myAssignmentSpans: Map[Int, List[Span]] | Null = null

compiler/src/dotty/tools/dotc/Compiler.scala

@@ -41,13 +41,13 @@ class Compiler {
     List(new semanticdb.ExtractSemanticDB.ExtractSemanticInfo) :: // Extract info into .semanticdb files
     List(new PostTyper) ::          // Additional checks and cleanups after type checking
     List(new sjs.PrepJSInterop) ::  // Additional checks and transformations for Scala.js (Scala.js only)
-    List(new sbt.ExtractAPI) ::     // Sends a representation of the API of classes to sbt via callbacks
     List(new SetRootTree) ::        // Set the `rootTreeOrProvider` on class symbols
     Nil
 
   /** Phases dealing with TASTY tree pickling and unpickling */
   protected def picklerPhases: List[List[Phase]] =
     List(new Pickler) ::            // Generate TASTY info
+    List(new sbt.ExtractAPI) ::     // Sends a representation of the API of classes to sbt via callbacks
     List(new Inlining) ::           // Inline and execute macros
     List(new PostInlining) ::       // Add mirror support for inlined code
     List(new CheckUnused.PostInlining) ::  // Check for unused elements

compiler/src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -14,6 +14,7 @@ import Setting.ChoiceWithHelp
 import scala.util.chaining.*
 
 import java.util.zip.Deflater
+import dotty.tools.io.NoAbstractFile
 
 class ScalaSettings extends SettingGroup with AllScalaSettings
 
@@ -373,6 +374,7 @@ private sealed trait YSettings:
   val YprintPos: Setting[Boolean] = BooleanSetting("-Yprint-pos", "Show tree positions.")
   val YprintPosSyms: Setting[Boolean] = BooleanSetting("-Yprint-pos-syms", "Show symbol definitions positions.")
   val YnoDeepSubtypes: Setting[Boolean] = BooleanSetting("-Yno-deep-subtypes", "Throw an exception on deep subtyping call stacks.")
+  val YnoSuspendedUnits: Setting[Boolean] = BooleanSetting("-Yno-suspended-units", "Do not suspend units, e.g. when calling a macro defined in the same run. This will error instead of suspending.")
   val YnoPatmatOpt: Setting[Boolean] = BooleanSetting("-Yno-patmat-opt", "Disable all pattern matching optimizations.")
   val YplainPrinter: Setting[Boolean] = BooleanSetting("-Yplain-printer", "Pretty-print using a plain printer.")
   val YprintSyms: Setting[Boolean] = BooleanSetting("-Yprint-syms", "When printing trees print info in symbols instead of corresponding info in trees.")
@@ -432,4 +434,8 @@ private sealed trait YSettings:
   val YforceInlineWhileTyping: Setting[Boolean] = BooleanSetting("-Yforce-inline-while-typing", "Make non-transparent inline methods inline when typing. Emulates the old inlining behavior of 3.0.0-M3.")
 
   val YdebugMacros: Setting[Boolean] = BooleanSetting("-Ydebug-macros", "Show debug info when quote pattern match fails")
+
+  // Pipeline compilation options
+  val YpickleJava: Setting[Boolean] = BooleanSetting("-Ypickle-java", "Pickler phase should compute pickles for .java defined symbols for use by build tools")
+  val YpickleWrite: Setting[AbstractFile] = OutputSetting("-Ypickle-write", "directory|jar", "destination for generated .sig files containing type signatures.", NoAbstractFile, aliases = List("-Yearly-tasty"))
 end YSettings

compiler/src/dotty/tools/dotc/config/Settings.scala

@@ -308,8 +308,8 @@ object Settings:
     def MultiStringSetting(name: String, helpArg: String, descr: String, default: List[String] = Nil, aliases: List[String] = Nil): Setting[List[String]] =
       publish(Setting(name, descr, default, helpArg, aliases = aliases))
 
-    def OutputSetting(name: String, helpArg: String, descr: String, default: AbstractFile): Setting[AbstractFile] =
-      publish(Setting(name, descr, default, helpArg))
+    def OutputSetting(name: String, helpArg: String, descr: String, default: AbstractFile, aliases: List[String] = Nil): Setting[AbstractFile] =
+      publish(Setting(name, descr, default, helpArg, aliases = aliases))
 
     def PathSetting(name: String, descr: String, default: String, aliases: List[String] = Nil): Setting[String] =
       publish(Setting(name, descr, default, aliases = aliases))

compiler/src/dotty/tools/dotc/core/SymbolLoaders.scala

@@ -451,7 +451,8 @@ class TastyLoader(val tastyFile: AbstractFile) extends SymbolLoader {
       val tastyUUID = new TastyHeaderUnpickler(TastyUnpickler.scala3CompilerConfig, tastyBytes).readHeader()
       new ClassfileTastyUUIDParser(classfile)(ctx).checkTastyUUID(tastyUUID)
     else
-      // This will be the case in any of our tests that compile with `-Youtput-only-tasty`
+      // This will be the case in any of our tests that compile with `-Youtput-only-tasty`, or when
+      // tasty file compiled by `-Ypickle-write` comes from an early output jar.
       report.inform(s"No classfiles found for $tastyFile when checking TASTy UUID")
 
   private def mayLoadTreesFromTasty(using Context): Boolean =

compiler/src/dotty/tools/dotc/inlines/Inliner.scala

@@ -1043,6 +1043,9 @@ class Inliner(val call: tpd.Tree)(using Context):
       for sym <- dependencies do
         if ctx.compilationUnit.source.file == sym.associatedFile then
           report.error(em"Cannot call macro $sym defined in the same source file", call.srcPos)
+        else if ctx.settings.YnoSuspendedUnits.value then
+          val addendum = ", suspension prevented by -Yno-suspended-units"
+          report.error(em"Cannot call macro $sym defined in the same compilation run$addendum", call.srcPos)
         if (suspendable && ctx.settings.XprintSuspension.value)
           report.echo(i"suspension triggered by macro call to ${sym.showLocated} in ${sym.associatedFile}", call.srcPos)
       if suspendable then

compiler/src/dotty/tools/dotc/sbt/ExtractAPI.scala

@@ -20,7 +20,8 @@ import NameOps.*
 import inlines.Inlines
 import transform.ValueClasses
 import transform.SymUtils.*
-import dotty.tools.io.{File, FileExtension, JarArchive}
+import transform.Pickler
+import dotty.tools.io.{File, FileExtension, JarArchive, ClassfileWriterOps}
 import util.{Property, SourceFile}
 import java.io.PrintWriter
 
@@ -63,16 +64,68 @@ class ExtractAPI extends Phase {
   // after `PostTyper` (unlike `ExtractDependencies`, the simplication to trees
   // done by `PostTyper` do not affect this phase because it only cares about
   // definitions, and `PostTyper` does not change definitions).
-  override def runsAfter: Set[String] = Set(transform.PostTyper.name)
+  override def runsAfter: Set[String] = Set(transform.Pickler.name)
 
   override def runOn(units: List[CompilationUnit])(using Context): List[CompilationUnit] =
+    val sigWriter: Option[Pickler.EarlyFileWriter] = ctx.settings.YpickleWrite.value match
+      case earlyOut if earlyOut.isDirectory && earlyOut.exists =>
+        Some(Pickler.EarlyFileWriter(ClassfileWriterOps(earlyOut)))
+      case _ =>
+        None
     val nonLocalClassSymbols = new mutable.HashSet[Symbol]
-    val ctx0 = ctx.withProperty(NonLocalClassSymbolsInCurrentUnits, Some(nonLocalClassSymbols))
-    val units0 = super.runOn(units)(using ctx0)
-    ctx.withIncCallback(recordNonLocalClasses(nonLocalClassSymbols, _))
-    units0
+    val ctx0 = ctx
+      .withProperty(NonLocalClassSymbolsInCurrentUnits, Some(nonLocalClassSymbols))
+    inContext(ctx0) {
+      val units0 = super.runOn(units)
+      sigWriter.foreach(writeSigFiles(units0, _))
+      ctx.withIncCallback(recordNonLocalClasses(nonLocalClassSymbols, _))
+      units0
+    }
   end runOn
 
+  // Why we only write to early output in the first run?
+  // ===================================================
+  // TL;DR the point of pipeline compilation is to start downstream projects early,
+  // so we don't want to wait for suspended units to be compiled.
+  //
+  // But why is it safe to ignore suspended units?
+  // If this project contains a transparent macro that is called in the same project,
+  // the compilation unit of that call will be suspended (if the macro implementation
+  // is also in this project), causing a second run.
+  // However before we do that run, we will have already requested sbt to begin
+  // early downstream compilation. This means that the suspended definitions will not
+  // be visible in *early* downstream compilation.
+  //
+  // However, sbt will by default prevent downstream compilation happening in this scenario,
+  // due to the existence of macro definitions. So we are protected from failure if user tries
+  // to use the suspended definitions.
+  //
+  // Additionally, it is recommended for the user to move macro implementations to another project
+  // if they want to force early output. In this scenario the suspensions will no longer occur, so now
+  // they will become visible in the early-output.
+  //
+  // See `sbt-test/pipelining/pipelining-scala-macro` and `sbt-test/pipelining/pipelining-scala-macro-force`
+  // for examples of this in action.
+  //
+  // Therefore we only need to write to early output in the first run. We also provide the option
+  // to diagnose suspensions with the `-Yno-suspended-units` flag.
+  private def writeSigFiles(units: List[CompilationUnit], writer: Pickler.EarlyFileWriter)(using Context): Unit = {
+    try
+      for
+        unit <- units
+        (cls, pickled) <- unit.pickled
+        if cls.isDefinedInCurrentRun
+      do
+        val binaryName = cls.binaryClassName.replace('.', java.io.File.separatorChar).nn
+        val binaryClassName = if (cls.is(Module)) binaryName.stripSuffix(str.MODULE_SUFFIX).nn else binaryName
+        writer.writeTasty(binaryClassName, pickled())
+    finally
+      writer.close()
+      if ctx.settings.verbose.value then
+        report.echo("[sig files written]")
+    end try
+  }
+
   private def recordNonLocalClasses(nonLocalClassSymbols: mutable.HashSet[Symbol], cb: interfaces.IncrementalCallback)(using Context): Unit =
     for cls <- nonLocalClassSymbols if !cls.isLocal do
       val sourceFile = cls.source

compiler/src/dotty/tools/dotc/transform/Pickler.scala

@@ -15,6 +15,8 @@ import Flags.Module
 import reporting.{ThrowingReporter, Profile, Message}
 import collection.mutable
 import util.concurrent.{Executor, Future}
+import util.Property
+import io.ClassfileWriterOps
 import compiletime.uninitialized
 
 object Pickler {
@@ -26,6 +28,9 @@ object Pickler {
    *  only in backend.
    */
   inline val ParallelPickling = true
+
+  class EarlyFileWriter(writer: ClassfileWriterOps):
+    export writer.{writeTasty, close}
 }
 
 /** This phase pickles trees */

compiler/src/dotty/tools/io/ClassfileWriterOps.scala

@@ -0,0 +1,108 @@
+package dotty.tools.io
+
+import java.io.{DataOutputStream, IOException, PrintWriter, StringWriter}
+import java.nio.file.Files
+
+import dotty.tools.io.*
+import dotty.tools.dotc.core.Decorators.*
+import dotty.tools.dotc.core.Contexts.*
+import dotty.tools.dotc.report
+import java.nio.channels.ClosedByInterruptException
+import scala.language.unsafeNulls
+import scala.annotation.constructorOnly
+
+
+class ClassfileWriterOps(outputDir: AbstractFile)(using @constructorOnly ictx: Context) {
+
+  type InternalName = String
+
+  // if non-null, classfiles are written to a jar instead of the output directory
+  private val jarWriter: JarWriter | Null =
+    val localCtx = ictx
+    outputDir match {
+      case jar: JarArchive =>
+        jar.underlyingSource.map { source =>
+          if jar.isEmpty then
+            new Jar(source.file).jarWriter()
+          else inContext(localCtx) {
+            // Writing to non-empty JAR might be an undefined behaviour, e.g. in case if other files where
+            // created using `AbstractFile.bufferedOutputStream`instead of JarWriter
+            report.warning(em"Tried to write to non-empty JAR: $source")
+            null
+          }
+        }.orNull
+
+      case _ => null
+  }
+
+  private def getFile(base: AbstractFile, clsName: String, suffix: String): AbstractFile = {
+    if (base.file != null) {
+      fastGetFile(base, clsName, suffix)
+    } else {
+      def ensureDirectory(dir: AbstractFile): AbstractFile =
+        if (dir.isDirectory) dir
+        else throw new FileConflictException(s"${base.path}/$clsName$suffix: ${dir.path} is not a directory", dir)
+      var dir = base
+      val pathParts = clsName.split("[./]").toList
+      for (part <- pathParts.init) dir = ensureDirectory(dir) subdirectoryNamed part
+      ensureDirectory(dir) fileNamed pathParts.last + suffix
+    }
+  }
+
+  private def fastGetFile(base: AbstractFile, clsName: String, suffix: String) = {
+    val index = clsName.lastIndexOf('/')
+    val (packageName, simpleName) = if (index > 0) {
+      (clsName.substring(0, index), clsName.substring(index + 1))
+    } else ("", clsName)
+    val directory = base.file.toPath.resolve(packageName)
+    new PlainFile(Path(directory.resolve(simpleName + suffix)))
+  }
+
+  private def writeBytes(outFile: AbstractFile, bytes: Array[Byte]): Unit = {
+    if (outFile.file != null) {
+      val outPath = outFile.file.toPath
+      try Files.write(outPath, bytes)
+      catch {
+        case _: java.nio.file.NoSuchFileException =>
+          Files.createDirectories(outPath.getParent)
+          Files.write(outPath, bytes)
+      }
+    } else {
+      val out = new DataOutputStream(outFile.bufferedOutput)
+      try out.write(bytes, 0, bytes.length)
+      finally out.close()
+    }
+  }
+
+  def writeTasty(className: InternalName, bytes: Array[Byte]): Unit =
+    writeToJarOrFile(className, bytes, ".tasty")
+
+  private def writeToJarOrFile(className: InternalName, bytes: Array[Byte], suffix: String): AbstractFile | Null = {
+    if jarWriter == null then
+      val outFolder = outputDir
+      val outFile = getFile(outFolder, className, suffix)
+      try writeBytes(outFile, bytes)
+      catch case ex: ClosedByInterruptException =>
+        try outFile.delete() // don't leave an empty or half-written files around after an interrupt
+        catch case _: Throwable => ()
+        finally throw ex
+      outFile
+    else
+      val path = className + suffix
+      val out = jarWriter.newOutputStream(path)
+      try out.write(bytes, 0, bytes.length)
+      finally out.flush()
+      null
+  }
+
+  def close(): Unit = {
+    if (jarWriter != null) jarWriter.close()
+    outputDir match
+      case jar: JarArchive => jar.close()
+      case _ =>
+  }
+}
+
+
+/** Can't output a file due to the state of the file system. */
+class FileConflictException(msg: String, val file: AbstractFile) extends IOException(msg)

compiler/src/dotty/tools/io/JarArchive.scala

@@ -12,6 +12,7 @@ import scala.jdk.CollectionConverters.*
  */
 class JarArchive private (root: Directory) extends PlainDirectory(root) {
   def close(): Unit = jpath.getFileSystem().close()
+  override def exists: Boolean = jpath.getFileSystem().isOpen() && super.exists
   def allFileNames(): Iterator[String] =
     java.nio.file.Files.walk(jpath).iterator().asScala.map(_.toString)
 }

sbt-test/pipelining/early-out-inline/a/src/main/scala/a/A.scala

@@ -0,0 +1,10 @@
+package a
+
+import scala.quoted.*
+
+object A {
+  inline def power(x: Double, inline n: Int): Double =
+    inline if (n == 0) 1.0
+    else inline if (n % 2 == 1) x * power(x, n - 1)
+    else power(x * x, n / 2)
+}

sbt-test/pipelining/early-out-inline/b/src/main/scala/b/B.scala

@@ -0,0 +1,10 @@
+package b
+
+import a.A
+
+object B {
+  @main def run =
+    assert(A.power(2.0, 2) == 4.0)
+    assert(A.power(2.0, 3) == 8.0)
+    assert(A.power(2.0, 4) == 16.0)
+}

sbt-test/pipelining/early-out-inline/build.sbt

@@ -0,0 +1,14 @@
+// defines a inline method
+lazy val a = project.in(file("a"))
+  .settings(
+    scalacOptions ++= Seq("-Yearly-tasty", ((ThisBuild / baseDirectory).value / "a-early.jar").toString),
+    scalacOptions += "-Ystop-after:firstTransform",
+    scalacOptions += "-Ycheck:all",
+  )
+
+// uses the inline method, this is fine as there is no macro classloader involved
+lazy val b = project.in(file("b"))
+  .settings(
+    Compile / unmanagedClasspath += Attributed.blank((ThisBuild / baseDirectory).value / "a-early.jar"),
+    scalacOptions += "-Ycheck:all",
+  )

sbt-test/pipelining/early-out-inline/project/DottyInjectedPlugin.scala

@@ -0,0 +1,12 @@
+import sbt._
+import Keys._
+
+object DottyInjectedPlugin extends AutoPlugin {
+  override def requires = plugins.JvmPlugin
+  override def trigger = allRequirements
+
+  override val projectSettings = Seq(
+    scalaVersion := sys.props("plugin.scalaVersion"),
+    scalacOptions += "-source:3.0-migration"
+  )
+}

sbt-test/pipelining/early-out-inline/test

@@ -0,0 +1,3 @@
+> a/compile
+# uses the early output jar of a
+> b/run

sbt-test/pipelining/early-out/a/src/main/scala/a/A.scala

@@ -0,0 +1,5 @@
+package a
+
+object A {
+  val foo: (1,2,3) = (1,2,3)
+}

sbt-test/pipelining/early-out/b-early-out/.keep

@@ -0,0 +1,5 @@
+package b
+
+object B {
+  val bar: (1,2,3) = (1,2,3)
+}