Port Scala#6057 - Pipeline code gen and post-processing

Author: WojciechMazur

compiler/src/dotty/tools/backend/jvm/CodeGen.scala

@@ -40,7 +40,10 @@ class CodeGen(val int: DottyBackendInterface, val primitives: DottyPrimitives)(
 
   private lazy val mirrorCodeGen = Impl.JMirrorBuilder()
 
-  def gen(unit: CompilationUnit): Unit = {
+  def genUnit(unit: CompilationUnit): GeneratedDefs = {
+    val generatedClasses = mutable.ListBuffer.empty[GeneratedClass]
+    val generatedTasty = mutable.ListBuffer.empty[GeneratedTasty]
+
     val postProcessor = Phases.genBCodePhase match {
       case genBCode: GenBCode => genBCode.postProcessor
       case _ => null
@@ -52,7 +55,7 @@ class CodeGen(val int: DottyBackendInterface, val primitives: DottyPrimitives)(
         val sourceFile = unit.source.file
 
         def registerGeneratedClass(classNode: ClassNode, isArtifact: Boolean) = if (classNode != null){
-          postProcessor.generatedClasses += GeneratedClass(classNode, sourceFile, isArtifact, onFileCreated(classNode, sym, unit.source))
+          generatedClasses += GeneratedClass(classNode, sourceFile, isArtifact, onFileCreated(classNode, sym, unit.source))
         }
 
         val plainC = genClass(cd, unit)
@@ -68,12 +71,33 @@ class CodeGen(val int: DottyBackendInterface, val primitives: DottyPrimitives)(
 
         if sym.isClass then 
           val attrNode = if (mirrorC ne null) mirrorC else plainC
-          setTastyAttributes(sym, attrNode, postProcessor, unit)
+          genTastyAndSetAttributes(sym, attrNode)
       catch 
         case ex: Throwable =>
           ex.printStackTrace()
           report.error(s"Error while emitting ${unit.source}\n${ex.getMessage}", NoSourcePosition)
 
+
+    def genTastyAndSetAttributes(claszSymbol: Symbol, store: ClassNode): Unit = 
+      import Impl.createJAttribute
+      for (binary <- unit.pickled.get(claszSymbol.asClass)) {
+        generatedTasty += GeneratedTasty(store, binary)
+        val tasty =
+          val uuid = new TastyHeaderUnpickler(binary()).readHeader()
+          val lo = uuid.getMostSignificantBits
+          val hi = uuid.getLeastSignificantBits
+
+          // TASTY attribute is created but only the UUID bytes are stored in it.
+          // A TASTY attribute has length 16 if and only if the .tasty file exists.
+          val buffer = new TastyBuffer(16)
+          buffer.writeUncompressedLong(lo)
+          buffer.writeUncompressedLong(hi)
+          buffer.bytes
+
+        val dataAttr = createJAttribute(nme.TASTYATTR.mangledString, tasty, 0, tasty.length)
+        store.visitAttribute(dataAttr)
+      }
+
     def genClassDefs(tree: Tree): Unit =
       tree match {
         case EmptyTree => ()
@@ -83,8 +107,10 @@ class CodeGen(val int: DottyBackendInterface, val primitives: DottyPrimitives)(
       }
 
     genClassDefs(unit.tpdTree)
+    GeneratedDefs(generatedClasses.toList, generatedTasty.toList)
   }
 
+  // Creates a callback that would be evaluated in PostProcessor after creating a file
   private def onFileCreated(cls: ClassNode, claszSymbol: Symbol, sourceFile: interfaces.SourceFile): AbstractFile => Unit = clsFile => {
     val (fullClassName, isLocal) = atPhase(sbtExtractDependenciesPhase) {
       (ExtractDependencies.classNameAsString(claszSymbol), claszSymbol.isLocal)
@@ -112,25 +138,6 @@ class CodeGen(val int: DottyBackendInterface, val primitives: DottyPrimitives)(
       override def jfile = Optional.ofNullable(absfile.file)
     }
 
-  private def setTastyAttributes(claszSymbol: Symbol, store: ClassNode, postProcessor: PostProcessor, unit: CompilationUnit): Unit = 
-    import Impl.createJAttribute
-    for (binary <- unit.pickled.get(claszSymbol.asClass)) {
-      postProcessor.generatedTasty += GeneratedTasty(store, binary)
-      val tasty =
-        val uuid = new TastyHeaderUnpickler(binary()).readHeader()
-        val lo = uuid.getMostSignificantBits
-        val hi = uuid.getLeastSignificantBits
-
-        // TASTY attribute is created but only the UUID bytes are stored in it.
-        // A TASTY attribute has length 16 if and only if the .tasty file exists.
-        val buffer = new TastyBuffer(16)
-        buffer.writeUncompressedLong(lo)
-        buffer.writeUncompressedLong(hi)
-        buffer.bytes
-
-      val dataAttr = createJAttribute(nme.TASTYATTR.mangledString, tasty, 0, tasty.length)
-      store.visitAttribute(dataAttr)
-    }
 
   private def genClass(cd: TypeDef, unit: CompilationUnit): ClassNode = {
     val b = new Impl.SyncAndTryBuilder(unit) {}
@@ -175,4 +182,4 @@ class CodeGen(val int: DottyBackendInterface, val primitives: DottyPrimitives)(
     protected val primitives: DottyPrimitives = self.primitives
   }
   object Impl extends ImplEarlyInit with BCodeSyncAndTry
-}
+}

compiler/src/dotty/tools/backend/jvm/GenBCode.scala

@@ -64,14 +64,16 @@ class GenBCode extends Phase { self =>
 
   override def run(using Context): Unit =
     backendInterface.ctx = summon[Context]
-    codeGen.gen(ctx.compilationUnit)
+    val generated = codeGen.genUnit(ctx.compilationUnit)
+    // In Scala 2 backend depening on either global optimizations are enabled
+    // post processing might be delayed to build the call graph, or run imiedietlly if opts are disabled
+    // In Scala 3 we don't perform backend optimizations and though always post process without the delay
+    // https://github.com/scala/scala/pull/6057
+    postProcessor.postProcessAndSendToDisk(generated)
 
   override def runOn(units: List[CompilationUnit])(using ctx:Context): List[CompilationUnit] = {
-    try 
-      val result = super.runOn(units)
-      postProcessor.postProcessAndSendToDisk()
-      result
-    finally 
+    try super.runOn(units)
+    finally
       frontendAccess.compilerSettings.outputDirectory match {
       case jar: JarArchive =>
         if (ctx.run.nn.suspendedUnits.nonEmpty)
@@ -81,7 +83,8 @@ class GenBCode extends Phase { self =>
 
         jar.close()
       case _ =>
-    }
+      }
+      postProcessor.classfileWriter.close()
   }
 }
 

compiler/src/dotty/tools/backend/jvm/PostProcessor.scala

@@ -19,11 +19,10 @@ class PostProcessor(val frontendAccess: PostProcessorFrontendAccess, val bTypes:
 
   val backendUtils = new BackendUtils(this)
   lazy val classfileWriter  = ClassfileWriter(frontendAccess)
-  lazy val generatedClasses = ListBuffer.empty[GeneratedClass]
-  lazy val generatedTasty   = ListBuffer.empty[GeneratedTasty]
 
-  def postProcessAndSendToDisk(): Unit = {
-    for (GeneratedClass(classNode, sourceFile, isArtifact, onFileCreated) <- generatedClasses) {
+  def postProcessAndSendToDisk(generatedDefs: GeneratedDefs): Unit = {
+    val GeneratedDefs(classes, tasty) = generatedDefs
+    for (GeneratedClass(classNode, sourceFile, isArtifact, onFileCreated) <- classes) {
       val bytes = 
         try 
           if !isArtifact then setSerializableLambdas(classNode)
@@ -50,11 +49,9 @@ class PostProcessor(val frontendAccess: PostProcessorFrontendAccess, val bTypes:
       }
     }
 
-    for (GeneratedTasty(classNode, binaryGen) <- generatedTasty){
+    for (GeneratedTasty(classNode, binaryGen) <- tasty){
       classfileWriter.writeTasty(classNode.name.nn, binaryGen())
     }
-
-    classfileWriter.close()
   }
 
   private def setSerializableLambdas(classNode: ClassNode): Unit = {
@@ -113,7 +110,7 @@ class PostProcessor(val frontendAccess: PostProcessorFrontendAccess, val bTypes:
  */
 case class GeneratedClass(classNode: ClassNode, sourceFile: AbstractFile, isArtifact: Boolean, onFileCreated: AbstractFile => Unit)
 case class GeneratedTasty(classNode: ClassNode, tastyGen: () => Array[Byte])
-
+case class GeneratedDefs(classes: List[GeneratedClass], tasty: List[GeneratedTasty])
 
 // Temporary class, will be refactored in a future commit
 trait ClassWriterForPostProcessor {