[compiler, cbh] Put the in-memory document compiler behind a registry key

#SCL-22338
#SCL-22736 fixed

- The physical temporary file document compiler (previous implementation) is used as the default.
- This is done due to issues with macro expansion and in-memory files in MUnit and possibly other libraries.
- scala/scala3#20591

Author: vasilmkd

scala/compiler-integration/resources/META-INF/compiler-integration.xml

@@ -35,6 +35,9 @@
         <registryKey key="scala.erase.compiler.process.jdk.once" defaultValue="true" restartRequired="false"
                      description="Erase compiler.process.jdk value before compilation. A dirty hack needed because of removal of SetSameJdkToBuildProcessAsInCompileServer #SCL-17676"/>
 
+        <registryKey key="scala.compiler.highlighting.document.use.in.memory.file" defaultValue="false" restartRequired="false"
+                     description="When enabled, uses an in-memory file for document compilation"/>
+
         <buildProcess.parametersProvider implementation="org.jetbrains.plugins.scala.compiler.references.CompilerIndicesBuildProcessParametersProvider"/>
 
         <projectConfigurable id="Bytecode Indices"

scala/compiler-integration/src/org/jetbrains/plugins/scala/compiler/highlighting/DocumentCompiler.scala

@@ -7,16 +7,17 @@ import com.intellij.openapi.editor.Document
 import com.intellij.openapi.module.Module
 import com.intellij.openapi.project.Project
 import com.intellij.openapi.util.io.FileUtil
+import com.intellij.openapi.util.registry.Registry
 import com.intellij.openapi.vfs.VirtualFile
-import org.jetbrains.jps.incremental.scala.Client
 import org.jetbrains.jps.incremental.scala.remote.{CommandIds, SourceScope}
+import org.jetbrains.jps.incremental.scala.{Client, DelegateClient}
 import org.jetbrains.plugins.scala.compiler.data.{CompilerData, CompilerJarsFactory, DocumentCompilationArguments, DocumentCompilationData, IncrementalityType}
 import org.jetbrains.plugins.scala.compiler.{RemoteServerConnectorBase, RemoteServerRunner}
 import org.jetbrains.plugins.scala.editor.DocumentExt
-import org.jetbrains.plugins.scala.project.{ModuleExt, ScalaLanguageLevel}
+import org.jetbrains.plugins.scala.project.{ModuleExt, ScalaLanguageLevel, VirtualFileExt}
 
 import java.io.File
-import java.nio.file.Path
+import java.nio.file.{Files, Path}
 
 @Service(Array(Service.Level.PROJECT))
 private final class DocumentCompiler(project: Project) {
@@ -42,21 +43,64 @@ private final class DocumentCompiler(project: Project) {
     virtualFile: VirtualFile,
     client: Client
   ): Unit = {
-    compileDocumentContent(
-      sourcePath = virtualFile.toNioPath,
-      sourceContent = document.textWithConvertedSeparators(virtualFile),
-      module = module,
-      sourceScope = sourceScope,
-      client = client
-    )
+    val useInMemoryFile = Registry.is("scala.compiler.highlighting.document.use.in.memory.file")
+    val originalSourceFile = virtualFile.toFile
+    val sourceContent = document.textWithConvertedSeparators(virtualFile)
+
+    if (useInMemoryFile) {
+      compileInMemoryFile(
+        sourcePath = originalSourceFile.toPath,
+        sourceContent = sourceContent,
+        module = module,
+        sourceScope = sourceScope,
+        client = client
+      )
+    } else {
+      compilePhysicalFile(
+        originalSourceFile = originalSourceFile,
+        content = sourceContent,
+        module = module,
+        sourceScope = sourceScope,
+        client = client
+      )
+    }
   }
 
-  private def compileDocumentContent(sourcePath: Path,
-                                     sourceContent: String,
-                                     module: Module,
-                                     sourceScope: SourceScope,
-                                     client: Client): Unit = {
-    val connector = new RemoteServerConnector(sourcePath, sourceContent, module, sourceScope)
+  private def compilePhysicalFile(originalSourceFile: File,
+                                  content: String,
+                                  module: Module,
+                                  sourceScope: SourceScope,
+                                  client: Client): Unit = {
+    val tempSourceFile = workingDirectory.resolve("tempSourceFile").toFile
+    Files.writeString(tempSourceFile.toPath, content)
+    val connector =
+      try new PhysicalFileConnector(tempSourceFile, module, sourceScope)
+      catch {
+        case t: Throwable =>
+          // Remove the temporary source file if creating the connector failed.
+          FileUtil.delete(tempSourceFile)
+          throw t
+      }
+
+    try connector.compile(originalSourceFile, client)
+    finally {
+      if (connector.requiresCleanup) {
+        val files = workingDirectory.toFile.listFiles()
+        if (files ne null) {
+          files.foreach(FileUtil.delete)
+        }
+      } else {
+        FileUtil.delete(tempSourceFile)
+      }
+    }
+  }
+
+  private def compileInMemoryFile(sourcePath: Path,
+                                  sourceContent: String,
+                                  module: Module,
+                                  sourceScope: SourceScope,
+                                  client: Client): Unit = {
+    val connector = new InMemoryFileConnector(sourcePath, sourceContent, module, sourceScope)
     try connector.compile(client)
     finally {
       if (connector.requiresCleanup) {
@@ -68,8 +112,64 @@ private final class DocumentCompiler(project: Project) {
     }
   }
 
-  private class RemoteServerConnector(sourcePath: Path, sourceContent: String, module: Module, sourceScope: SourceScope)
-    extends RemoteServerConnectorBase(module, None, workingDirectory.toFile) {
+  private final class PhysicalFileConnector(tempSourceFile: File, module: Module, sourceScope: SourceScope)
+    extends AbstractRemoteServerConnector(Some(Seq(tempSourceFile)), module, sourceScope) {
+
+    def compile(originalSourceFile: File, client: Client): Unit = {
+      val fixedClient = new DelegateClient(client) {
+        override def message(msg: Client.ClientMsg): Unit = {
+          /**
+           * NOTE: some compiler errors can be with empty `source`<br>
+           * Example: `bad option '-g:vars' was ignored`<br>
+           * We do not want to loose such message.
+           * We rely that they will be reported in the beginning of the file<br>
+           * see [[org.jetbrains.plugins.scala.compiler.highlighting.ExternalHighlightersService.toHighlightInfo]]
+           * (we assume that `from` and `to` are also empty for such files)
+           */
+          val fixedSource = Some(originalSourceFile) //msg.source.map(_ => originalSourceFile)
+          val fixedMsg = msg.copy(source = fixedSource)
+          client.message(fixedMsg)
+        }
+
+        override def compilationEnd(sources: Set[File]): Unit = {
+          val fixedSources = Set(originalSourceFile)
+          client.compilationEnd(fixedSources)
+        }
+      }
+      new RemoteServerRunner()
+        .buildProcess(CommandIds.Compile, arguments.asStrings, fixedClient)
+        .runSync()
+    }
+  }
+
+  private final class InMemoryFileConnector(sourcePath: Path, sourceContent: String, module: Module, sourceScope: SourceScope)
+    extends AbstractRemoteServerConnector(None, module, sourceScope) {
+
+    def compile(client: Client): Unit = {
+      val arguments = DocumentCompilationArguments(
+        sbtData = sbtData,
+        compilerData = CompilerData(
+          compilerJars = CompilerJarsFactory.fromFiles(compilerClasspath, module.customScalaCompilerBridgeJar).toOption,
+          javaHome = Some(findJdk),
+          incrementalType = IncrementalityType.IDEA
+        ),
+        compilationData = DocumentCompilationData(
+          sourcePath = sourcePath,
+          sourceContent = sourceContent,
+          output = workingDirectory,
+          classpath = runtimeClasspath.map(_.toPath),
+          scalacOptions = scalaParameters
+        )
+      )
+
+      new RemoteServerRunner()
+        .buildProcess(CommandIds.CompileDocument, DocumentCompilationArguments.serialize(arguments), client)
+        .runSync()
+    }
+  }
+
+  private abstract class AbstractRemoteServerConnector(filesToCompile: Option[Seq[File]], module: Module, sourceScope: SourceScope)
+    extends RemoteServerConnectorBase(module, filesToCompile, workingDirectory.toFile) {
 
     var requiresCleanup: Boolean = false
 
@@ -112,32 +212,10 @@ private final class DocumentCompiler(project: Project) {
           .map(Path.of(_).toFile)
       (fromSuper ++ outputDir).distinct
     }
-
-    def compile(client: Client): Unit = {
-      val arguments = DocumentCompilationArguments(
-        sbtData = sbtData,
-        compilerData = CompilerData(
-          compilerJars = CompilerJarsFactory.fromFiles(compilerClasspath, module.customScalaCompilerBridgeJar).toOption,
-          javaHome = Some(findJdk),
-          incrementalType = IncrementalityType.IDEA
-        ),
-        compilationData = DocumentCompilationData(
-          sourcePath = sourcePath,
-          sourceContent = sourceContent,
-          output = workingDirectory,
-          classpath = runtimeClasspath.map(_.toPath),
-          scalacOptions = scalaParameters
-        )
-      )
-
-      new RemoteServerRunner()
-        .buildProcess(CommandIds.CompileDocument, DocumentCompilationArguments.serialize(arguments), client)
-        .runSync()
-    }
   }
 }
 
 private object DocumentCompiler {
   def get(project: Project): DocumentCompiler =
     project.getService(classOf[DocumentCompiler])
-}
+}