Introduce best-effort compilation for IDEs

2 new experimental options are introduces for the compiler:
`-Ybest-effort` and `-Ywith-best-effort-tasty`. A related
Best Effort TASTy (.betasty) format, a TASTy aligned file format able
to hold some errored trees. Behaviour of the options and the format is
documented as part of this commit in the `best-effort-compilation.md`
docs file.

`-Ybest-effort` is used to produce `.betasty` files in the
`<output>/META-INF/best-effort`. `-Ywith-best-effort-tasty` allows to
use them during compilation, limiting it to the frontend phases if such
file is used. If any .betasty is used, transparent inline macros also
cease to be expanded by the compiler.

Since best-effort compilation can fail (e.g. due to cyclic reference
errors which sometimes are not able to be pickled or unpickled),
the crashes caused by it are wrapped into an additional descriptive
error message in the aim to fail more gracefully (and not pollute our
issue tracker with known problems).

The feature is tested in two ways:
 * with a set of pairs of dependent projects, one of which is
 meant to produce .betasty by using `-Ybest-effort`, and the other
 tries to consume it using `-Ywith-best-effort-tasty`.
 * by reusing the compiler nonbootstrapped neg tests, first by running
 them with `-Ybest-effort` option, and then by running read-tasty tests
 on the produced betasty files to thest best-effort tastt unpickling

Additionally, `-Ywith-best-effort-tasty` allows to print `.betasty` via
`-print-tasty`.

Author: jchyb

compiler/src/dotty/tools/backend/jvm/GenBCode.scala

@@ -16,6 +16,8 @@ class GenBCode extends Phase { self =>
 
   override def description: String = GenBCode.description
 
+  override def isRunnable(using Context) = super.isRunnable && !ctx.usesBestEffortTasty
+
   private val superCallsMap = new MutableSymbolMap[Set[ClassSymbol]]
   def registerSuperCall(sym: Symbol, calls: ClassSymbol): Unit = {
     val old = superCallsMap.getOrElse(sym, Set.empty)

compiler/src/dotty/tools/backend/sjs/GenSJSIR.scala

@@ -12,7 +12,7 @@ class GenSJSIR extends Phase {
   override def description: String = GenSJSIR.description
 
   override def isRunnable(using Context): Boolean =
-    super.isRunnable && ctx.settings.scalajs.value
+    super.isRunnable && ctx.settings.scalajs.value && !ctx.usesBestEffortTasty
 
   def run(using Context): Unit =
     new JSCodeGen().run()

compiler/src/dotty/tools/dotc/Driver.scala

@@ -39,6 +39,8 @@ class Driver {
       catch
         case ex: FatalError =>
           report.error(ex.getMessage.nn) // signals that we should fail compilation.
+        case ex: Throwable if ctx.usesBestEffortTasty =>
+          report.bestEffortError(ex, "Some best-effort tasty files were not able to be read.")
         case ex: TypeError if !runOrNull.enrichedErrorMessage =>
           println(runOrNull.enrichErrorMessage(s"${ex.toMessage} while compiling ${files.map(_.path).mkString(", ")}"))
           throw ex
@@ -100,8 +102,8 @@ class Driver {
             None
           else file.extension match
             case "jar" => Some(file.path)
-            case "tasty" =>
-              TastyFileUtil.getClassPath(file) match
+            case "tasty" | "betasty" =>
+              TastyFileUtil.getClassPath(file, ctx.withBestEffortTasty) match
                 case Some(classpath) => Some(classpath)
                 case _ =>
                   report.error(em"Could not load classname from: ${file.path}")

compiler/src/dotty/tools/dotc/Run.scala

@@ -230,6 +230,10 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint
       ctx.settings.Yskip.value, ctx.settings.YstopBefore.value, stopAfter, ctx.settings.Ycheck.value)
     ctx.base.usePhases(phases)
 
+    var forceReachPhaseMaybe =
+      if (ctx.isBestEffort && phases.exists(_.phaseName == "typer")) Some("typer")
+      else None
+
     if ctx.settings.YnoDoubleBindings.value then
       ctx.base.checkNoDoubleBindings = true
 
@@ -239,7 +243,7 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint
       var phasesWereAdjusted = false
 
       for (phase <- ctx.base.allPhases)
-        if (phase.isRunnable)
+        if (phase.isRunnable || forceReachPhaseMaybe.nonEmpty)
           Stats.trackTime(s"phase time ms/$phase") {
             val start = System.currentTimeMillis
             val profileBefore = profiler.beforePhase(phase)
@@ -249,6 +253,13 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint
               for (unit <- units)
                 lastPrintedTree =
                   printTree(lastPrintedTree)(using ctx.fresh.setPhase(phase.next).setCompilationUnit(unit))
+
+            forceReachPhaseMaybe match {
+              case Some(forceReachPhase) if phase.phaseName == forceReachPhase =>
+                forceReachPhaseMaybe = None
+              case _ =>
+            }
+
             report.informTime(s"$phase ", start)
             Stats.record(s"total trees at end of $phase", ast.Trees.ntrees)
             for (unit <- units)

compiler/src/dotty/tools/dotc/ast/TreeInfo.scala

@@ -893,12 +893,12 @@ trait TypedTreeInfo extends TreeInfo[Type] { self: Trees.Instance[Type] =>
       else cpy.PackageDef(tree)(pid, slicedStats) :: Nil
     case tdef: TypeDef =>
       val sym = tdef.symbol
-      assert(sym.isClass)
+      if !ctx.isBestEffort then assert(sym.isClass)
       if (cls == sym || cls == sym.linkedClass) tdef :: Nil
       else Nil
     case vdef: ValDef =>
       val sym = vdef.symbol
-      assert(sym.is(Module))
+      if !ctx.isBestEffort then assert(sym.is(Module))
       if (cls == sym.companionClass || cls == sym.moduleClass) vdef :: Nil
       else Nil
     case tree =>

compiler/src/dotty/tools/dotc/ast/tpd.scala

@@ -48,7 +48,7 @@ object tpd extends Trees.Instance[Type] with TypedTreeInfo {
     case _: RefTree | _: GenericApply | _: Inlined | _: Hole =>
       ta.assignType(untpd.Apply(fn, args), fn, args)
     case _ =>
-      assert(ctx.reporter.errorsReported)
+      assert(ctx.isBestEffort || ctx.usesBestEffortTasty || ctx.reporter.errorsReported)
       ta.assignType(untpd.Apply(fn, args), fn, args)
 
   def TypeApply(fn: Tree, args: List[Tree])(using Context): TypeApply = fn match
@@ -57,7 +57,7 @@ object tpd extends Trees.Instance[Type] with TypedTreeInfo {
     case _: RefTree | _: GenericApply =>
       ta.assignType(untpd.TypeApply(fn, args), fn, args)
     case _ =>
-      assert(ctx.reporter.errorsReported)
+      assert(ctx.isBestEffort || ctx.usesBestEffortTasty || ctx.reporter.errorsReported)
       ta.assignType(untpd.TypeApply(fn, args), fn, args)
 
   def Literal(const: Constant)(using Context): Literal =

compiler/src/dotty/tools/dotc/classpath/DirectoryClassPath.scala

@@ -288,7 +288,7 @@ case class DirectoryClassPath(dir: JFile) extends JFileDirectoryLookup[ClassFile
 
   protected def createFileEntry(file: AbstractFile): ClassFileEntryImpl = ClassFileEntryImpl(file)
   protected def isMatchingFile(f: JFile): Boolean =
-    f.isTasty || (f.isClass && f.classToTasty.isEmpty)
+    f.isTasty || f.isBestEffortTasty || (f.isClass && f.classToTasty.isEmpty)
 
   private[dotty] def classes(inPackage: PackageName): Seq[ClassFileEntry] = files(inPackage)
 }

compiler/src/dotty/tools/dotc/classpath/FileUtils.scala

@@ -24,8 +24,12 @@ object FileUtils {
 
     def hasTastyExtension: Boolean = file.hasExtension("tasty")
 
+    def hasBetastyExtension: Boolean = file.hasExtension("betasty")
+
     def isTasty: Boolean = !file.isDirectory && hasTastyExtension
 
+    def isBestEffortTasty: Boolean = !file.isDirectory && hasBetastyExtension
+
     def isScalaBinary: Boolean = file.isClass || file.isTasty
 
     def isScalaOrJavaSource: Boolean = !file.isDirectory && (file.hasExtension("scala") || file.hasExtension("java"))
@@ -54,6 +58,8 @@ object FileUtils {
 
     def isTasty: Boolean = file.isFile && file.getName.endsWith(SUFFIX_TASTY)
 
+    def isBestEffortTasty: Boolean = file.isFile && file.getName.endsWith(SUFFIX_BETASTY)
+
     /** Returns the tasty file associated with this class file */
     def classToTasty: Option[JFile] =
       assert(file.isClass, s"non-class: $file")
@@ -66,6 +72,7 @@ object FileUtils {
   private val SUFFIX_CLASS = ".class"
   private val SUFFIX_SCALA = ".scala"
   private val SUFFIX_TASTY = ".tasty"
+  private val SUFFIX_BETASTY = ".betasty"
   private val SUFFIX_JAVA = ".java"
   private val SUFFIX_SIG = ".sig"
 

compiler/src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -380,6 +380,9 @@ private sealed trait YSettings:
   val YprofileRunGcBetweenPhases: Setting[List[String]] = PhasesSetting("-Yprofile-run-gc", "Run a GC between phases - this allows heap size to be accurate at the expense of more time. Specify a list of phases, or *", "_")
       //.withPostSetHook( _ => YprofileEnabled.value = true )
 
+  val YbestEffort: Setting[Boolean] = BooleanSetting("-Ybest-effort", "Enable best-effort compilation attempting to produce betasty to the META-INF/best-effort directory, regardless of errors, as part of the pickler phase.")
+  val YwithBestEffortTasty: Setting[Boolean] = BooleanSetting("-Ywith-best-effort-tasty", "Allow to compile using best-effort tasty files. If such file is used, the compiler will stop after the pickler phase.")
+
   // Experimental language features
   val YnoKindPolymorphism: Setting[Boolean] = BooleanSetting("-Yno-kind-polymorphism", "Disable kind polymorphism.")
   val YexplicitNulls: Setting[Boolean] = BooleanSetting("-Yexplicit-nulls", "Make reference types non-nullable. Nullable types can be expressed with unions: e.g. String|Null.")

compiler/src/dotty/tools/dotc/core/Contexts.scala

@@ -463,6 +463,18 @@ object Contexts {
     /** Is the explicit nulls option set? */
     def explicitNulls: Boolean = base.settings.YexplicitNulls.value
 
+    /** Is best-effort-dir option set? */
+    def isBestEffort: Boolean = base.settings.YbestEffort.value
+
+    /** Is the from-best-effort-tasty option set to true? */
+    def withBestEffortTasty: Boolean = base.settings.YwithBestEffortTasty.value
+
+    /** Were any best effort tasty dependencies used during compilation? */
+    def usesBestEffortTasty: Boolean = base.usedBestEffortTasty
+
+    /** Confirm that a best effort tasty dependency was used during compilation. */
+    def setUsesBestEffortTasty(): Unit = base.usedBestEffortTasty = true
+
     /** A fresh clone of this context embedded in this context. */
     def fresh: FreshContext = freshOver(this)
 
@@ -949,6 +961,9 @@ object Contexts {
     val sources: util.HashMap[AbstractFile, SourceFile] = util.HashMap[AbstractFile, SourceFile]()
     val files: util.HashMap[TermName, AbstractFile] = util.HashMap()
 
+    /** Was best effort file used during compilation? */
+    private[core] var usedBestEffortTasty = false
+
     // Types state
     /** A table for hash consing unique types */
     private[core] val uniques: Uniques = Uniques()

compiler/src/dotty/tools/dotc/core/DenotTransformers.scala

@@ -28,6 +28,8 @@ object DenotTransformers {
 
     /** The transformation method */
     def transform(ref: SingleDenotation)(using Context): SingleDenotation
+
+    override def isRunnable(using Context) = super.isRunnable && !ctx.usesBestEffortTasty 
   }
 
   /** A transformer that only transforms the info field of denotations */

compiler/src/dotty/tools/dotc/core/Denotations.scala

@@ -717,7 +717,8 @@ object Denotations {
         ctx.runId >= validFor.runId
         || ctx.settings.YtestPickler.value // mixing test pickler with debug printing can travel back in time
         || ctx.mode.is(Mode.Printing)  // no use to be picky when printing error messages
-        || symbol.isOneOf(ValidForeverFlags),
+        || symbol.isOneOf(ValidForeverFlags)
+        || ctx.isBestEffort || ctx.usesBestEffortTasty,
         s"denotation $this invalid in run ${ctx.runId}. ValidFor: $validFor")
       var d: SingleDenotation = this
       while ({

compiler/src/dotty/tools/dotc/core/SymDenotations.scala

@@ -711,12 +711,16 @@ object SymDenotations {
      *  TODO: Find a more robust way to characterize self symbols, maybe by
      *       spending a Flag on them?
      */
-    final def isSelfSym(using Context): Boolean = owner.infoOrCompleter match {
-      case ClassInfo(_, _, _, _, selfInfo) =>
-        selfInfo == symbol ||
-          selfInfo.isInstanceOf[Type] && name == nme.WILDCARD
-      case _ => false
-    }
+    final def isSelfSym(using Context): Boolean = 
+      if !ctx.isBestEffort || exists then
+        owner.infoOrCompleter match {
+          case ClassInfo(_, _, _, _, selfInfo) =>
+            selfInfo == symbol ||
+              selfInfo.isInstanceOf[Type] && name == nme.WILDCARD
+          case _ => false
+        }
+      else false
+
 
     /** Is this definition contained in `boundary`?
      *  Same as `ownersIterator contains boundary` but more efficient.
@@ -1990,7 +1994,7 @@ object SymDenotations {
         case p :: parents1 =>
           p.classSymbol match {
             case pcls: ClassSymbol => builder.addAll(pcls.baseClasses)
-            case _ => assert(isRefinementClass || p.isError || ctx.mode.is(Mode.Interactive), s"$this has non-class parent: $p")
+            case _ => assert(isRefinementClass || p.isError || ctx.mode.is(Mode.Interactive) || ctx.isBestEffort || ctx.usesBestEffortTasty, s"$this has non-class parent: $p")
           }
           traverse(parents1)
         case nil =>

compiler/src/dotty/tools/dotc/core/SymbolLoaders.scala

@@ -7,7 +7,7 @@ import java.nio.channels.ClosedByInterruptException
 
 import scala.util.control.NonFatal
 
-import dotty.tools.dotc.classpath.FileUtils.isTasty
+import dotty.tools.dotc.classpath.FileUtils.{isTasty, isBestEffortTasty}
 import dotty.tools.io.{ ClassPath, ClassRepresentation, AbstractFile }
 import dotty.tools.backend.jvm.DottyBackendInterface.symExtensions
 
@@ -25,6 +25,7 @@ import ast.desugar
 import parsing.JavaParsers.OutlineJavaParser
 import parsing.Parsers.OutlineParser
 import dotty.tools.tasty.TastyHeaderUnpickler
+import dotty.tools.tasty.besteffort.BestEffortTastyHeaderUnpickler
 
 
 object SymbolLoaders {
@@ -198,7 +199,7 @@ object SymbolLoaders {
         enterToplevelsFromSource(owner, nameOf(classRep), src)
       case (Some(bin), _) =>
         val completer =
-          if bin.isTasty then ctx.platform.newTastyLoader(bin)
+          if bin.isTasty || bin.isBestEffortTasty then ctx.platform.newTastyLoader(bin)
           else ctx.platform.newClassLoader(bin)
         enterClassAndModule(owner, nameOf(classRep), completer)
     }
@@ -261,7 +262,8 @@ object SymbolLoaders {
       (idx + str.TOPLEVEL_SUFFIX.length + 1 != name.length || !name.endsWith(str.TOPLEVEL_SUFFIX))
     }
 
-    def maybeModuleClass(classRep: ClassRepresentation): Boolean = classRep.name.last == '$'
+    def maybeModuleClass(classRep: ClassRepresentation): Boolean =
+      classRep.name.nonEmpty && classRep.name.last == '$'
 
     private def enterClasses(root: SymDenotation, packageName: String, flat: Boolean)(using Context) = {
       def isAbsent(classRep: ClassRepresentation) =
@@ -418,17 +420,27 @@ class TastyLoader(val tastyFile: AbstractFile) extends SymbolLoader {
 
   override def sourceFileOrNull: AbstractFile | Null = tastyFile
 
-  def description(using Context): String = "TASTy file " + tastyFile.toString
+  def description(using Context): String =
+    if tastyFile.extension == ".betasty" then "Best Effort TASTy file " + tastyFile.toString
+    else "TASTy file " + tastyFile.toString
 
   override def doComplete(root: SymDenotation)(using Context): Unit =
     val (classRoot, moduleRoot) = rootDenots(root.asClass)
-    val tastyBytes = tastyFile.toByteArray
-    val unpickler = new tasty.DottyUnpickler(tastyBytes)
-    unpickler.enter(roots = Set(classRoot, moduleRoot, moduleRoot.sourceModule))(using ctx.withSource(util.NoSource))
-    if mayLoadTreesFromTasty then
-      classRoot.classSymbol.rootTreeOrProvider = unpickler
-      moduleRoot.classSymbol.rootTreeOrProvider = unpickler
-    checkTastyUUID(tastyFile, tastyBytes)
+    val isBestEffortTasty = tastyFile.name.endsWith(".betasty")
+    if (!isBestEffortTasty || ctx.withBestEffortTasty) then
+      val tastyBytes = tastyFile.toByteArray
+      val unpickler = new tasty.DottyUnpickler(tastyBytes, isBestEffortTasty = isBestEffortTasty)
+      unpickler.enter(roots = Set(classRoot, moduleRoot, moduleRoot.sourceModule))(using ctx.withSource(util.NoSource))
+      if mayLoadTreesFromTasty || (isBestEffortTasty && ctx.withBestEffortTasty) then
+        classRoot.classSymbol.rootTreeOrProvider = unpickler
+        moduleRoot.classSymbol.rootTreeOrProvider = unpickler
+      if isBestEffortTasty then
+        checkBeTastyUUID(tastyFile, tastyBytes)
+        ctx.setUsesBestEffortTasty()
+      else
+        checkTastyUUID(tastyFile, tastyBytes)
+    else
+      report.error(em"$tastyFile Best Effort TASTy file could not be read.")
 
 
   private def checkTastyUUID(tastyFile: AbstractFile, tastyBytes: Array[Byte])(using Context): Unit =
@@ -442,6 +454,9 @@ class TastyLoader(val tastyFile: AbstractFile) extends SymbolLoader {
       // This will be the case in any of our tests that compile with `-Youtput-only-tasty`
       report.inform(s"No classfiles found for $tastyFile when checking TASTy UUID")
 
+  private def checkBeTastyUUID(tastyFile: AbstractFile, tastyBytes: Array[Byte])(using Context): Unit =
+    new BestEffortTastyHeaderUnpickler(tastyBytes).readHeader()
+
   private def mayLoadTreesFromTasty(using Context): Boolean =
     ctx.settings.YretainTrees.value || ctx.settings.fromTasty.value
 }

compiler/src/dotty/tools/dotc/core/TypeErasure.scala

@@ -759,7 +759,8 @@ class TypeErasure(sourceLanguage: SourceLanguage, semiEraseVCs: Boolean, isConst
           MissingType(tycon.prefix, tycon.name).toMessage.message
         case _ =>
           i"Cannot resolve reference to $tp"
-      throw FatalError(msg)
+      if ctx.isBestEffort then report.error(msg)
+      else throw FatalError(msg)
     tp1
 
   /** Widen term ref, skipping any `()` parameter of an eventual getter. Used to erase a TermRef.

compiler/src/dotty/tools/dotc/core/Types.scala

@@ -3051,7 +3051,8 @@ object Types {
       if (ctx.erasedTypes) tref
       else cls.info match {
         case cinfo: ClassInfo => cinfo.selfType
-        case _: ErrorType | NoType if ctx.mode.is(Mode.Interactive) => cls.info
+        case _: ErrorType | NoType
+          if ctx.mode.is(Mode.Interactive) || ctx.isBestEffort || ctx.usesBestEffortTasty => cls.info
           // can happen in IDE if `cls` is stale
       }
 
@@ -3574,8 +3575,8 @@ object Types {
 
     def apply(tp1: Type, tp2: Type, soft: Boolean)(using Context): OrType = {
       def where = i"in union $tp1 | $tp2"
-      expectValueTypeOrWildcard(tp1, where)
-      expectValueTypeOrWildcard(tp2, where)
+      if (!ctx.usesBestEffortTasty) expectValueTypeOrWildcard(tp1, where)
+      if (!ctx.usesBestEffortTasty) expectValueTypeOrWildcard(tp2, where)
       assertUnerased()
       unique(new CachedOrType(tp1, tp2, soft))
     }

compiler/src/dotty/tools/dotc/core/tasty/BestEffortTastyWriter.scala

@@ -0,0 +1,43 @@
+package dotty.tools.dotc
+package core
+package tasty
+
+import scala.language.unsafeNulls
+import java.nio.file.{Path as JPath, Files as JFiles}
+import java.nio.channels.ClosedByInterruptException
+import java.io.DataOutputStream
+import dotty.tools.io.{File, PlainFile}
+import dotty.tools.dotc.core.Contexts.Context
+
+object BestEffortTastyWriter:
+
+  def write(dir: JPath, units: List[CompilationUnit])(using Context): Unit =
+    if JFiles.exists(dir) then JFiles.createDirectories(dir)
+
+    units.foreach { unit =>
+      unit.pickled.foreach { (clz, binary) =>
+        val parts = clz.fullName.mangledString.split('.')
+        val outPath = outputPath(parts.toList, dir)
+        val outTastyFile = new PlainFile(new File(outPath))
+        val outstream = new DataOutputStream(outTastyFile.bufferedOutput)
+        try outstream.write(binary())
+        catch case ex: ClosedByInterruptException =>
+          try
+            outTastyFile.delete() // don't leave an empty or half-written tastyfile around after an interrupt
+          catch
+            case _: Throwable =>
+          throw ex
+        finally outstream.close()
+      }
+    }
+
+  def outputPath(parts: List[String], acc: JPath): JPath =
+    parts match
+      case Nil => throw new Exception("Invalid class name")
+      case last :: Nil =>
+        val name = last.stripSuffix("$")
+        acc.resolve(s"$name.betasty")
+      case pkg :: tail =>
+        val next = acc.resolve(pkg)
+        if !JFiles.exists(next) then JFiles.createDirectory(next)
+        outputPath(tail, next)