Introduce best-effort compilation for IDEs

2 new experimental options are introduces for the compiler:
`-Ybest-effort` and `-Ywith-best-effort-tasty`. A related
Best Effort TASTy (.betasty) format, a TASTy aligned file format able
to hold some errored trees was also added. Behaviour of the options and
the format is documented as part of this commit in the
`best-effort-compilation.md` docs file.

`-Ybest-effort` is used to produce `.betasty` files in the
`<output>/META-INF/best-effort`. `-Ywith-best-effort-tasty` allows to
use them during compilation, limiting it to the frontend phases if such
file is used. If any .betasty is used, transparent inline macros also
cease to be expanded by the compiler.

Since best-effort compilation can fail (e.g. due to cyclic reference
errors which sometimes are not able to be pickled or unpickled),
the crashes caused by it are wrapped into an additional descriptive
error message in the aim to fail more gracefully (and not pollute our
issue tracker with known problems).

The feature is tested in two ways:
 * with a set of pairs of dependent projects, one of which is
 meant to produce .betasty by using `-Ybest-effort`, and the other
 tries to consume it using `-Ywith-best-effort-tasty`.
 * by reusing the compiler nonbootstrapped neg tests, first by running
 them with `-Ybest-effort` option, and then by running read-tasty tests
 on the produced betasty files to thest best-effort tastt unpickling

Additionally, `-Ywith-best-effort-tasty` allows to print `.betasty` via
`-print-tasty`.

Author: jchyb

compiler/src/dotty/tools/backend/jvm/GenBCode.scala

@@ -17,6 +17,8 @@ class GenBCode extends Phase { self =>
 
   override def description: String = GenBCode.description
 
+  override def isRunnable(using Context) = super.isRunnable && !ctx.usesBestEffortTasty
+
   private val superCallsMap = new MutableSymbolMap[Set[ClassSymbol]]
   def registerSuperCall(sym: Symbol, calls: ClassSymbol): Unit = {
     val old = superCallsMap.getOrElse(sym, Set.empty)

compiler/src/dotty/tools/backend/sjs/GenSJSIR.scala

@@ -12,7 +12,7 @@ class GenSJSIR extends Phase {
   override def description: String = GenSJSIR.description
 
   override def isRunnable(using Context): Boolean =
-    super.isRunnable && ctx.settings.scalajs.value
+    super.isRunnable && ctx.settings.scalajs.value && !ctx.usesBestEffortTasty
 
   def run(using Context): Unit =
     new JSCodeGen().run()

compiler/src/dotty/tools/dotc/Driver.scala

@@ -39,6 +39,8 @@ class Driver {
       catch
         case ex: FatalError =>
           report.error(ex.getMessage.nn) // signals that we should fail compilation.
+        case ex: Throwable if ctx.usesBestEffortTasty =>
+          report.bestEffortError(ex, "Some best-effort tasty files were not able to be read.")
         case ex: TypeError if !runOrNull.enrichedErrorMessage =>
           println(runOrNull.enrichErrorMessage(s"${ex.toMessage} while compiling ${files.map(_.path).mkString(", ")}"))
           throw ex
@@ -102,8 +104,8 @@ class Driver {
             None
           else file.ext match
             case FileExtension.Jar => Some(file.path)
-            case FileExtension.Tasty =>
-              TastyFileUtil.getClassPath(file) match
+            case FileExtension.Tasty | FileExtension.Betasty =>
+              TastyFileUtil.getClassPath(file, ctx.withBestEffortTasty) match
                 case Some(classpath) => Some(classpath)
                 case _ =>
                   report.error(em"Could not load classname from: ${file.path}")

compiler/src/dotty/tools/dotc/Run.scala

@@ -301,6 +301,10 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint
       ctx.settings.Yskip.value, ctx.settings.YstopBefore.value, stopAfter, ctx.settings.Ycheck.value)
     ctx.base.usePhases(phases, runCtx)
 
+    var forceReachPhaseMaybe =
+      if (ctx.isBestEffort && phases.exists(_.phaseName == "typer")) Some("typer")
+      else None
+
     if ctx.settings.YnoDoubleBindings.value then
       ctx.base.checkNoDoubleBindings = true
 
@@ -311,7 +315,7 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint
 
       for phase <- allPhases do
         doEnterPhase(phase)
-        val phaseWillRun = phase.isRunnable
+        val phaseWillRun = phase.isRunnable || forceReachPhaseMaybe.nonEmpty
         if phaseWillRun then
           Stats.trackTime(s"phase time ms/$phase") {
             val start = System.currentTimeMillis
@@ -324,6 +328,13 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint
                 def printCtx(unit: CompilationUnit) = phase.printingContext(
                   ctx.fresh.setPhase(phase.next).setCompilationUnit(unit))
                 lastPrintedTree = printTree(lastPrintedTree)(using printCtx(unit))
+
+            forceReachPhaseMaybe match {
+              case Some(forceReachPhase) if phase.phaseName == forceReachPhase =>
+                forceReachPhaseMaybe = None
+              case _ =>
+            }
+
             report.informTime(s"$phase ", start)
             Stats.record(s"total trees at end of $phase", ast.Trees.ntrees)
             for (unit <- units)

compiler/src/dotty/tools/dotc/ast/TreeInfo.scala

@@ -919,12 +919,12 @@ trait TypedTreeInfo extends TreeInfo[Type] { self: Trees.Instance[Type] =>
       else cpy.PackageDef(tree)(pid, slicedStats) :: Nil
     case tdef: TypeDef =>
       val sym = tdef.symbol
-      assert(sym.isClass)
+      if !ctx.isBestEffort then assert(sym.isClass)
       if (cls == sym || cls == sym.linkedClass) tdef :: Nil
       else Nil
     case vdef: ValDef =>
       val sym = vdef.symbol
-      assert(sym.is(Module))
+      if !ctx.isBestEffort then assert(sym.is(Module))
       if (cls == sym.companionClass || cls == sym.moduleClass) vdef :: Nil
       else Nil
     case tree =>

compiler/src/dotty/tools/dotc/ast/tpd.scala

@@ -47,7 +47,7 @@ object tpd extends Trees.Instance[Type] with TypedTreeInfo {
     case _: RefTree | _: GenericApply | _: Inlined | _: Hole =>
       ta.assignType(untpd.Apply(fn, args), fn, args)
     case _ =>
-      assert(ctx.reporter.errorsReported)
+      assert(ctx.isBestEffort || ctx.usesBestEffortTasty || ctx.reporter.errorsReported)
       ta.assignType(untpd.Apply(fn, args), fn, args)
 
   def TypeApply(fn: Tree, args: List[Tree])(using Context): TypeApply = fn match
@@ -56,7 +56,7 @@ object tpd extends Trees.Instance[Type] with TypedTreeInfo {
     case _: RefTree | _: GenericApply =>
       ta.assignType(untpd.TypeApply(fn, args), fn, args)
     case _ =>
-      assert(ctx.reporter.errorsReported, s"unexpected tree for type application: $fn")
+      assert(ctx.isBestEffort || ctx.usesBestEffortTasty || ctx.reporter.errorsReported, s"unexpected tree for type application: $fn")
       ta.assignType(untpd.TypeApply(fn, args), fn, args)
 
   def Literal(const: Constant)(using Context): Literal =

compiler/src/dotty/tools/dotc/classpath/DirectoryClassPath.scala

@@ -285,7 +285,7 @@ case class DirectoryClassPath(dir: JFile) extends JFileDirectoryLookup[BinaryFil
   protected def createFileEntry(file: AbstractFile): BinaryFileEntry = BinaryFileEntry(file)
 
   protected def isMatchingFile(f: JFile): Boolean =
-    f.isTasty || (f.isClass && !f.hasSiblingTasty)
+    f.isTasty || f.isBestEffortTasty || (f.isClass && f.hasSiblingTasty)
 
   private[dotty] def classes(inPackage: PackageName): Seq[BinaryFileEntry] = files(inPackage)
 }

compiler/src/dotty/tools/dotc/classpath/FileUtils.scala

@@ -23,8 +23,12 @@ object FileUtils {
 
     def hasTastyExtension: Boolean = file.ext.isTasty
 
+    def hasBetastyExtension: Boolean = file.ext.isBetasty
+
     def isTasty: Boolean = !file.isDirectory && hasTastyExtension
 
+    def isBestEffortTasty: Boolean = !file.isDirectory && hasBetastyExtension
+
     def isScalaBinary: Boolean = file.isClass || file.isTasty
 
     def isScalaOrJavaSource: Boolean = !file.isDirectory && file.ext.isScalaOrJava
@@ -55,6 +59,9 @@ object FileUtils {
 
     def isTasty: Boolean = file.isFile && file.getName.endsWith(SUFFIX_TASTY)
 
+    def isBestEffortTasty: Boolean = file.isFile && file.getName.endsWith(SUFFIX_BETASTY)
+
+
     /**
      * Returns if there is an existing sibling `.tasty` file.
      */
@@ -69,6 +76,7 @@ object FileUtils {
   private val SUFFIX_CLASS = ".class"
   private val SUFFIX_SCALA = ".scala"
   private val SUFFIX_TASTY = ".tasty"
+  private val SUFFIX_BETASTY = ".betasty"
   private val SUFFIX_JAVA = ".java"
   private val SUFFIX_SIG = ".sig"
 

compiler/src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -414,6 +414,9 @@ private sealed trait YSettings:
   val YprofileRunGcBetweenPhases: Setting[List[String]] = PhasesSetting(ForkSetting, "Yprofile-run-gc", "Run a GC between phases - this allows heap size to be accurate at the expense of more time. Specify a list of phases, or *", "_")
       //.withPostSetHook( _ => YprofileEnabled.value = true )
 
+  val YbestEffort: Setting[Boolean] = BooleanSetting(ForkSetting, "Ybest-effort", "Enable best-effort compilation attempting to produce betasty to the META-INF/best-effort directory, regardless of errors, as part of the pickler phase.")
+  val YwithBestEffortTasty: Setting[Boolean] = BooleanSetting(ForkSetting, "Ywith-best-effort-tasty", "Allow to compile using best-effort tasty files. If such file is used, the compiler will stop after the pickler phase.")
+
   // Experimental language features
   val YnoKindPolymorphism: Setting[Boolean] = BooleanSetting(ForkSetting, "Yno-kind-polymorphism", "Disable kind polymorphism.")
   val YexplicitNulls: Setting[Boolean] = BooleanSetting(ForkSetting, "Yexplicit-nulls", "Make reference types non-nullable. Nullable types can be expressed with unions: e.g. String|Null.")

compiler/src/dotty/tools/dotc/core/Contexts.scala

@@ -474,6 +474,18 @@ object Contexts {
 
     /** Is the flexible types option set? */
     def flexibleTypes: Boolean = base.settings.YexplicitNulls.value && !base.settings.YnoFlexibleTypes.value
+    
+    /** Is best-effort-dir option set? */
+    def isBestEffort: Boolean = base.settings.YbestEffort.value
+
+    /** Is the from-best-effort-tasty option set to true? */
+    def withBestEffortTasty: Boolean = base.settings.YwithBestEffortTasty.value
+
+    /** Were any best effort tasty dependencies used during compilation? */
+    def usesBestEffortTasty: Boolean = base.usedBestEffortTasty
+
+    /** Confirm that a best effort tasty dependency was used during compilation. */
+    def setUsesBestEffortTasty(): Unit = base.usedBestEffortTasty = true
 
     /** A fresh clone of this context embedded in this context. */
     def fresh: FreshContext = freshOver(this)
@@ -959,6 +971,9 @@ object Contexts {
     val sources: util.HashMap[AbstractFile, SourceFile] = util.HashMap[AbstractFile, SourceFile]()
     val files: util.HashMap[TermName, AbstractFile] = util.HashMap()
 
+    /** Was best effort file used during compilation? */
+    private[core] var usedBestEffortTasty = false
+
     // Types state
     /** A table for hash consing unique types */
     private[core] val uniques: Uniques = Uniques()

compiler/src/dotty/tools/dotc/core/DenotTransformers.scala

@@ -28,6 +28,8 @@ object DenotTransformers {
 
     /** The transformation method */
     def transform(ref: SingleDenotation)(using Context): SingleDenotation
+
+    override def isRunnable(using Context) = super.isRunnable && !ctx.usesBestEffortTasty 
   }
 
   /** A transformer that only transforms the info field of denotations */

compiler/src/dotty/tools/dotc/core/Denotations.scala

@@ -719,7 +719,8 @@ object Denotations {
         ctx.runId >= validFor.runId
         || ctx.settings.YtestPickler.value // mixing test pickler with debug printing can travel back in time
         || ctx.mode.is(Mode.Printing)  // no use to be picky when printing error messages
-        || symbol.isOneOf(ValidForeverFlags),
+        || symbol.isOneOf(ValidForeverFlags)
+        || ctx.isBestEffort || ctx.usesBestEffortTasty,
         s"denotation $this invalid in run ${ctx.runId}. ValidFor: $validFor")
       var d: SingleDenotation = this
       while ({

compiler/src/dotty/tools/dotc/core/SymDenotations.scala

@@ -720,12 +720,16 @@ object SymDenotations {
      *  TODO: Find a more robust way to characterize self symbols, maybe by
      *       spending a Flag on them?
      */
-    final def isSelfSym(using Context): Boolean = owner.infoOrCompleter match {
-      case ClassInfo(_, _, _, _, selfInfo) =>
-        selfInfo == symbol ||
-          selfInfo.isInstanceOf[Type] && name == nme.WILDCARD
-      case _ => false
-    }
+    final def isSelfSym(using Context): Boolean = 
+      if !ctx.isBestEffort || exists then
+        owner.infoOrCompleter match {
+          case ClassInfo(_, _, _, _, selfInfo) =>
+            selfInfo == symbol ||
+              selfInfo.isInstanceOf[Type] && name == nme.WILDCARD
+          case _ => false
+        }
+      else false
+
 
     /** Is this definition contained in `boundary`?
      *  Same as `ownersIterator contains boundary` but more efficient.
@@ -2003,7 +2007,7 @@ object SymDenotations {
         case p :: parents1 =>
           p.classSymbol match {
             case pcls: ClassSymbol => builder.addAll(pcls.baseClasses)
-            case _ => assert(isRefinementClass || p.isError || ctx.mode.is(Mode.Interactive), s"$this has non-class parent: $p")
+            case _ => assert(isRefinementClass || p.isError || ctx.mode.is(Mode.Interactive) || ctx.isBestEffort || ctx.usesBestEffortTasty, s"$this has non-class parent: $p")
           }
           traverse(parents1)
         case nil =>

compiler/src/dotty/tools/dotc/core/SymbolLoaders.scala

@@ -7,7 +7,7 @@ import java.nio.channels.ClosedByInterruptException
 
 import scala.util.control.NonFatal
 
-import dotty.tools.dotc.classpath.FileUtils.hasTastyExtension
+import dotty.tools.dotc.classpath.FileUtils.{hasTastyExtension, hasBetastyExtension}
 import dotty.tools.io.{ ClassPath, ClassRepresentation, AbstractFile }
 import dotty.tools.backend.jvm.DottyBackendInterface.symExtensions
 
@@ -26,6 +26,7 @@ import parsing.JavaParsers.OutlineJavaParser
 import parsing.Parsers.OutlineParser
 import dotty.tools.tasty.{TastyHeaderUnpickler, UnpickleException, UnpicklerConfig, TastyVersion}
 import dotty.tools.dotc.core.tasty.TastyUnpickler
+import dotty.tools.tasty.besteffort.BestEffortTastyHeaderUnpickler
 
 object SymbolLoaders {
   import ast.untpd.*
@@ -198,7 +199,7 @@ object SymbolLoaders {
         enterToplevelsFromSource(owner, nameOf(classRep), src)
       case (Some(bin), _) =>
         val completer =
-          if bin.hasTastyExtension then ctx.platform.newTastyLoader(bin)
+          if bin.hasTastyExtension || bin.hasBetastyExtension then ctx.platform.newTastyLoader(bin)
           else ctx.platform.newClassLoader(bin)
         enterClassAndModule(owner, nameOf(classRep), completer)
     }
@@ -261,7 +262,8 @@ object SymbolLoaders {
       (idx + str.TOPLEVEL_SUFFIX.length + 1 != name.length || !name.endsWith(str.TOPLEVEL_SUFFIX))
     }
 
-    def maybeModuleClass(classRep: ClassRepresentation): Boolean = classRep.name.last == '$'
+    def maybeModuleClass(classRep: ClassRepresentation): Boolean =
+      classRep.name.nonEmpty && classRep.name.last == '$'
 
     private def enterClasses(root: SymDenotation, packageName: String, flat: Boolean)(using Context) = {
       def isAbsent(classRep: ClassRepresentation) =
@@ -424,26 +426,39 @@ class TastyLoader(val tastyFile: AbstractFile) extends SymbolLoader {
 
   val compilationUnitInfo: CompilationUnitInfo | Null = unpickler.compilationUnitInfo
 
-  def description(using Context): String = "TASTy file " + tastyFile.toString
+  val isBestEffortTasty = tastyFile.name.endsWith(".betasty")
+
+  def description(using Context): String =
+    if tastyFile.extension == ".betasty" then "Best Effort TASTy file " + tastyFile.toString
+    else "TASTy file " + tastyFile.toString
 
   override def doComplete(root: SymDenotation)(using Context): Unit =
     handleUnpicklingExceptions:
-      checkTastyUUID()
       val (classRoot, moduleRoot) = rootDenots(root.asClass)
-      unpickler.enter(roots = Set(classRoot, moduleRoot, moduleRoot.sourceModule))(using ctx.withSource(util.NoSource))
-      if mayLoadTreesFromTasty then
-        classRoot.classSymbol.rootTreeOrProvider = unpickler
-        moduleRoot.classSymbol.rootTreeOrProvider = unpickler
-
+      if (!isBestEffortTasty || ctx.withBestEffortTasty) then
+        val tastyBytes = tastyFile.toByteArray
+        val unpickler = new tasty.DottyUnpickler(tastyFile, tastyBytes, isBestEffortTasty = isBestEffortTasty)
+        unpickler.enter(roots = Set(classRoot, moduleRoot, moduleRoot.sourceModule))(using ctx.withSource(util.NoSource))
+        if mayLoadTreesFromTasty || (isBestEffortTasty && ctx.withBestEffortTasty) then
+          classRoot.classSymbol.rootTreeOrProvider = unpickler
+          moduleRoot.classSymbol.rootTreeOrProvider = unpickler
+        if isBestEffortTasty then
+          checkBeTastyUUID(tastyFile, tastyBytes)
+          ctx.setUsesBestEffortTasty()
+        else
+          checkTastyUUID()
+      else
+        report.error(em"Best Effort TASTy $tastyFile file could not be read.")
   private def handleUnpicklingExceptions[T](thunk: =>T): T =
     try thunk
     catch case e: RuntimeException =>
+      val tastyType = if (isBestEffortTasty) "Best Effort TASTy" else "TASTy"
       val message = e match
         case e: UnpickleException =>
-          s"""TASTy file ${tastyFile.canonicalPath} could not be read, failing with:
+          s"""$tastyType file ${tastyFile.canonicalPath} could not be read, failing with:
             |  ${Option(e.getMessage).getOrElse("")}""".stripMargin
         case _ =>
-          s"""TASTy file ${tastyFile.canonicalPath} is broken, reading aborted with ${e.getClass}
+          s"""$tastyFile file ${tastyFile.canonicalPath} is broken, reading aborted with ${e.getClass}
             |  ${Option(e.getMessage).getOrElse("")}""".stripMargin
       throw IOException(message, e)
 
@@ -460,6 +475,9 @@ class TastyLoader(val tastyFile: AbstractFile) extends SymbolLoader {
       // tasty file compiled by `-Yearly-tasty-output-write` comes from an early output jar.
       report.inform(s"No classfiles found for $tastyFile when checking TASTy UUID")
 
+  private def checkBeTastyUUID(tastyFile: AbstractFile, tastyBytes: Array[Byte])(using Context): Unit =
+    new BestEffortTastyHeaderUnpickler(tastyBytes).readHeader()
+
   private def mayLoadTreesFromTasty(using Context): Boolean =
     ctx.settings.YretainTrees.value || ctx.settings.fromTasty.value
 }

compiler/src/dotty/tools/dotc/core/TypeErasure.scala

@@ -756,7 +756,8 @@ class TypeErasure(sourceLanguage: SourceLanguage, semiEraseVCs: Boolean, isConst
           MissingType(tycon.prefix, tycon.name)
         case _ =>
           TypeError(em"Cannot resolve reference to $tp")
-      throw typeErr
+      if ctx.isBestEffort then report.error(typeErr.toMessage)
+      else throw typeErr
     tp1
 
   /** Widen term ref, skipping any `()` parameter of an eventual getter. Used to erase a TermRef.

compiler/src/dotty/tools/dotc/core/Types.scala

@@ -3149,7 +3149,8 @@ object Types extends TypeUtils {
       if (ctx.erasedTypes) tref
       else cls.info match {
         case cinfo: ClassInfo => cinfo.selfType
-        case _: ErrorType | NoType if ctx.mode.is(Mode.Interactive) => cls.info
+        case _: ErrorType | NoType
+          if ctx.mode.is(Mode.Interactive) || ctx.isBestEffort || ctx.usesBestEffortTasty => cls.info
           // can happen in IDE if `cls` is stale
       }
 
@@ -3719,8 +3720,8 @@ object Types extends TypeUtils {
 
     def apply(tp1: Type, tp2: Type, soft: Boolean)(using Context): OrType = {
       def where = i"in union $tp1 | $tp2"
-      expectValueTypeOrWildcard(tp1, where)
-      expectValueTypeOrWildcard(tp2, where)
+      if (!ctx.usesBestEffortTasty) expectValueTypeOrWildcard(tp1, where)
+      if (!ctx.usesBestEffortTasty) expectValueTypeOrWildcard(tp2, where)
       assertUnerased()
       unique(new CachedOrType(tp1, tp2, soft))
     }