Introduce best-effort compilation for IDEs

2 new experimental options are introduces for the compiler:
`-Ybest-effort-dir` and `-Ywith-best-effort-tasty`. A related
Best Effort TASTy format, a TASTy aligned file format able to hold some
errored trees. Behaviour of the options and the format is documented as
part of this commit in the `best-effort-compilation.md` docs file.

Author: jchyb

compiler/src/dotty/tools/backend/jvm/GenBCode.scala

@@ -16,6 +16,8 @@ class GenBCode extends Phase { self =>
 
   override def description: String = GenBCode.description
 
+  override def isRunnable(using Context) = super.isRunnable && !ctx.usesBestEffortTasty
+
   private val superCallsMap = new MutableSymbolMap[Set[ClassSymbol]]
   def registerSuperCall(sym: Symbol, calls: ClassSymbol): Unit = {
     val old = superCallsMap.getOrElse(sym, Set.empty)

compiler/src/dotty/tools/backend/sjs/GenSJSIR.scala

@@ -12,7 +12,7 @@ class GenSJSIR extends Phase {
   override def description: String = GenSJSIR.description
 
   override def isRunnable(using Context): Boolean =
-    super.isRunnable && ctx.settings.scalajs.value
+    super.isRunnable && ctx.settings.scalajs.value && !ctx.usesBestEffortTasty
 
   def run(using Context): Unit =
     new JSCodeGen().run()

compiler/src/dotty/tools/dotc/Driver.scala

@@ -100,8 +100,8 @@ class Driver {
             None
           else file.extension match
             case "jar" => Some(file.path)
-            case "tasty" =>
-              TastyFileUtil.getClassPath(file) match
+            case "tasty" | "betasty" =>
+              TastyFileUtil.getClassPath(file, ctx.withBestEffortTasty) match
                 case Some(classpath) => Some(classpath)
                 case _ =>
                   report.error(em"Could not load classname from: ${file.path}")

compiler/src/dotty/tools/dotc/Run.scala

@@ -225,6 +225,13 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint
       if (ctx.settings.YtestPickler.value) List("pickler")
       else ctx.settings.YstopAfter.value
 
+    var forceReachPhaseMaybe =
+      if (ctx.isBestEffort) Some("typer")
+      else None
+
+    var reachedSemanticDB = false
+    var reachedPickler = false
+
     val pluginPlan = ctx.base.addPluginPhases(ctx.base.phasePlan)
     val phases = ctx.base.fusePhases(pluginPlan,
       ctx.settings.Yskip.value, ctx.settings.YstopBefore.value, stopAfter, ctx.settings.Ycheck.value)
@@ -239,7 +246,7 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint
       var phasesWereAdjusted = false
 
       for (phase <- ctx.base.allPhases)
-        if (phase.isRunnable)
+        if (phase.isRunnable || forceReachPhaseMaybe.nonEmpty)
           Stats.trackTime(s"$phase ms ") {
             val start = System.currentTimeMillis
             val profileBefore = profiler.beforePhase(phase)
@@ -249,6 +256,24 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint
               for (unit <- units)
                 lastPrintedTree =
                   printTree(lastPrintedTree)(using ctx.fresh.setPhase(phase.next).setCompilationUnit(unit))
+            
+            forceReachPhaseMaybe match {
+              case Some(forceReachPhase) if phase.phaseName == forceReachPhase =>
+                forceReachPhaseMaybe = None
+              case _ =>
+            }
+
+            if phase.phaseName == "extractSemanticDB" then reachedSemanticDB = true
+            if phase.phaseName == "pickler" then reachedPickler = true
+
+            if !reachedSemanticDB && forceReachPhaseMaybe.isEmpty && ctx.reporter.hasErrors && ctx.isBestEffort then
+              ctx.base.allPhases.find(_.phaseName == "extractSemanticDB").foreach(_.runOn(units))
+              reachedSemanticDB = true
+
+            if !reachedPickler && forceReachPhaseMaybe.isEmpty && ctx.reporter.hasErrors && ctx.isBestEffort then
+              ctx.base.allPhases.find(_.phaseName == "pickler").foreach(_.runOn(units))
+              reachedPickler = true
+
             report.informTime(s"$phase ", start)
             Stats.record(s"total trees at end of $phase", ast.Trees.ntrees)
             for (unit <- units)

compiler/src/dotty/tools/dotc/ast/TreeInfo.scala

@@ -886,12 +886,12 @@ trait TypedTreeInfo extends TreeInfo[Type] { self: Trees.Instance[Type] =>
       else cpy.PackageDef(tree)(pid, slicedStats) :: Nil
     case tdef: TypeDef =>
       val sym = tdef.symbol
-      assert(sym.isClass)
+      if !ctx.isBestEffort then assert(sym.isClass)
       if (cls == sym || cls == sym.linkedClass) tdef :: Nil
       else Nil
     case vdef: ValDef =>
       val sym = vdef.symbol
-      assert(sym.is(Module))
+      if !ctx.isBestEffort then assert(sym.is(Module))
       if (cls == sym.companionClass || cls == sym.moduleClass) vdef :: Nil
       else Nil
     case tree =>

compiler/src/dotty/tools/dotc/ast/tpd.scala

@@ -48,7 +48,7 @@ object tpd extends Trees.Instance[Type] with TypedTreeInfo {
     case _: RefTree | _: GenericApply | _: Inlined | _: Hole =>
       ta.assignType(untpd.Apply(fn, args), fn, args)
     case _ =>
-      assert(ctx.reporter.errorsReported)
+      assert(ctx.isBestEffort || ctx.usesBestEffortTasty || ctx.reporter.errorsReported)
       ta.assignType(untpd.Apply(fn, args), fn, args)
 
   def TypeApply(fn: Tree, args: List[Tree])(using Context): TypeApply = fn match
@@ -57,7 +57,7 @@ object tpd extends Trees.Instance[Type] with TypedTreeInfo {
     case _: RefTree | _: GenericApply =>
       ta.assignType(untpd.TypeApply(fn, args), fn, args)
     case _ =>
-      assert(ctx.reporter.errorsReported)
+      assert(ctx.isBestEffort || ctx.usesBestEffortTasty || ctx.reporter.errorsReported)
       ta.assignType(untpd.TypeApply(fn, args), fn, args)
 
   def Literal(const: Constant)(using Context): Literal =

compiler/src/dotty/tools/dotc/classpath/DirectoryClassPath.scala

@@ -286,7 +286,7 @@ case class DirectoryClassPath(dir: JFile) extends JFileDirectoryLookup[ClassFile
   }
 
   protected def createFileEntry(file: AbstractFile): ClassFileEntryImpl = ClassFileEntryImpl(file)
-  protected def isMatchingFile(f: JFile): Boolean = f.isClass
+  protected def isMatchingFile(f: JFile): Boolean = f.isClass || f.isBestEffortTasty
 
   private[dotty] def classes(inPackage: PackageName): Seq[ClassFileEntry] = files(inPackage)
 }

compiler/src/dotty/tools/dotc/classpath/FileUtils.scala

@@ -37,6 +37,10 @@ object FileUtils {
 
     def isClass: Boolean = file.isFile && file.getName.endsWith(".class") && !file.getName.endsWith("$class.class")
       // FIXME: drop last condition when we stop being compatible with Scala 2.11
+
+    def isTasty: Boolean = file.isFile && file.getName.endsWith(".tasty")
+
+    def isBestEffortTasty: Boolean = file.isFile && file.getName.endsWith(".betasty")
   }
 
   private val SUFFIX_CLASS = ".class"

compiler/src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -376,6 +376,9 @@ private sealed trait YSettings:
       //.withPostSetHook( _ => YprofileEnabled.value = true )
   val YprofileRunGcBetweenPhases: Setting[List[String]] = PhasesSetting("-Yprofile-run-gc", "Run a GC between phases - this allows heap size to be accurate at the expense of more time. Specify a list of phases, or *", "_")
       //.withPostSetHook( _ => YprofileEnabled.value = true )
+  
+  val YbestEffortDir: Setting[String] = StringSetting("-Ybest-effort-dir", "dir", "Enable best-effort compilation attempting to produce tasty in case of failure to specified directory, as part of the pickler phase.", "")
+  val YwithBestEffortTasty: Setting[Boolean] = BooleanSetting("-Ywith-best-effort-tasty", "Allow to compile from a best effort tasty files. If such file is used, the compiler will stop after the pickler phase.")
 
   // Experimental language features
   val YnoKindPolymorphism: Setting[Boolean] = BooleanSetting("-Yno-kind-polymorphism", "Disable kind polymorphism.")

compiler/src/dotty/tools/dotc/core/Contexts.scala

@@ -453,6 +453,18 @@ object Contexts {
     /** Is the explicit nulls option set? */
     def explicitNulls: Boolean = base.settings.YexplicitNulls.value
 
+    /** Is best-effort-dir option set? */
+    def isBestEffort: Boolean = base.settings.YbestEffortDir.value.nonEmpty
+
+    /** Is the from-best-effort-tasty option set to true? */
+    def withBestEffortTasty: Boolean = base.settings.YwithBestEffortTasty.value
+
+    /** Were any best effort tasty dependencies used during compilation? */
+    def usesBestEffortTasty: Boolean = base.usedBestEffortTasty
+
+    /** Confirm that a best effort tasty dependency was used during compilation. */
+    def setUsesBestEffortTasty(): Unit = base.usedBestEffortTasty = true
+
     /** A fresh clone of this context embedded in this context. */
     def fresh: FreshContext = freshOver(this)
 
@@ -939,6 +951,9 @@ object Contexts {
     val sources: util.HashMap[AbstractFile, SourceFile] = util.HashMap[AbstractFile, SourceFile]()
     val files: util.HashMap[TermName, AbstractFile] = util.HashMap()
 
+    /** Was best effort file used during compilation? */
+    private[core] var usedBestEffortTasty = false
+
     // Types state
     /** A table for hash consing unique types */
     private[core] val uniques: Uniques = Uniques()

compiler/src/dotty/tools/dotc/core/DenotTransformers.scala

@@ -28,6 +28,8 @@ object DenotTransformers {
 
     /** The transformation method */
     def transform(ref: SingleDenotation)(using Context): SingleDenotation
+
+    override def isRunnable(using Context) = super.isRunnable && !ctx.usesBestEffortTasty 
   }
 
   /** A transformer that only transforms the info field of denotations */

compiler/src/dotty/tools/dotc/core/Denotations.scala

@@ -713,7 +713,8 @@ object Denotations {
         ctx.runId >= validFor.runId
         || ctx.settings.YtestPickler.value // mixing test pickler with debug printing can travel back in time
         || ctx.mode.is(Mode.Printing)  // no use to be picky when printing error messages
-        || symbol.isOneOf(ValidForeverFlags),
+        || symbol.isOneOf(ValidForeverFlags)
+        || ctx.isBestEffort, 
         s"denotation $this invalid in run ${ctx.runId}. ValidFor: $validFor")
       var d: SingleDenotation = this
       while ({

compiler/src/dotty/tools/dotc/core/SymDenotations.scala

@@ -711,12 +711,16 @@ object SymDenotations {
      *  TODO: Find a more robust way to characterize self symbols, maybe by
      *       spending a Flag on them?
      */
-    final def isSelfSym(using Context): Boolean = owner.infoOrCompleter match {
-      case ClassInfo(_, _, _, _, selfInfo) =>
-        selfInfo == symbol ||
-          selfInfo.isInstanceOf[Type] && name == nme.WILDCARD
-      case _ => false
-    }
+    final def isSelfSym(using Context): Boolean = 
+      if !ctx.isBestEffort || exists then
+        owner.infoOrCompleter match {
+          case ClassInfo(_, _, _, _, selfInfo) =>
+            selfInfo == symbol ||
+              selfInfo.isInstanceOf[Type] && name == nme.WILDCARD
+          case _ => false
+        }
+      else false
+
 
     /** Is this definition contained in `boundary`?
      *  Same as `ownersIterator contains boundary` but more efficient.
@@ -1986,7 +1990,7 @@ object SymDenotations {
         case p :: parents1 =>
           p.classSymbol match {
             case pcls: ClassSymbol => builder.addAll(pcls.baseClasses)
-            case _ => assert(isRefinementClass || p.isError || ctx.mode.is(Mode.Interactive), s"$this has non-class parent: $p")
+            case _ => assert(isRefinementClass || p.isError || ctx.mode.is(Mode.Interactive) || ctx.isBestEffort || ctx.usesBestEffortTasty, s"$this has non-class parent: $p")
           }
           traverse(parents1)
         case nil =>

compiler/src/dotty/tools/dotc/core/SymbolLoaders.scala

@@ -14,6 +14,7 @@ import Contexts._, Symbols._, Flags._, SymDenotations._, Types._, Scopes._, Name
 import NameOps._
 import StdNames._
 import classfile.ClassfileParser
+import classfile.BestEffortTastyParser
 import Decorators._
 
 import util.Stats
@@ -256,7 +257,8 @@ object SymbolLoaders {
       (idx + str.TOPLEVEL_SUFFIX.length + 1 != name.length || !name.endsWith(str.TOPLEVEL_SUFFIX))
     }
 
-    def maybeModuleClass(classRep: ClassRepresentation): Boolean = classRep.name.last == '$'
+    def maybeModuleClass(classRep: ClassRepresentation): Boolean = 
+      classRep.name.nonEmpty && classRep.name.last == '$'
 
     private def enterClasses(root: SymDenotation, packageName: String, flat: Boolean)(using Context) = {
       def isAbsent(classRep: ClassRepresentation) =
@@ -408,9 +410,14 @@ class ClassfileLoader(val classfile: AbstractFile) extends SymbolLoader {
 
   def load(root: SymDenotation)(using Context): Unit = {
     val (classRoot, moduleRoot) = rootDenots(root.asClass)
-    val classfileParser = new ClassfileParser(classfile, classRoot, moduleRoot)(ctx)
-    val result = classfileParser.run()
-    if (mayLoadTreesFromTasty)
+    val isBestEffortTasty = classfile.name.endsWith(".betasty")
+    val result =
+      if isBestEffortTasty then
+        new BestEffortTastyParser(classfile, classRoot, moduleRoot)(ctx).run()
+      else
+        new ClassfileParser(classfile, classRoot, moduleRoot)(ctx).run()
+
+    if (mayLoadTreesFromTasty || (isBestEffortTasty && ctx.withBestEffortTasty))
       result match {
         case Some(unpickler: tasty.DottyUnpickler) =>
           classRoot.classSymbol.rootTreeOrProvider = unpickler

compiler/src/dotty/tools/dotc/core/TypeErasure.scala

@@ -709,12 +709,13 @@ class TypeErasure(sourceLanguage: SourceLanguage, semiEraseVCs: Boolean, isConst
   private def checkedSuperType(tp: TypeProxy)(using Context): Type =
     val tp1 = tp.translucentSuperType
     if !tp1.exists then
-      val msg = tp.typeConstructor match
-        case tycon: TypeRef =>
-          MissingType(tycon.prefix, tycon.name).toMessage.message
-        case _ =>
-          i"Cannot resolve reference to $tp"
-      throw FatalError(msg)
+        val msg = tp.typeConstructor match
+          case tycon: TypeRef =>
+            MissingType(tycon.prefix, tycon.name).toMessage.message
+          case _ =>
+            i"Cannot resolve reference to $tp"
+        if ctx.isBestEffort then report.error(msg)
+        else throw FatalError(msg)
     tp1
 
   /** Widen term ref, skipping any `()` parameter of an eventual getter. Used to erase a TermRef.

compiler/src/dotty/tools/dotc/core/Types.scala

@@ -699,6 +699,7 @@ object Types {
      *  the type of the member as seen from given prefix `pre`.
      */
     final def findMember(name: Name, pre: Type, required: FlagSet = EmptyFlags, excluded: FlagSet = EmptyFlags)(using Context): Denotation = {
+      // println("findMember " + name + " " + pre + " " + required)
       @tailrec def go(tp: Type): Denotation = tp match {
         case tp: TermRef =>
           go (tp.underlying match {
@@ -3511,8 +3512,8 @@ object Types {
 
     def apply(tp1: Type, tp2: Type, soft: Boolean)(using Context): OrType = {
       def where = i"in union $tp1 | $tp2"
-      expectValueTypeOrWildcard(tp1, where)
-      expectValueTypeOrWildcard(tp2, where)
+      if (!ctx.usesBestEffortTasty) expectValueTypeOrWildcard(tp1, where)
+      if (!ctx.usesBestEffortTasty) expectValueTypeOrWildcard(tp2, where)
       assertUnerased()
       unique(new CachedOrType(tp1, tp2, soft))
     }

compiler/src/dotty/tools/dotc/core/classfile/BestEffortTastyParser.scala

@@ -0,0 +1,46 @@
+package dotty.tools.dotc.core.classfile
+
+import java.io.ByteArrayOutputStream
+import dotty.tools.io
+import dotty.tools.io._
+import dotty.tools.dotc.core.Contexts.Context
+import dotty.tools.dotc.core.tasty.DottyUnpickler
+import dotty.tools.dotc.core.classfile.ClassfileParser.Embedded
+import dotty.tools.dotc.core.SymDenotations.ClassDenotation
+
+class BestEffortTastyParser(
+  bestEffortTastyFile: AbstractFile,
+  classRoot: ClassDenotation,
+  moduleRoot: ClassDenotation
+)(ictx: Context):
+
+  private def readFile(file: AbstractFile): Array[Byte] =
+    file match 
+      case zipEntry: io.ZipArchive#Entry => // We are in a jar
+        val stream = file.input
+        try
+          val tastyOutStream = new ByteArrayOutputStream()
+          val buffer = new Array[Byte](1024)
+          var read = stream.read(buffer, 0, buffer.length)
+          while read != -1 do 
+            tastyOutStream.write(buffer, 0, read)
+            read = stream.read(buffer, 0, buffer.length)
+          
+          tastyOutStream.flush()
+          tastyOutStream.toByteArray.nn
+        finally
+          stream.close()
+      case _ =>
+        file.toByteArray
+
+  private def unpickleTASTY(bytes: Array[Byte])(using ctx: Context): Some[Embedded] = {
+    ctx.setUsesBestEffortTasty()
+    val unpickler = new DottyUnpickler(bytes, withBestEffortTasty = true)
+    unpickler.enter(roots = Set(classRoot, moduleRoot, moduleRoot.sourceModule))(using ctx.withSource(dotty.tools.dotc.util.NoSource))
+    Some(unpickler)
+  }
+
+  def run()(using Context): Option[Embedded] =
+    val bytes = readFile(bestEffortTastyFile)
+    if bytes.nonEmpty then unpickleTASTY(bytes)
+    else None

compiler/src/dotty/tools/dotc/core/tasty/BestEffortTastyWriter.scala

@@ -0,0 +1,43 @@
+package dotty.tools.dotc
+package core
+package tasty
+
+import scala.language.unsafeNulls
+import java.nio.file.{Path as JPath, Files as JFiles}
+import java.nio.channels.ClosedByInterruptException
+import java.io.DataOutputStream
+import dotty.tools.io.{File, PlainFile}
+import dotty.tools.dotc.core.Contexts.Context
+
+object BestEffortTastyWriter:
+
+  def write(dir: JPath, units: List[CompilationUnit])(using Context): Unit = 
+    if JFiles.exists(dir) then JFiles.createDirectories(dir)
+
+    units.foreach { unit =>
+      unit.pickled.foreach { (clz, binary) =>
+        val parts = clz.fullName.mangledString.split('.')
+        val outPath = outputPath(parts.toList, dir)
+        val outTastyFile = new PlainFile(new File(outPath))
+        val outstream = new DataOutputStream(outTastyFile.bufferedOutput)
+        try outstream.write(binary())
+        catch case ex: ClosedByInterruptException =>
+          try
+            outTastyFile.delete() // don't leave an empty or half-written tastyfile around after an interrupt
+          catch
+            case _: Throwable =>
+          throw ex
+        finally outstream.close()
+      }
+    }
+
+  def outputPath(parts: List[String], acc: JPath): JPath =
+    parts match
+      case Nil => throw new Exception("Invalid class name")
+      case last :: Nil =>
+        val name = last.stripSuffix("$")
+        acc.resolve(s"$name.betasty")
+      case pkg :: tail => 
+        val next = acc.resolve(pkg)
+        if !JFiles.exists(next) then JFiles.createDirectory(next)
+        outputPath(tail, next)