detect missing class file for classpath dependency in macro

Previously, it was assumed that a missing class (that is valid in current run)
during macro evaluation was due to the symbol being defined in the same project.
If this condition is met, then compilation is suspended.

This assumption breaks when the symbol comes from the classpath, but without a
corresponding class file, leading a situation where the same file is always suspended,
until it is the only one left, leading to the "cyclic macro dependencies" error.
In this case we should assume that the class file will never become available because class path entries
are supposed to be immutable. Therefore we should not suspend in this case.

This commit therefore detects this situation. Instead of suspending the unit,
the compiler aborts the macro expansion, reporting an error that
the user will have to deal with - likely by changing the build definition

Author: bishabosha

compiler/src/dotty/tools/dotc/core/Symbols.scala

@@ -165,6 +165,10 @@ object Symbols extends SymUtils {
     final def isDefinedInSource(using Context): Boolean =
       span.exists && isValidInCurrentRun && associatedFileMatches(!_.isScalaBinary)
 
+    /** Is this symbol valid in the current run, but comes from the classpath? */
+    final def isDefinedInBinary(using Context): Boolean =
+      isValidInCurrentRun && associatedFileMatches(_.isScalaBinary)
+
     /** Is symbol valid in current run? */
     final def isValidInCurrentRun(using Context): Boolean =
       (lastDenot.validFor.runId == ctx.runId || stillValid(lastDenot)) &&

compiler/src/dotty/tools/dotc/quoted/Interpreter.scala

@@ -166,8 +166,8 @@ class Interpreter(pos: SrcPos, classLoader0: ClassLoader)(using Context):
     val inst =
       try loadModule(moduleClass)
       catch
-        case MissingClassDefinedInCurrentRun(sym) =>
-          suspendOnMissing(sym, pos)
+        case MissingClassValidInCurrentRun(sym, origin) =>
+          suspendOnMissing(sym, origin, pos)
     val clazz = inst.getClass
     val name = fn.name.asTermName
     val method = getMethod(clazz, name, paramsSig(fn))
@@ -213,8 +213,8 @@ class Interpreter(pos: SrcPos, classLoader0: ClassLoader)(using Context):
   private def loadClass(name: String): Class[?] =
     try classLoader.loadClass(name)
     catch
-      case MissingClassDefinedInCurrentRun(sym) =>
-        suspendOnMissing(sym, pos)
+      case MissingClassValidInCurrentRun(sym, origin) =>
+        suspendOnMissing(sym, origin, pos)
 
 
   private def getMethod(clazz: Class[?], name: Name, paramClasses: List[Class[?]]): JLRMethod =
@@ -223,8 +223,8 @@ class Interpreter(pos: SrcPos, classLoader0: ClassLoader)(using Context):
       case _: NoSuchMethodException =>
         val msg = em"Could not find method ${clazz.getCanonicalName}.$name with parameters ($paramClasses%, %)"
         throw new StopInterpretation(msg, pos)
-      case MissingClassDefinedInCurrentRun(sym) =>
-        suspendOnMissing(sym, pos)
+      case MissingClassValidInCurrentRun(sym, origin) =>
+        suspendOnMissing(sym, origin, pos)
     }
 
   private def stopIfRuntimeException[T](thunk: => T, method: JLRMethod): T =
@@ -242,8 +242,8 @@ class Interpreter(pos: SrcPos, classLoader0: ClassLoader)(using Context):
         ex.getTargetException match {
           case ex: scala.quoted.runtime.StopMacroExpansion =>
             throw ex
-          case MissingClassDefinedInCurrentRun(sym) =>
-            suspendOnMissing(sym, pos)
+          case MissingClassValidInCurrentRun(sym, origin) =>
+            suspendOnMissing(sym, origin, pos)
           case targetException =>
             val sw = new StringWriter()
             sw.write("Exception occurred while executing macro expansion.\n")
@@ -348,8 +348,11 @@ object Interpreter:
     }
   end Call
 
-  object MissingClassDefinedInCurrentRun {
-    def unapply(targetException: Throwable)(using Context): Option[Symbol] = {
+  enum ClassOrigin:
+    case Classpath, Source
+
+  object MissingClassValidInCurrentRun {
+    def unapply(targetException: Throwable)(using Context): Option[(Symbol, ClassOrigin)] = {
       if !ctx.compilationUnit.isSuspendable then None
       else targetException match
         case _: NoClassDefFoundError | _: ClassNotFoundException =>
@@ -358,17 +361,34 @@ object Interpreter:
           else
             val className = message.replace('/', '.')
             val sym =
-              if className.endsWith(str.MODULE_SUFFIX) then staticRef(className.toTermName).symbol.moduleClass
-              else staticRef(className.toTypeName).symbol
-            // If the symbol does not a a position we assume that it came from the current run and it has an error
-            if sym.isDefinedInCurrentRun || (sym.exists && !sym.srcPos.span.exists) then Some(sym)
-            else None
+              if className.endsWith(str.MODULE_SUFFIX) then
+                staticRef(className.stripSuffix(str.MODULE_SUFFIX).toTermName).symbol.moduleClass
+              else
+                staticRef(className.toTypeName).symbol
+            if sym.isDefinedInBinary then
+              // i.e. the associated file is `.tasty`, if the macro classloader is not able to find the class,
+              // possibly it indicates that it comes from a pipeline-compiled dependency.
+              Some((sym, ClassOrigin.Classpath))
+            else if sym.isDefinedInCurrentRun || (sym.exists && !sym.srcPos.span.exists) then
+              // If the symbol does not a a position we assume that it came from the current run and it has an error
+              Some((sym, ClassOrigin.Source))
+            else
+              None
         case _ => None
     }
   }
 
-  def suspendOnMissing(sym: Symbol, pos: SrcPos)(using Context): Nothing =
-    if ctx.settings.YnoSuspendedUnits.value then
-      throw StopInterpretation(em"suspension triggered by a dependency on missing $sym not allowed with -Yno-suspended-units", pos)
+  def suspendOnMissing(sym: Symbol, origin: ClassOrigin, pos: SrcPos)(using Context): Nothing =
+    if origin == ClassOrigin.Classpath then
+      throw StopInterpretation(
+          em"""Macro code depends on ${sym.showLocated} found on the classpath, but could not be loaded while evaluating the macro.
+              |  This is likely because class files could not be found in the classpath entry for the symbol.
+              |
+              |  A possible cause is if the origin of this symbol was built with pipelined compilation;
+              |  in which case, this problem may go away by disabling pipelining for that origin.
+              |
+              |  $sym is defined in file ${sym.associatedFile}""", pos)
+    else if ctx.settings.YnoSuspendedUnits.value then
+      throw StopInterpretation(em"suspension triggered by a dependency on missing ${sym.showLocated} not allowed with -Yno-suspended-units", pos)
     else
-      ctx.compilationUnit.suspend(i"suspension triggered by a dependency on missing $sym") // this throws a SuspendException
+      ctx.compilationUnit.suspend(i"suspension triggered by a dependency on missing ${sym.showLocated}") // this throws a SuspendException

compiler/src/dotty/tools/dotc/transform/MacroAnnotations.scala

@@ -107,8 +107,8 @@ class MacroAnnotations(phase: IdentityDenotTransformer):
             if !ctx.reporter.hasErrors then
               report.error("Macro expansion was aborted by the macro without any errors reported. Macros should issue errors to end-users when aborting a macro expansion with StopMacroExpansion.", annot.tree)
             List(tree)
-          case Interpreter.MissingClassDefinedInCurrentRun(sym) =>
-            Interpreter.suspendOnMissing(sym, annot.tree)
+          case Interpreter.MissingClassValidInCurrentRun(sym, origin) =>
+            Interpreter.suspendOnMissing(sym, origin, annot.tree)
           case NonFatal(ex) =>
             val stack0 = ex.getStackTrace.takeWhile(_.getClassName != "dotty.tools.dotc.transform.MacroAnnotations")
             val stack = stack0.take(1 + stack0.lastIndexWhere(_.getMethodName == "transform"))

sbt-test/pipelining/pipelining-scala-macro-splice-ok/build.sbt

@@ -0,0 +1,9 @@
+ThisBuild / usePipelining := true
+
+// m defines a macro depending on b.B, it also tries to use the macro in the same project,
+// which will succeed even though B.class is not available when running the macro,
+// because compilation can suspend until B is available.
+lazy val m = project.in(file("m"))
+  .settings(
+    scalacOptions += "-Ycheck:all",
+  )

sbt-test/pipelining/pipelining-scala-macro-splice-ok/m/src/main/scala/a/A.scala

@@ -0,0 +1,3 @@
+package a
+
+class A(val i: Int)

sbt-test/pipelining/pipelining-scala-macro-splice-ok/m/src/main/scala/b/B.scala

@@ -0,0 +1,26 @@
+package b
+
+import a.A
+import scala.quoted.*
+
+object B {
+
+  transparent inline def transparentPower(x: Double, inline n: Int): Double =
+    ${ powerCode('x, 'n)  }
+
+  def powerCode(x: Expr[Double], n: Expr[Int])(using Quotes): Expr[Double] = {
+    // this macro will cause a suspension in compilation of C.scala, because it calls
+    // transparentPower. This will try to invoke the macro but fail because A.class
+    // is not yet available until the run for A.scala completes.
+
+    // see sbt-test/pipelining/pipelining-scala-macro-splice/m/src/main/scala/b/B.scala
+    // for a corresponding implementation that uses a class from an upstream project
+    // instead, and fails because pipelining is turned on for the upstream project.
+    def impl(x: Double, n: A): Double =
+      if (n.i == 0) 1.0
+      else if (n.i % 2 == 1) x * impl(x, A(n.i - 1))
+      else impl(x * x, A(n.i / 2))
+
+    Expr(impl(x.valueOrError, A(n.valueOrError)))
+  }
+}

sbt-test/pipelining/pipelining-scala-macro-splice-ok/m/src/main/scala/c/C.scala

@@ -0,0 +1,11 @@
+package c
+
+import b.B
+
+object C {
+  @main def run = {
+    assert(B.transparentPower(2.0, 2) == 4.0)
+    assert(B.transparentPower(2.0, 3) == 8.0)
+    assert(B.transparentPower(2.0, 4) == 16.0)
+  }
+}

sbt-test/pipelining/pipelining-scala-macro-splice-ok/project/DottyInjectedPlugin.scala

@@ -0,0 +1,11 @@
+import sbt._
+import Keys._
+
+object DottyInjectedPlugin extends AutoPlugin {
+  override def requires = plugins.JvmPlugin
+  override def trigger = allRequirements
+
+  override val projectSettings = Seq(
+    scalaVersion := sys.props("plugin.scalaVersion"),
+  )
+}

sbt-test/pipelining/pipelining-scala-macro-splice-ok/test

@@ -0,0 +1,3 @@
+# shows that it is ok to depend on a class, defined in the same project,
+# in a macro implementation. Compilation will suspend at typer.
+> m/run

sbt-test/pipelining/pipelining-scala-macro-splice/a/src/main/scala/a/A.scala

@@ -0,0 +1,3 @@
+package a
+
+class A(val i: Int)

sbt-test/pipelining/pipelining-scala-macro-splice/a_alt/.keep

@@ -0,0 +1,32 @@
+ThisBuild / usePipelining := true
+
+lazy val a = project.in(file("a"))
+  .settings(
+    scalacOptions += "-Ycheck:all",
+  )
+
+// same as a, but does not use pipelining
+lazy val a_alt = project.in(file("a_alt"))
+  .settings(
+    Compile / sources := (a / Compile / sources).value,
+    Compile / exportPipelining := false,
+  )
+
+
+// m defines a macro depending on a, it also tries to use the macro in the same project,
+// which will fail because A.class is not available when running the macro,
+// because the dependency on a is pipelined.
+lazy val m = project.in(file("m"))
+  .dependsOn(a)
+  .settings(
+    scalacOptions += "-Ycheck:all",
+  )
+
+// same as m, but depends on a_alt, so it will compile
+// because A.class will be available when running the macro.
+lazy val m_alt = project.in(file("m_alt"))
+  .dependsOn(a_alt)
+  .settings(
+    Compile / sources := (m / Compile / sources).value,
+    scalacOptions += "-Ycheck:all",
+  )

sbt-test/pipelining/pipelining-scala-macro-splice/build.sbt

@@ -0,0 +1,26 @@
+package b
+
+import a.A
+import scala.quoted.*
+
+object B {
+
+  transparent inline def transparentPower(x: Double, inline n: Int): Double =
+    ${ powerCode('x, 'n)  }
+
+  def powerCode(x: Expr[Double], n: Expr[Int])(using Quotes): Expr[Double] = {
+    // this macro is invoked during compilation of C.scala. When project a is pipelined
+    // This will fail because A.class will never be available, because the classpath entry
+    // is the early-output jar. The compiler detects this and aborts macro expansion with an error.
+
+    // see sbt-test/pipelining/pipelining-scala-macro-splice-ok/m/src/main/scala/b/B.scala
+    // for a corresponding implementation that uses a class from the same project
+    // instead, but succeeds because it can suspend compilation until classes become available.
+    def impl(x: Double, n: A): Double =
+      if (n.i == 0) 1.0
+      else if (n.i % 2 == 1) x * impl(x, A(n.i - 1))
+      else impl(x * x, A(n.i / 2))
+
+    Expr(impl(x.valueOrError, A(n.valueOrError)))
+  }
+}

sbt-test/pipelining/pipelining-scala-macro-splice/m/src/main/scala/b/B.scala

@@ -0,0 +1,11 @@
+package c
+
+import b.B
+
+object C {
+  @main def run = {
+    assert(B.transparentPower(2.0, 2) == 4.0)
+    assert(B.transparentPower(2.0, 3) == 8.0)
+    assert(B.transparentPower(2.0, 4) == 16.0)
+  }
+}

sbt-test/pipelining/pipelining-scala-macro-splice/m/src/main/scala/c/C.scala

@@ -0,0 +1,11 @@
+import sbt._
+import Keys._
+
+object DottyInjectedPlugin extends AutoPlugin {
+  override def requires = plugins.JvmPlugin
+  override def trigger = allRequirements
+
+  override val projectSettings = Seq(
+    scalaVersion := sys.props("plugin.scalaVersion"),
+  )
+}

sbt-test/pipelining/pipelining-scala-macro-splice/m_alt/.keep

@@ -0,0 +1,10 @@
+# as described in build.sbt, this will fail to compile.
+# m defines a macro, depending on a.A, defined in upstream project a
+# however because m also tries to run the macro in the same project,
+# a/A.class is not available yet, so a reflection error will occur.
+# This is caught by the compiler and presents a pretty diagnostic to the user,
+# suggesting to disable pipelining in the project defining A.
+-> m/compile
+# This will run, simulating a user following the suggestion to
+# disable pipelining in project a.
+> m_alt/run