Merge pull request #9984 from dotty-staging/move-blackbox-macro-exapnsion-after-typer

Expand non-transparent macros after Typer

Author: nicolasstucki

.gitignore

@@ -89,5 +89,9 @@ community-build/sbt-scalajs-sbt
 *.check.out
 !/dist/bin/
 
+# semanticdb test output files
+*.expect.scala.out
+*.expect.out
+
 # Bloop
 .bsp

bench-run/src/main/scala/dotty/tools/benchmarks/tuples/TupleOps.scala

@@ -50,7 +50,7 @@ class TupleOps {
   def tupleMergeSort(tuple: Tuple): Tuple =
     if (tuple.size <= 1) tuple
     else {
-      val (tuple1, tuple2) = tuple.splitAt(tuple.size / 2)
+      val (tuple1, tuple2) = tuple.splitAt(tuple.size / 2): (Tuple, Tuple)// TODO remove ascriptions (issue with type variable constraints)
       val (sorted1, sorted2) = (tupleMergeSort(tuple1), tupleMergeSort(tuple2))
       tupleMerge(sorted1, sorted2)
     }

community-build/community-projects/izumi-reflect

@@ -38,15 +38,26 @@ class CompilationUnit protected (val source: SourceFile) {
    */
   val freshNames: FreshNameCreator = new FreshNameCreator.Default
 
+  /** Will be set to `true` if there are inline call that must be inlined after typer.
+   *  The information is used in phase `Inlining` in order to avoid traversing trees that need no transformations.
+   */
+  var needsInlining: Boolean = false
+
   /** Will be set to `true` if contains `Quote`.
    *  The information is used in phase `Staging` in order to avoid traversing trees that need no transformations.
    */
   var needsStaging: Boolean = false
 
+  /** Will be set to `true` if contains `Quote` that needs to be pickled
+   *  The information is used in phase `PickleQuotes` in order to avoid traversing trees that need no transformations.
+   */
+  var needsQuotePickling: Boolean = false
+
   /** A structure containing a temporary map for generating inline accessors */
   val inlineAccessors: InlineAccessors = new InlineAccessors
 
   var suspended: Boolean = false
+  var suspendedAtInliningPhase: Boolean = false
 
   /** Can this compilation unit be suspended */
   def isSuspendable: Boolean = true
@@ -61,6 +72,8 @@ class CompilationUnit protected (val source: SourceFile) {
         report.echo(i"suspended: $this")
       suspended = true
       ctx.run.suspendedUnits += this
+      if ctx.phase == Phases.inliningPhase then
+        suspendedAtInliningPhase = true
     throw CompilationUnit.SuspendException()
 
   private var myAssignmentSpans: Map[Int, List[Span]] = null
@@ -90,7 +103,9 @@ object CompilationUnit {
     if (forceTrees) {
       val force = new Force
       force.traverse(unit1.tpdTree)
-      unit1.needsStaging = force.needsStaging
+      unit1.needsStaging = force.containsQuote
+      unit1.needsQuotePickling = force.containsQuote
+      unit1.needsInlining = force.containsInline
     }
     unit1
   }
@@ -116,10 +131,13 @@ object CompilationUnit {
 
   /** Force the tree to be loaded */
   private class Force extends TreeTraverser {
-    var needsStaging = false
+    var containsQuote = false
+    var containsInline = false
     def traverse(tree: Tree)(using Context): Unit = {
       if (tree.symbol.isQuote)
-        needsStaging = true
+        containsQuote = true
+      if tree.symbol.is(Flags.Inline) then
+        containsInline = true
       traverseChildren(tree)
     }
   }

community-build/community-projects/scalatest

@@ -50,6 +50,7 @@ class Compiler {
   /** Phases dealing with TASTY tree pickling and unpickling */
   protected def picklerPhases: List[List[Phase]] =
     List(new Pickler) ::            // Generate TASTY info
+    List(new Inlining) ::           // Inline and execute macros
     List(new PickleQuotes) ::       // Turn quoted trees into explicit run-time data structures
     Nil
 
@@ -61,6 +62,7 @@ class Compiler {
          new CookComments,           // Cook the comments: expand variables, doc, etc.
          new CheckStatic,            // Check restrictions that apply to @static members
          new BetaReduce,             // Reduce closure applications
+         new InlineVals,             // Check right hand-sides of an `inline val`s
          new ExpandSAMs,             // Expand single abstract method closures to anonymous classes
          new init.Checker) ::        // Check initialization of objects
     List(new ElimRepeated,           // Rewrite vararg parameters and arguments

community-build/community-projects/shapeless

@@ -101,11 +101,27 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint
    */
   def units: List[CompilationUnit] = myUnits
 
-  var suspendedUnits: mutable.ListBuffer[CompilationUnit] = mutable.ListBuffer()
-
   private def units_=(us: List[CompilationUnit]): Unit =
     myUnits = us
 
+  var suspendedUnits: mutable.ListBuffer[CompilationUnit] = mutable.ListBuffer()
+
+  def checkSuspendedUnits(newUnits: List[CompilationUnit])(using Context): Unit =
+    if newUnits.isEmpty && suspendedUnits.nonEmpty && !ctx.reporter.errorsReported then
+      val where =
+        if suspendedUnits.size == 1 then i"in ${suspendedUnits.head}."
+        else i"""among
+                |
+                |  ${suspendedUnits.toList}%, %
+                |"""
+      val enableXprintSuspensionHint =
+        if ctx.settings.XprintSuspension.value then ""
+        else "\n\nCompiling with  -Xprint-suspension   gives more information."
+      report.error(em"""Cyclic macro dependencies $where
+                    |Compilation stopped since no further progress can be made.
+                    |
+                    |To fix this, place macros in one set of files and their callers in another.$enableXprintSuspensionHint""")
+
   /** The files currently being compiled (active or suspended).
    *  This may return different results over time.
    *  These files do not have to be source files since it's possible to compile

community-build/community-projects/utest

@@ -227,6 +227,8 @@ class ScalaSettings extends Settings.SettingGroup with CommonScalaSettings {
   val Yinstrument: Setting[Boolean] = BooleanSetting("-Yinstrument", "Add instrumentation code that counts allocations and closure creations.")
   val YinstrumentDefs: Setting[Boolean] = BooleanSetting("-Yinstrument-defs", "Add instrumentation code that counts method calls; needs -Yinstrument to be set, too.")
 
+  val YforceInlineWhileTyping: Setting[Boolean] = BooleanSetting("-Yforce-inline-while-typing", "Make non-transparent inline methods inline when typing. Emulates the old inlining behavior of 3.0.0-M3.")
+
   /** Dottydoc specific settings that are not used in scaladoc */
   val docSnapshot: Setting[Boolean] = BooleanSetting("-doc-snapshot", "Generate a documentation snapshot for the current Dotty version")
 

compiler/src/dotty/tools/dotc/CompilationUnit.scala

@@ -122,4 +122,9 @@ object Mode {
 
   /** Are we enforcing null safety */
   val SafeNulls = newMode(28, "SafeNulls")
+
+  /** We are typing the body of the condition of an `inline if` or the scrutinee of an `inline match`
+   *  This mode forces expansion of inline calls in those positions even during typing.
+   */
+  val ForceInline: Mode = newMode(29, "ForceInline")
 }

compiler/src/dotty/tools/dotc/Compiler.scala

@@ -198,6 +198,7 @@ object Phases {
     private var myPostTyperPhase: Phase = _
     private var mySbtExtractDependenciesPhase: Phase = _
     private var myPicklerPhase: Phase = _
+    private var myInliningPhase: Phase = _
     private var myPickleQuotesPhase: Phase = _
     private var myFirstTransformPhase: Phase = _
     private var myCollectNullableFieldsPhase: Phase = _
@@ -218,6 +219,7 @@ object Phases {
     final def postTyperPhase: Phase = myPostTyperPhase
     final def sbtExtractDependenciesPhase: Phase = mySbtExtractDependenciesPhase
     final def picklerPhase: Phase = myPicklerPhase
+    final def inliningPhase: Phase = myInliningPhase
     final def pickleQuotesPhase: Phase = myPickleQuotesPhase
     final def firstTransformPhase: Phase = myFirstTransformPhase
     final def collectNullableFieldsPhase: Phase = myCollectNullableFieldsPhase
@@ -241,6 +243,7 @@ object Phases {
       myPostTyperPhase = phaseOfClass(classOf[PostTyper])
       mySbtExtractDependenciesPhase = phaseOfClass(classOf[sbt.ExtractDependencies])
       myPicklerPhase = phaseOfClass(classOf[Pickler])
+      myInliningPhase = phaseOfClass(classOf[Inlining])
       myPickleQuotesPhase = phaseOfClass(classOf[PickleQuotes])
       myFirstTransformPhase = phaseOfClass(classOf[FirstTransform])
       myCollectNullableFieldsPhase = phaseOfClass(classOf[CollectNullableFields])
@@ -406,6 +409,7 @@ object Phases {
   def postTyperPhase(using Context): Phase              = ctx.base.postTyperPhase
   def sbtExtractDependenciesPhase(using Context): Phase = ctx.base.sbtExtractDependenciesPhase
   def picklerPhase(using Context): Phase                = ctx.base.picklerPhase
+  def inliningPhase(using Context): Phase               = ctx.base.inliningPhase
   def pickleQuotesPhase(using Context): Phase           = ctx.base.pickleQuotesPhase
   def firstTransformPhase(using Context): Phase         = ctx.base.firstTransformPhase
   def refchecksPhase(using Context): Phase              = ctx.base.refchecksPhase

compiler/src/dotty/tools/dotc/Run.scala

@@ -77,10 +77,11 @@ class ExtractAPI extends Phase {
       } finally pw.close()
     }
 
-    if (ctx.sbtCallback != null) {
+    if ctx.sbtCallback != null &&
+      !ctx.compilationUnit.suspendedAtInliningPhase // already registered before this unit was suspended
+    then
       classes.foreach(ctx.sbtCallback.api(sourceFile.file, _))
       mainClasses.foreach(ctx.sbtCallback.mainClass(sourceFile.file, _))
-    }
   }
 }
 

compiler/src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -0,0 +1,43 @@
+package dotty.tools
+package dotc
+package transform
+
+import dotty.tools.dotc.core.Contexts._
+import dotty.tools.dotc.core.Decorators._
+import dotty.tools.dotc.core.Flags._
+import dotty.tools.dotc.core.Types._
+import dotty.tools.dotc.transform.MegaPhase.MiniPhase
+import dotty.tools.dotc.typer.Inliner
+
+/** Check that `tree.rhs` can be right hand-side of an `inline` value definition. */
+class InlineVals extends MiniPhase:
+  import ast.tpd._
+
+  def phaseName: String = "inlineVals"
+
+  override def checkPostCondition(tree: Tree)(using Context): Unit =
+    if !ctx.erasedTypes then
+      tree match
+        case tree: ValDef => checkInlineConformant(tree)
+        case _ =>
+
+  override def transformValDef(tree: ValDef)(using Context): Tree =
+    checkInlineConformant(tree)
+    tree
+
+  /** Check that `tree.rhs` can be right hand-side of an `inline` value definition. */
+  private def checkInlineConformant(tree: ValDef)(using Context): Unit = {
+    if tree.symbol.is(Inline, butNot = DeferredOrTermParamOrAccessor)
+      && !Inliner.inInlineMethod
+    then
+      val rhs = tree.rhs
+      val tpt = tree.tpt
+      tpt.tpe.widenTermRefExpr.dealias.normalized match
+        case tp: ConstantType =>
+          if !isPureExpr(rhs) then
+            val details = if enclosingInlineds.isEmpty then "" else em"but was: $rhs"
+            report.error(s"inline value must be pure$details", rhs.srcPos)
+        case _ =>
+          val pos = if tpt.span.isZeroExtent then rhs.srcPos else tpt.srcPos
+          report.error(em"inline value must have a literal constant type", pos)
+  }

compiler/src/dotty/tools/dotc/core/Mode.scala

@@ -33,11 +33,22 @@ import scala.annotation.constructorOnly
 /** Inlines all calls to inline methods that are not in an inline method or a quote */
 class Inlining extends MacroTransform {
   import tpd._
+  import Inlining._
 
   override def phaseName: String = Inlining.name
 
   override def allowsImplicitSearch: Boolean = true
 
+  override def run(using Context): Unit =
+    if ctx.compilationUnit.needsInlining then
+      try super.run
+      catch case _: CompilationUnit.SuspendException => ()
+
+  override def runOn(units: List[CompilationUnit])(using Context): List[CompilationUnit] =
+    val newUnits = super.runOn(units).filterNot(_.suspended)
+    ctx.run.checkSuspendedUnits(newUnits)
+    newUnits
+
   override def checkPostCondition(tree: Tree)(using Context): Unit =
     tree match {
       case PackageDef(pid, _) if tree.symbol.owner == defn.RootClass =>
@@ -69,13 +80,13 @@ class Inlining extends MacroTransform {
           else super.transform(tree)
         case _: Typed | _: Block =>
           super.transform(tree)
-        case _ if Inliner.isInlineable(tree) && !tree.tpe.widen.isInstanceOf[MethodOrPoly] && StagingContext.level == 0 =>
+        case _ if Inliner.needsInlining(tree) =>
           val tree1 = super.transform(tree)
-          val inlined = Inliner.inlineCall(tree1)
-          if tree1 eq inlined then inlined
-          else transform(inlined)
+          if tree1.tpe.isError then tree1
+          else Inliner.inlineCall(tree1)
         case _: GenericApply if tree.symbol.isQuote =>
-          ctx.compilationUnit.needsStaging = true
+          if level == 0 then
+            ctx.compilationUnit.needsQuotePickling = true
           super.transform(tree)(using StagingContext.quoteContext)
         case _: GenericApply if tree.symbol.isExprSplice =>
           super.transform(tree)(using StagingContext.spliceContext)
@@ -85,6 +96,5 @@ class Inlining extends MacroTransform {
   }
 }
 
-object Inlining {
+object Inlining:
   val name: String = "inlining"
-}

compiler/src/dotty/tools/dotc/core/Phases.scala

@@ -88,7 +88,7 @@ class PickleQuotes extends MacroTransform {
     }
 
   override def run(using Context): Unit =
-    if (ctx.compilationUnit.needsStaging) super.run(using freshStagingContext)
+    if (ctx.compilationUnit.needsQuotePickling) super.run(using freshStagingContext)
 
   protected def newTransformer(using Context): Transformer = new Transformer {
     override def transform(tree: tpd.Tree)(using Context): tpd.Tree =

compiler/src/dotty/tools/dotc/sbt/ExtractAPI.scala

@@ -258,12 +258,16 @@ class PostTyper extends MacroTransform with IdentityDenotTransformer { thisPhase
     override def transform(tree: Tree)(using Context): Tree =
       try tree match {
         case tree: Ident if !tree.isType =>
+          if tree.symbol.is(Inline) && !Inliner.inInlineMethod then
+            ctx.compilationUnit.needsInlining = true
           checkNoConstructorProxy(tree)
           tree.tpe match {
             case tpe: ThisType => This(tpe.cls).withSpan(tree.span)
             case _ => tree
           }
         case tree @ Select(qual, name) =>
+          if tree.symbol.is(Inline) then
+            ctx.compilationUnit.needsInlining = true
           if (name.isTypeName) {
             Checking.checkRealizable(qual.tpe, qual.srcPos)
             withMode(Mode.Type)(super.transform(tree))
@@ -302,6 +306,9 @@ class PostTyper extends MacroTransform with IdentityDenotTransformer { thisPhase
         case tree: TypeApply =>
           if tree.symbol.isQuote then
             ctx.compilationUnit.needsStaging = true
+            ctx.compilationUnit.needsQuotePickling = true
+          if tree.symbol.is(Inline) then
+            ctx.compilationUnit.needsInlining = true
           val tree1 @ TypeApply(fn, args) = normalizeTypeArgs(tree)
           args.foreach(checkInferredWellFormed)
           if (fn.symbol != defn.ChildAnnot.primaryConstructor)

compiler/src/dotty/tools/dotc/transform/InlineVals.scala

@@ -173,7 +173,7 @@ class TupleOptimizations extends MiniPhase with IdentityDenotTransformer {
         val size = tpes.size
         if (size == 0)
           // Array.emptyObjectArray
-          ref(defn.ArrayModule).select("emptyObjectArray".toTermName).ensureApplied
+          ref(defn.ArrayModule).select("emptyObjectArray".toTermName).ensureApplied.withSpan(tree.span)
         else if (size <= MaxTupleArity)
           // scala.runtime.Tuples.productToArray(tup.asInstanceOf[Product])
           ref(defn.RuntimeTuples_productToArray).appliedTo(tup.asInstance(defn.ProductClass.typeRef))

compiler/src/dotty/tools/dotc/transform/Inlining.scala

@@ -27,11 +27,12 @@ class YCheckPositions extends Phase {
 
             // Check current context is correct
             assert(ctx.source == sources.head)
-            if (!tree.isEmpty && !tree.isInstanceOf[untpd.TypedSplice] && ctx.typerState.isGlobalCommittable)
-              if (!tree.isType) { // TODO also check types, currently we do not add Inlined(EmptyTree, _, _) for types. We should.
+            if (!tree.isEmpty && !tree.isInstanceOf[untpd.TypedSplice] && !tree.isInstanceOf[Inlined] && ctx.typerState.isGlobalCommittable)
+              if !tree.isType // TODO also check types, currently we do not add Inlined(EmptyTree, _, _) for types. We should.
+                && !tree.symbol.is(InlineProxy) // TODO check inline proxies (see tests/tun/lst)
+              then
                 val currentSource = sources.head
-                assert(tree.source == currentSource, i"wrong source set for $tree # ${tree.uniqueId} of ${tree.getClass}, set to ${tree.source} but context had $currentSource")
-              }
+                assert(tree.source == currentSource, i"wrong source set for $tree # ${tree.uniqueId} of ${tree.getClass}, set to ${tree.source} but context had $currentSource\n ${tree.symbol.flagsString}")
 
             // Recursivlely check children while keeping track of current source
             tree match {
@@ -42,7 +43,7 @@ class YCheckPositions extends Phase {
                 traverse(expansion)(using inlineContext(EmptyTree).withSource(sources.head))
                 sources = old
               case Inlined(call, bindings, expansion) =>
-                bindings.foreach(traverse(_))
+                // bindings.foreach(traverse(_)) // TODO check inline proxies (see tests/tun/lst)
                 sources = call.symbol.topLevelClass.source :: sources
                 if (!isMacro(call)) // FIXME macro implementations can drop Inlined nodes. We should reinsert them after macro expansion based on the positions of the trees
                   traverse(expansion)(using inlineContext(call).withSource(sources.head))
@@ -55,8 +56,10 @@ class YCheckPositions extends Phase {
     }
 
   private def isMacro(call: Tree)(using Context) =
-    if (ctx.phase <= postTyperPhase) call.symbol.is(Macro)
-    else call.isInstanceOf[Select] // The call of a macro after typer is encoded as a Select while other inlines are Ident
-                                   // TODO remove this distinction once Inline nodes of expanded macros can be trusted (also in Inliner.inlineCallTrace)
+    call.symbol.is(Macro) ||
+    // The call of a macro after typer is encoded as a Select while other inlines are Ident
+    // TODO remove this distinction once Inline nodes of expanded macros can be trusted (also in Inliner.inlineCallTrace)
+    (!(ctx.phase <= postTyperPhase) && call.isInstanceOf[Select])
+
 }