diff --git a/compiler/src/dotty/tools/dotc/inlines/PrepareInlineable.scala b/compiler/src/dotty/tools/dotc/inlines/PrepareInlineable.scala index eb1a97ab93c0..7a0d3f61cb33 100644 --- a/compiler/src/dotty/tools/dotc/inlines/PrepareInlineable.scala +++ b/compiler/src/dotty/tools/dotc/inlines/PrepareInlineable.scala @@ -18,7 +18,7 @@ import inlines.Inlines import NameOps._ import Annotations._ import transform.{AccessProxies, Splicer} -import staging.PCPCheckAndHeal +import staging.CrossStageSafety import transform.SymUtils.* import config.Printers.inlining import util.Property @@ -294,7 +294,7 @@ object PrepareInlineable { if (code.symbol.flags.is(Inline)) report.error("Macro cannot be implemented with an `inline` method", code.srcPos) Splicer.checkValidMacroBody(code) - (new PCPCheckAndHeal).transform(body) // Ignore output, only check PCP + (new CrossStageSafety).transform(body) // Ignore output, only check cross-stage safety case Block(List(stat), Literal(Constants.Constant(()))) => checkMacro(stat) case Block(Nil, expr) => checkMacro(expr) case Typed(expr, _) => checkMacro(expr) diff --git a/compiler/src/dotty/tools/dotc/staging/PCPCheckAndHeal.scala b/compiler/src/dotty/tools/dotc/staging/CrossStageSafety.scala similarity index 96% rename from compiler/src/dotty/tools/dotc/staging/PCPCheckAndHeal.scala rename to compiler/src/dotty/tools/dotc/staging/CrossStageSafety.scala index 2a1371d8eb0d..219b428ca8d4 100644 --- a/compiler/src/dotty/tools/dotc/staging/PCPCheckAndHeal.scala +++ b/compiler/src/dotty/tools/dotc/staging/CrossStageSafety.scala @@ -17,14 +17,14 @@ import dotty.tools.dotc.util.Property import dotty.tools.dotc.util.Spans._ import dotty.tools.dotc.util.SrcPos -/** Checks that the Phase Consistency Principle (PCP) holds and heals types. +/** Checks that staging level consistency holds and heals staged types . * - * Local term references are phase consistent if and only if they are used at the same level as their definition. + * Local term references are level consistent if and only if they are used at the same level as their definition. * * Local type references can be used at the level of their definition or lower. If used used at a higher level, * it will be healed if possible, otherwise it is inconsistent. * - * Type healing consists in transforming a phase inconsistent type `T` into `summon[Type[T]].Underlying`. + * Type healing consists in transforming a level inconsistent type `T` into `summon[Type[T]].Underlying`. * * As references to types do not necessarily have an associated tree it is not always possible to replace the types directly. * Instead we always generate a type alias for it and place it at the start of the surrounding quote. This also avoids duplication. @@ -43,7 +43,7 @@ import dotty.tools.dotc.util.SrcPos * } * */ -class PCPCheckAndHeal extends TreeMapWithStages { +class CrossStageSafety extends TreeMapWithStages { import tpd._ private val InAnnotation = Property.Key[Unit]() @@ -97,7 +97,7 @@ class PCPCheckAndHeal extends TreeMapWithStages { super.transform(tree) } - /** Transform quoted trees while maintaining phase correctness */ + /** Transform quoted trees while maintaining level correctness */ override protected def transformQuotation(body: Tree, quote: Apply)(using Context): Tree = { val taggedTypes = new QuoteTypeTags(quote.span) diff --git a/compiler/src/dotty/tools/dotc/staging/QuoteContext.scala b/compiler/src/dotty/tools/dotc/staging/QuoteContext.scala index 542e8ecc2b9f..8e25bba7110c 100644 --- a/compiler/src/dotty/tools/dotc/staging/QuoteContext.scala +++ b/compiler/src/dotty/tools/dotc/staging/QuoteContext.scala @@ -4,7 +4,6 @@ import dotty.tools.dotc.ast.tpd import dotty.tools.dotc.core.Contexts._ import dotty.tools.dotc.ast.tpd import dotty.tools.dotc.util.Property -import dotty.tools.dotc.staging.PCPCheckAndHeal import dotty.tools.dotc.staging.StagingLevel.* object QuoteContext { diff --git a/compiler/src/dotty/tools/dotc/transform/Splicing.scala b/compiler/src/dotty/tools/dotc/transform/Splicing.scala index 7ecde9400445..6067a0b4ff96 100644 --- a/compiler/src/dotty/tools/dotc/transform/Splicing.scala +++ b/compiler/src/dotty/tools/dotc/transform/Splicing.scala @@ -21,7 +21,6 @@ import dotty.tools.dotc.core.Names._ import dotty.tools.dotc.core.StdNames._ import dotty.tools.dotc.quoted._ import dotty.tools.dotc.config.ScalaRelease.* -import dotty.tools.dotc.staging.PCPCheckAndHeal import dotty.tools.dotc.staging.QuoteContext.* import dotty.tools.dotc.staging.StagingLevel.* import dotty.tools.dotc.staging.QuoteTypeTags diff --git a/compiler/src/dotty/tools/dotc/transform/Staging.scala b/compiler/src/dotty/tools/dotc/transform/Staging.scala index 5ebfa25eeacd..83b2bcdbcaa6 100644 --- a/compiler/src/dotty/tools/dotc/transform/Staging.scala +++ b/compiler/src/dotty/tools/dotc/transform/Staging.scala @@ -12,12 +12,12 @@ import dotty.tools.dotc.util.SrcPos import dotty.tools.dotc.transform.SymUtils._ import dotty.tools.dotc.staging.QuoteContext.* import dotty.tools.dotc.staging.StagingLevel.* -import dotty.tools.dotc.staging.PCPCheckAndHeal +import dotty.tools.dotc.staging.CrossStageSafety import dotty.tools.dotc.staging.HealType -/** Checks that the Phase Consistency Principle (PCP) holds and heals types. +/** Checks that staging level consistency holds and heals types used in higher levels. * - * Type healing consists in transforming a phase inconsistent type `T` into `${ implicitly[Type[T]] }`. + * See `CrossStageSafety` */ class Staging extends MacroTransform { import tpd._ @@ -32,10 +32,10 @@ class Staging extends MacroTransform { override def checkPostCondition(tree: Tree)(using Context): Unit = if (ctx.phase <= splicingPhase) { - // Recheck that PCP holds but do not heal any inconsistent types as they should already have been heald + // Recheck that staging level consistency holds but do not heal any inconsistent types as they should already have been heald tree match { case PackageDef(pid, _) if tree.symbol.owner == defn.RootClass => - val checker = new PCPCheckAndHeal { + val checker = new CrossStageSafety { override protected def healType(pos: SrcPos)(using Context) = new HealType(pos) { override protected def tryHeal(sym: Symbol, tp: TypeRef, pos: SrcPos): TypeRef = { def symStr = @@ -72,7 +72,7 @@ class Staging extends MacroTransform { protected def newTransformer(using Context): Transformer = new Transformer { override def transform(tree: tpd.Tree)(using Context): tpd.Tree = - (new PCPCheckAndHeal).transform(tree) + (new CrossStageSafety).transform(tree) } } diff --git a/compiler/src/dotty/tools/dotc/transform/TreeChecker.scala b/compiler/src/dotty/tools/dotc/transform/TreeChecker.scala index 05d3ce5679a3..6d904d1f3cc6 100644 --- a/compiler/src/dotty/tools/dotc/transform/TreeChecker.scala +++ b/compiler/src/dotty/tools/dotc/transform/TreeChecker.scala @@ -513,7 +513,7 @@ object TreeChecker { val inliningPhase = ctx.base.inliningPhase inliningPhase.exists && ctx.phase.id > inliningPhase.id if isAfterInlining then - // The staging phase destroys in PCPCheckAndHeal the property that + // The staging phase destroys in CrossStageSafety the property that // tree.expr.tpe <:< pt1. A test case where this arises is run-macros/enum-nat-macro. // We should follow up why this happens. If the problem is fixed, we can // drop the isAfterInlining special case. To reproduce the problem, just diff --git a/docs/_docs/internals/overall-structure.md b/docs/_docs/internals/overall-structure.md index f50ab6bf03a7..5bb43eb946a8 100644 --- a/docs/_docs/internals/overall-structure.md +++ b/docs/_docs/internals/overall-structure.md @@ -104,7 +104,6 @@ phases. The current list of phases is specified in class [Compiler] as follows: List(new semanticdb.ExtractSemanticDB) :: // Extract info into .semanticdb files List(new PostTyper) :: // Additional checks and cleanups after type checking List(new sjs.PrepJSInterop) :: // Additional checks and transformations for Scala.js (Scala.js only) - List(new Staging) :: // Check PCP, heal quoted types and expand macros List(new sbt.ExtractAPI) :: // Sends a representation of the API of classes to sbt via callbacks List(new SetRootTree) :: // Set the `rootTreeOrProvider` on class symbols Nil @@ -112,6 +111,10 @@ phases. The current list of phases is specified in class [Compiler] as follows: /** Phases dealing with TASTY tree pickling and unpickling */ protected def picklerPhases: List[List[Phase]] = List(new Pickler) :: // Generate TASTY info + List(new Inlining) :: // Inline and execute macros + List(new PostInlining) :: // Add mirror support for inlined code + List(new Staging) :: // Check staging levels and heal staged types + List(new Splicing) :: // Replace level 1 splices with holes List(new PickleQuotes) :: // Turn quoted trees into explicit run-time data structures Nil diff --git a/docs/_docs/reference/metaprogramming/macros-spec.md b/docs/_docs/reference/metaprogramming/macros-spec.md index d4221365454d..35a1b4b3d43a 100644 --- a/docs/_docs/reference/metaprogramming/macros-spec.md +++ b/docs/_docs/reference/metaprogramming/macros-spec.md @@ -4,251 +4,711 @@ title: "Macros Spec" nightlyOf: https://docs.scala-lang.org/scala3/reference/metaprogramming/macros-spec.html --- +## Formalization + +* Multi-stage programming with generative and analytical macros[^2] +* Multi-Stage Macro Calculus, Chapter 4 of Scalable Metaprogramming in Scala 3[^1]. + Contains and extends the calculus of _Multi-stage programming with generative and analytical macros_ with type polymorphism. + +## Syntax + +The quotation syntax using `'` and `$` was chosen to mimic the string interpolation syntax of Scala. +Like a string double-quotation, a single-quote block can contain splices. +However, unlike strings, splices can contain quotes using the same rules. + +```scala +s" Hello $name" s" Hello ${name}" +'{ hello($name) } '{ hello(${name}) } +${ hello('name) } ${ hello('{name}) } +``` + +### Quotes +Quotes come in four flavors: quoted identifiers, quoted blocks, quoted block patterns and quoted type patterns. +Scala 2 used quoted identifiers to represent `Symbol` literals. They were deprecated in Scala 3, allowing the syntax to be used for quotation. +```scala +SimpleExpr ::= ... + | `'` alphaid // quoted identifier + | `'` `{` Block `}` // quoted block +Pattern ::= ... + | `'` `{` Block `}` // quoted block pattern + | `'` `[` Type `]` // quoted type pattern +``` + +Quoted blocks and quoted block patterns contain an expression equivalent to a normal block of code. +When entering either of those we track the fact that we are in a quoted block (`inQuoteBlock`) which is used for spliced identifiers. +When entering a quoted block pattern we additionally track the fact that we are in a quoted pattern (`inQuotePattern`) which is used to distinguish spliced blocks and splice patterns. +Lastly, the quoted type pattern simply contains a type. + +### Splices +Splices come in three flavors: spliced identifiers, spliced blocks and splice patterns. +Scala specifies identifiers containing `$` as valid identifiers but reserves them for compiler and standard library use only. +Unfortunately, many libraries have used such identifiers in Scala 2. Therefore to mitigate the cost of migration, we still support them. +We work around this by only allowing spliced identifiers[^3] within quoted blocks or quoted block patterns (`inQuoteBlock`). +Splice blocks and splice patterns can contain an arbitrary block or pattern respectively. +They are distinguished based on their surrounding quote (`inQuotePattern`), a quote block will contain spliced blocks, and a quote block pattern will contain splice patterns. + +```scala +SimpleExpr ::= ... + | `$` alphaid if inQuoteBlock // spliced identifier + | `$` `{` Block `}` if !inQuotePattern // spliced block + | `$` `{` Pattern `}` if inQuotePattern // splice pattern +``` + +### Quoted Pattern Type Variables +Quoted pattern type variables in quoted patterns and quoted type patterns do not require additional syntax. +Any type definition or reference with a name composed of lower cases is assumed to be a pattern type variable definition while typing. +A backticked type name with lower cases is interpreted as a reference to the type with that name. + + ## Implementation -### Syntax - -Compared to the [Scala 3 reference grammar](../syntax.md) -there are the following syntax changes: -``` -SimpleExpr ::= ... - | ‘'’ ‘{’ Block ‘}’ - | ‘'’ ‘[’ Type ‘]’ - | ‘$’ ‘{’ Block ‘}’ -SimpleType ::= ... - | ‘$’ ‘{’ Block ‘}’ -``` -In addition, an identifier `$x` starting with a `$` that appears inside -a quoted expression or type is treated as a splice `${x}` and a quoted identifier -`'x` that appears inside a splice is treated as a quote `'{x}` - -### Implementation in `scalac` - -Quotes and splices are primitive forms in the generated abstract syntax trees. -Top-level splices are eliminated during macro expansion while typing. On the -other hand, top-level quotes are eliminated in an expansion phase `PickleQuotes` -phase (after typing and pickling). PCP checking occurs while preparing the RHS -of an inline method for top-level splices and in the `Staging` phase (after -typing and before pickling). - -Macro-expansion works outside-in. If the outermost scope is a splice, -the spliced AST will be evaluated in an interpreter. A call to a -previously compiled method can be implemented as a reflective call to -that method. With the restrictions on splices that are currently in -place that’s all that’s needed. We might allow more interpretation in -splices in the future, which would allow us to loosen the -restriction. Quotes in spliced, interpreted code are kept as they -are, after splices nested in the quotes are expanded. - -If the outermost scope is a quote, we need to generate code that -constructs the quoted tree at run-time. We implement this by -serializing the tree as a TASTy structure, which is stored -in a string literal. At runtime, an unpickler method is called to -deserialize the string into a tree. - -Splices inside quoted code insert the spliced tree as is, after -expanding any quotes in the spliced code recursively. +### Run-Time Representation -## Formalization +The standard library defines the `Quotes` interface which contains all the logic and the abstract classes `Expr` and `Type`. +The compiler implements the `Quotes` interface and provides the implementation of `Expr` and `Type`. + +##### `class Expr` +Expressions of type `Expr[T]` are represented by the following abstract class: +```scala +abstract class Expr[+T] private[scala] +``` +The only implementation of `Expr` is in the compiler along with the implementation of `Quotes`. +It is a class that wraps a typed AST and a `Scope` object with no methods of its own. +The `Scope` object is used to track the current splice scope and detect scope extrusions. + +##### `object Expr` +The companion object of `Expr` contains a few useful static methods; +the `apply`/`unapply` methods to use `ToExpr`/`FromExpr` with ease; +the `betaReduce` and `summon` methods. +It also contains methods to create expressions out of lists or sequences of expressions: `block`, `ofSeq`, `ofList`, `ofTupleFromSeq` and `ofTuple`. + +```scala +object Expr: + def apply[T](x: T)(using ToExpr[T])(using Quotes): Expr[T] = ... + def unapply[T](x: Expr[T])(using FromExpr[T])(using Quotes): Option[T] = ... + def betaReduce[T](e: Expr[T])(using Quotes): Expr[T] = ... + def summon[T: Type](using Quotes): Option[Expr[T]] = ... + def block[T](stats: List[Expr[Any]], e: Expr[T])(using Quotes): Expr[T] = ... + def ofSeq[T: Type](xs: Seq[Expr[T]])(using Quotes): Expr[Seq[T]] = ... + def ofList[T: Type](xs: Seq[Expr[T]])(using Quotes): Expr[List[T]] = ... + def ofTupleFromSeq(xs: Seq[Expr[Any]])(using Quotes): Expr[Tuple] = ... + def ofTuple[T <: Tuple: Tuple.IsMappedBy[Expr]: Type](tup: T)(using Quotes): + Expr[Tuple.InverseMap[T, Expr]] = ... +``` + +##### `class Type` +Types of type `Type[T]` are represented by the following abstract class: +```scala +abstract class Type[T <: AnyKind] private[scala]: + type Underlying = T +``` + +The only implementation of `Type` is in the compiler along with the implementation of `Quotes`. +It is a class that wraps the AST of a type and a `Scope` object with no methods of its own. +The upper bound of `T` is `AnyKind` which implies that `T` may be a higher-kinded type. +The `Underlying` alias is used to select the type from an instance of `Type`. +Users never need to use this alias as they can always use `T` directly. +`Underlying` is used for internal encoding while compiling the code (see _Type Healing_). + +##### `object Type` +The companion object of `Type` contains a few useful static methods. +The first and most important one is the `Type.of` given definition. +This instance of `Type[T]` is summoned by default when no other instance is available. +The `of` operation is an intrinsic operation that the compiler will transform into code that will generate the `Type[T]` at run-time. +Secondly, the `Type.show[T]` operation will show a string representation of the type, which is often useful when debugging. +Finally, the object defines `valueOfConstant` (and `valueOfTuple`) which can transform singleton types (or tuples of singleton types) into their value. + + +```scala +object Type: + given of[T <: AnyKind](using Quotes): Type[T] = ... + def show[T <: AnyKind](using Type[T])(using Quotes): String = ... + def valueOfConstant[T](using Type[T])(using Quotes): Option[T] = ... + def valueOfTuple[T <: Tuple](using Type[T])(using Quotes): Option[T] = ... +``` + +##### `Quotes` +The `Quotes` interface is where most of the primitive operations of the quotation system are defined. + +Quotes define all the `Expr[T]` methods as extension methods. +`Type[T]` does not have methods and therefore does not appear here. +These methods are available as long as `Quotes` is implicitly given in the current scope. + +The `Quotes` instance is also the entry point to the [reflection API](./refelction.md) through the `reflect` object. + +Finally, `Quotes` provides the internal logic used in quote un-pickling (`QuoteUnpickler`) in quote pattern matching (`QuoteMatching`). +These interfaces are added to the self-type of the trait to make sure they are implemented on this object but not visible to users of `Quotes`. + +Internally, the implementation of `Quotes` will also track its current splicing scope `Scope`. +This scope will be attached to any expression that is created using this `Quotes` instance. + +```scala +trait Quotes: + this: runtime.QuoteUnpickler & runtime.QuoteMatching => + + extension [T](self: Expr[T]) + def show: String + def matches(that: Expr[Any]): Boolean + def value(using FromExpr[T]): Option[T] + def valueOrAbort(using FromExpr[T]): T + end extension + + extension (self: Expr[Any]) + def isExprOf[X](using Type[X]): Boolean + def asExprOf[X](using Type[X]): Expr[X] + end extension + + // abstract object reflect ... +``` + + +##### `Scope` +The splice context is represented as a stack (immutable list) of `Scope` objects. +Each `Scope` contains the position of the splice (used for error reporting) and a reference to the enclosing splice scope `Scope`. +A scope is a sub-scope of another if the other is contained in its parents. +This check is performed when an expression is spliced into another using the `Scope` provided in the current scope in `Quotes` and the one in the `Expr` or `Type`. + +### Entry Points +The two entry points for multi-stage programming are macros and the `run` operation. + +#### Macros +Inline macro definitions will inline a top-level splice (a splice not nested in a quote). +This splice needs to be evaluated at compile-time. +In _Avoiding a complete interpreter_[^1], we stated the following restrictions: + + * The top-level splice must contain a single call to a compiled static method. + * Arguments to the function are either literal constants, quoted expressions (parameters), `Type.of` for type parameters and a reference to `Quotes`. + +These restrictions make the implementation of the interpreter quite simple. +Java Reflection is used to call the single function call in the top-level splice. +The execution of that function is entirely done on compiled bytecode. +These are Scala static methods and may not always become Java static methods, they might be inside module objects. +As modules are encoded as class instances, we need to interpret the prefix of the method to instantiate it before we can invoke the method. + +The code of the arguments has not been compiled and therefore needs to be interpreted by the compiler. +Interpreting literal constants is as simple as extracting the constant from the AST that represents literals. +When interpreting a quoted expression, the contents of the quote is kept as an AST which is wrapped inside the implementation of `Expr`. +Calls to `Type.of[T]` also wrap the AST of the type inside the implementation of `Type`. +Finally, the reference to `Quotes` is supposed to be the reference to the quotes provided by the splice. +This reference is interpreted as a new instance of `Quotes` that contains a fresh initial `Scope` with no parents. + +The result of calling the method via Java Reflection will return an `Expr` containing a new AST that was generated by the implementation of that macro. +The scope of this `Expr` is checked to make sure it did not extrude from some splice or `run` operation. +Then the AST is extracted from the `Expr` and it is inserted as replacement for the AST that contained the top-level splice. + + +#### Run-time Multi-Stage Programming + +To be able to compile the code, the `scala.quoted.staging` library defines the `Compiler` trait. +An instance of `staging.Compiler` is a wrapper over the normal Scala~3 compiler. +To be instantiated it requires an instance of the JVM _classloader_ of the application. + +```scala +import scala.quoted.staging.* +given Compiler = Compiler.make(getClass.getClassLoader) +``` + +The classloader is needed for the compiler to know which dependencies have been loaded and to load the generated code using the same classloader. Below is an example method `mkPower2` that is passed to `staging.run`: + +```scala +def mkPower2()(using Quotes): Expr[Double => Double] = ... + +run(mkPower2()) +``` +To run the previous example, the compiler will create code equivalent to the following class and compile it using a new `Scope` without parents. + +```scala +class RunInstance: + def exec(): Double => Double = ${ mkPower2() } +``` +Finally, `run` will interpret `(new RunInstance).exec()` to evaluate the contents of the quote. +To do this, the resulting `RunInstance` class is loaded in the JVM using Java Reflection, instantiated and then the `exec` method is invoked. + + +### Compilation + +Quotes and splices are primitive forms in the generated typed abstract syntax trees. +These need to be type-checked with some extra rules, e.g., staging levels need to be checked and the references to generic types need to be adapted. +Finally, quoted expressions that will be generated at run-time need to be encoded (serialized/pickled) and decoded (deserialized/unpickled). + +#### Typing Quoted Expressions + +The typing process for quoted expressions and splices with `Expr` is relatively straightforward. +At its core, quotes are desugared into calls to `quote`, splices are desugared into calls to `splice`. +We track the quotation level when desugaring into these methods. + + +```scala +def quote[T](x: T): Quotes ?=> Expr[T] + +def splice[T](x: Quotes ?=> Expr[T]): T +``` + +It would be impossible to track the quotation levels if users wrote calls to these methods directly. +To know if it is a call to one of those methods we would need to type it first, but to type it we would need to know if it is one of these methods to update the quotation level. +Therefore these methods can only be used by the compiler. + +At run-time, the splice needs to have a reference to the `Quotes` that created its surrounding quote. +To simplify this for later phases, we track the current `Quotes` and encode a reference directly in the splice using `nestedSplice` instead of `splice`. + +```scala +def nestedSplice[T](q: Quotes)(x: q.Nested ?=> Expr[T]): T +``` +With this addition, the original `splice` is only used for top-level splices. + +The levels are mostly used to identify top-level splices that need to be evaluated while typing. +We do not use the quotation level to influence the typing process. +Level checking is performed at a later phase. +This ensures that a source expression in a quote will have the same elaboration as a source expression outside the quote. + + + +#### Quote Pattern Matching + +Pattern matching is defined in the trait `QuoteMatching`, which is part of the self type of `Quotes`. +It is implemented by `Quotes` but not available to users of `Quotes`. +To access it, the compiler generates a cast from `Quotes` to `QuoteMatching` and then selects one of its two members: `ExprMatch` or `TypeMatch`. +`ExprMatch` defines an `unapply` extractor method that is used to encode quote patterns and `TypeMatch` defines an `unapply` method for quoted type patterns. + +```scala +trait Quotes: + self: runtime.QuoteMatching & ... => + ... + +trait QuoteMatching: + object ExprMatch: + def unapply[TypeBindings <: Tuple, Tup <: Tuple] + (scrutinee: Expr[Any]) + (using pattern: Expr[Any]): Option[Tup] = ... + object TypeMatch: + ... +``` + +These extractor methods are only meant to be used in code generated by the compiler. +The call to the extractor that is generated has an already elaborated form that cannot be written in source, namely explicit type parameters and explicit contextual parameters. + +This extractor returns a tuple type `Tup` which cannot be inferred from the types in the method signature. +This type will be computed when typing the quote pattern and will be explicitly added to the extractor call. +To refer to type variables in arbitrary places of `Tup`, we need to define them all before their use, hence we have `TypeBindings`, which will contain all pattern type variable definitions. +The extractor also receives a given parameter of type `Expr[Any]` that will contain an expression that represents the pattern. +The compiler will explicitly add this pattern expression. +We use a given parameter because these are the only parameters we are allowed to add to the extractor call in a pattern position. + +This extractor is a bit convoluted, but it encodes away all the quotation-specific features. +It compiles the pattern down into a representation that the pattern matcher compiler phase understands. + +The quote patterns are encoded into two parts: a tuple pattern that is tasked with extracting the result of the match and a quoted expression representing the pattern. +For example, if the pattern has no `$` we will have an `EmptyTuple` as the pattern and `'{1}` to represent the pattern. + +```scala + case '{ 1 } => +// is elaborated to + case ExprMatch(EmptyTuple)(using '{1}) => +// ^^^^^^^^^^ ^^^^^^^^^^ +// pattern expression +``` +When extracting expressions, each pattern that is contained in a splice `${..}` will be placed in order in the tuple pattern. +In the following case, the `f` and `x` are placed in a tuple pattern `(f, x)`. +The type of the tuple is encoded in the `Tup` and not only in the tuple itself. +Otherwise, the extractor would return a tuple `Tuple` for which the types need to be tested which is in turn not possible due to type erasure. + +```scala + case '{ ((y: Int) => $f(y)).apply($x) } => +// is elaborated to + case ExprMatch[.., (Expr[Int => Int], Expr[Int])]((f, x))(using pattern) => +// pattern = '{ ((y: Int) => pat[Int](y)).apply(pat[Int]()) } +``` +The contents of the quote are transformed into a valid quote expression by replacing the splice with a marker expression `pat[T](..)`. +The type `T` is taken from the type of the splice and the arguments are the HOAS arguments. +This implies that a `pat[T]()` is a closed pattern and `pat[T](y)` is an HOAS pattern that can refer to `y`. + + +Type variables in quoted patterns are first normalized to have all definitions at the start of the pattern. +For each definition of a type variable `t` in the pattern we will add a type variable definition in `TypeBindings`. +Each one will have a corresponding `Type[t]` that will get extracted if the pattern matches. +These `Type[t]` are also listed in the `Tup` and added in the tuple pattern. +It is additionally marked as `using` in the pattern to make it implicitly available in this case branch. + + +```scala + case '{ type t; ($xs: List[t]).map[t](identity[t]) } => +// is elaborated to + case ExprMatch[(t), (Type[t], Expr[List[t]])]((using t, xs))(using p) => +// ^^^ ^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^ ^^^^^^^ +// type bindings result type pattern expression +// p = '{ @patternType type u; pat[List[u]]().map[u](identity[u]) } +``` + +The contents of the quote are transformed into a valid quote expression by replacing type variables with fresh ones that do not escape the quote scope. +These are also annotated to be easily identifiable as pattern variables. + +#### Level Consistency Checking +Level consistency checking is performed after typing the program as a static check. +To check level consistency we traverse the tree top-down remembering the context staging level. +Each local definition in scope is recorded with its level and each term reference to a definition is checked against the current staging level. +```scala +// level 0 +'{ // level 1 + val x = ... // level 1 with (x -> 1) + ${ // level 0 (x -> 1) + val y = ... // level 0 with (x -> 1, y -> 0) + x // error: defined at level 1 but used in level 0 + } + // level 1 (x -> 1) + x // x is ok +} +``` + +#### Type Healing + +When using a generic type `T` in a future stage, it is necessary to have a given `Type[T]` in scope. +The compiler needs to identify those references and link them with the instance of `Type[T]`. +For instance consider the following example: + +```scala +def emptyList[T](using t: Type[T])(using Quotes): Expr[List[T]] = + '{ List.empty[T] } +``` + +For each reference to a generic type `T` that is defined at level 0 and used at level 1 or greater, the compiler will summon a `Type[T]`. +This is usually the given type that is provided as parameter, `t` in this case. +We can use the type `t.Underlying` to replace `T` as it is an alias of that type. +But `t.Underlying` contains the extra information that it is `t` that will be used in the evaluation of the quote. +In a sense, `Underlying` acts like a splice for types. + +```scala +def emptyList[T](using t: Type[T])(using Quotes): Expr[List[T]] = + '{ List.empty[t.Underlying] } +``` + +Due to some technical limitations, it is not always possible to replace the type reference with the AST containing `t.Underlying`. +To overcome this limitation, we can simply define a list of type aliases at the start of the quote and insert the `t.Underlying` there. +This has the added advantage that we do not have to repeatedly insert the `t.Underlying` in the quote. + +```scala +def emptyList[T](using t: Type[T])(using Quotes): Expr[List[T]] = + '{ type U = t.Underlying; List.empty[U] } +``` +These aliases can be used at any level within the quote and this transformation is only performed on quotes that are at level 0. + +```scala + '{ List.empty[T] ... '{ List.empty[T] } ... } +// becomes + '{ type U = t.Underlying; List.empty[U] ... '{ List.empty[U] } ... } +``` +If we define a generic type at level 1 or greater, it will not be subject to this transformation. +In some future compilation stage, when the definition of the generic type is at level 0, it will be subject to this transformation. +This simplifies the transformation logic and avoids leaking the encoding into code that a macro could inspect. + +```scala +'{ + def emptyList[T: Type](using Quotes): Expr[List[T]] = '{ List.empty[T] } + ... +} +``` +A similar transformation is performed on `Type.of[T]`. +Any generic type in `T` needs to have an implicitly given `Type[T]` in scope, which will also be used as a path. +The example: + +```scala +def empty[T](using t: Type[T])(using Quotes): Expr[T] = + Type.of[T] match ... +// becomes +def empty[T](using t: Type[T])(using Quotes): Expr[T] = + Type.of[t.Underlying] match ... +// then becomes +def empty[T](using t: Type[T])(using Quotes): Expr[T] = + t match ... +``` + +The operation `Type.of[t.Underlying]` can be optimized to just `t`. +But this is not always the case. +If the generic reference is nested in the type, we will need to keep the `Type.of`. -The phase consistency principle can be formalized in a calculus that -extends simply-typed lambda calculus with quotes and splices. +```scala +def matchOnList[T](using t: Type[T])(using Quotes): Expr[List[T]] = + Type.of[List[T]] match ... +// becomes +def matchOnList[T](using t: Type[T])(using Quotes): Expr[List[T]] = + Type.of[List[t.Underlying]] match ... +``` -### Syntax +By doing this transformation, we ensure that each abstract type `U` used in `Type.of` has an implicit `Type[U]` in scope. +This representation makes it simpler to identify parts of the type that are statically known from those that are known dynamically. +Type aliases are also added within the type of the `Type.of` though these are not valid source code. +These would look like `Type.of[{type U = t.Underlying; Map[U, U]}]` if written in source code. -The syntax of terms, values, and types is given as follows: -```ebnf -Terms t ::= x variable - (x: T) => t lambda - t t application - 't quote - $t splice -Values v ::= (x: T) => t lambda - 'u quote +#### Splice Normalization -Simple terms u ::= x | (x: T) => u | u u | 't +The contents of a splice may refer to variables defined in the enclosing quote. +This complicates the process of serialization of the contents of the quotes. +To make serialization simple, we first transform the contents of each level 1 splice. +Consider the following example: -Types T ::= A base type - T -> T function type - expr T quoted +```scala +def power5to(n: Expr[Int]): Expr[Double] = '{ + val x: Int = 5 + ${ powerCode('{x}, n) } +} ``` -Typing rules are formulated using a stack of environments -`Es`. Individual environments `E` consist as usual of variable -bindings `x: T`. Environments can be combined using the two -combinators `'` and `$`. -```ebnf -Environment E ::= () empty - E, x: T -Env. stack Es ::= () empty - E simple - Es * Es combined +The variable `x` is defined in the quote and used in the splice. +The normal form will extract all references to `x` and replace them with a staged version of `x`. +We will replace the reference to `x` of type `T` with a `$y` where `y` is of type `Expr[T]`. +Then we wrap the new contents of the splice in a lambda that defines `y` and apply it to the quoted version of `x`. +After this transformation we have 2 parts, a lambda without references to the quote, which knows how to compute the contents of the splice, and a sequence of quoted arguments that refer to variables defined in the lambda. -Separator * ::= ' - $ +```scala +def power5to(n: Expr[Int]): Expr[Double] = '{ + val x: Int = 5 + ${ ((y: Expr[Int]) => powerCode('{$y}, n)).apply('x) } +} ``` -The two environment combinators are both associative with left and -right identity `()`. -### Operational semantics +In general, the splice normal form has the shape `${ .apply(*) }` and the following constraints: + * `` a lambda expression that does not refer to variables defined in the outer quote + * `` sequence of quoted expressions or `Type.of` containing references to variables defined in the enclosing quote and no references to local variables defined outside the enclosing quote -We define a small step reduction relation `-->` with the following rules: + +##### Function references normalization +A reference to a function `f` that receives parameters is not a valid value in Scala. +Such a function reference `f` can be eta-expanded as `x => f(x)` to be used as a lambda value. +Therefore function references cannot be transformed by the normalization as directly as other expressions as we cannot represent `'{f}` with a method reference type. +We can use the eta-expanded form of `f` in the normalized form. +For example, consider the reference to `f` below. + +```scala +'{ + def f(a: Int)(b: Int, c: Int): Int = 2 + a + b + c + ${ '{ f(3)(4, 5) } } +} ``` - ((x: T) => t) v --> [x := v]t - ${'u} --> u +To normalize this code, we can eta-expand the reference to `f` and place it in a quote containing a proper expression. +Therefore the normalized form of the argument `'{f}` becomes the quoted lambda `'{ (a: Int) => (b: Int, c: Int) => f(a)(b, c) }` and is an expression of type `Expr[Int => (Int, Int) => Int]`. +The eta-expansion produces one curried lambda per parameter list. +The application `f(3)(4, 5)` does not become `$g(3)(4, 5)` but `$g.apply(3).apply(4, 5)`. +We add the `apply` because `g` is not a quoted reference to a function but a curried lambda. - t1 --> t2 - ----------------- - e[t1] --> e[t2] +```scala +'{ + def f(a: Int)(b: Int, c: Int): Int = 2 + a + b + c + ${ + ( + (g: Expr[Int => (Int, Int) => Int]) => '{$g.apply(3).apply(4, 5)} + ).apply('{ (a: Int) => (b: Int, c: Int) => f(a)(b, c) }) + } +} ``` -The first rule is standard call-by-value beta-reduction. The second -rule says that splice and quotes cancel each other out. The third rule -is a context rule; it says that reduction is allowed in the hole `[ ]` -position of an evaluation context. Evaluation contexts `e` and -splice evaluation context `e_s` are defined syntactically as follows: -```ebnf -Eval context e ::= [ ] | e t | v e | 'e_s[${e}] -Splice context e_s ::= [ ] | (x: T) => e_s | e_s t | u e_s + +Then we can apply it and beta-reduce the application when generating the code. + +```scala + (g: Expr[Int => Int => Int]) => betaReduce('{$g.apply(3).apply(4)}) ``` -### Typing rules -Typing judgments are of the form `Es |- t: T`. There are two -substructural rules which express the fact that quotes and splices -cancel each other out: +##### Variable assignment normalization +A reference to a mutable variable in the left-hand side of an assignment cannot be transformed directly as it is not in an expression position. +```scala +'{ + var x: Int = 5 + ${ g('{x = 2}) } +} ``` - Es1 * Es2 |- t: T - --------------------------- - Es1 $ E1 ' E2 * Es2 |- t: T +We can use the same strategy used for function references by eta-expanding the assignment operation `x = _` into `y => x = y`. - Es1 * Es2 |- t: T - --------------------------- - Es1 ' E1 $ E2 * Es2 |- t: T +```scala +'{ + var x: Int = 5 + ${ + g( + ( + (f: Expr[Int => Unit]) => betaReduce('{$f(2)}) + ).apply('{ (y: Int) => x = $y }) + ) + } +} ``` -The lambda calculus fragment of the rules is standard, except that we -use a stack of environments. The rules only interact with the topmost -environment of the stack. + + +##### Type normalization +Types defined in the quote are subject to a similar transformation. +In this example, `T` is defined within the quote at level 1 and used in the splice again at level 1. + +```scala +'{ def f[T] = ${ '{g[T]} } } ``` - x: T in E - -------------- - Es * E |- x: T +The normalization will add a `Type[T]` to the lambda, and we will insert this reference. +The difference is that it will add an alias similar to the one used in type healing. +In this example, we create a `type U` that aliases the staged type. + +```scala +'{ + def f[T] = ${ + ( + (t: Type[T]) => '{type U = t.Underling; g[U]} + ).apply(Type.of[T]) + } +} +``` - Es * E, x: T1 |- t: T2 - ------------------------------- - Es * E |- (x: T1) => t: T -> T2 +#### Serialization +Quoted code needs to be pickled to make it available at run-time in the next compilation phase. +We implement this by pickling the AST as a TASTy binary. - Es |- t1: T2 -> T Es |- t2: T2 - --------------------------------- - Es |- t1 t2: T +##### TASTy +The TASTy format is the typed abstract syntax tree serialization format of Scala 3. +It usually pickles the fully elaborated code after type-checking and is kept along the generated Java classfiles. + + +##### Pickling +We use TASTy as a serialization format for the contents of the quotes. +To show how serialization is performed, we will use the following example. +```scala +'{ + val (x, n): (Double, Int) = (5, 2) + ${ powerCode('{x}, '{n}) } * ${ powerCode('{2}, '{n}) } +} ``` -The rules for quotes and splices map between `expr T` and `T` by trading `'` and `$` between -environments and terms. + +This quote is transformed into the following code when normalizing the splices. + +```scala +'{ + val (x, n): (Double, Int) = (5, 2) + ${ + ((y: Expr[Double], m: Expr[Int]) => powerCode(y, m)).apply('x, 'n) + } * ${ + ((m: Expr[Int]) => powerCode('{2}, m)).apply('n) + } +} ``` - Es $ () |- t: expr T - -------------------- - Es |- $t: T +Splice normalization is a key part of the serialization process as it only allows references to variables defined in the quote in the arguments of the lambda in the splice. +This makes it possible to create a closed representation of the quote without much effort. +The first step is to remove all the splices and replace them with holes. +A hole is like a splice but it lacks the knowledge of how to compute the contents of the splice. +Instead, it knows the index of the hole and the contents of the arguments of the splice. +We can see this transformation in the following example where a hole is represented by `<< idx; holeType; args* >>`. - Es ' () |- t: T - ---------------- - Es |- 't: expr T +```scala + ${ ((y: Expr[Double], m: Expr[Int]) => powerCode(y, m)).apply('x, 'n) } +// becomes + << 0; Double; x, n >> ``` -The meta theory of a slightly simplified 2-stage variant of this calculus -is studied [separately](./simple-smp.md). -## Going Further +As this was the first hole it has index 0. +The hole type is `Double`, which needs to be remembered now that we cannot infer it from the contents of the splice. +The arguments of the splice are `x` and `n`; note that they do not require quoting because they were moved out of the splice. -The metaprogramming framework as presented and currently implemented is quite restrictive -in that it does not allow for the inspection of quoted expressions and -types. It’s possible to work around this by providing all necessary -information as normal, unquoted inline parameters. But we would gain -more flexibility by allowing for the inspection of quoted code with -pattern matching. This opens new possibilities. +References to healed types are handled in a similar way. +Consider the `emptyList` example, which shows the type aliases that are inserted into the quote. +```scala +'{ List.empty[T] } +// type healed to +'{ type U = t.Underlying; List.empty[U] } +``` +Instead of replacing a splice, we replace the `t.Underlying` type with a type hole. +The type hole is represented by `<< idx; bounds >>`. +```scala +'{ type U = << 0; Nothing..Any >>; List.empty[U] } +``` +Here, the bounds of `Nothing..Any` are the bounds of the original `T` type. +The types of a `Type.of` are transformed in the same way. + + +With these transformations, the contents of the quote or `Type.of` are guaranteed to be closed and therefore can be pickled. +The AST is pickled into TASTy, which is a sequence of bytes. +This sequence of bytes needs to be instantiated in the bytecode, but unfortunately it cannot be dumped into the classfile as bytes. +To reify it we encode the bytes into a Java `String`. +In the following examples we display this encoding in human readable form with the fictitious `|tasty"..."|` string literal. -For instance, here is a version of `power` that generates the multiplications -directly if the exponent is statically known and falls back to the dynamic -implementation of `power` otherwise. ```scala -import scala.quoted.* +// pickled AST bytes encoded in a base64 string +tasty""" + val (x, n): (Double, Int) = (5, 2) + << 0; Double; x, n >> * << 1; Double; n >> +""" +// or +tasty""" + type U = << 0; Nothing..Any; >> + List.empty[U] +""" +``` +The contents of a quote or `Type.of` are not always pickled. +In some cases it is better to generate equivalent (smaller and/or faster) code that will compute the expression. +Literal values are compiled into a call to `Expr()` using the implementation of `ToExpr` to create the quoted expression. +This is currently performed only on literal values, but can be extended to any value for which we have a `ToExpr` defined in the standard library. +Similarly, for non-generic types we can use their respective `java.lang.Class` and convert them into a `Type` using a primitive operation `typeConstructorOf` defined in the reflection API. -inline def power(x: Double, n: Int): Double = - ${ powerExpr('x, 'n) } +##### Unpickling -private def powerExpr(x: Expr[Double], n: Expr[Int]) - (using Quotes): Expr[Double] = - n.value match - case Some(m) => powerExpr(x, m) - case _ => '{ dynamicPower($x, $n) } +Now that we have seen how a quote is pickled, we can look at how to unpickle it. +We will continue with the previous example. -private def powerExpr(x: Expr[Double], n: Int) - (using Quotes): Expr[Double] = - if n == 0 then '{ 1.0 } - else if n == 1 then x - else if n % 2 == 0 then '{ val y = $x * $x; ${ powerExpr('y, n / 2) } } - else '{ $x * ${ powerExpr(x, n - 1) } } +Holes were used to replace the splices in the quote. +When we perform this transformation we also need to remember the lambdas from the splices and their hole index. +When unpickling a hole, the corresponding splice lambda will be used to compute the contents of the hole. +The lambda will receive as parameters quoted versions of the arguments of the hole. +For example to compute the contents of `<< 0; Double; x, n >>` we will evaluate the following code -private def dynamicPower(x: Double, n: Int): Double = - if n == 0 then 1.0 - else if n % 2 == 0 then dynamicPower(x * x, n / 2) - else x * dynamicPower(x, n - 1) +```scala + ((y: Expr[Double], m: Expr[Int]) => powerCode(y, m)).apply('x, 'n) ``` -In the above, the method `.value` maps a constant expression of the type -`Expr[T]` to its value of the type `T`. +The evaluation is not as trivial as it looks, because the lambda comes from compiled code and the rest is code that must be interpreted. +We put the AST of `x` and `n` into `Expr` objects to simulate the quotes and then we use Java Reflection to call the `apply` method. + +We may have many holes in a quote and therefore as many lambdas. +To avoid the instantiation of many lambdas, we can join them together into a single lambda. +Apart from the list of arguments, this lambda will also take the index of the hole that is being evaluated. +It will perform a switch match on the index and call the corresponding lambda in each branch. +Each branch will also extract the arguments depending on the definition of the lambda. +The application of the original lambdas are beta-reduced to avoid extra overhead. -With the right extractors, the "AsFunction" conversion -that maps expressions over functions to functions over expressions can -be implemented in user code: ```scala -given AsFunction1[T, U]: Conversion[Expr[T => U], Expr[T] => Expr[U]] with - def apply(f: Expr[T => U]): Expr[T] => Expr[U] = - (x: Expr[T]) => f match - case Lambda(g) => g(x) - case _ => '{ ($f)($x) } +(idx: Int, args: Seq[Any]) => + idx match + case 0 => // for << 0; Double; x, n >> + val x = args(0).asInstanceOf[Expr[Double]] + val n = args(1).asInstanceOf[Expr[Int]] + powerCode(x, n) + case 1 => // for << 1; Double; n >> + val n = args(0).asInstanceOf[Expr[Int]] + powerCode('{2}, n) ``` -This assumes an extractor + +This is similar to what we do for splices when we replace the type aliased with holes we keep track of the index of the hole. +Instead of lambdas, we will have a list of references to instances of `Type`. +From the following example we would extract `t`, `u`, ... . + ```scala -object Lambda: - def unapply[T, U](x: Expr[T => U]): Option[Expr[T] => Expr[U]] + '{ type T1 = t1.Underlying; type Tn = tn.Underlying; ... } +// with holes + '{ type T1 = << 0; ... >>; type Tn = << n-1; ... >>; ... } ``` -Once we allow inspection of code via extractors, it’s tempting to also -add constructors that create typed trees directly without going -through quotes. Most likely, those constructors would work over `Expr` -types which lack a known type argument. For instance, an `Apply` -constructor could be typed as follows: + +As the type holes are at the start of the quote, they will have the first `N` indices. +This implies that we can place the references in a sequence `Seq(t, u, ...)` where the index in the sequence is the same as the hole index. + +Lastly, the quote itself is replaced by a call to `QuoteUnpickler.unpickleExpr` which will unpickle the AST, evaluate the holes, i.e., splices, and wrap the resulting AST in an `Expr[Int]`. +This method takes takes the pickled `|tasty"..."|`, the types and the hole lambda. +Similarly, `Type.of` is replaced with a call to `QuoteUnpickler.unpickleType` but only receives the pickled `|tasty"..."|` and the types. +Because `QuoteUnpickler` is part of the self-type of the `Quotes` class, we have to cast the instance but know that this cast will always succeed. + ```scala -def Apply(fn: Expr[Any], args: List[Expr[Any]]): Expr[Any] +quotes.asInstanceOf[runtime.QuoteUnpickler].unpickleExpr[T]( + pickled = tasty"...", + types = Seq(...), + holes = (idx: Int, args: Seq[Any]) => idx match ... +) ``` -This would allow constructing applications from lists of arguments -without having to match the arguments one-by-one with the -corresponding formal parameter types of the function. We then need "at -the end" a method to convert an `Expr[Any]` to an `Expr[T]` where `T` is -given from the outside. For instance, if `code` yields a `Expr[Any]`, then -`code.atType[T]` yields an `Expr[T]`. The `atType` method has to be -implemented as a primitive; it would check that the computed type -structure of `Expr` is a subtype of the type structure representing -`T`. -Before going down that route, we should evaluate in detail the tradeoffs it -presents. Constructing trees that are only verified _a posteriori_ -to be type correct loses a lot of guidance for constructing the right -trees. So we should wait with this addition until we have more -use-cases that help us decide whether the loss in type-safety is worth -the gain in flexibility. In this context, it seems that deconstructing types is -less error-prone than deconstructing terms, so one might also -envisage a solution that allows the former but not the latter. - -## Conclusion - -Metaprogramming has a reputation of being difficult and confusing. -But with explicit `Expr/Type` types and quotes and splices it can become -downright pleasant. A simple strategy first defines the underlying quoted or unquoted -values using `Expr` and `Type` and then inserts quotes and splices to make the types -line up. Phase consistency is at the same time a great guideline -where to insert a splice or a quote and a vital sanity check that -the result makes sense. +[^1]: [Scalable Metaprogramming in Scala 3](https://infoscience.epfl.ch/record/299370) +[^2]: [Multi-stage programming with generative and analytical macros](https://dl.acm.org/doi/10.1145/3486609.3487203). +[^3]: In quotes, identifiers starting with `$` must be surrounded by backticks (`` `$` ``). For example `$conforms` from `scala.Predef`. diff --git a/docs/_docs/reference/metaprogramming/macros.md b/docs/_docs/reference/metaprogramming/macros.md index 0be48ef2baf8..a91e69d985f0 100644 --- a/docs/_docs/reference/metaprogramming/macros.md +++ b/docs/_docs/reference/metaprogramming/macros.md @@ -6,843 +6,617 @@ nightlyOf: https://docs.scala-lang.org/scala3/reference/metaprogramming/macros.h > When developing macros enable `-Xcheck-macros` scalac option flag to have extra runtime checks. -## Macros: Quotes and Splices +## Multi-Staging -Macros are built on two well-known fundamental operations: quotation and splicing. -Quotation is expressed as `'{...}` for expressions and splicing is expressed as `${ ... }`. -Additionally, within a quote or a splice we can quote or splice identifiers directly (i.e. `'e` and `$e`). -Readers may notice the resemblance of the two aforementioned syntactic -schemes with the familiar string interpolation syntax. +#### Quoted expressions +Multi-stage programming in Scala 3 uses quotes `'{..}` to delay, i.e., stage, execution of code and splices `${..}` to evaluate and insert code into quotes. +Quoted expressions are typed as `Expr[T]` with a covariant type parameter `T`. +It is easy to write statically safe code generators with these two concepts. +The following example shows a naive implementation of the $x^n$ mathematical operation. ```scala -println(s"Hello, $name, here is the result of 1 + 1 = ${1 + 1}") +import scala.quoted.* +def unrolledPowerCode(x: Expr[Double], n: Int)(using Quotes): Expr[Double] = + if n == 0 then '{ 1.0 } + else if n == 1 then x + else '{ $x * ${ unrolledPowerCode(x, n-1) } } ``` -In string interpolation we _quoted_ a string and then we _spliced_ into it, two others. The first, `name`, is a reference to a value of type [`String`](https://scala-lang.org/api/3.x/scala/Predef$.html#String-0), and the second is an arithmetic expression that will be _evaluated_ followed by the splicing of its string representation. - -Quotes and splices in this section allow us to treat code in a similar way, -effectively supporting macros. The entry point for macros is an inline method -with a top-level splice. We call it a top-level because it is the only occasion -where we encounter a splice outside a quote (consider as a quote the -compilation-unit at the call-site). For example, the code below presents an -`inline` method `assert` which calls at compile-time a method `assertImpl` with -a boolean expression tree as argument. `assertImpl` evaluates the expression and -prints it again in an error message if it evaluates to `false`. - ```scala -import scala.quoted.* - -inline def assert(inline expr: Boolean): Unit = - ${ assertImpl('expr) } - -def assertImpl(expr: Expr[Boolean])(using Quotes) = '{ - if !$expr then - throw AssertionError(s"failed assertion: ${${ showExpr(expr) }}") +'{ + val x = ... + ${ unrolledPowerCode('{x}, 3) } // evaluates to: x * x * x } - -def showExpr(expr: Expr[Boolean])(using Quotes): Expr[String] = - '{ [actual implementation later in this document] } ``` -If `e` is an expression, then `'{e}` represents the typed -abstract syntax tree representing `e`. If `T` is a type, then `Type.of[T]` -represents the type structure representing `T`. The precise -definitions of "typed abstract syntax tree" or "type-structure" do not -matter for now, the terms are used only to give some -intuition. Conversely, `${e}` evaluates the expression `e`, which must -yield a typed abstract syntax tree or type structure, and embeds the -result as an expression (respectively, type) in the enclosing program. +Quotes and splices are duals of each other. +For an arbitrary expression `x` of type `T` we have `${'{x}} = x` and for an arbitrary expression `e` of type `Expr[T]` we have `'{${e}} = e`. -Quotations can have spliced parts in them; in this case the embedded -splices are evaluated and embedded as part of the formation of the -quotation. +#### Abstract types +Quotes can handle generic and abstract types using the type class `Type[T]`. +A quote that refers to a generic or abstract type `T` requires a given `Type[T]` to be provided in the implicit scope. +The following examples show how `T` is annotated with a context bound (`: Type`) to provide an implicit `Type[T]`, or the equivalent `using Type[T]` parameter. -Quotes and splices can also be applied directly to identifiers. An identifier -`$x` starting with a `$` that appears inside a quoted expression or type is treated as a -splice `${x}`. Analogously, an quoted identifier `'x` that appears inside a splice -is treated as a quote `'{x}`. See the Syntax section below for details. +```scala +import scala.quoted.* +def singletonListExpr[T: Type](x: Expr[T])(using Quotes): Expr[List[T]] = + '{ List[T]($x) } // generic T used within a quote -Quotes and splices are duals of each other. -For arbitrary expressions `e` we have: +def emptyListExpr[T](using Type[T], Quotes): Expr[List[T]] = + '{ List.empty[T] } // generic T used within a quote +``` +If no other instance is found, the default `Type.of[T]` is used. +The following example implicitly uses `Type.of[String]` and `Type.of[Option[U]]`. ```scala -${'{e}} = e -'{${e}} = e +val list1: Expr[List[String]] = + singletonListExpr('{"hello"}) // requires a given `Type[Sting]` +val list0: Expr[List[Option[T]]] = + emptyListExpr[Option[U]] // requires a given `Type[Option[U]]` ``` -## Types for Quotations - -The type signatures of quotes and splices can be described using -two fundamental types: -- `Expr[T]`: abstract syntax trees representing expressions of type `T` -- `Type[T]`: non erased representation of type `T`. - -Quoting takes expressions of type `T` to expressions of type `Expr[T]` -and it takes types `T` to expressions of type `Type[T]`. Splicing -takes expressions of type `Expr[T]` to expressions of type `T` and it -takes expressions of type `Type[T]` to types `T`. - -The two types can be defined in package [`scala.quoted`](https://scala-lang.org/api/3.x/scala/quoted.html) as follows: +The `Type.of[T]` method is a primitive operation that the compiler will handle specially. +It will provide the implicit if the type `T` is statically known, or if `T` contains some other types `Ui` for which we have an implicit `Type[Ui]`. +In the example, `Type.of[String]` has a statically known type and `Type.of[Option[U]]` requires an implicit `Type[U]` in scope. +#### Quote context +We also track the current quotation context using a given `Quotes` instance. +To create a quote `'{..}` we require a given `Quotes` context, which should be passed as a contextual parameter `(using Quotes)` to the function. +Each splice will provide a new `Quotes` context within the scope of the splice. +Therefore quotes and splices can be seen as methods with the following signatures, but with special semantics. ```scala -package scala.quoted +def '[T](x: T): Quotes ?=> Expr[T] // def '[T](x: T)(using Quotes): Expr[T] -sealed trait Expr[+T] -sealed trait Type[T] +def $[T](x: Quotes ?=> Expr[T]): T ``` -Both `Expr` and `Type` are abstract and sealed, so all constructors for -these types are provided by the system. One way to construct values of -these types is by quoting, the other is by type-specific lifting -operations that will be discussed later on. +The lambda with a question mark `?=>` is a contextual function; it is a lambda that takes its argument implicitly and provides it implicitly in the implementation the lambda. +`Quotes` are used for a variety of purposes that will be mentioned when covering those topics. -## The Phase Consistency Principle +## Quoted Values -A fundamental *phase consistency principle* (PCP) regulates accesses -to free variables in quoted and spliced code: +#### Lifting +While it is not possible to use cross-stage persistence of local variables, it is possible to lift them to the next stage. +To this end, we provide the `Expr.apply` method, which can take a value and lift it into a quoted representation of the value. -- _For any free variable reference `x`, the number of quoted scopes and the number of spliced scopes between the reference to `x` and the definition of `x` must be equal_. - -Here, `this`-references count as free variables. On the other -hand, we assume that all imports are fully expanded and that `_root_` is -not a free variable. So references to global definitions are -allowed everywhere. +```scala +val expr1plus1: Expr[Int] = '{ 1 + 1 } -The phase consistency principle can be motivated as follows: First, -suppose the result of a program `P` is some quoted text `'{ ... x -... }` that refers to a free variable `x` in `P`. This can be -represented only by referring to the original variable `x`. Hence, the -result of the program will need to persist the program state itself as -one of its parts. We don’t want to do this, hence this situation -should be made illegal. Dually, suppose a top-level part of a program -is a spliced text `${ ... x ... }` that refers to a free variable `x` -in `P`. This would mean that we refer during _construction_ of `P` to -a value that is available only during _execution_ of `P`. This is of -course impossible and therefore needs to be ruled out. Now, the -small-step evaluation of a program will reduce quotes and splices in -equal measure using the cancellation rules above. But it will neither -create nor remove quotes or splices individually. So the PCP ensures -that program elaboration will lead to neither of the two unwanted -situations described above. +val expr2: Expr[Int] = Expr(1 + 1) // lift 2 into '{ 2 } +``` -In what concerns the range of features it covers, this form of macros introduces -a principled metaprogramming framework that is quite close to the MetaML family of -languages. One difference is that MetaML does not have an equivalent of the PCP - -quoted code in MetaML _can_ access variables in its immediately enclosing -environment, with some restrictions and caveats since such accesses involve -serialization. However, this does not constitute a fundamental gain in -expressiveness. +While it looks type wise similar to `'{ 1 + 1 }`, the semantics of `Expr(1 + 1)` are quite different. +`Expr(1 + 1)` will not stage or delay any computation; the argument is evaluated to a value and then lifted into a quote. +The quote will contain code that will create a copy of this value in the next stage. +`Expr` is polymorphic and user-extensible via the `ToExpr` type class. -## From `Expr`s to Functions and Back +```scala +trait ToExpr[T]: + def apply(x: T)(using Quotes): Expr[T] +``` -It is possible to convert any `Expr[T => R]` into `Expr[T] => Expr[R]` and back. -These conversions can be implemented as follows: +We can implement a `ToExpr` using a `given` definition that will add the definition to the implicits in scope. +In the following example we show how to implement a `ToExpr[Option[T]]` for any liftable type `T. ```scala -def to[T: Type, R: Type](f: Expr[T] => Expr[R])(using Quotes): Expr[T => R] = - '{ (x: T) => ${ f('x) } } - -def from[T: Type, R: Type](f: Expr[T => R])(using Quotes): Expr[T] => Expr[R] = - (x: Expr[T]) => '{ $f($x) } +given OptionToExpr[T: Type: ToExpr]: ToExpr[Option[T]] with + def apply(opt: Option[T])(using Quotes): Expr[Option[T]] = + opt match + case Some(x) => '{ Some[T]( ${Expr(x)} ) } + case None => '{ None } ``` -Note how the fundamental phase consistency principle works in two -different directions here for `f` and `x`. In the method `to`, the reference to `f` is -legal because it is quoted, then spliced, whereas the reference to `x` -is legal because it is spliced, then quoted. +The `ToExpr` for primitive types must be implemented as primitive operations in the system. +In our case, we use the reflection API to implement them. -They can be used as follows: +#### Extracting values from quotes +To be able to generate optimized code using the method `unrolledPowerCode`, the macro implementation `powerCode` needs to first +determine whether the argument passed as parameter `n` is a known constant value. +This can be achieved via _unlifting_ using the `Expr.unapply` extractor from our library implementation, which will only match if `n` is a quoted constant and extracts its value. ```scala -val f1: Expr[Int => String] = - to((x: Expr[Int]) => '{ $x.toString }) // '{ (x: Int) => x.toString } - -val f2: Expr[Int] => Expr[String] = - from('{ (x: Int) => x.toString }) // (x: Expr[Int]) => '{ ((x: Int) => x.toString)($x) } -f2('{2}) // '{ ((x: Int) => x.toString)(2) } +def powerCode(x: Expr[Double], n: Expr[Int])(using Quotes): Expr[Double] = + n match + case Expr(m) => // it is a constant: unlift code n='{m} into number m + unrolledPowerCode(x, m) + case _ => // not known: call power at run-time + '{ power($x, $n) } ``` -One limitation of `from` is that it does not β-reduce when a lambda is called immediately, as evidenced in the code `{ ((x: Int) => x.toString)(2) }`. -In some cases we want to remove the lambda from the code, for this we provide the method `Expr.betaReduce` that turns a tree -describing a function into a function mapping trees to trees. - +Alternatively, the `n.value` method can be used to get an `Option[Int]` with the value or `n.valueOrAbort` to get the value directly. ```scala -object Expr: - ... - def betaReduce[T](expr: Expr[T])(using Quotes): Expr[T] +def powerCode(x: Expr[Double], n: Expr[Int])(using Quotes): Expr[Double] = + // emits an error message if `n` is not a constant + unrolledPowerCode(x, n.valueOrAbort) ``` -`Expr.betaReduce` returns an expression that is functionally equivalent to e, however if e is of the form `((y1, ..., yn) => e2)(e1, ..., en)` then it optimizes the top most call by returning the result of beta-reducing the application. Otherwise returns expr. - -## Lifting Types - -Types are not directly affected by the phase consistency principle. -It is possible to use types defined at any level in any other level. -But, if a type is used in a subsequent stage it will need to be lifted to a `Type`. -Indeed, the definition of `to` above uses `T` in the next stage, there is a -quote but no splice between the parameter binding of `T` and its -usage. But the code can be rewritten by adding an explicit binding of a `Type[T]`: +`Expr.unapply` and all variants of `value` are polymorphic and user-extensible via a given `FromExpr` type class. ```scala -def to[T, R](f: Expr[T] => Expr[R])(using t: Type[T], r: Type[R])(using Quotes): Expr[T => R] = - '{ (x: t.Underlying) => ${ f('x) } } +trait FromExpr[T]: + def unapply(x: Expr[T])(using Quotes): Option[T] ``` -In this version of `to`, the type of `x` is now the result of -inserting the type `Type[T]` and selecting its `Underlying`. +We can use `given` definitions to implement the `FromExpr` as we did for `ToExpr`. +The `FromExpr` for primitive types must be implemented as primitive operations in the system. +In our case, we use the reflection API to implement them. +To implement `FromExpr` for non-primitive types we use quote pattern matching (for example `OptionFromExpr`). + + +## Macros and Multi-Stage Programming -To avoid clutter, the compiler converts any type reference to -a type `T` in subsequent phases to `summon[Type[T]].Underlying`. +The system supports multi-stage macros and run-time multi-stage programming using the same quotation abstractions. -And to avoid duplication it does it once per type, and creates -an alias for that type at the start of the quote. +### Multi-Stage Macros -For instance, the user-level definition of `to`: +#### Macros +We can generalize the splicing abstraction to express macros. +A macro consists of a top-level splice that is not nested in any quote. +Conceptually, the contents of the splice are evaluated one stage earlier than the program. +In other words, the contents are evaluated while compiling the program. The generated code resulting from the macro replaces the splice in the program. ```scala -def to[T, R](f: Expr[T] => Expr[R])(using t: Type[T], r: Type[R])(using Quotes): Expr[T => R] = - '{ (x: T) => ${ f('x) } } +def power2(x: Double): Double = + ${ unrolledPowerCode('x, 2) } // x * x ``` -would be rewritten to +#### Inline macros +Since using the splices in the middle of a program is not as ergonomic as calling a function; we hide the staging mechanism from end-users of macros. We have a uniform way of calling macros and normal functions. +For this, _we restrict the use of top-level splices to only appear in inline methods_[^1][^2]. ```scala -def to[T, R](f: Expr[T] => Expr[R])(using t: Type[T], r: Type[R])(using Quotes): Expr[T => R] = - '{ - type T = summon[Type[T]].Underlying - (x: T) => ${ f('x) } - } +// inline macro definition +inline def powerMacro(x: Double, inline n: Int): Double = + ${ powerCode('x, 'n) } + +// user code +def power2(x: Double): Double = + powerMacro(x, 2) // x * x ``` -The `summon` query succeeds because there is a using parameter of -type `Type[T]`, and the reference to that value is -phase-correct. If that was not the case, the phase inconsistency for -`T` would be reported as an error. +The evaluation of the macro will only happen when the code is inlined into `power2`. +When inlined, the code is equivalent to the previous definition of `power2`. +A consequence of using inline methods is that none of the arguments nor the return type of the macro will have to mention the `Expr` types; this hides all aspects of metaprogramming from the end-users. -## Lifting Expressions +#### Avoiding a complete interpreter +When evaluating a top-level splice, the compiler needs to interpret the code that is within the splice. +Providing an interpreter for the entire language is quite tricky, and it is even more challenging to make that interpreter run efficiently. +To avoid needing a complete interpreter, we can impose the following restrictions on splices to simplify the evaluation of the code in top-level splices. + * The top-level splice must contain a single call to a compiled static method. + * Arguments to the function are literal constants, quoted expressions (parameters), calls to `Type.of` for type parameters and a reference to `Quotes`. -Consider the following implementation of a staged interpreter that implements -a compiler through staging. +In particular, these restrictions disallow the use of splices in top-level splices. +Such a splice would require several stages of interpretation which would be unnecessarily inefficient. +#### Compilation stages +The macro implementation (i.e., the method called in the top-level splice) can come from any pre-compiled library. +This provides a clear difference between the stages of the compilation process. +Consider the following 3 source files defined in distinct libraries. ```scala -import scala.quoted.* - -enum Exp: - case Num(n: Int) - case Plus(e1: Exp, e2: Exp) - case Var(x: String) - case Let(x: String, e: Exp, in: Exp) - -import Exp.* +// Macro.scala +def powerCode(x: Expr[Double], n: Expr[Int])(using Quotes): Expr[Double] = ... +inline def powerMacro(x: Double, inline n: Int): Double = + ${ powerCode('x, 'n) } ``` -The interpreted language consists of numbers `Num`, addition `Plus`, and variables -`Var` which are bound by `Let`. Here are two sample expressions in the language: - ```scala -val exp = Plus(Plus(Num(2), Var("x")), Num(4)) -val letExp = Let("x", Num(3), exp) +// Lib.scala (depends on Macro.scala) +def power2(x: Double) = + ${ powerCode('x, '{2}) } // inlined from a call to: powerMacro(x, 2) ``` -Here’s a compiler that maps an expression given in the interpreted -language to quoted Scala code of type `Expr[Int]`. -The compiler takes an environment that maps variable names to Scala `Expr`s. - ```scala -import scala.quoted.* - -def compile(e: Exp, env: Map[String, Expr[Int]])(using Quotes): Expr[Int] = - e match - case Num(n) => - Expr(n) - case Plus(e1, e2) => - '{ ${ compile(e1, env) } + ${ compile(e2, env) } } - case Var(x) => - env(x) - case Let(x, e, body) => - '{ val y = ${ compile(e, env) }; ${ compile(body, env + (x -> 'y)) } } +// App.scala (depends on Lib.scala) +@main def app() = power2(3.14) ``` - -Running `compile(letExp, Map())` would yield the following Scala code: +One way to syntactically visualize this is to put the application in a quote that delays the compilation of the application. +Then the application dependencies can be placed in an outer quote that contains the quoted application, and we repeat this recursively for dependencies of dependencies. ```scala -'{ val y = 3; (2 + y) + 4 } +'{ // macro library (compilation stage 1) + def powerCode(x: Expr[Double], n: Expr[Int])(using Quotes): Expr[Double] = + ... + inline def powerMacro(x: Double, inline n: Int): Double = + ${ powerCode('x, 'n) } + '{ // library using macros (compilation stage 2) + def power2(x: Double) = + ${ powerCode('x, '{2}) } // inlined from a call to: powerMacro(x, 2) + '{ power2(3.14) /* app (compilation stage 3) */ } + } +} ``` -The body of the first clause, `case Num(n) => Expr(n)`, looks suspicious. `n` -is declared as an `Int`, yet it is converted to an `Expr[Int]` with `Expr()`. -Shouldn’t `n` be quoted? In fact this would not -work since replacing `n` by `'n` in the clause would not be phase -correct. +To make the system more versatile, we allow calling macros in the project where it is defined, with some restrictions. +For example, to compile `Macro.scala` and `Lib.scala` together in the same library. +To this end, we do not follow the simpler syntactic model and rely on semantic information from the source files. +When compiling a source, if we detect a call to a macro that is not compiled yet, we delay the compilation of this source to the following compilation stage. +In the example, we would delay the compilation of `Lib.scala` because it contains a compile-time call to `powerCode`. +Compilation stages are repeated until all sources are compiled, or no progress can be made. +If no progress is made, there was a cyclic dependency between the definition and the use of the macro. +We also need to detect if at runtime the macro depends on sources that have not been compiled yet. +These are detected by executing the macro and checking for JVM linking errors to classes that have not been compiled yet. -The `Expr.apply` method is defined in package `quoted`: +### Run-Time Multi-Stage Programming -```scala -package quoted +See [Run-Time Multi-Stage Programming](./staging.md) -object Expr: - ... - def apply[T: ToExpr](x: T)(using Quotes): Expr[T] = - summon[ToExpr[T]].toExpr(x) -``` +## Safety -This method says that values of types implementing the `ToExpr` type class can be -converted to `Expr` values using `Expr.apply`. +Multi-stage programming is by design statically safe and cross-stage safe. -Scala 3 comes with given instances of `ToExpr` for -several types including `Boolean`, `String`, and all primitive number -types. For example, `Int` values can be converted to `Expr[Int]` -values by wrapping the value in a `Literal` tree node. This makes use -of the underlying tree representation in the compiler for -efficiency. But the `ToExpr` instances are nevertheless not _magic_ -in the sense that they could all be defined in a user program without -knowing anything about the representation of `Expr` trees. For -instance, here is a possible instance of `ToExpr[Boolean]`: +### Static Safety -```scala -given ToExpr[Boolean] with - def toExpr(b: Boolean) = - if b then '{ true } else '{ false } -``` +#### Hygiene +All identifier names are interpreted as symbolic references to the corresponding variable in the context of the quote. +Therefore, while evaluating the quote, it is not possible to accidentally rebind a reference to a new variable with the same textual name. -Once we can lift bits, we can work our way up. For instance, here is a -possible implementation of `ToExpr[Int]` that does not use the underlying -tree machinery: +#### Well-typed +If a quote is well typed, then the generated code is well typed. +This is a simple consequence of tracking the type of each expression. +An `Expr[T]` can only be created from a quote that contains an expression of type `T`. +Conversely, an `Expr[T]` can only be spliced in a location that expects a type `T. +As mentioned before, `Expr` is covariant in its type parameter. +This means that an `Expr[T]` can contain an expression of a subtype of `T`. +When spliced in a location that expects a type `T, these expressions also have a valid type. -```scala -given ToExpr[Int] with - def toExpr(n: Int) = n match - case Int.MinValue => '{ Int.MinValue } - case _ if n < 0 => '{ - ${ toExpr(-n) } } - case 0 => '{ 0 } - case _ if n % 2 == 0 => '{ ${ toExpr(n / 2) } * 2 } - case _ => '{ ${ toExpr(n / 2) } * 2 + 1 } -``` +### Cross-Stage Safety + +#### Level consistency +We define the _staging level_ of some code as the number of quotes minus the number of splices surrounding said code. +Local variables must be defined and used in the same staging level. -Since `ToExpr` is a type class, its instances can be conditional. For example, -a `List` is liftable if its element type is: +It is never possible to access a local variable from a lower staging level as it does not yet exist. ```scala -given [T: ToExpr : Type]: ToExpr[List[T]] with - def toExpr(xs: List[T]) = xs match - case head :: tail => '{ ${ Expr(head) } :: ${ toExpr(tail) } } - case Nil => '{ Nil: List[T] } +def badPower(x: Double, n: Int): Double = + ${ unrolledPowerCode('x, n) } // error: value of `n` not known yet ``` -In the end, `ToExpr` resembles very much a serialization -framework. Like the latter it can be derived systematically for all -collections, case classes and enums. Note also that the synthesis -of _type-tag_ values of type `Type[T]` is essentially the type-level -analogue of lifting. -Using lifting, we can now give the missing definition of `showExpr` in the introductory example: +In the context of macros and _cross-platform portability_, that is, +macros compiled on one machine but potentially executed on another, +we cannot support cross-stage persistence of local variables. +Therefore, local variables can only be accessed at precisely the same staging level in our system. ```scala -def showExpr[T](expr: Expr[T])(using Quotes): Expr[String] = - val code: String = expr.show - Expr(code) +def badPowerCode(x: Expr[Double], n: Int)(using Quotes): Expr[Double] = + // error: `n` potentially not available in the next execution environment + '{ power($x, n) } ``` -That is, the `showExpr` method converts its `Expr` argument to a string (`code`), and lifts -the result back to an `Expr[String]` using `Expr.apply`. -## Lifting Types +The rules are slightly different for global definitions, such as `unrolledPowerCode`. +It is possible to generate code that contains a reference to a _global_ definition such as in `'{ power(2, 4) }`. +This is a limited form of cross-stage persistence that does not impede cross-platform portability, where we refer to the already compiled code for `power`. +Each compilation step will lower the staging level by one while keeping global definitions. +In consequence, we can refer to compiled definitions in macros such as `unrolledPowerCode` in `${ unrolledPowerCode('x, 2) }`. -The previous section has shown that the metaprogramming framework has -to be able to take a type `T` and convert it to a type tree of type -`Type[T]` that can be reified. This means that all free variables of -the type tree refer to types and values defined in the current stage. +We can sumarize level consistency in two rules: + * Local variables can be used only at the same staging level as their definition + * Global variables can be used at any staging level -For a reference to a global class, this is easy: Just issue the fully -qualified name of the class. Members of reifiable types are handled by -just reifying the containing type together with the member name. But -what to do for references to type parameters or local type definitions -that are not defined in the current stage? Here, we cannot construct -the `Type[T]` tree directly, so we need to get it from a recursive -implicit search. For instance, to implement +#### Type consistency +As Scala uses type erasure, generic types will be erased at run-time and hence in any following stage. +To ensure any quoted expression that refers to a generic type `T` does not lose the information it needs, we require a given `Type[T]` in scope. +The `Type[T]` will carry over the non-erased representation of the type into the next phase. +Therefore any generic type used at a higher staging level than its definition will require its `Type`. + +#### Scope extrusion +Within the contents of a splice, it is possible to have a quote that refers to a local variable defined in the outer quote. +If this quote is used within the splice, the variable will be in scope. +However, if the quote is somehow _extruded_ outside the splice, then variables might not be in scope anymore. +Quoted expressions can be extruded using side effects such as mutable state and exceptions. +The following example shows how a quote can be extruded using mutable state. ```scala -summon[Type[List[T]]] +var x: Expr[T] = null +'{ (y: T) => ${ x = 'y; 1 } } +x // has value '{y} but y is not in scope ``` -where `T` is not defined in the current stage, we construct the type constructor -of `List` applied to the splice of the result of searching for a given instance for `Type[T]`: +A second way a variable can be extruded is through the `run` method. +If `run` consumes a quoted variable reference, it will not be in scope anymore. +The result will reference a variable that is defined in the next stage. ```scala -Type.of[ List[ summon[Type[T]].Underlying ] ] +'{ (x: Int) => ${ run('x); ... } } +// evaluates to: '{ (x: Int) => ${ x; ... } 1 ``` -This is exactly the algorithm that Scala 2 uses to search for type tags. -In fact Scala 2's type tag feature can be understood as a more ad-hoc version of -`quoted.Type`. As was the case for type tags, the implicit search for a `quoted.Type` -is handled by the compiler, using the algorithm sketched above. +To catch both scope extrusion scenarios, our system restricts the use of quotes by only allowing a quote to be spliced if it was not extruded from a splice scope. +Unlike level consistency, this is checked at run-time[^4] rather than compile-time to avoid making the static type system too complicated. -## Relationship with `inline` +Each `Quotes` instance contains a unique scope identifier and refers to its parent scope, forming a stack of identifiers. +The parent of the scope of a `Quotes` is the scope of the `Quotes` used to create the enclosing quote. +Top-level splices and `run` create new scope stacks. +Every `Expr` knows in which scope it was created. +When it is spliced, we check that the quote scope is either the same as the splice scope, or a parent scope thereof. -Seen by itself, principled metaprogramming looks more like a framework for -runtime metaprogramming than one for compile-time metaprogramming with macros. -But combined with Scala 3’s `inline` feature it can be turned into a compile-time -system. The idea is that macro elaboration can be understood as a combination of -a macro library and a quoted program. For instance, here’s the `assert` macro -again together with a program that calls `assert`. -```scala -object Macros: +## Staged Lambdas - inline def assert(inline expr: Boolean): Unit = - ${ assertImpl('expr) } +When staging programs in a functional language there are two fundamental abstractions: a staged lambda `Expr[T => U]` and a staging lambda `Expr[T] => Expr[U]`. +The first is a function that will exist in the next stage, whereas the second is a function that exists in the current stage. +It is often convenient to have a mechanism to go from `Expr[T => U]` to `Expr[T] => Expr[U]` and vice versa. - def assertImpl(expr: Expr[Boolean])(using Quotes) = - val failMsg: Expr[String] = Expr("failed assertion: " + expr.show) - '{ if !($expr) then throw new AssertionError($failMsg) } +```scala +def later[T: Type, U: Type](f: Expr[T] => Expr[U]): Expr[T => U] = + '{ (x: T) => ${ f('x) } } -@main def program = - val x = 1 - Macros.assert(x != 0) +def now[T: Type, U: Type](f: Expr[T => U]): Expr[T] => Expr[U] = + (x: Expr[T]) => '{ $f($x) } ``` -Inlining the `assert` function would give the following program: +Both conversions can be performed out of the box with quotes and splices. +But if `f` is a known lambda function, `'{ $f($x) }` will not beta-reduce the lambda in place. +This optimization is performed in a later phase of the compiler. +Not reducing the application immediately can simplify analysis of generated code. +Nevertheless, it is possible to beta-reduce the lambda in place using the `Expr.betaReduce` method. ```scala -@main def program = - val x = 1 - ${ Macros.assertImpl('{ x != 0}) } +def now[T: Type, U: Type](f: Expr[T => U]): Expr[T] => Expr[U] = + (x: Expr[T]) => Expr.betaReduce('{ $f($x) }) ``` -The example is only phase correct because `Macros` is a global value and -as such not subject to phase consistency checking. Conceptually that’s -a bit unsatisfactory. If the PCP is so fundamental, it should be -applicable without the global value exception. But in the example as -given this does not hold since both `assert` and `program` call -`assertImpl` with a splice but no quote. +The `betaReduce` method will beta-reduce the outermost application of the expression if possible (regardless of arity). +If it is not possible to beta-reduce the expression, then it will return the original expression. -However, one could argue that the example is really missing -an important aspect: The macro library has to be compiled in a phase -prior to the program using it, but in the code above, macro -and program are defined together. A more accurate view of -macros would be to have the user program be in a phase after the macro -definitions, reflecting the fact that macros have to be defined and -compiled before they are used. Hence, conceptually the program part -should be treated by the compiler as if it was quoted: +## Staged Constructors +To create new class instances in a later stage, we can create them using factory methods (usually `apply` methods of an `object`), or we can instantiate them with a `new`. +For example, we can write `Some(1)` or `new Some(1)`, creating the same value. +In Scala 3, using the factory method call notation will fall back to a `new` if no `apply` method is found. +We follow the usual staging rules when calling a factory method. +Similarly, when we use a `new C`, the constructor of `C` is implicitly called, which also follows the usual staging rules. +Therefore for an arbitrary known class `C`, we can use both `'{ C(...) }` or `'{ new C(...) }` as constructors. +## Staged Classes +Quoted code can contain any valid expression including local class definitions. +This allows the creation of new classes with specialized implementations. +For example, we can implement a new version of `Runnable` that will perform some optimized operation. ```scala -@main def program = '{ - val x = 1 - ${ Macros.assertImpl('{ x != 0 }) } +def mkRunnable(x: Int)(using Quotes): Expr[Runnable] = '{ + class MyRunnable extends Runnable: + def run(): Unit = ... // generate some custom code that uses `x` + new MyRunnable } ``` -If `program` is treated as a quoted expression, the call to -`Macro.assertImpl` becomes phase correct even if macro library and -program are conceptualized as local definitions. +The quoted class is a local class and its type cannot escape the enclosing quote. +The class must be used inside the quote or an instance of it can be returned using a known interface (`Runnable` in this case). -But what about the call from `assert` to `assertImpl`? Here, we need a -tweak of the typing rules. An inline function such as `assert` that -contains a splice operation outside an enclosing quote is called a -_macro_. Macros are supposed to be expanded in a subsequent phase, -i.e. in a quoted context. Therefore, they are also type checked as if -they were in a quoted context. For instance, the definition of -`assert` is typechecked as if it appeared inside quotes. This makes -the call from `assert` to `assertImpl` phase-correct, even if we -assume that both definitions are local. +## Quote Pattern Matching -The `inline` modifier is used to declare a `val` that is -either a constant or is a parameter that will be a constant when instantiated. This -aspect is also important for macro expansion. - -To get values out of expressions containing constants `Expr` provides the method -`value` (or `valueOrError`). This will convert the `Expr[T]` into a `Some[T]` (or `T`) when the -expression contains value. Otherwise it will return `None` (or emit an error). -To avoid having incidental val bindings generated by the inlining of the `def` -it is recommended to use an inline parameter. To illustrate this, consider an -implementation of the `power` function that makes use of a statically known exponent: +It is sometimes necessary to analyze the structure of the code or decompose the code into its sub-expressions. +A classic example is an embedded DSL, where a macro knows a set of definitions that it can reinterpret while compiling the code (for instance, to perform optimizations). +In the following example, we extend our previous implementation of `powCode` to look into `x` to perform further optimizations. ```scala -inline def power(x: Double, inline n: Int) = ${ powerCode('x, 'n) } - -private def powerCode(x: Expr[Double], n: Expr[Int])(using Quotes): Expr[Double] = - n.value match - case Some(m) => powerCode(x, m) - case None => '{ Math.pow($x, $n.toDouble) } - -private def powerCode(x: Expr[Double], n: Int)(using Quotes): Expr[Double] = - if n == 0 then '{ 1.0 } - else if n == 1 then x - else if n % 2 == 0 then '{ val y = $x * $x; ${ powerCode('y, n / 2) } } - else '{ $x * ${ powerCode(x, n - 1) } } +def fusedPowCode(x: Expr[Double], n: Expr[Int])(using Quotes): Expr[Double] = + x match + case '{ power($y, $m) } => // we have (y^m)^n + fusedPowCode(y, '{ $n * $m }) // generate code for y^(n*m) + case _ => + '{ power($x, $n) } ``` -## Scope Extrusion -Quotes and splices are duals as far as the PCP is concerned. But there is an -additional restriction that needs to be imposed on splices to guarantee -soundness: code in splices must be free of side effects. The restriction -prevents code like this: +#### Sub-patterns -```scala -var x: Expr[T] = ... -'{ (y: T) => ${ x = 'y; 1 } } -``` - -This code, if it was accepted, would _extrude_ a reference to a quoted variable -`y` from its scope. This would subsequently allow access to a variable outside the -scope where it is defined, which is likely problematic. The code is clearly -phase consistent, so we cannot use PCP to rule it out. Instead, we postulate a -future effect system that can guarantee that splices are pure. In the absence of -such a system we simply demand that spliced expressions are pure by convention, -and allow for undefined compiler behavior if they are not. This is analogous to -the status of pattern guards in Scala, which are also required, but not -verified, to be pure. - -[Multi-Stage Programming](./staging.md) introduces one additional method where -you can expand code at runtime with a method `run`. There is also a problem with -that invocation of `run` in splices. Consider the following expression: +In quoted patterns, the `$` binds the sub-expression to an expression `Expr` that can be used in that `case` branch. +The contents of `${..}` in a quote pattern are regular Scala patterns. +For example, we can use the `Expr(_)` pattern within the `${..}` to only match if it is a known value and extract it. ```scala -'{ (x: Int) => ${ run('x); 1 } } +def fusedUnrolledPowCode(x: Expr[Double], n: Int)(using Quotes): Expr[Double] = + x match + case '{ power($y, ${Expr(m)}) } => // we have (y^m)^n + fusedUnrolledPowCode(y, n * m) // generate code for y * ... * y + case _ => // ( n*m times ) + unrolledPowerCode(x, n) ``` -This is again phase correct, but will lead us into trouble. Indeed, evaluating -the splice will reduce the expression `run('x)` to `x`. But then the result +These value extraction sub-patterns can be polymorphic using an instance of `FromExpr`. +In the following example, we show the implementation of `OptionFromExpr` which internally uses the `FromExpr[T]` to extract the value using the `Expr(x)` pattern. ```scala -'{ (x: Int) => ${ x; 1 } } +given OptionFromExpr[T](using Type[T], FromExpr[T]): FromExpr[Option[T]] with + def unapply(x: Expr[Option[T]])(using Quotes): Option[Option[T]] = + x match + case '{ Some( ${Expr(x)} ) } => Some(Some(x)) + case '{ None } => Some(None) + case _ => None ``` -is no longer phase correct. To prevent this soundness hole it seems easiest to -classify `run` as a side-effecting operation. It would thus be prevented from -appearing in splices. In a base language with side effects we would have to do this -anyway: Since `run` runs arbitrary code it can always produce a side effect if -the code it runs produces one. -## Example Expansion -Assume we have two methods, `foreach` that takes an `Expr[Array[T]]` and a -consumer `f`, and `sum` that performs a sum by delegating to `foreach`. +#### Closed patterns +Patterns may contain two kinds of references: global references such as the call to the `power` method in `'{ power(...) }`, or references to bindings defined in the pattern such as `x` in `case '{ (x: Int) => x }`. +When extracting an expression from a quote, we need to ensure that we do not extrude any variable from the scope where it is defined. ```scala -object Macros: - - def foreach[T](arr: Expr[Array[T]], f: Expr[T] => Expr[Unit]) - (using Type[T], Quotes): Expr[Unit] = '{ - var i: Int = 0 - while i < ($arr).length do - val element: T = ($arr)(i) - ${f('element)} - i += 1 - } - - def sum(arr: Expr[Array[Int]])(using Quotes): Expr[Int] = '{ - var sum = 0 - ${ foreach(arr, x => '{sum += $x}) } - sum - } - - inline def sum_m(arr: Array[Int]): Int = ${sum('arr)} - -end Macros +'{ (x: Int) => x + 1 } match + case '{ (y: Int) => $z } => + // should not match, otherwise: z = '{ x + 1 } ``` -A call to `sum_m(Array(1, 2, 3))` will first inline `sum_m`: - -```scala -val arr: Array[Int] = Array.apply(1, 2, 3) -${ _root_.Macros.sum('arr) } -``` +In this example, we see that the pattern should not match. +Otherwise, any use of the expression `z` would contain an unbound reference to `x`. +To avoid any such extrusion, we only match on a `${..}` if its expression is closed under the definitions within the pattern. +Therefore, the pattern will not match if the expression is not closed. -then it will call `sum`: +#### HOAS patterns +To allow extracting expressions that may contain extruded references we offer a _higher-order abstract syntax_ (HOAS) pattern `$f(y)` (or `$f(y1,...,yn)`). +This pattern will eta-expand the sub-expression with respect to `y` and bind it to `f`. +The lambda arguments will replace the variables that might have been extruded. ```scala -val arr: Array[Int] = Array.apply(1, 2, 3) -${ '{ - var sum = 0 - ${ foreach('arr, x => '{sum += $x}) } - sum -} } +'{ ((x: Int) => x + 1).apply(2) } match + case '{ ((y: Int) => $f(y)).apply($z: Int) } => + // f may contain references to `x` (replaced by `$y`) + // f = (y: Expr[Int]) => '{ $y + 1 } + f(z) // generates '{ 2 + 1 } ``` -and cancel the `${'{...}}`: - -```scala -val arr: Array[Int] = Array.apply(1, 2, 3) - -var sum = 0 -${ foreach('arr, x => '{sum += $x}) } -sum -``` -then it will extract `x => '{sum += $x}` into `f`, to have a value: +A HOAS pattern `$x(y1,...,yn)` will only match the expression if it does not contain references to variables defined in the pattern that are not in the set `y1,...,yn`. +In other words, the pattern will match if the expression only contains references to variables defined in the pattern that are in `y1,...,yn`. +Note that the HOAS patterns `$x()` are semantically equivalent to closed patterns `$x`. -```scala -val arr: Array[Int] = Array.apply(1, 2, 3) -var sum = 0 -val f = x => '{sum += $x} -${ _root_.Macros.foreach('arr, 'f)(Type.of[Int]) } -sum -``` +#### Type variables -and then call `foreach`: +Expressions may contain types that are not statically known. +For example, an `Expr[List[Int]]` may contain `list.map(_.toInt)` where `list` is a `List` of some type. +To cover all the possible cases we would need to explicitly match `list` on all possible types (`List[Int]`, `List[Int => Int]`, ...). +This is an infinite set of types and therefore pattern cases. +Even if we would know all possible types that a specific program could use, we may still end up with an unmanageable number of cases. +To overcome this, we introduce type variables in quoted patterns, which will match any type. +In the following example, we show how type variables `t` and `u` match all possible pairs of consecutive calls to `map` on lists. +In the quoted patterns, types named with lower cases are identified as type variables. +This follows the same notation as type variables used in normal patterns. ```scala -val arr: Array[Int] = Array.apply(1, 2, 3) +def fuseMapCode(x: Expr[List[Int]]): Expr[List[Int]] = + x match + case '{ ($ls: List[t]).map[u]($f).map[Int]($g) } => + '{ $ls.map($g.compose($f)) } + ... -var sum = 0 -val f = x => '{sum += $x} -${ '{ - var i: Int = 0 - while i < arr.length do - val element: Int = (arr)(i) - sum += element - i += 1 - sum -} } +fuseMapCode('{ List(1.2).map(f).map(g) }) // '{ List(1.2).map(g.compose(f)) } +fuseMapCode('{ List('a').map(h).map(i) }) // '{ List('a').map(i.compose(h)) } ``` +Variables `f` and `g` are inferred to be of type `Expr[t => u]` and `Expr[u => Int]` respectively. +Subsequently, we can infer `$g.compose($f)` to be of type `Expr[t => Int]` which is the type of the argument of `$ls.map(..)`. -and cancel the `${'{...}}` again: +Type variables are abstract types that will be erased; this implies that to reference them in the second quote we need a given `Type[t]` and `Type[u]`. +The quoted pattern will implicitly provide those given types. +At run-time, when the pattern matches, the type of `t` and `u` will be known, and the `Type[t]` and `Type[u]` will contain the precise types in the expression. +As `Expr` is covariant, the statically known type of the expression might not be the actual type. +Type variables can also be used to recover the precise type of the expression. ```scala -val arr: Array[Int] = Array.apply(1, 2, 3) +def let(x: Expr[Any])(using Quotes): Expr[Any] = + x match + case '{ $x: t } => + '{ val y: t = $x; y } -var sum = 0 -val f = x => '{sum += $x} -var i: Int = 0 -while i < arr.length do - val element: Int = (arr)(i) - sum += element - i += 1 -sum +let('{1}) // will return a `Expr[Any]` that contains an `Expr[Int]]` ``` -Finally cleanups and dead code elimination: - +While we can define the type variable in the middle of the pattern, their normal form is to define them as a `type` with a lower case name at the start of the pattern. +We use the Scala backquote `` `t` `` naming convention which interprets the string within the backquote as a literal name identifier. +This is typically used when we have names that contain special characters that are not allowed for normal Scala identifiers. +But we use it to explicitly state that this is a reference to that name and not the introduction of a new variable. ```scala -val arr: Array[Int] = Array.apply(1, 2, 3) -var sum = 0 -var i: Int = 0 -while i < arr.length do - val element: Int = arr(i) - sum += element - i += 1 -sum + case '{ type t; $x: `t` } => ``` - -## Find implicits within a macro - -Similarly to the `summonFrom` construct, it is possible to make implicit search available -in a quote context. For this we simply provide `scala.quoted.Expr.summon`: +This is a bit more verbose but has some expressivity advantages such as allowing to define bounds on the variables and be able to refer to them several times in any scope of the pattern. ```scala -import scala.collection.immutable.{ TreeSet, HashSet } -inline def setFor[T]: Set[T] = ${ setForExpr[T] } - -def setForExpr[T: Type](using Quotes): Expr[Set[T]] = - Expr.summon[Ordering[T]] match - case Some(ord) => '{ new TreeSet[T]()($ord) } - case _ => '{ new HashSet[T] } + case '{ type t >: List[Int] <: Seq[Int]; $x: `t` } => + case '{ type t; $x: (`t`, `t`) } => ``` -## Relationship with Transparent Inline -[Inline](./inline.md) documents inlining. The code below introduces a transparent -inline method that can calculate either a value of type `Int` or a value of type -`String`. +#### Type patterns +It is possible to only have a type and no expression of that type. +To be able to inspect a type, we introduce quoted type pattern `case '[..] =>`. +It works the same way as a quoted pattern but is restricted to contain a type. +Type variables can be used in quoted type patterns to extract a type. ```scala -transparent inline def defaultOf(inline str: String) = - ${ defaultOfImpl('str) } - -def defaultOfImpl(strExpr: Expr[String])(using Quotes): Expr[Any] = - strExpr.valueOrError match - case "int" => '{1} - case "string" => '{"a"} - -// in a separate file -val a: Int = defaultOf("int") -val b: String = defaultOf("string") - +def empty[T: Type]: Expr[T] = + Type.of[T] match + case '[String] => '{ "" } + case '[List[t]] => '{ List.empty[t] } + ... ``` -## Defining a macro and using it in a single project +`Type.of[T]` is used to summon the given instance of `Type[T]` in scope, it is equivalent to `summon[Type[T]]`. -It is possible to define macros and use them in the same project as long as the implementation -of the macros does not have run-time dependencies on code in the file where it is used. -It might still have compile-time dependencies on types and quoted code that refers to the use-site file. +#### Type testing and casting +It is important to note that instance checks and casts on `Expr`, such as `isInstanceOf[Expr[T]]` and `asInstanceOf[Expr[T]]`, will only check if the instance is of the class `Expr` but will not be able to check the `T` argument. +These cases will issue a warning at compile-time, but if they are ignored, they can result in unexpected behavior. -To provide this functionality Scala 3 provides a transparent compilation mode where files that -try to expand a macro but fail because the macro has not been compiled yet are suspended. -If there are any suspended files when the compilation ends, the compiler will automatically restart -compilation of the suspended files using the output of the previous (partial) compilation as macro classpath. -In case all files are suspended due to cyclic dependencies the compilation will fail with an error. +These operations can be supported correctly in the system. +For a simple type test it is possible to use the `isExprOf[T]` method of `Expr` to check if it is an instance of that type. +Similarly, it is possible to use `asExprOf[T]` to cast an expression to a given type. +These operations use a given `Type[T]` to work around type erasure. -## Pattern matching on quoted expressions -It is possible to deconstruct or extract values out of `Expr` using pattern matching. +## Sub-Expression Transformation -`scala.quoted` contains objects that can help extracting values from `Expr`. - -- `scala.quoted.Expr`/`scala.quoted.Exprs`: matches an expression of a value (resp. list of values) and returns the value (resp. list of values). -- `scala.quoted.Const`/`scala.quoted.Consts`: Same as `Expr`/`Exprs` but only works on primitive values. -- `scala.quoted.Varargs`: matches an explicit sequence of expressions and returns them. These sequences are useful to get individual `Expr[T]` out of a varargs expression of type `Expr[Seq[T]]`. - -These could be used in the following way to optimize any call to `sum` that has statically known values. +The system provides a mechanism to transform all sub-expressions of an expression. +This is useful when the sub-expressions we want to transform are deep in the expression. +It is also necessary if the expression contains sub-expressions that cannot be matched using quoted patterns (such as local class definitions). ```scala -inline def sum(inline args: Int*): Int = ${ sumExpr('args) } -private def sumExpr(argsExpr: Expr[Seq[Int]])(using Quotes): Expr[Int] = - argsExpr match - case Varargs(args @ Exprs(argValues)) => - // args is of type Seq[Expr[Int]] - // argValues is of type Seq[Int] - Expr(argValues.sum) // precompute result of sum - case Varargs(argExprs) => // argExprs is of type Seq[Expr[Int]] - val staticSum: Int = argExprs.map(_.value.getOrElse(0)).sum - val dynamicSum: Seq[Expr[Int]] = argExprs.filter(_.value.isEmpty) - dynamicSum.foldLeft(Expr(staticSum))((acc, arg) => '{ $acc + $arg }) - case _ => - '{ $argsExpr.sum } - -sum(1, 2, 3) // gets matched by Varargs - -val xs = List(1, 2, 3) -sum(xs*) // doesn't get matched by Varargs +trait ExprMap: + def transform[T](e: Expr[T])(using Type[T])(using Quotes): Expr[T] + def transformChildren[T](e: Expr[T])(using Type[T])(using Quotes): Expr[T] = + ... ``` -### Quoted patterns - -Quoted pattens allow deconstructing complex code that contains a precise structure, types or methods. -Patterns `'{ ... }` can be placed in any location where Scala expects a pattern. - -For example - -```scala -optimize { - sum(sum(1, a, 2), 3, b) -} // should be optimized to 6 + a + b -``` +Users can extend the `ExprMap` trait and implement the `transform` method. +This interface is flexible and can implement top-down, bottom-up, or other transformations. ```scala -def sum(args: Int*): Int = args.sum -inline def optimize(inline arg: Int): Int = ${ optimizeExpr('arg) } -private def optimizeExpr(body: Expr[Int])(using Quotes): Expr[Int] = - body match - // Match a call to sum without any arguments - case '{ sum() } => Expr(0) - // Match a call to sum with an argument $n of type Int. - // n will be the Expr[Int] representing the argument. - case '{ sum($n) } => n - // Match a call to sum and extracts all its args in an `Expr[Seq[Int]]` - case '{ sum(${Varargs(args)}: _*) } => sumExpr(args) - case body => body - -private def sumExpr(args1: Seq[Expr[Int]])(using Quotes): Expr[Int] = - def flatSumArgs(arg: Expr[Int]): Seq[Expr[Int]] = arg match - case '{ sum(${Varargs(subArgs)}: _*) } => subArgs.flatMap(flatSumArgs) - case arg => Seq(arg) - val args2 = args1.flatMap(flatSumArgs) - val staticSum: Int = args2.map(_.value.getOrElse(0)).sum - val dynamicSum: Seq[Expr[Int]] = args2.filter(_.value.isEmpty) - dynamicSum.foldLeft(Expr(staticSum))((acc, arg) => '{ $acc + $arg }) +object OptimizeIdentity extends ExprMap: + def transform[T](e: Expr[T])(using Type[T])(using Quotes): Expr[T] = + transformChildren(e) match // bottom-up transformation + case '{ identity($x) } => x + case _ => e ``` -### Recovering precise types using patterns +The `transformChildren` method is implemented as a primitive that knows how to reach all the direct sub-expressions and calls `transform` on each one. +The type passed to `transform` is the expected type of this sub-expression in its expression. +For example while transforming `Some(1)` in `'{ val x: Option[Int] = Some(1); ...}` the type will be `Option[Int]` and not `Some[Int]`. +This implies that we can safely transform `Some(1)` into `None`. -Sometimes it is necessary to get a more precise type for an expression. This can be achieved using the following pattern match. +## Staged Implicit Summoning +When summoning implicit arguments using `summon`, we will find the given instances in the current scope. +It is possible to use `summon` to get staged implicit arguments by explicitly staging them first. +In the following example, we can pass an implicit `Ordering[T]` in a macro as an `Expr[Ordering[T]]` to its implementation. +Then we can splice it and give it implicitly in the next stage. ```scala -def f(expr: Expr[Any])(using Quotes) = expr match - case '{ $x: t } => - // If the pattern match succeeds, then there is - // some type `t` such that - // - `x` is bound to a variable of type `Expr[t]` - // - `t` is bound to a new type `t` and a given - // instance `Type[t]` is provided for it - // That is, we have `x: Expr[t]` and `given Type[t]`, - // for some (unknown) type `t`. -``` - -This might be used to then perform an implicit search as in: - -```scala -extension (inline sc: StringContext) - inline def showMe(inline args: Any*): String = ${ showMeExpr('sc, 'args) } - -private def showMeExpr(sc: Expr[StringContext], argsExpr: Expr[Seq[Any]])(using Quotes): Expr[String] = - import quotes.reflect.report - argsExpr match - case Varargs(argExprs) => - val argShowedExprs = argExprs.map { - case '{ $arg: tp } => - Expr.summon[Show[tp]] match - case Some(showExpr) => - '{ $showExpr.show($arg) } - case None => - report.error(s"could not find implicit for ${Type.show[Show[tp]]}", arg); '{???} - } - val newArgsExpr = Varargs(argShowedExprs) - '{ $sc.s($newArgsExpr: _*) } - case _ => - // `new StringContext(...).showMeExpr(args: _*)` not an explicit `showMeExpr"..."` - report.error(s"Args must be explicit", argsExpr) - '{???} - -trait Show[-T]: - def show(x: T): String - -// in a different file -given Show[Boolean] with - def show(b: Boolean) = "boolean!" +inline def treeSetFor[T](using ord: Ordering[T]): Set[T] = + ${ setExpr[T](using 'ord) } -println(showMe"${true}") +def setExpr[T:Type](using ord: Expr[Ordering[T]])(using Quotes): Expr[Set[T]] = + '{ given Ordering[T] = $ord; new TreeSet[T]() } ``` -### Open code patterns +We pass it as an implicit `Expr[Ordering[T]]` because there might be intermediate methods that can pass it along implicitly. -Quoted pattern matching also provides higher-order patterns to match open terms. If a quoted term contains a definition, -then the rest of the quote can refer to this definition. +An alternative is to summon implicit values in the scope where the macro is invoked. +Using the `Expr.summon` method we get an optional expression containing the implicit instance. +This provides the ability to search for implicit instances conditionally. ```scala -'{ - val x: Int = 4 - x * x -} -``` - -To match such a term we need to match the definition and the rest of the code, but we need to explicitly state that the rest of the code may refer to this definition. - -```scala -case '{ val y: Int = $x; $body(y): Int } => +def summon[T: Type](using Quotes): Option[Expr[T]] ``` -Here `$x` will match any closed expression while `$body(y)` will match an expression that is closed under `y`. Then -the subexpression of type `Expr[Int]` is bound to `body` as an `Expr[Int => Int]`. The extra argument represents the references to `y`. Usually this expression is used in combination with `Expr.betaReduce` to replace the extra argument. - ```scala -inline def eval(inline e: Int): Int = ${ evalExpr('e) } +inline def setFor[T]: Set[T] = + ${ setForExpr[T] } -private def evalExpr(e: Expr[Int])(using Quotes): Expr[Int] = e match - case '{ val y: Int = $x; $body(y): Int } => - // body: Expr[Int => Int] where the argument represents - // references to y - evalExpr(Expr.betaReduce('{$body(${evalExpr(x)})})) - case '{ ($x: Int) * ($y: Int) } => - (x.value, y.value) match - case (Some(a), Some(b)) => Expr(a * b) - case _ => e - case _ => e +def setForExpr[T: Type]()(using Quotes): Expr[Set[T]] = + Expr.summon[Ordering[T]] match + case Some(ord) => + '{ new TreeSet[T]()($ord) } + case _ => + '{ new HashSet[T] } ``` -```scala -eval { // expands to the code: (16: Int) - val x: Int = 4 - x * x -} -``` +## More details -We can also close over several bindings using `$b(a1, a2, ..., an)`. -To match an actual application we can use braces on the function part `${b}(a1, a2, ..., an)`. +* [Specification](./macros-spec.md) +* Scalable Metaprogramming in Scala 3[^1] -## More details -[More details](./macros-spec.md) +[^1]: [Scalable Metaprogramming in Scala 3](https://infoscience.epfl.ch/record/299370) +[^2]: [Semantics-preserving inlining for metaprogramming](https://dl.acm.org/doi/10.1145/3426426.3428486) +[^3]: Implemented in the Scala 3 Dotty project https://github.com/lampepfl/dotty. sbt library dependency `"org.scala-lang" %% "scala3-staging" % scalaVersion.value` +[^4]: Using the `-Xcheck-macros` compiler flag diff --git a/docs/_docs/reference/metaprogramming/staging.md b/docs/_docs/reference/metaprogramming/staging.md index e74d491402b5..1c154e09f50e 100644 --- a/docs/_docs/reference/metaprogramming/staging.md +++ b/docs/_docs/reference/metaprogramming/staging.md @@ -1,6 +1,6 @@ --- layout: doc-page -title: "Runtime Multi-Stage Programming" +title: "Run-Time Multi-Stage Programming" nightlyOf: https://docs.scala-lang.org/scala3/reference/metaprogramming/staging.html --- @@ -60,7 +60,7 @@ impose the following restrictions on the use of splices. The framework as discussed so far allows code to be staged, i.e. be prepared to be executed at a later stage. To run that code, there is another method in class `Expr` called `run`. Note that `$` and `run` both map from `Expr[T]` -to `T` but only `$` is subject to the [PCP](./macros.md#the-phase-consistency-principle), whereas `run` is just a normal method. +to `T` but only `$` is subject to [Cross-Stage Safety](./macros.md#cross-stage-safety), whereas `run` is just a normal method. `scala.quoted.staging.run` provides a `Quotes` that can be used to show the expression in its scope. On the other hand `scala.quoted.staging.withQuotes` provides a `Quotes` without evaluating the expression. diff --git a/docs/_spec/TODOreference/metaprogramming/macros-spec.md b/docs/_spec/TODOreference/metaprogramming/macros-spec.md index aa8f94a9a1f7..6045354fdbbc 100644 --- a/docs/_spec/TODOreference/metaprogramming/macros-spec.md +++ b/docs/_spec/TODOreference/metaprogramming/macros-spec.md @@ -4,251 +4,711 @@ title: "Macros Spec" nightlyOf: https://docs.scala-lang.org/scala3/reference/metaprogramming/macros-spec.html --- +## Formalization + +* Multi-stage programming with generative and analytical macros[^2] +* Multi-Stage Macro Calculus, Chapter 4 of Scalable Metaprogramming in Scala 3[^1]. + Contains and extends the calculus of _Multi-stage programming with generative and analytical macros_ with type polymorphism. + +## Syntax + +The quotation syntax using `'` and `$` was chosen to mimic the string interpolation syntax of Scala. +Like a string double-quotation, a single-quote block can contain splices. +However, unlike strings, splices can contain quotes using the same rules. + +```scala +s" Hello $name" s" Hello ${name}" +'{ hello($name) } '{ hello(${name}) } +${ hello('name) } ${ hello('{name}) } +``` + +### Quotes +Quotes come in four flavors: quoted identifiers, quoted blocks, quoted block patterns and quoted type patterns. +Scala 2 used quoted identifiers to represent `Symbol` literals. They were deprecated in Scala 3, allowing to use them for quotation. +```scala +SimpleExpr ::= ... + | `'` alphaid // quoted identifier + | `'` `{` Block `}` // quoted block +Pattern ::= ... + | `'` `{` Block `}` // quoted block pattern + | `'` `[` Type `]` // quoted type pattern +``` + +Quoted blocks and quoted block patterns contain an expression equivalent to a normal block of code. +When entering either of those we track the fact that we are in a quoted block (`inQuoteBlock`) which is used for spliced identifiers. +When entering a quoted block pattern we additionally track the fact that we are in a quoted pattern (`inQuotePattern`) which is used to distinguish spliced blocks and splice patterns. +Lastly, the quoted type pattern simply contains a type. + +### Splices +Splices come in three flavors: spliced identifiers, spliced blocks and splice patterns. +Scala specifies identifiers containing `$` as valid identifiers but reserves them for compiler and standard library use only. +Unfortunately, many libraries have used such identifiers in Scala~2. Therefore to mitigate the cost of migration, we still support them. +We work around this by only allowing spliced identifiers[^3] within quoted blocks or quoted block patterns (`inQuoteBlock`). +Splice blocks and splice patterns can contain an arbitrary block or pattern respectively. +They are distinguished based on their surrounding quote (`inQuotePattern`), a quote block will contain spliced blocks, and a quote block pattern will contain splice patterns. + +```scala +SimpleExpr ::= ... + | `$` alphaid if inQuoteBlock // spliced identifier + | `$` `{` Block `}` if !inQuotePattern // spliced block + | `$` `{` Pattern `}` if inQuotePattern // splice pattern +``` + +### Quoted Pattern Type Variables +Quoted pattern type variables in quoted patterns and quoted type patterns do not require additional syntax. +Any type definition or reference with a name composed of lower cases is assumed to be a pattern type variable definition while typing. +A backticked type name with lower cases is interpreted as a reference to the type with that name. + + ## Implementation -### Syntax - -Compared to the [Scala 3 reference grammar](../syntax.md) -there are the following syntax changes: -``` -SimpleExpr ::= ... - | ‘'’ ‘{’ Block ‘}’ - | ‘'’ ‘[’ Type ‘]’ - | ‘$’ ‘{’ Block ‘}’ -SimpleType ::= ... - | ‘$’ ‘{’ Block ‘}’ -``` -In addition, an identifier `$x` starting with a `$` that appears inside -a quoted expression or type is treated as a splice `${x}` and a quoted identifier -`'x` that appears inside a splice is treated as a quote `'{x}` - -### Implementation in `scalac` - -Quotes and splices are primitive forms in the generated abstract syntax trees. -Top-level splices are eliminated during macro expansion while typing. On the -other hand, top-level quotes are eliminated in an expansion phase `PickleQuotes` -phase (after typing and pickling). PCP checking occurs while preparing the RHS -of an inline method for top-level splices and in the `Staging` phase (after -typing and before pickling). - -Macro-expansion works outside-in. If the outermost scope is a splice, -the spliced AST will be evaluated in an interpreter. A call to a -previously compiled method can be implemented as a reflective call to -that method. With the restrictions on splices that are currently in -place that’s all that’s needed. We might allow more interpretation in -splices in the future, which would allow us to loosen the -restriction. Quotes in spliced, interpreted code are kept as they -are, after splices nested in the quotes are expanded. - -If the outermost scope is a quote, we need to generate code that -constructs the quoted tree at run-time. We implement this by -serializing the tree as a TASTy structure, which is stored -in a string literal. At runtime, an unpickler method is called to -deserialize the string into a tree. - -Splices inside quoted code insert the spliced tree as is, after -expanding any quotes in the spliced code recursively. +### Run-Time Representation -## Formalization +The standard library defines the `Quotes` interface which contains all the logic and the abstract classes `Expr` and `Type`. +The compiler implements the `Quotes` interface and provides the implementation of `Expr` and `Type`. -The phase consistency principle can be formalized in a calculus that -extends simply-typed lambda calculus with quotes and splices. +##### `class Expr` +Expressions of type `Expr[T]` are represented by the following abstract class: +```scala +abstract class Expr[+T] private[scala] +``` +The only implementation of `Expr` is in the compiler along with the implementation of `Quotes`. +It is a class that wraps a typed AST and a `Scope` object with no methods of its own. +The `Scope` object is used to track the current splice scope and detect scope extrusions. -### Syntax +##### `object Expr` +The companion object of `Expr` contains a few useful static methods; +the `apply`/`unapply` methods to use `ToExpr`/`FromExpr` with ease; +the `betaReduce` and `summon` methods. +It also contains methods to create expressions out of lists or sequences of expressions: `block`, `ofSeq`, `ofList`, `ofTupleFromSeq` and `ofTuple`. -The syntax of terms, values, and types is given as follows: +```scala +object Expr: + def apply[T](x: T)(using ToExpr[T])(using Quotes): Expr[T] = ... + def unapply[T](x: Expr[T])(using FromExpr[T])(using Quotes): Option[T] = ... + def betaReduce[T](e: Expr[T])(using Quotes): Expr[T] = ... + def summon[T: Type](using Quotes): Option[Expr[T]] = ... + def block[T](stats: List[Expr[Any]], e: Expr[T])(using Quotes): Expr[T] = ... + def ofSeq[T: Type](xs: Seq[Expr[T]])(using Quotes): Expr[Seq[T]] = ... + def ofList[T: Type](xs: Seq[Expr[T]])(using Quotes): Expr[List[T]] = ... + def ofTupleFromSeq(xs: Seq[Expr[Any]])(using Quotes): Expr[Tuple] = ... + def ofTuple[T <: Tuple: Tuple.IsMappedBy[Expr]: Type](tup: T)(using Quotes): + Expr[Tuple.InverseMap[T, Expr]] = ... ``` -Terms t ::= x variable - (x: T) => t lambda - t t application - 't quote - $t splice -Values v ::= (x: T) => t lambda - 'u quote +##### `class Type` +Types of type `Type[T]` are represented by the following abstract class: +```scala +abstract class Type[T <: AnyKind] private[scala]: + type Underlying = T +``` + +The only implementation of `Type` is in the compiler along with the implementation of `Quotes`. +It is a class that wraps the AST of a type and a `Scope` object with no methods of its own. +The upper bound of `T` is `AnyKind` which implies that `T` may be a higher-kinded type. +The `Underlying` alias is used to select the type from an instance of `Type`. +Users never need to use this alias as they can always use `T` directly. +`Underlying` is used for internal encoding while compiling the code (see _Type Healing_). -Simple terms u ::= x | (x: T) => u | u u | 't +##### `object Type` +The companion object of `Type` contains a few useful static methods. +The first and most important one is the `Type.of` given definition. +This instance of `Type[T]` is summoned by default when no other instance is available. +The `of` operation is an intrinsic operation that the compiler will transform into code that will generate the `Type[T]` at run-time. +Secondly, the `Type.show[T]` operation will show a string representation of the type, which is often useful when debugging. +Finally, the object defines `valueOfConstant` (and `valueOfTuple`) which can transform singleton types (or tuples of singleton types) into their value. -Types T ::= A base type - T -> T function type - expr T quoted + +```scala +object Type: + given of[T <: AnyKind](using Quotes): Type[T] = ... + def show[T <: AnyKind](using Type[T])(using Quotes): String = ... + def valueOfConstant[T](using Type[T])(using Quotes): Option[T] = ... + def valueOfTuple[T <: Tuple](using Type[T])(using Quotes): Option[T] = ... ``` -Typing rules are formulated using a stack of environments -`Es`. Individual environments `E` consist as usual of variable -bindings `x: T`. Environments can be combined using the two -combinators `'` and `$`. + +##### `Quotes` +The `Quotes` interface is where most of the primitive operations of the quotation system are defined. + +Quotes define all the `Expr[T]` methods as extension methods. +`Type[T]` does not have methods and therefore does not appear here. +These methods are available as long as `Quotes` is implicitly given in the current scope. + +The `Quotes` instance is also the entry point to the [reflection API](./refelction.md) through the `reflect` object. + +Finally, `Quotes` provides the internal logic used in quote un-pickling (`QuoteUnpickler`) in quote pattern matching (`QuoteMatching`). +These interfaces are added to the self-type of the trait to make sure they are implemented on this object but not visible to users of `Quotes`. + +Internally, the implementation of `Quotes` will also track its current splicing scope `Scope`. +This scope will be attached to any expression that is created using this `Quotes` instance. + +```scala +trait Quotes: + this: runtime.QuoteUnpickler & runtime.QuoteMatching => + + extension [T](self: Expr[T]) + def show: String + def matches(that: Expr[Any]): Boolean + def value(using FromExpr[T]): Option[T] + def valueOrAbort(using FromExpr[T]): T + end extension + + extension (self: Expr[Any]) + def isExprOf[X](using Type[X]): Boolean + def asExprOf[X](using Type[X]): Expr[X] + end extension + + // abstract object reflect ... ``` -Environment E ::= () empty - E, x: T -Env. stack Es ::= () empty - E simple - Es * Es combined -Separator * ::= ' - $ +##### `Scope` +The splice context is represented as a stack (immutable list) of `Scope` objects. +Each `Scope` contains the position of the splice (used for error reporting) and a reference to the enclosing splice scope `Scope`. +A scope is a sub-scope of another if the other is contained in its parents. +This check is performed when an expression is spliced into another using the `Scope` provided in the current scope in `Quotes` and the one in the `Expr` or `Type`. + +### Entry Points +The two entry points for multi-stage programming are macros and the `run` operation. + +#### Macros +Inline macro definitions will inline a top-level splice (a splice not nested in a quote). +This splice needs to be evaluated at compile-time. +In _Avoiding a complete interpreter_[^1], we stated the following restrictions: + + * The top-level splice must contain a single call to a compiled static method. + * Arguments to the function are either literal constants, quoted expressions (parameters), `Type.of` for type parameters and a reference to `Quotes`. + +These restrictions make the implementation of the interpreter quite simple. +Java Reflection is used to call the single function call in the top-level splice. +The execution of that function is entirely done on compiled bytecode. +These are Scala static methods and may not always become Java static methods, they might be inside module objects. +As modules are encoded as class instances, we need to interpret the prefix of the method to instantiate it before we can invoke the method. + +The code of the arguments has not been compiled and therefore needs to be interpreted by the compiler. +Interpreting literal constants is as simple as extracting the constant from the AST that represents literals. +When interpreting a quoted expression, the contents of the quote is kept as an AST which is wrapped inside the implementation of `Expr`. +Calls to `Type.of[T]` also wrap the AST of the type inside the implementation of `Type`. +Finally, the reference to `Quotes` is supposed to be the reference to the quotes provided by the splice. +This reference is interpreted as a new instance of `Quotes` that contains a fresh initial `Scope` with no parents. + +The result of calling the method via Java Reflection will return an `Expr` containing a new AST that was generated by the implementation of that macro. +The scope of this `Expr` is checked to make sure it did not extrude from some splice or `run` operation. +Then the AST is extracted from the `Expr` and it is inserted as replacement for the AST that contained the top-level splice. + + +#### Run-time Multi-Stage Programming + +To be able to compile the code, the `scala.quoted.staging` library defines the `Compiler` trait. +An instance of `staging.Compiler` is a wrapper over the normal Scala~3 compiler. +To be instantiated it requires an instance of the JVM _classloader_ of the application. + +```scala +import scala.quoted.staging.* +given Compiler = Compiler.make(getClass.getClassLoader) ``` -The two environment combinators are both associative with left and -right identity `()`. -### Operational semantics +The classloader is needed for the compiler to know which dependencies have been loaded and to load the generated code using the same classloader. + +```scala +def mkPower2()(using Quotes): Expr[Double => Double] = ... -We define a small step reduction relation `-->` with the following rules: +run(mkPower2()) ``` - ((x: T) => t) v --> [x := v]t +To run the previous example, the compiler will create code equivalent to the following class and compile it using a new `Scope` without parents. + +```scala +class RunInstance: + def exec(): Double => Double = ${ mkPower2() } +``` +Finally, `run` will interpret `(new RunInstance).exec()` to evaluate the contents of the quote. +To do this, the resulting `RunInstance` class is loaded in the JVM using Java Reflection, instantiated and then the `exec` method is invoked. + + +### Compilation + +Quotes and splices are primitive forms in the generated typed abstract syntax trees. +These need to be type-checked with some extra rules, e.g., staging levels need to be checked and the references to generic types need to be adapted. +Finally, quoted expressions that will be generated at run-time need to be encoded (serialized) and decoded (deserialized). + +#### Typing Quoted Expressions - ${'u} --> u +The typing process for quoted expressions and splices with `Expr` is relatively straightforward. +At its core, quotes are desugared into calls to `quote`, splices are desugared into calls to `splice`. +We track the quotation level when desugaring into these methods. - t1 --> t2 - ----------------- - e[t1] --> e[t2] + +```scala +def quote[T](x: T): Quotes ?=> Expr[T] + +def splice[T](x: Quotes ?=> Expr[T]): T ``` -The first rule is standard call-by-value beta-reduction. The second -rule says that splice and quotes cancel each other out. The third rule -is a context rule; it says that reduction is allowed in the hole `[ ]` -position of an evaluation context. Evaluation contexts `e` and -splice evaluation context `e_s` are defined syntactically as follows: + +It would be impossible to track the quotation levels if users wrote calls to these methods directly. +To know if it is a call to one of those methods we would need to type it first, but to type it we would need to know if it is one of these methods to update the quotation level. +Therefore these methods can only be used by the compiler. + +At run-time, the splice needs to have a reference to the `Quotes` that created its surrounding quote. +To simplify this for later phases, we track the current `Quotes` and encode a reference directly in the splice using `nestedSplice` instead of `splice`. + +```scala +def nestedSplice[T](q: Quotes)(x: q.Nested ?=> Expr[T]): T ``` -Eval context e ::= [ ] | e t | v e | 'e_s[${e}] -Splice context e_s ::= [ ] | (x: T) => e_s | e_s t | u e_s +With this addition, the original `splice` is only used for top-level splices. + +The levels are mostly used to identify top-level splices that need to be evaluated while typing. +We do not use the quotation level to influence the typing process. +Level checking is performed at a later phase. +This ensures that a source expression in a quote will have the same elaboration as a source expression outside the quote. + + + +#### Quote Pattern Matching + +Pattern matching is defined in the trait `QuoteMatching`, which is part of the self type of `Quotes`. +It is implemented by `Quotes` but not available to users of `Quotes`. +To access it, the compiler generates a cast from `Quotes` to `QuoteMatching` and then selects one of its two members: `ExprMatch` or `TypeMatch`. +`ExprMatch` defines an `unapply` extractor method that is used to encode quote patterns and `TypeMatch` defines an `unapply` method for quoted type patterns. + +```scala +trait Quotes: + self: runtime.QuoteMatching & ... => + ... + +trait QuoteMatching: + object ExprMatch: + def unapply[TypeBindings <: Tuple, Tup <: Tuple] + (scrutinee: Expr[Any]) + (using pattern: Expr[Any]): Option[Tup] = ... + object TypeMatch: + ... ``` -### Typing rules +These extractor methods are only meant to be used in code generated by the compiler. +The call to the extractor that is generated has an already elaborated form that cannot be written in source, namely explicit type parameters and explicit contextual parameters. + +This extractor returns a tuple type `Tup` which cannot be inferred from the types in the method signature. +This type will be computed when typing the quote pattern and will be explicitly added to the extractor call. +To refer to type variables in arbitrary places of `Tup`, we need to define them all before their use, hence we have `TypeBindings`, which will contain all pattern type variable definitions. +The extractor also receives a given parameter of type `Expr[Any]` that will contain an expression that represents the pattern. +The compiler will explicitly add this pattern expression. +We use a given parameter because these are the only parameters we are allowed to add to the extractor call in a pattern position. + +This extractor is a bit convoluted, but it encodes away all the quotation-specific features. +It compiles the pattern down into a representation that the pattern matcher compiler phase understands. -Typing judgments are of the form `Es |- t: T`. There are two -substructural rules which express the fact that quotes and splices -cancel each other out: +The quote patterns are encoded into two parts: a tuple pattern that is tasked with extracting the result of the match and a quoted expression representing the pattern. +For example, if the pattern has no `$` we will have an `EmptyTuple` as the pattern and `'{1}` to represent the pattern. + +```scala + case '{ 1 } => +// is elaborated to + case ExprMatch(EmptyTuple)(using '{1}) => +// ^^^^^^^^^^ ^^^^^^^^^^ +// pattern expression +``` +When extracting expressions, each pattern that is contained in a splice `${..}` will be placed in order in the tuple pattern. +In the following case, the `f` and `x` are placed in a tuple pattern `(f, x)`. +The type of the tuple is encoded in the `Tup` and not only in the tuple itself. +Otherwise, the extractor would return a tuple `Tuple` for which the types need to be tested which is in turn not possible due to type erasure. + +```scala + case '{ ((y: Int) => $f(y)).apply($x) } => +// is elaborated to + case ExprMatch[.., (Expr[Int => Int], Expr[Int])]((f, x))(using pattern) => +// pattern = '{ ((y: Int) => pat[Int](y)).apply(pat[Int]()) } ``` - Es1 * Es2 |- t: T - --------------------------- - Es1 $ E1 ' E2 * Es2 |- t: T +The contents of the quote are transformed into a valid quote expression by replacing the splice with a marker expression `pat[T](..)`. +The type `T` is taken from the type of the splice and the arguments are the HOAS arguments. +This implies that a `pat[T]()` is a closed pattern and `pat[T](y)` is an HOAS pattern that can refer to `y`. - Es1 * Es2 |- t: T - --------------------------- - Es1 ' E1 $ E2 * Es2 |- t: T +Type variables in quoted patterns are first normalized to have all definitions at the start of the pattern. +For each definition of a type variable `t` in the pattern we will add a type variable definition in `TypeBindings`. +Each one will have a corresponding `Type[t]` that will get extracted if the pattern matches. +These `Type[t]` are also listed in the `Tup` and added in the tuple pattern. +It is additionally marked as `using` in the pattern to make it implicitly available in this case branch. + + +```scala + case '{ type t; ($xs: List[t]).map[t](identity[t]) } => +// is elaborated to + case ExprMatch[(t), (Type[t], Expr[List[t]])]((using t, xs))(using p) => +// ^^^ ^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^ ^^^^^^^ +// type bindings result type pattern expression +// p = '{ @patternType type u; pat[List[u]]().map[u](identity[u]) } ``` -The lambda calculus fragment of the rules is standard, except that we -use a stack of environments. The rules only interact with the topmost -environment of the stack. + +The contents of the quote are transformed into a valid quote expression by replacing type variables with fresh ones that do not escape the quote scope. +These are also annotated to be easily identifiable as pattern variables. + +#### Level Consistency Checking +Level consistency checking is performed after typing the program as a static check. +To check level consistency we traverse the tree top-down remembering the context staging level. +Each local definition in scope is recorded with its level and each term reference to a definition is checked against the current staging level. +```scala +// level 0 +'{ // level 1 + val x = ... // level 1 with (x -> 1) + ${ // level 0 (x -> 1) + val y = ... // level 0 with (x -> 1, y -> 0) + x // error: defined at level 1 but used in level 0 + } + // level 1 (x -> 1) + x // x is ok +} ``` - x: T in E - -------------- - Es * E |- x: T +#### Type Healing - Es * E, x: T1 |- t: T2 - ------------------------------- - Es * E |- (x: T1) => t: T -> T2 +When using a generic type `T` in a future stage, it is necessary to have a given `Type[T]` in scope. +The compiler needs to identify those references and link them with the instance of `Type[T]`. +For instance consider the following example: +```scala +def emptyList[T](using t: Type[T])(using Quotes): Expr[List[T]] = + '{ List.empty[T] } +``` - Es |- t1: T2 -> T Es |- t2: T2 - --------------------------------- - Es |- t1 t2: T +For each reference to a generic type `T` that is defined at level 0 and used at level 1 or greater, the compiler will summon a `Type[T]`. +This is usually the given type that is provided as parameter, `t` in this case. +We can use the type `t.Underlying` to replace `T` as it is an alias of that type. +But `t.Underlying` contains the extra information that it is `t` that will be used in the evaluation of the quote. +In a sense, `Underlying` acts like a splice for types. + +```scala +def emptyList[T](using t: Type[T])(using Quotes): Expr[List[T]] = + '{ List.empty[t.Underlying] } ``` -The rules for quotes and splices map between `expr T` and `T` by trading `'` and `$` between -environments and terms. + +Due to some technical limitations, it is not always possible to replace the type reference with the AST containing `t.Underlying`. +To overcome this limitation, we can simply define a list of type aliases at the start of the quote and insert the `t.Underlying` there. +This has the added advantage that we do not have to repeatedly insert the `t.Underlying` in the quote. + +```scala +def emptyList[T](using t: Type[T])(using Quotes): Expr[List[T]] = + '{ type U = t.Underlying; List.empty[U] } +``` +These aliases can be used at any level within the quote and this transformation is only performed on quotes that are at level 0. + +```scala + '{ List.empty[T] ... '{ List.empty[T] } ... } +// becomes + '{ type U = t.Underlying; List.empty[U] ... '{ List.empty[U] } ... } +``` +If we define a generic type at level 1 or greater, it will not be subject to this transformation. +In some future compilation stage, when the definition of the generic type is at level 0, it will be subject to this transformation. +This simplifies the transformation logic and avoids leaking the encoding into code that a macro could inspect. + +```scala +'{ + def emptyList[T: Type](using Quotes): Expr[List[T]] = '{ List.empty[T] } + ... +} +``` +A similar transformation is performed on `Type.of[T]`. +Any generic type in `T` needs to have an implicitly given `Type[T]` in scope, which will also be used as a path. +The example: + +```scala +def empty[T](using t: Type[T])(using Quotes): Expr[T] = + Type.of[T] match ... +// becomes +def empty[T](using t: Type[T])(using Quotes): Expr[T] = + Type.of[t.Underlying] match ... +// then becomes +def empty[T](using t: Type[T])(using Quotes): Expr[T] = + t match ... +``` + +The operation `Type.of[t.Underlying]` can be optimized to just `t`. +But this is not always the case. +If the generic reference is nested in the type, we will need to keep the `Type.of`. + +```scala +def matchOnList[T](using t: Type[T])(using Quotes): Expr[List[T]] = + Type.of[List[T]] match ... +// becomes +def matchOnList[T](using t: Type[T])(using Quotes): Expr[List[T]] = + Type.of[List[t.Underlying]] match ... +``` + +By doing this transformation, we ensure that each abstract type `U` used in `Type.of` has an implicit `Type[U]` in scope. +This representation makes it simpler to identify parts of the type that are statically known from those that are known dynamically. +Type aliases are also added within the type of the `Type.of` though these are not valid source code. +These would look like `Type.of[{type U = t.Underlying; Map[U, U]}]` if written in source code. + + +#### Splice Normalization + +The contents of a splice may refer to variables defined in the enclosing quote. +This complicates the process of serialization of the contents of the quotes. +To make serialization simple, we first transform the contents of each level 1 splice. +Consider the following example: + +```scala +def power5to(n: Expr[Int]): Expr[Double] = '{ + val x: Int = 5 + ${ powerCode('{x}, n) } +} +``` + +The variable `x` is defined in the quote and used in the splice. +The normal form will extract all references to `x` and replace them with a staged version of `x`. +We will replace the reference to `x` of type `T` with a `$y` where `y` is of type `Expr[T]`. +Then we wrap the new contents of the splice in a lambda that defines `y` and apply it to the quoted version of `x`. +After this transformation we have 2 parts, a lambda without references to the quote, which knows how to compute the contents of the splice, and a sequence of quoted arguments that refer to variables defined in the lambda. + +```scala +def power5to(n: Expr[Int]): Expr[Double] = '{ + val x: Int = 5 + ${ ((y: Expr[Int]) => powerCode('{$y}, n)).apply('x) } +} +``` + +In general, the splice normal form has the shape `${ .apply(*) }` and the following constraints: + * `` a lambda expression that does not refer to variables defined in the outer quote + * `` sequence of quoted expressions or `Type.of` containing references to variables defined in the enclosing quote and no references to local variables defined outside the enclosing quote + + +##### Function references normalization +A reference to a function `f` that receives parameters is not a valid value in Scala. +Such a function reference `f` can be eta-expaned as `x => f(x)` to be used as a lambda value. +Therefore function references cannot be transformed by the normalization as directly as other expressions as we cannot represent `'{f}` with a method reference type. +We can use the eta-expanded form of `f` in the normalized form. +For example, consider the reference to `f` below. + +```scala +'{ + def f(a: Int)(b: Int, c: Int): Int = 2 + a + b + c + ${ '{ f(3)(4, 5) } } +} +``` + +To normalize this code, we can eta-expand the reference to `f` and place it in a quote containing a proper expression. +Therefore the normalized form of the argument `'{f}` becomes the quoted lambda `'{ (a: Int) => (b: Int, c: Int) => f(a)(b, c) }` and is an expression of type `Expr[Int => (Int, Int) => Int]`. +The eta-expansion produces one curried lambda per parameter list. +The application `f(3)(4, 5)` does not become `$g(3)(4, 5)` but `$g.apply(3).apply(4, 5)`. +We add the `apply` because `g` is not a quoted reference to a function but a curried lambda. + +```scala +'{ + def f(a: Int)(b: Int, c: Int): Int = 2 + a + b + c + ${ + ( + (g: Expr[Int => (Int, Int) => Int]) => '{$g.apply(3).apply(4, 5)} + ).apply('{ (a: Int) => (b: Int, c: Int) => f(a)(b, c) }) + } +} +``` + +Then we can apply it and beta-reduce the application when generating the code. + +```scala + (g: Expr[Int => Int => Int]) => betaReduce('{$g.apply(3).apply(4)}) +``` + + +##### Variable assignment normalization +A reference to a mutable variable in the left-hand side of an assignment cannot be transformed directly as it is not in an expression position. +```scala +'{ + var x: Int = 5 + ${ g('{x = 2}) } +} +``` + +We can use the same strategy used for function references by eta-expanding the assignment operation `x = _` into `y => x = y`. + +```scala +'{ + var x: Int = 5 + ${ + g( + ( + (f: Expr[Int => Unit]) => betaReduce('{$f(2)}) + ).apply('{ (y: Int) => x = $y }) + ) + } +} +``` + + +##### Type normalization +Types defined in the quote are subject to a similar transformation. +In this example, `T` is defined within the quote at level 1 and used in the splice again at level 1. + +```scala +'{ def f[T] = ${ '{g[T]} } } +``` + +The normalization will add a `Type[T]` to the lambda, and we will insert this reference. +The difference is that it will add an alias similar to the one used in type healing. +In this example, we create a `type U` that aliases the staged type. + +```scala +'{ + def f[T] = ${ + ( + (t: Type[T]) => '{type U = t.Underling; g[U]} + ).apply(Type.of[T]) + } +} +``` + +#### Serialization + +Quoted code needs to be pickled to make it available at run-time in the next compilation phase. +We implement this by pickling the AST as a TASTy binary. + +##### TASTy +The TASTy format is the typed abstract syntax tree serialization format of Scala 3. +It usually pickles the fully elaborated code after type-checking and is kept along the generated Java classfiles. + + +##### Pickling +We use TASTy as a serialization format for the contents of the quotes. +To show how serialization is performed, we will use the following example. +```scala +'{ + val (x, n): (Double, Int) = (5, 2) + ${ powerCode('{x}, '{n}) } * ${ powerCode('{2}, '{n}) } +} ``` - Es $ () |- t: expr T - -------------------- - Es |- $t: T +This quote is transformed into the following code when normalizing the splices. - Es ' () |- t: T - ---------------- - Es |- 't: expr T +```scala +'{ + val (x, n): (Double, Int) = (5, 2) + ${ + ((y: Expr[Double], m: Expr[Int]) => powerCode(y, m)).apply('x, 'n) + } * ${ + ((m: Expr[Int]) => powerCode('{2}, m)).apply('n) + } +} ``` -The meta theory of a slightly simplified 2-stage variant of this calculus -is studied [separately](./simple-smp.md). -## Going Further +Splice normalization is a key part of the serialization process as it only allows references to variables defined in the quote in the arguments of the lambda in the splice. +This makes it possible to create a closed representation of the quote without much effort. +The first step is to remove all the splices and replace them with holes. +A hole is like a splice but it lacks the knowledge of how to compute the contents of the splice. +Instead, it knows the index of the hole and the contents of the arguments of the splice. +We can see this transformation in the following example where a hole is represented by `<< idx; holeType; args* >>`. + +```scala + ${ ((y: Expr[Double], m: Expr[Int]) => powerCode(y, m)).apply('x, 'n) } +// becomes + << 0; Double; x, n >> +``` -The metaprogramming framework as presented and currently implemented is quite restrictive -in that it does not allow for the inspection of quoted expressions and -types. It’s possible to work around this by providing all necessary -information as normal, unquoted inline parameters. But we would gain -more flexibility by allowing for the inspection of quoted code with -pattern matching. This opens new possibilities. +As this was the first hole it has index 0. +The hole type is `Double`, which needs to be remembered now that we cannot infer it from the contents of the splice. +The arguments of the splice are `x` and `n`; note that they do not require quoting because they were moved out of the splice. -For instance, here is a version of `power` that generates the multiplications -directly if the exponent is statically known and falls back to the dynamic -implementation of `power` otherwise. +References to healed types are handled in a similar way. +Consider the `emptyList` example, which shows the type aliases that are inserted into the quote. ```scala -import scala.quoted.* +'{ List.empty[T] } +// type healed to +'{ type U = t.Underlying; List.empty[U] } +``` +Instead of replacing a splice, we replace the `t.Underlying` type with a type hole. +The type hole is represented by `<< idx; bounds >>`. +```scala +'{ type U = << 0; Nothing..Any >>; List.empty[U] } +``` +Here, the bounds of `Nothing..Any` are the bounds of the original `T` type. +The types of a `Type.of` are transformed in the same way. -inline def power(x: Double, n: Int): Double = - ${ powerExpr('x, 'n) } -private def powerExpr(x: Expr[Double], n: Expr[Int]) - (using Quotes): Expr[Double] = - n.value match - case Some(m) => powerExpr(x, m) - case _ => '{ dynamicPower($x, $n) } +With these transformations, the contents of the quote or `Type.of` are guaranteed to be closed and therefore can be pickled. +The AST is pickled into TASTy, which is a sequence of bytes. +This sequence of bytes needs to be instantiated in the bytecode, but unfortunately it cannot be dumped into the classfile as bytes. +To reify it we encode the bytes into a Java `String`. +In the following examples we display this encoding in human readable form with the fictitious |tasty"..."| string literal. -private def powerExpr(x: Expr[Double], n: Int) - (using Quotes): Expr[Double] = - if n == 0 then '{ 1.0 } - else if n == 1 then x - else if n % 2 == 0 then '{ val y = $x * $x; ${ powerExpr('y, n / 2) } } - else '{ $x * ${ powerExpr(x, n - 1) } } +```scala +// pickled AST bytes encoded in a base64 string +tasty""" + val (x, n): (Double, Int) = (5, 2) + << 0; Double; x, n >> * << 1; Double; n >> +""" +// or +tasty""" + type U = << 0; Nothing..Any; >> + List.empty[U] +""" +``` +The contents of a quote or `Type.of` are not always pickled. +In some cases it is better to generate equivalent (smaller and/or faster) code that will compute the expression. +Literal values are compiled into a call to `Expr()` using the implementation of `ToExpr` to create the quoted expression. +This is currently performed only on literal values, but can be extended to any value for which we have a `ToExpr` defined in the standard library. +Similarly, for non-generic types we can use their respective `java.lang.Class` and convert them into a `Type` using a primitive operation `typeConstructorOf` defined in the reflection API. -private def dynamicPower(x: Double, n: Int): Double = - if n == 0 then 1.0 - else if n % 2 == 0 then dynamicPower(x * x, n / 2) - else x * dynamicPower(x, n - 1) +##### Unpickling + +Now that we have seen how a quote is pickled, we can look at how to unpickle it. +We will continue with the previous example. + +Holes were used to replace the splices in the quote. +When we perform this transformation we also need to remember the lambdas from the splices and their hole index. +When unpickling a hole, the corresponding splice lambda will be used to compute the contents of the hole. +The lambda will receive as parameters quoted versions of the arguments of the hole. +For example to compute the contents of `<< 0; Double; x, n >>` we will evaluate the following code + +```scala + ((y: Expr[Double], m: Expr[Int]) => powerCode(y, m)).apply('x, 'n) ``` -In the above, the method `.value` maps a constant expression of the type -`Expr[T]` to its value of the type `T`. +The evaluation is not as trivial as it looks, because the lambda comes from compiled code and the rest is code that must be interpreted. +We put the AST of `x` and `n` into `Expr` objects to simulate the quotes and then we use Java Reflection to call the `apply` method. + +We may have many holes in a quote and therefore as many lambdas. +To avoid the instantiation of many lambdas, we can join them together into a single lambda. +Apart from the list of arguments, this lambda will also take the index of the hole that is being evaluated. +It will perform a switch match on the index and call the corresponding lambda in each branch. +Each branch will also extract the arguments depending on the definition of the lambda. +The application of the original lambdas are beta-reduced to avoid extra overhead. -With the right extractors, the "AsFunction" conversion -that maps expressions over functions to functions over expressions can -be implemented in user code: ```scala -given AsFunction1[T, U]: Conversion[Expr[T => U], Expr[T] => Expr[U]] with - def apply(f: Expr[T => U]): Expr[T] => Expr[U] = - (x: Expr[T]) => f match - case Lambda(g) => g(x) - case _ => '{ ($f)($x) } +(idx: Int, args: Seq[Any]) => + idx match + case 0 => // for << 0; Double; x, n >> + val x = args(0).asInstanceOf[Expr[Double]] + val n = args(1).asInstanceOf[Expr[Int]] + powerCode(x, n) + case 1 => // for << 1; Double; n >> + val n = args(0).asInstanceOf[Expr[Int]] + powerCode('{2}, n) ``` -This assumes an extractor + +This is similar to what we do for splices when we replace the type aliased with holes we keep track of the index of the hole. +Instead of lambdas, we will have a list of references to instances of `Type`. +From the following example we would extract `t`, `u`, ... . + ```scala -object Lambda: - def unapply[T, U](x: Expr[T => U]): Option[Expr[T] => Expr[U]] + '{ type T1 = t1.Underlying; type Tn = tn.Underlying; ... } +// with holes + '{ type T1 = << 0; ... >>; type Tn = << n-1; ... >>; ... } ``` -Once we allow inspection of code via extractors, it’s tempting to also -add constructors that create typed trees directly without going -through quotes. Most likely, those constructors would work over `Expr` -types which lack a known type argument. For instance, an `Apply` -constructor could be typed as follows: + +As the type holes are at the start of the quote, they will have the first `N` indices. +This implies that we can place the references in a sequence `Seq(t, u, ...)` where the index in the sequence is the same as the hole index. + +Lastly, the quote itself is replaced by a call to `QuoteUnpickler.unpickleExpr` which will unpickle the AST, evaluate the holes, i.e., splices, and wrap the resulting AST in an `Expr[Int]`. +This method takes takes the pickled |tasty"..."|, the types and the hole lambda. +Similarly, `Type.of` is replaced with a call to `QuoteUnpickler.unpickleType` but only receives the pickled |tasty"..."| and the types. +Because `QuoteUnpickler` is part of the self-type of the `Quotes` class, we have to cast the instance but know that this cast will always succeed. + ```scala -def Apply(fn: Expr[Any], args: List[Expr[Any]]): Expr[Any] +quotes.asInstanceOf[runtime.QuoteUnpickler].unpickleExpr[T]( + pickled = tasty"...", + types = Seq(...), + holes = (idx: Int, args: Seq[Any]) => idx match ... +) ``` -This would allow constructing applications from lists of arguments -without having to match the arguments one-by-one with the -corresponding formal parameter types of the function. We then need "at -the end" a method to convert an `Expr[Any]` to an `Expr[T]` where `T` is -given from the outside. For instance, if `code` yields a `Expr[Any]`, then -`code.atType[T]` yields an `Expr[T]`. The `atType` method has to be -implemented as a primitive; it would check that the computed type -structure of `Expr` is a subtype of the type structure representing -`T`. -Before going down that route, we should evaluate in detail the tradeoffs it -presents. Constructing trees that are only verified _a posteriori_ -to be type correct loses a lot of guidance for constructing the right -trees. So we should wait with this addition until we have more -use-cases that help us decide whether the loss in type-safety is worth -the gain in flexibility. In this context, it seems that deconstructing types is -less error-prone than deconstructing terms, so one might also -envisage a solution that allows the former but not the latter. - -## Conclusion - -Metaprogramming has a reputation of being difficult and confusing. -But with explicit `Expr/Type` types and quotes and splices it can become -downright pleasant. A simple strategy first defines the underlying quoted or unquoted -values using `Expr` and `Type` and then inserts quotes and splices to make the types -line up. Phase consistency is at the same time a great guideline -where to insert a splice or a quote and a vital sanity check that -the result makes sense. +[^1]: [Scalable Metaprogramming in Scala 3](https://infoscience.epfl.ch/record/299370) +[^2]: [Multi-stage programming with generative and analytical macros](https://dl.acm.org/doi/10.1145/3486609.3487203). +[^3]: In quotes, identifiers starting with `$` must be surrounded by backticks (`` `$` ``). For example `$conforms` from `scala.Predef`. diff --git a/docs/_spec/TODOreference/metaprogramming/macros.md b/docs/_spec/TODOreference/metaprogramming/macros.md index 8045794d1143..e39f6f1022b8 100644 --- a/docs/_spec/TODOreference/metaprogramming/macros.md +++ b/docs/_spec/TODOreference/metaprogramming/macros.md @@ -6,818 +6,616 @@ nightlyOf: https://docs.scala-lang.org/scala3/reference/metaprogramming/macros.h > When developing macros enable `-Xcheck-macros` scalac option flag to have extra runtime checks. -## Macros: Quotes and Splices +## Multi-Staging -Macros are built on two well-known fundamental operations: quotation and splicing. -Quotation is expressed as `'{...}` for expressions and splicing is expressed as `${ ... }`. -Additionally, within a quote or a splice we can quote or splice identifiers directly (i.e. `'e` and `$e`). -Readers may notice the resemblance of the two aforementioned syntactic -schemes with the familiar string interpolation syntax. +#### Quoted expressions +Multi-stage programming in Scala 3 uses quotes `'{..}` to delay, i.e., stage, execution of code and splices `${..}` to evaluate and insert code into quotes. +Quoted expressions are typed as `Expr[T]` with a covariant type parameter `T`. +It is easy to write statically safe code generators with these two concepts. +The following example shows a naive implementation of the $x^n$ mathematical operation. ```scala -println(s"Hello, $name, here is the result of 1 + 1 = ${1 + 1}") +import scala.quoted.* +def unrolledPowerCode(x: Expr[Double], n: Int)(using Quotes): Expr[Double] = + if n == 0 then '{ 1.0 } + else if n == 1 then x + else '{ $x * ${ unrolledPowerCode(x, n-1) } } ``` -In string interpolation we _quoted_ a string and then we _spliced_ into it, two others. The first, `name`, is a reference to a value of type [`String`](https://scala-lang.org/api/3.x/scala/Predef$.html#String-0), and the second is an arithmetic expression that will be _evaluated_ followed by the splicing of its string representation. - -Quotes and splices in this section allow us to treat code in a similar way, -effectively supporting macros. The entry point for macros is an inline method -with a top-level splice. We call it a top-level because it is the only occasion -where we encounter a splice outside a quote (consider as a quote the -compilation-unit at the call-site). For example, the code below presents an -`inline` method `assert` which calls at compile-time a method `assertImpl` with -a boolean expression tree as argument. `assertImpl` evaluates the expression and -prints it again in an error message if it evaluates to `false`. - ```scala -import scala.quoted.* - -inline def assert(inline expr: Boolean): Unit = - ${ assertImpl('expr) } - -def assertImpl(expr: Expr[Boolean])(using Quotes) = '{ - if !$expr then - throw AssertionError(s"failed assertion: ${${ showExpr(expr) }}") +'{ + val x = ... + ${ unrolledPowerCode('{x}, 3) } // evaluates to: x * x * x } - -def showExpr(expr: Expr[Boolean])(using Quotes): Expr[String] = - '{ [actual implementation later in this document] } ``` -If `e` is an expression, then `'{e}` represents the typed -abstract syntax tree representing `e`. If `T` is a type, then `Type.of[T]` -represents the type structure representing `T`. The precise -definitions of "typed abstract syntax tree" or "type-structure" do not -matter for now, the terms are used only to give some -intuition. Conversely, `${e}` evaluates the expression `e`, which must -yield a typed abstract syntax tree or type structure, and embeds the -result as an expression (respectively, type) in the enclosing program. +Quotes and splices are duals of each other. +For an arbitrary expression `x` of type `T` we have `${'{x}} = x` and for an arbitrary expression `e` of type `Expr[T]` we have `'{${e}} = e`. -Quotations can have spliced parts in them; in this case the embedded -splices are evaluated and embedded as part of the formation of the -quotation. +#### Abstract types +Quotes can handle generic and abstract types using the type class `Type[T]`. +A quote that refers to a generic or abstract type `T` requires a given `Type[T]` to be provided in the implicit scope. +The following examples show how `T` is annotated with a context bound (`: Type`) to provide an implicit `Type[T]`, or the equivalent `using Type[T]` parameter. -Quotes and splices can also be applied directly to identifiers. An identifier -`$x` starting with a `$` that appears inside a quoted expression or type is treated as a -splice `${x}`. Analogously, an quoted identifier `'x` that appears inside a splice -is treated as a quote `'{x}`. See the Syntax section below for details. +```scala +import scala.quoted.* +def singletonListExpr[T: Type](x: Expr[T])(using Quotes): Expr[List[T]] = + '{ List[T]($x) } // generic T used within a quote -Quotes and splices are duals of each other. -For arbitrary expressions `e` we have: +def emptyListExpr[T](using Type[T], Quotes): Expr[List[T]] = + '{ List.empty[T] } // generic T used within a quote +``` +If no other instance is found, the default `Type.of[T]` is used. +The following example implicitly uses `Type.of[String]` and `Type.of[Option[U]]`. ```scala -${'{e}} = e -'{${e}} = e +val list1: Expr[List[String]] = + singletonListExpr('{"hello"}) // requires a given `Type[Sting]` +val list0: Expr[List[Option[T]]] = + emptyListExpr[Option[U]] // requires a given `Type[Option[U]]` ``` -## Types for Quotations - -The type signatures of quotes and splices can be described using -two fundamental types: -- `Expr[T]`: abstract syntax trees representing expressions of type `T` -- `Type[T]`: non erased representation of type `T`. - -Quoting takes expressions of type `T` to expressions of type `Expr[T]` -and it takes types `T` to expressions of type `Type[T]`. Splicing -takes expressions of type `Expr[T]` to expressions of type `T` and it -takes expressions of type `Type[T]` to types `T`. - -The two types can be defined in package [`scala.quoted`](https://scala-lang.org/api/3.x/scala/quoted.html) as follows: +The `Type.of[T]` method is a primitive operation that the compiler will handle specially. +It will provide the implicit if the type `T` is statically known, or if `T` contains some other types `Ui` for which we have an implicit `Type[Ui]`. +In the example, `Type.of[String]` has a statically known type and `Type.of[Option[U]]` requires an implicit `Type[U]` in scope. +#### Quote context +We also track the current quotation context using a given `Quotes` instance. +To create a quote `'{..}` we require a given `Quotes` context, which should be passed as a contextual parameter `(using Quotes)` to the function. +Each splice will provide a new `Quotes` context within the scope of the splice. +Therefore quotes and splices can be seen as methods with the following signatures, but with special semantics. ```scala -package scala.quoted +def '[T](x: T): Quotes ?=> Expr[T] // def '[T](x: T)(using Quotes): Expr[T] -sealed trait Expr[+T] -sealed trait Type[T] +def $[T](x: Quotes ?=> Expr[T]): T ``` -Both `Expr` and `Type` are abstract and sealed, so all constructors for -these types are provided by the system. One way to construct values of -these types is by quoting, the other is by type-specific lifting -operations that will be discussed later on. +The lambda with a question mark `?=>` is a contextual function; it is a lambda that takes its argument implicitly and provides it implicitly in the implementation the lambda. +`Quotes` are used for a variety of purposes that will be mentioned when covering those topics. -## The Phase Consistency Principle +## Quoted Values -A fundamental *phase consistency principle* (PCP) regulates accesses -to free variables in quoted and spliced code: +#### Lifting +While it is not possible to use cross-stage persistence of local variables, it is possible to lift them to the next stage. +To this end, we provide the `Expr.apply` method, which can take a value and lift it into a quoted representation of the value. -- _For any free variable reference `x`, the number of quoted scopes and the number of spliced scopes between the reference to `x` and the definition of `x` must be equal_. - -Here, `this`-references count as free variables. On the other -hand, we assume that all imports are fully expanded and that `_root_` is -not a free variable. So references to global definitions are -allowed everywhere. +```scala +val expr1plus1: Expr[Int] = '{ 1 + 1 } -The phase consistency principle can be motivated as follows: First, -suppose the result of a program `P` is some quoted text `'{ ... x -... }` that refers to a free variable `x` in `P`. This can be -represented only by referring to the original variable `x`. Hence, the -result of the program will need to persist the program state itself as -one of its parts. We don’t want to do this, hence this situation -should be made illegal. Dually, suppose a top-level part of a program -is a spliced text `${ ... x ... }` that refers to a free variable `x` -in `P`. This would mean that we refer during _construction_ of `P` to -a value that is available only during _execution_ of `P`. This is of -course impossible and therefore needs to be ruled out. Now, the -small-step evaluation of a program will reduce quotes and splices in -equal measure using the cancellation rules above. But it will neither -create nor remove quotes or splices individually. So the PCP ensures -that program elaboration will lead to neither of the two unwanted -situations described above. +val expr2: Expr[Int] = Expr(1 + 1) // lift 2 into '{ 2 } +``` -In what concerns the range of features it covers, this form of macros introduces -a principled metaprogramming framework that is quite close to the MetaML family of -languages. One difference is that MetaML does not have an equivalent of the PCP - -quoted code in MetaML _can_ access variables in its immediately enclosing -environment, with some restrictions and caveats since such accesses involve -serialization. However, this does not constitute a fundamental gain in -expressiveness. +While it looks type wise similar to `'{ 1 + 1 }`, the semantics of `Expr(1 + 1)` are quite different. +`Expr(1 + 1)` will not stage or delay any computation; the argument is evaluated to a value and then lifted into a quote. +The quote will contain code that will create a copy of this value in the next stage. +`Expr` is polymorphic and user-extensible via the `ToExpr` type class. -## From `Expr`s to Functions and Back +```scala +trait ToExpr[T]: + def apply(x: T)(using Quotes): Expr[T] +``` -It is possible to convert any `Expr[T => R]` into `Expr[T] => Expr[R]` and back. -These conversions can be implemented as follows: +We can implement a `ToExpr` using a `given` definition that will add the definition to the implicits in scope. +In the following example we show how to implement a `ToExpr[Option[T]]` for any liftable type `T. ```scala -def to[T: Type, R: Type](f: Expr[T] => Expr[R])(using Quotes): Expr[T => R] = - '{ (x: T) => ${ f('x) } } - -def from[T: Type, R: Type](f: Expr[T => R])(using Quotes): Expr[T] => Expr[R] = - (x: Expr[T]) => '{ $f($x) } +given OptionToExpr[T: Type: ToExpr]: ToExpr[Option[T]] with + def apply(opt: Option[T])(using Quotes): Expr[Option[T]] = + opt match + case Some(x) => '{ Some[T]( ${Expr(x)} ) } + case None => '{ None } ``` -Note how the fundamental phase consistency principle works in two -different directions here for `f` and `x`. In the method `to`, the reference to `f` is -legal because it is quoted, then spliced, whereas the reference to `x` -is legal because it is spliced, then quoted. +The `ToExpr` for primitive types must be implemented as primitive operations in the system. +In our case, we use the reflection API to implement them. -They can be used as follows: +#### Extracting values from quotes +To be able to generate optimized code using the method `unrolledPowerCode`, the macro implementation `powerCode` needs to first +determine whether the argument passed as parameter `n` is a known constant value. +This can be achieved via _unlifting_ using the `Expr.unapply` extractor from our library implementation, which will only match if `n` is a quoted constant and extracts its value. ```scala -val f1: Expr[Int => String] = - to((x: Expr[Int]) => '{ $x.toString }) // '{ (x: Int) => x.toString } - -val f2: Expr[Int] => Expr[String] = - from('{ (x: Int) => x.toString }) // (x: Expr[Int]) => '{ ((x: Int) => x.toString)($x) } -f2('{2}) // '{ ((x: Int) => x.toString)(2) } +def powerCode(x: Expr[Double], n: Expr[Int])(using Quotes): Expr[Double] = + n match + case Expr(m) => // it is a constant: unlift code n='{m} into number m + unrolledPowerCode(x, m) + case _ => // not known: call power at run-time + '{ power($x, $n) } ``` -One limitation of `from` is that it does not β-reduce when a lambda is called immediately, as evidenced in the code `{ ((x: Int) => x.toString)(2) }`. -In some cases we want to remove the lambda from the code, for this we provide the method `Expr.betaReduce` that turns a tree -describing a function into a function mapping trees to trees. - +Alternatively, the `n.value` method can be used to get an `Option[Int]` with the value or `n.valueOrAbort` to get the value directly. ```scala -object Expr: - ... - def betaReduce[...](...)(...): ... = ... +def powerCode(x: Expr[Double], n: Expr[Int])(using Quotes): Expr[Double] = + // emits an error message if `n` is not a constant + unrolledPowerCode(x, n.valueOrAbort) ``` -The definition of `Expr.betaReduce(f)(x)` is assumed to be functionally the same as -`'{($f)($x)}`, however it should optimize this call by returning the -result of beta-reducing `f(x)` if `f` is a known lambda expression. -`Expr.betaReduce` distributes applications of `Expr` over function arrows: +`Expr.unapply` and all variants of `value` are polymorphic and user-extensible via a given `FromExpr` type class. ```scala -Expr.betaReduce(_): Expr[(T1, ..., Tn) => R] => ((Expr[T1], ..., Expr[Tn]) => Expr[R]) +trait FromExpr[T]: + def unapply(x: Expr[T])(using Quotes): Option[T] ``` -## Lifting Types +We can use `given` definitions to implement the `FromExpr` as we did for `ToExpr`. +The `FromExpr` for primitive types must be implemented as primitive operations in the system. +In our case, we use the reflection API to implement them. +To implement `FromExpr` for non-primitive types we use quote pattern matching (for example `OptionFromExpr`). -Types are not directly affected by the phase consistency principle. -It is possible to use types defined at any level in any other level. -But, if a type is used in a subsequent stage it will need to be lifted to a `Type`. -Indeed, the definition of `to` above uses `T` in the next stage, there is a -quote but no splice between the parameter binding of `T` and its -usage. But the code can be rewritten by adding an explicit binding of a `Type[T]`: - -```scala -def to[T, R](f: Expr[T] => Expr[R])(using t: Type[T])(using Type[R], Quotes): Expr[T => R] = - '{ (x: t.Underlying) => ${ f('x) } } -``` -In this version of `to`, the type of `x` is now the result of -inserting the type `Type[T]` and selecting its `Underlying`. +## Macros and Multi-Stage Programming -To avoid clutter, the compiler converts any type reference to -a type `T` in subsequent phases to `summon[Type[T]].Underlying`. +The system supports multi-stage macros and run-time multi-stage programming using the same quotation abstractions. -And to avoid duplication it does it once per type, and creates -an alias for that type at the start of the quote. +### Multi-Stage Macros -For instance, the user-level definition of `to`: +#### Macros +We can generalize the splicing abstraction to express macros. +A macro consists of a top-level splice that is not nested in any quote. +Conceptually, the contents of the splice are evaluated one stage earlier than the program. +In other words, the contents are evaluated while compiling the program. The generated code resulting from the macro replaces the splice in the program. ```scala -def to[T, R](f: Expr[T] => Expr[R])(using t: Type[T], r: Type[R])(using Quotes): Expr[T => R] = - '{ (x: T) => ${ f('x) } } +def power2(x: Double): Double = + ${ unrolledPowerCode('x, 2) } // x * x ``` -would be rewritten to +#### Inline macros +Since using the splices in the middle of a program is not as ergonomic as calling a function; we hide the staging mechanism from end-users of macros. We have a uniform way of calling macros and normal functions. +For this, _we restrict the use of top-level splices to only appear in inline methods_[^1][^2]. ```scala -def to[T, R](f: Expr[T] => Expr[R])(using t: Type[T], r: Type[R])(using Quotes): Expr[T => R] = - '{ - type T = t.Underlying - (x: T) => ${ f('x) } - } +// inline macro definition +inline def powerMacro(x: Double, inline n: Int): Double = + ${ powerCode('x, 'n) } + +// user code +def power2(x: Double): Double = + powerMacro(x, 2) // x * x ``` -The `summon` query succeeds because there is a given instance of -type `Type[T]` available (namely the given parameter corresponding -to the context bound `: Type`), and the reference to that value is -phase-correct. If that was not the case, the phase inconsistency for -`T` would be reported as an error. +The evaluation of the macro will only happen when the code is inlined into `power2`. +When inlined, the code is equivalent to the previous definition of `power2`. +A consequence of using inline methods is that none of the arguments nor the return type of the macro will have to mention the `Expr` types; this hides all aspects of metaprogramming from the end-users. -## Lifting Expressions +#### Avoiding a complete interpreter +When evaluating a top-level splice, the compiler needs to interpret the code that is within the splice. +Providing an interpreter for the entire language is quite tricky, and it is even more challenging to make that interpreter run efficiently. +To avoid needing a complete interpreter, we can impose the following restrictions on splices to simplify the evaluation of the code in top-level splices. + * The top-level splice must contain a single call to a compiled static method. + * Arguments to the function are literal constants, quoted expressions (parameters), calls to `Type.of` for type parameters and a reference to `Quotes`. -Consider the following implementation of a staged interpreter that implements -a compiler through staging. +In particular, these restrictions disallow the use of splices in top-level splices. +Such a splice would require several stages of interpretation which would be unnecessarily inefficient. +#### Compilation stages +The macro implementation (i.e., the method called in the top-level splice) can come from any pre-compiled library. +This provides a clear difference between the stages of the compilation process. +Consider the following 3 source files defined in distinct libraries. ```scala -import scala.quoted.* - -enum Exp: - case Num(n: Int) - case Plus(e1: Exp, e2: Exp) - case Var(x: String) - case Let(x: String, e: Exp, in: Exp) - -import Exp.* +// Macro.scala +def powerCode(x: Expr[Double], n: Expr[Int])(using Quotes): Expr[Double] = ... +inline def powerMacro(x: Double, inline n: Int): Double = + ${ powerCode('x, 'n) } ``` -The interpreted language consists of numbers `Num`, addition `Plus`, and variables -`Var` which are bound by `Let`. Here are two sample expressions in the language: - ```scala -val exp = Plus(Plus(Num(2), Var("x")), Num(4)) -val letExp = Let("x", Num(3), exp) +// Lib.scala (depends on Macro.scala) +def power2(x: Double) = + ${ powerCode('x, '{2}) } // inlined from a call to: powerMacro(x, 2) ``` -Here’s a compiler that maps an expression given in the interpreted -language to quoted Scala code of type `Expr[Int]`. -The compiler takes an environment that maps variable names to Scala `Expr`s. - ```scala -import scala.quoted.* - -def compile(e: Exp, env: Map[String, Expr[Int]])(using Quotes): Expr[Int] = - e match - case Num(n) => - Expr(n) - case Plus(e1, e2) => - '{ ${ compile(e1, env) } + ${ compile(e2, env) } } - case Var(x) => - env(x) - case Let(x, e, body) => - '{ val y = ${ compile(e, env) }; ${ compile(body, env + (x -> 'y)) } } +// App.scala (depends on Lib.scala) +@main def app() = power2(3.14) ``` - -Running `compile(letExp, Map())` would yield the following Scala code: +One way to syntactically visualize this is to put the application in a quote that delays the compilation of the application. +Then the application dependencies can be placed in an outer quote that contains the quoted application, and we repeat this recursively for dependencies of dependencies. ```scala -'{ val y = 3; (2 + y) + 4 } +'{ // macro library (compilation stage 1) + def powerCode(x: Expr[Double], n: Expr[Int])(using Quotes): Expr[Double] = + ... + inline def powerMacro(x: Double, inline n: Int): Double = + ${ powerCode('x, 'n) } + '{ // library using macros (compilation stage 2) + def power2(x: Double) = + ${ powerCode('x, '{2}) } // inlined from a call to: powerMacro(x, 2) + '{ power2(3.14) /* app (compilation stage 3) */ } + } +} ``` -The body of the first clause, `case Num(n) => Expr(n)`, looks suspicious. `n` -is declared as an `Int`, yet it is converted to an `Expr[Int]` with `Expr()`. -Shouldn’t `n` be quoted? In fact this would not -work since replacing `n` by `'n` in the clause would not be phase -correct. +To make the system more versatile, we allow calling macros in the project where it is defined, with some restrictions. +For example, to compile `Macro.scala` and `Lib.scala` together in the same library. +To this end, we do not follow the simpler syntactic model and rely on semantic information from the source files. +When compiling a source, if we detect a call to a macro that is not compiled yet, we delay the compilation of this source to the following compilation stage. +In the example, we would delay the compilation of `Lib.scala` because it contains a compile-time call to `powerCode`. +Compilation stages are repeated until all sources are compiled, or no progress can be made. +If no progress is made, there was a cyclic dependency between the definition and the use of the macro. +We also need to detect if at runtime the macro depends on sources that have not been compiled yet. +These are detected by executing the macro and checking for JVM linking errors to classes that have not been compiled yet. -The `Expr.apply` method is defined in package `quoted`: +### Run-Time Multi-Stage Programming -```scala -package quoted +See [Run-Time Multi-Stage Programming](./staging.md) -object Expr: - ... - def apply[T: ToExpr](x: T)(using Quotes): Expr[T] = - summon[ToExpr[T]].toExpr(x) -``` +## Safety -This method says that values of types implementing the `ToExpr` type class can be -converted to `Expr` values using `Expr.apply`. +Multi-stage programming is by design statically safe and cross-stage safe. -Scala 3 comes with given instances of `ToExpr` for -several types including `Boolean`, `String`, and all primitive number -types. For example, `Int` values can be converted to `Expr[Int]` -values by wrapping the value in a `Literal` tree node. This makes use -of the underlying tree representation in the compiler for -efficiency. But the `ToExpr` instances are nevertheless not _magic_ -in the sense that they could all be defined in a user program without -knowing anything about the representation of `Expr` trees. For -instance, here is a possible instance of `ToExpr[Boolean]`: +### Static Safety -```scala -given ToExpr[Boolean] with - def toExpr(b: Boolean) = - if b then '{ true } else '{ false } -``` +#### Hygiene +All identifier names are interpreted as symbolic references to the corresponding variable in the context of the quote. +Therefore, while evaluating the quote, it is not possible to accidentally rebind a reference to a new variable with the same textual name. -Once we can lift bits, we can work our way up. For instance, here is a -possible implementation of `ToExpr[Int]` that does not use the underlying -tree machinery: +#### Well-typed +If a quote is well typed, then the generated code is well typed. +This is a simple consequence of tracking the type of each expression. +An `Expr[T]` can only be created from a quote that contains an expression of type `T. +Conversely, an `Expr[T]` can only be spliced in a location that expects a type `T. +As mentioned before, `Expr` is covariant in its type parameter. +This means that an `Expr[T]` can contain an expression of a subtype of `T. +When spliced in a location that expects a type `T, these expressions also have a valid type. -```scala -given ToExpr[Int] with - def toExpr(n: Int) = n match - case Int.MinValue => '{ Int.MinValue } - case _ if n < 0 => '{ - ${ toExpr(-n) } } - case 0 => '{ 0 } - case _ if n % 2 == 0 => '{ ${ toExpr(n / 2) } * 2 } - case _ => '{ ${ toExpr(n / 2) } * 2 + 1 } -``` +### Cross-Stage Safety -Since `ToExpr` is a type class, its instances can be conditional. For example, -a `List` is liftable if its element type is: +#### Level consistency +We define the _staging level_ of some code as the number of quotes minus the number of splices surrounding said code. +Local variables must be defined and used in the same staging level. + +It is never possible to access a local variable from a lower staging level as it does not yet exist. ```scala -given [T: ToExpr : Type]: ToExpr[List[T]] with - def toExpr(xs: List[T]) = xs match - case head :: tail => '{ ${ Expr(head) } :: ${ toExpr(tail) } } - case Nil => '{ Nil: List[T] } +def badPower(x: Double, n: Int): Double = + ${ unrolledPowerCode('x, n) } // error: value of `n` not known yet ``` -In the end, `ToExpr` resembles very much a serialization -framework. Like the latter it can be derived systematically for all -collections, case classes and enums. Note also that the synthesis -of _type-tag_ values of type `Type[T]` is essentially the type-level -analogue of lifting. -Using lifting, we can now give the missing definition of `showExpr` in the introductory example: +In the context of macros and _cross-platform portability_, that is, +macros compiled on one machine but potentially executed on another, +we cannot support cross-stage persistence of local variables. +Therefore, local variables can only be accessed at precisely the same staging level in our system. ```scala -def showExpr[T](expr: Expr[T])(using Quotes): Expr[String] = - val code: String = expr.show - Expr(code) +def badPowerCode(x: Expr[Double], n: Int)(using Quotes): Expr[Double] = + // error: `n` potentially not available in the next execution environment + '{ power($x, n) } ``` -That is, the `showExpr` method converts its `Expr` argument to a string (`code`), and lifts -the result back to an `Expr[String]` using `Expr.apply`. -## Lifting Types +The rules are slightly different for global definitions, such as `unrolledPowerCode`. +It is possible to generate code that contains a reference to a _global_ definition such as in `'{ power(2, 4) }`. +This is a limited form of cross-stage persistence that does not impede cross-platform portability, where we refer to the already compiled code for `power`. +Each compilation step will lower the staging level by one while keeping global definitions. +In consequence, we can refer to compiled definitions in macros such as `unrolledPowerCode` in `${ unrolledPowerCode('x, 2) }`. + +We can sumarize level consistency in two rules: + * Local variables can be used only at the same staging level as their definition + * Global variables can be used at any staging level -The previous section has shown that the metaprogramming framework has -to be able to take a type `T` and convert it to a type tree of type -`Type[T]` that can be reified. This means that all free variables of -the type tree refer to types and values defined in the current stage. -For a reference to a global class, this is easy: Just issue the fully -qualified name of the class. Members of reifiable types are handled by -just reifying the containing type together with the member name. But -what to do for references to type parameters or local type definitions -that are not defined in the current stage? Here, we cannot construct -the `Type[T]` tree directly, so we need to get it from a recursive -implicit search. For instance, to implement +#### Type consistency +As Scala uses type erasure, generic types will be erased at run-time and hence in any following stage. +To ensure any quoted expression that refers to a generic type `T` does not lose the information it needs, we require a given `Type[T]` in scope. +The `Type[T]` will carry over the non-erased representation of the type into the next phase. +Therefore any generic type used at a higher staging level than its definition will require its `Type`. +#### Scope extrusion +Within the contents of a splice, it is possible to have a quote that refers to a local variable defined in the outer quote. +If this quote is used within the splice, the variable will be in scope. +However, if the quote is somehow _extruded_ outside the splice, then variables might not be in scope anymore. +Quoted expressions can be extruded using side effects such as mutable state and exceptions. +The following example shows how a quote can be extruded using mutable state. ```scala -summon[Type[List[T]]] +var x: Expr[T] = null +'{ (y: T) => ${ x = 'y; 1 } } +x // has value '{y} but y is not in scope ``` -where `T` is not defined in the current stage, we construct the type constructor -of `List` applied to the splice of the result of searching for a given instance for `Type[T]`: +A second way a variable can be extruded is through the `run` method. +If `run` consumes a quoted variable reference, it will not be in scope anymore. +The result will reference a variable that is defined in the next stage. ```scala -Type.of[ List[ summon[Type[T]].Underlying ] ] +'{ (x: Int) => ${ run('x); ... } } +// evaluates to: '{ (x: Int) => ${ x; ... } 1 ``` -This is exactly the algorithm that Scala 2 uses to search for type tags. -In fact Scala 2's type tag feature can be understood as a more ad-hoc version of -`quoted.Type`. As was the case for type tags, the implicit search for a `quoted.Type` -is handled by the compiler, using the algorithm sketched above. +To catch both scope extrusion scenarios, our system restricts the use of quotes by only allowing a quote to be spliced if it was not extruded from a splice scope. +Unlike level consistency, this is checked at run-time[^4] rather than compile-time to avoid making the static type system too complicated. -## Relationship with `inline` +Each `Quotes` instance contains a unique scope identifier and refers to its parent scope, forming a stack of identifiers. +The parent of the scope of a `Quotes` is the scope of the `Quotes` used to create the enclosing quote. +Top-level splices and `run` create new scope stacks. +Every `Expr` knows in which scope it was created. +When it is spliced, we check that the quote scope is either the same as the splice scope, or a parent scope thereof. -Seen by itself, principled metaprogramming looks more like a framework for -runtime metaprogramming than one for compile-time metaprogramming with macros. -But combined with Scala 3’s `inline` feature it can be turned into a compile-time -system. The idea is that macro elaboration can be understood as a combination of -a macro library and a quoted program. For instance, here’s the `assert` macro -again together with a program that calls `assert`. -```scala -object Macros: +## Staged Lambdas - inline def assert(inline expr: Boolean): Unit = - ${ assertImpl('expr) } +When staging programs in a functional language there are two fundamental abstractions: a staged lambda `Expr[T => U]` and a staging lambda `Expr[T] => Expr[U]`. +The first is a function that will exist in the next stage, whereas the second is a function that exists in the current stage. +It is often convenient to have a mechanism to go from `Expr[T => U]` to `Expr[T] => Expr[U]` and vice versa. - def assertImpl(expr: Expr[Boolean])(using Quotes) = - val failMsg: Expr[String] = Expr("failed assertion: " + expr.show) - '{ if !($expr) then throw new AssertionError($failMsg) } +```scala +def later[T: Type, U: Type](f: Expr[T] => Expr[U]): Expr[T => U] = + '{ (x: T) => ${ f('x) } } -@main def program = - val x = 1 - Macros.assert(x != 0) +def now[T: Type, U: Type](f: Expr[T => U]): Expr[T] => Expr[U] = + (x: Expr[T]) => '{ $f($x) } ``` -Inlining the `assert` function would give the following program: +Both conversions can be performed out of the box with quotes and splices. +But if `f` is a known lambda function, `'{ $f($x) }` will not beta-reduce the lambda in place. +This optimization is performed in a later phase of the compiler. +Not reducing the application immediately can simplify analysis of generated code. +Nevertheless, it is possible to beta-reduce the lambda in place using the `Expr.betaReduce` method. ```scala -@main def program = - val x = 1 - ${ Macros.assertImpl('{ x != 0}) } +def now[T: Type, U: Type](f: Expr[T => U]): Expr[T] => Expr[U] = + (x: Expr[T]) => Expr.betaReduce('{ $f($x) }) ``` -The example is only phase correct because `Macros` is a global value and -as such not subject to phase consistency checking. Conceptually that’s -a bit unsatisfactory. If the PCP is so fundamental, it should be -applicable without the global value exception. But in the example as -given this does not hold since both `assert` and `program` call -`assertImpl` with a splice but no quote. +The `betaReduce` method will beta-reduce the outermost application of the expression if possible (regardless of arity). +If it is not possible to beta-reduce the expression, then it will return the original expression. -However, one could argue that the example is really missing -an important aspect: The macro library has to be compiled in a phase -prior to the program using it, but in the code above, macro -and program are defined together. A more accurate view of -macros would be to have the user program be in a phase after the macro -definitions, reflecting the fact that macros have to be defined and -compiled before they are used. Hence, conceptually the program part -should be treated by the compiler as if it was quoted: +## Staged Constructors +To create new class instances in a later stage, we can create them using factory methods (usually `apply` methods of an `object`), or we can instantiate them with a `new`. +For example, we can write `Some(1)` or `new Some(1)`, creating the same value. +In Scala 3, using the factory method call notation will fall back to a `new` if no `apply` method is found. +We follow the usual staging rules when calling a factory method. +Similarly, when we use a `new C`, the constructor of `C` is implicitly called, which also follows the usual staging rules. +Therefore for an arbitrary known class `C`, we can use both `'{ C(...) }` or `'{ new C(...) }` as constructors. +## Staged Classes +Quoted code can contain any valid expression including local class definitions. +This allows the creation of new classes with specialized implementations. +For example, we can implement a new version of `Runnable` that will perform some optimized operation. ```scala -@main def program = '{ - val x = 1 - ${ Macros.assertImpl('{ x != 0 }) } +def mkRunnable(x: Int)(using Quotes): Expr[Runnable] = '{ + class MyRunnable extends Runnable: + def run(): Unit = ... // generate some custom code that uses `x` + new MyRunnable } ``` -If `program` is treated as a quoted expression, the call to -`Macro.assertImpl` becomes phase correct even if macro library and -program are conceptualized as local definitions. - -But what about the call from `assert` to `assertImpl`? Here, we need a -tweak of the typing rules. An inline function such as `assert` that -contains a splice operation outside an enclosing quote is called a -_macro_. Macros are supposed to be expanded in a subsequent phase, -i.e. in a quoted context. Therefore, they are also type checked as if -they were in a quoted context. For instance, the definition of -`assert` is typechecked as if it appeared inside quotes. This makes -the call from `assert` to `assertImpl` phase-correct, even if we -assume that both definitions are local. +The quoted class is a local class and its type cannot escape the enclosing quote. +The class must be used inside the quote or an instance of it can be returned using a known interface (`Runnable` in this case). -The `inline` modifier is used to declare a `val` that is -either a constant or is a parameter that will be a constant when instantiated. This -aspect is also important for macro expansion. +## Quote Pattern Matching -To get values out of expressions containing constants `Expr` provides the method -`value` (or `valueOrError`). This will convert the `Expr[T]` into a `Some[T]` (or `T`) when the -expression contains value. Otherwise it will return `None` (or emit an error). -To avoid having incidental val bindings generated by the inlining of the `def` -it is recommended to use an inline parameter. To illustrate this, consider an -implementation of the `power` function that makes use of a statically known exponent: +It is sometimes necessary to analyze the structure of the code or decompose the code into its sub-expressions. +A classic example is an embedded DSL, where a macro knows a set of definitions that it can reinterpret while compiling the code (for instance, to perform optimizations). +In the following example, we extend our previous implementation of `powCode` to look into `x` to perform further optimizations. ```scala -inline def power(x: Double, inline n: Int) = ${ powerCode('x, 'n) } - -private def powerCode(x: Expr[Double], n: Expr[Int])(using Quotes): Expr[Double] = - n.value match - case Some(m) => powerCode(x, m) - case None => '{ Math.pow($x, $n.toDouble) } - -private def powerCode(x: Expr[Double], n: Int)(using Quotes): Expr[Double] = - if n == 0 then '{ 1.0 } - else if n == 1 then x - else if n % 2 == 0 then '{ val y = $x * $x; ${ powerCode('y, n / 2) } } - else '{ $x * ${ powerCode(x, n - 1) } } +def fusedPowCode(x: Expr[Double], n: Expr[Int])(using Quotes): Expr[Double] = + x match + case '{ power($y, $m) } => // we have (y^m)^n + fusedPowCode(y, '{ $n * $m }) // generate code for y^(n*m) + case _ => + '{ power($x, $n) } ``` -## Scope Extrusion - -Quotes and splices are duals as far as the PCP is concerned. But there is an -additional restriction that needs to be imposed on splices to guarantee -soundness: code in splices must be free of side effects. The restriction -prevents code like this: - -```scala -var x: Expr[T] = ... -'{ (y: T) => ${ x = 'y; 1 } } -``` -This code, if it was accepted, would _extrude_ a reference to a quoted variable -`y` from its scope. This would subsequently allow access to a variable outside the -scope where it is defined, which is likely problematic. The code is clearly -phase consistent, so we cannot use PCP to rule it out. Instead, we postulate a -future effect system that can guarantee that splices are pure. In the absence of -such a system we simply demand that spliced expressions are pure by convention, -and allow for undefined compiler behavior if they are not. This is analogous to -the status of pattern guards in Scala, which are also required, but not -verified, to be pure. +#### Sub-patterns -[Multi-Stage Programming](./staging.md) introduces one additional method where -you can expand code at runtime with a method `run`. There is also a problem with -that invocation of `run` in splices. Consider the following expression: +In quoted patterns, the `$` binds the sub-expression to an expression `Expr` that can be used in that `case` branch. +The contents of `${..}` in a quote pattern are regular Scala patterns. +For example, we can use the `Expr(_)` pattern within the `${..}` to only match if it is a known value and extract it. ```scala -'{ (x: Int) => ${ run('x); 1 } } +def fusedUnrolledPowCode(x: Expr[Double], n: Int)(using Quotes): Expr[Double] = + x match + case '{ power($y, ${Expr(m)}) } => // we have (y^m)^n + fusedUnrolledPowCode(y, n * m) // generate code for y * ... * y + case _ => // ( n*m times ) + unrolledPowerCode(x, n) ``` -This is again phase correct, but will lead us into trouble. Indeed, evaluating -the splice will reduce the expression `run('x)` to `x`. But then the result +These value extraction sub-patterns can be polymorphic using an instance of `FromExpr`. +In the following example, we show the implementation of `OptionFromExpr` which internally uses the `FromExpr[T]` to extract the value using the `Expr(x)` pattern. ```scala -'{ (x: Int) => ${ x; 1 } } +given OptionFromExpr[T](using Type[T], FromExpr[T]): FromExpr[Option[T]] with + def unapply(x: Expr[Option[T]])(using Quotes): Option[Option[T]] = + x match + case '{ Some( ${Expr(x)} ) } => Some(Some(x)) + case '{ None } => Some(None) + case _ => None ``` -is no longer phase correct. To prevent this soundness hole it seems easiest to -classify `run` as a side-effecting operation. It would thus be prevented from -appearing in splices. In a base language with side effects we would have to do this -anyway: Since `run` runs arbitrary code it can always produce a side effect if -the code it runs produces one. -## Example Expansion -Assume we have two methods, one `map` that takes an `Expr[Array[T]]` and a -function `f` and one `sum` that performs a sum by delegating to `map`. +#### Closed patterns +Patterns may contain two kinds of references: global references such as the call to the `power` method in `'{ power(...) }`, or references to bindings defined in the pattern such as `x` in `case '{ (x: Int) => x }`. +When extracting an expression from a quote, we need to ensure that we do not extrude any variable from the scope where it is defined. ```scala -object Macros: - - def map[T](arr: Expr[Array[T]], f: Expr[T] => Expr[Unit]) - (using Type[T], Quotes): Expr[Unit] = '{ - var i: Int = 0 - while i < ($arr).length do - val element: T = ($arr)(i) - ${f('element)} - i += 1 - } - - def sum(arr: Expr[Array[Int]])(using Quotes): Expr[Int] = '{ - var sum = 0 - ${ map(arr, x => '{sum += $x}) } - sum - } - - inline def sum_m(arr: Array[Int]): Int = ${sum('arr)} - -end Macros +'{ (x: Int) => x + 1 } match + case '{ (y: Int) => $z } => + // should not match, otherwise: z = '{ x + 1 } ``` -A call to `sum_m(Array(1,2,3))` will first inline `sum_m`: +In this example, we see that the pattern should not match. +Otherwise, any use of the expression `z` would contain an unbound reference to `x`. +To avoid any such extrusion, we only match on a `${..}` if its expression is closed under the definitions within the pattern. +Therefore, the pattern will not match if the expression is not closed. + +#### HOAS patterns +To allow extracting expressions that may contain extruded references we offer a _higher-order abstract syntax_ (HOAS) pattern `$f(y)` (or `$f(y1,...,yn)`). +This pattern will eta-expand the sub-expression with respect to `y` and bind it to `f`. +The lambda arguments will replace the variables that might have been extruded. ```scala -val arr: Array[Int] = Array.apply(1, [2,3 : Int]:Int*) -${_root_.Macros.sum('arr)} +'{ ((x: Int) => x + 1).apply(2) } match + case '{ ((y: Int) => $f(y)).apply($z: Int) } => + // f may contain references to `x` (replaced by `$y`) + // f = (y: Expr[Int]) => '{ $y + 1 } + f(z) // generates '{ 2 + 1 } ``` -then it will splice `sum`: -```scala -val arr: Array[Int] = Array.apply(1, [2,3 : Int]:Int*) +A HOAS pattern `$x(y1,...,yn)` will only match the expression if it does not contain references to variables defined in the pattern that are not in the set `y1,...,yn`. +In other words, the pattern will match if the expression only contains references to variables defined in the pattern that are in `y1,...,yn`. +Note that the HOAS patterns `$x()` are semantically equivalent to closed patterns `$x`. -var sum = 0 -${ map('arr, x => '{sum += $x}) } -sum -``` -then it will inline `map`: +#### Type variables +Expressions may contain types that are not statically known. +For example, an `Expr[List[Int]]` may contain `list.map(_.toInt)` where `list` is a `List` of some type. +To cover all the possible cases we would need to explicitly match `list` on all possible types (`List[Int]`, `List[Int => Int]`, ...). +This is an infinite set of types and therefore pattern cases. +Even if we would know all possible types that a specific program could use, we may still end up with an unmanageable number of cases. +To overcome this, we introduce type variables in quoted patterns, which will match any type. + +In the following example, we show how type variables `t` and `u` match all possible pairs of consecutive calls to `map` on lists. +In the quoted patterns, types named with lower cases are identified as type variables. +This follows the same notation as type variables used in normal patterns. ```scala -val arr: Array[Int] = Array.apply(1, [2,3 : Int]:Int*) +def fuseMapCode(x: Expr[List[Int]]): Expr[List[Int]] = + x match + case '{ ($ls: List[t]).map[u]($f).map[Int]($g) } => + '{ $ls.map($g.compose($f)) } + ... -var sum = 0 -val f = x => '{sum += $x} -${ _root_.Macros.map('arr, 'f)(Type.of[Int])} -sum +fuseMapCode('{ List(1.2).map(f).map(g) }) // '{ List(1.2).map(g.compose(f)) } +fuseMapCode('{ List('a').map(h).map(i) }) // '{ List('a').map(i.compose(h)) } ``` +Variables `f` and `g` are inferred to be of type `Expr[t => u]` and `Expr[u => Int]` respectively. +Subsequently, we can infer `$g.compose($f)` to be of type `Expr[t => Int]` which is the type of the argument of `$ls.map(..)`. -then it will expand and splice inside quotes `map`: +Type variables are abstract types that will be erased; this implies that to reference them in the second quote we need a given `Type[t]` and `Type[u]`. +The quoted pattern will implicitly provide those given types. +At run-time, when the pattern matches, the type of `t` and `u` will be known, and the `Type[t]` and `Type[u]` will contain the precise types in the expression. -```scala -val arr: Array[Int] = Array.apply(1, [2,3 : Int]:Int*) +As `Expr` is covariant, the statically known type of the expression might not be the actual type. +Type variables can also be used to recover the precise type of the expression. +``scala +def let(x: Expr[Any])(using Quotes): Expr[Any] = + x match + case '{ $x: t } => + '{ val y: t = $x; y } -var sum = 0 -val f = x => '{sum += $x} -var i: Int = 0 -while i < arr.length do - val element: Int = (arr)(i) - sum += element - i += 1 -sum +let('{1}) // will return a `Expr[Any]` that contains an `Expr[Int]]` ``` -Finally cleanups and dead code elimination: - +While we can define the type variable in the middle of the pattern, their normal form is to define them as a `type` with a lower case name at the start of the pattern. +We use the Scala backquote `` `t` `` naming convention which interprets the string within the backquote as a literal name identifier. +This is typically used when we have names that contain special characters that are not allowed for normal Scala identifiers. +But we use it to explicitly state that this is a reference to that name and not the introduction of a new variable. ```scala -val arr: Array[Int] = Array.apply(1, [2,3 : Int]:Int*) -var sum = 0 -var i: Int = 0 -while i < arr.length do - val element: Int = arr(i) - sum += element - i += 1 -sum + case '{ type t; $x: `t` } => ``` - -## Find implicits within a macro - -Similarly to the `summonFrom` construct, it is possible to make implicit search available -in a quote context. For this we simply provide `scala.quoted.Expr.summon`: +This is a bit more verbose but has some expressivity advantages such as allowing to define bounds on the variables and be able to refer to them several times in any scope of the pattern. ```scala -import scala.collection.immutable.{ TreeSet, HashSet } -inline def setFor[T]: Set[T] = ${ setForExpr[T] } - -def setForExpr[T: Type](using Quotes): Expr[Set[T]] = - Expr.summon[Ordering[T]] match - case Some(ord) => '{ new TreeSet[T]()($ord) } - case _ => '{ new HashSet[T] } + case '{ type t >: List[Int] <: Seq[Int]; $x: `t` } => + case '{ type t; $x: (`t`, `t`) } => ``` -## Relationship with Transparent Inline -[Inline](./inline.md) documents inlining. The code below introduces a transparent -inline method that can calculate either a value of type `Int` or a value of type -`String`. +#### Type patterns +It is possible to only have a type and no expression of that type. +To be able to inspect a type, we introduce quoted type pattern `case '[..] =>`. +It works the same way as a quoted pattern but is restricted to contain a type. +Type variables can be used in quoted type patterns to extract a type. ```scala -transparent inline def defaultOf(inline str: String) = - ${ defaultOfImpl('str) } - -def defaultOfImpl(strExpr: Expr[String])(using Quotes): Expr[Any] = - strExpr.valueOrError match - case "int" => '{1} - case "string" => '{"a"} - -// in a separate file -val a: Int = defaultOf("int") -val b: String = defaultOf("string") - +def empty[T: Type]: Expr[T] = + Type.of[T] match + case '[String] => '{ "" } + case '[List[t]] => '{ List.empty[t] } + ... ``` -## Defining a macro and using it in a single project - -It is possible to define macros and use them in the same project as long as the implementation -of the macros does not have run-time dependencies on code in the file where it is used. -It might still have compile-time dependencies on types and quoted code that refers to the use-site file. - -To provide this functionality Scala 3 provides a transparent compilation mode where files that -try to expand a macro but fail because the macro has not been compiled yet are suspended. -If there are any suspended files when the compilation ends, the compiler will automatically restart -compilation of the suspended files using the output of the previous (partial) compilation as macro classpath. -In case all files are suspended due to cyclic dependencies the compilation will fail with an error. +`Type.of[T]` is used to summon the given instance of `Type[T]` in scope, it is equivalent to `summon[Type[T]]`. -## Pattern matching on quoted expressions +#### Type testing and casting +It is important to note that instance checks and casts on `Expr`, such as `isInstanceOf[Expr[T]]` and `asInstanceOf[Expr[T]]`, will only check if the instance is of the class `Expr` but will not be able to check the `T` argument. +These cases will issue a warning at compile-time, but if they are ignored, they can result in unexpected behavior. -It is possible to deconstruct or extract values out of `Expr` using pattern matching. +These operations can be supported correctly in the system. +For a simple type test it is possible to use the `isExprOf[T]` method of `Expr` to check if it is an instance of that type. +Similarly, it is possible to use `asExprOf[T]` to cast an expression to a given type. +These operations use a given `Type[T]` to work around type erasure. -`scala.quoted` contains objects that can help extracting values from `Expr`. -- `scala.quoted.Expr`/`scala.quoted.Exprs`: matches an expression of a value (or list of values) and returns the value (or list of values). -- `scala.quoted.Const`/`scala.quoted.Consts`: Same as `Expr`/`Exprs` but only works on primitive values. -- `scala.quoted.Varargs`: matches an explicit sequence of expressions and returns them. These sequences are useful to get individual `Expr[T]` out of a varargs expression of type `Expr[Seq[T]]`. +## Sub-Expression Transformation -These could be used in the following way to optimize any call to `sum` that has statically known values. +The system provides a mechanism to transform all sub-expressions of an expression. +This is useful when the sub-expressions we want to transform are deep in the expression. +It is also necessary if the expression contains sub-expressions that cannot be matched using quoted patterns (such as local class definitions). ```scala -inline def sum(inline args: Int*): Int = ${ sumExpr('args) } -private def sumExpr(argsExpr: Expr[Seq[Int]])(using Quotes): Expr[Int] = - argsExpr match - case Varargs(args @ Exprs(argValues)) => - // args is of type Seq[Expr[Int]] - // argValues is of type Seq[Int] - Expr(argValues.sum) // precompute result of sum - case Varargs(argExprs) => // argExprs is of type Seq[Expr[Int]] - val staticSum: Int = argExprs.map(_.value.getOrElse(0)).sum - val dynamicSum: Seq[Expr[Int]] = argExprs.filter(_.value.isEmpty) - dynamicSum.foldLeft(Expr(staticSum))((acc, arg) => '{ $acc + $arg }) - case _ => - '{ $argsExpr.sum } +trait ExprMap: + def transform[T](e: Expr[T])(using Type[T])(using Quotes): Expr[T] + def transformChildren[T](e: Expr[T])(using Type[T])(using Quotes): Expr[T] = + ... ``` -### Quoted patterns - -Quoted pattens allow deconstructing complex code that contains a precise structure, types or methods. -Patterns `'{ ... }` can be placed in any location where Scala expects a pattern. - -For example +Users can extend the `ExprMap` trait and implement the `transform` method. +This interface is flexible and can implement top-down, bottom-up, or other transformations. ```scala -optimize { - sum(sum(1, a, 2), 3, b) -} // should be optimized to 6 + a + b -``` - -```scala -def sum(args: Int*): Int = args.sum -inline def optimize(inline arg: Int): Int = ${ optimizeExpr('arg) } -private def optimizeExpr(body: Expr[Int])(using Quotes): Expr[Int] = - body match - // Match a call to sum without any arguments - case '{ sum() } => Expr(0) - // Match a call to sum with an argument $n of type Int. - // n will be the Expr[Int] representing the argument. - case '{ sum($n) } => n - // Match a call to sum and extracts all its args in an `Expr[Seq[Int]]` - case '{ sum(${Varargs(args)}: _*) } => sumExpr(args) - case body => body - -private def sumExpr(args1: Seq[Expr[Int]])(using Quotes): Expr[Int] = - def flatSumArgs(arg: Expr[Int]): Seq[Expr[Int]] = arg match - case '{ sum(${Varargs(subArgs)}: _*) } => subArgs.flatMap(flatSumArgs) - case arg => Seq(arg) - val args2 = args1.flatMap(flatSumArgs) - val staticSum: Int = args2.map(_.value.getOrElse(0)).sum - val dynamicSum: Seq[Expr[Int]] = args2.filter(_.value.isEmpty) - dynamicSum.foldLeft(Expr(staticSum))((acc, arg) => '{ $acc + $arg }) +object OptimizeIdentity extends ExprMap: + def transform[T](e: Expr[T])(using Type[T])(using Quotes): Expr[T] = + transformChildren(e) match // bottom-up transformation + case '{ identity($x) } => x + case _ => e ``` -### Recovering precise types using patterns +The `transformChildren` method is implemented as a primitive that knows how to reach all the direct sub-expressions and calls `transform` on each one. +The type passed to `transform` is the expected type of this sub-expression in its expression. +For example while transforming `Some(1)` in `'{ val x: Option[Int] = Some(1); ...}` the type will be `Option[Int]` and not `Some[Int]`. +This implies that we can safely transform `Some(1)` into `None`. -Sometimes it is necessary to get a more precise type for an expression. This can be achieved using the following pattern match. +## Staged Implicit Summoning +When summoning implicit arguments using `summon`, we will find the given instances in the current scope. +It is possible to use `summon` to get staged implicit arguments by explicitly staging them first. +In the following example, we can pass an implicit `Ordering[T]` in a macro as an `Expr[Ordering[T]]` to its implementation. +Then we can splice it and give it implicitly in the next stage. ```scala -def f(expr: Expr[Any])(using Quotes) = expr match - case '{ $x: t } => - // If the pattern match succeeds, then there is - // some type `t` such that - // - `x` is bound to a variable of type `Expr[t]` - // - `t` is bound to a new type `t` and a given - // instance `Type[t]` is provided for it - // That is, we have `x: Expr[t]` and `given Type[t]`, - // for some (unknown) type `t`. -``` +inline def treeSetFor[T](using ord: Ordering[T]): Set[T] = + ${ setExpr[T](using 'ord) } -This might be used to then perform an implicit search as in: - -```scala -extension (inline sc: StringContext) - inline def showMe(inline args: Any*): String = ${ showMeExpr('sc, 'args) } - -private def showMeExpr(sc: Expr[StringContext], argsExpr: Expr[Seq[Any]])(using Quotes): Expr[String] = - import quotes.reflect.report - argsExpr match - case Varargs(argExprs) => - val argShowedExprs = argExprs.map { - case '{ $arg: tp } => - Expr.summon[Show[tp]] match - case Some(showExpr) => - '{ $showExpr.show($arg) } - case None => - report.error(s"could not find implicit for ${Type.show[Show[tp]]}", arg); '{???} - } - val newArgsExpr = Varargs(argShowedExprs) - '{ $sc.s($newArgsExpr: _*) } - case _ => - // `new StringContext(...).showMeExpr(args: _*)` not an explicit `showMeExpr"..."` - report.error(s"Args must be explicit", argsExpr) - '{???} - -trait Show[-T]: - def show(x: T): String - -// in a different file -given Show[Boolean] with - def show(b: Boolean) = "boolean!" - -println(showMe"${true}") +def setExpr[T:Type](using ord: Expr[Ordering[T]])(using Quotes): Expr[Set[T]] = + '{ given Ordering[T] = $ord; new TreeSet[T]() } ``` -### Open code patterns - -Quoted pattern matching also provides higher-order patterns to match open terms. If a quoted term contains a definition, -then the rest of the quote can refer to this definition. - -```scala -'{ - val x: Int = 4 - x * x -} -``` +We pass it as an implicit `Expr[Ordering[T]]` because there might be intermediate methods that can pass it along implicitly. -To match such a term we need to match the definition and the rest of the code, but we need to explicitly state that the rest of the code may refer to this definition. +An alternative is to summon implicit values in the scope where the macro is invoked. +Using the `Expr.summon` method we get an optional expression containing the implicit instance. +This provides the ability to search for implicit instances conditionally. ```scala -case '{ val y: Int = $x; $body(y): Int } => +def summon[T: Type](using Quotes): Option[Expr[T]] ``` -Here `$x` will match any closed expression while `$body(y)` will match an expression that is closed under `y`. Then -the subexpression of type `Expr[Int]` is bound to `body` as an `Expr[Int => Int]`. The extra argument represents the references to `y`. Usually this expression is used in combination with `Expr.betaReduce` to replace the extra argument. - ```scala -inline def eval(inline e: Int): Int = ${ evalExpr('e) } - -private def evalExpr(e: Expr[Int])(using Quotes): Expr[Int] = e match - case '{ val y: Int = $x; $body(y): Int } => - // body: Expr[Int => Int] where the argument represents - // references to y - evalExpr(Expr.betaReduce('{$body(${evalExpr(x)})})) - case '{ ($x: Int) * ($y: Int) } => - (x.value, y.value) match - case (Some(a), Some(b)) => Expr(a * b) - case _ => e - case _ => e -``` +inline def setFor[T]: Set[T] = + ${ setForExpr[T] } -```scala -eval { // expands to the code: (16: Int) - val x: Int = 4 - x * x -} +def setForExpr[T: Type]()(using Quotes): Expr[Set[T]] = + Expr.summon[Ordering[T]] match + case Some(ord) => + '{ new TreeSet[T]()($ord) } + case _ => + '{ new HashSet[T] } ``` -We can also close over several bindings using `$b(a1, a2, ..., an)`. -To match an actual application we can use braces on the function part `${b}(a1, a2, ..., an)`. - ## More details [More details](./macros-spec.md) + + +[^1]: [Scalable Metaprogramming in Scala 3](https://infoscience.epfl.ch/record/299370) +[^2]: [Semantics-preserving inlining for metaprogramming](https://dl.acm.org/doi/10.1145/3426426.3428486) +[^3]: Implemented in the Scala 3 Dotty project https://github.com/lampepfl/dotty. sbt library dependency `"org.scala-lang" %% "scala3-staging" % scalaVersion.value` +[^4]: Using the `-Xcheck-macros` compiler flag diff --git a/docs/_spec/TODOreference/metaprogramming/staging.md b/docs/_spec/TODOreference/metaprogramming/staging.md index e74d491402b5..6d9166e8249e 100644 --- a/docs/_spec/TODOreference/metaprogramming/staging.md +++ b/docs/_spec/TODOreference/metaprogramming/staging.md @@ -60,7 +60,7 @@ impose the following restrictions on the use of splices. The framework as discussed so far allows code to be staged, i.e. be prepared to be executed at a later stage. To run that code, there is another method in class `Expr` called `run`. Note that `$` and `run` both map from `Expr[T]` -to `T` but only `$` is subject to the [PCP](./macros.md#the-phase-consistency-principle), whereas `run` is just a normal method. +to `T` but only `$` is subject to the [Cross-Stage Safety](./macros.md#cross-stage-safety), whereas `run` is just a normal method. `scala.quoted.staging.run` provides a `Quotes` that can be used to show the expression in its scope. On the other hand `scala.quoted.staging.withQuotes` provides a `Quotes` without evaluating the expression. diff --git a/tests/pos-with-compiler-cc/dotc/core/StagingContext.scala b/tests/pos-with-compiler-cc/dotc/core/StagingContext.scala index 9e0bb95394a3..4ca53e02a831 100644 --- a/tests/pos-with-compiler-cc/dotc/core/StagingContext.scala +++ b/tests/pos-with-compiler-cc/dotc/core/StagingContext.scala @@ -4,7 +4,7 @@ import dotty.tools.dotc.ast.tpd import dotty.tools.dotc.core.Contexts._ import dotty.tools.dotc.ast.tpd import dotty.tools.dotc.util.Property -import dotty.tools.dotc.transform.PCPCheckAndHeal +import dotty.tools.dotc.transform.CrossStageSafety object StagingContext { @@ -16,7 +16,7 @@ object StagingContext { */ private val QuotesStack = new Property.Key[List[tpd.Tree]] - private val TaggedTypes = new Property.Key[PCPCheckAndHeal.QuoteTypeTags] + private val TaggedTypes = new Property.Key[CrossStageSafety.QuoteTypeTags] /** All enclosing calls that are currently inlined, from innermost to outermost. */ def level(using Context): Int = @@ -36,10 +36,10 @@ object StagingContext { def spliceContext(using Context): Context = ctx.fresh.setProperty(QuotationLevel, level - 1) - def contextWithQuoteTypeTags(taggedTypes: PCPCheckAndHeal.QuoteTypeTags)(using Context) = + def contextWithQuoteTypeTags(taggedTypes: CrossStageSafety.QuoteTypeTags)(using Context) = ctx.fresh.setProperty(TaggedTypes, taggedTypes) - def getQuoteTypeTags(using Context): PCPCheckAndHeal.QuoteTypeTags = + def getQuoteTypeTags(using Context): CrossStageSafety.QuoteTypeTags = ctx.property(TaggedTypes).get /** Context with a decremented quotation level and pops the Some of top of the quote context stack or None if the stack is empty. diff --git a/tests/pos-with-compiler-cc/dotc/inlines/PrepareInlineable.scala b/tests/pos-with-compiler-cc/dotc/inlines/PrepareInlineable.scala index ecf24ff8264e..9bb0bacd7a78 100644 --- a/tests/pos-with-compiler-cc/dotc/inlines/PrepareInlineable.scala +++ b/tests/pos-with-compiler-cc/dotc/inlines/PrepareInlineable.scala @@ -17,7 +17,7 @@ import NameKinds.{InlineAccessorName, UniqueInlineName} import inlines.Inlines import NameOps._ import Annotations._ -import transform.{AccessProxies, PCPCheckAndHeal, Splicer} +import transform.{AccessProxies, CrossStageSafety, Splicer} import transform.SymUtils.* import config.Printers.inlining import util.Property @@ -294,7 +294,7 @@ object PrepareInlineable { if (code.symbol.flags.is(Inline)) report.error("Macro cannot be implemented with an `inline` method", code.srcPos) Splicer.checkValidMacroBody(code) - new PCPCheckAndHeal(freshStagingContext).transform(body) // Ignore output, only check PCP + new CrossStageSafety(freshStagingContext).transform(body) // Ignore output, only check staging levels case Block(List(stat), Literal(Constants.Constant(()))) => checkMacro(stat) case Block(Nil, expr) => checkMacro(expr) case Typed(expr, _) => checkMacro(expr) diff --git a/tests/pos-with-compiler-cc/dotc/transform/PCPCheckAndHeal.scala b/tests/pos-with-compiler-cc/dotc/transform/CrossStageSafety.scala similarity index 95% rename from tests/pos-with-compiler-cc/dotc/transform/PCPCheckAndHeal.scala rename to tests/pos-with-compiler-cc/dotc/transform/CrossStageSafety.scala index 90128500374e..ca00c87161ef 100644 --- a/tests/pos-with-compiler-cc/dotc/transform/PCPCheckAndHeal.scala +++ b/tests/pos-with-compiler-cc/dotc/transform/CrossStageSafety.scala @@ -22,14 +22,14 @@ import dotty.tools.dotc.util.Property import scala.annotation.constructorOnly -/** Checks that the Phase Consistency Principle (PCP) holds and heals types. +/** Checks that staging level consistency holds and heals staged types . * - * Local term references are phase consistent if and only if they are used at the same level as their definition. + * Local term references are level consistent if and only if they are used at the same level as their definition. * * Local type references can be used at the level of their definition or lower. If used used at a higher level, * it will be healed if possible, otherwise it is inconsistent. * - * Type healing consists in transforming a phase inconsistent type `T` into `summon[Type[T]].Underlying`. + * Type healing consists in transforming a level inconsistent type `T` into `summon[Type[T]].Underlying`. * * As references to types do not necessarily have an associated tree it is not always possible to replace the types directly. * Instead we always generate a type alias for it and place it at the start of the surrounding quote. This also avoids duplication. @@ -48,7 +48,7 @@ import scala.annotation.constructorOnly * } * */ -class PCPCheckAndHeal(@constructorOnly ictx: DetachedContext) extends TreeMapWithStages(ictx), Checking, caps.Pure { +class CrossStageSafety(@constructorOnly ictx: DetachedContext) extends TreeMapWithStages(ictx), Checking, caps.Pure { import tpd._ private val InAnnotation = Property.Key[Unit]() @@ -96,9 +96,9 @@ class PCPCheckAndHeal(@constructorOnly ictx: DetachedContext) extends TreeMapWit super.transform(tree) } - /** Transform quoted trees while maintaining phase correctness */ + /** Transform quoted trees while maintaining level correctness */ override protected def transformQuotation(body: Tree, quote: Apply)(using Context): Tree = { - val taggedTypes = new PCPCheckAndHeal.QuoteTypeTags(quote.span) + val taggedTypes = new CrossStageSafety.QuoteTypeTags(quote.span) if (ctx.property(InAnnotation).isDefined) report.error("Cannot have a quote in an annotation", quote.srcPos) @@ -215,7 +215,7 @@ class PCPCheckAndHeal(@constructorOnly ictx: DetachedContext) extends TreeMapWit mapOver(tp) } - /** Check phase consistency of terms and heal inconsistent type references. */ + /** Check level consistency of terms and heal inconsistent type references. */ private def healTypeOfTerm(pos: SrcPos)(using Context) = new TypeMap { def apply(tp: Type): Type = tp match @@ -275,7 +275,7 @@ class PCPCheckAndHeal(@constructorOnly ictx: DetachedContext) extends TreeMapWit } -object PCPCheckAndHeal { +object CrossStageSafety { import tpd._ class QuoteTypeTags(span: Span)(using DetachedContext) extends caps.Pure { diff --git a/tests/pos-with-compiler-cc/dotc/transform/Splicing.scala b/tests/pos-with-compiler-cc/dotc/transform/Splicing.scala index df6128d249d2..3a3aa7e89445 100644 --- a/tests/pos-with-compiler-cc/dotc/transform/Splicing.scala +++ b/tests/pos-with-compiler-cc/dotc/transform/Splicing.scala @@ -191,7 +191,7 @@ class Splicing extends MacroTransform: private var refBindingMap = mutable.Map.empty[Symbol, (Tree, Symbol)] /** Reference to the `Quotes` instance of the current level 1 splice */ private var quotes: Tree | Null = null // TODO: add to the context - private var healedTypes: PCPCheckAndHeal.QuoteTypeTags | Null = null // TODO: add to the context + private var healedTypes: CrossStageSafety.QuoteTypeTags | Null = null // TODO: add to the context def transformSplice(tree: tpd.Tree, tpe: Type, holeIdx: Int)(using Context): tpd.Tree = assert(level == 0) @@ -254,7 +254,7 @@ class Splicing extends MacroTransform: private def transformLevel0QuoteContent(tree: Tree)(using Context): Tree = // transform and collect new healed types val old = healedTypes - healedTypes = new PCPCheckAndHeal.QuoteTypeTags(tree.span) + healedTypes = new CrossStageSafety.QuoteTypeTags(tree.span) val tree1 = transform(tree) val newHealedTypes = healedTypes.nn.getTypeTags healedTypes = old diff --git a/tests/pos-with-compiler-cc/dotc/transform/Staging.scala b/tests/pos-with-compiler-cc/dotc/transform/Staging.scala index 12c5c8215cad..c2c6f76cd0fc 100644 --- a/tests/pos-with-compiler-cc/dotc/transform/Staging.scala +++ b/tests/pos-with-compiler-cc/dotc/transform/Staging.scala @@ -15,7 +15,7 @@ import dotty.tools.dotc.transform.TreeMapWithStages._ -/** Checks that the Phase Consistency Principle (PCP) holds and heals types. +/** Checks that staging level consistency holds and heals staged types. * * Type healing consists in transforming a phase inconsistent type `T` into `${ implicitly[Type[T]] }`. */ @@ -32,12 +32,12 @@ class Staging extends MacroTransform { override def checkPostCondition(tree: Tree)(using Context): Unit = if (ctx.phase <= splicingPhase) { - // Recheck that PCP holds but do not heal any inconsistent types as they should already have been heald + // Recheck that staging levels hold but do not heal any inconsistent types as they should already have been heald tree match { case PackageDef(pid, _) if tree.symbol.owner == defn.RootClass => val stagingCtx = freshStagingContext - val checker = new PCPCheckAndHeal(stagingCtx) { - // !cc! type error is checker is defined as val checker = new PCPCheckAndHeal { ... } + val checker = new CrossStageSafety(stagingCtx) { + // !cc! type error is checker is defined as val checker = new CrossStageSafety { ... } override protected def tryHeal(sym: Symbol, tp: TypeRef, pos: SrcPos)(using Context): TypeRef = { def symStr = if (sym.is(ModuleClass)) sym.sourceModule.show @@ -72,7 +72,7 @@ class Staging extends MacroTransform { protected def newTransformer(using Context): Transformer = new Transformer { override def transform(tree: tpd.Tree)(using Context): tpd.Tree = - new PCPCheckAndHeal(ctx.detach).transform(tree) + new CrossStageSafety(ctx.detach).transform(tree) } } diff --git a/tests/pos-with-compiler-cc/dotc/transform/TreeChecker.scala b/tests/pos-with-compiler-cc/dotc/transform/TreeChecker.scala index e7607d8e59c6..b0d7fb89985f 100644 --- a/tests/pos-with-compiler-cc/dotc/transform/TreeChecker.scala +++ b/tests/pos-with-compiler-cc/dotc/transform/TreeChecker.scala @@ -458,7 +458,7 @@ class TreeChecker extends Phase with SymTransformer { val inliningPhase = ctx.base.inliningPhase inliningPhase.exists && ctx.phase.id > inliningPhase.id if isAfterInlining then - // The staging phase destroys in PCPCheckAndHeal the property that + // The staging phase destroys in CrossStageSafety the property that // tree.expr.tpe <:< pt1. A test case where this arises is run-macros/enum-nat-macro. // We should follow up why this happens. If the problem is fixed, we can // drop the isAfterInlining special case. To reproduce the problem, just