Add syntax for higher-order quote pattern holes

* Make typer explicitly type higher-order quasiquotes
* Add higher-order quasi-pattern concept to the Matcher (avoid ambiguities with splice applied to arguments)
* Introduce unambiguous syntax for higher-order quasi-patterns
* Add documentation
* Warn if old syntax is detected

Author: nicolasstucki

compiler/src/dotty/tools/dotc/core/Definitions.scala

@@ -695,6 +695,8 @@ class Definitions {
     @tu lazy val InternalQuoted_exprNestedSplice : Symbol = InternalQuotedModule.requiredMethod("exprNestedSplice")
     @tu lazy val InternalQuoted_typeQuote  : Symbol = InternalQuotedModule.requiredMethod("typeQuote")
     @tu lazy val InternalQuoted_patternHole: Symbol = InternalQuotedModule.requiredMethod("patternHole")
+    @tu lazy val InternalQuoted_patternHigherOrderHole: Symbol = InternalQuotedModule.requiredMethod("patternHigherOrderHole")
+    @tu lazy val InternalQuoted_higherOrderHole: Symbol = InternalQuotedModule.requiredMethod("higherOrderHole")
     @tu lazy val InternalQuoted_patternTypeAnnot: ClassSymbol = InternalQuotedModule.requiredClass("patternType")
     @tu lazy val InternalQuoted_QuoteTypeTagAnnot: ClassSymbol = InternalQuotedModule.requiredClass("quoteTypeTag")
     @tu lazy val InternalQuoted_fromAboveAnnot: ClassSymbol = InternalQuotedModule.requiredClass("fromAbove")

compiler/src/dotty/tools/dotc/tastyreflect/ReflectionCompilerInterface.scala

@@ -1983,6 +1983,7 @@ class ReflectionCompilerInterface(val rootContext: core.Contexts.Context) extend
   def Definitions_isTupleClass(sym: Symbol): Boolean = defn.isTupleClass(sym)
 
   def Definitions_InternalQuoted_patternHole: Symbol = defn.InternalQuoted_patternHole
+  def Definitions_InternalQuoted_higherOrderHole: Symbol = defn.InternalQuoted_higherOrderHole
   def Definitions_InternalQuoted_patternTypeAnnot: Symbol = defn.InternalQuoted_patternTypeAnnot
   def Definitions_InternalQuoted_fromAboveAnnot: Symbol = defn.InternalQuoted_fromAboveAnnot
 

compiler/src/dotty/tools/dotc/typer/QuotesAndSplices.scala

@@ -75,13 +75,41 @@ trait QuotesAndSplices {
     }
     if (ctx.mode.is(Mode.QuotedPattern))
       if (isFullyDefined(pt, ForceDegree.flipBottom)) {
-        def spliceOwner(ctx: Context): Symbol =
-          if (ctx.mode.is(Mode.QuotedPattern)) spliceOwner(ctx.outer) else ctx.owner
-        val pat = typedPattern(tree.expr, defn.QuotedExprClass.typeRef.appliedTo(pt))(
-          using spliceContext.retractMode(Mode.QuotedPattern).withOwner(spliceOwner(ctx)))
-        val baseType = pat.tpe.baseType(defn.QuotedExprClass)
-        val argType = if baseType != NoType then baseType.argTypesHi.head else defn.NothingType
-        ref(defn.InternalQuoted_exprSplice).appliedToType(argType).appliedTo(pat)
+        object HigherOrderQuasipattern {
+          /** Matches a splice of the form `${<pat> $$ <args*>}` */
+          def unapply(tree: untpd.Tree): Option[(untpd.Tree, List[untpd.Tree])] =
+            tree match
+              case untpd.InfixOp(pat, untpd.Ident(nme.EXPAND_SEPARATOR), args0) =>
+                val args = args0 match
+                  case untpd.Tuple(args) => args
+                  case arg => List(arg)
+                Some((pat, args))
+              case Apply(Select(pat, nme.EXPAND_SEPARATOR), args) => Some((pat, args))
+              case _ => None
+        }
+        tree.expr match
+          case HigherOrderQuasipattern(pat, args) =>
+            val typedArgs = args.map {
+              case arg: untpd.Ident =>
+                typedExpr(arg)
+              case arg =>
+                ctx.error("Exprected an identifier", arg.sourcePos)
+                EmptyTree
+            }
+            if args.isEmpty then
+              ctx.error("Missing arguments for open pattern", tree.expr.sourcePos)
+            val argTypes = typedArgs.map(_.tpe.widenTermRefExpr)
+            val splice1 = untpd.cpy.Splice(tree)(pat).asInstanceOf[untpd.Splice]
+            val typedPat = typedSplice(splice1, defn.FunctionOf(argTypes, pt))
+            ref(defn.InternalQuoted_patternHigherOrderHole).appliedToType(pt).appliedTo(typedPat, SeqLiteral(typedArgs, TypeTree(defn.AnyType)))
+          case _ =>
+            def spliceOwner(ctx: Context): Symbol =
+              if (ctx.mode.is(Mode.QuotedPattern)) spliceOwner(ctx.outer) else ctx.owner
+            val pat = typedPattern(tree.expr, defn.QuotedExprClass.typeRef.appliedTo(pt))(
+              using spliceContext.retractMode(Mode.QuotedPattern).withOwner(spliceOwner(ctx)))
+            val baseType = pat.tpe.baseType(defn.QuotedExprClass)
+            val argType = if baseType != NoType then baseType.argTypesHi.head else defn.NothingType
+            ref(defn.InternalQuoted_exprSplice).appliedToType(argType).appliedTo(pat)
       }
       else {
         ctx.error(i"Type must be fully defined.\nConsider annotating the splice using a type ascription:\n  ($tree: XYZ).", tree.expr.sourcePos)
@@ -119,6 +147,14 @@ trait QuotesAndSplices {
   def typedAppliedSplice(tree: untpd.Apply, pt: Type)(using Context): Tree = {
     assert(ctx.mode.is(Mode.QuotedPattern))
     val untpd.Apply(splice: untpd.Splice, args) = tree
+    if args.forall(_.isInstanceOf[untpd.Ident]) then
+      val suggestionArgs = args match
+        case arg :: Nil => arg.show
+        case args => args.map(_.show).mkString("(", ", ", ")")
+      val suggestion = s"$${${splice.expr.show} $$$$ $suggestionArgs}"
+      ctx.warning(s"""Possibly using open pattern syntax instead application pattern.
+                     |If this is a open pattern replace it with $suggestion.
+                     |""".stripMargin, tree.sourcePos)
     if (isFullyDefined(pt, ForceDegree.flipBottom)) then
       val typedArgs = args.map(arg => typedExpr(arg))
       val argTypes = typedArgs.map(_.tpe.widenTermRefExpr)
@@ -224,6 +260,14 @@ trait QuotesAndSplices {
           val exprTpt = AppliedTypeTree(TypeTree(defn.QuotedExprClass.typeRef), tpt1 :: Nil)
           val newSplice = ref(defn.InternalQuoted_exprSplice).appliedToType(tpt1.tpe).appliedTo(Typed(pat, exprTpt))
           transform(newSplice)
+        case Apply(TypeApply(fn, targs), Apply(sp, pat :: Nil) :: args :: Nil) if fn.symbol == defn.InternalQuoted_patternHigherOrderHole =>
+          try ref(defn.InternalQuoted_higherOrderHole.termRef).appliedToTypeTrees(targs).appliedTo(args).withSpan(tree.span)
+          finally {
+            val patType = pat.tpe.widen
+            val patType1 = patType.translateFromRepeated(toArray = false)
+            val pat1 = if (patType eq patType1) pat else pat.withType(patType1)
+            patBuf += pat1
+          }
         case Apply(fn, pat :: Nil) if fn.symbol == defn.InternalQuoted_exprSplice =>
           try ref(defn.InternalQuoted_patternHole.termRef).appliedToType(tree.tpe).withSpan(tree.span)
           finally {

docs/docs/reference/metaprogramming/macros.md

@@ -744,5 +744,50 @@ trait Show[-T] {
 }
 ```
 
+#### Open code patterns
+
+Quote pattern matching also provides higher-order patterns to match open terms. If a quoted term contains a definition,
+then the rest of the quote can refer to this definition.
+```
+'{
+  val x: Int = 4
+  x * x
+}
+```
+
+To match such a term we need to match the definition and the rest of the code, but we need to expicilty state that the rest of the code may refer to this definition.
+```scala
+case '{ val y: Int = $x; ${body $$ y}: Int } =>
+```
+Here `$x` will match any closed expression while `${body $$ y}` will match expression that is closed under `y`. Then
+the subxpression of type `Expr[Int]` is bound to `body` as an `Expr[Int => Int]`. The extra argument represents the references to `y`. Usually this expression is used in compination with `Expr.betaReduce` to replace the extra argument.
+
+```scala
+inline def eval(inline e: Int): Int = ${ evalExpr('e) }
+
+private def evalExpr(using QuoteContext)(e: Expr[Int]): Expr[Int] = {
+  e match {
+    case '{ val y: Int = $x; ${body $$ y}: Int } =>
+      // body: Expr[Int => Int] where the argument represents references to y
+      evalExpr(Expr.betaReduce(body)(evalExpr(x)))
+    case '{ ($x: Int) * ($y: Int) } =>
+      (x, y) match
+        case (Const(a), Const(b)) => Expr(a * b)
+        case _ => e
+    case _ => e
+  }
+}
+```
+
+```scala
+eval { // expands to the code: (16: Int)
+  val x: Int = 4
+  x * x
+}
+```
+
+We can also close over several bindings using `${b $$ (a1, a2, ..., an)}`.
+
+
 ### More details
 [More details](./macros-spec.md)

library/src-bootstrapped/scala/internal/quoted/CompileTime.scala

@@ -24,19 +24,31 @@ object CompileTime {
   @compileTimeOnly("Illegal reference to `scala.internal.quoted.CompileTime.typeQuote`")
   def typeQuote[T <: AnyKind]: QuoteContext ?=> Type[T] = ???
 
+  // TODO move to internal.Quoted.Matcher
   /** A splice in a quoted pattern is desugared by the compiler into a call to this method */
   @compileTimeOnly("Illegal reference to `scala.internal.quoted.CompileTime.patternHole`")
   def patternHole[T]: T = ???
 
+  @compileTimeOnly("Illegal reference to `scala.internal.quoted.CompileTime.patternHigherOrderHole`")
+  /** A higher order splice in a quoted pattern is desugared by the compiler into a call to this method */
+  def patternHigherOrderHole[U](pat: Any, args: Any*): U = ???
+
+  // TODO move to internal.Quoted.Matcher
+  @compileTimeOnly("Illegal reference to `scala.internal.quoted.CompileTime.higherOrderHole`")
+  /** A higher order splice in a quoted pattern is desugared by the compiler into a call to this method */
+  def higherOrderHole[U](args: Any*): U = ???
+
   // TODO remove
   /** A splice of a name in a quoted pattern is desugared by wrapping getting this annotation */
   @compileTimeOnly("Illegal reference to `scala.internal.quoted.CompileTime.patternBindHole`")
   class patternBindHole extends Annotation
 
+  // TODO move to internal.Quoted.Matcher
   /** A splice of a name in a quoted pattern is that marks the definition of a type splice */
   @compileTimeOnly("Illegal reference to `scala.internal.quoted.CompileTime.patternType`")
   class patternType extends Annotation
 
+  // TODO move to internal.Quoted.Matcher
   /** A type pattern that must be aproximated from above */
   @compileTimeOnly("Illegal reference to `scala.internal.quoted.CompileTime.fromAbove`")
   class fromAbove extends Annotation

library/src/scala/internal/quoted/Matcher.scala

@@ -240,7 +240,8 @@ private[quoted] object Matcher {
 
           /* Higher order term hole */
           // Matches an open term and wraps it into a lambda that provides the free variables
-          // TODO do not encode with `hole`. Maybe use `higherOrderHole[(T1, ..., Tn) => R]((x1: T1, ..., xn: Tn)): R`
+          // DEPRECATED: replaced with `higherOrderHole`
+          // TODO: remove case
           case (scrutinee, pattern @ Apply(Select(TypeApply(patternHole, List(Inferred())), "apply"), args0 @ IdentArgs(args)))
               if patternHole.symbol == internal.Definitions_InternalQuoted_patternHole =>
             def bodyFn(lambdaArgs: List[Tree]): Tree = {
@@ -258,6 +259,29 @@ private[quoted] object Matcher {
             val res = Lambda(MethodType(names)(_ => argTypes, _ => resType), bodyFn)
             matched(res.seal)
 
+          /* Higher order term hole */
+          // Matches an open term and wraps it into a lambda that provides the free variables
+          case (scrutinee, pattern @ Apply(TypeApply(Ident("higherOrderHole"), List(Inferred())), Repeated(args, _) :: Nil))
+              if pattern.symbol == internal.Definitions_InternalQuoted_higherOrderHole =>
+
+            def bodyFn(lambdaArgs: List[Tree]): Tree = {
+              val argsMap = args.map(_.symbol).zip(lambdaArgs.asInstanceOf[List[Term]]).toMap
+              new TreeMap {
+                override def transformTerm(tree: Term)(using ctx: Context): Term =
+                  tree match
+                    case tree: Ident => summon[Env].get(tree.symbol).flatMap(argsMap.get).getOrElse(tree)
+                    case tree => super.transformTerm(tree)
+              }.transformTree(scrutinee)
+            }
+            val names = args.map {
+              case Block(List(DefDef("$anonfun", _, _, _, Some(Apply(Ident(name), _)))), _) => name
+              case arg => arg.symbol.name
+            }
+            val argTypes = args.map(x => x.tpe.widenTermRefExpr)
+            val resType = pattern.tpe
+            val res = Lambda(MethodType(names)(_ => argTypes, _ => resType), bodyFn)
+            matched(res.seal)
+
           //
           // Match two equivalent trees
           //

library/src/scala/tasty/reflect/CompilerInterface.scala

@@ -1511,13 +1511,16 @@ trait CompilerInterface {
   def Definitions_TupleClass(arity: Int): Symbol
   def Definitions_isTupleClass(sym: Symbol): Boolean
 
-  /** Symbol of scala.internal.Quoted.patternHole */
+  /** Symbol of scala.internal.CompileTime.patternHole */
   def Definitions_InternalQuoted_patternHole: Symbol
 
-  /** Symbol of scala.internal.Quoted.patternType */
+  /** Symbol of scala.internal.CompileTime.higherOrderHole */
+  def Definitions_InternalQuoted_higherOrderHole: Symbol
+
+  /** Symbol of scala.internal.CompileTime.patternType */
   def Definitions_InternalQuoted_patternTypeAnnot: Symbol
 
-  /** Symbol of scala.internal.Quoted.fromAbove */
+  /** Symbol of scala.internal.CompileTime.fromAbove */
   def Definitions_InternalQuoted_fromAboveAnnot: Symbol
 
   def Definitions_UnitType: Type

tests/neg-custom-args/fatal-warnings/quoted-pattern-old-open-expr.scala

@@ -0,0 +1,6 @@
+import scala.quoted._
+
+def f(using QuoteContext)(x: Expr[Any]): Unit = {
+  x match
+    case '{ val a = 3; $b(a): Int } => // error: Possibly using old-higher order pattern syntax instead application pattern.
+}

tests/neg/quote-open-patterns-stages.scala

@@ -0,0 +1,5 @@
+import scala.quoted._
+
+def f(using QuoteContext)(x: Expr[Any]) = x match {
+  case '{ identity(${y $$ x}) } => // error: access to value x from wrong staging level
+}

tests/neg/quote-open-patterns-typer.scala

@@ -0,0 +1,6 @@
+import scala.quoted._
+
+def f(using QuoteContext)(x: Expr[Any]) = x match {
+  case '{ val a: Int = 3; ${y $$ identity(a)} } => // error: Exprected an identifier
+  case '{ identity(${y $$ () }) } => // error: Missing arguments for open pattern
+}

tests/run-macros/quote-matcher-symantics-3/quoted_1.scala

@@ -45,17 +45,17 @@ object Macros {
       case '{ (if ($cond) $thenp else $elsep): $t } =>
         '{ $sym.ifThenElse[$t](${lift(cond)}, ${lift(thenp)}, ${lift(elsep)}) }.asInstanceOf[Expr[R[T]]]
 
-      case '{ (x0: Int) => $bodyFn(x0): Any } =>
+      case '{ (x0: Int) => ${bodyFn $$ x0}: Any } =>
         val (i, nEnvVar) = freshEnvVar[Int]()
         val body2 = UnsafeExpr.open(bodyFn) { (body1, close) => close(body1)(nEnvVar) }
         '{ $sym.lam((x: R[Int]) => ${given Env = envWith(i, 'x)(using env); lift(body2)}).asInstanceOf[R[T]] }
 
-      case '{ (x0: Boolean) => $bodyFn(x0): Any } =>
+      case '{ (x0: Boolean) => ${bodyFn $$ x0}: Any } =>
         val (i, nEnvVar) = freshEnvVar[Boolean]()
         val body2 = UnsafeExpr.open(bodyFn) { (body1, close) => close(body1)(nEnvVar) }
         '{ $sym.lam((x: R[Boolean]) => ${given Env = envWith(i, 'x)(using env); lift(body2)}).asInstanceOf[R[T]] }
 
-      case '{ (x0: Int => Int) => $bodyFn(x0): Any } =>
+      case '{ (x0: Int => Int) => ${bodyFn $$ x0}: Any } =>
         val (i, nEnvVar) = freshEnvVar[Int => Int]()
         val body2 = UnsafeExpr.open(bodyFn) { (body1, close) => close(body1)(nEnvVar) }
         '{ $sym.lam((x: R[Int => Int]) => ${given Env = envWith(i, 'x)(using env); lift(body2)}).asInstanceOf[R[T]] }

tests/run-macros/quote-matching-open/Macro_1.scala

@@ -6,9 +6,9 @@ object Macro {
 
   def impl(x: Expr[Any])(using QuoteContext): Expr[Any] = {
     x match {
-      case '{ (x: Int) => $body(x): Int } => UnsafeExpr.open(body) { (body, close) => close(body)(Expr(2)) }
-      case '{ (x1: Int, x2: Int) => $body(x1, x2): Int } => UnsafeExpr.open(body) { (body, close) => close(body)(Expr(2), Expr(3)) }
-      case '{ (x1: Int, x2: Int, x3: Int) => $body(x1, x2, x3): Int } => UnsafeExpr.open(body) { (body, close) => close(body)(Expr(2), Expr(3), Expr(4)) }
+      case '{ (x: Int) => ${body $$ x}: Int } => UnsafeExpr.open(body) { (body, close) => close(body)(Expr(2)) }
+      case '{ (x1: Int, x2: Int) => ${body $$ (x1, x2)}: Int } => UnsafeExpr.open(body) { (body, close) => close(body)(Expr(2), Expr(3)) }
+      case '{ (x1: Int, x2: Int, x3: Int) => ${body $$ (x1, x2, x3)}: Int } => UnsafeExpr.open(body) { (body, close) => close(body)(Expr(2), Expr(3), Expr(4)) }
     }
   }
 

tests/run-macros/quoted-pattern-open-expr-0.check

@@ -0,0 +1,2 @@
+Matched open
+((y: scala.Int) => y.*(y))

tests/run-macros/quoted-pattern-open-expr-0/Macro_1.scala

@@ -0,0 +1,9 @@
+import scala.quoted._
+
+inline def test(inline e: Int): String = ${testExpr('e)}
+
+private def testExpr(e: Expr[Int])(using QuoteContext): Expr[String] = {
+  e match {
+    case '{ val y: Int = 4; ${body $$ y}: Int } => Expr("Matched open\n" + body.show)
+  }
+}

tests/run-macros/quoted-pattern-open-expr-0/Test_2.scala

@@ -0,0 +1,6 @@
+
+object Test {
+  def main(args: Array[String]): Unit = {
+    println(test { val x: Int = 4; x * x })
+  }
+}

tests/run-macros/quoted-pattern-open-expr-simple-eval/Macro_1.scala

@@ -0,0 +1,15 @@
+import scala.quoted._
+
+inline def eval(inline e: Int): Int = ${ evalExpr('e) }
+
+private def evalExpr(using QuoteContext)(e: Expr[Int]): Expr[Int] = {
+  e match {
+    case '{ val y: Int = $x; ${body $$ y}: Int } =>
+      evalExpr(Expr.betaReduce(body)(evalExpr(x)))
+    case '{ ($x: Int) * ($y: Int) } =>
+      (x, y) match
+        case (Const(a), Const(b)) => Expr(a * b)
+        case _ => e
+    case _ => e
+  }
+}

tests/run-macros/quoted-pattern-open-expr-simple-eval/Test_2.scala

@@ -0,0 +1,9 @@
+
+object Test {
+  def main(args: Array[String]): Unit = {
+    println(eval {
+      val x: Int = 4
+      x * x
+    })
+  }
+}

tests/run-macros/quoted-pattern-open-expr/Macro_1.scala

@@ -5,8 +5,8 @@ inline def test(inline e: Int): String = ${testExpr('e)}
 private def testExpr(e: Expr[Int])(using QuoteContext): Expr[String] = {
   e match {
     case '{ val y: Int = 4; $body } => Expr("Matched closed\n" + body.show)
-    case '{ val y: Int = 4; $body(y): Int } => Expr("Matched open\n" + body.show)
-    case '{ val y: Int => Int = x => x + 1; $body(y): Int } => Expr("Matched open\n" + body.show)
-    case '{ def g(x: Int): Int = $body(g, x); g(5) } => Expr("Matched open\n" + body.show)
+    case '{ val y: Int = 4; ${body $$ y}: Int } => Expr("Matched open\n" + body.show)
+    case '{ val y: Int => Int = x => x + 1; ${body $$ y}: Int } => Expr("Matched open\n" + body.show)
+    case '{ def g(x: Int): Int = ${body $$ (g, x)}; g(5) } => Expr("Matched open\n" + body.show)
   }
 }