Merge pull request #9180 from dotty-staging/structural-new

Change structural types scheme

Author: odersky

compiler/src/dotty/tools/dotc/typer/Dynamic.scala

@@ -131,18 +131,24 @@ trait Dynamic {
    *  and `x.a` is of type `U`, map `x.a` to the equivalent of:
    *
    *  ```scala
-   *  (x: Selectable).selectDynamic("a").asInstanceOf[U]
+   *  x1.selectDynamic("a").asInstanceOf[U]
    *  ```
+   *  where `x1` is `x` adapted to `Selectable`.
    *
    *  Given `x.a(a11, ..., a1n)...(aN1, ..., aNn)`, where `x.a` is of (widened) type
-   *  `(T11, ..., T1n)...(TN1, ..., TNn) => R`, it is desugared to:
+   *  `(T11, ..., T1n)...(TN1, ..., TNn): R`, it is desugared to:
    *
    *  ```scala
-   *  (x:selectable).applyDynamic("a", CT11, ..., CT1n, ..., CTN1, ... CTNn)
-   *                             (a11, ..., a1n, ..., aN1, ..., aNn)
-   *                .asInstanceOf[R]
+   *  x1.applyDynamic("a")(a11, ..., a1n, ..., aN1, ..., aNn)
+   *    .asInstanceOf[R]
+   *  ```
+   *  If this call resolves to an `applyDynamic` method that takes a `ClassTag[?]*` as second
+   *  parameter, we further rewrite this call to
+   *  scala```
+   *  x1.applyDynamic("a", CT11, ..., CT1n, ..., CTN1, ... CTNn)
+   *                  (a11, ..., a1n, ..., aN1, ..., aNn)
+   *    .asInstanceOf[R]
    *  ```
-   *
    *  where CT11, ..., CTNn are the class tags representing the erasure of T11, ..., TNn.
    *
    *  It's an error if U is neither a value nor a method type, or a dependent method
@@ -151,20 +157,37 @@ trait Dynamic {
   def handleStructural(tree: Tree)(using Context): Tree = {
     val (fun @ Select(qual, name), targs, vargss) = decomposeCall(tree)
 
-    def structuralCall(selectorName: TermName, ctags: List[Tree]) = {
+    def structuralCall(selectorName: TermName, ctags: => List[Tree]) = {
       val selectable = adapt(qual, defn.SelectableClass.typeRef)
 
-      // ($qual: Selectable).$selectorName("$name", ..$ctags)
+      // ($qual: Selectable).$selectorName("$name")
       val base =
         untpd.Apply(
           untpd.TypedSplice(selectable.select(selectorName)).withSpan(fun.span),
-          (Literal(Constant(name.toString)) :: ctags).map(untpd.TypedSplice(_)))
+          (Literal(Constant(name.toString)) :: Nil).map(untpd.TypedSplice(_)))
 
       val scall =
         if (vargss.isEmpty) base
         else untpd.Apply(base, vargss.flatten)
 
-      typed(scall)
+      // If function is an `applyDynamic` that takes a ClassTag* parameter,
+      // add `ctags`.
+      def addClassTags(tree: Tree): Tree = tree match
+        case Apply(fn: Apply, args) =>
+          cpy.Apply(tree)(addClassTags(fn), args)
+        case Apply(fn @ Select(_, nme.applyDynamic), nameArg :: _ :: Nil) =>
+          fn.tpe.widen match
+            case mt: MethodType => mt.paramInfos match
+              case _ :: ctagsParam :: Nil
+              if ctagsParam.isRepeatedParam
+                 && ctagsParam.argInfos.head.isRef(defn.ClassTagClass) =>
+                  val ctagType = defn.ClassTagClass.typeRef.appliedTo(TypeBounds.empty)
+                  cpy.Apply(tree)(fn,
+                    nameArg :: seqToRepeated(SeqLiteral(ctags, TypeTree(ctagType))) :: Nil)
+              case _ => tree
+            case other => tree
+        case _ => tree
+      addClassTags(typed(scall))
     }
 
     def fail(name: Name, reason: String) =
@@ -187,7 +210,7 @@ trait Dynamic {
         if (isDependentMethod(tpe))
           fail(name, i"has a method type with inter-parameter dependencies")
         else {
-          val ctags = tpe.paramInfoss.flatten.map(pt =>
+          def ctags = tpe.paramInfoss.flatten.map(pt =>
             implicitArgTree(defn.ClassTagClass.typeRef.appliedTo(pt.widenDealias :: Nil), fun.span.endPos))
           structuralCall(nme.applyDynamic, ctags).cast(tpe.finalResultType)
         }

docs/docs/reference/changed-features/structural-types-spec.md

@@ -14,61 +14,63 @@ RefineStat    ::= ‘val’ VarDcl | ‘def’ DefDcl | ‘type’ {nl} TypeDcl
 
 ## Implementation of structural types
 
-The standard library defines a trait `Selectable` in the package
-`scala`, defined as follows:
+The standard library defines a universal marker trait `Selectable` in the package `scala`:
 
 ```scala
-trait Selectable extends Any {
-  def selectDynamic(name: String): Any
-  def applyDynamic(name: String, paramClasses: ClassTag[_]*)(args: Any*): Any =
-    new UnsupportedOperationException("applyDynamic")
-}
+trait Selectable extends Any
 ```
 
 An implementation of `Selectable` that relies on Java reflection is
 available in the standard library: `scala.reflect.Selectable`. Other
 implementations can be envisioned for platforms where Java reflection
 is not available.
 
-`selectDynamic` takes a field name and returns the value associated
-with that name in the `Selectable`. Similarly, `applyDynamic`
-takes a method name, `ClassTag`s representing its parameters types and
-the arguments to pass to the function. It will return the result of
-calling this function with the given arguments.
+Implementations of `Selectable` have to make available one or both of
+the methods `selectDynamic` and `applyDynamic`. The methods could be members of the `Selectable` implementation or they could be extension methods.
+
+The `selectDynamic` method takes a field name and returns the value associated with that name in the `Selectable`.
+It should have a signature of the form:
+```scala
+def selectDynamic(name: String): T
+```
+Often, the return type `T` is `Any`.
+
+The `applyDynamic` method is used for selections that are applied to arguments. It takes a method name and possibly `ClassTag`s representing its parameters types as well as the arguments to pass to the function.
+Its signature should be of one of the two following forms:
+```scala
+def applyDynamic(name: String)(args: Any*): T
+def applyDynamic(name: String, ctags: ClassTag[?]*)(args: Any*): T
+```
+Both versions are passed the actual arguments in the `args` parameter. The second version takes in addition a vararg argument of class tags that identify the method's parameter classes. Such an argument is needed
+if `applyDynamic` is implemented using Java reflection, but it could be
+useful in other cases as well. `selectDynamic` and `applyDynamic` can also take additional context parameters in using clauses. These are resolved in the normal way at the callsite.
 
 Given a value `v` of type `C { Rs }`, where `C` is a class reference
-and `Rs` are refinement declarations, and given `v.a` of type `U`, we
-consider three distinct cases:
+and `Rs` are structural refinement declarations, and given `v.a` of type `U`, we consider three distinct cases:
 
-- If `U` is a value type, we map `v.a` to the equivalent of:
+- If `U` is a value type, we map `v.a` to:
   ```scala
-  v.a
-     --->
-  (v: Selectable).selectDynamic("a").asInstanceOf[U]
+  v.selectDynamic("a").asInstanceOf[U]
   ```
 
-- If `U` is a method type `(T11, ..., T1n)...(TN1, ..., TNn) => R` and it is not
-a dependent method type, we map `v.a(a11, ..., a1n)...(aN1, aNn)` to
-  the  equivalent of:
+- If `U` is a method type `(T11, ..., T1n)...(TN1, ..., TNn): R` and it is not a dependent method type, we map `v.a(a11, ..., a1n)...(aN1, ..., aNn)` to:
+  ```scala
+  v.applyDynamic("a")(a11, ..., a1n, ..., aN1, ..., aNn)
+    .asInstanceOf[R]
+  ```
+  If this call resolves to an `applyDynamic` method of the second form that takes a `ClassTag[?]*` argument, we further rewrite this call to
   ```scala
-  v.a(arg1, ..., argn)
-     --->
-  (v: Selectable).applyDynamic("a", CT11, ..., CTn, ..., CTN1, ... CTNn)
-                              (a11, ..., a1n, ..., aN1, ..., aNn)
-                 .asInstanceOf[R]
+  v.applyDynamic("a", CT11, ..., CT1n, ..., CTN1, ... CTNn)(
+    a11, ..., a1n, ..., aN1, ..., aNn)
+    .asInstanceOf[R]
   ```
+   where each `CT_ij` is the class tag of the type of the formal parameter `Tij`
 
 - If `U` is neither a value nor a method type, or a dependent method
   type, an error is emitted.
 
-We make sure that `r` conforms to type `Selectable`, potentially by
-introducing an implicit conversion, and then call either
-`selectDynamic` or `applyDynamic`, passing the name of the
-member to access, along with the class tags of the formal parameters
-and the arguments in the case of a method call. These parameters
-could be used to disambiguate one of several overload variants in the
-future, but overloads are not supported in structural types at the
-moment.
+Note that `v`'s static type does not necessarily have to conform to `Selectable`, nor does it need to have `selectDynamic` and `applyDynamic` as members. It suffices that there is an implicit
+conversion that can turn `v` into a `Selectable`, and the selection methods could also be available as extension methods.
 
 ## Limitations of structural types
 
@@ -85,14 +87,8 @@ moment.
 - In Scala 2, mutable `var`s are allowed in refinements. In Scala 3,
   they are no longer allowed.
 
-## Migration
-
-Receivers of structural calls need to be instances of `Selectable`. A
-conversion from `Any` to `Selectable` is available in the standard
-library, in `scala.reflect.Selectable.reflectiveSelectable`. This is
-similar to the implementation of structural types in Scala 2.
 
-## Reference
+## Context
 
 For more info, see [Rethink Structural
 Types](https://github.com/lampepfl/dotty/issues/1886).

docs/docs/reference/changed-features/structural-types.md

@@ -3,6 +3,8 @@ layout: doc-page
 title: "Programmatic Structural Types"
 ---
 
+## Motivation
+
 Some usecases, such as modelling database access, are more awkward in
 statically typed languages than in dynamically typed languages: With
 dynamically typed languages, it's quite natural to model a row as a
@@ -29,25 +31,91 @@ configure how fields and methods should be resolved.
 
 ## Example
 
+Here's an example of a structural type `Person`:
 ```scala
-object StructuralTypeExample {
-
-  case class Record(elems: (String, Any)*) extends Selectable {
-    def selectDynamic(name: String): Any = elems.find(_._1 == name).get._2
+  class Record(elems: (String, Any)*) extends Selectable {
+    private val fields = elems.toMap
+    def selectDynamic(name: String): Any = fields(name)
   }
-
   type Person = Record {
     val name: String
     val age: Int
   }
+```
+The person type adds a _refinement_ to its parent type `Record` that defines `name` and `age` fields. We say the refinement is _structural_ since  `name` and `age` are not defined in the parent type. But they exist nevertheless as members of class `Person`. For instance, the following
+program would print  "Emma is 42 years old.":
+```scala
+  val person = Record("name" -> "Emma", "age" -> 42).asInstanceOf[Person]
+  println(s"${person.name} is ${person.age} years old.")
+```
+The parent type `Record` in this example is a generic class that can represent arbitrary records in its `elems` argument. This argument is a
+sequence of pairs of labels of type `String` and values of type `Any`.
+When we create a `Person` as a `Record` we have to assert with a typecast
+that the record defines the right fields of the right types. `Record`
+itself is too weakly typed so the compiler cannot know this without
+help from the user. In practice, the connection between a structural type
+and its underlying generic representation would most likely be done by
+a database layer, and therefore would not be a concern of the end user.
+
+`Record` extends the marker trait `scala.Selectable` and defines
+a method `selectDynamic`, which maps a field name to its value.
+Selecting a structural type member is done by calling this method.
+The `person.name` and `person.age` selections are translated by
+the Scala compiler to:
+```scala
+  person.selectDynamic("name").asInstanceOf[String]
+  person.selectDynamic("age").asInstanceOf[Int]
+```
+
+Besides `selectDynamic`, a `Selectable` class sometimes also defines a method `applyDynamic`. This can then be used to translate function calls of structural members. So, if `a` is an instance of `Selectable`, a structural call like `a.f(b, c)` would translate to
+```scala
+  a.applyDynamic("f")(b, c)
+```
+
+## Using Java Reflection
 
-  def main(args: Array[String]): Unit = {
-    val person = Record("name" -> "Emma", "age" -> 42).asInstanceOf[Person]
-    println(s"${person.name} is ${person.age} years old.")
-    // Prints: Emma is 42 years old.
+Structural types can also be accessed using Java reflection. Example:
+```scala
+  type Closeable = {
+    def close(): Unit
+  }
+  class FileInputStream {
+    def close(): Unit
+  }
+  class Channel {
+    def close(): Unit
   }
-}
 ```
+Here, we define a structural type `Closeable` that defines a `close` method. There are various classes that have `close` methods, we just list `FileInputStream` and `Channel` as two examples. It would be easiest if the two classes shared a common interface that factors out the `close` method. But such factorings are often not possible if different libraries are combined in one application. Yet, we can still have methods that work on
+all classes with a `close` method by using the `Closeable` type. For instance,
+```scala
+  import scala.reflect.Selectable.reflectiveSelectable
+
+  def autoClose(f: Closeable)(op: Closeable => Unit): Unit =
+    try op(f) finally f.close()
+```
+The call `f.close()` has to use Java reflection to identify and call the `close` method in the receiver `f`. This needs to be enabled by an import
+of `reflectiveSelectable` shown above. What happens "under the hood" is then the following:
+
+ - The import makes available an implicit conversion that turns any type into a
+   `Selectable`. `f` is wrapped in this conversion.
+
+ - The compiler then transforms the `close` call on the wrapped `f`
+   to an `applyDynamic` call. The end result is:
+
+   ```scala
+     reflectiveSelectable(f).applyDynamic("close")()
+   ```
+ - The implementation of `applyDynamic` in `reflectiveSelectable`'s result
+uses Java reflection to find and call a method `close` with zero parameters in the value referenced by `f` at runtime.
+
+Structural calls like this tend to be much slower than normal method calls. The mandatory import of `reflectiveSelectable` serves as a signpost that something inefficient is going on.
+
+**Note:** In Scala 2, Java reflection is the only mechanism available for structural types and it is automatically enabled without needing the
+`reflectiveSelectable` conversion. However, to warn against inefficient
+dispatch, Scala 2 requires a language import `import scala.language.reflectiveCalls`.
+
+Before resorting to structural calls with Java reflection one should consider alternatives. For instance, sometimes a more a modular _and_ efficient architecture can be obtained using typeclasses.
 
 ## Extensibility
 

library/src/scala/Selectable.scala

@@ -1,8 +1,24 @@
 package scala
 import scala.reflect.ClassTag
 
-trait Selectable extends Any {
-  def selectDynamic(name: String): Any
-  def applyDynamic(name: String, paramClasses: ClassTag[_]*)(args: Any*): Any =
-    new UnsupportedOperationException("applyDynamic")
-}
+/** A marker trait for objects that support structural selection via
+ *  `selectDynamic` and `applyDynamic`
+ *
+ *  Implementation classes should define, or make available as extension
+ *  methods, the following two method signatures:
+ *
+ *    def selectDynamic(name: String): Any
+ *    def applyDynamic(name: String)(args: Any*): Any =
+ *
+ *  `selectDynamic` is invoked for simple selections `v.m`, whereas
+ *  `applyDynamic` is invoked for selections with arguments `v.m(...)`.
+ *  If there's only one kind of selection, the method supporting the
+ *  other may be omitted. The `applyDynamic` can also have a second parameter
+ *  list of class tag arguments, i.e. it may alternatively have the signature
+ *
+ *    def applyDynamic(name: String, paramClasses: ClassTag[_]*)(args: Any*): Any
+ *
+ *  In this case the call will synthesize `ClassTag` arguments for all formal parameter
+ *  types of the method in the structural type.
+ */
+trait Selectable extends Any

library/src/scala/reflect/Selectable.scala

@@ -14,7 +14,7 @@ class Selectable(val receiver: Any) extends AnyVal with scala.Selectable {
     }
   }
 
-  override def applyDynamic(name: String, paramTypes: ClassTag[_]*)(args: Any*): Any = {
+  def applyDynamic(name: String, paramTypes: ClassTag[_]*)(args: Any*): Any = {
     val rcls = receiver.getClass
     val paramClasses = paramTypes.map(_.runtimeClass)
     val mth = rcls.getMethod(name, paramClasses: _*)
@@ -24,8 +24,6 @@ class Selectable(val receiver: Any) extends AnyVal with scala.Selectable {
 }
 
 object Selectable {
-  implicit def reflectiveSelectable(receiver: Any): scala.Selectable = receiver match {
-    case receiver: scala.Selectable => receiver
-    case _                          => new Selectable(receiver)
-  }
+  implicit def reflectiveSelectable(receiver: Any): Selectable =
+    new Selectable(receiver)
 }

tests/pos/reference/structural-closeable.scala

@@ -0,0 +1,14 @@
+type Closeable = {
+  def close(): Unit
+}
+
+class FileInputStream:
+  def close(): Unit = ()
+
+class Channel:
+  def close(): Unit = ()
+
+import scala.reflect.Selectable.reflectiveSelectable
+
+def autoClose(f: Closeable)(op: Closeable => Unit): Unit =
+  try op(f) finally f.close()

tests/run-macros/refined-selectable-macro/Macro_1.scala

@@ -2,6 +2,9 @@ import scala.quoted._
 
 object Macro {
 
+  trait Selectable extends scala.Selectable:
+    def selectDynamic(name: String): Any
+
   trait SelectableRecord extends Selectable {
     transparent inline def toTuple: Tuple = ${ toTupleImpl('this)}
   }