Expand value references to packages to their underlying package objects (#22011)

A package object can be seen as the facade of a package. For instance,
it is the logical place where we want to write doc comments that explain
a package.

So far references to packages cannot be used as values. But if the
package has a package object, it would make sense to allow the package
reference with the meaning that it refers to this object. For instance,
let's say we have

```scala
package a
object b
```
Of course, we can use `a.b` as a value. But if we change that to
```scala
package a
package object b
```
we can't anymore. This PR changes that so that we still allow a
reference `a.b` as a value to mean the package object. Due to the way
package objects are encoded the `a.b` reference expands to
`a.b.package`.

Author: odersky

compiler/src/dotty/tools/dotc/config/Feature.scala

@@ -37,6 +37,7 @@ object Feature:
   val betterMatchTypeExtractors = experimental("betterMatchTypeExtractors")
   val quotedPatternsWithPolymorphicFunctions = experimental("quotedPatternsWithPolymorphicFunctions")
   val betterFors = experimental("betterFors")
+  val packageObjectValues = experimental("packageObjectValues")
 
   def experimentalAutoEnableFeatures(using Context): List[TermName] =
     defn.languageExperimentalFeatures

compiler/src/dotty/tools/dotc/reporting/ErrorMessageID.scala

@@ -132,7 +132,7 @@ enum ErrorMessageID(val isActive: Boolean = true) extends java.lang.Enum[ErrorMe
   case MissingCompanionForStaticID // errorNumber: 116
   case PolymorphicMethodMissingTypeInParentID // errorNumber: 117
   case ParamsNoInlineID // errorNumber: 118
-  case JavaSymbolIsNotAValueID // errorNumber: 119
+  case SymbolIsNotAValueID // errorNumber: 119
   case DoubleDefinitionID // errorNumber: 120
   case MatchCaseOnlyNullWarningID // errorNumber: 121
   case ImportedTwiceID // errorNumber: 122

compiler/src/dotty/tools/dotc/reporting/messages.scala

@@ -2317,7 +2317,7 @@ class ParamsNoInline(owner: Symbol)(using Context)
   def explain(using Context) = ""
 }
 
-class JavaSymbolIsNotAValue(symbol: Symbol)(using Context) extends TypeMsg(JavaSymbolIsNotAValueID) {
+class SymbolIsNotAValue(symbol: Symbol)(using Context) extends TypeMsg(SymbolIsNotAValueID) {
   def msg(using Context) =
     val kind =
       if symbol is Package then i"$symbol"

compiler/src/dotty/tools/dotc/transform/Erasure.scala

@@ -24,7 +24,6 @@ import typer.NoChecking
 import inlines.Inlines
 import typer.ProtoTypes.*
 import typer.ErrorReporting.errorTree
-import typer.Checking.checkValue
 import core.TypeErasure.*
 import core.Decorators.*
 import dotty.tools.dotc.ast.{tpd, untpd}
@@ -676,7 +675,7 @@ object Erasure {
       if tree.name == nme.apply && integrateSelect(tree) then
         return typed(tree.qualifier, pt)
 
-      val qual1 = typed(tree.qualifier, AnySelectionProto)
+      var qual1 = typed(tree.qualifier, AnySelectionProto)
 
       def mapOwner(sym: Symbol): Symbol =
         if !sym.exists && tree.name == nme.apply then
@@ -725,7 +724,8 @@ object Erasure {
 
       assert(sym.exists, i"no owner from $owner/${origSym.showLocated} in $tree")
 
-      if owner == defn.ObjectClass then checkValue(qual1)
+      if owner == defn.ObjectClass then
+        qual1 = checkValue(qual1)
 
       def select(qual: Tree, sym: Symbol): Tree =
         untpd.cpy.Select(tree)(qual, sym.name).withType(NamedType(qual.tpe, sym))

compiler/src/dotty/tools/dotc/typer/Checking.scala

@@ -804,24 +804,6 @@ object Checking {
           else "Cannot override non-inline parameter with an inline parameter",
           p1.srcPos)
 
-  def checkValue(tree: Tree)(using Context): Unit =
-    val sym = tree.tpe.termSymbol
-    if sym.isNoValue && !ctx.isJava then
-      report.error(JavaSymbolIsNotAValue(sym), tree.srcPos)
-
-  /** Check that `tree` refers to a value, unless `tree` is selected or applied
-   *  (singleton types x.type don't count as selections).
-   */
-  def checkValue(tree: Tree, proto: Type)(using Context): tree.type =
-    tree match
-      case tree: RefTree if tree.name.isTermName =>
-        proto match
-          case _: SelectionProto if proto ne SingletonTypeProto => // no value check
-          case _: FunOrPolyProto => // no value check
-          case _ => checkValue(tree)
-      case _ =>
-    tree
-
   /** Check that experimental language imports in `trees`
    *  are done only in experimental scopes. For top-level
    *  experimental imports, all top-level definitions are transformed

compiler/src/dotty/tools/dotc/typer/Typer.scala

@@ -618,10 +618,8 @@ class Typer(@constructorOnly nestingLevel: Int = 0) extends Namer
     // Shortcut for the root package, this is not just a performance
     // optimization, it also avoids forcing imports thus potentially avoiding
     // cyclic references.
-    if (name == nme.ROOTPKG)
-      val tree2 = tree.withType(defn.RootPackage.termRef)
-      checkLegalValue(tree2, pt)
-      return tree2
+    if name == nme.ROOTPKG then
+      return checkLegalValue(tree.withType(defn.RootPackage.termRef), pt)
 
     val rawType =
       val saved1 = unimported
@@ -681,9 +679,7 @@ class Typer(@constructorOnly nestingLevel: Int = 0) extends Namer
             cpy.Ident(tree)(tree.name.unmangleClassName).withType(checkedType)
           else
             tree.withType(checkedType)
-      val tree2 = toNotNullTermRef(tree1, pt)
-      checkLegalValue(tree2, pt)
-      tree2
+      checkLegalValue(toNotNullTermRef(tree1, pt), pt)
 
     def isLocalExtensionMethodRef: Boolean = rawType match
       case rawType: TermRef =>
@@ -723,21 +719,47 @@ class Typer(@constructorOnly nestingLevel: Int = 0) extends Namer
       errorTree(tree, MissingIdent(tree, kind, name, pt))
   end typedIdent
 
+  def checkValue(tree: Tree)(using Context): Tree =
+    val sym = tree.tpe.termSymbol
+    if sym.isNoValue && !ctx.isJava then
+      if sym.is(Package)
+          && Feature.enabled(Feature.packageObjectValues)
+          && tree.tpe.member(nme.PACKAGE).hasAltWith(_.symbol.isPackageObject)
+      then
+        typed(untpd.Select(untpd.TypedSplice(tree), nme.PACKAGE))
+      else
+        report.error(SymbolIsNotAValue(sym), tree.srcPos)
+        tree
+    else tree
+
+  /** Check that `tree` refers to a value, unless `tree` is selected or applied
+   *  (singleton types x.type don't count as selections).
+   */
+  def checkValue(tree: Tree, proto: Type)(using Context): Tree =
+    tree match
+      case tree: RefTree if tree.name.isTermName =>
+        proto match
+          case _: SelectionProto if proto ne SingletonTypeProto => tree // no value check
+          case _: FunOrPolyProto => tree // no value check
+          case _ => checkValue(tree)
+      case _ => tree
+
   /** (1) If this reference is neither applied nor selected, check that it does
    *      not refer to a package or Java companion object.
    *  (2) Check that a stable identifier pattern is indeed stable (SLS 8.1.5)
    */
-  private def checkLegalValue(tree: Tree, pt: Type)(using Context): Unit =
-    checkValue(tree, pt)
+  private def checkLegalValue(tree: Tree, pt: Type)(using Context): Tree =
+    val tree1 = checkValue(tree, pt)
     if ctx.mode.is(Mode.Pattern)
-       && !tree.isType
+       && !tree1.isType
        && !pt.isInstanceOf[ApplyingProto]
-       && !tree.tpe.match
+       && !tree1.tpe.match
         case tp: NamedType => tp.denot.hasAltWith(_.symbol.isStableMember && tp.prefix.isStable || tp.info.isStable)
         case tp            => tp.isStable
-       && !isWildcardArg(tree)
+       && !isWildcardArg(tree1)
     then
-      report.error(StableIdentPattern(tree, pt), tree.srcPos)
+      report.error(StableIdentPattern(tree1, pt), tree1.srcPos)
+    tree1
 
   def typedSelectWithAdapt(tree0: untpd.Select, pt: Type, qual: Tree)(using Context): Tree =
     val selName = tree0.name
@@ -751,8 +773,7 @@ class Typer(@constructorOnly nestingLevel: Int = 0) extends Namer
       if checkedType.exists then
         val select = toNotNullTermRef(assignType(tree, checkedType), pt)
         if selName.isTypeName then checkStable(qual.tpe, qual.srcPos, "type prefix")
-        checkLegalValue(select, pt)
-        ConstFold(select)
+        ConstFold(checkLegalValue(select, pt))
       else EmptyTree
 
     // Otherwise, simplify `m.apply(...)` to `m(...)`

docs/_docs/reference/experimental/package-object-values.md

@@ -0,0 +1,40 @@
+---
+layout: doc-page
+title: "Reference-able Package Objects"
+redirectFrom: /docs/reference/experimental/package-object-values.html
+nightlyOf: https://docs.scala-lang.org/scala3/reference/experimental/package-object-values.html
+---
+
+One limitation with `package object`s is that we cannot currently assign them to values: `a.b` fails to compile when `b` is a `package object`, even though it succeeds when `b` is a normal `object`. The workaround is to call
+```scala
+  a.b.`package`
+```
+But this is ugly and non-obvious. Or one could use a normal `object`, which is not always possible.
+
+The `packageObjectValues` language extension drops this limitation. The extension is enabled by the language import `import scala.language.experimental.packageObjectValues` or by setting the command line option `-language:experimental.packageObjectValues`.
+
+The extension, turns the following into valid code:
+
+```scala
+package a
+package object b
+
+val z = a.b // Currently fails with "package is not a value"
+```
+
+Currently the workaround is to use a `.package` suffix:
+
+```scala
+val z = a.b.`package`
+```
+
+With the extension, a reference such as `a.b` where `b` is a `package` containing a `package object`, expands to `a.b.package` automatically
+
+## Limitations
+
+* `a.b` only expands to `a.b.package` when used "standalone", i.e. not when part of a larger select chain `a.b.c` or equivalent postfix expression `a.b c`, prefix expression `!a.b`, or infix expression `a.b c d`.
+
+* `a.b` expands to `a.b.package` of the type `a.b.package.type`, and only contains the contents of the `package object`. It does not contain other things in the `package` `a.b` that are outside of the `package object`
+
+Both these requirements are necessary for backwards compatibility, and anyway do not impact the main goal of removing the irregularity between `package object`s and normal `object`s.
+

docs/_docs/reference/other-new-features/toplevel-definitions.md

@@ -0,0 +1,41 @@
+---
+layout: doc-page
+title: "Toplevel Definitions"
+nightlyOf: https://docs.scala-lang.org/scala3/reference/dropped-features/toplevel-definitions.html
+---
+
+All kind of definitions can now be written at the top-level.
+Example:
+```scala
+package p
+type Labelled[T] = (String, T)
+val a: Labelled[Int] = ("count", 1)
+def b = a._2
+
+case class C()
+
+extension (x: C) def pair(y: C) = (x, y)
+```
+Previously, `type`, `val` or `def` definitions had to be wrapped in a package object. Now,
+there may be several source files in a package containing such top-level definitions, and source files can freely mix top-level value, method, and type definitions with classes and objects.
+
+The compiler generates synthetic objects that wrap top-level definitions falling into one of the following categories:
+
+ - all pattern, value, method, and type definitions,
+ - implicit classes and objects,
+ - companion objects of opaque type aliases.
+
+If a source file `src.scala` contains such top-level definitions, they will be put in a synthetic object named `src$package`. The wrapping is transparent, however. The definitions in `src` can still be accessed as members of the enclosing package. The synthetic object will be placed last in the file,
+after any other package clauses, imports, or object and class definitions.
+
+**Note:** This means that
+1. The name of a source file containing wrapped top-level definitions is relevant for binary compatibility. If the name changes, so does the name of the generated object and its class.
+
+2. A top-level main method `def main(args: Array[String]): Unit = ...` is wrapped as any other method. If it appears
+in a source file `src.scala`, it could be invoked from the command line using a command like `scala src$package`. Since the
+"program name" is mangled it is recommended to always put `main` methods in explicitly named objects.
+
+3. The notion of `private` is independent of whether a definition is wrapped or not. A `private` top-level definition is always visible from everywhere in the enclosing package.
+
+4. If several top-level definitions are overloaded variants with the same name,
+they must all come from the same source file.

docs/sidebar.yml

@@ -123,7 +123,6 @@ subsection:
           - page: reference/dropped-features/type-projection.md
           - page: reference/dropped-features/do-while.md
           - page: reference/dropped-features/procedure-syntax.md
-          - page: reference/dropped-features/package-objects.md
           - page: reference/dropped-features/early-initializers.md
           - page: reference/dropped-features/class-shadowing.md
           - page: reference/dropped-features/class-shadowing-spec.md
@@ -164,6 +163,7 @@ subsection:
           - page: reference/experimental/runtimeChecked.md
           - page: reference/experimental/better-fors.md
           - page: reference/experimental/unrolled-defs.md
+          - page: reference/experimental/package-object-values.md
       - page: reference/syntax.md
       - title: Language Versions
         index: reference/language-versions/language-versions.md

library/src/scala/runtime/stdLibPatches/language.scala

@@ -140,6 +140,11 @@ object language:
      */
     @compileTimeOnly("`betterFors` can only be used at compile time in import statements")
     object betterFors
+
+    /** Experimental support for package object values
+     */
+    @compileTimeOnly("`packageObjectValues` can only be used at compile time in import statements")
+    object packageObjectValues
   end experimental
 
   /** The deprecated object contains features that are no longer officially suypported in Scala.

project/MiMaFilters.scala

@@ -13,6 +13,8 @@ object MiMaFilters {
         ProblemFilters.exclude[MissingFieldProblem]("scala.runtime.stdLibPatches.language#experimental.quotedPatternsWithPolymorphicFunctions"),
         ProblemFilters.exclude[MissingClassProblem]("scala.runtime.stdLibPatches.language$experimental$quotedPatternsWithPolymorphicFunctions$"),
         ProblemFilters.exclude[DirectMissingMethodProblem]("scala.quoted.runtime.Patterns.higherOrderHoleWithTypes"),
+        ProblemFilters.exclude[MissingFieldProblem]("scala.runtime.stdLibPatches.language#experimental.packageObjectValues"),
+        ProblemFilters.exclude[MissingClassProblem]("scala.runtime.stdLibPatches.language$experimental$packageObjectValues$"),
       ),
 
       // Additions since last LTS

tests/run/pkgobjvals.check

@@ -0,0 +1,4 @@
+Foo was created
+Foo was created
+Foo was created
+Foo was created

tests/run/pkgobjvals.scala

@@ -0,0 +1,22 @@
+import language.experimental.packageObjectValues
+
+package a:
+  package object b:
+    class Foo:
+      println("Foo was created")
+
+    def foo() = Foo()
+  end b
+
+  def test =
+    val bb = b
+    bb.foo()
+    new bb.Foo()
+end a
+
+@main def Test =
+  a.test
+  val ab: a.b.type = a.b
+  ab.foo()
+  new ab.Foo()
+