Enable experimental mode when experimental feature is imported (#19807)

Make it easier to use experimental language features out of the box. We
already have `-experimental` that enables experimental mode globally by
adding the `@experimental` annotation to all top-level definitions. The
complication was that this flag was necessary to use experimental
language features. It was not always easy to set (for example in the
REPL) and it is all or nothing. Users can also annotate explicitly all
top-level definitions that use language features with `@experimental`.

Now, the import of an experimental language feature will automatically
enable experimental mode. If the import is at the top-level, all
definitions in the file are experimental.

We also remove the special case that makes all code generated with a
nightly or snapshot compiler compile as if in experimental mode. This
implies that now the behavior of nightlies is identical to the behavior
of release versions. The `-Yno-experimental` becomes a no-op and will be
removed in a later PR.

[test_scala2_library_tasty]

Author: odersky

compiler/src/dotty/tools/dotc/Driver.scala

@@ -80,7 +80,6 @@ class Driver {
     val ictx = rootCtx.fresh
     val summary = command.distill(args, ictx.settings)(ictx.settingsState)(using ictx)
     ictx.setSettings(summary.sstate)
-    Feature.checkExperimentalSettings(using ictx)
     MacroClassLoader.init(ictx)
     Positioned.init(using ictx)
 

compiler/src/dotty/tools/dotc/config/Feature.scala

@@ -34,6 +34,11 @@ object Feature:
   val captureChecking = experimental("captureChecking")
   val into = experimental("into")
 
+  def experimentalAutoEnableFeatures(using Context): List[TermName] =
+    defn.languageExperimentalFeatures
+      .map(sym => experimental(sym.name))
+      .filterNot(_ == captureChecking) // TODO is this correct?
+
   /** Is `feature` enabled by by a command-line setting? The enabling setting is
    *
    *       -language:<prefix>feature
@@ -157,18 +162,19 @@ object Feature:
   private def experimentalUseSite(which: String): String =
     s"""Experimental $which may only be used under experimental mode:
        |  1. in a definition marked as @experimental, or
-       |  2. compiling with the -experimental compiler flag, or
-       |  3. with a nightly or snapshot version of the compiler.
+       |  2. an experimental feature is imported at the package level, or
+       |  3. compiling with the -experimental compiler flag.
        |""".stripMargin
 
-  /** Check that experimental compiler options are only set for snapshot or nightly compiler versions. */
-  def checkExperimentalSettings(using Context): Unit =
-    for setting <- ctx.settings.language.value
-        if setting.startsWith("experimental.") && setting != "experimental.macros"
-    do checkExperimentalFeature(s"feature $setting", NoSourcePosition)
-
   def isExperimentalEnabled(using Context): Boolean =
-    (Properties.unstableExperimentalEnabled && !ctx.settings.YnoExperimental.value) || ctx.settings.experimental.value
+    ctx.settings.experimental.value ||
+    experimentalAutoEnableFeatures.exists(enabled)
+
+  def experimentalEnabledByLanguageSetting(using Context): Option[TermName] =
+    experimentalAutoEnableFeatures.find(enabledBySetting)
+
+  def isExperimentalEnabledByImport(using Context): Boolean =
+    experimentalAutoEnableFeatures.exists(enabledByImport)
 
   /** Handle language import `import language.<prefix>.<imported>` if it is one
    *  of the global imports `pureFunctions` or `captureChecking`. In this case

compiler/src/dotty/tools/dotc/config/Properties.scala

@@ -84,12 +84,6 @@ trait PropertiesTrait {
    */
   val versionString: String = "version " + simpleVersionString
 
-  /** Whether the current version of compiler is experimental
-   *
-   *  Snapshot, nightly releases and non-bootstrapped compiler are experimental.
-   */
-  val unstableExperimentalEnabled: Boolean = versionString.contains("SNAPSHOT") || versionString.contains("NIGHTLY") || versionString.contains("nonbootstrapped")
-
   /** Whether the current version of compiler supports research plugins. */
   val researchPluginEnabled: Boolean = versionString.contains("SNAPSHOT") || versionString.contains("NIGHTLY") || versionString.contains("nonbootstrapped")
 

compiler/src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -406,7 +406,6 @@ private sealed trait YSettings:
   val YretainTrees: Setting[Boolean] = BooleanSetting(ForkSetting, "Yretain-trees", "Retain trees for top-level classes, accessible from ClassSymbol#tree")
   val YshowTreeIds: Setting[Boolean] = BooleanSetting(ForkSetting, "Yshow-tree-ids", "Uniquely tag all tree nodes in debugging output.")
   val YfromTastyIgnoreList: Setting[List[String]] = MultiStringSetting(ForkSetting, "Yfrom-tasty-ignore-list", "file", "List of `tasty` files in jar files that will not be loaded when using -from-tasty.")
-  val YnoExperimental: Setting[Boolean] = BooleanSetting(ForkSetting, "Yno-experimental", "Disable experimental language features by default in NIGHTLY/SNAPSHOT versions of the compiler.")
   val YlegacyLazyVals: Setting[Boolean] = BooleanSetting(ForkSetting, "Ylegacy-lazy-vals", "Use legacy (pre 3.3.0) implementation of lazy vals.")
   val YcompileScala2Library: Setting[Boolean] = BooleanSetting(ForkSetting, "Ycompile-scala2-library", "Used when compiling the Scala 2 standard library.")
   val YoutputOnlyTasty: Setting[Boolean] = BooleanSetting(ForkSetting, "Youtput-only-tasty", "Used to only generate the TASTy file without the classfiles")

compiler/src/dotty/tools/dotc/core/Definitions.scala

@@ -2008,6 +2008,13 @@ class Definitions {
     CapabilityAnnot, RequiresCapabilityAnnot,
     RetainsAnnot, RetainsCapAnnot, RetainsByNameAnnot)
 
+  /** Experimental language features defined in `scala.runtime.stdLibPatches.language.experimental`.
+   *
+   *  This list does not include `scala.language.experimental.macros`.
+   */
+  @tu lazy val languageExperimentalFeatures: List[TermSymbol] =
+    LanguageExperimentalModule.moduleClass.info.decls.toList.filter(_.isAllOf(Lazy | Module)).map(_.asTerm)
+
   // ----- primitive value class machinery ------------------------------------------
 
   class PerRun[T](generate: Context ?=> T) {

compiler/src/dotty/tools/dotc/transform/PostTyper.scala

@@ -417,7 +417,7 @@ class PostTyper extends MacroTransform with InfoTransformer { thisPhase =>
             )
           }
         case tree: ValDef =>
-          annotateExperimental(tree.symbol)
+          annotateExperimentalCompanion(tree.symbol)
           registerIfHasMacroAnnotations(tree)
           checkErasedDef(tree)
           Checking.checkPolyFunctionType(tree.tpt)
@@ -426,7 +426,6 @@ class PostTyper extends MacroTransform with InfoTransformer { thisPhase =>
             checkStableSelection(tree.rhs)
           processValOrDefDef(super.transform(tree1))
         case tree: DefDef =>
-          annotateExperimental(tree.symbol)
           registerIfHasMacroAnnotations(tree)
           checkErasedDef(tree)
           Checking.checkPolyFunctionType(tree.tpt)
@@ -438,7 +437,7 @@ class PostTyper extends MacroTransform with InfoTransformer { thisPhase =>
           val sym = tree.symbol
           if (sym.isClass)
             VarianceChecker.check(tree)
-            annotateExperimental(sym)
+            annotateExperimentalCompanion(sym)
             checkMacroAnnotation(sym)
             if sym.isOneOf(GivenOrImplicit) then
               sym.keepAnnotationsCarrying(thisPhase, Set(defn.CompanionClassMetaAnnot), orNoneOf = defn.MetaAnnots)
@@ -546,8 +545,8 @@ class PostTyper extends MacroTransform with InfoTransformer { thisPhase =>
       }
 
     override def transformStats[T](trees: List[Tree], exprOwner: Symbol, wrapResult: List[Tree] => Context ?=> T)(using Context): T =
-      try super.transformStats(trees, exprOwner, wrapResult)
-      finally Checking.checkExperimentalImports(trees)
+      Checking.checkAndAdaptExperimentalImports(trees)
+      super.transformStats(trees, exprOwner, wrapResult)
 
     /** Transforms the rhs tree into a its default tree if it is in an `erased` val/def.
      *  Performed to shrink the tree that is known to be erased later.
@@ -584,14 +583,9 @@ class PostTyper extends MacroTransform with InfoTransformer { thisPhase =>
         else if tpe.derivesFrom(defn.NullClass) then
           report.error("`erased` definition cannot be implemented with en expression of type Null", tree.srcPos)
 
-    private def annotateExperimental(sym: Symbol)(using Context): Unit =
-      def isTopLevelDefinitionInSource(sym: Symbol) =
-        !sym.is(Package) && !sym.name.isPackageObjectName &&
-        (sym.owner.is(Package) || (sym.owner.isPackageObject && !sym.isConstructor))
+    private def annotateExperimentalCompanion(sym: Symbol)(using Context): Unit =
       if sym.is(Module) then
         ExperimentalAnnotation.copy(sym.companionClass).foreach(sym.addAnnotation)
-      if !sym.hasAnnotation(defn.ExperimentalAnnot) && ctx.settings.experimental.value && isTopLevelDefinitionInSource(sym) then
-        sym.addAnnotation(ExperimentalAnnotation("Added by -experimental", sym.span))
 
     // It needs to run at the phase of the postTyper --- otherwise, the test of the symbols will use
     // the transformed denotation with added `Serializable` and `AbstractFunction1`.

compiler/src/dotty/tools/dotc/typer/Checking.scala

@@ -42,6 +42,7 @@ import cc.{isCaptureChecking, isRetainsLike}
 
 import collection.mutable
 import reporting.*
+import Annotations.ExperimentalAnnotation
 
 object Checking {
   import tpd.*
@@ -797,50 +798,57 @@ object Checking {
     tree
 
   /** Check that experimental language imports in `trees`
-   *  are done only in experimental scopes, or in a top-level
-   *  scope with only @experimental definitions.
+   *  are done only in experimental scopes. For top-level
+   *  experimental imports, all top-level definitions are transformed
+   *  to @experimental definitions.
+   *
    */
-  def checkExperimentalImports(trees: List[Tree])(using Context): Unit =
-
-    def nonExperimentalStat(trees: List[Tree]): Tree = trees match
-      case (_: Import | EmptyTree) :: rest =>
-        nonExperimentalStat(rest)
-      case (tree @ TypeDef(_, impl: Template)) :: rest if tree.symbol.isPackageObject =>
-        nonExperimentalStat(impl.body).orElse(nonExperimentalStat(rest))
-      case (tree: PackageDef) :: rest =>
-        nonExperimentalStat(tree.stats).orElse(nonExperimentalStat(rest))
-      case (tree: MemberDef) :: rest =>
-        if tree.symbol.isExperimental || tree.symbol.is(Synthetic) then
-          nonExperimentalStat(rest)
-        else
-          tree
-      case tree :: rest =>
-        tree
-      case Nil =>
-        EmptyTree
-
-    for case imp @ Import(qual, selectors) <- trees do
+  def checkAndAdaptExperimentalImports(trees: List[Tree])(using Context): Unit =
+    def nonExperimentalTopLevelDefs(pack: Symbol): Iterator[Symbol] =
+      def isNonExperimentalTopLevelDefinition(sym: Symbol) =
+        !sym.isExperimental
+        && sym.source == ctx.compilationUnit.source
+        && !sym.isConstructor // not constructor of package object
+        && !sym.is(Package) && !sym.name.isPackageObjectName
+
+      pack.info.decls.toList.iterator.flatMap: sym =>
+        if sym.isClass && (sym.is(Package) || sym.isPackageObject) then
+          nonExperimentalTopLevelDefs(sym)
+        else if isNonExperimentalTopLevelDefinition(sym) then
+          sym :: Nil
+        else Nil
+
+    def unitExperimentalLanguageImports =
       def isAllowedImport(sel: untpd.ImportSelector) =
         val name = Feature.experimental(sel.name)
         name == Feature.scala2macros
-        || name == Feature.erasedDefinitions
         || name == Feature.captureChecking
+      trees.filter {
+        case Import(qual, selectors) =>
+          languageImport(qual) match
+            case Some(nme.experimental) =>
+              !selectors.forall(isAllowedImport) && !ctx.owner.isInExperimentalScope
+            case _ => false
+        case _ => false
+      }
 
-      languageImport(qual) match
-        case Some(nme.experimental)
-        if !ctx.owner.isInExperimentalScope && !selectors.forall(isAllowedImport) =>
-          def check(stable: => String) =
-            Feature.checkExperimentalFeature("features", imp.srcPos,
-              s"\n\nNote: the scope enclosing the import is not considered experimental because it contains the\nnon-experimental $stable")
-          if ctx.owner.is(Package) then
-            // allow top-level experimental imports if all definitions are @experimental
-            nonExperimentalStat(trees) match
-              case EmptyTree =>
-              case tree: MemberDef => check(i"${tree.symbol}")
-              case tree => check(i"expression ${tree}")
-          else Feature.checkExperimentalFeature("features", imp.srcPos)
+    if ctx.owner.is(Package) || ctx.owner.name.startsWith(str.REPL_SESSION_LINE) then
+      def markTopLevelDefsAsExperimental(why: String): Unit =
+        for sym <- nonExperimentalTopLevelDefs(ctx.owner) do
+          sym.addAnnotation(ExperimentalAnnotation(s"Added by $why", sym.span))
+
+      unitExperimentalLanguageImports match
+        case imp :: _ => markTopLevelDefsAsExperimental(i"top level $imp")
         case _ =>
-  end checkExperimentalImports
+          Feature.experimentalEnabledByLanguageSetting match
+            case Some(sel) => markTopLevelDefsAsExperimental(i"-language:experimental.$sel")
+            case _ if ctx.settings.experimental.value => markTopLevelDefsAsExperimental(i"-experimental")
+            case _ =>
+    else
+      for imp <- unitExperimentalLanguageImports do
+        Feature.checkExperimentalFeature("feature local import", imp.srcPos)
+
+  end checkAndAdaptExperimentalImports
 
   /** Checks that PolyFunction only have valid refinements.
    *

compiler/src/dotty/tools/dotc/typer/CrossVersionChecks.scala

@@ -132,12 +132,14 @@ class CrossVersionChecks extends MiniPhase:
   }
 
   override def transformOther(tree: Tree)(using Context): Tree =
-    tree.foreachSubTree { // Find references in type trees and imports
-      case tree: Ident => transformIdent(tree)
-      case tree: Select => transformSelect(tree)
-      case tree: TypeTree => transformTypeTree(tree)
-      case _ =>
-    }
+    val inPackage = ctx.owner.is(Package) || ctx.owner.isPackageObject
+    if !(inPackage && tree.isInstanceOf[ImportOrExport] && Feature.isExperimentalEnabledByImport) then
+      tree.foreachSubTree { // Find references in type trees and imports
+        case tree: Ident => transformIdent(tree)
+        case tree: Select => transformSelect(tree)
+        case tree: TypeTree => transformTypeTree(tree)
+        case _ =>
+      }
     tree
 
 end CrossVersionChecks

compiler/test/dotty/tools/backend/jvm/PublicInBinaryTests.scala

@@ -42,7 +42,6 @@ class PublicInBinaryTests extends DottyBytecodeTest {
   override def initCtx =
     val ctx0 = super.initCtx
     ctx0.setSetting(ctx0.settings.experimental, true)
-    ctx0.setSetting(ctx0.settings.YnoExperimental, true)
 
   @Test
   def publicInBinaryDef(): Unit = {

docs/_docs/reference/other-new-features/experimental-defs.md

@@ -265,8 +265,7 @@ Experimental definitions can only be referenced in an experimental scope. Experi
 
    </details>
 
-6. Any code compiled using a [_Nightly_](https://search.maven.org/artifact/org.scala-lang/scala3-compiler_3) or _Snapshot_ version of the compiler is considered to be in an experimental scope.
-Can use the `-Yno-experimental` compiler flag to disable it and run as a proper release.
+6. An experimental language feature is imported in at the package level. All top-level definitions will be marked as `@experimental`.
 
 In any other situation, a reference to an experimental definition will cause a compilation error.
 

project/Build.scala

@@ -1083,7 +1083,6 @@ object Build {
       Compile / doc / scalacOptions += "-Ydocument-synthetic-types",
       scalacOptions += "-Ycompile-scala2-library",
       scalacOptions += "-Yscala2Unpickler:never",
-      scalacOptions += "-Yno-experimental",
       scalacOptions -= "-Xfatal-warnings",
       Compile / compile / logLevel.withRank(KeyRanks.Invisible) := Level.Error,
       ivyConfigurations += SourceDeps.hide,
@@ -1757,6 +1756,9 @@ object Build {
       SourceLinksIntegrationTest / scalaSource := baseDirectory.value / "test-source-links",
       SourceLinksIntegrationTest / test:= ((SourceLinksIntegrationTest / test) dependsOn generateScalaDocumentation.toTask("")).value,
     ).
+    settings(
+      scalacOptions += "-experimental" // workaround use of experimental .info in Scaladoc2AnchorCreator
+    ).
     settings(
       Compile / resourceGenerators ++= Seq(
         generateStaticAssetsTask.taskValue,
@@ -2170,9 +2172,6 @@ object Build {
           settings(
             versionScheme := Some("semver-spec"),
             libraryDependencies += "org.scala-lang" % "scala-library" % stdlibVersion,
-            // Make sure we do not refer to experimental features outside an experimental scope.
-            // In other words, disable NIGHTLY/SNAPSHOT experimental scope.
-            scalacOptions += "-Yno-experimental",
           ).
           settings(dottyLibrarySettings)
       if (mode == Bootstrapped) {

scaladoc-testcases/src/tests/hugetype.scala

@@ -31,7 +31,7 @@ trait E:
   @deprecated
   protected implicit def same[A](a: A): A
 
-trait XD extends E:
+@experimental trait XD extends E:
   /**
    * Some important information :o
    *

scaladoc-testcases/src/tests/methodsAndConstructors.scala

@@ -1,5 +1,7 @@
 package tests.methodsAndConstructors
 
+import scala.language.experimental.clauseInterleaving
+
 class A
 class B extends A
 class C
@@ -60,8 +62,6 @@ class Methods:
  def withImplicitParam2(v: String)(implicit ab: Double, a: Int, b: String): String
  = ???
 
- import scala.language.experimental.clauseInterleaving
-
  def clauseInterleaving[T](x: T)[U](y: U)(using (T, U)): (T, U)
  = ???
 

scaladoc/src/dotty/tools/scaladoc/snippets/SnippetCompiler.scala

@@ -27,7 +27,6 @@ class SnippetCompiler(
   object SnippetDriver extends Driver:
     val currentCtx =
       val rootCtx = initCtx.fresh.addMode(Mode.ReadPositions).addMode(Mode.Interactive)
-      rootCtx.setSetting(rootCtx.settings.YnoExperimental, true)
       rootCtx.setSetting(rootCtx.settings.experimental, true)
       rootCtx.setSetting(rootCtx.settings.YretainTrees, true)
       rootCtx.setSetting(rootCtx.settings.YcookComments, true)

tests/init-global/pos/global-region1.scala

@@ -1,4 +1,4 @@
-//> using options -experimental -Yno-experimental
+//> using options -experimental
 
 import scala.annotation.init.region
 

tests/init-global/warn/i18628_3.scala

@@ -1,4 +1,4 @@
-//> using options -experimental -Yno-experimental
+//> using options -experimental
 
 import scala.annotation.init.widen
 

tests/neg-macros/i18677-a/Macro_1.scala

@@ -1,4 +1,4 @@
-//> using options -experimental -Yno-experimental
+//> using options -experimental
 
 import annotation.MacroAnnotation
 import quoted.*

tests/neg-macros/i18677-a/Test_2.scala

@@ -1,4 +1,4 @@
-//> using options -experimental -Yno-experimental
+//> using options -experimental
 
 @extendFoo
 class AFoo // error

tests/neg-macros/i18677-b/Macro_1.scala

@@ -1,4 +1,4 @@
-//> using options -experimental -Yno-experimental
+//> using options -experimental
 
 import annotation.MacroAnnotation
 import quoted.*

tests/neg-macros/i18677-b/Test_2.scala

@@ -1,4 +1,4 @@
-//> using options -experimental -Yno-experimental
+//> using options -experimental
 
 @extendFoo
 class AFoo // error

tests/neg-macros/i19676/Macro_1.scala

@@ -1,4 +1,4 @@
-//> using options -experimental -Yno-experimental
+//> using options -experimental
 
 import scala.annotation.MacroAnnotation
 import scala.quoted.*

tests/neg-macros/i19676/Test_2.scala

@@ -1,4 +1,4 @@
-//> using options -experimental -Yno-experimental
+//> using options -experimental
 
 @buggy // error
 case class Foo()

tests/neg-macros/i19842-a/Macro.scala

@@ -1,4 +1,4 @@
-//> using options -experimental -Yno-experimental
+//> using options -experimental
 
 import scala.annotation.{experimental, targetName}
 import scala.quoted.*