Add capture checking to some standard library classes

compiler/src/dotty/tools/dotc/cc/CaptureOps.scala

@@ -15,6 +15,12 @@ import config.Feature
 private val Captures: Key[CaptureSet] = Key()
 private val BoxedType: Key[BoxedTypeCache] = Key()
 
+/** Switch whether unpickled function types and byname types should be mapped to
+ *  impure types. With the new gradual typing using Fluid capture sets, this should
+ *  be no longer needed. Also, it has bad interactions with pickling tests.
+ */
+private val adaptUnpickledFunctionTypes = false
+
 /** The arguments of a @retains or @retainsByName annotation */
 private[cc] def retainedElems(tree: Tree)(using Context): List[Tree] = tree match
   case Apply(_, Typed(SeqLiteral(elems, _), _) :: Nil) => elems
@@ -49,7 +55,7 @@ extension (tree: Tree)
    *  a by name parameter type, turning the latter into an impure by name parameter type.
    */
   def adaptByNameArgUnderPureFuns(using Context): Tree =
-    if Feature.pureFunsEnabledSomewhere then
+    if adaptUnpickledFunctionTypes && Feature.pureFunsEnabledSomewhere then
       val rbn = defn.RetainsByNameAnnot
       Annotated(tree,
         New(rbn.typeRef).select(rbn.primaryConstructor).appliedTo(
@@ -145,7 +151,7 @@ extension (tp: Type)
    */
   def adaptFunctionTypeUnderPureFuns(using Context): Type = tp match
     case AppliedType(fn, args)
-    if Feature.pureFunsEnabledSomewhere && defn.isFunctionClass(fn.typeSymbol) =>
+    if adaptUnpickledFunctionTypes && Feature.pureFunsEnabledSomewhere && defn.isFunctionClass(fn.typeSymbol) =>
       val fname = fn.typeSymbol.name
       defn.FunctionType(
         fname.functionArity,

compiler/src/dotty/tools/dotc/cc/CaptureSet.scala

@@ -356,6 +356,20 @@ object CaptureSet:
     override def toString = elems.toString
   end Const
 
+  /** A special capture set that gets added to the types of symbols that were not
+   *  themselves capture checked, in order to admit arbitrary corresponding capture
+   *  sets in subcapturing comparisons. Similar to platform types for explicit
+   *  nulls, this provides more lenient checking against compilation units that
+   *  were not yet compiled with capture checking on.
+   */
+  object Fluid extends Const(emptySet):
+    override def isAlwaysEmpty = false
+    override def addNewElems(elems: Refs, origin: CaptureSet)(using Context, VarState) = CompareResult.OK
+    override def accountsFor(x: CaptureRef)(using Context): Boolean = true
+    override def mightAccountFor(x: CaptureRef)(using Context): Boolean = true
+    override def toString = "<fluid>"
+  end Fluid
+
   /** The subclass of captureset variables with given initial elements */
   class Var(initialElems: Refs = emptySet) extends CaptureSet:
 
@@ -863,7 +877,7 @@ object CaptureSet:
         case CapturingType(parent, refs) =>
           recur(parent) ++ refs
         case tpd @ RefinedType(parent, _, rinfo: MethodType)
-        if followResult && defn.isFunctionType(tpd) =>
+        if followResult && defn.isFunctionNType(tpd) =>
           ofType(parent, followResult = false)                 // pick up capture set from parent type
           ++ (recur(rinfo.resType)                             // add capture set of result
           -- CaptureSet(rinfo.paramRefs.filter(_.isTracked)*)) // but disregard bound parameters

compiler/src/dotty/tools/dotc/cc/CheckCaptures.scala

@@ -290,14 +290,46 @@ class CheckCaptures extends Recheck, SymTransformer:
     def includeCallCaptures(sym: Symbol, pos: SrcPos)(using Context): Unit =
       if sym.exists && curEnv.isOpen then markFree(capturedVars(sym), pos)
 
+    private def handleBackwardsCompat(tp: Type, sym: Symbol, initialVariance: Int = 1)(using Context): Type =
+      val fluidify = new TypeMap with IdempotentCaptRefMap:
+        variance = initialVariance
+        def apply(t: Type): Type = t match
+          case t: MethodType =>
+            mapOver(t)
+          case t: TypeLambda =>
+            t.derivedLambdaType(resType = this(t.resType))
+          case CapturingType(_, _) =>
+            t
+          case _ =>
+            val t1  = t match
+              case t @ RefinedType(parent, rname, rinfo: MethodType) if defn.isFunctionType(t) =>
+                t.derivedRefinedType(parent, rname, this(rinfo))
+              case _ =>
+                mapOver(t)
+            if variance > 0 then t1
+            else Setup.decorate(t1, Function.const(CaptureSet.Fluid))
+
+      def isPreCC(sym: Symbol): Boolean =
+        sym.isTerm && sym.maybeOwner.isClass
+        && !defn.isFunctionSymbol(sym.owner)
+        && !sym.owner.is(CaptureChecked)
+
+      if isPreCC(sym) then
+        val tpw = tp.widen
+        val fluidTp = fluidify(tpw)
+        if fluidTp eq tpw then tp
+        else fluidTp.showing(i"fluid for ${sym.showLocated}, ${sym.is(JavaDefined)}: $tp --> $result", capt)
+      else tp
+    end handleBackwardsCompat
+
     override def recheckIdent(tree: Ident)(using Context): Type =
       if tree.symbol.is(Method) then
         if tree.symbol.info.isParameterless then
           // there won't be an apply; need to include call captures now
           includeCallCaptures(tree.symbol, tree.srcPos)
       else
         markFree(tree.symbol, tree.srcPos)
-      super.recheckIdent(tree)
+      handleBackwardsCompat(super.recheckIdent(tree), tree.symbol)
 
     /** A specialized implementation of the selection rule.
      *
@@ -327,7 +359,7 @@ class CheckCaptures extends Recheck, SymTransformer:
       val selType = recheckSelection(tree, qualType, name, disambiguate)
       val selCs = selType.widen.captureSet
       if selCs.isAlwaysEmpty || selType.widen.isBoxedCapturing || qualType.isBoxedCapturing then
-        selType
+        handleBackwardsCompat(selType, tree.symbol)
       else
         val qualCs = qualType.captureSet
         capt.println(i"pick one of $qualType, ${selType.widen}, $qualCs, $selCs in $tree")
@@ -362,7 +394,15 @@ class CheckCaptures extends Recheck, SymTransformer:
         val argType0 = f(recheckStart(arg, pt))
         val argType = super.recheckFinish(argType0, arg, pt)
         super.recheckFinish(argType, tree, pt)
-      if meth == defn.Caps_unsafeBox then
+
+      if meth == defn.Caps_unsafeAssumePure then
+        val arg :: Nil = tree.args: @unchecked
+        val argType0 = recheck(arg, pt.capturing(CaptureSet.universal))
+        val argType = if argType0.captureSet.isAlwaysEmpty then argType0
+        else argType0.widen.stripCapturing
+        capt.println(i"rechecking $arg with ${pt.capturing(CaptureSet.universal)}: $argType")
+        super.recheckFinish(argType, tree, pt)
+      else if meth == defn.Caps_unsafeBox then
         mapArgUsing(_.forceBoxStatus(true))
       else if meth == defn.Caps_unsafeUnbox then
         mapArgUsing(_.forceBoxStatus(false))
@@ -662,8 +702,10 @@ class CheckCaptures extends Recheck, SymTransformer:
     /** Turn `expected` into a dependent function when `actual` is dependent. */
     private def alignDependentFunction(expected: Type, actual: Type)(using Context): Type =
       def recur(expected: Type): Type = expected.dealias match
-        case expected @ CapturingType(eparent, refs) =>
-          CapturingType(recur(eparent), refs, boxed = expected.isBoxed)
+        case expected0 @ CapturingType(eparent, refs) =>
+          val eparent1 = recur(eparent)
+          if eparent1 eq eparent then expected
+          else CapturingType(eparent1, refs, boxed = expected0.isBoxed)
         case expected @ defn.FunctionOf(args, resultType, isContextual)
           if defn.isNonRefinedFunction(expected) && defn.isFunctionNType(actual) && !defn.isNonRefinedFunction(actual) =>
           val expected1 = toDepFun(args, resultType, isContextual)
@@ -883,7 +925,7 @@ class CheckCaptures extends Recheck, SymTransformer:
     *  But maybe we can then elide the check during the RefChecks phase under captureChecking?
     */
     def checkOverrides = new TreeTraverser:
-      class OverridingPairsCheckerCC(clazz: ClassSymbol, self: Type, srcPos: SrcPos)(using Context) extends OverridingPairsChecker(clazz, self) {
+      class OverridingPairsCheckerCC(clazz: ClassSymbol, self: Type, srcPos: SrcPos)(using Context) extends OverridingPairsChecker(clazz, self):
         /** Check subtype with box adaptation.
         *  This function is passed to RefChecks to check the compatibility of overriding pairs.
         *  @param sym  symbol of the field definition that is being checked
@@ -905,7 +947,11 @@ class CheckCaptures extends Recheck, SymTransformer:
                 case _ => adapted
             finally curEnv = saved
           actual1 frozen_<:< expected1
-      }
+
+        override def adjustOtherType(tp: Type, other: Symbol)(using Context): Type =
+          handleBackwardsCompat(tp, other, initialVariance = 0)
+            //.showing(i"adjust $other: $tp --> $result")
+      end OverridingPairsCheckerCC
 
       def traverse(t: Tree)(using Context) =
         t match

compiler/src/dotty/tools/dotc/cc/Setup.scala

@@ -114,88 +114,6 @@ extends tpd.TreeTraverser:
           case _ => tp
       case _ => tp
 
-    private def superTypeIsImpure(tp: Type): Boolean = {
-      tp.dealias match
-        case CapturingType(_, refs) =>
-          !refs.isAlwaysEmpty
-        case tp: (TypeRef | AppliedType) =>
-          val sym = tp.typeSymbol
-          if sym.isClass then
-            sym == defn.AnyClass
-              // we assume Any is a shorthand of {cap} Any, so if Any is an upper
-              // bound, the type is taken to be impure.
-          else superTypeIsImpure(tp.superType)
-        case tp: (RefinedOrRecType | MatchType) =>
-          superTypeIsImpure(tp.underlying)
-        case tp: AndType =>
-          superTypeIsImpure(tp.tp1) || needsVariable(tp.tp2)
-        case tp: OrType =>
-          superTypeIsImpure(tp.tp1) && superTypeIsImpure(tp.tp2)
-        case _ =>
-          false
-    }.showing(i"super type is impure $tp = $result", capt)
-
-    /** Should a capture set variable be added on type `tp`? */
-    def needsVariable(tp: Type): Boolean = {
-      tp.typeParams.isEmpty && tp.match
-        case tp: (TypeRef | AppliedType) =>
-          val tp1 = tp.dealias
-          if tp1 ne tp then needsVariable(tp1)
-          else
-            val sym = tp1.typeSymbol
-            if sym.isClass then
-              !sym.isPureClass && sym != defn.AnyClass
-            else superTypeIsImpure(tp1)
-        case tp: (RefinedOrRecType | MatchType) =>
-          needsVariable(tp.underlying)
-        case tp: AndType =>
-          needsVariable(tp.tp1) && needsVariable(tp.tp2)
-        case tp: OrType =>
-          needsVariable(tp.tp1) || needsVariable(tp.tp2)
-        case CapturingType(parent, refs) =>
-          needsVariable(parent)
-          && refs.isConst      // if refs is a variable, no need to add another
-          && !refs.isUniversal // if refs is {cap}, an added variable would not change anything
-        case _ =>
-          false
-    }.showing(i"can have inferred capture $tp = $result", capt)
-
-    /** Add a capture set variable to `tp` if necessary, or maybe pull out
-     *  an embedded capture set variable from a part of `tp`.
-     */
-    def addVar(tp: Type) = tp match
-      case tp @ RefinedType(parent @ CapturingType(parent1, refs), rname, rinfo) =>
-        CapturingType(tp.derivedRefinedType(parent1, rname, rinfo), refs, parent.isBoxed)
-      case tp: RecType =>
-        tp.parent match
-          case parent @ CapturingType(parent1, refs) =>
-            CapturingType(tp.derivedRecType(parent1), refs, parent.isBoxed)
-          case _ =>
-            tp // can return `tp` here since unlike RefinedTypes, RecTypes are never created
-                // by `mapInferred`. Hence if the underlying type admits capture variables
-                // a variable was already added, and the first case above would apply.
-      case AndType(tp1 @ CapturingType(parent1, refs1), tp2 @ CapturingType(parent2, refs2)) =>
-        assert(refs1.asVar.elems.isEmpty)
-        assert(refs2.asVar.elems.isEmpty)
-        assert(tp1.isBoxed == tp2.isBoxed)
-        CapturingType(AndType(parent1, parent2), refs1 ** refs2, tp1.isBoxed)
-      case tp @ OrType(tp1 @ CapturingType(parent1, refs1), tp2 @ CapturingType(parent2, refs2)) =>
-        assert(refs1.asVar.elems.isEmpty)
-        assert(refs2.asVar.elems.isEmpty)
-        assert(tp1.isBoxed == tp2.isBoxed)
-        CapturingType(OrType(parent1, parent2, tp.isSoft), refs1 ++ refs2, tp1.isBoxed)
-      case tp @ OrType(tp1 @ CapturingType(parent1, refs1), tp2) =>
-        CapturingType(OrType(parent1, tp2, tp.isSoft), refs1, tp1.isBoxed)
-      case tp @ OrType(tp1, tp2 @ CapturingType(parent2, refs2)) =>
-        CapturingType(OrType(tp1, parent2, tp.isSoft), refs2, tp2.isBoxed)
-      case _ if needsVariable(tp) =>
-        val cs = tp.dealias match
-          case CapturingType(_, refs) => CaptureSet.Var(refs.elems)
-          case _ => CaptureSet.Var()
-        CapturingType(tp, cs)
-      case _ =>
-        tp
-
     private var isTopLevel = true
 
     private def mapNested(ts: List[Type]): List[Type] =
@@ -246,7 +164,7 @@ extends tpd.TreeTraverser:
             resType = this(tp.resType))
         case _ =>
           mapOver(tp)
-      addVar(addCaptureRefinements(tp1))
+      Setup.addVar(addCaptureRefinements(tp1))
     end apply
   end mapInferred
 
@@ -474,4 +392,97 @@ object Setup:
 
   def isDuringSetup(using Context): Boolean =
     ctx.property(IsDuringSetupKey).isDefined
+
+  private def superTypeIsImpure(tp: Type)(using Context): Boolean = {
+    tp.dealias match
+      case CapturingType(_, refs) =>
+        !refs.isAlwaysEmpty
+      case tp: (TypeRef | AppliedType) =>
+        val sym = tp.typeSymbol
+        if sym.isClass then
+          sym == defn.AnyClass
+            // we assume Any is a shorthand of {cap} Any, so if Any is an upper
+            // bound, the type is taken to be impure.
+        else superTypeIsImpure(tp.superType)
+      case tp: (RefinedOrRecType | MatchType) =>
+        superTypeIsImpure(tp.underlying)
+      case tp: AndType =>
+        superTypeIsImpure(tp.tp1) || needsVariable(tp.tp2)
+      case tp: OrType =>
+        superTypeIsImpure(tp.tp1) && superTypeIsImpure(tp.tp2)
+      case _ =>
+        false
+  }.showing(i"super type is impure $tp = $result", capt)
+
+  /** Should a capture set variable be added on type `tp`? */
+  def needsVariable(tp: Type)(using Context): Boolean = {
+    tp.typeParams.isEmpty && tp.match
+      case tp: (TypeRef | AppliedType) =>
+        val sym = tp.typeSymbol
+        if sym.isClass then
+          !sym.isPureClass && sym != defn.AnyClass
+        else
+          sym != defn.FromJavaObjectSymbol
+            // For capture checking, we assume Object from Java is the same as Any
+          && {
+            val tp1 = tp.dealias
+            if tp1 ne tp then needsVariable(tp1)
+            else superTypeIsImpure(tp1)
+          }
+      case tp: (RefinedOrRecType | MatchType) =>
+        needsVariable(tp.underlying)
+      case tp: AndType =>
+        needsVariable(tp.tp1) && needsVariable(tp.tp2)
+      case tp: OrType =>
+        needsVariable(tp.tp1) || needsVariable(tp.tp2)
+      case CapturingType(parent, refs) =>
+        needsVariable(parent)
+        && refs.isConst      // if refs is a variable, no need to add another
+        && !refs.isUniversal // if refs is {cap}, an added variable would not change anything
+      case _ =>
+        false
+  }.showing(i"can have inferred capture $tp = $result", capt)
+
+  /** Add a capture set variable to `tp` if necessary, or maybe pull out
+   *  an embedded capture set variable from a part of `tp`.
+   */
+  def decorate(tp: Type, addedSet: Type => CaptureSet)(using Context): Type = tp match
+    case tp @ RefinedType(parent @ CapturingType(parent1, refs), rname, rinfo) =>
+      CapturingType(tp.derivedRefinedType(parent1, rname, rinfo), refs, parent.isBoxed)
+    case tp: RecType =>
+      tp.parent match
+        case parent @ CapturingType(parent1, refs) =>
+          CapturingType(tp.derivedRecType(parent1), refs, parent.isBoxed)
+        case _ =>
+          tp // can return `tp` here since unlike RefinedTypes, RecTypes are never created
+              // by `mapInferred`. Hence if the underlying type admits capture variables
+              // a variable was already added, and the first case above would apply.
+    case AndType(tp1 @ CapturingType(parent1, refs1), tp2 @ CapturingType(parent2, refs2)) =>
+      assert(refs1.elems.isEmpty)
+      assert(refs2.elems.isEmpty)
+      assert(tp1.isBoxed == tp2.isBoxed)
+      CapturingType(AndType(parent1, parent2), refs1 ** refs2, tp1.isBoxed)
+    case tp @ OrType(tp1 @ CapturingType(parent1, refs1), tp2 @ CapturingType(parent2, refs2)) =>
+      assert(refs1.elems.isEmpty)
+      assert(refs2.elems.isEmpty)
+      assert(tp1.isBoxed == tp2.isBoxed)
+      CapturingType(OrType(parent1, parent2, tp.isSoft), refs1 ++ refs2, tp1.isBoxed)
+    case tp @ OrType(tp1 @ CapturingType(parent1, refs1), tp2) =>
+      CapturingType(OrType(parent1, tp2, tp.isSoft), refs1, tp1.isBoxed)
+    case tp @ OrType(tp1, tp2 @ CapturingType(parent2, refs2)) =>
+      CapturingType(OrType(tp1, parent2, tp.isSoft), refs2, tp2.isBoxed)
+    case _ if needsVariable(tp) =>
+      CapturingType(tp, addedSet(tp))
+    case _ =>
+      tp
+
+  /** Add a capture set variable to `tp` if necessary, or maybe pull out
+   *  an embedded capture set variable from a part of `tp`.
+   */
+  def addVar(tp: Type)(using Context): Type =
+    decorate(tp,
+      addedSet = _.dealias.match
+        case CapturingType(_, refs) => CaptureSet.Var(refs.elems)
+        case _ => CaptureSet.Var())
+
 end Setup