Fix mapping TypeMaps over annotations

Avoids orphan parameters when pickling

Fixes #15922

I am not sure about the status of the test in custome-args/captures. Should it
pass or be rejected? But in any case it does not crash anymore.

Author: odersky

compiler/src/dotty/tools/dotc/core/Annotations.scala

@@ -1,4 +1,5 @@
-package dotty.tools.dotc
+package dotty.tools
+package dotc
 package core
 
 import Symbols._, Types._, Contexts._, Constants._
@@ -58,7 +59,7 @@ object Annotations {
             if tm.isRange(x) then x
             else
               val tp1 = tm(tree.tpe)
-              foldOver(if tp1 =:= tree.tpe then x else tp1, tree)
+              foldOver(if tp1 frozen_=:= tree.tpe then x else tp1, tree)
         val diff = findDiff(NoType, args)
         if tm.isRange(diff) then EmptyAnnotation
         else if diff.exists then derivedAnnotation(tm.mapOver(tree))
@@ -69,7 +70,7 @@ object Annotations {
       val args = arguments
       if args.isEmpty then false
       else tree.existsSubTree {
-        case id: Ident => id.tpe match
+        case id: Ident => id.tpe.stripped match
           case TermParamRef(tl1, _) => tl eq tl1
           case _ => false
         case _ => false

tests/pos-custom-args/captures/i15922.scala

@@ -0,0 +1,14 @@
+trait Cap { def use(): Int }
+type Id[X] = [T] -> (op: X => T) -> T
+def mkId[X](x: X): Id[X] = [T] => (op: X => T) => op(x)
+
+def withCap[X](op: ({*} Cap) => X): X = {
+  val cap: {*} Cap = new Cap { def use() = { println("cap is used"); 0 } }
+  val result = op(cap)
+  result
+}
+
+def leaking(c: {*} Cap): Id[{c} Cap] = mkId(c)
+
+def test =
+  val bad = withCap(leaking)

tests/pos/i15922.scala

@@ -0,0 +1,14 @@
+trait Cap:
+  type M
+class Id[X]
+
+object Test:
+  def withCap[X](op: Cap => X): X = ???
+
+  class retains1(xs: Any*) extends annotation.StaticAnnotation
+
+  def leaking1(c: Cap): Id[Cap @retains1(c)] = ??? // used to crash with orphan parameter on pickling
+  def leaking2(c: Cap): Id[c.type] = ???
+
+  val bad1 = withCap(leaking1)
+  val bad2 = withCap(leaking2)