Fix cycle detection for type aliases with wildcard arguments

compiler/src/dotty/tools/dotc/typer/Typer.scala

@@ -2027,12 +2027,19 @@ class Typer(@constructorOnly nestingLevel: Int = 0) extends Namer
       var checkedArgs = preCheckKinds(args1, paramBounds)
         // check that arguments conform to bounds is done in phase PostTyper
       val tycon = tpt1.symbol
-      if (tycon == defn.andType)
-        checkedArgs = checkedArgs.mapconserve(arg =>
-          checkSimpleKinded(checkNoWildcard(arg)))
-      else if (tycon == defn.orType)
+      if tycon == defn.andType || tycon == defn.orType then
         checkedArgs = checkedArgs.mapconserve(arg =>
           checkSimpleKinded(checkNoWildcard(arg)))
+      else if tycon.flagsUNSAFE.is(Provisional) then
+        // A type with Provisional flag is either an alias or abstract type.
+        // If it is an alias type, it would mean the type is cyclic
+        // If it is an abstract type, it would mean the type is an irreducible
+        // application of a higher-kinded type to a wildcard argument.
+        // Either way, the wildcard argument is illegal.
+        // The early test here is needed, so that we do not accidentally reduce
+        // an application of a Provisional type away so that the type constructor
+        // is no longer present on the roght hand side. See neg/i15507.scala.
+        checkedArgs = checkedArgs.mapconserve(checkNoWildcard)
       else if tycon == defn.throwsAlias
           && checkedArgs.length == 2
           && checkedArgs(1).tpe.derivesFrom(defn.RuntimeExceptionClass)

tests/neg/i15507.scala

@@ -0,0 +1,15 @@
+object TestNested:
+  type _NestedSet1[X] = Set[_NestedSet1[?]] // error // error
+  type _NestedSet2[X] <: Set[_NestedSet2[?]] // error
+  type _NestedSet3[X] <: Set[_NestedSet3[X]] // ok
+  type _NestedSet4[X] >: Set[_NestedSet4[X]] // error
+  type _NestedSet5[X] = Set[_NestedSet5[X]] // error
+  type _NestedSet6[X] = Set[_NestedSet6[Int]] // error
+
+  type _NestedList1[X] = List[_NestedList1[?]] // error // error
+  type _NestedList2[X] <: List[_NestedList2[?]] // error
+  type _NestedList3[X] <: List[_NestedList3[X]] // ok
+  type _NestedList4[X] >: List[_NestedList4[X]] // error
+  type _NestedList5[X] = List[_NestedList5[X]] // error
+  type _NestedList6[X] = List[_NestedList6[Int]] // error
+