[WIP] Fix scala#2874: implement check for expansive cycles

Expansive cycles characterize situations where the class hierarchy
of a generic class is potentially infinite. This can lead to nontermination
during subtype checks.

This PR implements the algorithm in
Kennedy, Andrew J., and Benjamin C. Pierce. "On decidability of nominal subtyping with variance." (2006).
for detecting (and disallowing) expansive cycles.

Author: abeln

compiler/src/dotty/tools/dotc/reporting/diagnostic/ErrorMessageID.java

@@ -96,6 +96,7 @@ public enum ErrorMessageID {
     WrongNumberOfParametersID,
     DuplicatePrivateProtectedQualifierID,
     ExpectedStartOfTopLevelDefinitionID,
+    TypeParamInExpansiveCycleID,
     ;
 
     public int errorNumber() {

compiler/src/dotty/tools/dotc/reporting/diagnostic/messages.scala

@@ -1710,4 +1710,38 @@ object messages {
       hl"you have to provide either ${"class"}, ${"trait"}, ${"object"}, or ${"enum"} definitions after qualifiers"
   }
 
+  case class TypeParamInExpansiveCycle(cinfo: ClassInfo, tparam: Symbol)(implicit ctx: Context)
+    extends Message(TypeParamInExpansiveCycleID) {
+
+    val kind = "Syntax"
+
+    val msg = hl"type parameter ${tparam.showName} in ${cinfo.classSymbol} is part of an expansive cycle"
+
+    val explanation =
+      hl"""| If a class has a type parameter that is part of an expansive cycle, then a subtype test involving the class
+           | might fail to terminate. Expansive cycles are therefore disallowed.
+           |
+           | For the definition of expansive cycles, see:
+           | ${Green("Kennedy, Andrew J., and Benjamin C. Pierce. \"On decidability of nominal subtyping with variance.\" (2006)")}
+           |
+           | Example
+           | In the class definitions:
+           |
+           | ${"class N[-Z]"}
+           | ${"class C[X] extends N[N[C[C[X]]]]"}
+           |
+           | ${"X"} is part of an expansive cycle. When we try to check ${"C[X] <: N[C[X]]"}, we get the following
+           | sequence of subtype tests:
+           |
+           | ${"C[X] <: N[C[X]]"}
+           | ${"N[N[C[C[X]]]] <: N[C[X]]"} (inheritance)
+           | ${"C[X] <: N[C[C[X]]]"} (contravariance)
+           | ${"N[N[C[C[X]]]} <: N[C[C[X]]]"} (inheritance)
+           | ${"C[C[X]] <: N[C[C[X]]]"} (contravariance)
+           | ...
+           |
+           | Notice how the last test is just like the first, but with an additional ${"C"}, which means we will
+           | loop forever.
+        """.stripMargin
+  }
 }

compiler/src/dotty/tools/dotc/typer/Checking.scala

@@ -707,6 +707,88 @@ trait Checking {
       checkNoForwardDependencies(vparams1)
     case Nil =>
   }
+
+  /*** Check that none of the type parameters of a class are part of an expansive cycle.
+    *
+    *  For the definition of expansive cycles, see Kennedy, Andrew J., and Benjamin C. Pierce.
+    *  "On decidability of nominal subtyping with variance." (2006).
+    */
+  def checkExpansiveCycles(classDef: TypeDef)(implicit ctx: Context): Unit = {
+    classDef.denot.info match {
+      case cinfo: ClassInfo =>
+        cinfo.typeParams foreach {
+          case tparam: Symbol =>
+            if (expansiveParams(tparam) contains tparam) {
+              ctx.error(TypeParamInExpansiveCycle(cinfo, tparam), tparam.pos)
+            }
+          case _ =>
+        }
+      case _ =>
+    }
+  }
+
+  /*** A state in the type parameter dependency graph indicates the referenced type parameter and whether we got
+    *  to it through a path that includes at least one expansive edge.
+    */
+  private case class ExpansiveSt(tparam: Symbol, expansive: Boolean)
+
+  /*** Calculates the set of type parameters that can be reached from the given type parameter by following a
+    *  path containing at least one expansive edge. The set is calculated via BFS.
+    */
+  private def expansiveParams(tparam: Symbol)(implicit ctx: Context): Set[Symbol] = {
+    val worklist = mutable.Queue[ExpansiveSt]()
+    val seen = mutable.Set[ExpansiveSt]()
+    def enqueue(st: ExpansiveSt): Unit = {
+      if (!(seen contains st)) {
+        worklist += st
+        seen += st
+      }
+    }
+    enqueue(ExpansiveSt(tparam, expansive=false))
+    while (worklist.nonEmpty) {
+      val st@ExpansiveSt(fromParam, expansive) = worklist.dequeue()
+      val findNext = new ExpansiveStAcc(st)
+      fromParam.owner.info match {
+        case cinfo: ClassInfo =>
+          for {
+            parent <- cinfo.parentsWithArgs
+            toParams = findNext(List.empty[ExpansiveSt], parent)
+            toParam <- toParams
+          } enqueue(toParam)
+        case _ =>
+      }
+    }
+    Set.empty[Symbol] ++ seen.filter((st) => st.expansive).map((st) => st.tparam)
+  }
+
+  /*** Given a type parameter and a type, finds all refinements in the type that reference the type parameter,
+    *  and whether they do so in an expansive manner.
+    */
+  private class ExpansiveStAcc(state: ExpansiveSt)(implicit ctx: Context) extends TypeAccumulator[List[ExpansiveSt]] {
+    val ExpansiveSt(tparam, expansive) = state
+
+    /** Does the type reference the type parameter somewhere in it? */
+    def hasReference(typ: Type): Boolean = {
+      typ.namedPartsWith(_.symbol == tparam).nonEmpty
+    }
+
+    override def apply(acc: List[ExpansiveSt], tp: Type): List[ExpansiveSt] = {
+      tp match {
+        case RefinedType(parent, name: TypeName, info) =>
+          val toParam = parent.typeParamNamed(name)
+          if (info.typeSymbol == tparam) {
+            // A non-expansive edge: `tparam` replaces `toParam` directly.
+            ExpansiveSt(toParam, expansive = expansive) :: acc
+          } else if (hasReference(info)) {
+            // An expansive edge: `tparam` is passed as *part* of the argument replacing `toParam`.
+            this (ExpansiveSt(toParam, expansive = true) :: acc, info)
+          } else {
+            acc
+          }
+        case _ => foldOver(acc, tp)
+      }
+    }
+  }
 }
 
 trait NoChecking extends Checking {

compiler/src/dotty/tools/dotc/typer/Typer.scala

@@ -1429,6 +1429,8 @@ class Typer extends Namer with TypeAssigner with Applications with Implicits wit
       // check value class constraints
       checkDerivedValueClass(cls, body1)
 
+      checkExpansiveCycles(cdef1)
+
       if (ctx.settings.YretainTrees.value) {
         cls.myTree = cdef1
       }