isSubtype: try to dealias TypeRefs before recursively checking the prefixes

As demonstrated by tests/pos/hk-deep-subtype.scala, we can avoid some
deep subtype recursions that result in stack overflows by doing this.

Author: smarter

src/dotty/tools/dotc/core/TypeComparer.scala

@@ -153,6 +153,7 @@ class TypeComparer(initctx: Context) extends DotClass with ConstraintHandling {
         tp1 match {
           case tp1: NamedType =>
             val sym1 = tp1.symbol
+            compareAlias(tp1.info) ||
             (if ((sym1 ne NoSymbol) && (sym1 eq tp2.symbol))
                ctx.erasedTypes || sym1.isStaticOwner || isSubType(tp1.prefix, tp2.prefix)
             else
@@ -163,8 +164,7 @@ class TypeComparer(initctx: Context) extends DotClass with ConstraintHandling {
               !tp2.isInstanceOf[WithFixedSym]
             ) ||
             compareHK(tp1, tp2, inOrder = true) ||
-            compareHK(tp2, tp1, inOrder = false) ||
-            compareAlias(tp1.info)
+            compareHK(tp2, tp1, inOrder = false)
           case _ =>
             compareHK(tp2, tp1, inOrder = false) ||
             compareAlias(NoType)

test/dotc/scala-collections.whitelist

@@ -263,8 +263,7 @@
 ./scala-scala/src/library/scala/collection/generic/GenericSeqCompanion.scala
 ./scala-scala/src/library/scala/collection/generic/GenericSetTemplate.scala
 
-# deep subtype
-#./scala-scala/src/library/scala/collection/generic/GenericTraversableTemplate.scala
+./scala-scala/src/library/scala/collection/generic/GenericTraversableTemplate.scala
 
 ./scala-scala/src/library/scala/collection/generic/HasNewBuilder.scala
 ./scala-scala/src/library/scala/collection/generic/HasNewCombiner.scala

tests/pos/hk-deep-subtype.scala

@@ -0,0 +1,15 @@
+// Minimized from scala.collection.generic.GenTraversableFactory plus dependencies
+import scala.annotation.unchecked.uncheckedVariance
+
+trait GT[A] extends GTT[A, GT]
+
+trait HNB[B]
+trait GTT[+C, DD[X] <: GT[X]] extends HNB[DD[C] @uncheckedVariance] // Can be any annotation and still crash
+
+class GTF[EE[X] <: GT[X] with GTT[X, EE]]
+{
+  def foo[F]: EE[F] = ???
+  def bar[G](f: G): EE[G] = ???
+
+  def tabulate: EE[EE[Int]] = bar(foo)
+}