Avoid inifinite loop when comparing & types with class types

The problem is that in two cases of subtype checking we form the base type
of the lhs, and, if that yields a new type continue with that one. But
this is not enough to ensure progress. Concrete test cases were in `run/mixins1`
and pos/intersections.scala. In `mixins1` we had an infinite cycle in the situation

     A[LazyRef(This)] & B <: A[B]

where `This <: A[This]` and `B extended A[B]`. We got `A[LazyRef(This)] & A[B]` as a base type,
which simplified to `A[_ >: LayzRef(This) | B <: LazyRef(This) & B]`. Then,  subsequent base
types cycled back and forth until the original type was reached.

The change in TypeComparer avoids the simplification (which is really a complication) and
keeps instead ``A[LazyRef(This)] & A[B]`. We then need the change in SymDenotations to
make sure that the base type of this type wrt `A` is referentially the same type, so that
we avoid the loop.

I wish there was a more robust measure for determining whether taking a base type represents
progress.

Author: odersky

compiler/src/dotty/tools/dotc/core/SymDenotations.scala

@@ -1633,7 +1633,7 @@ object SymDenotations {
           case tp: TypeVar => tp.inst.exists && inCache(tp.inst)
           //case tp: TypeProxy => inCache(tp.underlying) // disabled, can re-enable insyead of last two lines for performance testing
           //case tp: AndOrType => inCache(tp.tp1) && inCache(tp.tp2)
-          case tp: TypeProxy => isCachable(tp.underlying, btrCache) 
+          case tp: TypeProxy => isCachable(tp.underlying, btrCache)
           case tp: AndOrType => isCachable(tp.tp1, btrCache) && isCachable(tp.tp2, btrCache)
           case _ => true
         }
@@ -1678,9 +1678,15 @@ object SymDenotations {
           case tp: TypeProxy =>
             baseTypeOf(tp.superType)
           case AndType(tp1, tp2) =>
-            baseTypeOf(tp1) & baseTypeOf(tp2)
+            baseTypeOf(tp1) & baseTypeOf(tp2) match {
+              case AndType(tp1a, tp2a) if (tp1a eq tp1) && (tp2a eq tp2) => tp
+              case res => res
+            }
           case OrType(tp1, tp2) =>
-            baseTypeOf(tp1) | baseTypeOf(tp2)
+            baseTypeOf(tp1) | baseTypeOf(tp2) match {
+              case OrType(tp1a, tp2a) if (tp1a eq tp1) && (tp2a eq tp2) => tp
+              case res => res
+            }
           case JavaArrayType(_) if symbol == defn.ObjectClass =>
             this.typeRef
           case _ =>

compiler/src/dotty/tools/dotc/core/TypeComparer.scala

@@ -1448,10 +1448,13 @@ class TypeComparer(initctx: Context) extends DotClass with ConstraintHandling {
         val arg2 :: args2Rest = args2
         val v = tparam.paramVariance
         val glbArg =
-          if (v > 0) glb(arg1.hiBound, arg2.hiBound)
+          if (isSameTypeWhenFrozen(arg1, arg2)) arg1
+          else if (v > 0) glb(arg1.hiBound, arg2.hiBound)
           else if (v < 0) lub(arg1.loBound, arg2.loBound)
-          else TypeBounds(lub(arg1.loBound, arg2.loBound),
-                          glb(arg1.hiBound, arg2.hiBound))
+          else if (arg1.isInstanceOf[TypeBounds] || arg2.isInstanceOf[TypeBounds])
+            TypeBounds(lub(arg1.loBound, arg2.loBound),
+                       glb(arg1.hiBound, arg2.hiBound))
+          else NoType
         glbArg :: glbArgs(args1Rest, args2Rest, tparamsRest)
       case nil =>
         Nil
@@ -1599,7 +1602,9 @@ class TypeComparer(initctx: Context) extends DotClass with ConstraintHandling {
     case tp1 @ AppliedType(tycon1, args1) =>
       tp2 match {
         case AppliedType(tycon2, args2) if tycon1.typeSymbol == tycon2.typeSymbol =>
-          (tycon1 & tycon2).appliedTo(glbArgs(args1, args2, tycon1.typeParams))
+          val jointArgs = glbArgs(args1, args2, tycon1.typeParams)
+          if (jointArgs.forall(_.exists)) (tycon1 & tycon2).appliedTo(jointArgs)
+          else NoType
         case _ =>
           NoType
       }
@@ -1730,15 +1735,17 @@ class TypeComparer(initctx: Context) extends DotClass with ConstraintHandling {
 
   /** Show subtype goal that led to an assertion failure */
   def showGoal(tp1: Type, tp2: Type)(implicit ctx: Context) = {
-    println(ex"assertion failure for $tp1 <:< $tp2, frozen = $frozenConstraint")
+    println(i"assertion failure for $tp1 <:< $tp2, frozen = $frozenConstraint")
     def explainPoly(tp: Type) = tp match {
       case tp: TypeParamRef => ctx.echo(s"TypeParamRef ${tp.show} found in ${tp.binder.show}")
       case tp: TypeRef if tp.symbol.exists => ctx.echo(s"typeref ${tp.show} found in ${tp.symbol.owner.show}")
       case tp: TypeVar => ctx.echo(s"typevar ${tp.show}, origin = ${tp.origin}")
       case _ => ctx.echo(s"${tp.show} is a ${tp.getClass}")
     }
-    explainPoly(tp1)
-    explainPoly(tp2)
+    if (Config.verboseExplainSubtype) {
+      explainPoly(tp1)
+      explainPoly(tp2)
+    }
   }
 
   /** Record statistics about the total number of subtype checks