Merge pull request #12250 from dotty-staging/fix-12050

Fix #12050: catch StackOverflow in tryNormalize

Author: odersky

compiler/src/dotty/tools/dotc/core/Types.scala

@@ -4550,7 +4550,12 @@ object Types {
     private var myReduced: Type = null
     private var reductionContext: util.MutableMap[Type, Type] = null
 
-    override def tryNormalize(using Context): Type = reduced.normalized
+    override def tryNormalize(using Context): Type =
+      try
+        reduced.normalized
+      catch
+        case ex: Throwable =>
+          handleRecursive("normalizing", s"${scrutinee.show} match ..." , ex)
 
     def reduced(using Context): Type = {
 

tests/neg/12050.scala

@@ -0,0 +1,34 @@
+class P[X, Y]
+
+type Swap[X] = X match
+  case P[x, y] => Swap[P[y, x]]
+
+val z: P[String, Int] = ??? : Swap[P[Int, String]] // error
+//                            ^
+// Recursion limit exceeded.
+// Maybe there is an illegal cyclic reference?
+// If that's not the case, you could also try to increase the stacksize using the -Xss JVM option.
+// A recurring operation is (inner to outer):
+//
+//   normalizing P[String, Int] match ...
+//   normalizing P[Int, String] match ...
+//   normalizing P[String, Int] match ...
+//   normalizing P[Int, String] match ...
+//   normalizing P[String, Int] match ...
+//   normalizing P[Int, String] match ...
+//   normalizing P[String, Int] match ...
+//   normalizing P[Int, String] match ...
+//   normalizing P[String, Int] match ...
+//   normalizing P[Int, String] match ...
+//   ...
+//
+//   normalizing P[String, Int] match ...
+//   normalizing P[Int, String] match ...
+//   normalizing P[String, Int] match ...
+//   normalizing P[Int, String] match ...
+//   normalizing P[String, Int] match ...
+//   normalizing P[Int, String] match ...
+//   normalizing P[String, Int] match ...
+//   normalizing P[Int, String] match ...
+//   normalizing P[String, Int] match ...
+//   normalizing P[Int, String] match ...