Merge pull request #2543 from ahoppen/ahoppen/loop-progress-condition-false-negative

Fix a false negative assertion failure in `LoopProgressCondition` for unterminated nested string interpolation

Author: ahoppen

Sources/SwiftParser/Lexer/Cursor.swift

@@ -52,7 +52,7 @@ extension Lexer.Cursor {
   ///  - A string interpolation inside is entered
   ///  - A regex literal is being lexed
   ///  - A narrow case for 'try?' and 'try!' to ensure correct regex lexing
-  enum State {
+  enum State: Equatable {
     /// Normal top-level lexing mode
     case normal
 
@@ -203,6 +203,11 @@ extension Lexer.Cursor {
         }
       }
     }
+
+    /// See `Lexer.Cursor.hasProgressed(comparedTo:)`.
+    fileprivate func hasProgressed(comparedTo other: StateStack) -> Bool {
+      return currentState != other.currentState || stateStack?.count != other.stateStack?.count
+    }
   }
 
   /// An error that was discovered in a lexeme while lexing it.
@@ -257,6 +262,16 @@ extension Lexer {
       self.position = Position(input: input, previous: previous)
     }
 
+    /// Returns `true` if this cursor is sufficiently different to `other` in a way that indicates that the lexer has
+    /// made progress.
+    ///
+    /// This is the case if the lexer advanced its position in the source file or if it has performed a state
+    /// transition.
+    func hasProgressed(comparedTo other: Cursor) -> Bool {
+      return position.input.baseAddress != other.position.input.baseAddress
+        || stateStack.hasProgressed(comparedTo: other.stateStack)
+    }
+
     var input: UnsafeBufferPointer<UInt8> { position.input }
     var previous: UInt8 { position.previous }
 

Sources/SwiftParser/LoopProgressCondition.swift

@@ -29,14 +29,7 @@ struct LoopProgressCondition {
     guard let previousToken = self.currentToken else {
       return true
     }
-    // The loop has made progress if either
-    //  - the parser is now pointing at a different location in the source file
-    //  - the parser is still pointing at the same position in the source file
-    //     but now has a different token kind (and thus consumed a zero-length
-    //     token like an empty string interpolation
-    let hasMadeProgress =
-      previousToken.tokenText.baseAddress != currentToken.tokenText.baseAddress
-      || (previousToken.byteLength == 0 && previousToken.rawTokenKind != currentToken.rawTokenKind)
+    let hasMadeProgress = currentToken.cursor.hasProgressed(comparedTo: previousToken.cursor)
     assert(hasMadeProgress, "Loop should always make progress")
     return hasMadeProgress
   }

Tests/SwiftParserTest/LexerTests.swift

@@ -1561,4 +1561,26 @@ class LexerTests: ParserTestCase {
       ]
     )
   }
+
+  func testNestedUnterminatedStringInterpolations() {
+    assertLexemes(
+      #"""
+      "\("\(
+
+      """#,
+      lexemes: [
+        LexemeSpec(.stringQuote, text: #"""#),
+        LexemeSpec(.stringSegment, text: ""),
+        LexemeSpec(.backslash, text: #"\"#),
+        LexemeSpec(.leftParen, text: "("),
+        LexemeSpec(.stringQuote, text: #"""#),
+        LexemeSpec(.stringSegment, text: ""),
+        LexemeSpec(.backslash, text: #"\"#),
+        LexemeSpec(.leftParen, text: "("),
+        LexemeSpec(.stringSegment, text: ""),
+        LexemeSpec(.stringSegment, text: ""),
+        LexemeSpec(.endOfFile, leading: "\n", text: "", flags: [.isAtStartOfLine]),
+      ]
+    )
+  }
 }

Tests/SwiftParserTest/translated/UnclosedStringInterpolationTests.swift

@@ -247,4 +247,38 @@ final class UnclosedStringInterpolationTests: ParserTestCase {
         """#
     )
   }
+
+  func testNestedUnterminatedStringInterpolation() {
+    assertParse(
+      #"""
+      1️⃣"\2️⃣(3️⃣"\(4️⃣
+
+      """#,
+      diagnostics: [
+        DiagnosticSpec(locationMarker: "4️⃣", message: "expected value and ')' in string literal", fixIts: ["insert value and ')'"]),
+        DiagnosticSpec(
+          locationMarker: "4️⃣",
+          message: #"expected '"' to end string literal"#,
+          notes: [NoteSpec(locationMarker: "3️⃣", message: #"to match this opening '"'"#)],
+          fixIts: [#"insert '"'"#]
+        ),
+        DiagnosticSpec(
+          locationMarker: "4️⃣",
+          message: "expected ')' in string literal",
+          notes: [NoteSpec(locationMarker: "2️⃣", message: "to match this opening '('")],
+          fixIts: ["insert ')'"]
+        ),
+        DiagnosticSpec(
+          locationMarker: "4️⃣",
+          message: #"expected '"' to end string literal"#,
+          notes: [NoteSpec(locationMarker: "1️⃣", message: #"to match this opening '"'"#)],
+          fixIts: [#"insert '"'"#]
+        ),
+      ],
+      fixedSource: #"""
+        "\("\(<#expression#>)")"
+
+        """#
+    )
+  }
 }