Complete regex literal lexing

The previous regex literal lexing logic didn't
implement all the heuristics needed to parse all
the regex literals that the old C++ parser handles.
Update it with the heuristics it needs, and better
handle diagnostics for invalid cases.

This unfortunately requires a bit of lexical
hackery, including a heuristic to classify previous
token kinds to determine if we're in regex literal
position, but it's needed to handle pathological
cases such as `x /^ y/` where there's an ambiguity
with a binary operator.

Author: hamishknight

CodeGeneration/Sources/SyntaxSupport/Classification.swift

@@ -58,6 +58,7 @@ public let SYNTAX_CLASSIFICATIONS: [SyntaxClassification] = [
   SyntaxClassification(name: "ObjectLiteral", description: "An image, color, etc. literal."),
   SyntaxClassification(name: "OperatorIdentifier", description: "An identifier referring to an operator."),
   SyntaxClassification(name: "PoundDirectiveKeyword", description: "A `#` keyword like `#warning`."),
+  SyntaxClassification(name: "RegexLiteral", description: "A regex literal, including multiline regex literals."),
   SyntaxClassification(name: "StringInterpolationAnchor", description: "The opening and closing parenthesis of string interpolation."),
   SyntaxClassification(name: "StringLiteral", description: "A string literal including multiline string literals."),
   SyntaxClassification(name: "TypeIdentifier", description: "An identifier referring to a type."),

CodeGeneration/Sources/SyntaxSupport/ExprNodes.swift

@@ -1199,9 +1199,27 @@ public let EXPR_NODES: [Node] = [
     kind: "Expr",
     children: [
       Child(
-        name: "Regex",
-        kind: .token(choices: [.token(tokenKind: "RegexLiteralToken")])
-      )
+        name: "OpeningPounds",
+        kind: .token(choices: [.token(tokenKind: "ExtendedRegexDelimiterToken")]),
+        isOptional: true
+      ),
+      Child(
+        name: "OpenSlash",
+        kind: .token(choices: [.token(tokenKind: "RegexSlashToken")])
+      ),
+      Child(
+        name: "RegexPattern",
+        kind: .token(choices: [.token(tokenKind: "RegexLiteralPatternToken")])
+      ),
+      Child(
+        name: "CloseSlash",
+        kind: .token(choices: [.token(tokenKind: "RegexSlashToken")])
+      ),
+      Child(
+        name: "ClosingPounds",
+        kind: .token(choices: [.token(tokenKind: "ExtendedRegexDelimiterToken")]),
+        isOptional: true
+      ),
     ]
   ),
 

CodeGeneration/Sources/SyntaxSupport/TokenSpec.swift

@@ -178,6 +178,7 @@ public let SYNTAX_TOKENS: [TokenSpec] = [
   PunctuatorSpec(name: "Ellipsis", kind: "ellipsis", text: "..."),
   PunctuatorSpec(name: "Equal", kind: "equal", text: "=", requiresLeadingSpace: true, requiresTrailingSpace: true),
   PunctuatorSpec(name: "ExclamationMark", kind: "exclaim_postfix", text: "!"),
+  MiscSpec(name: "ExtendedRegexDelimiter", kind: "extended_regex_delimiter", nameForDiagnostics: "extended delimiter", classification: "RegexLiteral"),
   LiteralSpec(name: "FloatingLiteral", kind: "floating_literal", nameForDiagnostics: "floating literal", classification: "FloatingLiteral"),
   MiscSpec(name: "Identifier", kind: "identifier", nameForDiagnostics: "identifier", classification: "Identifier"),
   PunctuatorSpec(name: "InfixQuestionMark", kind: "question_infix", text: "?"),
@@ -202,7 +203,8 @@ public let SYNTAX_TOKENS: [TokenSpec] = [
   PunctuatorSpec(name: "PrefixAmpersand", kind: "amp_prefix", text: "&"),
   MiscSpec(name: "PrefixOperator", kind: "oper_prefix", nameForDiagnostics: "prefix operator", classification: "OperatorIdentifier"),
   MiscSpec(name: "RawStringDelimiter", kind: "raw_string_delimiter", nameForDiagnostics: "raw string delimiter"),
-  LiteralSpec(name: "RegexLiteral", kind: "regex_literal", nameForDiagnostics: "regex literal"),
+  MiscSpec(name: "RegexLiteralPattern", kind: "regex_literal_pattern", nameForDiagnostics: "regex pattern", classification: "RegexLiteral"),
+  PunctuatorSpec(name: "RegexSlash", kind: "regex_slash", text: "/", classification: "RegexLiteral"),
   PunctuatorSpec(name: "RightAngle", kind: "r_angle", text: ">"),
   PunctuatorSpec(name: "RightBrace", kind: "r_brace", text: "}"),
   PunctuatorSpec(name: "RightParen", kind: "r_paren", text: ")"),

Sources/SwiftParser/CMakeLists.txt

@@ -42,6 +42,7 @@ add_swift_host_library(SwiftParser
   Lexer/Lexeme.swift
   Lexer/LexemeSequence.swift
   Lexer/Lexer.swift
+  Lexer/RegexLiteralLexer.swift
   Lexer/UnicodeScalarExtensions.swift
 )
 

Sources/SwiftParser/Declarations.swift

@@ -1315,28 +1315,6 @@ extension Parser {
 }
 
 extension Parser {
-  /// Are we at a regular expression literal that could act as an operator?
-  private mutating func atRegexLiteralThatCouldBeAnOperator() -> Bool {
-    guard self.at(.regexLiteral) else {
-      return false
-    }
-
-    /// Try to re-lex at regex literal as an operator. If it succeeds and
-    /// consumes the entire regex literal, we're done.
-    return self.currentToken.tokenText.withBuffer {
-      (buffer: UnsafeBufferPointer<UInt8>) -> Bool in
-      var cursor = Lexer.Cursor(input: buffer, previous: 0)
-      guard buffer[0] == UInt8(ascii: "/") else { return false }
-      switch cursor.lexOperatorIdentifier(sourceBufferStart: cursor).tokenKind {
-      case .unknown:
-        return false
-
-      default:
-        return cursor.input.isEmpty
-      }
-    }
-  }
-
   @_spi(RawSyntax)
   public mutating func parseFuncDeclaration(
     _ attrs: DeclAttributes,
@@ -1345,7 +1323,7 @@ extension Parser {
     let (unexpectedBeforeFuncKeyword, funcKeyword) = self.eat(handle)
     let unexpectedBeforeIdentifier: RawUnexpectedNodesSyntax?
     let identifier: RawTokenSyntax
-    if self.at(anyIn: Operator.self) != nil || self.at(.exclamationMark, .prefixAmpersand) || self.atRegexLiteralThatCouldBeAnOperator() {
+    if self.at(anyIn: Operator.self) != nil || self.at(.exclamationMark, .prefixAmpersand) {
       var name = self.currentToken.tokenText
       if name.count > 1 && name.hasSuffix("<") && self.peek().rawTokenKind == .identifier {
         name = SyntaxText(rebasing: name.dropLast())

Sources/SwiftParser/Expressions.swift

@@ -1170,7 +1170,7 @@ extension Parser {
       )
     case (.rawStringDelimiter, _)?, (.stringQuote, _)?, (.multilineStringQuote, _)?, (.singleQuote, _)?:
       return RawExprSyntax(self.parseStringLiteral())
-    case (.regexLiteral, _)?:
+    case (.extendedRegexDelimiter, _)?, (.regexSlash, _)?:
       return RawExprSyntax(self.parseRegexLiteral())
     case (.nilKeyword, let handle)?:
       let nilKeyword = self.eat(handle)
@@ -1433,13 +1433,32 @@ extension Parser {
   /// Grammar
   /// =======
   ///
-  ///     regular-expression-literal → '\' `Any valid regular expression characters` '\'
+  ///     regular-expression-literal → '#'* '/' `Any valid regular expression characters` '/' '#'*
   @_spi(RawSyntax)
   public mutating func parseRegexLiteral() -> RawRegexLiteralExprSyntax {
-    let (unexpectedBeforeLiteral, literal) = self.expect(.regexLiteral)
+    // See if we have an opening set of pounds.
+    let openPounds = self.consume(if: .extendedRegexDelimiter)
+
+    // Parse the opening slash.
+    let (unexpectedBeforeSlash, openSlash) = self.expect(.regexSlash)
+
+    // Parse the pattern and closing slash, avoiding recovery or leading trivia
+    // as the lexer should provide the tokens exactly in order without trivia,
+    // otherwise they should be treated as missing.
+    let pattern = self.expectWithoutRecoveryOrLeadingTrivia(.regexLiteralPattern)
+    let closeSlash = self.expectWithoutRecoveryOrLeadingTrivia(.regexSlash)
+
+    // Finally, parse a closing set of pounds.
+    let (unexpectedBeforeClosePounds, closePounds) = parsePoundDelimiter(.extendedRegexDelimiter, matching: openPounds)
+
     return RawRegexLiteralExprSyntax(
-      unexpectedBeforeLiteral,
-      regex: literal,
+      openingPounds: openPounds,
+      unexpectedBeforeSlash,
+      openSlash: openSlash,
+      regexPattern: pattern,
+      closeSlash: closeSlash,
+      unexpectedBeforeClosePounds,
+      closingPounds: closePounds,
       arena: self.arena
     )
   }