Merge pull request #1910 from ahoppen/ahoppen/prefix-consumption

Simplify consumption of prefixes in a token

Author: ahoppen

Sources/SwiftParser/Declarations.swift

@@ -407,22 +407,6 @@ extension Parser {
 }
 
 extension Parser {
-  /// Attempt to consume an ellipsis prefix, splitting the current token if
-  /// necessary.
-  mutating func tryConsumeEllipsisPrefix() -> RawTokenSyntax? {
-    // It is not sufficient to check currentToken.isEllipsis here, as we may
-    // have something like '...>'.
-    // TODO: Recovery for different numbers of dots (which also needs to be
-    // done for regular variadics).
-    guard self.at(anyIn: Operator.self) != nil else { return nil }
-    let text = self.currentToken.tokenText
-    guard text.hasPrefix("...") else { return nil }
-    return self.consumePrefix(
-      SyntaxText(rebasing: text.prefix(3)),
-      as: .ellipsis
-    )
-  }
-
   mutating func parseGenericParameters() -> RawGenericParameterClauseSyntax {
     if let remainingTokens = remainingTokensIfMaximumNestingLevelReached() {
       return RawGenericParameterClauseSyntax(
@@ -435,12 +419,7 @@ extension Parser {
       )
     }
 
-    let langle: RawTokenSyntax
-    if self.currentToken.starts(with: "<") {
-      langle = self.consumePrefix("<", as: .leftAngle)
-    } else {
-      langle = missingToken(.leftAngle)
-    }
+    let langle = self.expectWithoutRecovery(prefix: "<", as: .leftAngle)
     var elements = [RawGenericParameterSyntax]()
     do {
       var keepGoing: RawTokenSyntax? = nil
@@ -452,14 +431,13 @@ extension Parser {
         var each = self.consume(if: .keyword(.each))
 
         let (unexpectedBetweenEachAndName, name) = self.expectIdentifier(allowSelfOrCapitalSelfAsIdentifier: true)
-        if attributes == nil && each == nil && unexpectedBetweenEachAndName == nil && name.isMissing && elements.isEmpty && !self.currentToken.starts(with: ">")
-        {
+        if attributes == nil && each == nil && unexpectedBetweenEachAndName == nil && name.isMissing && elements.isEmpty && !self.at(prefix: ">") {
           break
         }
 
         // Parse the unsupported ellipsis for a type parameter pack 'T...'.
         let unexpectedBetweenNameAndColon: RawUnexpectedNodesSyntax?
-        if let ellipsis = tryConsumeEllipsisPrefix() {
+        if let ellipsis = self.consume(ifPrefix: "...", as: .ellipsis) {
           unexpectedBetweenNameAndColon = RawUnexpectedNodesSyntax([ellipsis], arena: self.arena)
           if each == nil {
             each = missingToken(.each)
@@ -518,12 +496,7 @@ extension Parser {
       whereClause = nil
     }
 
-    let rangle: RawTokenSyntax
-    if self.currentToken.starts(with: ">") {
-      rangle = self.consumePrefix(">", as: .rightAngle)
-    } else {
-      rangle = RawTokenSyntax(missing: .rightAngle, arena: self.arena)
-    }
+    let rangle = expectWithoutRecovery(prefix: ">", as: .rightAngle)
 
     let parameters: RawGenericParameterListSyntax
     if elements.isEmpty && rangle.isMissing {
@@ -934,7 +907,7 @@ extension Parser {
     }
 
     // Detect an attempt to use (early syntax) type parameter pack.
-    let ellipsis = tryConsumeEllipsisPrefix()
+    let ellipsis = self.consume(ifPrefix: "...", as: .ellipsis)
 
     // Parse optional inheritance clause.
     let inheritance: RawTypeInheritanceClauseSyntax?
@@ -1011,7 +984,7 @@ extension Parser {
     }
 
     let generics: RawGenericParameterClauseSyntax?
-    if self.currentToken.starts(with: "<") {
+    if self.at(prefix: "<") {
       generics = self.parseGenericParameters()
     } else {
       generics = nil
@@ -1145,7 +1118,7 @@ extension Parser {
     }
 
     let genericParams: RawGenericParameterClauseSyntax?
-    if self.currentToken.starts(with: "<") {
+    if self.at(prefix: "<") {
       genericParams = self.parseGenericParameters()
     } else {
       genericParams = nil
@@ -1229,14 +1202,14 @@ extension Parser {
     let (unexpectedBeforeSubscriptKeyword, subscriptKeyword) = self.eat(handle)
 
     let unexpectedName: RawTokenSyntax?
-    if self.at(.identifier) && self.peek().starts(with: "<") || self.peek().rawTokenKind == .leftParen {
+    if self.at(.identifier) && self.peek().tokenText.hasPrefix("<") || self.peek().rawTokenKind == .leftParen {
       unexpectedName = self.consumeAnyToken()
     } else {
       unexpectedName = nil
     }
 
     let genericParameterClause: RawGenericParameterClauseSyntax?
-    if self.currentToken.starts(with: "<") {
+    if self.at(prefix: "<") {
       genericParameterClause = self.parseGenericParameters()
     } else {
       genericParameterClause = nil
@@ -1617,7 +1590,7 @@ extension Parser {
 
     // Parse a generic parameter list if it is present.
     let generics: RawGenericParameterClauseSyntax?
-    if self.currentToken.starts(with: "<") {
+    if self.at(prefix: "<") {
       generics = self.parseGenericParameters()
     } else {
       generics = nil
@@ -2012,7 +1985,7 @@ extension Parser {
 
     // Optional generic parameters.
     let genericParams: RawGenericParameterClauseSyntax?
-    if self.currentToken.starts(with: "<") {
+    if self.at(prefix: "<") {
       genericParams = self.parseGenericParameters()
     } else {
       genericParams = nil

Sources/SwiftParser/Expressions.swift

@@ -1023,21 +1023,12 @@ extension Parser {
 
     for _ in 0..<numComponents {
       // Consume a period, if there is one.
-      let period: RawTokenSyntax?
-      if self.currentToken.starts(with: ".") {
-        period = self.consumePrefix(".", as: .period)
-      } else {
-        period = nil
-      }
+      let period = self.consume(ifPrefix: ".", as: .period)
 
       // Consume the '!' or '?'.
-      let questionOrExclaim: RawTokenSyntax
-      if self.currentToken.starts(with: "!") {
-        questionOrExclaim = self.consumePrefix("!", as: .exclamationMark)
-      } else {
-        precondition(self.currentToken.starts(with: "?"))
-        questionOrExclaim = self.consumePrefix("?", as: .postfixQuestionMark)
-      }
+      let questionOrExclaim =
+        self.consume(ifPrefix: "!", as: .exclamationMark)
+        ?? self.expectWithoutRecovery(prefix: "?", as: .postfixQuestionMark)
 
       components.append(
         RawKeyPathComponentSyntax(
@@ -1078,7 +1069,7 @@ extension Parser {
     // operator token. Since keypath allows '.!' '.?' and '.[', consume '.'
     // the token is an operator starts with '.', or the following token is '['.
     let rootType: RawTypeSyntax?
-    if !self.currentToken.starts(with: ".") {
+    if !self.at(prefix: ".") {
       rootType = self.parseSimpleType(stopAtFirstPeriod: true)
     } else {
       rootType = nil

Sources/SwiftParser/Lookahead.swift

@@ -117,6 +117,10 @@ extension Parser.Lookahead {
   ///
   ///     <TOKEN> ... -> consumePrefix(<TOK>) -> [ <TOK> ] <EN> ...
   mutating func consumePrefix(_ prefix: SyntaxText, as tokenKind: RawTokenKind) {
+    precondition(
+      tokenKind.defaultText == nil || prefix == tokenKind.defaultText!,
+      "If tokenKind has a defaultText, the prefix needs to match it"
+    )
     let tokenText = self.currentToken.tokenText
 
     if tokenText == prefix {

Sources/SwiftParser/Names.swift

@@ -228,7 +228,7 @@ extension Parser.Lookahead {
     guard lookahead.canParseTypeIdentifier() else {
       return false
     }
-    return lookahead.currentToken.starts(with: ".")
+    return lookahead.at(prefix: ".")
   }
 }
 
@@ -289,12 +289,4 @@ extension Lexer.Lexeme {
     // constructed from them.
     return self.rawTokenKind == .keyword
   }
-
-  func starts(with symbol: SyntaxText) -> Bool {
-    guard Operator(lexeme: self) != nil || self.rawTokenKind.isPunctuation else {
-      return false
-    }
-
-    return self.tokenText.hasPrefix(symbol)
-  }
 }

Sources/SwiftParser/Nominals.swift

@@ -233,7 +233,7 @@ extension Parser {
     }
 
     let primaryOrGenerics: T.PrimaryOrGenerics?
-    if self.currentToken.starts(with: "<") {
+    if self.at(prefix: "<") {
       primaryOrGenerics = T.parsePrimaryOrGenerics(&self)
     } else {
       primaryOrGenerics = nil
@@ -352,18 +352,10 @@ extension Parser {
         )
       } while keepGoing != nil && loopProgress.evaluate(currentToken)
     }
-    let unexpectedBeforeRangle: RawUnexpectedNodesSyntax?
-    let rangle: RawTokenSyntax
-    if self.currentToken.starts(with: ">") {
-      unexpectedBeforeRangle = nil
-      rangle = self.consumePrefix(">", as: .rightAngle)
-    } else {
-      (unexpectedBeforeRangle, rangle) = self.expect(.rightAngle)
-    }
+    let rangle = self.expectWithoutRecovery(prefix: ">", as: .rightAngle)
     return RawPrimaryAssociatedTypeClauseSyntax(
       leftAngle: langle,
       primaryAssociatedTypeList: RawPrimaryAssociatedTypeListSyntax(elements: associatedTypes, arena: self.arena),
-      unexpectedBeforeRangle,
       rightAngle: rangle,
       arena: self.arena
     )

Sources/SwiftParser/Parser.swift

@@ -471,6 +471,17 @@ extension Parser {
     )
   }
 
+  /// If the current token starts with the given prefix, consume the prefis as the given token kind.
+  ///
+  /// Otherwise, synthesize a missing token of the given kind.
+  mutating func expectWithoutRecovery(prefix: SyntaxText, as tokenKind: RawTokenKind) -> Token {
+    if self.at(prefix: prefix) {
+      return consumePrefix(prefix, as: tokenKind)
+    } else {
+      return missingToken(tokenKind)
+    }
+  }
+
   /// - Parameters:
   ///   - keywordRecovery: If set to `true` and the parser is currently
   ///     positioned at a keyword instead of an identifier, this method recovers
@@ -594,6 +605,10 @@ extension Parser {
     _ prefix: SyntaxText,
     as tokenKind: RawTokenKind
   ) -> RawTokenSyntax {
+    precondition(
+      tokenKind.defaultText == nil || prefix == tokenKind.defaultText!,
+      "If tokenKind has a defaultText, the prefix needs to match it"
+    )
     let current = self.currentToken
     // Current token can be either one-character token we want to consume...
     let tokenText = current.tokenText

Sources/SwiftParser/TokenConsumer.swift

@@ -23,6 +23,13 @@ protocol TokenConsumer {
   /// Consume the current token and change its token kind to `remappedTokenKind`.
   mutating func consumeAnyToken(remapping remappedTokenKind: RawTokenKind) -> Token
 
+  /// Consumes a given token, or splits the current token into a leading token
+  /// matching the given token and a trailing token and consumes the leading
+  /// token.
+  ///
+  ///     <TOKEN> ... -> consumePrefix(<TOK>) -> [ <TOK> ] <EN> ...
+  mutating func consumePrefix(_ prefix: SyntaxText, as tokenKind: RawTokenKind) -> Token
+
   /// Synthesize a missing token with `kind`.
   /// If `text` is not `nil`, use it for the token's text, otherwise use the token's default text.
   mutating func missingToken(_ kind: RawTokenKind, text: SyntaxText?) -> Token
@@ -137,6 +144,12 @@ extension TokenConsumer {
     return nil
   }
 
+  /// Whether the current token’s text starts with the given prefix.
+  @inline(__always)
+  mutating func at(prefix: SyntaxText) -> Bool {
+    return self.currentToken.tokenText.hasPrefix(prefix)
+  }
+
   /// Eat a token that we know we are currently positioned at, based on `at(anyIn:)`.
   @inline(__always)
   mutating func eat(_ handle: TokenConsumptionHandle) -> Token {
@@ -248,6 +261,17 @@ extension TokenConsumer {
       return nil
     }
   }
+
+  /// If the current token starts with the given prefix, consume the prefis as the given token kind.
+  ///
+  /// Otherwise, return `nil`.
+  mutating func consume(ifPrefix prefix: SyntaxText, as tokenKind: RawTokenKind) -> Token? {
+    if self.at(prefix: prefix) {
+      return consumePrefix(prefix, as: tokenKind)
+    } else {
+      return nil
+    }
+  }
 }
 
 // MARK: Convenience functions

Sources/SwiftParser/Types.swift

@@ -422,8 +422,7 @@ extension Parser {
   ///     generic-argument-list → generic-argument | generic-argument ',' generic-argument-list
   ///     generic-argument → type
   mutating func parseGenericArguments() -> RawGenericArgumentClauseSyntax {
-    precondition(self.currentToken.starts(with: "<"))
-    let langle = self.consumePrefix("<", as: .leftAngle)
+    let langle = self.expectWithoutRecovery(prefix: "<", as: .leftAngle)
     var arguments = [RawGenericArgumentSyntax]()
     do {
       var keepGoing: RawTokenSyntax? = nil
@@ -444,12 +443,7 @@ extension Parser {
       } while keepGoing != nil && loopProgress.evaluate(currentToken)
     }
 
-    let rangle: RawTokenSyntax
-    if self.currentToken.starts(with: ">") {
-      rangle = self.consumePrefix(">", as: .rightAngle)
-    } else {
-      rangle = RawTokenSyntax(missing: .rightAngle, arena: self.arena)
-    }
+    let rangle = self.expectWithoutRecovery(prefix: ">", as: .rightAngle)
 
     let args: RawGenericArgumentListSyntax
     if arguments.isEmpty && rangle.isMissing {
@@ -656,7 +650,7 @@ extension Parser.Lookahead {
       return false
     }
 
-    if self.currentToken.isEllipsis {
+    if self.atContextualPunctuator("...") {
       self.consumeAnyToken()
     }
 
@@ -884,7 +878,7 @@ extension Parser.Lookahead {
     self.consumeAnyToken()
 
     // Parse an optional generic argument list.
-    if self.currentToken.starts(with: "<") && !self.consumeGenericArguments() {
+    if self.at(prefix: "<") && !self.consumeGenericArguments() {
       return false
     }
 
@@ -905,13 +899,11 @@ extension Parser.Lookahead {
 
   mutating func consumeGenericArguments() -> Bool {
     // Parse the opening '<'.
-    guard self.currentToken.starts(with: "<") else {
+    guard self.consume(ifPrefix: "<", as: .leftAngle) != nil else {
       return false
     }
 
-    self.consumePrefix("<", as: .leftAngle)
-
-    if !self.currentToken.starts(with: ">") {
+    if !self.at(prefix: ">") {
       var loopProgress = LoopProgressCondition()
       repeat {
         guard self.canParseType() else {
@@ -921,11 +913,10 @@ extension Parser.Lookahead {
       } while self.consume(if: .comma) != nil && loopProgress.evaluate(currentToken)
     }
 
-    guard self.currentToken.starts(with: ">") else {
+    guard self.consume(ifPrefix: ">", as: .rightAngle) != nil else {
       return false
     }
 
-    self.consumePrefix(">", as: .rightAngle)
     return true
   }
 }
@@ -1001,7 +992,7 @@ extension Parser {
 
 extension Parser {
   mutating func parseResultType() -> RawTypeSyntax {
-    if self.currentToken.starts(with: "<") {
+    if self.at(prefix: "<") {
       let generics = self.parseGenericParameters()
       let baseType = self.parseType()
       return RawTypeSyntax(
@@ -1076,10 +1067,6 @@ extension Lexer.Lexeme {
       || self.rawTokenKind == .prefixOperator
   }
 
-  var isEllipsis: Bool {
-    return self.isAnyOperator && self.tokenText == "..."
-  }
-
   var isGenericTypeDisambiguatingToken: Bool {
     switch self.rawTokenKind {
     case .rightParen,