Merge pull request #1475 from ahoppen/ahoppen/cherry-pick

Cherry-pick PRs from `main` to `release/5.9`

Author: ahoppen

CMakeLists.txt

@@ -68,6 +68,11 @@ message(STATUS "Module triple: ${SWIFT_MODULE_TRIPLE}")
 add_compile_definitions(
   $<$<COMPILE_LANGUAGE:Swift>:SWIFT_SYNTAX_ALWAYS_SINGLE_THREADED>
 )
+if (SWIFTSYNTAX_ENABLE_ASSERTIONS)
+  add_compile_definitions(
+    $<$<COMPILE_LANGUAGE:Swift>:SWIFTSYNTAX_ENABLE_ASSERTIONS>
+  )
+endif()
 
 add_subdirectory(Sources)
 

CodeGeneration/Package.swift

@@ -11,7 +11,7 @@ let package = Package(
     .executable(name: "generate-swiftsyntax", targets: ["generate-swiftsyntax"])
   ],
   dependencies: [
-    .package(url: "https://github.com/apple/swift-syntax.git", revision: "94b9021a2e461fc9a4b3bda6f4734119b52e8094"),
+    .package(url: "https://github.com/apple/swift-syntax.git", revision: "39b3336c3f3bfcd4ddbcbf6a111d8814ffe542f3"),
     .package(url: "https://github.com/apple/swift-argument-parser.git", .upToNextMinor(from: "1.2.2")),
   ],
   targets: [

CodeGeneration/Sources/SyntaxSupport/AttributeNodes.swift

@@ -301,7 +301,7 @@ public let ATTRIBUTE_NODES: [Node] = [
     children: [
       Child(
         name: "DeclBaseName",
-        kind: .token(choices: [.token(tokenKind: "IdentifierToken"), .token(tokenKind: "PrefixOperatorToken"), .keyword(text: "init")]),
+        kind: .token(choices: [.token(tokenKind: "IdentifierToken"), .token(tokenKind: "BinaryOperatorToken"), .keyword(text: "init"), .keyword(text: "self"), .keyword(text: "Self")]),
         nameForDiagnostics: "base name",
         description: "The base name of the protocol's requirement."
       ),
@@ -468,7 +468,7 @@ public let ATTRIBUTE_NODES: [Node] = [
     children: [
       Child(
         name: "DiffKind",
-        kind: .token(choices: [.keyword(text: "forward"), .keyword(text: "reverse"), .keyword(text: "linear")]),
+        kind: .token(choices: [.keyword(text: "_forward"), .keyword(text: "reverse"), .keyword(text: "_linear")]),
         isOptional: true
       ),
       Child(
@@ -644,7 +644,7 @@ public let ATTRIBUTE_NODES: [Node] = [
     children: [
       Child(
         name: "Label",
-        kind: .token(choices: [.token(tokenKind: "IdentifierToken"), .keyword(text: "available"), .keyword(text: "exported"), .keyword(text: "kind"), .keyword(text: "spi"), .keyword(text: "spiModule")]),
+        kind: .node(kind: "Token"),
         nameForDiagnostics: "label",
         description: "The label of the argument"
       ),
@@ -677,7 +677,7 @@ public let ATTRIBUTE_NODES: [Node] = [
     children: [
       Child(
         name: "Name",
-        kind: .token(choices: [.token(tokenKind: "IdentifierToken")]),
+        kind: .node(kind: "Token"),
         nameForDiagnostics: "name",
         isOptional: true
       ),
@@ -779,7 +779,7 @@ public let ATTRIBUTE_NODES: [Node] = [
       ),
       Child(
         name: "Name",
-        kind: .token(choices: [.token(tokenKind: "IdentifierToken"), .token(tokenKind: "BinaryOperatorToken"), .token(tokenKind: "PrefixOperatorToken"), .token(tokenKind: "PostfixOperatorToken")]),
+        kind: .token(choices: [.token(tokenKind: "IdentifierToken"), .keyword(text: "self"), .keyword(text: "Self"), .keyword(text: "init"), .token(tokenKind: "BinaryOperatorToken")]),
         nameForDiagnostics: "base name",
         description: "The base name of the referenced function."
       ),

CodeGeneration/Sources/SyntaxSupport/AvailabilityNodes.swift

@@ -128,9 +128,21 @@ public let AVAILABILITY_NODES: [Node] = [
     kind: "Syntax",
     children: [
       Child(
-        name: "MajorMinor",
-        kind: .token(choices: [.token(tokenKind: "IntegerLiteralToken"), .token(tokenKind: "FloatingLiteralToken")]),
-        description: "In case the version consists only of the major version, an integer literal that specifies the major version. In case the version consists of major and minor version number, a floating literal in which the decimal part is interpreted as the minor version."
+        name: "Major",
+        kind: .token(choices: [.token(tokenKind: "IntegerLiteralToken")]),
+        description: "The major version."
+      ),
+      Child(
+        name: "MinorPeriod",
+        kind: .token(choices: [.token(tokenKind: "PeriodToken")]),
+        description: "If the version contains a minor number, the period separating the major from the minor number.",
+        isOptional: true
+      ),
+      Child(
+        name: "Minor",
+        kind: .token(choices: [.token(tokenKind: "IntegerLiteralToken")]),
+        description: "The minor version if specified.",
+        isOptional: true
       ),
       Child(
         name: "PatchPeriod",
@@ -139,7 +151,7 @@ public let AVAILABILITY_NODES: [Node] = [
         isOptional: true
       ),
       Child(
-        name: "PatchVersion",
+        name: "Patch",
         kind: .token(choices: [.token(tokenKind: "IntegerLiteralToken")]),
         description: "The patch version if specified.",
         isOptional: true

CodeGeneration/Sources/SyntaxSupport/DeclNodes.swift

@@ -19,7 +19,7 @@ public let DECL_NODES: [Node] = [
     children: [
       Child(
         name: "Name",
-        kind: .token(choices: [.token(tokenKind: "IdentifierToken")]),
+        kind: .token(choices: [.token(tokenKind: "IdentifierToken"), .token(tokenKind: "BinaryOperatorToken"), .token(tokenKind: "PrefixOperatorToken"), .token(tokenKind: "PostfixOperatorToken")]),
         nameForDiagnostics: "name"
       ),
       Child(

CodeGeneration/Sources/SyntaxSupport/Node.swift

@@ -162,6 +162,6 @@ public class Node {
     self.elementsSeparatedByNewline = elementsSeparatedByNewline
 
     // For SyntaxCollections make sure that the elementName is set.
-    assert(!isSyntaxCollection || elementName != nil || element != "")
+    precondition(!isSyntaxCollection || elementName != nil || element != "")
   }
 }

CodeGeneration/Sources/Utils/SyntaxBuildableChild.swift

@@ -63,7 +63,7 @@ public extension Child {
   }
 
   /// If this node is a token that can't contain arbitrary text, generate a Swift
-  /// `assert` statement that verifies the variable with name var_name and of type
+  /// `precondition` statement that verifies the variable with name var_name and of type
   /// `TokenSyntax` contains one of the supported text options. Otherwise return `nil`.
   func generateAssertStmtTextChoices(varName: String) -> FunctionCallExprSyntax? {
     guard case .token(choices: let choices, requiresLeadingSpace: _, requiresTrailingSpace: _) = kind else {
@@ -79,7 +79,7 @@ public extension Child {
 
     let choicesTexts: [String]
     if tokenCanContainArbitraryText {
-      // Don't generate an assert statement if token can contain arbitrary text.
+      // Don't generate an precondition statement if token can contain arbitrary text.
       return nil
     } else if !choices.isEmpty {
       choicesTexts = choices.compactMap {
@@ -92,9 +92,9 @@ public extension Child {
       return nil
     }
 
-    var assertChoices: [ExprSyntax] = []
+    var preconditionChoices: [ExprSyntax] = []
     if type.isOptional {
-      assertChoices.append(
+      preconditionChoices.append(
         ExprSyntax(
           SequenceExprSyntax {
             IdentifierExprSyntax(identifier: .identifier(varName))
@@ -105,7 +105,7 @@ public extension Child {
       )
     }
     for textChoice in choicesTexts {
-      assertChoices.append(
+      preconditionChoices.append(
         ExprSyntax(
           SequenceExprSyntax {
             MemberAccessExprSyntax(base: type.forceUnwrappedIfNeeded(expr: IdentifierExprSyntax(identifier: .identifier(varName))), name: "text")
@@ -115,8 +115,8 @@ public extension Child {
         )
       )
     }
-    let disjunction = ExprListSyntax(assertChoices.flatMap { [$0, ExprSyntax(BinaryOperatorExprSyntax(text: "||"))] }.dropLast())
-    return FunctionCallExprSyntax(callee: ExprSyntax("assert")) {
+    let disjunction = ExprListSyntax(preconditionChoices.flatMap { [$0, ExprSyntax(BinaryOperatorExprSyntax(text: "||"))] }.dropLast())
+    return FunctionCallExprSyntax(callee: ExprSyntax("precondition")) {
       TupleExprElementSyntax(expression: SequenceExprSyntax(elements: disjunction))
     }
   }

CodeGeneration/Sources/Utils/SyntaxBuildableNode.swift

@@ -44,14 +44,14 @@ public extension Node {
 
   /// Assuming this node is a syntax collection, the type of its elements.
   var collectionElementType: SyntaxBuildableType {
-    assert(isSyntaxCollection)
+    precondition(isSyntaxCollection)
     return SyntaxBuildableType(syntaxKind: collectionElement)
   }
 
   /// Assuming this node has a single child without a default value, that child.
   var singleNonDefaultedChild: Child {
     let nonDefaultedParams = children.filter { $0.type.defaultInitialization == nil }
-    assert(nonDefaultedParams.count == 1)
+    precondition(nonDefaultedParams.count == 1)
     return nonDefaultedParams[0]
   }
 

CodeGeneration/Sources/generate-swiftsyntax/templates/swiftparser/ParserEntryFile.swift

@@ -78,8 +78,7 @@ let parserEntryFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
       """
       mutating func parseRemainder<R: RawSyntaxNodeProtocol>(into: R) -> R {
         guard !into.raw.kind.isSyntaxCollection, let layout = into.raw.layoutView else {
-          assertionFailure("Only support parsing of non-collection layout nodes")
-          return into
+          preconditionFailure("Only support parsing of non-collection layout nodes")
         }
 
         let remainingTokens = self.consumeRemainingTokens()
@@ -89,7 +88,7 @@ let parserEntryFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
 
         let existingUnexpected: [RawSyntax]
         if let unexpectedNode = layout.children[layout.children.count - 1] {
-           assert(unexpectedNode.is(RawUnexpectedNodesSyntax.self))
+           precondition(unexpectedNode.is(RawUnexpectedNodesSyntax.self))
            existingUnexpected = unexpectedNode.as(RawUnexpectedNodesSyntax.self).elements
         } else {
            existingUnexpected = []

CodeGeneration/Sources/generate-swiftsyntax/templates/swiftsyntax/RawSyntaxNodesFile.swift

@@ -129,7 +129,15 @@ let rawSyntaxNodesFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
       DeclSyntax(
         """
         init(raw: RawSyntax) {
-          assert(Self.isKindOf(raw))
+          precondition(Self.isKindOf(raw))
+          self.raw = raw
+        }
+        """
+      )
+
+      DeclSyntax(
+        """
+        private init(unchecked raw: RawSyntax) {
           self.raw = raw
         }
         """
@@ -139,7 +147,7 @@ let rawSyntaxNodesFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
         """
         public init?<Node: RawSyntaxNodeProtocol>(_ other: Node) {
           guard Self.isKindOf(other.raw) else { return nil }
-          self.init(raw: other.raw)
+          self.init(unchecked: other.raw)
         }
         """
       )
@@ -148,7 +156,7 @@ let rawSyntaxNodesFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
         DeclSyntax(
           """
           public init<Node: Raw\(raw: node.name)NodeProtocol>(_ other: Node) {
-            self.init(raw: other.raw)
+            self.init(unchecked: other.raw)
           }
           """
         )
@@ -167,7 +175,7 @@ let rawSyntaxNodesFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
                     ptr += 1
                   }
             }
-            self.init(raw: raw)
+            self.init(unchecked: raw)
           }
           """
         )
@@ -217,7 +225,7 @@ let rawSyntaxNodesFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
           } else {
             DeclSyntax("let raw = RawSyntax.makeEmptyLayout(kind: .\(raw: node.swiftSyntaxKind), arena: arena)")
           }
-          ExprSyntax("self.init(raw: raw)")
+          ExprSyntax("self.init(unchecked: raw)")
         }
 
         for (index, child) in node.children.enumerated() {

CodeGeneration/Sources/generate-swiftsyntax/templates/swiftsyntax/RawSyntaxValidationFile.swift

@@ -32,18 +32,39 @@ let rawSyntaxValidationFile = try! SourceFileSyntax(leadingTrivia: copyrightHead
           condition: ExprSyntax("DEBUG"),
           elements: .statements(
             try CodeBlockItemListSyntax {
+              DeclSyntax(
+                #"""
+                enum TokenChoice: CustomStringConvertible {
+                   case keyword(StaticString)
+                   case tokenKind(RawTokenKind)
+
+                   var description: String {
+                     switch self {
+                     case .keyword(let keyword):
+                       return "keyword('\(keyword)')"
+                     case .tokenKind(let kind):
+                       return "\(kind)"
+                     }
+                   }
+                 }
+                """#
+              )
+
               DeclSyntax(
                 #"""
                 enum ValidationError: CustomStringConvertible {
                   case expectedNonNil(expectedKind: RawSyntaxNodeProtocol.Type, file: StaticString, line: UInt)
                   case kindMismatch(expectedKind: RawSyntaxNodeProtocol.Type, actualKind: SyntaxKind, file: StaticString, line: UInt)
+                  case tokenMismatch(expectedTokenChoices: [TokenChoice], actualKind: RawTokenKind, actualText: SyntaxText, file: StaticString, line: UInt)
 
                   var description: String {
                     switch self {
                     case .expectedNonNil(expectedKind: let expectedKind, file: _, line: _):
                       return "Expected non-nil node of type \(expectedKind) but received nil"
                     case .kindMismatch(expectedKind: let expectedKind, actualKind: let actualKind, file: _, line: _):
                       return "Expected node of type \(expectedKind) but received \(actualKind)"
+                    case .tokenMismatch(expectedTokenChoices: let tokenChoices, actualKind: let actualKind, actualText: let actualText, file: _, line: _):
+                      return "Expected token with one of \(tokenChoices) but received \(actualKind) with text '\(actualText)'"
                     }
                   }
 
@@ -53,6 +74,8 @@ let rawSyntaxValidationFile = try! SourceFileSyntax(leadingTrivia: copyrightHead
                       return (file, line)
                     case .kindMismatch(expectedKind: _, actualKind: _, file: let file, line: let line):
                       return (file, line)
+                    case .tokenMismatch(expectedTokenChoices: _, actualKind: _, actualText: _, file: let file, line: let line):
+                      return (file, line)
                   }
                   }
                 }
@@ -84,6 +107,61 @@ let rawSyntaxValidationFile = try! SourceFileSyntax(leadingTrivia: copyrightHead
                 """
               )
 
+              DeclSyntax(
+                """
+                func verify(_ raw: RawSyntax?, as _: RawTokenSyntax?.Type, tokenChoices: [TokenChoice], file: StaticString = #file, line: UInt = #line) -> ValidationError? {
+                  // Validation of token choice is currently causing assertion failures where
+                  // the list of expected token choices in the syntax tree doesn't match those
+                  // the parser generates. Disable the verification for now until all issues
+                  // regarding it are fixed.
+                  #if VALIDATE_TOKEN_CHOICES
+                  if raw != nil {
+                    return verify(raw, as: RawTokenSyntax.self, tokenChoices: tokenChoices, file: file, line: line)
+                  }
+                  return nil
+                  #else
+                  return verify(raw, as: RawTokenSyntax?.self)
+                  #endif
+                }
+                """
+              )
+
+              DeclSyntax(
+                """
+                func verify(_ raw: RawSyntax?, as _: RawTokenSyntax.Type, tokenChoices: [TokenChoice], file: StaticString = #file, line: UInt = #line) -> ValidationError? {
+                  // Validation of token choice is currently causing assertion failures where
+                  // the list of expected token choices in the syntax tree doesn't match those
+                  // the parser generates. Disable the verification for now until all issues
+                  // regarding it are fixed.
+                  #if VALIDATE_TOKEN_CHOICES
+                  guard let raw = raw else {
+                    return .expectedNonNil(expectedKind: RawTokenSyntax.self, file: file, line: line)
+                  }
+                  if let error = verify(raw, as: RawTokenSyntax?.self) {
+                    return error
+                  }
+                  let tokenView = raw.tokenView!
+                  for tokenChoice in tokenChoices {
+                    switch tokenChoice {
+                    case .tokenKind(let tokenKind):
+                      if raw.tokenView?.rawKind == tokenKind {
+                        return nil
+                      }
+                    case .keyword(let keyword):
+                      if tokenView.rawKind == .keyword && tokenView.rawText == SyntaxText(keyword) {
+                        return nil
+                      }
+                    }
+                  }
+                  return ValidationError.tokenMismatch(expectedTokenChoices: tokenChoices, actualKind: tokenView.rawKind, actualText: tokenView.rawText, file: file, line: line)
+                  #else
+                  return verify(raw, as: RawTokenSyntax.self)
+                  #endif
+                }
+
+                """
+              )
+
               DeclSyntax(
                 #"""
                  func assertNoError(_ nodeKind: SyntaxKind, _ index: Int, _ error: ValidationError?) {
@@ -139,6 +217,19 @@ let rawSyntaxValidationFile = try! SourceFileSyntax(leadingTrivia: copyrightHead
                           }
 
                           ExprSyntax("assertAnyHasNoError(kind, \(raw: index), \(verifiedChoices))")
+                        case .token(choices: let choices, requiresLeadingSpace: _, requiresTrailingSpace: _):
+                          let choices = ArrayExprSyntax {
+                            for choice in choices {
+                              switch choice {
+                              case .keyword(text: let text):
+                                ArrayElementSyntax(expression: ExprSyntax(#".keyword("\#(raw: text)")"#))
+                              case .token(tokenKind: let tokenKind):
+                                ArrayElementSyntax(expression: ExprSyntax(".tokenKind(.\(raw: SYNTAX_TOKEN_MAP[tokenKind]!.swiftKind))"))
+                              }
+                            }
+                          }
+                          let verifyCall = ExprSyntax("verify(layout[\(raw: index)], as: Raw\(raw: child.type.buildable).self, tokenChoices: \(choices))")
+                          ExprSyntax("assertNoError(kind, \(raw: index), \(verifyCall))")
                         default:
                           ExprSyntax("assertNoError(kind, \(raw: index), verify(layout[\(raw: index)], as: Raw\(raw: child.type.buildable).self))")
                         }

CodeGeneration/Sources/generate-swiftsyntax/templates/swiftsyntax/SyntaxCollectionsFile.swift

@@ -158,7 +158,7 @@ let syntaxCollectionsFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
         /// that the `SyntaxData` is of the correct kind. If it is not, the behaviour
         /// is undefined.
         internal init(_ data: SyntaxData) {
-          assert(data.raw.kind == .\(raw: node.swiftSyntaxKind))
+          precondition(data.raw.kind == .\(raw: node.swiftSyntaxKind))
           self._syntaxNode = Syntax(data)
         }
         """