Merge pull request #2576 from adammcarter/adamcarter93/separate-backtick-tokens

Add Identifier wrapper that strips backticks from token text

Author: plemarquand

Release Notes/601.md

@@ -7,6 +7,10 @@
   - Issue: https://github.com/apple/swift-syntax/issues/405
   - Pull Request: https://github.com/apple/swift-syntax/pull/2605
 
+- `TokenSyntax.identifier`
+  - Description: Adds an `identifier` property to `TokenSyntax` which returns a canonicalized representation of an identifier that strips away backticks.
+  - Pull request: https://github.com/apple/swift-syntax/pull/2576
+
 ## API Behavior Changes
 
 ## Deprecations

Sources/SwiftSyntax/CMakeLists.txt

@@ -15,6 +15,7 @@ add_swift_syntax_library(SwiftSyntax
   CommonAncestor.swift
   Convenience.swift
   CustomTraits.swift
+  Identifier.swift
   MemoryLayout.swift
   MissingNodeInitializers.swift
   SourceEdit.swift

Sources/SwiftSyntax/Identifier.swift

@@ -0,0 +1,50 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+/// A canonicalized representation of an identifier that strips away backticks.
+public struct Identifier: Equatable, Hashable, Sendable {
+  /// The sanitized `text` of a token.
+  public var name: String {
+    String(syntaxText: raw.name)
+  }
+
+  @_spi(RawSyntax)
+  public let raw: RawIdentifier
+
+  private let arena: SyntaxArenaRef
+
+  public init?(_ token: TokenSyntax) {
+    guard case .identifier = token.tokenKind else {
+      return nil
+    }
+
+    self.raw = RawIdentifier(token.tokenView)
+    self.arena = token.tokenView.raw.arenaReference
+  }
+}
+
+@_spi(RawSyntax)
+public struct RawIdentifier: Equatable, Hashable, Sendable {
+  public let name: SyntaxText
+
+  @_spi(RawSyntax)
+  fileprivate init(_ raw: RawSyntaxTokenView) {
+    let backtick = SyntaxText("`")
+    if raw.rawText.count > 2 && raw.rawText.hasPrefix(backtick) && raw.rawText.hasSuffix(backtick) {
+      let startIndex = raw.rawText.index(after: raw.rawText.startIndex)
+      let endIndex = raw.rawText.index(before: raw.rawText.endIndex)
+      self.name = SyntaxText(rebasing: raw.rawText[startIndex..<endIndex])
+    } else {
+      self.name = raw.rawText
+    }
+  }
+}

Sources/SwiftSyntax/TokenSyntax.swift

@@ -153,6 +153,16 @@ public struct TokenSyntax: SyntaxProtocol, SyntaxHashable {
     }
   }
 
+  /// An identifier created from `self`.
+  public var identifier: Identifier? {
+    switch self.tokenKind {
+    case .identifier, .dollarIdentifier:
+      return Identifier(self)
+    default:
+      return nil
+    }
+  }
+
   /// A token by itself has no structure, so we represent its structure by an
   /// empty layout node.
   ///

Tests/SwiftSyntaxTest/IdentifierTests.swift

@@ -0,0 +1,53 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+@_spi(RawSyntax) import SwiftSyntax
+import XCTest
+
+class IdentifierTests: XCTestCase {
+  public func testIdentifierInit() {
+    let someToken = TokenSyntax(stringLiteral: "someToken")
+    XCTAssertNotNil(Identifier(someToken))
+
+    let nonIdentifierToken = DeclSyntax("let a = 1").firstToken(viewMode: .all)!
+    XCTAssertNil(Identifier(nonIdentifierToken))
+  }
+
+  public func testName() {
+    let basicToken = TokenSyntax(stringLiteral: "basicToken")
+    XCTAssertEqual(Identifier(basicToken)?.name, "basicToken")
+
+    let backtickedToken = TokenSyntax(stringLiteral: "`backtickedToken`")
+    XCTAssertEqual(Identifier(backtickedToken)?.name, "backtickedToken")
+
+    let multiBacktickedToken = TokenSyntax(stringLiteral: "```multiBacktickedToken```")
+    XCTAssertEqual(Identifier(multiBacktickedToken)?.name, "``multiBacktickedToken``")
+
+    let unicodeNormalizedToken = TokenSyntax(stringLiteral: "\u{e0}")  // "a`"
+    XCTAssertEqual(Identifier(unicodeNormalizedToken)?.name, "\u{61}\u{300}")  // "à"
+  }
+
+  public func testIdentifier() {
+    let token = TokenSyntax(stringLiteral: "sometoken")
+    withExtendedLifetime(token) { token in
+      XCTAssertEqual(token.identifier?.raw.name, SyntaxText("sometoken"))
+    }
+  }
+
+  public func testTokenSyntaxIdentifier() throws {
+    let tokenSyntax = TokenSyntax(stringLiteral: "sometoken")
+    XCTAssertEqual(tokenSyntax.identifier, Identifier(tokenSyntax))
+
+    let nonIdentifierToken = try XCTUnwrap(DeclSyntax("let a = 1").firstToken(viewMode: .all))
+    XCTAssertNil(nonIdentifierToken.identifier)
+  }
+}