From ed9f9db0e4c9d2ada2eeffe57c91d6176032c595 Mon Sep 17 00:00:00 2001 From: Rintaro Ishizaki Date: Wed, 22 May 2024 00:40:56 -0700 Subject: [PATCH 1/4] [SourceLocationConverter] Use SyntaxVisitor to visit tokens --- Sources/SwiftSyntax/SourceLocation.swift | 35 +++++++++++++++--------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/Sources/SwiftSyntax/SourceLocation.swift b/Sources/SwiftSyntax/SourceLocation.swift index e53fb9ce60f..377f9c22b3b 100644 --- a/Sources/SwiftSyntax/SourceLocation.swift +++ b/Sources/SwiftSyntax/SourceLocation.swift @@ -512,20 +512,29 @@ extension SyntaxProtocol { fileprivate func computeLines( tree: Syntax ) -> ([AbsolutePosition], AbsolutePosition) { - var lines: [AbsolutePosition] = [] - // First line starts from the beginning. - lines.append(.startOfFile) - var position: AbsolutePosition = .startOfFile - let addLine = { (lineLength: SourceLength) in - position += lineLength - lines.append(position) - } - var curPrefix: SourceLength = .zero - for token in tree.tokens(viewMode: .sourceAccurate) { - curPrefix = token.forEachLineLength(prefix: curPrefix, body: addLine) + class ComputeLineVisitor: SyntaxVisitor { + var lines: [AbsolutePosition] = [.startOfFile] + var position: AbsolutePosition = .startOfFile + var curPrefix: SourceLength = .zero + + init() { + super.init(viewMode: .sourceAccurate) + } + + private func addLine(_ lineLength: SourceLength) { + position += lineLength + lines.append(position) + } + + override func visit(_ token: TokenSyntax) -> SyntaxVisitorContinueKind { + curPrefix = token.forEachLineLength(prefix: curPrefix, body: addLine) + return .skipChildren + } } - position += curPrefix - return (lines, position) + + let visitor = ComputeLineVisitor() + visitor.walk(tree) + return (visitor.lines, visitor.position + visitor.curPrefix) } fileprivate func computeLines(_ source: SyntaxText) -> ([AbsolutePosition], AbsolutePosition) { From 00ba9d89b60cea95835c987f92d56fcd15a06816 Mon Sep 17 00:00:00 2001 From: Rintaro Ishizaki Date: Wed, 22 May 2024 00:41:09 -0700 Subject: [PATCH 2/4] [SourceLocationConverter] Avoid creating trivia piece array --- Sources/SwiftSyntax/SourceLocation.swift | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/Sources/SwiftSyntax/SourceLocation.swift b/Sources/SwiftSyntax/SourceLocation.swift index 377f9c22b3b..fd921636853 100644 --- a/Sources/SwiftSyntax/SourceLocation.swift +++ b/Sources/SwiftSyntax/SourceLocation.swift @@ -645,7 +645,7 @@ fileprivate extension RawTriviaPiece { } } -fileprivate extension Array where Element == RawTriviaPiece { +fileprivate extension RawTriviaPieceBuffer { /// Walks and passes to `body` the ``SourceLength`` for every detected line, /// with the newline character included. /// - Returns: The leftover ``SourceLength`` at the end of the walk. @@ -670,9 +670,16 @@ fileprivate extension TokenSyntax { body: (SourceLength) -> () ) -> SourceLength { var curPrefix = prefix - curPrefix = self.tokenView.leadingRawTriviaPieces.forEachLineLength(prefix: curPrefix, body: body) - curPrefix = self.tokenView.rawText.forEachLineLength(prefix: curPrefix, body: body) - curPrefix = self.tokenView.trailingRawTriviaPieces.forEachLineLength(prefix: curPrefix, body: body) + switch self.raw.rawData.payload { + case .parsedToken(let dat): + curPrefix = dat.wholeText.forEachLineLength(prefix: curPrefix, body: body) + case .materializedToken(let dat): + curPrefix = dat.leadingTrivia.forEachLineLength(prefix: curPrefix, body: body) + curPrefix = dat.tokenText.forEachLineLength(prefix: curPrefix, body: body) + curPrefix = dat.trailingTrivia.forEachLineLength(prefix: curPrefix, body: body) + case .layout(_): + preconditionFailure("forEachLineLength is called non-token raw syntax") + } return curPrefix } } From 34e7bb470bb5aefaa87d89b97e364cb700092b28 Mon Sep 17 00:00:00 2001 From: Rintaro Ishizaki Date: Wed, 22 May 2024 06:34:21 -0700 Subject: [PATCH 3/4] [SourceLocationConverter] Visit RawSyntax directly --- Sources/SwiftSyntax/SourceLocation.swift | 37 ++++++++---------------- 1 file changed, 12 insertions(+), 25 deletions(-) diff --git a/Sources/SwiftSyntax/SourceLocation.swift b/Sources/SwiftSyntax/SourceLocation.swift index fd921636853..d60086adcd0 100644 --- a/Sources/SwiftSyntax/SourceLocation.swift +++ b/Sources/SwiftSyntax/SourceLocation.swift @@ -512,29 +512,14 @@ extension SyntaxProtocol { fileprivate func computeLines( tree: Syntax ) -> ([AbsolutePosition], AbsolutePosition) { - class ComputeLineVisitor: SyntaxVisitor { - var lines: [AbsolutePosition] = [.startOfFile] - var position: AbsolutePosition = .startOfFile - var curPrefix: SourceLength = .zero - - init() { - super.init(viewMode: .sourceAccurate) - } - - private func addLine(_ lineLength: SourceLength) { - position += lineLength - lines.append(position) - } + var lines: [AbsolutePosition] = [.startOfFile] + var position: AbsolutePosition = .startOfFile - override func visit(_ token: TokenSyntax) -> SyntaxVisitorContinueKind { - curPrefix = token.forEachLineLength(prefix: curPrefix, body: addLine) - return .skipChildren - } + let lastLineLength = tree.raw.forEachLineLength { lineLength in + position += lineLength + lines.append(position) } - - let visitor = ComputeLineVisitor() - visitor.walk(tree) - return (visitor.lines, visitor.position + visitor.curPrefix) + return (lines, position + lastLineLength) } fileprivate func computeLines(_ source: SyntaxText) -> ([AbsolutePosition], AbsolutePosition) { @@ -661,7 +646,7 @@ fileprivate extension RawTriviaPieceBuffer { } } -fileprivate extension TokenSyntax { +fileprivate extension RawSyntax { /// Walks and passes to `body` the ``SourceLength`` for every detected line, /// with the newline character included. /// - Returns: The leftover ``SourceLength`` at the end of the walk. @@ -670,15 +655,17 @@ fileprivate extension TokenSyntax { body: (SourceLength) -> () ) -> SourceLength { var curPrefix = prefix - switch self.raw.rawData.payload { + switch self.rawData.payload { case .parsedToken(let dat): curPrefix = dat.wholeText.forEachLineLength(prefix: curPrefix, body: body) case .materializedToken(let dat): curPrefix = dat.leadingTrivia.forEachLineLength(prefix: curPrefix, body: body) curPrefix = dat.tokenText.forEachLineLength(prefix: curPrefix, body: body) curPrefix = dat.trailingTrivia.forEachLineLength(prefix: curPrefix, body: body) - case .layout(_): - preconditionFailure("forEachLineLength is called non-token raw syntax") + case .layout(let dat): + for case let node? in dat.layout where SyntaxTreeViewMode.sourceAccurate.shouldTraverse(node: node) { + curPrefix = node.forEachLineLength(prefix: curPrefix, body: body) + } } return curPrefix } From 6a81e4991592ad7251d13f4d567cc72906e9cd0e Mon Sep 17 00:00:00 2001 From: Rintaro Ishizaki Date: Wed, 22 May 2024 13:13:54 -0700 Subject: [PATCH 4/4] [SourceLocaitonConverter] Collect '#sourceLocationDirective' in 1 pass --- Sources/SwiftSyntax/SourceLocation.swift | 86 +++++++++++-------- .../SourceLocationConverterTests.swift | 28 ++++++ 2 files changed, 76 insertions(+), 38 deletions(-) diff --git a/Sources/SwiftSyntax/SourceLocation.swift b/Sources/SwiftSyntax/SourceLocation.swift index d60086adcd0..5efa83d7cec 100644 --- a/Sources/SwiftSyntax/SourceLocation.swift +++ b/Sources/SwiftSyntax/SourceLocation.swift @@ -103,22 +103,6 @@ public struct SourceRange: Hashable, Codable, Sendable { } } -/// Collects all `PoundSourceLocationSyntax` directives in a file. -fileprivate class SourceLocationCollector: SyntaxVisitor { - private var sourceLocationDirectives: [PoundSourceLocationSyntax] = [] - - override func visit(_ node: PoundSourceLocationSyntax) -> SyntaxVisitorContinueKind { - sourceLocationDirectives.append(node) - return .skipChildren - } - - static func collectSourceLocations(in tree: some SyntaxProtocol) -> [PoundSourceLocationSyntax] { - let collector = SourceLocationCollector(viewMode: .sourceAccurate) - collector.walk(tree) - return collector.sourceLocationDirectives - } -} - fileprivate struct SourceLocationDirectiveArguments { enum Error: Swift.Error, CustomStringConvertible { case nonDecimalLineNumber(TokenSyntax) @@ -169,8 +153,8 @@ public final class SourceLocationConverter { /// The information from all `#sourceLocation` directives in the file /// necessary to compute presumed locations. /// - /// - `sourceLine` is the line at which the `#sourceLocation` statement occurs - /// within the current file. + /// - `sourceLine` is the physical line number of the end of the last token of + /// `#sourceLocation(...)` directive within the current file. /// - `arguments` are the `file` and `line` arguments of the directive or `nil` /// if spelled as `#sourceLocation()` to reset the source location directive. private var sourceLocationDirectives: [(sourceLine: Int, arguments: SourceLocationDirectiveArguments?)] = [] @@ -189,21 +173,7 @@ public final class SourceLocationConverter { precondition(tree.parent == nil, "SourceLocationConverter must be passed the root of the syntax tree") self.fileName = fileName self.source = tree.syntaxTextBytes - (self.lines, endOfFile) = computeLines(tree: Syntax(tree)) - precondition(tree.totalLength.utf8Length == endOfFile.utf8Offset) - - for directive in SourceLocationCollector.collectSourceLocations(in: tree) { - let location = self.physicalLocation(for: directive.positionAfterSkippingLeadingTrivia) - if let args = directive.arguments { - if let parsedArgs = try? SourceLocationDirectiveArguments(args) { - // Ignore any malformed `#sourceLocation` directives. - sourceLocationDirectives.append((sourceLine: location.line, arguments: parsedArgs)) - } - } else { - // `#sourceLocation()` without any arguments resets the `#sourceLocation` directive. - sourceLocationDirectives.append((sourceLine: location.line, arguments: nil)) - } - } + (self.lines, self.endOfFile, self.sourceLocationDirectives) = computeLines(tree: Syntax(tree)) } /// Create a new ``SourceLocationConverter`` to convert between ``AbsolutePosition`` @@ -511,15 +481,21 @@ extension SyntaxProtocol { /// the end-of-file position. fileprivate func computeLines( tree: Syntax -) -> ([AbsolutePosition], AbsolutePosition) { +) -> ( + lines: [AbsolutePosition], + endOfFile: AbsolutePosition, + sourceLocationDirectives: [(sourceLine: Int, arguments: SourceLocationDirectiveArguments?)] +) { var lines: [AbsolutePosition] = [.startOfFile] var position: AbsolutePosition = .startOfFile - + var sourceLocationDirectives: [(sourceLine: Int, arguments: SourceLocationDirectiveArguments?)] = [] let lastLineLength = tree.raw.forEachLineLength { lineLength in position += lineLength lines.append(position) + } handleSourceLocationDirective: { lineOffset, args in + sourceLocationDirectives.append((sourceLine: lines.count + lineOffset, arguments: args)) } - return (lines, position + lastLineLength) + return (lines, position + lastLineLength, sourceLocationDirectives) } fileprivate func computeLines(_ source: SyntaxText) -> ([AbsolutePosition], AbsolutePosition) { @@ -652,7 +628,8 @@ fileprivate extension RawSyntax { /// - Returns: The leftover ``SourceLength`` at the end of the walk. func forEachLineLength( prefix: SourceLength = .zero, - body: (SourceLength) -> () + body: (SourceLength) -> (), + handleSourceLocationDirective: (_ lineOffset: Int, _ arguments: SourceLocationDirectiveArguments?) -> () ) -> SourceLength { var curPrefix = prefix switch self.rawData.payload { @@ -664,7 +641,40 @@ fileprivate extension RawSyntax { curPrefix = dat.trailingTrivia.forEachLineLength(prefix: curPrefix, body: body) case .layout(let dat): for case let node? in dat.layout where SyntaxTreeViewMode.sourceAccurate.shouldTraverse(node: node) { - curPrefix = node.forEachLineLength(prefix: curPrefix, body: body) + curPrefix = node.forEachLineLength( + prefix: curPrefix, + body: body, + handleSourceLocationDirective: handleSourceLocationDirective + ) + } + + // Handle '#sourceLocation' directive. + if dat.kind == .poundSourceLocation { + // Count newlines in the trailing trivia. The client want to get the + // line of the _end_ of '#sourceLocation()' directive. + var lineOffset = 0 + if let lastTok = self.lastToken(viewMode: .sourceAccurate) { + switch lastTok.raw.rawData.payload { + case .parsedToken(let dat): + _ = dat.trailingTriviaText.forEachLineLength(body: { _ in lineOffset -= 1 }) + case .materializedToken(let dat): + _ = dat.trailingTrivia.forEachLineLength(body: { _ in lineOffset -= 1 }) + case .layout(_): + preconditionFailure("lastToken(viewMode:) returned non-token") + } + } + + let directive = Syntax.forRoot(self, rawNodeArena: self.arenaReference.retained) + .cast(PoundSourceLocationSyntax.self) + if let args = directive.arguments { + if let parsedArgs = try? SourceLocationDirectiveArguments(args) { + // Ignore any malformed `#sourceLocation` directives. + handleSourceLocationDirective(lineOffset, parsedArgs) + } + } else { + // `#sourceLocation()` without any arguments resets the `#sourceLocation` directive. + handleSourceLocationDirective(lineOffset, nil) + } } } return curPrefix diff --git a/Tests/SwiftSyntaxTest/SourceLocationConverterTests.swift b/Tests/SwiftSyntaxTest/SourceLocationConverterTests.swift index bdc196b8d09..b826fcae29d 100644 --- a/Tests/SwiftSyntaxTest/SourceLocationConverterTests.swift +++ b/Tests/SwiftSyntaxTest/SourceLocationConverterTests.swift @@ -174,6 +174,34 @@ final class SourceLocationConverterTests: XCTestCase { ) } + func testMultiLineDirective() { + assertPresumedSourceLocation( + #""" + #sourceLocation( + file: "input.swift", + line: 10 + ) + + let a = 2 + """#, + presumedFile: "input.swift", + presumedLine: 11 + ) + } + + func testDirectiveWithTrailingBlockComment() { + assertPresumedSourceLocation( + #""" + #sourceLocation(file: "input.swift", line: 10) /* + comment + */ + + let a = 2 + """#, + presumedFile: "input.swift", + presumedLine: 13 + ) + } func testMultiLineStringLiteralAsFilename() { // FIXME: The current parser handles this fine but it’s a really bogus filename. // We ignore the directive because the multi-line string literal contains multiple segments.