Merge pull request #1091 from ahoppen/ahoppen/markdown-documentation

Emit raw documentation comments instead of parsing XML

Author: ahoppen

Sources/SKTestSupport/SkipUnless.swift

@@ -51,7 +51,7 @@ public enum SkipUnless {
   /// guaranteed to be up-to-date.
   private static func skipUnlessSupportedByToolchain(
     swiftVersion: SwiftVersion,
-    featureName: String,
+    featureName: String = #function,
     file: StaticString,
     line: UInt,
     featureCheck: () async throws -> Bool
@@ -99,12 +99,7 @@ public enum SkipUnless {
     file: StaticString = #file,
     line: UInt = #line
   ) async throws {
-    try await skipUnlessSupportedByToolchain(
-      swiftVersion: SwiftVersion(5, 11),
-      featureName: "semantic token support in sourcekitd",
-      file: file,
-      line: line
-    ) {
+    try await skipUnlessSupportedByToolchain(swiftVersion: SwiftVersion(5, 11), file: file, line: line) {
       let testClient = try await TestSourceKitLSPClient()
       let uri = DocumentURI.for(.swift)
       testClient.openDocument("0.bitPattern", uri: uri)
@@ -134,12 +129,7 @@ public enum SkipUnless {
     file: StaticString = #file,
     line: UInt = #line
   ) async throws {
-    try await skipUnlessSupportedByToolchain(
-      swiftVersion: SwiftVersion(5, 11),
-      featureName: "rename support in sourcekitd",
-      file: file,
-      line: line
-    ) {
+    try await skipUnlessSupportedByToolchain(swiftVersion: SwiftVersion(5, 11), file: file, line: line) {
       let testClient = try await TestSourceKitLSPClient()
       let uri = DocumentURI.for(.swift)
       let positions = testClient.openDocument("func 1️⃣test() {}", uri: uri)
@@ -159,12 +149,7 @@ public enum SkipUnless {
     file: StaticString = #file,
     line: UInt = #line
   ) async throws {
-    try await skipUnlessSupportedByToolchain(
-      swiftVersion: SwiftVersion(5, 11),
-      featureName: "rename support in clangd",
-      file: file,
-      line: line
-    ) {
+    try await skipUnlessSupportedByToolchain(swiftVersion: SwiftVersion(5, 11), file: file, line: line) {
       let testClient = try await TestSourceKitLSPClient()
       let uri = DocumentURI.for(.c)
       let positions = testClient.openDocument("void 1️⃣test() {}", uri: uri)
@@ -194,12 +179,7 @@ public enum SkipUnless {
     file: StaticString = #file,
     line: UInt = #line
   ) async throws {
-    try await skipUnlessSupportedByToolchain(
-      swiftVersion: SwiftVersion(5, 11),
-      featureName: "SwiftPM stores modules in subdirectory",
-      file: file,
-      line: line
-    ) {
+    try await skipUnlessSupportedByToolchain(swiftVersion: SwiftVersion(5, 11), file: file, line: line) {
       let workspace = try await SwiftPMTestWorkspace(
         files: ["test.swift": ""],
         build: true
@@ -217,16 +197,40 @@ public enum SkipUnless {
     file: StaticString = #file,
     line: UInt = #line
   ) async throws {
-    try await skipUnlessSupportedByToolchain(
-      swiftVersion: SwiftVersion(5, 11),
-      featureName: "Toolchain contains swift-format",
-      file: file,
-      line: line
-    ) {
+    try await skipUnlessSupportedByToolchain(swiftVersion: SwiftVersion(5, 11), file: file, line: line) {
       return await ToolchainRegistry.forTesting.default?.swiftFormat != nil
     }
   }
 
+  public static func sourcekitdReturnsRawDocumentationResponse(
+    file: StaticString = #file,
+    line: UInt = #line
+  ) async throws {
+    struct ExpectedMarkdownContentsError: Error {}
+
+    return try await skipUnlessSupportedByToolchain(swiftVersion: SwiftVersion(6, 0), file: file, line: line) {
+      // The XML-based doc comment conversion did not preserve `Precondition`.
+      let testClient = try await TestSourceKitLSPClient()
+      let uri = DocumentURI.for(.swift)
+      let positions = testClient.openDocument(
+        """
+        /// - Precondition: Must have an apple
+        func 1️⃣test() {}
+        """,
+        uri: uri
+      )
+      let response = try await testClient.send(
+        HoverRequest(textDocument: TextDocumentIdentifier(uri), position: positions["1️⃣"])
+      )
+      let hover = try XCTUnwrap(response, file: file, line: line)
+      XCTAssertNil(hover.range, file: file, line: line)
+      guard case .markupContent(let content) = hover.contents else {
+        throw ExpectedMarkdownContentsError()
+      }
+      return content.value.contains("Precondition")
+    }
+  }
+
   public static func longTestsEnabled() throws {
     if let value = ProcessInfo.processInfo.environment["SKIP_LONG_TESTS"], value == "1" || value == "YES" {
       throw XCTSkip("Long tests disabled using the `SKIP_LONG_TESTS` environment variable")
@@ -295,7 +299,7 @@ private func getSwiftVersion(_ swiftcPath: AbsolutePath) async throws -> SwiftVe
   let result = try await process.waitUntilExit()
   let output = String(bytes: try result.output.get(), encoding: .utf8)
   let regex = Regex {
-    "Apple Swift version "
+    "Swift version "
     Capture { OneOrMore(.digit) }
     "."
     Capture { OneOrMore(.digit) }

Sources/SourceKitD/sourcekitd_uids.swift

@@ -51,6 +51,8 @@ public struct sourcekitd_api_keys {
   public let genericRequirements: sourcekitd_api_uid_t
   /// `key.doc.full_as_xml`
   public let docFullAsXML: sourcekitd_api_uid_t
+  /// `key.doc_comment`
+  public let docComment: sourcekitd_api_uid_t
   /// `key.line`
   public let line: sourcekitd_api_uid_t
   /// `key.column`
@@ -403,6 +405,16 @@ public struct sourcekitd_api_keys {
   public let indexStorePath: sourcekitd_api_uid_t
   /// `key.index_unit_output_path`
   public let indexUnitOutputPath: sourcekitd_api_uid_t
+  /// `key.include_locals`
+  public let includeLocals: sourcekitd_api_uid_t
+  /// `key.ignore_clang_modules`
+  public let ignoreClangModules: sourcekitd_api_uid_t
+  /// `key.include_system_modules`
+  public let includeSystemModules: sourcekitd_api_uid_t
+  /// `key.ignore_stdlib`
+  public let ignoreStdlib: sourcekitd_api_uid_t
+  /// `key.disable_implicit_modules`
+  public let disableImplicitModules: sourcekitd_api_uid_t
   /// `key.codecomplete.sort.byname`
   public let sortByName: sourcekitd_api_uid_t
   /// `key.codecomplete.sort.useimportdepth`
@@ -473,6 +485,7 @@ public struct sourcekitd_api_keys {
     genericParams = api.uid_get_from_cstr("key.generic_params")!
     genericRequirements = api.uid_get_from_cstr("key.generic_requirements")!
     docFullAsXML = api.uid_get_from_cstr("key.doc.full_as_xml")!
+    docComment = api.uid_get_from_cstr("key.doc_comment")!
     line = api.uid_get_from_cstr("key.line")!
     column = api.uid_get_from_cstr("key.column")!
     receiverUSR = api.uid_get_from_cstr("key.receiver_usr")!
@@ -649,6 +662,11 @@ public struct sourcekitd_api_keys {
     expandedMacroReplacements = api.uid_get_from_cstr("key.expanded_macro_replacements")!
     indexStorePath = api.uid_get_from_cstr("key.index_store_path")!
     indexUnitOutputPath = api.uid_get_from_cstr("key.index_unit_output_path")!
+    includeLocals = api.uid_get_from_cstr("key.include_locals")!
+    ignoreClangModules = api.uid_get_from_cstr("key.ignore_clang_modules")!
+    includeSystemModules = api.uid_get_from_cstr("key.include_system_modules")!
+    ignoreStdlib = api.uid_get_from_cstr("key.ignore_stdlib")!
+    disableImplicitModules = api.uid_get_from_cstr("key.disable_implicit_modules")!
     sortByName = api.uid_get_from_cstr("key.codecomplete.sort.byname")!
     useImportDepth = api.uid_get_from_cstr("key.codecomplete.sort.useimportdepth")!
     groupOverloads = api.uid_get_from_cstr("key.codecomplete.group.overloads")!

Sources/SourceKitLSP/Swift/CursorInfo.swift

@@ -28,21 +28,31 @@ struct CursorInfo {
   /// The annotated declaration XML string.
   var annotatedDeclaration: String?
 
+  #if swift(>=6.2)
+  #warning(
+    "Documentation transitioned from XML to the raw string in Swift 6.0. We should be able to remove documentationXML now"
+  )
+  #endif
   /// The documentation comment XML string. The schema is at
   /// https://github.com/apple/swift/blob/main/bindings/xml/comment-xml-schema.rng
   var documentationXML: String?
 
+  /// The documentation as it is spelled in
+  var documentation: String?
+
   /// The refactor actions available at this position.
   var refactorActions: [SemanticRefactorCommand]? = nil
 
   init(
     _ symbolInfo: SymbolDetails,
     annotatedDeclaration: String?,
-    documentationXML: String?
+    documentationXML: String?,
+    documentation: String?
   ) {
     self.symbolInfo = symbolInfo
     self.annotatedDeclaration = annotatedDeclaration
     self.documentationXML = documentationXML
+    self.documentation = documentation
   }
 
   init?(
@@ -79,7 +89,8 @@ struct CursorInfo {
         receiverUsrs: dict[keys.receivers]?.compactMap { $0[keys.usr] as String? } ?? []
       ),
       annotatedDeclaration: dict[keys.annotatedDecl],
-      documentationXML: dict[keys.docFullAsXML]
+      documentationXML: dict[keys.docFullAsXML],
+      documentation: dict[keys.docComment]
     )
   }
 }

Sources/SourceKitLSP/Swift/SwiftLanguageServer.swift

@@ -597,7 +597,16 @@ extension SwiftLanguageServer {
       }
 
       var result = escapeNameMarkdown(name)
-      if let doc = cursorInfo.documentationXML {
+      if let documentation = cursorInfo.documentation {
+        if let annotatedDeclaration = cursorInfo.annotatedDeclaration {
+          let markdownDecl =
+            orLog("Convert XML declaration to Markdown") {
+              try xmlDocumentationToMarkdown(annotatedDeclaration)
+            } ?? annotatedDeclaration
+          result += "\n\(markdownDecl)"
+        }
+        result += documentation
+      } else if let doc = cursorInfo.documentationXML {
         result += """
 
           \(orLog("Convert XML to Markdown") { try xmlDocumentationToMarkdown(doc) } ?? doc)