Add automated test to check if scaladoc sourcelinks are correctly pointing to remote repository

.github/workflows/scaladoc.yaml

@@ -70,3 +70,25 @@ jobs:
             echo uplading docs to https://scala3doc.virtuslab.com/$DOC_DEST
             az storage container create --name $DOC_DEST --account-name scala3docstorage --public-access container
             az storage blob upload-batch -s scaladoc/output -d $DOC_DEST --account-name scala3docstorage
+
+  stdlib-sourcelinks-test:
+    runs-on: ubuntu-latest
+    if: "(    github.event_name == 'pull_request'
+           && !contains(github.event.pull_request.body, '[skip ci]')
+           && !contains(github.event.pull_request.body, '[skip docs]')
+         )
+         || contains(github.event.ref, 'scaladoc')
+         || contains(github.event.ref, 'scala3doc')
+         || contains(github.event.ref, 'master')"
+
+    steps:
+      - name: Git Checkout
+        uses: actions/checkout@v2
+
+      - name: Set up JDK 8
+        uses: actions/setup-java@v1
+        with:
+          java-version: 8
+
+      - name: Test sourcelinks to stdlib
+        run: ./project/scripts/sbt scaladoc/sourceLinksIntegrationTest:test

project/Build.scala

@@ -1182,6 +1182,7 @@ object Build {
   // Note: the two tasks below should be one, but a bug in Tasty prevents that
   val generateScalaDocumentation = inputKey[Unit]("Generate documentation for dotty lib")
   val generateTestcasesDocumentation  = taskKey[Unit]("Generate documentation for testcases, usefull for debugging tests")
+
   lazy val `scaladoc-testcases` = project.in(file("scaladoc-testcases")).
     dependsOn(`scala3-compiler-bootstrapped`).
     settings(commonBootstrappedSettings)
@@ -1217,10 +1218,18 @@ object Build {
       Def.task((s"$distLocation/bin/scaladoc" +: cmd).!)
     }
 
+  val SourceLinksIntegrationTest = config("sourceLinksIntegrationTest") extend Test
+
   lazy val scaladoc = project.in(file("scaladoc")).
+    configs(SourceLinksIntegrationTest).
     settings(commonBootstrappedSettings).
     dependsOn(`scala3-compiler-bootstrapped`).
     dependsOn(`scala3-tasty-inspector`).
+    settings(inConfig(SourceLinksIntegrationTest)(Defaults.testSettings)).
+    settings(
+      SourceLinksIntegrationTest / scalaSource := baseDirectory.value / "test-source-links",
+      SourceLinksIntegrationTest / test:= ((SourceLinksIntegrationTest / test) dependsOn generateScalaDocumentation.toTask("")).value,
+    ).
     settings(
       Compile / resourceGenerators += Def.task {
         val jsDestinationFile = (Compile / resourceManaged).value / "dotty_res" / "scripts" / "searchbar.js"

scaladoc-testcases/src/toplevel.scala

@@ -1,3 +1,3 @@
 def toplevelDef = 123
 
-class ToplevelClass
+class ToplevelClass

scaladoc/src/dotty/tools/scaladoc/SourceLinks.scala

@@ -140,4 +140,3 @@ object SourceLinks:
         )
         SourceLinks(sourceLinks)
     }
-

scaladoc/src/dotty/tools/scaladoc/tasty/NameNormalizer.scala

@@ -21,10 +21,9 @@ object NameNormalizer {
   private val ignoredKeywords: Set[String] = Set("this")
 
   private def escapedName(name: String) =
-    val simpleIdentifierRegex = raw"(?:\w+_[^\[\(\s_]+)|\w+|[^\[\(\s\w_]+".r
+    val complexIdentifierRegex = """([([{}]) ]|[^A-Za-z0-9$]_)""".r
     name match
       case n if ignoredKeywords(n) => n
-      case n if keywords(termName(n)) => s"`$n`"
-      case simpleIdentifierRegex() => name
-      case n => s"`$n`"
+      case n if keywords(termName(n)) || complexIdentifierRegex.findFirstIn(n).isDefined => s"`$n`"
+      case _ => name
 }

scaladoc/src/dotty/tools/scaladoc/tasty/SyntheticSupport.scala

@@ -119,4 +119,3 @@ object SyntheticsSupport:
     typeForClass(c).asInstanceOf[dotc.core.Types.Type]
       .memberInfo(symbol.asInstanceOf[dotc.core.Symbols.Symbol])
       .asInstanceOf[TypeRepr]
-

scaladoc/src/dotty/tools/scaladoc/tasty/comments/markdown/DocFlexmarkExtension.scala

@@ -66,4 +66,4 @@ case class DocFlexmarkRenderer(renderLink: (DocLink, String) => String)
 object DocFlexmarkRenderer:
   def render(node: Node)(renderLink: (DocLink, String) => String) =
     val opts = MarkdownParser.mkMarkdownOptions(Seq(DocFlexmarkRenderer(renderLink)))
-    HtmlRenderer.builder(opts).build().render(node)
+    HtmlRenderer.builder(opts).escapeHtml(true).build().render(node)

scaladoc/src/dotty/tools/scaladoc/util/html.scala

@@ -11,7 +11,17 @@ object HTML:
   case class Tag(name: String):
     def apply(tags: TagArg*): AppliedTag = apply()(tags:_*)
     def apply(first: AttrArg, rest: AttrArg*): AppliedTag = apply((first +: rest):_*)()
-    def apply(attrs: AttrArg*)(tags: TagArg*): AppliedTag = {
+    def apply(attrs: AttrArg*)(tags: TagArg*): AppliedTag =
+      def unpackTags(tags: TagArg*)(using sb: StringBuilder): StringBuilder =
+        tags.foreach {
+          case t: AppliedTag =>
+            sb.append(t)
+          case s: String =>
+            sb.append(s.escapeReservedTokens)
+          case s: Seq[AppliedTag | String] =>
+            unpackTags(s:_*)
+        }
+        sb
       val sb = StringBuilder()
       sb.append(s"<$name")
       attrs.filter(_ != Nil).foreach{
@@ -21,22 +31,9 @@ object HTML:
           sb.append(" ").append(e)
       }
       sb.append(">")
-      tags.foreach{
-        case t: AppliedTag =>
-          sb.append(t)
-        case s: String =>
-          sb.append(s.escapeReservedTokens)
-        case s: Seq[AppliedTag | String] =>
-          s.foreach{
-            case a: AppliedTag =>
-              sb.append(a)
-            case s: String =>
-              sb.append(s.escapeReservedTokens)
-          }
-      }
+      unpackTags(tags:_*)(using sb)
       sb.append(s"</$name>")
       sb
-    }
 
   extension (s: String) private def escapeReservedTokens: String =
     s.replace("&", "&amp;")

scaladoc/test-source-links/dotty/tools/scaladoc/source-links/RemoteLinksTest.scala

@@ -0,0 +1,84 @@
+package dotty.tools.scaladoc
+package sourcelinks
+
+import scala.util.Random
+import scala.io.Source
+import scala.jdk.CollectionConverters._
+import scala.util.matching.Regex
+import dotty.tools.scaladoc.test.BuildInfo
+import java.nio.file.Path
+import java.nio.file.Paths
+import org.jsoup.Jsoup
+import org.jsoup.nodes.Document
+import util.IO
+import org.junit.Assert.assertTrue
+import org.junit.Test
+
+class RemoteLinksTest:
+
+  class TimeoutException extends Exception
+
+  val randomGenerator = new Random(125L)
+  val mtslAll = membersToSourceLinks(using testDocContext())
+
+  @Test
+  def scala213XSourceLink =
+    assertTrue(mtslAll.find((k, _) => k == "AbstractMap").isDefined) // source link to Scala2.13.X stdlib class
+
+  @Test
+  def scala3SourceLink =
+    assertTrue(mtslAll.find((k, _) => k == "PolyFunction").isDefined) // source link to Scala3 stdlib class
+
+  @Test
+  def runTest =
+    assertTrue(mtslAll.nonEmpty)
+    val mtsl = randomGenerator.shuffle(mtslAll).take(20) // take 20 random entries
+    val pageToMtsl: Map[String, List[(String, String)]] = mtsl.groupMap(_._2.split("#L").head)(v => (v._1, v._2.split("#L").last))
+    pageToMtsl.foreach { case (link, members) =>
+      try
+        val doc = getDocumentFromUrl(link)
+        members.foreach { (member, line) =>
+          if !member.startsWith("given_") then // TODO: handle synthetic givens, for now we disable them from testing
+            val loc = doc.select(s"#LC$line").text
+            val memberToMatch = member.replace("`", "")
+            assertTrue(s"Expected to find $memberToMatch at $link at line $line", loc.contains(memberToMatch))
+        }
+      catch
+        case e: java.lang.IllegalArgumentException =>
+          report.error(s"Could not open link for $link - invalid URL")(using testContext)
+        case e: TimeoutException =>
+          report.error(s"Tried to open link $link 16 times but with no avail")(using testContext)
+        case e: org.jsoup.HttpStatusException => e.getStatusCode match
+          case 404 => throw AssertionError(s"Page $link does not exists")
+          case n   => report.warning(s"Could not open link for $link, return code $n")(using testContext)
+    }
+    assertNoErrors(testContext.reportedDiagnostics)
+
+  private def getDocumentFromUrl(link: String, retries: Int = 16): Document =
+    try
+      if retries == 0 then throw TimeoutException()
+      Jsoup.connect(link).get
+    catch
+      case e: org.jsoup.HttpStatusException => e.getStatusCode match
+          case 429 =>
+            Thread.sleep(10)
+            getDocumentFromUrl(link, retries - 1)
+          case n =>
+            throw e
+
+  private def membersToSourceLinks(using DocContext): List[(String, String)] =
+    val output = Paths.get("scaladoc", "output", "scala3", "api").toAbsolutePath
+    val mtsl = List.newBuilder[(String, String)]
+    def processFile(path: Path): Unit =
+      val document = Jsoup.parse(IO.read(path))
+      if document.select("span.kind").first.text == "package" then
+        document.select(".documentableElement").forEach { dElem =>
+          if dElem.select("span.kind").first.text != "package" then
+            dElem.select("dt").forEach { elem =>
+              val content = elem.text
+              if content == "Source" then
+                mtsl += dElem.select(".documentableName").first.text -> elem.nextSibling.childNode(0).attr("href")
+            }
+        }
+    IO.foreachFileIn(output, processFile)
+    mtsl.result

scaladoc/test/dotty/tools/scaladoc/ScaladocTest.scala

@@ -25,7 +25,8 @@ abstract class ScaladocTest(val name: String):
       name = "test",
       tastyFiles = tastyFiles(name),
       output = getTempDir().getRoot,
-      projectVersion = Some("1.0")
+      projectVersion = Some("1.0"),
+      sourceLinks = List("github://lampepfl/dotty/master")
     )
 
   @Test

scaladoc/test/dotty/tools/scaladoc/SourceLinksTests.scala → scaladoc/test/dotty/tools/scaladoc/source-links/SourceLinksTest.scala

@@ -1,4 +1,5 @@
 package dotty.tools.scaladoc
+package sourcelinks
 
 import java.nio.file._
 import org.junit.Assert._
@@ -155,4 +156,4 @@ class SourceLinksTest:
       ("src/lib/core.scala", 33, edit) -> "https://gitlab.com/lampepfl/dotty/-/edit/develop/lib/core.scala#L33",
       ("src/generated.scala", 33, edit) -> "https://gitlab.com/lampepfl/dotty/-/edit/develop/generated.scala#L33",
       ("src/generated/template.scala", 1, edit) -> "/template.scala#1"
-    )
+    )

scaladoc/test/dotty/tools/scaladoc/testUtils.scala

@@ -53,7 +53,7 @@ class TestReporter extends ConsoleReporter:
 def testArgs(files: Seq[File] = Nil, dest: File = new File("notUsed")) = Scaladoc.Args(
           name = "Test Project Name",
           output = dest,
-          tastyFiles = files,
+          tastyFiles = files
         )
 
 def testContext = (new ContextBase).initialCtx.fresh.setReporter(new TestReporter)