From 76ea80de1bee1a596cf411071541185d5f3503cb Mon Sep 17 00:00:00 2001 From: Lucas Date: Tue, 28 Mar 2023 15:33:25 +0200 Subject: [PATCH 1/4] Verify links in docs - Add a check of valid url and internal link - Add condition flag for the warning --- .../scaladoc/tasty/comments/Preparser.scala | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/scaladoc/src/dotty/tools/scaladoc/tasty/comments/Preparser.scala b/scaladoc/src/dotty/tools/scaladoc/tasty/comments/Preparser.scala index 95db8983626a..d2503481c668 100644 --- a/scaladoc/src/dotty/tools/scaladoc/tasty/comments/Preparser.scala +++ b/scaladoc/src/dotty/tools/scaladoc/tasty/comments/Preparser.scala @@ -4,6 +4,9 @@ package tasty.comments import scala.collection.mutable import scala.collection.immutable.SortedMap import scala.util.matching.Regex +import java.net.URL +import java.nio.file.{Paths, Files} +import scala.util.Try object Preparser { import Regexes._ @@ -11,7 +14,7 @@ object Preparser { /** Parses a raw comment string into a `Comment` object. */ def preparse( comment: List[String], - ): PreparsedComment = { + )(using DocContext): PreparsedComment = { /** Parses a comment (in the form of a list of lines) to a `Comment` * instance, recursively on lines. To do so, it splits the whole comment @@ -129,7 +132,38 @@ object Preparser { val stripTags = List(inheritDiagramTag, contentDiagramTag, SimpleTagKey("template"), SimpleTagKey("documentable")) val tagsWithoutDiagram = tags.filterNot(pair => stripTags.contains(pair._1)) + def processLink: Unit = + if (!summon[DocContext].args.noLinkWarnings) then tags.get(SimpleTagKey("see")).get.foreach(link => { + val newLink: String = link.replaceAll("\\[\\[|\\]\\]", "") + val isValid = Try(new URL(newLink)).isSuccess + isValid match { + case true => + val url = new URL(newLink) + url match { + // We check if it's an internal link + case s if s.getPath.contains("/docs/") => + if (newLink.contains("oracle")) then // exclude links containing "oracle" + None + else + // We check if the internal link to the static documentation is valid + val docPath = url.getPath.substring(url.getPath.indexOf("/docs/")).replaceFirst("/docs/", "docs/_docs/").replace(".html", ".md") + println(docPath) + val fileExists = Files.exists(Paths.get(docPath)) + if !fileExists then + //Si le fichier n'existe pas, on vérifie si le fichier existe avec l'extension .md + val newDocPath = docPath + ".md" + if !Files.exists(Paths.get(newDocPath)) then + report.warning(s"Link to $newLink will return a 404 not found") + case _ => None + } + case false => + None + } + }) + val bodyTags: mutable.Map[TagKey, List[String]] = + if tags.get(SimpleTagKey("see")).isDefined then + processLink mutable.Map((tagsWithoutDiagram).toSeq: _*) def allTags(key: SimpleTagKey): List[String] = From 18f167743d301a99fcd879833f2750862dd32d1f Mon Sep 17 00:00:00 2001 From: Lucas Date: Tue, 2 May 2023 10:16:08 +0200 Subject: [PATCH 2/4] Add a way to disable the warning for a specific link --- .../scaladoc/tasty/comments/Preparser.scala | 44 ++++++++----------- 1 file changed, 19 insertions(+), 25 deletions(-) diff --git a/scaladoc/src/dotty/tools/scaladoc/tasty/comments/Preparser.scala b/scaladoc/src/dotty/tools/scaladoc/tasty/comments/Preparser.scala index d2503481c668..f54a44cda774 100644 --- a/scaladoc/src/dotty/tools/scaladoc/tasty/comments/Preparser.scala +++ b/scaladoc/src/dotty/tools/scaladoc/tasty/comments/Preparser.scala @@ -134,32 +134,26 @@ object Preparser { def processLink: Unit = if (!summon[DocContext].args.noLinkWarnings) then tags.get(SimpleTagKey("see")).get.foreach(link => { - val newLink: String = link.replaceAll("\\[\\[|\\]\\]", "") - val isValid = Try(new URL(newLink)).isSuccess - isValid match { - case true => - val url = new URL(newLink) - url match { - // We check if it's an internal link - case s if s.getPath.contains("/docs/") => - if (newLink.contains("oracle")) then // exclude links containing "oracle" - None - else - // We check if the internal link to the static documentation is valid - val docPath = url.getPath.substring(url.getPath.indexOf("/docs/")).replaceFirst("/docs/", "docs/_docs/").replace(".html", ".md") - println(docPath) - val fileExists = Files.exists(Paths.get(docPath)) - if !fileExists then - //Si le fichier n'existe pas, on vérifie si le fichier existe avec l'extension .md - val newDocPath = docPath + ".md" - if !Files.exists(Paths.get(newDocPath)) then - report.warning(s"Link to $newLink will return a 404 not found") - case _ => None - } - case false => - None + val newLink = link.replaceAll("\\[\\[|\\]\\]", "") + val newUrl = new URL(newLink) + if(Try(newUrl).isSuccess) { + if(newUrl.getPath.contains("/docs/")) { + if (newLink.contains("oracle") || newLink.contains("excludeValidation")) then None + else + // We check if the internal link to the static documentation is valid + val docPath = newUrl.getPath.substring(newUrl.getPath.indexOf("/docs/")) + .replaceFirst("/docs/(docs/)?", "docs/_docs/") + .replace(".html", ".md") + println(docPath) + val fileExists = Files.exists(Paths.get(docPath)) + if !fileExists then + val newDocPath = docPath + ".md" + if !Files.exists(Paths.get(newDocPath)) then report.warning(s"Link to $newLink will return a 404 not found") } - }) + else None + } + else None + }) val bodyTags: mutable.Map[TagKey, List[String]] = if tags.get(SimpleTagKey("see")).isDefined then From cd73a8de89f20eb6f8f4b68afadecc8d9706f742 Mon Sep 17 00:00:00 2001 From: Lucas Date: Tue, 16 May 2023 15:04:17 +0200 Subject: [PATCH 3/4] WIP: Add Function ValidationLink --- .../scaladoc/renderers/MemberRenderer.scala | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/scaladoc/src/dotty/tools/scaladoc/renderers/MemberRenderer.scala b/scaladoc/src/dotty/tools/scaladoc/renderers/MemberRenderer.scala index 996b422b44fd..839659bb53aa 100644 --- a/scaladoc/src/dotty/tools/scaladoc/renderers/MemberRenderer.scala +++ b/scaladoc/src/dotty/tools/scaladoc/renderers/MemberRenderer.scala @@ -9,6 +9,9 @@ import scala.jdk.CollectionConverters.* import dotty.tools.scaladoc.translators.FilterAttributes import org.jsoup.Jsoup import translators.* +import java.net.URL +import scala.util.Try +import java.nio.file.{Files, Paths} class MemberRenderer(signatureRenderer: SignatureRenderer)(using DocContext) extends DocRender(signatureRenderer): import signatureRenderer._ @@ -100,6 +103,63 @@ class MemberRenderer(signatureRenderer: SignatureRenderer)(using DocContext) ext def typeParams(m: Member): Seq[AppliedTag] = m.docs.fold(Nil)(d => flattenedDocPart(d.typeParams)) def valueParams(m: Member): Seq[AppliedTag] = m.docs.fold(Nil)(d => flattenedDocPart(d.valueParams)) + def processLocalLinkWithGuard(str: String): String = + if str.startsWith("#") || str.isEmpty then + str + else + validationLink(str) + + def validationLink(str: String): String = + def asValidURL = Try(URL(str)).toOption.map(_ => str) + + def asAsset = + Option.when( + Files.exists(Paths.get("src/main/ressources").resolve(str.stripPrefix("/"))) + )( + s"src/main/ressources/$str" + ) + + def asStaticSite: Option[String] = + Option.when( + Files.exists(Paths.get("docs/_docs").resolve(str.stripPrefix("/"))) + )( + s"docs/_docs/$str" + ) + + def asApiLink: Option[String] = + val strWithoutHtml = if str.endsWith("$.html") then + str.stripSuffix("$.html") + else + str.stripSuffix(".html") + val sourceDir = Paths.get("src", "main", "scala") + val scalaPath = sourceDir.resolve(s"$strWithoutHtml.scala") + val scalaDirPath = sourceDir.resolve(strWithoutHtml) + Option.when( + Files.exists(scalaPath) || Files.exists(scalaDirPath)) + ( + s"api/$strWithoutHtml.html" + ) + + + asValidURL + .orElse(asStaticSite) + .orElse(asAsset) + .orElse(asApiLink) + .getOrElse{ + report.warning(s"Unable to resolve link '$str'") + str + } + + // println(asValidURL) + + // println(Paths.get("src/main/ressources").resolve(str.stripPrefix("/")).toAbsolutePath) + // println(Files.exists(Paths.get("src/main/ressources").resolve(str.stripPrefix("/")))) + // def asAsset = Option.when( + // Files.exists(Paths.get("docs/_assets").resolve(str.stripPrefix("/"))) + // )( + // s"docs/_assets/$str" + // ) + def memberInfo(m: Member, withBrief: Boolean = false, full: Boolean = false): Seq[AppliedTag] = val comment = m.docs val bodyContents = m.docs.fold(Nil)(e => renderDocPart(e.body) :: Nil) @@ -122,6 +182,21 @@ class MemberRenderer(signatureRenderer: SignatureRenderer)(using DocContext) ext case _ => true } + val document = Jsoup.parse(bodyContents.mkString) + val document2 = Jsoup.parse(attributes.mkString) + + document.select("img").forEach(element => + element.attr("src", validationLink(element.attr("src"))) + ) + + document.select("a").forEach(element => + element.attr("href", processLocalLinkWithGuard(element.attr("href"))) + ) + + // document2.select("a").forEach(element => + // println("BONJOUR"+element.attr("href")) + // ) <--- Take some href that I don't want + Seq( Option.when(withBrief && comment.flatMap(_.short).nonEmpty)(div(cls := "documentableBrief doc")(comment.flatMap(_.short).fold("")(renderDocPart))), Option.when(bodyContents.nonEmpty || attributes.nonEmpty)( From f2bbb111817fa24e69e426cd14fbc538b64bf92d Mon Sep 17 00:00:00 2001 From: Lucas Date: Tue, 23 May 2023 10:34:27 +0200 Subject: [PATCH 4/4] Refactor - Test only the assets and Api link --- .../scaladoc/renderers/MemberRenderer.scala | 53 ++----------------- .../scaladoc/tasty/comments/Comments.scala | 2 +- .../scaladoc/tasty/comments/Preparser.scala | 30 +---------- 3 files changed, 7 insertions(+), 78 deletions(-) diff --git a/scaladoc/src/dotty/tools/scaladoc/renderers/MemberRenderer.scala b/scaladoc/src/dotty/tools/scaladoc/renderers/MemberRenderer.scala index 839659bb53aa..08e0c6a9d7d0 100644 --- a/scaladoc/src/dotty/tools/scaladoc/renderers/MemberRenderer.scala +++ b/scaladoc/src/dotty/tools/scaladoc/renderers/MemberRenderer.scala @@ -112,54 +112,19 @@ class MemberRenderer(signatureRenderer: SignatureRenderer)(using DocContext) ext def validationLink(str: String): String = def asValidURL = Try(URL(str)).toOption.map(_ => str) - def asAsset = - Option.when( - Files.exists(Paths.get("src/main/ressources").resolve(str.stripPrefix("/"))) - )( - s"src/main/ressources/$str" - ) - - def asStaticSite: Option[String] = - Option.when( - Files.exists(Paths.get("docs/_docs").resolve(str.stripPrefix("/"))) - )( - s"docs/_docs/$str" - ) - - def asApiLink: Option[String] = - val strWithoutHtml = if str.endsWith("$.html") then - str.stripSuffix("$.html") - else - str.stripSuffix(".html") - val sourceDir = Paths.get("src", "main", "scala") - val scalaPath = sourceDir.resolve(s"$strWithoutHtml.scala") - val scalaDirPath = sourceDir.resolve(strWithoutHtml) - Option.when( - Files.exists(scalaPath) || Files.exists(scalaDirPath)) - ( - s"api/$strWithoutHtml.html" - ) - + def asAsset: Option[String] = Option.when( + Files.exists(Paths.get("docs/_assets").resolve(str)) + )( + s"docs/_assets/$str" + ) asValidURL - .orElse(asStaticSite) .orElse(asAsset) - .orElse(asApiLink) .getOrElse{ report.warning(s"Unable to resolve link '$str'") str } - // println(asValidURL) - - // println(Paths.get("src/main/ressources").resolve(str.stripPrefix("/")).toAbsolutePath) - // println(Files.exists(Paths.get("src/main/ressources").resolve(str.stripPrefix("/")))) - // def asAsset = Option.when( - // Files.exists(Paths.get("docs/_assets").resolve(str.stripPrefix("/"))) - // )( - // s"docs/_assets/$str" - // ) - def memberInfo(m: Member, withBrief: Boolean = false, full: Boolean = false): Seq[AppliedTag] = val comment = m.docs val bodyContents = m.docs.fold(Nil)(e => renderDocPart(e.body) :: Nil) @@ -189,14 +154,6 @@ class MemberRenderer(signatureRenderer: SignatureRenderer)(using DocContext) ext element.attr("src", validationLink(element.attr("src"))) ) - document.select("a").forEach(element => - element.attr("href", processLocalLinkWithGuard(element.attr("href"))) - ) - - // document2.select("a").forEach(element => - // println("BONJOUR"+element.attr("href")) - // ) <--- Take some href that I don't want - Seq( Option.when(withBrief && comment.flatMap(_.short).nonEmpty)(div(cls := "documentableBrief doc")(comment.flatMap(_.short).fold("")(renderDocPart))), Option.when(bodyContents.nonEmpty || attributes.nonEmpty)( diff --git a/scaladoc/src/dotty/tools/scaladoc/tasty/comments/Comments.scala b/scaladoc/src/dotty/tools/scaladoc/tasty/comments/Comments.scala index ff4405d3ec71..7496bce6e77f 100644 --- a/scaladoc/src/dotty/tools/scaladoc/tasty/comments/Comments.scala +++ b/scaladoc/src/dotty/tools/scaladoc/tasty/comments/Comments.scala @@ -112,7 +112,7 @@ abstract class MarkupConversion[T](val repr: Repr)(using dctx: DocContext) { case None => sym.dri DocLink.ToDRI(dri, targetText) case None => - val txt = s"No DRI found for query" + val txt = s"No DRI found for query, the link seems to be wrong or the file may not exist." val msg = s"$txt: $queryStr" if (!summon[DocContext].args.noLinkWarnings) then diff --git a/scaladoc/src/dotty/tools/scaladoc/tasty/comments/Preparser.scala b/scaladoc/src/dotty/tools/scaladoc/tasty/comments/Preparser.scala index f54a44cda774..95db8983626a 100644 --- a/scaladoc/src/dotty/tools/scaladoc/tasty/comments/Preparser.scala +++ b/scaladoc/src/dotty/tools/scaladoc/tasty/comments/Preparser.scala @@ -4,9 +4,6 @@ package tasty.comments import scala.collection.mutable import scala.collection.immutable.SortedMap import scala.util.matching.Regex -import java.net.URL -import java.nio.file.{Paths, Files} -import scala.util.Try object Preparser { import Regexes._ @@ -14,7 +11,7 @@ object Preparser { /** Parses a raw comment string into a `Comment` object. */ def preparse( comment: List[String], - )(using DocContext): PreparsedComment = { + ): PreparsedComment = { /** Parses a comment (in the form of a list of lines) to a `Comment` * instance, recursively on lines. To do so, it splits the whole comment @@ -132,32 +129,7 @@ object Preparser { val stripTags = List(inheritDiagramTag, contentDiagramTag, SimpleTagKey("template"), SimpleTagKey("documentable")) val tagsWithoutDiagram = tags.filterNot(pair => stripTags.contains(pair._1)) - def processLink: Unit = - if (!summon[DocContext].args.noLinkWarnings) then tags.get(SimpleTagKey("see")).get.foreach(link => { - val newLink = link.replaceAll("\\[\\[|\\]\\]", "") - val newUrl = new URL(newLink) - if(Try(newUrl).isSuccess) { - if(newUrl.getPath.contains("/docs/")) { - if (newLink.contains("oracle") || newLink.contains("excludeValidation")) then None - else - // We check if the internal link to the static documentation is valid - val docPath = newUrl.getPath.substring(newUrl.getPath.indexOf("/docs/")) - .replaceFirst("/docs/(docs/)?", "docs/_docs/") - .replace(".html", ".md") - println(docPath) - val fileExists = Files.exists(Paths.get(docPath)) - if !fileExists then - val newDocPath = docPath + ".md" - if !Files.exists(Paths.get(newDocPath)) then report.warning(s"Link to $newLink will return a 404 not found") - } - else None - } - else None - }) - val bodyTags: mutable.Map[TagKey, List[String]] = - if tags.get(SimpleTagKey("see")).isDefined then - processLink mutable.Map((tagsWithoutDiagram).toSeq: _*) def allTags(key: SimpleTagKey): List[String] =