Apply flat packages in cases where static site is not provided

Author: romanowski

scala3doc/src/dotty/dokka/DottyDokkaConfig.scala

@@ -19,6 +19,8 @@ case class DottyDokkaConfig(args: Scala3doc.Args, docContext: DocContext) extend
 
   lazy val sourceLinks: SourceLinks = SourceLinks.load(args)
 
+  lazy val displaySourceSets = getSourceSets.toDisplaySourceSet
+
   lazy val staticSiteContext = args.docsRoot.map(path => StaticSiteContext(
       File(path).getAbsoluteFile(),
       Set(mkSourceSet.asInstanceOf[SourceSetWrapper]),

scala3doc/src/dotty/dokka/DottyDokkaPlugin.scala

@@ -23,9 +23,9 @@ import dotty.dokka.model.api._
 import org.jetbrains.dokka.CoreExtensions
 import org.jetbrains.dokka.base.DokkaBase
 
+import dotty.dokka.site.NavigationCreator
 import dotty.dokka.site.SitePagesCreator
 import dotty.dokka.site.StaticSiteContext
-import dotty.dokka.site.RootIndexPageCreator
 import dotty.dokka.site.SiteResourceManager
 import dotty.dokka.site.StaticSiteLocationProviderFactory
 
@@ -49,7 +49,7 @@ class DottyDokkaPlugin extends DokkaJavaPlugin:
   // Just turn off another translator since multiple overrides does not work
   val disableDescriptorTranslator = extend(
     _.extensionPoint(CoreExtensions.INSTANCE.getSourceToDocumentableTranslator)
-    .fromRecipe(ctx => new ScalaModuleProvider(using ctx.docContext))
+    .fromRecipe(ctx => new ScalaModuleProvider(using ctx.config.docContext))
     .overrideExtension(dokkaBase.getDescriptorToDocumentableTranslator)
     .name("disableDescriptorTranslator")
   )
@@ -70,7 +70,7 @@ class DottyDokkaPlugin extends DokkaJavaPlugin:
 
   val scalaResourceInstaller = extend(
     _.extensionPoint(dokkaBase.getHtmlPreprocessors)
-      .fromRecipe(ctx => new ScalaResourceInstaller(ctx.args))
+      .fromRecipe(ctx => new ScalaResourceInstaller(ctx.config.args))
       .after(dokkaBase.getCustomResourceInstaller)
   )
 
@@ -111,7 +111,7 @@ class DottyDokkaPlugin extends DokkaJavaPlugin:
 
   val ourRenderer = extend(
     _.extensionPoint(CoreExtensions.INSTANCE.getRenderer)
-      .fromRecipe(ctx => ScalaHtmlRenderer(ctx, ctx.args))
+      .fromRecipe(ctx => ScalaHtmlRenderer(ctx, ctx.config.args))
       .overrideExtension(dokkaBase.getHtmlRenderer)
   )
 
@@ -129,7 +129,7 @@ class DottyDokkaPlugin extends DokkaJavaPlugin:
 
   val customDocumentationProvider = extend(
     _.extensionPoint(dokkaBase.getHtmlPreprocessors)
-      .fromRecipe(c => SitePagesCreator(c.siteContext))
+      .fromRecipe(c => SitePagesCreator(c.config.staticSiteContext))
       .name("customDocumentationProvider")
       .ordered(
         before = Seq(
@@ -142,25 +142,27 @@ class DottyDokkaPlugin extends DokkaJavaPlugin:
       )
   )
 
-  val customIndexRootProvider = extend(
+  val customNavigation = extend(
     _.extensionPoint(dokkaBase.getHtmlPreprocessors)
-      .fromRecipe(c => RootIndexPageCreator(c.siteContext))
-      .name("customIndexRootProvider")
+      .fromRecipe(c => NavigationCreator(c.config)) // Adding a parameter to constuctor to NavigationCreator failed inc compilation
+      .name("customNavigation")
       .ordered(
         before = Seq(
           dokkaBase.getScriptsInstaller,
           dokkaBase.getStylesInstaller,
         ),
-        after = Seq(dokkaBase.getNavigationPageInstaller)
+        after = Seq(customDocumentationProvider.getValue)
       )
+      .overrideExtension(dokkaBase.getNavigationPageInstaller)
   )
 
   val customDocumentationResources = extend(
     _.extensionPoint(dokkaBase.getHtmlPreprocessors)
-    .fromRecipe(c => SiteResourceManager(c.siteContext))
+    .fromRecipe(c => SiteResourceManager(c.config.staticSiteContext))
       .name("customDocumentationResources")
       .after(
-        scalaEmbeddedResourceAppender.getValue
+        scalaEmbeddedResourceAppender.getValue,
+        customDocumentationProvider.getValue
       )
   )
 
@@ -171,9 +173,7 @@ class DottyDokkaPlugin extends DokkaJavaPlugin:
   )
 
 extension (ctx: DokkaContext):
-  def siteContext: Option[StaticSiteContext] = ctx.getConfiguration.asInstanceOf[DottyDokkaConfig].staticSiteContext
-  def args: Scala3doc.Args = ctx.getConfiguration.asInstanceOf[DottyDokkaConfig].args
-  def docContext = ctx.getConfiguration.asInstanceOf[DottyDokkaConfig].docContext
+  def config = ctx.getConfiguration.asInstanceOf[DottyDokkaConfig]
 
 // TODO (https://github.com/lampepfl/scala3doc/issues/232): remove once problem is fixed in Dokka
 extension [T]  (builder: ExtensionBuilder[T]):

scala3doc/src/dotty/dokka/compat.scala

@@ -29,6 +29,7 @@ type JHashMap[K, V] = java.util.HashMap[K, V]
 type JMapEntry[K, V] = java.util.Map.Entry[K, V]
 
 type DRI = org.jetbrains.dokka.links.DRI
+val topLevelDri = org.jetbrains.dokka.links.DRI.Companion.getTopLevel
 
 type SourceSetWrapper = DokkaConfiguration$DokkaSourceSet
 type DokkaSourceSet = DokkaConfiguration.DokkaSourceSet

scala3doc/src/dotty/dokka/site/NaivationCreator.scala

@@ -0,0 +1,72 @@
+package dotty.dokka
+package site
+
+import org.jetbrains.dokka.base.renderers.html.{NavigationNode, NavigationPage}
+import org.jetbrains.dokka.model.DPackage
+import org.jetbrains.dokka.model.DModule
+import org.jetbrains.dokka.pages._
+import org.jetbrains.dokka.transformers.pages.PageTransformer
+
+import scala.collection.JavaConverters._
+
+class NavigationCreator(ctx: DottyDokkaConfig) extends PageTransformer:
+
+  private def processApiPages(pages: List[PageNode]): JList[NavigationNode] =
+    def flatMapPackages(pn: PageNode): List[NavigationNode] =
+      def processChildren = pn.getChildren.asScala.flatMap(flatMapPackages).toList
+      pn match
+        case cp: ContentPage => cp.getDocumentable match
+          case null =>
+            processChildren
+          // Left over package from dokka
+          case p: DPackage if p.getName == "<empty>" && p.getChildren.isEmpty =>
+            Nil
+          case p: DPackage =>
+            val ss = p.getSourceSets.asScala.toSet.toDisplay
+            List(new NavigationNode(p.getName, p.getDri, ss, JList())) ++ processChildren
+          case _: DModule =>
+            processChildren
+          case _ =>
+            Nil
+        case _ =>
+          Nil
+
+    pages.flatMap(flatMapPackages).sortBy(_.getName).asJava
+
+  private def processStaticPages(input: PageNode)(staticSiteContext: StaticSiteContext) =
+    def toNavigationNode(page: StaticPageNode): NavigationNode = NavigationNode(
+      page.title(),
+      page.getDri.asScala.head,
+      ctx.displaySourceSets,
+      page.getChildren.asScala
+        .collect { case p: StaticPageNode => toNavigationNode(p)}.asJava
+    )
+
+    def singleContentPage(p: PageNode) =
+      p.getChildren.asScala.collectFirst { case c: ContentPage => c }.get
+    val pageRoot = singleContentPage(input)
+    val docsRoot =
+      if !pageRoot.getDri.contains(topLevelDri) then pageRoot
+      else singleContentPage(pageRoot)
+    val apiPages = docsRoot.getChildren.asScala.filterNot(_.isInstanceOf[StaticPageNode])
+    val staticPages = staticSiteContext.mainPages.map(toNavigationNode).toList.asJava
+    val apiNodes = processApiPages(apiPages.toList)
+    staticPages ++
+      JList(new NavigationNode("API", apiPageDRI, ctx.displaySourceSets, apiNodes))
+
+  private def emptyNavigationJson =
+      val strategy = new RenderingStrategy.Write("[]")
+      new RendererSpecificResourcePage("scripts/navigation-pane.json", JList(), strategy)
+
+  final override def invoke(input: RootPageNode): RootPageNode =
+    def defaultApiPages = processApiPages(input.getChildren.asScala.toList)
+    val nodes = ctx.staticSiteContext.fold(defaultApiPages)(processStaticPages(input))
+
+    val navigationPage = new NavigationPage(new NavigationNode(
+      ctx.args.name,
+      ctx.staticSiteContext.fold(topLevelDri)(_ => docsRootDRI),
+      ctx.displaySourceSets,
+      nodes
+    ))
+    val newChildren = input.getChildren ++ JList(emptyNavigationJson, navigationPage)
+    input.modified(input.getName, newChildren)

scala3doc/src/dotty/dokka/site/SitePagesCreator.scala

@@ -0,0 +1,62 @@
+package dotty.dokka
+package site
+
+import java.io.File
+import java.nio.file.Files
+import java.nio.file.FileVisitOption
+
+import org.jetbrains.dokka.model.Documentable
+import org.jetbrains.dokka.pages._
+
+import scala.collection.JavaConverters._
+import dotty.dokka.model.api._
+
+
+class SitePagesCreator(ctx: Option[StaticSiteContext]) extends BaseStaticSiteProcessor(ctx):
+  private def mkRootPage(input: RootPageNode, children: List[PageNode]): AContentPage =
+    input match
+      case input: ContentPage =>
+        AContentPage(
+          input.getName,
+          children.asJava,
+          input.getContent,
+          JSet(apiPageDRI),
+          input.getEmbeddedResources
+        )
+      case _: RendererSpecificRootPage =>
+        children.filter(_.isInstanceOf[RootPageNode]) match
+            case List(nestedRoot: RootPageNode) =>
+              mkRootPage(nestedRoot, children.filter { _ != nestedRoot } ++
+                nestedRoot.getChildren.asScala)
+            case other =>
+              throw new RuntimeException(s"Expected single nested roor but get: $other")
+
+      case _ => throw new RuntimeException(s"UNSUPPORTED! ${input.getClass.getName}")
+
+
+  override def transform(input: RootPageNode, ctx: StaticSiteContext): RootPageNode =
+    val (contentPage, others) =
+      input.getChildren.asScala.toList.partition { _.isInstanceOf[ContentPage] }
+
+    val apiRoot = mkRootPage(input, contentPage)
+    val (indexes, children) = ctx.allPages.partition(f =>
+      f.template.isIndexPage() && f.template.file.toPath.getParent() == ctx.docsPath )
+    // TODO (https://github.com/lampepfl/scala3doc/issues/238): provide proper error handling
+    if (indexes.size > 1) println(s"ERROR: Multiple index pages found ${indexes.map(_.template.file)}")
+
+    def emptyContent = ctx.asContent(Text(), mkDRI(extra = "root_content")).get(0)
+
+    val root = ctx.indexPage().toList.map(_.copy(getDri = JSet(docsRootDRI)))
+    val docsRoot = AContentPage(
+      ctx.args.name,
+      (List(apiRoot.modified("API", apiRoot.getChildren)) ++ children ++ root).asJava,
+      indexes.headOption.fold(emptyContent)(_.getContent),
+      JSet(docsDRI),
+      JList()
+    )
+
+    new RendererSpecificRootPage(
+      apiRoot.getName,
+      (List(docsRoot) ++ others).asJava,
+      RenderingStrategy.DoNothing.INSTANCE
+    )

scala3doc/src/dotty/dokka/site/SiteResourceManager.scala

@@ -0,0 +1,50 @@
+package dotty.dokka
+package site
+
+import java.nio.file.Files
+import java.nio.file.FileVisitOption
+
+import org.jetbrains.dokka.model.Documentable
+import org.jetbrains.dokka.pages._
+
+import scala.collection.JavaConverters._
+import dotty.dokka.model.api._
+
+class SiteResourceManager(ctx: Option[StaticSiteContext])
+  extends BaseStaticSiteProcessor(ctx):
+    private def listResources(nodes: Seq[PageNode]): Set[String] =
+      nodes.flatMap {
+        case it: AContentPage =>
+          listResources(it.getChildren.asScala.toList)
+        case it: StaticPageNode =>
+          listResources(it.getChildren.asScala.toList) ++ it.resources()
+        case _ => Seq.empty
+      }.toSet
+
+    override def transform(input: RootPageNode, ctx: StaticSiteContext): RootPageNode =
+      val rootPath = ctx.root.toPath
+      val imgPath = rootPath.resolve("images")
+      val images =
+        if !Files.exists(imgPath) then Nil
+        else
+          val allPaths = Files.walk(imgPath, FileVisitOption.FOLLOW_LINKS)
+          val files = allPaths.filter(Files.isRegularFile(_)).iterator().asScala
+          files.map(p => rootPath.relativize(p).toString).toList
+
+      val resources = images ++ listResources(input.getChildren.asScala.toList)
+      println(s"#### $resources")
+      val resourcePages = resources.map { path =>
+        val strategy = new RenderingStrategy.Copy(rootPath.resolve(path).toString)
+        new RendererSpecificResourcePage(path, JList(), strategy)
+      }
+
+      val modified = input.transformContentPagesTree {
+        case it: StaticPageNode =>
+          it.copy(getEmbeddedResources =
+            if it.template.hasFrame then it.getEmbeddedResources ++ it.resources().asJava
+            else it.resources().asJava
+          )
+        case it => it
+      }
+      val newChildren = (resourcePages ++ modified.getChildren.asScala).asJava
+      modified.modified(modified.getName, newChildren)

scala3doc/src/dotty/dokka/site/StaticSiteLocationProvider.scala

@@ -7,6 +7,7 @@ import org.jetbrains.dokka.base.resolvers.local.LocationProviderFactory
 import org.jetbrains.dokka.pages.ContentPage
 import org.jetbrains.dokka.pages.PageNode
 import org.jetbrains.dokka.pages.RootPageNode
+import org.jetbrains.dokka.pages.ModulePage
 import org.jetbrains.dokka.plugability.DokkaContext
 
 import scala.collection.JavaConverters._
@@ -16,21 +17,21 @@ import java.nio.file.Path
 class StaticSiteLocationProviderFactory(private val ctx: DokkaContext) extends LocationProviderFactory:
   override def getLocationProvider(pageNode: RootPageNode): LocationProvider =
     try new StaticSiteLocationProvider(ctx, pageNode)
-    catch 
+    catch
       case e: Error =>
         // TODO (https://github.com/lampepfl/scala3doc/issues/238) error handling
         e.printStackTrace()
         // We encounter bug in Kotlin coroutines (race) when this method throws exception
         // In such case we want to return null to trigger NPE in other piece of code to fail properly coroutine context
-        // Making generated DRIs not-unique will reproduce this behavior 
+        // Making generated DRIs not-unique will reproduce this behavior
         null
 
 class StaticSiteLocationProvider(ctx: DokkaContext, pageNode: RootPageNode)
   extends DokkaLocationProvider(pageNode, ctx, ".html"):
     private def updatePageEntry(page: PageNode, jpath: JList[String]): JList[String] =
       page match
         case page: StaticPageNode =>
-          ctx.siteContext.fold(jpath) { context =>
+          ctx.config.staticSiteContext.fold(jpath) { context =>
             val rawFilePath = context.root.toPath.relativize(page.template.file.toPath)
             val pageName = page.template.file.getName
             val dotIndex = pageName.lastIndexOf('.')
@@ -60,7 +61,7 @@ class StaticSiteLocationProvider(ctx: DokkaContext, pageNode: RootPageNode)
         case _ if jpath.size() > 1 && jpath.get(0) ==   "--root--" && jpath.get(1) == "-a-p-i" =>
           (List("api") ++ jpath.asScala.drop(2)).asJava
 
-        case _: org.jetbrains.dokka.pages.ModulePage if ctx.siteContext.isEmpty =>
+        case _: ModulePage if ctx.config.staticSiteContext.isEmpty =>
           JList("index")
         case _ =>
           jpath

scala3doc/src/dotty/dokka/site/common.scala

@@ -14,7 +14,11 @@ import com.vladsch.flexmark.ext.yaml.front.matter.{AbstractYamlFrontMatterVisito
 import com.vladsch.flexmark.parser.{Parser, ParserEmulationProfile}
 import com.vladsch.flexmark.util.options.{DataHolder, MutableDataSet}
 import com.vladsch.flexmark.ext.wikilink.WikiLinkExtension
+
 import org.jetbrains.dokka.model.doc.Text
+import org.jetbrains.dokka.model.Documentable
+import org.jetbrains.dokka.transformers.pages.PageTransformer
+import org.jetbrains.dokka.pages._
 
 import scala.collection.JavaConverters._
 
@@ -112,3 +116,31 @@ def loadTemplateFile(file: File): TemplateFile = {
 }
 
 def Text(msg: String = "") = new Text(msg, JList(), JMap())
+
+abstract class BaseStaticSiteProcessor(staticSiteContext: Option[StaticSiteContext])
+  extends PageTransformer:
+    final override def invoke(input: RootPageNode): RootPageNode =
+      staticSiteContext.fold(input)(transform(input, _))
+
+    protected def transform(input: RootPageNode, ctx: StaticSiteContext): RootPageNode
+
+case class AContentPage(
+  override val getName: String,
+  override val getChildren: JList[PageNode],
+  override val getContent: ContentNode,
+  override val getDri: JSet[DRI],
+  override val getEmbeddedResources: JList[String] = JList(),
+) extends ContentPage:
+  override def getDocumentable: Documentable = null
+
+  override def modified(
+    name: String,
+    content: ContentNode,
+    dri: JSet[DRI],
+    embeddedResources: JList[String],
+    children: JList[_ <: PageNode]
+  ): ContentPage =
+    copy(name, children.asInstanceOf[JList[PageNode]], content, dri, embeddedResources)
+
+  override def modified(name: String, children: JList[_ <: PageNode]): PageNode =
+    copy(name, getChildren = children.asInstanceOf[JList[PageNode]])