Fix repeated traversal of packages when generating docs (5x speedup)

Author: felixmulder

dottydoc/jvm/src/dotty/tools/dottydoc/core/Phases.scala

@@ -94,10 +94,22 @@ object Phases {
     var packages: Map[String, Package] = Map.empty
 
     def addEntity(p: Package): Package = {
+      def mergedChildren(x1s: List[Entity], x2s: List[Entity]): List[Entity] = {
+        val (packs1, others1) = x1s.partition(_.kind == "package")
+        val (packs2, others2) = x2s.partition(_.kind == "package")
+
+        val others = others1 ::: others2
+        val packs  = (packs1 ::: packs2).groupBy(_.path).map(_._2.head)
+
+        (others ++ packs).sortBy(_.name)
+      }
+
       val path    = p.path.mkString(".")
-      val newPack = packages.get(path).map { ex =>
-        val children = (ex.children ::: p.children).distinct.sortBy(_.name)
-        PackageImpl(p.name, children, p.path, None)
+      val newPack = packages.get(path).map {
+        case ex: PackageImpl =>
+          if (!ex.comment.isDefined) ex.comment = p.comment
+          ex.members = mergedChildren(ex.members, p.members)
+          ex
       }.getOrElse(p)
 
       packages = packages + (path -> newPack)

dottydoc/jvm/test/CompilerTest.scala renamed to dottydoc/jvm/test/BaseTest.scala

@@ -3,6 +3,7 @@ package dottydoc
 
 import dotc.core.Contexts
 import Contexts.{ Context, ContextBase, FreshContext }
+import dotc.util.SourceFile
 import dotc.core.Phases.Phase
 import dotc.typer.FrontEnd
 import dottydoc.core.Phases.DocPhase
@@ -34,17 +35,24 @@ trait DottyTest {
       Nil
   }
 
-  def checkCompile(source: String)(assertion: DocPhase => Unit): Unit = {
+  def checkSource(source: String)(assertion: DocPhase => Unit): Unit = {
     val c = compilerWithChecker(assertion)
     c.rootContext(ctx)
     val run = c.newRun
     run.compile(source)
   }
 
-  def checkCompile(sources: List[String])(assertion: DocPhase => Unit): Unit = {
+  def checkFiles(sources: List[String])(assertion: DocPhase => Unit): Unit = {
     val c = compilerWithChecker(assertion)
     c.rootContext(ctx)
     val run = c.newRun
     run.compile(sources)
   }
+
+  def checkSources(sourceFiles: List[SourceFile])(assertion: DocPhase => Unit): Unit = {
+    val c = compilerWithChecker(assertion)
+    c.rootContext(ctx)
+    val run = c.newRun
+    run.compileSources(sourceFiles)
+  }
 }

dottydoc/jvm/test/PackageStructure.scala

@@ -0,0 +1,89 @@
+package dotty.tools
+package dottydoc
+
+import org.junit.Test
+import org.junit.Assert._
+
+import dotc.util.SourceFile
+import model.internal._
+
+class PackageStructure extends DottyTest {
+  @Test def multipleCompilationUnits = {
+    val source1 = new SourceFile(
+      "<test>",
+      """
+      |package scala
+      |
+      |trait A
+      """.stripMargin
+    )
+
+    val source2 = new SourceFile(
+      "<test>",
+      """
+      |package scala
+      |
+      |trait B
+      """.stripMargin
+    )
+
+    checkSources(source1 :: source2 :: Nil) { doc =>
+      doc.packages("scala") match {
+        case PackageImpl(_, List(tA, tB), _, _) =>
+          assert(
+            tA.name == "A" && tB.name == "B",
+            s"trait A had name '${tA.name}' and trait B had name '${tB.name}'"
+          )
+        case _ => fail("Incorrect package structure after run")
+      }
+    }
+  }
+
+
+  @Test def multiplePackages = {
+    val source1 = new SourceFile(
+      "<test>",
+      """
+      |package scala
+      |package collection
+      |
+      |trait A
+      """.stripMargin)
+
+    val source2 = new SourceFile(
+      "<test>",
+      """
+      |package scala
+      |package collection
+      |
+      |trait B
+      """.stripMargin)
+
+    checkSources(source1 :: source2 :: Nil) { doc =>
+      doc.packages("scala") match {
+        case PackageImpl(
+          "scala",
+          List(PackageImpl("scala.collection", List(tA, tB), _, _)),
+          _, _
+        ) =>
+          assert(
+            tA.name == "A" && tB.name == "B",
+            s"trait A had name '${tA.name}' and trait B had name '${tB.name}'"
+          )
+
+        case _ =>
+          fail(s"""Incorrect package structure for 'scala' package: ${doc.packages("scala")}""")
+      }
+
+      doc.packages("scala.collection") match {
+        case PackageImpl("scala.collection", List(tA, tB), _, _) =>
+          assert(
+            tA.name == "A" && tB.name == "B",
+            s"trait A had name '${tA.name}' and trait B had name '${tB.name}'"
+          )
+
+        case _ => fail("Incorrect package structure for 'scala.collection' package")
+      }
+    }
+  }
+}

dottydoc/jvm/test/TestSimpleComments.scala renamed to dottydoc/jvm/test/SimpleComments.scala

@@ -15,7 +15,7 @@ class TestSimpleComments extends DottyTest {
       |trait HelloWorld
       """.stripMargin
 
-    checkCompile(source) { doc =>
+    checkSource(source) { doc =>
       val traitCmt = doc
         .packages("scala")
         .children.find(_.path.mkString(".") == "scala.HelloWorld")

dottydoc/jvm/test/TestWhitelistedStdLib.scala

@@ -1,23 +1,17 @@
 package dotty.tools
 package dottydoc
 
-import scala.io.Source
+import org.junit.Test
+import org.junit.Assert._
 
-object WhitelistedStandardLib extends DottyDoc {
-  val files: List[String] = {
-    val whitelist = "../../test/dotc/scala-collections.whitelist"
+class TestWhitelistedCollections extends DottyTest {
+  @Test def arrayHasDocumentation =
+    checkFiles(WhitelistedStandardLib.files) { doc =>
+      val array = doc
+        .packages("scala")
+        .children.find(_.path.mkString(".") == "scala.Array")
+        .get
 
-    Source.fromFile(whitelist, "UTF8")
-      .getLines()
-      .map(_.trim) // allow identation
-      .filter(!_.startsWith("#")) // allow comment lines prefixed by #
-      .map(_.takeWhile(_ != '#').trim) // allow comments in the end of line
-      .filter(_.nonEmpty)
-      .filterNot(_.endsWith("package.scala"))
-      .map("../." + _)
-      .toList
-  }
-
-  override def main(args: Array[String]) =
-    super.main("-language:Scala2" +: files.toArray)
+      assert(array.comment.get.body.length > 0)
+    }
 }

dottydoc/jvm/test/WhitelistedStdLib.scala

@@ -0,0 +1,23 @@
+package dotty.tools
+package dottydoc
+
+import scala.io.Source
+
+object WhitelistedStandardLib extends DottyDoc {
+  val files: List[String] = {
+    val whitelist = "../../test/dotc/scala-collections.whitelist"
+
+    Source.fromFile(whitelist, "UTF8")
+      .getLines()
+      .map(_.trim) // allow identation
+      .filter(!_.startsWith("#")) // allow comment lines prefixed by #
+      .map(_.takeWhile(_ != '#').trim) // allow comments in the end of line
+      .filter(_.nonEmpty)
+      .filterNot(_.endsWith("package.scala"))
+      .map("../." + _)
+      .toList
+  }
+
+  override def main(args: Array[String]) =
+    super.main("-language:Scala2" +: files.toArray)
+}

dottydoc/jvm/test/WhitelistedStdLibMain.scala

@@ -11,11 +11,11 @@ object internal {
 
   final case class PackageImpl(
     name: String,
-    members: List[Entity],
+    var members: List[Entity],
     path: List[String],
     var comment: Option[Comment] = None
   ) extends Package with Impl {
-    val children: List[Entity with Members] =
+    def children: List[Entity with Members] =
       members.collect { case x: Entity with Members => x }
   }