community build can run doc on supported tasksFix sources in doc

romanowski · romanowski · commit 9d4bd15e38ea · 2020-11-30T22:47:04.000+01:00
diff --git a/community-build/src/scala/dotty/communitybuild/Main.scala b/community-build/src/scala/dotty/communitybuild/Main.scala
@@ -1,20 +1,91 @@
 package dotty.communitybuild
 
-object Main {
+import java.nio.file.Paths
+import java.nio.file.Path
+import java.nio.file.Files
+import scala.sys.process._
+
+
+object Main:
+  def allProjects = projects.projectMap.keys.toList.sorted
+
+  private def generateDocs(project: CommunityProject): Seq[Path] =
+    val name = project.project
+    try
+      project.doc()
+      val pathsOut = s"find community-projects/$name/ -name 'scala3doc.version'".!!
+      pathsOut.linesIterator.map(Paths.get(_).getParent).toList
+    catch
+      case e: Exception =>
+        e.printStackTrace()
+        Nil
+
   /** Allows running various commands on community build projects. */
   def main(args: Array[String]): Unit =
-    if args.length != 2 then
-      println("USAGE: <COMMAND> <PROJECT NAME>")
-      println("COMMAND is one of: publish doc")
-      println("Available projects are:")
-      projects.projectMap.keys.foreach { k =>
-        println(s"\t$k")
-      }
-      sys.exit(0)
-
-    val Array(cmd, proj) = args
-    cmd match {
-      case "doc" => projects(proj).doc()
-      case "publish" => projects(proj).publish()
-    }
-}
+    args.toList match
+      case "publish" :: name :: Nil =>
+      case "doc" :: "all" :: destStr :: Nil =>
+        val dest = Paths.get(destStr)
+        s"rm -rf $destStr".!
+        Files.createDirectory(dest)
+        val (toRun, ignored) =
+          allProjects.map(projects.projectMap).partition(_.docCommand != null)
+
+        val paths = toRun.map { project =>
+          val name = project.project
+          val projectDest = dest.resolve(name)
+          val projectRoot = Paths.get(s"community-projects/$name")
+          println(s"generating docs for $name into $projectDest")
+          val generatedDocs = generateDocs(project)
+          if !Files.exists(projectDest) && generatedDocs.nonEmpty then
+            Files.createDirectory(projectDest)
+
+          val docsFiles = generatedDocs.map { docsPath =>
+            val destFileName =
+              docsPath.subpath(2, docsPath.getNameCount).toString.replace('/', '_')
+
+            s"cp -r $docsPath $projectDest/$destFileName".!
+            destFileName
+          }
+          name -> docsFiles
+        }
+
+        val (failed, withDocs) = paths.partition{ case (_, paths) => paths.isEmpty }
+
+        val indexFile = withDocs.map { case (name, paths) =>
+          paths.map(p => s"""<a href="$name/$p">$p</a></br>\n""")
+            .mkString(s"<h1>$name</h1>","\n", "\n")
+        }.mkString("<html><body>\n", "\n", "\n</html></body>")
+
+        Files.write(dest.resolve("index.html"), indexFile.getBytes)
+
+        if ignored.nonEmpty then println(s"Ignored project without doc command: $ignored")
+
+        if failed.nonEmpty then
+          println(s"Documentation not found for ${failed.map(_._1).mkString(", ")}")
+          sys.exit(1)
+
+      case "doc" :: names if names.nonEmpty =>
+        val missing = names.filterNot(projects.projectMap.contains)
+        if missing.nonEmpty then
+          println(s"Missing projects: ${missing.mkString(", ")}. All projects: ${allProjects.mkString(", ")}")
+          sys.exit(0)
+
+        val failed = names.filter{ p =>
+          val docsRoots = generateDocs(projects.projectMap(p))
+          if docsRoots.nonEmpty then println(s"Docs for $p generated in $docsRoots")
+          docsRoots.isEmpty
+        }
+        if failed.nonEmpty then
+          println(s"Documentation not found for ${failed.mkString(", ")}")
+          sys.exit(0)
+
+      case args =>
+        println("USAGE: <COMMAND> <PROJECT NAME>")
+        println("COMMAND is one of: publish doc")
+        println("Available projects are:")
+        allProjects.foreach { k =>
+          println(s"\t$k")
+        }
+        sys.exit(0)
+
diff --git a/community-build/src/scala/dotty/communitybuild/projects.scala b/community-build/src/scala/dotty/communitybuild/projects.scala
@@ -123,6 +123,16 @@ final case class SbtCommunityProject(
 
 object projects:
 
+  private def forceDoc(projects: String*) =
+    projects.map(project =>
+      s""";set $project/Compile/doc/sources ++= file("a.scala") +: ($project/Compile/doc/tastyFiles).value ;$project/doc"""
+    ).mkString(" ")
+
+  private def aggregateDoc(in: String)(projects: String*) =
+    val tastyFiles =
+      (in +: projects).map(p => s"($p/Compile/doc/tastyFiles).value").mkString(" ++ ")
+    s""";set $in/Compile/doc/sources ++= file("a.scala") +: ($tastyFiles) ;$in/doc"""
+
   lazy val utest = MillCommunityProject(
     project = "utest",
     baseCommand = s"utest.jvm[$compilerVersion]",
@@ -201,14 +211,14 @@ object projects:
   lazy val algebra = SbtCommunityProject(
     project       = "algebra",
     sbtTestCommand   = "coreJVM/compile",
-    sbtDocCommand = "coreJVM/doc"
+    sbtDocCommand = forceDoc("coreJVM")
   )
 
   lazy val scalacheck = SbtCommunityProject(
     project       = "scalacheck",
     sbtTestCommand   = "jvm/test;js/test",
     sbtPublishCommand = "jvm/publishLocal;js/publishLocal",
-    sbtDocCommand = "jvm/doc"
+    sbtDocCommand = forceDoc("jvm")
   )
 
   lazy val scalatest = SbtCommunityProject(
@@ -251,13 +261,17 @@ object projects:
   lazy val ScalaPB = SbtCommunityProject(
     project       = "ScalaPB",
     sbtTestCommand   = "dotty-community-build/compile",
-    sbtDocCommand = "dotty-community-build/doc"
+    // aggregateDoc("runtimeJVM")("scalapbc", "grpcRuntime", "compilerPlugin") fails with
+    // module class ScalaPbCodeGenerator$ has non-class parent: TypeRef(TermRef(ThisType(TypeRef(NoPrefix,module class <root>)),module protocbridge),ProtocCodeGenerator)
+    // Also it seems that we do not handle correctly aggreagation projects
+    // sbtDocCommand = "dotty-community-build/doc"
+    sbtDocCommand = forceDoc("scalapbc", "grpcRuntime","runtimeJVM", "compilerPlugin")
   )
 
   lazy val minitest = SbtCommunityProject(
     project       = "minitest",
     sbtTestCommand   = "test",
-    sbtDocCommand = "dotty-community-build/doc",
+    sbtDocCommand = aggregateDoc("lawsJVM")("minitestJVM"),
     dependencies = List(scalacheck)
   )
 
@@ -280,7 +294,7 @@ object projects:
   lazy val shapeless = SbtCommunityProject(
     project       = "shapeless",
     sbtTestCommand   = "test",
-    sbtDocCommand = "doc"
+    sbtDocCommand = forceDoc("typeable", "deriving", "data")
   )
 
   lazy val xmlInterpolator = SbtCommunityProject(
@@ -314,13 +328,15 @@ object projects:
   lazy val sconfig = SbtCommunityProject(
     project       = "sconfig",
     sbtTestCommand   = "sconfigJVM/test",
-    sbtDocCommand = "sconfigJVM/doc",
+    // sbtDocCommand = "sconfigJVM/doc", // Fails with:
+    // Problem parsing sconfig/sharedScala3/src/main/scala/org/ekrich/config/ConfigSyntax.scala:[73..92..1340], documentation may not be generated.
+    // scala.MatchError: ValDef(JSON,TypeTree[TypeRef(TermRef(ThisType(TypeRef(NoPrefix,module class ekrich)),module config),class ConfigSyntax)],Apply(Ident($new),List(Literal(Constant(0)), Literal(Constant(JSON))))) (of class dotty.tools.dotc.ast.Trees$ValDef)
   )
 
   lazy val zio = SbtCommunityProject(
     project = "zio",
     sbtTestCommand = "testJVMDotty",
-    // sbtDocCommand  = "coreJVM/doc",
+    // sbtDocCommand  = forceDoc("coreJVM"),
     // Fails on tasty unpickling https://github.com/lampepfl/dotty/issues/10499
   )
 
@@ -349,19 +365,33 @@ object projects:
   lazy val scalaParserCombinators = SbtCommunityProject(
     project          = "scala-parser-combinators",
     sbtTestCommand   = "parserCombinatorsJVM/test",
-    sbtDocCommand   = "parserCombinatorsJVM/doc",
+    sbtDocCommand   = forceDoc("parserCombinatorsJVM"),
   )
 
   lazy val dottyCpsAsync = SbtCommunityProject(
     project          = "dotty-cps-async",
     sbtTestCommand   = "test",
-    sbtDocCommand = "doc",
+    // Does not compile (before reaches doc)
+    // sbtDocCommand = "cpsJVM/doc",
   )
 
   lazy val scalaz = SbtCommunityProject(
     project          = "scalaz",
     sbtTestCommand   = "rootJVM/test",
-    sbtDocCommand = "rootJVM/doc",
+
+    // sbtDocCommand = forceDoc("coreJVM"), // Fails with:
+    // [error] class scalaz.Conts cannot be unpickled because no class file was found
+    // [error] class scalaz.ContsT cannot be unpickled because no class file was found
+    // [error] class scalaz.IndexedCont cannot be unpickled because no class file was found
+
+    // aggregateDoc("rootJVM")("effectJVM", "iterateeJVM"), // Fails With
+    // [error] Caused by: java.lang.AssertionError: assertion failed:
+    // trait MonadIO has non-class parent: AppliedType(TypeRef(TermRef(ThisType(TypeRef(NoPrefix,module class <root>)),module scalaz),Monad),List(TypeRef(ThisType(TypeRef(TermRef(ThisType(TypeRef(NoPrefix,module class scalaz)),module effect),trait MonadIO)),type F)))
+
+    // sbtDocCommand = forceDoc("iterateeJVM"), // Fails with
+    // [error] class scalaz.iteratee.Iteratee cannot be unpickled because no class file was found
+
+    sbtDocCommand = forceDoc("effectJVM"),
     dependencies     = List(scalacheck)
   )
 
@@ -374,7 +404,8 @@ object projects:
   lazy val catsEffect2 = SbtCommunityProject(
     project        = "cats-effect-2",
     sbtTestCommand = "test",
-    sbtDocCommand = ";coreJVM/doc ;lawsJVM/doc",
+    // Currently is excluded from community build
+    // sbtDocCommand = ";coreJVM/doc ;lawsJVM/doc",
     forceUpgradeSbtScalajsPlugin = true
   )
 
@@ -388,8 +419,8 @@ object projects:
   lazy val scalaParallelCollections = SbtCommunityProject(
     project        = "scala-parallel-collections",
     sbtTestCommand = "test",
-    sbtDocCommand = "doc",
-    dependencies     = List(scalacheck)
+    sbtDocCommand  = forceDoc("core"),
+    dependencies   = List(scalacheck)
   )
 
   lazy val scalaCollectionCompat = SbtCommunityProject(
diff --git a/project/Build.scala b/project/Build.scala
@@ -1578,9 +1578,12 @@ object Build {
             Build.testcasesSourceRoot.in(Test),
             Build.testDocumentationRoot,
           ),
+          Compile / buildInfoKeys := Seq[BuildInfoKey](version),
+          Compile / buildInfoPackage := "dotty.dokka",
           testDocumentationRoot := (baseDirectory.value / "test-documentations").getAbsolutePath,
           buildInfoPackage in Test := "dotty.dokka",
           BuildInfoPlugin.buildInfoScopedSettings(Test),
+          BuildInfoPlugin.buildInfoScopedSettings(Compile),
           BuildInfoPlugin.buildInfoDefaultSettings,
           // Uncomment to debug dokka processing (require to run debug in listen mode on 5005 port)
           // javaOptions.in(run) += "-agentlib:jdwp=transport=dt_socket,server=n,address=localhost:5005,suspend=y"
diff --git a/sbt-dotty/src/dotty/tools/sbtplugin/DottyPlugin.scala b/sbt-dotty/src/dotty/tools/sbtplugin/DottyPlugin.scala
@@ -434,10 +434,12 @@ object DottyPlugin extends AutoPlugin {
   private val docSettings = inTask(doc)(Seq(
     tastyFiles := {
       val _ = compile.value // Ensure that everything is compiled, so TASTy is available.
-      (classDirectory.value ** "*.tasty").get.map(_.getAbsoluteFile)
+      // sbt is too smart and do not start doc task if there are no *.scala files defined
+      file("___fake___.scala") +:
+        (classDirectory.value ** "*.tasty").get.map(_.getAbsoluteFile)
     },
     sources := Def.taskDyn[Seq[File]] {
-      if (isDotty.value) Def.task { tastyFiles.value }
+      if (isDotty.value && useScala3doc.value) Def.task { tastyFiles.value }
       else Def.task { sources.value }
     }.value,
     scalacOptions ++= {
diff --git a/scala3doc/src/dotty/dokka/Scala3docArgs.scala b/scala3doc/src/dotty/dokka/Scala3docArgs.scala
@@ -58,8 +58,8 @@ object Scala3docArgs:
     def parseTastyRoots(roots: String) =
       roots.split(File.pathSeparatorChar).toList.map(new File(_))
 
-    val (existing, nonExisting) =
-      summary.arguments.map(File(_)).partition(_.exists)
+    val inFiles = summary.arguments.map(File(_)).filter(_.getName != "___fake___.scala")
+    val (existing, nonExisting) = inFiles.partition(_.exists)
 
     if nonExisting.nonEmpty then report.warning(
       s"Scala3doc will ignore following nonexisiten paths: ${nonExisting.mkString(", ")}"
diff --git a/scala3doc/src/dotty/dokka/preprocessors/ScalaResourceInstaller.scala b/scala3doc/src/dotty/dokka/preprocessors/ScalaResourceInstaller.scala
@@ -12,16 +12,24 @@ class ScalaResourceInstaller(args: Scala3doc.Args) extends PageTransformer:
     new RendererSpecificResourcePage(resourceName, java.util.ArrayList(), RenderingStrategy$Copy(s"/dotty_res/$resourceName"))
 
   override def invoke(input: RootPageNode): RootPageNode =
-    val defaultResources = input.getChildren.asScala ++ Seq("fonts", "images", "styles", "scripts", "hljs", "favicon.ico").map(dottyRes)
-    val newResources = projectLogo ++ defaultResources ++ Seq(dynamicJsData)
+    val dirs = Seq("fonts", "images", "styles", "scripts", "hljs", "favicon.ico")
+    val defaultResources = input.getChildren.asScala ++ dirs.map(dottyRes)
+    val newResources =
+      projectLogo ++ defaultResources ++ Seq(dynamicJsData, scala3docVersionFile)
     input.modified(input.getName, newResources.asJava)
 
+  private def textFile(path: String, content: String) =
+    val strategy = RenderingStrategy$Write(content)
+    new RendererSpecificResourcePage(path, java.util.ArrayList(), strategy)
+
   private def dynamicJsData =
-    // If data at any point will become more complex we should use a proper
-    val data: Map[String, Map[String, String]] = Map("filterDefaults" -> FilterAttributes.defaultValues)
+    // If data at any point will become more complex we should use a proper mapping
+    val data: Map[String, Map[String, String]] =
+      Map("filterDefaults" -> FilterAttributes.defaultValues)
     val str = new ObjectMapper().writeValueAsString(data.transform((_, v) => v.asJava).asJava)
+    textFile("scripts/data.js", s"var scala3DocData = $str")
 
-    new RendererSpecificResourcePage("scripts/data.js", java.util.ArrayList(), RenderingStrategy$Write(s"var scala3DocData = $str"))
+  private def scala3docVersionFile = textFile("scala3doc.version", BuildInfo.version)
 
   private def projectLogo = args.projectLogo.toSeq.map { path =>
       val fileName = Paths.get(path).getFileName()
