Specify unique ref tags in Dackka output (#4149)

* Add util methods for gradle projects

* Disable Javadoc generation on empty projects

* Fix ref path generation in Dackka output

* Add documentation for util methods

* Update the DackkaPlugin docs

* Reduced path to relative from tenant

* Reduced ref head path even more

* Fixed ref tag path to working solution

* Disabled publishJavadoc by default for tests

It should be enabled explicitly when being tested anyhow, and causes issues otherwise. Instead of disabling it for the tests that don't need it- this is much quicker and easier to manager.

Author: daymxn

buildSrc/src/main/java/com/google/firebase/gradle/plugins/DackkaPlugin.kt

@@ -5,9 +5,11 @@ import com.android.build.gradle.LibraryExtension
 import java.io.File
 import org.gradle.api.Plugin
 import org.gradle.api.Project
+import org.gradle.api.Task
 import org.gradle.api.provider.Provider
 import org.gradle.api.tasks.Copy
 import org.gradle.api.tasks.Delete
+import org.gradle.api.tasks.TaskProvider
 import org.gradle.kotlin.dsl.getByType
 import org.gradle.kotlin.dsl.register
 
@@ -90,8 +92,9 @@ tasks above). While we do not currently offer any configuration for the Dackka
 plugin, this could change in the future as needed. Currently, the DackkaPlugin
 provides sensible defaults to output directories, package lists, and so forth.
 
-The DackkaPlugin also provides three extra tasks:
+The DackkaPlugin also provides four extra tasks:
 [cleanDackkaDocumentation][registerCleanDackkaDocumentation],
+[separateJavadocAndKotlinDoc][registerSeparateJavadocAndKotlinDoc]
 [copyJavaDocToCommonDirectory][registerCopyJavaDocToCommonDirectoryTask] and
 [copyKotlinDocToCommonDirectory][registerCopyKotlinDocToCommonDirectoryTask].
 
@@ -100,6 +103,9 @@ the output of Dackka. This is useful when testing Dackka outputs itself- and
 shouldn't be apart of the normal flow. The reasoning is that it would otherwise
 invalidate the gradle cache.
 
+_separateJavadocAndKotlinDoc_ copies the Javadoc and Kotlindoc directories from Dackka into
+their own subdirectories- for easy and consistent differentiation.
+
 _copyJavaDocToCommonDirectory_ copies the JavaDoc variant of the Dackka output for each sdk,
 and pastes it in a common directory under the root project's build directory. This makes it easier
 to zip the doc files for staging.
@@ -119,24 +125,30 @@ abstract class DackkaPlugin : Plugin<Project> {
         registerCleanDackkaDocumentation(project)
         project.afterEvaluate {
             if (shouldWePublish(project)) {
-                val generateDocumentation = registerGenerateDackkaDocumentationTask(project)
-                val dackkaFilesDirectory = generateDocumentation.flatMap { it.outputDirectory }
-                val firesiteTransform = registerFiresiteTransformTask(project, dackkaFilesDirectory)
-                val transformedFilesDirectory = firesiteTransform.flatMap { it.outputDirectory }
-                val copyJavaDocToCommonDirectory = registerCopyJavaDocToCommonDirectoryTask(project, transformedFilesDirectory)
-                val copyKotlinDocToCommonDirectory = registerCopyKotlinDocToCommonDirectoryTask(project, transformedFilesDirectory)
+                val dackkaOutputDirectory = project.provider { fileFromBuildDir("dackkaRawOutput") }
+                val separatedFilesDirectory = project.provider { fileFromBuildDir("dackkaSeparatedFiles") }
+                val transformedDackkaFilesDirectory = project.provider { fileFromBuildDir("dackkaTransformedFiles") }
+
+                val generateDocumentation = registerGenerateDackkaDocumentationTask(project, dackkaOutputDirectory)
+                val separateJavadocAndKotlinDoc = registerSeparateJavadocAndKotlinDoc(project, dackkaOutputDirectory, separatedFilesDirectory)
+                val firesiteTransform = registerFiresiteTransformTask(project, separatedFilesDirectory, transformedDackkaFilesDirectory)
+                val copyJavaDocToCommonDirectory = registerCopyJavaDocToCommonDirectoryTask(project, transformedDackkaFilesDirectory)
+                val copyKotlinDocToCommonDirectory = registerCopyKotlinDocToCommonDirectoryTask(project, transformedDackkaFilesDirectory)
 
                 project.tasks.register("kotlindoc") {
                     group = "documentation"
                     dependsOn(
                         generateDocumentation,
+                        separateJavadocAndKotlinDoc,
                         firesiteTransform,
                         copyJavaDocToCommonDirectory,
                         copyKotlinDocToCommonDirectory
                     )
                 }
             } else {
-                project.tasks.register("kotlindoc")
+                project.tasks.register("kotlindoc") {
+                    group = "documentation"
+                }
             }
         }
     }
@@ -158,7 +170,10 @@ abstract class DackkaPlugin : Plugin<Project> {
     }
 
     // TODO(b/243324828): Refactor when fixed, so we no longer need stubs
-    private fun registerGenerateDackkaDocumentationTask(project: Project): Provider<GenerateDocumentationTask> {
+    private fun registerGenerateDackkaDocumentationTask(
+        project: Project,
+        targetDirectory: Provider<File>
+    ): Provider<GenerateDocumentationTask> {
         val docStubs = project.tasks.register<GenerateStubsTask>("docStubsForDackkaInput")
         val docsTask = project.tasks.register<GenerateDocumentationTask>("generateDackkaDocumentation")
         with(project.extensions.getByType<LibraryExtension>()) {
@@ -194,6 +209,8 @@ abstract class DackkaPlugin : Plugin<Project> {
                         kotlinSources.set(sourcesForKotlin)
                         dependencies.set(classpath)
 
+                        outputDirectory.set(targetDirectory)
+
                         applyCommonConfigurations()
                     }
                 }
@@ -232,20 +249,72 @@ abstract class DackkaPlugin : Plugin<Project> {
         dependsOnAndMustRunAfter("createFullJarRelease")
 
         val dackkaFile = project.provider { project.dackkaConfig.singleFile }
-        val dackkaOutputDirectory = File(project.buildDir, "dackkaDocumentation")
 
         dackkaJarFile.set(dackkaFile)
-        outputDirectory.set(dackkaOutputDirectory)
         clientName.set(project.firebaseConfigValue { artifactId })
     }
 
-    private fun registerFiresiteTransformTask(project: Project, dackkaFilesDirectory: Provider<File>) =
-        project.tasks.register<FiresiteTransformTask>("firesiteTransform") {
-            mustRunAfter("generateDackkaDocumentation")
+    // TODO(b/248302613): Remove when dackka exposes configuration for this
+    private fun registerSeparateJavadocAndKotlinDoc(
+        project: Project,
+        dackkaOutputDirectory: Provider<File>,
+        outputDirectory: Provider<File>
+    ): TaskProvider<Task> {
+        val outputJavadocFolder = project.childFile(outputDirectory, "android")
+        val outputKotlindocFolder = project.childFile(outputDirectory, "kotlin")
+
+        val separateJavadoc = project.tasks.register<Copy>("separateJavadoc") {
+            dependsOn("generateDackkaDocumentation")
+
+            val javadocClientFolder = project.childFile(dackkaOutputDirectory, "reference/client")
+            val javadocComFolder = project.childFile(dackkaOutputDirectory, "reference/com")
+
+            fromDirectory(javadocClientFolder)
+            fromDirectory(javadocComFolder)
+
+            into(outputJavadocFolder)
+        }
+
+        val separateKotlindoc = project.tasks.register<Copy>("separateKotlindoc") {
+            dependsOn("generateDackkaDocumentation")
+
+            val kotlindocFolder = project.childFile(dackkaOutputDirectory, "reference/kotlin")
+
+            from(kotlindocFolder)
+
+            into(outputKotlindocFolder)
+        }
+
+        return project.tasks.register("separateJavadocAndKotlinDoc") {
+            dependsOn(separateJavadoc, separateKotlindoc)
+        }
+    }
+
+    private fun registerFiresiteTransformTask(
+        project: Project,
+        separatedFilesDirectory: Provider<File>,
+        targetDirectory: Provider<File>
+    ): TaskProvider<Task> {
+        val transformJavadoc = project.tasks.register<FiresiteTransformTask>("firesiteTransformJavadoc") {
+            dependsOnAndMustRunAfter("separateJavadoc")
+
+            referenceHeadTagsPath.set("docs/reference/android")
+            dackkaFiles.set(project.childFile(separatedFilesDirectory, "android"))
+            outputDirectory.set(project.childFile(targetDirectory, "android"))
+        }
+
+        val transformKotlindoc = project.tasks.register<FiresiteTransformTask>("firesiteTransformKotlindoc") {
+            dependsOnAndMustRunAfter("separateKotlindoc")
+
+            referenceHeadTagsPath.set("docs/reference/kotlin")
+            dackkaFiles.set(project.childFile(separatedFilesDirectory, "kotlin"))
+            outputDirectory.set(project.childFile(targetDirectory, "kotlin"))
+        }
 
-            dackkaFiles.set(dackkaFilesDirectory)
-            outputDirectory.set(project.file("${project.buildDir}/dackkaTransformedFiles"))
+        return project.tasks.register("firesiteTransform") {
+            dependsOn(transformJavadoc, transformKotlindoc)
         }
+    }
 
     // TODO(b/246593212): Migrate doc files to single directory
     private fun registerCopyJavaDocToCommonDirectoryTask(project: Project, outputDirectory: Provider<File>) =
@@ -258,12 +327,10 @@ abstract class DackkaPlugin : Plugin<Project> {
             if (project.rootProject.findProperty("dackkaJavadoc") == "true") {
                 mustRunAfter("firesiteTransform")
 
-                val outputFolder = project.file("${project.rootProject.buildDir}/firebase-kotlindoc/android")
-                val clientFolder = outputDirectory.map { project.file("${it.path}/reference/client") }
-                val comFolder = outputDirectory.map { project.file("${it.path}/reference/com") }
+                val outputFolder = project.rootProject.fileFromBuildDir("firebase-kotlindoc/android")
+                val javaFolder = project.childFile(outputDirectory, "android")
 
-                fromDirectory(clientFolder)
-                fromDirectory(comFolder)
+                fromDirectory(javaFolder)
 
                 into(outputFolder)
             }
@@ -274,8 +341,8 @@ abstract class DackkaPlugin : Plugin<Project> {
         project.tasks.register<Copy>("copyKotlinDocToCommonDirectory") {
             mustRunAfter("firesiteTransform")
 
-            val outputFolder = project.file("${project.rootProject.buildDir}/firebase-kotlindoc")
-            val kotlinFolder = outputDirectory.map { project.file("${it.path}/reference/kotlin") }
+            val outputFolder = project.rootProject.fileFromBuildDir("firebase-kotlindoc")
+            val kotlinFolder = project.childFile(outputDirectory, "kotlin")
 
             fromDirectory(kotlinFolder)
 
@@ -287,8 +354,9 @@ abstract class DackkaPlugin : Plugin<Project> {
         project.tasks.register<Delete>("cleanDackkaDocumentation") {
             group = "cleanup"
 
-            delete("${project.buildDir}/dackkaDocumentation")
-            delete("${project.buildDir}/dackkaTransformedFiles")
-            delete("${project.rootProject.buildDir}/firebase-kotlindoc")
+            val outputDirs = listOf("dackkaRawOutput", "dackkaSeparatedFiles", "dackkaTransformedFiles")
+
+            delete(outputDirs.map { project.fileFromBuildDir(it) })
+            delete(project.rootProject.fileFromBuildDir("firebase-kotlindoc"))
         }
 }

buildSrc/src/main/java/com/google/firebase/gradle/plugins/FiresiteTransformTask.kt

@@ -4,6 +4,7 @@ import java.io.File
 import org.gradle.api.DefaultTask
 import org.gradle.api.provider.Property
 import org.gradle.api.tasks.CacheableTask
+import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.InputDirectory
 import org.gradle.api.tasks.OutputDirectory
 import org.gradle.api.tasks.PathSensitive
@@ -21,6 +22,7 @@ import org.gradle.api.tasks.TaskAction
  *  - Appends /docs/ to hyperlinks in html files
  *  - Removes the prefix path from book_path
  *  - Removes the firebase prefix from all links
+ *  - Changes the path for _reference-head-tags at the top of html files
  *
  *  **Please note:**
  *  This task is idempotent- meaning it can safely be ran multiple times on the same set of files.
@@ -31,6 +33,9 @@ abstract class FiresiteTransformTask : DefaultTask() {
     @get:PathSensitive(PathSensitivity.RELATIVE)
     abstract val dackkaFiles: Property<File>
 
+    @get:Input
+    abstract val referenceHeadTagsPath: Property<String>
+
     @get:OutputDirectory
     abstract val outputDirectory: Property<File>
 
@@ -60,7 +65,7 @@ abstract class FiresiteTransformTask : DefaultTask() {
     }
 
     private fun File.fixHTMLFile() {
-        val fixedContent = readText().fixBookPath().fixHyperlinks().removeLeadingFirebaseDomainInLinks()
+        val fixedContent = readText().fixBookPath().fixHyperlinks().removeLeadingFirebaseDomainInLinks().fixReferenceHeadTagsPath()
         writeText(fixedContent)
     }
 
@@ -69,28 +74,33 @@ abstract class FiresiteTransformTask : DefaultTask() {
         writeText(fixedContent)
     }
 
+    // We utilize difference reference head tags between Kotlin and Java docs
+    // TODO(b/248316730): Remove when dackka exposes configuration for this
+    private fun String.fixReferenceHeadTagsPath() =
+        replace(Regex("(?<=include \").*(?=/_reference-head-tags.html\" %})"), referenceHeadTagsPath.get())
+
     // We don't actually upload class or index files,
     // so these headers will throw not found errors if not removed.
-    // TODO(b/243674302): Remove when dackka supports this behavior
+    // TODO(b/243674302): Remove when dackka exposes configuration for this
     private fun String.removeClassHeader() =
         remove(Regex("- title: \"Class Index\"\n {2}path: \".+\"\n\n"))
     private fun String.removeIndexHeader() =
         remove(Regex("- title: \"Package Index\"\n {2}path: \".+\"\n\n"))
 
     // We use a common book for all sdks, wheres dackka expects each sdk to have its own book.
-    // TODO(b/243674303): Remove when dackka supports this behavior
+    // TODO(b/243674303): Remove when dackka exposes configuration for this
     private fun String.fixBookPath() =
         remove(Regex("(?<=setvar book_path ?%})(.+)(?=/_book.yaml\\{% ?endsetvar)"))
 
     // Our documentation lives under /docs/reference/ versus the expected /reference/
-    // TODO(b/243674305): Remove when dackka supports this behavior
+    // TODO(b/243674305): Remove when dackka exposes configuration for this
     private fun String.fixHyperlinks() =
         replace(Regex("(?<=href=\")(/)(?=reference/.*\\.html)"), "/docs/")
 
     // The documentation will work fine without this. This is primarily to make sure that links
     // resolve to their local counter part. Meaning when the docs are staged, they will resolve to
     // staged docs instead of prod docs- and vise versa.
-    // TODO(b/243673063): Remove when dackka supports this behavior
+    // TODO(b/243673063): Remove when dackka exposes configuration for this
     private fun String.removeLeadingFirebaseDomainInLinks() =
         remove(Regex("(?<=\")(https://firebase\\.google\\.com)(?=/docs/reference)"))
 }

buildSrc/src/main/java/com/google/firebase/gradle/plugins/GradleUtils.kt

@@ -1,10 +1,35 @@
 package com.google.firebase.gradle.plugins
 
 import java.io.File
+import org.gradle.api.Project
 import org.gradle.api.provider.Provider
 import org.gradle.api.tasks.Copy
 
 fun Copy.fromDirectory(directory: Provider<File>) =
     from(directory) {
         into(directory.map { it.name })
     }
+
+/**
+ * Creates a file at the buildDir for the given [Project].
+ *
+ * Syntax sugar for:
+ *
+ * ```
+ * project.file("${project.buildDir}/$path)
+ * ```
+ */
+fun Project.fileFromBuildDir(path: String) = file("$buildDir/$path")
+
+/**
+ * Maps a file provider to another file provider as a sub directory.
+ *
+ * Syntax sugar for:
+ *
+ * ```
+ * fileProvider.map { project.file("${it.path}/$path") }
+ * ```
+ */
+fun Project.childFile(provider: Provider<File>, childPath: String) = provider.map {
+    file("${it.path}/$childPath")
+}

buildSrc/src/test/kotlin/com/google/firebase/gradle/plugins/publishing.kt

@@ -29,6 +29,7 @@ data class Project(
     val externalDependencies: Set<Artifact> = setOf(),
     val releaseWith: Project? = null,
     val customizePom: String? = null,
+    val publishJavadoc: Boolean = false,
     val libraryType: LibraryType = LibraryType.ANDROID
 ) {
     fun generateBuildFile(): String {
@@ -42,6 +43,7 @@ data class Project(
             firebaseLibrary {
                 ${if (releaseWith != null) "releaseWith project(':${releaseWith.name}')" else ""}
                 ${if (customizePom != null) "customizePom {$customizePom}" else ""}
+                ${"publishJavadoc = $publishJavadoc"}
             }
             ${if (libraryType == LibraryType.ANDROID) "android.compileSdkVersion = 26" else ""}
 

firebase-abt/firebase-abt.gradle

@@ -19,6 +19,7 @@ plugins {
 firebaseLibrary {
     testLab.enabled = false
     publishSources = true
+    publishJavadoc = false
 }
 
 android {

firebase-components/firebase-components.gradle

@@ -18,6 +18,7 @@ plugins {
 
 firebaseLibrary {
     publishSources = true
+    publishJavadoc = false
 }
 
 android {

firebase-components/firebase-dynamic-module-support/firebase-dynamic-module-support.gradle

@@ -21,6 +21,7 @@ group = 'com.google.firebase'
 firebaseLibrary {
     testLab.enabled = false
     publishSources = true
+    publishJavadoc = false
 }
 
 android {

firebase-database-collection/firebase-database-collection.gradle

@@ -18,6 +18,7 @@ plugins {
 
 firebaseLibrary {
     publishSources = true
+    publishJavadoc = false
 }
 
 android {