Use a separate Knit tool (#1486)

* Use a separate Knit tool version 0.1.2
* Hierarchical knit.properties files are introduced which are
  gathered while walking up the directory tree with the properties
  in knit resources being top-level ones.
* Freemarker templates for "knit.include" (top-level of example files)
  and "test.template" (test template) are introduced with locations that
  are resolved from properties.
* KNIT and TEST_OUT directives are not supported anymore.
 - Knitting is controlled by "knit.dir" and "knit.pattern" props.
 - "test.name" prop or TEST_NAME directive drives test gen.
* All markdown files are now clean of top-level knit-related boilerplate
  (only TEST_NAME directive is remaining in some)
* All example files are renumbered. It affects example-basic-xx,
  which had the legacy numbering with later insertions.
* All auto-generated files now have the correct source-file name
  specified at their beginning, due to consistent use of template
  substitution.
* No need to customize knit.pattern anymore (works out-of-the box)
* "knit.name" is automatically generated based on example's file name
  "example-basic-01" -> "exampleBasic01"
* Not match-pattern-group woodoo anymore, "knit.pattern" cannot have any
  user-defined match groups.
* No need to look for "package xxx" in knitted sources, as the
  example's package is always ${knit.package}.${knit.name}
* Simpler test names:
  testKotlinxCoroutinesGuideBasic01 -> testExampleBasic01
* Replaced END_TOC with END directive
* Build-scripts are improved:
  - Consistent code to use mavenLocal when snapshots versions are used.
  - Proper substitution logic to use Android AAR files so that Dokka has correct links to Google APIs.
  - Google repository is added to all projects.

Author: elizarov

README.md

@@ -217,6 +217,12 @@ The `develop` branch is pushed to `master` during release.
 
 * Full release procedure checklist is [here](RELEASE.md).
 * Steps for contributing new integration modules are explained [here](integration/README.md#Contributing).
+* Use [Knit](https://github.com/Kotlin/kotlinx-knit/blob/master/README.md) for updates to documentation:
+  * In project root directory run `./gradlew knit`.
+  * Commit updated documents and examples together with other changes.
+* Use [Binary Compatibility Validator](https://github.com/Kotlin/binary-compatibility-validator/blob/master/README.md) for updates to public API:
+  * In project root directory run `./gradlew apiDump`. 
+  * Commit updated API index together with other changes.
 
 <!--- MODULE kotlinx-coroutines-core -->
 <!--- INDEX kotlinx.coroutines -->

build.gradle

@@ -1,5 +1,5 @@
 /*
- * Copyright 2016-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
+ * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
  */
 import org.jetbrains.kotlin.konan.target.HostManager
 
@@ -9,7 +9,7 @@ def rootModule = "kotlinx.coroutines"
 def coreModule = "kotlinx-coroutines-core"
 // Not applicable for Kotlin plugin
 def sourceless = ['kotlinx.coroutines', 'site', 'kotlinx-coroutines-bom']
-def internal = ['kotlinx.coroutines', 'site', 'benchmarks', 'knit', 'js-stub', 'stdlib-stubs']
+def internal = ['kotlinx.coroutines', 'site', 'benchmarks', 'js-stub', 'stdlib-stubs']
 // Not published
 def unpublished = internal + ['example-frontend-js', 'android-unit-tests']
 
@@ -41,7 +41,16 @@ buildscript {
         }
     }
 
-    if (build_snapshot_train || atomicfu_version.endsWith("-SNAPSHOT")) {
+    // Determine if any project dependency is using a snapshot version
+    ext.using_snapshot_version = build_snapshot_train
+    rootProject.properties.each { key, value ->
+        if (key.endsWith("_version") && value instanceof String && value.endsWith("-SNAPSHOT")) {
+            println("NOTE: USING SNAPSHOT VERSION: $key=$value")
+            ext.using_snapshot_version=true
+        }
+    }
+
+    if (using_snapshot_version) {
         repositories {
             mavenLocal()
             maven { url "https://oss.sonatype.org/content/repositories/snapshots" }
@@ -67,6 +76,7 @@ buildscript {
         classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
         classpath "org.jetbrains.dokka:dokka-gradle-plugin:$dokka_version"
         classpath "org.jetbrains.kotlinx:atomicfu-gradle-plugin:$atomicfu_version"
+        classpath "org.jetbrains.kotlinx:kotlinx-knit:$knit_version"
         classpath "com.moowork.gradle:gradle-node-plugin:$gradle_node_version"
         classpath "org.jetbrains.kotlinx:binary-compatibility-validator:$binary_compatibility_validator_version"
 
@@ -102,7 +112,7 @@ allprojects {
         kotlin_version = rootProject.properties['kotlin_snapshot_version']
     }
 
-    if (build_snapshot_train || atomicfu_version.endsWith("-SNAPSHOT") || lincheck_version.endsWith("-SNAPSHOT")) {
+    if (using_snapshot_version) {
         repositories {
             mavenLocal()
             maven { url "https://oss.sonatype.org/content/repositories/snapshots" }
@@ -122,7 +132,7 @@ allprojects {
 
 apply plugin: "binary-compatibility-validator"
 apiValidation {
-    ignoredProjects += ["stdlib-stubs", "benchmarks", "knit", "site",
+    ignoredProjects += ["stdlib-stubs", "benchmarks", "site",
                         "kotlinx-coroutines-bom", "android-unit-tests", "js-stub"]
     ignoredPackages += "kotlinx.coroutines.internal"
 }
@@ -136,9 +146,7 @@ allprojects {
          * google should be first in the repository list because some of the play services
          * transitive dependencies was removed from jcenter, thus breaking gradle dependency resolution
          */
-        if (projectName == "kotlinx-coroutines-play-services") {
-            google()
-        }
+        google()
         jcenter()
         maven {
             url "https://kotlin.bintray.com/kotlin-dev"
@@ -251,7 +259,7 @@ configure(subprojects.findAll { !unpublished.contains(it.name) }) {
             tasks.withType(dokka.getClass()) {
                 externalDocumentationLink {
                     url = new URL(core_docs_url)
-                    packageListUrl = new URL("file://$core_docs_file")
+                    packageListUrl = new File(core_docs_file).toURI().toURL()
                 }
             }
         }
@@ -267,3 +275,14 @@ task deploy(dependsOn: getTasksByName("publish", true) + getTasksByName("publish
 apply plugin: 'base'
 
 clean.dependsOn gradle.includedBuilds.collect { it.task(':clean') }
+
+// --------------- Knit configuration ---------------
+
+apply plugin: 'kotlinx-knit'
+
+knit {
+    siteRoot = "https://kotlin.github.io/kotlinx.coroutines"
+    moduleRoots = [".", "integration", "reactive", "ui"]
+}
+
+knitPrepare.dependsOn getTasksByName("dokka", true)

docs/basics.md

@@ -1,20 +1,4 @@
-<!--- INCLUDE .*/example-([a-z]+)-([0-9a-z]+)\.kt 
-/*
- * Copyright 2016-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
- */
-
-// This file was automatically generated from coroutines-guide.md by Knit tool. Do not edit.
-package kotlinx.coroutines.guide.$$1$$2
--->
-<!--- KNIT     ../kotlinx-coroutines-core/jvm/test/guide/.*\.kt -->
-<!--- TEST_OUT ../kotlinx-coroutines-core/jvm/test/guide/test/BasicsGuideTest.kt
-// This file was automatically generated from coroutines-guide.md by Knit tool. Do not edit.
-package kotlinx.coroutines.guide.test
-
-import org.junit.Test
-
-class BasicsGuideTest {
---> 
+<!--- TEST_NAME BasicsGuideTest -->
 
 **Table of contents**
 
@@ -30,8 +14,7 @@ class BasicsGuideTest {
   * [Coroutines ARE light-weight](#coroutines-are-light-weight)
   * [Global coroutines are like daemon threads](#global-coroutines-are-like-daemon-threads)
 
-<!--- END_TOC -->
-
+<!--- END -->
 
 ## Coroutine Basics
 
@@ -142,7 +125,7 @@ fun main() = runBlocking<Unit> { // start main coroutine
 
 </div>
 
-> You can get full code [here](../kotlinx-coroutines-core/jvm/test/guide/example-basic-02b.kt).
+> You can get full code [here](../kotlinx-coroutines-core/jvm/test/guide/example-basic-03.kt).
 
 <!--- TEST
 Hello,
@@ -197,7 +180,7 @@ fun main() = runBlocking {
 
 </div>
 
-> You can get full code [here](../kotlinx-coroutines-core/jvm/test/guide/example-basic-03.kt).
+> You can get full code [here](../kotlinx-coroutines-core/jvm/test/guide/example-basic-04.kt).
 
 <!--- TEST
 Hello,
@@ -242,7 +225,7 @@ fun main() = runBlocking { // this: CoroutineScope
 
 </div>
 
-> You can get full code [here](../kotlinx-coroutines-core/jvm/test/guide/example-basic-03s.kt).
+> You can get full code [here](../kotlinx-coroutines-core/jvm/test/guide/example-basic-05.kt).
 
 <!--- TEST
 Hello,
@@ -288,7 +271,7 @@ fun main() = runBlocking { // this: CoroutineScope
 
 </div>
 
-> You can get full code [here](../kotlinx-coroutines-core/jvm/test/guide/example-basic-04.kt).
+> You can get full code [here](../kotlinx-coroutines-core/jvm/test/guide/example-basic-06.kt).
 
 <!--- TEST
 Task from coroutine scope
@@ -327,7 +310,7 @@ suspend fun doWorld() {
 
 </div>
 
-> You can get full code [here](../kotlinx-coroutines-core/jvm/test/guide/example-basic-05.kt).
+> You can get full code [here](../kotlinx-coroutines-core/jvm/test/guide/example-basic-07.kt).
 
 <!--- TEST
 Hello,
@@ -364,7 +347,7 @@ fun main() = runBlocking {
 
 </div>
 
-> You can get full code [here](../kotlinx-coroutines-core/jvm/test/guide/example-basic-06.kt).
+> You can get full code [here](../kotlinx-coroutines-core/jvm/test/guide/example-basic-08.kt).
 
 <!--- TEST lines.size == 1 && lines[0] == ".".repeat(100_000) -->
 
@@ -396,7 +379,7 @@ fun main() = runBlocking {
 
 </div>
 
-> You can get full code [here](../kotlinx-coroutines-core/jvm/test/guide/example-basic-07.kt).
+> You can get full code [here](../kotlinx-coroutines-core/jvm/test/guide/example-basic-09.kt).
 
 You can run and see that it prints three lines and terminates:
 

docs/cancellation-and-timeouts.md

@@ -1,20 +1,5 @@
-<!--- INCLUDE .*/example-([a-z]+)-([0-9a-z]+)\.kt 
-/*
- * Copyright 2016-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
- */
+<!--- TEST_NAME CancellationGuideTest -->
 
-// This file was automatically generated from coroutines-guide.md by Knit tool. Do not edit.
-package kotlinx.coroutines.guide.$$1$$2
--->
-<!--- KNIT     ../kotlinx-coroutines-core/jvm/test/guide/.*\.kt -->
-<!--- TEST_OUT ../kotlinx-coroutines-core/jvm/test/guide/test/CancellationTimeOutsGuideTest.kt
-// This file was automatically generated from coroutines-guide.md by Knit tool. Do not edit.
-package kotlinx.coroutines.guide.test
-
-import org.junit.Test
-
-class CancellationTimeOutsGuideTest {
---> 
 **Table of contents**
 
 <!--- TOC -->
@@ -27,7 +12,7 @@ class CancellationTimeOutsGuideTest {
   * [Run non-cancellable block](#run-non-cancellable-block)
   * [Timeout](#timeout)
 
-<!--- END_TOC -->
+<!--- END -->
 
 ## Cancellation and Timeouts
 

docs/channels.md

@@ -1,20 +1,5 @@
-<!--- INCLUDE .*/example-([a-z]+)-([0-9a-z]+)\.kt 
-/*
- * Copyright 2016-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
- */
+<!--- TEST_NAME ChannelsGuideTest -->
 
-// This file was automatically generated from coroutines-guide.md by Knit tool. Do not edit.
-package kotlinx.coroutines.guide.$$1$$2
--->
-<!--- KNIT     ../kotlinx-coroutines-core/jvm/test/guide/.*\.kt -->
-<!--- TEST_OUT ../kotlinx-coroutines-core/jvm/test/guide/test/ChannelsGuideTest.kt
-// This file was automatically generated from coroutines-guide.md by Knit tool. Do not edit.
-package kotlinx.coroutines.guide.test
-
-import org.junit.Test
-
-class ChannelsGuideTest {
---> 
 **Table of contents**
 
 <!--- TOC -->
@@ -31,7 +16,7 @@ class ChannelsGuideTest {
   * [Channels are fair](#channels-are-fair)
   * [Ticker channels](#ticker-channels)
 
-<!--- END_TOC -->
+<!--- END -->
 
 ## Channels
 

docs/compatibility.md

@@ -1,9 +1,3 @@
-<!--- 
-/*
- * Copyright 2016-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
- */
----> 
-
 <!--- TOC -->
 
 * [Compatibility](#compatibility)
@@ -19,7 +13,7 @@
   * [Gradle](#gradle)
   * [Maven](#maven)
 
-<!--- END_TOC -->
+<!--- END -->
 
 ## Compatibility
 This document describes the compatibility policy of `kotlinx.coroutines` library since version 1.0.0 and semantics of compatibility-specific annotations.

docs/composing-suspending-functions.md

@@ -1,20 +1,4 @@
-<!--- INCLUDE .*/example-([a-z]+)-([0-9a-z]+)\.kt 
-/*
- * Copyright 2016-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
- */
-
-// This file was automatically generated from coroutines-guide.md by Knit tool. Do not edit.
-package kotlinx.coroutines.guide.$$1$$2
--->
-<!--- KNIT     ../kotlinx-coroutines-core/jvm/test/guide/.*\.kt -->
-<!--- TEST_OUT ../kotlinx-coroutines-core/jvm/test/guide/test/ComposingGuideTest.kt
-// This file was automatically generated from coroutines-guide.md by Knit tool. Do not edit.
-package kotlinx.coroutines.guide.test
-
-import org.junit.Test
-
-class ComposingGuideTest {
---> 
+<!--- TEST_NAME ComposingGuideTest -->
 
 **Table of contents**
 
@@ -27,7 +11,7 @@ class ComposingGuideTest {
   * [Async-style functions](#async-style-functions)
   * [Structured concurrency with async](#structured-concurrency-with-async)
 
-<!--- END_TOC -->
+<!--- END -->
 
 ## Composing Suspending Functions
 

docs/coroutine-context-and-dispatchers.md

@@ -1,20 +1,4 @@
-<!--- INCLUDE .*/example-([a-z]+)-([0-9a-z]+)\.kt 
-/*
- * Copyright 2016-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
- */
-
-// This file was automatically generated from coroutines-guide.md by Knit tool. Do not edit.
-package kotlinx.coroutines.guide.$$1$$2
--->
-<!--- KNIT     ../kotlinx-coroutines-core/jvm/test/guide/.*\.kt -->
-<!--- TEST_OUT ../kotlinx-coroutines-core/jvm/test/guide/test/DispatcherGuideTest.kt
-// This file was automatically generated from coroutines-guide.md by Knit tool. Do not edit.
-package kotlinx.coroutines.guide.test
-
-import org.junit.Test
-
-class DispatchersGuideTest {
---> 
+<!--- TEST_NAME DispatcherGuideTest -->
 
 **Table of contents**
 
@@ -33,7 +17,7 @@ class DispatchersGuideTest {
   * [Coroutine scope](#coroutine-scope)
   * [Thread-local data](#thread-local-data)
 
-<!--- END_TOC -->
+<!--- END -->
 
 ## Coroutine Context and Dispatchers
 

docs/debugging.md

@@ -9,8 +9,7 @@
 * [Debug agent](#debug-agent)
   * [Debug agent and Android](#debug-agent-and-android)
 
-<!--- END_TOC -->
-
+<!--- END -->
 
 ## Debugging coroutines
 Debugging asynchronous programs is challenging, because multiple concurrent coroutines are typically working at the same time.

docs/exception-handling.md

@@ -1,20 +1,5 @@
-<!--- INCLUDE .*/example-([a-z]+)-([0-9a-z]+)\.kt 
-/*
- * Copyright 2016-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
- */
+<!--- TEST_NAME ExceptionsGuideTest -->
 
-// This file was automatically generated from coroutines-guide.md by Knit tool. Do not edit.
-package kotlinx.coroutines.guide.$$1$$2
--->
-<!--- KNIT     ../kotlinx-coroutines-core/jvm/test/guide/.*\.kt -->
-<!--- TEST_OUT ../kotlinx-coroutines-core/jvm/test/guide/test/ExceptionsGuideTest.kt
-// This file was automatically generated from coroutines-guide.md by Knit tool. Do not edit.
-package kotlinx.coroutines.guide.test
-
-import org.junit.Test
-
-class ExceptionsGuideTest {
---> 
 **Table of contents**
 
 <!--- TOC -->
@@ -29,7 +14,7 @@ class ExceptionsGuideTest {
     * [Supervision scope](#supervision-scope)
     * [Exceptions in supervised coroutines](#exceptions-in-supervised-coroutines)
 
-<!--- END_TOC -->
+<!--- END -->
 
 ## Exception Handling
 
@@ -260,7 +245,6 @@ to leak to its exception handler.
 
 
 <!--- INCLUDE
-
 import kotlinx.coroutines.exceptions.*
 -->