From e04fc22ce546f00be2288c79ac86330e9af58283 Mon Sep 17 00:00:00 2001 From: Tyson Henning Date: Tue, 25 Jun 2019 13:50:16 -0700 Subject: [PATCH 01/24] Added CoroutineScope.asExecutor() to ListenableFuture.kt, and used it in place of directExecutor(). --- .../src/ListenableFuture.kt | 38 +++++++++++++++++-- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/integration/kotlinx-coroutines-guava/src/ListenableFuture.kt b/integration/kotlinx-coroutines-guava/src/ListenableFuture.kt index a924bb4aa3..4faea5bbe7 100644 --- a/integration/kotlinx-coroutines-guava/src/ListenableFuture.kt +++ b/integration/kotlinx-coroutines-guava/src/ListenableFuture.kt @@ -40,7 +40,7 @@ public fun CoroutineScope.future( val newContext = newCoroutineContext(context) val future = SettableFuture.create() val coroutine = ListenableFutureCoroutine(newContext, future) - Futures.addCallback(future, coroutine, MoreExecutors.directExecutor()) + Futures.addCallback(future, coroutine, CoroutineScope(newContext).asExecutor()) coroutine.start(start, coroutine, block) return future } @@ -122,7 +122,7 @@ public fun ListenableFuture.asDeferred(): Deferred { override fun onFailure(t: Throwable) { deferred.completeExceptionally(t) } - }, MoreExecutors.directExecutor()) + }, CoroutineScope(deferred).asExecutor()) deferred.invokeOnCompletion { cancel(false) } return deferred @@ -147,7 +147,7 @@ public suspend fun ListenableFuture.await(): T { return suspendCancellableCoroutine { cont: CancellableContinuation -> val callback = ContinuationCallback(cont) - Futures.addCallback(this, callback, MoreExecutors.directExecutor()) + Futures.addCallback(this, callback, CoroutineScope(cont.context).asExecutor()) cont.invokeOnCancellation { cancel(false) callback.cont = null // clear the reference to continuation from the future's callback @@ -162,3 +162,35 @@ private class ContinuationCallback( override fun onSuccess(result: T?) { cont?.resume(result as T) } override fun onFailure(t: Throwable) { cont?.resumeWithException(t) } } + +/** + * Returns an [Executor] that uses the [CoroutineScope] to run executed Runnables. + * + * Using this Executor ensures inputs pass through coroutine dispatch and + * continuation. + * + * This Executor can be used in place of + * [com.google.common.util.concurrent.MoreExecutors.directExecutor]. Using `directExecutor()` to + * schedule a [FutureCallback] has the negative property of non-deterministic execution scope: + * + * 1. If the [FutureCallback] is attached to a completed [ListenableFuture], the scheduling thread + * will execute the callback inline, reentering from the call stack. + * 2. If a [FutureCallback] is attached to an incomplete [ListenableFuture], the thread that + * completes the [ListenableFuture] will call the callback, and won't reenter from the call stack + * call stack. + * + * Coroutines (barring use of Start.UNDISPATCHED...) *don't* reenter on the stack when running a + * continuation, avoiding an entire class of bugs by design. + * + * Using this Executor instead of a `directExecutor()` avoids reintroducing + * non-deterministic-stack-reentrancy bugs to Coroutine-centric code. + */ +inline fun CoroutineScope.asExecutor(): Executor { + return Executor { + checkNotNull(it) + this.async { + it.run() + } + } +} + From 702139f2dc8f1642d537481c7a399b2c9fe5c157 Mon Sep 17 00:00:00 2001 From: Tyson Henning Date: Tue, 3 Sep 2019 16:36:40 -0700 Subject: [PATCH 02/24] Hid the SettableFuture of future {}. This should prevent successful casts to type SettableFuture, meaning client code can't access and complete the internal Future without resorting to reflection.. --- integration/kotlinx-coroutines-guava/src/ListenableFuture.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration/kotlinx-coroutines-guava/src/ListenableFuture.kt b/integration/kotlinx-coroutines-guava/src/ListenableFuture.kt index d247e82ca5..e502ff464f 100644 --- a/integration/kotlinx-coroutines-guava/src/ListenableFuture.kt +++ b/integration/kotlinx-coroutines-guava/src/ListenableFuture.kt @@ -46,14 +46,14 @@ public fun CoroutineScope.future( ): ListenableFuture { require(!start.isLazy) { "$start start is not supported" } val newContext = newCoroutineContext(context) - // TODO: It'd be nice not to leak this SettableFuture reference, which is easily blind-cast. val future = SettableFuture.create() val coroutine = ListenableFutureCoroutine(newContext, future) future.addListener( coroutine, MoreExecutors.directExecutor()) coroutine.start(start, coroutine, block) - return future + // Return hides the SettableFuture. This should prevent casting. + return object: ListenableFuture by future {} } /** From 85986c4ed381171da286fb9eb50f59714f44cdfa Mon Sep 17 00:00:00 2001 From: Tyson Henning Date: Tue, 14 Sep 2021 18:23:22 +0000 Subject: [PATCH 03/24] Implemented ThreadContextElement.copyForChildCoroutine() --- .../jvm/src/CoroutineContext.kt | 22 ++- .../jvm/src/ThreadContextElement.kt | 12 ++ .../jvm/test/ThreadContextElementTest.kt | 130 +++++++++++++++++- 3 files changed, 162 insertions(+), 2 deletions(-) diff --git a/kotlinx-coroutines-core/jvm/src/CoroutineContext.kt b/kotlinx-coroutines-core/jvm/src/CoroutineContext.kt index 702a0d83ea..4ff1274373 100644 --- a/kotlinx-coroutines-core/jvm/src/CoroutineContext.kt +++ b/kotlinx-coroutines-core/jvm/src/CoroutineContext.kt @@ -17,12 +17,32 @@ import kotlin.coroutines.jvm.internal.CoroutineStackFrame */ @ExperimentalCoroutinesApi public actual fun CoroutineScope.newCoroutineContext(context: CoroutineContext): CoroutineContext { - val combined = coroutineContext + context + val combined = coroutineContext.foldCopiesForChildCoroutine() + context val debug = if (DEBUG) combined + CoroutineId(COROUTINE_ID.incrementAndGet()) else combined return if (combined !== Dispatchers.Default && combined[ContinuationInterceptor] == null) debug + Dispatchers.Default else debug } +/** + * Returns a new [ThreadContextElement.copyForChildCoroutine] on each [ThreadContextElement] + * [ThreadContextElement], returning a new `CoroutineContext` with those values incorporated. + * + * Returns an equivalent but not reference-equal [CoroutineContext] if [this] has one or more + * [ThreadContextElement] in it, and no [ThreadContextElement] overrides + * [ThreadContextElement.copyForChildCoroutine]. + * + * Returns [this] if `this` has no [ThreadContextElement] in it. + */ +private fun CoroutineContext.foldCopiesForChildCoroutine(): CoroutineContext { + val jobElementCount = fold(0) { count, it -> + count + if (it is ThreadContextElement<*>) 1 else 0 + } + if (jobElementCount == 0) return this + return fold(EmptyCoroutineContext) { combined, it -> + combined + if (it is ThreadContextElement<*>) it.copyForChildCoroutine() else it + } +} + /** * Executes a block using a given coroutine context. */ diff --git a/kotlinx-coroutines-core/jvm/src/ThreadContextElement.kt b/kotlinx-coroutines-core/jvm/src/ThreadContextElement.kt index 37fd70a23e..3d3934d146 100644 --- a/kotlinx-coroutines-core/jvm/src/ThreadContextElement.kt +++ b/kotlinx-coroutines-core/jvm/src/ThreadContextElement.kt @@ -75,6 +75,18 @@ public interface ThreadContextElement : CoroutineContext.Element { * @param oldState the value returned by the previous invocation of [updateThreadContext]. */ public fun restoreThreadContext(context: CoroutineContext, oldState: S) + + /** + * Returns a [ThreadContextElement] to use in place of `this` ThreadContextElement in a child + * coroutine when `this` ThreadContextElement is inherited. + * + * Implement this method to return a new instance of this `ThreadContextElement` if access to it should be + * isolated to a single coroutine. + * + * Since this method is called whenever a new coroutine is launched in a context containing this + * `ThreadContextElement`, implementations of this method are performance-sensitive. + */ + public fun copyForChildCoroutine(): ThreadContextElement = this // default impl does not copy } /** diff --git a/kotlinx-coroutines-core/jvm/test/ThreadContextElementTest.kt b/kotlinx-coroutines-core/jvm/test/ThreadContextElementTest.kt index ea43c7ade2..6a818e0829 100644 --- a/kotlinx-coroutines-core/jvm/test/ThreadContextElementTest.kt +++ b/kotlinx-coroutines-core/jvm/test/ThreadContextElementTest.kt @@ -54,7 +54,6 @@ class ThreadContextElementTest : TestBase() { assertNull(myThreadLocal.get()) } - @Test fun testWithContext() = runTest { expect(1) @@ -86,6 +85,78 @@ class ThreadContextElementTest : TestBase() { finish(7) } + + @Test + fun testNonCopyableElementReferenceInheritedOnLaunch() = runTest { + var parentElement: MyElement? = null + var inheritedElement: MyElement? = null + + newSingleThreadContext("withContext").use { + withContext(it + MyElement(MyData())) { + parentElement = coroutineContext[MyElement.Key] + launch { + inheritedElement = coroutineContext[MyElement.Key] + } + } + } + + assertSame(inheritedElement, parentElement, + "Inner and outer coroutines did not have the same object reference to a" + + " ThreadContextElement that did not override `copyForChildCoroutine()`") + } + + @Test + fun testCopyableElementCopiedOnLaunch() = runTest { + var parentElement: MyElement? = null + var inheritedElement: MyElement? = null + + newSingleThreadContext("withContext").use { + withContext(it + CopyForChildCoroutineElement(MyData())) { + parentElement = coroutineContext[MyElement.Key] + launch { + inheritedElement = coroutineContext[MyElement.Key] + } + } + } + + assertNotSame(inheritedElement, parentElement, + "Inner coroutine did not copy its copyable ThreadContextElement.") + } + + @Test + fun testCopyableThreadContextElementImplementsWriteVisibility() = runTest { + newFixedThreadPoolContext(nThreads = 4, name = "withContext").use { + val startData = MyData() + withContext(it + CopyForChildCoroutineElement(startData)) { + val forBlockData = MyData() + myThreadLocal.setForBlock(forBlockData) { + assertSame(myThreadLocal.get(), forBlockData) + launch { + assertSame(myThreadLocal.get(), forBlockData) + } + launch { + assertSame(myThreadLocal.get(), forBlockData) + // Modify value in child coroutine. Writes to the ThreadLocal and + // the (copied) ThreadLocalElement's memory are not visible to peer or + // ancestor coroutines, so this write is both threadsafe and coroutinesafe. + val innerCoroutineData = MyData() + myThreadLocal.setForBlock(innerCoroutineData) { + assertSame(myThreadLocal.get(), innerCoroutineData) + } + assertSame(myThreadLocal.get(), forBlockData) // Asserts value was restored. + } + launch { + val innerCoroutineData = MyData() + myThreadLocal.setForBlock(innerCoroutineData) { + assertSame(myThreadLocal.get(), innerCoroutineData) + } + assertSame(myThreadLocal.get(), forBlockData) + } + } + assertSame(myThreadLocal.get(), startData) // Asserts value was restored. + } + } + } } class MyData @@ -114,3 +185,60 @@ class MyElement(val data: MyData) : ThreadContextElement { myThreadLocal.set(oldState) } } + + +/** + * A [ThreadContextElement] that implements copy semantics in [copyForChildCoroutine]. + */ +class CopyForChildCoroutineElement(val data: MyData?) : ThreadContextElement { + companion object Key : CoroutineContext.Key + + override val key: CoroutineContext.Key + get() = Key + + override fun updateThreadContext(context: CoroutineContext): MyData? { + val oldState = myThreadLocal.get() + myThreadLocal.set(data) + return oldState + } + + override fun restoreThreadContext(context: CoroutineContext, oldState: MyData?) { + myThreadLocal.set(oldState) + } + + /** + * At coroutine launch time, the _current value of the ThreadLocal_ is inherited by the new + * child coroutine, and that value is copied to a new, unique, ThreadContextElement memory + * reference for the child coroutine to use uniquely. + * + * n.b. the value copied to the child must be the __current value of the ThreadLocal__ and not + * the value initially passed to the ThreadContextElement in order to reflect writes made to the + * ThreadLocal between coroutine resumption and the child coroutine launch point. Those writes + * will be reflected in the parent coroutine's [CopyForChildCoroutineElement] when it yields the + * thread and calls [restoreThreadContext]. + */ + override fun copyForChildCoroutine(): ThreadContextElement { + return CopyForChildCoroutineElement(myThreadLocal.get()) + } +} + +/** + * Calls [block], setting the value of [this] [ThreadLocal] for the duration of [block]. + * + * When a [CopyForChildCoroutineElement] for `this` [ThreadLocal] is used within a + * [CoroutineContext], a ThreadLocal set this way will have the "correct" value expected lexically + * at every statement reached, whether that statement is reached immediately, across suspend and + * redispatch within one coroutine, or within a child coroutine. Writes made to the `ThreadLocal` + * by child coroutines will not be visible to the parent coroutine. Writes made to the `ThreadLocal` + * by the parent coroutine _after_ launching a child coroutine will not be visible to that child + * coroutine. + */ +private inline fun ThreadLocal.setForBlock( + value: ThreadLocalT, + crossinline block: () -> OutputT +) { + val priorValue = get() + set(value) + block() + set(priorValue) +} \ No newline at end of file From 07a0ab5ee35807cffc5912975465181f372146f3 Mon Sep 17 00:00:00 2001 From: Tyson Henning Date: Tue, 14 Sep 2021 19:09:23 +0000 Subject: [PATCH 04/24] Moved `copyForChildCoroutine()` to a `CopyableThreadContextElement`. This allows contexts using non-copyable `ThreadContextElements` to inherit a reference-equal context without constructing a new one. --- .../jvm/src/CoroutineContext.kt | 15 +++-- .../jvm/src/ThreadContextElement.kt | 59 +++++++++++++++++-- .../jvm/test/ThreadContextElementTest.kt | 4 +- 3 files changed, 62 insertions(+), 16 deletions(-) diff --git a/kotlinx-coroutines-core/jvm/src/CoroutineContext.kt b/kotlinx-coroutines-core/jvm/src/CoroutineContext.kt index 4ff1274373..36fd2b83dc 100644 --- a/kotlinx-coroutines-core/jvm/src/CoroutineContext.kt +++ b/kotlinx-coroutines-core/jvm/src/CoroutineContext.kt @@ -24,22 +24,21 @@ public actual fun CoroutineScope.newCoroutineContext(context: CoroutineContext): } /** - * Returns a new [ThreadContextElement.copyForChildCoroutine] on each [ThreadContextElement] - * [ThreadContextElement], returning a new `CoroutineContext` with those values incorporated. + * Returns the [CoroutineContext] for a child coroutine to inherit. * - * Returns an equivalent but not reference-equal [CoroutineContext] if [this] has one or more - * [ThreadContextElement] in it, and no [ThreadContextElement] overrides - * [ThreadContextElement.copyForChildCoroutine]. + * If any [CopyableThreadContextElement] is in the [this], calls + * [CopyableThreadContextElement.copyForChildCoroutine] on each, returning a new [CoroutineContext] + * by folding the returned copied elements into [this]. * - * Returns [this] if `this` has no [ThreadContextElement] in it. + * Returns [this] if `this` has zero [CopyableThreadContextElement] in it. */ private fun CoroutineContext.foldCopiesForChildCoroutine(): CoroutineContext { val jobElementCount = fold(0) { count, it -> - count + if (it is ThreadContextElement<*>) 1 else 0 + count + if (it is CopyableThreadContextElement<*>) 1 else 0 } if (jobElementCount == 0) return this return fold(EmptyCoroutineContext) { combined, it -> - combined + if (it is ThreadContextElement<*>) it.copyForChildCoroutine() else it + combined + if (it is CopyableThreadContextElement<*>) it.copyForChildCoroutine() else it } } diff --git a/kotlinx-coroutines-core/jvm/src/ThreadContextElement.kt b/kotlinx-coroutines-core/jvm/src/ThreadContextElement.kt index 3d3934d146..569dc939a6 100644 --- a/kotlinx-coroutines-core/jvm/src/ThreadContextElement.kt +++ b/kotlinx-coroutines-core/jvm/src/ThreadContextElement.kt @@ -75,18 +75,65 @@ public interface ThreadContextElement : CoroutineContext.Element { * @param oldState the value returned by the previous invocation of [updateThreadContext]. */ public fun restoreThreadContext(context: CoroutineContext, oldState: S) +} + +/** + * A [ThreadContextElement] copied whenever a child coroutine inherits a context containing it. + * + * When an API uses a _mutable_ `ThreadLocal` for consistency, a [CopyableThreadContextElement] + * can give coroutines "coroutine-safe" write access to that `ThreadLocal`. + * + * A write made to a `ThreadLocal` with a matching [CopyableThreadContextElement] by a coroutine + * will be visible to _itself_ and any child coroutine launched _after_ that write. + * + * Writes will not be visible to the parent coroutine, peer coroutines, or coroutines that happen + * to use the same thread. Writes made to the `ThreadLocal` by the parent coroutine _after_ + * launching a child coroutine will not be visible to that child coroutine. + * + * This can be used to allow a coroutine to use a mutable ThreadLocal API transparently and + * correctly, regardless of the coroutine's structured concurrency. + * + * ``` + * class TraceContextElement(val traceData: TraceData?) : CopyableThreadContextElement { + * companion object Key : CoroutineContext.Key + * override val key: CoroutineContext.Key + * get() = Key + * + * override fun updateThreadContext(context: CoroutineContext): TraceData? { + * val oldState = traceThreadLocal.get() + * traceThreadLocal.set(data) + * return oldState + * } + * + * override fun restoreThreadContext(context: CoroutineContext, oldData: TraceData?) { + * traceThreadLocal.set(oldState) + * } + * + * override fun copyForChildCoroutine(): CopyableThreadContextElement { + * // Copy from the ThreadLocal source of truth at child coroutine launch time. This makes + * // writes between resumption of the parent coroutine and the launch of the child + * // coroutine visible to the child. + * return CopyForChildCoroutineElement(traceThreadLocal.get()) + * } + * } + * ``` + * + * A coroutine using this mechanism can safely call Java code that assumes it's called using a + * `Thread`. + */ +public interface CopyableThreadContextElement : ThreadContextElement { /** - * Returns a [ThreadContextElement] to use in place of `this` ThreadContextElement in a child - * coroutine when `this` ThreadContextElement is inherited. + * Returns a [CopyableThreadContextElement] to replace of `this` `CopyableThreadContextElement` in the child + * coroutine's context that is under construction. * - * Implement this method to return a new instance of this `ThreadContextElement` if access to it should be - * isolated to a single coroutine. + * This function is called on the element each time a new coroutine inherits a context containing it, + * and the returned value is folded into the context for the child. * * Since this method is called whenever a new coroutine is launched in a context containing this - * `ThreadContextElement`, implementations of this method are performance-sensitive. + * [CopyableThreadContextElement], implementations are performance-sensitive. */ - public fun copyForChildCoroutine(): ThreadContextElement = this // default impl does not copy + public fun copyForChildCoroutine(): CopyableThreadContextElement } /** diff --git a/kotlinx-coroutines-core/jvm/test/ThreadContextElementTest.kt b/kotlinx-coroutines-core/jvm/test/ThreadContextElementTest.kt index 6a818e0829..e8a32cd32d 100644 --- a/kotlinx-coroutines-core/jvm/test/ThreadContextElementTest.kt +++ b/kotlinx-coroutines-core/jvm/test/ThreadContextElementTest.kt @@ -190,7 +190,7 @@ class MyElement(val data: MyData) : ThreadContextElement { /** * A [ThreadContextElement] that implements copy semantics in [copyForChildCoroutine]. */ -class CopyForChildCoroutineElement(val data: MyData?) : ThreadContextElement { +class CopyForChildCoroutineElement(val data: MyData?) : CopyableThreadContextElement { companion object Key : CoroutineContext.Key override val key: CoroutineContext.Key @@ -217,7 +217,7 @@ class CopyForChildCoroutineElement(val data: MyData?) : ThreadContextElement { + override fun copyForChildCoroutine(): CopyableThreadContextElement { return CopyForChildCoroutineElement(myThreadLocal.get()) } } From fe562f7c2cbe86aed5e8e64df2af82f89dabea9d Mon Sep 17 00:00:00 2001 From: Tyson Henning Date: Tue, 14 Sep 2021 19:09:23 +0000 Subject: [PATCH 05/24] Moved `copyForChildCoroutine()` to a `CopyableThreadContextElement`. This allows contexts using non-copyable `ThreadContextElements` to inherit a reference-equal context without constructing a new one. --- kotlinx-coroutines-core/jvm/test/ThreadContextElementTest.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kotlinx-coroutines-core/jvm/test/ThreadContextElementTest.kt b/kotlinx-coroutines-core/jvm/test/ThreadContextElementTest.kt index e8a32cd32d..f33a9e9980 100644 --- a/kotlinx-coroutines-core/jvm/test/ThreadContextElementTest.kt +++ b/kotlinx-coroutines-core/jvm/test/ThreadContextElementTest.kt @@ -241,4 +241,5 @@ private inline fun ThreadLocal.setForBlock set(value) block() set(priorValue) -} \ No newline at end of file +} + From 6a799b338d49479ad26598ec67387a3d82da9887 Mon Sep 17 00:00:00 2001 From: Tyson Henning Date: Tue, 14 Sep 2021 19:09:23 +0000 Subject: [PATCH 06/24] Moved `copyForChildCoroutine()` to a `CopyableThreadContextElement`. This allows contexts using non-copyable `ThreadContextElements` to inherit a reference-equal context without constructing a new one. --- kotlinx-coroutines-core/jvm/test/ThreadContextElementTest.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/kotlinx-coroutines-core/jvm/test/ThreadContextElementTest.kt b/kotlinx-coroutines-core/jvm/test/ThreadContextElementTest.kt index f33a9e9980..ae3a18ba70 100644 --- a/kotlinx-coroutines-core/jvm/test/ThreadContextElementTest.kt +++ b/kotlinx-coroutines-core/jvm/test/ThreadContextElementTest.kt @@ -186,7 +186,6 @@ class MyElement(val data: MyData) : ThreadContextElement { } } - /** * A [ThreadContextElement] that implements copy semantics in [copyForChildCoroutine]. */ From 9e2f622a07f11a98a2327bb14f5d0bae61eb6aa9 Mon Sep 17 00:00:00 2001 From: Tyson Henning Date: Tue, 14 Sep 2021 19:09:23 +0000 Subject: [PATCH 07/24] Moved `copyForChildCoroutine()` to a `CopyableThreadContextElement`. This allows contexts using non-copyable `ThreadContextElements` to inherit a reference-equal context without constructing a new one. --- .../jvm/src/ThreadContextElement.kt | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/kotlinx-coroutines-core/jvm/src/ThreadContextElement.kt b/kotlinx-coroutines-core/jvm/src/ThreadContextElement.kt index 569dc939a6..3da5b107da 100644 --- a/kotlinx-coroutines-core/jvm/src/ThreadContextElement.kt +++ b/kotlinx-coroutines-core/jvm/src/ThreadContextElement.kt @@ -93,6 +93,9 @@ public interface ThreadContextElement : CoroutineContext.Element { * This can be used to allow a coroutine to use a mutable ThreadLocal API transparently and * correctly, regardless of the coroutine's structured concurrency. * + * This example adapts a `ThreadLocal` method trace to be "coroutine local" while the method trace + * is in a coroutine: + * * ``` * class TraceContextElement(val traceData: TraceData?) : CopyableThreadContextElement { * companion object Key : CoroutineContext.Key @@ -111,8 +114,8 @@ public interface ThreadContextElement : CoroutineContext.Element { * * override fun copyForChildCoroutine(): CopyableThreadContextElement { * // Copy from the ThreadLocal source of truth at child coroutine launch time. This makes - * // writes between resumption of the parent coroutine and the launch of the child - * // coroutine visible to the child. + * // ThreadLocal writes between resumption of the parent coroutine and the launch of the + * // child coroutine visible to the child. * return CopyForChildCoroutineElement(traceThreadLocal.get()) * } * } @@ -124,11 +127,11 @@ public interface ThreadContextElement : CoroutineContext.Element { public interface CopyableThreadContextElement : ThreadContextElement { /** - * Returns a [CopyableThreadContextElement] to replace of `this` `CopyableThreadContextElement` in the child + * Returns a [CopyableThreadContextElement] to replace `this` `CopyableThreadContextElement` in the child * coroutine's context that is under construction. * * This function is called on the element each time a new coroutine inherits a context containing it, - * and the returned value is folded into the context for the child. + * and the returned value is folded into the context given to the child. * * Since this method is called whenever a new coroutine is launched in a context containing this * [CopyableThreadContextElement], implementations are performance-sensitive. From 1f1e8a3257c5eef4af21eca95a5554258022d80b Mon Sep 17 00:00:00 2001 From: Vsevolod Tolstopyatov Date: Sat, 11 Sep 2021 00:33:28 +0300 Subject: [PATCH 08/24] Fix benchmarks compilation after migration to Kotlin 1.5.30 --- .../kotlin/benchmarks/flow/scrabble/SequencePlaysScrabble.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/src/jmh/kotlin/benchmarks/flow/scrabble/SequencePlaysScrabble.kt b/benchmarks/src/jmh/kotlin/benchmarks/flow/scrabble/SequencePlaysScrabble.kt index e7bd1f5fb9..acfb3f3c6d 100644 --- a/benchmarks/src/jmh/kotlin/benchmarks/flow/scrabble/SequencePlaysScrabble.kt +++ b/benchmarks/src/jmh/kotlin/benchmarks/flow/scrabble/SequencePlaysScrabble.kt @@ -30,7 +30,7 @@ open class SequencePlaysScrabble : ShakespearePlaysScrabble() { val bonusForDoubleLetter: (String) -> Int = { word: String -> toBeMaxed(word) .map { letterScores[it - 'a'.toInt()] } - .max()!! + .maxOrNull()!! } val score3: (String) -> Int = { word: String -> From c69b75c5e2336d6b70bbc6bc1188001adda87c87 Mon Sep 17 00:00:00 2001 From: Nikita Nazarov Date: Mon, 20 Sep 2021 10:41:26 +0300 Subject: [PATCH 09/24] Add method that allows IDEA debugger to retrieve enhanced stack trace in a JSON format (#2933) --- .../jvm/src/debug/internal/DebugProbesImpl.kt | 35 ++++++++++--- ...nhanceStackTraceWithTreadDumpAsJsonTest.kt | 49 +++++++++++++++++++ 2 files changed, 78 insertions(+), 6 deletions(-) create mode 100644 kotlinx-coroutines-debug/test/EnhanceStackTraceWithTreadDumpAsJsonTest.kt diff --git a/kotlinx-coroutines-core/jvm/src/debug/internal/DebugProbesImpl.kt b/kotlinx-coroutines-core/jvm/src/debug/internal/DebugProbesImpl.kt index f0fc50e658..e6cddbca1c 100644 --- a/kotlinx-coroutines-core/jvm/src/debug/internal/DebugProbesImpl.kt +++ b/kotlinx-coroutines-core/jvm/src/debug/internal/DebugProbesImpl.kt @@ -183,7 +183,6 @@ internal object DebugProbesImpl { */ @OptIn(ExperimentalStdlibApi::class) public fun dumpCoroutinesInfoAsJsonAndReferences(): Array { - fun Any.toStringWithQuotes() = "\"$this\"" val coroutinesInfo = dumpCoroutinesInfo() val size = coroutinesInfo.size val lastObservedThreads = ArrayList(size) @@ -196,11 +195,11 @@ internal object DebugProbesImpl { coroutinesInfoAsJson.add( """ { - "name": $name, - "id": ${context[CoroutineId.Key]?.id}, - "dispatcher": $dispatcher, - "sequenceNumber": ${info.sequenceNumber}, - "state": "${info.state}" + "name": $name, + "id": ${context[CoroutineId.Key]?.id}, + "dispatcher": $dispatcher, + "sequenceNumber": ${info.sequenceNumber}, + "state": "${info.state}" } """.trimIndent() ) @@ -216,6 +215,30 @@ internal object DebugProbesImpl { ) } + /* + * Internal (JVM-public) method used by IDEA debugger as of 1.6.0-RC. + */ + public fun enhanceStackTraceWithThreadDumpAsJson(info: DebugCoroutineInfo): String { + val stackTraceElements = enhanceStackTraceWithThreadDump(info, info.lastObservedStackTrace) + val stackTraceElementsInfoAsJson = mutableListOf() + for (element in stackTraceElements) { + stackTraceElementsInfoAsJson.add( + """ + { + "declaringClass": "${element.className}", + "methodName": "${element.methodName}", + "fileName": ${element.fileName?.toStringWithQuotes()}, + "lineNumber": ${element.lineNumber} + } + """.trimIndent() + ) + } + + return "[${stackTraceElementsInfoAsJson.joinToString()}]" + } + + private fun Any.toStringWithQuotes() = "\"$this\"" + /* * Internal (JVM-public) method used by IDEA debugger as of 1.4-M3. */ diff --git a/kotlinx-coroutines-debug/test/EnhanceStackTraceWithTreadDumpAsJsonTest.kt b/kotlinx-coroutines-debug/test/EnhanceStackTraceWithTreadDumpAsJsonTest.kt new file mode 100644 index 0000000000..fcf9f1a9a9 --- /dev/null +++ b/kotlinx-coroutines-debug/test/EnhanceStackTraceWithTreadDumpAsJsonTest.kt @@ -0,0 +1,49 @@ +/* + * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ +@file:Suppress("INVISIBLE_MEMBER", "INVISIBLE_REFERENCE") +package kotlinx.coroutines.debug + +import com.google.gson.* +import kotlinx.coroutines.* +import kotlinx.coroutines.debug.internal.* +import org.junit.Test +import kotlin.test.* + +class EnhanceStackTraceWithTreadDumpAsJsonTest : DebugTestBase() { + private data class StackTraceElementInfoFromJson( + val declaringClass: String, + val methodName: String, + val fileName: String?, + val lineNumber: Int + ) + + @Test + fun testEnhancedStackTraceFormatWithDeferred() = runTest { + val deferred = async { + suspendingMethod() + assertTrue(true) + } + yield() + + val coroutineInfo = DebugProbesImpl.dumpCoroutinesInfo() + assertEquals(coroutineInfo.size, 2) + val info = coroutineInfo[1] + val enhancedStackTraceAsJsonString = DebugProbesImpl.enhanceStackTraceWithThreadDumpAsJson(info) + val enhancedStackTraceFromJson = Gson().fromJson(enhancedStackTraceAsJsonString, Array::class.java) + val enhancedStackTrace = DebugProbesImpl.enhanceStackTraceWithThreadDump(info, info.lastObservedStackTrace) + assertEquals(enhancedStackTrace.size, enhancedStackTraceFromJson.size) + for ((frame, frameFromJson) in enhancedStackTrace.zip(enhancedStackTraceFromJson)) { + assertEquals(frame.className, frameFromJson.declaringClass) + assertEquals(frame.methodName, frameFromJson.methodName) + assertEquals(frame.fileName, frameFromJson.fileName) + assertEquals(frame.lineNumber, frameFromJson.lineNumber) + } + + deferred.cancelAndJoin() + } + + private suspend fun suspendingMethod() { + delay(Long.MAX_VALUE) + } +} From 91188eaac29d04ea03f7317138d0dada82406ac6 Mon Sep 17 00:00:00 2001 From: Vsevolod Tolstopyatov Date: Tue, 21 Sep 2021 14:50:00 +0300 Subject: [PATCH 10/24] Update license year in NOTICE.txt Fixes #2934 --- license/NOTICE.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/license/NOTICE.txt b/license/NOTICE.txt index d1d00c1a87..8d1100a3a5 100644 --- a/license/NOTICE.txt +++ b/license/NOTICE.txt @@ -5,4 +5,4 @@ ========================================================================= kotlinx.coroutines library. -Copyright 2016-2019 JetBrains s.r.o and respective authors and developers \ No newline at end of file +Copyright 2016-2021 JetBrains s.r.o and respective authors and developers From 83c523ffc07d861bed134f6c6f9530b0a0f45394 Mon Sep 17 00:00:00 2001 From: Vsevolod Tolstopyatov Date: Thu, 23 Sep 2021 19:19:31 +0300 Subject: [PATCH 11/24] Promote newSingleThreadContext and newFixedThreadPoolContext to delicate API (#2922) * Mention CoroutineDispatcher.limitedParallelism as an intended replacement * Prepare the API to sharing with K/N new memory model where we _have_ to have this API Addresses #2919 --- .../jvm/src/ThreadPoolDispatcher.kt | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/kotlinx-coroutines-core/jvm/src/ThreadPoolDispatcher.kt b/kotlinx-coroutines-core/jvm/src/ThreadPoolDispatcher.kt index 99e3b46cce..32df8bfcdf 100644 --- a/kotlinx-coroutines-core/jvm/src/ThreadPoolDispatcher.kt +++ b/kotlinx-coroutines-core/jvm/src/ThreadPoolDispatcher.kt @@ -17,11 +17,12 @@ import java.util.concurrent.atomic.AtomicInteger * then the [Job] of the affected task is [cancelled][Job.cancel] and the task is submitted to the * [Dispatchers.IO], so that the affected coroutine can cleanup its resources and promptly complete. * - * **NOTE: This API will be replaced in the future**. A different API to create thread-limited thread pools - * that is based on a shared thread-pool and does not require the resulting dispatcher to be explicitly closed - * will be provided, thus avoiding potential thread leaks and also significantly improving performance, due - * to coroutine-oriented scheduling policy and thread-switch minimization. - * See [issue #261](https://github.com/Kotlin/kotlinx.coroutines/issues/261) for details. + * This is a **delicate** API. The result of this method is a closeable resource with the + * associated native resources (threads). It should not be allocated in place, + * should be closed at the end of its lifecycle, and has non-trivial memory and CPU footprint. + * If you do not need a separate thread-pool, but only have to limit effective parallelism of the dispatcher, + * it is recommended to use [CoroutineDispatcher.limitedParallelism] instead. + * * If you need a completely separate thread-pool with scheduling policy that is based on the standard * JDK executors, use the following expression: * `Executors.newSingleThreadExecutor().asCoroutineDispatcher()`. @@ -29,7 +30,7 @@ import java.util.concurrent.atomic.AtomicInteger * * @param name the base name of the created thread. */ -@ObsoleteCoroutinesApi +@DelicateCoroutinesApi public fun newSingleThreadContext(name: String): ExecutorCoroutineDispatcher = newFixedThreadPoolContext(1, name) @@ -43,11 +44,12 @@ public fun newSingleThreadContext(name: String): ExecutorCoroutineDispatcher = * then the [Job] of the affected task is [cancelled][Job.cancel] and the task is submitted to the * [Dispatchers.IO], so that the affected coroutine can cleanup its resources and promptly complete. * - * **NOTE: This API will be replaced in the future**. A different API to create thread-limited thread pools - * that is based on a shared thread-pool and does not require the resulting dispatcher to be explicitly closed - * will be provided, thus avoiding potential thread leaks and also significantly improving performance, due - * to coroutine-oriented scheduling policy and thread-switch minimization. - * See [issue #261](https://github.com/Kotlin/kotlinx.coroutines/issues/261) for details. + * This is a **delicate** API. The result of this method is a closeable resource with the + * associated native resources (threads). It should not be allocated in place, + * should be closed at the end of its lifecycle, and has non-trivial memory and CPU footprint. + * If you do not need a separate thread-pool, but only have to limit effective parallelism of the dispatcher, + * it is recommended to use [CoroutineDispatcher.limitedParallelism] instead. + * * If you need a completely separate thread-pool with scheduling policy that is based on the standard * JDK executors, use the following expression: * `Executors.newFixedThreadPool().asCoroutineDispatcher()`. @@ -56,7 +58,7 @@ public fun newSingleThreadContext(name: String): ExecutorCoroutineDispatcher = * @param nThreads the number of threads. * @param name the base name of the created threads. */ -@ObsoleteCoroutinesApi +@DelicateCoroutinesApi public fun newFixedThreadPoolContext(nThreads: Int, name: String): ExecutorCoroutineDispatcher { require(nThreads >= 1) { "Expected at least one thread, but $nThreads specified" } val threadNo = AtomicInteger() From 6a99a6dbf10d4ed0e808faf5639f8a38180f62a7 Mon Sep 17 00:00:00 2001 From: Vsevolod Tolstopyatov Date: Fri, 24 Sep 2021 17:49:49 +0300 Subject: [PATCH 12/24] Cleanup root Gradle script --- build.gradle | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/build.gradle b/build.gradle index e4b12ff3ad..6e0b250ea3 100644 --- a/build.gradle +++ b/build.gradle @@ -47,12 +47,6 @@ buildscript { } } - if (using_snapshot_version) { - repositories { - mavenLocal() - } - } - repositories { mavenCentral() maven { url "https://plugins.gradle.org/m2/" } @@ -102,13 +96,6 @@ allprojects { kotlin_version = rootProject.properties['kotlin_snapshot_version'] } - if (using_snapshot_version) { - repositories { - mavenLocal() - maven { url "https://oss.sonatype.org/content/repositories/snapshots" } - } - } - ext.unpublished = unpublished // This project property is set during nightly stress test @@ -139,6 +126,7 @@ allprojects { */ google() mavenCentral() + maven { url "https://maven.pkg.jetbrains.space/kotlin/p/kotlin/dev" } } } From 94c6fb787b03f82a9abb601865abb99e1df39c29 Mon Sep 17 00:00:00 2001 From: Vsevolod Tolstopyatov Date: Mon, 27 Sep 2021 11:55:48 +0300 Subject: [PATCH 13/24] Add version file to each module resources (#2950) * Add version file to each module resources * The approach with "Specification-Version" in Manifest doesn't work because Android merges all JARs into a single resource, trimming all the manifests Fixes #2941 --- build.gradle | 34 ++++++++++++++++ integration-testing/build.gradle | 1 + ...t => MavenPublicationAtomicfuValidator.kt} | 2 +- .../MavenPublicationVersionValidator.kt | 40 +++++++++++++++++++ 4 files changed, 76 insertions(+), 1 deletion(-) rename integration-testing/src/mavenTest/kotlin/{MavenPublicationValidator.kt => MavenPublicationAtomicfuValidator.kt} (97%) create mode 100644 integration-testing/src/mavenTest/kotlin/MavenPublicationVersionValidator.kt diff --git a/build.gradle b/build.gradle index 6e0b250ea3..f55e6c39d2 100644 --- a/build.gradle +++ b/build.gradle @@ -233,6 +233,40 @@ configure(subprojects.findAll { !unpublished.contains(it.name) }) { } } } + + def thisProject = it + if (thisProject.name in sourceless) { + return + } + + def versionFileTask = thisProject.tasks.register("versionFileTask") { + def name = thisProject.name.replace("-", "_") + def versionFile = thisProject.layout.buildDirectory.file("${name}.version") + it.outputs.file(versionFile) + + it.doLast { + versionFile.get().asFile.text = version.toString() + } + } + + List jarTasks + if (it.name == "kotlinx-coroutines-core") { + jarTasks = ["jvmJar", "metadataJar"] + } else if (it.name == "kotlinx-coroutines-debug") { + // We shadow debug module instead of just packaging it + jarTasks = ["shadowJar"] + } else { + jarTasks = ["jar"] + } + + for (name in jarTasks) { + thisProject.tasks.named(name, Jar) { + it.dependsOn versionFileTask + it.from(versionFileTask) { + into("META-INF") + } + } + } } // Report Kotlin compiler version when building project diff --git a/integration-testing/build.gradle b/integration-testing/build.gradle index 6efa3a14e6..d0286d7d55 100644 --- a/integration-testing/build.gradle +++ b/integration-testing/build.gradle @@ -58,6 +58,7 @@ task npmTest(type: Test) { } task mavenTest(type: Test) { + environment "version", version def sourceSet = sourceSets.mavenTest dependsOn(project(':').getTasksByName("publishToMavenLocal", true)) testClassesDirs = sourceSet.output.classesDirs diff --git a/integration-testing/src/mavenTest/kotlin/MavenPublicationValidator.kt b/integration-testing/src/mavenTest/kotlin/MavenPublicationAtomicfuValidator.kt similarity index 97% rename from integration-testing/src/mavenTest/kotlin/MavenPublicationValidator.kt rename to integration-testing/src/mavenTest/kotlin/MavenPublicationAtomicfuValidator.kt index 39d6598b55..dbb1921d80 100644 --- a/integration-testing/src/mavenTest/kotlin/MavenPublicationValidator.kt +++ b/integration-testing/src/mavenTest/kotlin/MavenPublicationAtomicfuValidator.kt @@ -8,7 +8,7 @@ import org.junit.* import org.junit.Assert.assertTrue import java.util.jar.* -class MavenPublicationValidator { +class MavenPublicationAtomicfuValidator { private val ATOMIC_FU_REF = "Lkotlinx/atomicfu/".toByteArray() @Test diff --git a/integration-testing/src/mavenTest/kotlin/MavenPublicationVersionValidator.kt b/integration-testing/src/mavenTest/kotlin/MavenPublicationVersionValidator.kt new file mode 100644 index 0000000000..da87d4cc59 --- /dev/null +++ b/integration-testing/src/mavenTest/kotlin/MavenPublicationVersionValidator.kt @@ -0,0 +1,40 @@ +/* + * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines.validator + +import org.junit.* +import org.junit.Test +import java.util.jar.* +import kotlin.test.* + +class MavenPublicationVersionValidator { + + @Test + fun testMppJar() { + val clazz = Class.forName("kotlinx.coroutines.Job") + JarFile(clazz.protectionDomain.codeSource.location.file).checkForVersion("kotlinx_coroutines_core.version") + } + + @Test + fun testAndroidJar() { + val clazz = Class.forName("kotlinx.coroutines.android.HandlerDispatcher") + JarFile(clazz.protectionDomain.codeSource.location.file).checkForVersion("kotlinx_coroutines_android.version") + } + + private fun JarFile.checkForVersion(file: String) { + val actualFile = "META-INF/$file" + val version = System.getenv("version") + use { + for (e in entries()) { + if (e.name == actualFile) { + val string = getInputStream(e).readAllBytes().decodeToString() + assertEquals(version, string) + return + } + } + error("File $file not found") + } + } +} From a740994ee36eaa936f5c2d0cc93a4b53d9f63bd3 Mon Sep 17 00:00:00 2001 From: Vsevolod Tolstopyatov Date: Mon, 27 Sep 2021 17:17:11 +0300 Subject: [PATCH 14/24] Migrate from deprecated packages in js-example (#2955) --- js/example-frontend-js/src/ExampleMain.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/js/example-frontend-js/src/ExampleMain.kt b/js/example-frontend-js/src/ExampleMain.kt index d4e530b04a..67c6ef04e7 100644 --- a/js/example-frontend-js/src/ExampleMain.kt +++ b/js/example-frontend-js/src/ExampleMain.kt @@ -8,7 +8,7 @@ import kotlinx.html.div import kotlinx.html.dom.* import kotlinx.html.js.onClickFunction import org.w3c.dom.* -import kotlin.browser.* +import kotlinx.browser.* import kotlin.coroutines.* import kotlin.math.* import kotlin.random.Random From 9edeb854e4e9548eb7c89b076056993aa582e7f6 Mon Sep 17 00:00:00 2001 From: Vadim Semenov <6957841+vadimsemenov@users.noreply.github.com> Date: Thu, 30 Sep 2021 13:52:56 +0100 Subject: [PATCH 15/24] Breaking change: Guava future coroutine builder shouldn't report to CoroutineExceptionHandler (#2840) This change makes `future` coroutine builder consistent with `java.util.concurrent.FutureTask` which also drops exceptions that happen after successful cancellation. Fixes #2774 Fixes #2791 --- .../src/ListenableFuture.kt | 45 ++++++----- .../test/ListenableFutureTest.kt | 79 +++++++++++++------ 2 files changed, 80 insertions(+), 44 deletions(-) diff --git a/integration/kotlinx-coroutines-guava/src/ListenableFuture.kt b/integration/kotlinx-coroutines-guava/src/ListenableFuture.kt index 8f11e0a916..d214cc6b1a 100644 --- a/integration/kotlinx-coroutines-guava/src/ListenableFuture.kt +++ b/integration/kotlinx-coroutines-guava/src/ListenableFuture.kt @@ -14,7 +14,7 @@ import kotlin.coroutines.* /** * Starts [block] in a new coroutine and returns a [ListenableFuture] pointing to its result. * - * The coroutine is immediately started. Passing [CoroutineStart.LAZY] to [start] throws + * The coroutine is started immediately. Passing [CoroutineStart.LAZY] to [start] throws * [IllegalArgumentException], because Futures don't have a way to start lazily. * * When the created coroutine [isCompleted][Job.isCompleted], it will try to @@ -35,10 +35,12 @@ import kotlin.coroutines.* * See [newCoroutineContext][CoroutineScope.newCoroutineContext] for a description of debugging * facilities. * - * Note that the error and cancellation semantics of [future] are _subtly different_ than [asListenableFuture]'s. - * In particular, any exception that happens in the coroutine after returned future is - * successfully cancelled will be passed to the [CoroutineExceptionHandler] from the [context]. - * See [ListenableFutureCoroutine] for details. + * Note that the error and cancellation semantics of [future] are _different_ than [async]'s. + * In contrast to [Deferred], [Future] doesn't have an intermediate `Cancelling` state. If + * the returned `Future` is successfully cancelled, and `block` throws afterward, the thrown + * error is dropped, and getting the `Future`'s value will throw a `CancellationException` with + * no cause. This is to match the specification and behavior of + * `java.util.concurrent.FutureTask`. * * @param context added overlaying [CoroutineScope.coroutineContext] to form the new context. * @param start coroutine start option. The default value is [CoroutineStart.DEFAULT]. @@ -241,8 +243,8 @@ public suspend fun ListenableFuture.await(): T { return suspendCancellableCoroutine { cont: CancellableContinuation -> addListener( - ToContinuation(this, cont), - MoreExecutors.directExecutor()) + ToContinuation(this, cont), + MoreExecutors.directExecutor()) cont.invokeOnCancellation { cancel(false) } @@ -284,16 +286,13 @@ private class ToContinuation( * By documented contract, a [Future] has been cancelled if * and only if its `isCancelled()` method returns true. * - * Any error that occurs after successfully cancelling a [ListenableFuture] will be passed - * to the [CoroutineExceptionHandler] from the context. The contract of [Future] does not permit - * it to return an error after it is successfully cancelled. - * - * By calling [asListenableFuture] on a [Deferred], any error that occurs after successfully - * cancelling the [ListenableFuture] representation of the [Deferred] will _not_ be passed to - * the [CoroutineExceptionHandler]. Cancelling a [Deferred] places that [Deferred] in the - * cancelling/cancelled states defined by [Job], which _can_ show the error. It's assumed that - * the [Deferred] pointing to the task will be used to observe any error outcome occurring after - * cancellation. + * Any error that occurs after successfully cancelling a [ListenableFuture] is lost. + * The contract of [Future] does not permit it to return an error after it is successfully cancelled. + * On the other hand, we can't report an unhandled exception to [CoroutineExceptionHandler], + * otherwise [Future.cancel] can lead to an app crash which arguably is a contract violation. + * In contrast to [Future] which can't change its outcome after a successful cancellation, + * cancelling a [Deferred] places that [Deferred] in the cancelling/cancelled states defined by [Job], + * which _can_ show the error. * * This may be counterintuitive, but it maintains the error and cancellation contracts of both * the [Deferred] and [ListenableFuture] types, while permitting both kinds of promise to point @@ -312,10 +311,14 @@ private class ListenableFutureCoroutine( } override fun onCancelled(cause: Throwable, handled: Boolean) { - if (!future.completeExceptionallyOrCancel(cause) && !handled) { - // prevents loss of exception that was not handled by parent & could not be set to JobListenableFuture - handleCoroutineException(context, cause) - } + // Note: if future was cancelled in a race with a cancellation of this + // coroutine, and the future was successfully cancelled first, the cause of coroutine + // cancellation is dropped in this promise. A Future can only be completed once. + // + // This is consistent with FutureTask behaviour. A race between a Future.cancel() and + // a FutureTask.setException() for the same Future will similarly drop the + // cause of a failure-after-cancellation. + future.completeExceptionallyOrCancel(cause) } } diff --git a/integration/kotlinx-coroutines-guava/test/ListenableFutureTest.kt b/integration/kotlinx-coroutines-guava/test/ListenableFutureTest.kt index 69ba193071..581e09abdd 100644 --- a/integration/kotlinx-coroutines-guava/test/ListenableFutureTest.kt +++ b/integration/kotlinx-coroutines-guava/test/ListenableFutureTest.kt @@ -555,11 +555,7 @@ class ListenableFutureTest : TestBase() { } @Test - fun testUnhandledExceptionOnExternalCancellation() = runTest( - unhandled = listOf( - { it -> it is TestException } // exception is unhandled because there is no parent - ) - ) { + fun testUnhandledExceptionOnExternalCancellation() = runTest { expect(1) // No parent here (NonCancellable), so nowhere to propagate exception val result = future(NonCancellable + Dispatchers.Unconfined) { @@ -567,7 +563,7 @@ class ListenableFutureTest : TestBase() { delay(Long.MAX_VALUE) } finally { expect(2) - throw TestException() // this exception cannot be handled + throw TestException() // this exception cannot be handled and is set to be lost. } } result.cancel(true) @@ -708,23 +704,6 @@ class ListenableFutureTest : TestBase() { assertEquals(testException, thrown.cause) } - @Test - fun stressTestJobListenableFutureIsCancelledDoesNotThrow() = runTest { - repeat(1000) { - val deferred = CompletableDeferred() - val asListenableFuture = deferred.asListenableFuture() - // We heed two threads to test a race condition. - withContext(Dispatchers.Default) { - val cancellationJob = launch { - asListenableFuture.cancel(false) - } - while (!cancellationJob.isCompleted) { - asListenableFuture.isCancelled // Shouldn't throw. - } - } - } - } - private inline fun ListenableFuture<*>.checkFutureException() { val e = assertFailsWith { get() } val cause = e.cause!! @@ -775,4 +754,58 @@ class ListenableFutureTest : TestBase() { assertEquals(count, completed.get()) } } + + @Test + fun futurePropagatesExceptionToParentAfterCancellation() = runTest { + val latch = CompletableDeferred() + val parent = Job() + val scope = CoroutineScope(parent) + val exception = TestException("propagated to parent") + val future = scope.future { + withContext(NonCancellable) { + latch.await() + throw exception + } + } + future.cancel(true) + latch.complete(true) + parent.join() + assertTrue(parent.isCancelled) + assertEquals(exception, parent.getCancellationException().cause) + } + + // Stress tests. + + @Test + fun testFutureDoesNotReportToCoroutineExceptionHandler() = runTest { + repeat(1000) { + supervisorScope { // Don't propagate failures in children to parent and other children. + val innerFuture = SettableFuture.create() + val outerFuture = async { innerFuture.await() } + + withContext(Dispatchers.Default) { + launch { innerFuture.setException(TestException("can be lost")) } + launch { outerFuture.cancel() } + // nothing should be reported to CoroutineExceptionHandler, otherwise `Future.cancel` contract violation. + } + } + } + } + + @Test + fun testJobListenableFutureIsCancelledDoesNotThrow() = runTest { + repeat(1000) { + val deferred = CompletableDeferred() + val asListenableFuture = deferred.asListenableFuture() + // We heed two threads to test a race condition. + withContext(Dispatchers.Default) { + val cancellationJob = launch { + asListenableFuture.cancel(false) + } + while (!cancellationJob.isCompleted) { + asListenableFuture.isCancelled // Shouldn't throw. + } + } + } + } } From 2cd1011aac25130f2ee0e0fd277c76f002e3c436 Mon Sep 17 00:00:00 2001 From: Vsevolod Tolstopyatov Date: Tue, 5 Oct 2021 18:03:15 +0300 Subject: [PATCH 16/24] Update binary compatibility validator (#2969) * Update binary compatibility validator * Fix race in testFuturePropagatesExceptionToParentAfterCancellation --- gradle.properties | 2 +- .../test/ListenableFutureTest.kt | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/gradle.properties b/gradle.properties index 26e5147c51..46eef4d76e 100644 --- a/gradle.properties +++ b/gradle.properties @@ -22,7 +22,7 @@ rxjava2_version=2.2.8 rxjava3_version=3.0.2 javafx_version=11.0.2 javafx_plugin_version=0.0.8 -binary_compatibility_validator_version=0.7.0 +binary_compatibility_validator_version=0.8.0-RC blockhound_version=1.0.2.RELEASE jna_version=5.5.0 diff --git a/integration/kotlinx-coroutines-guava/test/ListenableFutureTest.kt b/integration/kotlinx-coroutines-guava/test/ListenableFutureTest.kt index 581e09abdd..511b1b0322 100644 --- a/integration/kotlinx-coroutines-guava/test/ListenableFutureTest.kt +++ b/integration/kotlinx-coroutines-guava/test/ListenableFutureTest.kt @@ -756,19 +756,22 @@ class ListenableFutureTest : TestBase() { } @Test - fun futurePropagatesExceptionToParentAfterCancellation() = runTest { - val latch = CompletableDeferred() + fun testFuturePropagatesExceptionToParentAfterCancellation() = runTest { + val throwLatch = CompletableDeferred() + val cancelLatch = CompletableDeferred() val parent = Job() val scope = CoroutineScope(parent) val exception = TestException("propagated to parent") val future = scope.future { + cancelLatch.complete(true) withContext(NonCancellable) { - latch.await() + throwLatch.await() throw exception } } + cancelLatch.await() future.cancel(true) - latch.complete(true) + throwLatch.complete(true) parent.join() assertTrue(parent.isCancelled) assertEquals(exception, parent.getCancellationException().cause) From d014723ed96f60aebcdfa9030df01e2963e049d4 Mon Sep 17 00:00:00 2001 From: dkhalanskyjb <52952525+dkhalanskyjb@users.noreply.github.com> Date: Fri, 8 Oct 2021 10:04:45 +0300 Subject: [PATCH 17/24] Make kotlinx-coroutines-test an MPP (#2965) Change the build scripts and the file layout so that kotlinx-coroutines-test is built on all platforms. --- build.gradle | 54 ++-- buildSrc/src/main/kotlin/OptInPreset.kt | 13 + buildSrc/src/main/kotlin/SourceSets.kt | 19 ++ gradle/dokka.gradle.kts | 2 +- gradle/opt-in.gradle | 13 - gradle/publish.gradle | 2 +- kotlinx-coroutines-core/build.gradle | 15 +- .../common/src/Unconfined.kt | 2 +- .../common/src/flow/Migration.kt | 2 +- .../api/kotlinx-coroutines-test.api | 7 +- kotlinx-coroutines-test/build.gradle.kts | 10 +- .../{ => common}/src/DelayController.kt | 2 +- .../{ => common}/src/TestBuilders.kt | 21 +- .../src/TestCoroutineDispatcher.kt | 7 +- .../src/TestCoroutineExceptionHandler.kt | 14 +- .../{ => common}/src/TestCoroutineScope.kt | 4 +- .../common/src/TestDispatchers.kt | 26 ++ .../{ => common}/test/TestBuildersTest.kt | 3 +- .../test/TestCoroutineDispatcherOrderTest.kt | 25 +- .../test/TestCoroutineDispatcherTest.kt | 44 +--- .../test/TestCoroutineExceptionHandlerTest.kt | 3 +- .../test/TestCoroutineScopeTest.kt | 5 +- .../{ => common}/test/TestModuleHelpers.kt | 13 +- .../test/TestRunBlockingOrderTest.kt | 23 +- .../{ => common}/test/TestRunBlockingTest.kt | 244 +++++++++++------- .../js/src/TestDispatchers.kt | 16 ++ .../META-INF/proguard/coroutines.pro | 0 ....coroutines.internal.MainDispatcherFactory | 0 .../jvm/src/TestDispatchers.kt | 24 ++ .../src/internal/TestMainDispatcher.kt} | 2 +- .../jvm/test/MultithreadingTest.kt | 89 +++++++ .../jvm/test/TestDispatchersTest.kt | 72 ++++++ .../native/src/TestDispatchers.kt | 17 ++ kotlinx-coroutines-test/npm/README.md | 4 + kotlinx-coroutines-test/npm/package.json | 23 ++ .../src/TestDispatchers.kt | 38 --- .../test/TestDispatchersTest.kt | 89 ------- 37 files changed, 577 insertions(+), 370 deletions(-) create mode 100644 buildSrc/src/main/kotlin/OptInPreset.kt create mode 100644 buildSrc/src/main/kotlin/SourceSets.kt delete mode 100644 gradle/opt-in.gradle rename kotlinx-coroutines-test/{ => common}/src/DelayController.kt (97%) rename kotlinx-coroutines-test/{ => common}/src/TestBuilders.kt (85%) rename kotlinx-coroutines-test/{ => common}/src/TestCoroutineDispatcher.kt (98%) rename kotlinx-coroutines-test/{ => common}/src/TestCoroutineExceptionHandler.kt (81%) rename kotlinx-coroutines-test/{ => common}/src/TestCoroutineScope.kt (95%) create mode 100644 kotlinx-coroutines-test/common/src/TestDispatchers.kt rename kotlinx-coroutines-test/{ => common}/test/TestBuildersTest.kt (97%) rename kotlinx-coroutines-test/{ => common}/test/TestCoroutineDispatcherOrderTest.kt (58%) rename kotlinx-coroutines-test/{ => common}/test/TestCoroutineDispatcherTest.kt (69%) rename kotlinx-coroutines-test/{ => common}/test/TestCoroutineExceptionHandlerTest.kt (85%) rename kotlinx-coroutines-test/{ => common}/test/TestCoroutineScopeTest.kt (74%) rename kotlinx-coroutines-test/{ => common}/test/TestModuleHelpers.kt (60%) rename kotlinx-coroutines-test/{ => common}/test/TestRunBlockingOrderTest.kt (70%) rename kotlinx-coroutines-test/{ => common}/test/TestRunBlockingTest.kt (56%) create mode 100644 kotlinx-coroutines-test/js/src/TestDispatchers.kt rename kotlinx-coroutines-test/{ => jvm}/resources/META-INF/proguard/coroutines.pro (100%) rename kotlinx-coroutines-test/{ => jvm}/resources/META-INF/services/kotlinx.coroutines.internal.MainDispatcherFactory (100%) create mode 100644 kotlinx-coroutines-test/jvm/src/TestDispatchers.kt rename kotlinx-coroutines-test/{src/internal/MainTestDispatcher.kt => jvm/src/internal/TestMainDispatcher.kt} (95%) create mode 100644 kotlinx-coroutines-test/jvm/test/MultithreadingTest.kt create mode 100644 kotlinx-coroutines-test/jvm/test/TestDispatchersTest.kt create mode 100644 kotlinx-coroutines-test/native/src/TestDispatchers.kt create mode 100644 kotlinx-coroutines-test/npm/README.md create mode 100644 kotlinx-coroutines-test/npm/package.json delete mode 100644 kotlinx-coroutines-test/src/TestDispatchers.kt delete mode 100644 kotlinx-coroutines-test/test/TestDispatchersTest.kt diff --git a/build.gradle b/build.gradle index f55e6c39d2..26b598ff5d 100644 --- a/build.gradle +++ b/build.gradle @@ -4,14 +4,13 @@ import org.jetbrains.kotlin.gradle.plugin.KotlinPlatformType import org.jetbrains.kotlin.konan.target.HostManager -import org.gradle.util.VersionNumber import org.jetbrains.dokka.gradle.DokkaTaskPartial -import org.jetbrains.dokka.gradle.DokkaMultiModuleTask apply plugin: 'jdk-convention' -apply from: rootProject.file("gradle/opt-in.gradle") def coreModule = "kotlinx-coroutines-core" +def testModule = "kotlinx-coroutines-test" +def multiplatform = [coreModule, testModule] // Not applicable for Kotlin plugin def sourceless = ['kotlinx.coroutines', 'kotlinx-coroutines-bom', 'integration-testing'] def internal = ['kotlinx.coroutines', 'benchmarks', 'integration-testing'] @@ -112,7 +111,7 @@ apiValidation { ignoredProjects += unpublished + ["kotlinx-coroutines-bom"] if (build_snapshot_train) { ignoredProjects.remove("example-frontend-js") - ignoredProjects.add("kotlinx-coroutines-core") + ignoredProjects.add(coreModule) } ignoredPackages += "kotlinx.coroutines.internal" } @@ -133,13 +132,31 @@ allprojects { // Add dependency to core source sets. Core is configured in kx-core/build.gradle configure(subprojects.findAll { !sourceless.contains(it.name) && it.name != coreModule }) { evaluationDependsOn(":$coreModule") - def platform = PlatformKt.platformOf(it) - apply plugin: "kotlin-${platform}-conventions" - dependencies { - // See comment below for rationale, it will be replaced with "project" dependency - api project(":$coreModule") - // the only way IDEA can resolve test classes - testImplementation project(":$coreModule").kotlin.targets.jvm.compilations.test.output.allOutputs + if (it.name in multiplatform) { + apply plugin: "kotlin-multiplatform" + apply from: rootProject.file("gradle/compile-jvm-multiplatform.gradle") + apply from: rootProject.file("gradle/compile-common.gradle") + + if (rootProject.ext["native_targets_enabled"] as Boolean) { + apply from: rootProject.file("gradle/compile-native-multiplatform.gradle") + } + + apply from: rootProject.file("gradle/compile-js-multiplatform.gradle") + apply from: rootProject.file("gradle/publish-npm-js.gradle") + kotlin.sourceSets.commonMain.dependencies { + api project(":$coreModule") + } + kotlin.sourceSets.jvmTest.dependencies { + implementation project(":$coreModule").kotlin.targets.jvm.compilations.test.output.allOutputs + } + } else { + def platform = PlatformKt.platformOf(it) + apply plugin: "kotlin-${platform}-conventions" + dependencies { + api project(":$coreModule") + // the only way IDEA can resolve test classes + testImplementation project(":$coreModule").kotlin.targets.jvm.compilations.test.output.allOutputs + } } } @@ -150,7 +167,7 @@ configure(subprojects.findAll { !sourceless.contains(it.name) }) { // Configure options for all Kotlin compilation tasks tasks.withType(org.jetbrains.kotlin.gradle.tasks.AbstractKotlinCompile).all { - kotlinOptions.freeCompilerArgs += optInAnnotations.collect { "-Xopt-in=" + it } + kotlinOptions.freeCompilerArgs += OptInPresetKt.optInAnnotations.collect { "-Xopt-in=" + it } kotlinOptions.freeCompilerArgs += "-progressive" // Disable KT-36770 for RxJava2 integration kotlinOptions.freeCompilerArgs += "-XXLanguage:-ProhibitUsingNullableTypeParameterAgainstNotNullAnnotated" @@ -177,7 +194,7 @@ if (build_snapshot_train) { } println "Manifest of kotlin-compiler-embeddable.jar for coroutines" - configure(subprojects.findAll { it.name == "kotlinx-coroutines-core" }) { + configure(subprojects.findAll { it.name == coreModule }) { configurations.matching { it.name == "kotlinCompilerClasspath" }.all { resolvedConfiguration.getFiles().findAll { it.name.contains("kotlin-compiler-embeddable") }.each { def manifest = zipTree(it).matching { @@ -194,9 +211,8 @@ if (build_snapshot_train) { // Redefine source sets because we are not using 'kotlin/main/fqn' folder convention configure(subprojects.findAll { - !sourceless.contains(it.name) && + !sourceless.contains(it.name) && !multiplatform.contains(it.name) && it.name != "benchmarks" && - it.name != coreModule && it.name != "example-frontend-js" }) { // Pure JS and pure MPP doesn't have this notion and are configured separately @@ -250,7 +266,7 @@ configure(subprojects.findAll { !unpublished.contains(it.name) }) { } List jarTasks - if (it.name == "kotlinx-coroutines-core") { + if (it.name in multiplatform) { jarTasks = ["jvmJar", "metadataJar"] } else if (it.name == "kotlinx-coroutines-debug") { // We shadow debug module instead of just packaging it @@ -324,12 +340,12 @@ allprojects { subProject -> .matching { // Excluding substituted project itself because of circular dependencies, but still do it // for "*Test*" configurations - subProject.name != "kotlinx-coroutines-core" || it.name.contains("Test") + subProject.name != coreModule || it.name.contains("Test") } .configureEach { conf -> conf.resolutionStrategy.dependencySubstitution { - substitute(module("org.jetbrains.kotlinx:kotlinx-coroutines-core")) - .using(project(":kotlinx-coroutines-core")) + substitute(module("org.jetbrains.kotlinx:$coreModule")) + .using(project(":$coreModule")) .because("Because Kotlin compiler embeddable leaks coroutines into the runtime classpath, " + "triggering all sort of incompatible class changes errors") } diff --git a/buildSrc/src/main/kotlin/OptInPreset.kt b/buildSrc/src/main/kotlin/OptInPreset.kt new file mode 100644 index 0000000000..ee2aab11cf --- /dev/null +++ b/buildSrc/src/main/kotlin/OptInPreset.kt @@ -0,0 +1,13 @@ +/* + * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +val optInAnnotations = listOf( + "kotlin.RequiresOptIn", + "kotlin.experimental.ExperimentalTypeInference", + "kotlin.ExperimentalMultiplatform", + "kotlinx.coroutines.DelicateCoroutinesApi", + "kotlinx.coroutines.ExperimentalCoroutinesApi", + "kotlinx.coroutines.ObsoleteCoroutinesApi", + "kotlinx.coroutines.InternalCoroutinesApi", + "kotlinx.coroutines.FlowPreview") \ No newline at end of file diff --git a/buildSrc/src/main/kotlin/SourceSets.kt b/buildSrc/src/main/kotlin/SourceSets.kt new file mode 100644 index 0000000000..533ac70ac6 --- /dev/null +++ b/buildSrc/src/main/kotlin/SourceSets.kt @@ -0,0 +1,19 @@ +/* + * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ +import org.jetbrains.kotlin.gradle.plugin.* + +fun KotlinSourceSet.configureMultiplatform() { + val srcDir = if (name.endsWith("Main")) "src" else "test" + val platform = name.dropLast(4) + kotlin.srcDir("$platform/$srcDir") + if (name == "jvmMain") { + resources.srcDir("$platform/resources") + } else if (name == "jvmTest") { + resources.srcDir("$platform/test-resources") + } + languageSettings { + optInAnnotations.forEach { optIn(it) } + progressiveMode = true + } +} \ No newline at end of file diff --git a/gradle/dokka.gradle.kts b/gradle/dokka.gradle.kts index 659890a30b..a4926f7e61 100644 --- a/gradle/dokka.gradle.kts +++ b/gradle/dokka.gradle.kts @@ -37,7 +37,7 @@ tasks.withType(DokkaTaskPartial::class).configureEach { packageListUrl.set(rootProject.projectDir.toPath().resolve("site/stdlib.package.list").toUri().toURL()) } - if (project.name != "kotlinx-coroutines-core") { + if (project.name != "kotlinx-coroutines-core" && project.name != "kotlinx-coroutines-test") { dependsOn(project.configurations["compileClasspath"]) doFirst { // resolve classpath only during execution diff --git a/gradle/opt-in.gradle b/gradle/opt-in.gradle deleted file mode 100644 index 22f022dbb5..0000000000 --- a/gradle/opt-in.gradle +++ /dev/null @@ -1,13 +0,0 @@ -/* - * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. - */ - -ext.optInAnnotations = [ - "kotlin.RequiresOptIn", - "kotlin.experimental.ExperimentalTypeInference", - "kotlin.ExperimentalMultiplatform", - "kotlinx.coroutines.DelicateCoroutinesApi", - "kotlinx.coroutines.ExperimentalCoroutinesApi", - "kotlinx.coroutines.ObsoleteCoroutinesApi", - "kotlinx.coroutines.InternalCoroutinesApi", - "kotlinx.coroutines.FlowPreview"] diff --git a/gradle/publish.gradle b/gradle/publish.gradle index 3a0a4224ab..fa2bbb8544 100644 --- a/gradle/publish.gradle +++ b/gradle/publish.gradle @@ -12,7 +12,7 @@ apply plugin: 'signing' // ------------- tasks -def isMultiplatform = project.name == "kotlinx-coroutines-core" +def isMultiplatform = project.name == "kotlinx-coroutines-core" || project.name == "kotlinx-coroutines-test" def isBom = project.name == "kotlinx-coroutines-bom" if (!isBom) { diff --git a/kotlinx-coroutines-core/build.gradle b/kotlinx-coroutines-core/build.gradle index c45ca08cef..4435ad7fa1 100644 --- a/kotlinx-coroutines-core/build.gradle +++ b/kotlinx-coroutines-core/build.gradle @@ -70,19 +70,8 @@ if (rootProject.ext.native_targets_enabled) { * because JMV-only projects depend on core, thus core should always be initialized before configuration. */ kotlin { - configure(sourceSets) { - def srcDir = name.endsWith('Main') ? 'src' : 'test' - def platform = name[0..-5] - kotlin.srcDirs = ["$platform/$srcDir"] - if (name == "jvmMain") { - resources.srcDirs = ["$platform/resources"] - } else if (name == "jvmTest") { - resources.srcDirs = ["$platform/test-resources"] - } - languageSettings { - progressiveMode = true - optInAnnotations.each { useExperimentalAnnotation(it) } - } + sourceSets.forEach { + SourceSetsKt.configureMultiplatform(it) } configure(targets) { diff --git a/kotlinx-coroutines-core/common/src/Unconfined.kt b/kotlinx-coroutines-core/common/src/Unconfined.kt index 4f48645895..df0087100a 100644 --- a/kotlinx-coroutines-core/common/src/Unconfined.kt +++ b/kotlinx-coroutines-core/common/src/Unconfined.kt @@ -14,7 +14,7 @@ internal object Unconfined : CoroutineDispatcher() { override fun isDispatchNeeded(context: CoroutineContext): Boolean = false override fun dispatch(context: CoroutineContext, block: Runnable) { - // It can only be called by the "yield" function. See also code of "yield" function. + /** It can only be called by the [yield] function. See also code of [yield] function. */ val yieldContext = context[YieldContext] if (yieldContext != null) { // report to "yield" that it is an unconfined dispatcher and don't call "block.run()" diff --git a/kotlinx-coroutines-core/common/src/flow/Migration.kt b/kotlinx-coroutines-core/common/src/flow/Migration.kt index 6278081a5d..64effbf395 100644 --- a/kotlinx-coroutines-core/common/src/flow/Migration.kt +++ b/kotlinx-coroutines-core/common/src/flow/Migration.kt @@ -260,7 +260,7 @@ public fun Flow.skip(count: Int): Flow = noImpl() @Deprecated( level = DeprecationLevel.ERROR, message = "Flow analogue of 'forEach' is 'collect'", - replaceWith = ReplaceWith("collect(block)") + replaceWith = ReplaceWith("collect(action)") ) public fun Flow.forEach(action: suspend (value: T) -> Unit): Unit = noImpl() diff --git a/kotlinx-coroutines-test/api/kotlinx-coroutines-test.api b/kotlinx-coroutines-test/api/kotlinx-coroutines-test.api index c99ec5cbf1..707ee43df2 100644 --- a/kotlinx-coroutines-test/api/kotlinx-coroutines-test.api +++ b/kotlinx-coroutines-test/api/kotlinx-coroutines-test.api @@ -36,7 +36,7 @@ public final class kotlinx/coroutines/test/TestCoroutineDispatcher : kotlinx/cor public final class kotlinx/coroutines/test/TestCoroutineExceptionHandler : kotlin/coroutines/AbstractCoroutineContextElement, kotlinx/coroutines/CoroutineExceptionHandler, kotlinx/coroutines/test/UncaughtExceptionCaptor { public fun ()V - public fun cleanupTestCoroutines ()V + public fun cleanupTestCoroutinesCaptor ()V public fun getUncaughtExceptions ()Ljava/util/List; public fun handleException (Lkotlin/coroutines/CoroutineContext;Ljava/lang/Throwable;)V } @@ -56,12 +56,11 @@ public final class kotlinx/coroutines/test/TestDispatchers { } public abstract interface class kotlinx/coroutines/test/UncaughtExceptionCaptor { - public abstract fun cleanupTestCoroutines ()V + public abstract fun cleanupTestCoroutinesCaptor ()V public abstract fun getUncaughtExceptions ()Ljava/util/List; } public final class kotlinx/coroutines/test/UncompletedCoroutinesError : java/lang/AssertionError { - public fun (Ljava/lang/String;Ljava/lang/Throwable;)V - public synthetic fun (Ljava/lang/String;Ljava/lang/Throwable;ILkotlin/jvm/internal/DefaultConstructorMarker;)V + public fun (Ljava/lang/String;)V } diff --git a/kotlinx-coroutines-test/build.gradle.kts b/kotlinx-coroutines-test/build.gradle.kts index fef0a146f7..7b244bb091 100644 --- a/kotlinx-coroutines-test/build.gradle.kts +++ b/kotlinx-coroutines-test/build.gradle.kts @@ -2,6 +2,12 @@ * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. */ -dependencies { - implementation(project(":kotlinx-coroutines-debug")) +val experimentalAnnotations = listOf( + "kotlin.Experimental", + "kotlinx.coroutines.ExperimentalCoroutinesApi", + "kotlinx.coroutines.InternalCoroutinesApi" +) + +kotlin { + sourceSets.all { configureMultiplatform() } } diff --git a/kotlinx-coroutines-test/src/DelayController.kt b/kotlinx-coroutines-test/common/src/DelayController.kt similarity index 97% rename from kotlinx-coroutines-test/src/DelayController.kt rename to kotlinx-coroutines-test/common/src/DelayController.kt index 6e72222718..a4ab8c4aba 100644 --- a/kotlinx-coroutines-test/src/DelayController.kt +++ b/kotlinx-coroutines-test/common/src/DelayController.kt @@ -126,4 +126,4 @@ public interface DelayController { */ // todo: maybe convert into non-public class in 1.3.0 (need use-cases for a public exception type) @ExperimentalCoroutinesApi // Since 1.2.1, tentatively till 1.3.0 -public class UncompletedCoroutinesError(message: String, cause: Throwable? = null): AssertionError(message, cause) +public class UncompletedCoroutinesError(message: String): AssertionError(message) diff --git a/kotlinx-coroutines-test/src/TestBuilders.kt b/kotlinx-coroutines-test/common/src/TestBuilders.kt similarity index 85% rename from kotlinx-coroutines-test/src/TestBuilders.kt rename to kotlinx-coroutines-test/common/src/TestBuilders.kt index b40769ee97..dde9ac7b12 100644 --- a/kotlinx-coroutines-test/src/TestBuilders.kt +++ b/kotlinx-coroutines-test/common/src/TestBuilders.kt @@ -80,19 +80,16 @@ public fun TestCoroutineDispatcher.runBlockingTest(block: suspend TestCoroutineS runBlockingTest(this, block) private fun CoroutineContext.checkArguments(): Pair { - // TODO optimize it - val dispatcher = get(ContinuationInterceptor).run { - this?.let { require(this is DelayController) { "Dispatcher must implement DelayController: $this" } } - this ?: TestCoroutineDispatcher() + val dispatcher = when (val dispatcher = get(ContinuationInterceptor)) { + is DelayController -> dispatcher + null -> TestCoroutineDispatcher() + else -> throw IllegalArgumentException("Dispatcher must implement DelayController: $dispatcher") } - - val exceptionHandler = get(CoroutineExceptionHandler).run { - this?.let { - require(this is UncaughtExceptionCaptor) { "coroutineExceptionHandler must implement UncaughtExceptionCaptor: $this" } - } - this ?: TestCoroutineExceptionHandler() + val exceptionHandler = when (val handler = get(CoroutineExceptionHandler)) { + is UncaughtExceptionCaptor -> handler + null -> TestCoroutineExceptionHandler() + else -> throw IllegalArgumentException("coroutineExceptionHandler must implement UncaughtExceptionCaptor: $handler") } - val job = get(Job) ?: SupervisorJob() - return Pair(this + dispatcher + exceptionHandler + job, dispatcher as DelayController) + return Pair(this + dispatcher + exceptionHandler + job, dispatcher) } diff --git a/kotlinx-coroutines-test/src/TestCoroutineDispatcher.kt b/kotlinx-coroutines-test/common/src/TestCoroutineDispatcher.kt similarity index 98% rename from kotlinx-coroutines-test/src/TestCoroutineDispatcher.kt rename to kotlinx-coroutines-test/common/src/TestCoroutineDispatcher.kt index f6464789fc..55b92cd6b7 100644 --- a/kotlinx-coroutines-test/src/TestCoroutineDispatcher.kt +++ b/kotlinx-coroutines-test/common/src/TestCoroutineDispatcher.kt @@ -8,6 +8,7 @@ import kotlinx.atomicfu.* import kotlinx.coroutines.* import kotlinx.coroutines.internal.* import kotlin.coroutines.* +import kotlin.jvm.* import kotlin.math.* /** @@ -67,11 +68,7 @@ public class TestCoroutineDispatcher: CoroutineDispatcher(), Delay, DelayControl /** @suppress */ override fun invokeOnTimeout(timeMillis: Long, block: Runnable, context: CoroutineContext): DisposableHandle { val node = postDelayed(block, timeMillis) - return object : DisposableHandle { - override fun dispose() { - queue.remove(node) - } - } + return DisposableHandle { queue.remove(node) } } /** @suppress */ diff --git a/kotlinx-coroutines-test/src/TestCoroutineExceptionHandler.kt b/kotlinx-coroutines-test/common/src/TestCoroutineExceptionHandler.kt similarity index 81% rename from kotlinx-coroutines-test/src/TestCoroutineExceptionHandler.kt rename to kotlinx-coroutines-test/common/src/TestCoroutineExceptionHandler.kt index 66eb235906..b1296df12a 100644 --- a/kotlinx-coroutines-test/src/TestCoroutineExceptionHandler.kt +++ b/kotlinx-coroutines-test/common/src/TestCoroutineExceptionHandler.kt @@ -5,6 +5,7 @@ package kotlinx.coroutines.test import kotlinx.coroutines.* +import kotlinx.coroutines.internal.* import kotlin.coroutines.* /** @@ -16,7 +17,7 @@ public interface UncaughtExceptionCaptor { * List of uncaught coroutine exceptions. * * The returned list is a copy of the currently caught exceptions. - * During [cleanupTestCoroutines] the first element of this list is rethrown if it is not empty. + * During [cleanupTestCoroutinesCaptor] the first element of this list is rethrown if it is not empty. */ public val uncaughtExceptions: List @@ -28,7 +29,7 @@ public interface UncaughtExceptionCaptor { * * @throws Throwable the first uncaught exception, if there are any uncaught exceptions. */ - public fun cleanupTestCoroutines() + public fun cleanupTestCoroutinesCaptor() } /** @@ -39,21 +40,22 @@ public class TestCoroutineExceptionHandler : AbstractCoroutineContextElement(CoroutineExceptionHandler), UncaughtExceptionCaptor, CoroutineExceptionHandler { private val _exceptions = mutableListOf() + private val _lock = SynchronizedObject() /** @suppress **/ override fun handleException(context: CoroutineContext, exception: Throwable) { - synchronized(_exceptions) { + synchronized(_lock) { _exceptions += exception } } /** @suppress **/ override val uncaughtExceptions: List - get() = synchronized(_exceptions) { _exceptions.toList() } + get() = synchronized(_lock) { _exceptions.toList() } /** @suppress **/ - override fun cleanupTestCoroutines() { - synchronized(_exceptions) { + override fun cleanupTestCoroutinesCaptor() { + synchronized(_lock) { val exception = _exceptions.firstOrNull() ?: return // log the rest _exceptions.drop(1).forEach { it.printStackTrace() } diff --git a/kotlinx-coroutines-test/src/TestCoroutineScope.kt b/kotlinx-coroutines-test/common/src/TestCoroutineScope.kt similarity index 95% rename from kotlinx-coroutines-test/src/TestCoroutineScope.kt rename to kotlinx-coroutines-test/common/src/TestCoroutineScope.kt index 7c1ff872ec..da29cd22b4 100644 --- a/kotlinx-coroutines-test/src/TestCoroutineScope.kt +++ b/kotlinx-coroutines-test/common/src/TestCoroutineScope.kt @@ -14,7 +14,7 @@ import kotlin.coroutines.* public interface TestCoroutineScope: CoroutineScope, UncaughtExceptionCaptor, DelayController { /** * Call after the test completes. - * Calls [UncaughtExceptionCaptor.cleanupTestCoroutines] and [DelayController.cleanupTestCoroutines]. + * Calls [UncaughtExceptionCaptor.cleanupTestCoroutinesCaptor] and [DelayController.cleanupTestCoroutines]. * * @throws Throwable the first uncaught exception, if there are any uncaught exceptions. * @throws UncompletedCoroutinesError if any pending tasks are active, however it will not throw for suspended @@ -31,7 +31,7 @@ private class TestCoroutineScopeImpl ( DelayController by coroutineContext.delayController { override fun cleanupTestCoroutines() { - coroutineContext.uncaughtExceptionCaptor.cleanupTestCoroutines() + coroutineContext.uncaughtExceptionCaptor.cleanupTestCoroutinesCaptor() coroutineContext.delayController.cleanupTestCoroutines() } } diff --git a/kotlinx-coroutines-test/common/src/TestDispatchers.kt b/kotlinx-coroutines-test/common/src/TestDispatchers.kt new file mode 100644 index 0000000000..2f00331506 --- /dev/null +++ b/kotlinx-coroutines-test/common/src/TestDispatchers.kt @@ -0,0 +1,26 @@ +/* + * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines.test + +import kotlinx.coroutines.* + +/** + * Sets the given [dispatcher] as an underlying dispatcher of [Dispatchers.Main]. + * All subsequent usages of [Dispatchers.Main] will use given [dispatcher] under the hood. + * + * It is unsafe to call this method if alive coroutines launched in [Dispatchers.Main] exist. + */ +@ExperimentalCoroutinesApi +public expect fun Dispatchers.setMain(dispatcher: CoroutineDispatcher) + +/** + * Resets state of the [Dispatchers.Main] to the original main dispatcher. + * For example, in Android Main thread dispatcher will be set as [Dispatchers.Main]. + * Used to clean up all possible dependencies, should be used in tear down (`@After`) methods. + * + * It is unsafe to call this method if alive coroutines launched in [Dispatchers.Main] exist. + */ +@ExperimentalCoroutinesApi +public expect fun Dispatchers.resetMain() diff --git a/kotlinx-coroutines-test/test/TestBuildersTest.kt b/kotlinx-coroutines-test/common/test/TestBuildersTest.kt similarity index 97% rename from kotlinx-coroutines-test/test/TestBuildersTest.kt rename to kotlinx-coroutines-test/common/test/TestBuildersTest.kt index 27c8f5fb19..a3167e5876 100644 --- a/kotlinx-coroutines-test/test/TestBuildersTest.kt +++ b/kotlinx-coroutines-test/common/test/TestBuildersTest.kt @@ -5,7 +5,6 @@ package kotlinx.coroutines.test import kotlinx.coroutines.* -import org.junit.Test import kotlin.coroutines.* import kotlin.test.* @@ -59,7 +58,7 @@ class TestBuildersTest { } @Test - fun scopeRunBlocking_disablesImmedateOnExit() { + fun scopeRunBlocking_disablesImmediatelyOnExit() { val scope = TestCoroutineScope() scope.runBlockingTest { assertRunsFast { diff --git a/kotlinx-coroutines-test/test/TestCoroutineDispatcherOrderTest.kt b/kotlinx-coroutines-test/common/test/TestCoroutineDispatcherOrderTest.kt similarity index 58% rename from kotlinx-coroutines-test/test/TestCoroutineDispatcherOrderTest.kt rename to kotlinx-coroutines-test/common/test/TestCoroutineDispatcherOrderTest.kt index 116aadcf8d..c3686845de 100644 --- a/kotlinx-coroutines-test/test/TestCoroutineDispatcherOrderTest.kt +++ b/kotlinx-coroutines-test/common/test/TestCoroutineDispatcherOrderTest.kt @@ -1,11 +1,28 @@ +/* + * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + package kotlinx.coroutines.test +import kotlinx.atomicfu.* import kotlinx.coroutines.* -import org.junit.* -import kotlin.coroutines.* -import kotlin.test.assertEquals +import kotlin.test.* + +class TestCoroutineDispatcherOrderTest { + + private val actionIndex = atomic(0) + private val finished = atomic(false) -class TestCoroutineDispatcherOrderTest : TestBase() { + private fun expect(index: Int) { + val wasIndex = actionIndex.incrementAndGet() + // println("expect($index), wasIndex=$wasIndex") + check(index == wasIndex) { "Expecting action index $index but it is actually $wasIndex" } + } + + private fun finish(index: Int) { + expect(index) + check(!finished.getAndSet(true)) { "Should call 'finish(...)' at most once" } + } @Test fun testAdvanceTimeBy_progressesOnEachDelay() { diff --git a/kotlinx-coroutines-test/test/TestCoroutineDispatcherTest.kt b/kotlinx-coroutines-test/common/test/TestCoroutineDispatcherTest.kt similarity index 69% rename from kotlinx-coroutines-test/test/TestCoroutineDispatcherTest.kt rename to kotlinx-coroutines-test/common/test/TestCoroutineDispatcherTest.kt index 260edf9dc8..baf946f2e1 100644 --- a/kotlinx-coroutines-test/test/TestCoroutineDispatcherTest.kt +++ b/kotlinx-coroutines-test/common/test/TestCoroutineDispatcherTest.kt @@ -1,11 +1,10 @@ /* - * Copyright 2016-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. */ package kotlinx.coroutines.test import kotlinx.coroutines.* -import org.junit.Test import kotlin.test.* class TestCoroutineDispatcherTest { @@ -91,7 +90,7 @@ class TestCoroutineDispatcherTest { assertEquals(1, executed) } - @Test(expected = UncompletedCoroutinesError::class) + @Test fun whenDispatcherHasUncompletedCoroutines_itThrowsErrorInCleanup() { val subject = TestCoroutineDispatcher() subject.pauseDispatcher() @@ -99,44 +98,7 @@ class TestCoroutineDispatcherTest { scope.launch { delay(1_000) } - subject.cleanupTestCoroutines() - } - - @Test - fun whenDispatchCalled_runsOnCurrentThread() { - val currentThread = Thread.currentThread() - val subject = TestCoroutineDispatcher() - val scope = TestCoroutineScope(subject) - - val deferred = scope.async(Dispatchers.Default) { - withContext(subject) { - assertNotSame(currentThread, Thread.currentThread()) - 3 - } - } - - runBlocking { - // just to ensure the above code terminates - assertEquals(3, deferred.await()) - } + assertFailsWith { subject.cleanupTestCoroutines() } } - @Test - fun whenAllDispatchersMocked_runsOnSameThread() { - val currentThread = Thread.currentThread() - val subject = TestCoroutineDispatcher() - val scope = TestCoroutineScope(subject) - - val deferred = scope.async(subject) { - withContext(subject) { - assertSame(currentThread, Thread.currentThread()) - 3 - } - } - - runBlocking { - // just to ensure the above code terminates - assertEquals(3, deferred.await()) - } - } } \ No newline at end of file diff --git a/kotlinx-coroutines-test/test/TestCoroutineExceptionHandlerTest.kt b/kotlinx-coroutines-test/common/test/TestCoroutineExceptionHandlerTest.kt similarity index 85% rename from kotlinx-coroutines-test/test/TestCoroutineExceptionHandlerTest.kt rename to kotlinx-coroutines-test/common/test/TestCoroutineExceptionHandlerTest.kt index 1a0833af50..674fd288dd 100644 --- a/kotlinx-coroutines-test/test/TestCoroutineExceptionHandlerTest.kt +++ b/kotlinx-coroutines-test/common/test/TestCoroutineExceptionHandlerTest.kt @@ -4,12 +4,11 @@ package kotlinx.coroutines.test -import org.junit.Test import kotlin.test.* class TestCoroutineExceptionHandlerTest { @Test - fun whenExceptionsCaught_avaliableViaProperty() { + fun whenExceptionsCaught_availableViaProperty() { val subject = TestCoroutineExceptionHandler() val expected = IllegalArgumentException() subject.handleException(subject, expected) diff --git a/kotlinx-coroutines-test/test/TestCoroutineScopeTest.kt b/kotlinx-coroutines-test/common/test/TestCoroutineScopeTest.kt similarity index 74% rename from kotlinx-coroutines-test/test/TestCoroutineScopeTest.kt rename to kotlinx-coroutines-test/common/test/TestCoroutineScopeTest.kt index fa14c38409..4480cd99a3 100644 --- a/kotlinx-coroutines-test/test/TestCoroutineScopeTest.kt +++ b/kotlinx-coroutines-test/common/test/TestCoroutineScopeTest.kt @@ -5,13 +5,12 @@ package kotlinx.coroutines.test import kotlinx.coroutines.* -import org.junit.Test import kotlin.test.* class TestCoroutineScopeTest { @Test fun whenGivenInvalidExceptionHandler_throwsException() { - val handler = CoroutineExceptionHandler { _, _ -> Unit } + val handler = CoroutineExceptionHandler { _, _ -> } assertFails { TestCoroutineScope(handler) } @@ -20,7 +19,7 @@ class TestCoroutineScopeTest { @Test fun whenGivenInvalidDispatcher_throwsException() { assertFails { - TestCoroutineScope(newSingleThreadContext("incorrect call")) + TestCoroutineScope(Dispatchers.Default) } } } diff --git a/kotlinx-coroutines-test/test/TestModuleHelpers.kt b/kotlinx-coroutines-test/common/test/TestModuleHelpers.kt similarity index 60% rename from kotlinx-coroutines-test/test/TestModuleHelpers.kt rename to kotlinx-coroutines-test/common/test/TestModuleHelpers.kt index 12541bd90f..a34dbfd6c7 100644 --- a/kotlinx-coroutines-test/test/TestModuleHelpers.kt +++ b/kotlinx-coroutines-test/common/test/TestModuleHelpers.kt @@ -5,18 +5,21 @@ package kotlinx.coroutines.test import kotlinx.coroutines.* -import org.junit.* -import java.time.* +import kotlin.test.* +import kotlin.time.* const val SLOW = 10_000L /** * Assert a block completes within a second or fail the suite */ +@OptIn(ExperimentalTime::class) suspend fun CoroutineScope.assertRunsFast(block: suspend CoroutineScope.() -> Unit) { - val start = Instant.now().toEpochMilli() + val start = TimeSource.Monotonic.markNow() // don't need to be fancy with timeouts here since anything longer than a few ms is an error block() - val duration = Instant.now().minusMillis(start).toEpochMilli() - Assert.assertTrue("All tests must complete within 2000ms (use longer timeouts to cause failure)", duration < 2_000) + val duration = start.elapsedNow() + assertTrue("All tests must complete within 2000ms (use longer timeouts to cause failure)") { + duration.inWholeSeconds < 2 + } } diff --git a/kotlinx-coroutines-test/test/TestRunBlockingOrderTest.kt b/kotlinx-coroutines-test/common/test/TestRunBlockingOrderTest.kt similarity index 70% rename from kotlinx-coroutines-test/test/TestRunBlockingOrderTest.kt rename to kotlinx-coroutines-test/common/test/TestRunBlockingOrderTest.kt index e21c82b95c..064cb7cb71 100644 --- a/kotlinx-coroutines-test/test/TestRunBlockingOrderTest.kt +++ b/kotlinx-coroutines-test/common/test/TestRunBlockingOrderTest.kt @@ -1,14 +1,29 @@ /* - * Copyright 2016-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. */ package kotlinx.coroutines.test +import kotlinx.atomicfu.* import kotlinx.coroutines.* -import org.junit.* -import kotlin.coroutines.* +import kotlin.test.* + +class TestRunBlockingOrderTest { + + private val actionIndex = atomic(0) + private val finished = atomic(false) + + private fun expect(index: Int) { + val wasIndex = actionIndex.incrementAndGet() + // println("expect($index), wasIndex=$wasIndex") + check(index == wasIndex) { "Expecting action index $index but it is actually $wasIndex" } + } + + private fun finish(index: Int) { + expect(index) + check(!finished.getAndSet(true)) { "Should call 'finish(...)' at most once" } + } -class TestRunBlockingOrderTest : TestBase() { @Test fun testLaunchImmediate() = runBlockingTest { expect(1) diff --git a/kotlinx-coroutines-test/test/TestRunBlockingTest.kt b/kotlinx-coroutines-test/common/test/TestRunBlockingTest.kt similarity index 56% rename from kotlinx-coroutines-test/test/TestRunBlockingTest.kt rename to kotlinx-coroutines-test/common/test/TestRunBlockingTest.kt index e0c7091505..c93b50811f 100644 --- a/kotlinx-coroutines-test/test/TestRunBlockingTest.kt +++ b/kotlinx-coroutines-test/common/test/TestRunBlockingTest.kt @@ -5,7 +5,6 @@ package kotlinx.coroutines.test import kotlinx.coroutines.* -import kotlin.coroutines.* import kotlin.test.* class TestRunBlockingTest { @@ -53,22 +52,14 @@ class TestRunBlockingTest { } @Test - fun incorrectlyCalledRunblocking_doesNotHaveSameInterceptor() = runBlockingTest { - // this code is an error as a production test, please do not use this as an example - - // this test exists to document this error condition, if it's possible to make this code work please update - val outerInterceptor = coroutineContext[ContinuationInterceptor] - // runBlocking always requires an argument to pass the context in tests - runBlocking { - assertNotSame(coroutineContext[ContinuationInterceptor], outerInterceptor) - } - } - - @Test(expected = TimeoutCancellationException::class) - fun whenUsingTimeout_triggersWhenDelayed() = runBlockingTest { - assertRunsFast { - withTimeout(SLOW) { - delay(SLOW) + fun whenUsingTimeout_triggersWhenDelayed() { + assertFailsWith { + runBlockingTest { + assertRunsFast { + withTimeout(SLOW) { + delay(SLOW) + } + } } } } @@ -82,12 +73,16 @@ class TestRunBlockingTest { } } - @Test(expected = TimeoutCancellationException::class) - fun whenUsingTimeout_triggersWhenWaiting() = runBlockingTest { - val uncompleted = CompletableDeferred() - assertRunsFast { - withTimeout(SLOW) { - uncompleted.await() + @Test + fun whenUsingTimeout_triggersWhenWaiting() { + assertFailsWith { + runBlockingTest { + val uncompleted = CompletableDeferred() + assertRunsFast { + withTimeout(SLOW) { + uncompleted.await() + } + } } } } @@ -114,16 +109,20 @@ class TestRunBlockingTest { } } - @Test(expected = TimeoutCancellationException::class) - fun whenUsingTimeout_inAsync_triggersWhenDelayed() = runBlockingTest { - val deferred = async { - withTimeout(SLOW) { - delay(SLOW) - } - } + @Test + fun whenUsingTimeout_inAsync_triggersWhenDelayed() { + assertFailsWith { + runBlockingTest { + val deferred = async { + withTimeout(SLOW) { + delay(SLOW) + } + } - assertRunsFast { - deferred.await() + assertRunsFast { + deferred.await() + } + } } } @@ -141,18 +140,21 @@ class TestRunBlockingTest { } } - @Test(expected = TimeoutCancellationException::class) - fun whenUsingTimeout_inLaunch_triggersWhenDelayed() = runBlockingTest { - val job= launch { - withTimeout(1) { - delay(SLOW + 1) - 3 - } - } + @Test + fun whenUsingTimeout_inLaunch_triggersWhenDelayed() { + assertFailsWith { + runBlockingTest { + val job = launch { + withTimeout(1) { + delay(SLOW + 1) + } + } - assertRunsFast { - job.join() - throw job.getCancellationException() + assertRunsFast { + job.join() + throw job.getCancellationException() + } + } } } @@ -170,36 +172,48 @@ class TestRunBlockingTest { } } - @Test(expected = IllegalArgumentException::class) - fun throwingException_throws() = runBlockingTest { - assertRunsFast { - delay(SLOW) - throw IllegalArgumentException("Test") + @Test + fun throwingException_throws() { + assertFailsWith { + runBlockingTest { + assertRunsFast { + delay(SLOW) + throw IllegalArgumentException("Test") + } + } } } - @Test(expected = IllegalArgumentException::class) - fun throwingException_inLaunch_throws() = runBlockingTest { - val job = launch { - delay(SLOW) - throw IllegalArgumentException("Test") - } + @Test + fun throwingException_inLaunch_throws() { + assertFailsWith { + runBlockingTest { + val job = launch { + delay(SLOW) + throw IllegalArgumentException("Test") + } - assertRunsFast { - job.join() - throw job.getCancellationException().cause ?: assertFails { "expected exception" } + assertRunsFast { + job.join() + throw job.getCancellationException().cause ?: assertFails { "expected exception" } + } + } } } - @Test(expected = IllegalArgumentException::class) - fun throwingException__inAsync_throws() = runBlockingTest { - val deferred = async { - delay(SLOW) - throw IllegalArgumentException("Test") - } + @Test + fun throwingException__inAsync_throws() { + assertFailsWith { + runBlockingTest { + val deferred: Deferred = async { + delay(SLOW) + throw IllegalArgumentException("Test") + } - assertRunsFast { - deferred.await() + assertRunsFast { + deferred.await() + } + } } } @@ -273,25 +287,33 @@ class TestRunBlockingTest { job.join() } - @Test(expected = UncompletedCoroutinesError::class) - fun whenACoroutineLeaks_errorIsThrown() = runBlockingTest { - val uncompleted = CompletableDeferred() - launch { - uncompleted.await() + @Test + fun whenACoroutineLeaks_errorIsThrown() { + assertFailsWith { + runBlockingTest { + val uncompleted = CompletableDeferred() + launch { + uncompleted.await() + } + } } } - @Test(expected = java.lang.IllegalArgumentException::class) + @Test fun runBlockingTestBuilder_throwsOnBadDispatcher() { - runBlockingTest(newSingleThreadContext("name")) { + assertFailsWith { + runBlockingTest(Dispatchers.Default) { + } } } - @Test(expected = java.lang.IllegalArgumentException::class) + @Test fun runBlockingTestBuilder_throwsOnBadHandler() { - runBlockingTest(CoroutineExceptionHandler { _, _ -> Unit} ) { + assertFailsWith { + runBlockingTest(CoroutineExceptionHandler { _, _ -> }) { + } } } @@ -338,36 +360,48 @@ class TestRunBlockingTest { } - @Test(expected = IllegalAccessError::class) - fun testWithTestContextThrowingAnAssertionError() = runBlockingTest { - val expectedError = IllegalAccessError("hello") + @Test + fun testWithTestContextThrowingAnAssertionError() { + assertFailsWith { + runBlockingTest { + val expectedError = TestException("hello") - val job = launch { - throw expectedError - } + val job = launch { + throw expectedError + } - // don't rethrow or handle the exception + // don't rethrow or handle the exception + } + } } - @Test(expected = IllegalAccessError::class) - fun testExceptionHandlingWithLaunch() = runBlockingTest { - val expectedError = IllegalAccessError("hello") + @Test + fun testExceptionHandlingWithLaunch() { + assertFailsWith { + runBlockingTest { + val expectedError = TestException("hello") - launch { - throw expectedError + launch { + throw expectedError + } + } } } - @Test(expected = IllegalAccessError::class) - fun testExceptions_notThrownImmediately() = runBlockingTest { - val expectedException = IllegalAccessError("hello") - val result = runCatching { - launch { - throw expectedException + @Test + fun testExceptions_notThrownImmediately() { + assertFailsWith { + runBlockingTest { + val expectedException = TestException("hello") + val result = runCatching { + launch { + throw expectedException + } + } + runCurrent() + assertEquals(true, result.isSuccess) } } - runCurrent() - assertEquals(true, result.isSuccess) } @@ -380,9 +414,13 @@ class TestRunBlockingTest { assertNotSame(coroutineContext[CoroutineExceptionHandler], exceptionHandler) } - @Test(expected = IllegalArgumentException::class) - fun testPartialDispatcherOverride() = runBlockingTest(Dispatchers.Unconfined) { - fail("Unreached") + @Test + fun testPartialDispatcherOverride() { + assertFailsWith { + runBlockingTest(Dispatchers.Unconfined) { + fail("Unreached") + } + } } @Test @@ -390,8 +428,14 @@ class TestRunBlockingTest { assertSame(coroutineContext[CoroutineExceptionHandler], exceptionHandler) } - @Test(expected = IllegalArgumentException::class) - fun testOverrideExceptionHandlerError() = runBlockingTest(CoroutineExceptionHandler { _, _ -> }) { - fail("Unreached") + @Test + fun testOverrideExceptionHandlerError() { + assertFailsWith { + runBlockingTest(CoroutineExceptionHandler { _, _ -> }) { + fail("Unreached") + } + } } } + +private class TestException(message: String? = null): Exception(message) \ No newline at end of file diff --git a/kotlinx-coroutines-test/js/src/TestDispatchers.kt b/kotlinx-coroutines-test/js/src/TestDispatchers.kt new file mode 100644 index 0000000000..10322079d3 --- /dev/null +++ b/kotlinx-coroutines-test/js/src/TestDispatchers.kt @@ -0,0 +1,16 @@ +/* + * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines.test +import kotlinx.coroutines.* + +@ExperimentalCoroutinesApi +public actual fun Dispatchers.setMain(dispatcher: CoroutineDispatcher) { + throw UnsupportedOperationException("`setMain` is not supported on JS") +} + +@ExperimentalCoroutinesApi +public actual fun Dispatchers.resetMain() { + throw UnsupportedOperationException("`resetMain` is not supported on JS") +} diff --git a/kotlinx-coroutines-test/resources/META-INF/proguard/coroutines.pro b/kotlinx-coroutines-test/jvm/resources/META-INF/proguard/coroutines.pro similarity index 100% rename from kotlinx-coroutines-test/resources/META-INF/proguard/coroutines.pro rename to kotlinx-coroutines-test/jvm/resources/META-INF/proguard/coroutines.pro diff --git a/kotlinx-coroutines-test/resources/META-INF/services/kotlinx.coroutines.internal.MainDispatcherFactory b/kotlinx-coroutines-test/jvm/resources/META-INF/services/kotlinx.coroutines.internal.MainDispatcherFactory similarity index 100% rename from kotlinx-coroutines-test/resources/META-INF/services/kotlinx.coroutines.internal.MainDispatcherFactory rename to kotlinx-coroutines-test/jvm/resources/META-INF/services/kotlinx.coroutines.internal.MainDispatcherFactory diff --git a/kotlinx-coroutines-test/jvm/src/TestDispatchers.kt b/kotlinx-coroutines-test/jvm/src/TestDispatchers.kt new file mode 100644 index 0000000000..800eca5d1d --- /dev/null +++ b/kotlinx-coroutines-test/jvm/src/TestDispatchers.kt @@ -0,0 +1,24 @@ +/* + * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ +@file:JvmName("TestDispatchers") + +package kotlinx.coroutines.test + +import kotlinx.coroutines.* +import kotlinx.coroutines.test.internal.* + +@ExperimentalCoroutinesApi +public actual fun Dispatchers.setMain(dispatcher: CoroutineDispatcher) { + require(dispatcher !is TestMainDispatcher) { "Dispatchers.setMain(Dispatchers.Main) is prohibited, probably Dispatchers.resetMain() should be used instead" } + val mainDispatcher = Main + require(mainDispatcher is TestMainDispatcher) { "TestMainDispatcher is not set as main dispatcher, have $mainDispatcher instead." } + mainDispatcher.setDispatcher(dispatcher) +} + +@ExperimentalCoroutinesApi +public actual fun Dispatchers.resetMain() { + val mainDispatcher = Main + require(mainDispatcher is TestMainDispatcher) { "TestMainDispatcher is not set as main dispatcher, have $mainDispatcher instead." } + mainDispatcher.resetDispatcher() +} diff --git a/kotlinx-coroutines-test/src/internal/MainTestDispatcher.kt b/kotlinx-coroutines-test/jvm/src/internal/TestMainDispatcher.kt similarity index 95% rename from kotlinx-coroutines-test/src/internal/MainTestDispatcher.kt rename to kotlinx-coroutines-test/jvm/src/internal/TestMainDispatcher.kt index c85d27ea87..4c70ca3fcf 100644 --- a/kotlinx-coroutines-test/src/internal/MainTestDispatcher.kt +++ b/kotlinx-coroutines-test/jvm/src/internal/TestMainDispatcher.kt @@ -69,7 +69,7 @@ internal class TestMainDispatcherFactory : MainDispatcherFactory { /** * [Int.MAX_VALUE] -- test dispatcher always wins no matter what factories are present in the classpath. - * By default all actions are delegated to the second-priority dispatcher, so that it won't be the issue. + * By default, all actions are delegated to the second-priority dispatcher, so that it won't be the issue. */ override val loadPriority: Int get() = Int.MAX_VALUE diff --git a/kotlinx-coroutines-test/jvm/test/MultithreadingTest.kt b/kotlinx-coroutines-test/jvm/test/MultithreadingTest.kt new file mode 100644 index 0000000000..d06f2a35c6 --- /dev/null +++ b/kotlinx-coroutines-test/jvm/test/MultithreadingTest.kt @@ -0,0 +1,89 @@ +/* + * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +import kotlinx.coroutines.* +import kotlinx.coroutines.test.* +import kotlin.coroutines.* +import kotlin.test.* + +class MultithreadingTest : TestBase() { + + @Test + fun incorrectlyCalledRunblocking_doesNotHaveSameInterceptor() = runBlockingTest { + // this code is an error as a production test, please do not use this as an example + + // this test exists to document this error condition, if it's possible to make this code work please update + val outerInterceptor = coroutineContext[ContinuationInterceptor] + // runBlocking always requires an argument to pass the context in tests + runBlocking { + assertNotSame(coroutineContext[ContinuationInterceptor], outerInterceptor) + } + } + + @Test + fun testSingleThreadExecutor() = runTest { + val mainThread = Thread.currentThread() + Dispatchers.setMain(Dispatchers.Unconfined) + newSingleThreadContext("testSingleThread").use { threadPool -> + withContext(Dispatchers.Main) { + assertSame(mainThread, Thread.currentThread()) + } + + Dispatchers.setMain(threadPool) + withContext(Dispatchers.Main) { + assertNotSame(mainThread, Thread.currentThread()) + } + assertSame(mainThread, Thread.currentThread()) + + withContext(Dispatchers.Main.immediate) { + assertNotSame(mainThread, Thread.currentThread()) + } + assertSame(mainThread, Thread.currentThread()) + + Dispatchers.setMain(Dispatchers.Unconfined) + withContext(Dispatchers.Main.immediate) { + assertSame(mainThread, Thread.currentThread()) + } + assertSame(mainThread, Thread.currentThread()) + } + } + + @Test + fun whenDispatchCalled_runsOnCurrentThread() { + val currentThread = Thread.currentThread() + val subject = TestCoroutineDispatcher() + val scope = TestCoroutineScope(subject) + + val deferred = scope.async(Dispatchers.Default) { + withContext(subject) { + assertNotSame(currentThread, Thread.currentThread()) + 3 + } + } + + runBlocking { + // just to ensure the above code terminates + assertEquals(3, deferred.await()) + } + } + + @Test + fun whenAllDispatchersMocked_runsOnSameThread() { + val currentThread = Thread.currentThread() + val subject = TestCoroutineDispatcher() + val scope = TestCoroutineScope(subject) + + val deferred = scope.async(subject) { + withContext(subject) { + assertSame(currentThread, Thread.currentThread()) + 3 + } + } + + runBlocking { + // just to ensure the above code terminates + assertEquals(3, deferred.await()) + } + } +} \ No newline at end of file diff --git a/kotlinx-coroutines-test/jvm/test/TestDispatchersTest.kt b/kotlinx-coroutines-test/jvm/test/TestDispatchersTest.kt new file mode 100644 index 0000000000..3ab78987f0 --- /dev/null +++ b/kotlinx-coroutines-test/jvm/test/TestDispatchersTest.kt @@ -0,0 +1,72 @@ +/* + * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +import kotlinx.atomicfu.* +import kotlinx.coroutines.* +import kotlinx.coroutines.test.* +import kotlin.coroutines.* +import kotlin.test.* + +class TestDispatchersTest { + private val actionIndex = atomic(0) + private val finished = atomic(false) + + private fun expect(index: Int) { + val wasIndex = actionIndex.incrementAndGet() + println("expect($index), wasIndex=$wasIndex") + check(index == wasIndex) { "Expecting action index $index but it is actually $wasIndex" } + } + + private fun finish(index: Int) { + expect(index) + check(!finished.getAndSet(true)) { "Should call 'finish(...)' at most once" } + } + + @BeforeTest + fun setUp() { + Dispatchers.resetMain() + } + + @Test + fun testSelfSet() { + assertFailsWith { Dispatchers.setMain(Dispatchers.Main) } + } + + @Test + fun testImmediateDispatcher() = runBlockingTest { + Dispatchers.setMain(ImmediateDispatcher()) + expect(1) + withContext(Dispatchers.Main) { + expect(3) + } + + Dispatchers.setMain(RegularDispatcher()) + withContext(Dispatchers.Main) { + expect(6) + } + + finish(7) + } + + private inner class ImmediateDispatcher : CoroutineDispatcher() { + override fun isDispatchNeeded(context: CoroutineContext): Boolean { + expect(2) + return false + } + + override fun dispatch(context: CoroutineContext, block: Runnable) = throw RuntimeException("Shouldn't be reached") + } + + private inner class RegularDispatcher : CoroutineDispatcher() { + override fun isDispatchNeeded(context: CoroutineContext): Boolean { + expect(4) + return true + } + + override fun dispatch(context: CoroutineContext, block: Runnable) { + expect(5) + block.run() + } + } +} diff --git a/kotlinx-coroutines-test/native/src/TestDispatchers.kt b/kotlinx-coroutines-test/native/src/TestDispatchers.kt new file mode 100644 index 0000000000..44c7a72752 --- /dev/null +++ b/kotlinx-coroutines-test/native/src/TestDispatchers.kt @@ -0,0 +1,17 @@ +/* + * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines.test + +import kotlinx.coroutines.* + +@ExperimentalCoroutinesApi +public actual fun Dispatchers.setMain(dispatcher: CoroutineDispatcher) { + throw UnsupportedOperationException("`setMain` is not supported on Native") +} + +@ExperimentalCoroutinesApi +public actual fun Dispatchers.resetMain() { + throw UnsupportedOperationException("`resetMain` is not supported on Native") +} \ No newline at end of file diff --git a/kotlinx-coroutines-test/npm/README.md b/kotlinx-coroutines-test/npm/README.md new file mode 100644 index 0000000000..4df4825da9 --- /dev/null +++ b/kotlinx-coroutines-test/npm/README.md @@ -0,0 +1,4 @@ +# kotlinx-coroutines-test + +Testing support for `kotlinx-coroutines` in +[Kotlin/JS](https://kotlinlang.org/docs/js-overview.html). diff --git a/kotlinx-coroutines-test/npm/package.json b/kotlinx-coroutines-test/npm/package.json new file mode 100644 index 0000000000..b59d92fe03 --- /dev/null +++ b/kotlinx-coroutines-test/npm/package.json @@ -0,0 +1,23 @@ +{ + "name": "kotlinx-coroutines-test", + "version" : "$version", + "description" : "Test utilities for kotlinx-coroutines", + "main" : "kotlinx-coroutines-test.js", + "author": "JetBrains", + "license": "Apache-2.0", + "homepage": "https://github.com/Kotlin/kotlinx.coroutines", + "bugs": { + "url": "https://github.com/Kotlin/kotlinx.coroutines/issues" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/Kotlin/kotlinx.coroutines.git" + }, + "keywords": [ + "Kotlin", + "async", + "coroutines", + "JetBrains", + "test" + ] +} diff --git a/kotlinx-coroutines-test/src/TestDispatchers.kt b/kotlinx-coroutines-test/src/TestDispatchers.kt deleted file mode 100644 index bf068f9d7b..0000000000 --- a/kotlinx-coroutines-test/src/TestDispatchers.kt +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. - */ -@file:Suppress("unused") -@file:JvmName("TestDispatchers") - -package kotlinx.coroutines.test - -import kotlinx.coroutines.* -import kotlinx.coroutines.test.internal.* - -/** - * Sets the given [dispatcher] as an underlying dispatcher of [Dispatchers.Main]. - * All consecutive usages of [Dispatchers.Main] will use given [dispatcher] under the hood. - * - * It is unsafe to call this method if alive coroutines launched in [Dispatchers.Main] exist. - */ -@ExperimentalCoroutinesApi -public fun Dispatchers.setMain(dispatcher: CoroutineDispatcher) { - require(dispatcher !is TestMainDispatcher) { "Dispatchers.setMain(Dispatchers.Main) is prohibited, probably Dispatchers.resetMain() should be used instead" } - val mainDispatcher = Dispatchers.Main - require(mainDispatcher is TestMainDispatcher) { "TestMainDispatcher is not set as main dispatcher, have $mainDispatcher instead." } - mainDispatcher.setDispatcher(dispatcher) -} - -/** - * Resets state of the [Dispatchers.Main] to the original main dispatcher. - * For example, in Android Main thread dispatcher will be set as [Dispatchers.Main]. - * Used to clean up all possible dependencies, should be used in tear down (`@After`) methods. - * - * It is unsafe to call this method if alive coroutines launched in [Dispatchers.Main] exist. - */ -@ExperimentalCoroutinesApi -public fun Dispatchers.resetMain() { - val mainDispatcher = Dispatchers.Main - require(mainDispatcher is TestMainDispatcher) { "TestMainDispatcher is not set as main dispatcher, have $mainDispatcher instead." } - mainDispatcher.resetDispatcher() -} diff --git a/kotlinx-coroutines-test/test/TestDispatchersTest.kt b/kotlinx-coroutines-test/test/TestDispatchersTest.kt deleted file mode 100644 index 98d9705311..0000000000 --- a/kotlinx-coroutines-test/test/TestDispatchersTest.kt +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright 2016-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. - */ - -package kotlinx.coroutines.test - -import kotlinx.coroutines.* -import org.junit.* -import org.junit.Test -import kotlin.coroutines.* -import kotlin.test.* - -class TestDispatchersTest : TestBase() { - - @Before - fun setUp() { - Dispatchers.resetMain() - } - - @Test(expected = IllegalArgumentException::class) - fun testSelfSet() = runTest { - Dispatchers.setMain(Dispatchers.Main) - } - - @Test - fun testSingleThreadExecutor() = runTest { - val mainThread = Thread.currentThread() - Dispatchers.setMain(Dispatchers.Unconfined) - newSingleThreadContext("testSingleThread").use { threadPool -> - withContext(Dispatchers.Main) { - assertSame(mainThread, Thread.currentThread()) - } - - Dispatchers.setMain(threadPool) - withContext(Dispatchers.Main) { - assertNotSame(mainThread, Thread.currentThread()) - } - assertSame(mainThread, Thread.currentThread()) - - withContext(Dispatchers.Main.immediate) { - assertNotSame(mainThread, Thread.currentThread()) - } - assertSame(mainThread, Thread.currentThread()) - - Dispatchers.setMain(Dispatchers.Unconfined) - withContext(Dispatchers.Main.immediate) { - assertSame(mainThread, Thread.currentThread()) - } - assertSame(mainThread, Thread.currentThread()) - } - } - - @Test - fun testImmediateDispatcher() = runTest { - Dispatchers.setMain(ImmediateDispatcher()) - expect(1) - withContext(Dispatchers.Main) { - expect(3) - } - - Dispatchers.setMain(RegularDispatcher()) - withContext(Dispatchers.Main) { - expect(6) - } - - finish(7) - } - - private inner class ImmediateDispatcher : CoroutineDispatcher() { - override fun isDispatchNeeded(context: CoroutineContext): Boolean { - expect(2) - return false - } - - override fun dispatch(context: CoroutineContext, block: Runnable) = expectUnreached() - } - - private inner class RegularDispatcher : CoroutineDispatcher() { - override fun isDispatchNeeded(context: CoroutineContext): Boolean { - expect(4) - return true - } - - override fun dispatch(context: CoroutineContext, block: Runnable) { - expect(5) - block.run() - } - } -} From fbbe63b2ce0e2545b75e0846da5b07ae8258846d Mon Sep 17 00:00:00 2001 From: dkhalanskyjb <52952525+dkhalanskyjb@users.noreply.github.com> Date: Fri, 8 Oct 2021 12:38:34 +0300 Subject: [PATCH 18/24] Implement `setMain` in common code (#2967) Fixes #1720 --- kotlinx-coroutines-core/js/src/Dispatchers.kt | 16 +++- .../native/src/Dispatchers.kt | 18 ++++- .../common/src/TestDispatchers.kt | 12 ++- .../common/src/internal/TestMainDispatcher.kt | 42 ++++++++++ .../test/TestCoroutineDispatcherOrderTest.kt | 1 - .../test/TestDispatchersTest.kt | 3 +- .../common/test/TestRunBlockingOrderTest.kt | 1 - .../js/src/TestDispatchers.kt | 16 ---- .../js/src/internal/TestMainDispatcher.kt | 13 ++++ .../jvm/src/TestDispatchers.kt | 24 ------ .../jvm/src/internal/TestMainDispatcher.kt | 76 ------------------- .../jvm/src/internal/TestMainDispatcherJvm.kt | 31 ++++++++ .../native/src/TestDispatchers.kt | 17 ----- .../native/src/TestMainDispatcher.kt | 13 ++++ 14 files changed, 142 insertions(+), 141 deletions(-) create mode 100644 kotlinx-coroutines-test/common/src/internal/TestMainDispatcher.kt rename kotlinx-coroutines-test/{jvm => common}/test/TestDispatchersTest.kt (97%) delete mode 100644 kotlinx-coroutines-test/js/src/TestDispatchers.kt create mode 100644 kotlinx-coroutines-test/js/src/internal/TestMainDispatcher.kt delete mode 100644 kotlinx-coroutines-test/jvm/src/TestDispatchers.kt delete mode 100644 kotlinx-coroutines-test/jvm/src/internal/TestMainDispatcher.kt create mode 100644 kotlinx-coroutines-test/jvm/src/internal/TestMainDispatcherJvm.kt delete mode 100644 kotlinx-coroutines-test/native/src/TestDispatchers.kt create mode 100644 kotlinx-coroutines-test/native/src/TestMainDispatcher.kt diff --git a/kotlinx-coroutines-core/js/src/Dispatchers.kt b/kotlinx-coroutines-core/js/src/Dispatchers.kt index 8d3bac3209..3eec5408cc 100644 --- a/kotlinx-coroutines-core/js/src/Dispatchers.kt +++ b/kotlinx-coroutines-core/js/src/Dispatchers.kt @@ -8,8 +8,22 @@ import kotlin.coroutines.* public actual object Dispatchers { public actual val Default: CoroutineDispatcher = createDefaultDispatcher() - public actual val Main: MainCoroutineDispatcher = JsMainDispatcher(Default, false) + public actual val Main: MainCoroutineDispatcher + get() = injectedMainDispatcher ?: mainDispatcher public actual val Unconfined: CoroutineDispatcher = kotlinx.coroutines.Unconfined + + private val mainDispatcher = JsMainDispatcher(Default, false) + private var injectedMainDispatcher: MainCoroutineDispatcher? = null + + @PublishedApi + internal fun injectMain(dispatcher: MainCoroutineDispatcher) { + injectedMainDispatcher = dispatcher + } + + @PublishedApi + internal fun resetInjectedMain() { + injectedMainDispatcher = null + } } private class JsMainDispatcher( diff --git a/kotlinx-coroutines-core/native/src/Dispatchers.kt b/kotlinx-coroutines-core/native/src/Dispatchers.kt index 4e5facfeee..904483c2c0 100644 --- a/kotlinx-coroutines-core/native/src/Dispatchers.kt +++ b/kotlinx-coroutines-core/native/src/Dispatchers.kt @@ -6,10 +6,26 @@ package kotlinx.coroutines import kotlin.coroutines.* +/** Not inside [Dispatchers], as otherwise mutating this throws an `InvalidMutabilityException`. */ +private var injectedMainDispatcher: MainCoroutineDispatcher? = null + public actual object Dispatchers { public actual val Default: CoroutineDispatcher = createDefaultDispatcher() - public actual val Main: MainCoroutineDispatcher = NativeMainDispatcher(Default) + public actual val Main: MainCoroutineDispatcher + get() = injectedMainDispatcher ?: mainDispatcher public actual val Unconfined: CoroutineDispatcher get() = kotlinx.coroutines.Unconfined // Avoid freezing + + private val mainDispatcher = NativeMainDispatcher(Default) + + @PublishedApi + internal fun injectMain(dispatcher: MainCoroutineDispatcher) { + injectedMainDispatcher = dispatcher + } + + @PublishedApi + internal fun resetInjectedMain() { + injectedMainDispatcher = null + } } private class NativeMainDispatcher(val delegate: CoroutineDispatcher) : MainCoroutineDispatcher() { diff --git a/kotlinx-coroutines-test/common/src/TestDispatchers.kt b/kotlinx-coroutines-test/common/src/TestDispatchers.kt index 2f00331506..f8896d7278 100644 --- a/kotlinx-coroutines-test/common/src/TestDispatchers.kt +++ b/kotlinx-coroutines-test/common/src/TestDispatchers.kt @@ -1,10 +1,13 @@ /* * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. */ +@file:JvmName("TestDispatchers") package kotlinx.coroutines.test import kotlinx.coroutines.* +import kotlinx.coroutines.test.internal.* +import kotlin.jvm.* /** * Sets the given [dispatcher] as an underlying dispatcher of [Dispatchers.Main]. @@ -13,7 +16,10 @@ import kotlinx.coroutines.* * It is unsafe to call this method if alive coroutines launched in [Dispatchers.Main] exist. */ @ExperimentalCoroutinesApi -public expect fun Dispatchers.setMain(dispatcher: CoroutineDispatcher) +public fun Dispatchers.setMain(dispatcher: CoroutineDispatcher) { + require(dispatcher !is TestMainDispatcher) { "Dispatchers.setMain(Dispatchers.Main) is prohibited, probably Dispatchers.resetMain() should be used instead" } + getTestMainDispatcher().setDispatcher(dispatcher) +} /** * Resets state of the [Dispatchers.Main] to the original main dispatcher. @@ -23,4 +29,6 @@ public expect fun Dispatchers.setMain(dispatcher: CoroutineDispatcher) * It is unsafe to call this method if alive coroutines launched in [Dispatchers.Main] exist. */ @ExperimentalCoroutinesApi -public expect fun Dispatchers.resetMain() +public fun Dispatchers.resetMain() { + getTestMainDispatcher().resetDispatcher() +} diff --git a/kotlinx-coroutines-test/common/src/internal/TestMainDispatcher.kt b/kotlinx-coroutines-test/common/src/internal/TestMainDispatcher.kt new file mode 100644 index 0000000000..f2e5b7a168 --- /dev/null +++ b/kotlinx-coroutines-test/common/src/internal/TestMainDispatcher.kt @@ -0,0 +1,42 @@ +/* + * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines.test.internal +import kotlinx.coroutines.* +import kotlin.coroutines.* + +/** + * The testable main dispatcher used by kotlinx-coroutines-test. + * It is a [MainCoroutineDispatcher] that delegates all actions to a settable delegate. + */ +internal class TestMainDispatcher(private var delegate: CoroutineDispatcher): + MainCoroutineDispatcher(), + Delay by (delegate as? Delay ?: defaultDelay) +{ + private val mainDispatcher = delegate // the initial value passed to the constructor + + override val immediate: MainCoroutineDispatcher + get() = (delegate as? MainCoroutineDispatcher)?.immediate ?: this + + override fun dispatch(context: CoroutineContext, block: Runnable) = delegate.dispatch(context, block) + + override fun isDispatchNeeded(context: CoroutineContext): Boolean = delegate.isDispatchNeeded(context) + + override fun dispatchYield(context: CoroutineContext, block: Runnable) = delegate.dispatchYield(context, block) + + fun setDispatcher(dispatcher: CoroutineDispatcher) { + delegate = dispatcher + } + + fun resetDispatcher() { + delegate = mainDispatcher + } +} + +@Suppress("INVISIBLE_MEMBER") +private val defaultDelay + inline get() = DefaultDelay + +@Suppress("INVISIBLE_MEMBER") +internal expect fun Dispatchers.getTestMainDispatcher(): TestMainDispatcher \ No newline at end of file diff --git a/kotlinx-coroutines-test/common/test/TestCoroutineDispatcherOrderTest.kt b/kotlinx-coroutines-test/common/test/TestCoroutineDispatcherOrderTest.kt index c3686845de..e54ba21568 100644 --- a/kotlinx-coroutines-test/common/test/TestCoroutineDispatcherOrderTest.kt +++ b/kotlinx-coroutines-test/common/test/TestCoroutineDispatcherOrderTest.kt @@ -15,7 +15,6 @@ class TestCoroutineDispatcherOrderTest { private fun expect(index: Int) { val wasIndex = actionIndex.incrementAndGet() - // println("expect($index), wasIndex=$wasIndex") check(index == wasIndex) { "Expecting action index $index but it is actually $wasIndex" } } diff --git a/kotlinx-coroutines-test/jvm/test/TestDispatchersTest.kt b/kotlinx-coroutines-test/common/test/TestDispatchersTest.kt similarity index 97% rename from kotlinx-coroutines-test/jvm/test/TestDispatchersTest.kt rename to kotlinx-coroutines-test/common/test/TestDispatchersTest.kt index 3ab78987f0..b758c58636 100644 --- a/kotlinx-coroutines-test/jvm/test/TestDispatchersTest.kt +++ b/kotlinx-coroutines-test/common/test/TestDispatchersTest.kt @@ -14,7 +14,6 @@ class TestDispatchersTest { private fun expect(index: Int) { val wasIndex = actionIndex.incrementAndGet() - println("expect($index), wasIndex=$wasIndex") check(index == wasIndex) { "Expecting action index $index but it is actually $wasIndex" } } @@ -69,4 +68,4 @@ class TestDispatchersTest { block.run() } } -} +} \ No newline at end of file diff --git a/kotlinx-coroutines-test/common/test/TestRunBlockingOrderTest.kt b/kotlinx-coroutines-test/common/test/TestRunBlockingOrderTest.kt index 064cb7cb71..5d94bd2866 100644 --- a/kotlinx-coroutines-test/common/test/TestRunBlockingOrderTest.kt +++ b/kotlinx-coroutines-test/common/test/TestRunBlockingOrderTest.kt @@ -15,7 +15,6 @@ class TestRunBlockingOrderTest { private fun expect(index: Int) { val wasIndex = actionIndex.incrementAndGet() - // println("expect($index), wasIndex=$wasIndex") check(index == wasIndex) { "Expecting action index $index but it is actually $wasIndex" } } diff --git a/kotlinx-coroutines-test/js/src/TestDispatchers.kt b/kotlinx-coroutines-test/js/src/TestDispatchers.kt deleted file mode 100644 index 10322079d3..0000000000 --- a/kotlinx-coroutines-test/js/src/TestDispatchers.kt +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. - */ - -package kotlinx.coroutines.test -import kotlinx.coroutines.* - -@ExperimentalCoroutinesApi -public actual fun Dispatchers.setMain(dispatcher: CoroutineDispatcher) { - throw UnsupportedOperationException("`setMain` is not supported on JS") -} - -@ExperimentalCoroutinesApi -public actual fun Dispatchers.resetMain() { - throw UnsupportedOperationException("`resetMain` is not supported on JS") -} diff --git a/kotlinx-coroutines-test/js/src/internal/TestMainDispatcher.kt b/kotlinx-coroutines-test/js/src/internal/TestMainDispatcher.kt new file mode 100644 index 0000000000..4d865f83c0 --- /dev/null +++ b/kotlinx-coroutines-test/js/src/internal/TestMainDispatcher.kt @@ -0,0 +1,13 @@ +/* + * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines.test.internal +import kotlinx.coroutines.* + +@Suppress("INVISIBLE_MEMBER") +internal actual fun Dispatchers.getTestMainDispatcher(): TestMainDispatcher = + when (val mainDispatcher = Main) { + is TestMainDispatcher -> mainDispatcher + else -> TestMainDispatcher(mainDispatcher).also { injectMain(it) } + } diff --git a/kotlinx-coroutines-test/jvm/src/TestDispatchers.kt b/kotlinx-coroutines-test/jvm/src/TestDispatchers.kt deleted file mode 100644 index 800eca5d1d..0000000000 --- a/kotlinx-coroutines-test/jvm/src/TestDispatchers.kt +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. - */ -@file:JvmName("TestDispatchers") - -package kotlinx.coroutines.test - -import kotlinx.coroutines.* -import kotlinx.coroutines.test.internal.* - -@ExperimentalCoroutinesApi -public actual fun Dispatchers.setMain(dispatcher: CoroutineDispatcher) { - require(dispatcher !is TestMainDispatcher) { "Dispatchers.setMain(Dispatchers.Main) is prohibited, probably Dispatchers.resetMain() should be used instead" } - val mainDispatcher = Main - require(mainDispatcher is TestMainDispatcher) { "TestMainDispatcher is not set as main dispatcher, have $mainDispatcher instead." } - mainDispatcher.setDispatcher(dispatcher) -} - -@ExperimentalCoroutinesApi -public actual fun Dispatchers.resetMain() { - val mainDispatcher = Main - require(mainDispatcher is TestMainDispatcher) { "TestMainDispatcher is not set as main dispatcher, have $mainDispatcher instead." } - mainDispatcher.resetDispatcher() -} diff --git a/kotlinx-coroutines-test/jvm/src/internal/TestMainDispatcher.kt b/kotlinx-coroutines-test/jvm/src/internal/TestMainDispatcher.kt deleted file mode 100644 index 4c70ca3fcf..0000000000 --- a/kotlinx-coroutines-test/jvm/src/internal/TestMainDispatcher.kt +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. - */ - -package kotlinx.coroutines.test.internal - -import kotlinx.coroutines.* -import kotlinx.coroutines.internal.* -import kotlin.coroutines.* - -/** - * The testable main dispatcher used by kotlinx-coroutines-test. - * It is a [MainCoroutineDispatcher] which delegates all actions to a settable delegate. - */ -internal class TestMainDispatcher(private val mainFactory: MainDispatcherFactory) : MainCoroutineDispatcher(), Delay { - private var _delegate: CoroutineDispatcher? = null - private val delegate: CoroutineDispatcher get() { - _delegate?.let { return it } - mainFactory.tryCreateDispatcher(emptyList()).let { - // If we've failed to create a dispatcher, do no set _delegate - if (!isMissing()) { - _delegate = it - } - return it - } - } - - @Suppress("INVISIBLE_MEMBER") - private val delay: Delay get() = delegate as? Delay ?: DefaultDelay - - override val immediate: MainCoroutineDispatcher - get() = (delegate as? MainCoroutineDispatcher)?.immediate ?: this - - override fun dispatch(context: CoroutineContext, block: Runnable) { - delegate.dispatch(context, block) - } - - override fun isDispatchNeeded(context: CoroutineContext): Boolean = delegate.isDispatchNeeded(context) - - override fun scheduleResumeAfterDelay(timeMillis: Long, continuation: CancellableContinuation) { - delay.scheduleResumeAfterDelay(timeMillis, continuation) - } - - override suspend fun delay(time: Long) { - delay.delay(time) - } - - override fun invokeOnTimeout(timeMillis: Long, block: Runnable, context: CoroutineContext): DisposableHandle { - return delay.invokeOnTimeout(timeMillis, block, context) - } - - fun setDispatcher(dispatcher: CoroutineDispatcher) { - _delegate = dispatcher - } - - fun resetDispatcher() { - _delegate = null - } -} - -internal class TestMainDispatcherFactory : MainDispatcherFactory { - - override fun createDispatcher(allFactories: List): MainCoroutineDispatcher { - val originalFactory = allFactories.asSequence() - .filter { it !== this } - .maxByOrNull { it.loadPriority } ?: MissingMainCoroutineDispatcherFactory - return TestMainDispatcher(originalFactory) - } - - /** - * [Int.MAX_VALUE] -- test dispatcher always wins no matter what factories are present in the classpath. - * By default, all actions are delegated to the second-priority dispatcher, so that it won't be the issue. - */ - override val loadPriority: Int - get() = Int.MAX_VALUE -} diff --git a/kotlinx-coroutines-test/jvm/src/internal/TestMainDispatcherJvm.kt b/kotlinx-coroutines-test/jvm/src/internal/TestMainDispatcherJvm.kt new file mode 100644 index 0000000000..f86b08ea14 --- /dev/null +++ b/kotlinx-coroutines-test/jvm/src/internal/TestMainDispatcherJvm.kt @@ -0,0 +1,31 @@ +/* + * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines.test.internal + +import kotlinx.coroutines.* +import kotlinx.coroutines.internal.* + +internal class TestMainDispatcherFactory : MainDispatcherFactory { + + override fun createDispatcher(allFactories: List): MainCoroutineDispatcher { + val otherFactories = allFactories.filter { it !== this } + val secondBestFactory = otherFactories.maxByOrNull { it.loadPriority } ?: MissingMainCoroutineDispatcherFactory + val dispatcher = secondBestFactory.tryCreateDispatcher(otherFactories) + return TestMainDispatcher(dispatcher) + } + + /** + * [Int.MAX_VALUE] -- test dispatcher always wins no matter what factories are present in the classpath. + * By default, all actions are delegated to the second-priority dispatcher, so that it won't be the issue. + */ + override val loadPriority: Int + get() = Int.MAX_VALUE +} + +internal actual fun Dispatchers.getTestMainDispatcher(): TestMainDispatcher { + val mainDispatcher = Main + require(mainDispatcher is TestMainDispatcher) { "TestMainDispatcher is not set as main dispatcher, have $mainDispatcher instead." } + return mainDispatcher +} \ No newline at end of file diff --git a/kotlinx-coroutines-test/native/src/TestDispatchers.kt b/kotlinx-coroutines-test/native/src/TestDispatchers.kt deleted file mode 100644 index 44c7a72752..0000000000 --- a/kotlinx-coroutines-test/native/src/TestDispatchers.kt +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. - */ - -package kotlinx.coroutines.test - -import kotlinx.coroutines.* - -@ExperimentalCoroutinesApi -public actual fun Dispatchers.setMain(dispatcher: CoroutineDispatcher) { - throw UnsupportedOperationException("`setMain` is not supported on Native") -} - -@ExperimentalCoroutinesApi -public actual fun Dispatchers.resetMain() { - throw UnsupportedOperationException("`resetMain` is not supported on Native") -} \ No newline at end of file diff --git a/kotlinx-coroutines-test/native/src/TestMainDispatcher.kt b/kotlinx-coroutines-test/native/src/TestMainDispatcher.kt new file mode 100644 index 0000000000..4d865f83c0 --- /dev/null +++ b/kotlinx-coroutines-test/native/src/TestMainDispatcher.kt @@ -0,0 +1,13 @@ +/* + * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines.test.internal +import kotlinx.coroutines.* + +@Suppress("INVISIBLE_MEMBER") +internal actual fun Dispatchers.getTestMainDispatcher(): TestMainDispatcher = + when (val mainDispatcher = Main) { + is TestMainDispatcher -> mainDispatcher + else -> TestMainDispatcher(mainDispatcher).also { injectMain(it) } + } From 9dda95aa8bbc6f6f562bcb0175a2f913a6871df3 Mon Sep 17 00:00:00 2001 From: Vsevolod Tolstopyatov Date: Mon, 11 Oct 2021 16:57:33 +0300 Subject: [PATCH 19/24] Promote the following experimental API to stable (#2971) * transformWhile * awaitClose and ProducerScope (for callbackFlow and channelFlow) * merge * runningFold, runningReduce, and scan --- kotlinx-coroutines-core/common/src/channels/Produce.kt | 8 ++------ .../common/src/flow/operators/Limit.kt | 1 - .../common/src/flow/operators/Merge.kt | 2 -- .../common/src/flow/operators/Transform.kt | 3 --- reactive/kotlinx-coroutines-reactor/src/ReactorContext.kt | 1 - 5 files changed, 2 insertions(+), 13 deletions(-) diff --git a/kotlinx-coroutines-core/common/src/channels/Produce.kt b/kotlinx-coroutines-core/common/src/channels/Produce.kt index 3342fb6ec9..a03e3d8742 100644 --- a/kotlinx-coroutines-core/common/src/channels/Produce.kt +++ b/kotlinx-coroutines-core/common/src/channels/Produce.kt @@ -6,14 +6,11 @@ package kotlinx.coroutines.channels import kotlinx.coroutines.* import kotlin.coroutines.* +import kotlinx.coroutines.flow.* /** - * Scope for the [produce][CoroutineScope.produce] coroutine builder. - * - * **Note: This is an experimental api.** Behavior of producers that work as children in a parent scope with respect - * to cancellation and error handling may change in the future. + * Scope for the [produce][CoroutineScope.produce], [callbackFlow] and [channelFlow] builders. */ -@ExperimentalCoroutinesApi public interface ProducerScope : CoroutineScope, SendChannel { /** * A reference to the channel this coroutine [sends][send] elements to. @@ -45,7 +42,6 @@ public interface ProducerScope : CoroutineScope, SendChannel { * } * ``` */ -@ExperimentalCoroutinesApi public suspend fun ProducerScope<*>.awaitClose(block: () -> Unit = {}) { check(kotlin.coroutines.coroutineContext[Job] === this) { "awaitClose() can only be invoked from the producer context" } try { diff --git a/kotlinx-coroutines-core/common/src/flow/operators/Limit.kt b/kotlinx-coroutines-core/common/src/flow/operators/Limit.kt index 8fbf1a2b0e..734464b557 100644 --- a/kotlinx-coroutines-core/common/src/flow/operators/Limit.kt +++ b/kotlinx-coroutines-core/common/src/flow/operators/Limit.kt @@ -112,7 +112,6 @@ public fun Flow.takeWhile(predicate: suspend (T) -> Boolean): Flow = f * } * ``` */ -@ExperimentalCoroutinesApi public fun Flow.transformWhile( @BuilderInference transform: suspend FlowCollector.(value: T) -> Boolean ): Flow = diff --git a/kotlinx-coroutines-core/common/src/flow/operators/Merge.kt b/kotlinx-coroutines-core/common/src/flow/operators/Merge.kt index 432160f340..228cc9e245 100644 --- a/kotlinx-coroutines-core/common/src/flow/operators/Merge.kt +++ b/kotlinx-coroutines-core/common/src/flow/operators/Merge.kt @@ -90,7 +90,6 @@ public fun Flow>.flattenConcat(): Flow = flow { * Applications of [flowOn], [buffer], and [produceIn] _after_ this operator are fused with * its concurrent merging so that only one properly configured channel is used for execution of merging logic. */ -@ExperimentalCoroutinesApi public fun Iterable>.merge(): Flow { /* * This is a fuseable implementation of the following operator: @@ -114,7 +113,6 @@ public fun Iterable>.merge(): Flow { * Applications of [flowOn], [buffer], and [produceIn] _after_ this operator are fused with * its concurrent merging so that only one properly configured channel is used for execution of merging logic. */ -@ExperimentalCoroutinesApi public fun merge(vararg flows: Flow): Flow = flows.asIterable().merge() /** diff --git a/kotlinx-coroutines-core/common/src/flow/operators/Transform.kt b/kotlinx-coroutines-core/common/src/flow/operators/Transform.kt index a47ae776ca..9b97193227 100644 --- a/kotlinx-coroutines-core/common/src/flow/operators/Transform.kt +++ b/kotlinx-coroutines-core/common/src/flow/operators/Transform.kt @@ -85,7 +85,6 @@ public fun Flow.onEach(action: suspend (T) -> Unit): Flow = transform * * This function is an alias to [runningFold] operator. */ -@ExperimentalCoroutinesApi public fun Flow.scan(initial: R, @BuilderInference operation: suspend (accumulator: R, value: T) -> R): Flow = runningFold(initial, operation) /** @@ -97,7 +96,6 @@ public fun Flow.scan(initial: R, @BuilderInference operation: suspend * ``` * will produce `[], [1], [1, 2], [1, 2, 3]]`. */ -@ExperimentalCoroutinesApi public fun Flow.runningFold(initial: R, @BuilderInference operation: suspend (accumulator: R, value: T) -> R): Flow = flow { var accumulator: R = initial emit(accumulator) @@ -118,7 +116,6 @@ public fun Flow.runningFold(initial: R, @BuilderInference operation: s * ``` * will produce `[1, 3, 6, 10]` */ -@ExperimentalCoroutinesApi public fun Flow.runningReduce(operation: suspend (accumulator: T, value: T) -> T): Flow = flow { var accumulator: Any? = NULL collect { value -> diff --git a/reactive/kotlinx-coroutines-reactor/src/ReactorContext.kt b/reactive/kotlinx-coroutines-reactor/src/ReactorContext.kt index d9228409db..ce68091705 100644 --- a/reactive/kotlinx-coroutines-reactor/src/ReactorContext.kt +++ b/reactive/kotlinx-coroutines-reactor/src/ReactorContext.kt @@ -4,7 +4,6 @@ package kotlinx.coroutines.reactor -import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlin.coroutines.* import kotlinx.coroutines.reactive.* import reactor.util.context.* From 5b4b6126ea39be9c0c896284a772982fbd30dcdf Mon Sep 17 00:00:00 2001 From: Vsevolod Tolstopyatov Date: Tue, 12 Oct 2021 14:40:16 +0300 Subject: [PATCH 20/24] Introduce CoroutineDispatcher.limitedParallelism and make Dispatchers.IO unbounded for limited parallelism (#2918) * Introduce CoroutineDispatcher.limitedParallelism for granular concurrency control * Elastic Dispatchers.IO: * Extract Ktor-obsolete API to a separate file for backwards compatibility * Make Dispatchers.IO being a slice of unlimited blocking scheduler * Make Dispatchers.IO.limitParallelism take slices from the same internal scheduler Fixes #2943 Fixes #2919 --- .../benchmarks/ParametrizedDispatcherBase.kt | 2 +- .../kotlin/benchmarks/SemaphoreBenchmark.kt | 11 +- .../actors/PingPongWithBlockingContext.kt | 6 +- .../test/TaskTest.kt | 8 +- .../api/kotlinx-coroutines-core.api | 2 + .../common/src/CoroutineDispatcher.kt | 39 +++ .../common/src/EventLoop.common.kt | 6 +- .../common/src/MainCoroutineDispatcher.kt | 8 + .../common/src/Unconfined.kt | 6 + .../common/src/internal/LimitedDispatcher.kt | 105 +++++++ .../js/src/JSDispatcher.kt | 5 + .../jvm/src/Dispatchers.kt | 30 +- .../jvm/src/internal/MainDispatchers.kt | 3 + .../jvm/src/scheduling/Deprecated.kt | 212 ++++++++++++++ .../jvm/src/scheduling/Dispatcher.kt | 266 +++++------------- .../jvm/src/scheduling/Tasks.kt | 37 ++- .../jvm/test/LimitedParallelismStressTest.kt | 83 ++++++ .../jvm/test/LimitedParallelismTest.kt | 55 ++++ ...oroutineDispatcherTerminationStressTest.kt | 2 +- .../BlockingCoroutineDispatcherTest.kt | 65 ----- ...utineDispatcherWorkSignallingStressTest.kt | 2 +- .../scheduling/CoroutineDispatcherTest.kt | 2 +- .../CoroutineSchedulerCloseStressTest.kt | 26 +- .../CoroutineSchedulerStressTest.kt | 2 +- .../test/scheduling/CoroutineSchedulerTest.kt | 12 +- .../test/scheduling/DefaultDispatchersTest.kt | 75 +++++ .../test/scheduling/LimitingDispatcherTest.kt | 5 - .../jvm/test/scheduling/SchedulerTestBase.kt | 21 +- .../test/scheduling/SharingWorkerClassTest.kt | 6 +- 29 files changed, 755 insertions(+), 347 deletions(-) create mode 100644 kotlinx-coroutines-core/common/src/internal/LimitedDispatcher.kt create mode 100644 kotlinx-coroutines-core/jvm/src/scheduling/Deprecated.kt create mode 100644 kotlinx-coroutines-core/jvm/test/LimitedParallelismStressTest.kt create mode 100644 kotlinx-coroutines-core/jvm/test/LimitedParallelismTest.kt create mode 100644 kotlinx-coroutines-core/jvm/test/scheduling/DefaultDispatchersTest.kt diff --git a/benchmarks/src/jmh/kotlin/benchmarks/ParametrizedDispatcherBase.kt b/benchmarks/src/jmh/kotlin/benchmarks/ParametrizedDispatcherBase.kt index 9948a371bc..80e15a1b4f 100644 --- a/benchmarks/src/jmh/kotlin/benchmarks/ParametrizedDispatcherBase.kt +++ b/benchmarks/src/jmh/kotlin/benchmarks/ParametrizedDispatcherBase.kt @@ -30,7 +30,7 @@ abstract class ParametrizedDispatcherBase : CoroutineScope { coroutineContext = when { dispatcher == "fjp" -> ForkJoinPool.commonPool().asCoroutineDispatcher() dispatcher == "scheduler" -> { - ExperimentalCoroutineDispatcher(CORES_COUNT).also { closeable = it } + Dispatchers.Default } dispatcher.startsWith("ftp") -> { newFixedThreadPoolContext(dispatcher.substring(4).toInt(), dispatcher).also { closeable = it } diff --git a/benchmarks/src/jmh/kotlin/benchmarks/SemaphoreBenchmark.kt b/benchmarks/src/jmh/kotlin/benchmarks/SemaphoreBenchmark.kt index 40ddc8ec36..9e1bfc43bb 100644 --- a/benchmarks/src/jmh/kotlin/benchmarks/SemaphoreBenchmark.kt +++ b/benchmarks/src/jmh/kotlin/benchmarks/SemaphoreBenchmark.kt @@ -6,13 +6,10 @@ package benchmarks import benchmarks.common.* import kotlinx.coroutines.* -import kotlinx.coroutines.channels.Channel -import kotlinx.coroutines.scheduling.ExperimentalCoroutineDispatcher -import kotlinx.coroutines.sync.Semaphore -import kotlinx.coroutines.sync.withPermit +import kotlinx.coroutines.channels.* +import kotlinx.coroutines.sync.* import org.openjdk.jmh.annotations.* -import java.util.concurrent.ForkJoinPool -import java.util.concurrent.TimeUnit +import java.util.concurrent.* @Warmup(iterations = 3, time = 500, timeUnit = TimeUnit.MICROSECONDS) @Measurement(iterations = 10, time = 500, timeUnit = TimeUnit.MICROSECONDS) @@ -84,7 +81,7 @@ open class SemaphoreBenchmark { enum class SemaphoreBenchDispatcherCreator(val create: (parallelism: Int) -> CoroutineDispatcher) { FORK_JOIN({ parallelism -> ForkJoinPool(parallelism).asCoroutineDispatcher() }), - EXPERIMENTAL({ parallelism -> ExperimentalCoroutineDispatcher(corePoolSize = parallelism, maxPoolSize = parallelism) }) + EXPERIMENTAL({ parallelism -> Dispatchers.Default }) // TODO doesn't take parallelism into account } private const val WORK_INSIDE = 80 diff --git a/benchmarks/src/jmh/kotlin/benchmarks/scheduler/actors/PingPongWithBlockingContext.kt b/benchmarks/src/jmh/kotlin/benchmarks/scheduler/actors/PingPongWithBlockingContext.kt index a6f0a473c1..d874f3bbe1 100644 --- a/benchmarks/src/jmh/kotlin/benchmarks/scheduler/actors/PingPongWithBlockingContext.kt +++ b/benchmarks/src/jmh/kotlin/benchmarks/scheduler/actors/PingPongWithBlockingContext.kt @@ -27,10 +27,8 @@ import kotlin.coroutines.* @State(Scope.Benchmark) open class PingPongWithBlockingContext { - @UseExperimental(InternalCoroutinesApi::class) - private val experimental = ExperimentalCoroutineDispatcher(8) - @UseExperimental(InternalCoroutinesApi::class) - private val blocking = experimental.blocking(8) + private val experimental = Dispatchers.Default + private val blocking = Dispatchers.IO.limitedParallelism(8) private val threadPool = newFixedThreadPoolContext(8, "PongCtx") @TearDown diff --git a/integration/kotlinx-coroutines-play-services/test/TaskTest.kt b/integration/kotlinx-coroutines-play-services/test/TaskTest.kt index b125192e93..34fbe23b55 100644 --- a/integration/kotlinx-coroutines-play-services/test/TaskTest.kt +++ b/integration/kotlinx-coroutines-play-services/test/TaskTest.kt @@ -45,8 +45,8 @@ class TaskTest : TestBase() { } @Test - fun testCancelledAsTask() { - val deferred = GlobalScope.async { + fun testCancelledAsTask() = runTest { + val deferred = async(Dispatchers.Default) { delay(100) }.apply { cancel() } @@ -60,8 +60,8 @@ class TaskTest : TestBase() { } @Test - fun testThrowingAsTask() { - val deferred = GlobalScope.async { + fun testThrowingAsTask() = runTest({ e -> e is TestException }) { + val deferred = async(Dispatchers.Default) { throw TestException("Fail") } diff --git a/kotlinx-coroutines-core/api/kotlinx-coroutines-core.api b/kotlinx-coroutines-core/api/kotlinx-coroutines-core.api index fcd51ae2eb..495d11a804 100644 --- a/kotlinx-coroutines-core/api/kotlinx-coroutines-core.api +++ b/kotlinx-coroutines-core/api/kotlinx-coroutines-core.api @@ -156,6 +156,7 @@ public abstract class kotlinx/coroutines/CoroutineDispatcher : kotlin/coroutines public fun get (Lkotlin/coroutines/CoroutineContext$Key;)Lkotlin/coroutines/CoroutineContext$Element; public final fun interceptContinuation (Lkotlin/coroutines/Continuation;)Lkotlin/coroutines/Continuation; public fun isDispatchNeeded (Lkotlin/coroutines/CoroutineContext;)Z + public fun limitedParallelism (I)Lkotlinx/coroutines/CoroutineDispatcher; public fun minusKey (Lkotlin/coroutines/CoroutineContext$Key;)Lkotlin/coroutines/CoroutineContext; public final fun plus (Lkotlinx/coroutines/CoroutineDispatcher;)Lkotlinx/coroutines/CoroutineDispatcher; public final fun releaseInterceptedContinuation (Lkotlin/coroutines/Continuation;)V @@ -447,6 +448,7 @@ public class kotlinx/coroutines/JobSupport : kotlinx/coroutines/ChildJob, kotlin public abstract class kotlinx/coroutines/MainCoroutineDispatcher : kotlinx/coroutines/CoroutineDispatcher { public fun ()V public abstract fun getImmediate ()Lkotlinx/coroutines/MainCoroutineDispatcher; + public fun limitedParallelism (I)Lkotlinx/coroutines/CoroutineDispatcher; public fun toString ()Ljava/lang/String; protected final fun toStringInternalImpl ()Ljava/lang/String; } diff --git a/kotlinx-coroutines-core/common/src/CoroutineDispatcher.kt b/kotlinx-coroutines-core/common/src/CoroutineDispatcher.kt index d5613d4110..c91e944b91 100644 --- a/kotlinx-coroutines-core/common/src/CoroutineDispatcher.kt +++ b/kotlinx-coroutines-core/common/src/CoroutineDispatcher.kt @@ -61,6 +61,45 @@ public abstract class CoroutineDispatcher : */ public open fun isDispatchNeeded(context: CoroutineContext): Boolean = true + /** + * Creates a view of the current dispatcher that limits the parallelism to the given [value][parallelism]. + * The resulting view uses the original dispatcher for execution, but with the guarantee that + * no more than [parallelism] coroutines are executed at the same time. + * + * This method does not impose restrictions on the number of views or the total sum of parallelism values, + * each view controls its own parallelism independently with the guarantee that the effective parallelism + * of all views cannot exceed the actual parallelism of the original dispatcher. + * + * ### Limitations + * + * The default implementation of `limitedParallelism` does not support direct dispatchers, + * such as executing the given runnable in place during [dispatch] calls. + * Any dispatcher that may return `false` from [isDispatchNeeded] is considered direct. + * For direct dispatchers, it is recommended to override this method + * and provide a domain-specific implementation or to throw an [UnsupportedOperationException]. + * + * ### Example of usage + * ``` + * private val backgroundDispatcher = newFixedThreadPoolContext(4, "App Background") + * // At most 2 threads will be processing images as it is really slow and CPU-intensive + * private val imageProcessingDispatcher = backgroundDispatcher.limitedParallelism(2) + * // At most 3 threads will be processing JSON to avoid image processing starvation + * private val imageProcessingDispatcher = backgroundDispatcher.limitedParallelism(3) + * // At most 1 thread will be doing IO + * private val fileWriterDispatcher = backgroundDispatcher.limitedParallelism(1) + * ``` + * is 6. Yet at most 4 coroutines can be executed simultaneously as each view limits only its own parallelism. + * + * Note that this example was structured in such a way that it illustrates the parallelism guarantees. + * In practice, it is usually better to use [Dispatchers.IO] or [Dispatchers.Default] instead of creating a + * `backgroundDispatcher`. It is both possible and advised to call `limitedParallelism` on them. + */ + @ExperimentalCoroutinesApi + public open fun limitedParallelism(parallelism: Int): CoroutineDispatcher { + parallelism.checkParallelism() + return LimitedDispatcher(this, parallelism) + } + /** * Dispatches execution of a runnable [block] onto another thread in the given [context]. * This method should guarantee that the given [block] will be eventually invoked, diff --git a/kotlinx-coroutines-core/common/src/EventLoop.common.kt b/kotlinx-coroutines-core/common/src/EventLoop.common.kt index e2a1ffd69f..c4d4e272d7 100644 --- a/kotlinx-coroutines-core/common/src/EventLoop.common.kt +++ b/kotlinx-coroutines-core/common/src/EventLoop.common.kt @@ -115,6 +115,11 @@ internal abstract class EventLoop : CoroutineDispatcher() { } } + final override fun limitedParallelism(parallelism: Int): CoroutineDispatcher { + parallelism.checkParallelism() + return this + } + open fun shutdown() {} } @@ -525,4 +530,3 @@ internal expect fun nanoTime(): Long internal expect object DefaultExecutor { public fun enqueue(task: Runnable) } - diff --git a/kotlinx-coroutines-core/common/src/MainCoroutineDispatcher.kt b/kotlinx-coroutines-core/common/src/MainCoroutineDispatcher.kt index 602da6e0b5..a7065ccd15 100644 --- a/kotlinx-coroutines-core/common/src/MainCoroutineDispatcher.kt +++ b/kotlinx-coroutines-core/common/src/MainCoroutineDispatcher.kt @@ -4,6 +4,8 @@ package kotlinx.coroutines +import kotlinx.coroutines.internal.* + /** * Base class for special [CoroutineDispatcher] which is confined to application "Main" or "UI" thread * and used for any UI-based activities. Instance of `MainDispatcher` can be obtained by [Dispatchers.Main]. @@ -51,6 +53,12 @@ public abstract class MainCoroutineDispatcher : CoroutineDispatcher() { */ override fun toString(): String = toStringInternalImpl() ?: "$classSimpleName@$hexAddress" + override fun limitedParallelism(parallelism: Int): CoroutineDispatcher { + parallelism.checkParallelism() + // MainCoroutineDispatcher is single-threaded -- short-circuit any attempts to limit it + return this + } + /** * Internal method for more specific [toString] implementations. It returns non-null * string if this dispatcher is set in the platform as the main one. diff --git a/kotlinx-coroutines-core/common/src/Unconfined.kt b/kotlinx-coroutines-core/common/src/Unconfined.kt index df0087100a..24a401f702 100644 --- a/kotlinx-coroutines-core/common/src/Unconfined.kt +++ b/kotlinx-coroutines-core/common/src/Unconfined.kt @@ -11,6 +11,12 @@ import kotlin.jvm.* * A coroutine dispatcher that is not confined to any specific thread. */ internal object Unconfined : CoroutineDispatcher() { + + @ExperimentalCoroutinesApi + override fun limitedParallelism(parallelism: Int): CoroutineDispatcher { + throw UnsupportedOperationException("limitedParallelism is not supported for Dispatchers.Unconfined") + } + override fun isDispatchNeeded(context: CoroutineContext): Boolean = false override fun dispatch(context: CoroutineContext, block: Runnable) { diff --git a/kotlinx-coroutines-core/common/src/internal/LimitedDispatcher.kt b/kotlinx-coroutines-core/common/src/internal/LimitedDispatcher.kt new file mode 100644 index 0000000000..892375b89f --- /dev/null +++ b/kotlinx-coroutines-core/common/src/internal/LimitedDispatcher.kt @@ -0,0 +1,105 @@ +/* + * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines.internal + +import kotlinx.coroutines.* +import kotlin.coroutines.* +import kotlin.jvm.* + +/** + * The result of .limitedParallelism(x) call, a dispatcher + * that wraps the given dispatcher, but limits the parallelism level, while + * trying to emulate fairness. + */ +internal class LimitedDispatcher( + private val dispatcher: CoroutineDispatcher, + private val parallelism: Int +) : CoroutineDispatcher(), Runnable, Delay by (dispatcher as? Delay ?: DefaultDelay) { + + @Volatile + private var runningWorkers = 0 + + private val queue = LockFreeTaskQueue(singleConsumer = false) + + @ExperimentalCoroutinesApi + override fun limitedParallelism(parallelism: Int): CoroutineDispatcher { + parallelism.checkParallelism() + if (parallelism >= this.parallelism) return this + return super.limitedParallelism(parallelism) + } + + override fun run() { + var fairnessCounter = 0 + while (true) { + val task = queue.removeFirstOrNull() + if (task != null) { + try { + task.run() + } catch (e: Throwable) { + handleCoroutineException(EmptyCoroutineContext, e) + } + // 16 is our out-of-thin-air constant to emulate fairness. Used in JS dispatchers as well + if (++fairnessCounter >= 16 && dispatcher.isDispatchNeeded(this)) { + // Do "yield" to let other views to execute their runnable as well + // Note that we do not decrement 'runningWorkers' as we still committed to do our part of work + dispatcher.dispatch(this, this) + return + } + continue + } + + @Suppress("CAST_NEVER_SUCCEEDS") + synchronized(this as SynchronizedObject) { + --runningWorkers + if (queue.size == 0) return + ++runningWorkers + fairnessCounter = 0 + } + } + } + + override fun dispatch(context: CoroutineContext, block: Runnable) { + dispatchInternal(block) { + dispatcher.dispatch(this, this) + } + } + + @InternalCoroutinesApi + override fun dispatchYield(context: CoroutineContext, block: Runnable) { + dispatchInternal(block) { + dispatcher.dispatchYield(this, this) + } + } + + private inline fun dispatchInternal(block: Runnable, dispatch: () -> Unit) { + // Add task to queue so running workers will be able to see that + if (addAndTryDispatching(block)) return + /* + * Protect against the race when the number of workers is enough, + * but one (because of synchronized serialization) attempts to complete, + * and we just observed the number of running workers smaller than the actual + * number (hit right between `--runningWorkers` and `++runningWorkers` in `run()`) + */ + if (!tryAllocateWorker()) return + dispatch() + } + + private fun tryAllocateWorker(): Boolean { + @Suppress("CAST_NEVER_SUCCEEDS") + synchronized(this as SynchronizedObject) { + if (runningWorkers >= parallelism) return false + ++runningWorkers + return true + } + } + + private fun addAndTryDispatching(block: Runnable): Boolean { + queue.addLast(block) + return runningWorkers >= parallelism + } +} + +// Save a few bytecode ops +internal fun Int.checkParallelism() = require(this >= 1) { "Expected positive parallelism level, but got $this" } diff --git a/kotlinx-coroutines-core/js/src/JSDispatcher.kt b/kotlinx-coroutines-core/js/src/JSDispatcher.kt index 6ad7d41b15..603005d5a4 100644 --- a/kotlinx-coroutines-core/js/src/JSDispatcher.kt +++ b/kotlinx-coroutines-core/js/src/JSDispatcher.kt @@ -31,6 +31,11 @@ internal sealed class SetTimeoutBasedDispatcher: CoroutineDispatcher(), Delay { abstract fun scheduleQueueProcessing() + override fun limitedParallelism(parallelism: Int): CoroutineDispatcher { + parallelism.checkParallelism() + return this + } + override fun dispatch(context: CoroutineContext, block: Runnable) { messageQueue.enqueue(block) } diff --git a/kotlinx-coroutines-core/jvm/src/Dispatchers.kt b/kotlinx-coroutines-core/jvm/src/Dispatchers.kt index a3be9fa53c..4b1b03337d 100644 --- a/kotlinx-coroutines-core/jvm/src/Dispatchers.kt +++ b/kotlinx-coroutines-core/jvm/src/Dispatchers.kt @@ -86,7 +86,7 @@ public actual object Dispatchers { * Note that if you need your coroutine to be confined to a particular thread or a thread-pool after resumption, * but still want to execute it in the current call-frame until its first suspension, then you can use * an optional [CoroutineStart] parameter in coroutine builders like - * [launch][CoroutineScope.launch] and [async][CoroutineScope.async] setting it to the + * [launch][CoroutineScope.launch] and [async][CoroutineScope.async] setting it to * the value of [CoroutineStart.UNDISPATCHED]. */ @JvmStatic @@ -100,14 +100,30 @@ public actual object Dispatchers { * "`kotlinx.coroutines.io.parallelism`" ([IO_PARALLELISM_PROPERTY_NAME]) system property. * It defaults to the limit of 64 threads or the number of cores (whichever is larger). * - * Moreover, the maximum configurable number of threads is capped by the - * `kotlinx.coroutines.scheduler.max.pool.size` system property. - * If you need a higher number of parallel threads, - * you should use a custom dispatcher backed by your own thread pool. + * ### Elasticity for limited parallelism + * + * `Dispatchers.IO` has a unique property of elasticity: its views + * obtained with [CoroutineDispatcher.limitedParallelism] are + * not restricted by the `Dispatchers.IO` parallelism. Conceptually, there is + * a dispatcher backed by an unlimited pool of threads, and both `Dispatchers.IO` + * and views of `Dispatchers.IO` are actually views of that dispatcher. In practice + * this means that, despite not abiding by `Dispatchers.IO`'s parallelism + * restrictions, its views share threads and resources with it. + * + * In the following example + * ``` + * // 100 threads for MySQL connection + * val myMysqlDbDispatcher = Dispatchers.IO.limitedParallelism(100) + * // 60 threads for MongoDB connection + * val myMongoDbDispatcher = Dispatchers.IO.limitedParallelism(60) + * ``` + * the system may have up to `64 + 100 + 60` threads dedicated to blocking tasks during peak loads, + * but during its steady state there is only a small number of threads shared + * among `Dispatchers.IO`, `myMysqlDbDispatcher` and `myMongoDbDispatcher`. * * ### Implementation note * - * This dispatcher shares threads with the [Default][Dispatchers.Default] dispatcher, so using + * This dispatcher and its views share threads with the [Default][Dispatchers.Default] dispatcher, so using * `withContext(Dispatchers.IO) { ... }` when already running on the [Default][Dispatchers.Default] * dispatcher does not lead to an actual switching to another thread — typically execution * continues in the same thread. @@ -115,7 +131,7 @@ public actual object Dispatchers { * during operations over IO dispatcher. */ @JvmStatic - public val IO: CoroutineDispatcher = DefaultScheduler.IO + public val IO: CoroutineDispatcher = DefaultIoScheduler /** * Shuts down built-in dispatchers, such as [Default] and [IO], diff --git a/kotlinx-coroutines-core/jvm/src/internal/MainDispatchers.kt b/kotlinx-coroutines-core/jvm/src/internal/MainDispatchers.kt index 2d447413b8..2da633a6b6 100644 --- a/kotlinx-coroutines-core/jvm/src/internal/MainDispatchers.kt +++ b/kotlinx-coroutines-core/jvm/src/internal/MainDispatchers.kt @@ -93,6 +93,9 @@ private class MissingMainCoroutineDispatcher( override fun isDispatchNeeded(context: CoroutineContext): Boolean = missing() + override fun limitedParallelism(parallelism: Int): CoroutineDispatcher = + missing() + override suspend fun delay(time: Long) = missing() diff --git a/kotlinx-coroutines-core/jvm/src/scheduling/Deprecated.kt b/kotlinx-coroutines-core/jvm/src/scheduling/Deprecated.kt new file mode 100644 index 0000000000..86b0ade61a --- /dev/null +++ b/kotlinx-coroutines-core/jvm/src/scheduling/Deprecated.kt @@ -0,0 +1,212 @@ +/* + * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +@file:Suppress("unused") + +package kotlinx.coroutines.scheduling +import kotlinx.atomicfu.* +import kotlinx.coroutines.* +import java.util.concurrent.* +import kotlin.coroutines.* + +/** + * This API was "public @InternalApi" and leaked into Ktor enabled-by-default sources. + * Since then, we refactored scheduler sources and its API and decided to get rid of it in + * its current shape. + * + * To preserve backwards compatibility with Ktor 1.x, previous version of the code is + * extracted here as is and isolated from the rest of code base, so R8 can get rid of it. + * + * It should be removed after Kotlin 3.0.0 (EOL of Ktor 1.x) around 2022. + */ +@PublishedApi +internal open class ExperimentalCoroutineDispatcher( + private val corePoolSize: Int, + private val maxPoolSize: Int, + private val idleWorkerKeepAliveNs: Long, + private val schedulerName: String = "CoroutineScheduler" +) : ExecutorCoroutineDispatcher() { + public constructor( + corePoolSize: Int = CORE_POOL_SIZE, + maxPoolSize: Int = MAX_POOL_SIZE, + schedulerName: String = DEFAULT_SCHEDULER_NAME + ) : this(corePoolSize, maxPoolSize, IDLE_WORKER_KEEP_ALIVE_NS, schedulerName) + + @Deprecated(message = "Binary compatibility for Ktor 1.0-beta", level = DeprecationLevel.HIDDEN) + public constructor( + corePoolSize: Int = CORE_POOL_SIZE, + maxPoolSize: Int = MAX_POOL_SIZE + ) : this(corePoolSize, maxPoolSize, IDLE_WORKER_KEEP_ALIVE_NS) + + override val executor: Executor + get() = coroutineScheduler + + // This is variable for test purposes, so that we can reinitialize from clean state + private var coroutineScheduler = createScheduler() + + override fun dispatch(context: CoroutineContext, block: Runnable): Unit = + try { + coroutineScheduler.dispatch(block) + } catch (e: RejectedExecutionException) { + // CoroutineScheduler only rejects execution when it is being closed and this behavior is reserved + // for testing purposes, so we don't have to worry about cancelling the affected Job here. + DefaultExecutor.dispatch(context, block) + } + + override fun dispatchYield(context: CoroutineContext, block: Runnable): Unit = + try { + coroutineScheduler.dispatch(block, tailDispatch = true) + } catch (e: RejectedExecutionException) { + // CoroutineScheduler only rejects execution when it is being closed and this behavior is reserved + // for testing purposes, so we don't have to worry about cancelling the affected Job here. + DefaultExecutor.dispatchYield(context, block) + } + + override fun close(): Unit = coroutineScheduler.close() + + override fun toString(): String { + return "${super.toString()}[scheduler = $coroutineScheduler]" + } + + /** + * Creates a coroutine execution context with limited parallelism to execute tasks which may potentially block. + * Resulting [CoroutineDispatcher] doesn't own any resources (its threads) and provides a view of the original [ExperimentalCoroutineDispatcher], + * giving it additional hints to adjust its behaviour. + * + * @param parallelism parallelism level, indicating how many threads can execute tasks in the resulting dispatcher parallel. + */ + fun blocking(parallelism: Int = 16): CoroutineDispatcher { + require(parallelism > 0) { "Expected positive parallelism level, but have $parallelism" } + return LimitingDispatcher(this, parallelism, null, TASK_PROBABLY_BLOCKING) + } + + /** + * Creates a coroutine execution context with limited parallelism to execute CPU-intensive tasks. + * Resulting [CoroutineDispatcher] doesn't own any resources (its threads) and provides a view of the original [ExperimentalCoroutineDispatcher], + * giving it additional hints to adjust its behaviour. + * + * @param parallelism parallelism level, indicating how many threads can execute tasks in the resulting dispatcher parallel. + */ + fun limited(parallelism: Int): CoroutineDispatcher { + require(parallelism > 0) { "Expected positive parallelism level, but have $parallelism" } + require(parallelism <= corePoolSize) { "Expected parallelism level lesser than core pool size ($corePoolSize), but have $parallelism" } + return LimitingDispatcher(this, parallelism, null, TASK_NON_BLOCKING) + } + + internal fun dispatchWithContext(block: Runnable, context: TaskContext, tailDispatch: Boolean) { + try { + coroutineScheduler.dispatch(block, context, tailDispatch) + } catch (e: RejectedExecutionException) { + // CoroutineScheduler only rejects execution when it is being closed and this behavior is reserved + // for testing purposes, so we don't have to worry about cancelling the affected Job here. + // TaskContext shouldn't be lost here to properly invoke before/after task + DefaultExecutor.enqueue(coroutineScheduler.createTask(block, context)) + } + } + + private fun createScheduler() = CoroutineScheduler(corePoolSize, maxPoolSize, idleWorkerKeepAliveNs, schedulerName) +} + +private class LimitingDispatcher( + private val dispatcher: ExperimentalCoroutineDispatcher, + private val parallelism: Int, + private val name: String?, + override val taskMode: Int +) : ExecutorCoroutineDispatcher(), TaskContext, Executor { + + private val queue = ConcurrentLinkedQueue() + private val inFlightTasks = atomic(0) + + override val executor: Executor + get() = this + + override fun execute(command: Runnable) = dispatch(command, false) + + override fun close(): Unit = error("Close cannot be invoked on LimitingBlockingDispatcher") + + override fun dispatch(context: CoroutineContext, block: Runnable) = dispatch(block, false) + + private fun dispatch(block: Runnable, tailDispatch: Boolean) { + var taskToSchedule = block + while (true) { + // Commit in-flight tasks slot + val inFlight = inFlightTasks.incrementAndGet() + + // Fast path, if parallelism limit is not reached, dispatch task and return + if (inFlight <= parallelism) { + dispatcher.dispatchWithContext(taskToSchedule, this, tailDispatch) + return + } + + // Parallelism limit is reached, add task to the queue + queue.add(taskToSchedule) + + /* + * We're not actually scheduled anything, so rollback committed in-flight task slot: + * If the amount of in-flight tasks is still above the limit, do nothing + * If the amount of in-flight tasks is lesser than parallelism, then + * it's a race with a thread which finished the task from the current context, we should resubmit the first task from the queue + * to avoid starvation. + * + * Race example #1 (TN is N-th thread, R is current in-flight tasks number), execution is sequential: + * + * T1: submit task, start execution, R == 1 + * T2: commit slot for next task, R == 2 + * T1: finish T1, R == 1 + * T2: submit next task to local queue, decrement R, R == 0 + * Without retries, task from T2 will be stuck in the local queue + */ + if (inFlightTasks.decrementAndGet() >= parallelism) { + return + } + + taskToSchedule = queue.poll() ?: return + } + } + + override fun dispatchYield(context: CoroutineContext, block: Runnable) { + dispatch(block, tailDispatch = true) + } + + override fun toString(): String { + return name ?: "${super.toString()}[dispatcher = $dispatcher]" + } + + /** + * Tries to dispatch tasks which were blocked due to reaching parallelism limit if there is any. + * + * Implementation note: blocking tasks are scheduled in a fair manner (to local queue tail) to avoid + * non-blocking continuations starvation. + * E.g. for + * ``` + * foo() + * blocking() + * bar() + * ``` + * it's more profitable to execute bar at the end of `blocking` rather than pending blocking task + */ + override fun afterTask() { + var next = queue.poll() + // If we have pending tasks in current blocking context, dispatch first + if (next != null) { + dispatcher.dispatchWithContext(next, this, true) + return + } + inFlightTasks.decrementAndGet() + + /* + * Re-poll again and try to submit task if it's required otherwise tasks may be stuck in the local queue. + * Race example #2 (TN is N-th thread, R is current in-flight tasks number), execution is sequential: + * T1: submit task, start execution, R == 1 + * T2: commit slot for next task, R == 2 + * T1: finish T1, poll queue (it's still empty), R == 2 + * T2: submit next task to the local queue, decrement R, R == 1 + * T1: decrement R, finish. R == 0 + * + * The task from T2 is stuck is the local queue + */ + next = queue.poll() ?: return + dispatch(next, true) + } +} diff --git a/kotlinx-coroutines-core/jvm/src/scheduling/Dispatcher.kt b/kotlinx-coroutines-core/jvm/src/scheduling/Dispatcher.kt index 0b5a542c2c..d55edec94f 100644 --- a/kotlinx-coroutines-core/jvm/src/scheduling/Dispatcher.kt +++ b/kotlinx-coroutines-core/jvm/src/scheduling/Dispatcher.kt @@ -4,24 +4,16 @@ package kotlinx.coroutines.scheduling -import kotlinx.atomicfu.* import kotlinx.coroutines.* import kotlinx.coroutines.internal.* import java.util.concurrent.* import kotlin.coroutines.* -/** - * Default instance of coroutine dispatcher. - */ -internal object DefaultScheduler : ExperimentalCoroutineDispatcher() { - @JvmField - val IO: CoroutineDispatcher = LimitingDispatcher( - this, - systemProp(IO_PARALLELISM_PROPERTY_NAME, 64.coerceAtLeast(AVAILABLE_PROCESSORS)), - "Dispatchers.IO", - TASK_PROBABLY_BLOCKING - ) - +// Instance of Dispatchers.Default +internal object DefaultScheduler : SchedulerCoroutineDispatcher( + CORE_POOL_SIZE, MAX_POOL_SIZE, + IDLE_WORKER_KEEP_ALIVE_NS, DEFAULT_SCHEDULER_NAME +) { // Shuts down the dispatcher, used only by Dispatchers.shutdown() internal fun shutdown() { super.close() @@ -29,106 +21,91 @@ internal object DefaultScheduler : ExperimentalCoroutineDispatcher() { // Overridden in case anyone writes (Dispatchers.Default as ExecutorCoroutineDispatcher).close() override fun close() { - throw UnsupportedOperationException("$DEFAULT_DISPATCHER_NAME cannot be closed") + throw UnsupportedOperationException("Dispatchers.Default cannot be closed") } - override fun toString(): String = DEFAULT_DISPATCHER_NAME + override fun toString(): String = "Dispatchers.Default" +} + +// The unlimited instance of Dispatchers.IO that utilizes all the threads CoroutineScheduler provides +private object UnlimitedIoScheduler : CoroutineDispatcher() { @InternalCoroutinesApi - @Suppress("UNUSED") - public fun toDebugString(): String = super.toString() + override fun dispatchYield(context: CoroutineContext, block: Runnable) { + DefaultScheduler.dispatchWithContext(block, BlockingContext, true) + } + + override fun dispatch(context: CoroutineContext, block: Runnable) { + DefaultScheduler.dispatchWithContext(block, BlockingContext, false) + } } -/** - * @suppress **This is unstable API and it is subject to change.** - */ -// TODO make internal (and rename) after complete integration -@InternalCoroutinesApi -public open class ExperimentalCoroutineDispatcher( - private val corePoolSize: Int, - private val maxPoolSize: Int, - private val idleWorkerKeepAliveNs: Long, - private val schedulerName: String = "CoroutineScheduler" -) : ExecutorCoroutineDispatcher() { - public constructor( - corePoolSize: Int = CORE_POOL_SIZE, - maxPoolSize: Int = MAX_POOL_SIZE, - schedulerName: String = DEFAULT_SCHEDULER_NAME - ) : this(corePoolSize, maxPoolSize, IDLE_WORKER_KEEP_ALIVE_NS, schedulerName) - - @Deprecated(message = "Binary compatibility for Ktor 1.0-beta", level = DeprecationLevel.HIDDEN) - public constructor( - corePoolSize: Int = CORE_POOL_SIZE, - maxPoolSize: Int = MAX_POOL_SIZE - ) : this(corePoolSize, maxPoolSize, IDLE_WORKER_KEEP_ALIVE_NS) +// Dispatchers.IO +internal object DefaultIoScheduler : ExecutorCoroutineDispatcher(), Executor { + + private val default = UnlimitedIoScheduler.limitedParallelism( + systemProp( + IO_PARALLELISM_PROPERTY_NAME, + 64.coerceAtLeast(AVAILABLE_PROCESSORS) + ) + ) override val executor: Executor - get() = coroutineScheduler + get() = this - // This is variable for test purposes, so that we can reinitialize from clean state - private var coroutineScheduler = createScheduler() + override fun execute(command: java.lang.Runnable) = dispatch(EmptyCoroutineContext, command) - override fun dispatch(context: CoroutineContext, block: Runnable): Unit = - try { - coroutineScheduler.dispatch(block) - } catch (e: RejectedExecutionException) { - // CoroutineScheduler only rejects execution when it is being closed and this behavior is reserved - // for testing purposes, so we don't have to worry about cancelling the affected Job here. - DefaultExecutor.dispatch(context, block) - } + @ExperimentalCoroutinesApi + override fun limitedParallelism(parallelism: Int): CoroutineDispatcher { + // See documentation to Dispatchers.IO for the rationale + return UnlimitedIoScheduler.limitedParallelism(parallelism) + } - override fun dispatchYield(context: CoroutineContext, block: Runnable): Unit = - try { - coroutineScheduler.dispatch(block, tailDispatch = true) - } catch (e: RejectedExecutionException) { - // CoroutineScheduler only rejects execution when it is being closed and this behavior is reserved - // for testing purposes, so we don't have to worry about cancelling the affected Job here. - DefaultExecutor.dispatchYield(context, block) - } - - override fun close(): Unit = coroutineScheduler.close() - - override fun toString(): String { - return "${super.toString()}[scheduler = $coroutineScheduler]" + override fun dispatch(context: CoroutineContext, block: Runnable) { + default.dispatch(context, block) } - /** - * Creates a coroutine execution context with limited parallelism to execute tasks which may potentially block. - * Resulting [CoroutineDispatcher] doesn't own any resources (its threads) and provides a view of the original [ExperimentalCoroutineDispatcher], - * giving it additional hints to adjust its behaviour. - * - * @param parallelism parallelism level, indicating how many threads can execute tasks in the resulting dispatcher parallel. - */ - public fun blocking(parallelism: Int = BLOCKING_DEFAULT_PARALLELISM): CoroutineDispatcher { - require(parallelism > 0) { "Expected positive parallelism level, but have $parallelism" } - return LimitingDispatcher(this, parallelism, null, TASK_PROBABLY_BLOCKING) + @InternalCoroutinesApi + override fun dispatchYield(context: CoroutineContext, block: Runnable) { + default.dispatchYield(context, block) } - /** - * Creates a coroutine execution context with limited parallelism to execute CPU-intensive tasks. - * Resulting [CoroutineDispatcher] doesn't own any resources (its threads) and provides a view of the original [ExperimentalCoroutineDispatcher], - * giving it additional hints to adjust its behaviour. - * - * @param parallelism parallelism level, indicating how many threads can execute tasks in the resulting dispatcher parallel. - */ - public fun limited(parallelism: Int): CoroutineDispatcher { - require(parallelism > 0) { "Expected positive parallelism level, but have $parallelism" } - require(parallelism <= corePoolSize) { "Expected parallelism level lesser than core pool size ($corePoolSize), but have $parallelism" } - return LimitingDispatcher(this, parallelism, null, TASK_NON_BLOCKING) + override fun close() { + error("Cannot be invoked on Dispatchers.IO") } + override fun toString(): String = "Dispatchers.IO" +} + +// Instantiated in tests so we can test it in isolation +internal open class SchedulerCoroutineDispatcher( + private val corePoolSize: Int = CORE_POOL_SIZE, + private val maxPoolSize: Int = MAX_POOL_SIZE, + private val idleWorkerKeepAliveNs: Long = IDLE_WORKER_KEEP_ALIVE_NS, + private val schedulerName: String = "CoroutineScheduler", +) : ExecutorCoroutineDispatcher() { + + override val executor: Executor + get() = coroutineScheduler + + // This is variable for test purposes, so that we can reinitialize from clean state + private var coroutineScheduler = createScheduler() + + private fun createScheduler() = + CoroutineScheduler(corePoolSize, maxPoolSize, idleWorkerKeepAliveNs, schedulerName) + + override fun dispatch(context: CoroutineContext, block: Runnable): Unit = coroutineScheduler.dispatch(block) + + override fun dispatchYield(context: CoroutineContext, block: Runnable): Unit = + coroutineScheduler.dispatch(block, tailDispatch = true) + internal fun dispatchWithContext(block: Runnable, context: TaskContext, tailDispatch: Boolean) { - try { - coroutineScheduler.dispatch(block, context, tailDispatch) - } catch (e: RejectedExecutionException) { - // CoroutineScheduler only rejects execution when it is being closed and this behavior is reserved - // for testing purposes, so we don't have to worry about cancelling the affected Job here. - // TaskContext shouldn't be lost here to properly invoke before/after task - DefaultExecutor.enqueue(coroutineScheduler.createTask(block, context)) - } + coroutineScheduler.dispatch(block, context, tailDispatch) } - private fun createScheduler() = CoroutineScheduler(corePoolSize, maxPoolSize, idleWorkerKeepAliveNs, schedulerName) + override fun close() { + coroutineScheduler.close() + } // fot tests only @Synchronized @@ -146,106 +123,3 @@ public open class ExperimentalCoroutineDispatcher( // for tests only internal fun restore() = usePrivateScheduler() // recreate scheduler } - -private class LimitingDispatcher( - private val dispatcher: ExperimentalCoroutineDispatcher, - private val parallelism: Int, - private val name: String?, - override val taskMode: Int -) : ExecutorCoroutineDispatcher(), TaskContext, Executor { - - private val queue = ConcurrentLinkedQueue() - private val inFlightTasks = atomic(0) - - override val executor: Executor - get() = this - - override fun execute(command: Runnable) = dispatch(command, false) - - override fun close(): Unit = error("Close cannot be invoked on LimitingBlockingDispatcher") - - override fun dispatch(context: CoroutineContext, block: Runnable) = dispatch(block, false) - - private fun dispatch(block: Runnable, tailDispatch: Boolean) { - var taskToSchedule = block - while (true) { - // Commit in-flight tasks slot - val inFlight = inFlightTasks.incrementAndGet() - - // Fast path, if parallelism limit is not reached, dispatch task and return - if (inFlight <= parallelism) { - dispatcher.dispatchWithContext(taskToSchedule, this, tailDispatch) - return - } - - // Parallelism limit is reached, add task to the queue - queue.add(taskToSchedule) - - /* - * We're not actually scheduled anything, so rollback committed in-flight task slot: - * If the amount of in-flight tasks is still above the limit, do nothing - * If the amount of in-flight tasks is lesser than parallelism, then - * it's a race with a thread which finished the task from the current context, we should resubmit the first task from the queue - * to avoid starvation. - * - * Race example #1 (TN is N-th thread, R is current in-flight tasks number), execution is sequential: - * - * T1: submit task, start execution, R == 1 - * T2: commit slot for next task, R == 2 - * T1: finish T1, R == 1 - * T2: submit next task to local queue, decrement R, R == 0 - * Without retries, task from T2 will be stuck in the local queue - */ - if (inFlightTasks.decrementAndGet() >= parallelism) { - return - } - - taskToSchedule = queue.poll() ?: return - } - } - - override fun dispatchYield(context: CoroutineContext, block: Runnable) { - dispatch(block, tailDispatch = true) - } - - override fun toString(): String { - return name ?: "${super.toString()}[dispatcher = $dispatcher]" - } - - /** - * Tries to dispatch tasks which were blocked due to reaching parallelism limit if there is any. - * - * Implementation note: blocking tasks are scheduled in a fair manner (to local queue tail) to avoid - * non-blocking continuations starvation. - * E.g. for - * ``` - * foo() - * blocking() - * bar() - * ``` - * it's more profitable to execute bar at the end of `blocking` rather than pending blocking task - */ - override fun afterTask() { - var next = queue.poll() - // If we have pending tasks in current blocking context, dispatch first - if (next != null) { - dispatcher.dispatchWithContext(next, this, true) - return - } - inFlightTasks.decrementAndGet() - - /* - * Re-poll again and try to submit task if it's required otherwise tasks may be stuck in the local queue. - * Race example #2 (TN is N-th thread, R is current in-flight tasks number), execution is sequential: - * T1: submit task, start execution, R == 1 - * T2: commit slot for next task, R == 2 - * T1: finish T1, poll queue (it's still empty), R == 2 - * T2: submit next task to the local queue, decrement R, R == 1 - * T1: decrement R, finish. R == 0 - * - * The task from T2 is stuck is the local queue - */ - next = queue.poll() ?: return - dispatch(next, true) - } -} diff --git a/kotlinx-coroutines-core/jvm/src/scheduling/Tasks.kt b/kotlinx-coroutines-core/jvm/src/scheduling/Tasks.kt index da867c9853..5403cfc1fd 100644 --- a/kotlinx-coroutines-core/jvm/src/scheduling/Tasks.kt +++ b/kotlinx-coroutines-core/jvm/src/scheduling/Tasks.kt @@ -9,10 +9,6 @@ import kotlinx.coroutines.internal.* import java.util.concurrent.* -// TODO most of these fields will be moved to 'object ExperimentalDispatcher' - -// User-visible name -internal const val DEFAULT_DISPATCHER_NAME = "Dispatchers.Default" // Internal debuggability name + thread name prefixes internal const val DEFAULT_SCHEDULER_NAME = "DefaultDispatcher" @@ -22,27 +18,24 @@ internal val WORK_STEALING_TIME_RESOLUTION_NS = systemProp( "kotlinx.coroutines.scheduler.resolution.ns", 100000L ) -@JvmField -internal val BLOCKING_DEFAULT_PARALLELISM = systemProp( - "kotlinx.coroutines.scheduler.blocking.parallelism", 16 -) - -// NOTE: we coerce default to at least two threads to give us chances that multi-threading problems -// get reproduced even on a single-core machine, but support explicit setting of 1 thread scheduler if needed. +/** + * The maximum number of threads allocated for CPU-bound tasks at the default set of dispatchers. + * + * NOTE: we coerce default to at least two threads to give us chances that multi-threading problems + * get reproduced even on a single-core machine, but support explicit setting of 1 thread scheduler if needed + */ @JvmField internal val CORE_POOL_SIZE = systemProp( "kotlinx.coroutines.scheduler.core.pool.size", - AVAILABLE_PROCESSORS.coerceAtLeast(2), // !!! at least two here + AVAILABLE_PROCESSORS.coerceAtLeast(2), minValue = CoroutineScheduler.MIN_SUPPORTED_POOL_SIZE ) +/** The maximum number of threads allocated for blocking tasks at the default set of dispatchers. */ @JvmField internal val MAX_POOL_SIZE = systemProp( "kotlinx.coroutines.scheduler.max.pool.size", - (AVAILABLE_PROCESSORS * 128).coerceIn( - CORE_POOL_SIZE, - CoroutineScheduler.MAX_SUPPORTED_POOL_SIZE - ), + CoroutineScheduler.MAX_SUPPORTED_POOL_SIZE, maxValue = CoroutineScheduler.MAX_SUPPORTED_POOL_SIZE ) @@ -69,14 +62,18 @@ internal interface TaskContext { fun afterTask() } -internal object NonBlockingContext : TaskContext { - override val taskMode: Int = TASK_NON_BLOCKING - +private class TaskContextImpl(override val taskMode: Int): TaskContext { override fun afterTask() { - // Nothing for non-blocking context + // Nothing for non-blocking context } } +@JvmField +internal val NonBlockingContext: TaskContext = TaskContextImpl(TASK_NON_BLOCKING) + +@JvmField +internal val BlockingContext: TaskContext = TaskContextImpl(TASK_PROBABLY_BLOCKING) + internal abstract class Task( @JvmField var submissionTime: Long, @JvmField var taskContext: TaskContext diff --git a/kotlinx-coroutines-core/jvm/test/LimitedParallelismStressTest.kt b/kotlinx-coroutines-core/jvm/test/LimitedParallelismStressTest.kt new file mode 100644 index 0000000000..4de1862b0f --- /dev/null +++ b/kotlinx-coroutines-core/jvm/test/LimitedParallelismStressTest.kt @@ -0,0 +1,83 @@ +/* + * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines + +import org.junit.* +import org.junit.Test +import org.junit.runner.* +import org.junit.runners.* +import java.util.concurrent.* +import java.util.concurrent.atomic.* +import kotlin.test.* + +@RunWith(Parameterized::class) +class LimitedParallelismStressTest(private val targetParallelism: Int) : TestBase() { + + companion object { + @Parameterized.Parameters(name = "{0}") + @JvmStatic + fun params(): Collection> = listOf(1, 2, 3, 4).map { arrayOf(it) } + } + + @get:Rule + val executor = ExecutorRule(targetParallelism * 2) + private val iterations = 100_000 * stressTestMultiplier + + private val parallelism = AtomicInteger(0) + + private fun checkParallelism() { + val value = parallelism.incrementAndGet() + Thread.yield() + assertTrue { value <= targetParallelism } + parallelism.decrementAndGet() + } + + @Test + fun testLimitedExecutor() = runTest { + val view = executor.limitedParallelism(targetParallelism) + repeat(iterations) { + launch(view) { + checkParallelism() + } + } + } + + @Test + fun testLimitedDispatchersIo() = runTest { + val view = Dispatchers.IO.limitedParallelism(targetParallelism) + repeat(iterations) { + launch(view) { + checkParallelism() + } + } + } + + @Test + fun testLimitedDispatchersIoDispatchYield() = runTest { + val view = Dispatchers.IO.limitedParallelism(targetParallelism) + repeat(iterations) { + launch(view) { + yield() + checkParallelism() + } + } + } + + @Test + fun testLimitedExecutorReachesTargetParallelism() = runTest { + val view = executor.limitedParallelism(targetParallelism) + repeat(iterations) { + val barrier = CyclicBarrier(targetParallelism + 1) + repeat(targetParallelism) { + launch(view) { + barrier.await() + } + } + // Successfully awaited parallelism + 1 + barrier.await() + coroutineContext.job.children.toList().joinAll() + } + } +} diff --git a/kotlinx-coroutines-core/jvm/test/LimitedParallelismTest.kt b/kotlinx-coroutines-core/jvm/test/LimitedParallelismTest.kt new file mode 100644 index 0000000000..30c54117a9 --- /dev/null +++ b/kotlinx-coroutines-core/jvm/test/LimitedParallelismTest.kt @@ -0,0 +1,55 @@ +/* + * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines + +import org.junit.Test +import java.util.concurrent.* +import kotlin.coroutines.* +import kotlin.test.* + +class LimitedParallelismTest : TestBase() { + + @Test + fun testParallelismSpec() { + assertFailsWith { Dispatchers.Default.limitedParallelism(0) } + assertFailsWith { Dispatchers.Default.limitedParallelism(-1) } + assertFailsWith { Dispatchers.Default.limitedParallelism(Int.MIN_VALUE) } + Dispatchers.Default.limitedParallelism(Int.MAX_VALUE) + } + + @Test + fun testTaskFairness() = runTest { + val executor = newSingleThreadContext("test") + val view = executor.limitedParallelism(1) + val view2 = executor.limitedParallelism(1) + val j1 = launch(view) { + while (true) { + yield() + } + } + val j2 = launch(view2) { j1.cancel() } + joinAll(j1, j2) + executor.close() + } + + @Test + fun testUnhandledException() = runTest { + var caughtException: Throwable? = null + val executor = Executors.newFixedThreadPool( + 1 + ) { + Thread(it).also { + it.uncaughtExceptionHandler = Thread.UncaughtExceptionHandler { _, e -> caughtException = e } + } + }.asCoroutineDispatcher() + val view = executor.limitedParallelism(1) + view.dispatch(EmptyCoroutineContext, Runnable { throw TestException() }) + withContext(view) { + // Verify it is in working state and establish happens-before + } + assertTrue { caughtException is TestException } + executor.close() + } +} diff --git a/kotlinx-coroutines-core/jvm/test/scheduling/BlockingCoroutineDispatcherTerminationStressTest.kt b/kotlinx-coroutines-core/jvm/test/scheduling/BlockingCoroutineDispatcherTerminationStressTest.kt index 9c17e6988d..864ecdc087 100644 --- a/kotlinx-coroutines-core/jvm/test/scheduling/BlockingCoroutineDispatcherTerminationStressTest.kt +++ b/kotlinx-coroutines-core/jvm/test/scheduling/BlockingCoroutineDispatcherTerminationStressTest.kt @@ -10,7 +10,7 @@ import java.util.* import java.util.concurrent.* class BlockingCoroutineDispatcherTerminationStressTest : TestBase() { - private val baseDispatcher = ExperimentalCoroutineDispatcher( + private val baseDispatcher = SchedulerCoroutineDispatcher( 2, 20, TimeUnit.MILLISECONDS.toNanos(10) ) diff --git a/kotlinx-coroutines-core/jvm/test/scheduling/BlockingCoroutineDispatcherTest.kt b/kotlinx-coroutines-core/jvm/test/scheduling/BlockingCoroutineDispatcherTest.kt index fe09440f59..f8830feeef 100644 --- a/kotlinx-coroutines-core/jvm/test/scheduling/BlockingCoroutineDispatcherTest.kt +++ b/kotlinx-coroutines-core/jvm/test/scheduling/BlockingCoroutineDispatcherTest.kt @@ -125,71 +125,6 @@ class BlockingCoroutineDispatcherTest : SchedulerTestBase() { checkPoolThreadsCreated(101..100 + CORES_COUNT) } - @Test - fun testBlockingFairness() = runBlocking { - corePoolSize = 1 - maxPoolSize = 1 - - val blocking = blockingDispatcher(1) - val task = async(dispatcher) { - expect(1) - - val nonBlocking = async(dispatcher) { - expect(3) - } - - val firstBlocking = async(blocking) { - expect(2) - } - - val secondBlocking = async(blocking) { - // Already have 1 queued blocking task, so this one wouldn't be scheduled to head - expect(4) - } - - listOf(firstBlocking, nonBlocking, secondBlocking).joinAll() - finish(5) - } - - task.await() - } - - @Test - fun testBoundedBlockingFairness() = runBlocking { - corePoolSize = 1 - maxPoolSize = 1 - - val blocking = blockingDispatcher(2) - val task = async(dispatcher) { - expect(1) - - val nonBlocking = async(dispatcher) { - expect(3) - } - - val firstBlocking = async(blocking) { - expect(4) - } - - val secondNonBlocking = async(dispatcher) { - expect(5) - } - - val secondBlocking = async(blocking) { - expect(2) // <- last submitted blocking is executed first - } - - val thirdBlocking = async(blocking) { - expect(6) // parallelism level is reached before this task - } - - listOf(firstBlocking, nonBlocking, secondBlocking, secondNonBlocking, thirdBlocking).joinAll() - finish(7) - } - - task.await() - } - @Test(timeout = 1_000) fun testYield() = runBlocking { corePoolSize = 1 diff --git a/kotlinx-coroutines-core/jvm/test/scheduling/BlockingCoroutineDispatcherWorkSignallingStressTest.kt b/kotlinx-coroutines-core/jvm/test/scheduling/BlockingCoroutineDispatcherWorkSignallingStressTest.kt index 3280527f2a..3b3e085047 100644 --- a/kotlinx-coroutines-core/jvm/test/scheduling/BlockingCoroutineDispatcherWorkSignallingStressTest.kt +++ b/kotlinx-coroutines-core/jvm/test/scheduling/BlockingCoroutineDispatcherWorkSignallingStressTest.kt @@ -18,7 +18,7 @@ class BlockingCoroutineDispatcherWorkSignallingStressTest : SchedulerTestBase() val iterations = 1000 * stressTestMultiplier repeat(iterations) { // Create a dispatcher every iteration to increase probability of race - val dispatcher = ExperimentalCoroutineDispatcher(CORES_COUNT) + val dispatcher = SchedulerCoroutineDispatcher(CORES_COUNT) val blockingDispatcher = dispatcher.blocking(100) val blockingBarrier = CyclicBarrier(CORES_COUNT * 3 + 1) diff --git a/kotlinx-coroutines-core/jvm/test/scheduling/CoroutineDispatcherTest.kt b/kotlinx-coroutines-core/jvm/test/scheduling/CoroutineDispatcherTest.kt index 3cd77da74a..c95415a8df 100644 --- a/kotlinx-coroutines-core/jvm/test/scheduling/CoroutineDispatcherTest.kt +++ b/kotlinx-coroutines-core/jvm/test/scheduling/CoroutineDispatcherTest.kt @@ -134,7 +134,7 @@ class CoroutineDispatcherTest : SchedulerTestBase() { val initialCount = Thread.getAllStackTraces().keys.asSequence() .count { it is CoroutineScheduler.Worker && it.name.contains("SomeTestName") } assertEquals(0, initialCount) - val dispatcher = ExperimentalCoroutineDispatcher(1, 1, IDLE_WORKER_KEEP_ALIVE_NS, "SomeTestName") + val dispatcher = SchedulerCoroutineDispatcher(1, 1, IDLE_WORKER_KEEP_ALIVE_NS, "SomeTestName") dispatcher.use { launch(dispatcher) { }.join() diff --git a/kotlinx-coroutines-core/jvm/test/scheduling/CoroutineSchedulerCloseStressTest.kt b/kotlinx-coroutines-core/jvm/test/scheduling/CoroutineSchedulerCloseStressTest.kt index 473b429283..a50867d61c 100644 --- a/kotlinx-coroutines-core/jvm/test/scheduling/CoroutineSchedulerCloseStressTest.kt +++ b/kotlinx-coroutines-core/jvm/test/scheduling/CoroutineSchedulerCloseStressTest.kt @@ -22,15 +22,13 @@ class CoroutineSchedulerCloseStressTest(private val mode: Mode) : TestBase() { fun params(): Collection> = Mode.values().map { arrayOf(it) } } - private val N_REPEAT = 2 * stressTestMultiplier private val MAX_LEVEL = 5 private val N_COROS = (1 shl (MAX_LEVEL + 1)) - 1 private val N_THREADS = 4 private val rnd = Random() - private lateinit var closeableDispatcher: ExperimentalCoroutineDispatcher - private lateinit var dispatcher: ExecutorCoroutineDispatcher - private var closeIndex = -1 + private lateinit var closeableDispatcher: SchedulerCoroutineDispatcher + private lateinit var dispatcher: CoroutineDispatcher private val started = atomic(0) private val finished = atomic(0) @@ -44,20 +42,12 @@ class CoroutineSchedulerCloseStressTest(private val mode: Mode) : TestBase() { } } - @Test - fun testRacingClose() { - repeat(N_REPEAT) { - closeIndex = rnd.nextInt(N_COROS) - launchCoroutines() - } - } - private fun launchCoroutines() = runBlocking { - closeableDispatcher = ExperimentalCoroutineDispatcher(N_THREADS) + closeableDispatcher = SchedulerCoroutineDispatcher(N_THREADS) dispatcher = when (mode) { Mode.CPU -> closeableDispatcher - Mode.CPU_LIMITED -> closeableDispatcher.limited(N_THREADS) as ExecutorCoroutineDispatcher - Mode.BLOCKING -> closeableDispatcher.blocking(N_THREADS) as ExecutorCoroutineDispatcher + Mode.CPU_LIMITED -> closeableDispatcher.limitedParallelism(N_THREADS) + Mode.BLOCKING -> closeableDispatcher.blocking(N_THREADS) } started.value = 0 finished.value = 0 @@ -68,20 +58,16 @@ class CoroutineSchedulerCloseStressTest(private val mode: Mode) : TestBase() { assertEquals(N_COROS, finished.value) } + // Index and level are used only for debugging purpose private fun CoroutineScope.launchChild(index: Int, level: Int): Job = launch(start = CoroutineStart.ATOMIC) { started.incrementAndGet() try { - if (index == closeIndex) closeableDispatcher.close() if (level < MAX_LEVEL) { launchChild(2 * index + 1, level + 1) launchChild(2 * index + 2, level + 1) } else { if (rnd.nextBoolean()) { delay(1000) - val t = Thread.currentThread() - if (!t.name.contains("DefaultDispatcher-worker")) { - val a = 2 - } } else { yield() } diff --git a/kotlinx-coroutines-core/jvm/test/scheduling/CoroutineSchedulerStressTest.kt b/kotlinx-coroutines-core/jvm/test/scheduling/CoroutineSchedulerStressTest.kt index cb49f054ce..7aefd4f75c 100644 --- a/kotlinx-coroutines-core/jvm/test/scheduling/CoroutineSchedulerStressTest.kt +++ b/kotlinx-coroutines-core/jvm/test/scheduling/CoroutineSchedulerStressTest.kt @@ -15,7 +15,7 @@ import kotlin.coroutines.* import kotlin.test.* class CoroutineSchedulerStressTest : TestBase() { - private var dispatcher: ExperimentalCoroutineDispatcher = ExperimentalCoroutineDispatcher() + private var dispatcher: SchedulerCoroutineDispatcher = SchedulerCoroutineDispatcher() private val observedThreads = ConcurrentHashMap() private val tasksNum = 500_000 * stressMemoryMultiplier() diff --git a/kotlinx-coroutines-core/jvm/test/scheduling/CoroutineSchedulerTest.kt b/kotlinx-coroutines-core/jvm/test/scheduling/CoroutineSchedulerTest.kt index b0a5954b70..9d41c05d26 100644 --- a/kotlinx-coroutines-core/jvm/test/scheduling/CoroutineSchedulerTest.kt +++ b/kotlinx-coroutines-core/jvm/test/scheduling/CoroutineSchedulerTest.kt @@ -106,22 +106,22 @@ class CoroutineSchedulerTest : TestBase() { @Test(expected = IllegalArgumentException::class) fun testNegativeCorePoolSize() { - ExperimentalCoroutineDispatcher(-1, 4) + SchedulerCoroutineDispatcher(-1, 4) } @Test(expected = IllegalArgumentException::class) fun testNegativeMaxPoolSize() { - ExperimentalCoroutineDispatcher(1, -4) + SchedulerCoroutineDispatcher(1, -4) } @Test(expected = IllegalArgumentException::class) fun testCorePoolSizeGreaterThanMaxPoolSize() { - ExperimentalCoroutineDispatcher(4, 1) + SchedulerCoroutineDispatcher(4, 1) } @Test fun testSelfClose() { - val dispatcher = ExperimentalCoroutineDispatcher(1, 1) + val dispatcher = SchedulerCoroutineDispatcher(1, 1) val latch = CountDownLatch(1) dispatcher.dispatch(EmptyCoroutineContext, Runnable { dispatcher.close(); latch.countDown() @@ -131,7 +131,7 @@ class CoroutineSchedulerTest : TestBase() { @Test fun testInterruptionCleanup() { - ExperimentalCoroutineDispatcher(1, 1).use { + SchedulerCoroutineDispatcher(1, 1).use { val executor = it.executor var latch = CountDownLatch(1) executor.execute { @@ -171,4 +171,4 @@ class CoroutineSchedulerTest : TestBase() { private class TaskContextImpl(override val taskMode: Int) : TaskContext { override fun afterTask() {} } -} \ No newline at end of file +} diff --git a/kotlinx-coroutines-core/jvm/test/scheduling/DefaultDispatchersTest.kt b/kotlinx-coroutines-core/jvm/test/scheduling/DefaultDispatchersTest.kt new file mode 100644 index 0000000000..56c669547c --- /dev/null +++ b/kotlinx-coroutines-core/jvm/test/scheduling/DefaultDispatchersTest.kt @@ -0,0 +1,75 @@ +/* + * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines.scheduling + +import kotlinx.coroutines.* +import org.junit.Test +import java.util.concurrent.* +import java.util.concurrent.atomic.* +import kotlin.test.* + +class DefaultDispatchersTest : TestBase() { + + private /*const*/ val EXPECTED_PARALLELISM = 64 + + @Test(timeout = 10_000L) + fun testLimitedParallelismIsSeparatedFromDefaultIo() = runTest { + val barrier = CyclicBarrier(EXPECTED_PARALLELISM + 1) + val ioBlocker = CountDownLatch(1) + repeat(EXPECTED_PARALLELISM) { + launch(Dispatchers.IO) { + barrier.await() + ioBlocker.await() + } + } + + barrier.await() // Ensure all threads are occupied + barrier.reset() + val limited = Dispatchers.IO.limitedParallelism(EXPECTED_PARALLELISM) + repeat(EXPECTED_PARALLELISM) { + launch(limited) { + barrier.await() + } + } + barrier.await() + ioBlocker.countDown() + } + + @Test(timeout = 10_000L) + fun testDefaultDispatcherIsSeparateFromIO() = runTest { + val ioBarrier = CyclicBarrier(EXPECTED_PARALLELISM + 1) + val ioBlocker = CountDownLatch(1) + repeat(EXPECTED_PARALLELISM) { + launch(Dispatchers.IO) { + ioBarrier.await() + ioBlocker.await() + } + } + + ioBarrier.await() // Ensure all threads are occupied + val parallelism = Runtime.getRuntime().availableProcessors() + val defaultBarrier = CyclicBarrier(parallelism + 1) + repeat(parallelism) { + launch(Dispatchers.Default) { + defaultBarrier.await() + } + } + defaultBarrier.await() + ioBlocker.countDown() + } + + @Test + fun testHardCapOnParallelism() = runTest { + val iterations = 100_000 * stressTestMultiplierSqrt + val concurrency = AtomicInteger() + repeat(iterations) { + launch(Dispatchers.IO) { + val c = concurrency.incrementAndGet() + assertTrue("Got: $c") { c <= EXPECTED_PARALLELISM } + concurrency.decrementAndGet() + } + } + } +} diff --git a/kotlinx-coroutines-core/jvm/test/scheduling/LimitingDispatcherTest.kt b/kotlinx-coroutines-core/jvm/test/scheduling/LimitingDispatcherTest.kt index b4924277b5..e5705803c0 100644 --- a/kotlinx-coroutines-core/jvm/test/scheduling/LimitingDispatcherTest.kt +++ b/kotlinx-coroutines-core/jvm/test/scheduling/LimitingDispatcherTest.kt @@ -10,11 +10,6 @@ import java.util.concurrent.* class LimitingDispatcherTest : SchedulerTestBase() { - @Test(expected = IllegalArgumentException::class) - fun testTooLargeView() { - view(corePoolSize + 1) - } - @Test(expected = IllegalArgumentException::class) fun testNegativeView() { view(-1) diff --git a/kotlinx-coroutines-core/jvm/test/scheduling/SchedulerTestBase.kt b/kotlinx-coroutines-core/jvm/test/scheduling/SchedulerTestBase.kt index dd969bdd37..fc4436f418 100644 --- a/kotlinx-coroutines-core/jvm/test/scheduling/SchedulerTestBase.kt +++ b/kotlinx-coroutines-core/jvm/test/scheduling/SchedulerTestBase.kt @@ -6,7 +6,6 @@ package kotlinx.coroutines.scheduling -import kotlinx.atomicfu.* import kotlinx.coroutines.* import kotlinx.coroutines.internal.* import org.junit.* @@ -61,11 +60,11 @@ abstract class SchedulerTestBase : TestBase() { protected var maxPoolSize = 1024 protected var idleWorkerKeepAliveNs = IDLE_WORKER_KEEP_ALIVE_NS - private var _dispatcher: ExperimentalCoroutineDispatcher? = null + private var _dispatcher: SchedulerCoroutineDispatcher? = null protected val dispatcher: CoroutineDispatcher get() { if (_dispatcher == null) { - _dispatcher = ExperimentalCoroutineDispatcher( + _dispatcher = SchedulerCoroutineDispatcher( corePoolSize, maxPoolSize, idleWorkerKeepAliveNs @@ -86,7 +85,7 @@ abstract class SchedulerTestBase : TestBase() { protected fun view(parallelism: Int): CoroutineDispatcher { val intitialize = dispatcher - return _dispatcher!!.limited(parallelism) + return _dispatcher!!.limitedParallelism(parallelism) } @After @@ -98,3 +97,17 @@ abstract class SchedulerTestBase : TestBase() { } } } + +internal fun SchedulerCoroutineDispatcher.blocking(parallelism: Int = 16): CoroutineDispatcher { + return object : CoroutineDispatcher() { + + @InternalCoroutinesApi + override fun dispatchYield(context: CoroutineContext, block: Runnable) { + this@blocking.dispatchWithContext(block, BlockingContext, true) + } + + override fun dispatch(context: CoroutineContext, block: Runnable) { + this@blocking.dispatchWithContext(block, BlockingContext, false) + } + }.limitedParallelism(parallelism) +} diff --git a/kotlinx-coroutines-core/jvm/test/scheduling/SharingWorkerClassTest.kt b/kotlinx-coroutines-core/jvm/test/scheduling/SharingWorkerClassTest.kt index 6a66da9f5c..743b4a617f 100644 --- a/kotlinx-coroutines-core/jvm/test/scheduling/SharingWorkerClassTest.kt +++ b/kotlinx-coroutines-core/jvm/test/scheduling/SharingWorkerClassTest.kt @@ -13,8 +13,8 @@ class SharingWorkerClassTest : SchedulerTestBase() { @Test fun testSharedThread() = runTest { - val dispatcher = ExperimentalCoroutineDispatcher(1, schedulerName = "first") - val dispatcher2 = ExperimentalCoroutineDispatcher(1, schedulerName = "second") + val dispatcher = SchedulerCoroutineDispatcher(1, schedulerName = "first") + val dispatcher2 = SchedulerCoroutineDispatcher(1, schedulerName = "second") try { withContext(dispatcher) { @@ -39,7 +39,7 @@ class SharingWorkerClassTest : SchedulerTestBase() { val cores = Runtime.getRuntime().availableProcessors() repeat(cores + 1) { CoroutineScope(Dispatchers.Default).launch { - ExperimentalCoroutineDispatcher(1).close() + SchedulerCoroutineDispatcher(1).close() }.join() } } From bd44a9421be5b206a02a2341434513a2483610ba Mon Sep 17 00:00:00 2001 From: Vsevolod Tolstopyatov Date: Tue, 12 Oct 2021 19:05:44 +0300 Subject: [PATCH 21/24] Eagerly load CoroutineExceptionHandler and load corresponding service (#2957) * Eagerly load CoroutineExceptionHandler and load the corresponding service Partially addresses #2552 --- .../common/src/CoroutineExceptionHandler.kt | 8 +++++++- kotlinx-coroutines-core/common/src/Job.kt | 8 +++++++- .../js/src/CoroutineExceptionHandlerImpl.kt | 4 ++++ .../jvm/src/CoroutineExceptionHandlerImpl.kt | 6 ++++++ .../native/src/CoroutineExceptionHandlerImpl.kt | 4 ++++ 5 files changed, 28 insertions(+), 2 deletions(-) diff --git a/kotlinx-coroutines-core/common/src/CoroutineExceptionHandler.kt b/kotlinx-coroutines-core/common/src/CoroutineExceptionHandler.kt index 49923a92e7..819f205b17 100644 --- a/kotlinx-coroutines-core/common/src/CoroutineExceptionHandler.kt +++ b/kotlinx-coroutines-core/common/src/CoroutineExceptionHandler.kt @@ -1,13 +1,19 @@ /* * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. */ - package kotlinx.coroutines import kotlin.coroutines.* +import kotlin.jvm.* internal expect fun handleCoroutineExceptionImpl(context: CoroutineContext, exception: Throwable) +/** + * JVM kludge: trigger loading of all the classes and service loading + * **before** any exception occur because it may be OOM, SOE or VerifyError + */ +internal expect fun initializeDefaultExceptionHandlers() + /** * Helper function for coroutine builder implementations to handle uncaught and unexpected exceptions in coroutines, * that could not be otherwise handled in a normal way through structured concurrency, saving them to a future, and diff --git a/kotlinx-coroutines-core/common/src/Job.kt b/kotlinx-coroutines-core/common/src/Job.kt index 31d90eeef0..085ef7e8af 100644 --- a/kotlinx-coroutines-core/common/src/Job.kt +++ b/kotlinx-coroutines-core/common/src/Job.kt @@ -113,7 +113,13 @@ public interface Job : CoroutineContext.Element { /** * Key for [Job] instance in the coroutine context. */ - public companion object Key : CoroutineContext.Key + public companion object Key : CoroutineContext.Key { + init { + // `Job` will necessarily be accessed early, so this is as good a place as any for the + // initialization logic that we want to happen as soon as possible + initializeDefaultExceptionHandlers() + } + } // ------------ state query ------------ diff --git a/kotlinx-coroutines-core/js/src/CoroutineExceptionHandlerImpl.kt b/kotlinx-coroutines-core/js/src/CoroutineExceptionHandlerImpl.kt index 54a65e10a6..a4d671dc65 100644 --- a/kotlinx-coroutines-core/js/src/CoroutineExceptionHandlerImpl.kt +++ b/kotlinx-coroutines-core/js/src/CoroutineExceptionHandlerImpl.kt @@ -6,6 +6,10 @@ package kotlinx.coroutines import kotlin.coroutines.* +internal actual fun initializeDefaultExceptionHandlers() { + // Do nothing +} + internal actual fun handleCoroutineExceptionImpl(context: CoroutineContext, exception: Throwable) { // log exception console.error(exception) diff --git a/kotlinx-coroutines-core/jvm/src/CoroutineExceptionHandlerImpl.kt b/kotlinx-coroutines-core/jvm/src/CoroutineExceptionHandlerImpl.kt index 6d06969293..4c8c81b8db 100644 --- a/kotlinx-coroutines-core/jvm/src/CoroutineExceptionHandlerImpl.kt +++ b/kotlinx-coroutines-core/jvm/src/CoroutineExceptionHandlerImpl.kt @@ -22,6 +22,12 @@ private val handlers: List = ServiceLoader.load( CoroutineExceptionHandler::class.java.classLoader ).iterator().asSequence().toList() +internal actual fun initializeDefaultExceptionHandlers() { + // Load CEH and handlers + CoroutineExceptionHandler +} + + internal actual fun handleCoroutineExceptionImpl(context: CoroutineContext, exception: Throwable) { // use additional extension handlers for (handler in handlers) { diff --git a/kotlinx-coroutines-core/native/src/CoroutineExceptionHandlerImpl.kt b/kotlinx-coroutines-core/native/src/CoroutineExceptionHandlerImpl.kt index b0aa86339a..7fedbd9fac 100644 --- a/kotlinx-coroutines-core/native/src/CoroutineExceptionHandlerImpl.kt +++ b/kotlinx-coroutines-core/native/src/CoroutineExceptionHandlerImpl.kt @@ -6,6 +6,10 @@ package kotlinx.coroutines import kotlin.coroutines.* +internal actual fun initializeDefaultExceptionHandlers() { + // Do nothing +} + internal actual fun handleCoroutineExceptionImpl(context: CoroutineContext, exception: Throwable) { // log exception exception.printStackTrace() From d40f90cf136317777a05c20c7597bd6cfed4134d Mon Sep 17 00:00:00 2001 From: Vsevolod Tolstopyatov Date: Tue, 12 Oct 2021 20:01:10 +0300 Subject: [PATCH 22/24] Update Kotlin to 1.6.0-RC (#2980) --- gradle.properties | 2 +- .../common/src/flow/operators/Delay.kt | 1 + .../common/test/DelayDurationTest.kt | 2 ++ .../SharingStartedWhileSubscribedTest.kt | 7 +++++-- .../jvm/resources/DebugProbesKt.bin | Bin 1714 -> 1719 bytes .../jvm/test/examples/example-delay-01.kt | 1 + .../jvm/test/examples/example-delay-02.kt | 1 + .../jvm/test/examples/example-delay-03.kt | 1 + .../examples/example-delay-duration-01.kt | 1 + .../examples/example-delay-duration-02.kt | 1 + .../examples/example-delay-duration-03.kt | 1 + .../jvm/test/flow/SharedFlowStressTest.kt | 4 ++-- ui/kotlinx-coroutines-javafx/build.gradle.kts | 19 ++++++++++-------- 13 files changed, 28 insertions(+), 13 deletions(-) diff --git a/gradle.properties b/gradle.properties index 46eef4d76e..f0d49b5ef8 100644 --- a/gradle.properties +++ b/gradle.properties @@ -5,7 +5,7 @@ # Kotlin version=1.5.2-SNAPSHOT group=org.jetbrains.kotlinx -kotlin_version=1.5.30 +kotlin_version=1.6.0-RC # Dependencies junit_version=4.12 diff --git a/kotlinx-coroutines-core/common/src/flow/operators/Delay.kt b/kotlinx-coroutines-core/common/src/flow/operators/Delay.kt index fed5962bd5..e893f44ea5 100644 --- a/kotlinx-coroutines-core/common/src/flow/operators/Delay.kt +++ b/kotlinx-coroutines-core/common/src/flow/operators/Delay.kt @@ -23,6 +23,7 @@ import kotlin.time.* ----- INCLUDE .* import kotlinx.coroutines.* import kotlinx.coroutines.flow.* +import kotlin.time.Duration.Companion.milliseconds fun main() = runBlocking { ----- SUFFIX .* diff --git a/kotlinx-coroutines-core/common/test/DelayDurationTest.kt b/kotlinx-coroutines-core/common/test/DelayDurationTest.kt index 3dd55bde84..1c6c189a44 100644 --- a/kotlinx-coroutines-core/common/test/DelayDurationTest.kt +++ b/kotlinx-coroutines-core/common/test/DelayDurationTest.kt @@ -10,6 +10,8 @@ package kotlinx.coroutines import kotlin.test.* import kotlin.time.* +import kotlin.time.Duration.Companion.seconds +import kotlin.time.Duration.Companion.nanoseconds @ExperimentalTime class DelayDurationTest : TestBase() { diff --git a/kotlinx-coroutines-core/common/test/flow/sharing/SharingStartedWhileSubscribedTest.kt b/kotlinx-coroutines-core/common/test/flow/sharing/SharingStartedWhileSubscribedTest.kt index b3a3400389..3b961c5783 100644 --- a/kotlinx-coroutines-core/common/test/flow/sharing/SharingStartedWhileSubscribedTest.kt +++ b/kotlinx-coroutines-core/common/test/flow/sharing/SharingStartedWhileSubscribedTest.kt @@ -7,6 +7,8 @@ package kotlinx.coroutines.flow import kotlinx.coroutines.* import kotlin.test.* import kotlin.time.* +import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.Duration.Companion.seconds class SharingStartedWhileSubscribedTest : TestBase() { @Test // make sure equals works properly, or otherwise other tests don't make sense @@ -30,14 +32,15 @@ class SharingStartedWhileSubscribedTest : TestBase() { @Test fun testDurationParams() { assertEquals(SharingStarted.WhileSubscribed(0), SharingStarted.WhileSubscribed(Duration.ZERO)) - assertEquals(SharingStarted.WhileSubscribed(10), SharingStarted.WhileSubscribed(Duration.milliseconds(10))) + assertEquals(SharingStarted.WhileSubscribed(10), SharingStarted.WhileSubscribed(10.milliseconds)) assertEquals(SharingStarted.WhileSubscribed(1000), SharingStarted.WhileSubscribed(1.seconds)) assertEquals(SharingStarted.WhileSubscribed(Long.MAX_VALUE), SharingStarted.WhileSubscribed(Duration.INFINITE)) assertEquals(SharingStarted.WhileSubscribed(replayExpirationMillis = 0), SharingStarted.WhileSubscribed(replayExpiration = Duration.ZERO)) assertEquals(SharingStarted.WhileSubscribed(replayExpirationMillis = 3), SharingStarted.WhileSubscribed( replayExpiration = Duration.milliseconds(3) )) - assertEquals(SharingStarted.WhileSubscribed(replayExpirationMillis = 7000), SharingStarted.WhileSubscribed(replayExpiration = 7.seconds)) + assertEquals(SharingStarted.WhileSubscribed(replayExpirationMillis = 7000), + SharingStarted.WhileSubscribed(replayExpiration = 7.seconds)) assertEquals(SharingStarted.WhileSubscribed(replayExpirationMillis = Long.MAX_VALUE), SharingStarted.WhileSubscribed(replayExpiration = Duration.INFINITE)) } diff --git a/kotlinx-coroutines-core/jvm/resources/DebugProbesKt.bin b/kotlinx-coroutines-core/jvm/resources/DebugProbesKt.bin index 397aaf67ac52cdf6b99d39da8ddf1065bc098293..6c1500ae6e012b9ca4643da3671d498b39907235 100644 GIT binary patch delta 324 zcma)$OHKk&5Jk`9r5ou0aVQ`#Ad!y>3dsL6BfK^AGHL;g9rs2+O-vZToJA2!Vn7%% zxViTxRj2AB^_>cK`}GEBGhGE=*7vOzKDYMWm~P~};=HHb$EX7Onvc00Sj9numAu6 delta 282 zcmdnayNQ?U)W2Q(7#J8#80fY#tz+ zfq{>KlR*kdTLLi?13QBikOryZ0Mgcz7qE%|bqlabOg_uXZi6s^2gL+ABokyAtbsb& z8KFi Date: Wed, 13 Oct 2021 23:24:56 +0300 Subject: [PATCH 23/24] Handle exceptions on K/N by invoking 'processUnhandledException' (#2981) --- .../native/src/CoroutineExceptionHandlerImpl.kt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kotlinx-coroutines-core/native/src/CoroutineExceptionHandlerImpl.kt b/kotlinx-coroutines-core/native/src/CoroutineExceptionHandlerImpl.kt index 7fedbd9fac..d97743b4bf 100644 --- a/kotlinx-coroutines-core/native/src/CoroutineExceptionHandlerImpl.kt +++ b/kotlinx-coroutines-core/native/src/CoroutineExceptionHandlerImpl.kt @@ -5,12 +5,14 @@ package kotlinx.coroutines import kotlin.coroutines.* +import kotlin.native.* internal actual fun initializeDefaultExceptionHandlers() { // Do nothing } +@OptIn(ExperimentalStdlibApi::class) internal actual fun handleCoroutineExceptionImpl(context: CoroutineContext, exception: Throwable) { // log exception - exception.printStackTrace() + processUnhandledException(exception) } From 13ea81e2f754b0fee9ec38c2b0ed6f0e707a74d1 Mon Sep 17 00:00:00 2001 From: Tyson Henning Date: Wed, 13 Oct 2021 22:28:48 +0000 Subject: [PATCH 24/24] Implemented `CopyableThreadContextElement` with a `copyForChildCoroutine()`. This is a `ThreadContextElement` that is copy-constructed when a new coroutine is created and inherits the context. --- kotlinx-coroutines-core/jvm/src/CoroutineContext.kt | 6 +++--- kotlinx-coroutines-core/jvm/src/ThreadContextElement.kt | 1 + .../jvm/test/ThreadContextElementTest.kt | 8 ++++---- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/kotlinx-coroutines-core/jvm/src/CoroutineContext.kt b/kotlinx-coroutines-core/jvm/src/CoroutineContext.kt index 36fd2b83dc..4864c0abd6 100644 --- a/kotlinx-coroutines-core/jvm/src/CoroutineContext.kt +++ b/kotlinx-coroutines-core/jvm/src/CoroutineContext.kt @@ -33,10 +33,10 @@ public actual fun CoroutineScope.newCoroutineContext(context: CoroutineContext): * Returns [this] if `this` has zero [CopyableThreadContextElement] in it. */ private fun CoroutineContext.foldCopiesForChildCoroutine(): CoroutineContext { - val jobElementCount = fold(0) { count, it -> - count + if (it is CopyableThreadContextElement<*>) 1 else 0 + val hasToCopy = fold(false) { result, it -> + result || it is CopyableThreadContextElement<*> } - if (jobElementCount == 0) return this + if (!hasToCopy) return this return fold(EmptyCoroutineContext) { combined, it -> combined + if (it is CopyableThreadContextElement<*>) it.copyForChildCoroutine() else it } diff --git a/kotlinx-coroutines-core/jvm/src/ThreadContextElement.kt b/kotlinx-coroutines-core/jvm/src/ThreadContextElement.kt index 3da5b107da..1b825cef01 100644 --- a/kotlinx-coroutines-core/jvm/src/ThreadContextElement.kt +++ b/kotlinx-coroutines-core/jvm/src/ThreadContextElement.kt @@ -124,6 +124,7 @@ public interface ThreadContextElement : CoroutineContext.Element { * A coroutine using this mechanism can safely call Java code that assumes it's called using a * `Thread`. */ +@ExperimentalCoroutinesApi public interface CopyableThreadContextElement : ThreadContextElement { /** diff --git a/kotlinx-coroutines-core/jvm/test/ThreadContextElementTest.kt b/kotlinx-coroutines-core/jvm/test/ThreadContextElementTest.kt index ae3a18ba70..baba4aa8e6 100644 --- a/kotlinx-coroutines-core/jvm/test/ThreadContextElementTest.kt +++ b/kotlinx-coroutines-core/jvm/test/ThreadContextElementTest.kt @@ -107,14 +107,14 @@ class ThreadContextElementTest : TestBase() { @Test fun testCopyableElementCopiedOnLaunch() = runTest { - var parentElement: MyElement? = null - var inheritedElement: MyElement? = null + var parentElement: CopyForChildCoroutineElement? = null + var inheritedElement: CopyForChildCoroutineElement? = null newSingleThreadContext("withContext").use { withContext(it + CopyForChildCoroutineElement(MyData())) { - parentElement = coroutineContext[MyElement.Key] + parentElement = coroutineContext[CopyForChildCoroutineElement.Key] launch { - inheritedElement = coroutineContext[MyElement.Key] + inheritedElement = coroutineContext[CopyForChildCoroutineElement.Key] } } }