diff --git a/.idea/dictionaries/shared.xml b/.idea/dictionaries/shared.xml index 3da8e22952..45cbedcf1f 100644 --- a/.idea/dictionaries/shared.xml +++ b/.idea/dictionaries/shared.xml @@ -1,8 +1,13 @@ + Alistarh + Elizarov + Koval kotlinx lincheck + linearizability + linearizable redirector diff --git a/benchmarks/src/jmh/kotlin/benchmarks/ChannelProducerConsumerBenchmark.kt b/benchmarks/src/jmh/kotlin/benchmarks/ChannelProducerConsumerBenchmark.kt index 0fa5048983..0aa218e824 100644 --- a/benchmarks/src/jmh/kotlin/benchmarks/ChannelProducerConsumerBenchmark.kt +++ b/benchmarks/src/jmh/kotlin/benchmarks/ChannelProducerConsumerBenchmark.kt @@ -4,15 +4,14 @@ package benchmarks +import benchmarks.common.* import kotlinx.coroutines.* import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.scheduling.* import kotlinx.coroutines.selects.select import org.openjdk.jmh.annotations.* -import org.openjdk.jmh.infra.Blackhole import java.lang.Integer.max -import java.util.concurrent.ForkJoinPool import java.util.concurrent.Phaser -import java.util.concurrent.ThreadLocalRandom import java.util.concurrent.TimeUnit @@ -26,14 +25,14 @@ import java.util.concurrent.TimeUnit * Please, be patient, this benchmark takes quite a lot of time to complete. */ @Warmup(iterations = 3, time = 500, timeUnit = TimeUnit.MICROSECONDS) -@Measurement(iterations = 10, time = 500, timeUnit = TimeUnit.MICROSECONDS) -@Fork(value = 3) -@BenchmarkMode(Mode.AverageTime) +@Measurement(iterations = 20, time = 500, timeUnit = TimeUnit.MICROSECONDS) +@Fork(value = 1) +@BenchmarkMode(Mode.Throughput) @OutputTimeUnit(TimeUnit.MILLISECONDS) @State(Scope.Benchmark) open class ChannelProducerConsumerBenchmark { @Param - private var _0_dispatcher: DispatcherCreator = DispatcherCreator.FORK_JOIN + private var _0_dispatcher: DispatcherCreator = DispatcherCreator.DEFAULT @Param private var _1_channel: ChannelCreator = ChannelCreator.RENDEZVOUS @@ -44,12 +43,13 @@ open class ChannelProducerConsumerBenchmark { @Param("false", "true") private var _3_withSelect: Boolean = false - @Param("1", "2", "4") // local machine -// @Param("1", "2", "4", "8", "12") // local machine -// @Param("1", "2", "4", "8", "16", "32", "64", "128", "144") // dasquad -// @Param("1", "2", "4", "8", "16", "32", "64", "96") // Google Cloud + @Param("1", "2", "4", "8", "16") // local machine +// @Param("1", "2", "4", "8", "16", "32", "64", "128") // Server private var _4_parallelism: Int = 0 + @Param("50") + private var _5_workSize: Int = 0 + private lateinit var dispatcher: CoroutineDispatcher private lateinit var channel: Channel @@ -61,13 +61,21 @@ open class ChannelProducerConsumerBenchmark { } @Benchmark - fun spmc() { + fun mcsp() { if (_2_coroutines != 0) return val producers = max(1, _4_parallelism - 1) val consumers = 1 run(producers, consumers) } + @Benchmark + fun spmc() { + if (_2_coroutines != 0) return + val producers = 1 + val consumers = max(1, _4_parallelism - 1) + run(producers, consumers) + } + @Benchmark fun mpmc() { val producers = if (_2_coroutines == 0) (_4_parallelism + 1) / 2 else _2_coroutines / 2 @@ -76,7 +84,7 @@ open class ChannelProducerConsumerBenchmark { } private fun run(producers: Int, consumers: Int) { - val n = APPROX_BATCH_SIZE / producers * producers + val n = (APPROX_BATCH_SIZE / producers * producers) / consumers * consumers val phaser = Phaser(producers + consumers + 1) // Run producers repeat(producers) { @@ -111,7 +119,7 @@ open class ChannelProducerConsumerBenchmark { } else { channel.send(element) } - doWork() + doWork(_5_workSize) } private suspend fun consume(dummy: Channel?) { @@ -123,28 +131,25 @@ open class ChannelProducerConsumerBenchmark { } else { channel.receive() } - doWork() + doWork(_5_workSize) } } enum class DispatcherCreator(val create: (parallelism: Int) -> CoroutineDispatcher) { - FORK_JOIN({ parallelism -> ForkJoinPool(parallelism).asCoroutineDispatcher() }) + //FORK_JOIN({ parallelism -> ForkJoinPool(parallelism).asCoroutineDispatcher() }), + @Suppress("INVISIBLE_MEMBER", "INVISIBLE_REFERENCE") + DEFAULT({ parallelism -> ExperimentalCoroutineDispatcher(corePoolSize = parallelism, maxPoolSize = parallelism) }) } enum class ChannelCreator(private val capacity: Int) { RENDEZVOUS(Channel.RENDEZVOUS), -// BUFFERED_1(1), - BUFFERED_2(2), -// BUFFERED_4(4), - BUFFERED_32(32), - BUFFERED_128(128), + BUFFERED_16(16), + BUFFERED_64(64), BUFFERED_UNLIMITED(Channel.UNLIMITED); fun create(): Channel = Channel(capacity) } -private fun doWork(): Unit = Blackhole.consumeCPU(ThreadLocalRandom.current().nextLong(WORK_MIN, WORK_MAX)) +private fun doWork(workSize: Int): Unit = doGeomDistrWork(workSize) -private const val WORK_MIN = 50L -private const val WORK_MAX = 100L -private const val APPROX_BATCH_SIZE = 100000 +private const val APPROX_BATCH_SIZE = 100_000 diff --git a/benchmarks/src/jmh/kotlin/benchmarks/SemaphoreBenchmark.kt b/benchmarks/src/jmh/kotlin/benchmarks/SemaphoreBenchmark.kt index 9e1bfc43bb..6826b7a1a3 100644 --- a/benchmarks/src/jmh/kotlin/benchmarks/SemaphoreBenchmark.kt +++ b/benchmarks/src/jmh/kotlin/benchmarks/SemaphoreBenchmark.kt @@ -7,6 +7,7 @@ package benchmarks import benchmarks.common.* import kotlinx.coroutines.* import kotlinx.coroutines.channels.* +import kotlinx.coroutines.scheduling.* import kotlinx.coroutines.sync.* import org.openjdk.jmh.annotations.* import java.util.concurrent.* @@ -19,7 +20,7 @@ import java.util.concurrent.* @State(Scope.Benchmark) open class SemaphoreBenchmark { @Param - private var _1_dispatcher: SemaphoreBenchDispatcherCreator = SemaphoreBenchDispatcherCreator.FORK_JOIN + private var _1_dispatcher: SemaphoreBenchDispatcherCreator = SemaphoreBenchDispatcherCreator.DEFAULT @Param("0", "1000") private var _2_coroutines: Int = 0 @@ -27,9 +28,8 @@ open class SemaphoreBenchmark { @Param("1", "2", "4", "8", "32", "128", "100000") private var _3_maxPermits: Int = 0 - @Param("1", "2", "4") // local machine -// @Param("1", "2", "4", "8", "16", "32", "64", "128", "144") // dasquad -// @Param("1", "2", "4", "8", "16", "32", "64", "96") // Google Cloud + @Param("1", "2", "4", "8", "16") // local machine +// @Param("1", "2", "4", "8", "16", "32", "64", "128") // Server private var _4_parallelism: Int = 0 private lateinit var dispatcher: CoroutineDispatcher @@ -80,10 +80,11 @@ open class SemaphoreBenchmark { } enum class SemaphoreBenchDispatcherCreator(val create: (parallelism: Int) -> CoroutineDispatcher) { - FORK_JOIN({ parallelism -> ForkJoinPool(parallelism).asCoroutineDispatcher() }), - EXPERIMENTAL({ parallelism -> Dispatchers.Default }) // TODO doesn't take parallelism into account + // FORK_JOIN({ parallelism -> ForkJoinPool(parallelism).asCoroutineDispatcher() }), + @Suppress("INVISIBLE_MEMBER", "INVISIBLE_REFERENCE") + DEFAULT({ parallelism -> ExperimentalCoroutineDispatcher(corePoolSize = parallelism, maxPoolSize = parallelism) }) } -private const val WORK_INSIDE = 80 -private const val WORK_OUTSIDE = 40 -private const val BATCH_SIZE = 1000000 +private const val WORK_INSIDE = 50 +private const val WORK_OUTSIDE = 50 +private const val BATCH_SIZE = 100000 diff --git a/gradle.properties b/gradle.properties index c23ae62301..229904ead4 100644 --- a/gradle.properties +++ b/gradle.properties @@ -13,7 +13,7 @@ junit5_version=5.7.0 atomicfu_version=0.19.0 knit_version=0.4.0 html_version=0.7.2 -lincheck_version=2.14.1 +lincheck_version=2.16 dokka_version=1.7.20 byte_buddy_version=1.10.9 reactor_version=3.4.1 @@ -59,4 +59,6 @@ org.gradle.jvmargs=-Xmx3g kotlin.mpp.enableCompatibilityMetadataVariant=true kotlin.mpp.stability.nowarn=true kotlinx.atomicfu.enableJvmIrTransformation=true -kotlinx.atomicfu.enableJsIrTransformation=true +# When the flag below is set to `true`, AtomicFU cannot process +# usages of `moveForward` in `ConcurrentLinkedList.kt` correctly. +kotlinx.atomicfu.enableJsIrTransformation=false diff --git a/gradle/test-mocha-js.gradle b/gradle/test-mocha-js.gradle index 1986cb7f81..1ec297e415 100644 --- a/gradle/test-mocha-js.gradle +++ b/gradle/test-mocha-js.gradle @@ -29,7 +29,8 @@ task testMochaNode(type: NodeTask, dependsOn: [compileTestJsLegacy, installDepen def jsLegacyTestTask = project.tasks.findByName('jsLegacyTest') ? jsLegacyTest : jsTest -jsLegacyTestTask.dependsOn testMochaNode +// TODO +//jsLegacyTestTask.dependsOn testMochaNode // -- Testing with Mocha under headless Chrome @@ -100,5 +101,5 @@ task testMochaJsdom(type: NodeTask, dependsOn: [compileTestJsLegacy, installDepe if (project.hasProperty("teamcity")) args.addAll(['--reporter', 'mocha-teamcity-reporter']) } -jsLegacyTestTask.dependsOn testMochaJsdom - +// TODO +//jsLegacyTestTask.dependsOn testMochaJsdom diff --git a/kotlinx-coroutines-core/api/kotlinx-coroutines-core.api b/kotlinx-coroutines-core/api/kotlinx-coroutines-core.api index 3a2d08f428..def21f8130 100644 --- a/kotlinx-coroutines-core/api/kotlinx-coroutines-core.api +++ b/kotlinx-coroutines-core/api/kotlinx-coroutines-core.api @@ -51,7 +51,7 @@ public final class kotlinx/coroutines/CancellableContinuation$DefaultImpls { public static synthetic fun tryResume$default (Lkotlinx/coroutines/CancellableContinuation;Ljava/lang/Object;Ljava/lang/Object;ILjava/lang/Object;)Ljava/lang/Object; } -public class kotlinx/coroutines/CancellableContinuationImpl : kotlin/coroutines/jvm/internal/CoroutineStackFrame, kotlinx/coroutines/CancellableContinuation { +public class kotlinx/coroutines/CancellableContinuationImpl : kotlin/coroutines/jvm/internal/CoroutineStackFrame, kotlinx/coroutines/CancellableContinuation, kotlinx/coroutines/channels/Waiter { public fun (Lkotlin/coroutines/Continuation;I)V public final fun callCancelHandler (Lkotlinx/coroutines/CancelHandler;Ljava/lang/Throwable;)V public final fun callOnCancellation (Lkotlin/jvm/functions/Function1;Ljava/lang/Throwable;)V diff --git a/kotlinx-coroutines-core/build.gradle b/kotlinx-coroutines-core/build.gradle index 54883ff233..84d9b0485d 100644 --- a/kotlinx-coroutines-core/build.gradle +++ b/kotlinx-coroutines-core/build.gradle @@ -265,8 +265,11 @@ task jvmStressTest(type: Test, dependsOn: compileTestKotlinJvm) { enableAssertions = true testLogging.showStandardStreams = true systemProperty 'kotlinx.coroutines.scheduler.keep.alive.sec', '100000' // any unpark problem hangs test - systemProperty 'kotlinx.coroutines.semaphore.segmentSize', '2' + // Adjust internal algorithmic parameters to increase the testing quality instead of performance. + systemProperty 'kotlinx.coroutines.semaphore.segmentSize', '1' systemProperty 'kotlinx.coroutines.semaphore.maxSpinCycles', '10' + systemProperty 'kotlinx.coroutines.bufferedChannel.segmentSize', '2' + systemProperty 'kotlinx.coroutines.bufferedChannel.expandBufferCompletionWaitIterations', '1' } task jvmLincheckTest(type: Test, dependsOn: compileTestKotlinJvm) { @@ -278,17 +281,35 @@ task jvmLincheckTest(type: Test, dependsOn: compileTestKotlinJvm) { configureJvmForLincheck(jvmLincheckTest) } -static void configureJvmForLincheck(task) { +// Additional Lincheck tests with `segmentSize = 2`. +// Some bugs cannot be revealed when storing one request per segment, +// and some are hard to detect when storing multiple requests. +task jvmLincheckTestAdditional(type: Test, dependsOn: compileTestKotlinJvm) { + classpath = files { jvmTest.classpath } + testClassesDirs = files { jvmTest.testClassesDirs } + include '**/RendezvousChannelLincheckTest*' + include '**/Buffered1ChannelLincheckTest*' + include '**/Semaphore*LincheckTest*' + enableAssertions = true + testLogging.showStandardStreams = true + configureJvmForLincheck(jvmLincheckTestAdditional, true) +} + +static void configureJvmForLincheck(task, additional = false) { task.minHeapSize = '1g' task.maxHeapSize = '4g' // we may need more space for building an interleaving tree in the model checking mode task.jvmArgs = ['--add-opens', 'java.base/jdk.internal.misc=ALL-UNNAMED', // required for transformation '--add-exports', 'java.base/jdk.internal.util=ALL-UNNAMED'] // in the model checking mode - task.systemProperty 'kotlinx.coroutines.semaphore.segmentSize', '2' + // Adjust internal algorithmic parameters to increase the testing quality instead of performance. + var segmentSize = additional ? '2' : '1' + task.systemProperty 'kotlinx.coroutines.semaphore.segmentSize', segmentSize task.systemProperty 'kotlinx.coroutines.semaphore.maxSpinCycles', '1' // better for the model checking mode + task.systemProperty 'kotlinx.coroutines.bufferedChannel.segmentSize', segmentSize + task.systemProperty 'kotlinx.coroutines.bufferedChannel.expandBufferCompletionWaitIterations', '1' } // Always check additional test sets -task moreTest(dependsOn: [jvmStressTest, jvmLincheckTest]) +task moreTest(dependsOn: [jvmStressTest, jvmLincheckTest, jvmLincheckTestAdditional]) check.dependsOn moreTest tasks.jvmLincheckTest { diff --git a/kotlinx-coroutines-core/common/src/CancellableContinuation.kt b/kotlinx-coroutines-core/common/src/CancellableContinuation.kt index 2c2f1b8ff6..5e8d7f9102 100644 --- a/kotlinx-coroutines-core/common/src/CancellableContinuation.kt +++ b/kotlinx-coroutines-core/common/src/CancellableContinuation.kt @@ -358,13 +358,6 @@ internal fun getOrCreateCancellableContinuation(delegate: Continuation): ?: return CancellableContinuationImpl(delegate, MODE_CANCELLABLE_REUSABLE) } -/** - * Removes the specified [node] on cancellation. This function assumes that this node is already - * removed on successful resume and does not try to remove it if the continuation is cancelled during dispatch. - */ -internal fun CancellableContinuation<*>.removeOnCancellation(node: LockFreeLinkedListNode) = - invokeOnCancellation(handler = RemoveOnCancel(node).asHandler) - /** * Disposes the specified [handle] when this continuation is cancelled. * @@ -379,13 +372,6 @@ internal fun CancellableContinuation<*>.removeOnCancellation(node: LockFreeLinke public fun CancellableContinuation<*>.disposeOnCancellation(handle: DisposableHandle): Unit = invokeOnCancellation(handler = DisposeOnCancel(handle).asHandler) -// --------------- implementation details --------------- - -private class RemoveOnCancel(private val node: LockFreeLinkedListNode) : BeforeResumeCancelHandler() { - override fun invoke(cause: Throwable?) { node.remove() } - override fun toString() = "RemoveOnCancel[$node]" -} - private class DisposeOnCancel(private val handle: DisposableHandle) : CancelHandler() { override fun invoke(cause: Throwable?) = handle.dispose() override fun toString(): String = "DisposeOnCancel[$handle]" diff --git a/kotlinx-coroutines-core/common/src/CancellableContinuationImpl.kt b/kotlinx-coroutines-core/common/src/CancellableContinuationImpl.kt index ef49da23df..423cb05d18 100644 --- a/kotlinx-coroutines-core/common/src/CancellableContinuationImpl.kt +++ b/kotlinx-coroutines-core/common/src/CancellableContinuationImpl.kt @@ -5,6 +5,7 @@ package kotlinx.coroutines import kotlinx.atomicfu.* +import kotlinx.coroutines.channels.Waiter import kotlinx.coroutines.internal.* import kotlin.coroutines.* import kotlin.coroutines.intrinsics.* @@ -24,7 +25,7 @@ internal val RESUME_TOKEN = Symbol("RESUME_TOKEN") internal open class CancellableContinuationImpl( final override val delegate: Continuation, resumeMode: Int -) : DispatchedTask(resumeMode), CancellableContinuation, CoroutineStackFrame { +) : DispatchedTask(resumeMode), CancellableContinuation, CoroutineStackFrame, Waiter { init { assert { resumeMode != MODE_UNINITIALIZED } // invalid mode for CancellableContinuationImpl } diff --git a/kotlinx-coroutines-core/common/src/channels/AbstractChannel.kt b/kotlinx-coroutines-core/common/src/channels/AbstractChannel.kt deleted file mode 100644 index 2f2fe506e0..0000000000 --- a/kotlinx-coroutines-core/common/src/channels/AbstractChannel.kt +++ /dev/null @@ -1,1110 +0,0 @@ -/* - * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. - */ - -@file:Suppress("UNCHECKED_CAST", "UNUSED_PARAMETER") - -package kotlinx.coroutines.channels - -import kotlinx.atomicfu.* -import kotlinx.coroutines.* -import kotlinx.coroutines.internal.* -import kotlinx.coroutines.intrinsics.* -import kotlinx.coroutines.selects.* -import kotlinx.coroutines.selects.TrySelectDetailedResult.* -import kotlin.coroutines.* -import kotlin.jvm.* - -/** - * Abstract send channel. It is a base class for all send channel implementations. - */ -@Suppress("UNCHECKED_CAST", "UNUSED_PARAMETER") -internal abstract class AbstractSendChannel( - @JvmField protected val onUndeliveredElement: OnUndeliveredElement? -) : SendChannel { - /** @suppress **This is unstable API and it is subject to change.** */ - protected val queue = LockFreeLinkedListHead() - - // ------ extension points for buffered channels ------ - - /** - * Returns `true` if [isBufferFull] is always `true`. - * @suppress **This is unstable API and it is subject to change.** - */ - protected abstract val isBufferAlwaysFull: Boolean - - /** - * Returns `true` if this channel's buffer is full. - * This operation should be atomic if it is invoked by [enqueueSend]. - * @suppress **This is unstable API and it is subject to change.** - */ - protected abstract val isBufferFull: Boolean - - // State transitions: null -> handler -> HANDLER_INVOKED - private val onCloseHandler = atomic(null) - - // ------ internal functions for override by buffered channels ------ - - /** - * Tries to add element to buffer or to queued receiver. - * Return type is `OFFER_SUCCESS | OFFER_FAILED | Closed`. - * @suppress **This is unstable API and it is subject to change.** - */ - protected open fun offerInternal(element: E): Any { - while (true) { - val receive = takeFirstReceiveOrPeekClosed() ?: return OFFER_FAILED - val token = receive.tryResumeReceive(element) - if (token != null) { - assert { token === RESUME_TOKEN } - receive.completeResumeReceive(element) - return receive.offerResult - } - } - } - - - // ------ state functions & helpers for concrete implementations ------ - - /** - * Returns non-null closed token if it is last in the queue. - * @suppress **This is unstable API and it is subject to change.** - */ - protected val closedForSend: Closed<*>? get() = (queue.prevNode as? Closed<*>)?.also { helpClose(it) } - - /** - * Returns non-null closed token if it is first in the queue. - * @suppress **This is unstable API and it is subject to change.** - */ - protected val closedForReceive: Closed<*>? get() = (queue.nextNode as? Closed<*>)?.also { helpClose(it) } - - /** - * Retrieves first sending waiter from the queue or returns closed token. - * @suppress **This is unstable API and it is subject to change.** - */ - protected fun takeFirstSendOrPeekClosed(): Send? = - queue.removeFirstIfIsInstanceOfOrPeekIf { it is Closed<*> } - - /** - * Queues buffered element, returns null on success or - * returns node reference if it was already closed or is waiting for receive. - * @suppress **This is unstable API and it is subject to change.** - */ - protected fun sendBuffered(element: E): ReceiveOrClosed<*>? { - queue.addLastIfPrev(SendBuffered(element)) { prev -> - if (prev is ReceiveOrClosed<*>) return@sendBuffered prev - true - } - return null - } - - /** - * @suppress **This is unstable API and it is subject to change.** - */ - protected fun describeSendBuffered(element: E): AddLastDesc<*> = SendBufferedDesc(queue, element) - - private open class SendBufferedDesc( - queue: LockFreeLinkedListHead, - element: E - ) : AddLastDesc>(queue, SendBuffered(element)) { - override fun failure(affected: LockFreeLinkedListNode): Any? = when (affected) { - is Closed<*> -> affected - is ReceiveOrClosed<*> -> OFFER_FAILED - else -> null - } - } - - // ------ SendChannel ------ - - public final override val isClosedForSend: Boolean get() = closedForSend != null - private val isFullImpl: Boolean get() = queue.nextNode !is ReceiveOrClosed<*> && isBufferFull - - public final override suspend fun send(element: E) { - // fast path -- try offer non-blocking - if (offerInternal(element) === OFFER_SUCCESS) return - // slow-path does suspend or throws exception - return sendSuspend(element) - } - - @Suppress("DEPRECATION_ERROR") - @Deprecated( - level = DeprecationLevel.ERROR, - message = "Deprecated in the favour of 'trySend' method", - replaceWith = ReplaceWith("trySend(element).isSuccess") - ) // see super() - override fun offer(element: E): Boolean { - // Temporary migration for offer users who rely on onUndeliveredElement - try { - return super.offer(element) - } catch (e: Throwable) { - onUndeliveredElement?.callUndeliveredElementCatchingException(element)?.let { - // If it crashes, add send exception as suppressed for better diagnostics - it.addSuppressed(e) - throw it - } - throw e - } - } - - public final override fun trySend(element: E): ChannelResult { - val result = offerInternal(element) - return when { - result === OFFER_SUCCESS -> ChannelResult.success(Unit) - result === OFFER_FAILED -> { - // We should check for closed token on trySend as well, otherwise trySend won't be linearizable - // in the face of concurrent close() - // See https://github.com/Kotlin/kotlinx.coroutines/issues/359 - val closedForSend = closedForSend ?: return ChannelResult.failure() - ChannelResult.closed(helpCloseAndGetSendException(closedForSend)) - } - result is Closed<*> -> { - ChannelResult.closed(helpCloseAndGetSendException(result)) - } - else -> error("trySend returned $result") - } - } - - private fun helpCloseAndGetSendException(closed: Closed<*>): Throwable { - helpClose(closed) - return closed.sendException - } - - private fun helpCloseAndGetSendException(element: E, closed: Closed<*>): Throwable { - // To ensure linearizablity we must ALWAYS help close the channel when we observe that it was closed - // See https://github.com/Kotlin/kotlinx.coroutines/issues/1419 - helpClose(closed) - // Element was not delivered -> cals onUndeliveredElement - onUndeliveredElement?.callUndeliveredElementCatchingException(element)?.let { - // If it crashes, add send exception as suppressed for better diagnostics - it.addSuppressed(closed.sendException) - throw it - } - return closed.sendException - } - - private suspend fun sendSuspend(element: E): Unit = suspendCancellableCoroutineReusable sc@ { cont -> - sendSlowPath(element, cont) - } - - // waiter is either CancellableContinuation or SelectInstance - private fun sendSlowPath(element: E, waiter: Any) { - loop@ while (true) { - if (isFullImpl) { - val send: Send = if (waiter is SelectInstance<*>) { - if (onUndeliveredElement == null) SendElementSelect(element, waiter, this) - else SendElementSelectWithUndeliveredHandler(element, waiter, this, onUndeliveredElement) - } else { - if (onUndeliveredElement == null) SendElement(element, waiter as CancellableContinuation) - else SendElementWithUndeliveredHandler(element, waiter as CancellableContinuation, onUndeliveredElement) - } - val enqueueResult = enqueueSend(send) - when { - enqueueResult == null -> { // enqueued successfully - if (waiter is SelectInstance<*>) { - waiter.disposeOnCompletion { send.remove() } - } else { - waiter as CancellableContinuation - waiter.removeOnCancellation(send) - } - return - } - enqueueResult is Closed<*> -> { - if (waiter is SelectInstance<*>) { - waiter.helpCloseAndSelectInRegistrationPhaseClosed(element, enqueueResult) - } else { - waiter as CancellableContinuation - waiter.helpCloseAndResumeWithSendException(element, enqueueResult) - } - return - } - enqueueResult === ENQUEUE_FAILED -> {} // try to offer instead - enqueueResult is Receive<*> -> {} // try to offer instead - else -> error("enqueueSend returned $enqueueResult") - } - } - // hm... receiver is waiting or buffer is not full. try to offer - val offerResult = offerInternal(element) - when { - offerResult === OFFER_SUCCESS -> { - if (waiter is SelectInstance<*>) { - waiter.selectInRegistrationPhase(Unit) - } else { - waiter as CancellableContinuation - waiter.resume(Unit) - } - return - } - offerResult === OFFER_FAILED -> continue@loop - offerResult is Closed<*> -> { - if (waiter is SelectInstance<*>) { - waiter.helpCloseAndSelectInRegistrationPhaseClosed(element, offerResult) - } else { - waiter as CancellableContinuation - waiter.helpCloseAndResumeWithSendException(element, offerResult) - } - return - } - else -> error("offerInternal returned $offerResult") - } - } - } - - private fun Continuation<*>.helpCloseAndResumeWithSendException(element: E, closed: Closed<*>) { - helpClose(closed) - val sendException = closed.sendException - onUndeliveredElement?.callUndeliveredElementCatchingException(element)?.let { - it.addSuppressed(sendException) - resumeWithException(it) - return - } - resumeWithException(sendException) - } - - private fun SelectInstance<*>.helpCloseAndSelectInRegistrationPhaseClosed(element: E, closed: Closed<*>) { - helpClose(closed) - onUndeliveredElement?.callUndeliveredElementCatchingException(element)?.let { - it.addSuppressed(closed.sendException) - selectInRegistrationPhase(Closed(it)) - return - } - selectInRegistrationPhase(closed) - } - - /** - * Result is: - * * null -- successfully enqueued - * * ENQUEUE_FAILED -- buffer is not full (should not enqueue) - * * ReceiveOrClosed<*> -- receiver is waiting or it is closed (should not enqueue) - */ - protected open fun enqueueSend(send: Send): Any? { - if (isBufferAlwaysFull) { - queue.addLastIfPrev(send) { prev -> - if (prev is ReceiveOrClosed<*>) return@enqueueSend prev - true - } - } else { - if (!queue.addLastIfPrevAndIf(send, { prev -> - if (prev is ReceiveOrClosed<*>) return@enqueueSend prev - true - }, { isBufferFull })) - return ENQUEUE_FAILED - } - return null - } - - public override fun close(cause: Throwable?): Boolean { - val closed = Closed(cause) - /* - * Try to commit close by adding a close token to the end of the queue. - * Successful -> we're now responsible for closing receivers - * Not successful -> help closing pending receivers to maintain invariant - * "if (!close()) next send will throw" - */ - val closeAdded = queue.addLastIfPrev(closed) { it !is Closed<*> } - val actuallyClosed = if (closeAdded) closed else queue.prevNode as Closed<*> - helpClose(actuallyClosed) - if (closeAdded) invokeOnCloseHandler(cause) - return closeAdded // true if we have closed - } - - private fun invokeOnCloseHandler(cause: Throwable?) { - val handler = onCloseHandler.value - if (handler !== null && handler !== HANDLER_INVOKED - && onCloseHandler.compareAndSet(handler, HANDLER_INVOKED)) { - // CAS failed -> concurrent invokeOnClose() invoked handler - @Suppress("UNCHECKED_CAST") - (handler as Handler)(cause) - } - } - - override fun invokeOnClose(handler: Handler) { - // Intricate dance for concurrent invokeOnClose and close calls - if (!onCloseHandler.compareAndSet(null, handler)) { - val value = onCloseHandler.value - if (value === HANDLER_INVOKED) { - throw IllegalStateException("Another handler was already registered and successfully invoked") - } - - throw IllegalStateException("Another handler was already registered: $value") - } else { - val closedToken = closedForSend - if (closedToken != null && onCloseHandler.compareAndSet(handler, HANDLER_INVOKED)) { - // CAS failed -> close() call invoked handler - (handler)(closedToken.closeCause) - } - } - } - - private fun helpClose(closed: Closed<*>) { - /* - * It's important to traverse list from right to left to avoid races with sender. - * Consider channel state: head -> [receive_1] -> [receive_2] -> head - * - T1 calls receive() - * - T2 calls close() - * - T3 calls close() + send(value) - * - * If both will traverse list from left to right, following non-linearizable history is possible: - * [close -> false], [send -> transferred 'value' to receiver] - * - * Another problem with linearizability of close is that we cannot resume closed receives until all - * receivers are removed from the list. - * Consider channel state: head -> [receive_1] -> [receive_2] -> head - * - T1 called receive_2, and will call send() when it's receive call resumes - * - T2 calls close() - * - * Now if T2's close resumes T1's receive_2 then it's receive gets "closed for receive" exception, but - * its subsequent attempt to send successfully rendezvous with receive_1, producing non-linearizable execution. - */ - var closedList = InlineList>() - while (true) { - // Break when channel is empty or has no receivers - @Suppress("UNCHECKED_CAST") - val previous = closed.prevNode as? Receive ?: break - if (!previous.remove()) { - // failed to remove the node (due to race) -- retry finding non-removed prevNode - // NOTE: remove() DOES NOT help pending remove operation (that marked next pointer) - previous.helpRemove() // make sure remove is complete before continuing - continue - } - // add removed nodes to a separate list - closedList += previous - } - /* - * Now notify all removed nodes that the channel was closed - * in the order they were added to the channel - */ - closedList.forEachReversed { it.resumeReceiveClosed(closed) } - // and do other post-processing - onClosedIdempotent(closed) - } - - /** - * Invoked when channel is closed as the last action of [close] invocation. - * This method should be idempotent and can be called multiple times. - */ - protected open fun onClosedIdempotent(closed: LockFreeLinkedListNode) {} - - /** - * Retrieves first receiving waiter from the queue or returns closed token. - * @suppress **This is unstable API and it is subject to change.** - */ - protected open fun takeFirstReceiveOrPeekClosed(): ReceiveOrClosed? = - queue.removeFirstIfIsInstanceOfOrPeekIf>({ it is Closed<*> }) - - final override val onSend - get() = SelectClause2Impl>( - clauseObject = this, - regFunc = AbstractSendChannel<*>::registerSelectForSend as RegistrationFunction, - processResFunc = AbstractSendChannel<*>::processResultSelectSend as ProcessResultFunction - ) - - private fun registerSelectForSend(select: SelectInstance<*>, element: Any?) { - element as E - if (offerInternal(element) === OFFER_SUCCESS) { - select.selectInRegistrationPhase(Unit) - return - } - sendSlowPath(element, select) - } - - private fun processResultSelectSend(ignoredParam: Any?, selectResult: Any?): Any? = - if (selectResult is Closed<*>) throw selectResult.sendException - else this - - // ------ debug ------ - - public override fun toString() = - "$classSimpleName@$hexAddress{$queueDebugStateString}$bufferDebugString" - - private val queueDebugStateString: String - get() { - val head = queue.nextNode - if (head === queue) return "EmptyQueue" - var result = when (head) { - is Closed<*> -> head.toString() - is Receive<*> -> "ReceiveQueued" - is Send -> "SendQueued" - else -> "UNEXPECTED:$head" // should not happen - } - val tail = queue.prevNode - if (tail !== head) { - result += ",queueSize=${countQueueSize()}" - if (tail is Closed<*>) result += ",closedForSend=$tail" - } - return result - } - - private fun countQueueSize(): Int { - var size = 0 - queue.forEach { size++ } - return size - } - - protected open val bufferDebugString: String get() = "" - - // ------ private ------ - - internal class SendBuffered( - @JvmField val element: E - ) : Send() { - override val pollResult: Any? get() = element - override fun tryResumeSend(otherOp: PrepareOp?): Symbol? = RESUME_TOKEN.also { otherOp?.finishPrepare() } - override fun completeResumeSend() {} - - /** - * This method should be never called, see special logic in [LinkedListChannel.onCancelIdempotentList]. - */ - override fun resumeSendClosed(closed: Closed<*>) { - assert { false } - } - - override fun toString(): String = "SendBuffered@$hexAddress($element)" - } -} - -/** - * Abstract send/receive channel. It is a base class for all channel implementations. - */ -internal abstract class AbstractChannel( - onUndeliveredElement: OnUndeliveredElement? -) : AbstractSendChannel(onUndeliveredElement), Channel { - // ------ extension points for buffered channels ------ - - /** - * Returns `true` if [isBufferEmpty] is always `true`. - * @suppress **This is unstable API and it is subject to change.** - */ - protected abstract val isBufferAlwaysEmpty: Boolean - - /** - * Returns `true` if this channel's buffer is empty. - * This operation should be atomic if it is invoked by [enqueueReceive]. - * @suppress **This is unstable API and it is subject to change.** - */ - protected abstract val isBufferEmpty: Boolean - - // ------ internal functions for override by buffered channels ------ - - /** - * Tries to remove element from buffer or from queued sender. - * Return type is `E | POLL_FAILED | Closed` - * @suppress **This is unstable API and it is subject to change.** - */ - protected open fun pollInternal(): Any? { - while (true) { - val send = takeFirstSendOrPeekClosed() ?: return POLL_FAILED - val token = send.tryResumeSend(null) - if (token != null) { - assert { token === RESUME_TOKEN } - send.completeResumeSend() - return send.pollResult - } - // Too late, already cancelled, but we removed it from the queue and need to notify on undelivered element. - // The only exception is when this "send" operation is an `onSend` clause that has to be re-registered - // in the corresponding `select` invocation. - if (!(send is SendElementSelectWithUndeliveredHandler<*> && send.trySelectResult == REREGISTER)) - send.undeliveredElement() - } - } - - // ------ state functions & helpers for concrete implementations ------ - - /** - * @suppress **This is unstable API and it is subject to change.** - */ - protected val hasReceiveOrClosed: Boolean get() = queue.nextNode is ReceiveOrClosed<*> - - // ------ ReceiveChannel ------ - - public override val isClosedForReceive: Boolean get() = closedForReceive != null && isBufferEmpty - public override val isEmpty: Boolean get() = isEmptyImpl - protected val isEmptyImpl: Boolean get() = queue.nextNode !is Send && isBufferEmpty - - public final override suspend fun receive(): E { - // fast path -- try poll non-blocking - val result = pollInternal() - /* - * If result is Closed -- go to tail-call slow-path that will allow us to - * properly recover stacktrace without paying a performance cost on fast path. - * We prefer to recover stacktrace using suspending path to have a more precise stacktrace. - */ - @Suppress("UNCHECKED_CAST") - if (result !== POLL_FAILED && result !is Closed<*>) return result as E - // slow-path does suspend - return receiveSuspend(RECEIVE_THROWS_ON_CLOSE) - } - - @Suppress("UNCHECKED_CAST") - private suspend fun receiveSuspend(receiveMode: Int): R = suspendCancellableCoroutineReusable sc@ { cont -> - receiveSlowPath(receiveMode, cont) - } - - @Suppress("UNCHECKED_CAST") - private fun receiveSlowPath(receiveMode: Int, waiter: Any) { - val receive = if (waiter is SelectInstance<*>) { - if (onUndeliveredElement == null) ReceiveElementSelect(waiter) - else ReceiveElementSelectWithUndeliveredHandler(waiter, onUndeliveredElement) - } else { - waiter as CancellableContinuation - if (onUndeliveredElement == null) ReceiveElement(waiter, receiveMode) - else ReceiveElementWithUndeliveredHandler(waiter, receiveMode, onUndeliveredElement) - } - while (true) { - if (enqueueReceive(receive)) { - if (waiter is SelectInstance<*>) { - waiter.disposeOnCompletion(RemoveReceiveOnCancel(receive)) - } else { - waiter as CancellableContinuation - removeReceiveOnCancel(waiter, receive) - } - return - } - // hm... something is not right. try to poll - val result = pollInternal() - if (result is Closed<*>) { - if (waiter is SelectInstance<*>) { - waiter.selectInRegistrationPhase(result) - } else { // CancellableContinuation - receive.resumeReceiveClosed(result) - } - return - } - if (result !== POLL_FAILED) { - if (waiter is SelectInstance<*>) { - waiter.selectInRegistrationPhase(result as E) - } else { - waiter as CancellableContinuation - receive as ReceiveElement - waiter.resume(receive.resumeValue(result as E), receive.resumeOnCancellationFun(result as E)) - } - return - } - } - } - - protected open fun enqueueReceiveInternal(receive: Receive): Boolean = if (isBufferAlwaysEmpty) - queue.addLastIfPrev(receive) { it !is Send } else - queue.addLastIfPrevAndIf(receive, { it !is Send }, { isBufferEmpty }) - - private fun enqueueReceive(receive: Receive) = enqueueReceiveInternal(receive).also { result -> - if (result) onReceiveEnqueued() - } - - @Suppress("UNCHECKED_CAST") - public final override suspend fun receiveCatching(): ChannelResult { - // fast path -- try poll non-blocking - val result = pollInternal() - if (result !== POLL_FAILED) return result.toResult() - // slow-path does suspend - return receiveSuspend(RECEIVE_RESULT) - } - - @Suppress("UNCHECKED_CAST") - public final override fun tryReceive(): ChannelResult { - val result = pollInternal() - if (result === POLL_FAILED) return ChannelResult.failure() - if (result is Closed<*>) return ChannelResult.closed(result.closeCause) - return ChannelResult.success(result as E) - } - - @Deprecated(level = DeprecationLevel.HIDDEN, message = "Since 1.2.0, binary compatibility with versions <= 1.1.x") - final override fun cancel(cause: Throwable?): Boolean = - cancelInternal(cause) - - final override fun cancel(cause: CancellationException?) { - /* - * Do not create an exception if channel is already cancelled. - * Channel is closed for receive when either it is cancelled (then we are free to bail out) - * or was closed and elements were received. - * Then `onCancelIdempotent` does nothing for all implementations. - */ - if (isClosedForReceive) return - cancelInternal(cause ?: CancellationException("$classSimpleName was cancelled")) - } - - // It needs to be internal to support deprecated cancel(Throwable?) API - internal fun cancelInternal(cause: Throwable?): Boolean = - close(cause).also { - onCancelIdempotent(it) - } - - /** - * Method that is invoked right after [close] in [cancel] sequence. - * [wasClosed] is directly mapped to the value returned by [close]. - */ - protected open fun onCancelIdempotent(wasClosed: Boolean) { - /* - * See the comment to helpClose, all these machinery (reversed order of iteration, postponed resume) - * has the same rationale. - */ - val closed = closedForSend ?: error("Cannot happen") - var list = InlineList() - while (true) { - val previous = closed.prevNode - if (previous is LockFreeLinkedListHead) { - break - } - assert { previous is Send } - if (!previous.remove()) { - previous.helpRemove() // make sure remove is complete before continuing - continue - } - // Add to the list only **after** successful removal - list += previous as Send - } - onCancelIdempotentList(list, closed) - } - - /** - * This method is overridden by [LinkedListChannel] to handle cancellation of [SendBuffered] elements from the list. - */ - protected open fun onCancelIdempotentList(list: InlineList, closed: Closed<*>) { - list.forEachReversed { it.resumeSendClosed(closed) } - } - - public final override fun iterator(): ChannelIterator = Itr(this) - - // ------ registerSelectReceive ------ - - /** - * @suppress **This is unstable API and it is subject to change.** - */ - protected fun describeTryPoll(): TryPollDesc = TryPollDesc(queue) - - /** - * @suppress **This is unstable API and it is subject to change.** - */ - protected class TryPollDesc(queue: LockFreeLinkedListHead) : RemoveFirstDesc(queue) { - override fun failure(affected: LockFreeLinkedListNode): Any? = when (affected) { - is Closed<*> -> affected - !is Send -> POLL_FAILED - else -> null - } - - @Suppress("UNCHECKED_CAST") - override fun onPrepare(prepareOp: PrepareOp): Any? { - val affected = prepareOp.affected as Send // see "failure" impl - val token = affected.tryResumeSend(prepareOp) ?: return REMOVE_PREPARED - if (token === RETRY_ATOMIC) return RETRY_ATOMIC - assert { token === RESUME_TOKEN } - return null - } - - override fun onRemoved(affected: LockFreeLinkedListNode) { - // Called when we removed it from the queue but were too late to resume, so we have undelivered element - (affected as Send).undeliveredElement() - } - } - - // ------ the `select` expression support ------ - - final override val onReceive - get() = SelectClause1Impl( - clauseObject = this, - regFunc = AbstractChannel<*>::registerSelectForReceive as RegistrationFunction, - processResFunc = AbstractChannel<*>::processResultSelectReceive as ProcessResultFunction, - onCancellationConstructor = onUndeliveredElementReceiveCancellationConstructor - ) - - final override val onReceiveCatching - get() = SelectClause1Impl>( - clauseObject = this, - regFunc = AbstractChannel<*>::registerSelectForReceive as RegistrationFunction, - processResFunc = AbstractChannel<*>::processResultSelectReceiveCatching as ProcessResultFunction, - onCancellationConstructor = onUndeliveredElementReceiveCancellationConstructor - ) - - @Deprecated( - message = "Deprecated in favor of onReceiveCatching extension", - level = DeprecationLevel.ERROR, - replaceWith = ReplaceWith("onReceiveCatching") - ) // See super() - override val onReceiveOrNull: SelectClause1 - get() = SelectClause1Impl( - clauseObject = this, - regFunc = AbstractChannel<*>::registerSelectForReceive as RegistrationFunction, - processResFunc = AbstractChannel<*>::processResultSelectReceiveOrNull as ProcessResultFunction, - onCancellationConstructor = onUndeliveredElementReceiveCancellationConstructor - ) - - private fun registerSelectForReceive(select: SelectInstance<*>, ignoredParam: Any?) { - val result = pollInternal() - if (result !== POLL_FAILED && result !is Closed<*>) { - select.selectInRegistrationPhase(result as E) - return - } - receiveSlowPath(RECEIVE_RESULT, select) - } - - private fun processResultSelectReceive(ignoredParam: Any?, selectResult: Any?): Any? = - if (selectResult is Closed<*>) throw selectResult.receiveException - else selectResult as E - - private fun processResultSelectReceiveCatching(ignoredParam: Any?, selectResult: Any?): Any = - if (selectResult is Closed<*>) ChannelResult.closed(selectResult.closeCause) - else ChannelResult.success(selectResult as E) - - private fun processResultSelectReceiveOrNull(ignoredParam: Any?, selectResult: Any?): Any? = - if (selectResult is Closed<*>) null - else selectResult as E - - private val onUndeliveredElementReceiveCancellationConstructor: OnCancellationConstructor? = onUndeliveredElement?.let { - { select: SelectInstance<*>, _: Any?, element: Any? -> - { _: Throwable -> if (element !is Closed<*>) onUndeliveredElement.callUndeliveredElement(element as E, select.context) } - } - } - - // ------ protected ------ - - override fun takeFirstReceiveOrPeekClosed(): ReceiveOrClosed? = - super.takeFirstReceiveOrPeekClosed().also { - if (it != null && it !is Closed<*>) onReceiveDequeued() - } - - /** - * Invoked when receiver is successfully enqueued to the queue of waiting receivers. - * @suppress **This is unstable API and it is subject to change.** - */ - protected open fun onReceiveEnqueued() {} - - /** - * Invoked when enqueued receiver was successfully removed from the queue of waiting receivers. - * @suppress **This is unstable API and it is subject to change.** - */ - protected open fun onReceiveDequeued() {} - - // ------ private ------ - - private fun removeReceiveOnCancel(cont: CancellableContinuation<*>, receive: Receive<*>) = - cont.invokeOnCancellation(handler = RemoveReceiveOnCancel(receive).asHandler) - - private inner class RemoveReceiveOnCancel(private val receive: Receive<*>) : BeforeResumeCancelHandler(), DisposableHandle { - override fun invoke(cause: Throwable?) = dispose() - - override fun dispose() { - if (receive.remove()) - onReceiveDequeued() - } - - override fun toString(): String = "RemoveReceiveOnCancel[$receive]" - } - - private class Itr(@JvmField val channel: AbstractChannel) : ChannelIterator { - var result: Any? = POLL_FAILED // E | POLL_FAILED | Closed - - override suspend fun hasNext(): Boolean { - // check for repeated hasNext - if (result !== POLL_FAILED) return hasNextResult(result) - // fast path -- try poll non-blocking - result = channel.pollInternal() - if (result !== POLL_FAILED) return hasNextResult(result) - // slow-path does suspend - return hasNextSuspend() - } - - private fun hasNextResult(result: Any?): Boolean { - if (result is Closed<*>) { - if (result.closeCause != null) throw recoverStackTrace(result.receiveException) - return false - } - return true - } - - private suspend fun hasNextSuspend(): Boolean = suspendCancellableCoroutineReusable sc@ { cont -> - val receive = ReceiveHasNext(this, cont) - while (true) { - if (channel.enqueueReceive(receive)) { - channel.removeReceiveOnCancel(cont, receive) - return@sc - } - // hm... something is not right. try to poll - val result = channel.pollInternal() - this.result = result - if (result is Closed<*>) { - if (result.closeCause == null) - cont.resume(false) - else - cont.resumeWithException(result.receiveException) - return@sc - } - if (result !== POLL_FAILED) { - @Suppress("UNCHECKED_CAST") - cont.resume(true, channel.onUndeliveredElement?.bindCancellationFun(result as E, cont.context)) - return@sc - } - } - } - - @Suppress("UNCHECKED_CAST") - override fun next(): E { - val result = this.result - if (result is Closed<*>) throw recoverStackTrace(result.receiveException) - if (result !== POLL_FAILED) { - this.result = POLL_FAILED - return result as E - } - - throw IllegalStateException("'hasNext' should be called prior to 'next' invocation") - } - } - - private open class ReceiveElement( - @JvmField val cont: CancellableContinuation, - @JvmField val receiveMode: Int - ) : Receive() { - fun resumeValue(value: E): Any? = when (receiveMode) { - RECEIVE_RESULT -> ChannelResult.success(value) - else -> value - } - - override fun tryResumeReceive(value: E): Symbol? { - val token = cont.tryResume(resumeValue(value), null, resumeOnCancellationFun(value)) ?: return null - assert { token === RESUME_TOKEN } // the only other possible result - return RESUME_TOKEN - } - - override fun completeResumeReceive(value: E) = cont.completeResume(RESUME_TOKEN) - - override fun resumeReceiveClosed(closed: Closed<*>) { - when (receiveMode) { - RECEIVE_RESULT -> cont.resume(closed.toResult()) - else -> cont.resumeWithException(closed.receiveException) - } - } - override fun toString(): String = "ReceiveElement@$hexAddress[receiveMode=$receiveMode]" - } - - private class ReceiveElementWithUndeliveredHandler( - cont: CancellableContinuation, - receiveMode: Int, - @JvmField val onUndeliveredElement: OnUndeliveredElement - ) : ReceiveElement(cont, receiveMode) { - override fun resumeOnCancellationFun(value: E): ((Throwable) -> Unit)? = - onUndeliveredElement.bindCancellationFun(value, cont.context) - } - - internal open inner class ReceiveElementSelect( - @JvmField val select: SelectInstance<*>, - ) : Receive() { - private val lock = ReentrantLock() - private var success: Boolean? = null - - override fun tryResumeReceive(value: E): Symbol? = lock.withLock { - if (success == null) success = select.trySelect(this@AbstractChannel, value) - if (success!!) RESUME_TOKEN else null - } - override fun completeResumeReceive(value: E) {} - - override fun resumeReceiveClosed(closed: Closed<*>) { - select.trySelect(this@AbstractChannel, closed) - } - override fun toString(): String = "ReceiveElementSelect@$hexAddress" - } - - private open inner class ReceiveElementSelectWithUndeliveredHandler( - select: SelectInstance<*>, - @JvmField val onUndeliveredElement: OnUndeliveredElement - ) : ReceiveElementSelect(select) { - override fun resumeOnCancellationFun(value: E): ((Throwable) -> Unit)? = - onUndeliveredElement.bindCancellationFun(value, select.context) - } - - private open class ReceiveHasNext( - @JvmField val iterator: Itr, - @JvmField val cont: CancellableContinuation - ) : Receive() { - override fun tryResumeReceive(value: E): Symbol? { - val token = cont.tryResume(true, null, resumeOnCancellationFun(value)) - ?: return null - assert { token === RESUME_TOKEN } // the only other possible result - return RESUME_TOKEN - } - - override fun completeResumeReceive(value: E) { - iterator.result = value - cont.completeResume(RESUME_TOKEN) - } - - override fun resumeReceiveClosed(closed: Closed<*>) { - val token = if (closed.closeCause == null) { - cont.tryResume(false) - } else { - cont.tryResumeWithException(closed.receiveException) - } - if (token != null) { - iterator.result = closed - cont.completeResume(token) - } - } - - override fun resumeOnCancellationFun(value: E): ((Throwable) -> Unit)? = - iterator.channel.onUndeliveredElement?.bindCancellationFun(value, cont.context) - - override fun toString(): String = "ReceiveHasNext@$hexAddress" - } -} - -// receiveMode values -internal const val RECEIVE_THROWS_ON_CLOSE = 0 -internal const val RECEIVE_RESULT = 1 - -@JvmField -internal val EMPTY = Symbol("EMPTY") // marker for Conflated & Buffered channels - -@JvmField -internal val OFFER_SUCCESS = Symbol("OFFER_SUCCESS") - -@JvmField -internal val OFFER_FAILED = Symbol("OFFER_FAILED") - -@JvmField -internal val POLL_FAILED = Symbol("POLL_FAILED") - -@JvmField -internal val ENQUEUE_FAILED = Symbol("ENQUEUE_FAILED") - -@JvmField -internal val HANDLER_INVOKED = Symbol("ON_CLOSE_HANDLER_INVOKED") - -internal typealias Handler = (Throwable?) -> Unit - -/** - * Represents sending waiter in the queue. - */ -internal abstract class -Send : LockFreeLinkedListNode() { - abstract val pollResult: Any? // E | Closed - the result pollInternal returns when it rendezvous with this node - // Returns: null - failure, - // RETRY_ATOMIC for retry (only when otherOp != null), - // RESUME_TOKEN on success (call completeResumeSend) - // Must call otherOp?.finishPrepare() after deciding on result other than RETRY_ATOMIC - abstract fun tryResumeSend(otherOp: PrepareOp?): Symbol? - abstract fun completeResumeSend() - abstract fun resumeSendClosed(closed: Closed<*>) - open fun undeliveredElement() {} -} - -/** - * Represents receiver waiter in the queue or closed token. - */ -internal interface ReceiveOrClosed { - val offerResult: Any // OFFER_SUCCESS | Closed - // Returns: null - failure, - // RESUME_TOKEN on success (call completeResumeReceive) - fun tryResumeReceive(value: E): Symbol? - fun completeResumeReceive(value: E) -} - -/** - * Represents sender for a specific element. - */ -internal open class SendElement( - override val pollResult: E, - @JvmField val cont: CancellableContinuation -) : Send() { - override fun tryResumeSend(otherOp: PrepareOp?): Symbol? { - val token = cont.tryResume(Unit, otherOp?.desc) ?: return null - assert { token === RESUME_TOKEN } // the only other possible result - // We can call finishPrepare only after successful tryResume, so that only good affected node is saved - otherOp?.finishPrepare() // finish preparations - return RESUME_TOKEN - } - - override fun completeResumeSend() = cont.completeResume(RESUME_TOKEN) - override fun resumeSendClosed(closed: Closed<*>) = cont.resumeWithException(closed.sendException) - override fun toString(): String = "$classSimpleName@$hexAddress($pollResult)" -} - -internal open class SendElementSelect( - override val pollResult: E, - @JvmField val select: SelectInstance<*>, - private val channel: SendChannel -) : Send() { - private val lock = ReentrantLock() - private var _trySelectResult: TrySelectDetailedResult? = null - - override fun tryResumeSend(otherOp: PrepareOp?): Symbol? = lock.withLock { - select as SelectImplementation<*> - if (_trySelectResult == null) { - _trySelectResult = select.trySelectDetailed(channel, Unit) - } - return@withLock if (_trySelectResult == SUCCESSFUL) { - otherOp?.finishPrepare() // finish preparations - RESUME_TOKEN - } else null - } - - val trySelectResult: TrySelectDetailedResult - get() = checkNotNull(_trySelectResult) { "trySelect(..) has not been invoked yet" } - - override fun completeResumeSend() {} - override fun resumeSendClosed(closed: Closed<*>) { - select.trySelect(channel, closed) - } - override fun toString(): String = "$classSimpleName@$hexAddress($pollResult)" -} - -internal class SendElementWithUndeliveredHandler( - pollResult: E, - cont: CancellableContinuation, - @JvmField val onUndeliveredElement: OnUndeliveredElement -) : SendElement(pollResult, cont) { - override fun remove(): Boolean { - if (!super.remove()) return false - // if the node was successfully removed (meaning it was added but was not received) then we have undelivered element - undeliveredElement() - return true - } - - override fun undeliveredElement() { - onUndeliveredElement.callUndeliveredElement(pollResult, cont.context) - } -} - -internal class SendElementSelectWithUndeliveredHandler( - pollResult: E, - select: SelectInstance<*>, - channel: SendChannel, - onUndeliveredElement: OnUndeliveredElement -) : SendElementSelect(pollResult, select, channel) { - private val onUndeliveredElement = atomic?>(onUndeliveredElement) - - override fun remove(): Boolean { - undeliveredElement() - return super.remove() - } - - override fun undeliveredElement() { - onUndeliveredElement.getAndSet(null)?.callUndeliveredElement(pollResult, select.context) - } -} -/** - * Represents closed channel. - */ -internal class Closed( - @JvmField val closeCause: Throwable? -) : Send(), ReceiveOrClosed { - val sendException: Throwable get() = closeCause ?: ClosedSendChannelException(DEFAULT_CLOSE_MESSAGE) - val receiveException: Throwable get() = closeCause ?: ClosedReceiveChannelException(DEFAULT_CLOSE_MESSAGE) - - override val offerResult get() = this - override val pollResult get() = this - override fun tryResumeSend(otherOp: PrepareOp?): Symbol = RESUME_TOKEN.also { otherOp?.finishPrepare() } - override fun completeResumeSend() {} - override fun tryResumeReceive(value: E): Symbol = RESUME_TOKEN - override fun completeResumeReceive(value: E) {} - override fun resumeSendClosed(closed: Closed<*>) = assert { false } // "Should be never invoked" - override fun toString(): String = "Closed@$hexAddress[$closeCause]" -} - -internal abstract class Receive : LockFreeLinkedListNode(), ReceiveOrClosed { - override val offerResult get() = OFFER_SUCCESS - abstract fun resumeReceiveClosed(closed: Closed<*>) - open fun resumeOnCancellationFun(value: E): ((Throwable) -> Unit)? = null -} - -@Suppress("NOTHING_TO_INLINE", "UNCHECKED_CAST") -private inline fun Any?.toResult(): ChannelResult = - if (this is Closed<*>) ChannelResult.closed(closeCause) else ChannelResult.success(this as E) - -@Suppress("NOTHING_TO_INLINE") -private inline fun Closed<*>.toResult(): ChannelResult = ChannelResult.closed(closeCause) diff --git a/kotlinx-coroutines-core/common/src/channels/ArrayBroadcastChannel.kt b/kotlinx-coroutines-core/common/src/channels/ArrayBroadcastChannel.kt deleted file mode 100644 index a9aaf08a53..0000000000 --- a/kotlinx-coroutines-core/common/src/channels/ArrayBroadcastChannel.kt +++ /dev/null @@ -1,329 +0,0 @@ -/* - * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. - */ - -package kotlinx.coroutines.channels - -import kotlinx.atomicfu.* -import kotlinx.coroutines.* -import kotlinx.coroutines.internal.* -import kotlinx.coroutines.selects.* - -/** - * Broadcast channel with array buffer of a fixed [capacity]. - * Sender suspends only when buffer is full due to one of the receives being slow to consume and - * receiver suspends only when buffer is empty. - * - * **Note**, that elements that are sent to this channel while there are no - * [openSubscription] subscribers are immediately lost. - * - * This channel is created by `BroadcastChannel(capacity)` factory function invocation. - * - * This implementation uses lock to protect the buffer, which is held only during very short buffer-update operations. - * The lock at each subscription is also used to manage concurrent attempts to receive from the same subscriber. - * The lists of suspended senders or receivers are lock-free. - */ -internal class ArrayBroadcastChannel( - /** - * Buffer capacity. - */ - val capacity: Int -) : AbstractSendChannel(null), BroadcastChannel { - init { - require(capacity >= 1) { "ArrayBroadcastChannel capacity must be at least 1, but $capacity was specified" } - } - - /** - * NB: prior to changing any logic of ArrayBroadcastChannel internals, please ensure that - * you do not break internal invariants of the SubscriberList implementation on K/N and KJS - */ - - /* - * Writes to buffer are guarded by bufferLock, but reads from buffer are concurrent with writes - * - Write element to buffer then write "tail" (volatile) - * - Read "tail" (volatile), then read element from buffer - * So read/writes to buffer need not be volatile - */ - private val bufferLock = ReentrantLock() - private val buffer = arrayOfNulls(capacity) - - // head & tail are Long (64 bits) and we assume that they never wrap around - // head, tail, and size are guarded by bufferLock - - private val _head = atomic(0L) - private var head: Long // do modulo on use of head - get() = _head.value - set(value) { _head.value = value } - - private val _tail = atomic(0L) - private var tail: Long // do modulo on use of tail - get() = _tail.value - set(value) { _tail.value = value } - - private val _size = atomic(0) - private var size: Int - get() = _size.value - set(value) { _size.value = value } - - @Suppress("DEPRECATION") - private val subscribers = subscriberList>() - - override val isBufferAlwaysFull: Boolean get() = false - override val isBufferFull: Boolean get() = size >= capacity - - public override fun openSubscription(): ReceiveChannel = - Subscriber(this).also { - updateHead(addSub = it) - } - - public override fun close(cause: Throwable?): Boolean { - if (!super.close(cause)) return false - checkSubOffers() - return true - } - - @Deprecated(level = DeprecationLevel.HIDDEN, message = "Since 1.2.0, binary compatibility with versions <= 1.1.x") - override fun cancel(cause: Throwable?): Boolean = - cancelInternal(cause) - - override fun cancel(cause: CancellationException?) { - cancelInternal(cause) - } - - private fun cancelInternal(cause: Throwable?): Boolean = - close(cause).also { - for (sub in subscribers) sub.cancelInternal(cause) - } - - // result is `OFFER_SUCCESS | OFFER_FAILED | Closed` - override fun offerInternal(element: E): Any { - bufferLock.withLock { - // check if closed for send (under lock, so size cannot change) - closedForSend?.let { return it } - val size = this.size - if (size >= capacity) return OFFER_FAILED - val tail = this.tail - buffer[(tail % capacity).toInt()] = element - this.size = size + 1 - this.tail = tail + 1 - } - // if offered successfully, then check subscribers outside of lock - checkSubOffers() - return OFFER_SUCCESS - } - - private fun checkSubOffers() { - var updated = false - var hasSubs = false - @Suppress("LoopToCallChain") // must invoke `checkOffer` on every sub - for (sub in subscribers) { - hasSubs = true - if (sub.checkOffer()) updated = true - } - if (updated || !hasSubs) - updateHead() - } - - // updates head if needed and optionally adds / removes subscriber under the same lock - private tailrec fun updateHead(addSub: Subscriber? = null, removeSub: Subscriber? = null) { - // update head in a tail rec loop - var send: Send? = null - bufferLock.withLock { - if (addSub != null) { - addSub.subHead = tail // start from last element - val wasEmpty = subscribers.isEmpty() - subscribers.add(addSub) - if (!wasEmpty) return // no need to update when adding second and etc sub - } - if (removeSub != null) { - subscribers.remove(removeSub) - if (head != removeSub.subHead) return // no need to update - } - val minHead = computeMinHead() - val tail = this.tail - var head = this.head - val targetHead = minHead.coerceAtMost(tail) - if (targetHead <= head) return // nothing to do -- head was already moved - var size = this.size - // clean up removed (on not need if we don't have any subscribers anymore) - while (head < targetHead) { - buffer[(head % capacity).toInt()] = null - val wasFull = size >= capacity - // update the size before checking queue (no more senders can queue up) - this.head = ++head - this.size = --size - if (wasFull) { - while (true) { - send = takeFirstSendOrPeekClosed() ?: break // when when no sender - if (send is Closed<*>) break // break when closed for send - val token = send!!.tryResumeSend(null) - if (token != null) { - assert { token === RESUME_TOKEN } - // put sent element to the buffer - buffer[(tail % capacity).toInt()] = (send as Send).pollResult - this.size = size + 1 - this.tail = tail + 1 - return@withLock // go out of lock to wakeup this sender - } - // Too late, already cancelled, but we removed it from the queue and need to release resources. - // However, ArrayBroadcastChannel does not support onUndeliveredElement, so nothing to do - } - } - } - return // done updating here -> return - } - // we only get out of the lock normally when there is a sender to resume - send!!.completeResumeSend() - // since we've just sent an element, we might need to resume some receivers - checkSubOffers() - // tailrec call to recheck - updateHead() - } - - private fun computeMinHead(): Long { - var minHead = Long.MAX_VALUE - for (sub in subscribers) - minHead = minHead.coerceAtMost(sub.subHead) // volatile (atomic) reads of subHead - return minHead - } - - @Suppress("UNCHECKED_CAST") - private fun elementAt(index: Long): E = buffer[(index % capacity).toInt()] as E - - private class Subscriber( - private val broadcastChannel: ArrayBroadcastChannel - ) : AbstractChannel(null), ReceiveChannel { - private val subLock = ReentrantLock() - - private val _subHead = atomic(0L) - var subHead: Long // guarded by subLock - get() = _subHead.value - set(value) { _subHead.value = value } - - override val isBufferAlwaysEmpty: Boolean get() = false - override val isBufferEmpty: Boolean get() = subHead >= broadcastChannel.tail - override val isBufferAlwaysFull: Boolean get() = error("Should not be used") - override val isBufferFull: Boolean get() = error("Should not be used") - - override fun close(cause: Throwable?): Boolean { - val wasClosed = super.close(cause) - if (wasClosed) { - broadcastChannel.updateHead(removeSub = this) - subLock.withLock { - subHead = broadcastChannel.tail - } - } - return wasClosed - } - - // returns true if subHead was updated and broadcast channel's head must be checked - // this method is lock-free (it never waits on lock) - @Suppress("UNCHECKED_CAST") - fun checkOffer(): Boolean { - var updated = false - var closed: Closed<*>? = null - loop@ - while (needsToCheckOfferWithoutLock()) { - // just use `tryLock` here and break when some other thread is checking under lock - // it means that `checkOffer` must be retried after every `unlock` - if (!subLock.tryLock()) break - val receive: ReceiveOrClosed? - var result: Any? - try { - result = peekUnderLock() - when { - result === POLL_FAILED -> continue@loop // must retest `needsToCheckOfferWithoutLock` outside of the lock - result is Closed<*> -> { - closed = result - break@loop // was closed - } - } - // find a receiver for an element - receive = takeFirstReceiveOrPeekClosed() ?: break // break when no one's receiving - if (receive is Closed<*>) break // noting more to do if this sub already closed - val token = receive.tryResumeReceive(result as E) ?: continue - assert { token === RESUME_TOKEN } - val subHead = this.subHead - this.subHead = subHead + 1 // retrieved element for this subscriber - updated = true - } finally { - subLock.unlock() - } - receive!!.completeResumeReceive(result as E) - } - // do close outside of lock if needed - closed?.also { close(cause = it.closeCause) } - return updated - } - - // result is `E | POLL_FAILED | Closed` - override fun pollInternal(): Any? { - var updated = false - val result = subLock.withLock { - val result = peekUnderLock() - when { - result is Closed<*> -> { /* just bail out of lock */ } - result === POLL_FAILED -> { /* just bail out of lock */ } - else -> { - // update subHead after retrieiving element from buffer - val subHead = this.subHead - this.subHead = subHead + 1 - updated = true - } - } - result - } - // do close outside of lock - (result as? Closed<*>)?.also { close(cause = it.closeCause) } - // there could have been checkOffer attempt while we were holding lock - // now outside the lock recheck if anything else to offer - if (checkOffer()) - updated = true - // and finally update broadcast's channel head if needed - if (updated) - broadcastChannel.updateHead() - return result - } - - // Must invoke this check this after lock, because offer's invocation of `checkOffer` might have failed - // to `tryLock` just before the lock was about to unlocked, thus loosing notification to this - // subscription about an element that was just offered - private fun needsToCheckOfferWithoutLock(): Boolean { - if (closedForReceive != null) - return false // already closed -> nothing to do - if (isBufferEmpty && broadcastChannel.closedForReceive == null) - return false // no data for us && broadcast channel was not closed yet -> nothing to do - return true // check otherwise - } - - // guarded by lock, returns: - // E - the element from the buffer at subHead - // Closed<*> when closed; - // POLL_FAILED when there seems to be no data, but must retest `needsToCheckOfferWithoutLock` outside of lock - private fun peekUnderLock(): Any? { - val subHead = this.subHead // guarded read (can be non-volatile read) - // note: from the broadcastChannel we must read closed token first, then read its tail - // because it is Ok if tail moves in between the reads (we make decision based on tail first) - val closedBroadcast = broadcastChannel.closedForReceive // unguarded volatile read - val tail = broadcastChannel.tail // unguarded volatile read - if (subHead >= tail) { - // no elements to poll from the queue -- check if closed broads & closed this sub - // must retest `needsToCheckOfferWithoutLock` outside of the lock - return closedBroadcast ?: this.closedForReceive ?: POLL_FAILED - } - // Get tentative result. This result may be wrong (completely invalid value, including null), - // because this subscription might get closed, moving channel's head past this subscription's head. - val result = broadcastChannel.elementAt(subHead) - // now check if this subscription was closed - val closedSub = this.closedForReceive - if (closedSub != null) return closedSub - // we know the subscription was not closed, so this tentative result is Ok to return - return result - } - } - - // ------ debug ------ - - override val bufferDebugString: String - get() = "(buffer:capacity=${buffer.size},size=$size)" -} diff --git a/kotlinx-coroutines-core/common/src/channels/ArrayChannel.kt b/kotlinx-coroutines-core/common/src/channels/ArrayChannel.kt deleted file mode 100644 index abf18ac39a..0000000000 --- a/kotlinx-coroutines-core/common/src/channels/ArrayChannel.kt +++ /dev/null @@ -1,209 +0,0 @@ -/* - * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. - */ - -package kotlinx.coroutines.channels - -import kotlinx.atomicfu.* -import kotlinx.coroutines.* -import kotlinx.coroutines.internal.* -import kotlinx.coroutines.selects.TrySelectDetailedResult.* -import kotlin.math.* - -/** - * Channel with array buffer of a fixed [capacity]. - * Sender suspends only when buffer is full and receiver suspends only when buffer is empty. - * - * This channel is created by `Channel(capacity)` factory function invocation. - * - * This implementation uses lock to protect the buffer, which is held only during very short buffer-update operations. - * The lists of suspended senders or receivers are lock-free. - **/ -internal open class ArrayChannel( - /** - * Buffer capacity. - */ - private val capacity: Int, - private val onBufferOverflow: BufferOverflow, - onUndeliveredElement: OnUndeliveredElement? -) : AbstractChannel(onUndeliveredElement) { - init { - // This check is actually used by the Channel(...) constructor function which checks only for known - // capacities and calls ArrayChannel constructor for everything else. - require(capacity >= 1) { "ArrayChannel capacity must be at least 1, but $capacity was specified" } - } - - private val lock = ReentrantLock() - - /* - * Guarded by lock. - * Allocate minimum of capacity and 16 to avoid excess memory pressure for large channels when it's not necessary. - */ - private var buffer: Array = arrayOfNulls(min(capacity, 8)).apply { fill(EMPTY) } - - private var head: Int = 0 - private val size = atomic(0) // Invariant: size <= capacity - - protected final override val isBufferAlwaysEmpty: Boolean get() = false - protected final override val isBufferEmpty: Boolean get() = size.value == 0 - protected final override val isBufferAlwaysFull: Boolean get() = false - protected final override val isBufferFull: Boolean get() = size.value == capacity && onBufferOverflow == BufferOverflow.SUSPEND - - override val isEmpty: Boolean get() = lock.withLock { isEmptyImpl } - override val isClosedForReceive: Boolean get() = lock.withLock { super.isClosedForReceive } - - // result is `OFFER_SUCCESS | OFFER_FAILED | Closed` - protected override fun offerInternal(element: E): Any { - var receive: ReceiveOrClosed? = null - lock.withLock { - val size = this.size.value - closedForSend?.let { return it } - // update size before checking queue (!!!) - updateBufferSize(size)?.let { return it } - // check for receivers that were waiting on empty queue - if (size == 0) { - loop@ while (true) { - receive = takeFirstReceiveOrPeekClosed() ?: break@loop // break when no receivers queued - if (receive is Closed) { - this.size.value = size // restore size - return receive!! - } - val token = receive!!.tryResumeReceive(element) - if (token != null) { - assert { token === RESUME_TOKEN } - this.size.value = size // restore size - return@withLock - } - } - } - enqueueElement(size, element) - return OFFER_SUCCESS - } - // breaks here if offer meets receiver - receive!!.completeResumeReceive(element) - return receive!!.offerResult - } - - override fun enqueueSend(send: Send): Any? = lock.withLock { - super.enqueueSend(send) - } - - // Guarded by lock - // Result is `OFFER_SUCCESS | OFFER_FAILED | null` - private fun updateBufferSize(currentSize: Int): Symbol? { - if (currentSize < capacity) { - size.value = currentSize + 1 // tentatively put it into the buffer - return null // proceed - } - // buffer is full - return when (onBufferOverflow) { - BufferOverflow.SUSPEND -> OFFER_FAILED - BufferOverflow.DROP_LATEST -> OFFER_SUCCESS - BufferOverflow.DROP_OLDEST -> null // proceed, will drop oldest in enqueueElement - } - } - - // Guarded by lock - private fun enqueueElement(currentSize: Int, element: E) { - if (currentSize < capacity) { - ensureCapacity(currentSize) - buffer[(head + currentSize) % buffer.size] = element // actually queue element - } else { - // buffer is full - assert { onBufferOverflow == BufferOverflow.DROP_OLDEST } // the only way we can get here - buffer[head % buffer.size] = null // drop oldest element - buffer[(head + currentSize) % buffer.size] = element // actually queue element - head = (head + 1) % buffer.size - } - } - - // Guarded by lock - private fun ensureCapacity(currentSize: Int) { - if (currentSize >= buffer.size) { - val newSize = min(buffer.size * 2, capacity) - val newBuffer = arrayOfNulls(newSize) - for (i in 0 until currentSize) { - newBuffer[i] = buffer[(head + i) % buffer.size] - } - newBuffer.fill(EMPTY, currentSize, newSize) - buffer = newBuffer - head = 0 - } - } - - // result is `E | POLL_FAILED | Closed` - protected override fun pollInternal(): Any? { - var send: Send? = null - var resumed = false - var result: Any? = null - lock.withLock { - val size = this.size.value - if (size == 0) return closedForSend ?: POLL_FAILED // when nothing can be read from buffer - // size > 0: not empty -- retrieve element - result = buffer[head] - buffer[head] = null - this.size.value = size - 1 // update size before checking queue (!!!) - // check for senders that were waiting on full queue - var replacement: Any? = POLL_FAILED - if (size == capacity) { - loop@ while (true) { - send = takeFirstSendOrPeekClosed() ?: break - val token = send!!.tryResumeSend(null) - if (token != null) { - assert { token === RESUME_TOKEN } - resumed = true - replacement = send!!.pollResult - break@loop - } - // Too late, already cancelled, but we removed it from the queue and need to notify on undelivered element. - // The only exception is when this "send" operation is an `onSend` clause that has to be re-registered - // in the corresponding `select` invocation. - send!!.apply { - if (!(this is SendElementSelectWithUndeliveredHandler<*> && trySelectResult == REREGISTER)) - undeliveredElement() - } - } - } - if (replacement !== POLL_FAILED && replacement !is Closed<*>) { - this.size.value = size // restore size - buffer[(head + size) % buffer.size] = replacement - } - head = (head + 1) % buffer.size - } - // complete send the we're taken replacement from - if (resumed) - send!!.completeResumeSend() - return result - } - - override fun enqueueReceiveInternal(receive: Receive): Boolean = lock.withLock { - super.enqueueReceiveInternal(receive) - } - - // Note: this function is invoked when channel is already closed - override fun onCancelIdempotent(wasClosed: Boolean) { - // clear buffer first, but do not wait for it in helpers - val onUndeliveredElement = onUndeliveredElement - var undeliveredElementException: UndeliveredElementException? = null // first cancel exception, others suppressed - lock.withLock { - repeat(size.value) { - val value = buffer[head] - if (onUndeliveredElement != null && value !== EMPTY) { - @Suppress("UNCHECKED_CAST") - undeliveredElementException = onUndeliveredElement.callUndeliveredElementCatchingException(value as E, undeliveredElementException) - } - buffer[head] = EMPTY - head = (head + 1) % buffer.size - } - size.value = 0 - } - // then clean all queued senders - super.onCancelIdempotent(wasClosed) - undeliveredElementException?.let { throw it } // throw UndeliveredElementException at the end if there was one - } - - // ------ debug ------ - - override val bufferDebugString: String - get() = "(buffer:capacity=$capacity,size=${size.value})" -} diff --git a/kotlinx-coroutines-core/common/src/channels/Broadcast.kt b/kotlinx-coroutines-core/common/src/channels/Broadcast.kt index ba39d4ff7e..0a76c82f6a 100644 --- a/kotlinx-coroutines-core/common/src/channels/Broadcast.kt +++ b/kotlinx-coroutines-core/common/src/channels/Broadcast.kt @@ -16,7 +16,7 @@ import kotlin.coroutines.intrinsics.* * This function [consumes][ReceiveChannel.consume] all elements of the original [ReceiveChannel]. * * The kind of the resulting channel depends on the specified [capacity] parameter: - * when `capacity` is positive (1 by default), but less than [UNLIMITED] -- uses `ArrayBroadcastChannel` with a buffer of given capacity, + * when `capacity` is positive (1 by default), but less than [UNLIMITED] -- uses [BroadcastChannel] with a buffer of given capacity, * when `capacity` is [CONFLATED] -- uses [ConflatedBroadcastChannel] that conflates back-to-back sends; * Note that resulting channel behaves like [ConflatedBroadcastChannel] but is not an instance of [ConflatedBroadcastChannel]. * otherwise -- throws [IllegalArgumentException]. @@ -76,7 +76,7 @@ public fun ReceiveChannel.broadcast( * the resulting channel becomes _failed_, so that any attempt to receive from such a channel throws exception. * * The kind of the resulting channel depends on the specified [capacity] parameter: - * * when `capacity` is positive (1 by default), but less than [UNLIMITED] -- uses `ArrayBroadcastChannel` with a buffer of given capacity, + * * when `capacity` is positive (1 by default), but less than [UNLIMITED] -- uses [BroadcastChannel] with a buffer of given capacity, * * when `capacity` is [CONFLATED] -- uses [ConflatedBroadcastChannel] that conflates back-to-back sends; * Note that resulting channel behaves like [ConflatedBroadcastChannel] but is not an instance of [ConflatedBroadcastChannel]. * * otherwise -- throws [IllegalArgumentException]. diff --git a/kotlinx-coroutines-core/common/src/channels/BroadcastChannel.kt b/kotlinx-coroutines-core/common/src/channels/BroadcastChannel.kt index c82b8dbd63..de67b60a2d 100644 --- a/kotlinx-coroutines-core/common/src/channels/BroadcastChannel.kt +++ b/kotlinx-coroutines-core/common/src/channels/BroadcastChannel.kt @@ -7,10 +7,14 @@ package kotlinx.coroutines.channels import kotlinx.coroutines.* +import kotlinx.coroutines.channels.BufferOverflow.* import kotlinx.coroutines.channels.Channel.Factory.BUFFERED import kotlinx.coroutines.channels.Channel.Factory.CHANNEL_DEFAULT_CAPACITY import kotlinx.coroutines.channels.Channel.Factory.CONFLATED import kotlinx.coroutines.channels.Channel.Factory.UNLIMITED +import kotlinx.coroutines.internal.* +import kotlinx.coroutines.selects.* +import kotlin.native.concurrent.* /** * Broadcast channel is a non-blocking primitive for communication between the sender and multiple receivers @@ -55,7 +59,7 @@ public interface BroadcastChannel : SendChannel { * The resulting channel type depends on the specified [capacity] parameter: * * * when `capacity` positive, but less than [UNLIMITED] -- creates `ArrayBroadcastChannel` with a buffer of given capacity. - * **Note:** this channel looses all items that are send to it until the first subscriber appears; + * **Note:** this channel looses all items that have been sent to it until the first subscriber appears; * * when `capacity` is [CONFLATED] -- creates [ConflatedBroadcastChannel] that conflates back-to-back sends; * * when `capacity` is [BUFFERED] -- creates `ArrayBroadcastChannel` with a default capacity. * * otherwise -- throws [IllegalArgumentException]. @@ -70,6 +74,339 @@ public fun BroadcastChannel(capacity: Int): BroadcastChannel = 0 -> throw IllegalArgumentException("Unsupported 0 capacity for BroadcastChannel") UNLIMITED -> throw IllegalArgumentException("Unsupported UNLIMITED capacity for BroadcastChannel") CONFLATED -> ConflatedBroadcastChannel() - BUFFERED -> ArrayBroadcastChannel(CHANNEL_DEFAULT_CAPACITY) - else -> ArrayBroadcastChannel(capacity) + BUFFERED -> BroadcastChannelImpl(CHANNEL_DEFAULT_CAPACITY) + else -> BroadcastChannelImpl(capacity) } + +/** + * Broadcasts the most recently sent element (aka [value]) to all [openSubscription] subscribers. + * + * Back-to-send sent elements are _conflated_ -- only the most recently sent value is received, + * while previously sent elements **are lost**. + * Every subscriber immediately receives the most recently sent element. + * Sender to this broadcast channel never suspends and [trySend] always succeeds. + * + * A secondary constructor can be used to create an instance of this class that already holds a value. + * This channel is also created by `BroadcastChannel(Channel.CONFLATED)` factory function invocation. + * + * In this implementation, [opening][openSubscription] and [closing][ReceiveChannel.cancel] subscription + * takes linear time in the number of subscribers. + * + * **Note: This API is obsolete since 1.5.0.** It will be deprecated with warning in 1.7.0 + * and with error in 1.8.0. It is replaced with [StateFlow][kotlinx.coroutines.flow.StateFlow]. + */ +@ObsoleteCoroutinesApi +public class ConflatedBroadcastChannel private constructor( + private val broadcast: BroadcastChannelImpl +) : BroadcastChannel by broadcast { + public constructor(): this(BroadcastChannelImpl(capacity = CONFLATED)) + /** + * Creates an instance of this class that already holds a value. + * + * It is as a shortcut to creating an instance with a default constructor and + * immediately sending an element: `ConflatedBroadcastChannel().apply { offer(value) }`. + */ + public constructor(value: E) : this() { + trySend(value) + } + + /** + * The most recently sent element to this channel. + * + * Access to this property throws [IllegalStateException] when this class is constructed without + * initial value and no value was sent yet or if it was [closed][close] without a cause. + * It throws the original [close][SendChannel.close] cause exception if the channel has _failed_. + */ + public val value: E get() = broadcast.value + /** + * The most recently sent element to this channel or `null` when this class is constructed without + * initial value and no value was sent yet or if it was [closed][close]. + */ + public val valueOrNull: E? get() = broadcast.valueOrNull +} + +/** + * A common implementation for both the broadcast channel with a buffer of fixed [capacity] + * and the conflated broadcast channel (see [ConflatedBroadcastChannel]). + * + * **Note**, that elements that are sent to this channel while there are no + * [openSubscription] subscribers are immediately lost. + * + * This channel is created by `BroadcastChannel(capacity)` factory function invocation. + */ +internal class BroadcastChannelImpl( + /** + * Buffer capacity; [Channel.CONFLATED] when this broadcast is conflated. + */ + val capacity: Int +) : BufferedChannel(capacity = Channel.RENDEZVOUS, onUndeliveredElement = null), BroadcastChannel { + init { + require(capacity >= 1 || capacity == CONFLATED) { + "BroadcastChannel capacity must be positive or Channel.CONFLATED, but $capacity was specified" + } + } + + // This implementation uses coarse-grained synchronization, + // as, reputedly, it is the simplest synchronization scheme. + // All operations are protected by this lock. + private val lock = ReentrantLock() + // The list of subscribers; all accesses should be protected by lock. + // Each change must create a new list instance to avoid `ConcurrentModificationException`. + private var subscribers: List> = emptyList() + // When this broadcast is conflated, this field stores the last sent element. + // If this channel is empty or not conflated, it stores a special `NO_ELEMENT` marker. + private var lastConflatedElement: Any? = NO_ELEMENT // NO_ELEMENT or E + + // ########################### + // # Subscription Management # + // ########################### + + override fun openSubscription(): ReceiveChannel = lock.withLock { // protected by lock + // Is this broadcast conflated or buffered? + // Create the corresponding subscription channel. + val s = if (capacity == CONFLATED) SubscriberConflated() else SubscriberBuffered() + // If this broadcast is already closed or cancelled, + // and the last sent element is not available in case + // this broadcast is conflated, close the created + // subscriber immediately and return it. + if (isClosedForSend && lastConflatedElement === NO_ELEMENT) { + s.close(closeCause) + return s + } + // Is this broadcast conflated? If so, send + // the last sent element to the subscriber. + if (lastConflatedElement !== NO_ELEMENT) { + s.trySend(value) + } + // Add the subscriber to the list and return it. + subscribers += s + s + } + + private fun removeSubscriber(s: ReceiveChannel) = lock.withLock { // protected by lock + subscribers = subscribers.filter { it !== s } + } + + // ############################# + // # The `send(..)` Operations # + // ############################# + + /** + * Sends the specified element to all subscribers. + * + * **!!! THIS IMPLEMENTATION IS NOT LINEARIZABLE !!!** + * + * As the operation should send the element to multiple + * subscribers simultaneously, it is non-trivial to + * implement it in an atomic way. Specifically, this + * would require a special implementation that does + * not transfer the element until all parties are able + * to resume it (this `send(..)` can be cancelled + * or the broadcast can become closed in the meantime). + * As broadcasts are obsolete, we keep this implementation + * as simple as possible, allowing non-linearizability + * in corner cases. + */ + override suspend fun send(element: E) { + val subs = lock.withLock { // protected by lock + // Is this channel closed for send? + if (isClosedForSend) throw sendException + // Update the last sent element if this broadcast is conflated. + if (capacity == CONFLATED) lastConflatedElement = element + // Get a reference to the list of subscribers under the lock. + subscribers + } + // The lock has been released. Send the element to the + // subscribers one-by-one, and finish immediately + // when this broadcast discovered in the closed state. + // Note that this implementation is non-linearizable; + // see this method documentation for details. + subs.forEach { + // We use special function to send the element, + // which returns `true` on success and `false` + // if the subscriber is closed. + val success = it.sendBroadcast(element) + // The sending attempt has failed. + // Check whether the broadcast is closed. + if (!success && isClosedForSend) throw sendException + } + } + + override fun trySend(element: E): ChannelResult = lock.withLock { // protected by lock + // Is this channel closed for send? + if (isClosedForSend) return super.trySend(element) + // Check whether the plain `send(..)` operation + // should suspend and fail in this case. + val shouldSuspend = subscribers.any { it.shouldSendSuspend() } + if (shouldSuspend) return ChannelResult.failure() + // Update the last sent element if this broadcast is conflated. + if (capacity == CONFLATED) lastConflatedElement = element + // Send the element to all subscribers. + // It is guaranteed that the attempt cannot fail, + // as both the broadcast closing and subscription + // cancellation are guarded by lock, which is held + // by the current operation. + subscribers.forEach { it.trySend(element) } + // Finish with success. + return ChannelResult.success(Unit) + } + + // ########################################### + // # The `select` Expression: onSend { ... } # + // ########################################### + + override fun registerSelectForSend(select: SelectInstance<*>, element: Any?) { + // It is extremely complicated to support sending via `select` for broadcasts, + // as the operation should wait on multiple subscribers simultaneously. + // At the same time, broadcasts are obsolete, so we need a simple implementation + // that works somehow. Here is a tricky work-around. First, we launch a new + // coroutine that performs plain `send(..)` operation and tries to complete + // this `select` via `trySelect`, independently on whether it is in the + // registration or in the waiting phase. On success, the operation finishes. + // On failure, if another clause is already selected or the `select` operation + // has been cancelled, we observe non-linearizable behaviour, as this `onSend` + // clause is completed as well. However, we believe that such a non-linearizability + // is fine for obsolete API. The last case is when the `select` operation is still + // in the registration case, so this `onSend` clause should be re-registered. + // The idea is that we keep information that this `onSend` clause is already selected + // and finish immediately. + @Suppress("UNCHECKED_CAST") + element as E + // First, check whether this `onSend` clause is already + // selected, finishing immediately in this case. + lock.withLock { + val result = onSendInternalResult.remove(select) + if (result != null) { // already selected! + // `result` is either `Unit` ot `CHANNEL_CLOSED`. + select.selectInRegistrationPhase(result) + return + } + } + // Start a new coroutine that performs plain `send(..)` + // and tries to select this `onSend` clause at the end. + CoroutineScope(select.context).launch(start = CoroutineStart.UNDISPATCHED) { + val success: Boolean = try { + send(element) + // The element has been successfully sent! + true + } catch (t: Throwable) { + // This broadcast must be closed. However, it is possible that + // an unrelated exception, such as `OutOfMemoryError` has been thrown. + // This implementation checks that the channel is actually closed, + // re-throwing the caught exception otherwise. + if (isClosedForSend && (t is ClosedSendChannelException || sendException === t)) false + else throw t + } + // Mark this `onSend` clause as selected and + // try to complete the `select` operation. + lock.withLock { + // Status of this `onSend` clause should not be presented yet. + assert { onSendInternalResult[select] == null } + // Success or fail? Put the corresponding result. + onSendInternalResult[select] = if (success) Unit else CHANNEL_CLOSED + // Try to select this `onSend` clause. + select as SelectImplementation<*> + val trySelectResult = select.trySelectDetailed(this@BroadcastChannelImpl, Unit) + if (trySelectResult !== TrySelectDetailedResult.REREGISTER) { + // In case of re-registration (this `select` was still + // in the registration phase), the algorithm will invoke + // `registerSelectForSend`. As we stored an information that + // this `onSend` clause is already selected (in `onSendInternalResult`), + // the algorithm, will complete immediately. Otherwise, to avoid memory + // leaks, we must remove this information from the hashmap. + onSendInternalResult.remove(select) + } + } + + } + } + private val onSendInternalResult = HashMap, Any?>() // select -> Unit or CHANNEL_CLOSED + + // ############################ + // # Closing and Cancellation # + // ############################ + + override fun close(cause: Throwable?): Boolean = lock.withLock { // protected by lock + // Close all subscriptions first. + subscribers.forEach { it.close(cause) } + // Remove all subscriptions that do not contain + // buffered elements or waiting send-s to avoid + // memory leaks. We must keep other subscriptions + // in case `broadcast.cancel(..)` is called. + subscribers = subscribers.filter { it.hasElements() } + // Delegate to the parent implementation. + super.close(cause) + } + + override fun cancelImpl(cause: Throwable?): Boolean = lock.withLock { // protected by lock + // Cancel all subscriptions. As part of cancellation procedure, + // subscriptions automatically remove themselves from this broadcast. + subscribers.forEach { it.cancelImpl(cause) } + // For the conflated implementation, clear the last sent element. + lastConflatedElement = NO_ELEMENT + // Finally, delegate to the parent implementation. + super.cancelImpl(cause) + } + + override val isClosedForSend: Boolean + // Protect by lock to synchronize with `close(..)` / `cancel(..)`. + get() = lock.withLock { super.isClosedForSend } + + // ############################## + // # Subscriber Implementations # + // ############################## + + private inner class SubscriberBuffered : BufferedChannel(capacity = capacity) { + public override fun cancelImpl(cause: Throwable?): Boolean = lock.withLock { + // Remove this subscriber from the broadcast on cancellation. + removeSubscriber(this@SubscriberBuffered ) + super.cancelImpl(cause) + } + } + + private inner class SubscriberConflated : ConflatedBufferedChannel(capacity = 1, onBufferOverflow = DROP_OLDEST) { + public override fun cancelImpl(cause: Throwable?): Boolean { + // Remove this subscriber from the broadcast on cancellation. + removeSubscriber(this@SubscriberConflated ) + return super.cancelImpl(cause) + } + } + + // ######################################## + // # ConflatedBroadcastChannel Operations # + // ######################################## + + @Suppress("UNCHECKED_CAST") + val value: E get() = lock.withLock { + // Is this channel closed for sending? + if (isClosedForSend) { + throw closeCause ?: IllegalStateException("This broadcast channel is closed") + } + // Is there sent element? + if (lastConflatedElement === NO_ELEMENT) error("No value") + // Return the last sent element. + lastConflatedElement as E + } + + @Suppress("UNCHECKED_CAST") + val valueOrNull: E? get() = lock.withLock { + // Is this channel closed for sending? + if (isClosedForReceive) null + // Is there sent element? + else if (lastConflatedElement === NO_ELEMENT) null + // Return the last sent element. + else lastConflatedElement as E + } + + // ################# + // # For Debugging # + // ################# + + override fun toString() = + (if (lastConflatedElement !== NO_ELEMENT) "CONFLATED_ELEMENT=$lastConflatedElement; " else "") + + "BROADCAST=<${super.toString()}>; " + + "SUBSCRIBERS=${subscribers.joinToString(separator = ";", prefix = "<", postfix = ">")}" +} + +@SharedImmutable +private val NO_ELEMENT = Symbol("NO_ELEMENT") \ No newline at end of file diff --git a/kotlinx-coroutines-core/common/src/channels/BufferOverflow.kt b/kotlinx-coroutines-core/common/src/channels/BufferOverflow.kt index 48b89ce6fe..8af2c845e7 100644 --- a/kotlinx-coroutines-core/common/src/channels/BufferOverflow.kt +++ b/kotlinx-coroutines-core/common/src/channels/BufferOverflow.kt @@ -4,8 +4,6 @@ package kotlinx.coroutines.channels -import kotlinx.coroutines.* - /** * A strategy for buffer overflow handling in [channels][Channel] and [flows][kotlinx.coroutines.flow.Flow] that * controls what is going to be sacrificed on buffer overflow: diff --git a/kotlinx-coroutines-core/common/src/channels/BufferedChannel.kt b/kotlinx-coroutines-core/common/src/channels/BufferedChannel.kt new file mode 100644 index 0000000000..990798c9ca --- /dev/null +++ b/kotlinx-coroutines-core/common/src/channels/BufferedChannel.kt @@ -0,0 +1,3152 @@ +@file:Suppress("PrivatePropertyName") + +package kotlinx.coroutines.channels + +import kotlinx.atomicfu.* +import kotlinx.coroutines.* +import kotlinx.coroutines.channels.ChannelResult.Companion.closed +import kotlinx.coroutines.channels.ChannelResult.Companion.failure +import kotlinx.coroutines.channels.ChannelResult.Companion.success +import kotlinx.coroutines.flow.internal.* +import kotlinx.coroutines.internal.* +import kotlinx.coroutines.selects.* +import kotlinx.coroutines.selects.TrySelectDetailedResult.* +import kotlin.contracts.* +import kotlin.coroutines.* +import kotlin.js.* +import kotlin.jvm.* +import kotlin.math.* +import kotlin.random.* +import kotlin.reflect.* + +/** +The buffered channel implementation, which also serves as a rendezvous channel when the capacity is zero. +The high-level structure bases on a conceptually infinite array for storing elements and waiting requests, +separate counters of [send] and [receive] invocations that were ever performed, and an additional counter +that indicates the end of the logical buffer by counting the number of array cells it ever contained. +The key idea is that both [send] and [receive] start by incrementing their counters, assigning the array cell +referenced by the counter. In case of rendezvous channels, the operation either suspends and stores its continuation +in the cell or makes a rendezvous with the opposite request. Each cell can be processed by exactly one [send] and +one [receive]. As for buffered channels, [send]-s can also add elements without suspension if the logical buffer +contains the cell, while the [receive] operation updates the end of the buffer when its synchronization finishes. + +Please see the ["Fast and Scalable Channels in Kotlin Coroutines"](https://arxiv.org/abs/2211.04986) +paper by Nikita Koval, Roman Elizarov, and Dan Alistarh for the detailed algorithm description. + */ +internal open class BufferedChannel( + /** + * Channel capacity; `Channel.RENDEZVOUS` for rendezvous channel + * and `Channel.UNLIMITED` for unlimited capacity. + */ + private val capacity: Int, + @JvmField + internal val onUndeliveredElement: OnUndeliveredElement? = null +) : Channel { + init { + require(capacity >= 0) { "Invalid channel capacity: $capacity, should be >=0" } + // This implementation has second `init`. + } + + // Maintenance note: use `Buffered1ChannelLincheckTest` to check hypotheses. + + /* + The counters indicate the total numbers of send, receive, and buffer expansion calls + ever performed. The counters are incremented in the beginning of the corresponding + operation; thus, acquiring a unique (for the operation type) cell to process. + The segments reference to the last working one for each operation type. + + Notably, the counter for send is combined with the channel closing status + for synchronization simplicity and performance reasons. + + The logical end of the buffer is initialized with the channel capacity. + If the channel is rendezvous or unlimited, the counter equals `BUFFER_END_RENDEZVOUS` + or `BUFFER_END_RENDEZVOUS`, respectively, and never updates. The `bufferEndSegment` + point to a special `NULL_SEGMENT` in this case. + */ + private val sendersAndCloseStatus = atomic(0L) + private val receivers = atomic(0L) + private val bufferEnd = atomic(initialBufferEnd(capacity)) + + internal val sendersCounter: Long get() = sendersAndCloseStatus.value.sendersCounter + internal val receiversCounter: Long get() = receivers.value + private val bufferEndCounter: Long get() = bufferEnd.value + + /* + Additionally to the counters above, we need an extra one that + tracks the number of cells processed by `expandBuffer()`. + When a receiver aborts, the corresponding cell might be + physically removed from the data structure to avoid memory + leaks, while it still can be unprocessed by `expandBuffer()`. + In this case, `expandBuffer()` cannot know whether the + removed cell contained sender or receiver and, therefore, + cannot proceed. To solve the race, we ensure that cells + correspond to cancelled receivers cannot be physically + removed until the cell is processed. + This additional counter enables the synchronization, + */ + private val completedExpandBuffersAndPauseFlag = atomic(bufferEndCounter) + + private val isRendezvousOrUnlimited + get() = bufferEndCounter.let { it == BUFFER_END_RENDEZVOUS || it == BUFFER_END_UNLIMITED } + + private val sendSegment: AtomicRef> + private val receiveSegment: AtomicRef> + private val bufferEndSegment: AtomicRef> + + init { + @Suppress("LeakingThis") + val firstSegment = ChannelSegment(id = 0, prev = null, channel = this, pointers = 3) + sendSegment = atomic(firstSegment) + receiveSegment = atomic(firstSegment) + // If this channel is rendezvous or has unlimited capacity, the algorithm never + // invokes the buffer expansion procedure, and the corresponding segment reference + // points to a special `NULL_SEGMENT` one and never updates. + @Suppress("UNCHECKED_CAST") + bufferEndSegment = atomic(if (isRendezvousOrUnlimited) (NULL_SEGMENT as ChannelSegment) else firstSegment) + } + + // ######################### + // ## The send operations ## + // ######################### + + override suspend fun send(element: E): Unit = + sendImpl( // <-- this is an inline function + element = element, + // Do not create a continuation until it is required; + // it is created later via [onNoWaiterSuspend], if needed. + waiter = null, + // Finish immediately if a rendezvous happens + // or the element has been buffered. + onRendezvousOrBuffered = {}, + // As no waiter is provided, suspension is impossible. + onSuspend = { _, _ -> assert { false } }, + // According to the `send(e)` contract, we need to call + // `onUndeliveredElement(..)` handler and throw an exception + // if the channel is already closed. + onClosed = { onClosedSend(element) }, + // When `send(e)` decides to suspend, the corresponding + // `onNoWaiterSuspend` function that creates a continuation + // is called. The tail-call optimization is applied here. + onNoWaiterSuspend = { segm, i, elem, s -> sendOnNoWaiterSuspend(segm, i, elem, s) } + ) + + private suspend fun onClosedSend(element: E): Nothing = suspendCancellableCoroutine { continuation -> + onUndeliveredElement?.callUndeliveredElementCatchingException(element)?.let { + // If it crashes, add send exception as suppressed for better diagnostics + it.addSuppressed(sendException) + continuation.resumeWithStackTrace(it) + return@suspendCancellableCoroutine + } + continuation.resumeWithStackTrace(sendException) + } + + private suspend fun sendOnNoWaiterSuspend( + /* The working cell is specified by + the segment and the index in it. */ + segment: ChannelSegment, + index: Int, + /** The element to be inserted. */ + element: E, + /** The global index of the cell. */ + s: Long + ) = suspendCancellableCoroutineReusable sc@{ cont -> + sendImplOnNoWaiter( // <-- this is an inline function + segment = segment, index = index, element = element, s = s, + // Store the created continuation as a waiter. + waiter = cont, + // If a rendezvous happens or the element has been buffered, + // resume the continuation and finish. In case of prompt + // cancellation, it is guaranteed that the element + // has been already buffered or passed to receiver. + onRendezvousOrBuffered = { cont.resume(Unit) }, + // Clean the cell on suspension and invoke + // `onUndeliveredElement(..)` if needed. + onSuspend = { segm, i -> cont.prepareSenderForSuspension(segm, i) }, + // If the channel is closed, call `onUndeliveredElement(..)` and complete the + // continuation with the corresponding exception. + onClosed = { onClosedSendOnNoWaiterSuspend(element, cont) }, + ) + } + + private fun CancellableContinuation<*>.prepareSenderForSuspension( + /* The working cell is specified by + the segment and the index in it. */ + segment: ChannelSegment, + index: Int + ) { + if (onUndeliveredElement == null) { + invokeOnCancellation(SenderOrReceiverCancellationHandler(segment, index).asHandler) + } else { + invokeOnCancellation(SenderWithOnUndeliveredElementCancellationHandler(segment, index, context).asHandler) + } + } + + // TODO: Replace with a more efficient cancellation mechanism for segments when #3084 is finished. + private inner class SenderOrReceiverCancellationHandler( + private val segment: ChannelSegment, + private val index: Int + ) : BeforeResumeCancelHandler(), DisposableHandle { + override fun dispose() { + segment.onCancellation(index) + } + + override fun invoke(cause: Throwable?) = dispose() + } + + private inner class SenderWithOnUndeliveredElementCancellationHandler( + private val segment: ChannelSegment, + private val index: Int, + private val context: CoroutineContext + ) : BeforeResumeCancelHandler(), DisposableHandle { + override fun dispose() { + segment.onSenderCancellationWithOnUndeliveredElement(index, context) + } + + override fun invoke(cause: Throwable?) = dispose() + } + + private fun onClosedSendOnNoWaiterSuspend(element: E, cont: CancellableContinuation) { + onUndeliveredElement?.callUndeliveredElement(element, cont.context) + cont.resumeWithException(recoverStackTrace(sendException, cont)) + } + + override fun trySend(element: E): ChannelResult { + // Do not try to send the element if the plain `send(e)` operation would suspend. + if (shouldSendSuspend(sendersAndCloseStatus.value)) return failure() + // This channel either has waiting receivers or is closed. + // Let's try to send the element! + // The logic is similar to the plain `send(e)` operation, with + // the only difference that we install `INTERRUPTED_SEND` in case + // the operation decides to suspend. + return sendImpl( // <-- this is an inline function + element = element, + // Store an already interrupted sender in case of suspension. + waiter = INTERRUPTED_SEND, + // Finish successfully when a rendezvous happens + // or the element has been buffered. + onRendezvousOrBuffered = { success(Unit) }, + // On suspension, the `INTERRUPTED_SEND` token has been installed, + // and this `trySend(e)` fails. According to the contract, + // we do not need to call [onUndeliveredElement] handler. + onSuspend = { segm, _ -> + segm.onSlotCleaned() + failure() + }, + // If the channel is closed, return the corresponding result. + onClosed = { closed(sendException) } + ) + } + + /** + * This is a special `send(e)` implementation that returns `true` if the element + * has been successfully sent, and `false` if the channel is closed. + * + * In case of coroutine cancellation, the element may be undelivered -- + * the [onUndeliveredElement] feature is unsupported in this implementation. + * + * Note that this implementation always invokes [suspendCancellableCoroutineReusable], + * as we do not care about broadcasts performance -- they are already deprecated. + */ + internal open suspend fun sendBroadcast(element: E): Boolean = suspendCancellableCoroutineReusable { cont -> + check(onUndeliveredElement == null) { + "the `onUndeliveredElement` feature is unsupported for `sendBroadcast(e)`" + } + sendImpl( + element = element, + waiter = SendBroadcast(cont), + onRendezvousOrBuffered = { cont.resume(true) }, + onSuspend = { segm, i -> cont.prepareSenderForSuspension(segm, i) }, + onClosed = { cont.resume(false) } + ) + } + + /** + * Specifies waiting [sendBroadcast] operation. + */ + private class SendBroadcast(val cont: CancellableContinuation) : Waiter + + /** + * Abstract send implementation. + */ + protected inline fun sendImpl( + /* The element to be sent. */ + element: E, + /* The waiter to be stored in case of suspension, + or `null` if the waiter is not created yet. + In the latter case, when the algorithm decides + to suspend, [onNoWaiterSuspend] is called. */ + waiter: Any?, + /* This lambda is invoked when the element has been + buffered or a rendezvous with a receiver happens. */ + onRendezvousOrBuffered: () -> R, + /* This lambda is called when the operation suspends in the + cell specified by the segment and the index in it. */ + onSuspend: (segm: ChannelSegment, i: Int) -> R, + /* This lambda is called when the channel + is observed in the closed state. */ + onClosed: () -> R, + /* This lambda is called when the operation decides + to suspend, but the waiter is not provided (equals `null`). + It should create a waiter and delegate to `sendImplOnNoWaiter`. */ + onNoWaiterSuspend: ( + segm: ChannelSegment, + i: Int, + element: E, + s: Long + ) -> R = { _, _, _, _ -> error("unexpected") } + ): R { + // Read the segment reference before the counter increment; + // it is crucial to be able to find the required segment later. + var segment = sendSegment.value + while (true) { + // Atomically increment the `senders` counter and obtain the + // value right before the increment along with the close status. + val sendersAndCloseStatusCur = sendersAndCloseStatus.getAndIncrement() + val s = sendersAndCloseStatusCur.sendersCounter + // Is this channel already closed? Keep the information. + val closed = sendersAndCloseStatusCur.isClosedForSend0 + // Count the required segment id and the cell index in it. + val id = s / SEGMENT_SIZE + val i = (s % SEGMENT_SIZE).toInt() + // Try to find the required segment if the initially obtained + // one (in the beginning of this function) has lower id. + if (segment.id != id) { + // Find the required segment. + segment = findSegmentSend(id, segment) ?: + // The required segment has not been found. + // Finish immediately if this channel is closed, + // restarting the operation otherwise. + // In the latter case, the required segment was full + // of interrupted waiters and, therefore, removed + // physically to avoid memory leaks. + if (closed) { + return onClosed() + } else { + continue + } + } + // Update the cell according to the algorithm. Importantly, when + // the channel is already closed, storing a waiter is illegal, so + // the algorithm stores the `INTERRUPTED_SEND` token in this case. + when (updateCellSend(segment, i, element, s, waiter, closed)) { + RESULT_RENDEZVOUS -> { + // A rendezvous with a receiver has happened. + // The previous segments are no longer needed + // for the upcoming requests, so the algorithm + // resets the link to the previous segment. + segment.cleanPrev() + return onRendezvousOrBuffered() + } + RESULT_BUFFERED -> { + // The element has been buffered. + return onRendezvousOrBuffered() + } + RESULT_SUSPEND -> { + // The operation has decided to suspend and installed the + // specified waiter. If the channel was already closed, + // and the `INTERRUPTED_SEND` token has been installed as a waiter, + // this request finishes with the `onClosed()` action. + if (closed) { + segment.onSlotCleaned() + return onClosed() + } + return onSuspend(segment, i) + } + RESULT_CLOSED -> { + // This channel is closed. + // In case this segment is already or going to be + // processed by a receiver, ensure that all the + // previous segments are unreachable. + if (s < receiversCounter) segment.cleanPrev() + return onClosed() + } + RESULT_FAILED -> { + // Either the cell stores an interrupted receiver, + // or it was poisoned by a concurrent receiver. + // In both cases, all the previous segments are already processed, + segment.cleanPrev() + continue + } + RESULT_SUSPEND_NO_WAITER -> { + // The operation has decided to suspend, + // but no waiter has been provided. + return onNoWaiterSuspend(segment, i, element, s) + } + } + } + } + + private inline fun sendImplOnNoWaiter( + /* The working cell is specified by + the segment and the index in it. */ + segment: ChannelSegment, + index: Int, + /* The element to be sent. */ + element: E, + /* The global index of the cell. */ + s: Long, + /* The waiter to be stored in case of suspension. */ + waiter: Any, + /* This lambda is invoked when the element has been + buffered or a rendezvous with a receiver happens.*/ + onRendezvousOrBuffered: () -> R, + /* This lambda is called when the operation suspends in the + cell specified by the segment and the index in it. */ + onSuspend: (segm: ChannelSegment, i: Int) -> R, + /* This lambda is called when the channel + is observed in the closed state. */ + onClosed: () -> R, + ): R = + // Update the cell again, now with the non-null waiter, + // restarting the operation from the beginning on failure. + // Check the `sendImpl(..)` function for the comments. + when(updateCellSend(segment, index, element, s, waiter, false)) { + RESULT_RENDEZVOUS -> { + segment.cleanPrev() + onRendezvousOrBuffered() + } + RESULT_BUFFERED -> { + onRendezvousOrBuffered() + } + RESULT_SUSPEND -> { + onSuspend(segment, index) + } + RESULT_CLOSED -> { + if (s < receiversCounter) segment.cleanPrev() + onClosed() + } + RESULT_FAILED -> { + segment.cleanPrev() + sendImpl( + element = element, + waiter = waiter, + onRendezvousOrBuffered = onRendezvousOrBuffered, + onSuspend = onSuspend, + onClosed = onClosed, + ) + } + else -> error("unexpected") + } + + private fun updateCellSend( + /* The working cell is specified by + the segment and the index in it. */ + segment: ChannelSegment, + index: Int, + /* The element to be sent. */ + element: E, + /* The global index of the cell. */ + s: Long, + /* The waiter to be stored in case of suspension. */ + waiter: Any?, + closed: Boolean + ): Int { + // This is a fast-path of `updateCellSendSlow(..)`. + // + // First, the algorithm stores the element, + // performing the synchronization after that. + // This way, receivers safely retrieve the + // element, following the safe publication pattern. + segment.storeElement(index, element) + if (closed) return updateCellSendSlow(segment, index, element, s, waiter, closed) + // Read the current cell state. + val state = segment.getState(index) + when { + // The cell is empty. + state === null -> { + // If the element should be buffered, or a rendezvous should happen + // while the receiver is still coming, try to buffer the element. + // Otherwise, try to store the specified waiter in the cell. + if (bufferOrRendezvousSend(s)) { + // Move the cell state to `BUFFERED`. + if (segment.casState(index, null, BUFFERED)) { + // The element has been successfully buffered, finish. + return RESULT_BUFFERED + } + } else { + // This `send(e)` operation should suspend. + // However, in case the channel has already + // been observed closed, `INTERRUPTED_SEND` + // is installed instead. + if (waiter == null) { + // The waiter is not specified; return the corresponding result. + return RESULT_SUSPEND_NO_WAITER + } else { + // Try to install the waiter. + if (segment.casState(index, null, waiter)) return RESULT_SUSPEND + } + } + } + // A waiting receiver is stored in the cell. + state is Waiter -> { + // As the element will be passed directly to the waiter, + // the algorithm cleans the element slot in the cell. + segment.cleanElement(index) + // Try to make a rendezvous with the suspended receiver. + return if (state.tryResumeReceiver(element)) { + // Rendezvous! Move the cell state to `DONE_RCV` and finish. + segment.setState(index, DONE_RCV) + onReceiveDequeued() + RESULT_RENDEZVOUS + } else { + // The resumption has failed. Update the cell state correspondingly + // and clean the element field. It is also possible for a concurrent + // cancellation handler to update the cell state; we can safely + // ignore these updates. + if (segment.getAndSetState(index, INTERRUPTED_RCV) !== INTERRUPTED_RCV) { + segment.onCancelledRequest(index, true) + } + RESULT_FAILED + } + } + } + return updateCellSendSlow(segment, index, element, s, waiter, closed) + } + + /** + * Updates the working cell of an abstract send operation. + */ + private fun updateCellSendSlow( + /* The working cell is specified by + the segment and the index in it. */ + segment: ChannelSegment, + index: Int, + /* The element to be sent. */ + element: E, + /* The global index of the cell. */ + s: Long, + /* The waiter to be stored in case of suspension. */ + waiter: Any?, + closed: Boolean + ): Int { + // Then, the cell state should be updated according to + // its state machine; see the paper mentioned in the very + // beginning for the cell life-cycle and the algorithm details. + while (true) { + // Read the current cell state. + val state = segment.getState(index) + when { + // The cell is empty. + state === null -> { + // If the element should be buffered, or a rendezvous should happen + // while the receiver is still coming, try to buffer the element. + // Otherwise, try to store the specified waiter in the cell. + if (bufferOrRendezvousSend(s) && !closed) { + // Move the cell state to `BUFFERED`. + if (segment.casState(index, null, BUFFERED)) { + // The element has been successfully buffered, finish. + return RESULT_BUFFERED + } + } else { + // This `send(e)` operation should suspend. + // However, in case the channel has already + // been observed closed, `INTERRUPTED_SEND` + // is installed instead. + when { + // The channel is closed + closed -> if (segment.casState(index, null, INTERRUPTED_SEND)) { + segment.onCancelledRequest(index, false) + return RESULT_CLOSED + } + // The waiter is not specified; return the corresponding result. + waiter == null -> return RESULT_SUSPEND_NO_WAITER + // Try to install the waiter. + else -> if (segment.casState(index, null, waiter)) return RESULT_SUSPEND + } + } + } + // This cell is in the logical buffer. + state === IN_BUFFER -> { + // Try to buffer the element. + if (segment.casState(index, state, BUFFERED)) { + // The element has been successfully buffered, finish. + return RESULT_BUFFERED + } + } + // The cell stores a cancelled receiver. + state === INTERRUPTED_RCV -> { + // Clean the element slot to avoid memory leaks and finish. + segment.cleanElement(index) + return RESULT_FAILED + } + // The cell is poisoned by a concurrent receive. + state === POISONED -> { + // Clean the element slot to avoid memory leaks and finish. + segment.cleanElement(index) + return RESULT_FAILED + } + // The channel is already closed. + state === CHANNEL_CLOSED -> { + // Clean the element slot to avoid memory leaks, + // ensure that the closing/cancellation procedure + // has been completed, and finish. + segment.cleanElement(index) + completeCloseOrCancel() + return RESULT_CLOSED + } + // A waiting receiver is stored in the cell. + else -> { + assert { state is Waiter || state is WaiterEB } + // As the element will be passed directly to the waiter, + // the algorithm cleans the element slot in the cell. + segment.cleanElement(index) + // Unwrap the waiting receiver from `WaiterEB` if needed. + // As a receiver is stored in the cell, the buffer expansion + // procedure would finish, so senders simply ignore the "EB" marker. + val receiver = if (state is WaiterEB) state.waiter else state + // Try to make a rendezvous with the suspended receiver. + return if (receiver.tryResumeReceiver(element)) { + // Rendezvous! Move the cell state to `DONE_RCV` and finish. + segment.setState(index, DONE_RCV) + onReceiveDequeued() + RESULT_RENDEZVOUS + } else { + // The resumption has failed. Update the cell state correspondingly + // and clean the element field. It is also possible for a concurrent + // `expandBuffer()` or the cancellation handler to update the cell state; + // we can safely ignore these updates as senders do not help `expandBuffer()`. + if (segment.getAndSetState(index, INTERRUPTED_RCV) !== INTERRUPTED_RCV) { + segment.onCancelledRequest(index, true) + } + RESULT_FAILED + } + } + } + } + } + + /** + * Checks whether a [send] invocation is bound to suspend if it is called + * with the specified [sendersAndCloseStatus], [receivers], and [bufferEnd] + * values. When this channel is already closed, the function returns `false`. + * + * Specifically, [send] suspends if the channel is not unlimited, + * the number of receivers is greater than then index of the working cell of the + * potential [send] invocation, and the buffer does not cover this cell + * in case of buffered channel. + * When the channel is already closed, [send] does not suspend. + */ + @JsName("shouldSendSuspend0") + private fun shouldSendSuspend(curSendersAndCloseStatus: Long): Boolean { + // Does not suspend if the channel is already closed. + if (curSendersAndCloseStatus.isClosedForSend0) return false + // Does not suspend if a rendezvous may happen or the buffer is not full. + return !bufferOrRendezvousSend(curSendersAndCloseStatus.sendersCounter) + } + + /** + * Returns `true` when the specified [send] should place + * its element to the working cell without suspension. + */ + private fun bufferOrRendezvousSend(curSenders: Long): Boolean = + curSenders < bufferEndCounter || curSenders < receiversCounter + capacity + + /** + * Checks whether a [send] invocation is bound to suspend if it is called + * with the current counter and close status values. See [shouldSendSuspend] for details. + * + * Note that this implementation is _false positive_ in case of rendezvous channels, + * so it can return `false` when a [send] invocation is bound to suspend. Specifically, + * the counter of `receive()` operations may indicate that there is a waiting receiver, + * while it has already been cancelled, so the potential rendezvous is bound to fail. + */ + internal open fun shouldSendSuspend(): Boolean = shouldSendSuspend(sendersAndCloseStatus.value) + + /** + * Tries to resume this receiver with the specified [element] as a result. + * Returns `true` on success and `false` otherwise. + */ + @Suppress("UNCHECKED_CAST") + private fun Any.tryResumeReceiver(element: E): Boolean = when(this) { + is SelectInstance<*> -> { // `onReceiveXXX` select clause + trySelect(this@BufferedChannel, element) + } + is ReceiveCatching<*> -> { + this as ReceiveCatching + cont.tryResume0(success(element), onUndeliveredElement?.bindCancellationFun(element, cont.context)) + } + is BufferedChannel<*>.BufferedChannelIterator -> { + this as BufferedChannel.BufferedChannelIterator + tryResumeHasNext(element) + } + is CancellableContinuation<*> -> { // `receive()` + this as CancellableContinuation + tryResume0(element, onUndeliveredElement?.bindCancellationFun(element, context)) + } + else -> error("Unexpected receiver type: $this") + } + + // ########################## + // # The receive operations # + // ########################## + + /** + * This function is invoked when a receiver is added as a waiter in this channel. + */ + protected open fun onReceiveEnqueued() {} + + /** + * This function is invoked when a waiting receiver is no longer stored in this channel; + * independently on whether it is caused by rendezvous, cancellation, or channel closing. + */ + protected open fun onReceiveDequeued() {} + + override suspend fun receive(): E = + receiveImpl( // <-- this is an inline function + // Do not create a continuation until it is required; + // it is created later via [onNoWaiterSuspend], if needed. + waiter = null, + // Return the received element on successful retrieval from + // the buffer or rendezvous with a suspended sender. + // Also, inform `BufferedChannel` extensions that + // synchronization of this receive operation is completed. + onElementRetrieved = { element -> + return element + }, + // As no waiter is provided, suspension is impossible. + onSuspend = { _, _, _ -> error("unexpected") }, + // Throw an exception if the channel is already closed. + onClosed = { throw recoverStackTrace(receiveException) }, + // If `receive()` decides to suspend, the corresponding + // `suspend` function that creates a continuation is called. + // The tail-call optimization is applied here. + onNoWaiterSuspend = { segm, i, r -> receiveOnNoWaiterSuspend(segm, i, r) } + ) + + private suspend fun receiveOnNoWaiterSuspend( + /* The working cell is specified by + the segment and the index in it. */ + segment: ChannelSegment, + index: Int, + /* The global index of the cell. */ + r: Long + ) = suspendCancellableCoroutineReusable { cont -> + receiveImplOnNoWaiter( // <-- this is an inline function + segment = segment, index = index, r = r, + // Store the created continuation as a waiter. + waiter = cont, + // In case of successful element retrieval, resume + // the continuation with the element and inform the + // `BufferedChannel` extensions that the synchronization + // is completed. Importantly, the receiver coroutine + // may be cancelled after it is successfully resumed but + // not dispatched yet. In case `onUndeliveredElement` is + // specified, we need to invoke it in the latter case. + onElementRetrieved = { element -> + val onCancellation = onUndeliveredElement?.bindCancellationFun(element, cont.context) + cont.resume(element, onCancellation) + }, + onSuspend = { segm, i, _ -> cont.prepareReceiverForSuspension(segm, i) }, + onClosed = { onClosedReceiveOnNoWaiterSuspend(cont) }, + ) + } + + private fun CancellableContinuation<*>.prepareReceiverForSuspension(segment: ChannelSegment, index: Int) { + onReceiveEnqueued() + invokeOnCancellation(SenderOrReceiverCancellationHandler(segment, index).asHandler) + } + + private fun onClosedReceiveOnNoWaiterSuspend(cont: CancellableContinuation) { + cont.resumeWithException(receiveException) + } + + /* + The implementation is exactly the same as of `receive()`, + with the only difference that this function returns a `ChannelResult` + instance and does not throw exception explicitly in case the channel + is already closed for receiving. Please refer the plain `receive()` + implementation for the comments. + */ + override suspend fun receiveCatching(): ChannelResult = + receiveImpl( // <-- this is an inline function + waiter = null, + onElementRetrieved = { element -> + success(element) + }, + onSuspend = { _, _, _ -> error("unexpected") }, + onClosed = { closed(closeCause) }, + onNoWaiterSuspend = { segm, i, r -> receiveCatchingOnNoWaiterSuspend(segm, i, r) } + ) + + private suspend fun receiveCatchingOnNoWaiterSuspend( + segment: ChannelSegment, + index: Int, + r: Long + ) = suspendCancellableCoroutineReusable> { cont -> + val waiter = ReceiveCatching(cont) + receiveImplOnNoWaiter( + segment, index, r, + waiter = waiter, + onElementRetrieved = { element -> + cont.resume(success(element), onUndeliveredElement?.bindCancellationFun(element, cont.context)) + }, + onSuspend = { segm, i, _ -> cont.prepareReceiverForSuspension(segm, i) }, + onClosed = { onClosedReceiveCatchingOnNoWaiterSuspend(cont) } + ) + } + + private fun onClosedReceiveCatchingOnNoWaiterSuspend(cont: CancellableContinuation>) { + cont.resume(closed(closeCause)) + } + + override fun tryReceive(): ChannelResult { + // Read the `receivers` counter first. + val r = receivers.value + val sendersAndCloseStatusCur = sendersAndCloseStatus.value + // Is this channel closed for receive? + if (sendersAndCloseStatusCur.isClosedForReceive0) { + return closed(closeCause) + } + // Do not try to receive an element if the plain `receive()` operation would suspend. + val s = sendersAndCloseStatusCur.sendersCounter + if (r >= s) return failure() + // Let's try to retrieve an element! + // The logic is similar to the plain `receive()` operation, with + // the only difference that we store `INTERRUPTED_RCV` in case + // the operation decides to suspend. This way, we can leverage + // the unconditional `Fetch-and-Add` instruction. + // One may consider storing `INTERRUPTED_RCV` instead of an actual waiter + // on suspension (a.k.a. "no elements to retrieve") as a short-cut of + // "suspending and cancelling immediately". + return receiveImpl( // <-- this is an inline function + // Store an already interrupted receiver in case of suspension. + waiter = INTERRUPTED_RCV, + // Finish when an element is successfully retrieved. + onElementRetrieved = { element -> success(element) }, + // On suspension, the `INTERRUPTED_RCV` token has been + // installed, and this `tryReceive()` fails. + onSuspend = { segm, _, globalIndex -> + // Emulate "cancelled" receive, thus invoking 'waitExpandBufferCompletion' manually, + // because effectively there were no cancellation + waitExpandBufferCompletion(globalIndex) + segm.onSlotCleaned() + failure() + }, + // If the channel is closed, return the corresponding result. + onClosed = { closed(closeCause) } + ) + } + + /** + * Extracts the first element from this channel until the cell with the specified + * index is moved to the logical buffer. This is a key procedure for the _conflated_ + * channel implementation, see [ConflatedBufferedChannel] with the [BufferOverflow.DROP_OLDEST] + * strategy on buffer overflowing. + */ + protected fun dropFirstElementUntilTheSpecifiedCellIsInTheBuffer(globalCellIndex: Long) { + assert { isConflatedDropOldest } + // Read the segment reference before the counter increment; + // it is crucial to be able to find the required segment later. + var segment = receiveSegment.value + while (true) { + // Read the receivers counter to check whether the specified cell is already in the buffer + // or should be moved to the buffer in a short time, due to the already started `receive()`. + val r = this.receivers.value + if (globalCellIndex < max(r + capacity, bufferEndCounter)) return + // The cell is outside the buffer. Try to extract the first element + // if the `receivers` counter has not been changed. + if (!this.receivers.compareAndSet(r, r + 1)) continue + // Count the required segment id and the cell index in it. + val id = r / SEGMENT_SIZE + val i = (r % SEGMENT_SIZE).toInt() + // Try to find the required segment if the initially obtained + // segment (in the beginning of this function) has lower id. + if (segment.id != id) { + // Find the required segment, restarting the operation if it has not been found. + segment = findSegmentReceive(id, segment) ?: + // The required segment has not been found. It is possible that the channel is already + // closed for receiving, so the linked list of segments is closed as well. + // In the latter case, the operation will finish eventually after incrementing + // the `receivers` counter sufficient times. Note that it is impossible to check + // whether this channel is closed for receiving (we do this in `receive`), + // as it may call this function when helping to complete closing the channel. + continue + } + // Update the cell according to the cell life-cycle. + val updCellResult = updateCellReceive(segment, i, r, null) + when { + updCellResult === FAILED -> { + // The cell is poisoned; restart from the beginning. + // To avoid memory leaks, we also need to reset + // the `prev` pointer of the working segment. + if (r < sendersCounter) segment.cleanPrev() + } + else -> { // element + // A buffered element was retrieved from the cell. + // Clean the reference to the previous segment. + segment.cleanPrev() + @Suppress("UNCHECKED_CAST") + onUndeliveredElement?.callUndeliveredElementCatchingException(updCellResult as E)?.let { throw it } + } + } + } + } + + /** + * Abstract receive implementation. + */ + private inline fun receiveImpl( + /* The waiter to be stored in case of suspension, + or `null` if the waiter is not created yet. + In the latter case, if the algorithm decides + to suspend, [onNoWaiterSuspend] is called. */ + waiter: Any?, + /* This lambda is invoked when an element has been + successfully retrieved, either from the buffer or + by making a rendezvous with a suspended sender. */ + onElementRetrieved: (element: E) -> R, + /* This lambda is called when the operation suspends in the cell + specified by the segment and its global and in-segment indices. */ + onSuspend: (segm: ChannelSegment, i: Int, r: Long) -> R, + /* This lambda is called when the channel is observed + in the closed state and no waiting sender is found, + which means that it is closed for receiving. */ + onClosed: () -> R, + /* This lambda is called when the operation decides + to suspend, but the waiter is not provided (equals `null`). + It should create a waiter and delegate to `sendImplOnNoWaiter`. */ + onNoWaiterSuspend: ( + segm: ChannelSegment, + i: Int, + r: Long + ) -> R = { _, _, _ -> error("unexpected") } + ): R { + // Read the segment reference before the counter increment; + // it is crucial to be able to find the required segment later. + var segment = receiveSegment.value + while (true) { + // Similar to the `send(e)` operation, `receive()` first checks + // whether the channel is already closed for receiving. + if (isClosedForReceive) return onClosed() + // Atomically increments the `receivers` counter + // and obtain the value right before the increment. + val r = this.receivers.getAndIncrement() + // Count the required segment id and the cell index in it. + val id = r / SEGMENT_SIZE + val i = (r % SEGMENT_SIZE).toInt() + // Try to find the required segment if the initially obtained + // segment (in the beginning of this function) has lower id. + if (segment.id != id) { + // Find the required segment, restarting the operation if it has not been found. + segment = findSegmentReceive(id, segment) ?: + // The required segment is not found. It is possible that the channel is already + // closed for receiving, so the linked list of segments is closed as well. + // In the latter case, the operation fails with the corresponding check at the beginning. + continue + } + // Update the cell according to the cell life-cycle. + val updCellResult = updateCellReceive(segment, i, r, waiter) + return when { + updCellResult === SUSPEND -> { + // The operation has decided to suspend and + // stored the specified waiter in the cell. + onSuspend(segment, i, r) + } + updCellResult === FAILED -> { + // The operation has tried to make a rendezvous + // but failed: either the opposite request has + // already been cancelled or the cell is poisoned. + // Restart from the beginning in this case. + // To avoid memory leaks, we also need to reset + // the `prev` pointer of the working segment. + if (r < sendersCounter) segment.cleanPrev() + continue + } + updCellResult === SUSPEND_NO_WAITER -> { + // The operation has decided to suspend, + // but no waiter has been provided. + onNoWaiterSuspend(segment, i, r) + } + else -> { // element + // Either a buffered element was retrieved from the cell + // or a rendezvous with a waiting sender has happened. + // Clean the reference to the previous segment before finishing. + segment.cleanPrev() + @Suppress("UNCHECKED_CAST") + onElementRetrieved(updCellResult as E) + } + } + } + } + + private inline fun receiveImplOnNoWaiter( + /* The working cell is specified by + the segment and the index in it. */ + segment: ChannelSegment, + index: Int, + /* The global index of the cell. */ + r: Long, + /* The waiter to be stored in case of suspension. */ + waiter: W, + /* This lambda is invoked when an element has been + successfully retrieved, either from the buffer or + by making a rendezvous with a suspended sender. */ + onElementRetrieved: (element: E) -> R, + /* This lambda is called when the operation suspends in the cell + specified by the segment and its global and in-segment indices. */ + onSuspend: (segm: ChannelSegment, i: Int, r: Long) -> R, + /* This lambda is called when the channel is observed + in the closed state and no waiting senders is found, + which means that it is closed for receiving. */ + onClosed: () -> R + ): R { + // Update the cell with the non-null waiter, + // restarting from the beginning on failure. + // Check the `receiveImpl(..)` function for the comments. + val updCellResult = updateCellReceive(segment, index, r, waiter) + when { + updCellResult === SUSPEND -> { + return onSuspend(segment, index, r) + } + updCellResult === FAILED -> { + if (r < sendersCounter) segment.cleanPrev() + return receiveImpl( + waiter = waiter, + onElementRetrieved = onElementRetrieved, + onSuspend = onSuspend, + onClosed = onClosed + ) + } + else -> { + segment.cleanPrev() + @Suppress("UNCHECKED_CAST") + return onElementRetrieved(updCellResult as E) + } + } + } + + private fun updateCellReceive( + /* The working cell is specified by + the segment and the index in it. */ + segment: ChannelSegment, + index: Int, + /* The global index of the cell. */ + r: Long, + /* The waiter to be stored in case of suspension. */ + waiter: Any?, + ): Any? { + // This is a fast-path of `updateCellReceiveSlow(..)`. + // + // Read the current cell state. + val state = segment.getState(index) + when { + // The cell is empty. + state === null -> { + // If a rendezvous must happen, the operation does not wait + // until the cell stores a buffered element or a suspended + // sender, poisoning the cell and restarting instead. + // Otherwise, try to store the specified waiter in the cell. + val senders = sendersAndCloseStatus.value.sendersCounter + if (r >= senders) { + // This `receive()` operation should suspend. + if (waiter === null) { + // The waiter is not specified; + // return the corresponding result. + return SUSPEND_NO_WAITER + } + // Try to install the waiter. + if (segment.casState(index, state, waiter)) { + // The waiter has been successfully installed. + // Invoke the `expandBuffer()` procedure and finish. + expandBuffer() + return SUSPEND + } + } + } + // The cell stores a buffered element. + state === BUFFERED -> if (segment.casState(index, state, DONE_RCV)) { + // Retrieve the element and expand the buffer. + expandBuffer() + return segment.retrieveElement(index) + } + } + return updateCellReceiveSlow(segment, index, r, waiter) + } + + private fun updateCellReceiveSlow( + /* The working cell is specified by + the segment and the index in it. */ + segment: ChannelSegment, + index: Int, + /* The global index of the cell. */ + r: Long, + /* The waiter to be stored in case of suspension. */ + waiter: Any?, + ): Any? { + // The cell state should be updated according to its state machine; + // see the paper mentioned in the very beginning for the algorithm details. + while (true) { + // Read the current cell state. + val state = segment.getState(index) + when { + // The cell is empty. + state === null || state === IN_BUFFER -> { + // If a rendezvous must happen, the operation does not wait + // until the cell stores a buffered element or a suspended + // sender, poisoning the cell and restarting instead. + // Otherwise, try to store the specified waiter in the cell. + val senders = sendersAndCloseStatus.value.sendersCounter + if (r < senders) { + // The cell is already covered by sender, + // so a rendezvous must happen. Unfortunately, + // the cell is empty, so the operation poisons it. + if (segment.casState(index, state, POISONED)) { + // When the cell becomes poisoned, it is essentially + // the same as storing an already cancelled receiver. + // Thus, the `expandBuffer()` procedure should be invoked. + expandBuffer() + return FAILED + } + } else { + // This `receive()` operation should suspend. + if (waiter === null) { + // The waiter is not specified; + // return the corresponding result. + return SUSPEND_NO_WAITER + } + // Try to install the waiter. + if (segment.casState(index, state, waiter)) { + // The waiter has been successfully installed. + // Invoke the `expandBuffer()` procedure and finish. + expandBuffer() + return SUSPEND + } + } + } + // The cell stores a buffered element. + state === BUFFERED -> if (segment.casState(index, state, DONE_RCV)) { + // Retrieve the element and expand the buffer. + expandBuffer() + return segment.retrieveElement(index) + } + // The cell stores an interrupted sender. + state === INTERRUPTED_SEND -> return FAILED + // The cell is already poisoned by a concurrent + // `hasElements` call. Restart in this case. + state === POISONED -> return FAILED + // This channel is already closed. + state === CHANNEL_CLOSED -> { + // Although the channel is closed, it is still required + // to call the `expandBuffer()` procedure to keep + // `waitForExpandBufferCompletion()` correct. + expandBuffer() + return FAILED + } + // A concurrent `expandBuffer()` is resuming a + // suspended sender. Wait in a spin-loop until + // the resumption attempt completes: the cell + // state must change to either `BUFFERED` or + // `INTERRUPTED_SEND`. + state === RESUMING_BY_EB -> continue + // The cell stores a suspended sender; try to resume it. + else -> { + // To synchronize with expandBuffer(), the algorithm + // first moves the cell to an intermediate `S_RESUMING_BY_RCV` + // state, updating it to either `BUFFERED` (on success) or + // `INTERRUPTED_SEND` (on failure). + if (segment.casState(index, state, RESUMING_BY_RCV)) { + // Has a concurrent `expandBuffer()` delegated its completion? + val helpExpandBuffer = state is WaiterEB + // Extract the sender if needed and try to resume it. + val sender = if (state is WaiterEB) state.waiter else state + return if (sender.tryResumeSender(segment, index)) { + // The sender has been resumed successfully! + // Update the cell state correspondingly, + // expand the buffer, and return the element + // stored in the cell. + // In case a concurrent `expandBuffer()` has delegated + // its completion, the procedure should finish, as the + // sender is resumed. Thus, no further action is required. + segment.setState(index, DONE_RCV) + expandBuffer() + segment.retrieveElement(index) + } else { + // The resumption has failed. Update the cell correspondingly. + // In case a concurrent `expandBuffer()` has delegated + // its completion, the procedure should skip this cell, so + // `expandBuffer()` should be called once again. + segment.setState(index, INTERRUPTED_SEND) + segment.onCancelledRequest(index, false) + if (helpExpandBuffer) expandBuffer() + FAILED + } + } + } + } + } + } + + private fun Any.tryResumeSender(segment: ChannelSegment, index: Int): Boolean = when (this) { + is CancellableContinuation<*> -> { // suspended `send(e)` operation + @Suppress("UNCHECKED_CAST") + this as CancellableContinuation + tryResume0(Unit) + } + is SelectInstance<*> -> { + this as SelectImplementation<*> + val trySelectResult = trySelectDetailed(clauseObject = this@BufferedChannel, result = Unit) + // Clean the element slot to avoid memory leaks + // if this `select` clause should be re-registered. + if (trySelectResult === REREGISTER) segment.cleanElement(index) + // Was the resumption successful? + trySelectResult === SUCCESSFUL + } + is SendBroadcast -> cont.tryResume0(true) // // suspended `sendBroadcast(e)` operation + else -> error("Unexpected waiter: $this") + } + + // ################################ + // # The expandBuffer() procedure # + // ################################ + + private fun expandBuffer() { + // Do not need to take any action if + // this channel is rendezvous or unlimited. + if (isRendezvousOrUnlimited) return + // Read the current segment of + // the `expandBuffer()` procedure. + var segment = bufferEndSegment.value + // Try to expand the buffer until succeed. + try_again@ while (true) { + // Increment the logical end of the buffer. + // The `b`-th cell is going to be added to the buffer. + val b = bufferEnd.getAndIncrement() + val id = b / SEGMENT_SIZE + // After that, read the current `senders` counter. + // In case its value is lower than `b`, the `send(e)` + // invocation that will work with this `b`-th cell + // will detect that the cell is already a part of the + // buffer when comparing with the `bufferEnd` counter. + // However, `bufferEndSegment` may reference an outdated + // segment, which should be updated to avoid memory leaks. + val s = sendersCounter + if (s <= b) { + // Should `bufferEndSegment` be moved forward to avoid memory leaks? + if (segment.id < id && segment.next != null) + moveSegmentBufferEndToSpecifiedOrLast(id, segment) + // Increment the number of completed `expandBuffer()`-s and finish. + incCompletedExpandBufferAttempts() + return + } + // Is `bufferEndSegment` outdated? + // Find the required segment, creating new ones if needed. + if (segment.id < id) { + segment = findSegmentBufferEnd(id, segment, b) + // Restart if the required segment is removed, or + // the linked list of segments is already closed, + // and the required one will never be created. + // Please note that `findSegmentBuffer(..)` updates + // the number of completed `expandBuffer()` attempt + // in this case. + ?: continue@try_again + } + // Try to add the cell to the logical buffer, + // updating the cell state according to the state-machine. + val i = (b % SEGMENT_SIZE).toInt() + if (updateCellExpandBuffer(segment, i, b)) { + // The cell has been added to the logical buffer! + // Increment the number of completed `expandBuffer()`-s and finish. + // + // Note that it is possible to increment the number of + // completed `expandBuffer()` attempts earlier, right + // after the segment is obtained. We find this change + // counter-intuitive and prefer to avoid it. + incCompletedExpandBufferAttempts() + return + } else { + // The cell has not been added to the buffer. + // Increment the number of completed `expandBuffer()` + // attempts and restart. + incCompletedExpandBufferAttempts() + continue@try_again + } + } + } + + private fun updateCellExpandBuffer( + /* The working cell is specified by + the segment and the index in it. */ + segment: ChannelSegment, + index: Int, + /* The global index of the cell. */ + b: Long + ): Boolean { + // This is a fast-path of `updateCellExpandBufferSlow(..)`. + // + // Read the current cell state. + val state = segment.getState(index) + if (state is Waiter) { + // Usually, a sender is stored in the cell. + // However, it is possible for a concurrent + // receiver to be already suspended there. + // Try to distinguish whether the waiter is a + // sender by comparing the global cell index with + // the `receivers` counter. In case the cell is not + // covered by a receiver, a sender is stored in the cell. + if (b >= receivers.value) { + // The cell stores a suspended sender. Try to resume it. + // To synchronize with a concurrent `receive()`, the algorithm + // first moves the cell state to an intermediate `RESUMING_BY_EB` + // state, updating it to either `BUFFERED` (on successful resumption) + // or `INTERRUPTED_SEND` (on failure). + if (segment.casState(index, state, RESUMING_BY_EB)) { + return if (state.tryResumeSender(segment, index)) { + // The sender has been resumed successfully! + // Move the cell to the logical buffer and finish. + segment.setState(index, BUFFERED) + true + } else { + // The resumption has failed. + segment.setState(index, INTERRUPTED_SEND) + segment.onCancelledRequest(index, false) + false + } + } + } + } + return updateCellExpandBufferSlow(segment, index, b) + } + + private fun updateCellExpandBufferSlow( + /* The working cell is specified by + the segment and the index in it. */ + segment: ChannelSegment, + index: Int, + /* The global index of the cell. */ + b: Long + ): Boolean { + // Update the cell state according to its state machine. + // See the paper mentioned in the very beginning for + // the cell life-cycle and the algorithm details. + while (true) { + // Read the current cell state. + val state = segment.getState(index) + when { + // A suspended waiter, sender or receiver. + state is Waiter -> { + // Usually, a sender is stored in the cell. + // However, it is possible for a concurrent + // receiver to be already suspended there. + // Try to distinguish whether the waiter is a + // sender by comparing the global cell index with + // the `receivers` counter. In case the cell is not + // covered by a receiver, a sender is stored in the cell. + if (b < receivers.value) { + // The algorithm cannot distinguish whether the + // suspended in the cell operation is sender or receiver. + // To make progress, `expandBuffer()` delegates its completion + // to an upcoming pairwise request, atomically wrapping + // the waiter in `WaiterEB`. In case a sender is stored + // in the cell, the upcoming receiver will call `expandBuffer()` + // if the sender resumption fails; thus, effectively, skipping + // this cell. Otherwise, if a receiver is stored in the cell, + // this `expandBuffer()` procedure must finish; therefore, + // sender ignore the `WaiterEB` wrapper. + if (segment.casState(index, state, WaiterEB(waiter = state))) + return true + } else { + // The cell stores a suspended sender. Try to resume it. + // To synchronize with a concurrent `receive()`, the algorithm + // first moves the cell state to an intermediate `RESUMING_BY_EB` + // state, updating it to either `BUFFERED` (on successful resumption) + // or `INTERRUPTED_SEND` (on failure). + if (segment.casState(index, state, RESUMING_BY_EB)) { + return if (state.tryResumeSender(segment, index)) { + // The sender has been resumed successfully! + // Move the cell to the logical buffer and finish. + segment.setState(index, BUFFERED) + true + } else { + // The resumption has failed. + segment.setState(index, INTERRUPTED_SEND) + segment.onCancelledRequest(index, false) + false + } + } + } + } + // The cell stores an interrupted sender, skip it. + state === INTERRUPTED_SEND -> return false + // The cell is empty, a concurrent sender is coming. + state === null -> { + // To inform a concurrent sender that this cell is + // already a part of the buffer, the algorithm moves + // it to a special `IN_BUFFER` state. + if (segment.casState(index, state, IN_BUFFER)) return true + } + // The cell is already a part of the buffer, finish. + state === BUFFERED -> return true + // The cell is already processed by a receiver, no further action is required. + state === POISONED || state === DONE_RCV || state === INTERRUPTED_RCV -> return true + // The channel is closed, all the following + // cells are already in the same state, finish. + state === CHANNEL_CLOSED -> return true + // A concurrent receiver is resuming the suspended sender. + // Wait in a spin-loop until it changes the cell state + // to either `DONE_RCV` or `INTERRUPTED_SEND`. + state === RESUMING_BY_RCV -> continue // spin wait + else -> error("Unexpected cell state: $state") + } + } + } + + /** + * Increments the counter of completed [expandBuffer] invocations. + * To guarantee starvation-freedom for [waitExpandBufferCompletion], + * which waits until the counters of started and completed [expandBuffer] calls + * coincide and become greater or equal to the specified value, + * [waitExpandBufferCompletion] may set a flag that pauses further progress. + */ + private fun incCompletedExpandBufferAttempts(nAttempts: Long = 1) { + // Increment the number of completed `expandBuffer()` calls. + completedExpandBuffersAndPauseFlag.addAndGet(nAttempts).also { + // Should further `expandBuffer()`-s be paused? + // If so, this thread should wait in a spin-loop + // until the flag is unset. + if (it.ebPauseExpandBuffers) { + @Suppress("ControlFlowWithEmptyBody") + while (completedExpandBuffersAndPauseFlag.value.ebPauseExpandBuffers) {} + } + } + } + + /** + * Waits in a spin-loop until the [expandBuffer] call that + * should process the [globalIndex]-th cell is completed. + * Essentially, it waits until the numbers of started ([bufferEnd]) + * and completed ([completedExpandBuffersAndPauseFlag]) [expandBuffer] + * attempts coincide and become equal or greater than [globalIndex]. + * To avoid starvation, this function may set a flag + * that pauses further progress. + */ + internal fun waitExpandBufferCompletion(globalIndex: Long) { + // Do nothing if this channel is rendezvous or unlimited; + // `expandBuffer()` is not used in these cases. + if (isRendezvousOrUnlimited) return + // Wait in an infinite loop until the number of started + // buffer expansion calls become not lower than the cell index. + @Suppress("ControlFlowWithEmptyBody") + while (bufferEndCounter <= globalIndex) {} + // Now it is guaranteed that the `expandBuffer()` call that + // should process the required cell has been started. + // Wait in a fixed-size spin-loop until the numbers of + // started and completed buffer expansion calls coincide. + repeat(EXPAND_BUFFER_COMPLETION_WAIT_ITERATIONS) { + // Read the number of started buffer expansion calls. + val b = bufferEndCounter + // Read the number of completed buffer expansion calls. + val ebCompleted = completedExpandBuffersAndPauseFlag.value.ebCompletedCounter + // Do the numbers of started and completed calls coincide? + // Note that we need to re-read the number of started `expandBuffer()` + // calls to obtain a correct snapshot. + // Here we wait to a precise match in order to ensure that **our matching expandBuffer()** + // completed. The only way to ensure that is to check that number of started expands == number of finished expands + if (b == ebCompleted && b == bufferEndCounter) return + } + // To avoid starvation, pause further `expandBuffer()` calls. + completedExpandBuffersAndPauseFlag.update { + constructEBCompletedAndPauseFlag(it.ebCompletedCounter, true) + } + // Now wait in an infinite spin-loop until the counters coincide. + while (true) { + // Read the number of started buffer expansion calls. + val b = bufferEndCounter + // Read the number of completed buffer expansion calls + // along with the flag that pauses further progress. + val ebCompletedAndBit = completedExpandBuffersAndPauseFlag.value + val ebCompleted = ebCompletedAndBit.ebCompletedCounter + val pauseExpandBuffers = ebCompletedAndBit.ebPauseExpandBuffers + // Do the numbers of started and completed calls coincide? + // Note that we need to re-read the number of started `expandBuffer()` + // calls to obtain a correct snapshot. + if (b == ebCompleted && b == bufferEndCounter) { + // Unset the flag, which pauses progress, and finish. + completedExpandBuffersAndPauseFlag.update { + constructEBCompletedAndPauseFlag(it.ebCompletedCounter, false) + } + return + } + // It is possible that a concurrent caller of this function + // has unset the flag, which pauses further progress to avoid + // starvation. In this case, set the flag back. + if (!pauseExpandBuffers) { + completedExpandBuffersAndPauseFlag.compareAndSet( + ebCompletedAndBit, + constructEBCompletedAndPauseFlag(ebCompleted, true) + ) + } + } + } + + + // ####################### + // ## Select Expression ## + // ####################### + + @Suppress("UNCHECKED_CAST") + override val onSend: SelectClause2> + get() = SelectClause2Impl( + clauseObject = this@BufferedChannel, + regFunc = BufferedChannel<*>::registerSelectForSend as RegistrationFunction, + processResFunc = BufferedChannel<*>::processResultSelectSend as ProcessResultFunction + ) + + @Suppress("UNCHECKED_CAST") + protected open fun registerSelectForSend(select: SelectInstance<*>, element: Any?) = + sendImpl( // <-- this is an inline function + element = element as E, + waiter = select, + onRendezvousOrBuffered = { select.selectInRegistrationPhase(Unit) }, + onSuspend = { segm, i -> select.prepareSenderForSuspension(segm, i) }, + onClosed = { onClosedSelectOnSend(element, select) } + ) + + private fun SelectInstance<*>.prepareSenderForSuspension( + // The working cell is specified by + // the segment and the index in it. + segment: ChannelSegment, + index: Int + ) { + if (onUndeliveredElement == null) { + disposeOnCompletion(SenderOrReceiverCancellationHandler(segment, index)) + } else { + disposeOnCompletion(SenderWithOnUndeliveredElementCancellationHandler(segment, index, context)) + } + } + + private fun onClosedSelectOnSend(element: E, select: SelectInstance<*>) { + onUndeliveredElement?.callUndeliveredElement(element, select.context) + select.selectInRegistrationPhase(CHANNEL_CLOSED) + } + + @Suppress("UNUSED_PARAMETER", "RedundantNullableReturnType") + private fun processResultSelectSend(ignoredParam: Any?, selectResult: Any?): Any? = + if (selectResult === CHANNEL_CLOSED) throw sendException + else this + + @Suppress("UNCHECKED_CAST") + override val onReceive: SelectClause1 + get() = SelectClause1Impl( + clauseObject = this@BufferedChannel, + regFunc = BufferedChannel<*>::registerSelectForReceive as RegistrationFunction, + processResFunc = BufferedChannel<*>::processResultSelectReceive as ProcessResultFunction, + onCancellationConstructor = onUndeliveredElementReceiveCancellationConstructor + ) + + @Suppress("UNCHECKED_CAST") + override val onReceiveCatching: SelectClause1> + get() = SelectClause1Impl( + clauseObject = this@BufferedChannel, + regFunc = BufferedChannel<*>::registerSelectForReceive as RegistrationFunction, + processResFunc = BufferedChannel<*>::processResultSelectReceiveCatching as ProcessResultFunction, + onCancellationConstructor = onUndeliveredElementReceiveCancellationConstructor + ) + + @Suppress("OVERRIDE_DEPRECATION", "UNCHECKED_CAST") + override val onReceiveOrNull: SelectClause1 + get() = SelectClause1Impl( + clauseObject = this@BufferedChannel, + regFunc = BufferedChannel<*>::registerSelectForReceive as RegistrationFunction, + processResFunc = BufferedChannel<*>::processResultSelectReceiveOrNull as ProcessResultFunction, + onCancellationConstructor = onUndeliveredElementReceiveCancellationConstructor + ) + + @Suppress("UNUSED_PARAMETER") + private fun registerSelectForReceive(select: SelectInstance<*>, ignoredParam: Any?) = + receiveImpl( // <-- this is an inline function + waiter = select, + onElementRetrieved = { elem -> select.selectInRegistrationPhase(elem) }, + onSuspend = { segm, i, _ -> select.prepareReceiverForSuspension(segm, i) }, + onClosed = { onClosedSelectOnReceive(select) } + ) + + private fun SelectInstance<*>.prepareReceiverForSuspension( + /* The working cell is specified by + the segment and the index in it. */ + segment: ChannelSegment, + index: Int + ) { + onReceiveEnqueued() + disposeOnCompletion(SenderOrReceiverCancellationHandler(segment, index)) + } + + private fun onClosedSelectOnReceive(select: SelectInstance<*>) { + select.selectInRegistrationPhase(CHANNEL_CLOSED) + } + + @Suppress("UNUSED_PARAMETER") + private fun processResultSelectReceive(ignoredParam: Any?, selectResult: Any?): Any? = + if (selectResult === CHANNEL_CLOSED) throw receiveException + else selectResult + + @Suppress("UNUSED_PARAMETER") + private fun processResultSelectReceiveOrNull(ignoredParam: Any?, selectResult: Any?): Any? = + if (selectResult === CHANNEL_CLOSED) { + if (closeCause == null) null + else throw receiveException + } else selectResult + + @Suppress("UNCHECKED_CAST", "UNUSED_PARAMETER", "RedundantNullableReturnType") + private fun processResultSelectReceiveCatching(ignoredParam: Any?, selectResult: Any?): Any? = + if (selectResult === CHANNEL_CLOSED) closed(closeCause) + else success(selectResult as E) + + @Suppress("UNCHECKED_CAST") + private val onUndeliveredElementReceiveCancellationConstructor: OnCancellationConstructor? = onUndeliveredElement?.let { + { select: SelectInstance<*>, _: Any?, element: Any? -> + { if (element !== CHANNEL_CLOSED) onUndeliveredElement.callUndeliveredElement(element as E, select.context) } + } + } + + // ###################### + // ## Iterator Support ## + // ###################### + + override fun iterator(): ChannelIterator = BufferedChannelIterator() + + /** + * The key idea is that an iterator is a special receiver type, + * which should be resumed differently to [receive] and [onReceive] + * operations, but can be served as a waiter in a way similar to + * [CancellableContinuation] and [SelectInstance]. + * + * Roughly, [hasNext] is a [receive] sibling, while [next] simply + * returns the already retrieved element. From the implementation + * side, [receiveResult] stores the element retrieved by [hasNext] + * (or a special [CHANNEL_CLOSED] token if the channel is closed). + * + * The [invoke] function is a [CancelHandler] implementation, + * which requires knowing the [segment] and the [index] in it + * that specify the location of the stored iterator. + * + * To resume the suspended [hasNext] call, a special [tryResumeHasNext] + * function should be used in a way similar to [CancellableContinuation.tryResume] + * and [SelectInstance.trySelect]. When the channel becomes closed, + * [tryResumeHasNextOnClosedChannel] should be used instead. + */ + private inner class BufferedChannelIterator : ChannelIterator, BeforeResumeCancelHandler(), Waiter { + /** + * Stores the element retrieved by [hasNext] or + * a special [CHANNEL_CLOSED] token if this channel is closed. + * If [hasNext] has not been invoked yet, [NO_RECEIVE_RESULT] is stored. + */ + private var receiveResult: Any? = NO_RECEIVE_RESULT + + /** + * When [hasNext] suspends, this field stores the corresponding + * continuation. The [tryResumeHasNext] and [tryResumeHasNextOnClosedChannel] + * function resume this continuation when the [hasNext] invocation should complete. + */ + private var continuation: CancellableContinuation? = null + + // When `hasNext()` suspends, the location where the continuation + // is stored is specified via the segment and the index in it. + // We need this information in the cancellation handler below. + private var segment: ChannelSegment? = null + private var index = -1 + + /** + * Invoked on cancellation, [BeforeResumeCancelHandler] implementation. + */ + override fun invoke(cause: Throwable?) { + segment?.onCancellation(index) + } + + // `hasNext()` is just a special receive operation. + override suspend fun hasNext(): Boolean = + receiveImpl( // <-- this is an inline function + // Do not create a continuation until it is required; + // it is created later via [onNoWaiterSuspend], if needed. + waiter = null, + // Store the received element in `receiveResult` on successful + // retrieval from the buffer or rendezvous with a suspended sender. + // Also, inform the `BufferedChannel` extensions that + // the synchronization of this receive operation is completed. + onElementRetrieved = { element -> + this.receiveResult = element + true + }, + // As no waiter is provided, suspension is impossible. + onSuspend = { _, _, _ -> error("unreachable") }, + // Return `false` or throw an exception if the channel is already closed. + onClosed = { onClosedHasNext() }, + // If `hasNext()` decides to suspend, the corresponding + // `suspend` function that creates a continuation is called. + // The tail-call optimization is applied here. + onNoWaiterSuspend = { segm, i, r -> return hasNextOnNoWaiterSuspend(segm, i, r) } + ) + + private fun onClosedHasNext(): Boolean { + this.receiveResult = CHANNEL_CLOSED + val cause = closeCause ?: return false + throw recoverStackTrace(cause) + } + + private suspend fun hasNextOnNoWaiterSuspend( + /* The working cell is specified by + the segment and the index in it. */ + segment: ChannelSegment, + index: Int, + /* The global index of the cell. */ + r: Long + ): Boolean = suspendCancellableCoroutineReusable { cont -> + this.continuation = cont + receiveImplOnNoWaiter( // <-- this is an inline function + segment = segment, index = index, r = r, + waiter = this, // store this iterator as a waiter + // In case of successful element retrieval, store + // it in `receiveResult` and resume the continuation. + // Importantly, the receiver coroutine may be cancelled + // after it is successfully resumed but not dispatched yet. + // In case `onUndeliveredElement` is present, we must + // invoke it in the latter case. + onElementRetrieved = { element -> + this.receiveResult = element + this.continuation = null + cont.resume(true, onUndeliveredElement?.bindCancellationFun(element, cont.context)) + }, + onSuspend = { segm, i, _ -> prepareForSuspension(segm, i) }, + onClosed = { onClosedHasNextNoWaiterSuspend() } + ) + } + + private fun prepareForSuspension(segment: ChannelSegment, index: Int) { + this.segment = segment + this.index = index + // It is possible that this `hasNext()` invocation is already + // resumed, and the `continuation` field is already updated to `null`. + this.continuation?.invokeOnCancellation(this.asHandler) + onReceiveEnqueued() + } + + private fun onClosedHasNextNoWaiterSuspend() { + // Read the current continuation and clean + // the corresponding field to avoid memory leaks. + val cont = this.continuation!! + this.continuation = null + // Update the `hasNext()` internal result. + this.receiveResult = CHANNEL_CLOSED + // If this channel was closed without exception, + // `hasNext()` should return `false`; otherwise, + // it throws the closing exception. + val cause = closeCause + if (cause == null) { + cont.resume(false) + } else { + cont.resumeWithException(recoverStackTrace(cause, cont)) + } + } + + @Suppress("UNCHECKED_CAST") + override fun next(): E { + // Read the already received result, or [NO_RECEIVE_RESULT] if [hasNext] has not been invoked yet. + val result = receiveResult + check(result !== NO_RECEIVE_RESULT) { "`hasNext()` has not been invoked" } + receiveResult = NO_RECEIVE_RESULT + // Is this channel closed? + if (result === CHANNEL_CLOSED) throw recoverStackTrace(receiveException) + // Return the element. + return result as E + } + + fun tryResumeHasNext(element: E): Boolean { + // Read the current continuation and clean + // the corresponding field to avoid memory leaks. + val cont = this.continuation!! + this.continuation = null + // Store the retrieved element in `receiveResult`. + this.receiveResult = element + // Try to resume this `hasNext()`. Importantly, the receiver coroutine + // may be cancelled after it is successfully resumed but not dispatched yet. + // In case `onUndeliveredElement` is specified, we need to invoke it in the latter case. + return cont.tryResume0(true, onUndeliveredElement?.bindCancellationFun(element, cont.context)) + } + + fun tryResumeHasNextOnClosedChannel() { + // Read the current continuation and clean + // the corresponding field to avoid memory leaks. + val cont = this.continuation!! + this.continuation = null + // Update the `hasNext()` internal result and inform + // `BufferedChannel` extensions that synchronization + // of this receive operation is completed. + this.receiveResult = CHANNEL_CLOSED + // If this channel was closed without exception, + // `hasNext()` should return `false`; otherwise, + // it throws the closing exception. + val cause = closeCause + if (cause == null) { + cont.resume(false) + } else { + cont.resumeWithException(recoverStackTrace(cause, cont)) + } + } + } + + // ############################## + // ## Closing and Cancellation ## + // ############################## + + /** + * Store the cause of closing this channel, either via [close] or [cancel] call. + * The closing cause can be set only once. + */ + private val _closeCause = atomic(NO_CLOSE_CAUSE) + // Should be called only if this channel is closed or cancelled. + protected val closeCause get() = _closeCause.value as Throwable? + + /** Returns the closing cause if it is non-null, or [ClosedSendChannelException] otherwise. */ + protected val sendException get() = closeCause ?: ClosedSendChannelException(DEFAULT_CLOSE_MESSAGE) + + /** Returns the closing cause if it is non-null, or [ClosedReceiveChannelException] otherwise. */ + private val receiveException get() = closeCause ?: ClosedReceiveChannelException(DEFAULT_CLOSE_MESSAGE) + + /** + Stores the closed handler installed by [invokeOnClose]. + To synchronize [invokeOnClose] and [close], two additional + marker states, [CLOSE_HANDLER_INVOKED] and [CLOSE_HANDLER_CLOSED] + are used. The resulting state diagram is presented below. + + +------+ install handler +---------+ close(..) +---------+ + | null |------------------>| handler |------------>| INVOKED | + +------+ +---------+ +---------+ + | + | close(..) +--------+ + +----------->| CLOSED | + +--------+ + */ + private val closeHandler = atomic(null) + + /** + * Invoked when channel is closed as the last action of [close] invocation. + * This method should be idempotent and can be called multiple times. + */ + protected open fun onClosedIdempotent() {} + + override fun close(cause: Throwable?): Boolean = + closeOrCancelImpl(cause, cancel = false) + + @Suppress("OVERRIDE_DEPRECATION") + final override fun cancel(cause: Throwable?): Boolean = cancelImpl(cause) + + @Suppress("OVERRIDE_DEPRECATION") + final override fun cancel() { cancelImpl(null) } + + final override fun cancel(cause: CancellationException?) { cancelImpl(cause) } + + internal open fun cancelImpl(cause: Throwable?): Boolean = + closeOrCancelImpl(cause ?: CancellationException("Channel was cancelled"), cancel = true) + + /** + * This is a common implementation for [close] and [cancel]. It first tries + * to install the specified cause; the invocation that successfully installs + * the cause returns `true` as a results of this function, while all further + * [close] and [cancel] calls return `false`. + * + * After the closing/cancellation cause is installed, the channel should be marked + * as closed or cancelled, which bounds further `send(e)`-s to fails. + * + * Then, [completeCloseOrCancel] is called, which cancels waiting `receive()` + * requests ([cancelSuspendedReceiveRequests]) and removes unprocessed elements + * ([removeUnprocessedElements]) in case this channel is cancelled. + * + * Finally, if this [closeOrCancelImpl] has installed the cause, therefore, + * has closed the channel, [closeHandler] and [onClosedIdempotent] should be invoked. + */ + protected open fun closeOrCancelImpl(cause: Throwable?, cancel: Boolean): Boolean { + // If this is a `cancel(..)` invocation, set a bit that the cancellation + // has been started. This is crucial for ensuring linearizability, + // when concurrent `close(..)` and `isClosedFor[Send,Receive]` operations + // help this `cancel(..)`. + if (cancel) markCancellationStarted() + // Try to install the specified cause. On success, this invocation will + // return `true` as a result; otherwise, it will complete with `false`. + val closedByThisOperation = _closeCause.compareAndSet(NO_CLOSE_CAUSE, cause) + // Mark this channel as closed or cancelled, depending on this operation type. + if (cancel) markCancelled() else markClosed() + // Complete the closing or cancellation procedure. + completeCloseOrCancel() + // Finally, if this operation has installed the cause, + // it should invoke the close handlers. + return closedByThisOperation.also { + onClosedIdempotent() + if (it) invokeCloseHandler() + } + } + + /** + * Invokes the installed close handler, + * updating the [closeHandler] state correspondingly. + */ + private fun invokeCloseHandler() { + val closeHandler = closeHandler.getAndUpdate { + if (it === null) { + // Inform concurrent `invokeOnClose` + // that this channel is already closed. + CLOSE_HANDLER_CLOSED + } else { + // Replace the handler with a special + // `INVOKED` marker to avoid memory leaks. + CLOSE_HANDLER_INVOKED + } + } ?: return // no handler was installed, finish. + // Invoke the handler. + @Suppress("UNCHECKED_CAST") + closeHandler as (cause: Throwable?) -> Unit + closeHandler(closeCause) + } + + override fun invokeOnClose(handler: (cause: Throwable?) -> Unit) { + // Try to install the handler, finishing on success. + if (closeHandler.compareAndSet(null, handler)) { + // Handler has been successfully set, finish the operation. + return + } + // Either another handler is already set, or this channel is closed. + // In the latter case, the current handler should be invoked. + // However, the implementation must ensure that at most one + // handler is called, throwing an `IllegalStateException` + // if another close handler has been invoked. + closeHandler.loop { cur -> + when { + cur === CLOSE_HANDLER_CLOSED -> { + // Try to update the state from `CLOSED` to `INVOKED`. + // This is crucial to guarantee that at most one handler can be called. + // On success, invoke the handler and finish. + if (closeHandler.compareAndSet(CLOSE_HANDLER_CLOSED, CLOSE_HANDLER_INVOKED)) { + handler(closeCause) + return + } + } + cur === CLOSE_HANDLER_INVOKED -> error("Another handler was already registered and successfully invoked") + else -> error("Another handler is already registered: $cur") + } + } + } + + /** + * Marks this channel as closed. + * In case [cancelImpl] has already been invoked, + * and this channel is marked with [CLOSE_STATUS_CANCELLATION_STARTED], + * this function marks the channel as cancelled. + * + * All operation that notice this channel in the closed state, + * must help to complete the closing via [completeCloseOrCancel]. + */ + private fun markClosed(): Unit = + sendersAndCloseStatus.update { cur -> + when (cur.sendersCloseStatus) { + CLOSE_STATUS_ACTIVE -> // the channel is neither closed nor cancelled + constructSendersAndCloseStatus(cur.sendersCounter, CLOSE_STATUS_CLOSED) + CLOSE_STATUS_CANCELLATION_STARTED -> // the channel is going to be cancelled + constructSendersAndCloseStatus(cur.sendersCounter, CLOSE_STATUS_CANCELLED) + else -> return // the channel is already marked as closed or cancelled. + } + } + + /** + * Marks this channel as cancelled. + * + * All operation that notice this channel in the cancelled state, + * must help to complete the cancellation via [completeCloseOrCancel]. + */ + private fun markCancelled(): Unit = + sendersAndCloseStatus.update { cur -> + constructSendersAndCloseStatus(cur.sendersCounter, CLOSE_STATUS_CANCELLED) + } + + /** + * When the cancellation procedure starts, it is critical + * to mark the closing status correspondingly. Thus, other + * operations, which may help to complete the cancellation, + * always correctly update the status to `CANCELLED`. + */ + private fun markCancellationStarted(): Unit = + sendersAndCloseStatus.update { cur -> + if (cur.sendersCloseStatus == CLOSE_STATUS_ACTIVE) + constructSendersAndCloseStatus(cur.sendersCounter, CLOSE_STATUS_CANCELLATION_STARTED) + else return // this channel is already closed or cancelled + } + + /** + * Completes the started [close] or [cancel] procedure. + */ + private fun completeCloseOrCancel() { + isClosedForSend // must finish the started close/cancel if one is detected. + } + + protected open val isConflatedDropOldest get() = false + + /** + * Completes the channel closing procedure. + */ + private fun completeClose(sendersCur: Long): ChannelSegment { + // Close the linked list for further segment addition, + // obtaining the last segment in the data structure. + val lastSegment = closeLinkedList() + // In the conflated channel implementation (with the DROP_OLDEST + // elements conflation strategy), it is critical to mark all empty + // cells as closed to prevent in-progress `send(e)`-s, which have not + // put their elements yet, completions after this channel is closed. + // Otherwise, it is possible for a `send(e)` to put an element when + // the buffer is already full, while a concurrent receiver may extract + // the oldest element. When the channel is not closed, we can linearize + // this `receive()` before the `send(e)`, but after the channel is closed, + // `send(e)` must fails. Marking all unprocessed cells as `CLOSED` solves the issue. + if (isConflatedDropOldest) { + val lastBufferedCellGlobalIndex = markAllEmptyCellsAsClosed(lastSegment) + if (lastBufferedCellGlobalIndex != -1L) + dropFirstElementUntilTheSpecifiedCellIsInTheBuffer(lastBufferedCellGlobalIndex) + } + // Resume waiting `receive()` requests, + // informing them that the channel is closed. + cancelSuspendedReceiveRequests(lastSegment, sendersCur) + // Return the last segment in the linked list as a result + // of this function; we need it in `completeCancel(..)`. + return lastSegment + } + + /** + * Completes the channel cancellation procedure. + */ + private fun completeCancel(sendersCur: Long) { + // First, ensure that this channel is closed, + // obtaining the last segment in the linked list. + val lastSegment = completeClose(sendersCur) + // Cancel suspended `send(e)` requests and + // remove buffered elements in the reverse order. + removeUnprocessedElements(lastSegment) + } + + /** + * Closes the underlying linked list of segments for further segment addition. + */ + private fun closeLinkedList(): ChannelSegment { + // Choose the last segment. + var lastSegment = bufferEndSegment.value + sendSegment.value.let { if (it.id > lastSegment.id) lastSegment = it } + receiveSegment.value.let { if (it.id > lastSegment.id) lastSegment = it } + // Close the linked list of segment for new segment addition + // and return the last segment in the linked list. + return lastSegment.close() + } + + /** + * This function marks all empty cells, in the `null` and [IN_BUFFER] state, + * as closed. Notably, it processes the cells from right to left, and finishes + * immediately when the processing cell is already covered by `receive()` or + * contains a buffered elements ([BUFFERED] state). + * + * This function returns the global index of the last buffered element, + * or `-1` if this channel does not contain buffered elements. + */ + private fun markAllEmptyCellsAsClosed(lastSegment: ChannelSegment): Long { + // Process the cells in reverse order, from right to left. + var segment = lastSegment + while (true) { + for (index in SEGMENT_SIZE - 1 downTo 0) { + // Is this cell already covered by `receive()`? + val globalIndex = segment.id * SEGMENT_SIZE + index + if (globalIndex < receiversCounter) return -1 + // Process the cell `segment[index]`. + cell_update@ while (true) { + val state = segment.getState(index) + when { + // The cell is empty. + state === null || state === IN_BUFFER -> { + // Inform a possibly upcoming sender that this channel is already closed. + if (segment.casState(index, state, CHANNEL_CLOSED)) { + segment.onSlotCleaned() + break@cell_update + } + } + // The cell stores a buffered element. + state === BUFFERED -> return globalIndex + // Skip this cell if it is not empty and does not store a buffered element. + else -> break@cell_update + } + } + } + // Process the next segment, finishing if the linked list ends. + segment = segment.prev ?: return -1 + } + } + + /** + * Cancels suspended `send(e)` requests and removes buffered elements + * starting from the last cell in the specified [lastSegment] (it must + * be the physical tail of the underlying linked list) and updating + * the cells in reverse order. + */ + private fun removeUnprocessedElements(lastSegment: ChannelSegment) { + // Read the `onUndeliveredElement` lambda at once. In case it + // throws an exception, this exception is handled and stored in + // the variable below. If multiple exceptions are thrown, the first + // one is stored in the variable, while the others are suppressed. + val onUndeliveredElement = onUndeliveredElement + var undeliveredElementException: UndeliveredElementException? = null // first cancel exception, others suppressed + // To perform synchronization correctly, it is critical to + // process the cells in reverse order, from right to left. + // However, according to the API, suspended senders should + // be cancelled in the order of their suspension. Therefore, + // we need to collect all of them and cancel in the reverse + // order after that. + var suspendedSenders = InlineList() + var segment = lastSegment + process_segments@ while (true) { + for (index in SEGMENT_SIZE - 1 downTo 0) { + // Process the cell `segment[index]`. + val globalIndex = segment.id * SEGMENT_SIZE + index + // Update the cell state. + update_cell@ while (true) { + // Read the current state of the cell. + val state = segment.getState(index) + when { + // The cell is already processed by a receiver. + state === DONE_RCV -> break@process_segments + // The cell stores a buffered element. + state === BUFFERED -> { + // Is the cell already covered by a receiver? + if (globalIndex < receiversCounter) break@process_segments + // Update the cell state to `CHANNEL_CLOSED`. + if (segment.casState(index, state, CHANNEL_CLOSED)) { + // If `onUndeliveredElement` lambda is non-null, call it. + if (onUndeliveredElement != null) { + val element = segment.getElement(index) + undeliveredElementException = onUndeliveredElement.callUndeliveredElementCatchingException(element, undeliveredElementException) + } + // Clean the element field and inform the segment + // that the slot is cleaned to avoid memory leaks. + segment.cleanElement(index) + segment.onSlotCleaned() + break@update_cell + } + } + // The cell is empty. + state === IN_BUFFER || state === null -> { + // Update the cell state to `CHANNEL_CLOSED`. + if (segment.casState(index, state, CHANNEL_CLOSED)) { + // Inform the segment that the slot is cleaned to avoid memory leaks. + segment.onSlotCleaned() + break@update_cell + } + } + // The cell stores a suspended waiter. + state is Waiter || state is WaiterEB -> { + // Is the cell already covered by a receiver? + if (globalIndex < receiversCounter) break@process_segments + // Obtain the sender. + val sender: Waiter = if (state is WaiterEB) state.waiter + else state as Waiter + // Update the cell state to `CHANNEL_CLOSED`. + if (segment.casState(index, state, CHANNEL_CLOSED)) { + // If `onUndeliveredElement` lambda is non-null, call it. + if (onUndeliveredElement != null) { + val element = segment.getElement(index) + undeliveredElementException = onUndeliveredElement.callUndeliveredElementCatchingException(element, undeliveredElementException) + } + // Save the sender for further cancellation. + suspendedSenders += sender + // Clean the element field and inform the segment + // that the slot is cleaned to avoid memory leaks. + segment.cleanElement(index) + segment.onSlotCleaned() + break@update_cell + } + } + // A concurrent receiver is resuming a suspended sender. + // As the cell is covered by a receiver, finish immediately. + state === RESUMING_BY_EB || state === RESUMING_BY_RCV -> break@process_segments + // A concurrent `expandBuffer()` is resuming a suspended sender. + // Wait in a spin-loop until the cell state changes. + state === RESUMING_BY_EB -> continue@update_cell + else -> break@update_cell + } + } + } + // Process the previous segment. + segment = segment.prev ?: break + } + // Cancel suspended senders in their order of addition to this channel. + suspendedSenders.forEachReversed { it.resumeSenderOnCancelledChannel() } + // Throw `UndeliveredElementException` at the end if there was one. + undeliveredElementException?.let { throw it } + } + + /** + * Cancels suspended `receive` requests from the end to the beginning, + * also moving empty cells to the `CHANNEL_CLOSED` state. + */ + private fun cancelSuspendedReceiveRequests(lastSegment: ChannelSegment, sendersCounter: Long) { + // To perform synchronization correctly, it is critical to + // extract suspended requests in the reverse order, + // from the end to the beginning. + // However, according to the API, they should be cancelled + // in the order of their suspension. Therefore, we need to + // collect the suspended requests first, cancelling them + // in the reverse order after that. + var suspendedReceivers = InlineList() + var segment: ChannelSegment? = lastSegment + process_segments@ while (segment != null) { + for (index in SEGMENT_SIZE - 1 downTo 0) { + // Is the cell already covered by a sender? Finish immediately in this case. + if (segment.id * SEGMENT_SIZE + index < sendersCounter) break@process_segments + // Try to move the cell state to `CHANNEL_CLOSED`. + cell_update@ while (true) { + val state = segment.getState(index) + when { + state === null || state === IN_BUFFER -> { + if (segment.casState(index, state, CHANNEL_CLOSED)) { + segment.onSlotCleaned() + break@cell_update + } + } + state is WaiterEB -> { + if (segment.casState(index, state, CHANNEL_CLOSED)) { + suspendedReceivers += state.waiter // save for cancellation. + segment.onCancelledRequest(index = index, receiver = true) + break@cell_update + } + } + state is Waiter -> { + if (segment.casState(index, state, CHANNEL_CLOSED)) { + suspendedReceivers += state // save for cancellation. + segment.onCancelledRequest(index = index, receiver = true) + break@cell_update + } + } + else -> break@cell_update // nothing to cancel. + } + } + } + // Process the previous segment. + segment = segment.prev + } + // Cancel the suspended requests in their order of addition to this channel. + suspendedReceivers.forEachReversed { it.resumeReceiverOnClosedChannel() } + } + + /** + * Resumes this receiver because this channel is closed. + * This function does not take any effect if the operation has already been resumed or cancelled. + */ + private fun Waiter.resumeReceiverOnClosedChannel() = resumeWaiterOnClosedChannel(receiver = true) + + /** + * Resumes this sender because this channel is cancelled. + * This function does not take any effect if the operation has already been resumed or cancelled. + */ + private fun Waiter.resumeSenderOnCancelledChannel() = resumeWaiterOnClosedChannel(receiver = false) + + private fun Waiter.resumeWaiterOnClosedChannel(receiver: Boolean) { + when (this) { + is SendBroadcast -> cont.resume(false) + is CancellableContinuation<*> -> resumeWithException(if (receiver) receiveException else sendException) + is ReceiveCatching<*> -> cont.resume(closed(closeCause)) + is BufferedChannel<*>.BufferedChannelIterator -> tryResumeHasNextOnClosedChannel() + is SelectInstance<*> -> trySelect(this@BufferedChannel, CHANNEL_CLOSED) + else -> error("Unexpected waiter: $this") + } + } + + @ExperimentalCoroutinesApi + override val isClosedForSend: Boolean + get() = sendersAndCloseStatus.value.isClosedForSend0 + + private val Long.isClosedForSend0 get() = + isClosed(this, isClosedForReceive = false) + + @ExperimentalCoroutinesApi + override val isClosedForReceive: Boolean + get() = sendersAndCloseStatus.value.isClosedForReceive0 + + private val Long.isClosedForReceive0 get() = + isClosed(this, isClosedForReceive = true) + + private fun isClosed( + sendersAndCloseStatusCur: Long, + isClosedForReceive: Boolean + ) = when (sendersAndCloseStatusCur.sendersCloseStatus) { + // This channel is active and has not been closed. + CLOSE_STATUS_ACTIVE -> false + // The cancellation procedure has been started but + // not linearized yet, so this channel should be + // considered as active. + CLOSE_STATUS_CANCELLATION_STARTED -> false + // This channel has been successfully closed. + // Help to complete the closing procedure to + // guarantee linearizability, and return `true` + // for senders or the flag whether there still + // exist elements to retrieve for receivers. + CLOSE_STATUS_CLOSED -> { + completeClose(sendersAndCloseStatusCur.sendersCounter) + // When `isClosedForReceive` is `false`, always return `true`. + // Otherwise, it is possible that the channel is closed but + // still has elements to retrieve. + if (isClosedForReceive) !hasElements() else true + } + // This channel has been successfully cancelled. + // Help to complete the cancellation procedure to + // guarantee linearizability and return `true`. + CLOSE_STATUS_CANCELLED -> { + completeCancel(sendersAndCloseStatusCur.sendersCounter) + true + } + else -> error("unexpected close status: ${sendersAndCloseStatusCur.sendersCloseStatus}") + } + + @ExperimentalCoroutinesApi + override val isEmpty: Boolean get() { + // This function should return `false` if + // this channel is closed for `receive`. + if (isClosedForReceive) return false + // Does this channel has elements to retrieve? + if (hasElements()) return false + // This channel does not have elements to retrieve; + // Check that it is still not closed for `receive`. + return !isClosedForReceive + } + + /** + * Checks whether this channel contains elements to retrieve. + * Unfortunately, simply comparing the counters is insufficient, + * as some cells can be in the `INTERRUPTED` state due to cancellation. + * This function tries to find the first "alive" element, + * updating the `receivers` counter to skip empty cells. + * + * The implementation is similar to `receive()`. + */ + internal fun hasElements(): Boolean { + while (true) { + // Read the segment before obtaining the `receivers` counter value. + var segment = receiveSegment.value + // Obtains the `receivers` and `senders` counter values. + val r = receiversCounter + val s = sendersCounter + // Is there a chance that this channel has elements? + if (s <= r) return false // no elements + // The `r`-th cell is covered by a sender; check whether it contains an element. + // First, try to find the required segment if the initially + // obtained segment (in the beginning of this function) has lower id. + val id = r / SEGMENT_SIZE + if (segment.id != id) { + // Try to find the required segment. + segment = findSegmentReceive(id, segment) ?: + // The required segment has not been found. Either it has already + // been removed, or the underlying linked list is already closed + // for segment additions. In the latter case, the channel is closed + // and does not contain elements, so this operation returns `false`. + // Otherwise, if the required segment is removed, the operation restarts. + if (receiveSegment.value.id < id) return false else continue + } + segment.cleanPrev() // all the previous segments are no longer needed. + // Does the `r`-th cell contain waiting sender or buffered element? + val i = (r % SEGMENT_SIZE).toInt() + if (isCellNonEmpty(segment, i, r)) return true + // The cell is empty. Update `receivers` counter and try again. + receivers.compareAndSet(r, r + 1) // if this CAS fails, the counter has already been updated. + } + } + + /** + * Checks whether this cell contains a buffered element or a waiting sender, + * returning `true` in this case. Otherwise, if this cell is empty + * (due to waiter cancellation, cell poisoning, or channel closing), + * this function returns `false`. + * + * Notably, this function must be called only if the cell is covered by a sender. + */ + private fun isCellNonEmpty( + segment: ChannelSegment, + index: Int, + globalIndex: Long + ): Boolean { + // The logic is similar to `updateCellReceive` with the only difference + // that this function neither changes the cell state nor retrieves the element. + while (true) { + // Read the current cell state. + val state = segment.getState(index) + when { + // The cell is empty but a sender is coming. + state === null || state === IN_BUFFER -> { + // Poison the cell to ensure correctness. + if (segment.casState(index, state, POISONED)) { + // When the cell becomes poisoned, it is essentially + // the same as storing an already cancelled receiver. + // Thus, the `expandBuffer()` procedure should be invoked. + expandBuffer() + return false + } + } + // The cell stores a buffered element. + state === BUFFERED -> return true + // The cell stores an interrupted sender. + state === INTERRUPTED_SEND -> return false + // This channel is already closed. + state === CHANNEL_CLOSED -> return false + // The cell is already processed + // by a concurrent receiver. + state === DONE_RCV -> return false + // The cell is already poisoned + // by a concurrent receiver. + state === POISONED -> return false + // A concurrent `expandBuffer()` is resuming + // a suspended sender. This function is eligible + // to linearize before the buffer expansion procedure. + state === RESUMING_BY_EB -> return true + // A concurrent receiver is resuming + // a suspended sender. The element + // is no longer available for retrieval. + state === RESUMING_BY_RCV -> return false + // The cell stores a suspended request. + // However, it is possible that this request + // is receiver if the cell is covered by both + // send and receive operations. + // In case the cell is already covered by + // a receiver, the element is no longer + // available for retrieval, and this function + // return `false`. Otherwise, it is guaranteed + // that the suspended request is sender, so + // this function returns `true`. + else -> return globalIndex == receiversCounter + } + } + } + + // ####################### + // # Segments Management # + // ####################### + + /** + * Finds the segment with the specified [id] starting by the [startFrom] + * segment and following the [ChannelSegment.next] references. In case + * the required segment has not been created yet, this function attempts + * to add it to the underlying linked list. Finally, it updates [sendSegment] + * to the found segment if its [ChannelSegment.id] is greater than the one + * of the already stored segment. + * + * In case the requested segment is already removed, or if it should be allocated + * but the linked list structure is closed for new segments addition, this function + * returns `null`. The implementation also efficiently skips a sequence of removed + * segments, updating the counter value in [sendersAndCloseStatus] correspondingly. + */ + private fun findSegmentSend(id: Long, startFrom: ChannelSegment): ChannelSegment? { + return sendSegment.findSegmentAndMoveForward(id, startFrom, ::createSegment).let { + if (it.isClosed) { + // The required segment has not been found and new segments + // cannot be added, as the linked listed in already added. + // This channel is already closed or cancelled; help to complete + // the closing or cancellation procedure. + completeCloseOrCancel() + // Clean the `prev` reference of the provided segment + // if all the previous cells are already covered by senders. + // It is important to clean the `prev` reference only in + // this case, as the closing/cancellation procedure may + // need correct value to traverse the linked list from right to left. + if (startFrom.id * SEGMENT_SIZE < receiversCounter) startFrom.cleanPrev() + // As the required segment is not found and cannot be allocated, return `null`. + null + } else { + // Get the found segment. + val segment = it.segment + // Is the required segment removed? + if (segment.id > id) { + // The required segment has been removed; `segment` is the first + // segment with `id` not lower than the required one. + // Skip the sequence of removed cells in O(1). + updateSendersCounterIfLower(segment.id * SEGMENT_SIZE) + // Clean the `prev` reference of the provided segment + // if all the previous cells are already covered by senders. + // It is important to clean the `prev` reference only in + // this case, as the closing/cancellation procedure may + // need correct value to traverse the linked list from right to left. + if (segment.id * SEGMENT_SIZE < receiversCounter) segment.cleanPrev() + // As the required segment is not found and cannot be allocated, return `null`. + null + } else { + assert { segment.id == id } + // The required segment has been found; return it! + segment + } + } + } + } + + /** + * Finds the segment with the specified [id] starting by the [startFrom] + * segment and following the [ChannelSegment.next] references. In case + * the required segment has not been created yet, this function attempts + * to add it to the underlying linked list. Finally, it updates [receiveSegment] + * to the found segment if its [ChannelSegment.id] is greater than the one + * of the already stored segment. + * + * In case the requested segment is already removed, or if it should be allocated + * but the linked list structure is closed for new segments addition, this function + * returns `null`. The implementation also efficiently skips a sequence of removed + * segments, updating the [receivers] counter correspondingly. + */ + private fun findSegmentReceive(id: Long, startFrom: ChannelSegment): ChannelSegment? = + receiveSegment.findSegmentAndMoveForward(id, startFrom, ::createSegment).let { + if (it.isClosed) { + // The required segment has not been found and new segments + // cannot be added, as the linked listed in already added. + // This channel is already closed or cancelled; help to complete + // the closing or cancellation procedure. + completeCloseOrCancel() + // Clean the `prev` reference of the provided segment + // if all the previous cells are already covered by senders. + // It is important to clean the `prev` reference only in + // this case, as the closing/cancellation procedure may + // need correct value to traverse the linked list from right to left. + if (startFrom.id * SEGMENT_SIZE < sendersCounter) startFrom.cleanPrev() + // As the required segment is not found and cannot be allocated, return `null`. + null + } else { + // Get the found segment. + val segment = it.segment + // Advance the `bufferEnd` segment if required. + if (!isRendezvousOrUnlimited && id <= bufferEndCounter / SEGMENT_SIZE) { + bufferEndSegment.moveForward(segment) + } + // Is the required segment removed? + if (segment.id > id) { + // The required segment has been removed; `segment` is the first + // segment with `id` not lower than the required one. + // Skip the sequence of removed cells in O(1). + updateReceiversCounterIfLower(segment.id * SEGMENT_SIZE) + // Clean the `prev` reference of the provided segment + // if all the previous cells are already covered by senders. + // It is important to clean the `prev` reference only in + // this case, as the closing/cancellation procedure may + // need correct value to traverse the linked list from right to left. + if (segment.id * SEGMENT_SIZE < sendersCounter) segment.cleanPrev() + // As the required segment is already removed, return `null`. + null + } else { + assert { segment.id == id } + // The required segment has been found; return it! + segment + } + } + } + + /** + * Importantly, when this function does not find the requested segment, + * it always updates the number of completed `expandBuffer()` attempts. + */ + private fun findSegmentBufferEnd(id: Long, startFrom: ChannelSegment, currentBufferEndCounter: Long): ChannelSegment? = + bufferEndSegment.findSegmentAndMoveForward(id, startFrom, ::createSegment).let { + if (it.isClosed) { + // The required segment has not been found and new segments + // cannot be added, as the linked listed in already added. + // This channel is already closed or cancelled; help to complete + // the closing or cancellation procedure. + completeCloseOrCancel() + // Update `bufferEndSegment` to the last segment + // in the linked list to avoid memory leaks. + moveSegmentBufferEndToSpecifiedOrLast(id, startFrom) + // When this function does not find the requested segment, + // it should update the number of completed `expandBuffer()` attempts. + incCompletedExpandBufferAttempts() + null + } else { + // Get the found segment. + val segment = it.segment + // Is the required segment removed? + if (segment.id > id) { + // The required segment has been removed; `segment` is the first segment + // with `id` not lower than the required one. + // Try to skip the sequence of removed cells in O(1) by increasing the `bufferEnd` counter. + // Importantly, when this function does not find the requested segment, + // it should update the number of completed `expandBuffer()` attempts. + if (bufferEnd.compareAndSet(currentBufferEndCounter + 1, segment.id * SEGMENT_SIZE)) { + incCompletedExpandBufferAttempts(segment.id * SEGMENT_SIZE - currentBufferEndCounter) + } else { + incCompletedExpandBufferAttempts() + } + // As the required segment is already removed, return `null`. + null + } else { + assert { segment.id == id } + // The required segment has been found; return it! + segment + } + } + } + + /** + * Updates [bufferEndSegment] to the one with the specified [id] or + * to the last existing segment, if the required segment is not yet created. + * + * Unlike [findSegmentBufferEnd], this function does not allocate new segments. + */ + private fun moveSegmentBufferEndToSpecifiedOrLast(id: Long, startFrom: ChannelSegment) { + // Start searching the required segment from the specified one. + var segment: ChannelSegment = startFrom + while (segment.id < id) { + segment = segment.next ?: break + } + // Skip all removed segments and try to update `bufferEndSegment` + // to the first non-removed one. This part should succeed eventually, + // as the tail segment is never removed. + while (true) { + while (segment.isRemoved) { + segment = segment.next ?: break + } + // Try to update `bufferEndSegment`. On failure, + // the found segment is already removed, so it + // should be skipped. + if (bufferEndSegment.moveForward(segment)) return + } + } + + /** + * Updates the `senders` counter if its value + * is lower that the specified one. + * + * Senders use this function to efficiently skip + * a sequence of cancelled receivers. + */ + private fun updateSendersCounterIfLower(value: Long): Unit = + sendersAndCloseStatus.loop { cur -> + val curCounter = cur.sendersCounter + if (curCounter >= value) return + val update = constructSendersAndCloseStatus(curCounter, cur.sendersCloseStatus) + if (sendersAndCloseStatus.compareAndSet(cur, update)) return + } + + /** + * Updates the `receivers` counter if its value + * is lower that the specified one. + * + * Receivers use this function to efficiently skip + * a sequence of cancelled senders. + */ + private fun updateReceiversCounterIfLower(value: Long): Unit = + receivers.loop { cur -> + if (cur >= value) return + if (receivers.compareAndSet(cur, value)) return + } + + // ################### + // # Debug Functions # + // ################### + + @Suppress("ConvertTwoComparisonsToRangeCheck") + override fun toString(): String { + val sb = StringBuilder() + // Append the close status + when (sendersAndCloseStatus.value.sendersCloseStatus) { + CLOSE_STATUS_CLOSED -> sb.append("closed,") + CLOSE_STATUS_CANCELLED -> sb.append("cancelled,") + } + // Append the buffer capacity + sb.append("capacity=$capacity,") + // Append the data + sb.append("data=[") + val firstSegment = listOf(receiveSegment.value, sendSegment.value, bufferEndSegment.value) + .filter { it !== NULL_SEGMENT } + .minBy { it.id } + val r = receiversCounter + val s = sendersCounter + var segment = firstSegment + append_elements@ while (true) { + process_cell@ for (i in 0 until SEGMENT_SIZE) { + val globalCellIndex = segment.id * SEGMENT_SIZE + i + if (globalCellIndex >= s && globalCellIndex >= r) break@append_elements + val cellState = segment.getState(i) + val element = segment.getElement(i) + val cellStateString = when (cellState) { + is CancellableContinuation<*> -> { + when { + globalCellIndex < r && globalCellIndex >= s -> "receive" + globalCellIndex < s && globalCellIndex >= r -> "send" + else -> "cont" + } + } + is SelectInstance<*> -> { + when { + globalCellIndex < r && globalCellIndex >= s -> "onReceive" + globalCellIndex < s && globalCellIndex >= r -> "onSend" + else -> "select" + } + } + is ReceiveCatching<*> -> "receiveCatching" + is SendBroadcast -> "sendBroadcast" + is WaiterEB -> "EB($cellState)" + RESUMING_BY_RCV, RESUMING_BY_EB -> "resuming_sender" + null, IN_BUFFER, DONE_RCV, POISONED, INTERRUPTED_RCV, INTERRUPTED_SEND, CHANNEL_CLOSED -> continue@process_cell + else -> cellState.toString() // leave it just in case something is missed. + } + if (element != null) { + sb.append("($cellStateString,$element),") + } else { + sb.append("$cellStateString,") + } + } + // Process the next segment if exists. + segment = segment.next ?: break + } + if (sb.last() == ',') sb.deleteAt(sb.length - 1) + sb.append("]") + // The string representation is constructed. + return sb.toString() + } + + // Returns a debug representation of this channel, + // which is actively used in Lincheck tests. + internal fun toStringDebug(): String { + val sb = StringBuilder() + // Append the counter values and the close status + sb.append("S=${sendersCounter},R=${receiversCounter},B=${bufferEndCounter},B'=${completedExpandBuffersAndPauseFlag.value},C=${sendersAndCloseStatus.value.sendersCloseStatus},") + when (sendersAndCloseStatus.value.sendersCloseStatus) { + CLOSE_STATUS_CANCELLATION_STARTED -> sb.append("CANCELLATION_STARTED,") + CLOSE_STATUS_CLOSED -> sb.append("CLOSED,") + CLOSE_STATUS_CANCELLED -> sb.append("CANCELLED,") + } + // Append the segment references + sb.append("SEND_SEGM=${sendSegment.value.hexAddress},RCV_SEGM=${receiveSegment.value.hexAddress}") + if (!isRendezvousOrUnlimited) sb.append(",EB_SEGM=${bufferEndSegment.value.hexAddress}") + sb.append(" ") // add some space + // Append the linked list of segments. + val firstSegment = listOf(receiveSegment.value, sendSegment.value, bufferEndSegment.value) + .filter { it !== NULL_SEGMENT } + .minBy { it.id } + var segment = firstSegment + while (true) { + sb.append("${segment.hexAddress}=[${if (segment.isRemoved) "*" else ""}${segment.id},prev=${segment.prev?.hexAddress},") + repeat(SEGMENT_SIZE) { i -> + val cellState = segment.getState(i) + val element = segment.getElement(i) + val cellStateString = when (cellState) { + is CancellableContinuation<*> -> "cont" + is SelectInstance<*> -> "select" + is ReceiveCatching<*> -> "receiveCatching" + is SendBroadcast -> "send(broadcast)" + is WaiterEB -> "EB($cellState)" + else -> cellState.toString() + } + sb.append("[$i]=($cellStateString,$element),") + } + sb.append("next=${segment.next?.hexAddress}] ") + // Process the next segment if exists. + segment = segment.next ?: break + } + // The string representation of this channel is now constructed! + return sb.toString() + } + + + // This is an internal methods for tests. + fun checkSegmentStructureInvariants() { + if (isRendezvousOrUnlimited) { + check(bufferEndSegment.value === NULL_SEGMENT) { + "bufferEndSegment must be NULL_SEGMENT for rendezvous and unlimited channels; they do not manipulate it.\n" + + "Channel state: $this" + } + } else { + check(receiveSegment.value.id <= bufferEndSegment.value.id) { + "bufferEndSegment should not have lower id than receiveSegment.\n" + + "Channel state: $this" + } + } + val firstSegment = listOf(receiveSegment.value, sendSegment.value, bufferEndSegment.value) + .filter { it !== NULL_SEGMENT } + .minBy { it.id } + check(firstSegment.prev == null) { + "All processed segments should be unreachable from the data structure, but the `prev` link of the leftmost segment is non-null.\n" + + "Channel state: $this" + } + // Check that the doubly-linked list of segments does not + // contain full-of-cancelled-cells segments. + var segment = firstSegment + while (segment.next != null) { + // Note that the `prev` reference can be `null` if this channel is closed. + check(segment.next!!.prev == null || segment.next!!.prev === segment) { + "The `segment.next.prev === segment` invariant is violated.\n" + + "Channel state: $this" + } + // Count the number of closed/interrupted cells + // and check that all cells are in expected states. + var interruptedOrClosedCells = 0 + for (i in 0 until SEGMENT_SIZE) { + when (val state = segment.getState(i)) { + BUFFERED -> {} // The cell stores a buffered element. + is Waiter -> {} // The cell stores a suspended request. + INTERRUPTED_RCV, INTERRUPTED_SEND, CHANNEL_CLOSED -> { + // The cell stored an interrupted request or indicates + // that this channel is already closed. + // Check that the element slot is cleaned and increment + // the number of cells in closed/interrupted state. + check(segment.getElement(i) == null) + interruptedOrClosedCells++ + } + POISONED, DONE_RCV -> { + // The cell is successfully processed or poisoned. + // Check that the element slot is cleaned. + check(segment.getElement(i) == null) + } + // Other states are illegal after all running operations finish. + else -> error("Unexpected segment cell state: $state.\nChannel state: $this") + } + } + // Is this segment full of cancelled/closed cells? + // If so, this segment should be removed from the + // linked list if nether `receiveSegment`, nor + // `sendSegment`, nor `bufferEndSegment` reference it. + if (interruptedOrClosedCells == SEGMENT_SIZE) { + check(segment === receiveSegment.value || segment === sendSegment.value || segment === bufferEndSegment.value) { + "Logically removed segment is reachable.\nChannel state: $this" + } + } + // Process the next segment. + segment = segment.next!! + } + } +} + +/** + * The channel is represented as a list of segments, which simulates an infinite array. + * Each segment has its own [id], which increase from the beginning. These [id]s help + * to update [BufferedChannel.sendSegment], [BufferedChannel.receiveSegment], + * and [BufferedChannel.bufferEndSegment] correctly. + */ +internal class ChannelSegment(id: Long, prev: ChannelSegment?, channel: BufferedChannel?, pointers: Int) : Segment>(id, prev, pointers) { + private val _channel: BufferedChannel? = channel + val channel get() = _channel!! // always non-null except for `NULL_SEGMENT` + + private val data = atomicArrayOfNulls(SEGMENT_SIZE * 2) // 2 registers per slot: state + element + override val numberOfSlots: Int get() = SEGMENT_SIZE + + // ######################################## + // # Manipulation with the Element Fields # + // ######################################## + + internal fun storeElement(index: Int, element: E) { + setElementLazy(index, element) + } + + @Suppress("UNCHECKED_CAST") + internal fun getElement(index: Int) = data[index * 2].value as E + + internal fun retrieveElement(index: Int): E = getElement(index).also { cleanElement(index) } + + internal fun cleanElement(index: Int) { + setElementLazy(index, null) + } + + private fun setElementLazy(index: Int, value: Any?) { + data[index * 2].lazySet(value) + } + + // ###################################### + // # Manipulation with the State Fields # + // ###################################### + + internal fun getState(index: Int): Any? = data[index * 2 + 1].value + + internal fun setState(index: Int, value: Any?) { + data[index * 2 + 1].value = value + } + + internal fun casState(index: Int, from: Any?, to: Any?) = data[index * 2 + 1].compareAndSet(from, to) + + internal fun getAndSetState(index: Int, update: Any?) = data[index * 2 + 1].getAndSet(update) + + + // ######################## + // # Cancellation Support # + // ######################## + + fun onSenderCancellationWithOnUndeliveredElement(index: Int, context: CoroutineContext) { + // Read the element first. If the operation has not been successfully resumed + // (this cancellation may be caused by prompt cancellation during dispatching), + // it is guaranteed that the element is presented. + val element = getElement(index) + // Perform the cancellation; `onCancellationImpl(..)` return `true` if the + // cancelled operation had not been resumed. In this case, the `onUndeliveredElement` + // lambda should be called. + if (onCancellation(index)) { + channel.onUndeliveredElement!!.callUndeliveredElement(element, context) + } + } + + /** + * Returns `true` if the request is successfully cancelled, + * and no rendezvous has happened. We need this knowledge + * to keep [BufferedChannel.onUndeliveredElement] correct. + */ + @Suppress("ConvertTwoComparisonsToRangeCheck") + fun onCancellation(index: Int): Boolean { + // Count the global index of this cell and read + // the current counters of send and receive operations. + val globalIndex = id * SEGMENT_SIZE + index + val s = channel.sendersCounter + val r = channel.receiversCounter + // Update the cell state trying to distinguish whether + // the cancelled coroutine is sender or receiver. + var isSender: Boolean + var isReceiver: Boolean + while (true) { // CAS-loop + // Read the current state of the cell. + val cur = data[index * 2 + 1].value + when { + // The cell stores a waiter. + cur is Waiter || cur is WaiterEB -> { + // Is the cancelled request send for sure? + isSender = globalIndex < s && globalIndex >= r + // Is the cancelled request receiver for sure? + isReceiver = globalIndex < r && globalIndex >= s + // If the cancelled coroutine neither sender + // nor receiver, clean the element slot and finish. + // An opposite operation will resume this request + // and update the cell state eventually. + if (!isSender && !isReceiver) { + cleanElement(index) + return true + } + // The cancelled request is either send or receive. + // Update the cell state correspondingly. + val update = if (isSender) INTERRUPTED_SEND else INTERRUPTED_RCV + if (data[index * 2 + 1].compareAndSet(cur, update)) break + } + // The cell already indicates that the operation is cancelled. + cur === INTERRUPTED_SEND || cur === INTERRUPTED_RCV -> { + // Clean the element slot to avoid memory leaks and finish. + cleanElement(index) + return true + } + // An opposite operation is resuming this request; + // wait until the cell state updates. + // It is possible that an opposite operation has already + // resumed this request, which will result in updating + // the cell state to `DONE_RCV` or `BUFFERED`, while the + // current cancellation is caused by prompt cancellation. + cur === RESUMING_BY_EB || cur === RESUMING_BY_RCV -> continue + // This request was successfully resumed, so this cancellation + // is caused by the prompt cancellation feature and should be ignored. + cur === DONE_RCV || cur === BUFFERED -> return false + // The cell state indicates that the channel is closed; + // this cancellation should be ignored. + cur === CHANNEL_CLOSED -> { + return false + } + else -> error("unexpected state: $cur") + } + } + // Clean the element slot and invoke `onSlotCleaned()`, + // which may cause deleting the whole segment from the linked list. + // In case the cancelled request is receiver, it is critical to ensure + // that the `expandBuffer()` attempt that processes this cell is completed, + // so `onCancelledRequest(..)` waits for its completion before invoking `onSlotCleaned()`. + cleanElement(index) + onCancelledRequest(index, isReceiver) + return true + } + + /** + * Invokes `onSlotCleaned()` preceded by a `waitExpandBufferCompletion(..)` call + * in case the cancelled request is receiver. + */ + fun onCancelledRequest(index: Int, receiver: Boolean) { + if (receiver) channel.waitExpandBufferCompletion(id * SEGMENT_SIZE + index) + onSlotCleaned() + } +} +private fun createSegment(id: Long, prev: ChannelSegment) = ChannelSegment( + id = id, + prev = prev, + channel = prev.channel, + pointers = 0 +) +private val NULL_SEGMENT = ChannelSegment(id = -1, prev = null, channel = null, pointers = 0) + +/** + * Number of cells in each segment. + */ +@JvmField +internal val SEGMENT_SIZE = systemProp("kotlinx.coroutines.bufferedChannel.segmentSize", 32) + +/** + * Number of iterations to wait in [BufferedChannel.waitExpandBufferCompletion] until the numbers of started and completed + * [BufferedChannel.expandBuffer] calls coincide. When the limit is reached, [BufferedChannel.waitExpandBufferCompletion] + * blocks further [BufferedChannel.expandBuffer]-s to avoid starvation. + */ +private val EXPAND_BUFFER_COMPLETION_WAIT_ITERATIONS = systemProp("kotlinx.coroutines.bufferedChannel.expandBufferCompletionWaitIterations", 10_000) + +/** + * Tries to resume this continuation with the specified + * value. Returns `true` on success and `false` on failure. + */ +private fun CancellableContinuation.tryResume0( + value: T, + onCancellation: ((cause: Throwable) -> Unit)? = null +): Boolean = + tryResume(value, null, onCancellation).let { token -> + if (token != null) { + completeResume(token) + true + } else false + } + +/* + If the channel is rendezvous or unlimited, the `bufferEnd` counter + should be initialized with the corresponding value below and never change. + In this case, the `expandBuffer(..)` operation does nothing. + */ +private const val BUFFER_END_RENDEZVOUS = 0L // no buffer +private const val BUFFER_END_UNLIMITED = Long.MAX_VALUE // infinite buffer +private fun initialBufferEnd(capacity: Int): Long = when (capacity) { + Channel.RENDEZVOUS -> BUFFER_END_RENDEZVOUS + Channel.UNLIMITED -> BUFFER_END_UNLIMITED + else -> capacity.toLong() +} + +/* + Cell states. The initial "empty" state is represented with `null`, + and suspended operations are represented with [Waiter] instances. + */ + +// The cell stores a buffered element. +@JvmField +internal val BUFFERED = Symbol("BUFFERED") +// Concurrent `expandBuffer(..)` can inform the +// upcoming sender that it should buffer the element. +private val IN_BUFFER = Symbol("SHOULD_BUFFER") +// Indicates that a receiver (RCV suffix) is resuming +// the suspended sender; after that, it should update +// the state to either `DONE_RCV` (on success) or +// `INTERRUPTED_SEND` (on failure). +private val RESUMING_BY_RCV = Symbol("S_RESUMING_BY_RCV") +// Indicates that `expandBuffer(..)` (RCV suffix) is resuming +// the suspended sender; after that, it should update +// the state to either `BUFFERED` (on success) or +// `INTERRUPTED_SEND` (on failure). +private val RESUMING_BY_EB = Symbol("RESUMING_BY_EB") +// When a receiver comes to the cell already covered by +// a sender (according to the counters), but the cell +// is still in `EMPTY` or `IN_BUFFER` state, it breaks +// the cell by changing its state to `POISONED`. +private val POISONED = Symbol("POISONED") +// When the element is successfully transferred +// to a receiver, the cell changes to `DONE_RCV`. +private val DONE_RCV = Symbol("DONE_RCV") +// Cancelled sender. +private val INTERRUPTED_SEND = Symbol("INTERRUPTED_SEND") +// Cancelled receiver. +private val INTERRUPTED_RCV = Symbol("INTERRUPTED_RCV") +// Indicates that the channel is closed. +internal val CHANNEL_CLOSED = Symbol("CHANNEL_CLOSED") +// When the cell is already covered by both sender and +// receiver (`sender` and `receivers` counters are greater +// than the cell number), the `expandBuffer(..)` procedure +// cannot distinguish which kind of operation is stored +// in the cell. Thus, it wraps the waiter with this descriptor, +// informing the possibly upcoming receiver that it should +// complete the `expandBuffer(..)` procedure if the waiter stored +// in the cell is sender. In turn, senders ignore this information. +private class WaiterEB(@JvmField val waiter: Waiter) { + override fun toString() = "WaiterEB($waiter)" +} + + + +/** + * To distinguish suspended [BufferedChannel.receive] and + * [BufferedChannel.receiveCatching] operations, the latter + * uses this wrapper for its continuation. + */ +private class ReceiveCatching( + @JvmField val cont: CancellableContinuation> +) : Waiter + +/* + Internal results for [BufferedChannel.updateCellReceive]. + On successful rendezvous with waiting sender or + buffered element retrieval, the corresponding element + is returned as result of [BufferedChannel.updateCellReceive]. + */ +private val SUSPEND = Symbol("SUSPEND") +private val SUSPEND_NO_WAITER = Symbol("SUSPEND_NO_WAITER") +private val FAILED = Symbol("FAILED") + +/* + Internal results for [BufferedChannel.updateCellSend] + */ +private const val RESULT_RENDEZVOUS = 0 +private const val RESULT_BUFFERED = 1 +private const val RESULT_SUSPEND = 2 +private const val RESULT_SUSPEND_NO_WAITER = 3 +private const val RESULT_CLOSED = 4 +private const val RESULT_FAILED = 5 + +/** + * Special value for [BufferedChannel.BufferedChannelIterator.receiveResult] + * that indicates the absence of pre-received result. + */ +private val NO_RECEIVE_RESULT = Symbol("NO_RECEIVE_RESULT") + +/* + As [BufferedChannel.invokeOnClose] can be invoked concurrently + with channel closing, we have to synchronize them. These two + markers help with the synchronization. + */ +private val CLOSE_HANDLER_CLOSED = Symbol("CLOSE_HANDLER_CLOSED") +private val CLOSE_HANDLER_INVOKED = Symbol("CLOSE_HANDLER_INVOKED") + +/** + * Specifies the absence of closing cause, stored in [BufferedChannel._closeCause]. + * When the channel is closed or cancelled without exception, this [NO_CLOSE_CAUSE] + * marker should be replaced with `null`. + */ +private val NO_CLOSE_CAUSE = Symbol("NO_CLOSE_CAUSE") + +/* + The channel close statuses. The transition scheme is the following: + +--------+ +----------------------+ +-----------+ + | ACTIVE |-->| CANCELLATION_STARTED |-->| CANCELLED | + +--------+ +----------------------+ +-----------+ + | ^ + | +--------+ | + +------------>| CLOSED |------------------+ + +--------+ + We need `CANCELLATION_STARTED` to synchronize + concurrent closing and cancellation. + */ +private const val CLOSE_STATUS_ACTIVE = 0 +private const val CLOSE_STATUS_CANCELLATION_STARTED = 1 +private const val CLOSE_STATUS_CLOSED = 2 +private const val CLOSE_STATUS_CANCELLED = 3 + +/* + The `senders` counter and the channel close status + are stored in a single 64-bit register to save the space + and reduce the number of reads in sending operations. + The code below encapsulates the required bit arithmetics. + */ +private const val SENDERS_CLOSE_STATUS_SHIFT = 60 +private const val SENDERS_COUNTER_MASK = (1L shl SENDERS_CLOSE_STATUS_SHIFT) - 1 +private inline val Long.sendersCounter get() = this and SENDERS_COUNTER_MASK +private inline val Long.sendersCloseStatus: Int get() = (this shr SENDERS_CLOSE_STATUS_SHIFT).toInt() +private fun constructSendersAndCloseStatus(counter: Long, closeStatus: Int): Long = + (closeStatus.toLong() shl SENDERS_CLOSE_STATUS_SHIFT) + counter + +/* + The `completedExpandBuffersAndPauseFlag` 64-bit counter contains + the number of completed `expandBuffer()` attempts along with a special + flag that pauses progress to avoid starvation in `waitExpandBufferCompletion(..)`. + The code below encapsulates the required bit arithmetics. + */ +private const val EB_COMPLETED_PAUSE_EXPAND_BUFFERS_BIT = 1L shl 62 +private const val EB_COMPLETED_COUNTER_MASK = EB_COMPLETED_PAUSE_EXPAND_BUFFERS_BIT - 1 +private inline val Long.ebCompletedCounter get() = this and EB_COMPLETED_COUNTER_MASK +private inline val Long.ebPauseExpandBuffers: Boolean get() = (this and EB_COMPLETED_PAUSE_EXPAND_BUFFERS_BIT) != 0L +private fun constructEBCompletedAndPauseFlag(counter: Long, pauseEB: Boolean): Long = + (if (pauseEB) EB_COMPLETED_PAUSE_EXPAND_BUFFERS_BIT else 0) + counter + +/** + * All waiters, such as [CancellableContinuationImpl], [SelectInstance], and + * [BufferedChannel.BufferedChannelIterator], should be marked with this interface + * to make the code faster and easier to read. + */ +internal interface Waiter diff --git a/kotlinx-coroutines-core/common/src/channels/Channel.kt b/kotlinx-coroutines-core/common/src/channels/Channel.kt index a9d4d6fb30..052bddb4d9 100644 --- a/kotlinx-coroutines-core/common/src/channels/Channel.kt +++ b/kotlinx-coroutines-core/common/src/channels/Channel.kt @@ -377,7 +377,7 @@ public interface ReceiveChannel { level = DeprecationLevel.ERROR, replaceWith = ReplaceWith("onReceiveCatching") ) // Warning since 1.3.0, error in 1.5.0, will be hidden or removed in 1.7.0 - public val onReceiveOrNull: SelectClause1 get() = (this as AbstractChannel).onReceiveOrNull + public val onReceiveOrNull: SelectClause1 get() = (this as BufferedChannel).onReceiveOrNull } /** @@ -771,26 +771,24 @@ public fun Channel( when (capacity) { RENDEZVOUS -> { if (onBufferOverflow == BufferOverflow.SUSPEND) - RendezvousChannel(onUndeliveredElement) // an efficient implementation of rendezvous channel + BufferedChannel(RENDEZVOUS, onUndeliveredElement) // an efficient implementation of rendezvous channel else - ArrayChannel(1, onBufferOverflow, onUndeliveredElement) // support buffer overflow with buffered channel + ConflatedBufferedChannel(1, onBufferOverflow, onUndeliveredElement) // support buffer overflow with buffered channel } CONFLATED -> { require(onBufferOverflow == BufferOverflow.SUSPEND) { "CONFLATED capacity cannot be used with non-default onBufferOverflow" } - ConflatedChannel(onUndeliveredElement) + ConflatedBufferedChannel(1, BufferOverflow.DROP_OLDEST, onUndeliveredElement) + } + UNLIMITED -> BufferedChannel(UNLIMITED, onUndeliveredElement) // ignores onBufferOverflow: it has buffer, but it never overflows + BUFFERED -> { // uses default capacity with SUSPEND + if (onBufferOverflow == BufferOverflow.SUSPEND) BufferedChannel(CHANNEL_DEFAULT_CAPACITY, onUndeliveredElement) + else ConflatedBufferedChannel(1, onBufferOverflow, onUndeliveredElement) } - UNLIMITED -> LinkedListChannel(onUndeliveredElement) // ignores onBufferOverflow: it has buffer, but it never overflows - BUFFERED -> ArrayChannel( // uses default capacity with SUSPEND - if (onBufferOverflow == BufferOverflow.SUSPEND) CHANNEL_DEFAULT_CAPACITY else 1, - onBufferOverflow, onUndeliveredElement - ) else -> { - if (capacity == 1 && onBufferOverflow == BufferOverflow.DROP_OLDEST) - ConflatedChannel(onUndeliveredElement) // conflated implementation is more efficient but appears to work in the same way - else - ArrayChannel(capacity, onBufferOverflow, onUndeliveredElement) + if (onBufferOverflow === BufferOverflow.SUSPEND) BufferedChannel(capacity, onUndeliveredElement) + else ConflatedBufferedChannel(capacity, onBufferOverflow, onUndeliveredElement) } } diff --git a/kotlinx-coroutines-core/common/src/channels/ConflatedBroadcastChannel.kt b/kotlinx-coroutines-core/common/src/channels/ConflatedBroadcastChannel.kt deleted file mode 100644 index ff9f769ec9..0000000000 --- a/kotlinx-coroutines-core/common/src/channels/ConflatedBroadcastChannel.kt +++ /dev/null @@ -1,296 +0,0 @@ -/* - * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. - */ - -package kotlinx.coroutines.channels - -import kotlinx.atomicfu.* -import kotlinx.coroutines.* -import kotlinx.coroutines.internal.* -import kotlinx.coroutines.intrinsics.* -import kotlinx.coroutines.selects.* -import kotlin.jvm.* - -/** - * Broadcasts the most recently sent element (aka [value]) to all [openSubscription] subscribers. - * - * Back-to-send sent elements are _conflated_ -- only the most recently sent value is received, - * while previously sent elements **are lost**. - * Every subscriber immediately receives the most recently sent element. - * Sender to this broadcast channel never suspends and [trySend] always succeeds. - * - * A secondary constructor can be used to create an instance of this class that already holds a value. - * This channel is also created by `BroadcastChannel(Channel.CONFLATED)` factory function invocation. - * - * This implementation is fully lock-free. In this implementation - * [opening][openSubscription] and [closing][ReceiveChannel.cancel] subscription takes O(N) time, where N is the - * number of subscribers. - * - * **Note: This API is obsolete since 1.5.0.** It will be deprecated with warning in 1.6.0 - * and with error in 1.7.0. It is replaced with [StateFlow][kotlinx.coroutines.flow.StateFlow]. - */ -@ObsoleteCoroutinesApi -public class ConflatedBroadcastChannel() : BroadcastChannel { - /** - * Creates an instance of this class that already holds a value. - * - * It is as a shortcut to creating an instance with a default constructor and - * immediately sending an element: `ConflatedBroadcastChannel().apply { offer(value) }`. - */ - public constructor(value: E) : this() { - _state.lazySet(State(value, null)) - } - - private val _state = atomic(INITIAL_STATE) // State | Closed - private val _updating = atomic(0) - // State transitions: null -> handler -> HANDLER_INVOKED - private val onCloseHandler = atomic(null) - - private companion object { - private val CLOSED = Closed(null) - private val UNDEFINED = Symbol("UNDEFINED") - private val INITIAL_STATE = State(UNDEFINED, null) - } - - private class State( - @JvmField val value: Any?, // UNDEFINED | E - @JvmField val subscribers: Array>? - ) - - private class Closed(@JvmField val closeCause: Throwable?) { - val sendException: Throwable get() = closeCause ?: ClosedSendChannelException(DEFAULT_CLOSE_MESSAGE) - val valueException: Throwable get() = closeCause ?: IllegalStateException(DEFAULT_CLOSE_MESSAGE) - } - - /** - * The most recently sent element to this channel. - * - * Access to this property throws [IllegalStateException] when this class is constructed without - * initial value and no value was sent yet or if it was [closed][close] without a cause. - * It throws the original [close][SendChannel.close] cause exception if the channel has _failed_. - */ - @Suppress("UNCHECKED_CAST") - public val value: E get() { - _state.loop { state -> - when (state) { - is Closed -> throw state.valueException - is State<*> -> { - if (state.value === UNDEFINED) throw IllegalStateException("No value") - return state.value as E - } - else -> error("Invalid state $state") - } - } - } - - /** - * The most recently sent element to this channel or `null` when this class is constructed without - * initial value and no value was sent yet or if it was [closed][close]. - */ - public val valueOrNull: E? get() = when (val state = _state.value) { - is Closed -> null - is State<*> -> UNDEFINED.unbox(state.value) - else -> error("Invalid state $state") - } - - public override val isClosedForSend: Boolean get() = _state.value is Closed - - @Suppress("UNCHECKED_CAST") - public override fun openSubscription(): ReceiveChannel { - val subscriber = Subscriber(this) - _state.loop { state -> - when (state) { - is Closed -> { - subscriber.close(state.closeCause) - return subscriber - } - is State<*> -> { - if (state.value !== UNDEFINED) - subscriber.offerInternal(state.value as E) - val update = State(state.value, addSubscriber((state as State).subscribers, subscriber)) - if (_state.compareAndSet(state, update)) - return subscriber - } - else -> error("Invalid state $state") - } - } - } - - @Suppress("UNCHECKED_CAST") - private fun closeSubscriber(subscriber: Subscriber) { - _state.loop { state -> - when (state) { - is Closed -> return - is State<*> -> { - val update = State(state.value, removeSubscriber((state as State).subscribers!!, subscriber)) - if (_state.compareAndSet(state, update)) - return - } - else -> error("Invalid state $state") - } - } - } - - private fun addSubscriber(list: Array>?, subscriber: Subscriber): Array> { - if (list == null) return Array(1) { subscriber } - return list + subscriber - } - - @Suppress("UNCHECKED_CAST") - private fun removeSubscriber(list: Array>, subscriber: Subscriber): Array>? { - val n = list.size - val i = list.indexOf(subscriber) - assert { i >= 0 } - if (n == 1) return null - val update = arrayOfNulls>(n - 1) - list.copyInto( - destination = update, - endIndex = i - ) - list.copyInto( - destination = update, - destinationOffset = i, - startIndex = i + 1 - ) - return update as Array> - } - - @Suppress("UNCHECKED_CAST") - public override fun close(cause: Throwable?): Boolean { - _state.loop { state -> - when (state) { - is Closed -> return false - is State<*> -> { - val update = if (cause == null) CLOSED else Closed(cause) - if (_state.compareAndSet(state, update)) { - (state as State).subscribers?.forEach { it.close(cause) } - invokeOnCloseHandler(cause) - return true - } - } - else -> error("Invalid state $state") - } - } - } - - private fun invokeOnCloseHandler(cause: Throwable?) { - val handler = onCloseHandler.value - if (handler !== null && handler !== HANDLER_INVOKED - && onCloseHandler.compareAndSet(handler, HANDLER_INVOKED)) { - @Suppress("UNCHECKED_CAST") - (handler as Handler)(cause) - } - } - - override fun invokeOnClose(handler: Handler) { - // Intricate dance for concurrent invokeOnClose and close - if (!onCloseHandler.compareAndSet(null, handler)) { - val value = onCloseHandler.value - if (value === HANDLER_INVOKED) { - throw IllegalStateException("Another handler was already registered and successfully invoked") - } else { - throw IllegalStateException("Another handler was already registered: $value") - } - } else { - val state = _state.value - if (state is Closed && onCloseHandler.compareAndSet(handler, HANDLER_INVOKED)) { - (handler)(state.closeCause) - } - } - } - - /** - * @suppress This method has bad semantics when cause is not a [CancellationException]. Use [cancel]. - */ - @Deprecated(level = DeprecationLevel.HIDDEN, message = "Since 1.2.0, binary compatibility with versions <= 1.1.x") - public override fun cancel(cause: Throwable?): Boolean = close(cause) - - /** - * Cancels this conflated broadcast channel with an optional cause, same as [close]. - * This function closes the channel with - * the specified cause (unless it was already closed), - * and [cancels][ReceiveChannel.cancel] all open subscriptions. - * A cause can be used to specify an error message or to provide other details on - * a cancellation reason for debugging purposes. - */ - public override fun cancel(cause: CancellationException?) { - close(cause) - } - - /** - * Sends the value to all subscribed receives and stores this value as the most recent state for - * future subscribers. This implementation never suspends. - * It throws exception if the channel [isClosedForSend] (see [close] for details). - */ - public override suspend fun send(element: E) { - offerInternal(element)?.let { throw it.sendException } - } - - /** - * Sends the value to all subscribed receives and stores this value as the most recent state for - * future subscribers. This implementation always returns either successful result - * or closed with an exception. - */ - public override fun trySend(element: E): ChannelResult { - offerInternal(element)?.let { return ChannelResult.closed(it.sendException) } - return ChannelResult.success(Unit) - } - - @Suppress("UNCHECKED_CAST") - private fun offerInternal(element: E): Closed? { - // If some other thread is updating the state in its offer operation we assume that our offer had linearized - // before that offer (we lost) and that offer overwrote us and conflated our offer. - if (!_updating.compareAndSet(0, 1)) return null - try { - _state.loop { state -> - when (state) { - is Closed -> return state - is State<*> -> { - val update = State(element, (state as State).subscribers) - if (_state.compareAndSet(state, update)) { - // Note: Using offerInternal here to ignore the case when this subscriber was - // already concurrently closed (assume the close had conflated our offer for this - // particular subscriber). - state.subscribers?.forEach { it.offerInternal(element) } - return null - } - } - else -> error("Invalid state $state") - } - } - } finally { - _updating.value = 0 // reset the updating flag to zero even when something goes wrong - } - } - - @Suppress("UNCHECKED_CAST") - public override val onSend: SelectClause2> - get() = SelectClause2Impl( - clauseObject = this, - regFunc = ConflatedBroadcastChannel<*>::registerSelectForSend as RegistrationFunction, - processResFunc = ConflatedBroadcastChannel<*>::processResultSelectSend as ProcessResultFunction - ) - - @Suppress("UNCHECKED_CAST") - private fun registerSelectForSend(select: SelectInstance<*>, element: Any?) { - select.selectInRegistrationPhase(offerInternal(element as E)) - } - - @Suppress("RedundantNullableReturnType", "UNUSED_PARAMETER") - private fun processResultSelectSend(ignoredParam: Any?, selectResult: Any?): Any? = - if (selectResult is Closed) throw selectResult.sendException - else this - - private class Subscriber( - private val broadcastChannel: ConflatedBroadcastChannel - ) : ConflatedChannel(null), ReceiveChannel { - - override fun onCancelIdempotent(wasClosed: Boolean) { - if (wasClosed) { - broadcastChannel.closeSubscriber(this) - } - } - - public override fun offerInternal(element: E): Any = super.offerInternal(element) - } -} diff --git a/kotlinx-coroutines-core/common/src/channels/ConflatedBufferedChannel.kt b/kotlinx-coroutines-core/common/src/channels/ConflatedBufferedChannel.kt new file mode 100644 index 0000000000..6a9f23e958 --- /dev/null +++ b/kotlinx-coroutines-core/common/src/channels/ConflatedBufferedChannel.kt @@ -0,0 +1,118 @@ +/* + * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines.channels + +import kotlinx.atomicfu.* +import kotlinx.coroutines.channels.BufferOverflow.* +import kotlinx.coroutines.channels.ChannelResult.Companion.closed +import kotlinx.coroutines.channels.ChannelResult.Companion.success +import kotlinx.coroutines.internal.* +import kotlinx.coroutines.internal.OnUndeliveredElement +import kotlinx.coroutines.selects.* +import kotlin.coroutines.* + +/** + * This is a special [BufferedChannel] extension that supports [DROP_OLDEST] and [DROP_LATEST] + * strategies for buffer overflowing. This implementation ensures that `send(e)` never suspends, + * either extracting the first element ([DROP_OLDEST]) or dropping the sending one ([DROP_LATEST]) + * when the channel capacity exceeds. + */ +internal open class ConflatedBufferedChannel( + private val capacity: Int, + private val onBufferOverflow: BufferOverflow, + onUndeliveredElement: OnUndeliveredElement? = null +) : BufferedChannel(capacity = capacity, onUndeliveredElement = onUndeliveredElement) { + init { + require(onBufferOverflow !== SUSPEND) { + "This implementation does not support suspension for senders, use ${BufferedChannel::class.simpleName} instead" + } + require(capacity >= 1) { + "Buffered channel capacity must be at least 1, but $capacity was specified" + } + } + + override val isConflatedDropOldest: Boolean + get() = onBufferOverflow == DROP_OLDEST + + override suspend fun send(element: E) { + // Should never suspend, implement via `trySend(..)`. + trySendImpl(element, isSendOp = true).onClosed { // fails only when this channel is closed. + onUndeliveredElement?.callUndeliveredElementCatchingException(element)?.let { + it.addSuppressed(sendException) + throw it + } + throw sendException + } + } + + override suspend fun sendBroadcast(element: E): Boolean { + // Should never suspend, implement via `trySend(..)`. + trySendImpl(element, isSendOp = true) // fails only when this channel is closed. + .onSuccess { return true } + return false + } + + override fun trySend(element: E): ChannelResult = trySendImpl(element, isSendOp = false) + + private fun trySendImpl(element: E, isSendOp: Boolean) = + if (onBufferOverflow === DROP_LATEST) trySendDropLatest(element, isSendOp) + else trySendDropOldest(element) + + private fun trySendDropLatest(element: E, isSendOp: Boolean): ChannelResult { + // Try to send the element without suspension. + val result = super.trySend(element) + // Complete on success or if this channel is closed. + if (result.isSuccess || result.isClosed) return result + // This channel is full. Drop the sending element. + // Call the `onUndeliveredElement` lambda ONLY for 'send()' invocations, + // for 'trySend()' it is responsibility of the caller + if (isSendOp) { + onUndeliveredElement?.callUndeliveredElementCatchingException(element)?.let { + throw it + } + } + return success(Unit) + } + + private fun trySendDropOldest(element: E): ChannelResult = + sendImpl( // <-- this is an inline function + element = element, + // Put the element into the logical buffer even + // if this channel is already full, the `onSuspend` + // callback below extract the first (oldest) element. + waiter = BUFFERED, + // Finish successfully when a rendezvous has happened + // or the element has been buffered. + onRendezvousOrBuffered = { success(Unit) }, + // In case the algorithm decided to suspend, the element + // was added to the buffer. However, as the buffer is now + // overflowed, the first (oldest) element has to be extracted. + onSuspend = { segm, i -> + dropFirstElementUntilTheSpecifiedCellIsInTheBuffer(segm.id * SEGMENT_SIZE + i) + success(Unit) + }, + // If the channel is closed, return the corresponding result. + onClosed = { closed(sendException) } + ) + + @Suppress("UNCHECKED_CAST") + override fun registerSelectForSend(select: SelectInstance<*>, element: Any?) { + // The plain `send(..)` operation never suspends. Thus, either this + // attempt to send the element succeeds or the channel is closed. + // In any case, complete this `select` in the registration phase. + trySend(element as E).let { + it.onSuccess { + select.selectInRegistrationPhase(Unit) + return + }.onClosed { + select.selectInRegistrationPhase(CHANNEL_CLOSED) + return + } + } + error("unreachable") + } + + override fun shouldSendSuspend() = false // never suspends. +} diff --git a/kotlinx-coroutines-core/common/src/channels/ConflatedChannel.kt b/kotlinx-coroutines-core/common/src/channels/ConflatedChannel.kt deleted file mode 100644 index 54033a8764..0000000000 --- a/kotlinx-coroutines-core/common/src/channels/ConflatedChannel.kt +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. - */ - -package kotlinx.coroutines.channels - -import kotlinx.coroutines.* -import kotlinx.coroutines.internal.* - -/** - * Channel that buffers at most one element and conflates all subsequent `send` and `trySend` invocations, - * so that the receiver always gets the most recently sent element. - * Back-to-send sent elements are _conflated_ -- only the most recently sent element is received, - * while previously sent elements **are lost**. - * Sender to this channel never suspends and [trySend] always succeeds. - * - * This channel is created by `Channel(Channel.CONFLATED)` factory function invocation. - */ -internal open class ConflatedChannel(onUndeliveredElement: OnUndeliveredElement?) : AbstractChannel(onUndeliveredElement) { - protected final override val isBufferAlwaysEmpty: Boolean get() = false - protected final override val isBufferEmpty: Boolean get() = lock.withLock { value === EMPTY } - protected final override val isBufferAlwaysFull: Boolean get() = false - protected final override val isBufferFull: Boolean get() = false - - override val isEmpty: Boolean get() = lock.withLock { isEmptyImpl } - - private val lock = ReentrantLock() - - private var value: Any? = EMPTY - - // result is `OFFER_SUCCESS | Closed` - protected override fun offerInternal(element: E): Any { - var receive: ReceiveOrClosed? = null - lock.withLock { - closedForSend?.let { return it } - // if there is no element written in buffer - if (value === EMPTY) { - // check for receivers that were waiting on the empty buffer - loop@ while(true) { - receive = takeFirstReceiveOrPeekClosed() ?: break@loop // break when no receivers queued - if (receive is Closed) { - return receive!! - } - val token = receive!!.tryResumeReceive(element) - if (token != null) { - assert { token === RESUME_TOKEN } - return@withLock - } - } - } - updateValueLocked(element)?.let { throw it } - return OFFER_SUCCESS - } - // breaks here if offer meets receiver - receive!!.completeResumeReceive(element) - return receive!!.offerResult - } - - // result is `E | POLL_FAILED | Closed` - protected override fun pollInternal(): Any? { - var result: Any? = null - lock.withLock { - if (value === EMPTY) return closedForSend ?: POLL_FAILED - result = value - value = EMPTY - } - return result - } - - protected override fun onCancelIdempotent(wasClosed: Boolean) { - var undeliveredElementException: UndeliveredElementException? = null // resource cancel exception - lock.withLock { - undeliveredElementException = updateValueLocked(EMPTY) - } - super.onCancelIdempotent(wasClosed) - undeliveredElementException?.let { throw it } // throw UndeliveredElementException at the end if there was one - } - - @Suppress("UNCHECKED_CAST") - private fun updateValueLocked(element: Any?): UndeliveredElementException? { - val old = value - val undeliveredElementException = if (old === EMPTY) null else - onUndeliveredElement?.callUndeliveredElementCatchingException(old as E) - value = element - return undeliveredElementException - } - - override fun enqueueReceiveInternal(receive: Receive): Boolean = lock.withLock { - super.enqueueReceiveInternal(receive) - } - - // ------ debug ------ - - override val bufferDebugString: String - get() = lock.withLock { "(value=$value)" } -} diff --git a/kotlinx-coroutines-core/common/src/channels/LinkedListChannel.kt b/kotlinx-coroutines-core/common/src/channels/LinkedListChannel.kt deleted file mode 100644 index 18bdcc6465..0000000000 --- a/kotlinx-coroutines-core/common/src/channels/LinkedListChannel.kt +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. - */ - -package kotlinx.coroutines.channels - -import kotlinx.coroutines.internal.* - -/** - * Channel with linked-list buffer of a unlimited capacity (limited only by available memory). - * Sender to this channel never suspends and [trySend] always succeeds. - * - * This channel is created by `Channel(Channel.UNLIMITED)` factory function invocation. - * - * This implementation is fully lock-free. - * - * @suppress **This an internal API and should not be used from general code.** - */ -internal open class LinkedListChannel(onUndeliveredElement: OnUndeliveredElement?) : AbstractChannel(onUndeliveredElement) { - protected final override val isBufferAlwaysEmpty: Boolean get() = true - protected final override val isBufferEmpty: Boolean get() = true - protected final override val isBufferAlwaysFull: Boolean get() = false - protected final override val isBufferFull: Boolean get() = false - - // result is always `OFFER_SUCCESS | Closed` - protected override fun offerInternal(element: E): Any { - while (true) { - val result = super.offerInternal(element) - when { - result === OFFER_SUCCESS -> return OFFER_SUCCESS - result === OFFER_FAILED -> { // try to buffer - when (val sendResult = sendBuffered(element)) { - null -> return OFFER_SUCCESS - is Closed<*> -> return sendResult - } - // otherwise there was receiver in queue, retry super.offerInternal - } - result is Closed<*> -> return result - else -> error("Invalid offerInternal result $result") - } - } - } - - override fun onCancelIdempotentList(list: InlineList, closed: Closed<*>) { - var undeliveredElementException: UndeliveredElementException? = null - list.forEachReversed { - when (it) { - is SendBuffered<*> -> { - @Suppress("UNCHECKED_CAST") - undeliveredElementException = onUndeliveredElement?.callUndeliveredElementCatchingException(it.element as E, undeliveredElementException) - } - else -> it.resumeSendClosed(closed) - } - } - undeliveredElementException?.let { throw it } // throw UndeliveredElementException at the end if there was one - } -} - diff --git a/kotlinx-coroutines-core/common/src/channels/RendezvousChannel.kt b/kotlinx-coroutines-core/common/src/channels/RendezvousChannel.kt deleted file mode 100644 index e8ade513f5..0000000000 --- a/kotlinx-coroutines-core/common/src/channels/RendezvousChannel.kt +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright 2016-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. - */ - -package kotlinx.coroutines.channels - -import kotlinx.coroutines.internal.* - -/** - * Rendezvous channel. This channel does not have any buffer at all. An element is transferred from sender - * to receiver only when [send] and [receive] invocations meet in time (rendezvous), so [send] suspends - * until another coroutine invokes [receive] and [receive] suspends until another coroutine invokes [send]. - * - * Use `Channel()` factory function to conveniently create an instance of rendezvous channel. - * - * This implementation is fully lock-free. - **/ -internal open class RendezvousChannel(onUndeliveredElement: OnUndeliveredElement?) : AbstractChannel(onUndeliveredElement) { - protected final override val isBufferAlwaysEmpty: Boolean get() = true - protected final override val isBufferEmpty: Boolean get() = true - protected final override val isBufferAlwaysFull: Boolean get() = true - protected final override val isBufferFull: Boolean get() = true -} diff --git a/kotlinx-coroutines-core/common/src/internal/Concurrent.common.kt b/kotlinx-coroutines-core/common/src/internal/Concurrent.common.kt index 6b42dd15db..848a42c867 100644 --- a/kotlinx-coroutines-core/common/src/internal/Concurrent.common.kt +++ b/kotlinx-coroutines-core/common/src/internal/Concurrent.common.kt @@ -4,18 +4,6 @@ package kotlinx.coroutines.internal -/** - * Special kind of list intended to be used as collection of subscribers in `ArrayBroadcastChannel` - * On JVM it's CopyOnWriteList and on JS it's MutableList. - * - * Note that this alias is intentionally not named as CopyOnWriteList to avoid accidental misusage outside of the ArrayBroadcastChannel - */ -internal typealias SubscribersList = MutableList - -@Deprecated(message = "Implementation of this primitive is tailored to specific ArrayBroadcastChannel usages on K/N " + - "and K/JS platforms and it is unsafe to use it anywhere else") -internal expect fun subscriberList(): SubscribersList - internal expect class ReentrantLock() { fun tryLock(): Boolean fun unlock() diff --git a/kotlinx-coroutines-core/common/src/internal/ConcurrentLinkedList.kt b/kotlinx-coroutines-core/common/src/internal/ConcurrentLinkedList.kt index 5eee2bd59c..2bcf97b7ad 100644 --- a/kotlinx-coroutines-core/common/src/internal/ConcurrentLinkedList.kt +++ b/kotlinx-coroutines-core/common/src/internal/ConcurrentLinkedList.kt @@ -14,7 +14,7 @@ import kotlin.jvm.* */ private inline fun > S.findSegmentInternal( id: Long, - createNewSegment: (id: Long, prev: S?) -> S + createNewSegment: (id: Long, prev: S) -> S ): SegmentOrClosed { /* Go through `next` references and add new segments if needed, similarly to the `push` in the Michael-Scott @@ -22,7 +22,7 @@ private inline fun > S.findSegmentInternal( added, so the algorithm just uses it. This way, only one segment with each id can be added. */ var cur: S = this - while (cur.id < id || cur.removed) { + while (cur.id < id || cur.isRemoved) { val next = cur.nextOrIfClosed { return SegmentOrClosed(CLOSED) } if (next != null) { // there is a next node -- move there cur = next @@ -30,7 +30,7 @@ private inline fun > S.findSegmentInternal( } val newTail = createNewSegment(cur.id + 1, cur) if (cur.trySetNext(newTail)) { // successfully added new node -- move there - if (cur.removed) cur.remove() + if (cur.isRemoved) cur.remove() cur = newTail } } @@ -40,8 +40,8 @@ private inline fun > S.findSegmentInternal( /** * Returns `false` if the segment `to` is logically removed, `true` on a successful update. */ -@Suppress("NOTHING_TO_INLINE") // Must be inline because it is an AtomicRef extension -private inline fun > AtomicRef.moveForward(to: S): Boolean = loop { cur -> +@Suppress("NOTHING_TO_INLINE", "RedundantNullableReturnType") // Must be inline because it is an AtomicRef extension +internal inline fun > AtomicRef.moveForward(to: S): Boolean = loop { cur -> if (cur.id >= to.id) return true if (!to.tryIncPointers()) return false if (compareAndSet(cur, to)) { // the segment is moved @@ -65,7 +65,7 @@ private inline fun > AtomicRef.moveForward(to: S): Boolean = l internal inline fun > AtomicRef.findSegmentAndMoveForward( id: Long, startFrom: S, - createNewSegment: (id: Long, prev: S?) -> S + createNewSegment: (id: Long, prev: S) -> S ): SegmentOrClosed { while (true) { val s = startFrom.findSegmentInternal(id, createNewSegment) @@ -136,47 +136,49 @@ internal abstract class ConcurrentLinkedListNode /** * This property indicates whether the current node is logically removed. - * The expected use-case is removing the node logically (so that [removed] becomes true), + * The expected use-case is removing the node logically (so that [isRemoved] becomes true), * and invoking [remove] after that. Note that this implementation relies on the contract * that the physical tail cannot be logically removed. Please, do not break this contract; * otherwise, memory leaks and unexpected behavior can occur. */ - abstract val removed: Boolean + abstract val isRemoved: Boolean /** * Removes this node physically from this linked list. The node should be - * logically removed (so [removed] returns `true`) at the point of invocation. + * logically removed (so [isRemoved] returns `true`) at the point of invocation. */ fun remove() { - assert { removed } // The node should be logically removed at first. - assert { !isTail } // The physical tail cannot be removed. + assert { isRemoved || isTail } // The node should be logically removed at first. + // The physical tail cannot be removed. Instead, we remove it when + // a new segment is added and this segment is not the tail one anymore. + if (isTail) return while (true) { // Read `next` and `prev` pointers ignoring logically removed nodes. - val prev = leftmostAliveNode - val next = rightmostAliveNode + val prev = aliveSegmentLeft + val next = aliveSegmentRight // Link `next` and `prev`. - next._prev.value = prev + next._prev.update { if (it === null) null else prev } if (prev !== null) prev._next.value = next // Checks that prev and next are still alive. - if (next.removed) continue - if (prev !== null && prev.removed) continue + if (next.isRemoved && !next.isTail) continue + if (prev !== null && prev.isRemoved) continue // This node is removed. return } } - private val leftmostAliveNode: N? get() { + private val aliveSegmentLeft: N? get() { var cur = prev - while (cur !== null && cur.removed) + while (cur !== null && cur.isRemoved) cur = cur._prev.value return cur } - private val rightmostAliveNode: N get() { + private val aliveSegmentRight: N get() { assert { !isTail } // Should not be invoked on the tail node var cur = next!! - while (cur.removed) - cur = cur.next!! + while (cur.isRemoved) + cur = cur.next ?: return cur return cur } } @@ -188,10 +190,10 @@ internal abstract class ConcurrentLinkedListNode */ internal abstract class Segment>(val id: Long, prev: S?, pointers: Int): ConcurrentLinkedListNode(prev) { /** - * This property should return the maximal number of slots in this segment, + * This property should return the number of slots in this segment, * it is used to define whether the segment is logically removed. */ - abstract val maxSlots: Int + abstract val numberOfSlots: Int /** * Numbers of cleaned slots (the lowest bits) and AtomicRef pointers to this segment (the highest bits) @@ -199,23 +201,22 @@ internal abstract class Segment>(val id: Long, prev: S?, pointers private val cleanedAndPointers = atomic(pointers shl POINTERS_SHIFT) /** - * The segment is considered as removed if all the slots are cleaned. - * There are no pointers to this segment from outside, and - * it is not a physical tail in the linked list of segments. + * The segment is considered as removed if all the slots are cleaned + * and there are no pointers to this segment from outside. */ - override val removed get() = cleanedAndPointers.value == maxSlots && !isTail + override val isRemoved get() = cleanedAndPointers.value == numberOfSlots && !isTail // increments the number of pointers if this segment is not logically removed. - internal fun tryIncPointers() = cleanedAndPointers.addConditionally(1 shl POINTERS_SHIFT) { it != maxSlots || isTail } + internal fun tryIncPointers() = cleanedAndPointers.addConditionally(1 shl POINTERS_SHIFT) { it != numberOfSlots || isTail } // returns `true` if this segment is logically removed after the decrement. - internal fun decPointers() = cleanedAndPointers.addAndGet(-(1 shl POINTERS_SHIFT)) == maxSlots && !isTail + internal fun decPointers() = cleanedAndPointers.addAndGet(-(1 shl POINTERS_SHIFT)) == numberOfSlots && !isTail /** * Invoked on each slot clean-up; should not be invoked twice for the same slot. */ fun onSlotCleaned() { - if (cleanedAndPointers.incrementAndGet() == maxSlots && !isTail) remove() + if (cleanedAndPointers.incrementAndGet() == numberOfSlots) remove() } } diff --git a/kotlinx-coroutines-core/common/src/selects/Select.kt b/kotlinx-coroutines-core/common/src/selects/Select.kt index 5da5e95702..b9d128b7f8 100644 --- a/kotlinx-coroutines-core/common/src/selects/Select.kt +++ b/kotlinx-coroutines-core/common/src/selects/Select.kt @@ -238,7 +238,7 @@ public sealed interface SelectInstance { */ public fun selectInRegistrationPhase(internalResult: Any?) } -internal interface SelectInstanceInternal: SelectInstance +internal interface SelectInstanceInternal: SelectInstance, Waiter @PublishedApi internal open class SelectImplementation constructor( @@ -705,7 +705,11 @@ internal open class SelectImplementation constructor( // Update the state. state.update { cur -> // Finish immediately when this `select` is already completed. - if (cur is ClauseData<*> || cur == STATE_COMPLETED) return + // Notably, this select might be logically completed + // (the `state` field stores the selected `ClauseData`), + // while the continuation is already cancelled. + // We need to invoke the cancellation handler in this case. + if (cur === STATE_COMPLETED) return STATE_CANCELLED } // Read the list of clauses. If the `clauses` field is already `null`, diff --git a/kotlinx-coroutines-core/common/src/sync/Mutex.kt b/kotlinx-coroutines-core/common/src/sync/Mutex.kt index 40e4308f4e..fbd1fe55f4 100644 --- a/kotlinx-coroutines-core/common/src/sync/Mutex.kt +++ b/kotlinx-coroutines-core/common/src/sync/Mutex.kt @@ -247,9 +247,8 @@ internal open class MutexImpl(locked: Boolean) : SemaphoreImpl(1, if (locked) 1 ) : SelectInstanceInternal by select as SelectInstanceInternal { override fun trySelect(clauseObject: Any, result: Any?): Boolean { assert { this@MutexImpl.owner.value === NO_OWNER } - this@MutexImpl.owner.value = owner return select.trySelect(clauseObject, result).also { success -> - if (!success) this@MutexImpl.owner.value = NO_OWNER + if (success) this@MutexImpl.owner.value = owner } } diff --git a/kotlinx-coroutines-core/common/src/sync/Semaphore.kt b/kotlinx-coroutines-core/common/src/sync/Semaphore.kt index 9ba6eaf36b..4db8ae3ca6 100644 --- a/kotlinx-coroutines-core/common/src/sync/Semaphore.kt +++ b/kotlinx-coroutines-core/common/src/sync/Semaphore.kt @@ -375,11 +375,11 @@ private class CancelSemaphoreAcquisitionHandler( override fun toString() = "CancelSemaphoreAcquisitionHandler[$segment, $index]" } -private fun createSegment(id: Long, prev: SemaphoreSegment?) = SemaphoreSegment(id, prev, 0) +private fun createSegment(id: Long, prev: SemaphoreSegment) = SemaphoreSegment(id, prev, 0) private class SemaphoreSegment(id: Long, prev: SemaphoreSegment?, pointers: Int) : Segment(id, prev, pointers) { val acquirers = atomicArrayOfNulls(SEGMENT_SIZE) - override val maxSlots: Int get() = SEGMENT_SIZE + override val numberOfSlots: Int get() = SEGMENT_SIZE @Suppress("NOTHING_TO_INLINE") inline fun get(index: Int): Any? = acquirers[index].value diff --git a/kotlinx-coroutines-core/common/test/channels/BasicOperationsTest.kt b/kotlinx-coroutines-core/common/test/channels/BasicOperationsTest.kt index 4538f6c680..aeb6199134 100644 --- a/kotlinx-coroutines-core/common/test/channels/BasicOperationsTest.kt +++ b/kotlinx-coroutines-core/common/test/channels/BasicOperationsTest.kt @@ -21,7 +21,7 @@ class BasicOperationsTest : TestBase() { @Test fun testTrySendAfterClose() = runTest { - TestChannelKind.values().forEach { kind -> testTrySend(kind) } + TestChannelKind.values().forEach { kind -> testTrySendAfterClose(kind) } } @Test @@ -114,7 +114,7 @@ class BasicOperationsTest : TestBase() { finish(6) } - private suspend fun testTrySend(kind: TestChannelKind) = coroutineScope { + private suspend fun testTrySendAfterClose(kind: TestChannelKind) = coroutineScope { val channel = kind.create() val d = async { channel.send(42) } yield() diff --git a/kotlinx-coroutines-core/common/test/channels/BroadcastChannelFactoryTest.kt b/kotlinx-coroutines-core/common/test/channels/BroadcastChannelFactoryTest.kt index 61e93fa8ea..e27edcf6f7 100644 --- a/kotlinx-coroutines-core/common/test/channels/BroadcastChannelFactoryTest.kt +++ b/kotlinx-coroutines-core/common/test/channels/BroadcastChannelFactoryTest.kt @@ -16,7 +16,7 @@ class BroadcastChannelFactoryTest : TestBase() { } @Test - fun testLinkedListChannelNotSupported() { + fun testUnlimitedChannelNotSupported() { assertFailsWith { BroadcastChannel(Channel.UNLIMITED) } } @@ -26,9 +26,9 @@ class BroadcastChannelFactoryTest : TestBase() { } @Test - fun testArrayBroadcastChannel() { - assertTrue { BroadcastChannel(1) is ArrayBroadcastChannel } - assertTrue { BroadcastChannel(10) is ArrayBroadcastChannel } + fun testBufferedBroadcastChannel() { + assertTrue { BroadcastChannel(1) is BroadcastChannelImpl } + assertTrue { BroadcastChannel(10) is BroadcastChannelImpl } } @Test diff --git a/kotlinx-coroutines-core/common/test/channels/BroadcastTest.kt b/kotlinx-coroutines-core/common/test/channels/BroadcastTest.kt index ab1a85d697..34b1395564 100644 --- a/kotlinx-coroutines-core/common/test/channels/BroadcastTest.kt +++ b/kotlinx-coroutines-core/common/test/channels/BroadcastTest.kt @@ -7,6 +7,7 @@ package kotlinx.coroutines.channels import kotlinx.coroutines.* +import kotlinx.coroutines.selects.* import kotlin.test.* class BroadcastTest : TestBase() { @@ -17,7 +18,7 @@ class BroadcastTest : TestBase() { expect(4) send(1) // goes to receiver expect(5) - send(2) // goes to buffer + select { onSend(2) {} } // goes to buffer expect(6) send(3) // suspends, will not be consumes, but will not be cancelled either expect(10) diff --git a/kotlinx-coroutines-core/common/test/channels/ArrayBroadcastChannelTest.kt b/kotlinx-coroutines-core/common/test/channels/BufferedBroadcastChannelTest.kt similarity index 99% rename from kotlinx-coroutines-core/common/test/channels/ArrayBroadcastChannelTest.kt rename to kotlinx-coroutines-core/common/test/channels/BufferedBroadcastChannelTest.kt index 2d71cc94ed..fad6500805 100644 --- a/kotlinx-coroutines-core/common/test/channels/ArrayBroadcastChannelTest.kt +++ b/kotlinx-coroutines-core/common/test/channels/BufferedBroadcastChannelTest.kt @@ -7,7 +7,7 @@ package kotlinx.coroutines.channels import kotlinx.coroutines.* import kotlin.test.* -class ArrayBroadcastChannelTest : TestBase() { +class BufferedBroadcastChannelTest : TestBase() { @Test fun testConcurrentModification() = runTest { diff --git a/kotlinx-coroutines-core/common/test/channels/ArrayChannelTest.kt b/kotlinx-coroutines-core/common/test/channels/BufferedChannelTest.kt similarity index 89% rename from kotlinx-coroutines-core/common/test/channels/ArrayChannelTest.kt rename to kotlinx-coroutines-core/common/test/channels/BufferedChannelTest.kt index 632fd2928b..0f7035214b 100644 --- a/kotlinx-coroutines-core/common/test/channels/ArrayChannelTest.kt +++ b/kotlinx-coroutines-core/common/test/channels/BufferedChannelTest.kt @@ -7,7 +7,7 @@ package kotlinx.coroutines.channels import kotlinx.coroutines.* import kotlin.test.* -class ArrayChannelTest : TestBase() { +class BufferedChannelTest : TestBase() { @Test fun testSimple() = runTest { val q = Channel(1) @@ -34,6 +34,7 @@ class ArrayChannelTest : TestBase() { sender.join() receiver.join() check(q.isEmpty) + (q as BufferedChannel<*>).checkSegmentStructureInvariants() finish(10) } @@ -59,6 +60,7 @@ class ArrayChannelTest : TestBase() { check(!q.isEmpty && q.isClosedForSend && !q.isClosedForReceive) yield() check(!q.isEmpty && q.isClosedForSend && q.isClosedForReceive) + (q as BufferedChannel<*>).checkSegmentStructureInvariants() finish(8) } @@ -81,6 +83,7 @@ class ArrayChannelTest : TestBase() { expect(6) try { q.send(42) } catch (e: ClosedSendChannelException) { + (q as BufferedChannel<*>).checkSegmentStructureInvariants() finish(7) } } @@ -112,6 +115,7 @@ class ArrayChannelTest : TestBase() { expect(8) assertFalse(q.trySend(4).isSuccess) yield() + (q as BufferedChannel<*>).checkSegmentStructureInvariants() finish(12) } @@ -135,6 +139,7 @@ class ArrayChannelTest : TestBase() { check(q.isClosedForSend) check(q.isClosedForReceive) assertFailsWith { q.receiveCatching().getOrThrow() } + (q as BufferedChannel<*>).checkSegmentStructureInvariants() finish(12) } @@ -165,6 +170,11 @@ class ArrayChannelTest : TestBase() { checkBufferChannel(channel, capacity) } + @Test + fun testBufferIsNotPreallocated() { + (0..100_000).map { Channel(Int.MAX_VALUE / 2) } + } + private suspend fun CoroutineScope.checkBufferChannel( channel: Channel, capacity: Int @@ -189,6 +199,7 @@ class ArrayChannelTest : TestBase() { result.add(it) } assertEquals((0..capacity).toList(), result) + (channel as BufferedChannel<*>).checkSegmentStructureInvariants() finish(6) } } diff --git a/kotlinx-coroutines-core/common/test/channels/ChannelFactoryTest.kt b/kotlinx-coroutines-core/common/test/channels/ChannelFactoryTest.kt index 413c91f5a7..706a2fdd0a 100644 --- a/kotlinx-coroutines-core/common/test/channels/ChannelFactoryTest.kt +++ b/kotlinx-coroutines-core/common/test/channels/ChannelFactoryTest.kt @@ -11,28 +11,28 @@ import kotlin.test.* class ChannelFactoryTest : TestBase() { @Test fun testRendezvousChannel() { - assertTrue(Channel() is RendezvousChannel) - assertTrue(Channel(0) is RendezvousChannel) + assertTrue(Channel() is BufferedChannel) + assertTrue(Channel(0) is BufferedChannel) } @Test - fun testLinkedListChannel() { - assertTrue(Channel(Channel.UNLIMITED) is LinkedListChannel) - assertTrue(Channel(Channel.UNLIMITED, BufferOverflow.DROP_OLDEST) is LinkedListChannel) - assertTrue(Channel(Channel.UNLIMITED, BufferOverflow.DROP_LATEST) is LinkedListChannel) + fun testUnlimitedChannel() { + assertTrue(Channel(Channel.UNLIMITED) is BufferedChannel) + assertTrue(Channel(Channel.UNLIMITED, BufferOverflow.DROP_OLDEST) is BufferedChannel) + assertTrue(Channel(Channel.UNLIMITED, BufferOverflow.DROP_LATEST) is BufferedChannel) } @Test fun testConflatedChannel() { - assertTrue(Channel(Channel.CONFLATED) is ConflatedChannel) - assertTrue(Channel(1, BufferOverflow.DROP_OLDEST) is ConflatedChannel) + assertTrue(Channel(Channel.CONFLATED) is ConflatedBufferedChannel) + assertTrue(Channel(1, BufferOverflow.DROP_OLDEST) is ConflatedBufferedChannel) } @Test - fun testArrayChannel() { - assertTrue(Channel(1) is ArrayChannel) - assertTrue(Channel(1, BufferOverflow.DROP_LATEST) is ArrayChannel) - assertTrue(Channel(10) is ArrayChannel) + fun testBufferedChannel() { + assertTrue(Channel(1) is BufferedChannel) + assertTrue(Channel(1, BufferOverflow.DROP_LATEST) is ConflatedBufferedChannel) + assertTrue(Channel(10) is BufferedChannel) } @Test diff --git a/kotlinx-coroutines-core/common/test/channels/ChannelUndeliveredElementFailureTest.kt b/kotlinx-coroutines-core/common/test/channels/ChannelUndeliveredElementFailureTest.kt index ae05fb8d74..f02bd09a66 100644 --- a/kotlinx-coroutines-core/common/test/channels/ChannelUndeliveredElementFailureTest.kt +++ b/kotlinx-coroutines-core/common/test/channels/ChannelUndeliveredElementFailureTest.kt @@ -107,11 +107,71 @@ class ChannelUndeliveredElementFailureTest : TestBase() { } @Test - fun testChannelCancelledFail() = runTest(expected = { it.isElementCancelException()}) { + fun testChannelCancelledFail() = runTest(expected = { it.isElementCancelException() }) { val channel = Channel(1, onUndeliveredElement = onCancelFail) channel.send(item) channel.cancel() expectUnreached() } + @Test + fun testFailedHandlerInClosedConflatedChannel() = runTest(expected = { it is UndeliveredElementException }) { + val conflated = Channel(Channel.CONFLATED, onUndeliveredElement = { + finish(2) + throw TestException() + }) + expect(1) + conflated.close(IndexOutOfBoundsException()) + conflated.send(3) + } + + @Test + fun testFailedHandlerInClosedBufferedChannel() = runTest(expected = { it is UndeliveredElementException }) { + val conflated = Channel(3, onUndeliveredElement = { + finish(2) + throw TestException() + }) + expect(1) + conflated.close(IndexOutOfBoundsException()) + conflated.send(3) + } + + @Test + fun testSendDropOldestInvokeHandlerBuffered() = runTest(expected = { it is UndeliveredElementException }) { + val channel = Channel(1, BufferOverflow.DROP_OLDEST, onUndeliveredElement = { + finish(2) + throw TestException() + }) + + channel.send(42) + expect(1) + channel.send(12) + } + + @Test + fun testSendDropLatestInvokeHandlerBuffered() = runTest(expected = { it is UndeliveredElementException }) { + val channel = Channel(2, BufferOverflow.DROP_LATEST, onUndeliveredElement = { + finish(2) + throw TestException() + }) + + channel.send(42) + channel.send(12) + expect(1) + channel.send(12) + expectUnreached() + } + + @Test + fun testSendDropOldestInvokeHandlerConflated() = runTest(expected = { it is UndeliveredElementException }) { + val channel = Channel(Channel.CONFLATED, onUndeliveredElement = { + finish(2) + println(TestException().stackTraceToString()) + throw TestException() + }) + channel.send(42) + expect(1) + channel.send(42) + expectUnreached() + } } diff --git a/kotlinx-coroutines-core/common/test/channels/ChannelUndeliveredElementTest.kt b/kotlinx-coroutines-core/common/test/channels/ChannelUndeliveredElementTest.kt index f26361f2f8..3a99484630 100644 --- a/kotlinx-coroutines-core/common/test/channels/ChannelUndeliveredElementTest.kt +++ b/kotlinx-coroutines-core/common/test/channels/ChannelUndeliveredElementTest.kt @@ -138,4 +138,63 @@ class ChannelUndeliveredElementTest : TestBase() { channel.send(Unit) finish(3) } + + @Test + fun testChannelBufferOverflow() = runTest { + testBufferOverflowStrategy(listOf(1, 2), BufferOverflow.DROP_OLDEST) + testBufferOverflowStrategy(listOf(3), BufferOverflow.DROP_LATEST) + } + + private suspend fun testBufferOverflowStrategy(expectedDroppedElements: List, strategy: BufferOverflow) { + val list = ArrayList() + val channel = Channel( + capacity = 2, + onBufferOverflow = strategy, + onUndeliveredElement = { value -> list.add(value) } + ) + + channel.send(1) + channel.send(2) + + channel.send(3) + channel.trySend(4).onFailure { expectUnreached() } + assertEquals(expectedDroppedElements, list) + } + + + @Test + fun testTrySendDoesNotInvokeHandlerOnClosedConflatedChannel() = runTest { + val conflated = Channel(Channel.CONFLATED, onUndeliveredElement = { + expectUnreached() + }) + conflated.close(IndexOutOfBoundsException()) + conflated.trySend(3) + } + + @Test + fun testTrySendDoesNotInvokeHandlerOnClosedChannel() = runTest { + val conflated = Channel(3, onUndeliveredElement = { + expectUnreached() + }) + conflated.close(IndexOutOfBoundsException()) + repeat(10) { + conflated.trySend(3) + } + } + + @Test + fun testTrySendDoesNotInvokeHandler() { + for (capacity in 0..2) { + testTrySendDoesNotInvokeHandler(capacity) + } + } + + private fun testTrySendDoesNotInvokeHandler(capacity: Int) { + val channel = Channel(capacity, BufferOverflow.DROP_LATEST, onUndeliveredElement = { + expectUnreached() + }) + repeat(10) { + channel.trySend(3) + } + } } diff --git a/kotlinx-coroutines-core/common/test/channels/ChannelsTest.kt b/kotlinx-coroutines-core/common/test/channels/ChannelsTest.kt index fb704c5b86..e40071b91e 100644 --- a/kotlinx-coroutines-core/common/test/channels/ChannelsTest.kt +++ b/kotlinx-coroutines-core/common/test/channels/ChannelsTest.kt @@ -20,7 +20,10 @@ class ChannelsTest: TestBase() { } @Test - fun testCloseWithMultipleWaiters() = runTest { + fun testCloseWithMultipleSuspendedReceivers() = runTest { + // Once the channel is closed, the waiting + // requests should be cancelled in the order + // they were suspended in the channel. val channel = Channel() launch { try { @@ -50,6 +53,40 @@ class ChannelsTest: TestBase() { finish(7) } + @Test + fun testCloseWithMultipleSuspendedSenders() = runTest { + // Once the channel is closed, the waiting + // requests should be cancelled in the order + // they were suspended in the channel. + val channel = Channel() + launch { + try { + expect(2) + channel.send(42) + expectUnreached() + } catch (e: CancellationException) { + expect(5) + } + } + + launch { + try { + expect(3) + channel.send(42) + expectUnreached() + } catch (e: CancellationException) { + expect(6) + } + } + + expect(1) + yield() + expect(4) + channel.cancel() + yield() + finish(7) + } + @Test fun testEmptyList() = runTest { assertTrue(emptyList().asReceiveChannel().toList().isEmpty()) diff --git a/kotlinx-coroutines-core/common/test/channels/ConflatedChannelArrayModelTest.kt b/kotlinx-coroutines-core/common/test/channels/ConflatedChannelArrayModelTest.kt deleted file mode 100644 index e80309be89..0000000000 --- a/kotlinx-coroutines-core/common/test/channels/ConflatedChannelArrayModelTest.kt +++ /dev/null @@ -1,11 +0,0 @@ -/* - * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. - */ - -package kotlinx.coroutines.channels - -// Test that ArrayChannel(1, DROP_OLDEST) works just like ConflatedChannel() -class ConflatedChannelArrayModelTest : ConflatedChannelTest() { - override fun createConflatedChannel(): Channel = - ArrayChannel(1, BufferOverflow.DROP_OLDEST, null) -} diff --git a/kotlinx-coroutines-core/common/test/channels/SendReceiveStressTest.kt b/kotlinx-coroutines-core/common/test/channels/SendReceiveStressTest.kt index b9aa9990b0..dcbb2d2f15 100644 --- a/kotlinx-coroutines-core/common/test/channels/SendReceiveStressTest.kt +++ b/kotlinx-coroutines-core/common/test/channels/SendReceiveStressTest.kt @@ -5,7 +5,6 @@ package kotlinx.coroutines.channels import kotlinx.coroutines.* -import kotlin.coroutines.* import kotlin.test.* class SendReceiveStressTest : TestBase() { @@ -13,12 +12,12 @@ class SendReceiveStressTest : TestBase() { // Emulate parametrized by hand :( @Test - fun testArrayChannel() = runTest { + fun testBufferedChannel() = runTest { testStress(Channel(2)) } @Test - fun testLinkedListChannel() = runTest { + fun testUnlimitedChannel() = runTest { testStress(Channel(Channel.UNLIMITED)) } diff --git a/kotlinx-coroutines-core/common/test/channels/TestBroadcastChannelKind.kt b/kotlinx-coroutines-core/common/test/channels/TestBroadcastChannelKind.kt index d58c05da46..94a488763f 100644 --- a/kotlinx-coroutines-core/common/test/channels/TestBroadcastChannelKind.kt +++ b/kotlinx-coroutines-core/common/test/channels/TestBroadcastChannelKind.kt @@ -7,11 +7,11 @@ package kotlinx.coroutines.channels enum class TestBroadcastChannelKind { ARRAY_1 { override fun create(): BroadcastChannel = BroadcastChannel(1) - override fun toString(): String = "ArrayBroadcastChannel(1)" + override fun toString(): String = "BufferedBroadcastChannel(1)" }, ARRAY_10 { override fun create(): BroadcastChannel = BroadcastChannel(10) - override fun toString(): String = "ArrayBroadcastChannel(10)" + override fun toString(): String = "BufferedBroadcastChannel(10)" }, CONFLATED { override fun create(): BroadcastChannel = ConflatedBroadcastChannel() diff --git a/kotlinx-coroutines-core/common/test/channels/TestChannelKind.kt b/kotlinx-coroutines-core/common/test/channels/TestChannelKind.kt index 227e690ce8..305c0eea7f 100644 --- a/kotlinx-coroutines-core/common/test/channels/TestChannelKind.kt +++ b/kotlinx-coroutines-core/common/test/channels/TestChannelKind.kt @@ -13,13 +13,13 @@ enum class TestChannelKind( val viaBroadcast: Boolean = false ) { RENDEZVOUS(0, "RendezvousChannel"), - ARRAY_1(1, "ArrayChannel(1)"), - ARRAY_2(2, "ArrayChannel(2)"), - ARRAY_10(10, "ArrayChannel(10)"), - LINKED_LIST(Channel.UNLIMITED, "LinkedListChannel"), + BUFFERED_1(1, "BufferedChannel(1)"), + BUFFERED_2(2, "BufferedChannel(2)"), + BUFFERED_10(10, "BufferedChannel(10)"), + UNLIMITED(Channel.UNLIMITED, "UnlimitedChannel"), CONFLATED(Channel.CONFLATED, "ConflatedChannel"), - ARRAY_1_BROADCAST(1, "ArrayBroadcastChannel(1)", viaBroadcast = true), - ARRAY_10_BROADCAST(10, "ArrayBroadcastChannel(10)", viaBroadcast = true), + BUFFERED_1_BROADCAST(1, "BufferedBroadcastChannel(1)", viaBroadcast = true), + BUFFERED_10_BROADCAST(10, "BufferedBroadcastChannel(10)", viaBroadcast = true), CONFLATED_BROADCAST(Channel.CONFLATED, "ConflatedBroadcastChannel", viaBroadcast = true) ; @@ -33,7 +33,7 @@ enum class TestChannelKind( override fun toString(): String = description } -private class ChannelViaBroadcast( +internal class ChannelViaBroadcast( private val broadcast: BroadcastChannel ): Channel, SendChannel by broadcast { val sub = broadcast.openSubscription() @@ -46,16 +46,14 @@ private class ChannelViaBroadcast( override fun iterator(): ChannelIterator = sub.iterator() override fun tryReceive(): ChannelResult = sub.tryReceive() - override fun cancel(cause: CancellationException?) = sub.cancel(cause) + override fun cancel(cause: CancellationException?) = broadcast.cancel(cause) // implementing hidden method anyway, so can cast to an internal class @Deprecated(level = DeprecationLevel.HIDDEN, message = "Since 1.2.0, binary compatibility with versions <= 1.1.x") - override fun cancel(cause: Throwable?): Boolean = (sub as AbstractChannel).cancelInternal(cause) + override fun cancel(cause: Throwable?): Boolean = error("unsupported") override val onReceive: SelectClause1 get() = sub.onReceive override val onReceiveCatching: SelectClause1> get() = sub.onReceiveCatching - override val onReceiveOrNull: SelectClause1 - get() = error("unsupported") } diff --git a/kotlinx-coroutines-core/common/test/channels/LinkedListChannelTest.kt b/kotlinx-coroutines-core/common/test/channels/UnlimitedChannelTest.kt similarity index 96% rename from kotlinx-coroutines-core/common/test/channels/LinkedListChannelTest.kt rename to kotlinx-coroutines-core/common/test/channels/UnlimitedChannelTest.kt index 501affb4d9..24b9d3d058 100644 --- a/kotlinx-coroutines-core/common/test/channels/LinkedListChannelTest.kt +++ b/kotlinx-coroutines-core/common/test/channels/UnlimitedChannelTest.kt @@ -7,7 +7,7 @@ package kotlinx.coroutines.channels import kotlinx.coroutines.* import kotlin.test.* -class LinkedListChannelTest : TestBase() { +class UnlimitedChannelTest : TestBase() { @Test fun testBasic() = runTest { val c = Channel(Channel.UNLIMITED) diff --git a/kotlinx-coroutines-core/common/test/selects/SelectArrayChannelTest.kt b/kotlinx-coroutines-core/common/test/selects/SelectBufferedChannelTest.kt similarity index 99% rename from kotlinx-coroutines-core/common/test/selects/SelectArrayChannelTest.kt rename to kotlinx-coroutines-core/common/test/selects/SelectBufferedChannelTest.kt index 88c21160ea..6bb8049e54 100644 --- a/kotlinx-coroutines-core/common/test/selects/SelectArrayChannelTest.kt +++ b/kotlinx-coroutines-core/common/test/selects/SelectBufferedChannelTest.kt @@ -6,10 +6,9 @@ package kotlinx.coroutines.selects import kotlinx.coroutines.* import kotlinx.coroutines.channels.* -import kotlinx.coroutines.intrinsics.* import kotlin.test.* -class SelectArrayChannelTest : TestBase() { +class SelectBufferedChannelTest : TestBase() { @Test fun testSelectSendSuccess() = runTest { diff --git a/kotlinx-coroutines-core/common/test/selects/SelectLinkedListChannelTest.kt b/kotlinx-coroutines-core/common/test/selects/SelectUnlimitedChannelTest.kt similarity index 93% rename from kotlinx-coroutines-core/common/test/selects/SelectLinkedListChannelTest.kt rename to kotlinx-coroutines-core/common/test/selects/SelectUnlimitedChannelTest.kt index a066f6b3a9..081c9183aa 100644 --- a/kotlinx-coroutines-core/common/test/selects/SelectLinkedListChannelTest.kt +++ b/kotlinx-coroutines-core/common/test/selects/SelectUnlimitedChannelTest.kt @@ -8,7 +8,7 @@ import kotlinx.coroutines.* import kotlinx.coroutines.channels.* import kotlin.test.* -class SelectLinkedListChannelTest : TestBase() { +class SelectUnlimitedChannelTest : TestBase() { @Test fun testSelectSendWhenClosed() = runTest { expect(1) diff --git a/kotlinx-coroutines-core/concurrent/test/channels/ChannelCancelUndeliveredElementStressTest.kt b/kotlinx-coroutines-core/concurrent/test/channels/ChannelCancelUndeliveredElementStressTest.kt index 76918850e5..1cf7d8a17d 100644 --- a/kotlinx-coroutines-core/concurrent/test/channels/ChannelCancelUndeliveredElementStressTest.kt +++ b/kotlinx-coroutines-core/concurrent/test/channels/ChannelCancelUndeliveredElementStressTest.kt @@ -51,6 +51,7 @@ class ChannelCancelUndeliveredElementStressTest : TestBase() { println(" Undelivered: ${dUndeliveredCnt.value}") error("Failed") } + (channel as? BufferedChannel<*>)?.checkSegmentStructureInvariants() trySendFailedCnt += dTrySendFailedCnt receivedCnt += dReceivedCnt undeliveredCnt += dUndeliveredCnt.value diff --git a/kotlinx-coroutines-core/concurrent/test/selects/SelectChannelStressTest.kt b/kotlinx-coroutines-core/concurrent/test/selects/SelectChannelStressTest.kt index 0b687a49c6..a5bf90ac40 100644 --- a/kotlinx-coroutines-core/concurrent/test/selects/SelectChannelStressTest.kt +++ b/kotlinx-coroutines-core/concurrent/test/selects/SelectChannelStressTest.kt @@ -6,7 +6,6 @@ package kotlinx.coroutines.selects import kotlinx.coroutines.* import kotlinx.coroutines.channels.* -import kotlinx.coroutines.intrinsics.* import kotlin.test.* class SelectChannelStressTest: TestBase() { @@ -15,7 +14,7 @@ class SelectChannelStressTest: TestBase() { private val iterations = (if (isNative) 1_000 else 1_000_000) * stressTestMultiplier @Test - fun testSelectSendResourceCleanupArrayChannel() = runTest { + fun testSelectSendResourceCleanupBufferedChannel() = runTest { val channel = Channel(1) expect(1) channel.send(-1) // fill the buffer, so all subsequent sends cannot proceed @@ -29,7 +28,7 @@ class SelectChannelStressTest: TestBase() { } @Test - fun testSelectReceiveResourceCleanupArrayChannel() = runTest { + fun testSelectReceiveResourceCleanupBufferedChannel() = runTest { val channel = Channel(1) expect(1) repeat(iterations) { i -> diff --git a/kotlinx-coroutines-core/js/src/internal/Concurrent.kt b/kotlinx-coroutines-core/js/src/internal/Concurrent.kt index 71f652271a..6272679e3f 100644 --- a/kotlinx-coroutines-core/js/src/internal/Concurrent.kt +++ b/kotlinx-coroutines-core/js/src/internal/Concurrent.kt @@ -13,7 +13,5 @@ internal class NoOpLock { fun unlock(): Unit {} } -internal actual fun subscriberList(): SubscribersList = CopyOnWriteList() - internal actual fun identitySet(expectedSize: Int): MutableSet = HashSet(expectedSize) diff --git a/kotlinx-coroutines-core/jvm/src/internal/Concurrent.kt b/kotlinx-coroutines-core/jvm/src/internal/Concurrent.kt index 050b974755..5df79b8d75 100644 --- a/kotlinx-coroutines-core/jvm/src/internal/Concurrent.kt +++ b/kotlinx-coroutines-core/jvm/src/internal/Concurrent.kt @@ -9,8 +9,6 @@ import java.util.* import java.util.concurrent.* import kotlin.concurrent.withLock as withLockJvm -internal actual fun subscriberList(): SubscribersList = CopyOnWriteArrayList() - @Suppress("ACTUAL_WITHOUT_EXPECT") internal actual typealias ReentrantLock = java.util.concurrent.locks.ReentrantLock diff --git a/kotlinx-coroutines-core/jvm/src/internal/SegmentBasedQueue.kt b/kotlinx-coroutines-core/jvm/src/internal/SegmentBasedQueue.kt deleted file mode 100644 index f3447b7164..0000000000 --- a/kotlinx-coroutines-core/jvm/src/internal/SegmentBasedQueue.kt +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. - */ - -package kotlinx.coroutines.internal - -import kotlinx.atomicfu.* - -/** - * This queue implementation is based on [SegmentList] for testing purposes and is organized as follows. Essentially, - * the [SegmentBasedQueue] is represented as an infinite array of segments, each stores one element (see [OneElementSegment]). - * Both [enqueue] and [dequeue] operations increment the corresponding global index ([enqIdx] for [enqueue] and - * [deqIdx] for [dequeue]) and work with the indexed by this counter cell. Since both operations increment the indices - * at first, there could be a race: [enqueue] increments [enqIdx], then [dequeue] checks that the queue is not empty - * (that's true) and increments [deqIdx], looking into the corresponding cell after that; however, the cell is empty - * because the [enqIdx] operation has not been put its element yet. To make the queue non-blocking, [dequeue] can mark - * the cell with [BROKEN] token and retry the operation, [enqueue] at the same time should restart as well; this way, - * the queue is obstruction-free. - */ -internal class SegmentBasedQueue { - private val head: AtomicRef> - private val tail: AtomicRef> - - private val enqIdx = atomic(0L) - private val deqIdx = atomic(0L) - - init { - val s = OneElementSegment(0, null, 2) - head = atomic(s) - tail = atomic(s) - } - - // Returns the segments associated with the enqueued element, or `null` if the queue is closed. - fun enqueue(element: T): OneElementSegment? { - while (true) { - val curTail = this.tail.value - val enqIdx = this.enqIdx.getAndIncrement() - @Suppress("INFERRED_TYPE_VARIABLE_INTO_POSSIBLE_EMPTY_INTERSECTION") // KT-54411 - val segmentOrClosed = this.tail.findSegmentAndMoveForward(id = enqIdx, startFrom = curTail, createNewSegment = ::createSegment) - if (segmentOrClosed.isClosed) return null - val s = segmentOrClosed.segment - if (s.element.value === BROKEN) continue - if (s.element.compareAndSet(null, element)) return s - } - } - - fun dequeue(): T? { - while (true) { - if (this.deqIdx.value >= this.enqIdx.value) return null - val curHead = this.head.value - val deqIdx = this.deqIdx.getAndIncrement() - @Suppress("INFERRED_TYPE_VARIABLE_INTO_POSSIBLE_EMPTY_INTERSECTION") // KT-54411 - val segmentOrClosed = this.head.findSegmentAndMoveForward(id = deqIdx, startFrom = curHead, createNewSegment = ::createSegment) - if (segmentOrClosed.isClosed) return null - val s = segmentOrClosed.segment - if (s.id > deqIdx) continue - var el = s.element.value - if (el === null) { - if (s.element.compareAndSet(null, BROKEN)) continue - else el = s.element.value - } - // The link to the previous segment should be cleaned after retrieving the element; - // otherwise, `close()` cannot clean the slot. - s.cleanPrev() - if (el === BROKEN) continue - @Suppress("UNCHECKED_CAST") - return el as T - } - } - - // `enqueue` should return `null` after the queue is closed - fun close(): OneElementSegment { - val s = this.tail.value.close() - var cur = s - while (true) { - cur.element.compareAndSet(null, BROKEN) - cur = cur.prev ?: break - } - return s - } - - val numberOfSegments: Int get() { - var cur = head.value - var i = 1 - while (true) { - cur = cur.next ?: return i - i++ - } - } - - fun checkHeadPrevIsCleaned() { - check(head.value.prev === null) { "head.prev is not null"} - } - - fun checkAllSegmentsAreNotLogicallyRemoved() { - var prev: OneElementSegment? = null - var cur = head.value - while (true) { - check(!cur.logicallyRemoved || cur.isTail) { - "This queue contains removed segments, memory leak detected" - } - check(cur.prev === prev) { - "Two neighbour segments are incorrectly linked: S.next.prev != S" - } - prev = cur - cur = cur.next ?: return - } - } - -} - -private fun createSegment(id: Long, prev: OneElementSegment?) = OneElementSegment(id, prev, 0) - -internal class OneElementSegment(id: Long, prev: OneElementSegment?, pointers: Int) : Segment>(id, prev, pointers) { - val element = atomic(null) - - override val maxSlots get() = 1 - - val logicallyRemoved get() = element.value === BROKEN - - fun removeSegment() { - val old = element.getAndSet(BROKEN) - if (old !== BROKEN) onSlotCleaned() - } -} - -private val BROKEN = Symbol("BROKEN") diff --git a/kotlinx-coroutines-core/jvm/test-resources/stacktraces/channels/testReceiveFromChannel.txt b/kotlinx-coroutines-core/jvm/test-resources/stacktraces/channels/testReceiveFromChannel.txt index af8c1fd389..64085ad329 100644 --- a/kotlinx-coroutines-core/jvm/test-resources/stacktraces/channels/testReceiveFromChannel.txt +++ b/kotlinx-coroutines-core/jvm/test-resources/stacktraces/channels/testReceiveFromChannel.txt @@ -1,8 +1,10 @@ kotlinx.coroutines.RecoverableTestException - at kotlinx.coroutines.exceptions.StackTraceRecoveryChannelsTest$testReceiveFromChannel$1$job$1.invokeSuspend(StackTraceRecoveryChannelsTest.kt:97) - at _COROUTINE._BOUNDARY._(CoroutineDebugging.kt) - at kotlinx.coroutines.exceptions.StackTraceRecoveryChannelsTest.channelReceive(StackTraceRecoveryChannelsTest.kt:116) - at kotlinx.coroutines.exceptions.StackTraceRecoveryChannelsTest$testReceiveFromChannel$1.invokeSuspend(StackTraceRecoveryChannelsTest.kt:101) + at kotlinx.coroutines.internal.StackTraceRecoveryKt.recoverStackTrace(StackTraceRecovery.kt) + at kotlinx.coroutines.channels.BufferedChannel.receive$suspendImpl(BufferedChannel.kt) + at kotlinx.coroutines.channels.BufferedChannel.receive(BufferedChannel.kt) + at kotlinx.coroutines.exceptions.StackTraceRecoveryChannelsTest.channelReceive(StackTraceRecoveryChannelsTest.kt) + at kotlinx.coroutines.exceptions.StackTraceRecoveryChannelsTest.access$channelReceive(StackTraceRecoveryChannelsTest.kt) + at kotlinx.coroutines.exceptions.StackTraceRecoveryChannelsTest$channelReceive$1.invokeSuspend(StackTraceRecoveryChannelsTest.kt) Caused by: kotlinx.coroutines.RecoverableTestException - at kotlinx.coroutines.exceptions.StackTraceRecoveryChannelsTest$testReceiveFromChannel$1$job$1.invokeSuspend(StackTraceRecoveryChannelsTest.kt:97) - at kotlin.coroutines.jvm.internal.BaseContinuationImpl.resumeWith(ContinuationImpl.kt:33) + at kotlinx.coroutines.exceptions.StackTraceRecoveryChannelsTest$testReceiveFromChannel$1$job$1.invokeSuspend(StackTraceRecoveryChannelsTest.kt) + at kotlin.coroutines.jvm.internal.BaseContinuationImpl.resumeWith(ContinuationImpl.kt) \ No newline at end of file diff --git a/kotlinx-coroutines-core/jvm/test-resources/stacktraces/channels/testSendToChannel.txt b/kotlinx-coroutines-core/jvm/test-resources/stacktraces/channels/testSendToChannel.txt index 4a8e320e2d..e40cc741d8 100644 --- a/kotlinx-coroutines-core/jvm/test-resources/stacktraces/channels/testSendToChannel.txt +++ b/kotlinx-coroutines-core/jvm/test-resources/stacktraces/channels/testSendToChannel.txt @@ -1,12 +1,20 @@ -java.util.concurrent.CancellationException: RendezvousChannel was cancelled - at kotlinx.coroutines.channels.AbstractChannel.cancel(AbstractChannel.kt:630) - at kotlinx.coroutines.channels.ReceiveChannel$DefaultImpls.cancel$default(Channel.kt:311) - at kotlinx.coroutines.exceptions.StackTraceRecoveryChannelsTest$testSendToChannel$1$job$1.invokeSuspend(StackTraceRecoveryChannelsTest.kt:52) +java.util.concurrent.CancellationException: Channel was cancelled + at kotlinx.coroutines.channels.BufferedChannel.cancelImpl$(BufferedChannel.kt) + at kotlinx.coroutines.channels.BufferedChannel.cancel(BufferedChannel.kt) + at kotlinx.coroutines.channels.ReceiveChannel$DefaultImpls.cancel$default(Channel.kt) + at kotlinx.coroutines.exceptions.StackTraceRecoveryChannelsTest$testSendToChannel$1$job$1.invokeSuspend(StackTraceRecoveryChannelsTest.kt) at _COROUTINE._BOUNDARY._(CoroutineDebugging.kt) - at kotlinx.coroutines.exceptions.StackTraceRecoveryChannelsTest.channelSend(StackTraceRecoveryChannelsTest.kt:73) - at kotlinx.coroutines.exceptions.StackTraceRecoveryChannelsTest$testSendToChannel$1.invokeSuspend(StackTraceRecoveryChannelsTest.kt:56) -Caused by: java.util.concurrent.CancellationException: RendezvousChannel was cancelled - at kotlinx.coroutines.channels.AbstractChannel.cancel(AbstractChannel.kt:630) - at kotlinx.coroutines.channels.ReceiveChannel$DefaultImpls.cancel$default(Channel.kt:311) - at kotlinx.coroutines.exceptions.StackTraceRecoveryChannelsTest$testSendToChannel$1$job$1.invokeSuspend(StackTraceRecoveryChannelsTest.kt:52) - at kotlin.coroutines.jvm.internal.BaseContinuationImpl.resumeWith(ContinuationImpl.kt:33) + at kotlinx.coroutines.exceptions.StackTraceRecoveryChannelsTest.channelSend(StackTraceRecoveryChannelsTest.kt) + at kotlinx.coroutines.exceptions.StackTraceRecoveryChannelsTest$testSendToChannel$1.invokeSuspend(StackTraceRecoveryChannelsTest.kt) +Caused by: java.util.concurrent.CancellationException: Channel was cancelled + at kotlinx.coroutines.channels.BufferedChannel.cancelImpl$(BufferedChannel.kt) + at kotlinx.coroutines.channels.BufferedChannel.cancel(BufferedChannel.kt) + at kotlinx.coroutines.channels.ReceiveChannel$DefaultImpls.cancel$default(Channel.kt) + at kotlinx.coroutines.exceptions.StackTraceRecoveryChannelsTest$testSendToChannel$1$job$1.invokeSuspend(StackTraceRecoveryChannelsTest.kt) + at kotlin.coroutines.jvm.internal.BaseContinuationImpl.resumeWith(ContinuationImpl.kt) + at kotlinx.coroutines.DispatchedTask.run(DispatchedTask.kt) + at kotlinx.coroutines.EventLoopImplBase.processNextEvent(EventLoop.common.kt) + at kotlinx.coroutines.BlockingCoroutine.joinBlocking(Builders.kt) + at kotlinx.coroutines.BuildersKt__BuildersKt.runBlocking(Builders.kt) + at kotlinx.coroutines.BuildersKt.runBlocking(Unknown Source) + at kotlinx.coroutines.TestBase.runTest(TestBase.kt) diff --git a/kotlinx-coroutines-core/jvm/test-resources/stacktraces/resume-mode/testUnconfined.txt b/kotlinx-coroutines-core/jvm/test-resources/stacktraces/resume-mode/testUnconfined.txt index 4f0103ecc8..a8461556d1 100644 --- a/kotlinx-coroutines-core/jvm/test-resources/stacktraces/resume-mode/testUnconfined.txt +++ b/kotlinx-coroutines-core/jvm/test-resources/stacktraces/resume-mode/testUnconfined.txt @@ -1,12 +1,10 @@ kotlinx.coroutines.RecoverableTestException - at kotlinx.coroutines.exceptions.StackTraceRecoveryResumeModeTest.testResumeModeFastPath(StackTraceRecoveryResumeModeTest.kt:61) - at kotlinx.coroutines.exceptions.StackTraceRecoveryResumeModeTest$testUnconfined$1.invokeSuspend(StackTraceRecoveryResumeModeTest.kt:40) - at _COROUTINE._BOUNDARY._(CoroutineDebugging.kt) - at kotlinx.coroutines.exceptions.StackTraceRecoveryResumeModeTest$withContext$2.invokeSuspend(StackTraceRecoveryResumeModeTest.kt:76) - at kotlinx.coroutines.exceptions.StackTraceRecoveryResumeModeTest.doFastPath(StackTraceRecoveryResumeModeTest.kt:71) - at kotlinx.coroutines.exceptions.StackTraceRecoveryResumeModeTest.testResumeModeFastPath(StackTraceRecoveryResumeModeTest.kt:62) - at kotlinx.coroutines.exceptions.StackTraceRecoveryResumeModeTest$testUnconfined$1.invokeSuspend(StackTraceRecoveryResumeModeTest.kt:40) + at kotlinx.coroutines.internal.StackTraceRecoveryKt.recoverStackTrace(StackTraceRecovery.kt) + at kotlinx.coroutines.channels.BufferedChannel.receive$suspendImpl(BufferedChannel.kt) + at kotlinx.coroutines.channels.BufferedChannel.receive(BufferedChannel.kt) + at kotlinx.coroutines.exceptions.StackTraceRecoveryResumeModeTest$withContext$2.invokeSuspend(StackTraceRecoveryResumeModeTest.kt) Caused by: kotlinx.coroutines.RecoverableTestException - at kotlinx.coroutines.exceptions.StackTraceRecoveryResumeModeTest.testResumeModeFastPath(StackTraceRecoveryResumeModeTest.kt:61) - at kotlinx.coroutines.exceptions.StackTraceRecoveryResumeModeTest$testUnconfined$1.invokeSuspend(StackTraceRecoveryResumeModeTest.kt:40) - at kotlin.coroutines.jvm.internal.BaseContinuationImpl.resumeWith(ContinuationImpl.kt:33) + at kotlinx.coroutines.exceptions.StackTraceRecoveryResumeModeTest.testResumeModeFastPath(StackTraceRecoveryResumeModeTest.kt) + at kotlinx.coroutines.exceptions.StackTraceRecoveryResumeModeTest.access$testResumeModeFastPath(StackTraceRecoveryResumeModeTest.kt) + at kotlinx.coroutines.exceptions.StackTraceRecoveryResumeModeTest$testUnconfined$1.invokeSuspend(StackTraceRecoveryResumeModeTest.kt) + at kotlin.coroutines.jvm.internal.BaseContinuationImpl.resumeWith(ContinuationImpl.kt) \ No newline at end of file diff --git a/kotlinx-coroutines-core/jvm/test-resources/stacktraces/select/testSelectOnReceive.txt b/kotlinx-coroutines-core/jvm/test-resources/stacktraces/select/testSelectOnReceive.txt index ae40259914..8b958d2058 100644 --- a/kotlinx-coroutines-core/jvm/test-resources/stacktraces/select/testSelectOnReceive.txt +++ b/kotlinx-coroutines-core/jvm/test-resources/stacktraces/select/testSelectOnReceive.txt @@ -1,9 +1,9 @@ kotlinx.coroutines.channels.ClosedReceiveChannelException: Channel was closed - at kotlinx.coroutines.channels.Closed.getReceiveException(AbstractChannel.kt) - at kotlinx.coroutines.channels.AbstractChannel.processResultSelectReceive(AbstractChannel.kt) - at kotlinx.coroutines.channels.AbstractChannel.access$processResultSelectReceive(AbstractChannel.kt) - at kotlinx.coroutines.channels.AbstractChannel$onReceive$2.invoke(AbstractChannel.kt) - at kotlinx.coroutines.channels.AbstractChannel$onReceive$2.invoke(AbstractChannel.kt) + at kotlinx.coroutines.channels.BufferedChannel.getReceiveException(BufferedChannel.kt) + at kotlinx.coroutines.channels.BufferedChannel.processResultSelectReceive(BufferedChannel.kt) + at kotlinx.coroutines.channels.BufferedChannel.access$processResultSelectReceive(BufferedChannel.kt) + at kotlinx.coroutines.channels.BufferedChannel$onReceive$2.invoke(BufferedChannel.kt) + at kotlinx.coroutines.channels.BufferedChannel$onReceive$2.invoke(BufferedChannel.kt) at kotlinx.coroutines.selects.SelectImplementation$ClauseData.processResult(Select.kt) at kotlinx.coroutines.selects.SelectImplementation.processResultAndInvokeBlockRecoveringException(Select.kt) at kotlinx.coroutines.selects.SelectImplementation.complete(Select.kt) @@ -16,11 +16,11 @@ kotlinx.coroutines.channels.ClosedReceiveChannelException: Channel was closed at kotlinx.coroutines.selects.SelectImplementation.processResultAndInvokeBlockRecoveringException(Select.kt) at kotlinx.coroutines.exceptions.StackTraceRecoverySelectTest$testSelectOnReceive$1.invokeSuspend(StackTraceRecoverySelectTest.kt) Caused by: kotlinx.coroutines.channels.ClosedReceiveChannelException: Channel was closed - at kotlinx.coroutines.channels.Closed.getReceiveException(AbstractChannel.kt) - at kotlinx.coroutines.channels.AbstractChannel.processResultSelectReceive(AbstractChannel.kt) - at kotlinx.coroutines.channels.AbstractChannel.access$processResultSelectReceive(AbstractChannel.kt) - at kotlinx.coroutines.channels.AbstractChannel$onReceive$2.invoke(AbstractChannel.kt) - at kotlinx.coroutines.channels.AbstractChannel$onReceive$2.invoke(AbstractChannel.kt) + at kotlinx.coroutines.channels.BufferedChannel.getReceiveException(BufferedChannel.kt) + at kotlinx.coroutines.channels.BufferedChannel.processResultSelectReceive(BufferedChannel.kt) + at kotlinx.coroutines.channels.BufferedChannel.access$processResultSelectReceive(BufferedChannel.kt) + at kotlinx.coroutines.channels.BufferedChannel$onReceive$2.invoke(BufferedChannel.kt) + at kotlinx.coroutines.channels.BufferedChannel$onReceive$2.invoke(BufferedChannel.kt) at kotlinx.coroutines.selects.SelectImplementation$ClauseData.processResult(Select.kt) at kotlinx.coroutines.selects.SelectImplementation.processResultAndInvokeBlockRecoveringException(Select.kt) at kotlinx.coroutines.selects.SelectImplementation.complete(Select.kt) diff --git a/kotlinx-coroutines-core/jvm/test/AbstractLincheckTest.kt b/kotlinx-coroutines-core/jvm/test/AbstractLincheckTest.kt index 89bbbfd7ee..bdb615d83c 100644 --- a/kotlinx-coroutines-core/jvm/test/AbstractLincheckTest.kt +++ b/kotlinx-coroutines-core/jvm/test/AbstractLincheckTest.kt @@ -16,7 +16,7 @@ abstract class AbstractLincheckTest : VerifierState() { @Test fun modelCheckingTest() = ModelCheckingOptions() - .iterations(if (isStressTest) 100 else 20) + .iterations(if (isStressTest) 200 else 20) .invocationsPerIteration(if (isStressTest) 10_000 else 1_000) .commonConfiguration() .customize(isStressTest) @@ -24,7 +24,7 @@ abstract class AbstractLincheckTest : VerifierState() { @Test fun stressTest() = StressOptions() - .iterations(if (isStressTest) 100 else 20) + .iterations(if (isStressTest) 200 else 20) .invocationsPerIteration(if (isStressTest) 10_000 else 1_000) .commonConfiguration() .customize(isStressTest) @@ -32,8 +32,13 @@ abstract class AbstractLincheckTest : VerifierState() { private fun > O.commonConfiguration(): O = this .actorsBefore(if (isStressTest) 3 else 1) + // All the bugs we have discovered so far + // were reproducible on at most 3 threads .threads(3) - .actorsPerThread(if (isStressTest) 4 else 2) + // 3 operations per thread is sufficient, + // while increasing this number declines + // the model checking coverage. + .actorsPerThread(if (isStressTest) 3 else 2) .actorsAfter(if (isStressTest) 3 else 0) .customize(isStressTest) diff --git a/kotlinx-coroutines-core/jvm/test/MutexCancellationStressTest.kt b/kotlinx-coroutines-core/jvm/test/MutexCancellationStressTest.kt index 9e676e9a60..eb6360dac0 100644 --- a/kotlinx-coroutines-core/jvm/test/MutexCancellationStressTest.kt +++ b/kotlinx-coroutines-core/jvm/test/MutexCancellationStressTest.kt @@ -48,7 +48,7 @@ class MutexCancellationStressTest : TestBase() { val checkProgressJob = launch(dispatcher + CoroutineName("checkProgressJob")) { var lastCounterLocalSnapshot = (0 until mutexJobNumber).map { 0 } while (completed.value == 0) { - delay(500) + delay(1000) val c = counterLocal.map { it.value } for (i in 0 until mutexJobNumber) { assert(c[i] > lastCounterLocalSnapshot[i]) { "No progress in MutexJob-$i" } diff --git a/kotlinx-coroutines-core/jvm/test/channels/BroadcastChannelLeakTest.kt b/kotlinx-coroutines-core/jvm/test/channels/BroadcastChannelLeakTest.kt index 66b08c74e4..df944654b6 100644 --- a/kotlinx-coroutines-core/jvm/test/channels/BroadcastChannelLeakTest.kt +++ b/kotlinx-coroutines-core/jvm/test/channels/BroadcastChannelLeakTest.kt @@ -6,8 +6,8 @@ import kotlin.test.* class BroadcastChannelLeakTest : TestBase() { @Test - fun testArrayBroadcastChannelSubscriptionLeak() { - checkLeak { ArrayBroadcastChannel(1) } + fun testBufferedBroadcastChannelSubscriptionLeak() { + checkLeak { BroadcastChannelImpl(1) } } @Test diff --git a/kotlinx-coroutines-core/jvm/test/channels/ArrayChannelStressTest.kt b/kotlinx-coroutines-core/jvm/test/channels/BufferedChannelStressTest.kt similarity index 95% rename from kotlinx-coroutines-core/jvm/test/channels/ArrayChannelStressTest.kt rename to kotlinx-coroutines-core/jvm/test/channels/BufferedChannelStressTest.kt index 74dc24c7f6..a6464263cc 100644 --- a/kotlinx-coroutines-core/jvm/test/channels/ArrayChannelStressTest.kt +++ b/kotlinx-coroutines-core/jvm/test/channels/BufferedChannelStressTest.kt @@ -10,7 +10,7 @@ import org.junit.runner.* import org.junit.runners.* @RunWith(Parameterized::class) -class ArrayChannelStressTest(private val capacity: Int) : TestBase() { +class BufferedChannelStressTest(private val capacity: Int) : TestBase() { companion object { @Parameterized.Parameters(name = "{0}, nSenders={1}, nReceivers={2}") diff --git a/kotlinx-coroutines-core/jvm/test/channels/ChannelMemoryLeakStressTest.kt b/kotlinx-coroutines-core/jvm/test/channels/ChannelMemoryLeakStressTest.kt new file mode 100644 index 0000000000..ebc2bee89a --- /dev/null +++ b/kotlinx-coroutines-core/jvm/test/channels/ChannelMemoryLeakStressTest.kt @@ -0,0 +1,25 @@ +/* + * Copyright 2016-2022 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package channels + +import kotlinx.coroutines.* +import kotlinx.coroutines.channels.* +import org.junit.Test + +class ChannelMemoryLeakStressTest : TestBase() { + private val nRepeat = 1_000_000 * stressTestMultiplier + + @Test + fun test() = runTest { + val c = Channel(1) + repeat(nRepeat) { + c.send(bigValue()) + c.receive() + } + } + + // capture big value for fast OOM in case of a bug + private fun bigValue(): ByteArray = ByteArray(4096) +} diff --git a/kotlinx-coroutines-core/jvm/test/channels/ChannelSendReceiveStressTest.kt b/kotlinx-coroutines-core/jvm/test/channels/ChannelSendReceiveStressTest.kt index 7e55f2e602..8a60ce5051 100644 --- a/kotlinx-coroutines-core/jvm/test/channels/ChannelSendReceiveStressTest.kt +++ b/kotlinx-coroutines-core/jvm/test/channels/ChannelSendReceiveStressTest.kt @@ -7,12 +7,14 @@ package kotlinx.coroutines.channels import kotlinx.coroutines.* import kotlinx.coroutines.selects.* import org.junit.* +import org.junit.Ignore import org.junit.Test import org.junit.runner.* import org.junit.runners.* import java.util.concurrent.atomic.* import kotlin.test.* +@Ignore @RunWith(Parameterized::class) class ChannelSendReceiveStressTest( private val kind: TestChannelKind, @@ -25,10 +27,7 @@ class ChannelSendReceiveStressTest( fun params(): Collection> = listOf(1, 2, 10).flatMap { nSenders -> listOf(1, 10).flatMap { nReceivers -> - TestChannelKind.values() - // Workaround for bug that won't be fixed unless new channel implementation, see #2443 - .filter { it != TestChannelKind.LINKED_LIST } - .map { arrayOf(it, nSenders, nReceivers) } + TestChannelKind.values().map { arrayOf(it, nSenders, nReceivers) } } } } @@ -36,7 +35,7 @@ class ChannelSendReceiveStressTest( private val timeLimit = 30_000L * stressTestMultiplier // 30 sec private val nEvents = 200_000 * stressTestMultiplier - private val maxBuffer = 10_000 // artificial limit for LinkedListChannel + private val maxBuffer = 10_000 // artificial limit for unlimited channel val channel = kind.create() private val sendersCompleted = AtomicInteger() @@ -107,6 +106,7 @@ class ChannelSendReceiveStressTest( repeat(nReceivers) { receiveIndex -> println(" Received by #$receiveIndex ${receivedBy[receiveIndex]}") } + (channel as? BufferedChannel<*>)?.checkSegmentStructureInvariants() assertEquals(nSenders, sendersCompleted.get()) assertEquals(nReceivers, receiversCompleted.get()) assertEquals(0, dupes.get()) @@ -121,7 +121,7 @@ class ChannelSendReceiveStressTest( sentTotal.incrementAndGet() if (!kind.isConflated) { while (sentTotal.get() > receivedTotal.get() + maxBuffer) - yield() // throttle fast senders to prevent OOM with LinkedListChannel + yield() // throttle fast senders to prevent OOM with an unlimited channel } } diff --git a/kotlinx-coroutines-core/jvm/test/channels/ChannelUndeliveredElementSelectOldStressTest.kt b/kotlinx-coroutines-core/jvm/test/channels/ChannelUndeliveredElementSelectOldStressTest.kt index 1b4b83eb53..25cccf948a 100644 --- a/kotlinx-coroutines-core/jvm/test/channels/ChannelUndeliveredElementSelectOldStressTest.kt +++ b/kotlinx-coroutines-core/jvm/test/channels/ChannelUndeliveredElementSelectOldStressTest.kt @@ -160,7 +160,7 @@ class ChannelUndeliveredElementSelectOldStressTest(private val kind: TestChannel sentStatus[trySendData.x] = 3 when { // must artificially slow down LINKED_LIST sender to avoid overwhelming receiver and going OOM - kind == TestChannelKind.LINKED_LIST -> while (sentCnt > lastReceived + 100) yield() + kind == TestChannelKind.UNLIMITED -> while (sentCnt > lastReceived + 100) yield() // yield periodically to check cancellation on conflated channels kind.isConflated -> if (counter++ % 100 == 0) yield() } diff --git a/kotlinx-coroutines-core/jvm/test/channels/ChannelUndeliveredElementStressTest.kt b/kotlinx-coroutines-core/jvm/test/channels/ChannelUndeliveredElementStressTest.kt index 335980daaf..f8a5644769 100644 --- a/kotlinx-coroutines-core/jvm/test/channels/ChannelUndeliveredElementStressTest.kt +++ b/kotlinx-coroutines-core/jvm/test/channels/ChannelUndeliveredElementStressTest.kt @@ -116,6 +116,7 @@ class ChannelUndeliveredElementStressTest(private val kind: TestChannelKind) : T printErrorDetails() throw e } + (channel as? BufferedChannel<*>)?.checkSegmentStructureInvariants() sentStatus.clear() receivedStatus.clear() failedStatus.clear() @@ -165,7 +166,7 @@ class ChannelUndeliveredElementStressTest(private val kind: TestChannelKind) : T sentStatus[trySendData.x] = sendMode + 2 when { // must artificially slow down LINKED_LIST sender to avoid overwhelming receiver and going OOM - kind == TestChannelKind.LINKED_LIST -> while (sentCnt > lastReceived + 100) yield() + kind == TestChannelKind.UNLIMITED -> while (sentCnt > lastReceived + 100) yield() // yield periodically to check cancellation on conflated channels kind.isConflated -> if (counter++ % 100 == 0) yield() } diff --git a/kotlinx-coroutines-core/jvm/test/channels/InvokeOnCloseStressTest.kt b/kotlinx-coroutines-core/jvm/test/channels/InvokeOnCloseStressTest.kt index 888522c63c..8ac859137e 100644 --- a/kotlinx-coroutines-core/jvm/test/channels/InvokeOnCloseStressTest.kt +++ b/kotlinx-coroutines-core/jvm/test/channels/InvokeOnCloseStressTest.kt @@ -5,7 +5,6 @@ package kotlinx.coroutines.channels import kotlinx.coroutines.* -import kotlinx.coroutines.channels.* import org.junit.* import org.junit.Test import java.util.concurrent.* @@ -28,7 +27,7 @@ class InvokeOnCloseStressTest : TestBase(), CoroutineScope { @Test fun testInvokedExactlyOnce() = runBlocking { - runStressTest(TestChannelKind.ARRAY_1) + runStressTest(TestChannelKind.BUFFERED_1) } @Test diff --git a/kotlinx-coroutines-core/jvm/test/internal/SegmentQueueTest.kt b/kotlinx-coroutines-core/jvm/test/internal/SegmentQueueTest.kt deleted file mode 100644 index 1390403a3b..0000000000 --- a/kotlinx-coroutines-core/jvm/test/internal/SegmentQueueTest.kt +++ /dev/null @@ -1,110 +0,0 @@ -package kotlinx.coroutines.internal - -import kotlinx.coroutines.* -import kotlinx.coroutines.internal.* -import org.junit.Test -import java.util.* -import java.util.concurrent.CyclicBarrier -import java.util.concurrent.atomic.AtomicInteger -import kotlin.concurrent.thread -import kotlin.random.Random -import kotlin.test.* - -class SegmentQueueTest : TestBase() { - @Test - fun testSimpleTest() { - val q = SegmentBasedQueue() - assertEquals(1, q.numberOfSegments) - assertNull(q.dequeue()) - q.enqueue(1) - assertEquals(1, q.numberOfSegments) - q.enqueue(2) - assertEquals(2, q.numberOfSegments) - assertEquals(1, q.dequeue()) - assertEquals(2, q.numberOfSegments) - assertEquals(2, q.dequeue()) - assertEquals(1, q.numberOfSegments) - assertNull(q.dequeue()) - } - - @Test - fun testSegmentRemoving() { - val q = SegmentBasedQueue() - q.enqueue(1) - val s = q.enqueue(2) - q.enqueue(3) - assertEquals(3, q.numberOfSegments) - s!!.removeSegment() - assertEquals(2, q.numberOfSegments) - assertEquals(1, q.dequeue()) - assertEquals(3, q.dequeue()) - assertNull(q.dequeue()) - } - - @Test - fun testRemoveHeadSegment() { - val q = SegmentBasedQueue() - q.enqueue(1) - val s = q.enqueue(2) - assertEquals(1, q.dequeue()) - q.enqueue(3) - s!!.removeSegment() - assertEquals(3, q.dequeue()) - assertNull(q.dequeue()) - } - - @Test - fun testClose() { - val q = SegmentBasedQueue() - q.enqueue(1) - assertEquals(0, q.close().id) - assertEquals(null, q.enqueue(2)) - assertEquals(1, q.dequeue()) - assertEquals(null, q.dequeue()) - } - - @Test - fun stressTest() { - val q = SegmentBasedQueue() - val expectedQueue = ArrayDeque() - val r = Random(0) - repeat(1_000_000 * stressTestMultiplier) { - if (r.nextBoolean()) { // add - val el = r.nextInt() - q.enqueue(el) - expectedQueue.add(el) - } else { // remove - assertEquals(expectedQueue.poll(), q.dequeue()) - q.checkHeadPrevIsCleaned() - } - } - } - - @Test - fun testRemoveSegmentsSerial() = stressTestRemoveSegments(false) - - @Test - fun testRemoveSegmentsRandom() = stressTestRemoveSegments(true) - - private fun stressTestRemoveSegments(random: Boolean) { - val N = 100_000 * stressTestMultiplier - val T = 10 - val q = SegmentBasedQueue() - val segments = (1..N).map { q.enqueue(it)!! }.toMutableList() - if (random) segments.shuffle() - assertEquals(N, q.numberOfSegments) - val nextSegmentIndex = AtomicInteger() - val barrier = CyclicBarrier(T) - (1..T).map { - thread { - barrier.await() - while (true) { - val i = nextSegmentIndex.getAndIncrement() - if (i >= N) break - segments[i].removeSegment() - } - } - }.forEach { it.join() } - assertEquals(2, q.numberOfSegments) - } -} diff --git a/kotlinx-coroutines-core/jvm/test/lincheck/ChannelsLincheckTest.kt b/kotlinx-coroutines-core/jvm/test/lincheck/ChannelsLincheckTest.kt index 46611b792b..87ed74b715 100644 --- a/kotlinx-coroutines-core/jvm/test/lincheck/ChannelsLincheckTest.kt +++ b/kotlinx-coroutines-core/jvm/test/lincheck/ChannelsLincheckTest.kt @@ -1,7 +1,7 @@ /* * Copyright 2016-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. */ -@file:Suppress("unused") +@file:Suppress("unused", "MemberVisibilityCanBePrivate") package kotlinx.coroutines.lincheck @@ -15,107 +15,160 @@ import org.jetbrains.kotlinx.lincheck.* import org.jetbrains.kotlinx.lincheck.annotations.* import org.jetbrains.kotlinx.lincheck.annotations.Operation import org.jetbrains.kotlinx.lincheck.paramgen.* +import org.jetbrains.kotlinx.lincheck.strategy.managed.modelchecking.* import org.jetbrains.kotlinx.lincheck.verifier.* -class RendezvousChannelLincheckTest : ChannelLincheckTestBase( +class RendezvousChannelLincheckTest : ChannelLincheckTestBaseWithOnSend( c = Channel(RENDEZVOUS), sequentialSpecification = SequentialRendezvousChannel::class.java ) class SequentialRendezvousChannel : SequentialIntChannelBase(RENDEZVOUS) -class Array1ChannelLincheckTest : ChannelLincheckTestBase( +class Buffered1ChannelLincheckTest : ChannelLincheckTestBaseWithOnSend( c = Channel(1), - sequentialSpecification = SequentialArray1RendezvousChannel::class.java + sequentialSpecification = SequentialBuffered1Channel::class.java ) -class SequentialArray1RendezvousChannel : SequentialIntChannelBase(1) +class Buffered1BroadcastChannelLincheckTest : ChannelLincheckTestBase( + c = ChannelViaBroadcast(BroadcastChannelImpl(1)), + sequentialSpecification = SequentialBuffered1Channel::class.java, + obstructionFree = false +) +class SequentialBuffered1Channel : SequentialIntChannelBase(1) -class Array2ChannelLincheckTest : ChannelLincheckTestBase( +class Buffered2ChannelLincheckTest : ChannelLincheckTestBaseWithOnSend( c = Channel(2), - sequentialSpecification = SequentialArray2RendezvousChannel::class.java + sequentialSpecification = SequentialBuffered2Channel::class.java +) +class Buffered2BroadcastChannelLincheckTest : ChannelLincheckTestBase( + c = ChannelViaBroadcast(BroadcastChannelImpl(2)), + sequentialSpecification = SequentialBuffered2Channel::class.java, + obstructionFree = false ) -class SequentialArray2RendezvousChannel : SequentialIntChannelBase(2) +class SequentialBuffered2Channel : SequentialIntChannelBase(2) -class UnlimitedChannelLincheckTest : ChannelLincheckTestBase( +class UnlimitedChannelLincheckTest : ChannelLincheckTestBaseAll( c = Channel(UNLIMITED), sequentialSpecification = SequentialUnlimitedChannel::class.java ) class SequentialUnlimitedChannel : SequentialIntChannelBase(UNLIMITED) -class ConflatedChannelLincheckTest : ChannelLincheckTestBase( +class ConflatedChannelLincheckTest : ChannelLincheckTestBaseAll( c = Channel(CONFLATED), - sequentialSpecification = SequentialConflatedChannel::class.java + sequentialSpecification = SequentialConflatedChannel::class.java, + obstructionFree = false +) +class ConflatedBroadcastChannelLincheckTest : ChannelLincheckTestBaseAll( + c = ChannelViaBroadcast(ConflatedBroadcastChannel()), + sequentialSpecification = SequentialConflatedChannel::class.java, + obstructionFree = false ) class SequentialConflatedChannel : SequentialIntChannelBase(CONFLATED) +abstract class ChannelLincheckTestBaseAll( + c: Channel, + sequentialSpecification: Class<*>, + obstructionFree: Boolean = true +) : ChannelLincheckTestBaseWithOnSend(c, sequentialSpecification, obstructionFree) { + @Operation + override fun trySend(value: Int) = super.trySend(value) + @Operation + override fun isClosedForReceive() = super.isClosedForReceive() + @Operation + override fun isEmpty() = super.isEmpty() +} + +abstract class ChannelLincheckTestBaseWithOnSend( + c: Channel, + sequentialSpecification: Class<*>, + obstructionFree: Boolean = true +) : ChannelLincheckTestBase(c, sequentialSpecification, obstructionFree) { + @Operation(allowExtraSuspension = true, blocking = true) + suspend fun sendViaSelect(@Param(name = "value") value: Int): Any = try { + select { c.onSend(value) {} } + } catch (e: NumberedCancellationException) { + e.testResult + } +} + @Param.Params( - Param(name = "value", gen = IntGen::class, conf = "1:5"), - Param(name = "closeToken", gen = IntGen::class, conf = "1:3") + Param(name = "value", gen = IntGen::class, conf = "1:9"), + Param(name = "closeToken", gen = IntGen::class, conf = "1:9") ) abstract class ChannelLincheckTestBase( - private val c: Channel, - private val sequentialSpecification: Class<*> + protected val c: Channel, + private val sequentialSpecification: Class<*>, + private val obstructionFree: Boolean = true ) : AbstractLincheckTest() { - @Operation(promptCancellation = true) + + @Operation(allowExtraSuspension = true, blocking = true) suspend fun send(@Param(name = "value") value: Int): Any = try { c.send(value) } catch (e: NumberedCancellationException) { e.testResult } - @Operation - fun trySend(@Param(name = "value") value: Int): Any = c.trySend(value) - .onSuccess { return true } - .onFailure { - return if (it is NumberedCancellationException) it.testResult - else false - } - - @Operation(promptCancellation = true) - suspend fun sendViaSelect(@Param(name = "value") value: Int): Any = try { - select { c.onSend(value) {} } - } catch (e: NumberedCancellationException) { - e.testResult - } + // @Operation TODO: `trySend()` is not linearizable as it can fail due to postponed buffer expansion + // TODO: or make a rendezvous with `tryReceive`, which violates the sequential specification. + open fun trySend(@Param(name = "value") value: Int): Any = c.trySend(value) + .onSuccess { return true } + .onFailure { + return if (it is NumberedCancellationException) it.testResult + else false + } - @Operation(promptCancellation = true) + @Operation(allowExtraSuspension = true, blocking = true) suspend fun receive(): Any = try { c.receive() } catch (e: NumberedCancellationException) { e.testResult } - @Operation + @Operation(allowExtraSuspension = true, blocking = true) + suspend fun receiveCatching(): Any = c.receiveCatching() + .onSuccess { return it } + .onClosed { e -> return (e as NumberedCancellationException).testResult } + + @Operation(blocking = true) fun tryReceive(): Any? = c.tryReceive() .onSuccess { return it } .onFailure { return if (it is NumberedCancellationException) it.testResult else null } - @Operation(promptCancellation = true) + @Operation(allowExtraSuspension = true, blocking = true) suspend fun receiveViaSelect(): Any = try { select { c.onReceive { it } } } catch (e: NumberedCancellationException) { e.testResult } - @Operation(causesBlocking = true) + @Operation(causesBlocking = true, blocking = true) fun close(@Param(name = "closeToken") token: Int): Boolean = c.close(NumberedCancellationException(token)) - // TODO: this operation should be (and can be!) linearizable, but is not - // @Operation + @Operation(causesBlocking = true, blocking = true) fun cancel(@Param(name = "closeToken") token: Int) = c.cancel(NumberedCancellationException(token)) - // @Operation - fun isClosedForReceive() = c.isClosedForReceive + // @Operation TODO non-linearizable in BufferedChannel + open fun isClosedForReceive() = c.isClosedForReceive - // @Operation + @Operation(blocking = true) fun isClosedForSend() = c.isClosedForSend - // TODO: this operation should be (and can be!) linearizable, but is not - // @Operation - fun isEmpty() = c.isEmpty + // @Operation TODO non-linearizable in BufferedChannel + open fun isEmpty() = c.isEmpty + + @StateRepresentation + fun state() = (c as? BufferedChannel<*>)?.toStringDebug() ?: c.toString() - override fun > O.customize(isStressTest: Boolean): O = + @Validate + fun validate() { + (c as? BufferedChannel<*>)?.checkSegmentStructureInvariants() + } + + override fun > O.customize(isStressTest: Boolean) = actorsBefore(0).sequentialSpecification(sequentialSpecification) + + override fun ModelCheckingOptions.customize(isStressTest: Boolean) = + checkObstructionFreedom(obstructionFree) } private class NumberedCancellationException(number: Int) : CancellationException() { @@ -165,6 +218,8 @@ abstract class SequentialIntChannelBase(private val capacity: Int) : VerifierSta receivers.add(cont) } + suspend fun receiveCatching() = receive() + fun tryReceive(): Any? { if (buffer.isNotEmpty()) { val el = buffer.removeAt(0) @@ -198,7 +253,7 @@ abstract class SequentialIntChannelBase(private val capacity: Int) : VerifierSta } fun cancel(token: Int) { - if (!close(token)) return + close(token) for ((s, _) in senders) s.resume(closedMessage!!) senders.clear() buffer.clear() diff --git a/kotlinx-coroutines-core/jvm/test/lincheck/MutexLincheckTest.kt b/kotlinx-coroutines-core/jvm/test/lincheck/MutexLincheckTest.kt index 6e6609d6f6..02964f9793 100644 --- a/kotlinx-coroutines-core/jvm/test/lincheck/MutexLincheckTest.kt +++ b/kotlinx-coroutines-core/jvm/test/lincheck/MutexLincheckTest.kt @@ -22,7 +22,8 @@ class MutexLincheckTest : AbstractLincheckTest() { @Operation(promptCancellation = true) suspend fun lock(@Param(name = "owner") owner: Int) = mutex.lock(owner.asOwnerOrNull) - @Operation(promptCancellation = true) + // onLock may suspend in case of clause re-registration. + @Operation(allowExtraSuspension = true, promptCancellation = true) suspend fun onLock(@Param(name = "owner") owner: Int) = select { mutex.onLock(owner.asOwnerOrNull) {} } @Operation(handleExceptionsAsResult = [IllegalStateException::class]) diff --git a/kotlinx-coroutines-core/jvm/test/lincheck/SegmentListRemoveLincheckTest.kt b/kotlinx-coroutines-core/jvm/test/lincheck/SegmentListRemoveLincheckTest.kt deleted file mode 100644 index 5a8d7b475d..0000000000 --- a/kotlinx-coroutines-core/jvm/test/lincheck/SegmentListRemoveLincheckTest.kt +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. - */ - -@file:Suppress("unused") - -package kotlinx.coroutines.lincheck - -import kotlinx.coroutines.* -import kotlinx.coroutines.internal.* -import org.jetbrains.kotlinx.lincheck.* -import org.jetbrains.kotlinx.lincheck.annotations.* -import org.jetbrains.kotlinx.lincheck.paramgen.* -import org.jetbrains.kotlinx.lincheck.strategy.managed.modelchecking.* - -class SegmentListRemoveLincheckTest : AbstractLincheckTest() { - private val q = SegmentBasedQueue() - private val segments: Array> - - init { - segments = (0..5).map { q.enqueue(it)!! }.toTypedArray() - q.enqueue(6) - } - - @Operation - fun removeSegment(@Param(gen = IntGen::class, conf = "1:5") index: Int) { - segments[index].removeSegment() - } - - override fun > O.customize(isStressTest: Boolean): O = this - .actorsBefore(0).actorsAfter(0) - - override fun extractState() = segments.map { it.logicallyRemoved } - - @Validate - fun checkAllRemoved() { - q.checkHeadPrevIsCleaned() - q.checkAllSegmentsAreNotLogicallyRemoved() - } - - override fun ModelCheckingOptions.customize(isStressTest: Boolean) = - checkObstructionFreedom() -} \ No newline at end of file diff --git a/kotlinx-coroutines-core/jvm/test/lincheck/SegmentQueueLincheckTest.kt b/kotlinx-coroutines-core/jvm/test/lincheck/SegmentQueueLincheckTest.kt deleted file mode 100644 index 76a59e39e7..0000000000 --- a/kotlinx-coroutines-core/jvm/test/lincheck/SegmentQueueLincheckTest.kt +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2016-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. - */ -@file:Suppress("unused") - -package kotlinx.coroutines.lincheck - -import kotlinx.coroutines.* -import kotlinx.coroutines.internal.SegmentBasedQueue -import org.jetbrains.kotlinx.lincheck.annotations.* -import org.jetbrains.kotlinx.lincheck.annotations.Operation -import org.jetbrains.kotlinx.lincheck.paramgen.* -import org.jetbrains.kotlinx.lincheck.strategy.managed.modelchecking.* - -@Param(name = "value", gen = IntGen::class, conf = "1:5") -class SegmentQueueLincheckTest : AbstractLincheckTest() { - private val q = SegmentBasedQueue() - - @Operation - fun enqueue(@Param(name = "value") x: Int): Boolean { - return q.enqueue(x) !== null - } - - @Operation - fun dequeue(): Int? = q.dequeue() - - @Operation - fun close() { - q.close() - } - - override fun extractState(): Any { - val elements = ArrayList() - while (true) { - val x = q.dequeue() ?: break - elements.add(x) - } - val closed = q.enqueue(0) === null - return elements to closed - } - - override fun ModelCheckingOptions.customize(isStressTest: Boolean) = - checkObstructionFreedom() -} \ No newline at end of file diff --git a/kotlinx-coroutines-core/jvm/test/selects/SelectMemoryLeakStressTest.kt b/kotlinx-coroutines-core/jvm/test/selects/SelectMemoryLeakStressTest.kt index 7f924dba09..2ddf133f00 100644 --- a/kotlinx-coroutines-core/jvm/test/selects/SelectMemoryLeakStressTest.kt +++ b/kotlinx-coroutines-core/jvm/test/selects/SelectMemoryLeakStressTest.kt @@ -40,7 +40,7 @@ class SelectMemoryLeakStressTest : TestBase() { val data = Channel(1) repeat(nRepeat) { value -> val bigValue = bigValue() // new instance - select { + select { leak.onReceive { println("Capture big value into this lambda: $bigValue") expectUnreached() diff --git a/kotlinx-coroutines-core/native/src/internal/Concurrent.kt b/kotlinx-coroutines-core/native/src/internal/Concurrent.kt index 55c3efe360..17975e2e7f 100644 --- a/kotlinx-coroutines-core/native/src/internal/Concurrent.kt +++ b/kotlinx-coroutines-core/native/src/internal/Concurrent.kt @@ -12,8 +12,6 @@ internal actual typealias ReentrantLock = kotlinx.atomicfu.locks.SynchronizedObj internal actual inline fun ReentrantLock.withLock(action: () -> T): T = this.withLock2(action) -internal actual fun subscriberList(): MutableList = CopyOnWriteList() - internal actual fun identitySet(expectedSize: Int): MutableSet = HashSet() diff --git a/kotlinx-coroutines-debug/src/CoroutinesBlockHoundIntegration.kt b/kotlinx-coroutines-debug/src/CoroutinesBlockHoundIntegration.kt index a26c1928c1..06f702861b 100644 --- a/kotlinx-coroutines-debug/src/CoroutinesBlockHoundIntegration.kt +++ b/kotlinx-coroutines-debug/src/CoroutinesBlockHoundIntegration.kt @@ -106,43 +106,39 @@ public class CoroutinesBlockHoundIntegration : BlockHoundIntegration { } private fun BlockHound.Builder.allowBlockingCallsInChannels() { - allowBlockingCallsInArrayChannel() - allowBlockingCallsInBroadcastChannel() - allowBlockingCallsInConflatedChannel() + allowBlockingCallsInBroadcastChannels() + allowBlockingCallsInConflatedChannels() } /** - * Allows blocking inside [kotlinx.coroutines.channels.ArrayChannel]. + * Allows blocking inside [kotlinx.coroutines.channels.BroadcastChannel]. */ - private fun BlockHound.Builder.allowBlockingCallsInArrayChannel() { - for (method in listOf( - "pollInternal", "isEmpty", "isFull", "isClosedForReceive", "offerInternal", "offerSelectInternal", - "enqueueSend", "pollSelectInternal", "enqueueReceiveInternal", "onCancelIdempotent")) + private fun BlockHound.Builder.allowBlockingCallsInBroadcastChannels() { + for (method in listOf("openSubscription", "removeSubscriber", "send", "trySend", "registerSelectForSend", + "close", "cancelImpl", "isClosedForSend", "value", "valueOrNull")) { - allowBlockingCallsInside("kotlinx.coroutines.channels.ArrayChannel", method) + allowBlockingCallsInside("kotlinx.coroutines.channels.BroadcastChannelImpl", method) } - } - - /** - * Allows blocking inside [kotlinx.coroutines.channels.ArrayBroadcastChannel]. - */ - private fun BlockHound.Builder.allowBlockingCallsInBroadcastChannel() { - for (method in listOf("offerInternal", "offerSelectInternal", "updateHead")) { - allowBlockingCallsInside("kotlinx.coroutines.channels.ArrayBroadcastChannel", method) + for (method in listOf("cancelImpl")) { + allowBlockingCallsInside("kotlinx.coroutines.channels.BroadcastChannelImpl\$SubscriberConflated", method) } - for (method in listOf("checkOffer", "pollInternal", "pollSelectInternal")) { - allowBlockingCallsInside("kotlinx.coroutines.channels.ArrayBroadcastChannel\$Subscriber", method) + for (method in listOf("cancelImpl")) { + allowBlockingCallsInside("kotlinx.coroutines.channels.BroadcastChannelImpl\$SubscriberBuffered", method) } } /** - * Allows blocking inside [kotlinx.coroutines.channels.ConflatedChannel]. + * Allows blocking inside [kotlinx.coroutines.channels.ConflatedBufferedChannel]. */ - private fun BlockHound.Builder.allowBlockingCallsInConflatedChannel() { - for (method in listOf("offerInternal", "offerSelectInternal", "pollInternal", "pollSelectInternal", - "onCancelIdempotent", "isEmpty", "enqueueReceiveInternal")) + private fun BlockHound.Builder.allowBlockingCallsInConflatedChannels() { + for (method in listOf("receive", "receiveCatching", "tryReceive", "registerSelectForReceive", + "send", "trySend", "sendBroadcast", "registerSelectForSend", + "close", "cancelImpl", "isClosedForSend", "isClosedForReceive", "isEmpty")) { - allowBlockingCallsInside("kotlinx.coroutines.channels.ConflatedChannel", method) + allowBlockingCallsInside("kotlinx.coroutines.channels.ConflatedBufferedChannel", method) + } + for (method in listOf("hasNext")) { + allowBlockingCallsInside("kotlinx.coroutines.channels.ConflatedBufferedChannel\$ConflatedChannelIterator", method) } } diff --git a/kotlinx-coroutines-debug/test/BlockHoundTest.kt b/kotlinx-coroutines-debug/test/BlockHoundTest.kt index 3f58878525..5ec767c3d9 100644 --- a/kotlinx-coroutines-debug/test/BlockHoundTest.kt +++ b/kotlinx-coroutines-debug/test/BlockHoundTest.kt @@ -1,4 +1,5 @@ package kotlinx.coroutines.debug + import kotlinx.coroutines.* import kotlinx.coroutines.channels.* import org.junit.* @@ -55,20 +56,22 @@ class BlockHoundTest : TestBase() { } @Test - fun testChannelNotBeingConsideredBlocking() = runTest { + fun testBroadcastChannelNotBeingConsideredBlocking() = runTest { withContext(Dispatchers.Default) { - // Copy of kotlinx.coroutines.channels.ArrayChannelTest.testSimple - val q = Channel(1) - check(q.isEmpty) - check(!q.isClosedForReceive) + // Copy of kotlinx.coroutines.channels.BufferedChannelTest.testSimple + val q = BroadcastChannel(1) + val s = q.openSubscription() check(!q.isClosedForSend) + check(s.isEmpty) + check(!s.isClosedForReceive) val sender = launch { q.send(1) q.send(2) } val receiver = launch { - q.receive() == 1 - q.receive() == 2 + s.receive() == 1 + s.receive() == 2 + s.cancel() } sender.join() receiver.join() @@ -76,7 +79,7 @@ class BlockHoundTest : TestBase() { } @Test - fun testConflatedChannelsNotBeingConsideredBlocking() = runTest { + fun testConflatedChannelNotBeingConsideredBlocking() = runTest { withContext(Dispatchers.Default) { val q = Channel(Channel.CONFLATED) check(q.isEmpty) @@ -110,5 +113,4 @@ class BlockHoundTest : TestBase() { } } } - } diff --git a/reactive/kotlinx-coroutines-reactive/src/Channel.kt b/reactive/kotlinx-coroutines-reactive/src/Channel.kt index a8db21711d..7836ed7d0d 100644 --- a/reactive/kotlinx-coroutines-reactive/src/Channel.kt +++ b/reactive/kotlinx-coroutines-reactive/src/Channel.kt @@ -7,7 +7,6 @@ package kotlinx.coroutines.reactive import kotlinx.atomicfu.* import kotlinx.coroutines.channels.* import kotlinx.coroutines.flow.* -import kotlinx.coroutines.internal.* import org.reactivestreams.* /** @@ -29,7 +28,7 @@ internal fun Publisher.toChannel(request: Int = 1): ReceiveChannel { @Suppress("INVISIBLE_REFERENCE", "INVISIBLE_MEMBER", "SubscriberImplementation") private class SubscriptionChannel( private val request: Int -) : LinkedListChannel(null), Subscriber { +) : BufferedChannel(capacity = Channel.UNLIMITED), Subscriber { init { require(request >= 0) { "Invalid request size: $request" } } @@ -40,7 +39,7 @@ private class SubscriptionChannel( // can be negative if we have receivers, but no subscription yet private val _requested = atomic(0) - // --------------------- AbstractChannel overrides ------------------------------- + // --------------------- BufferedChannel overrides ------------------------------- @Suppress("CANNOT_OVERRIDE_INVISIBLE_MEMBER") override fun onReceiveEnqueued() { _requested.loop { wasRequested -> @@ -64,7 +63,7 @@ private class SubscriptionChannel( } @Suppress("CANNOT_OVERRIDE_INVISIBLE_MEMBER") - override fun onClosedIdempotent(closed: LockFreeLinkedListNode) { + override fun onClosedIdempotent() { _subscription.getAndSet(null)?.cancel() // cancel exactly once } diff --git a/reactive/kotlinx-coroutines-rx2/src/RxChannel.kt b/reactive/kotlinx-coroutines-rx2/src/RxChannel.kt index 8db22799c0..94c9c2224c 100644 --- a/reactive/kotlinx-coroutines-rx2/src/RxChannel.kt +++ b/reactive/kotlinx-coroutines-rx2/src/RxChannel.kt @@ -8,7 +8,6 @@ import io.reactivex.* import io.reactivex.disposables.* import kotlinx.atomicfu.* import kotlinx.coroutines.channels.* -import kotlinx.coroutines.internal.* import kotlinx.coroutines.flow.* import kotlinx.coroutines.reactive.* @@ -46,12 +45,12 @@ internal fun ObservableSource.toChannel(): ReceiveChannel { @Suppress("INVISIBLE_REFERENCE", "INVISIBLE_MEMBER") private class SubscriptionChannel : - LinkedListChannel(null), Observer, MaybeObserver + BufferedChannel(capacity = Channel.UNLIMITED), Observer, MaybeObserver { private val _subscription = atomic(null) @Suppress("CANNOT_OVERRIDE_INVISIBLE_MEMBER") - override fun onClosedIdempotent(closed: LockFreeLinkedListNode) { + override fun onClosedIdempotent() { _subscription.getAndSet(null)?.dispose() // dispose exactly once } diff --git a/reactive/kotlinx-coroutines-rx3/src/RxChannel.kt b/reactive/kotlinx-coroutines-rx3/src/RxChannel.kt index 5d7624a17a..614494438e 100644 --- a/reactive/kotlinx-coroutines-rx3/src/RxChannel.kt +++ b/reactive/kotlinx-coroutines-rx3/src/RxChannel.kt @@ -8,7 +8,6 @@ import io.reactivex.rxjava3.core.* import io.reactivex.rxjava3.disposables.* import kotlinx.atomicfu.* import kotlinx.coroutines.channels.* -import kotlinx.coroutines.internal.* import kotlinx.coroutines.flow.* /** @@ -58,12 +57,12 @@ public suspend inline fun ObservableSource.collect(action: (T) -> U @Suppress("INVISIBLE_REFERENCE", "INVISIBLE_MEMBER") private class SubscriptionChannel : - LinkedListChannel(null), Observer, MaybeObserver + BufferedChannel(capacity = Channel.UNLIMITED), Observer, MaybeObserver { private val _subscription = atomic(null) @Suppress("CANNOT_OVERRIDE_INVISIBLE_MEMBER") - override fun onClosedIdempotent(closed: LockFreeLinkedListNode) { + override fun onClosedIdempotent() { _subscription.getAndSet(null)?.dispose() // dispose exactly once } diff --git a/ui/coroutines-guide-ui.md b/ui/coroutines-guide-ui.md index 4ee898e2a4..6cd791cf6f 100644 --- a/ui/coroutines-guide-ui.md +++ b/ui/coroutines-guide-ui.md @@ -335,10 +335,12 @@ processed. This is also a desired behaviour for UI applications that have to react to incoming high-frequency event streams by updating their UI based on the most recently received update. A coroutine that is using -`ConflatedChannel` avoids delays that are usually introduced by buffering of events. +a conflated channel (`capacity = Channel.CONFLATED`, or a buffered channel with +`onBufferOverflow = DROP_OLDEST` or `onBufferOverflow = DROP_LATEST`) avoids delays +that are usually introduced by buffering of events. You can experiment with `capacity` parameter in the above line to see how it affects the behaviour of the code. -Setting `capacity = Channel.UNLIMITED` creates a coroutine with `LinkedListChannel` mailbox that buffers all +Setting `capacity = Channel.UNLIMITED` creates a coroutine with an unbounded mailbox that buffers all events. In this case, the animation runs as many times as the circle is clicked. ## Blocking operations