[K/N] Make STMS use separate threads for GC and finalizers.

Moves GC into a separate thread, and finalizers processing into yet
another thread. This makes STMS more similar to CMS.

Author: projedi

kotlin-native/runtime/src/custom_alloc/cpp/AtomicStack.hpp

@@ -67,7 +67,7 @@ class AtomicStack: private kotlin::MoveOnly {
         } while (!stack_.compare_exchange_weak(thisHead, otherHead, std::memory_order_acq_rel));
     }
 
-    bool isEmpty() noexcept { return stack_.load(std::memory_order_relaxed) == nullptr; }
+    bool isEmpty() const noexcept { return stack_.load(std::memory_order_relaxed) == nullptr; }
 
     // Not thread-safe. Named like this to make AtomicStack compatible with FinalizerQueue
     size_t size() {

kotlin-native/runtime/src/custom_alloc/cpp/CustomFinalizerProcessor.cpp

@@ -6,41 +6,27 @@
 #ifndef CUSTOM_ALLOC_CPP_CUSTOMFINALIZERPROCESSOR_HPP_
 #define CUSTOM_ALLOC_CPP_CUSTOMFINALIZERPROCESSOR_HPP_
 
-#include <cstdint>
-#include <condition_variable>
-
 #include "AtomicStack.hpp"
+#include "ExtraObjectData.hpp"
 #include "ExtraObjectPage.hpp"
-#include "ScopedThread.hpp"
+#include "FinalizerHooks.hpp"
 
 namespace kotlin::alloc {
 
-class CustomFinalizerProcessor : Pinned {
-public:
-    using Queue = typename kotlin::alloc::AtomicStack<kotlin::alloc::ExtraObjectCell>;
-    explicit CustomFinalizerProcessor(std::function<void(int64_t)> epochDoneCallback) : epochDoneCallback_(std::move(epochDoneCallback)) {}
-    void ScheduleTasks(Queue&& tasks, int64_t epoch) noexcept;
-    void StopFinalizerThread() noexcept;
-    bool IsRunning() noexcept;
-    void StartFinalizerThreadIfNone() noexcept;
-    void WaitFinalizerThreadInitialized() noexcept;
-    ~CustomFinalizerProcessor();
-
-private:
-    ScopedThread finalizerThread_;
-    Queue finalizerQueue_;
-    std::condition_variable finalizerQueueCondVar_;
-    std::mutex finalizerQueueMutex_;
-    std::function<void(int64_t)> epochDoneCallback_;
-    int64_t finalizerQueueEpoch_ = 0;
-    bool shutdownFlag_ = false;
-    bool newTasksAllowed_ = true;
-
-    std::mutex initializedMutex_;
-    std::condition_variable initializedCondVar_;
-    bool initialized_ = false;
-
-    std::mutex threadCreatingMutex_;
+using FinalizerQueue = kotlin::alloc::AtomicStack<kotlin::alloc::ExtraObjectCell>;
+
+struct FinalizerQueueTraits {
+    static bool isEmpty(const FinalizerQueue& queue) noexcept { return queue.isEmpty(); }
+
+    static void add(FinalizerQueue& into, FinalizerQueue from) noexcept { into.TransferAllFrom(std::move(from)); }
+
+    static void process(FinalizerQueue queue) noexcept {
+        while (auto* cell = queue.Pop()) {
+            auto* extraObject = cell->Data();
+            auto* baseObject = extraObject->GetBaseObject();
+            RunFinalizers(baseObject);
+        }
+    }
 };
 
 } // namespace kotlin::alloc

kotlin-native/runtime/src/custom_alloc/cpp/CustomFinalizerProcessor.hpp

@@ -19,11 +19,9 @@
 #include "ThreadRegistry.hpp"
 #include "ThreadSuspension.hpp"
 #include "GCState.hpp"
-#include "FinalizerProcessor.hpp"
 #include "GCStatistics.hpp"
 
 #ifdef CUSTOM_ALLOCATOR
-#include "CustomFinalizerProcessor.hpp"
 #include "Heap.hpp"
 #endif
 
@@ -113,15 +111,14 @@ void gc::ConcurrentMarkAndSweep::ThreadData::OnSuspendForGC() noexcept {
 gc::ConcurrentMarkAndSweep::ConcurrentMarkAndSweep(
         mm::ObjectFactory<ConcurrentMarkAndSweep>& objectFactory, GCScheduler& gcScheduler) noexcept :
     objectFactory_(objectFactory),
-    gcScheduler_(gcScheduler),
-    finalizerProcessor_(std_support::make_unique<FinalizerProcessor>([this](int64_t epoch) {
 #else
 gc::ConcurrentMarkAndSweep::ConcurrentMarkAndSweep(GCScheduler& gcScheduler) noexcept :
-    gcScheduler_(gcScheduler), finalizerProcessor_(std_support::make_unique<alloc::CustomFinalizerProcessor>([this](int64_t epoch) {
 #endif
+    gcScheduler_(gcScheduler),
+    finalizerProcessor_([this](int64_t epoch) {
         GCHandle::getByEpoch(epoch).finalizersDone();
         state_.finalized(epoch);
-    })) {
+    }) {
     gcScheduler_.SetScheduleGC([this]() NO_INLINE {
         // This call acquires a lock, so we need to ensure that we're in the safe state.
         NativeOrUnregisteredThreadGuard guard(/* reentrant = */ true);
@@ -147,31 +144,31 @@ gc::ConcurrentMarkAndSweep::~ConcurrentMarkAndSweep() {
 
 void gc::ConcurrentMarkAndSweep::StartFinalizerThreadIfNeeded() noexcept {
     NativeOrUnregisteredThreadGuard guard(true);
-    finalizerProcessor_->StartFinalizerThreadIfNone();
-    finalizerProcessor_->WaitFinalizerThreadInitialized();
+    finalizerProcessor_.StartFinalizerThreadIfNone();
+    finalizerProcessor_.WaitFinalizerThreadInitialized();
 }
 
 void gc::ConcurrentMarkAndSweep::StopFinalizerThreadIfRunning() noexcept {
     NativeOrUnregisteredThreadGuard guard(true);
-    finalizerProcessor_->StopFinalizerThread();
+    finalizerProcessor_.StopFinalizerThread();
 }
 
 bool gc::ConcurrentMarkAndSweep::FinalizersThreadIsRunning() noexcept {
-    return finalizerProcessor_->IsRunning();
+    return finalizerProcessor_.IsRunning();
 }
 
 void gc::ConcurrentMarkAndSweep::SetMarkingBehaviorForTests(MarkingBehavior markingBehavior) noexcept {
     markingBehavior_ = markingBehavior;
 }
 
-bool gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
+void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
     auto gcHandle = GCHandle::create(epoch);
     SetMarkingRequested(epoch);
     bool didSuspend = mm::RequestThreadsSuspension();
     RuntimeAssert(didSuspend, "Only GC thread can request suspension");
     gcHandle.suspensionRequested();
 
-    RuntimeAssert(!kotlin::mm::IsCurrentThreadRegistered(), "Concurrent GC must run on unregistered thread");
+    RuntimeAssert(!kotlin::mm::IsCurrentThreadRegistered(), "GC must run on unregistered thread");
     WaitForThreadsReadyToMark();
     gcHandle.threadsAreSuspended();
 
@@ -218,11 +215,11 @@ bool gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
     state_.finish(epoch);
     gcHandle.finalizersScheduled(finalizerQueue.size());
     gcHandle.finished();
+
     // This may start a new thread. On some pthreads implementations, this may block waiting for concurrent thread
     // destructors running. So, it must ensured that no locks are held by this point.
     // TODO: Consider having an always on sleeping finalizer thread.
-    finalizerProcessor_->ScheduleTasks(std::move(finalizerQueue), epoch);
-    return true;
+    finalizerProcessor_.ScheduleTasks(std::move(finalizerQueue), epoch);
 }
 
 namespace {

kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp

@@ -9,34 +9,28 @@
 #include <cstddef>
 
 #include "Allocator.hpp"
+#include "FinalizerProcessor.hpp"
 #include "GCScheduler.hpp"
+#include "GCState.hpp"
+#include "GCStatistics.hpp"
 #include "IntrusiveList.hpp"
 #include "MarkAndSweepUtils.hpp"
 #include "ObjectFactory.hpp"
 #include "ScopedThread.hpp"
 #include "ThreadData.hpp"
 #include "Types.h"
 #include "Utils.hpp"
-#include "GCState.hpp"
 #include "std_support/Memory.hpp"
-#include "GCStatistics.hpp"
 
 #ifdef CUSTOM_ALLOCATOR
 #include "CustomAllocator.hpp"
+#include "CustomFinalizerProcessor.hpp"
 #include "Heap.hpp"
-
-namespace kotlin::alloc {
-class CustomFinalizerProcessor;
-}
 #endif
 
 namespace kotlin {
 namespace gc {
 
-#ifndef CUSTOM_ALLOCATOR
-class FinalizerProcessor;
-#endif
-
 // Stop-the-world parallel mark + concurrent sweep. The GC runs in a separate thread, finalizers run in another thread of their own.
 // TODO: Also make marking run concurrently with Kotlin threads.
 class ConcurrentMarkAndSweep : private Pinned {
@@ -108,6 +102,14 @@ class ConcurrentMarkAndSweep : private Pinned {
 
     using Allocator = ThreadData::Allocator;
 
+#ifndef CUSTOM_ALLOCATOR
+    using FinalizerQueue = mm::ObjectFactory<ConcurrentMarkAndSweep>::FinalizerQueue;
+    using FinalizerQueueTraits = mm::ObjectFactory<ConcurrentMarkAndSweep>::FinalizerQueueTraits;
+#else
+    using FinalizerQueue = alloc::FinalizerQueue;
+    using FinalizerQueueTraits = alloc::FinalizerQueueTraits;
+#endif
+
 #ifdef CUSTOM_ALLOCATOR
     explicit ConcurrentMarkAndSweep(GCScheduler& scheduler) noexcept;
 #else
@@ -128,8 +130,7 @@ class ConcurrentMarkAndSweep : private Pinned {
 #endif
 
 private:
-    // Returns `true` if GC has happened, and `false` if not (because someone else has suspended the threads).
-    bool PerformFullGC(int64_t epoch) noexcept;
+    void PerformFullGC(int64_t epoch) noexcept;
 
 #ifndef CUSTOM_ALLOCATOR
     mm::ObjectFactory<ConcurrentMarkAndSweep>& objectFactory_;
@@ -140,11 +141,7 @@ class ConcurrentMarkAndSweep : private Pinned {
 
     GCStateHolder state_;
     ScopedThread gcThread_;
-#ifndef CUSTOM_ALLOCATOR
-    std_support::unique_ptr<FinalizerProcessor> finalizerProcessor_;
-#else
-    std_support::unique_ptr<alloc::CustomFinalizerProcessor> finalizerProcessor_;
-#endif
+    FinalizerProcessor<FinalizerQueue, FinalizerQueueTraits> finalizerProcessor_;
 
     MarkQueue markQueue_;
     MarkingBehavior markingBehavior_;