[Router] Fixed No source in route tree in ParallelConnectionRouter

The `No source in route tree` bug in ParallelConnectionRouter (since
commit 875b98e) has been fixed. It turns out that putting another member
variable `MultiQueueDAryHeap<HeapImplementation::arg_D> heap_` in the
derived class ParallelConnectionRouter together with the existing
`HeapImplementation heap_` in the base class ConnectionRouter causes the
issue. The solution is to keep `heap_` only in the base class and use
`ConnectionRouter<MultiQueueDAryHeap<HeapImplementation::arg_D>>` rather
than `ConnectionRouter<HeapImplementation>` for deriving the parallel
connection router.

Please note that ParallelConnectionRouter still has some bugs (i.e.,
getting stuck in the MultiQueue pop). This commit is not fully working.
Please do not use it for any experiments.

Updated the previously incorrect command-line options for the parallel
connection router in the regression tests.

Author: ueqri

vpr/src/route/NestedNetlistRouter.h

@@ -6,6 +6,7 @@
 #include "vtr_thread_pool.h"
 #include "serial_connection_router.h"
 #include "parallel_connection_router.h"
+#include <memory>
 #include <unordered_map>
 
 /* Add cmd line option for this later */
@@ -51,11 +52,7 @@ class NestedNetlistRouter : public NetlistRouter {
         , _choking_spots(choking_spots)
         , _is_flat(is_flat)
         , _thread_pool(MAX_THREADS) {}
-    ~NestedNetlistRouter() {
-        for (auto& [_, router] : _routers_th) {
-            delete router;
-        }
-    }
+    ~NestedNetlistRouter() {}
 
     /** Run a single iteration of netlist routing for this->_net_list. This usually means calling
      * \ref route_net for each net, which will handle other global updates.
@@ -73,15 +70,15 @@ class NestedNetlistRouter : public NetlistRouter {
     /** Route all nets in a PartitionTree node and add its children to the task queue. */
     void route_partition_tree_node(PartitionTreeNode& node);
 
-    ConnectionRouter<HeapType>* _make_router(const RouterLookahead* router_lookahead,
-                                             const t_router_opts& router_opts,
-                                             bool is_flat) {
+    std::shared_ptr<ConnectionRouterInterface> _make_router(const RouterLookahead* router_lookahead,
+                                                            const t_router_opts& router_opts,
+                                                            bool is_flat) {
         auto& device_ctx = g_vpr_ctx.device();
         auto& route_ctx = g_vpr_ctx.mutable_routing();
 
         if (!router_opts.enable_parallel_connection_router) {
             // Serial Connection Router
-            return new SerialConnectionRouter<HeapType>(
+            return std::make_shared<SerialConnectionRouter<HeapType>>(
                 device_ctx.grid,
                 *router_lookahead,
                 device_ctx.rr_graph.rr_nodes(),
@@ -92,7 +89,7 @@ class NestedNetlistRouter : public NetlistRouter {
                 is_flat);
         } else {
             // Parallel Connection Router
-            return new ParallelConnectionRouter<HeapType>(
+            return std::make_shared<ParallelConnectionRouter<HeapType>>(
                 device_ctx.grid,
                 *router_lookahead,
                 device_ctx.rr_graph.rr_nodes(),
@@ -134,13 +131,13 @@ class NestedNetlistRouter : public NetlistRouter {
 
     /* Thread-local storage.
      * These are maps because thread::id is a random integer instead of 1, 2, ... */
-    std::unordered_map<std::thread::id, ConnectionRouter<HeapType>*> _routers_th;
+    std::unordered_map<std::thread::id, std::shared_ptr<ConnectionRouterInterface>> _routers_th;
     std::unordered_map<std::thread::id, RouteIterResults> _results_th;
     std::mutex _storage_mutex;
 
     /** Get a thread-local ConnectionRouter. We lock the id->router lookup, but this is
      * accessed once per partition so the overhead should be small */
-    ConnectionRouter<HeapType>* get_thread_router() {
+    std::shared_ptr<ConnectionRouterInterface> get_thread_router() {
         auto id = std::this_thread::get_id();
         std::lock_guard<std::mutex> lock(_storage_mutex);
         if (!_routers_th.count(id)) {

vpr/src/route/SerialNetlistRouter.h

@@ -34,25 +34,23 @@ class SerialNetlistRouter : public NetlistRouter {
         , _routing_predictor(routing_predictor)
         , _choking_spots(choking_spots)
         , _is_flat(is_flat) {}
-    ~SerialNetlistRouter() {
-        delete _router;
-    }
+    ~SerialNetlistRouter() {}
 
     RouteIterResults route_netlist(int itry, float pres_fac, float worst_neg_slack);
     void handle_bb_updated_nets(const std::vector<ParentNetId>& nets);
     void set_rcv_enabled(bool x);
     void set_timing_info(std::shared_ptr<SetupHoldTimingInfo> timing_info);
 
   private:
-    ConnectionRouter<HeapType>* _make_router(const RouterLookahead* router_lookahead,
-                                             const t_router_opts& router_opts,
-                                             bool is_flat) {
+    std::unique_ptr<ConnectionRouterInterface> _make_router(const RouterLookahead* router_lookahead,
+                                                            const t_router_opts& router_opts,
+                                                            bool is_flat) {
         auto& device_ctx = g_vpr_ctx.device();
         auto& route_ctx = g_vpr_ctx.mutable_routing();
 
         if (!router_opts.enable_parallel_connection_router) {
             // Serial Connection Router
-            return new SerialConnectionRouter<HeapType>(
+            return std::make_unique<SerialConnectionRouter<HeapType>>(
                 device_ctx.grid,
                 *router_lookahead,
                 device_ctx.rr_graph.rr_nodes(),
@@ -63,7 +61,7 @@ class SerialNetlistRouter : public NetlistRouter {
                 is_flat);
         } else {
             // Parallel Connection Router
-            return new ParallelConnectionRouter<HeapType>(
+            return std::make_unique<ParallelConnectionRouter<HeapType>>(
                 device_ctx.grid,
                 *router_lookahead,
                 device_ctx.rr_graph.rr_nodes(),
@@ -78,7 +76,7 @@ class SerialNetlistRouter : public NetlistRouter {
         }
     }
     /* Context fields */
-    ConnectionRouter<HeapType>* _router;
+    std::unique_ptr<ConnectionRouterInterface> _router;
     const Netlist<>& _net_list;
     const t_router_opts& _router_opts;
     CBRR& _connections_inf;

vpr/src/route/connection_router.h

@@ -332,9 +332,6 @@ class ConnectionRouter : public ConnectionRouterInterface {
     /** Is flat router enabled or not? */
     bool is_flat_;
 
-    /** Node IDs of modified nodes in rr_node_route_inf */
-    std::vector<RRNodeId> modified_rr_node_inf_;
-
     /** Router statistics (e.g., heap push/pop counts) */
     RouterStats* router_stats_;
 

vpr/src/route/multi_queue_d_ary_heap.h

@@ -20,10 +20,11 @@
 #ifndef _MULTI_QUEUE_D_ARY_HEAP_H
 #define _MULTI_QUEUE_D_ARY_HEAP_H
 
-#include <tuple>
 #include "device_grid.h"
 #include "heap_type.h"
 #include "multi_queue_d_ary_heap.tpp"
+#include <tuple>
+#include <memory>
 
 using MQHeapNode = std::tuple<HeapNodePriority, uint32_t /*FIXME*/>;
 
@@ -40,20 +41,24 @@ class MultiQueueDAryHeap {
     using MQ_IO = MultiQueueIO<D, MQHeapNode, MQHeapNodeTupleComparator, HeapNodePriority>;
 
     MultiQueueDAryHeap() {
-        pq_ = new MQ_IO(2, 1, 0); // Serial (#threads=1, #queues=2) by default
+        set_num_threads_and_queues(2, 1); // Serial (#threads=1, #queues=2) by default
     }
 
     MultiQueueDAryHeap(size_t num_threads, size_t num_queues) {
-        pq_ = new MQ_IO(num_queues, num_threads, 0 /*Dont care (batch size for only popBatch)*/);
+        set_num_threads_and_queues(num_threads, num_queues);
     }
 
-    ~MultiQueueDAryHeap() {
-        delete pq_;
+    ~MultiQueueDAryHeap() {}
+
+    void set_num_threads_and_queues(size_t num_threads, size_t num_queues) {
+        pq_.reset();
+        pq_ = std::make_unique<MQ_IO>(num_threads, num_queues, 0 /*Dont care (batch size for only popBatch)*/);
     }
 
     void init_heap(const DeviceGrid& grid) {
         (void)grid;
         // TODO: Reserve storage for MQ_IO
+        // Note: This function could be called before setting num_threads/num_queues
     }
 
     bool try_pop(HeapNode& heap_node) {
@@ -119,7 +124,7 @@ class MultiQueueDAryHeap {
 #endif
 
   private:
-    MQ_IO* pq_;
+    std::unique_ptr<MQ_IO> pq_;
 };
 
 #endif

vpr/src/route/parallel_connection_router.cpp

@@ -394,12 +394,12 @@ void ParallelConnectionRouter<Heap>::timing_driven_add_to_heap(const t_conn_cost
     if (to_node == target_node) {
 #ifdef MQ_IO_ENABLE_CLEAR_FOR_POP
         if (multi_queue_direct_draining_) {
-            heap_.setMinPrioForPop(new_total_cost);
+            this->heap_.setMinPrioForPop(new_total_cost);
         }
 #endif
         return;
     }
-    heap_.add_to_heap({new_total_cost, to_node});
+    this->heap_.add_to_heap({new_total_cost, to_node});
 }
 
 template<typename Heap>

vpr/src/route/parallel_connection_router.h

@@ -153,7 +153,7 @@ using barrier_t = barrier_spin_t; // Using the spin-based thread barrier
  * Dec. 2024.
  */
 template<typename HeapImplementation>
-class ParallelConnectionRouter : public ConnectionRouter<HeapImplementation> {
+class ParallelConnectionRouter : public ConnectionRouter<MultiQueueDAryHeap<HeapImplementation::arg_D>> {
   public:
     ParallelConnectionRouter(
         const DeviceGrid& grid,
@@ -167,13 +167,14 @@ class ParallelConnectionRouter : public ConnectionRouter<HeapImplementation> {
         int multi_queue_num_threads,
         int multi_queue_num_queues,
         bool multi_queue_direct_draining)
-        : ConnectionRouter<HeapImplementation>(grid, router_lookahead, rr_nodes, rr_graph, rr_rc_data, rr_switch_inf, rr_node_route_inf, is_flat)
+        : ConnectionRouter<MultiQueueDAryHeap<HeapImplementation::arg_D>>(grid, router_lookahead, rr_nodes, rr_graph, rr_rc_data, rr_switch_inf, rr_node_route_inf, is_flat)
         , modified_rr_node_inf_(multi_queue_num_threads)
-        , heap_(multi_queue_num_threads, multi_queue_num_queues)
         , thread_barrier_(multi_queue_num_threads)
         , is_router_destroying_(false)
         , locks_(rr_node_route_inf.size())
         , multi_queue_direct_draining_(multi_queue_direct_draining) {
+        // Set the MultiQueue parameters
+        this->heap_.set_num_threads_and_queues(multi_queue_num_threads, multi_queue_num_queues);
         // Initialize the thread barrier
         this->thread_barrier_.init();
         // Instantiate (multi_queue_num_threads - 1) helper threads
@@ -400,9 +401,6 @@ class ParallelConnectionRouter : public ConnectionRouter<HeapImplementation> {
     /** Node IDs of modified nodes in rr_node_route_inf for each thread*/
     std::vector<std::vector<RRNodeId>> modified_rr_node_inf_;
 
-    /** MultiQueue-based parallel heap */
-    MultiQueueDAryHeap<HeapImplementation::arg_D> heap_;
-
     /** Helper threads */
     std::vector<std::thread> sub_threads_;
 

vpr/src/route/serial_connection_router.h

@@ -231,6 +231,9 @@ class SerialConnectionRouter : public ConnectionRouter<HeapImplementation> {
     vtr::vector<RRNodeId, RTExploredNode> timing_driven_find_all_shortest_paths_from_heap(
         const t_conn_cost_params& cost_params,
         const t_bb& bounding_box) final;
+
+    /** Node IDs of modified nodes in rr_node_route_inf */
+    std::vector<RRNodeId> modified_rr_node_inf_;
 };
 
 /** Construct a serial connection router that uses the specified heap type.

vtr_flow/tasks/regression_tests/vtr_reg_strong/koios_test/config/config.txt

@@ -38,6 +38,6 @@ pass_requirements_file=pass_requirements.txt
 script_params_common=-track_memory_usage
 script_params_list_add =
 script_params_list_add = --router_algorithm parallel
-script_params_list_add = --enable_parallel_connection_router
-script_params_list_add = --enable_parallel_connection_router --multi_queue_num_threads 4 --multi_queue_num_queues 16
-script_params_list_add = --enable_parallel_connection_router --multi_queue_num_threads 2 --multi_queue_num_queues 4 --multi_queue_direct_draining
+script_params_list_add = --enable_parallel_connection_router on
+script_params_list_add = --enable_parallel_connection_router on --multi_queue_num_threads 4 --multi_queue_num_queues 16
+script_params_list_add = --enable_parallel_connection_router on --multi_queue_num_threads 2 --multi_queue_num_queues 4 --multi_queue_direct_draining on

vtr_flow/tasks/regression_tests/vtr_reg_strong/strong_flat_router/config/config.txt

@@ -27,6 +27,6 @@ pass_requirements_file=pass_requirements.txt
 script_params_common=-track_memory_usage --route_chan_width 100 --max_router_iterations 100 --router_lookahead map --flat_routing on
 script_params_list_add =
 script_params_list_add = --router_algorithm parallel --num_workers 4
-script_params_list_add = --enable_parallel_connection_router
-script_params_list_add = --enable_parallel_connection_router --multi_queue_num_threads 4 --multi_queue_num_queues 16
-script_params_list_add = --enable_parallel_connection_router --multi_queue_num_threads 2 --multi_queue_num_queues 4 --multi_queue_direct_draining
+script_params_list_add = --enable_parallel_connection_router on
+script_params_list_add = --enable_parallel_connection_router on --multi_queue_num_threads 4 --multi_queue_num_queues 16
+script_params_list_add = --enable_parallel_connection_router on --multi_queue_num_threads 2 --multi_queue_num_queues 4 --multi_queue_direct_draining on

vtr_flow/tasks/regression_tests/vtr_reg_strong/strong_multiclock/config/config.txt

@@ -27,6 +27,6 @@ pass_requirements_file=pass_requirements_multiclock.txt
 script_params_common=-starting_stage vpr -sdc_file tasks/regression_tests/vtr_reg_strong/strong_multiclock/config/multiclock.sdc
 script_params_list_add =
 script_params_list_add = --router_algorithm parallel --num_workers 4
-script_params_list_add = --enable_parallel_connection_router
-script_params_list_add = --enable_parallel_connection_router --multi_queue_num_threads 4 --multi_queue_num_queues 16
-script_params_list_add = --enable_parallel_connection_router --multi_queue_num_threads 2 --multi_queue_num_queues 4 --multi_queue_direct_draining
+script_params_list_add = --enable_parallel_connection_router on
+script_params_list_add = --enable_parallel_connection_router on --multi_queue_num_threads 4 --multi_queue_num_queues 16
+script_params_list_add = --enable_parallel_connection_router on --multi_queue_num_threads 2 --multi_queue_num_queues 4 --multi_queue_direct_draining on

vtr_flow/tasks/regression_tests/vtr_reg_strong/strong_timing/config/config.txt

@@ -27,6 +27,6 @@ pass_requirements_file=pass_requirements.txt
 script_params_common = -track_memory_usage
 script_params_list_add =
 script_params_list_add = --router_algorithm parallel --num_workers 4
-script_params_list_add = --enable_parallel_connection_router
-script_params_list_add = --enable_parallel_connection_router --multi_queue_num_threads 4 --multi_queue_num_queues 16
-script_params_list_add = --enable_parallel_connection_router --multi_queue_num_threads 2 --multi_queue_num_queues 4 --multi_queue_direct_draining
+script_params_list_add = --enable_parallel_connection_router on
+script_params_list_add = --enable_parallel_connection_router on --multi_queue_num_threads 4 --multi_queue_num_queues 16
+script_params_list_add = --enable_parallel_connection_router on --multi_queue_num_threads 2 --multi_queue_num_queues 4 --multi_queue_direct_draining on

vtr_flow/tasks/regression_tests/vtr_reg_strong/strong_timing_update_type/config/config.txt

@@ -31,9 +31,9 @@ script_params_list_add = --timing_update_type incremental
 script_params_list_add = --timing_update_type incremental --quench_recompute_divider 999999999 #Do post-move incremental STA during quench
 script_params_list_add = --timing_update_type incremental --router_algorithm parallel --num_workers 4  # rarely exercised code path
 script_params_list_add = --timing_update_type full --router_algorithm parallel --num_workers 4
-script_params_list_add = --timing_update_type incremental --enable_parallel_connection_router
-script_params_list_add = --timing_update_type incremental --enable_parallel_connection_router --multi_queue_num_threads 4 --multi_queue_num_queues 16
-script_params_list_add = --timing_update_type incremental --enable_parallel_connection_router --multi_queue_num_threads 2 --multi_queue_num_queues 4 --multi_queue_direct_draining
-script_params_list_add = --timing_update_type full --enable_parallel_connection_router
-script_params_list_add = --timing_update_type full --enable_parallel_connection_router --multi_queue_num_threads 4 --multi_queue_num_queues 16
-script_params_list_add = --timing_update_type full --enable_parallel_connection_router --multi_queue_num_threads 2 --multi_queue_num_queues 4 --multi_queue_direct_draining
+script_params_list_add = --timing_update_type incremental --enable_parallel_connection_router on
+script_params_list_add = --timing_update_type incremental --enable_parallel_connection_router on --multi_queue_num_threads 4 --multi_queue_num_queues 16
+script_params_list_add = --timing_update_type incremental --enable_parallel_connection_router on --multi_queue_num_threads 2 --multi_queue_num_queues 4 --multi_queue_direct_draining on
+script_params_list_add = --timing_update_type full --enable_parallel_connection_router on
+script_params_list_add = --timing_update_type full --enable_parallel_connection_router on --multi_queue_num_threads 4 --multi_queue_num_queues 16
+script_params_list_add = --timing_update_type full --enable_parallel_connection_router on --multi_queue_num_threads 2 --multi_queue_num_queues 4 --multi_queue_direct_draining on