duplicate rr_node_route_inf for decomp router

Author: duck2

utils/route_diag/src/main.cpp

@@ -139,7 +139,7 @@ static void do_one_route(const Netlist<>& net_list,
         tree.print();
         VTR_LOG("\n");
 
-        VTR_ASSERT_MSG(route_ctx.rr_node_route_inf[tree.root().inode].occ() <= rr_graph.node_capacity(tree.root().inode), "SOURCE should never be congested");
+        VTR_ASSERT_MSG(route_ctx.rr_node_occ_inf[tree.root().inode].occ() <= rr_graph.node_capacity(tree.root().inode), "SOURCE should never be congested");
     } else {
         VTR_LOG("Routing failed");
     }

vpr/src/base/old_traceback.cpp

@@ -130,7 +130,7 @@ void print_traceback(const t_trace* trace) {
             VTR_LOG("*"); //Reached non-configurably
         }
 
-        if (route_ctx.rr_node_route_inf[inode].occ() > rr_graph.node_capacity(inode)) {
+        if (route_ctx.rr_node_occ_inf[inode].occ() > rr_graph.node_capacity(inode)) {
             VTR_LOG(" x"); //Overused
         }
         VTR_LOG("\n");

vpr/src/base/vpr_context.h

@@ -435,6 +435,8 @@ struct RoutingContext : public Context {
 
     vtr::vector<RRNodeId, t_rr_node_route_inf> rr_node_route_inf; /* [0..device_ctx.num_rr_nodes-1] */
 
+    vtr::vector<RRNodeId, t_rr_node_occ_inf> rr_node_occ_inf; /* [0..device_ctx.num_rr_nodes-1] */
+
     vtr::vector<ParentNetId, std::vector<std::vector<int>>> net_terminal_groups;
 
     vtr::vector<ParentNetId, std::vector<int>> net_terminal_group_num;

vpr/src/base/vpr_types.h

@@ -1723,31 +1723,36 @@ constexpr bool is_src_sink(e_rr_type type) { return (type == SOURCE || type == S
  *   @param prev_edge  Index of the edge (from 0 to num_edges-1 of prev_node)
  *                     that was used to reach this node from the previous node.
  *                     If there is no predecessor, prev_edge = NO_PREVIOUS.
- *   @param acc_cost   Accumulated cost term from previous Pathfinder iterations.
  *   @param path_cost  Total cost of the path up to and including this node +
  *                     the expected cost to the target if the timing_driven router
  *                     is being used.
  *   @param backward_path_cost  Total cost of the path up to and including this
  *                     node.
- *   @param target_flag  Is this node a target (sink) for the current routing?
- *                     Number of times this node must be reached to fully route.
- *   @param occ        The current occupancy of the associated rr node
  */
 struct t_rr_node_route_inf {
     RRNodeId prev_node;
     RREdgeId prev_edge;
 
-    float acc_cost;
     float path_cost;
     float backward_path_cost;
+};
 
-    short target_flag;
+/**
+ * @brief Occupancy related routing information about each rr_node.
+ *
+ *   @param acc_cost   Accumulated cost term from previous Pathfinder iterations.
+ *   @param target_flag  Is this node a target (sink) for the current routing?
+ *                     Number of times this node must be reached to fully route.
+ *   @param occ        The current occupancy of the associated rr node
+ */
+ struct t_rr_node_occ_inf {
+    float acc_cost;
 
   public: //Accessors
-    short occ() const { return occ_; }
+    inline short occ() const { return occ_; }
 
   public: //Mutators
-    void set_occ(int new_occ) { occ_ = new_occ; }
+    inline void set_occ(int new_occ) { occ_ = new_occ; }
 
   private: //Data
     short occ_ = 0;

vpr/src/draw/draw_basic.cpp

@@ -306,7 +306,7 @@ void draw_congestion(ezgl::renderer* g) {
     float max_congestion_ratio = min_congestion_ratio;
     auto congested_rr_nodes = collect_congested_rr_nodes();
     for (RRNodeId inode : congested_rr_nodes) {
-        short occ = route_ctx.rr_node_route_inf[inode].occ();
+        short occ = route_ctx.rr_node_occ_inf[inode].occ();
         short capacity = rr_graph.node_capacity(inode);
 
         float congestion_ratio = float(occ) / capacity;
@@ -328,10 +328,10 @@ void draw_congestion(ezgl::renderer* g) {
     //Sort the nodes in ascending order of value for drawing, this ensures high
     //valued nodes are not overdrawn by lower value ones (e.g-> when zoomed-out far)
     auto cmp_ascending_acc_cost = [&](RRNodeId lhs_node, RRNodeId rhs_node) {
-        short lhs_occ = route_ctx.rr_node_route_inf[lhs_node].occ();
+        short lhs_occ = route_ctx.rr_node_occ_inf[lhs_node].occ();
         short lhs_capacity = rr_graph.node_capacity(lhs_node);
 
-        short rhs_occ = route_ctx.rr_node_route_inf[rhs_node].occ();
+        short rhs_occ = route_ctx.rr_node_occ_inf[rhs_node].occ();
         short rhs_capacity = rr_graph.node_capacity(rhs_node);
 
         float lhs_cong_ratio = float(lhs_occ) / lhs_capacity;
@@ -369,7 +369,7 @@ void draw_congestion(ezgl::renderer* g) {
         int transparency_factor = get_rr_node_transparency(inode);
         if (!draw_state->draw_layer_display[layer_num].visible)
             continue;
-        short occ = route_ctx.rr_node_route_inf[inode].occ();
+        short occ = route_ctx.rr_node_occ_inf[inode].occ();
         short capacity = rr_graph.node_capacity(inode);
 
         float congestion_ratio = float(occ) / capacity;
@@ -414,7 +414,7 @@ void draw_routing_costs(ezgl::renderer* g) {
     auto& route_ctx = g_vpr_ctx.routing();
     g->set_line_width(0);
 
-    VTR_ASSERT(!route_ctx.rr_node_route_inf.empty());
+    VTR_ASSERT(!route_ctx.rr_node_occ_inf.empty());
 
     float min_cost = std::numeric_limits<float>::infinity();
     float max_cost = -min_cost;

vpr/src/route/DecompNetlistRouter.h

@@ -3,6 +3,7 @@
 /** @file Parallel and net-decomposing case for NetlistRouter. Works like
  * \see ParallelNetlistRouter, but tries to "decompose" nets and assign them to
  * the next level of the partition tree where possible. */
+#include "globals.h"
 #include "netlist_routers.h"
 
 #include <tbb/task_group.h>
@@ -37,7 +38,8 @@ class DecompNetlistRouter : public NetlistRouter {
         const RoutingPredictor& routing_predictor,
         const vtr::vector<ParentNetId, std::vector<std::unordered_map<RRNodeId, int>>>& choking_spots,
         bool is_flat)
-        : _routers_th(_make_router(router_lookahead, is_flat))
+        : _rr_node_route_infs_th(g_vpr_ctx.routing().rr_node_route_inf) /* Copy existing route_inf to all threads */
+        , _routers_th([router_lookahead, is_flat, this] { return _make_router(router_lookahead, is_flat); }) /* Call make_router on demand */
         , _net_list(net_list)
         , _router_opts(router_opts)
         , _connections_inf(connections_inf)
@@ -80,7 +82,6 @@ class DecompNetlistRouter : public NetlistRouter {
 
     ConnectionRouter<HeapType> _make_router(const RouterLookahead* router_lookahead, bool is_flat) {
         auto& device_ctx = g_vpr_ctx.device();
-        auto& route_ctx = g_vpr_ctx.mutable_routing();
 
         return ConnectionRouter<HeapType>(
             device_ctx.grid,
@@ -89,11 +90,13 @@ class DecompNetlistRouter : public NetlistRouter {
             &device_ctx.rr_graph,
             device_ctx.rr_rc_data,
             device_ctx.rr_graph.rr_switch(),
-            route_ctx.rr_node_route_inf,
+             _rr_node_route_infs_th.local(),
             is_flat);
     }
 
     /* Context fields. Most of them will be forwarded to route_net (see route_net.tpp) */
+    /** Per-thread storage for rr_node_route_infs (we need them to route 1 net with several threads) */
+    tbb::enumerable_thread_specific<vtr::vector<RRNodeId, t_rr_node_route_inf>> _rr_node_route_infs_th;
     /** Per-thread storage for ConnectionRouters. */
     tbb::enumerable_thread_specific<ConnectionRouter<HeapType>> _routers_th;
     const Netlist<>& _net_list;

vpr/src/route/DecompNetlistRouter.tpp

@@ -25,10 +25,13 @@ inline RouteIterResults DecompNetlistRouter<HeapType>::route_netlist(int itry, f
      * Nets in a given level of nodes are guaranteed to not have any overlapping bounding boxes, so they can be routed in parallel. */
     PartitionTree tree(_net_list);
 
+    vtr::Timer t;
+
     /* Put the root node on the task queue, which will add its child nodes when it's finished. Wait until the entire tree gets routed. */
     tbb::task_group g;
     route_partition_tree_node(g, tree.root());
     g.wait();
+    PartitionTreeDebug::log("Routing all nets took " + std::to_string(t.elapsed_sec()) + " s");
 
     /* Combine results from threads */
     RouteIterResults out;
@@ -37,6 +40,7 @@ inline RouteIterResults DecompNetlistRouter<HeapType>::route_netlist(int itry, f
         out.rerouted_nets.insert(out.rerouted_nets.end(), results.rerouted_nets.begin(), results.rerouted_nets.end());
         out.is_routable &= results.is_routable;
     }
+
     return out;
 }
 
@@ -147,7 +151,8 @@ void DecompNetlistRouter<HeapType>::route_partition_tree_node(tbb::task_group& g
                 _net_list,
                 _connections_inf,
                 _router_opts,
-                _worst_neg_slack);
+                _worst_neg_slack,
+                _rr_node_route_infs_th.local());
             /* Try decomposing the net. */
             if (should_decompose_net(net_id, node)) {
                 VirtualNet left_vnet, right_vnet;
@@ -177,6 +182,7 @@ void DecompNetlistRouter<HeapType>::route_partition_tree_node(tbb::task_group& g
                 _worst_neg_slack,
                 _routing_predictor,
                 _choking_spots[net_id],
+                _rr_node_route_infs_th.local(),
                 _is_flat,
                 route_ctx.route_bb[net_id],
                 false);
@@ -225,6 +231,7 @@ void DecompNetlistRouter<HeapType>::route_partition_tree_node(tbb::task_group& g
                 _worst_neg_slack,
                 _routing_predictor,
                 _choking_spots[vnet.net_id],
+                _rr_node_route_infs_th.local(),
                 _is_flat,
                 vnet.clipped_bb,
                 false,
@@ -301,6 +308,7 @@ bool DecompNetlistRouter<HeapType>::decompose_and_route_net(ParentNetId net_id,
         _worst_neg_slack,
         _routing_predictor,
         _choking_spots[net_id],
+        _rr_node_route_infs_th.local(),
         _is_flat,
         net_bb,
         false,
@@ -402,6 +410,7 @@ bool DecompNetlistRouter<HeapType>::decompose_and_route_vnet(VirtualNet& vnet, c
         _worst_neg_slack,
         _routing_predictor,
         _choking_spots[vnet.net_id],
+        _rr_node_route_infs_th.local(),
         _is_flat,
         vnet.clipped_bb,
         false,

vpr/src/route/ParallelNetlistRouter.tpp

@@ -65,6 +65,7 @@ void ParallelNetlistRouter<HeapType>::route_partition_tree_node(tbb::task_group&
             _worst_neg_slack,
             _routing_predictor,
             _choking_spots[net_id],
+            route_ctx.rr_node_route_inf,
             _is_flat,
             route_ctx.route_bb[net_id]);
 

vpr/src/route/SerialNetlistRouter.tpp

@@ -35,6 +35,7 @@ inline RouteIterResults SerialNetlistRouter<HeapType>::route_netlist(int itry, f
             worst_neg_slack,
             _routing_predictor,
             _choking_spots[net_id],
+            route_ctx.rr_node_route_inf,
             _is_flat,
             route_ctx.route_bb[net_id]);
 

vpr/src/route/check_route.cpp

@@ -480,7 +480,7 @@ void recompute_occupancy_from_scratch(const Netlist<>& net_list, bool is_flat) {
 
     /* First set the occupancy of everything to zero. */
     for (RRNodeId inode : device_ctx.rr_graph.nodes())
-        route_ctx.rr_node_route_inf[inode].set_occ(0);
+        route_ctx.rr_node_occ_inf[inode].set_occ(0);
 
     /* Now go through each net and count the tracks and pins used everywhere */
 
@@ -493,7 +493,7 @@ void recompute_occupancy_from_scratch(const Netlist<>& net_list, bool is_flat) {
 
         for (auto& rt_node : route_ctx.route_trees[net_id].value().all_nodes()) {
             RRNodeId inode = rt_node.inode;
-            route_ctx.rr_node_route_inf[inode].set_occ(route_ctx.rr_node_route_inf[inode].occ() + 1);
+            route_ctx.rr_node_occ_inf[inode].set_occ(route_ctx.rr_node_occ_inf[inode].occ() + 1);
         }
     }
 
@@ -510,7 +510,7 @@ void recompute_occupancy_from_scratch(const Netlist<>& net_list, bool is_flat) {
                 for (int ipin = 0; ipin < num_local_opins; ipin++) {
                     RRNodeId inode = route_ctx.clb_opins_used_locally[cluster_blk_id][iclass][ipin];
                     VTR_ASSERT(inode && size_t(inode) < device_ctx.rr_graph.num_nodes());
-                    route_ctx.rr_node_route_inf[inode].set_occ(route_ctx.rr_node_route_inf[inode].occ() + 1);
+                    route_ctx.rr_node_occ_inf[inode].set_occ(route_ctx.rr_node_occ_inf[inode].occ() + 1);
                 }
             }
         }

vpr/src/route/connection_router.cpp

@@ -900,10 +900,6 @@ void ConnectionRouter<Heap>::add_route_tree_node_to_heap(
     float backward_path_cost = cost_params.criticality * rt_node.Tdel;
     float R_upstream = rt_node.R_upstream;
 
-    /* Don't push to heap if not in bounding box: no-op for serial router, important for parallel router */
-    if (!inside_bb(rt_node.inode, net_bb))
-        return;
-
     // after budgets are loaded, calculate delay cost as described by RCV paper
     /* R. Fung, V. Betz and W. Chow, "Slack Allocation and Routing to Improve FPGA Timing While
      * Repairing Short-Path Violations," in IEEE Transactions on Computer-Aided Design of
@@ -1021,11 +1017,6 @@ t_bb ConnectionRouter<Heap>::add_high_fanout_route_tree_to_heap(
                         continue;
                 }
 
-                /* In case of the parallel router, we may be dealing with a virtual net
-                 * so prune the nodes from the HF lookup against the bounding box just in case */
-                if (!inside_bb(rr_node_to_add, net_bounding_box))
-                    continue;
-
                 if (!has_path_to_sink(rr_nodes_, rr_graph_, RRNodeId(rt_node.inode), target_node, only_opin_inter_layer)) {
                     continue;
                 }
@@ -1035,14 +1026,18 @@ t_bb ConnectionRouter<Heap>::add_high_fanout_route_tree_to_heap(
                 // Expand HF BB to include the node (clip by original BB)
                 expand_highfanout_bounding_box(highfanout_bb, net_bounding_box, rr_node_to_add, rr_graph_);
 
-                if (is_flat_) {
-                    if (rr_graph_->node_type(rr_node_to_add) == CHANY || rr_graph_->node_type(rr_node_to_add) == CHANX) {
+                /* In case of the parallel router, we may be dealing with a virtual net.
+                 * Still push nodes outside the BB to the heap, but don't count them towards nodes_added */
+                if(inside_bb(rr_node_to_add, net_bounding_box)){
+                    if (is_flat_) {
+                        if (rr_graph_->node_type(rr_node_to_add) == CHANY || rr_graph_->node_type(rr_node_to_add) == CHANX) {
+                            chan_nodes_added++;
+                        }
+                    } else {
                         chan_nodes_added++;
                     }
-                } else {
-                    chan_nodes_added++;
+                    nodes_added++;
                 }
-                nodes_added++;
             }
 
             constexpr int SINGLE_BIN_MIN_NODES = 2;

vpr/src/route/connection_router.h

@@ -56,9 +56,14 @@ class ConnectionRouter : public ConnectionRouterInterface {
         modified_rr_node_inf_.clear();
     }
 
-    // Reset modified data in rr_node_route_inf based on modified_rr_node_inf.
+    /** Reset modified data in rr_node_route_inf based on modified_rr_node_inf_ */
     void reset_path_costs() final {
-        ::reset_path_costs(modified_rr_node_inf_);
+        for (auto node : modified_rr_node_inf_) {
+            rr_node_route_inf_[node].path_cost = std::numeric_limits<float>::infinity();
+            rr_node_route_inf_[node].backward_path_cost = std::numeric_limits<float>::infinity();
+            rr_node_route_inf_[node].prev_node = RRNodeId::INVALID();
+            rr_node_route_inf_[node].prev_edge = RREdgeId::INVALID();
+        }
     }
 
     /** Finds a path from the route tree rooted at rt_root to sink_node.

vpr/src/route/overuse_report.cpp

@@ -53,7 +53,7 @@ void log_overused_nodes_status(int max_logged_overused_rr_nodes) {
     //Print overuse info body
     int overuse_index = 0;
     for (RRNodeId inode : rr_graph.nodes()) {
-        int overuse = route_ctx.rr_node_route_inf[inode].occ() - rr_graph.node_capacity(inode);
+        int overuse = route_ctx.rr_node_occ_inf[inode].occ() - rr_graph.node_capacity(inode);
 
         if (overuse > 0) {
             log_single_overused_node_status(overuse_index, inode);
@@ -103,7 +103,7 @@ void report_overused_nodes(const Netlist<>& net_list,
         /* Report basic rr node info */
         os << "Overused RR node #" << inode << '\n';
         os << "Node id = " << size_t(node_id) << '\n';
-        os << "Occupancy = " << route_ctx.rr_node_route_inf[node_id].occ() << '\n';
+        os << "Occupancy = " << route_ctx.rr_node_occ_inf[node_id].occ() << '\n';
         os << "Capacity = " << rr_graph.node_capacity(node_id) << "\n\n";
 
         /* Report selective info based on the rr node type */
@@ -179,7 +179,7 @@ void generate_overused_nodes_to_congested_net_lookup(const Netlist<>& net_list,
 
         for (auto& rt_node : route_ctx.route_trees[net_id].value().all_nodes()) {
             RRNodeId inode = rt_node.inode;
-            int overuse = route_ctx.rr_node_route_inf[inode].occ() - rr_graph.node_capacity(inode);
+            int overuse = route_ctx.rr_node_occ_inf[inode].occ() - rr_graph.node_capacity(inode);
             if (overuse > 0) {
                 nodes_to_nets_lookup[inode].insert(net_id);
             }
@@ -394,7 +394,7 @@ static void log_single_overused_node_status(int overuse_index, RRNodeId node_id)
     VTR_LOG(" %7d", size_t(node_id));
 
     //Occupancy
-    VTR_LOG(" %10d", route_ctx.rr_node_route_inf[node_id].occ());
+    VTR_LOG(" %10d", route_ctx.rr_node_occ_inf[node_id].occ());
 
     //Capacity
     VTR_LOG(" %9d", rr_graph.node_capacity(node_id));

vpr/src/route/route.cpp

@@ -309,6 +309,8 @@ bool route(const Netlist<>& net_list,
         float iter_cumm_time = iteration_timer.elapsed_sec();
         float iter_elapsed_time = iter_cumm_time - prev_iter_cumm_time;
 
+        PartitionTreeDebug::log("Iteration " + std::to_string(itry) + " took " +  std::to_string(iter_elapsed_time) + " s");
+
         //Output progress
         print_route_status(itry, iter_elapsed_time, pres_fac, num_net_bounding_boxes_updated, iter_results.stats, overuse_info, wirelength_info, timing_info, est_success_iteration);