Merge pull request #2345 from verilog-to-routing/restore_checkpoint_noc_costs

Recompte noc costs when restorting a checkpoint

Author: vaughnbetz

vpr/src/noc/noc_storage.cpp

@@ -23,6 +23,10 @@ const vtr::vector<NocLinkId, NocLink>& NocStorage::get_noc_links(void) const {
     return link_storage;
 }
 
+vtr::vector<NocLinkId, NocLink>& NocStorage::get_mutable_noc_links(void) {
+    return link_storage;
+}
+
 int NocStorage::get_number_of_noc_links(void) const {
     return link_storage.size();
 }
@@ -219,9 +223,9 @@ NocLinkId NocStorage::get_parallel_link(NocLinkId current_link) const {
     NocLinkId parallel_link = INVALID_LINK_ID;
 
     // go through the links of the sink router and the link that has the current source router as the sink router of the link is the parallel link we are looking for
-    for (auto link = sink_router_links->begin(); link != sink_router_links->end(); link++) {
-        if (link_storage[*link].get_sink_router() == curr_source_router) {
-            parallel_link = *link;
+    for (auto sink_router_link : *sink_router_links) {
+        if (link_storage[sink_router_link].get_sink_router() == curr_source_router) {
+            parallel_link = sink_router_link;
             break;
         }
     }
@@ -260,17 +264,17 @@ void NocStorage::echo_noc(char* file_name) const {
     fprintf(fp, "\n");
 
     // go through each router and print its information
-    for (auto router = router_storage.begin(); router != router_storage.end(); router++) {
-        fprintf(fp, "Router %d:\n", router->get_router_user_id());
+    for (const auto& router : router_storage) {
+        fprintf(fp, "Router %d:\n", router.get_router_user_id());
         // if the router tile is larger than a single grid, the position represents the bottom left corner of the tile
-        fprintf(fp, "Equivalent Physical Tile Grid Position -> (%d,%d)\n", router->get_router_grid_position_x(), router->get_router_grid_position_y());
+        fprintf(fp, "Equivalent Physical Tile Grid Position -> (%d,%d)\n", router.get_router_grid_position_x(), router.get_router_grid_position_y());
         fprintf(fp, "Router Connections ->");
 
-        auto& router_connections = this->get_noc_router_connections(this->convert_router_id(router->get_router_user_id()));
+        auto& router_connections = this->get_noc_router_connections(this->convert_router_id(router.get_router_user_id()));
 
         // go through the outgoing links of the current router and print the connecting router
-        for (auto link_id = router_connections.begin(); link_id != router_connections.end(); link_id++) {
-            const NocRouterId connecting_router_id = get_single_noc_link(*link_id).get_sink_router();
+        for (auto router_connection : router_connections) {
+            const NocRouterId connecting_router_id = get_single_noc_link(router_connection).get_sink_router();
 
             fprintf(fp, " %d", get_single_noc_router(connecting_router_id).get_router_user_id());
         }

vpr/src/noc/noc_storage.h

@@ -187,6 +187,15 @@ class NocStorage {
      */
     const vtr::vector<NocLinkId, NocLink>& get_noc_links(void) const;
 
+    /**
+     * @brief Get all the links in the NoC. The links themselves can
+     * be modified. This function should be used when information on
+     * every link needs to be modified.
+     *
+     * @return A vector of links.
+     */
+    vtr::vector<NocLinkId, NocLink>& get_mutable_noc_links(void);
+
     /**
      * @return An integer representing the total number of links within the
      * NoC.

vpr/src/noc/noc_traffic_flows.cpp

@@ -54,7 +54,7 @@ const std::vector<NocTrafficFlowId>& NocTrafficFlows::get_all_traffic_flow_id(vo
 
 // setters for the traffic flows
 
-void NocTrafficFlows::create_noc_traffic_flow(std::string source_router_module_name, std::string sink_router_module_name, ClusterBlockId source_router_cluster_id, ClusterBlockId sink_router_cluster_id, double traffic_flow_bandwidth, double traffic_flow_latency, int traffic_flow_priority) {
+void NocTrafficFlows::create_noc_traffic_flow(const std::string& source_router_module_name, const std::string& sink_router_module_name, ClusterBlockId source_router_cluster_id, ClusterBlockId sink_router_cluster_id, double traffic_flow_bandwidth, double traffic_flow_latency, int traffic_flow_priority) {
     VTR_ASSERT_MSG(!built_traffic_flows, "NoC traffic flows have already been added, cannot modify further.");
 
     // create and add the new traffic flow to the vector
@@ -106,7 +106,7 @@ void NocTrafficFlows::clear_traffic_flows(void) {
     return;
 }
 
-bool NocTrafficFlows::check_if_cluster_block_has_traffic_flows(ClusterBlockId block_id) {
+bool NocTrafficFlows::check_if_cluster_block_has_traffic_flows(ClusterBlockId block_id) const {
     auto traffic_flows = get_traffic_flows_associated_to_router_block(block_id);
 
     // indicate whether a vector of traffic flows were found that are associated to the current cluster block
@@ -149,15 +149,15 @@ void NocTrafficFlows::echo_noc_traffic_flows(char* file_name) {
     int traffic_flow_id = 0;
 
     // go through each traffic flow and print its information
-    for (auto traffic_flow = noc_traffic_flows.begin(); traffic_flow != noc_traffic_flows.end(); traffic_flow++) {
+    for (const auto& traffic_flow : noc_traffic_flows) {
         // print the current traffic flows data
         fprintf(fp, "Traffic flow ID: %d\n", traffic_flow_id);
-        fprintf(fp, "Traffic flow source router block name: %s\n", traffic_flow->source_router_module_name.c_str());
-        fprintf(fp, "Traffic flow source router block cluster ID: %lu\n", (size_t)traffic_flow->source_router_cluster_id);
-        fprintf(fp, "Traffic flow sink router block name: %s\n", traffic_flow->sink_router_module_name.c_str());
-        fprintf(fp, "Traffic flow sink router block cluster ID: %lu\n", (size_t)traffic_flow->sink_router_cluster_id);
-        fprintf(fp, "Traffic flow bandwidth: %f bps\n", traffic_flow->traffic_flow_bandwidth);
-        fprintf(fp, "Traffic flow latency: %f seconds\n", traffic_flow->max_traffic_flow_latency);
+        fprintf(fp, "Traffic flow source router block name: %s\n", traffic_flow.source_router_module_name.c_str());
+        fprintf(fp, "Traffic flow source router block cluster ID: %lu\n", (size_t)traffic_flow.source_router_cluster_id);
+        fprintf(fp, "Traffic flow sink router block name: %s\n", traffic_flow.sink_router_module_name.c_str());
+        fprintf(fp, "Traffic flow sink router block cluster ID: %lu\n", (size_t)traffic_flow.sink_router_cluster_id);
+        fprintf(fp, "Traffic flow bandwidth: %f bps\n", traffic_flow.traffic_flow_bandwidth);
+        fprintf(fp, "Traffic flow latency: %f seconds\n", traffic_flow.max_traffic_flow_latency);
 
         // separate the next link information
         fprintf(fp, "\n");
@@ -174,16 +174,16 @@ void NocTrafficFlows::echo_noc_traffic_flows(char* file_name) {
 
     // go through each router block cluster and print its associated traffic flows
     // we only print out the traffic flow ids
-    for (auto it = traffic_flows_associated_to_router_blocks.begin(); it != traffic_flows_associated_to_router_blocks.end(); it++) {
+    for (const auto& router_traffic_flows : traffic_flows_associated_to_router_blocks) {
         // print the current router cluster id
-        fprintf(fp, "Cluster ID %lu: ", (size_t)it->first);
+        fprintf(fp, "Cluster ID %lu: ", (size_t)router_traffic_flows.first);
 
         // get the vector of associated traffic flows
-        auto assoc_traffic_flows = &(it->second);
+        auto assoc_traffic_flows = &(router_traffic_flows.second);
 
         // now go through the associated traffic flows of the current router and print their ids
-        for (auto traffic_flow = assoc_traffic_flows->begin(); traffic_flow != assoc_traffic_flows->end(); traffic_flow++) {
-            fprintf(fp, "%lu ", (size_t)*traffic_flow);
+        for (auto assoc_traffic_flow : *assoc_traffic_flows) {
+            fprintf(fp, "%lu ", (size_t)assoc_traffic_flow);
         }
 
         // separate to the next cluster associated traffic flows information

vpr/src/noc/noc_traffic_flows.h

@@ -31,6 +31,7 @@
 #include <iostream>
 #include <unordered_map>
 #include <unordered_set>
+#include <utility>
 #include <vector>
 #include "clustered_netlist_fwd.h"
 #include "noc_data_types.h"
@@ -67,8 +68,8 @@ struct t_noc_traffic_flow {
 
     /** Constructor initializes all variables*/
     t_noc_traffic_flow(std::string source_router_name, std::string sink_router_name, ClusterBlockId source_router_id, ClusterBlockId sink_router_id, double flow_bandwidth, double max_flow_latency, int flow_priority)
-        : source_router_module_name(source_router_name)
-        , sink_router_module_name(sink_router_name)
+        : source_router_module_name(std::move(source_router_name))
+        , sink_router_module_name(std::move(sink_router_name))
         , source_router_cluster_id(source_router_id)
         , sink_router_cluster_id(sink_router_id)
         , traffic_flow_bandwidth(flow_bandwidth)
@@ -254,7 +255,7 @@ class NocTrafficFlows {
      * at the sink router.
      * @param traffic_flow_priority The importance of a given traffic flow.
      */
-    void create_noc_traffic_flow(std::string source_router_module_name, std::string sink_router_module_name, ClusterBlockId source_router_cluster_id, ClusterBlockId sink_router_cluster_id, double traffic_flow_bandwidth, double traffic_flow_latency, int traffic_flow_priority);
+    void create_noc_traffic_flow(const std::string& source_router_module_name, const std::string& sink_router_module_name, ClusterBlockId source_router_cluster_id, ClusterBlockId sink_router_cluster_id, double traffic_flow_bandwidth, double traffic_flow_latency, int traffic_flow_priority);
 
     /**
      * @brief Copies the passed in router_cluster_id_in_netlist vector to the
@@ -304,7 +305,7 @@ class NocTrafficFlows {
      * @return true The block has traffic flows that it is a part of
      * @return false The block has no traffic flows it is a prt of
      */
-    bool check_if_cluster_block_has_traffic_flows(ClusterBlockId block_id);
+    bool check_if_cluster_block_has_traffic_flows(ClusterBlockId block_id) const;
 
     /**
      * @brief Writes out the NocTrafficFlows class information to a file.

vpr/src/noc/xy_routing.cpp

@@ -106,9 +106,9 @@ bool XYRouting::move_to_next_router(NocRouterId& curr_router_id, int curr_router
     const std::vector<NocLinkId>& router_connections = noc_model.get_noc_router_connections(curr_router_id);
 
     // go through each outgoing link and determine whether the link leads towards the intended route direction
-    for (auto connecting_link = router_connections.begin(); connecting_link != router_connections.end(); connecting_link++) {
+    for (auto connecting_link : router_connections) {
         // get the current outgoing link which is being processed
-        const NocLink& curr_outgoing_link = noc_model.get_single_noc_link(*connecting_link);
+        const NocLink& curr_outgoing_link = noc_model.get_single_noc_link(connecting_link);
 
         // get the next router that we will visit if we travel across the current link
         next_router_id = curr_outgoing_link.get_sink_router();
@@ -154,7 +154,7 @@ bool XYRouting::move_to_next_router(NocRouterId& curr_router_id, int curr_router
         // If the next router was already visited, then this link is not valid, so indicate this and move onto processing the next link.
         if (found_next_router && !visited_next_router) {
             // if we are here then the link is legal to traverse, so add it to the found route and traverse the link by moving to the router connected by this link
-            flow_route.push_back(*connecting_link);
+            flow_route.push_back(connecting_link);
             curr_router_id = next_router_id;
 
             // we found a suitable router to visit next, so add it to the set of visited routers

vpr/src/place/initial_placement.cpp

@@ -1105,7 +1105,7 @@ void initial_placement(enum e_pad_loc_type pad_loc_type, const char* constraints
 
     // route all the traffic flows in the NoC now that all the router cluster block have been placed  (this is done only if the noc optimization is enabled by the user)
     if (noc_enabled) {
-        initial_noc_placement();
+        initial_noc_routing();
     }
 
     //#ifdef VERBOSE

vpr/src/place/noc_place_utils.cpp

@@ -9,7 +9,7 @@ static vtr::vector<NocTrafficFlowId, TrafficFlowPlaceCost> traffic_flow_costs, p
 static std::vector<NocTrafficFlowId> affected_traffic_flows;
 /*********************************************************** *****************************/
 
-void initial_noc_placement(void) {
+void initial_noc_routing(void) {
     // need to get placement information about where the router cluster blocks are placed on the device
     const auto& place_ctx = g_vpr_ctx.placement();
 
@@ -37,6 +37,22 @@ void initial_noc_placement(void) {
     return;
 }
 
+void reinitialize_noc_routing(const t_noc_opts& noc_opts, t_placer_costs& costs) {
+    auto& noc_ctx = g_vpr_ctx.mutable_noc();
+
+    // Zero out bandwidth usage for all links
+    for (auto& noc_link : noc_ctx.noc_model.get_mutable_noc_links()) {
+        noc_link.set_bandwidth_usage(0.0);
+    }
+
+    // Route traffic flows and update link bandwidth usage
+    initial_noc_routing();
+
+    // Initialize traffic_flow_costs
+    costs.noc_aggregate_bandwidth_cost = comp_noc_aggregate_bandwidth_cost();
+    costs.noc_latency_cost = comp_noc_latency_cost(noc_opts);
+}
+
 void find_affected_noc_routers_and_update_noc_costs(const t_pl_blocks_to_be_moved& blocks_affected, double& noc_aggregate_bandwidth_delta_c, double& noc_latency_delta_c, const t_noc_opts& noc_opts) {
     // provides the positions where the affected blocks have moved to
     auto& place_ctx = g_vpr_ctx.placement();

vpr/src/place/noc_place_utils.h

@@ -49,7 +49,23 @@ struct TrafficFlowPlaceCost {
  * routed. This is why this function should only be used once.
  * 
  */
-void initial_noc_placement(void);
+void initial_noc_routing(void);
+
+/**
+ * @brief Zeros out all link bandwidth usage an re-routes traffic flows.
+ * Initializes static variables in noc_place_utils.cpp that are used to
+ * keep track of NoC-related costs.
+ *
+ * This function should be called when a placement checkpoint is restored.
+ * If the router placement in the checkpoint is different from the last
+ * router placement before the checkpoint is restored, link bandwidth usage,
+ * traffic flow routes, and static variable in noc_place_utils.cpp are no
+ * longer valid and need to be re-initialized.
+ *
+ * @param noc_opts NoC-related options used to calculated NoC costs
+ * @param costs Used to get aggregate bandwidth and latency costs.
+ */
+void reinitialize_noc_routing(const t_noc_opts& noc_opts, t_placer_costs& costs);
 
 /**
  * @brief Goes through all the cluster blocks that were moved

vpr/src/place/place.cpp

@@ -953,7 +953,7 @@ void try_place(const Netlist<>& net_list,
     if (placer_opts.place_checkpointing)
         restore_best_placement(placement_checkpoint, timing_info, costs,
                                placer_criticalities, placer_setup_slacks, place_delay_model,
-                               pin_timing_invalidator, crit_params);
+                               pin_timing_invalidator, crit_params, noc_opts);
 
     if (placer_opts.placement_saves_per_temperature >= 1) {
         std::string filename = vtr::string_fmt("placement_%03d_%03d.place",

vpr/src/place/place_checkpoint.cpp

@@ -1,4 +1,5 @@
 #include "place_checkpoint.h"
+#include "noc_place_utils.h"
 
 float t_placement_checkpoint::get_cp_cpd() { return cpd; }
 double t_placement_checkpoint::get_cp_bb_cost() { return costs.bb_cost; }
@@ -26,7 +27,7 @@ void save_placement_checkpoint_if_needed(t_placement_checkpoint& placement_check
     }
 }
 
-void restore_best_placement(t_placement_checkpoint& placement_checkpoint, std::shared_ptr<SetupTimingInfo>& timing_info, t_placer_costs& costs, std::unique_ptr<PlacerCriticalities>& placer_criticalities, std::unique_ptr<PlacerSetupSlacks>& placer_setup_slacks, std::unique_ptr<PlaceDelayModel>& place_delay_model, std::unique_ptr<NetPinTimingInvalidator>& pin_timing_invalidator, PlaceCritParams crit_params) {
+void restore_best_placement(t_placement_checkpoint& placement_checkpoint, std::shared_ptr<SetupTimingInfo>& timing_info, t_placer_costs& costs, std::unique_ptr<PlacerCriticalities>& placer_criticalities, std::unique_ptr<PlacerSetupSlacks>& placer_setup_slacks, std::unique_ptr<PlaceDelayModel>& place_delay_model, std::unique_ptr<NetPinTimingInvalidator>& pin_timing_invalidator, PlaceCritParams crit_params, const t_noc_opts& noc_opts) {
     if (placement_checkpoint.cp_is_valid() && timing_info->least_slack_critical_path().delay() > placement_checkpoint.get_cp_cpd() && costs.bb_cost < 1.05 * placement_checkpoint.get_cp_bb_cost()) {
         //restore the latest placement checkpoint
         costs = placement_checkpoint.restore_placement();
@@ -43,6 +44,15 @@ void restore_best_placement(t_placement_checkpoint& placement_checkpoint, std::s
                                    timing_info.get(),
                                    &costs);
 
+        /* If NoC is enabled, re-compute NoC costs and re-initialize NoC internal data structures.
+         * If some routers have different locations than the last placement, NoC-related costs and
+         * internal data structures that are used to keep track of each flow's cost are no longer valid,
+         * and need to be re-computed from scratch.
+         */
+        if (noc_opts.noc) {
+            reinitialize_noc_routing(noc_opts, costs);
+        }
+
         VTR_LOG("\nCheckpoint restored\n");
     }
 }

vpr/src/place/place_checkpoint.h

@@ -50,5 +50,5 @@ class t_placement_checkpoint {
 void save_placement_checkpoint_if_needed(t_placement_checkpoint& placement_checkpoint, std::shared_ptr<SetupTimingInfo> timing_info, t_placer_costs& costs, float cpd);
 
 //restore the checkpoint if it's better than the latest placement solution
-void restore_best_placement(t_placement_checkpoint& placement_checkpoint, std::shared_ptr<SetupTimingInfo>& timing_info, t_placer_costs& costs, std::unique_ptr<PlacerCriticalities>& placer_criticalities, std::unique_ptr<PlacerSetupSlacks>& placer_setup_slacks, std::unique_ptr<PlaceDelayModel>& place_delay_model, std::unique_ptr<NetPinTimingInvalidator>& pin_timing_invalidator, PlaceCritParams crit_params);
+void restore_best_placement(t_placement_checkpoint& placement_checkpoint, std::shared_ptr<SetupTimingInfo>& timing_info, t_placer_costs& costs, std::unique_ptr<PlacerCriticalities>& placer_criticalities, std::unique_ptr<PlacerSetupSlacks>& placer_setup_slacks, std::unique_ptr<PlaceDelayModel>& place_delay_model, std::unique_ptr<NetPinTimingInvalidator>& pin_timing_invalidator, PlaceCritParams crit_params, const t_noc_opts& noc_opts);
 #endif

vpr/test/test_noc_place_utils.cpp

@@ -179,7 +179,7 @@ TEST_CASE("test_initial_noc_placement", "[noc_place_utils]") {
     }
 
     // now call the test function
-    initial_noc_placement();
+    initial_noc_routing();
 
     // now verify the function by comparing the link bandwidths in the noc model (should have been updated by the test function) to the golden set
     int number_of_links = golden_link_bandwidths.size();
@@ -357,7 +357,7 @@ TEST_CASE("test_initial_comp_cost_functions", "[noc_place_utils]") {
 
     // assume this works
     // this is needed to set up the global noc packet router and also global datastructures
-    initial_noc_placement();
+    initial_noc_routing();
 
     SECTION("test_comp_noc_aggregate_bandwidth_cost") {
         //initialize all the cost calculator datastructures
@@ -613,7 +613,7 @@ TEST_CASE("test_find_affected_noc_routers_and_update_noc_costs, test_commit_noc_
 
     // assume this works
     // this is needed to set up the global noc packet router and also global datastructures
-    initial_noc_placement();
+    initial_noc_routing();
 
     // datastructure below will store the bandwidth usages of all the links
     // and will be updated throughout this test.
@@ -1344,7 +1344,7 @@ TEST_CASE("test_revert_noc_traffic_flow_routes", "[noc_place_utils]") {
 
     // assume this works
     // this is needed to set up the global noc packet router and also global datastructures
-    initial_noc_placement();
+    initial_noc_routing();
 
     // datastructure below will store the bandwidth usages of all the links
     // and will be updated throughout this test.