Merge pull request #1570 from verilog-to-routing/directedMoves_and_RL

Directed moves and Reinforcement Learning agent

Author: vaughnbetz

vpr/src/base/CheckSetup.cpp

@@ -43,6 +43,16 @@ void CheckSetup(const t_packer_opts& PackerOpts,
                         "A block location file requires that placement is enabled.\n");
     }
 
+    if (PlacerOpts.place_static_move_prob.size() != NUM_PL_MOVE_TYPES) {
+        VPR_FATAL_ERROR(VPR_ERROR_OTHER,
+                        "The number of placer move probabilities should equal to the total number of supported moves. %d\n", PlacerOpts.place_static_move_prob.size());
+    }
+
+    if (PlacerOpts.place_static_notiming_move_prob.size() != NUM_PL_NONTIMING_MOVE_TYPES) {
+        VPR_FATAL_ERROR(VPR_ERROR_OTHER,
+                        "The number of placer non timing move probabilities should equal to the total number of supported moves. %d\n", PlacerOpts.place_static_notiming_move_prob.size());
+    }
+
     if (RouterOpts.doRouting) {
         if (!Timing.timing_analysis_enabled
             && (DEMAND_ONLY != RouterOpts.base_cost_type && DEMAND_ONLY_NORMALIZED_LENGTH != RouterOpts.base_cost_type)) {

vpr/src/base/SetupVPR.cpp

@@ -573,6 +573,18 @@ static void SetupPlacerOpts(const t_options& Options, t_placer_opts* PlacerOpts)
     PlacerOpts->effort_scaling = Options.place_effort_scaling;
     PlacerOpts->timing_update_type = Options.timing_update_type;
     PlacerOpts->enable_analytic_placer = Options.enable_analytic_placer;
+    PlacerOpts->place_static_move_prob = Options.place_static_move_prob;
+    PlacerOpts->place_static_notiming_move_prob = Options.place_static_notiming_move_prob;
+    PlacerOpts->place_high_fanout_net = Options.place_high_fanout_net;
+    PlacerOpts->RL_agent_placement = Options.RL_agent_placement;
+    PlacerOpts->place_agent_multistate = Options.place_agent_multistate;
+    PlacerOpts->place_checkpointing = Options.place_checkpointing;
+    PlacerOpts->place_agent_epsilon = Options.place_agent_epsilon;
+    PlacerOpts->place_agent_gamma = Options.place_agent_gamma;
+    PlacerOpts->place_dm_rlim = Options.place_dm_rlim;
+    PlacerOpts->place_reward_fun = Options.place_reward_fun;
+    PlacerOpts->place_crit_limit = Options.place_crit_limit;
+    PlacerOpts->place_agent_algorithm = Options.place_agent_algorithm;
 }
 
 static void SetupAnalysisOpts(const t_options& Options, t_analysis_opts& analysis_opts) {

vpr/src/base/read_options.cpp

@@ -393,6 +393,37 @@ struct ParsePlaceAlgorithm {
     }
 };
 
+struct ParsePlaceAgentAlgorithm {
+    ConvertedValue<e_agent_algorithm> from_str(std::string str) {
+        ConvertedValue<e_agent_algorithm> conv_value;
+        if (str == "e_greedy")
+            conv_value.set_value(E_GREEDY);
+        else if (str == "softmax")
+            conv_value.set_value(SOFTMAX);
+        else {
+            std::stringstream msg;
+            msg << "Invalid conversion from '" << str << "' to e_agent_algorithm (expected one of: " << argparse::join(default_choices(), ", ") << ")";
+            conv_value.set_error(msg.str());
+        }
+        return conv_value;
+    }
+
+    ConvertedValue<std::string> to_str(e_agent_algorithm val) {
+        ConvertedValue<std::string> conv_value;
+        if (val == E_GREEDY)
+            conv_value.set_value("e_greedy");
+        else {
+            VTR_ASSERT(val == SOFTMAX);
+            conv_value.set_value("softmax");
+        }
+        return conv_value;
+    }
+
+    std::vector<std::string> default_choices() {
+        return {"e_greedy", "softmax"};
+    }
+};
+
 struct ParseFixPins {
     ConvertedValue<e_pad_loc_type> from_str(std::string str) {
         ConvertedValue<e_pad_loc_type> conv_value;
@@ -1625,7 +1656,7 @@ argparse::ArgumentParser create_arg_parser(std::string prog_name, t_options& arg
 
     place_grp.add_argument(args.PlaceInnerNum, "--inner_num")
         .help("Controls number of moves per temperature: inner_num * num_blocks ^ (4/3)")
-        .default_value("1.0")
+        .default_value("0.5")
         .show_in(argparse::ShowIn::HELP_ONLY);
 
     place_grp.add_argument<e_place_effort_scaling, ParsePlaceEfforScaling>(args.place_effort_scaling, "--place_effort_scaling")
@@ -1758,6 +1789,103 @@ argparse::ArgumentParser create_arg_parser(std::string prog_name, t_options& arg
         .default_value("false")
         .show_in(argparse::ShowIn::HELP_ONLY);
 
+    place_grp.add_argument(args.place_static_move_prob, "--place_static_move_prob")
+        .help(
+            "The percentage probabilities of different moves in Simulated Annealing placement."
+            "This option is only effective for timing-driven placement."
+            "The numbers listed are interpreted as the percentage probabilities of {uniformMove, MedianMove, CentroidMove, WeightedCentroid, WeightedMedian, Timing feasible Region(TFR), Critical UniformMove}, in that order.")
+        .nargs('+')
+        .default_value({"100", "0", "0", "0", "0", "0", "0"})
+
+        .show_in(argparse::ShowIn::HELP_ONLY);
+
+    place_grp.add_argument(args.place_static_notiming_move_prob, "--place_static_notiming_move_prob")
+        .help(
+            "The Probability of different non timing move in Simulated Annealing."
+            "This option is only effective for nontiming driven placement."
+            " The numbers listed are interpreted as the percentage probabilities of {uniformMove, MedianMove, CentroidMove}, in that order.")
+        .nargs('+')
+        .default_value({"100", "0", "0"})
+
+        .show_in(argparse::ShowIn::HELP_ONLY);
+
+    place_grp.add_argument(args.place_high_fanout_net, "--place_high_fanout_net")
+        .help(
+            "Sets the assumed high fanout net during placement. "
+            "Any net with higher fanout would be ignored while calculating some of the directed moves: Median and WeightedMedian")
+        .default_value("10")
+        .show_in(argparse::ShowIn::HELP_ONLY);
+
+    place_grp.add_argument<bool, ParseOnOff>(args.RL_agent_placement, "--RL_agent_placement")
+        .help(
+            "Uses a Reinforcement Learning (RL) agent in choosing the appropiate move type in placement."
+            "It activates the RL agent placement instead of using fixed probability for each move type.")
+        .default_value("on")
+        .show_in(argparse::ShowIn::HELP_ONLY);
+
+    place_grp.add_argument<bool, ParseOnOff>(args.place_agent_multistate, "--place_agent_multistate")
+        .help(
+            "Enable multistate agent. "
+            "A second state will be activated late in the annealing and in the Quench that includes all the timing driven directed moves.")
+        .default_value("on")
+        .show_in(argparse::ShowIn::HELP_ONLY);
+
+    place_grp.add_argument<bool, ParseOnOff>(args.place_checkpointing, "--place_checkpointing")
+        .help(
+            "Enable Placement checkpoints. This means saving the placement and restore it if it's better than later placements."
+            "Only effective if agnet's 2nd state is activated.")
+        .default_value("on")
+        .show_in(argparse::ShowIn::HELP_ONLY);
+
+    place_grp.add_argument(args.place_agent_epsilon, "--place_agent_epsilon")
+        .help(
+            "Placement RL agent's epsilon for epsilon-greedy agent."
+            "Epsilon represents the percentage of exploration actions taken vs the exploitation ones.")
+        .default_value("0.3")
+        .show_in(argparse::ShowIn::HELP_ONLY);
+
+    place_grp.add_argument(args.place_agent_gamma, "--place_agent_gamma")
+        .help(
+            "Controls how quickly the agent's memory decays. "
+            "Values between [0., 1.] specify the fraction of weight in the exponentially weighted reward average applied to moves which occured greater than moves_per_temp moves ago."
+            "Values < 0 cause the unweighted reward sample average to be used (all samples are weighted equally)")
+        .default_value("0.05")
+        .show_in(argparse::ShowIn::HELP_ONLY);
+
+    place_grp.add_argument(args.place_dm_rlim, "--place_dm_rlim")
+        .help(
+            "The maximum range limit of any directed move other than the uniform move. "
+            "It also shrinks with the default rlim")
+        .default_value("3.0")
+        .show_in(argparse::ShowIn::HELP_ONLY);
+
+    place_grp.add_argument(args.place_reward_fun, "--place_reward_fun")
+        .help(
+            "The reward function used by placement RL agent."
+            "The available values are: basic, nonPenalizing_basic, runtime_aware, WLbiased_runtime_aware")
+        .default_value("WLbiased_runtime_aware")
+        .show_in(argparse::ShowIn::HELP_ONLY);
+
+    place_grp.add_argument(args.place_crit_limit, "--place_crit_limit")
+        .help(
+            "The criticality limit to count a block as a critical one (or have a critical connection). "
+            "It used in some directed moves that only move critical blocks like critical uniform and feasible region. "
+            "Its range equals to [0., 1.].")
+        .default_value("0.7")
+        .show_in(argparse::ShowIn::HELP_ONLY);
+    /*
+     * place_grp.add_argument(args.place_timing_cost_func, "--place_timing_cost_func")
+     * .help(
+     * "which timing cost function to use")
+     * .default_value("0")
+     * .show_in(argparse::ShowIn::HELP_ONLY);
+     */
+    place_grp.add_argument<e_agent_algorithm, ParsePlaceAgentAlgorithm>(args.place_agent_algorithm, "--place_agent_algorithm")
+        .help("Controls which placement RL agent is used")
+        .default_value("softmax")
+        .choices({"e_greedy", "softmax"})
+        .show_in(argparse::ShowIn::HELP_ONLY);
+
     auto& place_timing_grp = parser.add_argument_group("timing-driven placement options");
 
     place_timing_grp.add_argument(args.PlaceTimingTradeoff, "--timing_tradeoff")

vpr/src/base/read_options.h

@@ -114,6 +114,20 @@ struct t_options {
     argparse::ArgValue<e_place_effort_scaling> place_effort_scaling;
     argparse::ArgValue<e_place_delta_delay_algorithm> place_delta_delay_matrix_calculation_method;
     argparse::ArgValue<bool> enable_analytic_placer;
+    argparse::ArgValue<std::vector<float>> place_static_move_prob;
+    argparse::ArgValue<std::vector<float>> place_static_notiming_move_prob;
+    argparse::ArgValue<int> place_high_fanout_net;
+
+    argparse::ArgValue<bool> RL_agent_placement;
+    argparse::ArgValue<bool> place_agent_multistate;
+    argparse::ArgValue<bool> place_checkpointing;
+    argparse::ArgValue<float> place_agent_epsilon;
+    argparse::ArgValue<float> place_agent_gamma;
+    argparse::ArgValue<float> place_dm_rlim;
+    argparse::ArgValue<e_agent_algorithm> place_agent_algorithm;
+    argparse::ArgValue<std::string> place_reward_fun;
+    //argparse::ArgValue<int> place_timing_cost_func;
+    argparse::ArgValue<float> place_crit_limit;
 
     /* Timing-driven placement options only */
     argparse::ArgValue<float> PlaceTimingTradeoff;

vpr/src/base/vpr_api.cpp

@@ -590,7 +590,7 @@ bool vpr_pack(t_vpr_setup& vpr_setup, const t_arch& arch) {
 }
 
 void vpr_load_packing(t_vpr_setup& vpr_setup, const t_arch& arch) {
-    vtr::ScopedStartFinishTimer timer("Load Packing");
+    vtr::ScopedStartFinishTimer timer("Load packing");
 
     VTR_ASSERT_MSG(!vpr_setup.FileNameOpts.NetFile.empty(),
                    "Must have valid .net filename to load packing");

vpr/src/base/vpr_types.h

@@ -490,6 +490,11 @@ enum class e_timing_update_type {
  * Placement and routing data types
  ****************************************************************************/
 
+/* Values of number of placement available move types */
+#define NUM_PL_MOVE_TYPES 7
+#define NUM_PL_NONTIMING_MOVE_TYPES 3
+#define NUM_PL_1ST_STATE_MOVE_TYPES 4
+
 /* Timing data structures end */
 enum sched_type {
     AUTO_SCHED,
@@ -920,6 +925,18 @@ enum e_pad_loc_type {
     RANDOM
 };
 
+/**
+ * @brief Used to determine the RL agent's algorithm
+ *
+ * This algorithm controls the exploration-exploitation and how we select the new action
+ * Currently, the supported algorithms are: epsilon greedy and softmax
+ * For more details, check simpleRL_move_generator.cpp
+ */
+enum e_agent_algorithm {
+    E_GREEDY,
+    SOFTMAX
+};
+
 ///@brief Used to calculate the inner placer loop's block swapping limit move_lim.
 enum e_place_effort_scaling {
     CIRCUIT,       ///<Effort scales based on circuit size only
@@ -1033,6 +1050,19 @@ struct t_placer_opts {
 
     std::string write_placement_delay_lookup;
     std::string read_placement_delay_lookup;
+    std::vector<float> place_static_move_prob;
+    std::vector<float> place_static_notiming_move_prob;
+    bool RL_agent_placement;
+    bool place_agent_multistate;
+    bool place_checkpointing;
+    int place_high_fanout_net;
+    e_agent_algorithm place_agent_algorithm;
+    float place_agent_epsilon;
+    float place_agent_gamma;
+    float place_dm_rlim;
+    //int place_timing_cost_func;
+    std::string place_reward_fun;
+    float place_crit_limit;
 
     /**
      * @brief Tile types that should be used during delay sampling.

vpr/src/draw/draw.cpp

@@ -50,6 +50,10 @@
 
 #include "move_utils.h"
 
+#ifdef VTR_ENABLE_DEBUG_LOGGING
+#    include "move_utils.h"
+#endif
+
 #ifdef WIN32 /* For runtime tracking in WIN32. The clock() function defined in time.h will *
               * track CPU runtime.														   */
 #    include <time.h>