Add include_temp to vtr task syntax

When I pass rr graph and router lookahead files to VPR, it throws an error. capnproto uses mmap to open these files. It seems that multiple processes can access a single file using mmap. However, I cannot trust capnproto. The changes in this commit enhace the vtr task syntax by allowing copying arbitrary files to the temporay directory. This way, I can copy rr graph file and prevent multiple processes accessing the same file.

Author: soheilshahrouz

vtr_flow/scripts/python_libs/vtr/flow.py

@@ -1,6 +1,7 @@
 """
     Module to run the VTR flow. This module calls other modules that then access the tools like VPR.
 """
+import os
 import shutil
 from pathlib import Path
 from collections import OrderedDict
@@ -36,6 +37,7 @@ def run(
     circuit_file,
     power_tech_file=None,
     include_files=None,
+    include_temp_files=None,
     start_stage=VtrStage.PARMYS,
     end_stage=VtrStage.VPR,
     command_runner=vtr.CommandRunner(),
@@ -176,7 +178,7 @@ def run(
     shutil.copy(str(circuit_file), str(circuit_copy))
     shutil.copy(str(architecture_file), str(architecture_copy))
 
-    # Check whether any inclulde is specified
+    # Check whether any include is specified
     if include_files:
         # Verify include files are Paths or convert them to Path + check that they exist
         # Copy include files to the run directory
@@ -185,6 +187,17 @@ def run(
             include_copy = temp_dir / include_file.name
             shutil.copy(str(include), str(include_copy))
 
+
+    # Check whether any include is specified
+    if include_temp_files:
+        # Verify include files are Paths or convert them to Path + check that they exist
+        # Copy temp include files to the run directory
+        for include_temp in include_temp_files:
+            include_temp_file = vtr.util.verify_file(include_temp, "Temporary Include")
+            include_temp_copy = temp_dir / include_temp_file.name
+            shutil.copy(str(include_temp), str(include_temp_copy))
+
+
     # There are multiple potential paths for the netlist to reach a tool
     # We initialize it here to the user specified circuit and let downstream
     # stages update it
@@ -384,6 +397,15 @@ def run(
             power_tech_file,
         )
 
+    # Check whether any temporary include is specified
+    if include_temp_files:
+        # Verify temp include files are Paths or convert them to Path + check that they exist
+        # Then find
+        for include_temp in include_temp_files:
+            include_temp_file = vtr.util.verify_file(include_temp, "Temporary Include")
+            include_temp_copy = temp_dir / include_temp_file.name
+            os.remove(str(include_temp_copy))
+
 
 # pylint: enable=too-many-arguments, too-many-locals, too-many-branches, too-many-statements
 

vtr_flow/scripts/python_libs/vtr/task.py

@@ -39,6 +39,8 @@ def __init__(
         parse_file,
         includes_dir=None,
         include_list_add=None,
+        include_temp_dir=None,
+        include_temp_list_add=None,
         second_parse_file=None,
         script_path=None,
         script_params=None,
@@ -64,6 +66,8 @@ def __init__(
         self.archs = arch_list_add
         self.include_dir = includes_dir
         self.includes = include_list_add
+        self.include_temp_dir = include_temp_dir
+        self.include_temps = include_temp_list_add
         self.parse_file = parse_file
         self.second_parse_file = second_parse_file
         self.script_path = script_path
@@ -97,6 +101,7 @@ def __init__(
         arch,
         circuit,
         include,
+        include_temp,
         script_params,
         work_dir,
         run_command,
@@ -108,6 +113,7 @@ def __init__(
         self._arch = arch
         self._circuit = circuit
         self._include = include
+        self._include_temp = include_temp
         self._script_params = script_params
         self._run_command = run_command
         self._parse_command = parse_command
@@ -139,6 +145,12 @@ def include(self):
         """
         return self._include
 
+    def include_temp(self):
+        """
+        return the list of temporary include files of the job.
+        """
+        return self._include_temp
+
     def script_params(self):
         """
         return the script parameter of the job
@@ -197,6 +209,7 @@ def load_task_config(config_file) -> TaskConfig:
         [
             "circuits_dir",
             "includes_dir",
+            "include_temp_dir",
             "archs_dir",
             "additional_files",
             "parse_file",
@@ -264,6 +277,7 @@ def load_task_config(config_file) -> TaskConfig:
 
     check_required_fields(config_file, required_keys, key_values)
     check_include_fields(config_file, key_values)
+    check_include_temp_fields(config_file, key_values)
 
     # Useful meta-data about the config
     config_dir = str(Path(config_file).parent)
@@ -300,6 +314,18 @@ def check_include_fields(config_file, key_values):
                 )
             )
 
+def check_include_temp_fields(config_file, key_values):
+    """
+    Check that include_temp_dir was specified if some files to temporarily include
+    in the designs (include_temp_list_add) was specified.
+    """
+    if "include_temp_list_add" in key_values:
+        if "include_temp_dir" not in key_values:
+            raise VtrError(
+                "Missing required key '{key}' in config file {file}".format(
+                    key="include_temp_dir", file=config_file
+                )
+            )
 
 def shorten_task_names(configs, common_task_prefix):
     """
@@ -377,6 +403,17 @@ def create_cmd(
 
         cmd += includes
 
+    # Resolve and collect all include_temp paths in the config file
+    # as -include_temp ["include_temp1", "include_temp2", ..]
+    include_temps = []
+    if config.include_temps:
+        cmd += ["-include_temp"]
+        for include_temp in config.include_temps:
+            abs_include_filepath = resolve_vtr_source_file(config, include_temp, config.include_temp_dir)
+            include_temps.append(abs_include_filepath)
+
+        cmd += include_temps
+
     # Check if additional architectural data files are present
     if config.additional_files_list_add:
         for additional_file in config.additional_files_list_add:
@@ -466,7 +503,7 @@ def create_cmd(
             resolve_vtr_source_file(config, noc_traffic, config.noc_traffic_dir),
         ]
 
-    return includes, parse_cmd, second_parse_cmd, qor_parse_command, cmd
+    return includes, include_temps, parse_cmd, second_parse_cmd, qor_parse_command, cmd
 
 
 # pylint: disable=too-many-branches
@@ -509,7 +546,7 @@ def create_jobs(args, configs, after_run=False) -> List[Job]:
                 )
             )
 
-            includes, parse_cmd, second_parse_cmd, qor_parse_command, cmd = create_cmd(
+            includes, include_temps, parse_cmd, second_parse_cmd, qor_parse_command, cmd = create_cmd(
                 abs_circuit_filepath, abs_arch_filepath, config, args, circuit, noc_traffic
             )
 
@@ -521,6 +558,7 @@ def create_jobs(args, configs, after_run=False) -> List[Job]:
                             config,
                             circuit,
                             includes,
+                            include_temps,
                             arch,
                             noc_traffic,
                             value,
@@ -540,6 +578,7 @@ def create_jobs(args, configs, after_run=False) -> List[Job]:
                         config,
                         circuit,
                         includes,
+                        include_temps,
                         arch,
                         noc_traffic,
                         None,
@@ -561,6 +600,7 @@ def create_job(
     config,
     circuit,
     include,
+    include_temp,
     arch,
     noc_flow,
     param,
@@ -658,6 +698,7 @@ def create_job(
         arch,
         circuit,
         include,
+        include_temp,
         param_string,
         work_dir + "/" + param_string,
         current_cmd,

vtr_flow/scripts/run_vtr_flow.py

@@ -155,6 +155,14 @@ def vtr_command_argparser(prog=None):
         help="List of include files to a benchmark circuit (pass to VTR"
         + " frontends as a benchmark design set)",
     )
+    parser.add_argument(
+        "-include_temp",
+        nargs="*",
+        default=None,
+        dest="include_temp_list_file",
+        help="List of include files to be copied to the working directory"
+             + " and be removed when the task is done)",
+    )
 
     #
     # Power arguments
@@ -561,6 +569,7 @@ def vtr_command_main(arg_list, prog=None):
             Path(args.circuit_file),
             power_tech_file=args.power_tech,
             include_files=args.include_list_file,
+            include_temp_files=args.include_temp_list_file,
             temp_dir=temp_dir,
             start_stage=args.start,
             end_stage=args.end,