From 0e44b3982e2fc5a6e826fa24b253fd99dd13bfb4 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Mon, 4 May 2020 14:08:29 -0700 Subject: [PATCH 01/86] copied over old python scripts, fixed a spelling error with is_exacutable and reordered the arguments on vtr-flow.py to match master --- .../python_libs/verilogtorouting/__init__.py | 0 .../python_libs/verilogtorouting/error.py | 32 + .../python_libs/verilogtorouting/flow.py | 439 ++++++++++++ .../python_libs/verilogtorouting/inspect.py | 306 ++++++++ .../python_libs/verilogtorouting/task.py | 154 +++++ .../python_libs/verilogtorouting/util.py | 492 +++++++++++++ vtr_flow/scripts/vtr-flow.py | 292 ++++++++ vtr_flow/scripts/vtr-task.py | 654 ++++++++++++++++++ vtr_flow/scripts/vtr-test.py | 213 ++++++ 9 files changed, 2582 insertions(+) create mode 100644 vtr_flow/scripts/python_libs/verilogtorouting/__init__.py create mode 100644 vtr_flow/scripts/python_libs/verilogtorouting/error.py create mode 100644 vtr_flow/scripts/python_libs/verilogtorouting/flow.py create mode 100644 vtr_flow/scripts/python_libs/verilogtorouting/inspect.py create mode 100644 vtr_flow/scripts/python_libs/verilogtorouting/task.py create mode 100644 vtr_flow/scripts/python_libs/verilogtorouting/util.py create mode 100755 vtr_flow/scripts/vtr-flow.py create mode 100755 vtr_flow/scripts/vtr-task.py create mode 100755 vtr_flow/scripts/vtr-test.py diff --git a/vtr_flow/scripts/python_libs/verilogtorouting/__init__.py b/vtr_flow/scripts/python_libs/verilogtorouting/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/vtr_flow/scripts/python_libs/verilogtorouting/error.py b/vtr_flow/scripts/python_libs/verilogtorouting/error.py new file mode 100644 index 00000000000..5c37cc3366b --- /dev/null +++ b/vtr_flow/scripts/python_libs/verilogtorouting/error.py @@ -0,0 +1,32 @@ +class VtrError(Exception): + """ + Base class for VTR related exceptions + + Attributes: + msg -- An explanation of the error + """ + def __init__(self, msg=""): + self.msg = msg + +class CommandError(VtrError): + """ + Raised when an external command failed. + + Attributes: + returncode -- The return code from the command + cmd == The command run + """ + def __init__(self, msg, cmd, returncode, log=None): + super(CommandError, self).__init__(msg=msg) + self.returncode = returncode + self.cmd = cmd + self.log = log + +class InspectError(VtrError): + """ + Raised when some query (inspection) result is not found. + """ + def __init__(self, msg, filename=None): + super(InspectError, self).__init__(msg=msg) + self.filename=filename + diff --git a/vtr_flow/scripts/python_libs/verilogtorouting/flow.py b/vtr_flow/scripts/python_libs/verilogtorouting/flow.py new file mode 100644 index 00000000000..daf1b18c2bb --- /dev/null +++ b/vtr_flow/scripts/python_libs/verilogtorouting/flow.py @@ -0,0 +1,439 @@ +import os +import sys +import shutil +import subprocess +import time +import glob +from collections import OrderedDict + +from util import make_enum, print_verbose, mkdir_p, find_vtr_file, file_replace, relax_W, CommandRunner, write_tab_delimitted_csv +from inspect import determine_memory_addr_width, determine_lut_size, determine_min_W, load_parse_patterns +from error import * + +VTR_STAGE = make_enum("odin", "abc", 'ace', "vpr", "lec") +vtr_stages = VTR_STAGE.reverse_mapping.values() + +def run_vtr_flow(architecture_file, circuit_file, + power_tech_file=None, + start_stage=VTR_STAGE.odin, end_stage=VTR_STAGE.vpr, + command_runner=CommandRunner(), + parse_config_file=None, + work_dir=".", + verbosity=0, + vpr_args=None): + """ + Runs the VTR CAD flow to map the specificied circuit_file onto the target architecture_file + + Arguments + --------- + architecture_file: Architecture file to target + circuit_file : Circuit to implement + + power_tech_file : Technology power file. Enables power analysis and runs ace + + work_dir : Directory to run in (created if non-existant) + start_stage : Stage of the flow to start at + end_stage : Stage of the flow to finish at + command_runner : A CommandRunner object used to run system commands + parse_config_file: The configuration file defining how to parse metrics from results + verbosity : How much output to produce + vpr_args : A dictionary of keywork arguments to pass on to VPR + """ + if vpr_args == None: + vpr_args = OrderedDict() + + # + #Initial setup + # + architecture_file_basename = os.path.basename(architecture_file) + circuit_file_basename = os.path.basename(circuit_file) + + circuit_name, circuit_ext = os.path.splitext(circuit_file_basename) + architecture_name, architecture_ext = os.path.splitext(architecture_file_basename) + + mkdir_p(work_dir) + + #Define useful filenames + post_odin_netlist = circuit_name + '.odin.blif' + post_abc_netlist = circuit_name + '.abc.blif' + post_ace_netlist = circuit_name + ".ace.blif" + post_ace_activity_file = circuit_name + ".act" + pre_vpr_netlist = circuit_name + ".pre_vpr.blif" + post_vpr_netlist = "top_post_synthesis.blif" #circuit_name + ".vpr.blif" + lec_base_netlist = None #Reference netlist for LEC + + if circuit_ext == ".blif": + #If the user provided a .blif netlist, we use that as the baseline for LEC + #(ABC can't LEC behavioural verilog) + lec_base_netlist = circuit_file_basename + + #Copy the circuit and architecture + shutil.copy(circuit_file, os.path.join(work_dir, circuit_file_basename)) + shutil.copy(architecture_file, os.path.join(work_dir, architecture_file_basename)) + + + #There are multiple potential paths for the netlist to reach a tool + #We initialize it here to the user specified circuit and let downstream + #stages update it + next_stage_netlist = circuit_file_basename + + # + # RTL Elaboration & Synthesis + # + if should_run_stage(VTR_STAGE.odin, start_stage, end_stage): + if circuit_ext != ".blif": + print_verbose(1, verbosity, "Running Odin II") + + run_odin(architecture_file_basename, next_stage_netlist, + output_netlist=post_odin_netlist, + command_runner=command_runner, + work_dir=work_dir) + + next_stage_netlist = post_odin_netlist + + if not lec_base_netlist: + lec_base_netlist = post_odin_netlist + + # + # Logic Optimization & Technology Mapping + # + if should_run_stage(VTR_STAGE.abc, start_stage, end_stage): + print_verbose(1, verbosity, "Running ABC") + + run_abc(architecture_file_basename, next_stage_netlist, + output_netlist=post_abc_netlist, + command_runner=command_runner, + work_dir=work_dir) + + next_stage_netlist = post_abc_netlist + + if not lec_base_netlist: + lec_base_netlist = post_abc_netlist + + + # + # Power Activity Estimation + # + if power_tech_file: + #The user provided a tech file, so do power analysis + + if should_run_stage(VTR_STAGE.ace, start_stage, end_stage): + print_verbose(1, verbosity, "Running ACE") + + run_ace(next_stage_netlist, output_netlist=post_ace_netlist, + output_activity_file=post_ace_activity_file, + command_runner=command_runner, + work_dir=work_dir) + + #Use ACE's output netlist + next_stage_netlist = post_ace_netlist + + if not lec_base_netlist: + lec_base_netlist = post_ace_netlist + + #Enable power analysis in VPR + vpr_args["power"] = True + vpr_args["activity_file"] = post_ace_activity_file + vpr_args["tech_properties"] = power_tech_file + + # + # Pack/Place/Route + # + if should_run_stage(VTR_STAGE.vpr, start_stage, end_stage): + #Copy the input netlist for input to vpr + shutil.copyfile(os.path.join(work_dir, next_stage_netlist), os.path.join(work_dir, pre_vpr_netlist)) + + #Do we need to generate the post-synthesis netlist? (e.g. for LEC) + if should_run_stage(VTR_STAGE.lec, start_stage, end_stage): + if "gen_postsynthesis_netlist" not in vpr_args: + vpr_args["gen_postsynthesis_netlist"] = "on" + + if "route_chan_width" in vpr_args: + #The User specified a fixed channel width + print_verbose(1, verbosity, "Running VPR (at fixed channel width)") + run_vpr(architecture_file_basename, pre_vpr_netlist, + output_netlist=post_vpr_netlist, + command_runner=command_runner, + work_dir=work_dir, + vpr_args=vpr_args) + else: + #First find minW and then re-route at a relaxed W + run_vpr_relax_W(architecture_file_basename, pre_vpr_netlist, + output_netlist=post_vpr_netlist, + command_runner=command_runner, + work_dir=work_dir, + verbosity=verbosity, + vpr_args=vpr_args) + + if not lec_base_netlist: + lec_base_netlist = pre_vpr_netlist + + # + # Logical Equivalence Checks (LEC) + # + if should_run_stage(VTR_STAGE.lec, start_stage, end_stage): + print_verbose(1, verbosity, "Running ABC Logical Equivalence Check") + run_abc_lec(lec_base_netlist, post_vpr_netlist, command_runner=command_runner, log_filename="abc.lec.out") + +def parse_vtr_flow(work_dir, parse_config_file=None, metrics_filepath=None, verbosity=1): + print_verbose(1, verbosity, "Parsing results") + + if parse_config_file is None: + parse_config_file = find_vtr_file("vtr_benchmarks.min_chan_width.txt") + + parse_patterns = load_parse_patterns(parse_config_file) + + metrics = OrderedDict() + + #Set defaults + for parse_pattern in parse_patterns.values(): + + if parse_pattern.default_value() != None: + metrics[parse_pattern.name()] = parse_pattern.default_value() + else: + metrics[parse_pattern.name()] = "" + + #Process each pattern + for parse_pattern in parse_patterns.values(): + + #We interpret the parse pattern's filename as a glob pattern + filepattern = os.path.join(work_dir, parse_pattern.filename()) + filepaths = glob.glob(filepattern) + + num_files = len(filepaths) + + if num_files > 1: + raise InspectError("File pattern '{}' is ambiguous ({} files matched)".format(parse_pattern.filename()), num_files, filepaths) + + elif num_files == 1: + filepath = filepaths[0] + + assert os.path.exists(filepath) + + with open(filepath) as f: + for line in f: + match = parse_pattern.regex().match(line) + if match: + #Extract the first group value + metrics[parse_pattern.name()] = match.groups()[0] + else: + #No matching file, skip + assert num_files == 0 + + if metrics_filepath is None: + metrics_filepath = os.path.join(work_dir, "parse_results.txt") + + write_tab_delimitted_csv(metrics_filepath, [metrics]) + + return metrics + +def run_odin(architecture_file, circuit_file, + output_netlist, + command_runner, + work_dir=".", + log_filename="odin.out", + odin_exec=None, + odin_config=None, + min_hard_mult_size=3, + min_hard_adder_size=1): + mkdir_p(work_dir) + + if odin_exec == None: + odin_exec = find_vtr_file('odin_II', is_executabe=True) + + if odin_config == None: + odin_base_config = find_vtr_file('basic_odin_config_split.xml') + + #Copy the config file + odin_config = "odin_config.xml" + odin_config_full_path = os.path.abspath(os.path.join(work_dir, odin_config)) + shutil.copyfile(odin_base_config, odin_config_full_path) + + #Update the config file + file_replace(odin_config_full_path, { + "XXX": circuit_file, + "YYY": architecture_file, + "ZZZ": output_netlist, + "PPP": determine_memory_addr_width(os.path.join(work_dir, architecture_file)), + "MMM": min_hard_mult_size, + "AAA": min_hard_adder_size, + }) + + cmd = [odin_exec, "-c", odin_config] + + command_runner.run_system_command(cmd, work_dir=work_dir, log_filename=log_filename, indent_depth=1) + +def run_abc(architecture_file, circuit_file, output_netlist, command_runner, work_dir=".", log_filename="abc.opt_techmap.out", abc_exec=None, abc_script=None, abc_rc=None): + mkdir_p(work_dir) + + if abc_exec == None: + abc_exec = find_vtr_file('abc', is_executabe=True) + + if abc_rc == None: + abc_dir = os.path.dirname(abc_exec) + abc_rc = os.path.join(abc_dir, 'abc.rc') + + shutil.copyfile(abc_rc, os.path.join(work_dir, 'abc.rc')) + + lut_size = determine_lut_size(os.path.join(work_dir, architecture_file)) + + if abc_script == None: + abc_script = ['read {input_netlist}'.format(input_netlist=circuit_file), + 'time', + 'resyn', + 'resyn2', + 'if -K {lut_size}'.format(lut_size=lut_size), + 'time', + 'scleanup', + 'write_hie {input_netlist} {output_netlist}'.format(input_netlist=circuit_file, output_netlist=output_netlist), + 'print_stats'] + abc_script = "; ".join(abc_script) + + cmd = [abc_exec, '-c', abc_script] + + command_runner.run_system_command(cmd, work_dir=work_dir, log_filename=log_filename, indent_depth=1) + +def run_ace(circuit_file, output_netlist, output_activity_file, command_runner, work_dir=".", log_filename="ace.out", ace_exec=None): + + if ace_exec is None: + ace_exec = find_vtr_file('ace', is_executabe=True) + + cmd = [ace_exec, + "-b", circuit_file, + "-n", output_netlist, + "-o", output_activity_file] + + command_runner.run_system_command(cmd, work_dir=work_dir, log_filename=log_filename, indent_depth=1) + +def run_vpr_relax_W(architecture, circuit, command_runner=CommandRunner(), work_dir=".", + relax_W_factor=1.3, vpr_exec=None, verbosity=1, logfile_base="vpr", + vpr_args=None, output_netlist=None): + """ + Runs VPR twice: + 1st: To find the minimum channel width + 2nd: At relaxed channel width (e.g. for critical path delay) + + Arguments + --------- + architecture: Architecture file + circuit: Input circuit netlist + command_runner: CommandRunner object + work_dir: Directory to run in + + relax_W_factor: Factor by which to relax minimum channel width for critical path delay routing + verbosity: How much progress output to produce + logfile_base: Base name for log files (e.g. "vpr" produces vpr.min_W.out, vpr.relaxed_W.out) + vpr_args: Extra arguments for VPR + vpr_exec: Path to the VPR executable + output_netlist: Output implementation netlist to generate + """ + if vpr_args is None: + vpr_args = OrderedDict() + + mkdir_p(work_dir) + + vpr_min_W_log = '.'.join([logfile_base, "min_W", "out"]) + vpr_relaxed_W_log = '.'.join([logfile_base, "relaxed_W", "out"]) + + print_verbose(1, verbosity, "Running VPR (determining minimum channel width)" ) + + run_vpr(architecture, circuit, command_runner, work_dir, log_filename=vpr_min_W_log, vpr_exec=vpr_exec, vpr_args=vpr_args) + + if ('pack' in vpr_args or 'place' in vpr_args) and 'route' not in vpr_args: + #Don't look for min W if routing was not run + return + + + min_W = determine_min_W(os.path.join(work_dir, vpr_min_W_log)) + + relaxed_W = relax_W(min_W, relax_W_factor) + + print_verbose(1, verbosity, "Running VPR (at {fac}x relaxed minimum channel width)".format(fac=relax_W_factor)) + + vpr_args['route'] = True #Re-route only + vpr_args['route_chan_width'] = relaxed_W #At a fixed channel width + + #VPR does not support performing routing when fixed pins + # are specified, and placement is not run; so remove the option + if 'fix_pins' in vpr_args: + del vpr_args['fix_pins'] + + run_vpr(architecture, circuit, command_runner, work_dir, log_filename=vpr_relaxed_W_log, vpr_exec=vpr_exec, vpr_args=vpr_args) + + +def run_vpr(architecture, circuit, command_runner, work_dir, output_netlist=None, log_filename="vpr.out", vpr_exec=None, vpr_args=None): + """ + Runs VPR with the specified configuration + """ + if vpr_args is None: + vpr_args = OrderedDict() + + mkdir_p(work_dir) + + if vpr_exec == None: + vpr_exec = find_vtr_file('vpr', is_executabe=True) + + cmd = [vpr_exec, architecture, circuit] + + #Enable netlist generation + #if output_netlist: + #vpr_args['gen_postsynthesis_netlist'] = output_netlist + + #Translate arbitrary keyword arguments into options for VPR + for arg, value in vpr_args.iteritems(): + if value == True: + cmd += ["--" + arg] + elif value == False: + pass + else: + cmd += ["--" + arg, str(value)] + + command_runner.run_system_command(cmd, work_dir=work_dir, log_filename=log_filename, indent_depth=1) + +def run_abc_lec(reference_netlist, implementation_netlist, command_runner, work_dir=".", log_filename="abc.lec.out", abc_exec=None): + """ + Run Logical Equivalence Checking (LEC) between two netlists using ABC + """ + mkdir_p(work_dir) + + if abc_exec == None: + abc_exec = find_vtr_file('abc', is_executabe=True) + + abc_script = ['cec {ref} {imp}'.format(ref=reference_netlist, imp=implementation_netlist), + 'sec {ref} {imp}'.format(ref=reference_netlist, imp=implementation_netlist), + ] + abc_script = "; ".join(abc_script) + + cmd = [abc_exec, '-c', abc_script] + + output, returncode = command_runner.run_system_command(cmd, work_dir=work_dir, log_filename=log_filename, indent_depth=1) + + #Check if ABC's LEC engine passed + lec_passed = check_abc_lec_status(output) + + if lec_passed is None: + raise InspectError("Could not determine Logical Equivalence Check status between {input} <-> {output}".format(input=reference_netlist, output=implementation_netlist), filename=log_filename) + elif lec_passed is False: + raise InspectError("Logical Equivalence Check failed between {input} <-> {output}".format(input=reference_netlist, output=implementation_netlist), filename=log_filename) + + assert lec_passed + +def check_abc_lec_status(output): + equivalent = None + for line in output: + if line.startswith("Networks are NOT EQUIVALENT"): + equivalent = False + elif line.startswith("Networks are equivalent"): + equivalent = True + + #Returns None if could not determine LEC status + return equivalent + +def should_run_stage(stage, flow_start_stage, flow_end_stage): + """ + Returns True if stage falls between flow_start_stage and flow_end_stage + """ + if flow_start_stage <= stage <= flow_end_stage: + return True + return False + diff --git a/vtr_flow/scripts/python_libs/verilogtorouting/inspect.py b/vtr_flow/scripts/python_libs/verilogtorouting/inspect.py new file mode 100644 index 00000000000..bbde9263ce2 --- /dev/null +++ b/vtr_flow/scripts/python_libs/verilogtorouting/inspect.py @@ -0,0 +1,306 @@ +import re +from collections import OrderedDict + +try: + #Try for the fast c-based version first + import xml.etree.cElementTree as ET +except ImportError: + #Fall back on python implementation + import xml.etree.ElementTree as ET + +from util import load_config_lines +from error import InspectError + +class ParsePattern: + def __init__(self, name, filename, regex_str, default_value=None): + self._name = name + self._filename = filename + self._regex = re.compile(regex_str) + self._default_value = default_value + + def name(self): + return self._name + + def filename(self): + return self._filename + + def regex(self): + return self._regex + + def default_value(self): + return self._default_value + +class PassRequirement(object): + def __init__(self, metric): + self._metric = metric + self._type = type + + def metric(self): + return self._metric + + def type(self): + raise NotImplementedError() + + def check_passed(golden_value, check_value): + raise NotImplementedError() + +class EqualPassRequirement(PassRequirement): + def __init__(self, metric): + super(EqualPassRequirement, self).__init__(metric) + + def type(self): + return "Equal" + + def check_passed(self, golden_value, check_value): + if golden_value == check_value: + return True, "" + else: + return False, "Task value '{}' does not match golden value '{}'".format(golden_value, check_value) + +class RangePassRequirement(PassRequirement): + + def __init__(self, metric, min_value=None, max_value=None): + super(RangePassRequirement, self).__init__(metric) + + if max_value < min_value: + raise InspectError("Invalid range specification (max value larger than min value)") + + self._min_value = min_value + self._max_value = max_value + + def type(self): + return "Range" + + def min_value(self): + return self._min_value + + def max_value(self): + return self._max_value + + def check_passed(self, golden_value, check_value): + if golden_value == None and check_value == None: + return True, "both golden and check are None" + elif golden_value == None and check_value != None: + return False, "golden value is None, but check value is {}".format(check_value) + elif golden_value != None and check_value == None: + return False, "golden value is {}, but check value is None".format(golden_value) + + assert golden_value != None + assert check_value != None + + try: + golden_value = float(golden_value) + except ValueError as e: + raise InspectError("Failed to convert golden value '{}' to float".format(golden_value)) + + try: + check_value = float(check_value) + except ValueError as e: + raise InspectError("Failed to convert check value '{}' to float".format(check_value)) + + if golden_value == 0.: #Avoid division by zero + + if golden_value == check_value: + return True, "golden and check both equal 0" + else: + return False, "unable to normalize relative value (golden value is zero)".format(norm_check_value, min_value(), max_value()) + + else: + norm_check_value = check_value / golden_value + + if self.min_value() <= norm_check_value <= self.max_value(): + return True, "relative value within range" + else: + return False, "relative value {} outside of range [{},{}]".format(norm_check_value, min_value(), max_value()) + +class ParseResults: + def __init__(self): + self._metrics = OrderedDict() + + def primary_keys(self): + return ("architecture", "circuit") + + def add_result(self, arch, circuit, parse_result): + self._metrics[(arch, circuit)] = parse_result + + def metrics(self, arch, circuit): + if (arch, circuit) in self._metrics: + return self._metrics[(arch, circuit)] + else: + return None + + def all_metrics(self): + return self._metrics + + +def load_parse_patterns(parse_config_filepath): + parse_patterns = OrderedDict() + + for line in load_config_lines(parse_config_filepath): + + components = line.split(';') + + + if len(components) == 3 or len(components) == 4: + + name = components[0] + filepath = components[1] + regex_str = components[2] + + default_value = None + if len(components) == 4: + default_value = components[3] + + if name not in parse_patterns: + parse_patterns[name] = ParsePattern(name, filepath, regex_str, default_value) + else: + raise InspectError("Duplicate parse pattern name '{}'".format(name), parse_config_filepath) + + else: + raise InspectError("Invalid parse format line: '{}'".format(line), parse_config_filepath) + + return parse_patterns + +def load_pass_requirements(pass_requirements_filepath): + parse_patterns = OrderedDict() + + for line in load_config_lines(pass_requirements_filepath): + + components = line.split(';') + + if len(components) == 2: + + metric = components[0] + expr = components[1] + + if metric not in parse_patterns: + func, params_str = expr.split("(") + params_str = params_str.rstrip(")") + if params_str == "": + params = [] + else: + params = params_str.split(",") + + if func == "Range": + if len(params) != 2: + raise InspectError("Range() pass requirement function requires two arguments", pass_requirements_filepath) + + parse_patterns[metric] = RangePassRequirement(metric, float(params[0]), float(params[1])) + elif func == "Equal": + if len(params) != 0: + raise InspectError("Equal() pass requirement function requires no arguments", pass_requirements_filepath) + parse_patterns[metric] = EqualPassRequirement(metric) + else: + raise InspectError("Unexpected pass requirement function '{}' for metric '{}'".format(func, metric), pass_requirements_filepath) + + else: + raise InspectError("Duplicate pass requirement for '{}'".format(metric), pass_requirements_filepath) + + else: + raise InspectError("Invalid pass requirement format line: '{}'".format(line), pass_requirements_filepath) + + return parse_patterns + +def load_parse_results(parse_results_filepath, primary_key_set=None): + header = [] + + parse_results = ParseResults() + + with open(parse_results_filepath) as f: + for lineno, row in enumerate(f): + elements = [elem.strip() for elem in row.split("\t")] + + if lineno == 0: + header = elements + else: + primary_keys = OrderedDict() + result = OrderedDict() + + arch=None + circuit=None + for i, elem in enumerate(elements): + metric = header[i] + + if elem == "": + elem = None + + if metric == "arch": + metric = "architecture" + + if metric == "architecture": + arch = elem + elif metric == "circuit": + circuit = elem + + result[metric] = elem + + assert arch and circuit + + parse_results.add_result(arch, circuit, result) + + return parse_results + +def determine_lut_size(architecture_file): + """ + Determines the maximum LUT size (K) in an architecture file. + + Assumes LUTs are represented as BLIF '.names' + """ + arch_xml = ET.parse(architecture_file).getroot() + + lut_size = 0 + saw_blif_names = False + for elem in arch_xml.findall('.//pb_type'): #Xpath recrusive search for 'pb_type' + blif_model = elem.get('blif_model') + if blif_model and blif_model == ".names": + saw_blif_names = True + input_port = elem.find('input') + + input_width = int(input_port.get('num_pins')) + assert input_width > 0 + + #Keep the maximum lut size found (i.e. fracturable architectures) + lut_size = max(lut_size, input_width) + + if saw_blif_names and lut_size == 0: + raise InspectError(msg="Could not identify valid LUT size (K)", + filename=architecture_file) + + return lut_size + +def determine_memory_addr_width(architecture_file): + """ + Determines the maximum RAM block address width in an architecture file + + Assumes RAMS are represented using the standard VTR primitives (.subckt single_port_ram, .subckt dual_port_ram etc.) + """ + arch_xml = ET.parse(architecture_file).getroot() + + mem_addr_width = 0 + saw_ram = False + for elem in arch_xml.findall('.//pb_type'): #XPATH for recursive search + blif_model = elem.get('blif_model') + if blif_model and "port_ram" in blif_model: + saw_ram = True + for input_port in elem.findall('input'): + port_name = input_port.get('name') + if "addr" in port_name: + input_width = int(input_port.get('num_pins')) + mem_addr_width = max(mem_addr_width, input_width) + + if saw_ram and mem_addr_width == 0: + raise InspectError(msg="Could not identify RAM block address width", + filename=architecture_file) + + return mem_addr_width + +def determine_min_W(log_filename): + min_W_regex = re.compile(r"\s*Best routing used a channel width factor of (?P\d+).") + with open(log_filename) as f: + for line in f: + match = min_W_regex.match(line) + if match: + return int(match.group("min_W")) + + raise InspectError(msg="Failed to find minimum channel width.", + filename=log_filename) diff --git a/vtr_flow/scripts/python_libs/verilogtorouting/task.py b/vtr_flow/scripts/python_libs/verilogtorouting/task.py new file mode 100644 index 00000000000..8b1d398ae35 --- /dev/null +++ b/vtr_flow/scripts/python_libs/verilogtorouting/task.py @@ -0,0 +1,154 @@ +import os + +from verilogtorouting.error import * +from util import find_vtr_root, find_vtr_file, load_list_file + +class TaskConfig: + """ + An object representing a task config file + """ + def __init__(self, + task_name, + config_dir, + circuits_dir, + archs_dir, + circuit_list_add, + arch_list_add, + parse_file, + script_path=None, + script_params=None, + pass_requirements_file=None, + sdc_dir=None, + qor_parse_file=None, + cmos_tech_behavior=None, + pad_file=None): + self.task_name = task_name + self.config_dir = config_dir + self.circuit_dir = circuits_dir + self.arch_dir = archs_dir + self.circuits = circuit_list_add + self.archs = arch_list_add + self.parse_file = parse_file + self.script_path = script_path + self.script_params = script_params + self.pass_requirements_file = pass_requirements_file + self.sdc_dir = sdc_dir + self.qor_parse_file = qor_parse_file + self.cmos_tech_behavior = cmos_tech_behavior + self.pad_file = pad_file + +def load_task_config(config_file): + #Load the file stripping comments + values = load_list_file(config_file) + + #Keys that can appear only once + unique_keys = set(["circuits_dir", + "archs_dir", + "parse_file", + "script_path", + "script_params", + "pass_requirements_file", + "sdc_dir", + "qor_parse_file", + "cmos_tech_behavior", + "pad_file"]) + + #Keys that are repeated to create lists + repeated_keys = set(["circuit_list_add", + "arch_list_add"]) + + #Keys that are required + required_keys = set(["circuits_dir", + "archs_dir", + "circuit_list_add", + "arch_list_add", + "parse_file"]) + + #Interpret the file + key_values = {} + for line in values: + key, value = line.split('=') + + #Trim whitespace + key = key.strip() + value = value.strip() + + if key in unique_keys: + if key not in key_values: + key_values[key] = value + else: + raise VtrError("Duplicate {key} in config file {file}".format(key=key, file=config_file)) + + elif key in repeated_keys: + if key not in key_values: + key_values[key] = [] + key_values[key].append(value) + + else: + #All valid keys should have been collected by now + raise VtrError("Unrecognzied key '{key}' in config file {file}".format(key=key, file=config_file)) + + #We split the script params into a list + if 'script_params' in key_values: + key_values['script_params'] = key_values['script_params'].split(' ') + + #Check that all required fields were specified + for required_key in required_keys: + if required_key not in key_values: + raise VtrError("Missing required key '{key}' in config file {file}".format(key=required_key, file=config_file)) + + #Useful meta-data about the config + config_dir = os.path.dirname(config_file) + key_values['config_dir'] = config_dir + key_values['task_name'] = os.path.basename(os.path.dirname(config_dir)) + + #Create the task config object + return TaskConfig(**key_values) + + +def find_task_config_file(task_name): + # + # See if we can find the config.txt assuming the task name is an + # absolute/relative path + # + + base_dirs = [] + if os.path.isabs(task_name): + #Only check the root path since the path is aboslute + base_dirs.append('/') + else: + #Not absolute path, so check from the current directory first + base_dirs.append('.') + + vtr_root = find_vtr_root() + vtr_flow_tasks_dir = os.path.join(vtr_root, "vtr_flow", "tasks") + + #Then the VTR tasks directory + base_dirs.append(vtr_flow_tasks_dir) + + #Generate potential config files (from most to least specific) + potential_config_file_paths = [] + for base_dir in base_dirs: + #Assume points directly to a config.txt + assume_config_path = os.path.join(base_dir, task_name) + potential_config_file_paths.append(assume_config_path) + + #Assume points to a config dir (containing config.txt) + assume_config_dir_path = os.path.join(base_dir, task_name, "config.txt") + potential_config_file_paths.append(assume_config_dir_path) + + #Assume points to a task dir (containing config/config.txt) + assume_task_dir_path = os.path.join(base_dir, task_name, "config", "config.txt") + potential_config_file_paths.append(assume_task_dir_path) + + #Find the first potential file that is valid + for config_file in potential_config_file_paths: + is_file = os.path.isfile(config_file) + is_named_config = os.path.basename(config_file) == "config.txt" + is_in_config_dir = os.path.basename(os.path.dirname(config_file)) == "config" + + if is_file and is_named_config and is_in_config_dir: + return os.path.abspath(config_file) + + raise VtrError("Could not find config/config.txt for task {name}".format(name=task_name)) + diff --git a/vtr_flow/scripts/python_libs/verilogtorouting/util.py b/vtr_flow/scripts/python_libs/verilogtorouting/util.py new file mode 100644 index 00000000000..fa9da670af1 --- /dev/null +++ b/vtr_flow/scripts/python_libs/verilogtorouting/util.py @@ -0,0 +1,492 @@ +import os +import sys +import re +import time +import errno +import subprocess +import distutils.spawn as distutils_spawn +import argparse +import csv +from collections import OrderedDict + +from verilogtorouting.error import * + +VERBOSITY_CHOICES = range(5) + +class RawDefaultHelpFormatter(argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescriptionHelpFormatter): + """ + An argparse formatter which supports both default arguments and raw formatting of description/epilog + """ + pass + +class CommandRunner(object): + + def __init__(self, timeout_sec=None, max_memory_mb=None, track_memory=True, verbose_error=False, verbose=False, echo_cmd=False, indent="\t"): + """ + An object for running system commands with timeouts, memory limits and varying verbose-ness + + Arguments + ========= + timeout_sec: maximum walk-clock-time of the command in seconds. Default: None + max_memory_mb: maximum memory usage of the command in megabytes (if supported). Default: None + track_memory: Whether to track usage of the command (disabled if not supported). Default: True + verbose_error: Produce more verbose output if the commadn fails. Default: False + verbose: Produce more verbose output. Default: False + echo_cmd: Echo the command before running. Default: False + indent: The string specifying a single indent (used in verbose mode) + """ + self._timeout_sec = timeout_sec + self._max_memory_mb = max_memory_mb + self._track_memory = track_memory + self._verbose_error = verbose_error + self._verbose = verbose + self._echo_cmd = echo_cmd + self._indent = indent + + def run_system_command(self, cmd, work_dir, log_filename=None, expected_return_code=0, indent_depth=0): + """ + Runs the specified command in the system shell. + + Returns + ======= + A tuple of the command output (list of lines) and the command's return code. + + Arguments + ========= + cmd: list of tokens that form the command to be run + log_filename: name of the log file for the command's output. Default: derived from command + work_dir: The directory to run the command in. Default: None (uses object default). + expected_return_code: The expected return code from the command. If the actula return code does not match, will generate an exception. Default: 0 + indent_depth: How deep to indent the tool output in verbose mode. Default 0 + """ + #Save the original command + orig_cmd = cmd + + #If no log file is specified the name is based on the executed command + if log_filename == None: + log_filename = os.path.basename(orig_cmd[0]) + '.out' + + + #Limit memory usage? + memory_limit = ["ulimit", "-Sv", "{val};".format(val=self._max_memory_mb)] + if self._max_memory_mb != None and self.check_command(memory_limit[0]): + cmd = memory_limit + cmd + + #Enable memory tracking? + memory_tracking = ["/usr/bin/time", "-v"] + if self._track_memory and self.check_command(memory_tracking[0]): + cmd = memory_tracking + cmd + + #Flush before calling subprocess to ensure output is ordered + #correctly if stdout is buffered + sys.stdout.flush() + + #Echo the command? + if self._echo_cmd: + #print ' '.join(cmd) + print cmd + + #Begin timing + start_time = time.time() + + cmd_output=[] + cmd_returncode=None + proc = None + try: + #Call the command + proc = subprocess.Popen(cmd, + stdout=subprocess.PIPE, #We grab stdout + stderr=subprocess.STDOUT, #stderr redirected to stderr + universal_newlines=True, #Lines always end in \n + cwd=work_dir, #Where to run the command + ) + + # Read the output line-by-line and log it + # to a file. + # + # We do this rather than use proc.communicate() + # to get interactive output + with open(os.path.join(work_dir, log_filename), 'w') as log_f: + #Print the command at the top of the log + print >> log_f, " ".join(cmd) + + #Read from subprocess output + for line in proc.stdout: + + #Send to log file + print >> log_f, line, + + #Save the output + cmd_output.append(line) + + #Abort if over time limit + elapsed_time = time.time() - start_time + if self._timeout_sec and elapsed_time > self._timeout_sec: + proc.terminate() + + #Should now be finished (since we stopped reading from proc.stdout), + #sets the return code + proc.wait() + + finally: + #Clean-up if we did not exit cleanly + if proc: + if proc.returncode == None: + #Still running, stop it + proc.terminate() + + cmd_returncode = proc.returncode + + cmd_errored = (cmd_returncode != expected_return_code) + + #Send to stdout + if self._verbose or (cmd_errored and self._verbose_error): + for line in cmd_output: + print indent_depth*self._indent + line, + + + if cmd_errored: + raise CommandError(msg="Executable {exec_name} failed".format(exec_name=os.path.basename(orig_cmd[0])), + cmd=cmd, + log=os.path.join(work_dir, log_filename), + returncode=cmd_returncode) + + return cmd_output, cmd_returncode + + def check_command(self, command): + """ + Return True if command can be run, False otherwise. + """ + + #TODO: actually check for this + return True + + +def write_tab_delimitted_csv(filepath, rows): + """ + Write out the data provied in a tab-delimited CSV format + + filepath: The filepath to write the data to + rows: An iterable of dictionary-like elements; each element + provides a set key-value pairs corresponding to a row + in the output file + """ + #Calculate the max width of each column + columns = OrderedDict() + for row in rows: + for key, value in row.iteritems(): + + if key not in columns: + columns[key] = max(len(key), len(str(value))) + else: + columns[key] = max(columns[key], len(str(value))) + + #Write the elements + with open(filepath, 'w') as f: + writer = csv.writer(f, delimiter='\t') + + #Write out the header + header = [] + for col_name, col_width in columns.iteritems(): + header.append("{:{width}}".format(col_name, width=col_width)) + writer.writerow(header) + + #Write rows + for row in rows: + values = [] + for col_name, col_width in columns.iteritems(): + values.append("{:{width}}".format(row[col_name], width=col_width)) + writer.writerow(values) + +def load_tab_delimited_csv(filepath): + + data = [] + with open(filepath) as f: + reader = csv.reader(f, delimiter='\t') + + header = None + for csv_row in reader: + if header == None: + header = [val.strip() for val in csv_row] + else: + data_row = OrderedDict() + + for i, value in enumerate(csv_row): + data_row[header[i]] = value.strip() + + data.append(data_row) + + return data + +def make_enum(*sequential, **named): + """ + Makes an enumeration + + >> MY_ENUM = enum(ONE, TWO, THREE) + """ + enums = dict(zip(sequential, range(len(sequential))), **named) + reverse = dict((value, key) for key, value in enums.iteritems()) + enums['reverse_mapping'] = reverse + return type('Enum', (), enums) + +def mkdir_p(path): + """ + Makes a directory (including parents) at the specified path + """ + try: + os.makedirs(path) + except OSError as exc: # Python >2.5 + if exc.errno == errno.EEXIST and os.path.isdir(path): + pass + else: + raise + +def print_verbose(min_verbosity, curr_verbosity, string, endl=True): + """ + Print string if curr_verbosity is gteq min_verbosity + """ + if curr_verbosity >= min_verbosity: + if endl: + print string + else: + print string, + +def find_vtr_file(filename, is_executable=False): + """ + Attempts to find a VTR related file by searching the environment. + + Checking the following places: + 1) System path (if is_executable=True) + 2) The inferred vtr root from environment variables or the script file location + + """ + #We assume exectuables are specified in the unix style (no .exe), + # if it was specified with .exe, strip it off + filebase, ext = os.path.splitext(filename) + if ext == ".exe": + filename = filebase + + # + #Check if it is on the path (provided it is executable) + # + if is_executable: + #Search first for the non-exe version + result = distutils_spawn.find_executable(filename) + if result: + return result + + #If not found try the .exe version + result = distutils_spawn.find_executable(filename + '.exe') + if result: + return result + + vtr_root = find_vtr_root() + + #Check the inferred VTR root + result = find_file_from_vtr_root(filename, vtr_root, is_executable=is_executable) + if result: + return result + + #Since we stripped off the .exe, try looking for the .exe version as a last resort (i.e. on Windows/cygwin) + if is_executable: + result = find_file_from_vtr_root(filename + '.exe', vtr_root, is_executable=is_executable) + if result: + return result + + raise ValueError("Could not find {type} {file}".format(type="executable" if is_executable else "file",file=filename)) + +def find_file_from_vtr_root(filename, vtr_root, is_executable=False): + """ + Given a vtr_root and a filename searches for the file recursively under some common VTR directories + """ + for subdir in ['vpr', 'abc', 'abc_with_bb_support', 'ODIN_II', 'vtr_flow', 'ace2']: + + for root, dirs, files in os.walk(os.path.join(vtr_root, subdir)): + for file in files: + if file == filename: + full_file_path = os.path.join(root, file) + + if os.path.isfile(full_file_path): + if is_executable: + #Found an executable file as required + if os.access(full_file_path, os.X_OK): + return full_file_path + else: + #Found a file as required + return full_file_path + return None + +def find_vtr_root(): + for env_var in ['VTR_ROOT']: + if env_var in os.environ: + return os.environ[env_var] + + inferred_script_dir = os.path.dirname(os.path.abspath(__file__)) + + #We assume that this file is in /vtr_flow/python_libs/verilogtorouting + inferred_vtr_root = os.path.abspath(os.path.join(inferred_script_dir, '../../../..')) + + if os.path.isdir(inferred_vtr_root): + return inferred_vtr_root + else: + raise VtrError("Could not find VTR root directory. Try setting VTR_ROOT environment variable.") + +def file_replace(filename, search_replace_dict): + lines = [] + with open(filename, 'r') as f: + lines = f.readlines() + + with open(filename, 'w') as f: + for line in lines: + for search, replace in search_replace_dict.iteritems(): + line = line.replace(search, str(replace)) + print >>f, line, + +def relax_W(min_W, relax_factor, base=2): + """ + Scale min_W by relax_factor and round to the nearest multiple of base. + """ + relaxed_W = int(base * round(min_W*relax_factor/base)) + return relaxed_W + +def load_list_file(list_file): + """ + Loads a file containing a single value-per-line, + potentially with '#' comments + """ + values = [] + with open(list_file) as f: + for line in f: + line = line.strip() + #Handle comments + if '#' in line: + line = line.split('#')[0] + if line == "": + continue + values.append(line) + return values + +def load_config_lines(filepath, allow_includes=True): + """ + Loads the lines of a file, stripping blank lines and '#' comments. + + If allow_includes is set then lines of the form: + + @include "another_file.txt" + + will cause the specified file to be included in-line. + The @included filename is interpretted as relative to the directory + containing filepath. + + Returns a list of lines + """ + config_lines = [] + + blank_regex = re.compile(r"^\s*$") + try: + with open(filepath) as f: + for line in f: + #Trim '\n' + line = line.strip() + + #Trim comments + if '#' in line: + line = line.split('#')[0] + + #Skip blanks + if blank_regex.match(line): + continue + + if line.startswith("@include"): + if allow_includes: + components = line.split() + assert len(components) == 2 + + include_file = components[1].strip('"') #Strip quotes + include_file_abs = os.path.join(os.path.dirname(filepath), include_file) + + #Recursively load the config + config_lines += load_config_lines(include_file_abs, allow_includes=allow_includes) + else: + raise InspectError("@include not allowed in this file", filepath) + else: + config_lines.append(line) + except IOError as e: + raise InspectError("Error opening config file ({})".format(e)) + + return config_lines + +def format_elapsed_time(time_delta): + total_sec = int(round(time_delta.total_seconds())) + m, s = divmod(total_sec, 60) + h, m = divmod(m, 60) + + return "{:02}:{:02}:{:02}".format(h, m, s) + +def argparse_str2bool(str_val): + str_val = str_val.lower() + if str_val in ['yes', 'on', 'true', '1']: + return True + elif str_val in ['no', 'off', 'false', '0']: + return False + else: + raise argparse.ArgumentTypeError('Boolean value expected.') + + +def get_next_run_dir(base_dir): + """ + Returns the next unused run directory within base_dir. + + Does not create the directory + """ + return os.path.join(base_dir, run_dir_name(get_next_run_number(base_dir))) + +def get_latest_run_dir(base_dir): + """ + Returns the run directory with the highest run number in base_dir + """ + latest_run_number = get_latest_run_number(base_dir) + + if latest_run_number == None: + return None + + return os.path.join(base_dir, run_dir_name(latest_run_number)) + +def get_next_run_number(base_dir): + """ + Returns the next available (i.e. non-existing) run number in base_dir + """ + latest_run_number = get_latest_run_number(base_dir) + + if latest_run_number == None: + next_run_number = 0 + else: + next_run_number = latest_run_number + 1 + + return next_run_number + +def get_latest_run_number(base_dir): + """ + Returns the highest run number of all run directories with in base_dir + """ + run_number = 0 + run_dir = os.path.join(base_dir, run_dir_name(run_number)) + + if not os.path.exists(run_dir): + #No existing run directories + return None + + while os.path.exists(run_dir): + run_number += 1 + run_dir = os.path.join(base_dir, run_dir_name(run_number)) + + #Currently one-past the last existing run dir, + #to get latest existing, subtract one + return run_number - 1 + + +def run_dir_name(run_num): + """ + Returns the formatted directory name for a specific run number + """ + return "run{:03d}".format(run_num) diff --git a/vtr_flow/scripts/vtr-flow.py b/vtr_flow/scripts/vtr-flow.py new file mode 100755 index 00000000000..e1451139f6b --- /dev/null +++ b/vtr_flow/scripts/vtr-flow.py @@ -0,0 +1,292 @@ +#!/usr/bin/env python2 +import sys +import os +import errno +import argparse +import subprocess +import time +import shutil +import re +import textwrap +from datetime import datetime + +from collections import OrderedDict + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), 'python_libs')) + +from verilogtorouting.flow import run_vtr_flow, parse_vtr_flow, VTR_STAGE, vtr_stages, CommandRunner +from verilogtorouting.error import * +from verilogtorouting.util import print_verbose, RawDefaultHelpFormatter, VERBOSITY_CHOICES, find_vtr_file, format_elapsed_time + + +BASIC_VERBOSITY = 1 + + + +class VtrStageArgparseAction(argparse.Action): + def __call__(self, parser, namespace, value, option_string=None): + if value == "odin": + setattr(namespace, self.dest, VTR_STAGE.odin) + elif value == "abc": + setattr(namespace, self.dest, VTR_STAGE.abc) + elif value == "vpr": + setattr(namespace, self.dest, VTR_STAGE.vpr) + elif value == "lec": + setattr(namespace, self.dest, VTR_STAGE.lec) + else: + raise argparse.ArgumentError(self, "Invalid VTR stage '" + value + "'") + +class OnOffArgparseAction(argparse.Action): + def __call__(self, parser, namespace, value, option_string=None): + if value.lower() == "on": + setattr(namespace, self.dest, True) + elif value.lower() == "off": + setattr(namespace, self.dest, False) + else: + raise argparse.ArgumentError(self, "Invalid yes/no value '" + value + "'") + +on_off_choices = ['on', 'off'] + +def vtr_command_argparser(prog=None): + usage = "%(prog)s circuit_file architecture_file [options]" + description = textwrap.dedent( + """ + Runs a single architecture and circuit through the VTR CAD flow - mapping + the circuit onto the target archietcture. + + Any unrecognzied arguments are passed to VPR. + """ + ) + + epilog = textwrap.dedent( + """ + Examples + -------- + + Produce more output (e.g. individual tool outputs): + + %(prog)s arch.xml circuit.v -v 2 + + + Route at a fixed channel width: + + %(prog)s arch.xml circuit.v --route_chan_width 100 + + Passing '--route_chan_width 100' will force VPR to route at a fixed channel width, + rather then attempting to find the minimum channel width (VPR's default behaviour). + + + Run in the directory 'test' (rather than in a temporary directory): + + %(prog)s arch.xml circuit.v --work_dir test + + + Enable power analysis: + + %(prog)s arch.xml circuit.v --power_tech power_tech.xml + + + Only run a specific stage (assumes required results have already been generated): + + %(prog)s arch.xml circuit.blif --start vpr --end vpr + + """ + ) + + parser = argparse.ArgumentParser( + prog=prog, + usage=usage, + description=description, + epilog=epilog, + formatter_class=RawDefaultHelpFormatter, + ) + + # + # Major arguments + # + parser.add_argument('circuit_file', + help="The circuit to map to the target architecture.") + parser.add_argument('architecture_file', + help="The FPGA architecture to target.") + parser.add_argument("--start", "--starting_stage", + choices=VTR_STAGE.reverse_mapping.values(), + default=VTR_STAGE.odin, + action=VtrStageArgparseAction, + help="Starting stage of the VTR flow.") + + parser.add_argument("--end", "--ending_stage", + choices=VTR_STAGE.reverse_mapping.values(), + default=VTR_STAGE.vpr, + action=VtrStageArgparseAction, + help="Ending stage of the VTR flow.") + + parser.add_argument("-v", "--verbosity", + choices=VERBOSITY_CHOICES, + default=2, + type=int, + help="Verbosity of the script. Higher values produce more output.") + + parser.add_argument("--parse_config_file", + default=None, + help="Parse file to run after flow completion") + + parser.add_argument("--parse", + default=False, + action="store_true", + dest="parse_only", + help="Perform only parsing (assumes a previous run exists in work_dir)") + + + # + # Power arguments + # + power = parser.add_argument_group("Power", description="Power Analysis Related Options") + power.add_argument("--power", + default=None, + dest="power_tech", + metavar="POWER_TECH_FILE", + help="Enables power analysis using the specified technology file. Power analysis is disabled if not specified.") + + # + # House keeping arguments + # + house_keeping = parser.add_argument_group("House Keeping", description="Configuration related to how files/time/memory are used by the script.") + + house_keeping.add_argument("--work_dir", + default=".", + help="Directory to run the flow in (will be created if non-existant).") + + house_keeping.add_argument("--track_memory_usage", + choices=on_off_choices, + default="on", + action=OnOffArgparseAction, + help="Track the memory usage for each stage. Requires /usr/bin/time -v, disabled if not available.") + + house_keeping.add_argument("--memory_limit", + default=None, + metavar="MAX_MEMORY_MB", + help="Specifies the maximum memory usable by any stage. " + "Not supported on some platforms (requires ulimit).") + + house_keeping.add_argument("--timeout", + default=14*24*60*60, #14 days + type=float, + metavar="TIMEOUT_SECONDS", + help="Maximum time in seconds to spend on a single stage.") + + return parser + +def main(): + vtr_command_main(sys.argv[1:]) + +def vtr_command_main(arg_list, prog=None): + start = datetime.now() + + #Load the arguments + args, unkown_args = vtr_command_argparser(prog).parse_known_args(arg_list) + + print_verbose(BASIC_VERBOSITY, args.verbosity, "# {} {}\n".format(prog, ' '.join(arg_list))) + + abs_path_arch_file = os.path.abspath(args.architecture_file) + abs_path_circuit_file = os.path.abspath(args.circuit_file) + abs_work_dir = os.path.abspath(args.work_dir) + + + #Specify how command should be run + command_runner = CommandRunner(track_memory=args.track_memory_usage, + max_memory_mb=args.memory_limit, + timeout_sec=args.timeout, + verbose_error=True if args.verbosity == 2 else False, + verbose=True if args.verbosity > 2 else False, + echo_cmd=True if args.verbosity >= 4 else False) + exit_status = 0 + try: + if not args.parse_only: + try: + vpr_args = process_unkown_args(unkown_args) + + #Run the flow + run_vtr_flow(abs_path_arch_file, + abs_path_circuit_file, + power_tech_file=args.power_tech, + work_dir=abs_work_dir, + start_stage=args.start, + end_stage=args.end, + command_runner=command_runner, + verbosity=args.verbosity, + vpr_args=vpr_args + ) + except CommandError as e: + #An external command failed + print "Error: {msg}".format(msg=e.msg) + print "\tfull command: ", ' '.join(e.cmd) + print "\treturncode : ", e.returncode + print "\tlog file : ", e.log + exit_status = 1 + except InspectError as e: + #Something went wrong gathering information + print "Error: {msg}".format(msg=e.msg) + print "\tfile : ", e.filename + exit_status = 2 + + except VtrError as e: + #Generic VTR errors + print "Error: ", e.msg + exit_status = 3 + + except KeyboardInterrupt as e: + print "{} recieved keyboard interrupt".format(prog) + exit_status = 4 + + #Parse the flow results + try: + parse_vtr_flow(abs_work_dir, args.parse_config_file, verbosity=args.verbosity) + except InspectError as e: + print "Error: {msg}".format(msg=e.msg) + print "\tfile : ", e.filename + exit_status = 2 + + finally: + print_verbose(BASIC_VERBOSITY, args.verbosity, "\n# {} took {} (exiting {})".format(prog, format_elapsed_time(datetime.now() - start), exit_status)) + sys.exit(exit_status) + +def process_unkown_args(unkown_args): + #We convert the unkown_args into a dictionary, which is eventually + #used to generate arguments for VPR + vpr_args = OrderedDict() + while len(unkown_args) > 0: + #Get the first argument + arg = unkown_args.pop(0) + + if arg == '': + continue + + if not arg.startswith('-'): + raise VtrError("Extra argument '{}' intended for VPR does not start with '-'".format(arg)) + + #To make it a valid kwargs dictionary we trim the initial '-' or '--' from the + #argument name + assert len(arg) >= 2 + if arg[1] == '-': + #Double-dash prefix + arg = arg[2:] + else: + #Single-dash prefix + arg = arg[1:] + + + #Determine if there is a value associated with this argument + if len(unkown_args) == 0 or unkown_args[0].startswith('-'): + #Single value argument, we place these with value 'True' + #in vpr_args + vpr_args[arg] = True + else: + #Multivalue argument + val = unkown_args.pop(0) + vpr_args[arg] = val + + return vpr_args + +if __name__ == "__main__": + retval = main() + sys.exit(retval) diff --git a/vtr_flow/scripts/vtr-task.py b/vtr_flow/scripts/vtr-task.py new file mode 100755 index 00000000000..1df01ada571 --- /dev/null +++ b/vtr_flow/scripts/vtr-task.py @@ -0,0 +1,654 @@ +#!/usr/bin/env python2 +import os +import sys +import argparse +import itertools +import textwrap +import subprocess +import time +import shutil +from datetime import datetime +from multiprocessing import Pool + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), 'python_libs')) + +import verilogtorouting as vtr +from verilogtorouting.error import * +from verilogtorouting.util import load_list_file, find_vtr_file, mkdir_p, print_verbose, find_vtr_root, CommandRunner, format_elapsed_time, RawDefaultHelpFormatter, VERBOSITY_CHOICES, argparse_str2bool, get_next_run_dir, get_latest_run_dir +from verilogtorouting.task import load_task_config, TaskConfig, find_task_config_file +from verilogtorouting.flow import CommandRunner +from verilogtorouting.inspect import load_pass_requirements, load_parse_results + +BASIC_VERBOSITY = 1 +FAILED_LOG_VERBOSITY = 2 +ALL_LOG_VERBOSITY = 4 + +class Job: + + def __init__(self, task_name, arch, circuit, work_dir, command): + self._task_name = task_name + self._arch = arch + self._circuit = circuit + self._command = command + self._work_dir = work_dir + + def task_name(self): + return self._task_name + + def arch(self): + return self._arch + + def circuit(self): + return self._circuit + + def job_name(self): + return os.path.join(self.arch(), self.circuit()) + + def command(self): + return self._command + + def work_dir(self, run_dir): + return os.path.join(run_dir, self._work_dir) + +def vtr_command_argparser(prog=None): + description = textwrap.dedent( + """ + Runs one or more VTR tasks. + """ + ) + epilog = textwrap.dedent( + """ + Examples + -------- + + Run the task named 'timing_chain': + + %(prog)s timing_chain + + Run all the tasks listed in the file 'task_list.txt': + + %(prog)s -l task_list.txt + + Run 'timing_chain' with 4 jobs running in parallel: + + %(prog)s timing_chain -j4 + + Exit Code + --------- + The exit code equals the number failures (i.e. exit code 0 indicates no failures). + """ + ) + + parser = argparse.ArgumentParser( + prog=prog, + description=description, + epilog=epilog, + formatter_class=RawDefaultHelpFormatter, + ) + + # + # Major arguments + # + parser.add_argument('task', + nargs="*", + help="Tasks to be run") + + parser.add_argument('-l', '--list_file', + nargs="*", + default=[], + metavar="TASK_LIST_FILE", + help="A file listing tasks to be run") + + parser.add_argument("--parse", + default=False, + action="store_true", + dest="parse", + help="Perform only parsing on the latest task run") + + parser.add_argument("--create_golden", + default=False, + action="store_true", + dest="create_golden", + help="Update or create golden results for the specified task") + + parser.add_argument("--check_golden", + default=False, + action="store_true", + dest="check_golden", + help="Check the latest task run against golden results") + + parser.add_argument('--system', + choices=['local'], + default='local', + help="What system to run the tasks on.") + + parser.add_argument('-j', + default=1, + type=int, + metavar="NUM_PROC", + help="How many processors to use for execution.") + + parser.add_argument('--timeout', + default=30*24*60*60, #30 days + metavar="TIMEOUT_SECONDS", + help="Time limit for this script.") + + parser.add_argument("-v", "--verbosity", + choices=VERBOSITY_CHOICES, + default=2, + type=int, + help="Sets the verbosity of the script. Higher values produce more output.") + + parser.add_argument("--work_dir", + default=None, + help="Directory to store intermediate and result files." + "If None, set to the relevant directory under $VTR_ROOT/vtr_flow/tasks.") + + parser.add_argument("--print_metadata", + default=True, + type=argparse_str2bool, + help="Print meta-data like command-line arguments and run-time") + + return parser + +def main(): + vtr_command_main(sys.argv[1:]) + +def vtr_command_main(arg_list, prog=None): + start = datetime.now() + + #Load the arguments + args = vtr_command_argparser(prog).parse_args(arg_list) + + args.run = True + if args.parse or args.create_golden or args.check_golden: + #Don't run if parsing or handling golden results + args.run = False + + if args.run: + #Always parse if running + args.parse = True + + if args.print_metadata: + print "# {} {}\n".format(prog, ' '.join(arg_list)) + + num_failed = -1 + try: + task_names = args.task + + for list_file in args.list_file: + task_names += load_list_file(list_file) + + config_files = [find_task_config_file(task_name) for task_name in task_names] + + configs = [load_task_config(config_file) for config_file in config_files] + + num_failed = run_tasks(args, configs) + + except CommandError as e: + print "Error: {msg}".format(msg=e.msg) + print "\tfull command: ", e.cmd + print "\treturncode : ", e.returncode + print "\tlog file : ", e.log + except InspectError as e: + print "Error: {msg}".format(msg=e.msg) + if e.filename: + print "\tfile: ", e.filename + except VtrError as e: + print "Error:", e.msg + finally: + if args.print_metadata: + print "\n# {} took {} (exiting {})".format(prog, format_elapsed_time(datetime.now() - start), num_failed) + sys.exit(num_failed) + +def run_tasks(args, configs): + """ + Runs the specified set of tasks (configs) + """ + num_failed = 0 + + #We could potentially support other 'run' systems (e.g. a cluster), + #rather than just the local machine + if args.system == "local": + assert args.j > 0, "Invalid number of processors" + + #Generate the jobs, each corresponding to an invocation of vtr flow + jobs = create_jobs(args, configs) + + if args.run: + num_failed = run_parallel(args, configs, jobs) + + if args.parse: + print_verbose(BASIC_VERBOSITY, args.verbosity, "") + parse_tasks(args, configs, jobs) + + if args.create_golden: + print_verbose(BASIC_VERBOSITY, args.verbosity, "") + create_golden_results_for_tasks(args, configs) + + if args.check_golden: + print_verbose(BASIC_VERBOSITY, args.verbosity, "") + num_failed += check_golden_results_for_tasks(args, configs) + + else: + raise VtrError("Unrecognized run system {system}".format(system=args.system)) + + return num_failed + +def parse_tasks(args, configs, jobs): + """ + Parse the selection of tasks specified in configs and associated jobs + """ + for config in configs: + config_jobs = [job for job in jobs if job.task_name() == config.task_name] + parse_task(args, config, config_jobs) + +def parse_task(args, config, config_jobs, task_metrics_filepath=None, flow_metrics_basename="parse_results.txt"): + """ + Parse a single task run. + + This generates a file parse_results.txt in the task's working directory, + which is an amalgam of the parse_rests.txt's produced by each job (flow invocation) + """ + run_dir = find_latest_run_dir(args, config) + + print_verbose(BASIC_VERBOSITY, args.verbosity, "Parsing task run {}".format(run_dir)) + + for job in config_jobs: + #Re-run parsing only + cmd = job.command() + cmd += ['--parse'] + cmd += ['-v', str(max(0, args.verbosity-3))] + subprocess.check_call(cmd, cwd=job.work_dir(run_dir)) + + if task_metrics_filepath is None: + task_metrics_filepath = task_parse_results_filepath = os.path.join(run_dir, "parse_results.txt") + + #Record max widths for pretty printing + max_arch_len = len("architecture") + max_circuit_len = len("circuit") + for job in config_jobs: + max_arch_len = max(max_arch_len, len(job.arch())) + max_circuit_len = max(max_circuit_len, len(job.circuit())) + + with open(task_parse_results_filepath, "w") as out_f: + + #Start the header + print >>out_f, "{:<{arch_width}}\t{:<{circuit_width}}\t".format("architecture", "circuit", arch_width=max_arch_len, circuit_width=max_circuit_len), + header = True + + for job in config_jobs: + #Open the job results file + # + #The job results file is basically the same format, but excludes the architecture and circuit fields, + #which we prefix to each line of the task result file + job_parse_results_filepath = os.path.join(run_dir, job.arch(), job.circuit(), flow_metrics_basename) + if os.path.isfile(job_parse_results_filepath): + with open(job_parse_results_filepath) as in_f: + lines = in_f.readlines() + + assert len(lines) == 2 + + if header: + #First line is the header + print >>out_f, lines[0], + header = False + + #Second line is the data + print >>out_f, "{:<{arch_width}}\t{:<{circuit_width}}\t{}".format(job.arch(), job.circuit(), lines[1], arch_width=max_arch_len, circuit_width=max_circuit_len), + else: + print_verbose(BASIC_VERBOSITY, args.verbosity, "Warning: Flow result file not found (task QoR will be incomplete): {} ".format(job_parse_results_filepath)) + +def create_golden_results_for_tasks(args, configs): + for config in configs: + create_golden_results_for_task(args, config) + +def create_golden_results_for_task(args, config): + """ + Copies the latest task run's parse_results.txt into the config directory as golden_results.txt + """ + run_dir = find_latest_run_dir(args, config) + + task_results = os.path.join(run_dir, "parse_results.txt") + golden_results_filepath = os.path.join(config.config_dir, "golden_results.txt") + + print_verbose(BASIC_VERBOSITY, args.verbosity, "Creating golden task results from {} -> {}".format(run_dir, golden_results_filepath)) + + shutil.copy(task_results, golden_results_filepath) + +def check_golden_results_for_tasks(args, configs): + num_qor_failures = 0 + + print_verbose(BASIC_VERBOSITY, args.verbosity, "Checking QoR:") + for config in configs: + num_qor_failures += check_golden_results_for_task(args, config) + + return num_qor_failures + +def check_golden_results_for_task(args, config): + """ + Copies the latest task run's parse_results.txt into the config directory as golden_results.txt + """ + num_qor_failures = 0 + run_dir = find_latest_run_dir(args, config) + + if not config.pass_requirements_file: + print_verbose(BASIC_VERBOSITY, args.verbosity, + "Warning: no pass requirements file for task {}, QoR will not be checked".format(config.task_name)) + else: + + #Load the pass requirements file + pass_req_filepath = os.path.join(find_vtr_root(), 'vtr_flow', 'parse', 'pass_requirements', config.pass_requirements_file) + pass_requirements = load_pass_requirements(pass_req_filepath) + + #Load the task's parse results + task_results_filepath = os.path.join(run_dir, "parse_results.txt") + task_results = load_parse_results(task_results_filepath) + + #Load the golden reference + golden_results_filepath = os.path.join(config.config_dir, "golden_results.txt") + golden_results = load_parse_results(golden_results_filepath) + + #Verify that the architecture and circuit are specified + for param in ["architecture", "circuit"]: + if param not in task_results.primary_keys(): + raise InspectError("Required param '{}' missing from task results: {}".format(param, task_results_filepath), task_results_filepath) + + if param not in golden_results.primary_keys(): + raise InspectError("Required param '{}' missing from golden results: {}".format(param, golden_results_filepath), golden_results_filepath) + + #Verify that all params and pass requirement metric are included in both the golden and task results + # We do not worry about non-pass_requriements elements being different or missing + for metric in pass_requirements.keys(): + for (arch, circuit), result in task_results.all_metrics().iteritems(): + if metric not in result: + #print_verbose(BASIC_VERBOSITY, args.verbosity, + #"Warning: Required metric '{}' missing from task results".format(metric), task_results_filepath) + raise InspectError("Required metric '{}' missing from task results".format(metric), task_results_filepath) + + for (arch, circuit), result in golden_results.all_metrics().iteritems(): + if metric not in result: + #print_verbose(BASIC_VERBOSITY, args.verbosity, + #"Warning: Required metric '{}' missing from golden results".format(metric), golden_results_filepath) + raise InspectError("Required metric '{}' missing from golden results".format(metric), golden_results_filepath) + + #Load the primary keys for golden and task + golden_primary_keys = [] + for (arch, circuit), metrics in golden_results.all_metrics().iteritems(): + golden_primary_keys.append((arch, circuit)) + + task_primary_keys = [] + for (arch, circuit), metrics in task_results.all_metrics().iteritems(): + task_primary_keys.append((arch, circuit)) + + #Ensure that task has all the golden cases + for arch, circuit in golden_primary_keys: + if task_results.metrics(arch, circuit) == None: + raise InspectError("Required case {}/{} missing from task results: {}".format(arch, circuit), parse_results_filepath) + + #Warn about any elements in task that are not found in golden + for arch, circuit in task_primary_keys: + if golden_results.metrics(arch, circuit) == None: + print_verbose(BASIC_VERBOSITY, args.verbosity, + "Warning: Task includes result for {}/{} missing in golden results".format(arch, circuit)) + + #Verify that task results pass each metric for all cases in golden + for (arch, circuit) in golden_primary_keys: + golden_metrics = golden_results.metrics(arch, circuit) + task_metrics = task_results.metrics(arch, circuit) + + for metric in pass_requirements.keys(): + + if not metric in golden_metrics: + print_verbose(BASIC_VERBOSITY, args.verbosity, "Warning: Metric {} missing from golden results".format(metric)) + continue + + if not metric in task_metrics: + print_verbose(BASIC_VERBOSITY, args.verbosity, "Warning: Metric {} missing from task results".format(metric)) + continue + + try: + metric_passed, reason = pass_requirements[metric].check_passed(golden_metrics[metric], task_metrics[metric]) + except InspectError as e: + metric_passed = False + reason = e.msg + + if not metric_passed: + print_verbose(BASIC_VERBOSITY, args.verbosity, " FAILED {} {} {}/{}: {} {}".format(os.path.basename(run_dir), config.task_name, arch, circuit, metric, reason)) + num_qor_failures += 1 + + if num_qor_failures == 0: + print_verbose(BASIC_VERBOSITY, args.verbosity, + " PASSED {} {}".format(os.path.basename(run_dir), config.task_name)) + + return num_qor_failures + +def create_jobs(args, configs): + jobs = [] + for config in configs: + for arch, circuit in itertools.product(config.archs, config.circuits): + abs_arch_filepath = resolve_vtr_source_file(config, arch, config.arch_dir) + abs_circuit_filepath = resolve_vtr_source_file(config, circuit, config.circuit_dir) + + executable = None + if config.script_path: + #Custom flow script + executable = [config.script_path] + else: + #Default flow script + executable = [find_vtr_file('vtr', is_executabe=True), 'flow'] + + #Collect any extra script params from the config file + script_params = [abs_arch_filepath, abs_circuit_filepath] + script_params += config.script_params + + #Apply any special config based parameters + if config.cmos_tech_behavior: + script_params += ["--power", resolve_vtr_source_file(config, config.cmos_tech_behavior, "tech")] + + if config.pad_file: + script_params += ["--fix_pins", resolve_vtr_source_file(config, config.pad_file)] + + if config.parse_file: + script_params += ["--parse_config_file", resolve_vtr_source_file(config, config.parse_file, os.path.join("parse", "parse_config"))] + + #We specify less verbosity to the sub-script + # This keeps the amount of output reasonable + script_params += ["-v", str(max(0, args.verbosity - 1))] + + cmd = executable + script_params + + work_dir = os.path.join(arch, circuit) + + jobs.append(Job(config.task_name, arch, circuit, work_dir, cmd)) + + return jobs + +def find_latest_run_dir(args, config): + task_dir = find_task_dir(args, config) + + run_dir = get_latest_run_dir(task_dir) + + if not run_dir: + raise InspectError("Failed to find run directory for task '{}' in '{}'".format(config.task_name, task_dir)) + + assert os.path.isdir(run_dir) + + return run_dir + +def find_task_dir(args, config): + task_dir = None + if args.work_dir: + task_dir = os.path.join(args.work_dir, config.task_name) + + else: + #Task dir is just above the config directory + task_dir = os.path.dirname(config.config_dir) + assert os.path.isdir(task_dir) + + return task_dir + +def run_parallel(args, configs, queued_jobs): + """ + Run each external command in commands with at most args.j commands running in parllel + """ + #Determine the run dir for each config + run_dirs = {} + for config in configs: + task_dir = find_task_dir(args, config) + task_run_dir = get_next_run_dir(task_dir) + run_dirs[config.task_name] = task_run_dir + + + #We pop off the jobs of queued_jobs, which python does from the end, + #so reverse the list now so we get the expected order. This also ensures + #we are working with a copy of the jobs + queued_jobs = list(reversed(queued_jobs)) + + #Find the max taskname length for pretty printing + max_taskname_len = 0 + for job in queued_jobs: + max_taskname_len = max(max_taskname_len, len(job.task_name())) + + #Queue of currently running subprocesses + running_procs = [] + + num_failed = 0 + try: + while len(queued_jobs) > 0 or len(running_procs) > 0: #Outstanding or running jobs + + #Launch outstanding jobs if workers are available + while len(queued_jobs) > 0 and len(running_procs) < args.j: + job = queued_jobs.pop() + + #Make the working directory + work_dir = job.work_dir(run_dirs[job.task_name()]) + mkdir_p(work_dir) + + log_filepath = os.path.join(work_dir, "vtr_flow.log") + + log_file = open(log_filepath, 'w+') + + #print "Starting {}: {}".format(job.task_name(), job.job_name()) + #print job.command() + proc = subprocess.Popen(job.command(), + cwd=work_dir, + stderr=subprocess.STDOUT, + stdout=log_file) + + running_procs.append((proc, job, log_file)) + + + while len(running_procs) > 0: + #Are any of the workers finished? + procs_to_del = set() + for i in xrange(len(running_procs)): + proc, job, log_file = running_procs[i] + + status = proc.poll() + if status is not None: + #Process has completed + if status == 0: + print_verbose(BASIC_VERBOSITY, args.verbosity, + "Finished {:<{w}}: {}".format(job.task_name(), job.job_name(), w=max_taskname_len)) + + if args.verbosity >= ALL_LOG_VERBOSITY: + print_log(log_file) + else: + #Failed + num_failed += 1 + print_verbose(BASIC_VERBOSITY, args.verbosity, + "Failed {:<{w}}: {}".format(job.task_name(), job.job_name(), w=max_taskname_len)) + + if args.verbosity >= FAILED_LOG_VERBOSITY: + print_log(log_file) + + log_file.close() + + #Record jobs to be removed from the run queue + procs_to_del.add(proc) + + if len(procs_to_del) > 0: + #Remove jobs from run queue + running_procs = [info for info in running_procs if info[0] not in procs_to_del] + + assert len(running_procs) < args.j + + #There are idle workers, allow new jobs to start + break; + + #Don't constantly poll + time.sleep(0.1) + + except KeyboardInterrupt as e: + print "Recieved KeyboardInterrupt -- halting workers" + for proc, job, log_file in running_procs: + proc.terminate() + log_file.close() + + #Remove any procs finished after terminate + running_procs = [(proc, job, log_file) for proc, job, log_file in running_procs if proc.returncode is None] + + finally: + if len(running_procs) > 0: + print "Killing {} worker processes".format(len(running_procs)) + for proc, job, log_file in running_procs: + proc.kill() + log_file.close() + + return num_failed + +def print_log(log_file, indent=" "): + #Save position + curr_pos = log_file.tell() + + log_file.seek(0) #Rewind to start + + #Print log + for line in log_file: + line = line.rstrip() + print indent + line + print "" + + #Return to original position + log_file.seek(curr_pos) + + +def resolve_vtr_source_file(config, filename, base_dir=""): + """ + Resolves an filename with a base_dir + + Checks the following in order: + 1) filename as absolute path + 2) filename under config directory + 3) base_dir as absolute path (join filename with base_dir) + 4) filename and base_dir are relative paths (join under vtr_root) + """ + + #Absolute path + if os.path.isabs(filename): + return filename + + #Under config + assert os.path.isabs(config.config_dir) + joined_path = os.path.join(config.config_dir, filename) + if os.path.exists(joined_path): + return joined_path + + #Under base dir + if os.path.isabs(base_dir): + #Absolute base + joined_path = os.path.join(base_dir, filename) + if os.path.exists(joined_path): + return joined_path + else: + #Relative base under the VTR flow directory + joined_path = os.path.join(find_vtr_root(), 'vtr_flow', base_dir, filename) + if os.path.exists(joined_path): + return joined_path + + #Not found + raise InspectError("Failed to resolve VTR source file {}".format(filename)) + +if __name__ == "__main__": + main() diff --git a/vtr_flow/scripts/vtr-test.py b/vtr_flow/scripts/vtr-test.py new file mode 100755 index 00000000000..0a0748e5f61 --- /dev/null +++ b/vtr_flow/scripts/vtr-test.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python2 +import os +import sys +import argparse +import itertools +import textwrap +import subprocess +import time +from datetime import datetime +from multiprocessing import Pool + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), 'python_libs')) + +import verilogtorouting as vtr +from verilogtorouting.error import * +from verilogtorouting.util import load_list_file, find_vtr_file, mkdir_p, print_verbose, find_vtr_root, CommandRunner, format_elapsed_time, RawDefaultHelpFormatter, VERBOSITY_CHOICES +from verilogtorouting.task import load_task_config, TaskConfig, find_task_config_file +from verilogtorouting.flow import CommandRunner + +BASIC_VERBOSITY=1 + +def vtr_command_argparser(prog=None): + description = textwrap.dedent( + """ + Runs one or more VTR regression tests. + """ + ) + epilog = textwrap.dedent( + """ + Examples + -------- + + Run the regression test 'vtr_strong': + + %(prog)s vtr_strong + + Run the regression tests 'vtr_basic' and 'vtr_strong': + + %(prog)s vtr_basic vtr_strong + + Run the regression tests 'vtr_basic' and 'vtr_strong' with 8 parallel workers: + + %(prog)s vtr_basic vtr_strong -j8 + """ + ) + + parser = argparse.ArgumentParser( + prog=prog, + description=description, + epilog=epilog, + formatter_class=RawDefaultHelpFormatter, + ) + + # + # Major arguments + # + parser.add_argument('reg_test', + nargs="+", + choices=["vtr_basic", "vtr_strong", "vtr_nightly", "vtr_weekly", "odin_micro", "odin_full"], + help="Regression tests to be run") + + parser.add_argument("--create_golden", + default=False, + action="store_true", + help="Create golden reference results for the associated tasks") + + parser.add_argument('-j', + default=1, + type=int, + metavar="NUM_PROC", + help="How many processors to use for execution.") + + parser.add_argument("-v", "--verbosity", + choices=VERBOSITY_CHOICES + [5], + default=2, + type=int, + help="Sets the verbosity of the script. Higher values produce more output.") + + parser.add_argument("--work_dir", + default=None, + help="Directory to store intermediate and result files." + "If None, set to the relevante directory under $VTR_ROOT/vtr_flow/tasks.") + + parser.add_argument("--debug", + default=False, + action="store_true", + help="Produce additional debug output") + + return parser + +def main(): + vtr_command_main(sys.argv[1:]) + +def vtr_command_main(arg_list, prog=None): + start = datetime.now() + + #Load the arguments + args = vtr_command_argparser(prog).parse_args(arg_list) + + print_verbose(BASIC_VERBOSITY, args.verbosity, "# {} {}\n".format(prog, ' '.join(arg_list))) + + num_func_failures = 0 + num_qor_failures = 0 + + try: + if args.create_golden: + #Create golden results + num_qor_failures = 0 + create_tasks_golden_result(args, vtr_task_list_files) + else: + #Run any ODIN tests + for reg_test in args.reg_test: + if reg_test.startswith("odin"): + num_func_failures += run_odin_test(args, reg_test) + + #Collect the task lists + vtr_task_list_files = [] + for reg_test in args.reg_test: + if reg_test.startswith("vtr"): + + base_testname = reg_test.split('_')[-1] + task_list_filepath = os.path.join(find_vtr_root(), + 'vtr_flow', 'tasks', 'regression_tests', + 'vtr_reg_' + base_testname, 'task_list.txt') + vtr_task_list_files.append(task_list_filepath) + + #Run the actual tasks, recording functionality failures + num_func_failures += run_tasks(args, vtr_task_list_files) + + #Check against golden results + num_qor_failures += check_tasks_qor(args, vtr_task_list_files) + + + #Final summary + print_verbose(BASIC_VERBOSITY, args.verbosity, "") + if num_func_failures == 0 and num_qor_failures == 0: + print_verbose(BASIC_VERBOSITY, args.verbosity, + "PASSED All Test(s)") + elif num_func_failures != 0 or num_qor_failures != 0: + print_verbose(BASIC_VERBOSITY, args.verbosity, + "FAILED {} functionality and {} QoR tests".format(num_func_failures, num_qor_failures)) + + sys.exit(num_func_failures + num_qor_failures) + finally: + print_verbose(BASIC_VERBOSITY, args.verbosity, "\n# {} took {} (exiting {})".format(prog, format_elapsed_time(datetime.now() - start), num_func_failures + num_qor_failures)) + +def run_odin_test(args, test_name): + odin_reg_script = None + if test_name == "odin_micro": + odin_reg_script = find_vtr_file("verify_microbenchmarks.sh", is_executabe=True) + else: + assert test_name == "odin_full" + odin_reg_script = find_vtr_file("verify_regression_tests.sh", is_executabe=True) + + assert odin_reg_script + + odin_root = os.path.dirname(odin_reg_script) + + result = subprocess.call(odin_reg_script, cwd=odin_root) + + assert result != None + if result != 0: + #Error + print_verbose(BASIC_VERBOSITY, args.verbosity, "FAILED test '{}'".format(test_name)) + return 1 + + #Pass + print_verbose(BASIC_VERBOSITY, args.verbosity, "PASSED test '{}'".format(test_name)) + return 0 + +def run_tasks(args, task_lists): + #Call 'vtr task' + vtr_task_cmd = ['vtr', 'task'] + vtr_task_cmd += ['-l'] + task_lists + vtr_task_cmd += ['-j', str(args.j), + '-v', str(max(0, args.verbosity - 1)), + '--print_metadata', str(args.debug) + ] + if args.work_dir: + vtr_task_cmd += ["--work_dir", args.workdir] + + #Exit code is number of failures + return subprocess.call(vtr_task_cmd) + +def check_tasks_qor(args, task_lists): + num_qor_failures = 0 + vtr_task_cmd = ['vtr', 'task'] + vtr_task_cmd += ['-l'] + task_lists + vtr_task_cmd += ['-v', str(max(0, args.verbosity - 1)), + '--check_golden', + '--print_metadata', str(args.debug) + ] + if args.work_dir: + vtr_task_cmd += ["--work_dir", args.workdir] + + #Exit code is number of failures + return subprocess.call(vtr_task_cmd) + +def create_tasks_golden_result(args, task_lists): + vtr_task_cmd = ['vtr', 'task'] + vtr_task_cmd += ['-l'] + task_lists + vtr_task_cmd += ['-v', str(max(0, args.verbosity - 1)), + '--create_golden', + '--print_metadata', str(args.debug) + ] + if args.work_dir: + vtr_task_cmd += ["--work_dir", args.workdir] + + #Exit code is number of failures + return subprocess.call(vtr_task_cmd) + +if __name__ == "__main__": + main() From d5b6f90d19b9e00a8f594abc78fefa35a2631c21 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Mon, 4 May 2020 14:57:09 -0700 Subject: [PATCH 02/86] Fixing more instances of misspelt is_executable --- vtr_flow/scripts/python_libs/verilogtorouting/flow.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/vtr_flow/scripts/python_libs/verilogtorouting/flow.py b/vtr_flow/scripts/python_libs/verilogtorouting/flow.py index daf1b18c2bb..513109c0d54 100644 --- a/vtr_flow/scripts/python_libs/verilogtorouting/flow.py +++ b/vtr_flow/scripts/python_libs/verilogtorouting/flow.py @@ -239,7 +239,7 @@ def run_odin(architecture_file, circuit_file, mkdir_p(work_dir) if odin_exec == None: - odin_exec = find_vtr_file('odin_II', is_executabe=True) + odin_exec = find_vtr_file('odin_II', is_executable=True) if odin_config == None: odin_base_config = find_vtr_file('basic_odin_config_split.xml') @@ -267,7 +267,7 @@ def run_abc(architecture_file, circuit_file, output_netlist, command_runner, wor mkdir_p(work_dir) if abc_exec == None: - abc_exec = find_vtr_file('abc', is_executabe=True) + abc_exec = find_vtr_file('abc', is_executable=True) if abc_rc == None: abc_dir = os.path.dirname(abc_exec) @@ -296,7 +296,7 @@ def run_abc(architecture_file, circuit_file, output_netlist, command_runner, wor def run_ace(circuit_file, output_netlist, output_activity_file, command_runner, work_dir=".", log_filename="ace.out", ace_exec=None): if ace_exec is None: - ace_exec = find_vtr_file('ace', is_executabe=True) + ace_exec = find_vtr_file('ace', is_executable=True) cmd = [ace_exec, "-b", circuit_file, @@ -371,7 +371,7 @@ def run_vpr(architecture, circuit, command_runner, work_dir, output_netlist=None mkdir_p(work_dir) if vpr_exec == None: - vpr_exec = find_vtr_file('vpr', is_executabe=True) + vpr_exec = find_vtr_file('vpr', is_executable=True) cmd = [vpr_exec, architecture, circuit] @@ -397,7 +397,7 @@ def run_abc_lec(reference_netlist, implementation_netlist, command_runner, work_ mkdir_p(work_dir) if abc_exec == None: - abc_exec = find_vtr_file('abc', is_executabe=True) + abc_exec = find_vtr_file('abc', is_executable=True) abc_script = ['cec {ref} {imp}'.format(ref=reference_netlist, imp=implementation_netlist), 'sec {ref} {imp}'.format(ref=reference_netlist, imp=implementation_netlist), From 9d8f9d8123571b678064383ee831fd1cc9ccfde8 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Thu, 7 May 2020 14:48:31 -0700 Subject: [PATCH 03/86] vtr-flow.py: switch the old --work_dir to the new -temp_dir as used on master. flow.py: updated the commands to ODIN, ABC and VPR to match that of master. --- .../python_libs/verilogtorouting/flow.py | 107 +++++++++++------- .../python_libs/verilogtorouting/util.py | 10 +- vtr_flow/scripts/vtr-flow.py | 13 ++- 3 files changed, 77 insertions(+), 53 deletions(-) diff --git a/vtr_flow/scripts/python_libs/verilogtorouting/flow.py b/vtr_flow/scripts/python_libs/verilogtorouting/flow.py index 513109c0d54..4b58055be32 100644 --- a/vtr_flow/scripts/python_libs/verilogtorouting/flow.py +++ b/vtr_flow/scripts/python_libs/verilogtorouting/flow.py @@ -18,7 +18,7 @@ def run_vtr_flow(architecture_file, circuit_file, start_stage=VTR_STAGE.odin, end_stage=VTR_STAGE.vpr, command_runner=CommandRunner(), parse_config_file=None, - work_dir=".", + temp_dir="./temp", verbosity=0, vpr_args=None): """ @@ -31,7 +31,7 @@ def run_vtr_flow(architecture_file, circuit_file, power_tech_file : Technology power file. Enables power analysis and runs ace - work_dir : Directory to run in (created if non-existant) + temp_dir : Directory to run in (created if non-existant) start_stage : Stage of the flow to start at end_stage : Stage of the flow to finish at command_runner : A CommandRunner object used to run system commands @@ -51,7 +51,7 @@ def run_vtr_flow(architecture_file, circuit_file, circuit_name, circuit_ext = os.path.splitext(circuit_file_basename) architecture_name, architecture_ext = os.path.splitext(architecture_file_basename) - mkdir_p(work_dir) + mkdir_p(temp_dir) #Define useful filenames post_odin_netlist = circuit_name + '.odin.blif' @@ -68,8 +68,8 @@ def run_vtr_flow(architecture_file, circuit_file, lec_base_netlist = circuit_file_basename #Copy the circuit and architecture - shutil.copy(circuit_file, os.path.join(work_dir, circuit_file_basename)) - shutil.copy(architecture_file, os.path.join(work_dir, architecture_file_basename)) + shutil.copy(circuit_file, os.path.join(temp_dir, circuit_file_basename)) + shutil.copy(architecture_file, os.path.join(temp_dir, architecture_file_basename)) #There are multiple potential paths for the netlist to reach a tool @@ -87,7 +87,7 @@ def run_vtr_flow(architecture_file, circuit_file, run_odin(architecture_file_basename, next_stage_netlist, output_netlist=post_odin_netlist, command_runner=command_runner, - work_dir=work_dir) + temp_dir=temp_dir) next_stage_netlist = post_odin_netlist @@ -103,7 +103,7 @@ def run_vtr_flow(architecture_file, circuit_file, run_abc(architecture_file_basename, next_stage_netlist, output_netlist=post_abc_netlist, command_runner=command_runner, - work_dir=work_dir) + temp_dir=temp_dir) next_stage_netlist = post_abc_netlist @@ -123,7 +123,7 @@ def run_vtr_flow(architecture_file, circuit_file, run_ace(next_stage_netlist, output_netlist=post_ace_netlist, output_activity_file=post_ace_activity_file, command_runner=command_runner, - work_dir=work_dir) + temp_dir=temp_dir) #Use ACE's output netlist next_stage_netlist = post_ace_netlist @@ -141,7 +141,7 @@ def run_vtr_flow(architecture_file, circuit_file, # if should_run_stage(VTR_STAGE.vpr, start_stage, end_stage): #Copy the input netlist for input to vpr - shutil.copyfile(os.path.join(work_dir, next_stage_netlist), os.path.join(work_dir, pre_vpr_netlist)) + shutil.copyfile(os.path.join(temp_dir, next_stage_netlist), os.path.join(temp_dir, pre_vpr_netlist)) #Do we need to generate the post-synthesis netlist? (e.g. for LEC) if should_run_stage(VTR_STAGE.lec, start_stage, end_stage): @@ -151,17 +151,17 @@ def run_vtr_flow(architecture_file, circuit_file, if "route_chan_width" in vpr_args: #The User specified a fixed channel width print_verbose(1, verbosity, "Running VPR (at fixed channel width)") - run_vpr(architecture_file_basename, pre_vpr_netlist, + run_vpr(architecture_file_basename, circuit_name, pre_vpr_netlist, output_netlist=post_vpr_netlist, command_runner=command_runner, - work_dir=work_dir, + temp_dir=temp_dir, vpr_args=vpr_args) else: #First find minW and then re-route at a relaxed W - run_vpr_relax_W(architecture_file_basename, pre_vpr_netlist, + run_vpr_relax_W(architecture_file_basename, circuit_name, pre_vpr_netlist, output_netlist=post_vpr_netlist, command_runner=command_runner, - work_dir=work_dir, + temp_dir=temp_dir, verbosity=verbosity, vpr_args=vpr_args) @@ -175,11 +175,11 @@ def run_vtr_flow(architecture_file, circuit_file, print_verbose(1, verbosity, "Running ABC Logical Equivalence Check") run_abc_lec(lec_base_netlist, post_vpr_netlist, command_runner=command_runner, log_filename="abc.lec.out") -def parse_vtr_flow(work_dir, parse_config_file=None, metrics_filepath=None, verbosity=1): +def parse_vtr_flow(temp_dir, parse_config_file=None, metrics_filepath=None, verbosity=1): print_verbose(1, verbosity, "Parsing results") if parse_config_file is None: - parse_config_file = find_vtr_file("vtr_benchmarks.min_chan_width.txt") + parse_config_file = find_vtr_file("vtr_benchmarks.txt") parse_patterns = load_parse_patterns(parse_config_file) @@ -197,7 +197,7 @@ def parse_vtr_flow(work_dir, parse_config_file=None, metrics_filepath=None, verb for parse_pattern in parse_patterns.values(): #We interpret the parse pattern's filename as a glob pattern - filepattern = os.path.join(work_dir, parse_pattern.filename()) + filepattern = os.path.join(temp_dir, parse_pattern.filename()) filepaths = glob.glob(filepattern) num_files = len(filepaths) @@ -221,7 +221,7 @@ def parse_vtr_flow(work_dir, parse_config_file=None, metrics_filepath=None, verb assert num_files == 0 if metrics_filepath is None: - metrics_filepath = os.path.join(work_dir, "parse_results.txt") + metrics_filepath = os.path.join(temp_dir, "parse_results.txt") write_tab_delimitted_csv(metrics_filepath, [metrics]) @@ -230,13 +230,13 @@ def parse_vtr_flow(work_dir, parse_config_file=None, metrics_filepath=None, verb def run_odin(architecture_file, circuit_file, output_netlist, command_runner, - work_dir=".", + temp_dir=".", log_filename="odin.out", odin_exec=None, odin_config=None, min_hard_mult_size=3, min_hard_adder_size=1): - mkdir_p(work_dir) + mkdir_p(temp_dir) if odin_exec == None: odin_exec = find_vtr_file('odin_II', is_executable=True) @@ -246,7 +246,7 @@ def run_odin(architecture_file, circuit_file, #Copy the config file odin_config = "odin_config.xml" - odin_config_full_path = os.path.abspath(os.path.join(work_dir, odin_config)) + odin_config_full_path = os.path.abspath(os.path.join(temp_dir, odin_config)) shutil.copyfile(odin_base_config, odin_config_full_path) #Update the config file @@ -254,17 +254,17 @@ def run_odin(architecture_file, circuit_file, "XXX": circuit_file, "YYY": architecture_file, "ZZZ": output_netlist, - "PPP": determine_memory_addr_width(os.path.join(work_dir, architecture_file)), + "PPP": determine_memory_addr_width(os.path.join(temp_dir, architecture_file)), "MMM": min_hard_mult_size, "AAA": min_hard_adder_size, }) - cmd = [odin_exec, "-c", odin_config] + cmd = [odin_exec, "-c", odin_config, "--adder_type", "default", "-U0"] - command_runner.run_system_command(cmd, work_dir=work_dir, log_filename=log_filename, indent_depth=1) + command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) -def run_abc(architecture_file, circuit_file, output_netlist, command_runner, work_dir=".", log_filename="abc.opt_techmap.out", abc_exec=None, abc_script=None, abc_rc=None): - mkdir_p(work_dir) +def run_abc(architecture_file, circuit_file, output_netlist, command_runner, temp_dir=".", log_filename="abc.opt_techmap.out", abc_exec=None, abc_script=None, abc_rc=None,use_old_abc_script = False): + mkdir_p(temp_dir) if abc_exec == None: abc_exec = find_vtr_file('abc', is_executable=True) @@ -273,12 +273,32 @@ def run_abc(architecture_file, circuit_file, output_netlist, command_runner, wor abc_dir = os.path.dirname(abc_exec) abc_rc = os.path.join(abc_dir, 'abc.rc') - shutil.copyfile(abc_rc, os.path.join(work_dir, 'abc.rc')) + shutil.copyfile(abc_rc, os.path.join(temp_dir, 'abc.rc')) - lut_size = determine_lut_size(os.path.join(work_dir, architecture_file)) + lut_size = determine_lut_size(os.path.join(temp_dir, architecture_file)) if abc_script == None: abc_script = ['read {input_netlist}'.format(input_netlist=circuit_file), + 'time', + 'print_stats', + 'print_latch', + 'time', + 'print_lut', + 'time', + 'strash', + 'ifraig -v', + 'scorr -v', + 'dc2 -v', + 'dch -f', + 'if -K {lut_size} -v'.format(lut_size=lut_size), + 'mfs2 -v', + 'print_stats', + 'time', + 'write_hie {input_netlist} {output_netlist}'.format(input_netlist=circuit_file, output_netlist=output_netlist), + 'time;'] + + if(use_old_abc_script): + abc_script = ['read {input_netlist}'.format(input_netlist=circuit_file), 'time', 'resyn', 'resyn2', @@ -287,13 +307,14 @@ def run_abc(architecture_file, circuit_file, output_netlist, command_runner, wor 'scleanup', 'write_hie {input_netlist} {output_netlist}'.format(input_netlist=circuit_file, output_netlist=output_netlist), 'print_stats'] + abc_script = "; ".join(abc_script) cmd = [abc_exec, '-c', abc_script] - command_runner.run_system_command(cmd, work_dir=work_dir, log_filename=log_filename, indent_depth=1) + command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) -def run_ace(circuit_file, output_netlist, output_activity_file, command_runner, work_dir=".", log_filename="ace.out", ace_exec=None): +def run_ace(circuit_file, output_netlist, output_activity_file, command_runner, temp_dir=".", log_filename="ace.out", ace_exec=None): if ace_exec is None: ace_exec = find_vtr_file('ace', is_executable=True) @@ -303,9 +324,9 @@ def run_ace(circuit_file, output_netlist, output_activity_file, command_runner, "-n", output_netlist, "-o", output_activity_file] - command_runner.run_system_command(cmd, work_dir=work_dir, log_filename=log_filename, indent_depth=1) + command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) -def run_vpr_relax_W(architecture, circuit, command_runner=CommandRunner(), work_dir=".", +def run_vpr_relax_W(architecture, circuit_name, circuit, command_runner=CommandRunner(), temp_dir=".", relax_W_factor=1.3, vpr_exec=None, verbosity=1, logfile_base="vpr", vpr_args=None, output_netlist=None): """ @@ -318,7 +339,7 @@ def run_vpr_relax_W(architecture, circuit, command_runner=CommandRunner(), work_ architecture: Architecture file circuit: Input circuit netlist command_runner: CommandRunner object - work_dir: Directory to run in + temp_dir: Directory to run in relax_W_factor: Factor by which to relax minimum channel width for critical path delay routing verbosity: How much progress output to produce @@ -330,21 +351,21 @@ def run_vpr_relax_W(architecture, circuit, command_runner=CommandRunner(), work_ if vpr_args is None: vpr_args = OrderedDict() - mkdir_p(work_dir) + mkdir_p(temp_dir) vpr_min_W_log = '.'.join([logfile_base, "min_W", "out"]) vpr_relaxed_W_log = '.'.join([logfile_base, "relaxed_W", "out"]) print_verbose(1, verbosity, "Running VPR (determining minimum channel width)" ) - run_vpr(architecture, circuit, command_runner, work_dir, log_filename=vpr_min_W_log, vpr_exec=vpr_exec, vpr_args=vpr_args) + run_vpr(architecture, circuit_name, circuit, command_runner, temp_dir, log_filename=vpr_min_W_log, vpr_exec=vpr_exec, vpr_args=vpr_args) if ('pack' in vpr_args or 'place' in vpr_args) and 'route' not in vpr_args: #Don't look for min W if routing was not run return - min_W = determine_min_W(os.path.join(work_dir, vpr_min_W_log)) + min_W = determine_min_W(os.path.join(temp_dir, vpr_min_W_log)) relaxed_W = relax_W(min_W, relax_W_factor) @@ -358,22 +379,22 @@ def run_vpr_relax_W(architecture, circuit, command_runner=CommandRunner(), work_ if 'fix_pins' in vpr_args: del vpr_args['fix_pins'] - run_vpr(architecture, circuit, command_runner, work_dir, log_filename=vpr_relaxed_W_log, vpr_exec=vpr_exec, vpr_args=vpr_args) + run_vpr(architecture, circuit_name, circuit, command_runner, temp_dir, log_filename=vpr_relaxed_W_log, vpr_exec=vpr_exec, vpr_args=vpr_args) -def run_vpr(architecture, circuit, command_runner, work_dir, output_netlist=None, log_filename="vpr.out", vpr_exec=None, vpr_args=None): +def run_vpr(architecture, circuit_name, circuit, command_runner, temp_dir, output_netlist=None, log_filename="vpr.out", vpr_exec=None, vpr_args=None): """ Runs VPR with the specified configuration """ if vpr_args is None: vpr_args = OrderedDict() - mkdir_p(work_dir) + mkdir_p(temp_dir) if vpr_exec == None: vpr_exec = find_vtr_file('vpr', is_executable=True) - cmd = [vpr_exec, architecture, circuit] + cmd = [vpr_exec, architecture, circuit_name, "--circuit_file", circuit, "--route", "--route_chan_width", "76", "--max_router_iterations", "150"] #this needs updated with actual width and iteration numbers #Enable netlist generation #if output_netlist: @@ -388,13 +409,13 @@ def run_vpr(architecture, circuit, command_runner, work_dir, output_netlist=None else: cmd += ["--" + arg, str(value)] - command_runner.run_system_command(cmd, work_dir=work_dir, log_filename=log_filename, indent_depth=1) + command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) -def run_abc_lec(reference_netlist, implementation_netlist, command_runner, work_dir=".", log_filename="abc.lec.out", abc_exec=None): +def run_abc_lec(reference_netlist, implementation_netlist, command_runner, temp_dir=".", log_filename="abc.lec.out", abc_exec=None): """ Run Logical Equivalence Checking (LEC) between two netlists using ABC """ - mkdir_p(work_dir) + mkdir_p(temp_dir) if abc_exec == None: abc_exec = find_vtr_file('abc', is_executable=True) @@ -406,7 +427,7 @@ def run_abc_lec(reference_netlist, implementation_netlist, command_runner, work_ cmd = [abc_exec, '-c', abc_script] - output, returncode = command_runner.run_system_command(cmd, work_dir=work_dir, log_filename=log_filename, indent_depth=1) + output, returncode = command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) #Check if ABC's LEC engine passed lec_passed = check_abc_lec_status(output) diff --git a/vtr_flow/scripts/python_libs/verilogtorouting/util.py b/vtr_flow/scripts/python_libs/verilogtorouting/util.py index fa9da670af1..f2f7d395f63 100644 --- a/vtr_flow/scripts/python_libs/verilogtorouting/util.py +++ b/vtr_flow/scripts/python_libs/verilogtorouting/util.py @@ -43,7 +43,7 @@ def __init__(self, timeout_sec=None, max_memory_mb=None, track_memory=True, verb self._echo_cmd = echo_cmd self._indent = indent - def run_system_command(self, cmd, work_dir, log_filename=None, expected_return_code=0, indent_depth=0): + def run_system_command(self, cmd, temp_dir, log_filename=None, expected_return_code=0, indent_depth=0): """ Runs the specified command in the system shell. @@ -55,7 +55,7 @@ def run_system_command(self, cmd, work_dir, log_filename=None, expected_return_c ========= cmd: list of tokens that form the command to be run log_filename: name of the log file for the command's output. Default: derived from command - work_dir: The directory to run the command in. Default: None (uses object default). + temp_dir: The directory to run the command in. Default: None (uses object default). expected_return_code: The expected return code from the command. If the actula return code does not match, will generate an exception. Default: 0 indent_depth: How deep to indent the tool output in verbose mode. Default 0 """ @@ -98,7 +98,7 @@ def run_system_command(self, cmd, work_dir, log_filename=None, expected_return_c stdout=subprocess.PIPE, #We grab stdout stderr=subprocess.STDOUT, #stderr redirected to stderr universal_newlines=True, #Lines always end in \n - cwd=work_dir, #Where to run the command + cwd=temp_dir, #Where to run the command ) # Read the output line-by-line and log it @@ -106,7 +106,7 @@ def run_system_command(self, cmd, work_dir, log_filename=None, expected_return_c # # We do this rather than use proc.communicate() # to get interactive output - with open(os.path.join(work_dir, log_filename), 'w') as log_f: + with open(os.path.join(temp_dir, log_filename), 'w') as log_f: #Print the command at the top of the log print >> log_f, " ".join(cmd) @@ -148,7 +148,7 @@ def run_system_command(self, cmd, work_dir, log_filename=None, expected_return_c if cmd_errored: raise CommandError(msg="Executable {exec_name} failed".format(exec_name=os.path.basename(orig_cmd[0])), cmd=cmd, - log=os.path.join(work_dir, log_filename), + log=os.path.join(temp_dir, log_filename), returncode=cmd_returncode) return cmd_output, cmd_returncode diff --git a/vtr_flow/scripts/vtr-flow.py b/vtr_flow/scripts/vtr-flow.py index e1451139f6b..d83a28b37d3 100755 --- a/vtr_flow/scripts/vtr-flow.py +++ b/vtr_flow/scripts/vtr-flow.py @@ -78,7 +78,7 @@ def vtr_command_argparser(prog=None): Run in the directory 'test' (rather than in a temporary directory): - %(prog)s arch.xml circuit.v --work_dir test + %(prog)s arch.xml circuit.v -temp_dir test Enable power analysis: @@ -152,7 +152,7 @@ def vtr_command_argparser(prog=None): # house_keeping = parser.add_argument_group("House Keeping", description="Configuration related to how files/time/memory are used by the script.") - house_keeping.add_argument("--work_dir", + house_keeping.add_argument("-temp_dir", default=".", help="Directory to run the flow in (will be created if non-existant).") @@ -189,7 +189,10 @@ def vtr_command_main(arg_list, prog=None): abs_path_arch_file = os.path.abspath(args.architecture_file) abs_path_circuit_file = os.path.abspath(args.circuit_file) - abs_work_dir = os.path.abspath(args.work_dir) + if (args.temp_dir == "."): + temp_dir="./temp" + else: + temp_dir=args.temp_dir #Specify how command should be run @@ -209,7 +212,7 @@ def vtr_command_main(arg_list, prog=None): run_vtr_flow(abs_path_arch_file, abs_path_circuit_file, power_tech_file=args.power_tech, - work_dir=abs_work_dir, + temp_dir=temp_dir, start_stage=args.start, end_stage=args.end, command_runner=command_runner, @@ -240,7 +243,7 @@ def vtr_command_main(arg_list, prog=None): #Parse the flow results try: - parse_vtr_flow(abs_work_dir, args.parse_config_file, verbosity=args.verbosity) + parse_vtr_flow(temp_dir, args.parse_config_file, verbosity=args.verbosity) except InspectError as e: print "Error: {msg}".format(msg=e.msg) print "\tfile : ", e.filename From 430ac0dac7a3bc4c2be1b8371660184b4cc1c507 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Tue, 12 May 2020 08:30:27 -0700 Subject: [PATCH 04/86] enabled blackbox iterative flow for multi-clock circuits --- .../python_libs/verilogtorouting/flow.py | 143 ++++++++++++------ vtr_flow/scripts/vtr-flow.py | 46 +++++- 2 files changed, 142 insertions(+), 47 deletions(-) diff --git a/vtr_flow/scripts/python_libs/verilogtorouting/flow.py b/vtr_flow/scripts/python_libs/verilogtorouting/flow.py index 4b58055be32..e941550f48e 100644 --- a/vtr_flow/scripts/python_libs/verilogtorouting/flow.py +++ b/vtr_flow/scripts/python_libs/verilogtorouting/flow.py @@ -4,6 +4,7 @@ import subprocess import time import glob +import pathlib #replace os from above from collections import OrderedDict from util import make_enum, print_verbose, mkdir_p, find_vtr_file, file_replace, relax_W, CommandRunner, write_tab_delimitted_csv @@ -20,7 +21,9 @@ def run_vtr_flow(architecture_file, circuit_file, parse_config_file=None, temp_dir="./temp", verbosity=0, - vpr_args=None): + vpr_args=None, + flow_type=2, + use_new_latches_restoration_script=1): """ Runs the VTR CAD flow to map the specificied circuit_file onto the target architecture_file @@ -99,11 +102,12 @@ def run_vtr_flow(architecture_file, circuit_file, # if should_run_stage(VTR_STAGE.abc, start_stage, end_stage): print_verbose(1, verbosity, "Running ABC") - run_abc(architecture_file_basename, next_stage_netlist, output_netlist=post_abc_netlist, command_runner=command_runner, - temp_dir=temp_dir) + temp_dir=temp_dir, + flow_type=flow_type, + use_new_latches_restoration_script=use_new_latches_restoration_script) next_stage_netlist = post_abc_netlist @@ -263,57 +267,106 @@ def run_odin(architecture_file, circuit_file, command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) -def run_abc(architecture_file, circuit_file, output_netlist, command_runner, temp_dir=".", log_filename="abc.opt_techmap.out", abc_exec=None, abc_script=None, abc_rc=None,use_old_abc_script = False): +def populate_clock_list(circuit_file,blackbox_latches_script,clk_list,command_runner,temp_dir,log_filename): + clk_list_filename = "report_clk.out" + cmd = [blackbox_latches_script, "--input", circuit_file,"--output_list", clk_list_filename] + command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) + clk_list_path=pathlib.Path(temp_dir) / clk_list_filename + i=0 + with clk_list_path.open('r') as f: + clk_list.append(f.readline().strip('\n')) + i=i+1 + +def run_abc(architecture_file, circuit_file, output_netlist, command_runner, temp_dir=".", log_filename="abc.opt_techmap.out", abc_exec=None, abc_script=None, abc_rc=None,use_old_abc_script = False,flow_type=2,use_new_latches_restoration_script=1): mkdir_p(temp_dir) + blackbox_latches_script = find_vtr_file("blackbox_latches.pl") + clk_list = [] + clk_log_file = "report_clk_out.out" + if(flow_type): + populate_clock_list(circuit_file,blackbox_latches_script,clk_list,command_runner,temp_dir,log_filename) if abc_exec == None: abc_exec = find_vtr_file('abc', is_executable=True) - + if abc_rc == None: abc_dir = os.path.dirname(abc_exec) abc_rc = os.path.join(abc_dir, 'abc.rc') shutil.copyfile(abc_rc, os.path.join(temp_dir, 'abc.rc')) - lut_size = determine_lut_size(os.path.join(temp_dir, architecture_file)) - - if abc_script == None: - abc_script = ['read {input_netlist}'.format(input_netlist=circuit_file), - 'time', - 'print_stats', - 'print_latch', - 'time', - 'print_lut', - 'time', - 'strash', - 'ifraig -v', - 'scorr -v', - 'dc2 -v', - 'dch -f', - 'if -K {lut_size} -v'.format(lut_size=lut_size), - 'mfs2 -v', - 'print_stats', - 'time', - 'write_hie {input_netlist} {output_netlist}'.format(input_netlist=circuit_file, output_netlist=output_netlist), - 'time;'] - - if(use_old_abc_script): - abc_script = ['read {input_netlist}'.format(input_netlist=circuit_file), - 'time', - 'resyn', - 'resyn2', - 'if -K {lut_size}'.format(lut_size=lut_size), - 'time', - 'scleanup', - 'write_hie {input_netlist} {output_netlist}'.format(input_netlist=circuit_file, output_netlist=output_netlist), - 'print_stats'] - - abc_script = "; ".join(abc_script) - - cmd = [abc_exec, '-c', abc_script] - - command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) - + iterations=len(clk_list) + + if(iterations==0 or flow_type != 2): + iterations=1 + + for i in range(0, iterations): + pre_abc_blif= str(i)+"_" + circuit_file + post_abc_blif = str(i)+"_"+output_netlist + post_abc_raw_blif = str(i)+"_raw_"+output_netlist + if(flow_type==3): + cmd = [blackbox_latches_script, "--input", circuit_file,"--output",pre_abc_blif] + command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=str(i)+"_blackboxing_latch.out", indent_depth=1) + + elif(len(clk_list)>i): + cmd = [blackbox_latches_script,"--clk_list", clk_list[i], "--input", circuit_file,"--output",pre_abc_blif] + command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=str(i)+"_blackboxing_latch.out", indent_depth=1) + else: + pre_abc_blif = circuit_file + + if abc_script == None: + + abc_script = ['read {pre_abc_blif}'.format(pre_abc_blif=pre_abc_blif), + 'time', + 'print_stats', + 'print_latch', + 'time', + 'print_lut', + 'time', + 'strash', + 'ifraig -v', + 'scorr -v', + 'dc2 -v', + 'dch -f', + 'if -K {lut_size} -v'.format(lut_size=lut_size), + 'mfs2 -v', + 'print_stats', + 'time', + 'write_hie {pre_abc_blif} {post_abc_raw_blif}'.format(pre_abc_blif=pre_abc_blif, post_abc_raw_blif=post_abc_raw_blif), + 'time;'] + + if(use_old_abc_script): + abc_script = ['read {pre_abc_blif}'.format(pre_abc_blif=pre_abc_blif), + 'time', + 'resyn', + 'resyn2', + 'if -K {lut_size}'.format(lut_size=lut_size), + 'time', + 'scleanup', + 'write_hie {pre_abc_blif} {post_abc_raw_blif}'.format(pre_abc_blif=pre_abc_blif, post_abc_raw_blif=post_abc_raw_blif), + 'print_stats'] + + abc_script = "; ".join(abc_script) + + cmd = [abc_exec, '-c', abc_script] + + command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) + + if(flow_type != 3 and len(clk_list)>i): + cmd = [blackbox_latches_script,"--restore", clk_list[i], "--input", post_abc_raw_blif,"--output",post_abc_blif] + command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename="restore_latch"+str(i)+".out", indent_depth=1) + else: + if(use_new_latches_restoration_script): + restore_multiclock_info_script = find_vtr_file("restore_multiclock_latch.pl") + else: + restore_multiclock_info_script = find_vtr_file("restore_multiclock_latch_information.pl") + cmd = [restore_multiclock_info_script, pre_abc_blif, post_abc_raw_blif ,post_abc_blif] + command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename="restore_latch"+str(i)+".out", indent_depth=1) + if(flow_type != 2): + break + + cmd = [blackbox_latches_script, "--input", post_abc_blif,"--output",output_netlist,"--vanilla"] + command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename="restore_latch"+str(i)+".out", indent_depth=1) + def run_ace(circuit_file, output_netlist, output_activity_file, command_runner, temp_dir=".", log_filename="ace.out", ace_exec=None): if ace_exec is None: @@ -394,7 +447,7 @@ def run_vpr(architecture, circuit_name, circuit, command_runner, temp_dir, outpu if vpr_exec == None: vpr_exec = find_vtr_file('vpr', is_executable=True) - cmd = [vpr_exec, architecture, circuit_name, "--circuit_file", circuit, "--route", "--route_chan_width", "76", "--max_router_iterations", "150"] #this needs updated with actual width and iteration numbers + cmd = [vpr_exec, architecture, circuit_name, "--circuit_file", circuit] #Enable netlist generation #if output_netlist: diff --git a/vtr_flow/scripts/vtr-flow.py b/vtr_flow/scripts/vtr-flow.py index d83a28b37d3..20501c54cea 100755 --- a/vtr_flow/scripts/vtr-flow.py +++ b/vtr_flow/scripts/vtr-flow.py @@ -173,7 +173,30 @@ def vtr_command_argparser(prog=None): type=float, metavar="TIMEOUT_SECONDS", help="Maximum time in seconds to spend on a single stage.") - + # + # Iterative black-boxing flow arguments + # + iterative = parser.add_argument_group("Iterative", description="Iterative black-boxing flow for multi-clock circuits options") + iterative.add_argument("-iterative_bb", + default=False, + action="store_true", + dest="iterative_bb", + help="Use iterative black-boxing flow for multi-clock circuits") + iterative.add_argument("-once_bb", + default=False, + action="store_true", + dest="once_bb", + help="Use the black-boxing flow a single time") + iterative.add_argument("-blanket_bb", + default=False, + action="store_true", + dest="blanket_bb", + help="Use iterative black-boxing flow with out clocks") #not sure if this is a correct statement. + iterative.add_argument("-use_new_latches_restoration_script", + default=False, + action="store_true", + dest="use_new_latches_restoration_script", + help="Use the new latches restoration script") return parser def main(): @@ -203,6 +226,23 @@ def vtr_command_main(arg_list, prog=None): verbose=True if args.verbosity > 2 else False, echo_cmd=True if args.verbosity >= 4 else False) exit_status = 0 + flow_type = 2 #Use iterative black-boxing flow for multi-clock circuits + use_new_latches_restoration_script = 1 + + if(args.iterative_bb): + flow_type =2 + use_new_latches_restoration_script = 1 + + if(args.once_bb): + flow_type = 1 + use_new_latches_restoration_script = 1 + + if(args.blanket_bb): + flow_type = 3 + use_new_latches_restoration_script = 1 + + if(args.use_new_latches_restoration_script): + use_new_latches_restoration_script = 1 try: if not args.parse_only: try: @@ -217,7 +257,9 @@ def vtr_command_main(arg_list, prog=None): end_stage=args.end, command_runner=command_runner, verbosity=args.verbosity, - vpr_args=vpr_args + vpr_args=vpr_args, + flow_type=flow_type, + use_new_latches_restoration_script=use_new_latches_restoration_script ) except CommandError as e: #An external command failed From 6ee55ab7903735fee1d9c367f92c65979a205db4 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Wed, 13 May 2020 11:13:53 -0700 Subject: [PATCH 05/86] flow.py: replaced os.path to pathlib.path, vtr-flow.py: replaced and enabled use_new_latches_restoration_script with use_old_latches_restoration_script --- .../python_libs/verilogtorouting/flow.py | 132 +++++++++--------- vtr_flow/scripts/vtr-flow.py | 15 +- 2 files changed, 73 insertions(+), 74 deletions(-) diff --git a/vtr_flow/scripts/python_libs/verilogtorouting/flow.py b/vtr_flow/scripts/python_libs/verilogtorouting/flow.py index e941550f48e..78b3f8677ef 100644 --- a/vtr_flow/scripts/python_libs/verilogtorouting/flow.py +++ b/vtr_flow/scripts/python_libs/verilogtorouting/flow.py @@ -1,10 +1,9 @@ -import os import sys import shutil import subprocess import time import glob -import pathlib #replace os from above +from pathlib import Path from collections import OrderedDict from util import make_enum, print_verbose, mkdir_p, find_vtr_file, file_replace, relax_W, CommandRunner, write_tab_delimitted_csv @@ -23,7 +22,7 @@ def run_vtr_flow(architecture_file, circuit_file, verbosity=0, vpr_args=None, flow_type=2, - use_new_latches_restoration_script=1): + use_old_latches_restoration_script=1): """ Runs the VTR CAD flow to map the specificied circuit_file onto the target architecture_file @@ -48,21 +47,26 @@ def run_vtr_flow(architecture_file, circuit_file, # #Initial setup # - architecture_file_basename = os.path.basename(architecture_file) - circuit_file_basename = os.path.basename(circuit_file) - circuit_name, circuit_ext = os.path.splitext(circuit_file_basename) - architecture_name, architecture_ext = os.path.splitext(architecture_file_basename) + circuit_path = Path(circuit_file) + architecture_path = Path(architecture_file) + architecture_file_basename =architecture_path.name + circuit_file_basename = circuit_path.name + + circuit_name = circuit_path.stem + circuit_ext = circuit_path.suffixes + architecture_name = architecture_path.stem + architecture_ext = architecture_path.suffixes mkdir_p(temp_dir) #Define useful filenames - post_odin_netlist = circuit_name + '.odin.blif' - post_abc_netlist = circuit_name + '.abc.blif' - post_ace_netlist = circuit_name + ".ace.blif" - post_ace_activity_file = circuit_name + ".act" - pre_vpr_netlist = circuit_name + ".pre_vpr.blif" - post_vpr_netlist = "top_post_synthesis.blif" #circuit_name + ".vpr.blif" + post_odin_netlist = Path(temp_dir) / (circuit_name + '.odin.blif') + post_abc_netlist =Path(temp_dir) / (circuit_name + '.abc.blif') + post_ace_netlist =Path(temp_dir) / (circuit_name + ".ace.blif") + post_ace_activity_file = Path(temp_dir) / (circuit_name + ".act") + pre_vpr_netlist = Path(temp_dir) / (circuit_name + ".pre_vpr.blif") + post_vpr_netlist = Path(temp_dir) / "top_post_synthesis.blif" #circuit_name + ".vpr.blif" lec_base_netlist = None #Reference netlist for LEC if circuit_ext == ".blif": @@ -71,14 +75,16 @@ def run_vtr_flow(architecture_file, circuit_file, lec_base_netlist = circuit_file_basename #Copy the circuit and architecture - shutil.copy(circuit_file, os.path.join(temp_dir, circuit_file_basename)) - shutil.copy(architecture_file, os.path.join(temp_dir, architecture_file_basename)) + circuit_copy = Path(temp_dir) / circuit_path.name + architecture_copy = Path(temp_dir) / architecture_path.name + shutil.copy(circuit_file, str(circuit_copy)) + shutil.copy(architecture_file, str(architecture_copy)) #There are multiple potential paths for the netlist to reach a tool #We initialize it here to the user specified circuit and let downstream #stages update it - next_stage_netlist = circuit_file_basename + next_stage_netlist = circuit_copy # # RTL Elaboration & Synthesis @@ -87,7 +93,7 @@ def run_vtr_flow(architecture_file, circuit_file, if circuit_ext != ".blif": print_verbose(1, verbosity, "Running Odin II") - run_odin(architecture_file_basename, next_stage_netlist, + run_odin(architecture_copy, next_stage_netlist, output_netlist=post_odin_netlist, command_runner=command_runner, temp_dir=temp_dir) @@ -102,12 +108,11 @@ def run_vtr_flow(architecture_file, circuit_file, # if should_run_stage(VTR_STAGE.abc, start_stage, end_stage): print_verbose(1, verbosity, "Running ABC") - run_abc(architecture_file_basename, next_stage_netlist, + run_abc(architecture_copy, next_stage_netlist, output_netlist=post_abc_netlist, command_runner=command_runner, temp_dir=temp_dir, - flow_type=flow_type, - use_new_latches_restoration_script=use_new_latches_restoration_script) + flow_type=flow_type) next_stage_netlist = post_abc_netlist @@ -137,7 +142,7 @@ def run_vtr_flow(architecture_file, circuit_file, #Enable power analysis in VPR vpr_args["power"] = True - vpr_args["activity_file"] = post_ace_activity_file + vpr_args["activity_file"] = post_ace_activity_file.name vpr_args["tech_properties"] = power_tech_file # @@ -145,7 +150,7 @@ def run_vtr_flow(architecture_file, circuit_file, # if should_run_stage(VTR_STAGE.vpr, start_stage, end_stage): #Copy the input netlist for input to vpr - shutil.copyfile(os.path.join(temp_dir, next_stage_netlist), os.path.join(temp_dir, pre_vpr_netlist)) + shutil.copyfile(str(next_stage_netlist), str(pre_vpr_netlist)) #Do we need to generate the post-synthesis netlist? (e.g. for LEC) if should_run_stage(VTR_STAGE.lec, start_stage, end_stage): @@ -155,14 +160,14 @@ def run_vtr_flow(architecture_file, circuit_file, if "route_chan_width" in vpr_args: #The User specified a fixed channel width print_verbose(1, verbosity, "Running VPR (at fixed channel width)") - run_vpr(architecture_file_basename, circuit_name, pre_vpr_netlist, + run_vpr(architecture_copy, circuit_copy, pre_vpr_netlist, output_netlist=post_vpr_netlist, command_runner=command_runner, temp_dir=temp_dir, vpr_args=vpr_args) else: #First find minW and then re-route at a relaxed W - run_vpr_relax_W(architecture_file_basename, circuit_name, pre_vpr_netlist, + run_vpr_relax_W(architecture_copy, circuit_copy, pre_vpr_netlist, output_netlist=post_vpr_netlist, command_runner=command_runner, temp_dir=temp_dir, @@ -201,7 +206,7 @@ def parse_vtr_flow(temp_dir, parse_config_file=None, metrics_filepath=None, verb for parse_pattern in parse_patterns.values(): #We interpret the parse pattern's filename as a glob pattern - filepattern = os.path.join(temp_dir, parse_pattern.filename()) + filepattern = str(Path(temp_dir) / parse_pattern.filename()) filepaths = glob.glob(filepattern) num_files = len(filepaths) @@ -212,7 +217,7 @@ def parse_vtr_flow(temp_dir, parse_config_file=None, metrics_filepath=None, verb elif num_files == 1: filepath = filepaths[0] - assert os.path.exists(filepath) + assert Path(filepath).exists with open(filepath) as f: for line in f: @@ -225,7 +230,7 @@ def parse_vtr_flow(temp_dir, parse_config_file=None, metrics_filepath=None, verb assert num_files == 0 if metrics_filepath is None: - metrics_filepath = os.path.join(temp_dir, "parse_results.txt") + metrics_filepath = str(Path(temp_dir) / "parse_results.txt") write_tab_delimitted_csv(metrics_filepath, [metrics]) @@ -250,15 +255,15 @@ def run_odin(architecture_file, circuit_file, #Copy the config file odin_config = "odin_config.xml" - odin_config_full_path = os.path.abspath(os.path.join(temp_dir, odin_config)) + odin_config_full_path = str(Path(temp_dir) / odin_config) shutil.copyfile(odin_base_config, odin_config_full_path) #Update the config file file_replace(odin_config_full_path, { - "XXX": circuit_file, - "YYY": architecture_file, - "ZZZ": output_netlist, - "PPP": determine_memory_addr_width(os.path.join(temp_dir, architecture_file)), + "XXX": circuit_file.name, + "YYY": architecture_file.name, + "ZZZ": output_netlist.name, + "PPP": determine_memory_addr_width(str(architecture_file)), "MMM": min_hard_mult_size, "AAA": min_hard_adder_size, }) @@ -268,16 +273,13 @@ def run_odin(architecture_file, circuit_file, command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) def populate_clock_list(circuit_file,blackbox_latches_script,clk_list,command_runner,temp_dir,log_filename): - clk_list_filename = "report_clk.out" - cmd = [blackbox_latches_script, "--input", circuit_file,"--output_list", clk_list_filename] + clk_list_path = Path(temp_dir) / "report_clk.out" + cmd = [blackbox_latches_script, "--input", circuit_file.name,"--output_list", clk_list_path.name] command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) - clk_list_path=pathlib.Path(temp_dir) / clk_list_filename - i=0 with clk_list_path.open('r') as f: clk_list.append(f.readline().strip('\n')) - i=i+1 -def run_abc(architecture_file, circuit_file, output_netlist, command_runner, temp_dir=".", log_filename="abc.opt_techmap.out", abc_exec=None, abc_script=None, abc_rc=None,use_old_abc_script = False,flow_type=2,use_new_latches_restoration_script=1): +def run_abc(architecture_file, circuit_file, output_netlist, command_runner, temp_dir=".", log_filename="abc.opt_techmap.out", abc_exec=None, abc_script=None, abc_rc=None,use_old_abc_script = False,flow_type=2,use_old_latches_restoration_script=1): mkdir_p(temp_dir) blackbox_latches_script = find_vtr_file("blackbox_latches.pl") clk_list = [] @@ -289,33 +291,32 @@ def run_abc(architecture_file, circuit_file, output_netlist, command_runner, tem abc_exec = find_vtr_file('abc', is_executable=True) if abc_rc == None: - abc_dir = os.path.dirname(abc_exec) - abc_rc = os.path.join(abc_dir, 'abc.rc') + abc_rc = Path(abc_exec).parent / 'abc.rc' - shutil.copyfile(abc_rc, os.path.join(temp_dir, 'abc.rc')) - lut_size = determine_lut_size(os.path.join(temp_dir, architecture_file)) + shutil.copyfile(str(abc_rc), str(Path(temp_dir) / 'abc.rc')) + lut_size = determine_lut_size(str(architecture_file)) iterations=len(clk_list) if(iterations==0 or flow_type != 2): iterations=1 for i in range(0, iterations): - pre_abc_blif= str(i)+"_" + circuit_file - post_abc_blif = str(i)+"_"+output_netlist - post_abc_raw_blif = str(i)+"_raw_"+output_netlist + pre_abc_blif= Path(temp_dir) / (str(i)+"_" + circuit_file.name) + post_abc_blif = Path(temp_dir) / (str(i)+"_"+output_netlist.name) + post_abc_raw_blif = Path(temp_dir) / (str(i)+"_raw_"+output_netlist.name) if(flow_type==3): - cmd = [blackbox_latches_script, "--input", circuit_file,"--output",pre_abc_blif] + cmd = [blackbox_latches_script, "--input", circuit_file.name,"--output",pre_abc_blif.name] command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=str(i)+"_blackboxing_latch.out", indent_depth=1) elif(len(clk_list)>i): - cmd = [blackbox_latches_script,"--clk_list", clk_list[i], "--input", circuit_file,"--output",pre_abc_blif] + cmd = [blackbox_latches_script,"--clk_list", clk_list[i], "--input", circuit_file.name,"--output",pre_abc_blif.name] command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=str(i)+"_blackboxing_latch.out", indent_depth=1) else: pre_abc_blif = circuit_file if abc_script == None: - abc_script = ['read {pre_abc_blif}'.format(pre_abc_blif=pre_abc_blif), + abc_script = ['read {pre_abc_blif}'.format(pre_abc_blif=pre_abc_blif.name), 'time', 'print_stats', 'print_latch', @@ -331,18 +332,18 @@ def run_abc(architecture_file, circuit_file, output_netlist, command_runner, tem 'mfs2 -v', 'print_stats', 'time', - 'write_hie {pre_abc_blif} {post_abc_raw_blif}'.format(pre_abc_blif=pre_abc_blif, post_abc_raw_blif=post_abc_raw_blif), + 'write_hie {pre_abc_blif} {post_abc_raw_blif}'.format(pre_abc_blif=pre_abc_blif.name, post_abc_raw_blif=post_abc_raw_blif.name), 'time;'] if(use_old_abc_script): - abc_script = ['read {pre_abc_blif}'.format(pre_abc_blif=pre_abc_blif), + abc_script = ['read {pre_abc_blif}'.format(pre_abc_blif=pre_abc_blif.name), 'time', 'resyn', 'resyn2', 'if -K {lut_size}'.format(lut_size=lut_size), 'time', 'scleanup', - 'write_hie {pre_abc_blif} {post_abc_raw_blif}'.format(pre_abc_blif=pre_abc_blif, post_abc_raw_blif=post_abc_raw_blif), + 'write_hie {pre_abc_blif} {post_abc_raw_blif}'.format(pre_abc_blif=pre_abc_blif.name, post_abc_raw_blif=post_abc_raw_blif.name), 'print_stats'] abc_script = "; ".join(abc_script) @@ -352,20 +353,21 @@ def run_abc(architecture_file, circuit_file, output_netlist, command_runner, tem command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) if(flow_type != 3 and len(clk_list)>i): - cmd = [blackbox_latches_script,"--restore", clk_list[i], "--input", post_abc_raw_blif,"--output",post_abc_blif] - command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename="restore_latch"+str(i)+".out", indent_depth=1) + cmd = [blackbox_latches_script,"--restore", clk_list[i], "--input", post_abc_raw_blif.name,"--output",post_abc_blif.name] + command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename="restore_latch" + str(i) + ".out", indent_depth=1) else: - if(use_new_latches_restoration_script): - restore_multiclock_info_script = find_vtr_file("restore_multiclock_latch.pl") - else: + if(use_old_latches_restoration_script): restore_multiclock_info_script = find_vtr_file("restore_multiclock_latch_information.pl") - cmd = [restore_multiclock_info_script, pre_abc_blif, post_abc_raw_blif ,post_abc_blif] - command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename="restore_latch"+str(i)+".out", indent_depth=1) + else: + restore_multiclock_info_script = find_vtr_file("restore_multiclock_latch.pl") + + cmd = [restore_multiclock_info_script, pre_abc_blif.name, post_abc_raw_blif.name ,post_abc_blif.name] + command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename="restore_latch" + str(i) + ".out", indent_depth=1) if(flow_type != 2): break - cmd = [blackbox_latches_script, "--input", post_abc_blif,"--output",output_netlist,"--vanilla"] - command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename="restore_latch"+str(i)+".out", indent_depth=1) + cmd = [blackbox_latches_script, "--input", post_abc_blif.name,"--output",output_netlist.name,"--vanilla"] + command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename="restore_latch" + str(i) + ".out", indent_depth=1) def run_ace(circuit_file, output_netlist, output_activity_file, command_runner, temp_dir=".", log_filename="ace.out", ace_exec=None): @@ -373,9 +375,9 @@ def run_ace(circuit_file, output_netlist, output_activity_file, command_runner, ace_exec = find_vtr_file('ace', is_executable=True) cmd = [ace_exec, - "-b", circuit_file, - "-n", output_netlist, - "-o", output_activity_file] + "-b", circuit_file.name, + "-n", output_netlist.name, + "-o", output_activity_file.name] command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) @@ -418,7 +420,7 @@ def run_vpr_relax_W(architecture, circuit_name, circuit, command_runner=CommandR return - min_W = determine_min_W(os.path.join(temp_dir, vpr_min_W_log)) + min_W = determine_min_W(str(Path(temp_dir) / vpr_min_W_log)) relaxed_W = relax_W(min_W, relax_W_factor) @@ -447,7 +449,7 @@ def run_vpr(architecture, circuit_name, circuit, command_runner, temp_dir, outpu if vpr_exec == None: vpr_exec = find_vtr_file('vpr', is_executable=True) - cmd = [vpr_exec, architecture, circuit_name, "--circuit_file", circuit] + cmd = [vpr_exec, architecture.name, circuit_name.name, "--circuit_file", circuit.name] #Enable netlist generation #if output_netlist: diff --git a/vtr_flow/scripts/vtr-flow.py b/vtr_flow/scripts/vtr-flow.py index 20501c54cea..4ba161fb50f 100755 --- a/vtr_flow/scripts/vtr-flow.py +++ b/vtr_flow/scripts/vtr-flow.py @@ -192,10 +192,10 @@ def vtr_command_argparser(prog=None): action="store_true", dest="blanket_bb", help="Use iterative black-boxing flow with out clocks") #not sure if this is a correct statement. - iterative.add_argument("-use_new_latches_restoration_script", + iterative.add_argument("-use_old_latches_restoration_script", default=False, action="store_true", - dest="use_new_latches_restoration_script", + dest="use_old_latches_restoration_script", help="Use the new latches restoration script") return parser @@ -227,22 +227,19 @@ def vtr_command_main(arg_list, prog=None): echo_cmd=True if args.verbosity >= 4 else False) exit_status = 0 flow_type = 2 #Use iterative black-boxing flow for multi-clock circuits - use_new_latches_restoration_script = 1 + use_old_latches_restoration_script = 0 if(args.iterative_bb): flow_type =2 - use_new_latches_restoration_script = 1 if(args.once_bb): flow_type = 1 - use_new_latches_restoration_script = 1 if(args.blanket_bb): flow_type = 3 - use_new_latches_restoration_script = 1 - if(args.use_new_latches_restoration_script): - use_new_latches_restoration_script = 1 + if(args.use_old_latches_restoration_script): + use_old_latches_restoration_script = 1 try: if not args.parse_only: try: @@ -259,7 +256,7 @@ def vtr_command_main(arg_list, prog=None): verbosity=args.verbosity, vpr_args=vpr_args, flow_type=flow_type, - use_new_latches_restoration_script=use_new_latches_restoration_script + use_old_latches_restoration_script=use_old_latches_restoration_script ) except CommandError as e: #An external command failed From f503d6c7c47d2dbf0fb0548390f469dc98150b82 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Wed, 13 May 2020 11:50:09 -0700 Subject: [PATCH 06/86] vtr-flow.py:fixed arguments to match the current ones on master --- vtr_flow/scripts/vtr-flow.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/vtr_flow/scripts/vtr-flow.py b/vtr_flow/scripts/vtr-flow.py index 4ba161fb50f..a7f8f414493 100755 --- a/vtr_flow/scripts/vtr-flow.py +++ b/vtr_flow/scripts/vtr-flow.py @@ -83,12 +83,12 @@ def vtr_command_argparser(prog=None): Enable power analysis: - %(prog)s arch.xml circuit.v --power_tech power_tech.xml + %(prog)s arch.xml circuit.v -power_tech power_tech.xml Only run a specific stage (assumes required results have already been generated): - %(prog)s arch.xml circuit.blif --start vpr --end vpr + %(prog)s arch.xml circuit.blif -start vpr -end vpr """ ) @@ -108,19 +108,19 @@ def vtr_command_argparser(prog=None): help="The circuit to map to the target architecture.") parser.add_argument('architecture_file', help="The FPGA architecture to target.") - parser.add_argument("--start", "--starting_stage", + parser.add_argument("-start", "-starting_stage", choices=VTR_STAGE.reverse_mapping.values(), default=VTR_STAGE.odin, action=VtrStageArgparseAction, help="Starting stage of the VTR flow.") - parser.add_argument("--end", "--ending_stage", + parser.add_argument("-end", "-ending_stage", choices=VTR_STAGE.reverse_mapping.values(), default=VTR_STAGE.vpr, action=VtrStageArgparseAction, help="Ending stage of the VTR flow.") - parser.add_argument("-v", "--verbosity", + parser.add_argument("-v", "-verbose", choices=VERBOSITY_CHOICES, default=2, type=int, @@ -141,7 +141,7 @@ def vtr_command_argparser(prog=None): # Power arguments # power = parser.add_argument_group("Power", description="Power Analysis Related Options") - power.add_argument("--power", + power.add_argument("-power", default=None, dest="power_tech", metavar="POWER_TECH_FILE", @@ -156,19 +156,19 @@ def vtr_command_argparser(prog=None): default=".", help="Directory to run the flow in (will be created if non-existant).") - house_keeping.add_argument("--track_memory_usage", + house_keeping.add_argument("-track_memory_usage", choices=on_off_choices, default="on", action=OnOffArgparseAction, help="Track the memory usage for each stage. Requires /usr/bin/time -v, disabled if not available.") - house_keeping.add_argument("--memory_limit", + house_keeping.add_argument("-limit_memory_usage", default=None, metavar="MAX_MEMORY_MB", help="Specifies the maximum memory usable by any stage. " "Not supported on some platforms (requires ulimit).") - house_keeping.add_argument("--timeout", + house_keeping.add_argument("-timeout", default=14*24*60*60, #14 days type=float, metavar="TIMEOUT_SECONDS", @@ -220,7 +220,7 @@ def vtr_command_main(arg_list, prog=None): #Specify how command should be run command_runner = CommandRunner(track_memory=args.track_memory_usage, - max_memory_mb=args.memory_limit, + max_memory_mb=args.limit_memory_usage, timeout_sec=args.timeout, verbose_error=True if args.verbosity == 2 else False, verbose=True if args.verbosity > 2 else False, From 329607cb55474fdf85d0956b3306ea2ed42cbe04 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Sat, 16 May 2020 09:50:46 -0700 Subject: [PATCH 07/86] Reorganized and renamed scripts/python_libs/verilogtorouting and fixed imports in vtr-flow, vtr-task and vtr-test --- .../python_libs/verilogtorouting/__init__.py | 0 .../python_libs/verilogtorouting/flow.py | 515 ------------------ vtr_flow/scripts/python_libs/vtr/__init__.py | 9 + .../scripts/python_libs/vtr/abc/__init__.py | 1 + vtr_flow/scripts/python_libs/vtr/abc/abc.py | 140 +++++ .../scripts/python_libs/vtr/ace/__init__.py | 1 + vtr_flow/scripts/python_libs/vtr/ace/ace.py | 12 + .../{verilogtorouting => vtr}/error.py | 0 vtr_flow/scripts/python_libs/vtr/flow.py | 246 +++++++++ .../{verilogtorouting => vtr}/inspect.py | 3 +- .../scripts/python_libs/vtr/odin/__init__.py | 1 + vtr_flow/scripts/python_libs/vtr/odin/odin.py | 39 ++ .../{verilogtorouting => vtr}/task.py | 3 +- .../{verilogtorouting => vtr}/util.py | 4 +- .../scripts/python_libs/vtr/vpr/__init__.py | 1 + vtr_flow/scripts/python_libs/vtr/vpr/vpr.py | 88 +++ vtr_flow/scripts/vtr-flow.py | 74 ++- vtr_flow/scripts/vtr-task.py | 9 +- vtr_flow/scripts/vtr-test.py | 7 +- 19 files changed, 581 insertions(+), 572 deletions(-) delete mode 100644 vtr_flow/scripts/python_libs/verilogtorouting/__init__.py delete mode 100644 vtr_flow/scripts/python_libs/verilogtorouting/flow.py create mode 100644 vtr_flow/scripts/python_libs/vtr/__init__.py create mode 100644 vtr_flow/scripts/python_libs/vtr/abc/__init__.py create mode 100644 vtr_flow/scripts/python_libs/vtr/abc/abc.py create mode 100644 vtr_flow/scripts/python_libs/vtr/ace/__init__.py create mode 100644 vtr_flow/scripts/python_libs/vtr/ace/ace.py rename vtr_flow/scripts/python_libs/{verilogtorouting => vtr}/error.py (100%) create mode 100644 vtr_flow/scripts/python_libs/vtr/flow.py rename vtr_flow/scripts/python_libs/{verilogtorouting => vtr}/inspect.py (99%) create mode 100644 vtr_flow/scripts/python_libs/vtr/odin/__init__.py create mode 100644 vtr_flow/scripts/python_libs/vtr/odin/odin.py rename vtr_flow/scripts/python_libs/{verilogtorouting => vtr}/task.py (98%) rename vtr_flow/scripts/python_libs/{verilogtorouting => vtr}/util.py (99%) create mode 100644 vtr_flow/scripts/python_libs/vtr/vpr/__init__.py create mode 100644 vtr_flow/scripts/python_libs/vtr/vpr/vpr.py diff --git a/vtr_flow/scripts/python_libs/verilogtorouting/__init__.py b/vtr_flow/scripts/python_libs/verilogtorouting/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/vtr_flow/scripts/python_libs/verilogtorouting/flow.py b/vtr_flow/scripts/python_libs/verilogtorouting/flow.py deleted file mode 100644 index 78b3f8677ef..00000000000 --- a/vtr_flow/scripts/python_libs/verilogtorouting/flow.py +++ /dev/null @@ -1,515 +0,0 @@ -import sys -import shutil -import subprocess -import time -import glob -from pathlib import Path -from collections import OrderedDict - -from util import make_enum, print_verbose, mkdir_p, find_vtr_file, file_replace, relax_W, CommandRunner, write_tab_delimitted_csv -from inspect import determine_memory_addr_width, determine_lut_size, determine_min_W, load_parse_patterns -from error import * - -VTR_STAGE = make_enum("odin", "abc", 'ace', "vpr", "lec") -vtr_stages = VTR_STAGE.reverse_mapping.values() - -def run_vtr_flow(architecture_file, circuit_file, - power_tech_file=None, - start_stage=VTR_STAGE.odin, end_stage=VTR_STAGE.vpr, - command_runner=CommandRunner(), - parse_config_file=None, - temp_dir="./temp", - verbosity=0, - vpr_args=None, - flow_type=2, - use_old_latches_restoration_script=1): - """ - Runs the VTR CAD flow to map the specificied circuit_file onto the target architecture_file - - Arguments - --------- - architecture_file: Architecture file to target - circuit_file : Circuit to implement - - power_tech_file : Technology power file. Enables power analysis and runs ace - - temp_dir : Directory to run in (created if non-existant) - start_stage : Stage of the flow to start at - end_stage : Stage of the flow to finish at - command_runner : A CommandRunner object used to run system commands - parse_config_file: The configuration file defining how to parse metrics from results - verbosity : How much output to produce - vpr_args : A dictionary of keywork arguments to pass on to VPR - """ - if vpr_args == None: - vpr_args = OrderedDict() - - # - #Initial setup - # - - circuit_path = Path(circuit_file) - architecture_path = Path(architecture_file) - architecture_file_basename =architecture_path.name - circuit_file_basename = circuit_path.name - - circuit_name = circuit_path.stem - circuit_ext = circuit_path.suffixes - architecture_name = architecture_path.stem - architecture_ext = architecture_path.suffixes - - mkdir_p(temp_dir) - - #Define useful filenames - post_odin_netlist = Path(temp_dir) / (circuit_name + '.odin.blif') - post_abc_netlist =Path(temp_dir) / (circuit_name + '.abc.blif') - post_ace_netlist =Path(temp_dir) / (circuit_name + ".ace.blif") - post_ace_activity_file = Path(temp_dir) / (circuit_name + ".act") - pre_vpr_netlist = Path(temp_dir) / (circuit_name + ".pre_vpr.blif") - post_vpr_netlist = Path(temp_dir) / "top_post_synthesis.blif" #circuit_name + ".vpr.blif" - lec_base_netlist = None #Reference netlist for LEC - - if circuit_ext == ".blif": - #If the user provided a .blif netlist, we use that as the baseline for LEC - #(ABC can't LEC behavioural verilog) - lec_base_netlist = circuit_file_basename - - #Copy the circuit and architecture - circuit_copy = Path(temp_dir) / circuit_path.name - architecture_copy = Path(temp_dir) / architecture_path.name - shutil.copy(circuit_file, str(circuit_copy)) - shutil.copy(architecture_file, str(architecture_copy)) - - - #There are multiple potential paths for the netlist to reach a tool - #We initialize it here to the user specified circuit and let downstream - #stages update it - next_stage_netlist = circuit_copy - - # - # RTL Elaboration & Synthesis - # - if should_run_stage(VTR_STAGE.odin, start_stage, end_stage): - if circuit_ext != ".blif": - print_verbose(1, verbosity, "Running Odin II") - - run_odin(architecture_copy, next_stage_netlist, - output_netlist=post_odin_netlist, - command_runner=command_runner, - temp_dir=temp_dir) - - next_stage_netlist = post_odin_netlist - - if not lec_base_netlist: - lec_base_netlist = post_odin_netlist - - # - # Logic Optimization & Technology Mapping - # - if should_run_stage(VTR_STAGE.abc, start_stage, end_stage): - print_verbose(1, verbosity, "Running ABC") - run_abc(architecture_copy, next_stage_netlist, - output_netlist=post_abc_netlist, - command_runner=command_runner, - temp_dir=temp_dir, - flow_type=flow_type) - - next_stage_netlist = post_abc_netlist - - if not lec_base_netlist: - lec_base_netlist = post_abc_netlist - - - # - # Power Activity Estimation - # - if power_tech_file: - #The user provided a tech file, so do power analysis - - if should_run_stage(VTR_STAGE.ace, start_stage, end_stage): - print_verbose(1, verbosity, "Running ACE") - - run_ace(next_stage_netlist, output_netlist=post_ace_netlist, - output_activity_file=post_ace_activity_file, - command_runner=command_runner, - temp_dir=temp_dir) - - #Use ACE's output netlist - next_stage_netlist = post_ace_netlist - - if not lec_base_netlist: - lec_base_netlist = post_ace_netlist - - #Enable power analysis in VPR - vpr_args["power"] = True - vpr_args["activity_file"] = post_ace_activity_file.name - vpr_args["tech_properties"] = power_tech_file - - # - # Pack/Place/Route - # - if should_run_stage(VTR_STAGE.vpr, start_stage, end_stage): - #Copy the input netlist for input to vpr - shutil.copyfile(str(next_stage_netlist), str(pre_vpr_netlist)) - - #Do we need to generate the post-synthesis netlist? (e.g. for LEC) - if should_run_stage(VTR_STAGE.lec, start_stage, end_stage): - if "gen_postsynthesis_netlist" not in vpr_args: - vpr_args["gen_postsynthesis_netlist"] = "on" - - if "route_chan_width" in vpr_args: - #The User specified a fixed channel width - print_verbose(1, verbosity, "Running VPR (at fixed channel width)") - run_vpr(architecture_copy, circuit_copy, pre_vpr_netlist, - output_netlist=post_vpr_netlist, - command_runner=command_runner, - temp_dir=temp_dir, - vpr_args=vpr_args) - else: - #First find minW and then re-route at a relaxed W - run_vpr_relax_W(architecture_copy, circuit_copy, pre_vpr_netlist, - output_netlist=post_vpr_netlist, - command_runner=command_runner, - temp_dir=temp_dir, - verbosity=verbosity, - vpr_args=vpr_args) - - if not lec_base_netlist: - lec_base_netlist = pre_vpr_netlist - - # - # Logical Equivalence Checks (LEC) - # - if should_run_stage(VTR_STAGE.lec, start_stage, end_stage): - print_verbose(1, verbosity, "Running ABC Logical Equivalence Check") - run_abc_lec(lec_base_netlist, post_vpr_netlist, command_runner=command_runner, log_filename="abc.lec.out") - -def parse_vtr_flow(temp_dir, parse_config_file=None, metrics_filepath=None, verbosity=1): - print_verbose(1, verbosity, "Parsing results") - - if parse_config_file is None: - parse_config_file = find_vtr_file("vtr_benchmarks.txt") - - parse_patterns = load_parse_patterns(parse_config_file) - - metrics = OrderedDict() - - #Set defaults - for parse_pattern in parse_patterns.values(): - - if parse_pattern.default_value() != None: - metrics[parse_pattern.name()] = parse_pattern.default_value() - else: - metrics[parse_pattern.name()] = "" - - #Process each pattern - for parse_pattern in parse_patterns.values(): - - #We interpret the parse pattern's filename as a glob pattern - filepattern = str(Path(temp_dir) / parse_pattern.filename()) - filepaths = glob.glob(filepattern) - - num_files = len(filepaths) - - if num_files > 1: - raise InspectError("File pattern '{}' is ambiguous ({} files matched)".format(parse_pattern.filename()), num_files, filepaths) - - elif num_files == 1: - filepath = filepaths[0] - - assert Path(filepath).exists - - with open(filepath) as f: - for line in f: - match = parse_pattern.regex().match(line) - if match: - #Extract the first group value - metrics[parse_pattern.name()] = match.groups()[0] - else: - #No matching file, skip - assert num_files == 0 - - if metrics_filepath is None: - metrics_filepath = str(Path(temp_dir) / "parse_results.txt") - - write_tab_delimitted_csv(metrics_filepath, [metrics]) - - return metrics - -def run_odin(architecture_file, circuit_file, - output_netlist, - command_runner, - temp_dir=".", - log_filename="odin.out", - odin_exec=None, - odin_config=None, - min_hard_mult_size=3, - min_hard_adder_size=1): - mkdir_p(temp_dir) - - if odin_exec == None: - odin_exec = find_vtr_file('odin_II', is_executable=True) - - if odin_config == None: - odin_base_config = find_vtr_file('basic_odin_config_split.xml') - - #Copy the config file - odin_config = "odin_config.xml" - odin_config_full_path = str(Path(temp_dir) / odin_config) - shutil.copyfile(odin_base_config, odin_config_full_path) - - #Update the config file - file_replace(odin_config_full_path, { - "XXX": circuit_file.name, - "YYY": architecture_file.name, - "ZZZ": output_netlist.name, - "PPP": determine_memory_addr_width(str(architecture_file)), - "MMM": min_hard_mult_size, - "AAA": min_hard_adder_size, - }) - - cmd = [odin_exec, "-c", odin_config, "--adder_type", "default", "-U0"] - - command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) - -def populate_clock_list(circuit_file,blackbox_latches_script,clk_list,command_runner,temp_dir,log_filename): - clk_list_path = Path(temp_dir) / "report_clk.out" - cmd = [blackbox_latches_script, "--input", circuit_file.name,"--output_list", clk_list_path.name] - command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) - with clk_list_path.open('r') as f: - clk_list.append(f.readline().strip('\n')) - -def run_abc(architecture_file, circuit_file, output_netlist, command_runner, temp_dir=".", log_filename="abc.opt_techmap.out", abc_exec=None, abc_script=None, abc_rc=None,use_old_abc_script = False,flow_type=2,use_old_latches_restoration_script=1): - mkdir_p(temp_dir) - blackbox_latches_script = find_vtr_file("blackbox_latches.pl") - clk_list = [] - clk_log_file = "report_clk_out.out" - if(flow_type): - populate_clock_list(circuit_file,blackbox_latches_script,clk_list,command_runner,temp_dir,log_filename) - - if abc_exec == None: - abc_exec = find_vtr_file('abc', is_executable=True) - - if abc_rc == None: - abc_rc = Path(abc_exec).parent / 'abc.rc' - - shutil.copyfile(str(abc_rc), str(Path(temp_dir) / 'abc.rc')) - lut_size = determine_lut_size(str(architecture_file)) - iterations=len(clk_list) - - if(iterations==0 or flow_type != 2): - iterations=1 - - for i in range(0, iterations): - pre_abc_blif= Path(temp_dir) / (str(i)+"_" + circuit_file.name) - post_abc_blif = Path(temp_dir) / (str(i)+"_"+output_netlist.name) - post_abc_raw_blif = Path(temp_dir) / (str(i)+"_raw_"+output_netlist.name) - if(flow_type==3): - cmd = [blackbox_latches_script, "--input", circuit_file.name,"--output",pre_abc_blif.name] - command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=str(i)+"_blackboxing_latch.out", indent_depth=1) - - elif(len(clk_list)>i): - cmd = [blackbox_latches_script,"--clk_list", clk_list[i], "--input", circuit_file.name,"--output",pre_abc_blif.name] - command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=str(i)+"_blackboxing_latch.out", indent_depth=1) - else: - pre_abc_blif = circuit_file - - if abc_script == None: - - abc_script = ['read {pre_abc_blif}'.format(pre_abc_blif=pre_abc_blif.name), - 'time', - 'print_stats', - 'print_latch', - 'time', - 'print_lut', - 'time', - 'strash', - 'ifraig -v', - 'scorr -v', - 'dc2 -v', - 'dch -f', - 'if -K {lut_size} -v'.format(lut_size=lut_size), - 'mfs2 -v', - 'print_stats', - 'time', - 'write_hie {pre_abc_blif} {post_abc_raw_blif}'.format(pre_abc_blif=pre_abc_blif.name, post_abc_raw_blif=post_abc_raw_blif.name), - 'time;'] - - if(use_old_abc_script): - abc_script = ['read {pre_abc_blif}'.format(pre_abc_blif=pre_abc_blif.name), - 'time', - 'resyn', - 'resyn2', - 'if -K {lut_size}'.format(lut_size=lut_size), - 'time', - 'scleanup', - 'write_hie {pre_abc_blif} {post_abc_raw_blif}'.format(pre_abc_blif=pre_abc_blif.name, post_abc_raw_blif=post_abc_raw_blif.name), - 'print_stats'] - - abc_script = "; ".join(abc_script) - - cmd = [abc_exec, '-c', abc_script] - - command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) - - if(flow_type != 3 and len(clk_list)>i): - cmd = [blackbox_latches_script,"--restore", clk_list[i], "--input", post_abc_raw_blif.name,"--output",post_abc_blif.name] - command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename="restore_latch" + str(i) + ".out", indent_depth=1) - else: - if(use_old_latches_restoration_script): - restore_multiclock_info_script = find_vtr_file("restore_multiclock_latch_information.pl") - else: - restore_multiclock_info_script = find_vtr_file("restore_multiclock_latch.pl") - - cmd = [restore_multiclock_info_script, pre_abc_blif.name, post_abc_raw_blif.name ,post_abc_blif.name] - command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename="restore_latch" + str(i) + ".out", indent_depth=1) - if(flow_type != 2): - break - - cmd = [blackbox_latches_script, "--input", post_abc_blif.name,"--output",output_netlist.name,"--vanilla"] - command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename="restore_latch" + str(i) + ".out", indent_depth=1) - -def run_ace(circuit_file, output_netlist, output_activity_file, command_runner, temp_dir=".", log_filename="ace.out", ace_exec=None): - - if ace_exec is None: - ace_exec = find_vtr_file('ace', is_executable=True) - - cmd = [ace_exec, - "-b", circuit_file.name, - "-n", output_netlist.name, - "-o", output_activity_file.name] - - command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) - -def run_vpr_relax_W(architecture, circuit_name, circuit, command_runner=CommandRunner(), temp_dir=".", - relax_W_factor=1.3, vpr_exec=None, verbosity=1, logfile_base="vpr", - vpr_args=None, output_netlist=None): - """ - Runs VPR twice: - 1st: To find the minimum channel width - 2nd: At relaxed channel width (e.g. for critical path delay) - - Arguments - --------- - architecture: Architecture file - circuit: Input circuit netlist - command_runner: CommandRunner object - temp_dir: Directory to run in - - relax_W_factor: Factor by which to relax minimum channel width for critical path delay routing - verbosity: How much progress output to produce - logfile_base: Base name for log files (e.g. "vpr" produces vpr.min_W.out, vpr.relaxed_W.out) - vpr_args: Extra arguments for VPR - vpr_exec: Path to the VPR executable - output_netlist: Output implementation netlist to generate - """ - if vpr_args is None: - vpr_args = OrderedDict() - - mkdir_p(temp_dir) - - vpr_min_W_log = '.'.join([logfile_base, "min_W", "out"]) - vpr_relaxed_W_log = '.'.join([logfile_base, "relaxed_W", "out"]) - - print_verbose(1, verbosity, "Running VPR (determining minimum channel width)" ) - - run_vpr(architecture, circuit_name, circuit, command_runner, temp_dir, log_filename=vpr_min_W_log, vpr_exec=vpr_exec, vpr_args=vpr_args) - - if ('pack' in vpr_args or 'place' in vpr_args) and 'route' not in vpr_args: - #Don't look for min W if routing was not run - return - - - min_W = determine_min_W(str(Path(temp_dir) / vpr_min_W_log)) - - relaxed_W = relax_W(min_W, relax_W_factor) - - print_verbose(1, verbosity, "Running VPR (at {fac}x relaxed minimum channel width)".format(fac=relax_W_factor)) - - vpr_args['route'] = True #Re-route only - vpr_args['route_chan_width'] = relaxed_W #At a fixed channel width - - #VPR does not support performing routing when fixed pins - # are specified, and placement is not run; so remove the option - if 'fix_pins' in vpr_args: - del vpr_args['fix_pins'] - - run_vpr(architecture, circuit_name, circuit, command_runner, temp_dir, log_filename=vpr_relaxed_W_log, vpr_exec=vpr_exec, vpr_args=vpr_args) - - -def run_vpr(architecture, circuit_name, circuit, command_runner, temp_dir, output_netlist=None, log_filename="vpr.out", vpr_exec=None, vpr_args=None): - """ - Runs VPR with the specified configuration - """ - if vpr_args is None: - vpr_args = OrderedDict() - - mkdir_p(temp_dir) - - if vpr_exec == None: - vpr_exec = find_vtr_file('vpr', is_executable=True) - - cmd = [vpr_exec, architecture.name, circuit_name.name, "--circuit_file", circuit.name] - - #Enable netlist generation - #if output_netlist: - #vpr_args['gen_postsynthesis_netlist'] = output_netlist - - #Translate arbitrary keyword arguments into options for VPR - for arg, value in vpr_args.iteritems(): - if value == True: - cmd += ["--" + arg] - elif value == False: - pass - else: - cmd += ["--" + arg, str(value)] - - command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) - -def run_abc_lec(reference_netlist, implementation_netlist, command_runner, temp_dir=".", log_filename="abc.lec.out", abc_exec=None): - """ - Run Logical Equivalence Checking (LEC) between two netlists using ABC - """ - mkdir_p(temp_dir) - - if abc_exec == None: - abc_exec = find_vtr_file('abc', is_executable=True) - - abc_script = ['cec {ref} {imp}'.format(ref=reference_netlist, imp=implementation_netlist), - 'sec {ref} {imp}'.format(ref=reference_netlist, imp=implementation_netlist), - ] - abc_script = "; ".join(abc_script) - - cmd = [abc_exec, '-c', abc_script] - - output, returncode = command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) - - #Check if ABC's LEC engine passed - lec_passed = check_abc_lec_status(output) - - if lec_passed is None: - raise InspectError("Could not determine Logical Equivalence Check status between {input} <-> {output}".format(input=reference_netlist, output=implementation_netlist), filename=log_filename) - elif lec_passed is False: - raise InspectError("Logical Equivalence Check failed between {input} <-> {output}".format(input=reference_netlist, output=implementation_netlist), filename=log_filename) - - assert lec_passed - -def check_abc_lec_status(output): - equivalent = None - for line in output: - if line.startswith("Networks are NOT EQUIVALENT"): - equivalent = False - elif line.startswith("Networks are equivalent"): - equivalent = True - - #Returns None if could not determine LEC status - return equivalent - -def should_run_stage(stage, flow_start_stage, flow_end_stage): - """ - Returns True if stage falls between flow_start_stage and flow_end_stage - """ - if flow_start_stage <= stage <= flow_end_stage: - return True - return False - diff --git a/vtr_flow/scripts/python_libs/vtr/__init__.py b/vtr_flow/scripts/python_libs/vtr/__init__.py new file mode 100644 index 00000000000..1f025e0ad28 --- /dev/null +++ b/vtr_flow/scripts/python_libs/vtr/__init__.py @@ -0,0 +1,9 @@ +from .util import load_config_lines, mkdir_p, find_vtr_file, CommandRunner, print_verbose, relax_W, file_replace, make_enum, RawDefaultHelpFormatter, VERBOSITY_CHOICES,format_elapsed_time, write_tab_delimitted_csv, load_list_file, find_vtr_root, argparse_str2bool, get_next_run_dir, get_latest_run_dir +from .inspect import determine_lut_size, determine_min_W, determine_memory_addr_width, load_parse_patterns, load_pass_requirements, load_parse_results +from .abc import run, run_lec +from .vpr import run,run_relax_W +from .odin import run +from .ace import run +from .error import * +from .flow import run, VTR_STAGE, parse_vtr_flow +from .task import load_task_config, TaskConfig, find_task_config_file diff --git a/vtr_flow/scripts/python_libs/vtr/abc/__init__.py b/vtr_flow/scripts/python_libs/vtr/abc/__init__.py new file mode 100644 index 00000000000..1d722fcc0a1 --- /dev/null +++ b/vtr_flow/scripts/python_libs/vtr/abc/__init__.py @@ -0,0 +1 @@ +from .abc import run, run_lec \ No newline at end of file diff --git a/vtr_flow/scripts/python_libs/vtr/abc/abc.py b/vtr_flow/scripts/python_libs/vtr/abc/abc.py new file mode 100644 index 00000000000..278559e78a7 --- /dev/null +++ b/vtr_flow/scripts/python_libs/vtr/abc/abc.py @@ -0,0 +1,140 @@ +import shutil +from pathlib import Path +from vtr import mkdir_p, find_vtr_file, determine_lut_size, error + +def run(architecture_file, circuit_file, output_netlist, command_runner, temp_dir=".", log_filename="abc.opt_techmap.out", abc_exec=None, abc_script=None, abc_rc=None,use_old_abc_script = False,abc_flow_type=2,use_old_latches_restoration_script=1): + mkdir_p(temp_dir) + blackbox_latches_script = find_vtr_file("blackbox_latches.pl") + clk_list = [] + clk_log_file = "report_clk_out.out" + if(abc_flow_type): + populate_clock_list(circuit_file,blackbox_latches_script,clk_list,command_runner,temp_dir,log_filename) + + if abc_exec == None: + abc_exec = find_vtr_file('abc', is_executable=True) + + if abc_rc == None: + abc_rc = Path(abc_exec).parent / 'abc.rc' + + shutil.copyfile(str(abc_rc), str(Path(temp_dir) / 'abc.rc')) + lut_size = determine_lut_size(str(architecture_file)) + iterations=len(clk_list) + + if(iterations==0 or abc_flow_type != 2): + iterations=1 + + for i in range(0, iterations): + pre_abc_blif= Path(temp_dir) / (str(i)+"_" + circuit_file.name) + post_abc_blif = Path(temp_dir) / (str(i)+"_"+output_netlist.name) + post_abc_raw_blif = Path(temp_dir) / (str(i)+"_raw_"+output_netlist.name) + if(abc_flow_type==3): + cmd = [blackbox_latches_script, "--input", circuit_file.name,"--output",pre_abc_blif.name] + command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=str(i)+"_blackboxing_latch.out", indent_depth=1) + + elif(len(clk_list)>i): + cmd = [blackbox_latches_script,"--clk_list", clk_list[i], "--input", circuit_file.name,"--output",pre_abc_blif.name] + command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=str(i)+"_blackboxing_latch.out", indent_depth=1) + else: + pre_abc_blif = circuit_file + + if abc_script == None: + + abc_script = ['read {pre_abc_blif}'.format(pre_abc_blif=pre_abc_blif.name), + 'time', + 'print_stats', + 'print_latch', + 'time', + 'print_lut', + 'time', + 'strash', + 'ifraig -v', + 'scorr -v', + 'dc2 -v', + 'dch -f', + 'if -K {lut_size} -v'.format(lut_size=lut_size), + 'mfs2 -v', + 'print_stats', + 'time', + 'write_hie {pre_abc_blif} {post_abc_raw_blif}'.format(pre_abc_blif=pre_abc_blif.name, post_abc_raw_blif=post_abc_raw_blif.name), + 'time;'] + + if(use_old_abc_script): + abc_script = ['read {pre_abc_blif}'.format(pre_abc_blif=pre_abc_blif.name), + 'time', + 'resyn', + 'resyn2', + 'if -K {lut_size}'.format(lut_size=lut_size), + 'time', + 'scleanup', + 'write_hie {pre_abc_blif} {post_abc_raw_blif}'.format(pre_abc_blif=pre_abc_blif.name, post_abc_raw_blif=post_abc_raw_blif.name), + 'print_stats'] + + abc_script = "; ".join(abc_script) + + cmd = [abc_exec, '-c', abc_script] + + command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) + + if(abc_flow_type != 3 and len(clk_list)>i): + cmd = [blackbox_latches_script,"--restore", clk_list[i], "--input", post_abc_raw_blif.name,"--output",post_abc_blif.name] + command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename="restore_latch" + str(i) + ".out", indent_depth=1) + else: + if(use_old_latches_restoration_script): + restore_multiclock_info_script = find_vtr_file("restore_multiclock_latch_information.pl") + else: + restore_multiclock_info_script = find_vtr_file("restore_multiclock_latch.pl") + + cmd = [restore_multiclock_info_script, pre_abc_blif.name, post_abc_raw_blif.name ,post_abc_blif.name] + command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename="restore_latch" + str(i) + ".out", indent_depth=1) + if(abc_flow_type != 2): + break + + cmd = [blackbox_latches_script, "--input", post_abc_blif.name,"--output",output_netlist.name,"--vanilla"] + command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename="restore_latch" + str(i) + ".out", indent_depth=1) + +def populate_clock_list(circuit_file,blackbox_latches_script,clk_list,command_runner,temp_dir,log_filename): + clk_list_path = Path(temp_dir) / "report_clk.out" + cmd = [blackbox_latches_script, "--input", circuit_file.name,"--output_list", clk_list_path.name] + command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) + with clk_list_path.open('r') as f: + clk_list.append(f.readline().strip('\n')) + +def run_lec(reference_netlist, implementation_netlist, command_runner, temp_dir=".", log_filename="abc.lec.out", abc_exec=None): + """ + Run Logical Equivalence Checking (LEC) between two netlists using ABC + """ + mkdir_p(temp_dir) + + if abc_exec == None: + abc_exec = find_vtr_file('abc', is_executable=True) + + abc_script = ['cec {ref} {imp}'.format(ref=reference_netlist, imp=implementation_netlist), + 'sec {ref} {imp}'.format(ref=reference_netlist, imp=implementation_netlist), + ] + abc_script = "; ".join(abc_script) + + cmd = [abc_exec, '-c', abc_script] + + output, returncode = command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) + + #Check if ABC's LEC engine passed + lec_passed = check_abc_lec_status(output) + + if lec_passed is None: + raise InspectError("Could not determine Logical Equivalence Check status between {input} <-> {output}".format(input=reference_netlist, output=implementation_netlist), filename=log_filename) + elif lec_passed is False: + raise InspectError("Logical Equivalence Check failed between {input} <-> {output}".format(input=reference_netlist, output=implementation_netlist), filename=log_filename) + + assert lec_passed + +def check_abc_lec_status(output): + equivalent = None + for line in output: + if line.startswith("Networks are NOT EQUIVALENT"): + equivalent = False + elif line.startswith("Networks are equivalent"): + equivalent = True + + #Returns None if could not determine LEC status + return equivalent + diff --git a/vtr_flow/scripts/python_libs/vtr/ace/__init__.py b/vtr_flow/scripts/python_libs/vtr/ace/__init__.py new file mode 100644 index 00000000000..ffac7c866ed --- /dev/null +++ b/vtr_flow/scripts/python_libs/vtr/ace/__init__.py @@ -0,0 +1 @@ +from .ace import run \ No newline at end of file diff --git a/vtr_flow/scripts/python_libs/vtr/ace/ace.py b/vtr_flow/scripts/python_libs/vtr/ace/ace.py new file mode 100644 index 00000000000..5d75a69ed91 --- /dev/null +++ b/vtr_flow/scripts/python_libs/vtr/ace/ace.py @@ -0,0 +1,12 @@ +from vtr import find_vtr_file +def run(circuit_file, output_netlist, output_activity_file, command_runner, temp_dir=".", log_filename="ace.out", ace_exec=None): + + if ace_exec is None: + ace_exec = find_vtr_file('ace', is_executable=True) + + cmd = [ace_exec, + "-b", circuit_file.name, + "-n", output_netlist.name, + "-o", output_activity_file.name] + + command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) \ No newline at end of file diff --git a/vtr_flow/scripts/python_libs/verilogtorouting/error.py b/vtr_flow/scripts/python_libs/vtr/error.py similarity index 100% rename from vtr_flow/scripts/python_libs/verilogtorouting/error.py rename to vtr_flow/scripts/python_libs/vtr/error.py diff --git a/vtr_flow/scripts/python_libs/vtr/flow.py b/vtr_flow/scripts/python_libs/vtr/flow.py new file mode 100644 index 00000000000..a031a6ed313 --- /dev/null +++ b/vtr_flow/scripts/python_libs/vtr/flow.py @@ -0,0 +1,246 @@ +import sys +import shutil +import subprocess +import time +import glob +import vtr +from pathlib import Path +from collections import OrderedDict + +VTR_STAGE = vtr.make_enum("odin", "abc", 'ace', "vpr", "lec") +vtr_stages = VTR_STAGE.reverse_mapping.values() + +def run(architecture_file, circuit_file, + power_tech_file=None, + start_stage=VTR_STAGE.odin, end_stage=VTR_STAGE.vpr, + command_runner=vtr.CommandRunner(), + parse_config_file=None, + temp_dir="./temp", + verbosity=0, + vpr_args=None, + abc_flow_type=2, + use_old_latches_restoration_script=1): + """ + Runs the VTR CAD flow to map the specificied circuit_file onto the target architecture_file + + Arguments + --------- + architecture_file: Architecture file to target + circuit_file : Circuit to implement + + power_tech_file : Technology power file. Enables power analysis and runs ace + + temp_dir : Directory to run in (created if non-existant) + start_stage : Stage of the flow to start at + end_stage : Stage of the flow to finish at + command_runner : A CommandRunner object used to run system commands + parse_config_file: The configuration file defining how to parse metrics from results + verbosity : How much output to produce + vpr_args : A dictionary of keywork arguments to pass on to VPR + """ + if vpr_args == None: + vpr_args = OrderedDict() + + # + #Initial setup + # + + circuit_path = Path(circuit_file) + architecture_path = Path(architecture_file) + architecture_file_basename =architecture_path.name + circuit_file_basename = circuit_path.name + + circuit_name = circuit_path.stem + circuit_ext = circuit_path.suffixes + architecture_name = architecture_path.stem + architecture_ext = architecture_path.suffixes + + vtr.mkdir_p(temp_dir) + + #Define useful filenames + post_odin_netlist = Path(temp_dir) / (circuit_name + '.odin.blif') + post_abc_netlist =Path(temp_dir) / (circuit_name + '.abc.blif') + post_ace_netlist =Path(temp_dir) / (circuit_name + ".ace.blif") + post_ace_activity_file = Path(temp_dir) / (circuit_name + ".act") + pre_vpr_netlist = Path(temp_dir) / (circuit_name + ".pre_vpr.blif") + post_vpr_netlist = Path(temp_dir) / "top_post_synthesis.blif" #circuit_name + ".vpr.blif" + lec_base_netlist = None #Reference netlist for LEC + + if circuit_ext == ".blif": + #If the user provided a .blif netlist, we use that as the baseline for LEC + #(ABC can't LEC behavioural verilog) + lec_base_netlist = circuit_file_basename + + #Copy the circuit and architecture + circuit_copy = Path(temp_dir) / circuit_path.name + architecture_copy = Path(temp_dir) / architecture_path.name + shutil.copy(circuit_file, str(circuit_copy)) + shutil.copy(architecture_file, str(architecture_copy)) + + + #There are multiple potential paths for the netlist to reach a tool + #We initialize it here to the user specified circuit and let downstream + #stages update it + next_stage_netlist = circuit_copy + + # + # RTL Elaboration & Synthesis + # + if should_run_stage(VTR_STAGE.odin, start_stage, end_stage): + if circuit_ext != ".blif": + vtr.print_verbose(1, verbosity, "Running Odin II") + + vtr.odin.run(architecture_copy, next_stage_netlist, + output_netlist=post_odin_netlist, + command_runner=command_runner, + temp_dir=temp_dir) + + next_stage_netlist = post_odin_netlist + + if not lec_base_netlist: + lec_base_netlist = post_odin_netlist + + # + # Logic Optimization & Technology Mapping + # + if should_run_stage(VTR_STAGE.abc, start_stage, end_stage): + vtr.print_verbose(1, verbosity, "Running ABC") + vtr.abc.run(architecture_copy, next_stage_netlist, + output_netlist=post_abc_netlist, + command_runner=command_runner, + temp_dir=temp_dir, + abc_flow_type=abc_flow_type) + + next_stage_netlist = post_abc_netlist + + if not lec_base_netlist: + lec_base_netlist = post_abc_netlist + + + # + # Power Activity Estimation + # + if power_tech_file: + #The user provided a tech file, so do power analysis + + if should_run_stage(VTR_STAGE.ace, start_stage, end_stage): + vtr.print_verbose(1, verbosity, "Running ACE") + + vtr.ace.run(next_stage_netlist, output_netlist=post_ace_netlist, + output_activity_file=post_ace_activity_file, + command_runner=command_runner, + temp_dir=temp_dir) + + #Use ACE's output netlist + next_stage_netlist = post_ace_netlist + + if not lec_base_netlist: + lec_base_netlist = post_ace_netlist + + #Enable power analysis in VPR + vpr_args["power"] = True + vpr_args["activity_file"] = post_ace_activity_file.name + vpr_args["tech_properties"] = power_tech_file + + # + # Pack/Place/Route + # + if should_run_stage(VTR_STAGE.vpr, start_stage, end_stage): + #Copy the input netlist for input to vpr + shutil.copyfile(str(next_stage_netlist), str(pre_vpr_netlist)) + + #Do we need to generate the post-synthesis netlist? (e.g. for LEC) + if should_run_stage(VTR_STAGE.lec, start_stage, end_stage): + if "gen_postsynthesis_netlist" not in vpr_args: + vpr_args["gen_postsynthesis_netlist"] = "on" + + if "route_chan_width" in vpr_args: + #The User specified a fixed channel width + vtr.print_verbose(1, verbosity, "Running VPR (at fixed channel width)") + vtr.vpr.run(architecture_copy, circuit_copy, pre_vpr_netlist, + output_netlist=post_vpr_netlist, + command_runner=command_runner, + temp_dir=temp_dir, + vpr_args=vpr_args) + else: + #First find minW and then re-route at a relaxed W + vtr.vpr.run_relax_W(architecture_copy, circuit_copy, pre_vpr_netlist, + output_netlist=post_vpr_netlist, + command_runner=command_runner, + temp_dir=temp_dir, + verbosity=verbosity, + vpr_args=vpr_args) + + if not lec_base_netlist: + lec_base_netlist = pre_vpr_netlist + + # + # Logical Equivalence Checks (LEC) + # + if should_run_stage(VTR_STAGE.lec, start_stage, end_stage): + vtr.print_verbose(1, verbosity, "Running ABC Logical Equivalence Check") + vtr.abc.run_lec(lec_base_netlist, post_vpr_netlist, command_runner=command_runner, log_filename="abc.lec.out") + +def parse_vtr_flow(temp_dir, parse_config_file=None, metrics_filepath=None, verbosity=1): + vtr.print_verbose(1, verbosity, "Parsing results") + vtr.mkdir_p(temp_dir) + if parse_config_file is None: + parse_config_file = vtr.find_vtr_file("vtr_benchmarks.txt") + + parse_patterns = vtr.load_parse_patterns(parse_config_file) + + metrics = OrderedDict() + + #Set defaults + for parse_pattern in parse_patterns.values(): + + if parse_pattern.default_value() != None: + metrics[parse_pattern.name()] = parse_pattern.default_value() + else: + metrics[parse_pattern.name()] = "" + + #Process each pattern + for parse_pattern in parse_patterns.values(): + + #We interpret the parse pattern's filename as a glob pattern + filepattern = str(Path(temp_dir) / parse_pattern.filename()) + filepaths = glob.glob(filepattern) + + num_files = len(filepaths) + + if num_files > 1: + raise vtr.InspectError("File pattern '{}' is ambiguous ({} files matched)".format(parse_pattern.filename()), num_files, filepaths) + + elif num_files == 1: + filepath = filepaths[0] + + assert Path(filepath).exists + + with open(filepath) as f: + for line in f: + match = parse_pattern.regex().match(line) + if match: + #Extract the first group value + metrics[parse_pattern.name()] = match.groups()[0] + else: + #No matching file, skip + assert num_files == 0 + + if metrics_filepath is None: + metrics_filepath = str(Path(temp_dir) / "parse_results.txt") + + vtr.write_tab_delimitted_csv(metrics_filepath, [metrics]) + + return metrics + +def should_run_stage(stage, flow_start_stage, flow_end_stage): + """ + Returns True if stage falls between flow_start_stage and flow_end_stage + """ + if flow_start_stage <= stage <= flow_end_stage: + return True + return False + + + + diff --git a/vtr_flow/scripts/python_libs/verilogtorouting/inspect.py b/vtr_flow/scripts/python_libs/vtr/inspect.py similarity index 99% rename from vtr_flow/scripts/python_libs/verilogtorouting/inspect.py rename to vtr_flow/scripts/python_libs/vtr/inspect.py index bbde9263ce2..121b5127a76 100644 --- a/vtr_flow/scripts/python_libs/verilogtorouting/inspect.py +++ b/vtr_flow/scripts/python_libs/vtr/inspect.py @@ -8,8 +8,7 @@ #Fall back on python implementation import xml.etree.ElementTree as ET -from util import load_config_lines -from error import InspectError +from vtr import load_config_lines, InspectError class ParsePattern: def __init__(self, name, filename, regex_str, default_value=None): diff --git a/vtr_flow/scripts/python_libs/vtr/odin/__init__.py b/vtr_flow/scripts/python_libs/vtr/odin/__init__.py new file mode 100644 index 00000000000..541e332898d --- /dev/null +++ b/vtr_flow/scripts/python_libs/vtr/odin/__init__.py @@ -0,0 +1 @@ +from .odin import run \ No newline at end of file diff --git a/vtr_flow/scripts/python_libs/vtr/odin/odin.py b/vtr_flow/scripts/python_libs/vtr/odin/odin.py new file mode 100644 index 00000000000..01b98c224b1 --- /dev/null +++ b/vtr_flow/scripts/python_libs/vtr/odin/odin.py @@ -0,0 +1,39 @@ +import shutil +from pathlib import Path +from vtr import mkdir_p, find_vtr_file, file_replace, determine_memory_addr_width + +def run(architecture_file, circuit_file, + output_netlist, + command_runner, + temp_dir=".", + log_filename="odin.out", + odin_exec=None, + odin_config=None, + min_hard_mult_size=3, + min_hard_adder_size=1): + mkdir_p(temp_dir) + + if odin_exec == None: + odin_exec = find_vtr_file('odin_II', is_executable=True) + + if odin_config == None: + odin_base_config = find_vtr_file('basic_odin_config_split.xml') + + #Copy the config file + odin_config = "odin_config.xml" + odin_config_full_path = str(Path(temp_dir) / odin_config) + shutil.copyfile(odin_base_config, odin_config_full_path) + + #Update the config file + file_replace(odin_config_full_path, { + "XXX": circuit_file.name, + "YYY": architecture_file.name, + "ZZZ": output_netlist.name, + "PPP": determine_memory_addr_width(str(architecture_file)), + "MMM": min_hard_mult_size, + "AAA": min_hard_adder_size, + }) + + cmd = [odin_exec, "-c", odin_config, "--adder_type", "default", "-U0"] + + command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) \ No newline at end of file diff --git a/vtr_flow/scripts/python_libs/verilogtorouting/task.py b/vtr_flow/scripts/python_libs/vtr/task.py similarity index 98% rename from vtr_flow/scripts/python_libs/verilogtorouting/task.py rename to vtr_flow/scripts/python_libs/vtr/task.py index 8b1d398ae35..c93b1dc2daa 100644 --- a/vtr_flow/scripts/python_libs/verilogtorouting/task.py +++ b/vtr_flow/scripts/python_libs/vtr/task.py @@ -1,7 +1,6 @@ import os -from verilogtorouting.error import * -from util import find_vtr_root, find_vtr_file, load_list_file +from vtr import error, find_vtr_root, load_list_file class TaskConfig: """ diff --git a/vtr_flow/scripts/python_libs/verilogtorouting/util.py b/vtr_flow/scripts/python_libs/vtr/util.py similarity index 99% rename from vtr_flow/scripts/python_libs/verilogtorouting/util.py rename to vtr_flow/scripts/python_libs/vtr/util.py index f2f7d395f63..41cd2b4a0f4 100644 --- a/vtr_flow/scripts/python_libs/verilogtorouting/util.py +++ b/vtr_flow/scripts/python_libs/vtr/util.py @@ -9,7 +9,7 @@ import csv from collections import OrderedDict -from verilogtorouting.error import * +from vtr import error VERBOSITY_CHOICES = range(5) @@ -182,7 +182,7 @@ def write_tab_delimitted_csv(filepath, rows): columns[key] = max(columns[key], len(str(value))) #Write the elements - with open(filepath, 'w') as f: + with open(filepath, 'w+') as f: writer = csv.writer(f, delimiter='\t') #Write out the header diff --git a/vtr_flow/scripts/python_libs/vtr/vpr/__init__.py b/vtr_flow/scripts/python_libs/vtr/vpr/__init__.py new file mode 100644 index 00000000000..afa018cffa7 --- /dev/null +++ b/vtr_flow/scripts/python_libs/vtr/vpr/__init__.py @@ -0,0 +1 @@ +from .vpr import run,run_relax_W \ No newline at end of file diff --git a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py new file mode 100644 index 00000000000..f40f48c3b18 --- /dev/null +++ b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py @@ -0,0 +1,88 @@ +import shutil +from pathlib import Path +from vtr import mkdir_p, find_vtr_file, CommandRunner, print_verbose, relax_W, determine_lut_size, determine_min_W + +def run_relax_W(architecture, circuit_name, circuit, command_runner=CommandRunner(), temp_dir=".", + relax_W_factor=1.3, vpr_exec=None, verbosity=1, logfile_base="vpr", + vpr_args=None, output_netlist=None): + """ + Runs VPR twice: + 1st: To find the minimum channel width + 2nd: At relaxed channel width (e.g. for critical path delay) + + Arguments + --------- + architecture: Architecture file + circuit: Input circuit netlist + command_runner: CommandRunner object + temp_dir: Directory to run in + + relax_W_factor: Factor by which to relax minimum channel width for critical path delay routing + verbosity: How much progress output to produce + logfile_base: Base name for log files (e.g. "vpr" produces vpr.min_W.out, vpr.relaxed_W.out) + vpr_args: Extra arguments for VPR + vpr_exec: Path to the VPR executable + output_netlist: Output implementation netlist to generate + """ + if vpr_args is None: + vpr_args = OrderedDict() + + mkdir_p(temp_dir) + + vpr_min_W_log = '.'.join([logfile_base, "min_W", "out"]) + vpr_relaxed_W_log = '.'.join([logfile_base, "relaxed_W", "out"]) + + print_verbose(1, verbosity, "Running VPR (determining minimum channel width)" ) + + run(architecture, circuit_name, circuit, command_runner, temp_dir, log_filename=vpr_min_W_log, vpr_exec=vpr_exec, vpr_args=vpr_args) + + if ('pack' in vpr_args or 'place' in vpr_args) and 'route' not in vpr_args: + #Don't look for min W if routing was not run + return + + + min_W = determine_min_W(str(Path(temp_dir) / vpr_min_W_log)) + + relaxed_W = relax_W(min_W, relax_W_factor) + + print_verbose(1, verbosity, "Running VPR (at {fac}x relaxed minimum channel width)".format(fac=relax_W_factor)) + + vpr_args['route'] = True #Re-route only + vpr_args['route_chan_width'] = relaxed_W #At a fixed channel width + + #VPR does not support performing routing when fixed pins + # are specified, and placement is not run; so remove the option + if 'fix_pins' in vpr_args: + del vpr_args['fix_pins'] + + run(architecture, circuit_name, circuit, command_runner, temp_dir, log_filename=vpr_relaxed_W_log, vpr_exec=vpr_exec, vpr_args=vpr_args) + + +def run(architecture, circuit_name, circuit, command_runner, temp_dir, output_netlist=None, log_filename="vpr.out", vpr_exec=None, vpr_args=None): + """ + Runs VPR with the specified configuration + """ + if vpr_args is None: + vpr_args = OrderedDict() + + mkdir_p(temp_dir) + + if vpr_exec == None: + vpr_exec = find_vtr_file('vpr', is_executable=True) + + cmd = [vpr_exec, architecture.name, circuit_name.name, "--circuit_file", circuit.name] + + #Enable netlist generation + #if output_netlist: + #vpr_args['gen_postsynthesis_netlist'] = output_netlist + + #Translate arbitrary keyword arguments into options for VPR + for arg, value in vpr_args.iteritems(): + if value == True: + cmd += ["--" + arg] + elif value == False: + pass + else: + cmd += ["--" + arg, str(value)] + + command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) \ No newline at end of file diff --git a/vtr_flow/scripts/vtr-flow.py b/vtr_flow/scripts/vtr-flow.py index a7f8f414493..1849a48fe4e 100755 --- a/vtr_flow/scripts/vtr-flow.py +++ b/vtr_flow/scripts/vtr-flow.py @@ -1,6 +1,6 @@ #!/usr/bin/env python2 import sys -import os +from pathlib import Path import errno import argparse import subprocess @@ -12,27 +12,23 @@ from collections import OrderedDict -sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), 'python_libs')) - -from verilogtorouting.flow import run_vtr_flow, parse_vtr_flow, VTR_STAGE, vtr_stages, CommandRunner -from verilogtorouting.error import * -from verilogtorouting.util import print_verbose, RawDefaultHelpFormatter, VERBOSITY_CHOICES, find_vtr_file, format_elapsed_time +sys.path.insert(0, str(Path(__file__).resolve().parent / 'python_libs')) +import vtr BASIC_VERBOSITY = 1 - class VtrStageArgparseAction(argparse.Action): def __call__(self, parser, namespace, value, option_string=None): if value == "odin": - setattr(namespace, self.dest, VTR_STAGE.odin) + setattr(namespace, self.dest, vtr.VTR_STAGE.odin) elif value == "abc": - setattr(namespace, self.dest, VTR_STAGE.abc) + setattr(namespace, self.dest, vtr.VTR_STAGE.abc) elif value == "vpr": - setattr(namespace, self.dest, VTR_STAGE.vpr) + setattr(namespace, self.dest, vtr.VTR_STAGE.vpr) elif value == "lec": - setattr(namespace, self.dest, VTR_STAGE.lec) + setattr(namespace, self.dest, vtr.VTR_STAGE.lec) else: raise argparse.ArgumentError(self, "Invalid VTR stage '" + value + "'") @@ -98,7 +94,7 @@ def vtr_command_argparser(prog=None): usage=usage, description=description, epilog=epilog, - formatter_class=RawDefaultHelpFormatter, + formatter_class=vtr.RawDefaultHelpFormatter, ) # @@ -109,19 +105,19 @@ def vtr_command_argparser(prog=None): parser.add_argument('architecture_file', help="The FPGA architecture to target.") parser.add_argument("-start", "-starting_stage", - choices=VTR_STAGE.reverse_mapping.values(), - default=VTR_STAGE.odin, + choices=vtr.VTR_STAGE.reverse_mapping.values(), + default=vtr.VTR_STAGE.odin, action=VtrStageArgparseAction, help="Starting stage of the VTR flow.") parser.add_argument("-end", "-ending_stage", - choices=VTR_STAGE.reverse_mapping.values(), - default=VTR_STAGE.vpr, + choices=vtr.VTR_STAGE.reverse_mapping.values(), + default=vtr.VTR_STAGE.vpr, action=VtrStageArgparseAction, help="Ending stage of the VTR flow.") - parser.add_argument("-v", "-verbose", - choices=VERBOSITY_CHOICES, + parser.add_argument("-verbose", "-v", + choices=vtr.VERBOSITY_CHOICES, default=2, type=int, help="Verbosity of the script. Higher values produce more output.") @@ -208,10 +204,10 @@ def vtr_command_main(arg_list, prog=None): #Load the arguments args, unkown_args = vtr_command_argparser(prog).parse_known_args(arg_list) - print_verbose(BASIC_VERBOSITY, args.verbosity, "# {} {}\n".format(prog, ' '.join(arg_list))) + vtr.print_verbose(BASIC_VERBOSITY, args.verbose, "# {} {}\n".format(prog, ' '.join(arg_list))) - abs_path_arch_file = os.path.abspath(args.architecture_file) - abs_path_circuit_file = os.path.abspath(args.circuit_file) + abs_path_arch_file = str(Path(args.architecture_file)) + abs_path_circuit_file = str(Path(args.circuit_file)) if (args.temp_dir == "."): temp_dir="./temp" else: @@ -219,24 +215,24 @@ def vtr_command_main(arg_list, prog=None): #Specify how command should be run - command_runner = CommandRunner(track_memory=args.track_memory_usage, + command_runner = vtr.CommandRunner(track_memory=args.track_memory_usage, max_memory_mb=args.limit_memory_usage, timeout_sec=args.timeout, - verbose_error=True if args.verbosity == 2 else False, - verbose=True if args.verbosity > 2 else False, - echo_cmd=True if args.verbosity >= 4 else False) + verbose_error=True if args.verbose == 2 else False, + verbose=True if args.verbose > 2 else False, + echo_cmd=True if args.verbose >= 4 else False) exit_status = 0 - flow_type = 2 #Use iterative black-boxing flow for multi-clock circuits + abc_flow_type = 2 #Use iterative black-boxing flow for multi-clock circuits use_old_latches_restoration_script = 0 if(args.iterative_bb): - flow_type =2 + abc_flow_type =2 if(args.once_bb): - flow_type = 1 + abc_flow_type = 1 if(args.blanket_bb): - flow_type = 3 + abc_flow_type = 3 if(args.use_old_latches_restoration_script): use_old_latches_restoration_script = 1 @@ -246,32 +242,32 @@ def vtr_command_main(arg_list, prog=None): vpr_args = process_unkown_args(unkown_args) #Run the flow - run_vtr_flow(abs_path_arch_file, + vtr.run(abs_path_arch_file, abs_path_circuit_file, power_tech_file=args.power_tech, temp_dir=temp_dir, start_stage=args.start, end_stage=args.end, command_runner=command_runner, - verbosity=args.verbosity, + verbosity=args.verbose, vpr_args=vpr_args, - flow_type=flow_type, + abc_flow_type=abc_flow_type, use_old_latches_restoration_script=use_old_latches_restoration_script ) - except CommandError as e: + except vtr.CommandError as e: #An external command failed print "Error: {msg}".format(msg=e.msg) print "\tfull command: ", ' '.join(e.cmd) print "\treturncode : ", e.returncode print "\tlog file : ", e.log exit_status = 1 - except InspectError as e: + except vtr.InspectError as e: #Something went wrong gathering information print "Error: {msg}".format(msg=e.msg) print "\tfile : ", e.filename exit_status = 2 - except VtrError as e: + except vtr.VtrError as e: #Generic VTR errors print "Error: ", e.msg exit_status = 3 @@ -282,14 +278,14 @@ def vtr_command_main(arg_list, prog=None): #Parse the flow results try: - parse_vtr_flow(temp_dir, args.parse_config_file, verbosity=args.verbosity) - except InspectError as e: + vtr.parse_vtr_flow(temp_dir, args.parse_config_file, verbosity=args.verbose) + except vtr.InspectError as e: print "Error: {msg}".format(msg=e.msg) print "\tfile : ", e.filename exit_status = 2 finally: - print_verbose(BASIC_VERBOSITY, args.verbosity, "\n# {} took {} (exiting {})".format(prog, format_elapsed_time(datetime.now() - start), exit_status)) + vtr.print_verbose(BASIC_VERBOSITY, args.verbose, "\n# {} took {} (exiting {})".format(prog, vtr.format_elapsed_time(datetime.now() - start), exit_status)) sys.exit(exit_status) def process_unkown_args(unkown_args): @@ -304,7 +300,7 @@ def process_unkown_args(unkown_args): continue if not arg.startswith('-'): - raise VtrError("Extra argument '{}' intended for VPR does not start with '-'".format(arg)) + raise vtr.VtrError("Extra argument '{}' intended for VPR does not start with '-'".format(arg)) #To make it a valid kwargs dictionary we trim the initial '-' or '--' from the #argument name diff --git a/vtr_flow/scripts/vtr-task.py b/vtr_flow/scripts/vtr-task.py index 1df01ada571..ee358ef9553 100755 --- a/vtr_flow/scripts/vtr-task.py +++ b/vtr_flow/scripts/vtr-task.py @@ -12,12 +12,7 @@ sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), 'python_libs')) -import verilogtorouting as vtr -from verilogtorouting.error import * -from verilogtorouting.util import load_list_file, find_vtr_file, mkdir_p, print_verbose, find_vtr_root, CommandRunner, format_elapsed_time, RawDefaultHelpFormatter, VERBOSITY_CHOICES, argparse_str2bool, get_next_run_dir, get_latest_run_dir -from verilogtorouting.task import load_task_config, TaskConfig, find_task_config_file -from verilogtorouting.flow import CommandRunner -from verilogtorouting.inspect import load_pass_requirements, load_parse_results +from vtr import VtrError, InspectError, CommandError, load_list_file, find_vtr_file, mkdir_p, print_verbose, find_vtr_root, CommandRunner, format_elapsed_time, RawDefaultHelpFormatter, VERBOSITY_CHOICES, argparse_str2bool, get_next_run_dir, get_latest_run_dir, load_task_config, TaskConfig, find_task_config_file, CommandRunner, load_pass_requirements, load_parse_results BASIC_VERBOSITY = 1 FAILED_LOG_VERBOSITY = 2 @@ -436,7 +431,7 @@ def create_jobs(args, configs): executable = [config.script_path] else: #Default flow script - executable = [find_vtr_file('vtr', is_executabe=True), 'flow'] + executable = [find_vtr_file('vtr-flow.py', is_executable=True)] #Collect any extra script params from the config file script_params = [abs_arch_filepath, abs_circuit_filepath] diff --git a/vtr_flow/scripts/vtr-test.py b/vtr_flow/scripts/vtr-test.py index 0a0748e5f61..66cdf5b4add 100755 --- a/vtr_flow/scripts/vtr-test.py +++ b/vtr_flow/scripts/vtr-test.py @@ -11,11 +11,8 @@ sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), 'python_libs')) -import verilogtorouting as vtr -from verilogtorouting.error import * -from verilogtorouting.util import load_list_file, find_vtr_file, mkdir_p, print_verbose, find_vtr_root, CommandRunner, format_elapsed_time, RawDefaultHelpFormatter, VERBOSITY_CHOICES -from verilogtorouting.task import load_task_config, TaskConfig, find_task_config_file -from verilogtorouting.flow import CommandRunner +import vtr +from vtr import error, load_list_file, find_vtr_file, mkdir_p, print_verbose, find_vtr_root, CommandRunner, format_elapsed_time, RawDefaultHelpFormatter, VERBOSITY_CHOICES, load_task_config, TaskConfig, find_task_config_file, CommandRunner BASIC_VERBOSITY=1 From 8102fc5853c39cca7778c6ef73b058bc8dde74c9 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Sat, 16 May 2020 10:00:14 -0700 Subject: [PATCH 08/86] fixing minor import issue --- vtr_flow/scripts/python_libs/vtr/__init__.py | 2 +- vtr_flow/scripts/python_libs/vtr/inspect.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/vtr_flow/scripts/python_libs/vtr/__init__.py b/vtr_flow/scripts/python_libs/vtr/__init__.py index 1f025e0ad28..80b453a7c6c 100644 --- a/vtr_flow/scripts/python_libs/vtr/__init__.py +++ b/vtr_flow/scripts/python_libs/vtr/__init__.py @@ -4,6 +4,6 @@ from .vpr import run,run_relax_W from .odin import run from .ace import run -from .error import * +from .error import InspectError, VtrError, CommandError from .flow import run, VTR_STAGE, parse_vtr_flow from .task import load_task_config, TaskConfig, find_task_config_file diff --git a/vtr_flow/scripts/python_libs/vtr/inspect.py b/vtr_flow/scripts/python_libs/vtr/inspect.py index 121b5127a76..66a78401d61 100644 --- a/vtr_flow/scripts/python_libs/vtr/inspect.py +++ b/vtr_flow/scripts/python_libs/vtr/inspect.py @@ -8,7 +8,8 @@ #Fall back on python implementation import xml.etree.ElementTree as ET -from vtr import load_config_lines, InspectError +from vtr import load_config_lines +from vtr.error import InspectError class ParsePattern: def __init__(self, name, filename, regex_str, default_value=None): From 8d3b29d6a75620d2fbaacc45fc43df45d0bb2c2e Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Tue, 19 May 2020 13:56:36 -0700 Subject: [PATCH 09/86] replaced os with pathlib in vtr-task, vtr-flow, flow and util, also fixed some import issues along the way. --- vtr_flow/scripts/python_libs/vtr/task.py | 28 +++++---- vtr_flow/scripts/python_libs/vtr/util.py | 72 +++++++++++------------ vtr_flow/scripts/vtr-task.py | 75 ++++++++++++------------ vtr_flow/scripts/vtr-test.py | 10 ++-- 4 files changed, 93 insertions(+), 92 deletions(-) diff --git a/vtr_flow/scripts/python_libs/vtr/task.py b/vtr_flow/scripts/python_libs/vtr/task.py index c93b1dc2daa..901a6adf29d 100644 --- a/vtr_flow/scripts/python_libs/vtr/task.py +++ b/vtr_flow/scripts/python_libs/vtr/task.py @@ -1,6 +1,7 @@ -import os +from pathlib import Path +from pathlib import PurePath -from vtr import error, find_vtr_root, load_list_file +from vtr import VtrError, find_vtr_root, load_list_file class TaskConfig: """ @@ -97,9 +98,9 @@ def load_task_config(config_file): raise VtrError("Missing required key '{key}' in config file {file}".format(key=required_key, file=config_file)) #Useful meta-data about the config - config_dir = os.path.dirname(config_file) + config_dir = str(Path(config_file).parent) key_values['config_dir'] = config_dir - key_values['task_name'] = os.path.basename(os.path.dirname(config_dir)) + key_values['task_name'] = Path(config_dir).parent.name #Create the task config object return TaskConfig(**key_values) @@ -112,7 +113,7 @@ def find_task_config_file(task_name): # base_dirs = [] - if os.path.isabs(task_name): + if PurePath(task_name).is_absolute: #Only check the root path since the path is aboslute base_dirs.append('/') else: @@ -120,7 +121,7 @@ def find_task_config_file(task_name): base_dirs.append('.') vtr_root = find_vtr_root() - vtr_flow_tasks_dir = os.path.join(vtr_root, "vtr_flow", "tasks") + vtr_flow_tasks_dir = str(PurePath(vtr_root) / "vtr_flow" / "tasks") #Then the VTR tasks directory base_dirs.append(vtr_flow_tasks_dir) @@ -129,25 +130,26 @@ def find_task_config_file(task_name): potential_config_file_paths = [] for base_dir in base_dirs: #Assume points directly to a config.txt - assume_config_path = os.path.join(base_dir, task_name) + assume_config_path = str(PurePath(base_dir) / task_name) potential_config_file_paths.append(assume_config_path) #Assume points to a config dir (containing config.txt) - assume_config_dir_path = os.path.join(base_dir, task_name, "config.txt") + assume_config_dir_path = str(PurePath(base_dir) / task_name / "config.txt") potential_config_file_paths.append(assume_config_dir_path) #Assume points to a task dir (containing config/config.txt) - assume_task_dir_path = os.path.join(base_dir, task_name, "config", "config.txt") + assume_task_dir_path = str(PurePath(base_dir) / task_name / "config" / "config.txt") potential_config_file_paths.append(assume_task_dir_path) #Find the first potential file that is valid for config_file in potential_config_file_paths: - is_file = os.path.isfile(config_file) - is_named_config = os.path.basename(config_file) == "config.txt" - is_in_config_dir = os.path.basename(os.path.dirname(config_file)) == "config" + config_path = Path(config_file) + is_file = config_path.is_file() + is_named_config = config_path.name == "config.txt" + is_in_config_dir = config_path.parent.name == "config" if is_file and is_named_config and is_in_config_dir: - return os.path.abspath(config_file) + return config_path.resolve() raise VtrError("Could not find config/config.txt for task {name}".format(name=task_name)) diff --git a/vtr_flow/scripts/python_libs/vtr/util.py b/vtr_flow/scripts/python_libs/vtr/util.py index 41cd2b4a0f4..3b951927c2e 100644 --- a/vtr_flow/scripts/python_libs/vtr/util.py +++ b/vtr_flow/scripts/python_libs/vtr/util.py @@ -1,4 +1,6 @@ import os +from pathlib import PurePath +from pathlib import Path import sys import re import time @@ -64,7 +66,7 @@ def run_system_command(self, cmd, temp_dir, log_filename=None, expected_return_c #If no log file is specified the name is based on the executed command if log_filename == None: - log_filename = os.path.basename(orig_cmd[0]) + '.out' + log_filename = PurePath(orig_cmd[0]).name + '.out' #Limit memory usage? @@ -106,15 +108,15 @@ def run_system_command(self, cmd, temp_dir, log_filename=None, expected_return_c # # We do this rather than use proc.communicate() # to get interactive output - with open(os.path.join(temp_dir, log_filename), 'w') as log_f: + with (Path(temp_dir) / log_filename).open('w') as log_f: #Print the command at the top of the log - print >> log_f, " ".join(cmd) + log_f.write(" ".join(cmd).decode('utf-8')) #Read from subprocess output for line in proc.stdout: #Send to log file - print >> log_f, line, + log_f.write(line.decode('unicode_escape') ) #Save the output cmd_output.append(line) @@ -146,9 +148,9 @@ def run_system_command(self, cmd, temp_dir, log_filename=None, expected_return_c if cmd_errored: - raise CommandError(msg="Executable {exec_name} failed".format(exec_name=os.path.basename(orig_cmd[0])), + raise CommandError(msg="Executable {exec_name} failed".format(exec_name=PurePath(orig_cmd[0]).name), cmd=cmd, - log=os.path.join(temp_dir, log_filename), + log=str(PurePath(temp_dir).joinpath(log_filename)), returncode=cmd_returncode) return cmd_output, cmd_returncode @@ -233,10 +235,11 @@ def mkdir_p(path): """ Makes a directory (including parents) at the specified path """ + make_path = Path(path) try: - os.makedirs(path) + make_path.mkdir() except OSError as exc: # Python >2.5 - if exc.errno == errno.EEXIST and os.path.isdir(path): + if exc.errno == errno.EEXIST and make_path.is_dir(): pass else: raise @@ -262,9 +265,9 @@ def find_vtr_file(filename, is_executable=False): """ #We assume exectuables are specified in the unix style (no .exe), # if it was specified with .exe, strip it off - filebase, ext = os.path.splitext(filename) - if ext == ".exe": - filename = filebase + file_path = PurePath(filename) + if file_path.suffix == ".exe": + filename = file_path.name # #Check if it is on the path (provided it is executable) @@ -300,20 +303,17 @@ def find_file_from_vtr_root(filename, vtr_root, is_executable=False): Given a vtr_root and a filename searches for the file recursively under some common VTR directories """ for subdir in ['vpr', 'abc', 'abc_with_bb_support', 'ODIN_II', 'vtr_flow', 'ace2']: - - for root, dirs, files in os.walk(os.path.join(vtr_root, subdir)): - for file in files: - if file == filename: - full_file_path = os.path.join(root, file) - - if os.path.isfile(full_file_path): - if is_executable: - #Found an executable file as required - if os.access(full_file_path, os.X_OK): - return full_file_path - else: - #Found a file as required - return full_file_path + directory_path = (Path(vtr_root) / subdir) + for file_path in directory_path.glob('**/*'): + if file_path.name == filename: + if file_path.is_file: + if is_executable: + #Found an executable file as required + if os.access(str(file_path), os.X_OK): + return str(file_path) + else: + #Found a file as required + return str(file_path) return None def find_vtr_root(): @@ -321,13 +321,11 @@ def find_vtr_root(): if env_var in os.environ: return os.environ[env_var] - inferred_script_dir = os.path.dirname(os.path.abspath(__file__)) - #We assume that this file is in /vtr_flow/python_libs/verilogtorouting - inferred_vtr_root = os.path.abspath(os.path.join(inferred_script_dir, '../../../..')) + inferred_vtr_root = Path(__file__).parent / '../../../..' - if os.path.isdir(inferred_vtr_root): - return inferred_vtr_root + if inferred_vtr_root.is_dir: + return str(inferred_vtr_root) else: raise VtrError("Could not find VTR root directory. Try setting VTR_ROOT environment variable.") @@ -403,7 +401,7 @@ def load_config_lines(filepath, allow_includes=True): assert len(components) == 2 include_file = components[1].strip('"') #Strip quotes - include_file_abs = os.path.join(os.path.dirname(filepath), include_file) + include_file_abs = str(Path(filepath).paren / include_file) #Recursively load the config config_lines += load_config_lines(include_file_abs, allow_includes=allow_includes) @@ -439,7 +437,7 @@ def get_next_run_dir(base_dir): Does not create the directory """ - return os.path.join(base_dir, run_dir_name(get_next_run_number(base_dir))) + return str(PurePath(base_dir) / run_dir_name(get_next_run_number(base_dir))) def get_latest_run_dir(base_dir): """ @@ -450,7 +448,7 @@ def get_latest_run_dir(base_dir): if latest_run_number == None: return None - return os.path.join(base_dir, run_dir_name(latest_run_number)) + return str(PurePath(base_dir) / run_dir_name(latest_run_number)) def get_next_run_number(base_dir): """ @@ -470,15 +468,15 @@ def get_latest_run_number(base_dir): Returns the highest run number of all run directories with in base_dir """ run_number = 0 - run_dir = os.path.join(base_dir, run_dir_name(run_number)) + run_dir = Path(base_dir) / run_dir_name(run_number) - if not os.path.exists(run_dir): + if not run_dir.exists: #No existing run directories return None - while os.path.exists(run_dir): + while run_dir.exists: run_number += 1 - run_dir = os.path.join(base_dir, run_dir_name(run_number)) + run_dir = Path(base_dir) / run_dir_name(run_number) #Currently one-past the last existing run dir, #to get latest existing, subtract one diff --git a/vtr_flow/scripts/vtr-task.py b/vtr_flow/scripts/vtr-task.py index ee358ef9553..9422f80e094 100755 --- a/vtr_flow/scripts/vtr-task.py +++ b/vtr_flow/scripts/vtr-task.py @@ -1,5 +1,6 @@ #!/usr/bin/env python2 -import os +from pathlib import Path +from pathlib import PurePath import sys import argparse import itertools @@ -10,7 +11,7 @@ from datetime import datetime from multiprocessing import Pool -sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), 'python_libs')) +sys.path.insert(0, str(Path(__file__).resolve().parent / 'python_libs')) from vtr import VtrError, InspectError, CommandError, load_list_file, find_vtr_file, mkdir_p, print_verbose, find_vtr_root, CommandRunner, format_elapsed_time, RawDefaultHelpFormatter, VERBOSITY_CHOICES, argparse_str2bool, get_next_run_dir, get_latest_run_dir, load_task_config, TaskConfig, find_task_config_file, CommandRunner, load_pass_requirements, load_parse_results @@ -37,13 +38,13 @@ def circuit(self): return self._circuit def job_name(self): - return os.path.join(self.arch(), self.circuit()) + return str(PurePath(self.arch()).joinpath(self.circuit())) def command(self): return self._command def work_dir(self, run_dir): - return os.path.join(run_dir, self._work_dir) + return str(PurePath(run_dir).joinpath(self._work_dir)) def vtr_command_argparser(prog=None): description = textwrap.dedent( @@ -257,7 +258,7 @@ def parse_task(args, config, config_jobs, task_metrics_filepath=None, flow_metri subprocess.check_call(cmd, cwd=job.work_dir(run_dir)) if task_metrics_filepath is None: - task_metrics_filepath = task_parse_results_filepath = os.path.join(run_dir, "parse_results.txt") + task_metrics_filepath = task_parse_results_filepath = str(PurePath(run_dir).joinpath("parse_results.txt")) #Record max widths for pretty printing max_arch_len = len("architecture") @@ -277,9 +278,9 @@ def parse_task(args, config, config_jobs, task_metrics_filepath=None, flow_metri # #The job results file is basically the same format, but excludes the architecture and circuit fields, #which we prefix to each line of the task result file - job_parse_results_filepath = os.path.join(run_dir, job.arch(), job.circuit(), flow_metrics_basename) - if os.path.isfile(job_parse_results_filepath): - with open(job_parse_results_filepath) as in_f: + job_parse_results_filepath = Path(run_dir) / job.arch / job.circuit / flow_metrics_basename + if job_parse_results_filepath.exists: + with job_parse_results_filepath.open() as in_f: lines = in_f.readlines() assert len(lines) == 2 @@ -292,7 +293,7 @@ def parse_task(args, config, config_jobs, task_metrics_filepath=None, flow_metri #Second line is the data print >>out_f, "{:<{arch_width}}\t{:<{circuit_width}}\t{}".format(job.arch(), job.circuit(), lines[1], arch_width=max_arch_len, circuit_width=max_circuit_len), else: - print_verbose(BASIC_VERBOSITY, args.verbosity, "Warning: Flow result file not found (task QoR will be incomplete): {} ".format(job_parse_results_filepath)) + print_verbose(BASIC_VERBOSITY, args.verbosity, "Warning: Flow result file not found (task QoR will be incomplete): {} ".format(str(job_parse_results_filepath))) def create_golden_results_for_tasks(args, configs): for config in configs: @@ -304,8 +305,8 @@ def create_golden_results_for_task(args, config): """ run_dir = find_latest_run_dir(args, config) - task_results = os.path.join(run_dir, "parse_results.txt") - golden_results_filepath = os.path.join(config.config_dir, "golden_results.txt") + task_results = str(PurePath(run_dir).joinpath("parse_results.txt")) + golden_results_filepath = str(PurePath(config.config_dir).joinpath("golden_results.txt")) print_verbose(BASIC_VERBOSITY, args.verbosity, "Creating golden task results from {} -> {}".format(run_dir, golden_results_filepath)) @@ -333,15 +334,15 @@ def check_golden_results_for_task(args, config): else: #Load the pass requirements file - pass_req_filepath = os.path.join(find_vtr_root(), 'vtr_flow', 'parse', 'pass_requirements', config.pass_requirements_file) + pass_req_filepath = str(PurePath(find_vtr_root).joinpath('vtr_flow').joinpath('parse').joinpath('pass_requirements').joinpath(config.pass_requirements_file)) pass_requirements = load_pass_requirements(pass_req_filepath) #Load the task's parse results - task_results_filepath = os.path.join(run_dir, "parse_results.txt") + task_results_filepath = str(PurePath(run_dir).joinpath("parse_results.txt")) task_results = load_parse_results(task_results_filepath) #Load the golden reference - golden_results_filepath = os.path.join(config.config_dir, "golden_results.txt") + golden_results_filepath = str(PurePath(config.config_dir).joinpath("golden_results.txt")) golden_results = load_parse_results(golden_results_filepath) #Verify that the architecture and circuit are specified @@ -409,12 +410,12 @@ def check_golden_results_for_task(args, config): reason = e.msg if not metric_passed: - print_verbose(BASIC_VERBOSITY, args.verbosity, " FAILED {} {} {}/{}: {} {}".format(os.path.basename(run_dir), config.task_name, arch, circuit, metric, reason)) + print_verbose(BASIC_VERBOSITY, args.verbosity, " FAILED {} {} {}/{}: {} {}".format(PurePath(run_dir).name, config.task_name, arch, circuit, metric, reason)) num_qor_failures += 1 if num_qor_failures == 0: print_verbose(BASIC_VERBOSITY, args.verbosity, - " PASSED {} {}".format(os.path.basename(run_dir), config.task_name)) + " PASSED {} {}".format(PurePath(run_dir).name, config.task_name)) return num_qor_failures @@ -445,7 +446,7 @@ def create_jobs(args, configs): script_params += ["--fix_pins", resolve_vtr_source_file(config, config.pad_file)] if config.parse_file: - script_params += ["--parse_config_file", resolve_vtr_source_file(config, config.parse_file, os.path.join("parse", "parse_config"))] + script_params += ["--parse_config_file", resolve_vtr_source_file(config, config.parse_file, str(PurePath("parse").joinpath("parse_config")))] #We specify less verbosity to the sub-script # This keeps the amount of output reasonable @@ -453,7 +454,7 @@ def create_jobs(args, configs): cmd = executable + script_params - work_dir = os.path.join(arch, circuit) + work_dir = str(PurePath(arch).joinpath(circuit)) jobs.append(Job(config.task_name, arch, circuit, work_dir, cmd)) @@ -467,21 +468,21 @@ def find_latest_run_dir(args, config): if not run_dir: raise InspectError("Failed to find run directory for task '{}' in '{}'".format(config.task_name, task_dir)) - assert os.path.isdir(run_dir) + assert Path(run_dir).is_dir() return run_dir def find_task_dir(args, config): task_dir = None if args.work_dir: - task_dir = os.path.join(args.work_dir, config.task_name) + task_dir = str(PurePath(args.work_dir).joinpath(config.task_name)) else: #Task dir is just above the config directory - task_dir = os.path.dirname(config.config_dir) - assert os.path.isdir(task_dir) + task_dir = Path(config.config_dir).parent + assert task_dir.is_dir - return task_dir + return str(task_dir) def run_parallel(args, configs, queued_jobs): """ @@ -520,7 +521,7 @@ def run_parallel(args, configs, queued_jobs): work_dir = job.work_dir(run_dirs[job.task_name()]) mkdir_p(work_dir) - log_filepath = os.path.join(work_dir, "vtr_flow.log") + log_filepath = str(PurePath(work_dir).joinpath("vtr_flow.log")) log_file = open(log_filepath, 'w+') @@ -621,26 +622,28 @@ def resolve_vtr_source_file(config, filename, base_dir=""): """ #Absolute path - if os.path.isabs(filename): + if PurePath(filename).is_absolute(): return filename #Under config - assert os.path.isabs(config.config_dir) - joined_path = os.path.join(config.config_dir, filename) - if os.path.exists(joined_path): - return joined_path + config_path = Path(config.config_dir) + assert config_path.is_absolute() + joined_path = config_path / filename + if joined_path.exists(): + return str(joined_path) #Under base dir - if os.path.isabs(base_dir): + base_path = Path(base_dir) + if base_path.is_absolute(): #Absolute base - joined_path = os.path.join(base_dir, filename) - if os.path.exists(joined_path): - return joined_path + joined_path = base_path / filename + if joined_path.exists(): + return str(joined_path) else: #Relative base under the VTR flow directory - joined_path = os.path.join(find_vtr_root(), 'vtr_flow', base_dir, filename) - if os.path.exists(joined_path): - return joined_path + joined_path = Path(find_vtr_root()) / 'vtr_flow' / base_dir / filename + if joined_path.exists(): + return str(joined_path) #Not found raise InspectError("Failed to resolve VTR source file {}".format(filename)) diff --git a/vtr_flow/scripts/vtr-test.py b/vtr_flow/scripts/vtr-test.py index 66cdf5b4add..d17dacd290d 100755 --- a/vtr_flow/scripts/vtr-test.py +++ b/vtr_flow/scripts/vtr-test.py @@ -1,5 +1,5 @@ #!/usr/bin/env python2 -import os +from pathlib import Path import sys import argparse import itertools @@ -9,7 +9,7 @@ from datetime import datetime from multiprocessing import Pool -sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), 'python_libs')) +sys.path.insert(0, str(Path(__file__).resolve().parent / 'python_libs')) import vtr from vtr import error, load_list_file, find_vtr_file, mkdir_p, print_verbose, find_vtr_root, CommandRunner, format_elapsed_time, RawDefaultHelpFormatter, VERBOSITY_CHOICES, load_task_config, TaskConfig, find_task_config_file, CommandRunner @@ -116,9 +116,7 @@ def vtr_command_main(arg_list, prog=None): if reg_test.startswith("vtr"): base_testname = reg_test.split('_')[-1] - task_list_filepath = os.path.join(find_vtr_root(), - 'vtr_flow', 'tasks', 'regression_tests', - 'vtr_reg_' + base_testname, 'task_list.txt') + task_list_filepath = str(Path(find_vtr_root()) / 'vtr_flow' / 'tasks' / 'regression_tests' / ('vtr_reg_' + base_testname) / 'task_list.txt') vtr_task_list_files.append(task_list_filepath) #Run the actual tasks, recording functionality failures @@ -151,7 +149,7 @@ def run_odin_test(args, test_name): assert odin_reg_script - odin_root = os.path.dirname(odin_reg_script) + odin_root = str(Path(odin_reg_script).resolve().parent) result = subprocess.call(odin_reg_script, cwd=odin_root) From ea28ed6724f0b040852b89b9b0cb9c06a4098811 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Wed, 20 May 2020 10:29:08 -0700 Subject: [PATCH 10/86] upgraded vtr-flow, vtr-test and vtr-task to python3 also made the python libs used in those files compatable with python3 --- vtr_flow/scripts/python_libs/vtr/util.py | 26 +++++++++---------- vtr_flow/scripts/python_libs/vtr/vpr/vpr.py | 2 +- vtr_flow/scripts/vtr-flow.py | 22 ++++++++-------- vtr_flow/scripts/vtr-task.py | 28 ++++++++++----------- vtr_flow/scripts/vtr-test.py | 5 ++-- 5 files changed, 42 insertions(+), 41 deletions(-) diff --git a/vtr_flow/scripts/python_libs/vtr/util.py b/vtr_flow/scripts/python_libs/vtr/util.py index 3b951927c2e..63ff6cd8542 100644 --- a/vtr_flow/scripts/python_libs/vtr/util.py +++ b/vtr_flow/scripts/python_libs/vtr/util.py @@ -86,7 +86,7 @@ def run_system_command(self, cmd, temp_dir, log_filename=None, expected_return_c #Echo the command? if self._echo_cmd: #print ' '.join(cmd) - print cmd + print(cmd) #Begin timing start_time = time.time() @@ -110,13 +110,13 @@ def run_system_command(self, cmd, temp_dir, log_filename=None, expected_return_c # to get interactive output with (Path(temp_dir) / log_filename).open('w') as log_f: #Print the command at the top of the log - log_f.write(" ".join(cmd).decode('utf-8')) + log_f.write(" ".join(cmd)) #Read from subprocess output for line in proc.stdout: #Send to log file - log_f.write(line.decode('unicode_escape') ) + log_f.write(line) #Save the output cmd_output.append(line) @@ -144,10 +144,10 @@ def run_system_command(self, cmd, temp_dir, log_filename=None, expected_return_c #Send to stdout if self._verbose or (cmd_errored and self._verbose_error): for line in cmd_output: - print indent_depth*self._indent + line, + print (indent_depth*self._indent + line,) - if cmd_errored: + if (cmd_errored): raise CommandError(msg="Executable {exec_name} failed".format(exec_name=PurePath(orig_cmd[0]).name), cmd=cmd, log=str(PurePath(temp_dir).joinpath(log_filename)), @@ -176,7 +176,7 @@ def write_tab_delimitted_csv(filepath, rows): #Calculate the max width of each column columns = OrderedDict() for row in rows: - for key, value in row.iteritems(): + for key, value in row.items(): if key not in columns: columns[key] = max(len(key), len(str(value))) @@ -189,14 +189,14 @@ def write_tab_delimitted_csv(filepath, rows): #Write out the header header = [] - for col_name, col_width in columns.iteritems(): + for col_name, col_width in columns.items(): header.append("{:{width}}".format(col_name, width=col_width)) writer.writerow(header) #Write rows for row in rows: values = [] - for col_name, col_width in columns.iteritems(): + for col_name, col_width in columns.items(): values.append("{:{width}}".format(row[col_name], width=col_width)) writer.writerow(values) @@ -227,7 +227,7 @@ def make_enum(*sequential, **named): >> MY_ENUM = enum(ONE, TWO, THREE) """ enums = dict(zip(sequential, range(len(sequential))), **named) - reverse = dict((value, key) for key, value in enums.iteritems()) + reverse = dict((value, key) for key, value in enums.items()) enums['reverse_mapping'] = reverse return type('Enum', (), enums) @@ -250,9 +250,9 @@ def print_verbose(min_verbosity, curr_verbosity, string, endl=True): """ if curr_verbosity >= min_verbosity: if endl: - print string + print(string) else: - print string, + print (string,) def find_vtr_file(filename, is_executable=False): """ @@ -336,9 +336,9 @@ def file_replace(filename, search_replace_dict): with open(filename, 'w') as f: for line in lines: - for search, replace in search_replace_dict.iteritems(): + for search, replace in search_replace_dict.items(): line = line.replace(search, str(replace)) - print >>f, line, + print (line,file=f) def relax_W(min_W, relax_factor, base=2): """ diff --git a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py index f40f48c3b18..8bbe25369ff 100644 --- a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py +++ b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py @@ -77,7 +77,7 @@ def run(architecture, circuit_name, circuit, command_runner, temp_dir, output_ne #vpr_args['gen_postsynthesis_netlist'] = output_netlist #Translate arbitrary keyword arguments into options for VPR - for arg, value in vpr_args.iteritems(): + for arg, value in vpr_args.items(): if value == True: cmd += ["--" + arg] elif value == False: diff --git a/vtr_flow/scripts/vtr-flow.py b/vtr_flow/scripts/vtr-flow.py index 1849a48fe4e..118402d2f18 100755 --- a/vtr_flow/scripts/vtr-flow.py +++ b/vtr_flow/scripts/vtr-flow.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 import sys from pathlib import Path import errno @@ -256,32 +256,32 @@ def vtr_command_main(arg_list, prog=None): ) except vtr.CommandError as e: #An external command failed - print "Error: {msg}".format(msg=e.msg) - print "\tfull command: ", ' '.join(e.cmd) - print "\treturncode : ", e.returncode - print "\tlog file : ", e.log + print ("Error: {msg}".format(msg=e.msg)) + print ("\tfull command: ", ' '.join(e.cmd)) + print ("\treturncode : ", e.returncode) + print ("\tlog file : ", e.log) exit_status = 1 except vtr.InspectError as e: #Something went wrong gathering information - print "Error: {msg}".format(msg=e.msg) - print "\tfile : ", e.filename + print ("Error: {msg}".format(msg=e.msg)) + print ("\tfile : ", e.filename) exit_status = 2 except vtr.VtrError as e: #Generic VTR errors - print "Error: ", e.msg + print ("Error: ", e.msg) exit_status = 3 except KeyboardInterrupt as e: - print "{} recieved keyboard interrupt".format(prog) + print ("{} recieved keyboard interrupt".format(prog)) exit_status = 4 #Parse the flow results try: vtr.parse_vtr_flow(temp_dir, args.parse_config_file, verbosity=args.verbose) except vtr.InspectError as e: - print "Error: {msg}".format(msg=e.msg) - print "\tfile : ", e.filename + print ("Error: {msg}".format(msg=e.msg)) + print ("\tfile : ", e.filename) exit_status = 2 finally: diff --git a/vtr_flow/scripts/vtr-task.py b/vtr_flow/scripts/vtr-task.py index 9422f80e094..335682c5aa1 100755 --- a/vtr_flow/scripts/vtr-task.py +++ b/vtr_flow/scripts/vtr-task.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 from pathlib import Path from pathlib import PurePath import sys @@ -166,7 +166,7 @@ def vtr_command_main(arg_list, prog=None): args.parse = True if args.print_metadata: - print "# {} {}\n".format(prog, ' '.join(arg_list)) + print ("# {} {}\n".format(prog, ' '.join(arg_list))) num_failed = -1 try: @@ -182,19 +182,19 @@ def vtr_command_main(arg_list, prog=None): num_failed = run_tasks(args, configs) except CommandError as e: - print "Error: {msg}".format(msg=e.msg) - print "\tfull command: ", e.cmd - print "\treturncode : ", e.returncode - print "\tlog file : ", e.log + print ("Error: {msg}".format(msg=e.msg)) + print ("\tfull command: ", e.cmd) + print ("\treturncode : ", e.returncode) + print ("\tlog file : ", e.log) except InspectError as e: - print "Error: {msg}".format(msg=e.msg) + print ("Error: {msg}".format(msg=e.msg)) if e.filename: - print "\tfile: ", e.filename + print ("\tfile: ", e.filename) except VtrError as e: - print "Error:", e.msg + print ("Error:", e.msg) finally: if args.print_metadata: - print "\n# {} took {} (exiting {})".format(prog, format_elapsed_time(datetime.now() - start), num_failed) + print ("\n# {} took {} (exiting {})".format(prog, format_elapsed_time(datetime.now() - start), num_failed)) sys.exit(num_failed) def run_tasks(args, configs): @@ -577,7 +577,7 @@ def run_parallel(args, configs, queued_jobs): time.sleep(0.1) except KeyboardInterrupt as e: - print "Recieved KeyboardInterrupt -- halting workers" + print ("Recieved KeyboardInterrupt -- halting workers") for proc, job, log_file in running_procs: proc.terminate() log_file.close() @@ -587,7 +587,7 @@ def run_parallel(args, configs, queued_jobs): finally: if len(running_procs) > 0: - print "Killing {} worker processes".format(len(running_procs)) + print ("Killing {} worker processes".format(len(running_procs))) for proc, job, log_file in running_procs: proc.kill() log_file.close() @@ -603,8 +603,8 @@ def print_log(log_file, indent=" "): #Print log for line in log_file: line = line.rstrip() - print indent + line - print "" + print (indent + line) + print ("") #Return to original position log_file.seek(curr_pos) diff --git a/vtr_flow/scripts/vtr-test.py b/vtr_flow/scripts/vtr-test.py index d17dacd290d..d5aa5d1e0d6 100755 --- a/vtr_flow/scripts/vtr-test.py +++ b/vtr_flow/scripts/vtr-test.py @@ -1,5 +1,6 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 from pathlib import Path +from itertools import chain import sys import argparse import itertools @@ -68,7 +69,7 @@ def vtr_command_argparser(prog=None): help="How many processors to use for execution.") parser.add_argument("-v", "--verbosity", - choices=VERBOSITY_CHOICES + [5], + choices=list(chain(VERBOSITY_CHOICES, [5])), default=2, type=int, help="Sets the verbosity of the script. Higher values produce more output.") From 476a894f12ef8783417cc087aacb42df65b5b0eb Mon Sep 17 00:00:00 2001 From: Jeff Goeders Date: Thu, 21 May 2020 13:53:40 -0600 Subject: [PATCH 11/86] Adding unit testing for new run vtr flow py script --- .travis.yml | 4 + .../scripts/unittests/check_py_vtr_flow.py | 81 +++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100755 vtr_flow/scripts/unittests/check_py_vtr_flow.py diff --git a/.travis.yml b/.travis.yml index c99b0481202..ef798536df5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -87,6 +87,10 @@ jobs: #script: #- ./.github/travis/build.sh + - stage: Test + name: "PY scripts" + script: + - ./vtr_flow/scripts/unittests/check_py_vtr_flow.py - stage: Test name: "Code Formatting" script: diff --git a/vtr_flow/scripts/unittests/check_py_vtr_flow.py b/vtr_flow/scripts/unittests/check_py_vtr_flow.py new file mode 100755 index 00000000000..6aa5469dfb9 --- /dev/null +++ b/vtr_flow/scripts/unittests/check_py_vtr_flow.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 + +import subprocess +import pathlib +import sys +import shutil +import filecmp + +# VTR paths +scripts_path = pathlib.Path(__file__).resolve().parent.parent +vtr_flow_path = scripts_path.parent +temp_dir = pathlib.Path.cwd() / "temp" + +# run_vtr_flow.pl will be compared against vtr-flow.py using the following test cases as arguments: +arg_list = ["benchmarks/verilog/ch_intrinsics.v arch/timing/k6_N10_mem32K_40nm.xml -track_memory_usage --router_heap binary --min_route_chan_width_hint 38"] + +# The following output files will be compared to ensure they are identical (only one file must match each entry) +# The first list item is the glob pattern, the second is the number of lines to ignore at the beginning of the file +files_to_validate = [ ("*.route", 1), + ("report_timing.setup.rpt",0) + ] + +def error(*msg, return_code=-1): + print("!!!! ERROR:", " ".join(str(item) for item in msg)) + sys.exit(return_code) + +# Find a single file matching a pattern at a given path +def find_file_matching(path, pattern): + matches = [f for f in path.glob(pattern)] + if len(matches) == 0: + error("No files match", path / pattern) + elif len(matches) > 1: + error("Multiple files match", path / pattern) + return matches[0] + +# Run +def run_vtr_flow(script_name, run_name, temp_dir_run): + cmd = [scripts_path / script_name, *(arg_list[0].split()), "-temp_dir", temp_dir_run] + print(" Running", run_name,"flow:", cmd) + p = subprocess.run(cmd, cwd = vtr_flow_path, stdout = subprocess.PIPE, stderr=subprocess.STDOUT) + if (p.returncode): + print(p.stdout.decode()) + error("Run VTR failed") + +# Remove lines from start of a file +def remove_file_lines_head(path, num_lines): + with open(path, 'r') as fin: + data = fin.read().splitlines(True) + with open(path, 'w') as fout: + fout.writelines(data[num_lines:]) + +def main(): + + for test_str in arg_list: + # Delete all files in temp directory + if temp_dir.is_dir(): + shutil.rmtree(temp_dir) + temp_dir.mkdir() + + print("Testing", test_str) + temp_dir_pl = temp_dir / "pl" + temp_dir_py = temp_dir / "py" + + # Run with old and new flows + run_vtr_flow("run_vtr_flow.pl", "old pl", temp_dir_pl) + run_vtr_flow("run_vtr_flow.pl", "new py", temp_dir_py) + + # Check that output files match + for (pattern, skip_lines) in files_to_validate: + pl_file = find_file_matching(temp_dir_pl, pattern) + remove_file_lines_head(pl_file, skip_lines) + py_file = find_file_matching(temp_dir_py, pattern) + remove_file_lines_head(py_file, skip_lines) + + if not filecmp.cmp(pl_file, py_file): + error(pl_file, py_file, "differ") + else: + print(" ", pl_file, py_file, "match") + +if __name__ == "__main__": + main() \ No newline at end of file From 3dcb940a9775815b7a272930e1c9cfd7502437ff Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Thu, 21 May 2020 15:29:44 -0700 Subject: [PATCH 12/86] Reorganized abc arguments and fixed error handling --- vtr_flow/scripts/python_libs/vtr/__init__.py | 2 +- vtr_flow/scripts/python_libs/vtr/abc/abc.py | 26 ++++++++++++++++++-- vtr_flow/scripts/python_libs/vtr/flow.py | 7 +++--- vtr_flow/scripts/python_libs/vtr/util.py | 2 +- vtr_flow/scripts/vtr-flow.py | 14 +++++------ vtr_flow/scripts/vtr-task.py | 3 ++- 6 files changed, 37 insertions(+), 17 deletions(-) diff --git a/vtr_flow/scripts/python_libs/vtr/__init__.py b/vtr_flow/scripts/python_libs/vtr/__init__.py index 80b453a7c6c..9a7ae903f6e 100644 --- a/vtr_flow/scripts/python_libs/vtr/__init__.py +++ b/vtr_flow/scripts/python_libs/vtr/__init__.py @@ -4,6 +4,6 @@ from .vpr import run,run_relax_W from .odin import run from .ace import run -from .error import InspectError, VtrError, CommandError +from .error import * from .flow import run, VTR_STAGE, parse_vtr_flow from .task import load_task_config, TaskConfig, find_task_config_file diff --git a/vtr_flow/scripts/python_libs/vtr/abc/abc.py b/vtr_flow/scripts/python_libs/vtr/abc/abc.py index 278559e78a7..9cb04bd537b 100644 --- a/vtr_flow/scripts/python_libs/vtr/abc/abc.py +++ b/vtr_flow/scripts/python_libs/vtr/abc/abc.py @@ -1,12 +1,31 @@ import shutil from pathlib import Path -from vtr import mkdir_p, find_vtr_file, determine_lut_size, error +from vtr import mkdir_p, find_vtr_file, determine_lut_size +from vtr.error import VtrError, InspectError, CommandError -def run(architecture_file, circuit_file, output_netlist, command_runner, temp_dir=".", log_filename="abc.opt_techmap.out", abc_exec=None, abc_script=None, abc_rc=None,use_old_abc_script = False,abc_flow_type=2,use_old_latches_restoration_script=1): +def run(architecture_file, circuit_file, output_netlist, command_runner, temp_dir=".", log_filename="abc.opt_techmap.out", abc_exec=None, abc_script=None, abc_rc=None, use_old_abc_script = False, abc_args = ""): mkdir_p(temp_dir) blackbox_latches_script = find_vtr_file("blackbox_latches.pl") clk_list = [] clk_log_file = "report_clk_out.out" + # + # Parse arguments + # + abc_flow_type = 2 + abc_run_args = "" + use_old_latches_restoration_script = 0 + for i in abc_args: + if(i=="iterative_bb"): + abc_flow_type=2 + elif(i=="blanket_bb"): + abc_flow_type=3 + elif(i=="once_bb"): + abc_flow_type=1 + elif(i=="use_old_latches_restoration_script"): + use_old_latches_restoration_script = 1 + else: + abc_run_args += i + if(abc_flow_type): populate_clock_list(circuit_file,blackbox_latches_script,clk_list,command_runner,temp_dir,log_filename) @@ -72,6 +91,9 @@ def run(architecture_file, circuit_file, output_netlist, command_runner, temp_di abc_script = "; ".join(abc_script) cmd = [abc_exec, '-c', abc_script] + + if(abc_run_args !=""): + cmd.append(abc_run_args) command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) diff --git a/vtr_flow/scripts/python_libs/vtr/flow.py b/vtr_flow/scripts/python_libs/vtr/flow.py index a031a6ed313..c394acab322 100644 --- a/vtr_flow/scripts/python_libs/vtr/flow.py +++ b/vtr_flow/scripts/python_libs/vtr/flow.py @@ -18,8 +18,7 @@ def run(architecture_file, circuit_file, temp_dir="./temp", verbosity=0, vpr_args=None, - abc_flow_type=2, - use_old_latches_restoration_script=1): + abc_args=""): """ Runs the VTR CAD flow to map the specificied circuit_file onto the target architecture_file @@ -56,7 +55,7 @@ def run(architecture_file, circuit_file, architecture_ext = architecture_path.suffixes vtr.mkdir_p(temp_dir) - + #Define useful filenames post_odin_netlist = Path(temp_dir) / (circuit_name + '.odin.blif') post_abc_netlist =Path(temp_dir) / (circuit_name + '.abc.blif') @@ -109,7 +108,7 @@ def run(architecture_file, circuit_file, output_netlist=post_abc_netlist, command_runner=command_runner, temp_dir=temp_dir, - abc_flow_type=abc_flow_type) + abc_args=abc_args) next_stage_netlist = post_abc_netlist diff --git a/vtr_flow/scripts/python_libs/vtr/util.py b/vtr_flow/scripts/python_libs/vtr/util.py index 63ff6cd8542..b76ce9d106e 100644 --- a/vtr_flow/scripts/python_libs/vtr/util.py +++ b/vtr_flow/scripts/python_libs/vtr/util.py @@ -11,7 +11,7 @@ import csv from collections import OrderedDict -from vtr import error +from vtr.error import VtrError, InspectError, CommandError VERBOSITY_CHOICES = range(5) diff --git a/vtr_flow/scripts/vtr-flow.py b/vtr_flow/scripts/vtr-flow.py index 118402d2f18..d8fbef4038b 100755 --- a/vtr_flow/scripts/vtr-flow.py +++ b/vtr_flow/scripts/vtr-flow.py @@ -222,20 +222,19 @@ def vtr_command_main(arg_list, prog=None): verbose=True if args.verbose > 2 else False, echo_cmd=True if args.verbose >= 4 else False) exit_status = 0 - abc_flow_type = 2 #Use iterative black-boxing flow for multi-clock circuits - use_old_latches_restoration_script = 0 + abc_args = [] if(args.iterative_bb): - abc_flow_type =2 + abc_args.append("iterative_bb") if(args.once_bb): - abc_flow_type = 1 + abc_args.append("once_bb") if(args.blanket_bb): - abc_flow_type = 3 + abc_args.append("blanket_bb") if(args.use_old_latches_restoration_script): - use_old_latches_restoration_script = 1 + abc_args.append("use_old_latches_restoration_script") try: if not args.parse_only: try: @@ -251,8 +250,7 @@ def vtr_command_main(arg_list, prog=None): command_runner=command_runner, verbosity=args.verbose, vpr_args=vpr_args, - abc_flow_type=abc_flow_type, - use_old_latches_restoration_script=use_old_latches_restoration_script + abc_args=abc_args ) except vtr.CommandError as e: #An external command failed diff --git a/vtr_flow/scripts/vtr-task.py b/vtr_flow/scripts/vtr-task.py index 335682c5aa1..a6ce9b8d2fd 100755 --- a/vtr_flow/scripts/vtr-task.py +++ b/vtr_flow/scripts/vtr-task.py @@ -13,7 +13,8 @@ sys.path.insert(0, str(Path(__file__).resolve().parent / 'python_libs')) -from vtr import VtrError, InspectError, CommandError, load_list_file, find_vtr_file, mkdir_p, print_verbose, find_vtr_root, CommandRunner, format_elapsed_time, RawDefaultHelpFormatter, VERBOSITY_CHOICES, argparse_str2bool, get_next_run_dir, get_latest_run_dir, load_task_config, TaskConfig, find_task_config_file, CommandRunner, load_pass_requirements, load_parse_results +from vtr import load_list_file, find_vtr_file, mkdir_p, print_verbose, find_vtr_root, CommandRunner, format_elapsed_time, RawDefaultHelpFormatter, VERBOSITY_CHOICES, argparse_str2bool, get_next_run_dir, get_latest_run_dir, load_task_config, TaskConfig, find_task_config_file, CommandRunner, load_pass_requirements, load_parse_results +from vtr.error import VtrError, InspectError, CommandError BASIC_VERBOSITY = 1 FAILED_LOG_VERBOSITY = 2 From 092ea950307a3ea7d1e92e94cab3aa4e6c7981d9 Mon Sep 17 00:00:00 2001 From: Jeff Goeders Date: Thu, 21 May 2020 16:51:40 -0600 Subject: [PATCH 13/86] Fix py version bug in check_py_vtr_flow --- vtr_flow/scripts/unittests/check_py_vtr_flow.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/vtr_flow/scripts/unittests/check_py_vtr_flow.py b/vtr_flow/scripts/unittests/check_py_vtr_flow.py index 6aa5469dfb9..33ef5c2c8f0 100755 --- a/vtr_flow/scripts/unittests/check_py_vtr_flow.py +++ b/vtr_flow/scripts/unittests/check_py_vtr_flow.py @@ -12,7 +12,9 @@ temp_dir = pathlib.Path.cwd() / "temp" # run_vtr_flow.pl will be compared against vtr-flow.py using the following test cases as arguments: -arg_list = ["benchmarks/verilog/ch_intrinsics.v arch/timing/k6_N10_mem32K_40nm.xml -track_memory_usage --router_heap binary --min_route_chan_width_hint 38"] +arg_list = ["benchmarks/verilog/ch_intrinsics.v arch/timing/k6_N10_mem32K_40nm.xml -track_memory_usage --router_heap binary --min_route_chan_width_hint 38", + "benchmarks/verilog/diffeq1.v arch/timing/k6_N10_mem32K_40nm.xml -track_memory_usage --router_heap bucket --min_route_chan_width_hint 46" + ] # The following output files will be compared to ensure they are identical (only one file must match each entry) # The first list item is the glob pattern, the second is the number of lines to ignore at the beginning of the file @@ -34,8 +36,9 @@ def find_file_matching(path, pattern): return matches[0] # Run -def run_vtr_flow(script_name, run_name, temp_dir_run): - cmd = [scripts_path / script_name, *(arg_list[0].split()), "-temp_dir", temp_dir_run] +def run_vtr_flow(script_name, run_name, temp_dir_run, args): + arg_list = args.split() + cmd = [scripts_path / script_name,] + arg_list + ["-temp_dir", temp_dir_run] print(" Running", run_name,"flow:", cmd) p = subprocess.run(cmd, cwd = vtr_flow_path, stdout = subprocess.PIPE, stderr=subprocess.STDOUT) if (p.returncode): @@ -62,8 +65,8 @@ def main(): temp_dir_py = temp_dir / "py" # Run with old and new flows - run_vtr_flow("run_vtr_flow.pl", "old pl", temp_dir_pl) - run_vtr_flow("run_vtr_flow.pl", "new py", temp_dir_py) + run_vtr_flow("run_vtr_flow.pl", "old pl", temp_dir_pl, test_str) + run_vtr_flow("vtr-flow.py", "new py", temp_dir_py, test_str) # Check that output files match for (pattern, skip_lines) in files_to_validate: From dfbce12e1d211c4c72b93a6c607186fc3a07b906 Mon Sep 17 00:00:00 2001 From: Jeff Goeders Date: Thu, 21 May 2020 17:00:25 -0600 Subject: [PATCH 14/86] Add python3 to travis --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index ef798536df5..1640d149b74 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,6 +44,7 @@ addons: - libxml++2.6-dev - perl - python + - python3.6 - python-lxml - texinfo - time From abbb41fc1ac6e24d83a2391cf72e7494b925de19 Mon Sep 17 00:00:00 2001 From: Jeff Goeders Date: Thu, 21 May 2020 17:00:41 -0600 Subject: [PATCH 15/86] Update chekc_py_vtr_flow to use python3.6 --- vtr_flow/scripts/unittests/check_py_vtr_flow.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/vtr_flow/scripts/unittests/check_py_vtr_flow.py b/vtr_flow/scripts/unittests/check_py_vtr_flow.py index 33ef5c2c8f0..7c21c85a202 100755 --- a/vtr_flow/scripts/unittests/check_py_vtr_flow.py +++ b/vtr_flow/scripts/unittests/check_py_vtr_flow.py @@ -37,8 +37,7 @@ def find_file_matching(path, pattern): # Run def run_vtr_flow(script_name, run_name, temp_dir_run, args): - arg_list = args.split() - cmd = [scripts_path / script_name,] + arg_list + ["-temp_dir", temp_dir_run] + cmd = [scripts_path / script_name, *(args.split()), "-temp_dir", temp_dir_run] print(" Running", run_name,"flow:", cmd) p = subprocess.run(cmd, cwd = vtr_flow_path, stdout = subprocess.PIPE, stderr=subprocess.STDOUT) if (p.returncode): From 5410b07225f4c193a67f877b143953ccd29c8b98 Mon Sep 17 00:00:00 2001 From: Jeff Goeders Date: Thu, 21 May 2020 17:13:36 -0600 Subject: [PATCH 16/86] New ppa source for travis-ci and python3.6 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 1640d149b74..ebbcc767671 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,6 +15,7 @@ addons: - sourceline: 'deb http://apt.llvm.org/trusty llvm-toolchain-trusty-8 main' key_url: 'https://apt.llvm.org/llvm-snapshot.gpg.key' - sourceline: 'ppa:ubuntu-toolchain-r/test' + - sourceline: 'ppa:deadsnakes/ppa' # For python3.6 packages: - autoconf - automake From 3f048cbf507a71bf4ced0ca01f3af2a95c5368d1 Mon Sep 17 00:00:00 2001 From: Jeff Goeders Date: Thu, 21 May 2020 17:17:54 -0600 Subject: [PATCH 17/86] check_py_vtr_flow to use python3.6 --- vtr_flow/scripts/unittests/check_py_vtr_flow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vtr_flow/scripts/unittests/check_py_vtr_flow.py b/vtr_flow/scripts/unittests/check_py_vtr_flow.py index 7c21c85a202..8531db2caae 100755 --- a/vtr_flow/scripts/unittests/check_py_vtr_flow.py +++ b/vtr_flow/scripts/unittests/check_py_vtr_flow.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python3.6 import subprocess import pathlib From c9c11c24bffbac0fe0a8cfff51805cda7f2f922a Mon Sep 17 00:00:00 2001 From: Jeff Goeders Date: Thu, 21 May 2020 17:22:41 -0600 Subject: [PATCH 18/86] travis-ci for py scripts needs to run build first --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index ebbcc767671..fe6c7a72d75 100644 --- a/.travis.yml +++ b/.travis.yml @@ -92,6 +92,7 @@ jobs: - stage: Test name: "PY scripts" script: + - ./.github/travis/build.sh - ./vtr_flow/scripts/unittests/check_py_vtr_flow.py - stage: Test name: "Code Formatting" From d270635649996a06b0a2c3eea5c46e1f2289b371 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Thu, 21 May 2020 17:12:23 -0700 Subject: [PATCH 19/86] vtr-flow.py:fixing track_memory_usage issue --- vtr_flow/scripts/vtr-flow.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vtr_flow/scripts/vtr-flow.py b/vtr_flow/scripts/vtr-flow.py index d8fbef4038b..873276786e6 100755 --- a/vtr_flow/scripts/vtr-flow.py +++ b/vtr_flow/scripts/vtr-flow.py @@ -153,9 +153,9 @@ def vtr_command_argparser(prog=None): help="Directory to run the flow in (will be created if non-existant).") house_keeping.add_argument("-track_memory_usage", - choices=on_off_choices, - default="on", - action=OnOffArgparseAction, + default=False, + action="store_true", + dest="track_memory_usage", help="Track the memory usage for each stage. Requires /usr/bin/time -v, disabled if not available.") house_keeping.add_argument("-limit_memory_usage", From a6ab0d93afc04c0256b7e4c1d70b3a9d655a6fad Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Fri, 29 May 2020 12:57:44 -0700 Subject: [PATCH 20/86] Enabling Power estimation by fixing ACE script. also enabled the -delete_result_files and -delete_intermediate_files options in the flow --- vtr_flow/scripts/python_libs/vtr/__init__.py | 2 +- vtr_flow/scripts/python_libs/vtr/abc/abc.py | 21 ++++++++--- vtr_flow/scripts/python_libs/vtr/ace/ace.py | 12 ------- .../vtr/{ace => ace_flow}/__init__.py | 0 .../scripts/python_libs/vtr/ace_flow/ace.py | 29 +++++++++++++++ vtr_flow/scripts/python_libs/vtr/flow.py | 36 ++++++++++++++----- vtr_flow/scripts/python_libs/vtr/vpr/vpr.py | 2 +- vtr_flow/scripts/vtr-flow.py | 20 +++++++++-- 8 files changed, 94 insertions(+), 28 deletions(-) delete mode 100644 vtr_flow/scripts/python_libs/vtr/ace/ace.py rename vtr_flow/scripts/python_libs/vtr/{ace => ace_flow}/__init__.py (100%) create mode 100644 vtr_flow/scripts/python_libs/vtr/ace_flow/ace.py diff --git a/vtr_flow/scripts/python_libs/vtr/__init__.py b/vtr_flow/scripts/python_libs/vtr/__init__.py index 9a7ae903f6e..c40ea9394be 100644 --- a/vtr_flow/scripts/python_libs/vtr/__init__.py +++ b/vtr_flow/scripts/python_libs/vtr/__init__.py @@ -3,7 +3,7 @@ from .abc import run, run_lec from .vpr import run,run_relax_W from .odin import run -from .ace import run +from .ace_flow import run from .error import * from .flow import run, VTR_STAGE, parse_vtr_flow from .task import load_task_config, TaskConfig, find_task_config_file diff --git a/vtr_flow/scripts/python_libs/vtr/abc/abc.py b/vtr_flow/scripts/python_libs/vtr/abc/abc.py index 9cb04bd537b..0737ae04805 100644 --- a/vtr_flow/scripts/python_libs/vtr/abc/abc.py +++ b/vtr_flow/scripts/python_libs/vtr/abc/abc.py @@ -3,7 +3,12 @@ from vtr import mkdir_p, find_vtr_file, determine_lut_size from vtr.error import VtrError, InspectError, CommandError -def run(architecture_file, circuit_file, output_netlist, command_runner, temp_dir=".", log_filename="abc.opt_techmap.out", abc_exec=None, abc_script=None, abc_rc=None, use_old_abc_script = False, abc_args = ""): +def run(architecture_file, circuit_file, + output_netlist, command_runner, + temp_dir=".", log_filename="abc.out", + abc_exec=None, abc_script=None, abc_rc=None, + use_old_abc_script = False, abc_args = None,keep_intermediate_files=1): + mkdir_p(temp_dir) blackbox_latches_script = find_vtr_file("blackbox_latches.pl") clk_list = [] @@ -45,7 +50,7 @@ def run(architecture_file, circuit_file, output_netlist, command_runner, temp_di for i in range(0, iterations): pre_abc_blif= Path(temp_dir) / (str(i)+"_" + circuit_file.name) post_abc_blif = Path(temp_dir) / (str(i)+"_"+output_netlist.name) - post_abc_raw_blif = Path(temp_dir) / (str(i)+"_raw_"+output_netlist.name) + post_abc_raw_blif = Path(temp_dir) / (str(i)+"_"+output_netlist.with_suffix('').stem+".raw.abc.blif") if(abc_flow_type==3): cmd = [blackbox_latches_script, "--input", circuit_file.name,"--output",pre_abc_blif.name] command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=str(i)+"_blackboxing_latch.out", indent_depth=1) @@ -92,10 +97,10 @@ def run(architecture_file, circuit_file, output_netlist, command_runner, temp_di cmd = [abc_exec, '-c', abc_script] - if(abc_run_args !=""): + if(abc_run_args): cmd.append(abc_run_args) - command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) + command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=str(i)+"_"+log_filename, indent_depth=1) if(abc_flow_type != 3 and len(clk_list)>i): cmd = [blackbox_latches_script,"--restore", clk_list[i], "--input", post_abc_raw_blif.name,"--output",post_abc_blif.name] @@ -113,6 +118,14 @@ def run(architecture_file, circuit_file, output_netlist, command_runner, temp_di cmd = [blackbox_latches_script, "--input", post_abc_blif.name,"--output",output_netlist.name,"--vanilla"] command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename="restore_latch" + str(i) + ".out", indent_depth=1) + if(not keep_intermediate_files): + files = [] + for file in Path(temp_dir).iterdir(): + if file.suffix in ('.dot','.v','.rc'): + files.append(file) + for p in files: + p.unlink() + def populate_clock_list(circuit_file,blackbox_latches_script,clk_list,command_runner,temp_dir,log_filename): clk_list_path = Path(temp_dir) / "report_clk.out" diff --git a/vtr_flow/scripts/python_libs/vtr/ace/ace.py b/vtr_flow/scripts/python_libs/vtr/ace/ace.py deleted file mode 100644 index 5d75a69ed91..00000000000 --- a/vtr_flow/scripts/python_libs/vtr/ace/ace.py +++ /dev/null @@ -1,12 +0,0 @@ -from vtr import find_vtr_file -def run(circuit_file, output_netlist, output_activity_file, command_runner, temp_dir=".", log_filename="ace.out", ace_exec=None): - - if ace_exec is None: - ace_exec = find_vtr_file('ace', is_executable=True) - - cmd = [ace_exec, - "-b", circuit_file.name, - "-n", output_netlist.name, - "-o", output_activity_file.name] - - command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) \ No newline at end of file diff --git a/vtr_flow/scripts/python_libs/vtr/ace/__init__.py b/vtr_flow/scripts/python_libs/vtr/ace_flow/__init__.py similarity index 100% rename from vtr_flow/scripts/python_libs/vtr/ace/__init__.py rename to vtr_flow/scripts/python_libs/vtr/ace_flow/__init__.py diff --git a/vtr_flow/scripts/python_libs/vtr/ace_flow/ace.py b/vtr_flow/scripts/python_libs/vtr/ace_flow/ace.py new file mode 100644 index 00000000000..d5c16fc5d0f --- /dev/null +++ b/vtr_flow/scripts/python_libs/vtr/ace_flow/ace.py @@ -0,0 +1,29 @@ +from vtr import find_vtr_file +from pathlib import Path +def run(circuit_file, old_netlist, output_netlist, output_activity_file, command_runner, temp_dir=".", log_filename="ace.out", ace_exec=None, ace_seed = 1): + ace_clk_file = Path(temp_dir) / "ace_clk.txt" + ace_raw = Path(temp_dir) / (circuit_file.with_suffix('').stem + ".raw.ace.blif") + if ace_exec is None: + ace_exec = find_vtr_file('ace') + + ace_extraction_file = Path(find_vtr_file("extract_clk_from_blif.py")) + + cmd =[str(ace_extraction_file), ace_clk_file.name, circuit_file.name] + command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename="ace_clk_extraction.out", indent_depth=1) + ace_clk = "" + with ace_clk_file.open('r') as f: + ace_clk = f.readline().strip('\n') + cmd = [ace_exec, + "-b", circuit_file.name, + "-c", ace_clk, + "-n", ace_raw.name, + "-o", output_activity_file.name, + "-s", str(ace_seed)] + + command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) + + clock_script = find_vtr_file("restore_multiclock_latch.pl") + + cmd = [ clock_script, old_netlist.name, ace_raw.name, output_netlist.name] + command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename="ace_clk_restore.out", indent_depth=1) + diff --git a/vtr_flow/scripts/python_libs/vtr/flow.py b/vtr_flow/scripts/python_libs/vtr/flow.py index c394acab322..42cd11dee92 100644 --- a/vtr_flow/scripts/python_libs/vtr/flow.py +++ b/vtr_flow/scripts/python_libs/vtr/flow.py @@ -18,7 +18,9 @@ def run(architecture_file, circuit_file, temp_dir="./temp", verbosity=0, vpr_args=None, - abc_args=""): + abc_args=None, + keep_intermediate_files=True, + keep_result_files=True): """ Runs the VTR CAD flow to map the specificied circuit_file onto the target architecture_file @@ -61,7 +63,7 @@ def run(architecture_file, circuit_file, post_abc_netlist =Path(temp_dir) / (circuit_name + '.abc.blif') post_ace_netlist =Path(temp_dir) / (circuit_name + ".ace.blif") post_ace_activity_file = Path(temp_dir) / (circuit_name + ".act") - pre_vpr_netlist = Path(temp_dir) / (circuit_name + ".pre_vpr.blif") + pre_vpr_netlist = Path(temp_dir) / (circuit_name + ".pre-vpr.blif") post_vpr_netlist = Path(temp_dir) / "top_post_synthesis.blif" #circuit_name + ".vpr.blif" lec_base_netlist = None #Reference netlist for LEC @@ -108,7 +110,8 @@ def run(architecture_file, circuit_file, output_netlist=post_abc_netlist, command_runner=command_runner, temp_dir=temp_dir, - abc_args=abc_args) + abc_args=abc_args, + keep_intermediate_files=keep_intermediate_files) next_stage_netlist = post_abc_netlist @@ -125,21 +128,25 @@ def run(architecture_file, circuit_file, if should_run_stage(VTR_STAGE.ace, start_stage, end_stage): vtr.print_verbose(1, verbosity, "Running ACE") - vtr.ace.run(next_stage_netlist, output_netlist=post_ace_netlist, + vtr.ace_flow.run(next_stage_netlist, old_netlist = post_odin_netlist, output_netlist=post_ace_netlist, output_activity_file=post_ace_activity_file, command_runner=command_runner, temp_dir=temp_dir) - #Use ACE's output netlist + if not keep_intermediate_files: + next_stage_netlist.unlink() + post_odin_netlist.unlink() + + #Use ACE's output netlistf next_stage_netlist = post_ace_netlist if not lec_base_netlist: lec_base_netlist = post_ace_netlist - + #Enable power analysis in VPR vpr_args["power"] = True - vpr_args["activity_file"] = post_ace_activity_file.name - vpr_args["tech_properties"] = power_tech_file + #vpr_args["activity_file"] = post_ace_activity_file.name + vpr_args["tech_properties"] = str(Path(power_tech_file).resolve()) # # Pack/Place/Route @@ -179,6 +186,19 @@ def run(architecture_file, circuit_file, if should_run_stage(VTR_STAGE.lec, start_stage, end_stage): vtr.print_verbose(1, verbosity, "Running ABC Logical Equivalence Check") vtr.abc.run_lec(lec_base_netlist, post_vpr_netlist, command_runner=command_runner, log_filename="abc.lec.out") + if(not keep_intermediate_files): + next_stage_netlist.unlink() + exts = ('.xml','.sdf','.v') + if not keep_result_files: + exts += ('.net', '.place', '.route') + files = [] + for file in Path(temp_dir).iterdir(): + if file.suffix in exts: + files.append(file) + for p in files: + p.unlink() + if power_tech_file: + post_ace_activity_file.unlink() def parse_vtr_flow(temp_dir, parse_config_file=None, metrics_filepath=None, verbosity=1): vtr.print_verbose(1, verbosity, "Parsing results") diff --git a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py index 8bbe25369ff..4fd4c8d58eb 100644 --- a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py +++ b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py @@ -70,7 +70,7 @@ def run(architecture, circuit_name, circuit, command_runner, temp_dir, output_ne if vpr_exec == None: vpr_exec = find_vtr_file('vpr', is_executable=True) - cmd = [vpr_exec, architecture.name, circuit_name.name, "--circuit_file", circuit.name] + cmd = [vpr_exec, architecture.name, circuit_name.stem, "--circuit_file", circuit.name] #Enable netlist generation #if output_netlist: diff --git a/vtr_flow/scripts/vtr-flow.py b/vtr_flow/scripts/vtr-flow.py index 873276786e6..100f8235e4c 100755 --- a/vtr_flow/scripts/vtr-flow.py +++ b/vtr_flow/scripts/vtr-flow.py @@ -109,7 +109,16 @@ def vtr_command_argparser(prog=None): default=vtr.VTR_STAGE.odin, action=VtrStageArgparseAction, help="Starting stage of the VTR flow.") - + parser.add_argument("-delete_intermediate_files", + default=True, + action="store_false", + dest="keep_intermediate_files", + help="Determines if intermediate files are deleted.") + parser.add_argument("-delete_result_files", + default=True, + action="store_false", + dest="keep_result_files", + help="Determines if result files are deleted.") parser.add_argument("-end", "-ending_stage", choices=vtr.VTR_STAGE.reverse_mapping.values(), default=vtr.VTR_STAGE.vpr, @@ -138,6 +147,11 @@ def vtr_command_argparser(prog=None): # power = parser.add_argument_group("Power", description="Power Analysis Related Options") power.add_argument("-power", + default=False, + action="store_true", + dest="do_power", + help="Track the memory usage for each stage. Requires /usr/bin/time -v, disabled if not available.") + power.add_argument("-cmos_tech", default=None, dest="power_tech", metavar="POWER_TECH_FILE", @@ -250,7 +264,9 @@ def vtr_command_main(arg_list, prog=None): command_runner=command_runner, verbosity=args.verbose, vpr_args=vpr_args, - abc_args=abc_args + abc_args=abc_args, + keep_intermediate_files=args.keep_intermediate_files, + keep_result_files=args.keep_result_files ) except vtr.CommandError as e: #An external command failed From a860661f818ceb50f4c69c33845a6f7d4988ba8b Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Fri, 29 May 2020 13:17:59 -0700 Subject: [PATCH 21/86] adding comments to abc script --- vtr_flow/scripts/python_libs/vtr/abc/abc.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/vtr_flow/scripts/python_libs/vtr/abc/abc.py b/vtr_flow/scripts/python_libs/vtr/abc/abc.py index 0737ae04805..d07c94100e7 100644 --- a/vtr_flow/scripts/python_libs/vtr/abc/abc.py +++ b/vtr_flow/scripts/python_libs/vtr/abc/abc.py @@ -63,13 +63,25 @@ def run(architecture_file, circuit_file, if abc_script == None: - abc_script = ['read {pre_abc_blif}'.format(pre_abc_blif=pre_abc_blif.name), + abc_script = ['echo ""', + 'echo "Load Netlist"', + 'echo "============"', + 'read {pre_abc_blif}'.format(pre_abc_blif=pre_abc_blif.name), 'time', + 'echo ""', + 'echo "Circuit Info"', + 'echo "=========="', 'print_stats', 'print_latch', 'time', + 'echo ""', + 'echo "LUT Costs"', + 'echo "========="', 'print_lut', 'time', + 'echo ""', + 'echo "Logic Opt + Techmap"', + 'echo "==================="', 'strash', 'ifraig -v', 'scorr -v', @@ -79,6 +91,9 @@ def run(architecture_file, circuit_file, 'mfs2 -v', 'print_stats', 'time', + 'echo ""', + 'echo "Output Netlist"', + 'echo "=============="', 'write_hie {pre_abc_blif} {post_abc_raw_blif}'.format(pre_abc_blif=pre_abc_blif.name, post_abc_raw_blif=post_abc_raw_blif.name), 'time;'] From 2f5561c0ce344fc4b76233f5c5b71fce055f1dc8 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Fri, 29 May 2020 16:58:51 -0700 Subject: [PATCH 22/86] adding name argument and show_failures, show_failures still needs to be enabled but it has been added to make running with this argument not fail. --- vtr_flow/scripts/python_libs/vtr/flow.py | 19 ++++++++--------- vtr_flow/scripts/vtr-flow.py | 26 ++++++++++++++++++------ 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/vtr_flow/scripts/python_libs/vtr/flow.py b/vtr_flow/scripts/python_libs/vtr/flow.py index 42cd11dee92..de0874c5e19 100644 --- a/vtr_flow/scripts/python_libs/vtr/flow.py +++ b/vtr_flow/scripts/python_libs/vtr/flow.py @@ -45,16 +45,13 @@ def run(architecture_file, circuit_file, # #Initial setup # + architecture_file_basename =architecture_file.name + circuit_file_basename = circuit_file.name - circuit_path = Path(circuit_file) - architecture_path = Path(architecture_file) - architecture_file_basename =architecture_path.name - circuit_file_basename = circuit_path.name - - circuit_name = circuit_path.stem - circuit_ext = circuit_path.suffixes - architecture_name = architecture_path.stem - architecture_ext = architecture_path.suffixes + circuit_name = circuit_file.stem + circuit_ext = circuit_file.suffixes + architecture_name = architecture_file.stem + architecture_ext = architecture_file.suffixes vtr.mkdir_p(temp_dir) @@ -73,8 +70,8 @@ def run(architecture_file, circuit_file, lec_base_netlist = circuit_file_basename #Copy the circuit and architecture - circuit_copy = Path(temp_dir) / circuit_path.name - architecture_copy = Path(temp_dir) / architecture_path.name + circuit_copy = Path(temp_dir) / circuit_file.name + architecture_copy = Path(temp_dir) / architecture_file.name shutil.copy(circuit_file, str(circuit_copy)) shutil.copy(architecture_file, str(architecture_copy)) diff --git a/vtr_flow/scripts/vtr-flow.py b/vtr_flow/scripts/vtr-flow.py index 100f8235e4c..10539f308da 100755 --- a/vtr_flow/scripts/vtr-flow.py +++ b/vtr_flow/scripts/vtr-flow.py @@ -165,6 +165,10 @@ def vtr_command_argparser(prog=None): house_keeping.add_argument("-temp_dir", default=".", help="Directory to run the flow in (will be created if non-existant).") + + house_keeping.add_argument("-name", + default=None, + help="Directory to run the flow in (will be created if non-existant).") house_keeping.add_argument("-track_memory_usage", default=False, @@ -172,6 +176,12 @@ def vtr_command_argparser(prog=None): dest="track_memory_usage", help="Track the memory usage for each stage. Requires /usr/bin/time -v, disabled if not available.") + house_keeping.add_argument("-show_failures", + default=False, + action="store_true", + dest="show_failures", + help="Show failures")#needs updating + house_keeping.add_argument("-limit_memory_usage", default=None, metavar="MAX_MEMORY_MB", @@ -220,8 +230,8 @@ def vtr_command_main(arg_list, prog=None): vtr.print_verbose(BASIC_VERBOSITY, args.verbose, "# {} {}\n".format(prog, ' '.join(arg_list))) - abs_path_arch_file = str(Path(args.architecture_file)) - abs_path_circuit_file = str(Path(args.circuit_file)) + path_arch_file = Path(args.architecture_file) + path_circuit_file = Path(args.circuit_file) if (args.temp_dir == "."): temp_dir="./temp" else: @@ -237,7 +247,6 @@ def vtr_command_main(arg_list, prog=None): echo_cmd=True if args.verbose >= 4 else False) exit_status = 0 abc_args = [] - if(args.iterative_bb): abc_args.append("iterative_bb") @@ -253,10 +262,15 @@ def vtr_command_main(arg_list, prog=None): if not args.parse_only: try: vpr_args = process_unkown_args(unkown_args) - + name = "" + if(args.name): + name=args.name + else: + name=path_arch_file.name+"/"+path_circuit_file.name + print(name) #Run the flow - vtr.run(abs_path_arch_file, - abs_path_circuit_file, + vtr.run(path_arch_file, + path_circuit_file, power_tech_file=args.power_tech, temp_dir=temp_dir, start_stage=args.start, From c7fb72f04e96507147900d2713a2d9060690e8a5 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Fri, 29 May 2020 20:31:20 -0700 Subject: [PATCH 23/86] Updating regression tests to work with python scripts and adding python regression test to travis --- .travis.yml | 35 ++++++++++++++++++++++++++++++++ run_reg_test.pl | 5 ++++- vtr_flow/scripts/run_vtr_task.pl | 11 +++++++--- 3 files changed, 47 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index fe6c7a72d75..f3c43bc2180 100644 --- a/.travis.yml +++ b/.travis.yml @@ -94,6 +94,41 @@ jobs: script: - ./.github/travis/build.sh - ./vtr_flow/scripts/unittests/check_py_vtr_flow.py + - stage: Test + name: "Basic Regression Tests Python" + env: + - CMAKE_PARAMS="-DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on" + - MATRIX_EVAL="CC=gcc-5 && CXX=g++-5" + script: + - ./.github/travis/build.sh + - travis_wait 30 ./run_reg_test.pl vtr_reg_basic -show_failures -j2 -script vtr-flow.py + - stage: Test + name: "Strong Regression Tests Python" + env: + - CMAKE_PARAMS="-DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on" + - MATRIX_EVAL="CC=gcc-5 && CXX=g++-5" + script: + - ./.github/travis/build.sh + - travis_wait 30 ./run_reg_test.pl vtr_reg_strong -show_failures -j2 -script vtr-flow.py + - stage: Test + name: "Basic Valgrind Memory Tests Python" + env: + - CMAKE_PARAMS="-DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on" + - MATRIX_EVAL="CC=gcc-5 && CXX=g++-5" + script: + - ./.github/travis/build.sh + - travis_wait 30 ./run_reg_test.pl vtr_reg_valgrind_small -show_failures -j2 -script vtr-flow.py + - stage: Test + name: "Sanitized Basic Regression Tests Python" + env: + - CMAKE_PARAMS="-DVTR_ASSERT_LEVEL=3 -DVTR_ENABLE_SANITIZE=on -DVTR_IPO_BUILD=off -DWITH_BLIFEXPLORER=on" + - MATRIX_EVAL="CC=gcc-5 && CXX=g++-5" + - BUILD_TYPE=debug + - LSAN_OPTIONS="exitcode=42" #Use a non-standard exit code to ensure LSAN errors are detected + script: + - ./.github/travis/build.sh + #We skip QoR since we are only checking for errors in sanitizer runs + - travis_wait 30 ./run_reg_test.pl vtr_reg_basic -show_failures -skip_qor -j2 -script vtr-flow.py - stage: Test name: "Code Formatting" script: diff --git a/run_reg_test.pl b/run_reg_test.pl index 64486d296f9..e567f97c776 100755 --- a/run_reg_test.pl +++ b/run_reg_test.pl @@ -65,6 +65,7 @@ my $skip_qor = 0; my $show_failures = 0; my $num_cpu = 1; +my $script = "run_vtr_flow.pl"; my $long_task_names = 0; # Parse Input Arguments @@ -87,6 +88,8 @@ $long_task_names = 1; } elsif ( $token eq "-j" ) { #-j N $num_cpu = int(shift(@ARGV)); + } elsif ( $token eq "-script" ) { #-j N + $script = shift(@ARGV); } elsif ( $token =~ /^-j(\d+)$/ ) { #-jN $num_cpu = int($1); } elsif ($token eq "quick_test") { @@ -337,7 +340,7 @@ sub run_single_test { print "\nRunning $test\n"; print "-------------------------------------------------------------------------------\n"; - print "scripts/run_vtr_task.pl $run_params \n"; + print "scripts/run_vtr_task.pl $run_params -script $script \n"; print "\n"; chdir("$vtr_flow_path"); my $test_status = system("scripts/run_vtr_task.pl $run_params \n"); diff --git a/vtr_flow/scripts/run_vtr_task.pl b/vtr_flow/scripts/run_vtr_task.pl index 97409c1c276..ac972c361b7 100755 --- a/vtr_flow/scripts/run_vtr_task.pl +++ b/vtr_flow/scripts/run_vtr_task.pl @@ -68,6 +68,7 @@ my $run_prefix = "run"; my $show_runtime_estimates = 1; my $system_type = "local"; +my $script = "run_vtr_flow.pl"; my $shared_script_params = ""; my $verbosity = 0; my $short_task_names = 0; @@ -96,6 +97,10 @@ $verbosity = int( shift(@ARGV) ); } + elsif ( $token eq "-script") { + $script = shift(@ARGV); + } + # Treat the remainder of the command line options as script parameters shared by all tasks elsif ( $token eq "-s" ) { $shared_script_params = join(' ', @ARGV); @@ -105,6 +110,7 @@ print "shared script params: $shared_script_params\n" } + elsif ( $token eq "-system" ) { $system_type = shift(@ARGV); } @@ -191,7 +197,7 @@ my @all_task_actions; foreach my $task (@tasks) { chomp($task); - my $task_actions = generate_single_task_actions($task, $common_task_prefix); + my $task_actions = generate_single_task_actions($task, $common_task_prefix, $script); push(@all_task_actions, @$task_actions); } @@ -211,7 +217,6 @@ sub generate_single_task_actions { my $archs_dir; my $sdc_dir = undef; my $script_default = "run_vtr_flow.pl"; - my $script = $script_default; my $script_path; my $script_params_common = $shared_script_params; # start with the shared ones then build unique ones my @circuits_list; @@ -219,7 +224,7 @@ sub generate_single_task_actions { my @script_params_list; my $cmos_tech_path = ""; - my ($task, $common_prefix) = @_; + my ($task, $common_prefix,$script) = @_; (my $task_dir = "$vtr_flow_path/tasks/$task") =~ s/\s+$//; # trim right white spaces for chdir to work on Windows chdir($task_dir) or die "Task directory does not exist ($task_dir): $!\n"; From e41cbcf09a07c19576f017f5f4bc42e720986798 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Sat, 30 May 2020 12:52:17 -0700 Subject: [PATCH 24/86] fixing pathlib issue --- vtr_flow/scripts/python_libs/vtr/flow.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vtr_flow/scripts/python_libs/vtr/flow.py b/vtr_flow/scripts/python_libs/vtr/flow.py index de0874c5e19..bea61a43790 100644 --- a/vtr_flow/scripts/python_libs/vtr/flow.py +++ b/vtr_flow/scripts/python_libs/vtr/flow.py @@ -72,8 +72,8 @@ def run(architecture_file, circuit_file, #Copy the circuit and architecture circuit_copy = Path(temp_dir) / circuit_file.name architecture_copy = Path(temp_dir) / architecture_file.name - shutil.copy(circuit_file, str(circuit_copy)) - shutil.copy(architecture_file, str(architecture_copy)) + shutil.copy(str(circuit_file), str(circuit_copy)) + shutil.copy(str(architecture_file), str(architecture_copy)) #There are multiple potential paths for the netlist to reach a tool From fe0812c4a8341478a89a6dd4a04e473d5c56f709 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Sat, 30 May 2020 13:59:55 -0700 Subject: [PATCH 25/86] adding compatibility in python scripts to accept pathlib files or strings --- vtr_flow/scripts/python_libs/vtr/abc/abc.py | 10 ++++++++++ vtr_flow/scripts/python_libs/vtr/ace_flow/ace.py | 12 ++++++++++++ vtr_flow/scripts/python_libs/vtr/flow.py | 10 +++++++++- vtr_flow/scripts/python_libs/vtr/odin/odin.py | 9 +++++++++ vtr_flow/scripts/python_libs/vtr/vpr/vpr.py | 9 +++++++++ 5 files changed, 49 insertions(+), 1 deletion(-) diff --git a/vtr_flow/scripts/python_libs/vtr/abc/abc.py b/vtr_flow/scripts/python_libs/vtr/abc/abc.py index d07c94100e7..baf5ce90737 100644 --- a/vtr_flow/scripts/python_libs/vtr/abc/abc.py +++ b/vtr_flow/scripts/python_libs/vtr/abc/abc.py @@ -10,6 +10,16 @@ def run(architecture_file, circuit_file, use_old_abc_script = False, abc_args = None,keep_intermediate_files=1): mkdir_p(temp_dir) + + if(not isinstance(architecture_file,Path)): + architecture_file = Path(architecture_file) + + if(not isinstance(circuit_file,Path)): + circuit_file = Path(circuit_file) + + if(not isinstance(output_netlist,Path)): + output_netlist = Path(output_netlist) + blackbox_latches_script = find_vtr_file("blackbox_latches.pl") clk_list = [] clk_log_file = "report_clk_out.out" diff --git a/vtr_flow/scripts/python_libs/vtr/ace_flow/ace.py b/vtr_flow/scripts/python_libs/vtr/ace_flow/ace.py index d5c16fc5d0f..0bc544695c5 100644 --- a/vtr_flow/scripts/python_libs/vtr/ace_flow/ace.py +++ b/vtr_flow/scripts/python_libs/vtr/ace_flow/ace.py @@ -1,6 +1,18 @@ from vtr import find_vtr_file from pathlib import Path def run(circuit_file, old_netlist, output_netlist, output_activity_file, command_runner, temp_dir=".", log_filename="ace.out", ace_exec=None, ace_seed = 1): + if(not isinstance(circuit_file,Path)): + circuit_file = Path(circuit_file) + + if(not isinstance(old_netlist,Path)): + old_netlist = Path(old_netlist) + + if(not isinstance(output_netlist,Path)): + output_netlist = Path(output_netlist) + + if(not isinstance(output_activity_file,Path)): + output_activity_file = Path(output_activity_file) + ace_clk_file = Path(temp_dir) / "ace_clk.txt" ace_raw = Path(temp_dir) / (circuit_file.with_suffix('').stem + ".raw.ace.blif") if ace_exec is None: diff --git a/vtr_flow/scripts/python_libs/vtr/flow.py b/vtr_flow/scripts/python_libs/vtr/flow.py index bea61a43790..3af67593d6f 100644 --- a/vtr_flow/scripts/python_libs/vtr/flow.py +++ b/vtr_flow/scripts/python_libs/vtr/flow.py @@ -45,6 +45,12 @@ def run(architecture_file, circuit_file, # #Initial setup # + if(not isinstance(architecture_file,Path)): + architecture_file = Path(architecture_file) + + if(not isinstance(circuit_file,Path)): + circuit_file = Path(circuit_file) + architecture_file_basename =architecture_file.name circuit_file_basename = circuit_file.name @@ -121,6 +127,8 @@ def run(architecture_file, circuit_file, # if power_tech_file: #The user provided a tech file, so do power analysis + if(not isinstance(power_tech_file,Path)): + power_tech_file=Path(power_tech_file) if should_run_stage(VTR_STAGE.ace, start_stage, end_stage): vtr.print_verbose(1, verbosity, "Running ACE") @@ -143,7 +151,7 @@ def run(architecture_file, circuit_file, #Enable power analysis in VPR vpr_args["power"] = True #vpr_args["activity_file"] = post_ace_activity_file.name - vpr_args["tech_properties"] = str(Path(power_tech_file).resolve()) + vpr_args["tech_properties"] = str(power_tech_file.resolve()) # # Pack/Place/Route diff --git a/vtr_flow/scripts/python_libs/vtr/odin/odin.py b/vtr_flow/scripts/python_libs/vtr/odin/odin.py index 01b98c224b1..d9f89ef501f 100644 --- a/vtr_flow/scripts/python_libs/vtr/odin/odin.py +++ b/vtr_flow/scripts/python_libs/vtr/odin/odin.py @@ -13,6 +13,15 @@ def run(architecture_file, circuit_file, min_hard_adder_size=1): mkdir_p(temp_dir) + if(not isinstance(architecture_file,Path)): + architecture_file = Path(architecture_file) + + if(not isinstance(circuit_file,Path)): + circuit_file = Path(circuit_file) + + if(not isinstance(output_netlist,Path)): + output_netlist = Path(output_netlist) + if odin_exec == None: odin_exec = find_vtr_file('odin_II', is_executable=True) diff --git a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py index 4fd4c8d58eb..78a728ffce7 100644 --- a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py +++ b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py @@ -70,6 +70,15 @@ def run(architecture, circuit_name, circuit, command_runner, temp_dir, output_ne if vpr_exec == None: vpr_exec = find_vtr_file('vpr', is_executable=True) + if(not isinstance(architecture,Path)): + architecture = Path(architecture) + + if(not isinstance(circuit_name,Path)): + circuit_name = Path(circuit_name) + + if(not isinstance(circuit,Path)): + circuit = Path(circuit) + cmd = [vpr_exec, architecture.name, circuit_name.stem, "--circuit_file", circuit.name] #Enable netlist generation From 170d0d9ef872283ba851e09aa025752a5e37a517 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Mon, 1 Jun 2020 12:02:23 -0700 Subject: [PATCH 26/86] enabled show_failures in vtr-flow.py --- vtr_flow/scripts/python_libs/vtr/util.py | 7 ++++--- vtr_flow/scripts/vtr-flow.py | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/vtr_flow/scripts/python_libs/vtr/util.py b/vtr_flow/scripts/python_libs/vtr/util.py index b76ce9d106e..f08cca57dbd 100644 --- a/vtr_flow/scripts/python_libs/vtr/util.py +++ b/vtr_flow/scripts/python_libs/vtr/util.py @@ -23,7 +23,7 @@ class RawDefaultHelpFormatter(argparse.ArgumentDefaultsHelpFormatter, argparse.R class CommandRunner(object): - def __init__(self, timeout_sec=None, max_memory_mb=None, track_memory=True, verbose_error=False, verbose=False, echo_cmd=False, indent="\t"): + def __init__(self, timeout_sec=None, max_memory_mb=None, track_memory=True, verbose_error=False, verbose=False, echo_cmd=False, indent="\t", show_failures=False): """ An object for running system commands with timeouts, memory limits and varying verbose-ness @@ -44,6 +44,7 @@ def __init__(self, timeout_sec=None, max_memory_mb=None, track_memory=True, verb self._verbose = verbose self._echo_cmd = echo_cmd self._indent = indent + self._show_failures = show_failures def run_system_command(self, cmd, temp_dir, log_filename=None, expected_return_code=0, indent_depth=0): """ @@ -142,12 +143,12 @@ def run_system_command(self, cmd, temp_dir, log_filename=None, expected_return_c cmd_errored = (cmd_returncode != expected_return_code) #Send to stdout - if self._verbose or (cmd_errored and self._verbose_error): + if self._show_failures and (self._verbose or (cmd_errored and self._verbose_error)): for line in cmd_output: print (indent_depth*self._indent + line,) - if (cmd_errored): + if (self._show_failures and cmd_errored): raise CommandError(msg="Executable {exec_name} failed".format(exec_name=PurePath(orig_cmd[0]).name), cmd=cmd, log=str(PurePath(temp_dir).joinpath(log_filename)), diff --git a/vtr_flow/scripts/vtr-flow.py b/vtr_flow/scripts/vtr-flow.py index 10539f308da..7e8a298e0de 100755 --- a/vtr_flow/scripts/vtr-flow.py +++ b/vtr_flow/scripts/vtr-flow.py @@ -244,7 +244,8 @@ def vtr_command_main(arg_list, prog=None): timeout_sec=args.timeout, verbose_error=True if args.verbose == 2 else False, verbose=True if args.verbose > 2 else False, - echo_cmd=True if args.verbose >= 4 else False) + echo_cmd=True if args.verbose >= 4 else False, + show_failures = args.show_failures) exit_status = 0 abc_args = [] if(args.iterative_bb): From 808fb1e5a4991b4598b99ae562f9dce26eddb6b8 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Mon, 1 Jun 2020 12:57:17 -0700 Subject: [PATCH 27/86] Creating better file validation --- vtr_flow/scripts/python_libs/vtr/__init__.py | 2 +- vtr_flow/scripts/python_libs/vtr/abc/abc.py | 13 ++++--------- .../scripts/python_libs/vtr/ace_flow/ace.py | 17 +++++------------ vtr_flow/scripts/python_libs/vtr/flow.py | 11 +++++------ vtr_flow/scripts/python_libs/vtr/odin/odin.py | 13 ++++--------- vtr_flow/scripts/python_libs/vtr/util.py | 7 +++++++ vtr_flow/scripts/python_libs/vtr/vpr/vpr.py | 18 +++++++++--------- 7 files changed, 35 insertions(+), 46 deletions(-) diff --git a/vtr_flow/scripts/python_libs/vtr/__init__.py b/vtr_flow/scripts/python_libs/vtr/__init__.py index c40ea9394be..e474ed00d12 100644 --- a/vtr_flow/scripts/python_libs/vtr/__init__.py +++ b/vtr_flow/scripts/python_libs/vtr/__init__.py @@ -1,4 +1,4 @@ -from .util import load_config_lines, mkdir_p, find_vtr_file, CommandRunner, print_verbose, relax_W, file_replace, make_enum, RawDefaultHelpFormatter, VERBOSITY_CHOICES,format_elapsed_time, write_tab_delimitted_csv, load_list_file, find_vtr_root, argparse_str2bool, get_next_run_dir, get_latest_run_dir +from .util import load_config_lines, mkdir_p, find_vtr_file, CommandRunner, print_verbose, relax_W, file_replace, make_enum, RawDefaultHelpFormatter, VERBOSITY_CHOICES,format_elapsed_time, write_tab_delimitted_csv, load_list_file, find_vtr_root, argparse_str2bool, get_next_run_dir, get_latest_run_dir, verify_file from .inspect import determine_lut_size, determine_min_W, determine_memory_addr_width, load_parse_patterns, load_pass_requirements, load_parse_results from .abc import run, run_lec from .vpr import run,run_relax_W diff --git a/vtr_flow/scripts/python_libs/vtr/abc/abc.py b/vtr_flow/scripts/python_libs/vtr/abc/abc.py index baf5ce90737..a977de8d398 100644 --- a/vtr_flow/scripts/python_libs/vtr/abc/abc.py +++ b/vtr_flow/scripts/python_libs/vtr/abc/abc.py @@ -1,6 +1,6 @@ import shutil from pathlib import Path -from vtr import mkdir_p, find_vtr_file, determine_lut_size +from vtr import mkdir_p, find_vtr_file, determine_lut_size, verify_file from vtr.error import VtrError, InspectError, CommandError def run(architecture_file, circuit_file, @@ -11,14 +11,9 @@ def run(architecture_file, circuit_file, mkdir_p(temp_dir) - if(not isinstance(architecture_file,Path)): - architecture_file = Path(architecture_file) - - if(not isinstance(circuit_file,Path)): - circuit_file = Path(circuit_file) - - if(not isinstance(output_netlist,Path)): - output_netlist = Path(output_netlist) + verify_file(architecture_file, "Architecture") + verify_file(circuit_file, "Circuit") + verify_file(output_netlist, "Output netlist") blackbox_latches_script = find_vtr_file("blackbox_latches.pl") clk_list = [] diff --git a/vtr_flow/scripts/python_libs/vtr/ace_flow/ace.py b/vtr_flow/scripts/python_libs/vtr/ace_flow/ace.py index 0bc544695c5..32b4524233a 100644 --- a/vtr_flow/scripts/python_libs/vtr/ace_flow/ace.py +++ b/vtr_flow/scripts/python_libs/vtr/ace_flow/ace.py @@ -1,17 +1,10 @@ -from vtr import find_vtr_file +from vtr import find_vtr_file, verify_file from pathlib import Path def run(circuit_file, old_netlist, output_netlist, output_activity_file, command_runner, temp_dir=".", log_filename="ace.out", ace_exec=None, ace_seed = 1): - if(not isinstance(circuit_file,Path)): - circuit_file = Path(circuit_file) - - if(not isinstance(old_netlist,Path)): - old_netlist = Path(old_netlist) - - if(not isinstance(output_netlist,Path)): - output_netlist = Path(output_netlist) - - if(not isinstance(output_activity_file,Path)): - output_activity_file = Path(output_activity_file) + verify_file(circuit_file, "Circuit") + verify_file(old_netlist, "Previous netlist") + verify_file(output_netlist, "Output netlist") + verify_file(output_activity_file, "Output activity") ace_clk_file = Path(temp_dir) / "ace_clk.txt" ace_raw = Path(temp_dir) / (circuit_file.with_suffix('').stem + ".raw.ace.blif") diff --git a/vtr_flow/scripts/python_libs/vtr/flow.py b/vtr_flow/scripts/python_libs/vtr/flow.py index 3af67593d6f..258d9da4510 100644 --- a/vtr_flow/scripts/python_libs/vtr/flow.py +++ b/vtr_flow/scripts/python_libs/vtr/flow.py @@ -6,6 +6,7 @@ import vtr from pathlib import Path from collections import OrderedDict +from vtr import CommandError VTR_STAGE = vtr.make_enum("odin", "abc", 'ace', "vpr", "lec") vtr_stages = VTR_STAGE.reverse_mapping.values() @@ -45,12 +46,10 @@ def run(architecture_file, circuit_file, # #Initial setup # - if(not isinstance(architecture_file,Path)): - architecture_file = Path(architecture_file) - - if(not isinstance(circuit_file,Path)): - circuit_file = Path(circuit_file) - + vtr.util.verify_file(architecture_file, "Architecture") + vtr.util.verify_file(circuit_file, "Circuit") + if(power_tech_file): + vtr.util.verify_file(power_tech_file, "Power tech") architecture_file_basename =architecture_file.name circuit_file_basename = circuit_file.name diff --git a/vtr_flow/scripts/python_libs/vtr/odin/odin.py b/vtr_flow/scripts/python_libs/vtr/odin/odin.py index d9f89ef501f..d25afe8f55e 100644 --- a/vtr_flow/scripts/python_libs/vtr/odin/odin.py +++ b/vtr_flow/scripts/python_libs/vtr/odin/odin.py @@ -1,6 +1,6 @@ import shutil from pathlib import Path -from vtr import mkdir_p, find_vtr_file, file_replace, determine_memory_addr_width +from vtr import mkdir_p, find_vtr_file, file_replace, determine_memory_addr_width, verify_file def run(architecture_file, circuit_file, output_netlist, @@ -13,14 +13,9 @@ def run(architecture_file, circuit_file, min_hard_adder_size=1): mkdir_p(temp_dir) - if(not isinstance(architecture_file,Path)): - architecture_file = Path(architecture_file) - - if(not isinstance(circuit_file,Path)): - circuit_file = Path(circuit_file) - - if(not isinstance(output_netlist,Path)): - output_netlist = Path(output_netlist) + verify_file(architecture_file, "Architecture") + verify_file(circuit_file, "Circuit") + verify_file(output_netlist, "Output netlist") if odin_exec == None: odin_exec = find_vtr_file('odin_II', is_executable=True) diff --git a/vtr_flow/scripts/python_libs/vtr/util.py b/vtr_flow/scripts/python_libs/vtr/util.py index f08cca57dbd..87905460e60 100644 --- a/vtr_flow/scripts/python_libs/vtr/util.py +++ b/vtr_flow/scripts/python_libs/vtr/util.py @@ -415,6 +415,13 @@ def load_config_lines(filepath, allow_includes=True): return config_lines +def verify_file(file, file_type): + if(not isinstance(file,Path)): + file = Path(file) + + if(not file.is_file()): + raise Exception("{file_type} file does not exist: {file} ".format(file_type = file_type, file=file)) + def format_elapsed_time(time_delta): total_sec = int(round(time_delta.total_seconds())) m, s = divmod(total_sec, 60) diff --git a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py index 78a728ffce7..0dad2b37853 100644 --- a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py +++ b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py @@ -1,6 +1,6 @@ import shutil from pathlib import Path -from vtr import mkdir_p, find_vtr_file, CommandRunner, print_verbose, relax_W, determine_lut_size, determine_min_W +from vtr import mkdir_p, find_vtr_file, CommandRunner, print_verbose, relax_W, determine_lut_size, determine_min_W, verify_file def run_relax_W(architecture, circuit_name, circuit, command_runner=CommandRunner(), temp_dir=".", relax_W_factor=1.3, vpr_exec=None, verbosity=1, logfile_base="vpr", @@ -29,6 +29,10 @@ def run_relax_W(architecture, circuit_name, circuit, command_runner=CommandRunne mkdir_p(temp_dir) + verify_file(architecture, "Architecture") + verify_file(circuit_name, "Circuit") + verify_file(circuit, "Circuit") + vpr_min_W_log = '.'.join([logfile_base, "min_W", "out"]) vpr_relaxed_W_log = '.'.join([logfile_base, "relaxed_W", "out"]) @@ -62,6 +66,7 @@ def run(architecture, circuit_name, circuit, command_runner, temp_dir, output_ne """ Runs VPR with the specified configuration """ + if vpr_args is None: vpr_args = OrderedDict() @@ -70,14 +75,9 @@ def run(architecture, circuit_name, circuit, command_runner, temp_dir, output_ne if vpr_exec == None: vpr_exec = find_vtr_file('vpr', is_executable=True) - if(not isinstance(architecture,Path)): - architecture = Path(architecture) - - if(not isinstance(circuit_name,Path)): - circuit_name = Path(circuit_name) - - if(not isinstance(circuit,Path)): - circuit = Path(circuit) + verify_file(architecture, "Architecture") + verify_file(circuit_name, "Circuit") + verify_file(circuit, "Circuit") cmd = [vpr_exec, architecture.name, circuit_name.stem, "--circuit_file", circuit.name] From 4d61c7dc8455efdc089494860f48d1d3e4b1fcec Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Tue, 2 Jun 2020 08:02:44 -0700 Subject: [PATCH 28/86] Fixing issue that thought output files should exist before they should --- vtr_flow/scripts/python_libs/vtr/abc/abc.py | 2 +- vtr_flow/scripts/python_libs/vtr/ace_flow/ace.py | 4 ++-- vtr_flow/scripts/python_libs/vtr/odin/odin.py | 2 +- vtr_flow/scripts/python_libs/vtr/util.py | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/vtr_flow/scripts/python_libs/vtr/abc/abc.py b/vtr_flow/scripts/python_libs/vtr/abc/abc.py index a977de8d398..32a64aec90d 100644 --- a/vtr_flow/scripts/python_libs/vtr/abc/abc.py +++ b/vtr_flow/scripts/python_libs/vtr/abc/abc.py @@ -13,7 +13,7 @@ def run(architecture_file, circuit_file, verify_file(architecture_file, "Architecture") verify_file(circuit_file, "Circuit") - verify_file(output_netlist, "Output netlist") + verify_file(output_netlist, "Output netlist", False) blackbox_latches_script = find_vtr_file("blackbox_latches.pl") clk_list = [] diff --git a/vtr_flow/scripts/python_libs/vtr/ace_flow/ace.py b/vtr_flow/scripts/python_libs/vtr/ace_flow/ace.py index 32b4524233a..46229de8a7e 100644 --- a/vtr_flow/scripts/python_libs/vtr/ace_flow/ace.py +++ b/vtr_flow/scripts/python_libs/vtr/ace_flow/ace.py @@ -3,8 +3,8 @@ def run(circuit_file, old_netlist, output_netlist, output_activity_file, command_runner, temp_dir=".", log_filename="ace.out", ace_exec=None, ace_seed = 1): verify_file(circuit_file, "Circuit") verify_file(old_netlist, "Previous netlist") - verify_file(output_netlist, "Output netlist") - verify_file(output_activity_file, "Output activity") + verify_file(output_netlist, "Output netlist", False) + verify_file(output_activity_file, "Output activity", False) ace_clk_file = Path(temp_dir) / "ace_clk.txt" ace_raw = Path(temp_dir) / (circuit_file.with_suffix('').stem + ".raw.ace.blif") diff --git a/vtr_flow/scripts/python_libs/vtr/odin/odin.py b/vtr_flow/scripts/python_libs/vtr/odin/odin.py index d25afe8f55e..8e56e2dae0b 100644 --- a/vtr_flow/scripts/python_libs/vtr/odin/odin.py +++ b/vtr_flow/scripts/python_libs/vtr/odin/odin.py @@ -15,7 +15,7 @@ def run(architecture_file, circuit_file, verify_file(architecture_file, "Architecture") verify_file(circuit_file, "Circuit") - verify_file(output_netlist, "Output netlist") + verify_file(output_netlist, "Output netlist", False) if odin_exec == None: odin_exec = find_vtr_file('odin_II', is_executable=True) diff --git a/vtr_flow/scripts/python_libs/vtr/util.py b/vtr_flow/scripts/python_libs/vtr/util.py index 87905460e60..5eab747fd9d 100644 --- a/vtr_flow/scripts/python_libs/vtr/util.py +++ b/vtr_flow/scripts/python_libs/vtr/util.py @@ -415,11 +415,11 @@ def load_config_lines(filepath, allow_includes=True): return config_lines -def verify_file(file, file_type): +def verify_file(file, file_type,check_is_file=True): if(not isinstance(file,Path)): file = Path(file) - if(not file.is_file()): + if(check_is_file and not file.is_file()): raise Exception("{file_type} file does not exist: {file} ".format(file_type = file_type, file=file)) def format_elapsed_time(time_delta): From 4385a54f36550a6595eb36772f83a6aaaa1c5925 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Tue, 2 Jun 2020 14:41:41 -0700 Subject: [PATCH 29/86] fixing issue that was causing the tests to only be run on the perl script. --- run_reg_test.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_reg_test.pl b/run_reg_test.pl index e567f97c776..3eacc807ebd 100755 --- a/run_reg_test.pl +++ b/run_reg_test.pl @@ -343,7 +343,7 @@ sub run_single_test { print "scripts/run_vtr_task.pl $run_params -script $script \n"; print "\n"; chdir("$vtr_flow_path"); - my $test_status = system("scripts/run_vtr_task.pl $run_params \n"); + my $test_status = system("scripts/run_vtr_task.pl $run_params -script $script \n"); chdir(".."); #Perl is obtuse, and requires you to manually shift the return value by 8 bits From 130cddc07256afc4558cf39644f3d21add8bed54 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Tue, 2 Jun 2020 15:40:02 -0700 Subject: [PATCH 30/86] lining up output with original scripts. --- vtr_flow/scripts/python_libs/vtr/flow.py | 8 -------- vtr_flow/scripts/python_libs/vtr/util.py | 7 ++----- vtr_flow/scripts/python_libs/vtr/vpr/vpr.py | 4 ---- vtr_flow/scripts/vtr-flow.py | 2 +- 4 files changed, 3 insertions(+), 18 deletions(-) diff --git a/vtr_flow/scripts/python_libs/vtr/flow.py b/vtr_flow/scripts/python_libs/vtr/flow.py index 258d9da4510..9184ed19931 100644 --- a/vtr_flow/scripts/python_libs/vtr/flow.py +++ b/vtr_flow/scripts/python_libs/vtr/flow.py @@ -91,8 +91,6 @@ def run(architecture_file, circuit_file, # if should_run_stage(VTR_STAGE.odin, start_stage, end_stage): if circuit_ext != ".blif": - vtr.print_verbose(1, verbosity, "Running Odin II") - vtr.odin.run(architecture_copy, next_stage_netlist, output_netlist=post_odin_netlist, command_runner=command_runner, @@ -107,7 +105,6 @@ def run(architecture_file, circuit_file, # Logic Optimization & Technology Mapping # if should_run_stage(VTR_STAGE.abc, start_stage, end_stage): - vtr.print_verbose(1, verbosity, "Running ABC") vtr.abc.run(architecture_copy, next_stage_netlist, output_netlist=post_abc_netlist, command_runner=command_runner, @@ -130,8 +127,6 @@ def run(architecture_file, circuit_file, power_tech_file=Path(power_tech_file) if should_run_stage(VTR_STAGE.ace, start_stage, end_stage): - vtr.print_verbose(1, verbosity, "Running ACE") - vtr.ace_flow.run(next_stage_netlist, old_netlist = post_odin_netlist, output_netlist=post_ace_netlist, output_activity_file=post_ace_activity_file, command_runner=command_runner, @@ -166,7 +161,6 @@ def run(architecture_file, circuit_file, if "route_chan_width" in vpr_args: #The User specified a fixed channel width - vtr.print_verbose(1, verbosity, "Running VPR (at fixed channel width)") vtr.vpr.run(architecture_copy, circuit_copy, pre_vpr_netlist, output_netlist=post_vpr_netlist, command_runner=command_runner, @@ -188,7 +182,6 @@ def run(architecture_file, circuit_file, # Logical Equivalence Checks (LEC) # if should_run_stage(VTR_STAGE.lec, start_stage, end_stage): - vtr.print_verbose(1, verbosity, "Running ABC Logical Equivalence Check") vtr.abc.run_lec(lec_base_netlist, post_vpr_netlist, command_runner=command_runner, log_filename="abc.lec.out") if(not keep_intermediate_files): next_stage_netlist.unlink() @@ -205,7 +198,6 @@ def run(architecture_file, circuit_file, post_ace_activity_file.unlink() def parse_vtr_flow(temp_dir, parse_config_file=None, metrics_filepath=None, verbosity=1): - vtr.print_verbose(1, verbosity, "Parsing results") vtr.mkdir_p(temp_dir) if parse_config_file is None: parse_config_file = vtr.find_vtr_file("vtr_benchmarks.txt") diff --git a/vtr_flow/scripts/python_libs/vtr/util.py b/vtr_flow/scripts/python_libs/vtr/util.py index 5eab747fd9d..18ad783471c 100644 --- a/vtr_flow/scripts/python_libs/vtr/util.py +++ b/vtr_flow/scripts/python_libs/vtr/util.py @@ -423,11 +423,8 @@ def verify_file(file, file_type,check_is_file=True): raise Exception("{file_type} file does not exist: {file} ".format(file_type = file_type, file=file)) def format_elapsed_time(time_delta): - total_sec = int(round(time_delta.total_seconds())) - m, s = divmod(total_sec, 60) - h, m = divmod(m, 60) - - return "{:02}:{:02}:{:02}".format(h, m, s) + return "%.2f seconds" % time_delta.total_seconds() + def argparse_str2bool(str_val): str_val = str_val.lower() diff --git a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py index 0dad2b37853..99650cc6b4f 100644 --- a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py +++ b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py @@ -36,8 +36,6 @@ def run_relax_W(architecture, circuit_name, circuit, command_runner=CommandRunne vpr_min_W_log = '.'.join([logfile_base, "min_W", "out"]) vpr_relaxed_W_log = '.'.join([logfile_base, "relaxed_W", "out"]) - print_verbose(1, verbosity, "Running VPR (determining minimum channel width)" ) - run(architecture, circuit_name, circuit, command_runner, temp_dir, log_filename=vpr_min_W_log, vpr_exec=vpr_exec, vpr_args=vpr_args) if ('pack' in vpr_args or 'place' in vpr_args) and 'route' not in vpr_args: @@ -49,8 +47,6 @@ def run_relax_W(architecture, circuit_name, circuit, command_runner=CommandRunne relaxed_W = relax_W(min_W, relax_W_factor) - print_verbose(1, verbosity, "Running VPR (at {fac}x relaxed minimum channel width)".format(fac=relax_W_factor)) - vpr_args['route'] = True #Re-route only vpr_args['route_chan_width'] = relaxed_W #At a fixed channel width diff --git a/vtr_flow/scripts/vtr-flow.py b/vtr_flow/scripts/vtr-flow.py index 7e8a298e0de..be944507891 100755 --- a/vtr_flow/scripts/vtr-flow.py +++ b/vtr_flow/scripts/vtr-flow.py @@ -314,7 +314,7 @@ def vtr_command_main(arg_list, prog=None): exit_status = 2 finally: - vtr.print_verbose(BASIC_VERBOSITY, args.verbose, "\n# {} took {} (exiting {})".format(prog, vtr.format_elapsed_time(datetime.now() - start), exit_status)) + vtr.print_verbose(BASIC_VERBOSITY, args.verbose, "OK (took {})".format(vtr.format_elapsed_time(datetime.now() - start))) sys.exit(exit_status) def process_unkown_args(unkown_args): From bab4a8985e736eb84975eaa09728658634871650 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Tue, 2 Jun 2020 15:49:28 -0700 Subject: [PATCH 31/86] fixing one last output difference --- vtr_flow/scripts/vtr-flow.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/vtr_flow/scripts/vtr-flow.py b/vtr_flow/scripts/vtr-flow.py index be944507891..dc27135ac46 100755 --- a/vtr_flow/scripts/vtr-flow.py +++ b/vtr_flow/scripts/vtr-flow.py @@ -228,8 +228,6 @@ def vtr_command_main(arg_list, prog=None): #Load the arguments args, unkown_args = vtr_command_argparser(prog).parse_known_args(arg_list) - vtr.print_verbose(BASIC_VERBOSITY, args.verbose, "# {} {}\n".format(prog, ' '.join(arg_list))) - path_arch_file = Path(args.architecture_file) path_circuit_file = Path(args.circuit_file) if (args.temp_dir == "."): From c0c222d77ac87e7bbde5f7ac4a34aa8731edbe8a Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Wed, 24 Jun 2020 14:47:36 -0600 Subject: [PATCH 32/86] fixing file name issues on the python script. Also made the python script the default script to test with the regression tests. --- .travis.yml | 16 +++++++------- run_reg_test.pl | 2 +- vtr_flow/scripts/python_libs/vtr/abc/abc.py | 4 ++-- vtr_flow/scripts/python_libs/vtr/vpr/vpr.py | 4 ++-- vtr_flow/scripts/run_vtr_task.pl | 6 +++--- vtr_flow/scripts/vtr-flow.py | 23 +++++++++++++++++---- 6 files changed, 35 insertions(+), 20 deletions(-) diff --git a/.travis.yml b/.travis.yml index f3c43bc2180..b5d27744c74 100644 --- a/.travis.yml +++ b/.travis.yml @@ -101,7 +101,7 @@ jobs: - MATRIX_EVAL="CC=gcc-5 && CXX=g++-5" script: - ./.github/travis/build.sh - - travis_wait 30 ./run_reg_test.pl vtr_reg_basic -show_failures -j2 -script vtr-flow.py + - travis_wait 30 ./run_reg_test.pl vtr_reg_basic -show_failures -j2 - stage: Test name: "Strong Regression Tests Python" env: @@ -109,7 +109,7 @@ jobs: - MATRIX_EVAL="CC=gcc-5 && CXX=g++-5" script: - ./.github/travis/build.sh - - travis_wait 30 ./run_reg_test.pl vtr_reg_strong -show_failures -j2 -script vtr-flow.py + - travis_wait 30 ./run_reg_test.pl vtr_reg_strong -show_failures -j2 - stage: Test name: "Basic Valgrind Memory Tests Python" env: @@ -117,7 +117,7 @@ jobs: - MATRIX_EVAL="CC=gcc-5 && CXX=g++-5" script: - ./.github/travis/build.sh - - travis_wait 30 ./run_reg_test.pl vtr_reg_valgrind_small -show_failures -j2 -script vtr-flow.py + - travis_wait 30 ./run_reg_test.pl vtr_reg_valgrind_small -show_failures -j2 - stage: Test name: "Sanitized Basic Regression Tests Python" env: @@ -128,7 +128,7 @@ jobs: script: - ./.github/travis/build.sh #We skip QoR since we are only checking for errors in sanitizer runs - - travis_wait 30 ./run_reg_test.pl vtr_reg_basic -show_failures -skip_qor -j2 -script vtr-flow.py + - travis_wait 30 ./run_reg_test.pl vtr_reg_basic -show_failures -skip_qor -j2 - stage: Test name: "Code Formatting" script: @@ -157,7 +157,7 @@ jobs: - MATRIX_EVAL="CC=gcc-5 && CXX=g++-5" script: - ./.github/travis/build.sh - - travis_wait 30 ./run_reg_test.pl vtr_reg_basic -show_failures -j2 + - travis_wait 30 ./run_reg_test.pl vtr_reg_basic -show_failures -j2 -script run_vtr_flow.pl - stage: Test name: "Strong Regression Tests" env: @@ -165,7 +165,7 @@ jobs: - MATRIX_EVAL="CC=gcc-5 && CXX=g++-5" script: - ./.github/travis/build.sh - - travis_wait 30 ./run_reg_test.pl vtr_reg_strong -show_failures -j2 + - travis_wait 30 ./run_reg_test.pl vtr_reg_strong -show_failures -j2 -script run_vtr_flow.pl - stage: Test name: "Basic Valgrind Memory Tests" env: @@ -173,7 +173,7 @@ jobs: - MATRIX_EVAL="CC=gcc-5 && CXX=g++-5" script: - ./.github/travis/build.sh - - travis_wait 30 ./run_reg_test.pl vtr_reg_valgrind_small -show_failures -j2 + - travis_wait 30 ./run_reg_test.pl vtr_reg_valgrind_small -show_failures -j2 -script run_vtr_flow.pl - stage: Test name: "Sanitized Basic Regression Tests" env: @@ -184,7 +184,7 @@ jobs: script: - ./.github/travis/build.sh #We skip QoR since we are only checking for errors in sanitizer runs - - travis_wait 30 ./run_reg_test.pl vtr_reg_basic -show_failures -skip_qor -j2 + - travis_wait 30 ./run_reg_test.pl vtr_reg_basic -show_failures -skip_qor -j2 -script run_vtr_flow.pl #Currently strong regression with sanitizers is disabled as it exceeds the maximum travis job run-time #- stage: Test #name: "Sanitized Strong Regression Tests" diff --git a/run_reg_test.pl b/run_reg_test.pl index 3eacc807ebd..c3932fcf719 100755 --- a/run_reg_test.pl +++ b/run_reg_test.pl @@ -65,7 +65,7 @@ my $skip_qor = 0; my $show_failures = 0; my $num_cpu = 1; -my $script = "run_vtr_flow.pl"; +my $script = "vtr-flow.py"; my $long_task_names = 0; # Parse Input Arguments diff --git a/vtr_flow/scripts/python_libs/vtr/abc/abc.py b/vtr_flow/scripts/python_libs/vtr/abc/abc.py index 32a64aec90d..2f1ace64eee 100644 --- a/vtr_flow/scripts/python_libs/vtr/abc/abc.py +++ b/vtr_flow/scripts/python_libs/vtr/abc/abc.py @@ -119,8 +119,8 @@ def run(architecture_file, circuit_file, if(abc_run_args): cmd.append(abc_run_args) - - command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=str(i)+"_"+log_filename, indent_depth=1) + log_file = Path(log_filename) + command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_file.stem+str(i)+log_file.suffix, indent_depth=1) if(abc_flow_type != 3 and len(clk_list)>i): cmd = [blackbox_latches_script,"--restore", clk_list[i], "--input", post_abc_raw_blif.name,"--output",post_abc_blif.name] diff --git a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py index 99650cc6b4f..ceafa61c24d 100644 --- a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py +++ b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py @@ -33,8 +33,8 @@ def run_relax_W(architecture, circuit_name, circuit, command_runner=CommandRunne verify_file(circuit_name, "Circuit") verify_file(circuit, "Circuit") - vpr_min_W_log = '.'.join([logfile_base, "min_W", "out"]) - vpr_relaxed_W_log = '.'.join([logfile_base, "relaxed_W", "out"]) + vpr_min_W_log = '.'.join([logfile_base, "out"]) + vpr_relaxed_W_log = '.'.join([logfile_base, "crit_path", "out"]) run(architecture, circuit_name, circuit, command_runner, temp_dir, log_filename=vpr_min_W_log, vpr_exec=vpr_exec, vpr_args=vpr_args) diff --git a/vtr_flow/scripts/run_vtr_task.pl b/vtr_flow/scripts/run_vtr_task.pl index ac972c361b7..3a890f74db5 100755 --- a/vtr_flow/scripts/run_vtr_task.pl +++ b/vtr_flow/scripts/run_vtr_task.pl @@ -68,7 +68,7 @@ my $run_prefix = "run"; my $show_runtime_estimates = 1; my $system_type = "local"; -my $script = "run_vtr_flow.pl"; +my $script = "vtr-flow.py"; my $shared_script_params = ""; my $verbosity = 0; my $short_task_names = 0; @@ -216,7 +216,7 @@ sub generate_single_task_actions { my $circuits_dir; my $archs_dir; my $sdc_dir = undef; - my $script_default = "run_vtr_flow.pl"; + my $script_default = "vtr-flow.py run_vtr_flow.pl"; my $script_path; my $script_params_common = $shared_script_params; # start with the shared ones then build unique ones my @circuits_list; @@ -288,7 +288,7 @@ sub generate_single_task_actions { close(CONFIG_FH); # Using default script - if ( $script eq $script_default ) { + if (index($script_default, $script) != -1 ) { # This is hack to automatically add the option '-temp_dir .' if using the run_vtr_flow.pl script # This ensures that a 'temp' folder is not created in each circuit directory diff --git a/vtr_flow/scripts/vtr-flow.py b/vtr_flow/scripts/vtr-flow.py index dc27135ac46..e65de254254 100755 --- a/vtr_flow/scripts/vtr-flow.py +++ b/vtr_flow/scripts/vtr-flow.py @@ -8,6 +8,7 @@ import shutil import re import textwrap +import socket from datetime import datetime from collections import OrderedDict @@ -163,7 +164,7 @@ def vtr_command_argparser(prog=None): house_keeping = parser.add_argument_group("House Keeping", description="Configuration related to how files/time/memory are used by the script.") house_keeping.add_argument("-temp_dir", - default=".", + default=None, help="Directory to run the flow in (will be created if non-existant).") house_keeping.add_argument("-name", @@ -230,7 +231,7 @@ def vtr_command_main(arg_list, prog=None): path_arch_file = Path(args.architecture_file) path_circuit_file = Path(args.circuit_file) - if (args.temp_dir == "."): + if (args.temp_dir == None): temp_dir="./temp" else: temp_dir=args.temp_dir @@ -266,7 +267,7 @@ def vtr_command_main(arg_list, prog=None): name=args.name else: name=path_arch_file.name+"/"+path_circuit_file.name - print(name) + print(name, end ="\t\t") #Run the flow vtr.run(path_arch_file, path_circuit_file, @@ -312,7 +313,21 @@ def vtr_command_main(arg_list, prog=None): exit_status = 2 finally: - vtr.print_verbose(BASIC_VERBOSITY, args.verbose, "OK (took {})".format(vtr.format_elapsed_time(datetime.now() - start))) + seconds = datetime.now() - start + vtr.print_verbose(BASIC_VERBOSITY, args.verbose, "OK (took {})".format(vtr.format_elapsed_time(seconds)+"seconds")) + out = Path(temp_dir) / "output.txt" + out.touch() + with out.open('w') as f: + f.write("vpr_status=") + if(exit_status==0): + f.write("success\n") + else: + f.write("failure\n") + f.write("vpr_seconds=%d\nrundir=%s\nhostname=%s\nerror=" % (seconds.total_seconds(), str(Path.cwd()), socket.gethostname())) + if(exit_status!=0): + f.write(exit_status) + f.write("\n") + sys.exit(exit_status) def process_unkown_args(unkown_args): From ba96eb30887562fa501abfa1ea2837e09b4a6d56 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Thu, 25 Jun 2020 08:03:54 -0600 Subject: [PATCH 33/86] adding missing ODIN II arguments. --- vtr_flow/scripts/python_libs/vtr/flow.py | 10 ++-- vtr_flow/scripts/python_libs/vtr/odin/odin.py | 27 ++++++++-- vtr_flow/scripts/vtr-flow.py | 54 ++++++++++++++++--- 3 files changed, 77 insertions(+), 14 deletions(-) diff --git a/vtr_flow/scripts/python_libs/vtr/flow.py b/vtr_flow/scripts/python_libs/vtr/flow.py index 9184ed19931..bb239b5bc9b 100644 --- a/vtr_flow/scripts/python_libs/vtr/flow.py +++ b/vtr_flow/scripts/python_libs/vtr/flow.py @@ -18,10 +18,12 @@ def run(architecture_file, circuit_file, parse_config_file=None, temp_dir="./temp", verbosity=0, - vpr_args=None, + odin_args=None, abc_args=None, + vpr_args=None, keep_intermediate_files=True, - keep_result_files=True): + keep_result_files=True, + min_hard_mult_size=3): """ Runs the VTR CAD flow to map the specificied circuit_file onto the target architecture_file @@ -94,7 +96,9 @@ def run(architecture_file, circuit_file, vtr.odin.run(architecture_copy, next_stage_netlist, output_netlist=post_odin_netlist, command_runner=command_runner, - temp_dir=temp_dir) + temp_dir=temp_dir, + odin_args=odin_args, + min_hard_mult_size=min_hard_mult_size) next_stage_netlist = post_odin_netlist diff --git a/vtr_flow/scripts/python_libs/vtr/odin/odin.py b/vtr_flow/scripts/python_libs/vtr/odin/odin.py index 8e56e2dae0b..05a714abf7b 100644 --- a/vtr_flow/scripts/python_libs/vtr/odin/odin.py +++ b/vtr_flow/scripts/python_libs/vtr/odin/odin.py @@ -6,6 +6,7 @@ def run(architecture_file, circuit_file, output_netlist, command_runner, temp_dir=".", + odin_args="--adder_type default", log_filename="odin.out", odin_exec=None, odin_config=None, @@ -37,7 +38,25 @@ def run(architecture_file, circuit_file, "MMM": min_hard_mult_size, "AAA": min_hard_adder_size, }) - - cmd = [odin_exec, "-c", odin_config, "--adder_type", "default", "-U0"] - - command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) \ No newline at end of file + disable_odin_xml = False + if("disable_odin_xml" in odin_args): + disable_odin_xml=True + odin_args.remove("disable_odin_xml") + use_odin_simulation = False + if("use_odin_simulation" in odin_args): + use_odin_simulation=True + odin_args.remove("use_odin_simulation") + + cmd = [odin_exec, "-c", odin_config] + cmd.extend(odin_args) + cmd.extend(["-U0"]) + if(disable_odin_xml): + cmd.extend(["-a",architecture_file.name, "-V", circuit_file.name, "-o",output_netlist.name]) + + command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) + + if(use_odin_simulation): + sim_dir = Path(temp_dir) / "simulation_init" + sim_dir.mkdir() + cmd = [odin_exec, "-b", output_netlist.name, -a, architecture_file.name, "-sim_dir", str(sim_dir),"-g", "100", "--best_coverage", "-U0"] + command_runner.run_system_command(cmd,temp_dir=temp_dir, log_filename="sim_produce_vector.out",indent_depth=1) \ No newline at end of file diff --git a/vtr_flow/scripts/vtr-flow.py b/vtr_flow/scripts/vtr-flow.py index e65de254254..291deacdb1b 100755 --- a/vtr_flow/scripts/vtr-flow.py +++ b/vtr_flow/scripts/vtr-flow.py @@ -127,10 +127,10 @@ def vtr_command_argparser(prog=None): help="Ending stage of the VTR flow.") parser.add_argument("-verbose", "-v", - choices=vtr.VERBOSITY_CHOICES, - default=2, - type=int, - help="Verbosity of the script. Higher values produce more output.") + default=False, + action="store_true", + dest="verbose", + help="Verbosity of the script.") parser.add_argument("--parse_config_file", default=None, @@ -218,6 +218,33 @@ def vtr_command_argparser(prog=None): action="store_true", dest="use_old_latches_restoration_script", help="Use the new latches restoration script") + # + # ODIN II arguments + # + odin = parser.add_argument_group("Odin", description="Arguments to be passed to ODIN II") + odin.add_argument("-adder_type", + default="default", + help="Odin adder type") + odin.add_argument("-adder_cin_global", + default=False, + action="store_true", + dest="adder_cin_global", + help="Tells ODIN II to connect the first cin in an adder/subtractor chain to a global gnd/vdd net.") + odin.add_argument("-disable_odin_xml", + default=False, + action="store_true", + dest="disable_odin_xml", + help="Disables the ODIN xml") + odin.add_argument("-use_odin_simulation", + default=False, + action="store_true", + dest="use_odin_simulation", + help="Tells ODIN II to connect the first cin in an adder/subtractor chain to a global gnd/vdd net.") + odin.add_argument("-min_hard_mult_size", + default=3, + type=int, + metavar="min_hard_mult_size", + help="Tells ODIN II the minimum multiplier size that should be implemented using hard multiplier (if available).") return parser def main(): @@ -258,6 +285,17 @@ def vtr_command_main(arg_list, prog=None): if(args.use_old_latches_restoration_script): abc_args.append("use_old_latches_restoration_script") + odin_args= [] + odin_args.append("--adder_type") + odin_args.append(args.adder_type) + if(args.adder_cin_global): + odin_args.append("--adder_cin_global") + if(args.disable_odin_xml): + odin_args.append("disable_odin_xml") + if(args.use_odin_simulation): + odin_args.append("use_odin_simulation") + + try: if not args.parse_only: try: @@ -279,8 +317,10 @@ def vtr_command_main(arg_list, prog=None): verbosity=args.verbose, vpr_args=vpr_args, abc_args=abc_args, + odin_args=odin_args, keep_intermediate_files=args.keep_intermediate_files, - keep_result_files=args.keep_result_files + keep_result_files=args.keep_result_files, + min_hard_mult_size=args.min_hard_mult_size ) except vtr.CommandError as e: #An external command failed @@ -314,7 +354,7 @@ def vtr_command_main(arg_list, prog=None): finally: seconds = datetime.now() - start - vtr.print_verbose(BASIC_VERBOSITY, args.verbose, "OK (took {})".format(vtr.format_elapsed_time(seconds)+"seconds")) + print("OK (took {})".format(vtr.format_elapsed_time(seconds))) out = Path(temp_dir) / "output.txt" out.touch() with out.open('w') as f: @@ -325,7 +365,7 @@ def vtr_command_main(arg_list, prog=None): f.write("failure\n") f.write("vpr_seconds=%d\nrundir=%s\nhostname=%s\nerror=" % (seconds.total_seconds(), str(Path.cwd()), socket.gethostname())) if(exit_status!=0): - f.write(exit_status) + f.write(str(exit_status)) f.write("\n") sys.exit(exit_status) From f1b2c88ba7809e6ac5f17bc5cbc50480d155a368 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Thu, 25 Jun 2020 08:54:43 -0600 Subject: [PATCH 34/86] added crit_path_router_iterations argument and used it to impliment the vpr argument max_router_iterations --- vtr_flow/scripts/vtr-flow.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/vtr_flow/scripts/vtr-flow.py b/vtr_flow/scripts/vtr-flow.py index 291deacdb1b..3d34aefae0d 100755 --- a/vtr_flow/scripts/vtr-flow.py +++ b/vtr_flow/scripts/vtr-flow.py @@ -245,6 +245,14 @@ def vtr_command_argparser(prog=None): type=int, metavar="min_hard_mult_size", help="Tells ODIN II the minimum multiplier size that should be implemented using hard multiplier (if available).") + # + # VPR arguments + # + vpr = parser.add_argument_group("Vpr", description="Arguments to be parsed and then passed to VPR") + vpr.add_argument("-crit_path_router_iterations", + default=150, + type=int, + help="Tells ODIN II the minimum multiplier size that should be implemented using hard multiplier (if available).") return parser def main(): @@ -300,6 +308,7 @@ def vtr_command_main(arg_list, prog=None): if not args.parse_only: try: vpr_args = process_unkown_args(unkown_args) + vpr_args["max_router_iterations"] = args.crit_path_router_iterations name = "" if(args.name): name=args.name From fec5ad269f543735fc276a0127145b723a116819 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Thu, 25 Jun 2020 13:56:38 -0600 Subject: [PATCH 35/86] Adding lut_size argument and making extra abc and odin arguments cleaner. --- vtr_flow/scripts/python_libs/vtr/abc/abc.py | 20 ++++++--- vtr_flow/scripts/python_libs/vtr/odin/odin.py | 4 +- vtr_flow/scripts/vtr-flow.py | 43 +++++++++++-------- 3 files changed, 42 insertions(+), 25 deletions(-) diff --git a/vtr_flow/scripts/python_libs/vtr/abc/abc.py b/vtr_flow/scripts/python_libs/vtr/abc/abc.py index 2f1ace64eee..e1393e6f079 100644 --- a/vtr_flow/scripts/python_libs/vtr/abc/abc.py +++ b/vtr_flow/scripts/python_libs/vtr/abc/abc.py @@ -24,17 +24,23 @@ def run(architecture_file, circuit_file, abc_flow_type = 2 abc_run_args = "" use_old_latches_restoration_script = 0 - for i in abc_args: - if(i=="iterative_bb"): + lut_size = None + for key, value in abc_args.items(): + if(key=="iterative_bb"): abc_flow_type=2 - elif(i=="blanket_bb"): + elif(key=="blanket_bb"): abc_flow_type=3 - elif(i=="once_bb"): + elif(key=="once_bb"): abc_flow_type=1 - elif(i=="use_old_latches_restoration_script"): + elif(key=="use_old_latches_restoration_script"): use_old_latches_restoration_script = 1 + elif(key=="lut_size"): + lut_size=value else: abc_run_args += i + + if(lut_size == None): + lut_size = determine_lut_size(str(architecture_file)) if(abc_flow_type): populate_clock_list(circuit_file,blackbox_latches_script,clk_list,command_runner,temp_dir,log_filename) @@ -46,7 +52,9 @@ def run(architecture_file, circuit_file, abc_rc = Path(abc_exec).parent / 'abc.rc' shutil.copyfile(str(abc_rc), str(Path(temp_dir) / 'abc.rc')) - lut_size = determine_lut_size(str(architecture_file)) + + + iterations=len(clk_list) if(iterations==0 or abc_flow_type != 2): diff --git a/vtr_flow/scripts/python_libs/vtr/odin/odin.py b/vtr_flow/scripts/python_libs/vtr/odin/odin.py index 05a714abf7b..d460f2a1509 100644 --- a/vtr_flow/scripts/python_libs/vtr/odin/odin.py +++ b/vtr_flow/scripts/python_libs/vtr/odin/odin.py @@ -41,11 +41,11 @@ def run(architecture_file, circuit_file, disable_odin_xml = False if("disable_odin_xml" in odin_args): disable_odin_xml=True - odin_args.remove("disable_odin_xml") + del odin_args["disable_odin_xml"] use_odin_simulation = False if("use_odin_simulation" in odin_args): use_odin_simulation=True - odin_args.remove("use_odin_simulation") + del odin_args["use_odin_simulation"] cmd = [odin_exec, "-c", odin_config] cmd.extend(odin_args) diff --git a/vtr_flow/scripts/vtr-flow.py b/vtr_flow/scripts/vtr-flow.py index 3d34aefae0d..0a44f01e3c8 100755 --- a/vtr_flow/scripts/vtr-flow.py +++ b/vtr_flow/scripts/vtr-flow.py @@ -195,29 +195,32 @@ def vtr_command_argparser(prog=None): metavar="TIMEOUT_SECONDS", help="Maximum time in seconds to spend on a single stage.") # - # Iterative black-boxing flow arguments + # ABC arguments # - iterative = parser.add_argument_group("Iterative", description="Iterative black-boxing flow for multi-clock circuits options") - iterative.add_argument("-iterative_bb", + abc = parser.add_argument_group("abc", description="Iterative black-boxing flow for multi-clock circuits options") + abc.add_argument("-iterative_bb", default=False, action="store_true", dest="iterative_bb", help="Use iterative black-boxing flow for multi-clock circuits") - iterative.add_argument("-once_bb", + abc.add_argument("-once_bb", default=False, action="store_true", dest="once_bb", help="Use the black-boxing flow a single time") - iterative.add_argument("-blanket_bb", + abc.add_argument("-blanket_bb", default=False, action="store_true", dest="blanket_bb", help="Use iterative black-boxing flow with out clocks") #not sure if this is a correct statement. - iterative.add_argument("-use_old_latches_restoration_script", + abc.add_argument("-use_old_latches_restoration_script", default=False, action="store_true", dest="use_old_latches_restoration_script", help="Use the new latches restoration script") + abc.add_argument("-lut_size", + type=int, + help="Tells ABC the LUT size of the FPGA architecture") # # ODIN II arguments # @@ -281,27 +284,33 @@ def vtr_command_main(arg_list, prog=None): echo_cmd=True if args.verbose >= 4 else False, show_failures = args.show_failures) exit_status = 0 - abc_args = [] + abc_args = OrderedDict() if(args.iterative_bb): - abc_args.append("iterative_bb") + abc_args["iterative_bb"] = True if(args.once_bb): - abc_args.append("once_bb") + abc_args["once_bb"] = True if(args.blanket_bb): - abc_args.append("blanket_bb") + abc_args["blanket_bb"] = True if(args.use_old_latches_restoration_script): - abc_args.append("use_old_latches_restoration_script") - odin_args= [] - odin_args.append("--adder_type") - odin_args.append(args.adder_type) + abc_args["use_old_latches_restoration_script"] = True + + if(args.lut_size): + abc_args["lut_size"] = args.lut_size + + odin_args = OrderedDict() + odin_args["--adder_type"] = args.adder_type + if(args.adder_cin_global): - odin_args.append("--adder_cin_global") + odin_args["--adder_cin_global"] = True + if(args.disable_odin_xml): - odin_args.append("disable_odin_xml") + odin_args["disable_odin_xml"] = True + if(args.use_odin_simulation): - odin_args.append("use_odin_simulation") + odin_args["use_odin_simulation"] = True try: From 04280500cfbd8818b5015ca56ec22243d2f3009b Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Fri, 26 Jun 2020 09:06:00 -0600 Subject: [PATCH 36/86] Added check_equivalent argument and enabled it. Fixed issue with temp_dir if it had parent directories that also needed to be created. fixed odin argument issue --- vtr_flow/scripts/python_libs/vtr/abc/abc.py | 29 +++++++++++++------ vtr_flow/scripts/python_libs/vtr/flow.py | 19 ++++++------ vtr_flow/scripts/python_libs/vtr/odin/odin.py | 8 ++++- vtr_flow/scripts/python_libs/vtr/util.py | 2 +- vtr_flow/scripts/vtr-flow.py | 13 +++++++-- 5 files changed, 48 insertions(+), 23 deletions(-) diff --git a/vtr_flow/scripts/python_libs/vtr/abc/abc.py b/vtr_flow/scripts/python_libs/vtr/abc/abc.py index e1393e6f079..5f05b28b143 100644 --- a/vtr_flow/scripts/python_libs/vtr/abc/abc.py +++ b/vtr_flow/scripts/python_libs/vtr/abc/abc.py @@ -7,7 +7,7 @@ def run(architecture_file, circuit_file, output_netlist, command_runner, temp_dir=".", log_filename="abc.out", abc_exec=None, abc_script=None, abc_rc=None, - use_old_abc_script = False, abc_args = None,keep_intermediate_files=1): + use_old_abc_script = False, abc_args = None, keep_intermediate_files=1): mkdir_p(temp_dir) @@ -37,7 +37,12 @@ def run(architecture_file, circuit_file, elif(key=="lut_size"): lut_size=value else: - abc_run_args += i + if value == True: + abc_run_args += ["--" + arg] + elif value == False: + pass + else: + abc_run_args += ["--" + arg, str(value)] if(lut_size == None): lut_size = determine_lut_size(str(architecture_file)) @@ -162,7 +167,7 @@ def populate_clock_list(circuit_file,blackbox_latches_script,clk_list,command_ru with clk_list_path.open('r') as f: clk_list.append(f.readline().strip('\n')) -def run_lec(reference_netlist, implementation_netlist, command_runner, temp_dir=".", log_filename="abc.lec.out", abc_exec=None): +def run_lec(reference_netlist, implementation_netlist, command_runner, temp_dir=".", log_filename="abc.dec.out", abc_exec=None): """ Run Logical Equivalence Checking (LEC) between two netlists using ABC """ @@ -171,9 +176,7 @@ def run_lec(reference_netlist, implementation_netlist, command_runner, temp_dir= if abc_exec == None: abc_exec = find_vtr_file('abc', is_executable=True) - abc_script = ['cec {ref} {imp}'.format(ref=reference_netlist, imp=implementation_netlist), - 'sec {ref} {imp}'.format(ref=reference_netlist, imp=implementation_netlist), - ] + abc_script = 'dsec {ref} {imp}'.format(ref=reference_netlist, imp=implementation_netlist), abc_script = "; ".join(abc_script) cmd = [abc_exec, '-c', abc_script] @@ -181,8 +184,13 @@ def run_lec(reference_netlist, implementation_netlist, command_runner, temp_dir= output, returncode = command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) #Check if ABC's LEC engine passed - lec_passed = check_abc_lec_status(output) - + lec_passed, errored = check_abc_lec_status(output) + if errored: + abc_script = 'cec {ref} {imp}'.format(ref=reference_netlist, imp=implementation_netlist), + abc_script = "; ".join(abc_script) + cmd = [abc_exec, '-c', abc_script] + output, returncode = command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename="abc.cec.out", indent_depth=1) + lec_passed, errored = check_abc_lec_status(output) if lec_passed is None: raise InspectError("Could not determine Logical Equivalence Check status between {input} <-> {output}".format(input=reference_netlist, output=implementation_netlist), filename=log_filename) elif lec_passed is False: @@ -192,12 +200,15 @@ def run_lec(reference_netlist, implementation_netlist, command_runner, temp_dir= def check_abc_lec_status(output): equivalent = None + errored = False for line in output: + if "Error: The network has no latches." in line: + errored = True if line.startswith("Networks are NOT EQUIVALENT"): equivalent = False elif line.startswith("Networks are equivalent"): equivalent = True #Returns None if could not determine LEC status - return equivalent + return equivalent, errored diff --git a/vtr_flow/scripts/python_libs/vtr/flow.py b/vtr_flow/scripts/python_libs/vtr/flow.py index bb239b5bc9b..04250e28bb0 100644 --- a/vtr_flow/scripts/python_libs/vtr/flow.py +++ b/vtr_flow/scripts/python_libs/vtr/flow.py @@ -23,7 +23,8 @@ def run(architecture_file, circuit_file, vpr_args=None, keep_intermediate_files=True, keep_result_files=True, - min_hard_mult_size=3): + min_hard_mult_size=3, + check_equivalent=False): """ Runs the VTR CAD flow to map the specificied circuit_file onto the target architecture_file @@ -68,8 +69,9 @@ def run(architecture_file, circuit_file, post_ace_netlist =Path(temp_dir) / (circuit_name + ".ace.blif") post_ace_activity_file = Path(temp_dir) / (circuit_name + ".act") pre_vpr_netlist = Path(temp_dir) / (circuit_name + ".pre-vpr.blif") - post_vpr_netlist = Path(temp_dir) / "top_post_synthesis.blif" #circuit_name + ".vpr.blif" + post_vpr_netlist = Path(temp_dir) / "vpr.out" #circuit_name + ".vpr.blif" lec_base_netlist = None #Reference netlist for LEC + gen_postsynthesis_netlist = Path(temp_dir) / (circuit_name + "_post_synthesis.blif") if circuit_ext == ".blif": #If the user provided a .blif netlist, we use that as the baseline for LEC @@ -158,11 +160,6 @@ def run(architecture_file, circuit_file, #Copy the input netlist for input to vpr shutil.copyfile(str(next_stage_netlist), str(pre_vpr_netlist)) - #Do we need to generate the post-synthesis netlist? (e.g. for LEC) - if should_run_stage(VTR_STAGE.lec, start_stage, end_stage): - if "gen_postsynthesis_netlist" not in vpr_args: - vpr_args["gen_postsynthesis_netlist"] = "on" - if "route_chan_width" in vpr_args: #The User specified a fixed channel width vtr.vpr.run(architecture_copy, circuit_copy, pre_vpr_netlist, @@ -185,8 +182,12 @@ def run(architecture_file, circuit_file, # # Logical Equivalence Checks (LEC) # - if should_run_stage(VTR_STAGE.lec, start_stage, end_stage): - vtr.abc.run_lec(lec_base_netlist, post_vpr_netlist, command_runner=command_runner, log_filename="abc.lec.out") + if check_equivalent: + for file in Path(temp_dir).iterdir(): + if "post_synthesis.blif" in str(file) : + gen_postsynthesis_netlist = file.name + break + vtr.abc.run_lec(lec_base_netlist, gen_postsynthesis_netlist, command_runner=command_runner, temp_dir=temp_dir) if(not keep_intermediate_files): next_stage_netlist.unlink() exts = ('.xml','.sdf','.v') diff --git a/vtr_flow/scripts/python_libs/vtr/odin/odin.py b/vtr_flow/scripts/python_libs/vtr/odin/odin.py index d460f2a1509..1a8b2dcca8a 100644 --- a/vtr_flow/scripts/python_libs/vtr/odin/odin.py +++ b/vtr_flow/scripts/python_libs/vtr/odin/odin.py @@ -48,7 +48,13 @@ def run(architecture_file, circuit_file, del odin_args["use_odin_simulation"] cmd = [odin_exec, "-c", odin_config] - cmd.extend(odin_args) + for arg, value in odin_args.items(): + if value == True: + cmd += ["--" + arg] + elif value == False: + pass + else: + cmd += ["--" + arg, str(value)] cmd.extend(["-U0"]) if(disable_odin_xml): cmd.extend(["-a",architecture_file.name, "-V", circuit_file.name, "-o",output_netlist.name]) diff --git a/vtr_flow/scripts/python_libs/vtr/util.py b/vtr_flow/scripts/python_libs/vtr/util.py index 18ad783471c..504352714e7 100644 --- a/vtr_flow/scripts/python_libs/vtr/util.py +++ b/vtr_flow/scripts/python_libs/vtr/util.py @@ -238,7 +238,7 @@ def mkdir_p(path): """ make_path = Path(path) try: - make_path.mkdir() + make_path.mkdir(parents=True) except OSError as exc: # Python >2.5 if exc.errno == errno.EEXIST and make_path.is_dir(): pass diff --git a/vtr_flow/scripts/vtr-flow.py b/vtr_flow/scripts/vtr-flow.py index 0a44f01e3c8..6e6017da270 100755 --- a/vtr_flow/scripts/vtr-flow.py +++ b/vtr_flow/scripts/vtr-flow.py @@ -15,6 +15,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent / 'python_libs')) import vtr +from vtr import mkdir_p BASIC_VERBOSITY = 1 @@ -218,6 +219,10 @@ def vtr_command_argparser(prog=None): action="store_true", dest="use_old_latches_restoration_script", help="Use the new latches restoration script") + abc.add_argument("-check_equivalent", + default=False, + action="store_true", + help="Enables Logical Equivalence Checks") abc.add_argument("-lut_size", type=int, help="Tells ABC the LUT size of the FPGA architecture") @@ -301,10 +306,10 @@ def vtr_command_main(arg_list, prog=None): abc_args["lut_size"] = args.lut_size odin_args = OrderedDict() - odin_args["--adder_type"] = args.adder_type + odin_args["adder_type"] = args.adder_type if(args.adder_cin_global): - odin_args["--adder_cin_global"] = True + odin_args["adder_cin_global"] = True if(args.disable_odin_xml): odin_args["disable_odin_xml"] = True @@ -338,7 +343,8 @@ def vtr_command_main(arg_list, prog=None): odin_args=odin_args, keep_intermediate_files=args.keep_intermediate_files, keep_result_files=args.keep_result_files, - min_hard_mult_size=args.min_hard_mult_size + min_hard_mult_size=args.min_hard_mult_size, + check_equivalent = args.check_equivalent ) except vtr.CommandError as e: #An external command failed @@ -373,6 +379,7 @@ def vtr_command_main(arg_list, prog=None): finally: seconds = datetime.now() - start print("OK (took {})".format(vtr.format_elapsed_time(seconds))) + mkdir_p(temp_dir) out = Path(temp_dir) / "output.txt" out.touch() with out.open('w') as f: From 7eba52d81f716c04a7769868927f3916825e8a16 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Fri, 26 Jun 2020 10:12:32 -0600 Subject: [PATCH 37/86] fixing default issue --- vtr_flow/scripts/vtr-flow.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vtr_flow/scripts/vtr-flow.py b/vtr_flow/scripts/vtr-flow.py index 6e6017da270..c0838d52215 100755 --- a/vtr_flow/scripts/vtr-flow.py +++ b/vtr_flow/scripts/vtr-flow.py @@ -258,7 +258,6 @@ def vtr_command_argparser(prog=None): # vpr = parser.add_argument_group("Vpr", description="Arguments to be parsed and then passed to VPR") vpr.add_argument("-crit_path_router_iterations", - default=150, type=int, help="Tells ODIN II the minimum multiplier size that should be implemented using hard multiplier (if available).") return parser @@ -322,7 +321,8 @@ def vtr_command_main(arg_list, prog=None): if not args.parse_only: try: vpr_args = process_unkown_args(unkown_args) - vpr_args["max_router_iterations"] = args.crit_path_router_iterations + if(args.crit_path_router_iterations): + vpr_args["max_router_iterations"] = args.crit_path_router_iterations name = "" if(args.name): name=args.name From feef3dd3b9573df89851bb2cf8185681205fec97 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Fri, 26 Jun 2020 14:53:48 -0600 Subject: [PATCH 38/86] fixed the eblif issue --- vtr_flow/scripts/python_libs/vtr/flow.py | 16 +++++++++------- vtr_flow/scripts/vtr-flow.py | 2 +- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/vtr_flow/scripts/python_libs/vtr/flow.py b/vtr_flow/scripts/python_libs/vtr/flow.py index 04250e28bb0..aa8a39e6fe4 100644 --- a/vtr_flow/scripts/python_libs/vtr/flow.py +++ b/vtr_flow/scripts/python_libs/vtr/flow.py @@ -62,18 +62,20 @@ def run(architecture_file, circuit_file, architecture_ext = architecture_file.suffixes vtr.mkdir_p(temp_dir) - + netlist_ext = ".blif" + if ".eblif" in circuit_ext: + netlist_ext = ".eblif" #Define useful filenames - post_odin_netlist = Path(temp_dir) / (circuit_name + '.odin.blif') - post_abc_netlist =Path(temp_dir) / (circuit_name + '.abc.blif') - post_ace_netlist =Path(temp_dir) / (circuit_name + ".ace.blif") + post_odin_netlist = Path(temp_dir) / (circuit_name + '.odin' + netlist_ext) + post_abc_netlist =Path(temp_dir) / (circuit_name + '.abc' + netlist_ext) + post_ace_netlist =Path(temp_dir) / (circuit_name + ".ace" + netlist_ext) post_ace_activity_file = Path(temp_dir) / (circuit_name + ".act") - pre_vpr_netlist = Path(temp_dir) / (circuit_name + ".pre-vpr.blif") + pre_vpr_netlist = Path(temp_dir) / (circuit_name + ".pre-vpr" + netlist_ext) post_vpr_netlist = Path(temp_dir) / "vpr.out" #circuit_name + ".vpr.blif" lec_base_netlist = None #Reference netlist for LEC - gen_postsynthesis_netlist = Path(temp_dir) / (circuit_name + "_post_synthesis.blif") + gen_postsynthesis_netlist = Path(temp_dir) / (circuit_name + "_post_synthesis." + netlist_ext) - if circuit_ext == ".blif": + if "blif" in circuit_ext: #If the user provided a .blif netlist, we use that as the baseline for LEC #(ABC can't LEC behavioural verilog) lec_base_netlist = circuit_file_basename diff --git a/vtr_flow/scripts/vtr-flow.py b/vtr_flow/scripts/vtr-flow.py index c0838d52215..5aacafe8e2d 100755 --- a/vtr_flow/scripts/vtr-flow.py +++ b/vtr_flow/scripts/vtr-flow.py @@ -127,7 +127,7 @@ def vtr_command_argparser(prog=None): action=VtrStageArgparseAction, help="Ending stage of the VTR flow.") - parser.add_argument("-verbose", "-v", + parser.add_argument("-verbose", default=False, action="store_true", dest="verbose", From 5464661ecc1df46b3c2ff2dce1cb4c981994ed04 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Mon, 29 Jun 2020 09:53:01 -0600 Subject: [PATCH 39/86] added and enabled arguments fix_pins, verify_rr_graph, rr_graph_ext, check_route, check_place, and expect_fail --- vtr_flow/scripts/python_libs/vtr/vpr/vpr.py | 76 +++++++++++++- vtr_flow/scripts/vtr-flow.py | 106 ++++++++++++++++---- 2 files changed, 155 insertions(+), 27 deletions(-) diff --git a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py index ceafa61c24d..d8338a5d270 100644 --- a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py +++ b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py @@ -1,6 +1,7 @@ import shutil from pathlib import Path from vtr import mkdir_p, find_vtr_file, CommandRunner, print_verbose, relax_W, determine_lut_size, determine_min_W, verify_file +from vtr.error import InspectError def run_relax_W(architecture, circuit_name, circuit, command_runner=CommandRunner(), temp_dir=".", relax_W_factor=1.3, vpr_exec=None, verbosity=1, logfile_base="vpr", @@ -35,13 +36,28 @@ def run_relax_W(architecture, circuit_name, circuit, command_runner=CommandRunne vpr_min_W_log = '.'.join([logfile_base, "out"]) vpr_relaxed_W_log = '.'.join([logfile_base, "crit_path", "out"]) + max_router_iterations = None + + if "max_router_iterations" in vpr_args: + max_router_iterations = vpr_args["max_router_iterations"] + del vpr_args["max_router_iterations"] + + if "write_rr_graph" in vpr_args: + del vpr_args["write_rr_graph"] + + if "analysis" in vpr_args: + del vpr_args["analysis"] + + if "route" in vpr_args: + del vpr_args["route"] run(architecture, circuit_name, circuit, command_runner, temp_dir, log_filename=vpr_min_W_log, vpr_exec=vpr_exec, vpr_args=vpr_args) if ('pack' in vpr_args or 'place' in vpr_args) and 'route' not in vpr_args: #Don't look for min W if routing was not run return - + if max_router_iterations: + vpr_args["max_router_iterations"]=max_router_iterations min_W = determine_min_W(str(Path(temp_dir) / vpr_min_W_log)) @@ -52,8 +68,6 @@ def run_relax_W(architecture, circuit_name, circuit, command_runner=CommandRunne #VPR does not support performing routing when fixed pins # are specified, and placement is not run; so remove the option - if 'fix_pins' in vpr_args: - del vpr_args['fix_pins'] run(architecture, circuit_name, circuit, command_runner, temp_dir, log_filename=vpr_relaxed_W_log, vpr_exec=vpr_exec, vpr_args=vpr_args) @@ -82,12 +96,64 @@ def run(architecture, circuit_name, circuit, command_runner, temp_dir, output_ne #vpr_args['gen_postsynthesis_netlist'] = output_netlist #Translate arbitrary keyword arguments into options for VPR + do_second_run = False + second_run_args = vpr_args + + if "write_rr_graph" in vpr_args: + do_second_run = True + + if "analysis" in vpr_args: + do_second_run = True + del vpr_args["analysis"] + + if "route" in vpr_args: + dp_second_run = True + del vpr_args["route"] + for arg, value in vpr_args.items(): if value == True: cmd += ["--" + arg] elif value == False: pass else: - cmd += ["--" + arg, str(value)] + if isinstance(value,list): + cmd += ["--" + arg] + for i in range(len(value)): + cmd += [str(value[i])] + else: + cmd += ["--" + arg, str(value)] + + command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) + + if(do_second_run): + rr_graph_ext=".xml" + rr_graph_out_file = "" + if "write_rr_graph" in second_run_args: + rr_graph_out_file = second_run_args["write_rr_graph"] + rr_graph_ext = Path(rr_graph_out_file).suffix + + rr_graph_out_file2 = "rr_graph2" + rr_graph_ext + if "write_rr_graph" in second_run_args: + second_run_args["read_rr_graph"] = rr_graph_out_file + second_run_args["write_rr_graph"] = rr_graph_out_file2 + + second_run_log_file = "vpr_second_run.out" + cmd = [vpr_exec, architecture.name, circuit_name.stem, "--circuit_file", circuit.name] + + for arg, value in vpr_args.items(): + if value == True: + cmd += ["--" + arg] + elif value == False: + pass + else: + cmd += ["--" + arg, str(value)] + + command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=second_run_log_file, indent_depth=1) + + if "write_rr_graph" in second_run_args: + cmd = ["diff", rr_graph_out_file, rr_graph_out_file2] + output, diff_result = command_runner.run_system_command(cmd,temp_dir,log_filename="diff.rr_graph.out", indent_depth=1) + if diff_result: + raise InspectError("failed: vpr (RR Graph XML output not consistent when reloaded)") + - command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) \ No newline at end of file diff --git a/vtr_flow/scripts/vtr-flow.py b/vtr_flow/scripts/vtr-flow.py index 5aacafe8e2d..8ffaa0d12ff 100755 --- a/vtr_flow/scripts/vtr-flow.py +++ b/vtr_flow/scripts/vtr-flow.py @@ -195,6 +195,10 @@ def vtr_command_argparser(prog=None): type=float, metavar="TIMEOUT_SECONDS", help="Maximum time in seconds to spend on a single stage.") + house_keeping.add_argument("-expect_fail", + default=None, + type=str, + help="Informs VTR that this run is expected to fail with this message.") # # ABC arguments # @@ -259,26 +263,45 @@ def vtr_command_argparser(prog=None): vpr = parser.add_argument_group("Vpr", description="Arguments to be parsed and then passed to VPR") vpr.add_argument("-crit_path_router_iterations", type=int, - help="Tells ODIN II the minimum multiplier size that should be implemented using hard multiplier (if available).") + default=150, + help="") + vpr.add_argument("-fix_pins", + type=str, + help="") + vpr.add_argument("-verify_rr_graph", + default=False, + action="store_true", + help="Tells VPR to verify the routing resource graph.") + vpr.add_argument("-rr_graph_ext", + default=".xml", + type=str, + help="") + vpr.add_argument("-check_route", + default=False, + action="store_true", + help="") + vpr.add_argument("-check_place", + default=False, + action="store_true", + help="") + return parser def main(): - vtr_command_main(sys.argv[1:]) + vtr_command_main(sys.argv[1:],prog = sys.argv[0]) def vtr_command_main(arg_list, prog=None): start = datetime.now() - #Load the arguments - args, unkown_args = vtr_command_argparser(prog).parse_known_args(arg_list) - + args, unknown_args = vtr_command_argparser(prog).parse_known_args(arg_list) path_arch_file = Path(args.architecture_file) path_circuit_file = Path(args.circuit_file) + error_status = "OK" if (args.temp_dir == None): temp_dir="./temp" else: temp_dir=args.temp_dir - #Specify how command should be run command_runner = vtr.CommandRunner(track_memory=args.track_memory_usage, max_memory_mb=args.limit_memory_usage, @@ -288,6 +311,7 @@ def vtr_command_main(arg_list, prog=None): echo_cmd=True if args.verbose >= 4 else False, show_failures = args.show_failures) exit_status = 0 + return_status = 0 abc_args = OrderedDict() if(args.iterative_bb): abc_args["iterative_bb"] = True @@ -320,9 +344,21 @@ def vtr_command_main(arg_list, prog=None): try: if not args.parse_only: try: - vpr_args = process_unkown_args(unkown_args) + vpr_args = process_unknown_args(unknown_args) if(args.crit_path_router_iterations): vpr_args["max_router_iterations"] = args.crit_path_router_iterations + if(args.fix_pins): + new_file = str(Path(temp_dir) / Path(args.fix_pins).name) + shutil.copyfile(str((Path(prog).parent.parent / args.fix_pins)), new_file) + vpr_args["fix_pins"] = new_file + if args.verify_rr_graph: + rr_graph_out_file = "rr_graph" + args.rr_graph_ext + vpr_args["write_rr_graph"] = rr_graph_out_file + if args.check_place: + vpr_args["route"] = True + if args.check_route: + vpr_args["analysis"] = True + name = "" if(args.name): name=args.name @@ -348,25 +384,47 @@ def vtr_command_main(arg_list, prog=None): ) except vtr.CommandError as e: #An external command failed - print ("Error: {msg}".format(msg=e.msg)) - print ("\tfull command: ", ' '.join(e.cmd)) - print ("\treturncode : ", e.returncode) - print ("\tlog file : ", e.log) + return_status = 1 + if args.expect_fail: + expect_string = args.expect_fail + actual_error = None + if "exited with return code" in expect_string: + actual_error = "exited with return code {}".format(e.returncode) + else: + actual_error = e.msg + if expect_string != actual_error: + error_status = "failed: expected to fail with '{expected}' but was '{actual}'".format(expected=expect_string,actual = actual_error) + exit_status = 1 + else: + error_status = "OK" + return_status = 0 + if args.verbose: + error_status += " (as expected {})".format(expect_string) + else: + error_status += "*" + if not args.expect_fail or exit_status: + print ("Error: {msg}".format(msg=e.msg)) + print ("\tfull command: ", ' '.join(e.cmd)) + print ("\treturncode : ", e.returncode) + print ("\tlog file : ", e.log) exit_status = 1 except vtr.InspectError as e: #Something went wrong gathering information print ("Error: {msg}".format(msg=e.msg)) print ("\tfile : ", e.filename) exit_status = 2 + return_status = exit_status except vtr.VtrError as e: #Generic VTR errors print ("Error: ", e.msg) exit_status = 3 + return_status = exit_status except KeyboardInterrupt as e: print ("{} recieved keyboard interrupt".format(prog)) exit_status = 4 + return_status = exit_status #Parse the flow results try: @@ -378,7 +436,7 @@ def vtr_command_main(arg_list, prog=None): finally: seconds = datetime.now() - start - print("OK (took {})".format(vtr.format_elapsed_time(seconds))) + print("{status} (took {time})".format(status = error_status, time=vtr.format_elapsed_time(seconds))) mkdir_p(temp_dir) out = Path(temp_dir) / "output.txt" out.touch() @@ -387,21 +445,19 @@ def vtr_command_main(arg_list, prog=None): if(exit_status==0): f.write("success\n") else: - f.write("failure\n") + f.write("exited with return code {}\n".format(exit_status)) f.write("vpr_seconds=%d\nrundir=%s\nhostname=%s\nerror=" % (seconds.total_seconds(), str(Path.cwd()), socket.gethostname())) - if(exit_status!=0): - f.write(str(exit_status)) f.write("\n") - sys.exit(exit_status) + sys.exit(return_status) -def process_unkown_args(unkown_args): - #We convert the unkown_args into a dictionary, which is eventually +def process_unknown_args(unknown_args): + #We convert the unknown_args into a dictionary, which is eventually #used to generate arguments for VPR vpr_args = OrderedDict() - while len(unkown_args) > 0: + while len(unknown_args) > 0: #Get the first argument - arg = unkown_args.pop(0) + arg = unknown_args.pop(0) if arg == '': continue @@ -421,13 +477,19 @@ def process_unkown_args(unkown_args): #Determine if there is a value associated with this argument - if len(unkown_args) == 0 or unkown_args[0].startswith('-'): + if len(unknown_args) == 0 or (unknown_args[0].startswith('-') and arg != "target_ext_pin_util"): #Single value argument, we place these with value 'True' #in vpr_args vpr_args[arg] = True else: #Multivalue argument - val = unkown_args.pop(0) + val = unknown_args.pop(0) + if len(unknown_args) != 0 and not unknown_args[0].startswith('-'): + temp = val + val =[] + val.append(temp) + while len(unknown_args) != 0 and not unknown_args[0].startswith('-'): + val.append(unknown_args.pop(0)) vpr_args[arg] = val return vpr_args From e92a5514ae07511afac87daac2f0c5c084bdaadd Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Mon, 29 Jun 2020 14:39:06 -0600 Subject: [PATCH 40/86] fixing issue with routing argument --- vtr_flow/scripts/python_libs/vtr/vpr/vpr.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py index d8338a5d270..ea3a618140b 100644 --- a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py +++ b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py @@ -69,10 +69,10 @@ def run_relax_W(architecture, circuit_name, circuit, command_runner=CommandRunne #VPR does not support performing routing when fixed pins # are specified, and placement is not run; so remove the option - run(architecture, circuit_name, circuit, command_runner, temp_dir, log_filename=vpr_relaxed_W_log, vpr_exec=vpr_exec, vpr_args=vpr_args) + run(architecture, circuit_name, circuit, command_runner, temp_dir, log_filename=vpr_relaxed_W_log, vpr_exec=vpr_exec, vpr_args=vpr_args, check_for_second_run=False) -def run(architecture, circuit_name, circuit, command_runner, temp_dir, output_netlist=None, log_filename="vpr.out", vpr_exec=None, vpr_args=None): +def run(architecture, circuit_name, circuit, command_runner, temp_dir, output_netlist=None, log_filename="vpr.out", vpr_exec=None, vpr_args=None,check_for_second_run=True): """ Runs VPR with the specified configuration """ @@ -99,15 +99,15 @@ def run(architecture, circuit_name, circuit, command_runner, temp_dir, output_ne do_second_run = False second_run_args = vpr_args - if "write_rr_graph" in vpr_args: + if check_for_second_run and "write_rr_graph" in vpr_args: do_second_run = True - if "analysis" in vpr_args: + if check_for_second_run and "analysis" in vpr_args: do_second_run = True del vpr_args["analysis"] - if "route" in vpr_args: - dp_second_run = True + if check_for_second_run and "route" in vpr_args: + do_second_run = True del vpr_args["route"] for arg, value in vpr_args.items(): From 80f657ca8e35dc03038bf24edf1f470ada7f16f0 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Tue, 30 Jun 2020 08:53:56 -0600 Subject: [PATCH 41/86] fixing sdc_file issue and fixing timing issue --- vtr_flow/scripts/vtr-flow.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/vtr_flow/scripts/vtr-flow.py b/vtr_flow/scripts/vtr-flow.py index 8ffaa0d12ff..5390eb386bf 100755 --- a/vtr_flow/scripts/vtr-flow.py +++ b/vtr_flow/scripts/vtr-flow.py @@ -284,6 +284,10 @@ def vtr_command_argparser(prog=None): default=False, action="store_true", help="") + vpr.add_argument("-sdc_file", + default=None, + type=str, + help="") return parser @@ -301,9 +305,8 @@ def vtr_command_main(arg_list, prog=None): temp_dir="./temp" else: temp_dir=args.temp_dir - #Specify how command should be run - command_runner = vtr.CommandRunner(track_memory=args.track_memory_usage, + command_runner = vtr.CommandRunner(track_memory=True, max_memory_mb=args.limit_memory_usage, timeout_sec=args.timeout, verbose_error=True if args.verbose == 2 else False, @@ -358,6 +361,14 @@ def vtr_command_main(arg_list, prog=None): vpr_args["route"] = True if args.check_route: vpr_args["analysis"] = True + if args.sdc_file: + if not Path(args.sdc_file).exists(): + sdc_file = Path(str(Path(prog).parent.parent) + args.sdc_file) + if sdc_file.exists(): + args.sdc_file = str(sdc_file) + else: + raise vtr.InspectError("Sdc file {} was not found.".format(args.sdc_file)) + vpr_args["sdc_file"] = args.sdc_file name = "" if(args.name): From 308e4137d29bfbe340b87336b253c76714928292 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Tue, 30 Jun 2020 13:06:04 -0600 Subject: [PATCH 42/86] fixed sdc issue with format of input --- vtr_flow/scripts/vtr-flow.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/vtr_flow/scripts/vtr-flow.py b/vtr_flow/scripts/vtr-flow.py index 5390eb386bf..aba44fbe861 100755 --- a/vtr_flow/scripts/vtr-flow.py +++ b/vtr_flow/scripts/vtr-flow.py @@ -363,11 +363,16 @@ def vtr_command_main(arg_list, prog=None): vpr_args["analysis"] = True if args.sdc_file: if not Path(args.sdc_file).exists(): - sdc_file = Path(str(Path(prog).parent.parent) + args.sdc_file) + sdc_file = None + if args.sdc_file.startswith("/"): + sdc_file = Path(str(Path(prog).parent.parent) + args.sdc_file) + else: + sdc_file = Path(prog).parent.parent / args.sdc_file + if sdc_file.exists(): args.sdc_file = str(sdc_file) else: - raise vtr.InspectError("Sdc file {} was not found.".format(args.sdc_file)) + raise vtr.InspectError("Sdc file {arg},{sdc} was not found.".format(arg=args.sdc_file,sdc=str(sdc_file))) vpr_args["sdc_file"] = args.sdc_file name = "" @@ -413,6 +418,8 @@ def vtr_command_main(arg_list, prog=None): error_status += " (as expected {})".format(expect_string) else: error_status += "*" + else: + error_status = "failed: {}".format(e.msg) if not args.expect_fail or exit_status: print ("Error: {msg}".format(msg=e.msg)) print ("\tfull command: ", ' '.join(e.cmd)) @@ -425,12 +432,14 @@ def vtr_command_main(arg_list, prog=None): print ("\tfile : ", e.filename) exit_status = 2 return_status = exit_status + error_status = "failed: {}".format(e.msg) except vtr.VtrError as e: #Generic VTR errors print ("Error: ", e.msg) exit_status = 3 return_status = exit_status + error_status = "failed: {}".format(e.msg) except KeyboardInterrupt as e: print ("{} recieved keyboard interrupt".format(prog)) From 481ea928374438f39ce2d39a473c5b434997c986 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Tue, 30 Jun 2020 14:30:55 -0600 Subject: [PATCH 43/86] fixed an issue with max_router_iterations --- vtr_flow/scripts/python_libs/vtr/vpr/vpr.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py index ea3a618140b..33542f40ca5 100644 --- a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py +++ b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py @@ -39,7 +39,6 @@ def run_relax_W(architecture, circuit_name, circuit, command_runner=CommandRunne max_router_iterations = None if "max_router_iterations" in vpr_args: - max_router_iterations = vpr_args["max_router_iterations"] del vpr_args["max_router_iterations"] if "write_rr_graph" in vpr_args: @@ -56,8 +55,6 @@ def run_relax_W(architecture, circuit_name, circuit, command_runner=CommandRunne if ('pack' in vpr_args or 'place' in vpr_args) and 'route' not in vpr_args: #Don't look for min W if routing was not run return - if max_router_iterations: - vpr_args["max_router_iterations"]=max_router_iterations min_W = determine_min_W(str(Path(temp_dir) / vpr_min_W_log)) @@ -69,10 +66,10 @@ def run_relax_W(architecture, circuit_name, circuit, command_runner=CommandRunne #VPR does not support performing routing when fixed pins # are specified, and placement is not run; so remove the option - run(architecture, circuit_name, circuit, command_runner, temp_dir, log_filename=vpr_relaxed_W_log, vpr_exec=vpr_exec, vpr_args=vpr_args, check_for_second_run=False) + run(architecture, circuit_name, circuit, command_runner, temp_dir, log_filename=vpr_relaxed_W_log, vpr_exec=vpr_exec, vpr_args=vpr_args, specified_channel_width=False) -def run(architecture, circuit_name, circuit, command_runner, temp_dir, output_netlist=None, log_filename="vpr.out", vpr_exec=None, vpr_args=None,check_for_second_run=True): +def run(architecture, circuit_name, circuit, command_runner, temp_dir, output_netlist=None, log_filename="vpr.out", vpr_exec=None, vpr_args=None,specified_channel_width=True): """ Runs VPR with the specified configuration """ @@ -99,14 +96,14 @@ def run(architecture, circuit_name, circuit, command_runner, temp_dir, output_ne do_second_run = False second_run_args = vpr_args - if check_for_second_run and "write_rr_graph" in vpr_args: + if specified_channel_width and "write_rr_graph" in vpr_args: do_second_run = True - if check_for_second_run and "analysis" in vpr_args: + if specified_channel_width and "analysis" in vpr_args: do_second_run = True del vpr_args["analysis"] - if check_for_second_run and "route" in vpr_args: + if specified_channel_width and "route" in vpr_args: do_second_run = True del vpr_args["route"] From 01b5bd809ecfc0a13ab1a1e38da454fad4f1f3be Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Thu, 2 Jul 2020 10:19:44 -0400 Subject: [PATCH 44/86] reverted changes from last commit, fixed an route_chan_width issue, fixed an output spacing issue and fixed the abc script to work with multiple clocks --- vtr_flow/scripts/python_libs/vtr/abc/abc.py | 7 ++++++- vtr_flow/scripts/python_libs/vtr/flow.py | 8 ++++++-- vtr_flow/scripts/python_libs/vtr/util.py | 3 ++- vtr_flow/scripts/python_libs/vtr/vpr/vpr.py | 14 ++++++++------ 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/vtr_flow/scripts/python_libs/vtr/abc/abc.py b/vtr_flow/scripts/python_libs/vtr/abc/abc.py index 5f05b28b143..db8141ddd1d 100644 --- a/vtr_flow/scripts/python_libs/vtr/abc/abc.py +++ b/vtr_flow/scripts/python_libs/vtr/abc/abc.py @@ -65,6 +65,8 @@ def run(architecture_file, circuit_file, if(iterations==0 or abc_flow_type != 2): iterations=1 + original_script = abc_script + for i in range(0, iterations): pre_abc_blif= Path(temp_dir) / (str(i)+"_" + circuit_file.name) post_abc_blif = Path(temp_dir) / (str(i)+"_"+output_netlist.name) @@ -148,6 +150,8 @@ def run(architecture_file, circuit_file, command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename="restore_latch" + str(i) + ".out", indent_depth=1) if(abc_flow_type != 2): break + + abc_script = original_script cmd = [blackbox_latches_script, "--input", post_abc_blif.name,"--output",output_netlist.name,"--vanilla"] command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename="restore_latch" + str(i) + ".out", indent_depth=1) @@ -165,7 +169,8 @@ def populate_clock_list(circuit_file,blackbox_latches_script,clk_list,command_ru cmd = [blackbox_latches_script, "--input", circuit_file.name,"--output_list", clk_list_path.name] command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) with clk_list_path.open('r') as f: - clk_list.append(f.readline().strip('\n')) + for line in f.readlines(): + clk_list.append(line.strip('\n')) def run_lec(reference_netlist, implementation_netlist, command_runner, temp_dir=".", log_filename="abc.dec.out", abc_exec=None): """ diff --git a/vtr_flow/scripts/python_libs/vtr/flow.py b/vtr_flow/scripts/python_libs/vtr/flow.py index aa8a39e6fe4..6144b0f9786 100644 --- a/vtr_flow/scripts/python_libs/vtr/flow.py +++ b/vtr_flow/scripts/python_libs/vtr/flow.py @@ -161,8 +161,12 @@ def run(architecture_file, circuit_file, if should_run_stage(VTR_STAGE.vpr, start_stage, end_stage): #Copy the input netlist for input to vpr shutil.copyfile(str(next_stage_netlist), str(pre_vpr_netlist)) - - if "route_chan_width" in vpr_args: + route_fixed_W = "route_chan_width" in vpr_args + if ("route" in vpr_args or "place" in vpr_args) and not route_fixed_W: + vpr_args["route_chan_width"] = 300 + route_fixed_W = True + + if route_fixed_W: #The User specified a fixed channel width vtr.vpr.run(architecture_copy, circuit_copy, pre_vpr_netlist, output_netlist=post_vpr_netlist, diff --git a/vtr_flow/scripts/python_libs/vtr/util.py b/vtr_flow/scripts/python_libs/vtr/util.py index 504352714e7..309b061a6cb 100644 --- a/vtr_flow/scripts/python_libs/vtr/util.py +++ b/vtr_flow/scripts/python_libs/vtr/util.py @@ -76,7 +76,7 @@ def run_system_command(self, cmd, temp_dir, log_filename=None, expected_return_c cmd = memory_limit + cmd #Enable memory tracking? - memory_tracking = ["/usr/bin/time", "-v"] + memory_tracking = ["/usr/bin/env", "time", "-v"] if self._track_memory and self.check_command(memory_tracking[0]): cmd = memory_tracking + cmd @@ -112,6 +112,7 @@ def run_system_command(self, cmd, temp_dir, log_filename=None, expected_return_c with (Path(temp_dir) / log_filename).open('w') as log_f: #Print the command at the top of the log log_f.write(" ".join(cmd)) + log_f.write("\n") #Read from subprocess output for line in proc.stdout: diff --git a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py index 33542f40ca5..171fa919d2a 100644 --- a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py +++ b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py @@ -39,6 +39,7 @@ def run_relax_W(architecture, circuit_name, circuit, command_runner=CommandRunne max_router_iterations = None if "max_router_iterations" in vpr_args: + max_router_iterations = vpr_args["max_router_iterations"] del vpr_args["max_router_iterations"] if "write_rr_graph" in vpr_args: @@ -55,7 +56,8 @@ def run_relax_W(architecture, circuit_name, circuit, command_runner=CommandRunne if ('pack' in vpr_args or 'place' in vpr_args) and 'route' not in vpr_args: #Don't look for min W if routing was not run return - + if max_router_iterations: + vpr_args["max_router_iterations"]=max_router_iterations min_W = determine_min_W(str(Path(temp_dir) / vpr_min_W_log)) relaxed_W = relax_W(min_W, relax_W_factor) @@ -66,10 +68,10 @@ def run_relax_W(architecture, circuit_name, circuit, command_runner=CommandRunne #VPR does not support performing routing when fixed pins # are specified, and placement is not run; so remove the option - run(architecture, circuit_name, circuit, command_runner, temp_dir, log_filename=vpr_relaxed_W_log, vpr_exec=vpr_exec, vpr_args=vpr_args, specified_channel_width=False) + run(architecture, circuit_name, circuit, command_runner, temp_dir, log_filename=vpr_relaxed_W_log, vpr_exec=vpr_exec, vpr_args=vpr_args, check_for_second_run=False) -def run(architecture, circuit_name, circuit, command_runner, temp_dir, output_netlist=None, log_filename="vpr.out", vpr_exec=None, vpr_args=None,specified_channel_width=True): +def run(architecture, circuit_name, circuit, command_runner, temp_dir, output_netlist=None, log_filename="vpr.out", vpr_exec=None, vpr_args=None,check_for_second_run=True): """ Runs VPR with the specified configuration """ @@ -96,14 +98,14 @@ def run(architecture, circuit_name, circuit, command_runner, temp_dir, output_ne do_second_run = False second_run_args = vpr_args - if specified_channel_width and "write_rr_graph" in vpr_args: + if check_for_second_run and "write_rr_graph" in vpr_args: do_second_run = True - if specified_channel_width and "analysis" in vpr_args: + if check_for_second_run and "analysis" in vpr_args: do_second_run = True del vpr_args["analysis"] - if specified_channel_width and "route" in vpr_args: + if check_for_second_run and "route" in vpr_args: do_second_run = True del vpr_args["route"] From 70b209612698cd358c5b2a775c001c57e54e1561 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Thu, 2 Jul 2020 11:02:41 -0400 Subject: [PATCH 45/86] added valgrind functionality --- vtr_flow/scripts/python_libs/vtr/util.py | 15 ++++++++++++--- vtr_flow/scripts/vtr-flow.py | 7 ++++++- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/vtr_flow/scripts/python_libs/vtr/util.py b/vtr_flow/scripts/python_libs/vtr/util.py index 309b061a6cb..c5763b1d1bf 100644 --- a/vtr_flow/scripts/python_libs/vtr/util.py +++ b/vtr_flow/scripts/python_libs/vtr/util.py @@ -23,7 +23,7 @@ class RawDefaultHelpFormatter(argparse.ArgumentDefaultsHelpFormatter, argparse.R class CommandRunner(object): - def __init__(self, timeout_sec=None, max_memory_mb=None, track_memory=True, verbose_error=False, verbose=False, echo_cmd=False, indent="\t", show_failures=False): + def __init__(self, timeout_sec=None, max_memory_mb=None, track_memory=True, verbose_error=False, verbose=False, echo_cmd=False, indent="\t", show_failures=False,valgrind=False): """ An object for running system commands with timeouts, memory limits and varying verbose-ness @@ -36,6 +36,7 @@ def __init__(self, timeout_sec=None, max_memory_mb=None, track_memory=True, verb verbose: Produce more verbose output. Default: False echo_cmd: Echo the command before running. Default: False indent: The string specifying a single indent (used in verbose mode) + valgrind: Indicates if commands should be run with valgrind """ self._timeout_sec = timeout_sec self._max_memory_mb = max_memory_mb @@ -45,6 +46,7 @@ def __init__(self, timeout_sec=None, max_memory_mb=None, track_memory=True, verb self._echo_cmd = echo_cmd self._indent = indent self._show_failures = show_failures + self._valgrind = valgrind def run_system_command(self, cmd, temp_dir, log_filename=None, expected_return_code=0, indent_depth=0): """ @@ -78,7 +80,11 @@ def run_system_command(self, cmd, temp_dir, log_filename=None, expected_return_c #Enable memory tracking? memory_tracking = ["/usr/bin/env", "time", "-v"] if self._track_memory and self.check_command(memory_tracking[0]): - cmd = memory_tracking + cmd + if self._valgrind: + valgrind_args = ["valgrind", "--leak-check=full", "--suppressions=$vtr_flow_path/../vpr/valgrind.supp", "--error-exitcode=1", "--errors-for-leak-kinds=none", "--track-origins=yes", "--log-file=valgrind.log","--error-limit=no"] + cmd = memory_tracking + valgrind_args + cmd + else: + cmd = memory_tracking + cmd #Flush before calling subprocess to ensure output is ordered #correctly if stdout is buffered @@ -97,9 +103,12 @@ def run_system_command(self, cmd, temp_dir, log_filename=None, expected_return_c proc = None try: #Call the command + stderr = subprocess.STDOUT + if self._valgrind: + stderr = None proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, #We grab stdout - stderr=subprocess.STDOUT, #stderr redirected to stderr + stderr=stderr, #stderr redirected to stderr universal_newlines=True, #Lines always end in \n cwd=temp_dir, #Where to run the command ) diff --git a/vtr_flow/scripts/vtr-flow.py b/vtr_flow/scripts/vtr-flow.py index aba44fbe861..b0baaed6dab 100755 --- a/vtr_flow/scripts/vtr-flow.py +++ b/vtr_flow/scripts/vtr-flow.py @@ -199,6 +199,10 @@ def vtr_command_argparser(prog=None): default=None, type=str, help="Informs VTR that this run is expected to fail with this message.") + house_keeping.add_argument("-valgrind", + default=False, + action="store_true", + help="Runs the flow with valgrind with the following options (--leak-check=full, --errors-for-leak-kinds=none, --error-exitcode=1, --track-origins=yes)") # # ABC arguments # @@ -312,7 +316,8 @@ def vtr_command_main(arg_list, prog=None): verbose_error=True if args.verbose == 2 else False, verbose=True if args.verbose > 2 else False, echo_cmd=True if args.verbose >= 4 else False, - show_failures = args.show_failures) + show_failures = args.show_failures, + valgrind = args.valgrind) exit_status = 0 return_status = 0 abc_args = OrderedDict() From d0e31b0dd660b87b64ae8d9e22b8eea136e6a496 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Thu, 2 Jul 2020 12:10:55 -0400 Subject: [PATCH 46/86] fixed num_clb issue --- vtr_flow/scripts/python_libs/vtr/abc/abc.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/vtr_flow/scripts/python_libs/vtr/abc/abc.py b/vtr_flow/scripts/python_libs/vtr/abc/abc.py index db8141ddd1d..197a86eaf30 100644 --- a/vtr_flow/scripts/python_libs/vtr/abc/abc.py +++ b/vtr_flow/scripts/python_libs/vtr/abc/abc.py @@ -48,7 +48,7 @@ def run(architecture_file, circuit_file, lut_size = determine_lut_size(str(architecture_file)) if(abc_flow_type): - populate_clock_list(circuit_file,blackbox_latches_script,clk_list,command_runner,temp_dir,log_filename) + populate_clock_list(circuit_file,blackbox_latches_script,clk_list,command_runner,temp_dir,"report_clocks.abc.out") if abc_exec == None: abc_exec = find_vtr_file('abc', is_executable=True) @@ -66,20 +66,20 @@ def run(architecture_file, circuit_file, iterations=1 original_script = abc_script - + input_file = circuit_file.name for i in range(0, iterations): pre_abc_blif= Path(temp_dir) / (str(i)+"_" + circuit_file.name) post_abc_blif = Path(temp_dir) / (str(i)+"_"+output_netlist.name) post_abc_raw_blif = Path(temp_dir) / (str(i)+"_"+output_netlist.with_suffix('').stem+".raw.abc.blif") if(abc_flow_type==3): - cmd = [blackbox_latches_script, "--input", circuit_file.name,"--output",pre_abc_blif.name] + cmd = [blackbox_latches_script, "--input", input_file,"--output",pre_abc_blif.name] command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=str(i)+"_blackboxing_latch.out", indent_depth=1) elif(len(clk_list)>i): - cmd = [blackbox_latches_script,"--clk_list", clk_list[i], "--input", circuit_file.name,"--output",pre_abc_blif.name] + cmd = [blackbox_latches_script,"--clk_list", clk_list[i], "--input", input_file,"--output",pre_abc_blif.name] command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=str(i)+"_blackboxing_latch.out", indent_depth=1) else: - pre_abc_blif = circuit_file + pre_abc_blif = input_file if abc_script == None: @@ -152,6 +152,8 @@ def run(architecture_file, circuit_file, break abc_script = original_script + input_file = post_abc_blif.name + cmd = [blackbox_latches_script, "--input", post_abc_blif.name,"--output",output_netlist.name,"--vanilla"] command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename="restore_latch" + str(i) + ".out", indent_depth=1) From 51758f3d57241f411471f40dc602baa028ae43b7 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Thu, 2 Jul 2020 13:14:39 -0400 Subject: [PATCH 47/86] fixing valgrind file error --- vtr_flow/scripts/python_libs/vtr/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vtr_flow/scripts/python_libs/vtr/util.py b/vtr_flow/scripts/python_libs/vtr/util.py index c5763b1d1bf..4658c7b0568 100644 --- a/vtr_flow/scripts/python_libs/vtr/util.py +++ b/vtr_flow/scripts/python_libs/vtr/util.py @@ -81,7 +81,7 @@ def run_system_command(self, cmd, temp_dir, log_filename=None, expected_return_c memory_tracking = ["/usr/bin/env", "time", "-v"] if self._track_memory and self.check_command(memory_tracking[0]): if self._valgrind: - valgrind_args = ["valgrind", "--leak-check=full", "--suppressions=$vtr_flow_path/../vpr/valgrind.supp", "--error-exitcode=1", "--errors-for-leak-kinds=none", "--track-origins=yes", "--log-file=valgrind.log","--error-limit=no"] + valgrind_args = ["valgrind", "--leak-check=full", "--suppressions={}".format(find_vtr_file("valgrind.supp")), "--error-exitcode=1", "--errors-for-leak-kinds=none", "--track-origins=yes", "--log-file=valgrind.log","--error-limit=no"] cmd = memory_tracking + valgrind_args + cmd else: cmd = memory_tracking + cmd From 39a10c2c7d2ec2496d0074cb26200648b4a41f40 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Thu, 9 Jul 2020 10:06:53 -0600 Subject: [PATCH 48/86] rename vtr-flow.py to run_vtr_flow.py, added missing arguments, and added some comments. --- run_reg_test.pl | 2 +- vtr_flow/scripts/python_libs/vtr/flow.py | 12 ++++-- .../scripts/{vtr-flow.py => run_vtr_flow.py} | 41 +++++++++++++------ vtr_flow/scripts/run_vtr_task.pl | 6 +-- .../scripts/unittests/check_py_vtr_flow.py | 4 +- vtr_flow/scripts/vtr-task.py | 2 +- 6 files changed, 44 insertions(+), 23 deletions(-) rename vtr_flow/scripts/{vtr-flow.py => run_vtr_flow.py} (91%) diff --git a/run_reg_test.pl b/run_reg_test.pl index c3932fcf719..afdc4731cb3 100755 --- a/run_reg_test.pl +++ b/run_reg_test.pl @@ -65,7 +65,7 @@ my $skip_qor = 0; my $show_failures = 0; my $num_cpu = 1; -my $script = "vtr-flow.py"; +my $script = "run_vtr_flow.py"; my $long_task_names = 0; # Parse Input Arguments diff --git a/vtr_flow/scripts/python_libs/vtr/flow.py b/vtr_flow/scripts/python_libs/vtr/flow.py index 6144b0f9786..c2a520faee6 100644 --- a/vtr_flow/scripts/python_libs/vtr/flow.py +++ b/vtr_flow/scripts/python_libs/vtr/flow.py @@ -24,7 +24,10 @@ def run(architecture_file, circuit_file, keep_intermediate_files=True, keep_result_files=True, min_hard_mult_size=3, - check_equivalent=False): + check_equivalent=False, + min_hard_adder_size=1, + use_old_abc_script=False, + relax_W_factor=1.3): """ Runs the VTR CAD flow to map the specificied circuit_file onto the target architecture_file @@ -102,7 +105,8 @@ def run(architecture_file, circuit_file, command_runner=command_runner, temp_dir=temp_dir, odin_args=odin_args, - min_hard_mult_size=min_hard_mult_size) + min_hard_mult_size=min_hard_mult_size, + min_hard_adder_size=min_hard_adder_size) next_stage_netlist = post_odin_netlist @@ -118,7 +122,8 @@ def run(architecture_file, circuit_file, command_runner=command_runner, temp_dir=temp_dir, abc_args=abc_args, - keep_intermediate_files=keep_intermediate_files) + keep_intermediate_files=keep_intermediate_files, + use_old_abc_script=use_old_abc_script) next_stage_netlist = post_abc_netlist @@ -178,6 +183,7 @@ def run(architecture_file, circuit_file, vtr.vpr.run_relax_W(architecture_copy, circuit_copy, pre_vpr_netlist, output_netlist=post_vpr_netlist, command_runner=command_runner, + relax_W_factor=relax_W_factor, temp_dir=temp_dir, verbosity=verbosity, vpr_args=vpr_args) diff --git a/vtr_flow/scripts/vtr-flow.py b/vtr_flow/scripts/run_vtr_flow.py similarity index 91% rename from vtr_flow/scripts/vtr-flow.py rename to vtr_flow/scripts/run_vtr_flow.py index b0baaed6dab..3913156e1ed 100755 --- a/vtr_flow/scripts/vtr-flow.py +++ b/vtr_flow/scripts/run_vtr_flow.py @@ -170,7 +170,7 @@ def vtr_command_argparser(prog=None): house_keeping.add_argument("-name", default=None, - help="Directory to run the flow in (will be created if non-existant).") + help="Name for this run to be output.") house_keeping.add_argument("-track_memory_usage", default=False, @@ -182,7 +182,7 @@ def vtr_command_argparser(prog=None): default=False, action="store_true", dest="show_failures", - help="Show failures")#needs updating + help="Tells the flow to display failures in the console.") house_keeping.add_argument("-limit_memory_usage", default=None, @@ -206,7 +206,7 @@ def vtr_command_argparser(prog=None): # # ABC arguments # - abc = parser.add_argument_group("abc", description="Iterative black-boxing flow for multi-clock circuits options") + abc = parser.add_argument_group("abc", description="Arguments used by ABC") abc.add_argument("-iterative_bb", default=False, action="store_true", @@ -226,11 +226,15 @@ def vtr_command_argparser(prog=None): default=False, action="store_true", dest="use_old_latches_restoration_script", - help="Use the new latches restoration script") + help="Use the old latches restoration script") abc.add_argument("-check_equivalent", default=False, action="store_true", help="Enables Logical Equivalence Checks") + abc.add_argument("-use_old_abc_script", + default=False, + action="store_true", + help="Enables use of legacy ABC script adapted for new ABC") abc.add_argument("-lut_size", type=int, help="Tells ABC the LUT size of the FPGA architecture") @@ -240,7 +244,7 @@ def vtr_command_argparser(prog=None): odin = parser.add_argument_group("Odin", description="Arguments to be passed to ODIN II") odin.add_argument("-adder_type", default="default", - help="Odin adder type") + help="Tells ODIN II the adder type used in this configuration") odin.add_argument("-adder_cin_global", default=False, action="store_true", @@ -255,12 +259,16 @@ def vtr_command_argparser(prog=None): default=False, action="store_true", dest="use_odin_simulation", - help="Tells ODIN II to connect the first cin in an adder/subtractor chain to a global gnd/vdd net.") + help="Tells odin to run simulation.") odin.add_argument("-min_hard_mult_size", default=3, type=int, metavar="min_hard_mult_size", help="Tells ODIN II the minimum multiplier size that should be implemented using hard multiplier (if available).") + odin.add_argument("-min_hard_adder_size", + default=1, + type=int, + help="Tells ODIN II the minimum adder size that should be implemented using hard adder (if available).") # # VPR arguments # @@ -268,10 +276,14 @@ def vtr_command_argparser(prog=None): vpr.add_argument("-crit_path_router_iterations", type=int, default=150, - help="") + help="Tells VPR the amount of iterations allowed to obtain the critical path.") vpr.add_argument("-fix_pins", type=str, - help="") + help="Controls how the placer handles I/O pads during placement.") + vpr.add_argument("-relax_W_factor", + type=float, + default=1.3, + help="Factor by which to relax minimum channel width for critical path delay routing") vpr.add_argument("-verify_rr_graph", default=False, action="store_true", @@ -279,19 +291,19 @@ def vtr_command_argparser(prog=None): vpr.add_argument("-rr_graph_ext", default=".xml", type=str, - help="") + help="Determines the output rr_graph files' extention.") vpr.add_argument("-check_route", default=False, action="store_true", - help="") + help="Tells VPR to run final analysis stage.") vpr.add_argument("-check_place", default=False, action="store_true", - help="") + help="Tells VPR to run routing stage") vpr.add_argument("-sdc_file", default=None, type=str, - help="") + help="Path to SDC timing constraints file.") return parser @@ -401,7 +413,10 @@ def vtr_command_main(arg_list, prog=None): keep_intermediate_files=args.keep_intermediate_files, keep_result_files=args.keep_result_files, min_hard_mult_size=args.min_hard_mult_size, - check_equivalent = args.check_equivalent + check_equivalent = args.check_equivalent, + min_hard_adder_size = args.min_hard_adder_size, + use_old_abc_script=args.use_old_abc_script, + relax_W_factor=args.relax_W_factor ) except vtr.CommandError as e: #An external command failed diff --git a/vtr_flow/scripts/run_vtr_task.pl b/vtr_flow/scripts/run_vtr_task.pl index 3a890f74db5..101d546a4b0 100755 --- a/vtr_flow/scripts/run_vtr_task.pl +++ b/vtr_flow/scripts/run_vtr_task.pl @@ -68,7 +68,7 @@ my $run_prefix = "run"; my $show_runtime_estimates = 1; my $system_type = "local"; -my $script = "vtr-flow.py"; +my $script = "run_vtr_flow.py"; my $shared_script_params = ""; my $verbosity = 0; my $short_task_names = 0; @@ -216,7 +216,7 @@ sub generate_single_task_actions { my $circuits_dir; my $archs_dir; my $sdc_dir = undef; - my $script_default = "vtr-flow.py run_vtr_flow.pl"; + my $script_default = "run_vtr_flow"; my $script_path; my $script_params_common = $shared_script_params; # start with the shared ones then build unique ones my @circuits_list; @@ -288,7 +288,7 @@ sub generate_single_task_actions { close(CONFIG_FH); # Using default script - if (index($script_default, $script) != -1 ) { + if (index($script, $script_default) != -1 ) { # This is hack to automatically add the option '-temp_dir .' if using the run_vtr_flow.pl script # This ensures that a 'temp' folder is not created in each circuit directory diff --git a/vtr_flow/scripts/unittests/check_py_vtr_flow.py b/vtr_flow/scripts/unittests/check_py_vtr_flow.py index 8531db2caae..9ccb9b1f3dc 100755 --- a/vtr_flow/scripts/unittests/check_py_vtr_flow.py +++ b/vtr_flow/scripts/unittests/check_py_vtr_flow.py @@ -11,7 +11,7 @@ vtr_flow_path = scripts_path.parent temp_dir = pathlib.Path.cwd() / "temp" -# run_vtr_flow.pl will be compared against vtr-flow.py using the following test cases as arguments: +# run_vtr_flow.pl will be compared against run_vtr_flow.py using the following test cases as arguments: arg_list = ["benchmarks/verilog/ch_intrinsics.v arch/timing/k6_N10_mem32K_40nm.xml -track_memory_usage --router_heap binary --min_route_chan_width_hint 38", "benchmarks/verilog/diffeq1.v arch/timing/k6_N10_mem32K_40nm.xml -track_memory_usage --router_heap bucket --min_route_chan_width_hint 46" ] @@ -65,7 +65,7 @@ def main(): # Run with old and new flows run_vtr_flow("run_vtr_flow.pl", "old pl", temp_dir_pl, test_str) - run_vtr_flow("vtr-flow.py", "new py", temp_dir_py, test_str) + run_vtr_flow("run_vtr_flow.py", "new py", temp_dir_py, test_str) # Check that output files match for (pattern, skip_lines) in files_to_validate: diff --git a/vtr_flow/scripts/vtr-task.py b/vtr_flow/scripts/vtr-task.py index a6ce9b8d2fd..a960f6bb656 100755 --- a/vtr_flow/scripts/vtr-task.py +++ b/vtr_flow/scripts/vtr-task.py @@ -433,7 +433,7 @@ def create_jobs(args, configs): executable = [config.script_path] else: #Default flow script - executable = [find_vtr_file('vtr-flow.py', is_executable=True)] + executable = [find_vtr_file('run_vtr_flow.py', is_executable=True)] #Collect any extra script params from the config file script_params = [abs_arch_filepath, abs_circuit_filepath] From ce6baae7b890ba5d7d14866e920edc0878b17129 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Thu, 9 Jul 2020 10:24:53 -0600 Subject: [PATCH 49/86] updating documentation to use the python script --- dev/tutorial/NewDeveloperTutorial.txt | 6 +++--- .../dev/tutorials/new_developer_tutorial.md | 2 +- doc/src/tutorials/flow/basic_flow.rst | 2 +- doc/src/vtr/power_estimation/index.rst | 4 ++-- doc/src/vtr/run_vtr_flow.rst | 20 +++++++++---------- doc/src/vtr/running_vtr.rst | 4 ++-- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/dev/tutorial/NewDeveloperTutorial.txt b/dev/tutorial/NewDeveloperTutorial.txt index babf29e79dd..79ba611d3c6 100644 --- a/dev/tutorial/NewDeveloperTutorial.txt +++ b/dev/tutorial/NewDeveloperTutorial.txt @@ -80,14 +80,14 @@ Task #5 - Open the Black Box 1. Using the custom Verilog circuit and architecture created in Task #4, directly run Odin II on it to generate a blif netlist. You may need to skim the Odin II - readme file and the vtr_flow/scripts/run_vtr_flow.pl. + readme file and the vtr_flow/scripts/run_vtr_flow.py. 2. Using the output netlist of Odin II, run ABC to generate a technology-mapped blif file. - You may need to skim vtr_flow/scripts/run_vtr_flow.pl. + You may need to skim vtr_flow/scripts/run_vtr_flow.py. 3. Using the output of ABC, run VPR to complete the mapping of a user circuit to a target architecture. You may need to consult the VPR User Manual and skim - You may need to skim vtr_flow/scripts/run_vtr_flow.pl. + You may need to skim vtr_flow/scripts/run_vtr_flow.py. 4. Read the VPR section of the online documentation. diff --git a/doc/src/dev/tutorials/new_developer_tutorial.md b/doc/src/dev/tutorials/new_developer_tutorial.md index 6fdffafcb78..51519b39d5d 100644 --- a/doc/src/dev/tutorials/new_developer_tutorial.md +++ b/doc/src/dev/tutorials/new_developer_tutorial.md @@ -43,7 +43,7 @@ Perform a simple architecture experiment. Run an experiment that varies Fc_in fr At this stage, you have gotten a taste of how an FPGA architect would go about using VTR. As a developer though, you need a much deeper understanding of how this tool works. The purpose of this section is to have you to learn the details of the VTR CAD flow by having you manually do what the scripts do. -Using the custom Verilog circuit and architecture created in the previous step, directly run Odin II on it to generate a blif netlist. You may need to skim the `ODIN_II/README.rst` and the `vtr_flow/scripts/run_vtr_flow.pl`. +Using the custom Verilog circuit and architecture created in the previous step, directly run Odin II on it to generate a blif netlist. You may need to skim the `ODIN_II/README.rst` and the `vtr_flow/scripts/run_vtr_flow.py`. Using the output netlist of Odin II, run ABC to generate a technology-mapped blif file. You may need to skim the [ABC homepage](http://www.eecs.berkeley.edu/~alanmi/abc/). diff --git a/doc/src/tutorials/flow/basic_flow.rst b/doc/src/tutorials/flow/basic_flow.rst index 2552958beb3..bb0fb77319c 100644 --- a/doc/src/tutorials/flow/basic_flow.rst +++ b/doc/src/tutorials/flow/basic_flow.rst @@ -46,7 +46,7 @@ The following steps show you to run the VTR design flow to map a sample circuit Some also contain a ``golden_results.txt`` file that is used by the scripts to check for correctness. - The ``vtr_release/vtr_flow/scripts/run_vtr_flow.pl`` script describes the CAD flow employed in the test. + The ``vtr_release/vtr_flow/scripts/run_vtr_flow.py`` script describes the CAD flow employed in the test. You can modify the flow by editing this script. At this point, feel free to run any of the tasks pre-pended with "regression". diff --git a/doc/src/vtr/power_estimation/index.rst b/doc/src/vtr/power_estimation/index.rst index c5f45f8df2f..e971e9978d3 100644 --- a/doc/src/vtr/power_estimation/index.rst +++ b/doc/src/vtr/power_estimation/index.rst @@ -33,8 +33,8 @@ The easiest way to run the VTR flow is to use the :ref:`run_vtr_flow` script. In order to perform power estimation, you must add the following options: - * :option:`run_vtr_flow.pl -power` - * :option:`run_vtr_flow.pl -cmos_tech` ```` + * :option:`run_vtr_flow.py -power` + * :option:`run_vtr_flow.py -cmos_tech` ```` The CMOS technology properties file is an XML file that contains relevant process-dependent information needed for power estimation. XML files for 22nm, 45nm, and 130nm PTM models can be found here:: diff --git a/doc/src/vtr/run_vtr_flow.rst b/doc/src/vtr/run_vtr_flow.rst index 68e5346ddd0..bf9f504c4a2 100644 --- a/doc/src/vtr/run_vtr_flow.rst +++ b/doc/src/vtr/run_vtr_flow.rst @@ -7,16 +7,16 @@ This script runs the VTR flow for a single benchmark circuit and architecture fi The script is located at:: - $VTR_ROOT/vtr_flow/scripts/run_vtr_flow.pl + $VTR_ROOT/vtr_flow/scripts/run_vtr_flow.py -.. program:: run_vtr_flow.pl +.. program:: run_vtr_flow.py Basic Usage ~~~~~~~~~~~ -At a minimum ``run_vtr_flow.pl`` requires two command-line arguments:: +At a minimum ``run_vtr_flow.py`` requires two command-line arguments:: - run_vtr_flow.pl + run_vtr_flow.p where: @@ -44,20 +44,20 @@ The script will also produce an output files (\*.out) for each stage, containing Advanced Usage ~~~~~~~~~~~~~~ -Additional *optional* command arguments can also be passed to ``run_vtr_flow.pl``:: +Additional *optional* command arguments can also be passed to ``run_vtr_flow.py``:: - run_vtr_flow.pl [] [] + run_vtr_flow.py [] [] where: - * ```` are additional arguments passed to ``run_vtr_flow.pl`` (described below), - * ```` are any arguments not recognized by ``run_vtr_flow.pl``. These will be forwarded to VPR. + * ```` are additional arguments passed to ``run_vtr_flow.py`` (described below), + * ```` are any arguments not recognized by ``run_vtr_flow.py``. These will be forwarded to VPR. For example:: - run_vtr_flow.pl my_circuit.v my_arch.xml -track_memory_usage --pack --place + run_vtr_flow.py my_circuit.v my_arch.xml -track_memory_usage --pack --place -will run the VTR flow to map the circuit ``my_circuit.v`` onto the architecture ``my_arch.xml``; the arguments ``--pack`` and ``--place`` will be passed to VPR (since they are unrecognized arguments to ``run_vtr_flow.pl``). +will run the VTR flow to map the circuit ``my_circuit.v`` onto the architecture ``my_arch.xml``; the arguments ``--pack`` and ``--place`` will be passed to VPR (since they are unrecognized arguments to ``run_vtr_flow.py``). They will cause VPR to perform only :ref:`packing and placement `. Detailed Command-line Options diff --git a/doc/src/vtr/running_vtr.rst b/doc/src/vtr/running_vtr.rst index 1eb62707969..922d1fa5298 100644 --- a/doc/src/vtr/running_vtr.rst +++ b/doc/src/vtr/running_vtr.rst @@ -24,7 +24,7 @@ The :ref:`run_vtr_flow` script is provided to execute the VTR flow for a single .. code-block:: none - $VTR_ROOT/vtr_flow/scripts/run_vtr_flow.pl + $VTR_ROOT/vtr_flow/scripts/run_vtr_flow.py It requires two arguments: @@ -41,7 +41,7 @@ Architecture files can be found under:: The script can also be used to run parts of the VTR flow. -.. seealso:: :ref:`run_vtr_flow` for the detailed command line options of ``run_vtr_flow.pl``. +.. seealso:: :ref:`run_vtr_flow` for the detailed command line options of ``run_vtr_flow.py``. Running Multiple Benchmarks & Architectures with Tasks From 35be0d4520422d3f34584b10620a0fa2bda29e86 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Fri, 10 Jul 2020 15:01:21 -0600 Subject: [PATCH 50/86] adding auto doc to the python library and fixing various minor issues. --- vtr_flow/scripts/python_libs/doc/Makefile | 20 +++++++ vtr_flow/scripts/python_libs/doc/conf.py | 55 ++++++++++++++++++ vtr_flow/scripts/python_libs/doc/index.rst | 24 ++++++++ vtr_flow/scripts/python_libs/doc/make.bat | 35 ++++++++++++ vtr_flow/scripts/python_libs/vtr/__init__.py | 2 +- vtr_flow/scripts/python_libs/vtr/abc/abc.py | 56 ++++++++++++++++++- .../scripts/python_libs/vtr/ace/__init__.py | 1 + .../vtr/{ace_flow/ace.py => ace/ace_flow.py} | 33 ++++++++++- .../python_libs/vtr/ace_flow/__init__.py | 1 - vtr_flow/scripts/python_libs/vtr/flow.py | 51 +++++++++++++---- vtr_flow/scripts/python_libs/vtr/odin/odin.py | 35 +++++++++++- vtr_flow/scripts/python_libs/vtr/vpr/vpr.py | 51 +++++++++++++++-- vtr_flow/scripts/run_vtr_flow.py | 2 +- 13 files changed, 340 insertions(+), 26 deletions(-) create mode 100755 vtr_flow/scripts/python_libs/doc/Makefile create mode 100755 vtr_flow/scripts/python_libs/doc/conf.py create mode 100755 vtr_flow/scripts/python_libs/doc/index.rst create mode 100755 vtr_flow/scripts/python_libs/doc/make.bat create mode 100644 vtr_flow/scripts/python_libs/vtr/ace/__init__.py rename vtr_flow/scripts/python_libs/vtr/{ace_flow/ace.py => ace/ace_flow.py} (62%) delete mode 100644 vtr_flow/scripts/python_libs/vtr/ace_flow/__init__.py diff --git a/vtr_flow/scripts/python_libs/doc/Makefile b/vtr_flow/scripts/python_libs/doc/Makefile new file mode 100755 index 00000000000..d4bb2cbb9ed --- /dev/null +++ b/vtr_flow/scripts/python_libs/doc/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/vtr_flow/scripts/python_libs/doc/conf.py b/vtr_flow/scripts/python_libs/doc/conf.py new file mode 100755 index 00000000000..d84fa4eec16 --- /dev/null +++ b/vtr_flow/scripts/python_libs/doc/conf.py @@ -0,0 +1,55 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys +import sphinx_rtd_theme +sys.path.insert(0, os.path.abspath('..')) + + +# -- Project information ----------------------------------------------------- + +project = u'Verilog-to-Routing' +copyright = u'2016, VTR Developers' +author = u'VTR Developers' + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "sphinx_rtd_theme", + 'sphinx.ext.autodoc', + 'sphinx.ext.napoleon', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'sphinx_rtd_theme' + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] diff --git a/vtr_flow/scripts/python_libs/doc/index.rst b/vtr_flow/scripts/python_libs/doc/index.rst new file mode 100755 index 00000000000..4d02cba7824 --- /dev/null +++ b/vtr_flow/scripts/python_libs/doc/index.rst @@ -0,0 +1,24 @@ +Welcome to VTR's documentation! +=============================== +.. automodule:: vtr.flow + :members: run +.. automodule:: vtr.odin.odin + :members: +.. automodule:: vtr.abc.abc + :members: +.. automodule:: vtr.ace.ace_flow + :members: +.. automodule:: vtr.vpr.vpr + :members: +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/vtr_flow/scripts/python_libs/doc/make.bat b/vtr_flow/scripts/python_libs/doc/make.bat new file mode 100755 index 00000000000..2119f51099b --- /dev/null +++ b/vtr_flow/scripts/python_libs/doc/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/vtr_flow/scripts/python_libs/vtr/__init__.py b/vtr_flow/scripts/python_libs/vtr/__init__.py index e474ed00d12..da64c64cba7 100644 --- a/vtr_flow/scripts/python_libs/vtr/__init__.py +++ b/vtr_flow/scripts/python_libs/vtr/__init__.py @@ -3,7 +3,7 @@ from .abc import run, run_lec from .vpr import run,run_relax_W from .odin import run -from .ace_flow import run +from .ace import run from .error import * from .flow import run, VTR_STAGE, parse_vtr_flow from .task import load_task_config, TaskConfig, find_task_config_file diff --git a/vtr_flow/scripts/python_libs/vtr/abc/abc.py b/vtr_flow/scripts/python_libs/vtr/abc/abc.py index 197a86eaf30..0f01475a579 100644 --- a/vtr_flow/scripts/python_libs/vtr/abc/abc.py +++ b/vtr_flow/scripts/python_libs/vtr/abc/abc.py @@ -1,14 +1,46 @@ import shutil from pathlib import Path -from vtr import mkdir_p, find_vtr_file, determine_lut_size, verify_file +from vtr import mkdir_p, find_vtr_file, determine_lut_size, verify_file, CommandRunner from vtr.error import VtrError, InspectError, CommandError def run(architecture_file, circuit_file, - output_netlist, command_runner, + output_netlist, command_runner=CommandRunner(), temp_dir=".", log_filename="abc.out", abc_exec=None, abc_script=None, abc_rc=None, use_old_abc_script = False, abc_args = None, keep_intermediate_files=1): + """ + Runs ABC to optimize specified file. + + To run: + vtr.abc.run(args) + Required arguments: + architecture_file : Architecture file to target + + circuit_file : Circuit file to optimize + + output_netlist : File name to output the resulting circuit to + + Options: + command_runner : A CommandRunner object used to run system commands + + temp_dir : Directory to run in (created if non-existent) + + log_filename : File to log result to + + abc_exec : ABC executable to be run + + abc_script : the script to be run on abc + + abc_rc : the ABC rc file + + use_old_abc_script : Enables the use of the old ABC script + + abc_args : A dictionary of keyword arguments to pass on to ABC + + keep_intermediate_files : Determines if intermediate files are kept or deleted + + """ mkdir_p(temp_dir) verify_file(architecture_file, "Architecture") @@ -174,9 +206,27 @@ def populate_clock_list(circuit_file,blackbox_latches_script,clk_list,command_ru for line in f.readlines(): clk_list.append(line.strip('\n')) -def run_lec(reference_netlist, implementation_netlist, command_runner, temp_dir=".", log_filename="abc.dec.out", abc_exec=None): +def run_lec(reference_netlist, implementation_netlist, command_runner=CommandRunner(), temp_dir=".", log_filename="abc.dec.out", abc_exec=None): """ Run Logical Equivalence Checking (LEC) between two netlists using ABC + + To run: + vtr.abc.run_lec(args) + + Required arguments: + reference_netlist : The reference netlist to be commpared to + + implementation_netlist : The implemeted netlist to compare to the reference netlist + + Options: + command_runner : A CommandRunner object used to run system commands + + temp_dir : Directory to run in (created if non-existent) + + log_filename : File to log result to + + abc_exec : ABC executable to be run + """ mkdir_p(temp_dir) diff --git a/vtr_flow/scripts/python_libs/vtr/ace/__init__.py b/vtr_flow/scripts/python_libs/vtr/ace/__init__.py new file mode 100644 index 00000000000..8553fb3d888 --- /dev/null +++ b/vtr_flow/scripts/python_libs/vtr/ace/__init__.py @@ -0,0 +1 @@ +from .ace_flow import run \ No newline at end of file diff --git a/vtr_flow/scripts/python_libs/vtr/ace_flow/ace.py b/vtr_flow/scripts/python_libs/vtr/ace/ace_flow.py similarity index 62% rename from vtr_flow/scripts/python_libs/vtr/ace_flow/ace.py rename to vtr_flow/scripts/python_libs/vtr/ace/ace_flow.py index 46229de8a7e..2fda01df260 100644 --- a/vtr_flow/scripts/python_libs/vtr/ace_flow/ace.py +++ b/vtr_flow/scripts/python_libs/vtr/ace/ace_flow.py @@ -1,6 +1,35 @@ -from vtr import find_vtr_file, verify_file +from vtr import find_vtr_file, verify_file,CommandRunner from pathlib import Path -def run(circuit_file, old_netlist, output_netlist, output_activity_file, command_runner, temp_dir=".", log_filename="ace.out", ace_exec=None, ace_seed = 1): +def run(circuit_file, old_netlist, output_netlist, output_activity_file, + command_runner=CommandRunner(), temp_dir=".", log_filename="ace.out", + ace_exec=None, ace_seed = 1): + """ + Runs ACE for activity estimation + + To run: + vtr.ace.run(args) + + Required arguments: + circuit_file : Circuit file to optimize + + old_netlist : netlist to be anylized + + output_netlist : File name to output the resulting circuit to + + output_activity_file : The output activity file + + Options: + command_runner : A CommandRunner object used to run system commands + + temp_dir : Directory to run in (created if non-existent) + + log_filename : File to log result to + + ace_exec : ACE executable to be run + + ace_seed : The ACE seed + """ + verify_file(circuit_file, "Circuit") verify_file(old_netlist, "Previous netlist") verify_file(output_netlist, "Output netlist", False) diff --git a/vtr_flow/scripts/python_libs/vtr/ace_flow/__init__.py b/vtr_flow/scripts/python_libs/vtr/ace_flow/__init__.py deleted file mode 100644 index ffac7c866ed..00000000000 --- a/vtr_flow/scripts/python_libs/vtr/ace_flow/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .ace import run \ No newline at end of file diff --git a/vtr_flow/scripts/python_libs/vtr/flow.py b/vtr_flow/scripts/python_libs/vtr/flow.py index c2a520faee6..3d8b31f00d2 100644 --- a/vtr_flow/scripts/python_libs/vtr/flow.py +++ b/vtr_flow/scripts/python_libs/vtr/flow.py @@ -24,27 +24,56 @@ def run(architecture_file, circuit_file, keep_intermediate_files=True, keep_result_files=True, min_hard_mult_size=3, - check_equivalent=False, min_hard_adder_size=1, + check_equivalent=False, use_old_abc_script=False, relax_W_factor=1.3): """ - Runs the VTR CAD flow to map the specificied circuit_file onto the target architecture_file + Runs the VTR CAD flow to map the specified circuit_file onto the target architecture_file - Arguments - --------- - architecture_file: Architecture file to target - circuit_file : Circuit to implement + To run: + vtr.run(args) + Required arguments: + architecture_file : Architecture file to target + + circuit_file : Circuit to implement + + Options: power_tech_file : Technology power file. Enables power analysis and runs ace - - temp_dir : Directory to run in (created if non-existant) + start_stage : Stage of the flow to start at + end_stage : Stage of the flow to finish at + + temp_dir : Directory to run in (created if non-existent) + command_runner : A CommandRunner object used to run system commands - parse_config_file: The configuration file defining how to parse metrics from results + + parse_config_file : The configuration file defining how to parse metrics from results + verbosity : How much output to produce - vpr_args : A dictionary of keywork arguments to pass on to VPR + + odin_args : A dictionary of keyword arguments to pass on to ODIN II + + abc_args : A dictionary of keyword arguments to pass on to ABC + + vpr_args : A dictionary of keyword arguments to pass on to VPR + + keep_intermediate_files : Determines if intermediate files are kept or deleted + + keep_result_files : Determines if the result files are kept or deleted + + min_hard_mult_size : Tells ODIN II the minimum multiplier size that should be implemented using hard multiplier (if available) + + min_hard_adder_size : Tells ODIN II the minimum adder size that should be implemented using hard adder (if available). + + check_equivalent : Enables Logical Equivalence Checks + + use_old_abc_script : Enables the use of the old ABC script + + relax_W_factor : Factor by which to relax minimum channel width for critical path delay routing + """ if vpr_args == None: vpr_args = OrderedDict() @@ -140,7 +169,7 @@ def run(architecture_file, circuit_file, power_tech_file=Path(power_tech_file) if should_run_stage(VTR_STAGE.ace, start_stage, end_stage): - vtr.ace_flow.run(next_stage_netlist, old_netlist = post_odin_netlist, output_netlist=post_ace_netlist, + vtr.ace.run(next_stage_netlist, old_netlist = post_odin_netlist, output_netlist=post_ace_netlist, output_activity_file=post_ace_activity_file, command_runner=command_runner, temp_dir=temp_dir) diff --git a/vtr_flow/scripts/python_libs/vtr/odin/odin.py b/vtr_flow/scripts/python_libs/vtr/odin/odin.py index 1a8b2dcca8a..2e8a53d4574 100644 --- a/vtr_flow/scripts/python_libs/vtr/odin/odin.py +++ b/vtr_flow/scripts/python_libs/vtr/odin/odin.py @@ -1,10 +1,10 @@ import shutil from pathlib import Path -from vtr import mkdir_p, find_vtr_file, file_replace, determine_memory_addr_width, verify_file +from vtr import mkdir_p, find_vtr_file, file_replace, determine_memory_addr_width, verify_file, CommandRunner def run(architecture_file, circuit_file, output_netlist, - command_runner, + command_runner=CommandRunner(), temp_dir=".", odin_args="--adder_type default", log_filename="odin.out", @@ -12,6 +12,37 @@ def run(architecture_file, circuit_file, odin_config=None, min_hard_mult_size=3, min_hard_adder_size=1): + """ + Runs ODIN II on the specified architecture file and circuit file + + To run: + vtr.odin.run(args) + + Required arguments: + architecture_file : Architecture file to target + + circuit_file : Circuit file to optimize + + output_netlist : File name to output the resulting circuit to + + Options: + command_runner : A CommandRunner object used to run system commands + + temp_dir : Directory to run in (created if non-existent) + + odin_args: A dictionary of keyword arguments to pass on to ODIN II + + log_filename : File to log result to + + odin_exec: ODIN II executable to be run + + odin_config: The ODIN II configuration file + + min_hard_mult_size : Tells ODIN II the minimum multiplier size that should be implemented using hard multiplier (if available) + + min_hard_adder_size : Tells ODIN II the minimum adder size that should be implemented using hard adder (if available). + + """ mkdir_p(temp_dir) verify_file(architecture_file, "Architecture") diff --git a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py index 171fa919d2a..4b98eca7717 100644 --- a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py +++ b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py @@ -8,21 +8,35 @@ def run_relax_W(architecture, circuit_name, circuit, command_runner=CommandRunne vpr_args=None, output_netlist=None): """ Runs VPR twice: + 1st: To find the minimum channel width + 2nd: At relaxed channel width (e.g. for critical path delay) + + To run: + vtr.vpr.run_relax_W(args) - Arguments - --------- + Required arguments: architecture: Architecture file + + circuit_name: Curcuit netlist + circuit: Input circuit netlist + Options: command_runner: CommandRunner object + temp_dir: Directory to run in - + relax_W_factor: Factor by which to relax minimum channel width for critical path delay routing + + vpr_exec: Path to the VPR executable + verbosity: How much progress output to produce + logfile_base: Base name for log files (e.g. "vpr" produces vpr.min_W.out, vpr.relaxed_W.out) + vpr_args: Extra arguments for VPR - vpr_exec: Path to the VPR executable + output_netlist: Output implementation netlist to generate """ if vpr_args is None: @@ -71,9 +85,36 @@ def run_relax_W(architecture, circuit_name, circuit, command_runner=CommandRunne run(architecture, circuit_name, circuit, command_runner, temp_dir, log_filename=vpr_relaxed_W_log, vpr_exec=vpr_exec, vpr_args=vpr_args, check_for_second_run=False) -def run(architecture, circuit_name, circuit, command_runner, temp_dir, output_netlist=None, log_filename="vpr.out", vpr_exec=None, vpr_args=None,check_for_second_run=True): +def run(architecture, circuit_name, circuit, command_runner=CommandRunner(), temp_dir=".", output_netlist=None, + log_filename="vpr.out", vpr_exec=None, vpr_args=None,check_for_second_run=True): """ Runs VPR with the specified configuration + + To run: + vtr.vpr.run(args) + + Required arguments: + architecture: Architecture file + + circuit_name: Curcuit netlist + + circuit: Input circuit file + + Options: + command_runner: CommandRunner object + + temp_dir: Directory to run in + + output_netlist: Output implementation netlist to generate + + log_filename : File to log result to + + vpr_exec: Path to the VPR executable + + vpr_args: Extra arguments for VPR + + check_for_second_run: enables checking for arguments in vpr_args that require a second run of VPR ie analysis + """ if vpr_args is None: diff --git a/vtr_flow/scripts/run_vtr_flow.py b/vtr_flow/scripts/run_vtr_flow.py index 3913156e1ed..521944dae97 100755 --- a/vtr_flow/scripts/run_vtr_flow.py +++ b/vtr_flow/scripts/run_vtr_flow.py @@ -413,8 +413,8 @@ def vtr_command_main(arg_list, prog=None): keep_intermediate_files=args.keep_intermediate_files, keep_result_files=args.keep_result_files, min_hard_mult_size=args.min_hard_mult_size, - check_equivalent = args.check_equivalent, min_hard_adder_size = args.min_hard_adder_size, + check_equivalent = args.check_equivalent, use_old_abc_script=args.use_old_abc_script, relax_W_factor=args.relax_W_factor ) From f987b87ffe085f688a8778bad1b47fa190538a02 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Sat, 11 Jul 2020 09:43:34 -0600 Subject: [PATCH 51/86] adding check_incremental_sta_consistency argument and functionality, and fixing ace issue --- vtr_flow/scripts/python_libs/vtr/__init__.py | 2 +- vtr_flow/scripts/python_libs/vtr/flow.py | 18 ++++++- vtr_flow/scripts/python_libs/vtr/util.py | 2 +- .../scripts/python_libs/vtr/vpr/__init__.py | 2 +- vtr_flow/scripts/python_libs/vtr/vpr/vpr.py | 52 ++++++++++++++++++- vtr_flow/scripts/run_vtr_flow.py | 5 ++ 6 files changed, 74 insertions(+), 7 deletions(-) diff --git a/vtr_flow/scripts/python_libs/vtr/__init__.py b/vtr_flow/scripts/python_libs/vtr/__init__.py index da64c64cba7..4039844df1e 100644 --- a/vtr_flow/scripts/python_libs/vtr/__init__.py +++ b/vtr_flow/scripts/python_libs/vtr/__init__.py @@ -1,7 +1,7 @@ from .util import load_config_lines, mkdir_p, find_vtr_file, CommandRunner, print_verbose, relax_W, file_replace, make_enum, RawDefaultHelpFormatter, VERBOSITY_CHOICES,format_elapsed_time, write_tab_delimitted_csv, load_list_file, find_vtr_root, argparse_str2bool, get_next_run_dir, get_latest_run_dir, verify_file from .inspect import determine_lut_size, determine_min_W, determine_memory_addr_width, load_parse_patterns, load_pass_requirements, load_parse_results from .abc import run, run_lec -from .vpr import run,run_relax_W +from .vpr import run,run_relax_W,cmp_full_vs_incr_STA from .odin import run from .ace import run from .error import * diff --git a/vtr_flow/scripts/python_libs/vtr/flow.py b/vtr_flow/scripts/python_libs/vtr/flow.py index 3d8b31f00d2..dc193332014 100644 --- a/vtr_flow/scripts/python_libs/vtr/flow.py +++ b/vtr_flow/scripts/python_libs/vtr/flow.py @@ -26,6 +26,7 @@ def run(architecture_file, circuit_file, min_hard_mult_size=3, min_hard_adder_size=1, check_equivalent=False, + check_incremental_sta_consistency = False, use_old_abc_script=False, relax_W_factor=1.3): """ @@ -74,6 +75,8 @@ def run(architecture_file, circuit_file, relax_W_factor : Factor by which to relax minimum channel width for critical path delay routing + check_incremental_sta_consistency : Do a second-run of the incremental analysis to compare the result files + """ if vpr_args == None: vpr_args = OrderedDict() @@ -106,6 +109,7 @@ def run(architecture_file, circuit_file, post_vpr_netlist = Path(temp_dir) / "vpr.out" #circuit_name + ".vpr.blif" lec_base_netlist = None #Reference netlist for LEC gen_postsynthesis_netlist = Path(temp_dir) / (circuit_name + "_post_synthesis." + netlist_ext) + rr_graph_ext=".xml" if "blif" in circuit_ext: #If the user provided a .blif netlist, we use that as the baseline for LEC @@ -206,7 +210,8 @@ def run(architecture_file, circuit_file, output_netlist=post_vpr_netlist, command_runner=command_runner, temp_dir=temp_dir, - vpr_args=vpr_args) + vpr_args=vpr_args, + rr_graph_ext=rr_graph_ext) else: #First find minW and then re-route at a relaxed W vtr.vpr.run_relax_W(architecture_copy, circuit_copy, pre_vpr_netlist, @@ -228,7 +233,16 @@ def run(architecture_file, circuit_file, if "post_synthesis.blif" in str(file) : gen_postsynthesis_netlist = file.name break - vtr.abc.run_lec(lec_base_netlist, gen_postsynthesis_netlist, command_runner=command_runner, temp_dir=temp_dir) + vtr.abc.run_lec(lec_base_netlist, gen_postsynthesis_netlist, command_runner=command_runner, temp_dir=temp_dir,check_incremental_sta_consistency=check_incremental_sta_consistency) + + if check_incremental_sta_consistency: + vtr.vpr.cmp_full_vs_incr_STA(architecture_copy, circuit_copy, pre_vpr_netlist, + command_runner=command_runner, + vpr_args=vpr_args, + rr_graph_ext=rr_graph_ext, + temp_dir=temp_dir + ) + if(not keep_intermediate_files): next_stage_netlist.unlink() exts = ('.xml','.sdf','.v') diff --git a/vtr_flow/scripts/python_libs/vtr/util.py b/vtr_flow/scripts/python_libs/vtr/util.py index 4658c7b0568..7485e7b9a19 100644 --- a/vtr_flow/scripts/python_libs/vtr/util.py +++ b/vtr_flow/scripts/python_libs/vtr/util.py @@ -317,7 +317,7 @@ def find_file_from_vtr_root(filename, vtr_root, is_executable=False): directory_path = (Path(vtr_root) / subdir) for file_path in directory_path.glob('**/*'): if file_path.name == filename: - if file_path.is_file: + if file_path.is_file(): if is_executable: #Found an executable file as required if os.access(str(file_path), os.X_OK): diff --git a/vtr_flow/scripts/python_libs/vtr/vpr/__init__.py b/vtr_flow/scripts/python_libs/vtr/vpr/__init__.py index afa018cffa7..68018083ed2 100644 --- a/vtr_flow/scripts/python_libs/vtr/vpr/__init__.py +++ b/vtr_flow/scripts/python_libs/vtr/vpr/__init__.py @@ -1 +1 @@ -from .vpr import run,run_relax_W \ No newline at end of file +from .vpr import run,run_relax_W,cmp_full_vs_incr_STA \ No newline at end of file diff --git a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py index 4b98eca7717..9242d3af8e0 100644 --- a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py +++ b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py @@ -86,7 +86,7 @@ def run_relax_W(architecture, circuit_name, circuit, command_runner=CommandRunne def run(architecture, circuit_name, circuit, command_runner=CommandRunner(), temp_dir=".", output_netlist=None, - log_filename="vpr.out", vpr_exec=None, vpr_args=None,check_for_second_run=True): + log_filename="vpr.out", vpr_exec=None, vpr_args=None,check_for_second_run=True,rr_graph_ext=".xml",): """ Runs VPR with the specified configuration @@ -166,7 +166,6 @@ def run(architecture, circuit_name, circuit, command_runner=CommandRunner(), tem command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) if(do_second_run): - rr_graph_ext=".xml" rr_graph_out_file = "" if "write_rr_graph" in second_run_args: rr_graph_out_file = second_run_args["write_rr_graph"] @@ -196,4 +195,53 @@ def run(architecture, circuit_name, circuit, command_runner=CommandRunner(), tem if diff_result: raise InspectError("failed: vpr (RR Graph XML output not consistent when reloaded)") +def cmp_full_vs_incr_STA(architecture,circuit_name,circuit,command_runner=CommandRunner(),vpr_args=None,rr_graph_ext=".xml",temp_dir=".",vpr_exec=None): + """" + Sanity check that full STA and the incremental STA produce the same *.net, *.place, *.route files as well as identical timing report files + """ + verify_file(architecture, "Architecture") + verify_file(circuit_name, "Circuit") + verify_file(circuit, "Circuit") + + default_output_filenames = [ + "{}.net".format(circuit_name.stem), + "{}.place".format(circuit_name.stem), + "{}.route".format(circuit_name.stem), + "report_timing.setup.rpt", + "report_timing.hold.rpt", + "report_unconstrained_timing.setup.rpt", + "report_unconstrained_timing.hold.rpt"] + + dif_exec = "diff" + move_exec = "mv" + # The full STA flow should have already been run + # directly rename the output files + for filename in default_output_filenames: + cmd = [move_exec,filename,"full_sta_{}".format(filename)] + command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename="move.out", indent_depth=1) + + # run incremental STA flow + incremental_vpr_args = vpr_args + incremental_vpr_args["timing_update_type"]="incremental" + fixed_W_log_file = "vpr.incr_sta.out" + + run(architecture, circuit_name, circuit, command_runner, temp_dir, log_filename=fixed_W_log_file, vpr_exec=vpr_exec, vpr_args=incremental_vpr_args, check_for_second_run=False) + + # Rename the incremental STA output files + for filename in default_output_filenames: + cmd = [move_exec,filename,"incremental_sta_{}".format(filename)] + command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename="move.out", indent_depth=1) + + failed_msg = "Failed with these files (not identical):" + identical = True + + for filename in default_output_filenames: + cmd=[dif_exec,"full_sta_{}".format(filename),"incremental_sta_{}".filename] + cmd_output,cmd_return_code = command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename="diff.out", indent_depth=1) + if(cmd_return_code): + identical = False + failed_msg += " {}".format(filename) + + if not identical: + raise InspectError(failed_msg) diff --git a/vtr_flow/scripts/run_vtr_flow.py b/vtr_flow/scripts/run_vtr_flow.py index 521944dae97..6bd5d0a7fc3 100755 --- a/vtr_flow/scripts/run_vtr_flow.py +++ b/vtr_flow/scripts/run_vtr_flow.py @@ -304,6 +304,10 @@ def vtr_command_argparser(prog=None): default=None, type=str, help="Path to SDC timing constraints file.") + vpr.add_argument("-check_incremental_sta_consistency", + default=False, + action="store_true", + help="Do a second-run of the incremental analysis to compare the result files") return parser @@ -415,6 +419,7 @@ def vtr_command_main(arg_list, prog=None): min_hard_mult_size=args.min_hard_mult_size, min_hard_adder_size = args.min_hard_adder_size, check_equivalent = args.check_equivalent, + check_incremental_sta_consistency = args.check_incremental_sta_consistency, use_old_abc_script=args.use_old_abc_script, relax_W_factor=args.relax_W_factor ) From dd5cd2aea92196d205f1575eaeacf6d8375d9fb6 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Sat, 11 Jul 2020 10:26:01 -0600 Subject: [PATCH 52/86] fixed format issue --- vtr_flow/scripts/python_libs/vtr/vpr/vpr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py index 9242d3af8e0..9fa8a3035ae 100644 --- a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py +++ b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py @@ -237,7 +237,7 @@ def cmp_full_vs_incr_STA(architecture,circuit_name,circuit,command_runner=Comman identical = True for filename in default_output_filenames: - cmd=[dif_exec,"full_sta_{}".format(filename),"incremental_sta_{}".filename] + cmd=[dif_exec,"full_sta_{}".format(filename),"incremental_sta_{}".format(filename)] cmd_output,cmd_return_code = command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename="diff.out", indent_depth=1) if(cmd_return_code): identical = False From 16eed5e98e599e6fd06928500cb3ed5c7e10bb13 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Mon, 13 Jul 2020 07:21:06 -0600 Subject: [PATCH 53/86] fixed issue where if there was a syntax error, the test said it passed. --- vtr_flow/scripts/python_libs/vtr/flow.py | 2 +- vtr_flow/scripts/python_libs/vtr/vpr/vpr.py | 3 ++- vtr_flow/scripts/run_vtr_flow.py | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/vtr_flow/scripts/python_libs/vtr/flow.py b/vtr_flow/scripts/python_libs/vtr/flow.py index dc193332014..0396f6bf26e 100644 --- a/vtr_flow/scripts/python_libs/vtr/flow.py +++ b/vtr_flow/scripts/python_libs/vtr/flow.py @@ -233,7 +233,7 @@ def run(architecture_file, circuit_file, if "post_synthesis.blif" in str(file) : gen_postsynthesis_netlist = file.name break - vtr.abc.run_lec(lec_base_netlist, gen_postsynthesis_netlist, command_runner=command_runner, temp_dir=temp_dir,check_incremental_sta_consistency=check_incremental_sta_consistency) + vtr.abc.run_lec(lec_base_netlist, gen_postsynthesis_netlist, command_runner=command_runner, temp_dir=temp_dir) if check_incremental_sta_consistency: vtr.vpr.cmp_full_vs_incr_STA(architecture_copy, circuit_copy, pre_vpr_netlist, diff --git a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py index 9fa8a3035ae..e4f71de8851 100644 --- a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py +++ b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py @@ -121,7 +121,7 @@ def run(architecture, circuit_name, circuit, command_runner=CommandRunner(), tem vpr_args = OrderedDict() mkdir_p(temp_dir) - + if vpr_exec == None: vpr_exec = find_vtr_file('vpr', is_executable=True) @@ -177,6 +177,7 @@ def run(architecture, circuit_name, circuit, command_runner=CommandRunner(), tem second_run_args["write_rr_graph"] = rr_graph_out_file2 second_run_log_file = "vpr_second_run.out" + cmd = [vpr_exec, architecture.name, circuit_name.stem, "--circuit_file", circuit.name] for arg, value in vpr_args.items(): diff --git a/vtr_flow/scripts/run_vtr_flow.py b/vtr_flow/scripts/run_vtr_flow.py index 6bd5d0a7fc3..b9138754a7d 100755 --- a/vtr_flow/scripts/run_vtr_flow.py +++ b/vtr_flow/scripts/run_vtr_flow.py @@ -320,7 +320,7 @@ def vtr_command_main(arg_list, prog=None): args, unknown_args = vtr_command_argparser(prog).parse_known_args(arg_list) path_arch_file = Path(args.architecture_file) path_circuit_file = Path(args.circuit_file) - error_status = "OK" + error_status = "Error" if (args.temp_dir == None): temp_dir="./temp" else: @@ -423,6 +423,7 @@ def vtr_command_main(arg_list, prog=None): use_old_abc_script=args.use_old_abc_script, relax_W_factor=args.relax_W_factor ) + error_status = "OK" except vtr.CommandError as e: #An external command failed return_status = 1 From f21b0baaa97382b191226c294199ddc2d02288c2 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Mon, 13 Jul 2020 09:29:20 -0600 Subject: [PATCH 54/86] added auto doc to cmp_full_vs_incr_STA function --- vtr_flow/scripts/python_libs/vtr/vpr/vpr.py | 22 +++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py index e4f71de8851..1179f7f3412 100644 --- a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py +++ b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py @@ -121,7 +121,7 @@ def run(architecture, circuit_name, circuit, command_runner=CommandRunner(), tem vpr_args = OrderedDict() mkdir_p(temp_dir) - + if vpr_exec == None: vpr_exec = find_vtr_file('vpr', is_executable=True) @@ -177,7 +177,6 @@ def run(architecture, circuit_name, circuit, command_runner=CommandRunner(), tem second_run_args["write_rr_graph"] = rr_graph_out_file2 second_run_log_file = "vpr_second_run.out" - cmd = [vpr_exec, architecture.name, circuit_name.stem, "--circuit_file", circuit.name] for arg, value in vpr_args.items(): @@ -200,6 +199,25 @@ def cmp_full_vs_incr_STA(architecture,circuit_name,circuit,command_runner=Comman """" Sanity check that full STA and the incremental STA produce the same *.net, *.place, *.route files as well as identical timing report files + To run: + vtr.vpr.cmp_full_vs_incr_STA(args) + + Required arguments: + architecture: Architecture file + + circuit_name: Curcuit netlist + + circuit: Input circuit file + + Options: + command_runner: CommandRunner object + + temp_dir: Directory to run in + + vpr_exec: Path to the VPR executable + + vpr_args: Extra arguments for VPR + """ verify_file(architecture, "Architecture") verify_file(circuit_name, "Circuit") From 42138460fa102092bb93d9ee46588311407d855e Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Mon, 13 Jul 2020 09:55:25 -0600 Subject: [PATCH 55/86] remove vtr-task.py and vtr-test.py because I didn't actually work on these, these are to be implemented in a different branch --- vtr_flow/scripts/vtr-task.py | 653 ----------------------------------- vtr_flow/scripts/vtr-test.py | 209 ----------- 2 files changed, 862 deletions(-) delete mode 100755 vtr_flow/scripts/vtr-task.py delete mode 100755 vtr_flow/scripts/vtr-test.py diff --git a/vtr_flow/scripts/vtr-task.py b/vtr_flow/scripts/vtr-task.py deleted file mode 100755 index a960f6bb656..00000000000 --- a/vtr_flow/scripts/vtr-task.py +++ /dev/null @@ -1,653 +0,0 @@ -#!/usr/bin/env python3 -from pathlib import Path -from pathlib import PurePath -import sys -import argparse -import itertools -import textwrap -import subprocess -import time -import shutil -from datetime import datetime -from multiprocessing import Pool - -sys.path.insert(0, str(Path(__file__).resolve().parent / 'python_libs')) - -from vtr import load_list_file, find_vtr_file, mkdir_p, print_verbose, find_vtr_root, CommandRunner, format_elapsed_time, RawDefaultHelpFormatter, VERBOSITY_CHOICES, argparse_str2bool, get_next_run_dir, get_latest_run_dir, load_task_config, TaskConfig, find_task_config_file, CommandRunner, load_pass_requirements, load_parse_results -from vtr.error import VtrError, InspectError, CommandError - -BASIC_VERBOSITY = 1 -FAILED_LOG_VERBOSITY = 2 -ALL_LOG_VERBOSITY = 4 - -class Job: - - def __init__(self, task_name, arch, circuit, work_dir, command): - self._task_name = task_name - self._arch = arch - self._circuit = circuit - self._command = command - self._work_dir = work_dir - - def task_name(self): - return self._task_name - - def arch(self): - return self._arch - - def circuit(self): - return self._circuit - - def job_name(self): - return str(PurePath(self.arch()).joinpath(self.circuit())) - - def command(self): - return self._command - - def work_dir(self, run_dir): - return str(PurePath(run_dir).joinpath(self._work_dir)) - -def vtr_command_argparser(prog=None): - description = textwrap.dedent( - """ - Runs one or more VTR tasks. - """ - ) - epilog = textwrap.dedent( - """ - Examples - -------- - - Run the task named 'timing_chain': - - %(prog)s timing_chain - - Run all the tasks listed in the file 'task_list.txt': - - %(prog)s -l task_list.txt - - Run 'timing_chain' with 4 jobs running in parallel: - - %(prog)s timing_chain -j4 - - Exit Code - --------- - The exit code equals the number failures (i.e. exit code 0 indicates no failures). - """ - ) - - parser = argparse.ArgumentParser( - prog=prog, - description=description, - epilog=epilog, - formatter_class=RawDefaultHelpFormatter, - ) - - # - # Major arguments - # - parser.add_argument('task', - nargs="*", - help="Tasks to be run") - - parser.add_argument('-l', '--list_file', - nargs="*", - default=[], - metavar="TASK_LIST_FILE", - help="A file listing tasks to be run") - - parser.add_argument("--parse", - default=False, - action="store_true", - dest="parse", - help="Perform only parsing on the latest task run") - - parser.add_argument("--create_golden", - default=False, - action="store_true", - dest="create_golden", - help="Update or create golden results for the specified task") - - parser.add_argument("--check_golden", - default=False, - action="store_true", - dest="check_golden", - help="Check the latest task run against golden results") - - parser.add_argument('--system', - choices=['local'], - default='local', - help="What system to run the tasks on.") - - parser.add_argument('-j', - default=1, - type=int, - metavar="NUM_PROC", - help="How many processors to use for execution.") - - parser.add_argument('--timeout', - default=30*24*60*60, #30 days - metavar="TIMEOUT_SECONDS", - help="Time limit for this script.") - - parser.add_argument("-v", "--verbosity", - choices=VERBOSITY_CHOICES, - default=2, - type=int, - help="Sets the verbosity of the script. Higher values produce more output.") - - parser.add_argument("--work_dir", - default=None, - help="Directory to store intermediate and result files." - "If None, set to the relevant directory under $VTR_ROOT/vtr_flow/tasks.") - - parser.add_argument("--print_metadata", - default=True, - type=argparse_str2bool, - help="Print meta-data like command-line arguments and run-time") - - return parser - -def main(): - vtr_command_main(sys.argv[1:]) - -def vtr_command_main(arg_list, prog=None): - start = datetime.now() - - #Load the arguments - args = vtr_command_argparser(prog).parse_args(arg_list) - - args.run = True - if args.parse or args.create_golden or args.check_golden: - #Don't run if parsing or handling golden results - args.run = False - - if args.run: - #Always parse if running - args.parse = True - - if args.print_metadata: - print ("# {} {}\n".format(prog, ' '.join(arg_list))) - - num_failed = -1 - try: - task_names = args.task - - for list_file in args.list_file: - task_names += load_list_file(list_file) - - config_files = [find_task_config_file(task_name) for task_name in task_names] - - configs = [load_task_config(config_file) for config_file in config_files] - - num_failed = run_tasks(args, configs) - - except CommandError as e: - print ("Error: {msg}".format(msg=e.msg)) - print ("\tfull command: ", e.cmd) - print ("\treturncode : ", e.returncode) - print ("\tlog file : ", e.log) - except InspectError as e: - print ("Error: {msg}".format(msg=e.msg)) - if e.filename: - print ("\tfile: ", e.filename) - except VtrError as e: - print ("Error:", e.msg) - finally: - if args.print_metadata: - print ("\n# {} took {} (exiting {})".format(prog, format_elapsed_time(datetime.now() - start), num_failed)) - sys.exit(num_failed) - -def run_tasks(args, configs): - """ - Runs the specified set of tasks (configs) - """ - num_failed = 0 - - #We could potentially support other 'run' systems (e.g. a cluster), - #rather than just the local machine - if args.system == "local": - assert args.j > 0, "Invalid number of processors" - - #Generate the jobs, each corresponding to an invocation of vtr flow - jobs = create_jobs(args, configs) - - if args.run: - num_failed = run_parallel(args, configs, jobs) - - if args.parse: - print_verbose(BASIC_VERBOSITY, args.verbosity, "") - parse_tasks(args, configs, jobs) - - if args.create_golden: - print_verbose(BASIC_VERBOSITY, args.verbosity, "") - create_golden_results_for_tasks(args, configs) - - if args.check_golden: - print_verbose(BASIC_VERBOSITY, args.verbosity, "") - num_failed += check_golden_results_for_tasks(args, configs) - - else: - raise VtrError("Unrecognized run system {system}".format(system=args.system)) - - return num_failed - -def parse_tasks(args, configs, jobs): - """ - Parse the selection of tasks specified in configs and associated jobs - """ - for config in configs: - config_jobs = [job for job in jobs if job.task_name() == config.task_name] - parse_task(args, config, config_jobs) - -def parse_task(args, config, config_jobs, task_metrics_filepath=None, flow_metrics_basename="parse_results.txt"): - """ - Parse a single task run. - - This generates a file parse_results.txt in the task's working directory, - which is an amalgam of the parse_rests.txt's produced by each job (flow invocation) - """ - run_dir = find_latest_run_dir(args, config) - - print_verbose(BASIC_VERBOSITY, args.verbosity, "Parsing task run {}".format(run_dir)) - - for job in config_jobs: - #Re-run parsing only - cmd = job.command() - cmd += ['--parse'] - cmd += ['-v', str(max(0, args.verbosity-3))] - subprocess.check_call(cmd, cwd=job.work_dir(run_dir)) - - if task_metrics_filepath is None: - task_metrics_filepath = task_parse_results_filepath = str(PurePath(run_dir).joinpath("parse_results.txt")) - - #Record max widths for pretty printing - max_arch_len = len("architecture") - max_circuit_len = len("circuit") - for job in config_jobs: - max_arch_len = max(max_arch_len, len(job.arch())) - max_circuit_len = max(max_circuit_len, len(job.circuit())) - - with open(task_parse_results_filepath, "w") as out_f: - - #Start the header - print >>out_f, "{:<{arch_width}}\t{:<{circuit_width}}\t".format("architecture", "circuit", arch_width=max_arch_len, circuit_width=max_circuit_len), - header = True - - for job in config_jobs: - #Open the job results file - # - #The job results file is basically the same format, but excludes the architecture and circuit fields, - #which we prefix to each line of the task result file - job_parse_results_filepath = Path(run_dir) / job.arch / job.circuit / flow_metrics_basename - if job_parse_results_filepath.exists: - with job_parse_results_filepath.open() as in_f: - lines = in_f.readlines() - - assert len(lines) == 2 - - if header: - #First line is the header - print >>out_f, lines[0], - header = False - - #Second line is the data - print >>out_f, "{:<{arch_width}}\t{:<{circuit_width}}\t{}".format(job.arch(), job.circuit(), lines[1], arch_width=max_arch_len, circuit_width=max_circuit_len), - else: - print_verbose(BASIC_VERBOSITY, args.verbosity, "Warning: Flow result file not found (task QoR will be incomplete): {} ".format(str(job_parse_results_filepath))) - -def create_golden_results_for_tasks(args, configs): - for config in configs: - create_golden_results_for_task(args, config) - -def create_golden_results_for_task(args, config): - """ - Copies the latest task run's parse_results.txt into the config directory as golden_results.txt - """ - run_dir = find_latest_run_dir(args, config) - - task_results = str(PurePath(run_dir).joinpath("parse_results.txt")) - golden_results_filepath = str(PurePath(config.config_dir).joinpath("golden_results.txt")) - - print_verbose(BASIC_VERBOSITY, args.verbosity, "Creating golden task results from {} -> {}".format(run_dir, golden_results_filepath)) - - shutil.copy(task_results, golden_results_filepath) - -def check_golden_results_for_tasks(args, configs): - num_qor_failures = 0 - - print_verbose(BASIC_VERBOSITY, args.verbosity, "Checking QoR:") - for config in configs: - num_qor_failures += check_golden_results_for_task(args, config) - - return num_qor_failures - -def check_golden_results_for_task(args, config): - """ - Copies the latest task run's parse_results.txt into the config directory as golden_results.txt - """ - num_qor_failures = 0 - run_dir = find_latest_run_dir(args, config) - - if not config.pass_requirements_file: - print_verbose(BASIC_VERBOSITY, args.verbosity, - "Warning: no pass requirements file for task {}, QoR will not be checked".format(config.task_name)) - else: - - #Load the pass requirements file - pass_req_filepath = str(PurePath(find_vtr_root).joinpath('vtr_flow').joinpath('parse').joinpath('pass_requirements').joinpath(config.pass_requirements_file)) - pass_requirements = load_pass_requirements(pass_req_filepath) - - #Load the task's parse results - task_results_filepath = str(PurePath(run_dir).joinpath("parse_results.txt")) - task_results = load_parse_results(task_results_filepath) - - #Load the golden reference - golden_results_filepath = str(PurePath(config.config_dir).joinpath("golden_results.txt")) - golden_results = load_parse_results(golden_results_filepath) - - #Verify that the architecture and circuit are specified - for param in ["architecture", "circuit"]: - if param not in task_results.primary_keys(): - raise InspectError("Required param '{}' missing from task results: {}".format(param, task_results_filepath), task_results_filepath) - - if param not in golden_results.primary_keys(): - raise InspectError("Required param '{}' missing from golden results: {}".format(param, golden_results_filepath), golden_results_filepath) - - #Verify that all params and pass requirement metric are included in both the golden and task results - # We do not worry about non-pass_requriements elements being different or missing - for metric in pass_requirements.keys(): - for (arch, circuit), result in task_results.all_metrics().iteritems(): - if metric not in result: - #print_verbose(BASIC_VERBOSITY, args.verbosity, - #"Warning: Required metric '{}' missing from task results".format(metric), task_results_filepath) - raise InspectError("Required metric '{}' missing from task results".format(metric), task_results_filepath) - - for (arch, circuit), result in golden_results.all_metrics().iteritems(): - if metric not in result: - #print_verbose(BASIC_VERBOSITY, args.verbosity, - #"Warning: Required metric '{}' missing from golden results".format(metric), golden_results_filepath) - raise InspectError("Required metric '{}' missing from golden results".format(metric), golden_results_filepath) - - #Load the primary keys for golden and task - golden_primary_keys = [] - for (arch, circuit), metrics in golden_results.all_metrics().iteritems(): - golden_primary_keys.append((arch, circuit)) - - task_primary_keys = [] - for (arch, circuit), metrics in task_results.all_metrics().iteritems(): - task_primary_keys.append((arch, circuit)) - - #Ensure that task has all the golden cases - for arch, circuit in golden_primary_keys: - if task_results.metrics(arch, circuit) == None: - raise InspectError("Required case {}/{} missing from task results: {}".format(arch, circuit), parse_results_filepath) - - #Warn about any elements in task that are not found in golden - for arch, circuit in task_primary_keys: - if golden_results.metrics(arch, circuit) == None: - print_verbose(BASIC_VERBOSITY, args.verbosity, - "Warning: Task includes result for {}/{} missing in golden results".format(arch, circuit)) - - #Verify that task results pass each metric for all cases in golden - for (arch, circuit) in golden_primary_keys: - golden_metrics = golden_results.metrics(arch, circuit) - task_metrics = task_results.metrics(arch, circuit) - - for metric in pass_requirements.keys(): - - if not metric in golden_metrics: - print_verbose(BASIC_VERBOSITY, args.verbosity, "Warning: Metric {} missing from golden results".format(metric)) - continue - - if not metric in task_metrics: - print_verbose(BASIC_VERBOSITY, args.verbosity, "Warning: Metric {} missing from task results".format(metric)) - continue - - try: - metric_passed, reason = pass_requirements[metric].check_passed(golden_metrics[metric], task_metrics[metric]) - except InspectError as e: - metric_passed = False - reason = e.msg - - if not metric_passed: - print_verbose(BASIC_VERBOSITY, args.verbosity, " FAILED {} {} {}/{}: {} {}".format(PurePath(run_dir).name, config.task_name, arch, circuit, metric, reason)) - num_qor_failures += 1 - - if num_qor_failures == 0: - print_verbose(BASIC_VERBOSITY, args.verbosity, - " PASSED {} {}".format(PurePath(run_dir).name, config.task_name)) - - return num_qor_failures - -def create_jobs(args, configs): - jobs = [] - for config in configs: - for arch, circuit in itertools.product(config.archs, config.circuits): - abs_arch_filepath = resolve_vtr_source_file(config, arch, config.arch_dir) - abs_circuit_filepath = resolve_vtr_source_file(config, circuit, config.circuit_dir) - - executable = None - if config.script_path: - #Custom flow script - executable = [config.script_path] - else: - #Default flow script - executable = [find_vtr_file('run_vtr_flow.py', is_executable=True)] - - #Collect any extra script params from the config file - script_params = [abs_arch_filepath, abs_circuit_filepath] - script_params += config.script_params - - #Apply any special config based parameters - if config.cmos_tech_behavior: - script_params += ["--power", resolve_vtr_source_file(config, config.cmos_tech_behavior, "tech")] - - if config.pad_file: - script_params += ["--fix_pins", resolve_vtr_source_file(config, config.pad_file)] - - if config.parse_file: - script_params += ["--parse_config_file", resolve_vtr_source_file(config, config.parse_file, str(PurePath("parse").joinpath("parse_config")))] - - #We specify less verbosity to the sub-script - # This keeps the amount of output reasonable - script_params += ["-v", str(max(0, args.verbosity - 1))] - - cmd = executable + script_params - - work_dir = str(PurePath(arch).joinpath(circuit)) - - jobs.append(Job(config.task_name, arch, circuit, work_dir, cmd)) - - return jobs - -def find_latest_run_dir(args, config): - task_dir = find_task_dir(args, config) - - run_dir = get_latest_run_dir(task_dir) - - if not run_dir: - raise InspectError("Failed to find run directory for task '{}' in '{}'".format(config.task_name, task_dir)) - - assert Path(run_dir).is_dir() - - return run_dir - -def find_task_dir(args, config): - task_dir = None - if args.work_dir: - task_dir = str(PurePath(args.work_dir).joinpath(config.task_name)) - - else: - #Task dir is just above the config directory - task_dir = Path(config.config_dir).parent - assert task_dir.is_dir - - return str(task_dir) - -def run_parallel(args, configs, queued_jobs): - """ - Run each external command in commands with at most args.j commands running in parllel - """ - #Determine the run dir for each config - run_dirs = {} - for config in configs: - task_dir = find_task_dir(args, config) - task_run_dir = get_next_run_dir(task_dir) - run_dirs[config.task_name] = task_run_dir - - - #We pop off the jobs of queued_jobs, which python does from the end, - #so reverse the list now so we get the expected order. This also ensures - #we are working with a copy of the jobs - queued_jobs = list(reversed(queued_jobs)) - - #Find the max taskname length for pretty printing - max_taskname_len = 0 - for job in queued_jobs: - max_taskname_len = max(max_taskname_len, len(job.task_name())) - - #Queue of currently running subprocesses - running_procs = [] - - num_failed = 0 - try: - while len(queued_jobs) > 0 or len(running_procs) > 0: #Outstanding or running jobs - - #Launch outstanding jobs if workers are available - while len(queued_jobs) > 0 and len(running_procs) < args.j: - job = queued_jobs.pop() - - #Make the working directory - work_dir = job.work_dir(run_dirs[job.task_name()]) - mkdir_p(work_dir) - - log_filepath = str(PurePath(work_dir).joinpath("vtr_flow.log")) - - log_file = open(log_filepath, 'w+') - - #print "Starting {}: {}".format(job.task_name(), job.job_name()) - #print job.command() - proc = subprocess.Popen(job.command(), - cwd=work_dir, - stderr=subprocess.STDOUT, - stdout=log_file) - - running_procs.append((proc, job, log_file)) - - - while len(running_procs) > 0: - #Are any of the workers finished? - procs_to_del = set() - for i in xrange(len(running_procs)): - proc, job, log_file = running_procs[i] - - status = proc.poll() - if status is not None: - #Process has completed - if status == 0: - print_verbose(BASIC_VERBOSITY, args.verbosity, - "Finished {:<{w}}: {}".format(job.task_name(), job.job_name(), w=max_taskname_len)) - - if args.verbosity >= ALL_LOG_VERBOSITY: - print_log(log_file) - else: - #Failed - num_failed += 1 - print_verbose(BASIC_VERBOSITY, args.verbosity, - "Failed {:<{w}}: {}".format(job.task_name(), job.job_name(), w=max_taskname_len)) - - if args.verbosity >= FAILED_LOG_VERBOSITY: - print_log(log_file) - - log_file.close() - - #Record jobs to be removed from the run queue - procs_to_del.add(proc) - - if len(procs_to_del) > 0: - #Remove jobs from run queue - running_procs = [info for info in running_procs if info[0] not in procs_to_del] - - assert len(running_procs) < args.j - - #There are idle workers, allow new jobs to start - break; - - #Don't constantly poll - time.sleep(0.1) - - except KeyboardInterrupt as e: - print ("Recieved KeyboardInterrupt -- halting workers") - for proc, job, log_file in running_procs: - proc.terminate() - log_file.close() - - #Remove any procs finished after terminate - running_procs = [(proc, job, log_file) for proc, job, log_file in running_procs if proc.returncode is None] - - finally: - if len(running_procs) > 0: - print ("Killing {} worker processes".format(len(running_procs))) - for proc, job, log_file in running_procs: - proc.kill() - log_file.close() - - return num_failed - -def print_log(log_file, indent=" "): - #Save position - curr_pos = log_file.tell() - - log_file.seek(0) #Rewind to start - - #Print log - for line in log_file: - line = line.rstrip() - print (indent + line) - print ("") - - #Return to original position - log_file.seek(curr_pos) - - -def resolve_vtr_source_file(config, filename, base_dir=""): - """ - Resolves an filename with a base_dir - - Checks the following in order: - 1) filename as absolute path - 2) filename under config directory - 3) base_dir as absolute path (join filename with base_dir) - 4) filename and base_dir are relative paths (join under vtr_root) - """ - - #Absolute path - if PurePath(filename).is_absolute(): - return filename - - #Under config - config_path = Path(config.config_dir) - assert config_path.is_absolute() - joined_path = config_path / filename - if joined_path.exists(): - return str(joined_path) - - #Under base dir - base_path = Path(base_dir) - if base_path.is_absolute(): - #Absolute base - joined_path = base_path / filename - if joined_path.exists(): - return str(joined_path) - else: - #Relative base under the VTR flow directory - joined_path = Path(find_vtr_root()) / 'vtr_flow' / base_dir / filename - if joined_path.exists(): - return str(joined_path) - - #Not found - raise InspectError("Failed to resolve VTR source file {}".format(filename)) - -if __name__ == "__main__": - main() diff --git a/vtr_flow/scripts/vtr-test.py b/vtr_flow/scripts/vtr-test.py deleted file mode 100755 index d5aa5d1e0d6..00000000000 --- a/vtr_flow/scripts/vtr-test.py +++ /dev/null @@ -1,209 +0,0 @@ -#!/usr/bin/env python3 -from pathlib import Path -from itertools import chain -import sys -import argparse -import itertools -import textwrap -import subprocess -import time -from datetime import datetime -from multiprocessing import Pool - -sys.path.insert(0, str(Path(__file__).resolve().parent / 'python_libs')) - -import vtr -from vtr import error, load_list_file, find_vtr_file, mkdir_p, print_verbose, find_vtr_root, CommandRunner, format_elapsed_time, RawDefaultHelpFormatter, VERBOSITY_CHOICES, load_task_config, TaskConfig, find_task_config_file, CommandRunner - -BASIC_VERBOSITY=1 - -def vtr_command_argparser(prog=None): - description = textwrap.dedent( - """ - Runs one or more VTR regression tests. - """ - ) - epilog = textwrap.dedent( - """ - Examples - -------- - - Run the regression test 'vtr_strong': - - %(prog)s vtr_strong - - Run the regression tests 'vtr_basic' and 'vtr_strong': - - %(prog)s vtr_basic vtr_strong - - Run the regression tests 'vtr_basic' and 'vtr_strong' with 8 parallel workers: - - %(prog)s vtr_basic vtr_strong -j8 - """ - ) - - parser = argparse.ArgumentParser( - prog=prog, - description=description, - epilog=epilog, - formatter_class=RawDefaultHelpFormatter, - ) - - # - # Major arguments - # - parser.add_argument('reg_test', - nargs="+", - choices=["vtr_basic", "vtr_strong", "vtr_nightly", "vtr_weekly", "odin_micro", "odin_full"], - help="Regression tests to be run") - - parser.add_argument("--create_golden", - default=False, - action="store_true", - help="Create golden reference results for the associated tasks") - - parser.add_argument('-j', - default=1, - type=int, - metavar="NUM_PROC", - help="How many processors to use for execution.") - - parser.add_argument("-v", "--verbosity", - choices=list(chain(VERBOSITY_CHOICES, [5])), - default=2, - type=int, - help="Sets the verbosity of the script. Higher values produce more output.") - - parser.add_argument("--work_dir", - default=None, - help="Directory to store intermediate and result files." - "If None, set to the relevante directory under $VTR_ROOT/vtr_flow/tasks.") - - parser.add_argument("--debug", - default=False, - action="store_true", - help="Produce additional debug output") - - return parser - -def main(): - vtr_command_main(sys.argv[1:]) - -def vtr_command_main(arg_list, prog=None): - start = datetime.now() - - #Load the arguments - args = vtr_command_argparser(prog).parse_args(arg_list) - - print_verbose(BASIC_VERBOSITY, args.verbosity, "# {} {}\n".format(prog, ' '.join(arg_list))) - - num_func_failures = 0 - num_qor_failures = 0 - - try: - if args.create_golden: - #Create golden results - num_qor_failures = 0 - create_tasks_golden_result(args, vtr_task_list_files) - else: - #Run any ODIN tests - for reg_test in args.reg_test: - if reg_test.startswith("odin"): - num_func_failures += run_odin_test(args, reg_test) - - #Collect the task lists - vtr_task_list_files = [] - for reg_test in args.reg_test: - if reg_test.startswith("vtr"): - - base_testname = reg_test.split('_')[-1] - task_list_filepath = str(Path(find_vtr_root()) / 'vtr_flow' / 'tasks' / 'regression_tests' / ('vtr_reg_' + base_testname) / 'task_list.txt') - vtr_task_list_files.append(task_list_filepath) - - #Run the actual tasks, recording functionality failures - num_func_failures += run_tasks(args, vtr_task_list_files) - - #Check against golden results - num_qor_failures += check_tasks_qor(args, vtr_task_list_files) - - - #Final summary - print_verbose(BASIC_VERBOSITY, args.verbosity, "") - if num_func_failures == 0 and num_qor_failures == 0: - print_verbose(BASIC_VERBOSITY, args.verbosity, - "PASSED All Test(s)") - elif num_func_failures != 0 or num_qor_failures != 0: - print_verbose(BASIC_VERBOSITY, args.verbosity, - "FAILED {} functionality and {} QoR tests".format(num_func_failures, num_qor_failures)) - - sys.exit(num_func_failures + num_qor_failures) - finally: - print_verbose(BASIC_VERBOSITY, args.verbosity, "\n# {} took {} (exiting {})".format(prog, format_elapsed_time(datetime.now() - start), num_func_failures + num_qor_failures)) - -def run_odin_test(args, test_name): - odin_reg_script = None - if test_name == "odin_micro": - odin_reg_script = find_vtr_file("verify_microbenchmarks.sh", is_executabe=True) - else: - assert test_name == "odin_full" - odin_reg_script = find_vtr_file("verify_regression_tests.sh", is_executabe=True) - - assert odin_reg_script - - odin_root = str(Path(odin_reg_script).resolve().parent) - - result = subprocess.call(odin_reg_script, cwd=odin_root) - - assert result != None - if result != 0: - #Error - print_verbose(BASIC_VERBOSITY, args.verbosity, "FAILED test '{}'".format(test_name)) - return 1 - - #Pass - print_verbose(BASIC_VERBOSITY, args.verbosity, "PASSED test '{}'".format(test_name)) - return 0 - -def run_tasks(args, task_lists): - #Call 'vtr task' - vtr_task_cmd = ['vtr', 'task'] - vtr_task_cmd += ['-l'] + task_lists - vtr_task_cmd += ['-j', str(args.j), - '-v', str(max(0, args.verbosity - 1)), - '--print_metadata', str(args.debug) - ] - if args.work_dir: - vtr_task_cmd += ["--work_dir", args.workdir] - - #Exit code is number of failures - return subprocess.call(vtr_task_cmd) - -def check_tasks_qor(args, task_lists): - num_qor_failures = 0 - vtr_task_cmd = ['vtr', 'task'] - vtr_task_cmd += ['-l'] + task_lists - vtr_task_cmd += ['-v', str(max(0, args.verbosity - 1)), - '--check_golden', - '--print_metadata', str(args.debug) - ] - if args.work_dir: - vtr_task_cmd += ["--work_dir", args.workdir] - - #Exit code is number of failures - return subprocess.call(vtr_task_cmd) - -def create_tasks_golden_result(args, task_lists): - vtr_task_cmd = ['vtr', 'task'] - vtr_task_cmd += ['-l'] + task_lists - vtr_task_cmd += ['-v', str(max(0, args.verbosity - 1)), - '--create_golden', - '--print_metadata', str(args.debug) - ] - if args.work_dir: - vtr_task_cmd += ["--work_dir", args.workdir] - - #Exit code is number of failures - return subprocess.call(vtr_task_cmd) - -if __name__ == "__main__": - main() From be7f2ab1055d05771f976c9b6443bad0bea2bfec Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Mon, 13 Jul 2020 15:53:01 -0600 Subject: [PATCH 56/86] cleaning up random comments and fixing possible minor future bugs --- .../dev/tutorials/new_developer_tutorial.rst | 2 +- doc/src/vtr/run_vtr_flow.rst | 2 +- run_reg_test.pl | 2 +- vtr_flow/scripts/python_libs/doc/conf.py | 11 ----- vtr_flow/scripts/python_libs/vtr/abc/abc.py | 48 +++++++++++-------- .../scripts/python_libs/vtr/ace/__init__.py | 2 +- .../vtr/ace/{ace_flow.py => ace.py} | 9 ++-- vtr_flow/scripts/python_libs/vtr/flow.py | 31 +++++++----- vtr_flow/scripts/python_libs/vtr/odin/odin.py | 8 +++- vtr_flow/scripts/python_libs/vtr/util.py | 10 ++-- vtr_flow/scripts/python_libs/vtr/vpr/vpr.py | 7 +++ 11 files changed, 76 insertions(+), 56 deletions(-) rename vtr_flow/scripts/python_libs/vtr/ace/{ace_flow.py => ace.py} (84%) diff --git a/doc/src/dev/tutorials/new_developer_tutorial.rst b/doc/src/dev/tutorials/new_developer_tutorial.rst index 30fab6bd2dd..4aa694a3c70 100644 --- a/doc/src/dev/tutorials/new_developer_tutorial.rst +++ b/doc/src/dev/tutorials/new_developer_tutorial.rst @@ -57,7 +57,7 @@ Open the Black Box At this stage, you have gotten a taste of how an FPGA architect would go about using VTR. As a developer though, you need a much deeper understanding of how this tool works. The purpose of this section is to have you to learn the details of the VTR CAD flow by having you manually do what the scripts do. -Using the custom Verilog circuit and architecture created in the previous step, directly run Odin II on it to generate a blif netlist. You may need to skim the `ODIN_II/README.rst` and the `vtr_flow/scripts/run_vtr_flow.py`. +Using the custom Verilog circuit and architecture created in the previous step, directly run Odin II on it to generate a blif netlist. You may need to skim the ``ODIN_II/README.rst`` and the ``vtr_flow/scripts/run_vtr_flow.py``. Using the output netlist of Odin II, run ABC to generate a technology-mapped blif file. You may need to skim the ABC homepage (http://www.eecs.berkeley.edu/~alanmi/abc/). diff --git a/doc/src/vtr/run_vtr_flow.rst b/doc/src/vtr/run_vtr_flow.rst index bf9f504c4a2..6ce79a8e0ca 100644 --- a/doc/src/vtr/run_vtr_flow.rst +++ b/doc/src/vtr/run_vtr_flow.rst @@ -16,7 +16,7 @@ Basic Usage At a minimum ``run_vtr_flow.py`` requires two command-line arguments:: - run_vtr_flow.p + run_vtr_flow.py where: diff --git a/run_reg_test.pl b/run_reg_test.pl index afdc4731cb3..f04a22ee224 100755 --- a/run_reg_test.pl +++ b/run_reg_test.pl @@ -88,7 +88,7 @@ $long_task_names = 1; } elsif ( $token eq "-j" ) { #-j N $num_cpu = int(shift(@ARGV)); - } elsif ( $token eq "-script" ) { #-j N + } elsif ( $token eq "-script" ) { $script = shift(@ARGV); } elsif ( $token =~ /^-j(\d+)$/ ) { #-jN $num_cpu = int($1); diff --git a/vtr_flow/scripts/python_libs/doc/conf.py b/vtr_flow/scripts/python_libs/doc/conf.py index d84fa4eec16..10b5f0ace62 100755 --- a/vtr_flow/scripts/python_libs/doc/conf.py +++ b/vtr_flow/scripts/python_libs/doc/conf.py @@ -1,15 +1,4 @@ -# Configuration file for the Sphinx documentation builder. -# -# This file only contains a selection of the most common options. For a full -# list see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html - # -- Path setup -------------------------------------------------------------- - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# import os import sys import sphinx_rtd_theme diff --git a/vtr_flow/scripts/python_libs/vtr/abc/abc.py b/vtr_flow/scripts/python_libs/vtr/abc/abc.py index 0f01475a579..4c53aa40ef2 100644 --- a/vtr_flow/scripts/python_libs/vtr/abc/abc.py +++ b/vtr_flow/scripts/python_libs/vtr/abc/abc.py @@ -7,7 +7,7 @@ def run(architecture_file, circuit_file, output_netlist, command_runner=CommandRunner(), temp_dir=".", log_filename="abc.out", abc_exec=None, abc_script=None, abc_rc=None, - use_old_abc_script = False, abc_args = None, keep_intermediate_files=1): + use_old_abc_script = False, abc_args = None, keep_intermediate_files=True): """ Runs ABC to optimize specified file. @@ -43,9 +43,13 @@ def run(architecture_file, circuit_file, """ mkdir_p(temp_dir) + if abc_args == None: + abc_args = OrderedDict() + + #Verify that files are Paths or convert them to Paths and check that they exist verify_file(architecture_file, "Architecture") verify_file(circuit_file, "Circuit") - verify_file(output_netlist, "Output netlist", False) + verify_file(output_netlist, "Output netlist", should_exist=False) blackbox_latches_script = find_vtr_file("blackbox_latches.pl") clk_list = [] @@ -57,25 +61,31 @@ def run(architecture_file, circuit_file, abc_run_args = "" use_old_latches_restoration_script = 0 lut_size = None + + if("iterative_bb" in abc_args): + abc_flow_type=2 + del abc_args["iterative_bb"] + if("blanket_bb" in abc_args): + abc_flow_type=3 + del abc_args["blanket_bb"] + if("once_bb" in abc_args): + abc_flow_type=1 + del abc_args["once_bb"] + if("use_old_latches_restoration_script" in abc_args): + use_old_latches_restoration_script = 1 + del abc_args["use_old_latches_restoration_script"] + if("lut_size" in abc_args): + lut_size=abc_args["lut_size"] + del abc_args["lut_size"] + for key, value in abc_args.items(): - if(key=="iterative_bb"): - abc_flow_type=2 - elif(key=="blanket_bb"): - abc_flow_type=3 - elif(key=="once_bb"): - abc_flow_type=1 - elif(key=="use_old_latches_restoration_script"): - use_old_latches_restoration_script = 1 - elif(key=="lut_size"): - lut_size=value + if value == True: + abc_run_args += ["--" + arg] + elif value == False: + pass else: - if value == True: - abc_run_args += ["--" + arg] - elif value == False: - pass - else: - abc_run_args += ["--" + arg, str(value)] - + abc_run_args += ["--" + arg, str(value)] + if(lut_size == None): lut_size = determine_lut_size(str(architecture_file)) diff --git a/vtr_flow/scripts/python_libs/vtr/ace/__init__.py b/vtr_flow/scripts/python_libs/vtr/ace/__init__.py index 8553fb3d888..ffac7c866ed 100644 --- a/vtr_flow/scripts/python_libs/vtr/ace/__init__.py +++ b/vtr_flow/scripts/python_libs/vtr/ace/__init__.py @@ -1 +1 @@ -from .ace_flow import run \ No newline at end of file +from .ace import run \ No newline at end of file diff --git a/vtr_flow/scripts/python_libs/vtr/ace/ace_flow.py b/vtr_flow/scripts/python_libs/vtr/ace/ace.py similarity index 84% rename from vtr_flow/scripts/python_libs/vtr/ace/ace_flow.py rename to vtr_flow/scripts/python_libs/vtr/ace/ace.py index 2fda01df260..50b7bd0f359 100644 --- a/vtr_flow/scripts/python_libs/vtr/ace/ace_flow.py +++ b/vtr_flow/scripts/python_libs/vtr/ace/ace.py @@ -30,19 +30,20 @@ def run(circuit_file, old_netlist, output_netlist, output_activity_file, ace_seed : The ACE seed """ + #Verify that files are Paths or convert them to Paths and check that they exist verify_file(circuit_file, "Circuit") verify_file(old_netlist, "Previous netlist") - verify_file(output_netlist, "Output netlist", False) - verify_file(output_activity_file, "Output activity", False) + verify_file(output_netlist, "Output netlist", should_exist=False) + verify_file(output_activity_file, "Output activity", should_exist=False) ace_clk_file = Path(temp_dir) / "ace_clk.txt" ace_raw = Path(temp_dir) / (circuit_file.with_suffix('').stem + ".raw.ace.blif") if ace_exec is None: ace_exec = find_vtr_file('ace') - ace_extraction_file = Path(find_vtr_file("extract_clk_from_blif.py")) + ace_extraction_file = find_vtr_file("extract_clk_from_blif.py") - cmd =[str(ace_extraction_file), ace_clk_file.name, circuit_file.name] + cmd =[ace_extraction_file, ace_clk_file.name, circuit_file.name] command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename="ace_clk_extraction.out", indent_depth=1) ace_clk = "" with ace_clk_file.open('r') as f: diff --git a/vtr_flow/scripts/python_libs/vtr/flow.py b/vtr_flow/scripts/python_libs/vtr/flow.py index 0396f6bf26e..2ff95b10c93 100644 --- a/vtr_flow/scripts/python_libs/vtr/flow.py +++ b/vtr_flow/scripts/python_libs/vtr/flow.py @@ -8,7 +8,7 @@ from collections import OrderedDict from vtr import CommandError -VTR_STAGE = vtr.make_enum("odin", "abc", 'ace', "vpr", "lec") +VTR_STAGE = vtr.make_enum("odin", "abc", 'ace', "vpr") vtr_stages = VTR_STAGE.reverse_mapping.values() def run(architecture_file, circuit_file, @@ -78,23 +78,29 @@ def run(architecture_file, circuit_file, check_incremental_sta_consistency : Do a second-run of the incremental analysis to compare the result files """ + if odin_args == None: + odin_args = OrderedDict() + if abc_args == None: + abc_args = OrderedDict() if vpr_args == None: vpr_args = OrderedDict() - + # #Initial setup # + + #Verify that files are Paths or convert them to Paths and check that they exist vtr.util.verify_file(architecture_file, "Architecture") vtr.util.verify_file(circuit_file, "Circuit") if(power_tech_file): vtr.util.verify_file(power_tech_file, "Power tech") + architecture_file_basename =architecture_file.name circuit_file_basename = circuit_file.name circuit_name = circuit_file.stem circuit_ext = circuit_file.suffixes architecture_name = architecture_file.stem - architecture_ext = architecture_file.suffixes vtr.mkdir_p(temp_dir) netlist_ext = ".blif" @@ -112,7 +118,7 @@ def run(architecture_file, circuit_file, rr_graph_ext=".xml" if "blif" in circuit_ext: - #If the user provided a .blif netlist, we use that as the baseline for LEC + #If the user provided a .blif or .eblif netlist, we use that as the baseline for LEC #(ABC can't LEC behavioural verilog) lec_base_netlist = circuit_file_basename @@ -169,8 +175,8 @@ def run(architecture_file, circuit_file, # if power_tech_file: #The user provided a tech file, so do power analysis - if(not isinstance(power_tech_file,Path)): - power_tech_file=Path(power_tech_file) + + verify_file(power_tech_file, "power tech file") if should_run_stage(VTR_STAGE.ace, start_stage, end_stage): vtr.ace.run(next_stage_netlist, old_netlist = post_odin_netlist, output_netlist=post_ace_netlist, @@ -182,7 +188,7 @@ def run(architecture_file, circuit_file, next_stage_netlist.unlink() post_odin_netlist.unlink() - #Use ACE's output netlistf + #Use ACE's output netlist next_stage_netlist = post_ace_netlist if not lec_base_netlist: @@ -190,7 +196,6 @@ def run(architecture_file, circuit_file, #Enable power analysis in VPR vpr_args["power"] = True - #vpr_args["activity_file"] = post_ace_activity_file.name vpr_args["tech_properties"] = str(power_tech_file.resolve()) # @@ -235,6 +240,7 @@ def run(architecture_file, circuit_file, break vtr.abc.run_lec(lec_base_netlist, gen_postsynthesis_netlist, command_runner=command_runner, temp_dir=temp_dir) + # Do a second-run of the incremental analysis to compare the result files if check_incremental_sta_consistency: vtr.vpr.cmp_full_vs_incr_STA(architecture_copy, circuit_copy, pre_vpr_netlist, command_runner=command_runner, @@ -243,17 +249,16 @@ def run(architecture_file, circuit_file, temp_dir=temp_dir ) - if(not keep_intermediate_files): + if not keep_intermediate_files: next_stage_netlist.unlink() exts = ('.xml','.sdf','.v') if not keep_result_files: exts += ('.net', '.place', '.route') - files = [] + for file in Path(temp_dir).iterdir(): if file.suffix in exts: - files.append(file) - for p in files: - p.unlink() + file.unlink() + if power_tech_file: post_ace_activity_file.unlink() diff --git a/vtr_flow/scripts/python_libs/vtr/odin/odin.py b/vtr_flow/scripts/python_libs/vtr/odin/odin.py index 2e8a53d4574..88112f21c70 100644 --- a/vtr_flow/scripts/python_libs/vtr/odin/odin.py +++ b/vtr_flow/scripts/python_libs/vtr/odin/odin.py @@ -45,6 +45,10 @@ def run(architecture_file, circuit_file, """ mkdir_p(temp_dir) + if odin_args == None: + odin_args = OrderedDict() + + #Verify that files are Paths or convert them to Paths and check that they exist verify_file(architecture_file, "Architecture") verify_file(circuit_file, "Circuit") verify_file(output_netlist, "Output netlist", False) @@ -86,9 +90,9 @@ def run(architecture_file, circuit_file, pass else: cmd += ["--" + arg, str(value)] - cmd.extend(["-U0"]) + cmd += ["-U0"] if(disable_odin_xml): - cmd.extend(["-a",architecture_file.name, "-V", circuit_file.name, "-o",output_netlist.name]) + cmd += ["-a",architecture_file.name, "-V", circuit_file.name, "-o",output_netlist.name]) command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) diff --git a/vtr_flow/scripts/python_libs/vtr/util.py b/vtr_flow/scripts/python_libs/vtr/util.py index 7485e7b9a19..a31f8dffccb 100644 --- a/vtr_flow/scripts/python_libs/vtr/util.py +++ b/vtr_flow/scripts/python_libs/vtr/util.py @@ -92,7 +92,6 @@ def run_system_command(self, cmd, temp_dir, log_filename=None, expected_return_c #Echo the command? if self._echo_cmd: - #print ' '.join(cmd) print(cmd) #Begin timing @@ -425,11 +424,16 @@ def load_config_lines(filepath, allow_includes=True): return config_lines -def verify_file(file, file_type,check_is_file=True): +def verify_file(file, file_type,should_exist=True): + """ + Verifies that the file is a Pathlib object and if not makes it one. + Ensures that the file exists by default. + This makes it possible to pass files into the various files as strings or as pathlib objects. + """ if(not isinstance(file,Path)): file = Path(file) - if(check_is_file and not file.is_file()): + if(should_exist and not file.is_file()): raise Exception("{file_type} file does not exist: {file} ".format(file_type = file_type, file=file)) def format_elapsed_time(time_delta): diff --git a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py index 1179f7f3412..f6be82f520f 100644 --- a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py +++ b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py @@ -44,6 +44,7 @@ def run_relax_W(architecture, circuit_name, circuit, command_runner=CommandRunne mkdir_p(temp_dir) + #Verify that files are Paths or convert them to Paths and check that they exist verify_file(architecture, "Architecture") verify_file(circuit_name, "Circuit") verify_file(circuit, "Circuit") @@ -65,6 +66,9 @@ def run_relax_W(architecture, circuit_name, circuit, command_runner=CommandRunne if "route" in vpr_args: del vpr_args["route"] + if vpr_exec == None: + vpr_exec = find_vtr_file('vpr', is_executable=True) + run(architecture, circuit_name, circuit, command_runner, temp_dir, log_filename=vpr_min_W_log, vpr_exec=vpr_exec, vpr_args=vpr_args) if ('pack' in vpr_args or 'place' in vpr_args) and 'route' not in vpr_args: @@ -125,6 +129,7 @@ def run(architecture, circuit_name, circuit, command_runner=CommandRunner(), tem if vpr_exec == None: vpr_exec = find_vtr_file('vpr', is_executable=True) + #Verify that files are Paths or convert them to Paths and check that they exist verify_file(architecture, "Architecture") verify_file(circuit_name, "Circuit") verify_file(circuit, "Circuit") @@ -219,6 +224,8 @@ def cmp_full_vs_incr_STA(architecture,circuit_name,circuit,command_runner=Comman vpr_args: Extra arguments for VPR """ + + #Verify that files are Paths or convert them to Paths and check that they exist verify_file(architecture, "Architecture") verify_file(circuit_name, "Circuit") verify_file(circuit, "Circuit") From 0a20e74d269b61afbdca0e15e77a114a6e0ec992 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Tue, 14 Jul 2020 08:05:19 -0600 Subject: [PATCH 57/86] fixing small syntax error --- vtr_flow/scripts/python_libs/vtr/odin/odin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vtr_flow/scripts/python_libs/vtr/odin/odin.py b/vtr_flow/scripts/python_libs/vtr/odin/odin.py index 88112f21c70..65deed39caa 100644 --- a/vtr_flow/scripts/python_libs/vtr/odin/odin.py +++ b/vtr_flow/scripts/python_libs/vtr/odin/odin.py @@ -92,7 +92,7 @@ def run(architecture_file, circuit_file, cmd += ["--" + arg, str(value)] cmd += ["-U0"] if(disable_odin_xml): - cmd += ["-a",architecture_file.name, "-V", circuit_file.name, "-o",output_netlist.name]) + cmd += ["-a",architecture_file.name, "-V", circuit_file.name, "-o",output_netlist.name] command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) From 57c3c4cf0b9d0aaef2575a6014392842316b5b07 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Tue, 14 Jul 2020 09:40:30 -0600 Subject: [PATCH 58/86] fixing verify file issue --- vtr_flow/scripts/python_libs/vtr/abc/abc.py | 6 +++--- vtr_flow/scripts/python_libs/vtr/ace/ace.py | 8 ++++---- vtr_flow/scripts/python_libs/vtr/flow.py | 10 ++++------ vtr_flow/scripts/python_libs/vtr/odin/odin.py | 6 +++--- vtr_flow/scripts/python_libs/vtr/util.py | 3 ++- vtr_flow/scripts/python_libs/vtr/vpr/vpr.py | 18 +++++++++--------- 6 files changed, 25 insertions(+), 26 deletions(-) diff --git a/vtr_flow/scripts/python_libs/vtr/abc/abc.py b/vtr_flow/scripts/python_libs/vtr/abc/abc.py index 4c53aa40ef2..fe361a38ca9 100644 --- a/vtr_flow/scripts/python_libs/vtr/abc/abc.py +++ b/vtr_flow/scripts/python_libs/vtr/abc/abc.py @@ -47,9 +47,9 @@ def run(architecture_file, circuit_file, abc_args = OrderedDict() #Verify that files are Paths or convert them to Paths and check that they exist - verify_file(architecture_file, "Architecture") - verify_file(circuit_file, "Circuit") - verify_file(output_netlist, "Output netlist", should_exist=False) + architecture_file = verify_file(architecture_file, "Architecture") + circuit_file = verify_file(circuit_file, "Circuit") + output_netlist = verify_file(output_netlist, "Output netlist", should_exist=False) blackbox_latches_script = find_vtr_file("blackbox_latches.pl") clk_list = [] diff --git a/vtr_flow/scripts/python_libs/vtr/ace/ace.py b/vtr_flow/scripts/python_libs/vtr/ace/ace.py index 50b7bd0f359..0662be92c28 100644 --- a/vtr_flow/scripts/python_libs/vtr/ace/ace.py +++ b/vtr_flow/scripts/python_libs/vtr/ace/ace.py @@ -31,10 +31,10 @@ def run(circuit_file, old_netlist, output_netlist, output_activity_file, """ #Verify that files are Paths or convert them to Paths and check that they exist - verify_file(circuit_file, "Circuit") - verify_file(old_netlist, "Previous netlist") - verify_file(output_netlist, "Output netlist", should_exist=False) - verify_file(output_activity_file, "Output activity", should_exist=False) + circuit_file = verify_file(circuit_file, "Circuit") + old_netlist = verify_file(old_netlist, "Previous netlist") + output_netlist = verify_file(output_netlist, "Output netlist", should_exist=False) + output_activity_file = verify_file(output_activity_file, "Output activity", should_exist=False) ace_clk_file = Path(temp_dir) / "ace_clk.txt" ace_raw = Path(temp_dir) / (circuit_file.with_suffix('').stem + ".raw.ace.blif") diff --git a/vtr_flow/scripts/python_libs/vtr/flow.py b/vtr_flow/scripts/python_libs/vtr/flow.py index 2ff95b10c93..e1be15d1d6f 100644 --- a/vtr_flow/scripts/python_libs/vtr/flow.py +++ b/vtr_flow/scripts/python_libs/vtr/flow.py @@ -90,10 +90,10 @@ def run(architecture_file, circuit_file, # #Verify that files are Paths or convert them to Paths and check that they exist - vtr.util.verify_file(architecture_file, "Architecture") - vtr.util.verify_file(circuit_file, "Circuit") - if(power_tech_file): - vtr.util.verify_file(power_tech_file, "Power tech") + architecture_file = vtr.util.verify_file(architecture_file, "Architecture") + circuit_file = vtr.util.verify_file(circuit_file, "Circuit") + if power_tech_file != None: + power_tech_file = vtr.util.verify_file(power_tech_file, "Power tech") architecture_file_basename =architecture_file.name circuit_file_basename = circuit_file.name @@ -176,8 +176,6 @@ def run(architecture_file, circuit_file, if power_tech_file: #The user provided a tech file, so do power analysis - verify_file(power_tech_file, "power tech file") - if should_run_stage(VTR_STAGE.ace, start_stage, end_stage): vtr.ace.run(next_stage_netlist, old_netlist = post_odin_netlist, output_netlist=post_ace_netlist, output_activity_file=post_ace_activity_file, diff --git a/vtr_flow/scripts/python_libs/vtr/odin/odin.py b/vtr_flow/scripts/python_libs/vtr/odin/odin.py index 65deed39caa..79061396a30 100644 --- a/vtr_flow/scripts/python_libs/vtr/odin/odin.py +++ b/vtr_flow/scripts/python_libs/vtr/odin/odin.py @@ -49,9 +49,9 @@ def run(architecture_file, circuit_file, odin_args = OrderedDict() #Verify that files are Paths or convert them to Paths and check that they exist - verify_file(architecture_file, "Architecture") - verify_file(circuit_file, "Circuit") - verify_file(output_netlist, "Output netlist", False) + architecture_file = verify_file(architecture_file, "Architecture") + circuit_file = verify_file(circuit_file, "Circuit") + output_netlist = verify_file(output_netlist, "Output netlist", False) if odin_exec == None: odin_exec = find_vtr_file('odin_II', is_executable=True) diff --git a/vtr_flow/scripts/python_libs/vtr/util.py b/vtr_flow/scripts/python_libs/vtr/util.py index a31f8dffccb..1d60dab55d1 100644 --- a/vtr_flow/scripts/python_libs/vtr/util.py +++ b/vtr_flow/scripts/python_libs/vtr/util.py @@ -432,9 +432,10 @@ def verify_file(file, file_type,should_exist=True): """ if(not isinstance(file,Path)): file = Path(file) - if(should_exist and not file.is_file()): raise Exception("{file_type} file does not exist: {file} ".format(file_type = file_type, file=file)) + + return file def format_elapsed_time(time_delta): return "%.2f seconds" % time_delta.total_seconds() diff --git a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py index f6be82f520f..82850f41363 100644 --- a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py +++ b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py @@ -45,9 +45,9 @@ def run_relax_W(architecture, circuit_name, circuit, command_runner=CommandRunne mkdir_p(temp_dir) #Verify that files are Paths or convert them to Paths and check that they exist - verify_file(architecture, "Architecture") - verify_file(circuit_name, "Circuit") - verify_file(circuit, "Circuit") + architecture = verify_file(architecture, "Architecture") + circuit_name = verify_file(circuit_name, "Circuit") + circuit = verify_file(circuit, "Circuit") vpr_min_W_log = '.'.join([logfile_base, "out"]) vpr_relaxed_W_log = '.'.join([logfile_base, "crit_path", "out"]) @@ -130,9 +130,9 @@ def run(architecture, circuit_name, circuit, command_runner=CommandRunner(), tem vpr_exec = find_vtr_file('vpr', is_executable=True) #Verify that files are Paths or convert them to Paths and check that they exist - verify_file(architecture, "Architecture") - verify_file(circuit_name, "Circuit") - verify_file(circuit, "Circuit") + architecture = verify_file(architecture, "Architecture") + circuit_name = verify_file(circuit_name, "Circuit") + circuit = verify_file(circuit, "Circuit") cmd = [vpr_exec, architecture.name, circuit_name.stem, "--circuit_file", circuit.name] @@ -226,9 +226,9 @@ def cmp_full_vs_incr_STA(architecture,circuit_name,circuit,command_runner=Comman """ #Verify that files are Paths or convert them to Paths and check that they exist - verify_file(architecture, "Architecture") - verify_file(circuit_name, "Circuit") - verify_file(circuit, "Circuit") + architecture = verify_file(architecture, "Architecture") + circuit_name = verify_file(circuit_name, "Circuit") + circuit = verify_file(circuit, "Circuit") default_output_filenames = [ "{}.net".format(circuit_name.stem), From 1cdca8e41b54acdb04e665b49343283da823b872 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Tue, 14 Jul 2020 16:42:32 -0600 Subject: [PATCH 59/86] removed parse and task functionality to be implemented later --- vtr_flow/scripts/python_libs/vtr/__init__.py | 5 +- vtr_flow/scripts/python_libs/vtr/abc/abc.py | 8 +- vtr_flow/scripts/python_libs/vtr/ace/ace.py | 4 +- vtr_flow/scripts/python_libs/vtr/flow.py | 58 +---- vtr_flow/scripts/python_libs/vtr/inspect.py | 108 --------- vtr_flow/scripts/python_libs/vtr/odin/odin.py | 4 +- vtr_flow/scripts/python_libs/vtr/task.py | 155 ------------ vtr_flow/scripts/python_libs/vtr/vpr/vpr.py | 4 +- vtr_flow/scripts/run_vtr_flow.py | 222 ++++++++---------- 9 files changed, 114 insertions(+), 454 deletions(-) delete mode 100644 vtr_flow/scripts/python_libs/vtr/task.py diff --git a/vtr_flow/scripts/python_libs/vtr/__init__.py b/vtr_flow/scripts/python_libs/vtr/__init__.py index 4039844df1e..82e86a511be 100644 --- a/vtr_flow/scripts/python_libs/vtr/__init__.py +++ b/vtr_flow/scripts/python_libs/vtr/__init__.py @@ -1,9 +1,8 @@ from .util import load_config_lines, mkdir_p, find_vtr_file, CommandRunner, print_verbose, relax_W, file_replace, make_enum, RawDefaultHelpFormatter, VERBOSITY_CHOICES,format_elapsed_time, write_tab_delimitted_csv, load_list_file, find_vtr_root, argparse_str2bool, get_next_run_dir, get_latest_run_dir, verify_file -from .inspect import determine_lut_size, determine_min_W, determine_memory_addr_width, load_parse_patterns, load_pass_requirements, load_parse_results +from .inspect import determine_lut_size, determine_min_W, determine_memory_addr_width from .abc import run, run_lec from .vpr import run,run_relax_W,cmp_full_vs_incr_STA from .odin import run from .ace import run from .error import * -from .flow import run, VTR_STAGE, parse_vtr_flow -from .task import load_task_config, TaskConfig, find_task_config_file +from .flow import run, VTR_STAGE \ No newline at end of file diff --git a/vtr_flow/scripts/python_libs/vtr/abc/abc.py b/vtr_flow/scripts/python_libs/vtr/abc/abc.py index fe361a38ca9..37fa14b74d1 100644 --- a/vtr_flow/scripts/python_libs/vtr/abc/abc.py +++ b/vtr_flow/scripts/python_libs/vtr/abc/abc.py @@ -11,8 +11,8 @@ def run(architecture_file, circuit_file, """ Runs ABC to optimize specified file. - To run: - vtr.abc.run(args) + Usage: + vtr.abc.run(,,,[OPTIONS]) Required arguments: architecture_file : Architecture file to target @@ -30,9 +30,9 @@ def run(architecture_file, circuit_file, abc_exec : ABC executable to be run - abc_script : the script to be run on abc + abc_script : The script to be run on abc - abc_rc : the ABC rc file + abc_rc : The ABC rc file use_old_abc_script : Enables the use of the old ABC script diff --git a/vtr_flow/scripts/python_libs/vtr/ace/ace.py b/vtr_flow/scripts/python_libs/vtr/ace/ace.py index 0662be92c28..c8ddfac8ad8 100644 --- a/vtr_flow/scripts/python_libs/vtr/ace/ace.py +++ b/vtr_flow/scripts/python_libs/vtr/ace/ace.py @@ -6,8 +6,8 @@ def run(circuit_file, old_netlist, output_netlist, output_activity_file, """ Runs ACE for activity estimation - To run: - vtr.ace.run(args) + Usage: + vtr.ace.run(,,,[OPTIONS]) Required arguments: circuit_file : Circuit file to optimize diff --git a/vtr_flow/scripts/python_libs/vtr/flow.py b/vtr_flow/scripts/python_libs/vtr/flow.py index e1be15d1d6f..279351c35ae 100644 --- a/vtr_flow/scripts/python_libs/vtr/flow.py +++ b/vtr_flow/scripts/python_libs/vtr/flow.py @@ -15,7 +15,6 @@ def run(architecture_file, circuit_file, power_tech_file=None, start_stage=VTR_STAGE.odin, end_stage=VTR_STAGE.vpr, command_runner=vtr.CommandRunner(), - parse_config_file=None, temp_dir="./temp", verbosity=0, odin_args=None, @@ -32,8 +31,8 @@ def run(architecture_file, circuit_file, """ Runs the VTR CAD flow to map the specified circuit_file onto the target architecture_file - To run: - vtr.run(args) + Usage: + vtr.run(,,[OPTIONS]) Required arguments: architecture_file : Architecture file to target @@ -51,8 +50,6 @@ def run(architecture_file, circuit_file, command_runner : A CommandRunner object used to run system commands - parse_config_file : The configuration file defining how to parse metrics from results - verbosity : How much output to produce odin_args : A dictionary of keyword arguments to pass on to ODIN II @@ -260,57 +257,6 @@ def run(architecture_file, circuit_file, if power_tech_file: post_ace_activity_file.unlink() -def parse_vtr_flow(temp_dir, parse_config_file=None, metrics_filepath=None, verbosity=1): - vtr.mkdir_p(temp_dir) - if parse_config_file is None: - parse_config_file = vtr.find_vtr_file("vtr_benchmarks.txt") - - parse_patterns = vtr.load_parse_patterns(parse_config_file) - - metrics = OrderedDict() - - #Set defaults - for parse_pattern in parse_patterns.values(): - - if parse_pattern.default_value() != None: - metrics[parse_pattern.name()] = parse_pattern.default_value() - else: - metrics[parse_pattern.name()] = "" - - #Process each pattern - for parse_pattern in parse_patterns.values(): - - #We interpret the parse pattern's filename as a glob pattern - filepattern = str(Path(temp_dir) / parse_pattern.filename()) - filepaths = glob.glob(filepattern) - - num_files = len(filepaths) - - if num_files > 1: - raise vtr.InspectError("File pattern '{}' is ambiguous ({} files matched)".format(parse_pattern.filename()), num_files, filepaths) - - elif num_files == 1: - filepath = filepaths[0] - - assert Path(filepath).exists - - with open(filepath) as f: - for line in f: - match = parse_pattern.regex().match(line) - if match: - #Extract the first group value - metrics[parse_pattern.name()] = match.groups()[0] - else: - #No matching file, skip - assert num_files == 0 - - if metrics_filepath is None: - metrics_filepath = str(Path(temp_dir) / "parse_results.txt") - - vtr.write_tab_delimitted_csv(metrics_filepath, [metrics]) - - return metrics - def should_run_stage(stage, flow_start_stage, flow_end_stage): """ Returns True if stage falls between flow_start_stage and flow_end_stage diff --git a/vtr_flow/scripts/python_libs/vtr/inspect.py b/vtr_flow/scripts/python_libs/vtr/inspect.py index 66a78401d61..8561f7cf378 100644 --- a/vtr_flow/scripts/python_libs/vtr/inspect.py +++ b/vtr_flow/scripts/python_libs/vtr/inspect.py @@ -132,114 +132,6 @@ def metrics(self, arch, circuit): def all_metrics(self): return self._metrics - -def load_parse_patterns(parse_config_filepath): - parse_patterns = OrderedDict() - - for line in load_config_lines(parse_config_filepath): - - components = line.split(';') - - - if len(components) == 3 or len(components) == 4: - - name = components[0] - filepath = components[1] - regex_str = components[2] - - default_value = None - if len(components) == 4: - default_value = components[3] - - if name not in parse_patterns: - parse_patterns[name] = ParsePattern(name, filepath, regex_str, default_value) - else: - raise InspectError("Duplicate parse pattern name '{}'".format(name), parse_config_filepath) - - else: - raise InspectError("Invalid parse format line: '{}'".format(line), parse_config_filepath) - - return parse_patterns - -def load_pass_requirements(pass_requirements_filepath): - parse_patterns = OrderedDict() - - for line in load_config_lines(pass_requirements_filepath): - - components = line.split(';') - - if len(components) == 2: - - metric = components[0] - expr = components[1] - - if metric not in parse_patterns: - func, params_str = expr.split("(") - params_str = params_str.rstrip(")") - if params_str == "": - params = [] - else: - params = params_str.split(",") - - if func == "Range": - if len(params) != 2: - raise InspectError("Range() pass requirement function requires two arguments", pass_requirements_filepath) - - parse_patterns[metric] = RangePassRequirement(metric, float(params[0]), float(params[1])) - elif func == "Equal": - if len(params) != 0: - raise InspectError("Equal() pass requirement function requires no arguments", pass_requirements_filepath) - parse_patterns[metric] = EqualPassRequirement(metric) - else: - raise InspectError("Unexpected pass requirement function '{}' for metric '{}'".format(func, metric), pass_requirements_filepath) - - else: - raise InspectError("Duplicate pass requirement for '{}'".format(metric), pass_requirements_filepath) - - else: - raise InspectError("Invalid pass requirement format line: '{}'".format(line), pass_requirements_filepath) - - return parse_patterns - -def load_parse_results(parse_results_filepath, primary_key_set=None): - header = [] - - parse_results = ParseResults() - - with open(parse_results_filepath) as f: - for lineno, row in enumerate(f): - elements = [elem.strip() for elem in row.split("\t")] - - if lineno == 0: - header = elements - else: - primary_keys = OrderedDict() - result = OrderedDict() - - arch=None - circuit=None - for i, elem in enumerate(elements): - metric = header[i] - - if elem == "": - elem = None - - if metric == "arch": - metric = "architecture" - - if metric == "architecture": - arch = elem - elif metric == "circuit": - circuit = elem - - result[metric] = elem - - assert arch and circuit - - parse_results.add_result(arch, circuit, result) - - return parse_results - def determine_lut_size(architecture_file): """ Determines the maximum LUT size (K) in an architecture file. diff --git a/vtr_flow/scripts/python_libs/vtr/odin/odin.py b/vtr_flow/scripts/python_libs/vtr/odin/odin.py index 79061396a30..56ac9b74b5b 100644 --- a/vtr_flow/scripts/python_libs/vtr/odin/odin.py +++ b/vtr_flow/scripts/python_libs/vtr/odin/odin.py @@ -15,8 +15,8 @@ def run(architecture_file, circuit_file, """ Runs ODIN II on the specified architecture file and circuit file - To run: - vtr.odin.run(args) + Usage: + vtr.odin.run(,,,[OPTIONS]) Required arguments: architecture_file : Architecture file to target diff --git a/vtr_flow/scripts/python_libs/vtr/task.py b/vtr_flow/scripts/python_libs/vtr/task.py deleted file mode 100644 index 901a6adf29d..00000000000 --- a/vtr_flow/scripts/python_libs/vtr/task.py +++ /dev/null @@ -1,155 +0,0 @@ -from pathlib import Path -from pathlib import PurePath - -from vtr import VtrError, find_vtr_root, load_list_file - -class TaskConfig: - """ - An object representing a task config file - """ - def __init__(self, - task_name, - config_dir, - circuits_dir, - archs_dir, - circuit_list_add, - arch_list_add, - parse_file, - script_path=None, - script_params=None, - pass_requirements_file=None, - sdc_dir=None, - qor_parse_file=None, - cmos_tech_behavior=None, - pad_file=None): - self.task_name = task_name - self.config_dir = config_dir - self.circuit_dir = circuits_dir - self.arch_dir = archs_dir - self.circuits = circuit_list_add - self.archs = arch_list_add - self.parse_file = parse_file - self.script_path = script_path - self.script_params = script_params - self.pass_requirements_file = pass_requirements_file - self.sdc_dir = sdc_dir - self.qor_parse_file = qor_parse_file - self.cmos_tech_behavior = cmos_tech_behavior - self.pad_file = pad_file - -def load_task_config(config_file): - #Load the file stripping comments - values = load_list_file(config_file) - - #Keys that can appear only once - unique_keys = set(["circuits_dir", - "archs_dir", - "parse_file", - "script_path", - "script_params", - "pass_requirements_file", - "sdc_dir", - "qor_parse_file", - "cmos_tech_behavior", - "pad_file"]) - - #Keys that are repeated to create lists - repeated_keys = set(["circuit_list_add", - "arch_list_add"]) - - #Keys that are required - required_keys = set(["circuits_dir", - "archs_dir", - "circuit_list_add", - "arch_list_add", - "parse_file"]) - - #Interpret the file - key_values = {} - for line in values: - key, value = line.split('=') - - #Trim whitespace - key = key.strip() - value = value.strip() - - if key in unique_keys: - if key not in key_values: - key_values[key] = value - else: - raise VtrError("Duplicate {key} in config file {file}".format(key=key, file=config_file)) - - elif key in repeated_keys: - if key not in key_values: - key_values[key] = [] - key_values[key].append(value) - - else: - #All valid keys should have been collected by now - raise VtrError("Unrecognzied key '{key}' in config file {file}".format(key=key, file=config_file)) - - #We split the script params into a list - if 'script_params' in key_values: - key_values['script_params'] = key_values['script_params'].split(' ') - - #Check that all required fields were specified - for required_key in required_keys: - if required_key not in key_values: - raise VtrError("Missing required key '{key}' in config file {file}".format(key=required_key, file=config_file)) - - #Useful meta-data about the config - config_dir = str(Path(config_file).parent) - key_values['config_dir'] = config_dir - key_values['task_name'] = Path(config_dir).parent.name - - #Create the task config object - return TaskConfig(**key_values) - - -def find_task_config_file(task_name): - # - # See if we can find the config.txt assuming the task name is an - # absolute/relative path - # - - base_dirs = [] - if PurePath(task_name).is_absolute: - #Only check the root path since the path is aboslute - base_dirs.append('/') - else: - #Not absolute path, so check from the current directory first - base_dirs.append('.') - - vtr_root = find_vtr_root() - vtr_flow_tasks_dir = str(PurePath(vtr_root) / "vtr_flow" / "tasks") - - #Then the VTR tasks directory - base_dirs.append(vtr_flow_tasks_dir) - - #Generate potential config files (from most to least specific) - potential_config_file_paths = [] - for base_dir in base_dirs: - #Assume points directly to a config.txt - assume_config_path = str(PurePath(base_dir) / task_name) - potential_config_file_paths.append(assume_config_path) - - #Assume points to a config dir (containing config.txt) - assume_config_dir_path = str(PurePath(base_dir) / task_name / "config.txt") - potential_config_file_paths.append(assume_config_dir_path) - - #Assume points to a task dir (containing config/config.txt) - assume_task_dir_path = str(PurePath(base_dir) / task_name / "config" / "config.txt") - potential_config_file_paths.append(assume_task_dir_path) - - #Find the first potential file that is valid - for config_file in potential_config_file_paths: - config_path = Path(config_file) - is_file = config_path.is_file() - is_named_config = config_path.name == "config.txt" - is_in_config_dir = config_path.parent.name == "config" - - if is_file and is_named_config and is_in_config_dir: - return config_path.resolve() - - raise VtrError("Could not find config/config.txt for task {name}".format(name=task_name)) - diff --git a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py index 82850f41363..1c1a4ded319 100644 --- a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py +++ b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py @@ -13,8 +13,8 @@ def run_relax_W(architecture, circuit_name, circuit, command_runner=CommandRunne 2nd: At relaxed channel width (e.g. for critical path delay) - To run: - vtr.vpr.run_relax_W(args) + Usage: + vtr.vpr.run_relax_W(,,,[OPTIONS]) Required arguments: architecture: Architecture file diff --git a/vtr_flow/scripts/run_vtr_flow.py b/vtr_flow/scripts/run_vtr_flow.py index b9138754a7d..2e2ad6efe1a 100755 --- a/vtr_flow/scripts/run_vtr_flow.py +++ b/vtr_flow/scripts/run_vtr_flow.py @@ -133,16 +133,6 @@ def vtr_command_argparser(prog=None): dest="verbose", help="Verbosity of the script.") - parser.add_argument("--parse_config_file", - default=None, - help="Parse file to run after flow completion") - - parser.add_argument("--parse", - default=False, - action="store_true", - dest="parse_only", - help="Perform only parsing (assumes a previous run exists in work_dir)") - # # Power arguments @@ -363,122 +353,110 @@ def vtr_command_main(arg_list, prog=None): if(args.use_odin_simulation): odin_args["use_odin_simulation"] = True - - try: - if not args.parse_only: - try: - vpr_args = process_unknown_args(unknown_args) - if(args.crit_path_router_iterations): - vpr_args["max_router_iterations"] = args.crit_path_router_iterations - if(args.fix_pins): - new_file = str(Path(temp_dir) / Path(args.fix_pins).name) - shutil.copyfile(str((Path(prog).parent.parent / args.fix_pins)), new_file) - vpr_args["fix_pins"] = new_file - if args.verify_rr_graph: - rr_graph_out_file = "rr_graph" + args.rr_graph_ext - vpr_args["write_rr_graph"] = rr_graph_out_file - if args.check_place: - vpr_args["route"] = True - if args.check_route: - vpr_args["analysis"] = True - if args.sdc_file: - if not Path(args.sdc_file).exists(): - sdc_file = None - if args.sdc_file.startswith("/"): - sdc_file = Path(str(Path(prog).parent.parent) + args.sdc_file) - else: - sdc_file = Path(prog).parent.parent / args.sdc_file - - if sdc_file.exists(): - args.sdc_file = str(sdc_file) - else: - raise vtr.InspectError("Sdc file {arg},{sdc} was not found.".format(arg=args.sdc_file,sdc=str(sdc_file))) - vpr_args["sdc_file"] = args.sdc_file - - name = "" - if(args.name): - name=args.name + vpr_args = process_unknown_args(unknown_args) + if(args.crit_path_router_iterations): + vpr_args["max_router_iterations"] = args.crit_path_router_iterations + if(args.fix_pins): + new_file = str(Path(temp_dir) / Path(args.fix_pins).name) + shutil.copyfile(str((Path(prog).parent.parent / args.fix_pins)), new_file) + vpr_args["fix_pins"] = new_file + if args.verify_rr_graph: + rr_graph_out_file = "rr_graph" + args.rr_graph_ext + vpr_args["write_rr_graph"] = rr_graph_out_file + if args.check_place: + vpr_args["route"] = True + if args.check_route: + vpr_args["analysis"] = True + if args.sdc_file: + if not Path(args.sdc_file).exists(): + sdc_file = None + if args.sdc_file.startswith("/"): + sdc_file = Path(str(Path(prog).parent.parent) + args.sdc_file) else: - name=path_arch_file.name+"/"+path_circuit_file.name - print(name, end ="\t\t") - #Run the flow - vtr.run(path_arch_file, - path_circuit_file, - power_tech_file=args.power_tech, - temp_dir=temp_dir, - start_stage=args.start, - end_stage=args.end, - command_runner=command_runner, - verbosity=args.verbose, - vpr_args=vpr_args, - abc_args=abc_args, - odin_args=odin_args, - keep_intermediate_files=args.keep_intermediate_files, - keep_result_files=args.keep_result_files, - min_hard_mult_size=args.min_hard_mult_size, - min_hard_adder_size = args.min_hard_adder_size, - check_equivalent = args.check_equivalent, - check_incremental_sta_consistency = args.check_incremental_sta_consistency, - use_old_abc_script=args.use_old_abc_script, - relax_W_factor=args.relax_W_factor - ) - error_status = "OK" - except vtr.CommandError as e: - #An external command failed - return_status = 1 - if args.expect_fail: - expect_string = args.expect_fail - actual_error = None - if "exited with return code" in expect_string: - actual_error = "exited with return code {}".format(e.returncode) - else: - actual_error = e.msg - if expect_string != actual_error: - error_status = "failed: expected to fail with '{expected}' but was '{actual}'".format(expected=expect_string,actual = actual_error) - exit_status = 1 - else: - error_status = "OK" - return_status = 0 - if args.verbose: - error_status += " (as expected {})".format(expect_string) - else: - error_status += "*" + sdc_file = Path(prog).parent.parent / args.sdc_file + + if sdc_file.exists(): + args.sdc_file = str(sdc_file) else: - error_status = "failed: {}".format(e.msg) - if not args.expect_fail or exit_status: - print ("Error: {msg}".format(msg=e.msg)) - print ("\tfull command: ", ' '.join(e.cmd)) - print ("\treturncode : ", e.returncode) - print ("\tlog file : ", e.log) + raise vtr.InspectError("Sdc file {arg},{sdc} was not found.".format(arg=args.sdc_file,sdc=str(sdc_file))) + vpr_args["sdc_file"] = args.sdc_file + + name = "" + if(args.name): + name=args.name + else: + name=path_arch_file.name+"/"+path_circuit_file.name + print(name, end ="\t\t") + #Run the flow + vtr.run(path_arch_file, + path_circuit_file, + power_tech_file=args.power_tech, + temp_dir=temp_dir, + start_stage=args.start, + end_stage=args.end, + command_runner=command_runner, + verbosity=args.verbose, + vpr_args=vpr_args, + abc_args=abc_args, + odin_args=odin_args, + keep_intermediate_files=args.keep_intermediate_files, + keep_result_files=args.keep_result_files, + min_hard_mult_size=args.min_hard_mult_size, + min_hard_adder_size = args.min_hard_adder_size, + check_equivalent = args.check_equivalent, + check_incremental_sta_consistency = args.check_incremental_sta_consistency, + use_old_abc_script=args.use_old_abc_script, + relax_W_factor=args.relax_W_factor + ) + error_status = "OK" + except vtr.CommandError as e: + #An external command failed + return_status = 1 + if args.expect_fail: + expect_string = args.expect_fail + actual_error = None + if "exited with return code" in expect_string: + actual_error = "exited with return code {}".format(e.returncode) + else: + actual_error = e.msg + if expect_string != actual_error: + error_status = "failed: expected to fail with '{expected}' but was '{actual}'".format(expected=expect_string,actual = actual_error) exit_status = 1 - except vtr.InspectError as e: - #Something went wrong gathering information - print ("Error: {msg}".format(msg=e.msg)) - print ("\tfile : ", e.filename) - exit_status = 2 - return_status = exit_status - error_status = "failed: {}".format(e.msg) - - except vtr.VtrError as e: - #Generic VTR errors - print ("Error: ", e.msg) - exit_status = 3 - return_status = exit_status - error_status = "failed: {}".format(e.msg) - - except KeyboardInterrupt as e: - print ("{} recieved keyboard interrupt".format(prog)) - exit_status = 4 - return_status = exit_status - - #Parse the flow results - try: - vtr.parse_vtr_flow(temp_dir, args.parse_config_file, verbosity=args.verbose) - except vtr.InspectError as e: + else: + error_status = "OK" + return_status = 0 + if args.verbose: + error_status += " (as expected {})".format(expect_string) + else: + error_status += "*" + else: + error_status = "failed: {}".format(e.msg) + if not args.expect_fail or exit_status: print ("Error: {msg}".format(msg=e.msg)) - print ("\tfile : ", e.filename) - exit_status = 2 + print ("\tfull command: ", ' '.join(e.cmd)) + print ("\treturncode : ", e.returncode) + print ("\tlog file : ", e.log) + exit_status = 1 + except vtr.InspectError as e: + #Something went wrong gathering information + print ("Error: {msg}".format(msg=e.msg)) + print ("\tfile : ", e.filename) + exit_status = 2 + return_status = exit_status + error_status = "failed: {}".format(e.msg) + + except vtr.VtrError as e: + #Generic VTR errors + print ("Error: ", e.msg) + exit_status = 3 + return_status = exit_status + error_status = "failed: {}".format(e.msg) + + except KeyboardInterrupt as e: + print ("{} recieved keyboard interrupt".format(prog)) + exit_status = 4 + return_status = exit_status finally: seconds = datetime.now() - start @@ -494,7 +472,7 @@ def vtr_command_main(arg_list, prog=None): f.write("exited with return code {}\n".format(exit_status)) f.write("vpr_seconds=%d\nrundir=%s\nhostname=%s\nerror=" % (seconds.total_seconds(), str(Path.cwd()), socket.gethostname())) f.write("\n") - + sys.exit(return_status) def process_unknown_args(unknown_args): From cf1da8ae8165383ee1412db2e4d04589adb118b9 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Wed, 15 Jul 2020 13:15:34 -0600 Subject: [PATCH 60/86] removed dulicate tests and unnecessary test --- .travis.yml | 48 +---------- .../scripts/unittests/check_py_vtr_flow.py | 83 ------------------- 2 files changed, 4 insertions(+), 127 deletions(-) delete mode 100755 vtr_flow/scripts/unittests/check_py_vtr_flow.py diff --git a/.travis.yml b/.travis.yml index b5d27744c74..368ad0feea4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -89,46 +89,6 @@ jobs: #script: #- ./.github/travis/build.sh - - stage: Test - name: "PY scripts" - script: - - ./.github/travis/build.sh - - ./vtr_flow/scripts/unittests/check_py_vtr_flow.py - - stage: Test - name: "Basic Regression Tests Python" - env: - - CMAKE_PARAMS="-DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on" - - MATRIX_EVAL="CC=gcc-5 && CXX=g++-5" - script: - - ./.github/travis/build.sh - - travis_wait 30 ./run_reg_test.pl vtr_reg_basic -show_failures -j2 - - stage: Test - name: "Strong Regression Tests Python" - env: - - CMAKE_PARAMS="-DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on" - - MATRIX_EVAL="CC=gcc-5 && CXX=g++-5" - script: - - ./.github/travis/build.sh - - travis_wait 30 ./run_reg_test.pl vtr_reg_strong -show_failures -j2 - - stage: Test - name: "Basic Valgrind Memory Tests Python" - env: - - CMAKE_PARAMS="-DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on" - - MATRIX_EVAL="CC=gcc-5 && CXX=g++-5" - script: - - ./.github/travis/build.sh - - travis_wait 30 ./run_reg_test.pl vtr_reg_valgrind_small -show_failures -j2 - - stage: Test - name: "Sanitized Basic Regression Tests Python" - env: - - CMAKE_PARAMS="-DVTR_ASSERT_LEVEL=3 -DVTR_ENABLE_SANITIZE=on -DVTR_IPO_BUILD=off -DWITH_BLIFEXPLORER=on" - - MATRIX_EVAL="CC=gcc-5 && CXX=g++-5" - - BUILD_TYPE=debug - - LSAN_OPTIONS="exitcode=42" #Use a non-standard exit code to ensure LSAN errors are detected - script: - - ./.github/travis/build.sh - #We skip QoR since we are only checking for errors in sanitizer runs - - travis_wait 30 ./run_reg_test.pl vtr_reg_basic -show_failures -skip_qor -j2 - stage: Test name: "Code Formatting" script: @@ -157,7 +117,7 @@ jobs: - MATRIX_EVAL="CC=gcc-5 && CXX=g++-5" script: - ./.github/travis/build.sh - - travis_wait 30 ./run_reg_test.pl vtr_reg_basic -show_failures -j2 -script run_vtr_flow.pl + - travis_wait 50 ./run_reg_test.pl vtr_reg_basic -show_failures -j2 - stage: Test name: "Strong Regression Tests" env: @@ -165,7 +125,7 @@ jobs: - MATRIX_EVAL="CC=gcc-5 && CXX=g++-5" script: - ./.github/travis/build.sh - - travis_wait 30 ./run_reg_test.pl vtr_reg_strong -show_failures -j2 -script run_vtr_flow.pl + - travis_wait 30 ./run_reg_test.pl vtr_reg_strong -show_failures -j2 - stage: Test name: "Basic Valgrind Memory Tests" env: @@ -173,7 +133,7 @@ jobs: - MATRIX_EVAL="CC=gcc-5 && CXX=g++-5" script: - ./.github/travis/build.sh - - travis_wait 30 ./run_reg_test.pl vtr_reg_valgrind_small -show_failures -j2 -script run_vtr_flow.pl + - travis_wait 30 ./run_reg_test.pl vtr_reg_valgrind_small -show_failures -j2 - stage: Test name: "Sanitized Basic Regression Tests" env: @@ -184,7 +144,7 @@ jobs: script: - ./.github/travis/build.sh #We skip QoR since we are only checking for errors in sanitizer runs - - travis_wait 30 ./run_reg_test.pl vtr_reg_basic -show_failures -skip_qor -j2 -script run_vtr_flow.pl + - travis_wait 30 ./run_reg_test.pl vtr_reg_basic -show_failures -skip_qor -j2 #Currently strong regression with sanitizers is disabled as it exceeds the maximum travis job run-time #- stage: Test #name: "Sanitized Strong Regression Tests" diff --git a/vtr_flow/scripts/unittests/check_py_vtr_flow.py b/vtr_flow/scripts/unittests/check_py_vtr_flow.py deleted file mode 100755 index 9ccb9b1f3dc..00000000000 --- a/vtr_flow/scripts/unittests/check_py_vtr_flow.py +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env python3.6 - -import subprocess -import pathlib -import sys -import shutil -import filecmp - -# VTR paths -scripts_path = pathlib.Path(__file__).resolve().parent.parent -vtr_flow_path = scripts_path.parent -temp_dir = pathlib.Path.cwd() / "temp" - -# run_vtr_flow.pl will be compared against run_vtr_flow.py using the following test cases as arguments: -arg_list = ["benchmarks/verilog/ch_intrinsics.v arch/timing/k6_N10_mem32K_40nm.xml -track_memory_usage --router_heap binary --min_route_chan_width_hint 38", - "benchmarks/verilog/diffeq1.v arch/timing/k6_N10_mem32K_40nm.xml -track_memory_usage --router_heap bucket --min_route_chan_width_hint 46" - ] - -# The following output files will be compared to ensure they are identical (only one file must match each entry) -# The first list item is the glob pattern, the second is the number of lines to ignore at the beginning of the file -files_to_validate = [ ("*.route", 1), - ("report_timing.setup.rpt",0) - ] - -def error(*msg, return_code=-1): - print("!!!! ERROR:", " ".join(str(item) for item in msg)) - sys.exit(return_code) - -# Find a single file matching a pattern at a given path -def find_file_matching(path, pattern): - matches = [f for f in path.glob(pattern)] - if len(matches) == 0: - error("No files match", path / pattern) - elif len(matches) > 1: - error("Multiple files match", path / pattern) - return matches[0] - -# Run -def run_vtr_flow(script_name, run_name, temp_dir_run, args): - cmd = [scripts_path / script_name, *(args.split()), "-temp_dir", temp_dir_run] - print(" Running", run_name,"flow:", cmd) - p = subprocess.run(cmd, cwd = vtr_flow_path, stdout = subprocess.PIPE, stderr=subprocess.STDOUT) - if (p.returncode): - print(p.stdout.decode()) - error("Run VTR failed") - -# Remove lines from start of a file -def remove_file_lines_head(path, num_lines): - with open(path, 'r') as fin: - data = fin.read().splitlines(True) - with open(path, 'w') as fout: - fout.writelines(data[num_lines:]) - -def main(): - - for test_str in arg_list: - # Delete all files in temp directory - if temp_dir.is_dir(): - shutil.rmtree(temp_dir) - temp_dir.mkdir() - - print("Testing", test_str) - temp_dir_pl = temp_dir / "pl" - temp_dir_py = temp_dir / "py" - - # Run with old and new flows - run_vtr_flow("run_vtr_flow.pl", "old pl", temp_dir_pl, test_str) - run_vtr_flow("run_vtr_flow.py", "new py", temp_dir_py, test_str) - - # Check that output files match - for (pattern, skip_lines) in files_to_validate: - pl_file = find_file_matching(temp_dir_pl, pattern) - remove_file_lines_head(pl_file, skip_lines) - py_file = find_file_matching(temp_dir_py, pattern) - remove_file_lines_head(py_file, skip_lines) - - if not filecmp.cmp(pl_file, py_file): - error(pl_file, py_file, "differ") - else: - print(" ", pl_file, py_file, "match") - -if __name__ == "__main__": - main() \ No newline at end of file From 1a6409488b925c2d8f35c1e0a68715ccab39fe76 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Wed, 15 Jul 2020 13:19:37 -0600 Subject: [PATCH 61/86] removing accidental change --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 368ad0feea4..2c43291418c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -117,7 +117,7 @@ jobs: - MATRIX_EVAL="CC=gcc-5 && CXX=g++-5" script: - ./.github/travis/build.sh - - travis_wait 50 ./run_reg_test.pl vtr_reg_basic -show_failures -j2 + - travis_wait 30 ./run_reg_test.pl vtr_reg_basic -show_failures -j2 - stage: Test name: "Strong Regression Tests" env: @@ -144,7 +144,7 @@ jobs: script: - ./.github/travis/build.sh #We skip QoR since we are only checking for errors in sanitizer runs - - travis_wait 30 ./run_reg_test.pl vtr_reg_basic -show_failures -skip_qor -j2 + - travis_wait 50 ./run_reg_test.pl vtr_reg_basic -show_failures -skip_qor -j2 #Currently strong regression with sanitizers is disabled as it exceeds the maximum travis job run-time #- stage: Test #name: "Sanitized Strong Regression Tests" From b63446c8b4a63acb8a80f318e31a2a47f6f26fac Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Thu, 16 Jul 2020 13:31:30 -0600 Subject: [PATCH 62/86] adding newline to end of files --- vtr_flow/scripts/python_libs/vtr/__init__.py | 2 +- vtr_flow/scripts/python_libs/vtr/abc/__init__.py | 2 +- vtr_flow/scripts/python_libs/vtr/ace/__init__.py | 2 +- vtr_flow/scripts/python_libs/vtr/odin/__init__.py | 2 +- vtr_flow/scripts/python_libs/vtr/vpr/__init__.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/vtr_flow/scripts/python_libs/vtr/__init__.py b/vtr_flow/scripts/python_libs/vtr/__init__.py index 82e86a511be..f55219b239f 100644 --- a/vtr_flow/scripts/python_libs/vtr/__init__.py +++ b/vtr_flow/scripts/python_libs/vtr/__init__.py @@ -5,4 +5,4 @@ from .odin import run from .ace import run from .error import * -from .flow import run, VTR_STAGE \ No newline at end of file +from .flow import run, VTR_STAGE diff --git a/vtr_flow/scripts/python_libs/vtr/abc/__init__.py b/vtr_flow/scripts/python_libs/vtr/abc/__init__.py index 1d722fcc0a1..15cdd920af3 100644 --- a/vtr_flow/scripts/python_libs/vtr/abc/__init__.py +++ b/vtr_flow/scripts/python_libs/vtr/abc/__init__.py @@ -1 +1 @@ -from .abc import run, run_lec \ No newline at end of file +from .abc import run, run_lec diff --git a/vtr_flow/scripts/python_libs/vtr/ace/__init__.py b/vtr_flow/scripts/python_libs/vtr/ace/__init__.py index ffac7c866ed..9fb6597793e 100644 --- a/vtr_flow/scripts/python_libs/vtr/ace/__init__.py +++ b/vtr_flow/scripts/python_libs/vtr/ace/__init__.py @@ -1 +1 @@ -from .ace import run \ No newline at end of file +from .ace import run diff --git a/vtr_flow/scripts/python_libs/vtr/odin/__init__.py b/vtr_flow/scripts/python_libs/vtr/odin/__init__.py index 541e332898d..63d040329cd 100644 --- a/vtr_flow/scripts/python_libs/vtr/odin/__init__.py +++ b/vtr_flow/scripts/python_libs/vtr/odin/__init__.py @@ -1 +1 @@ -from .odin import run \ No newline at end of file +from .odin import run diff --git a/vtr_flow/scripts/python_libs/vtr/vpr/__init__.py b/vtr_flow/scripts/python_libs/vtr/vpr/__init__.py index 68018083ed2..6da4386cc41 100644 --- a/vtr_flow/scripts/python_libs/vtr/vpr/__init__.py +++ b/vtr_flow/scripts/python_libs/vtr/vpr/__init__.py @@ -1 +1 @@ -from .vpr import run,run_relax_W,cmp_full_vs_incr_STA \ No newline at end of file +from .vpr import run,run_relax_W,cmp_full_vs_incr_STA From c7f965d95d4358e7beb2b4c42b3e942042d3f07d Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Fri, 17 Jul 2020 09:25:03 -0600 Subject: [PATCH 63/86] moved python_lib documentation to be in masters documentation --- doc/src/conf.py | 6 +- doc/src/vtr/index.rst | 1 + doc/src/vtr/python_libs/vtr.rst | 38 ++++++ vtr_flow/scripts/python_libs/doc/Makefile | 20 --- vtr_flow/scripts/python_libs/doc/conf.py | 44 ------- vtr_flow/scripts/python_libs/doc/index.rst | 24 ---- vtr_flow/scripts/python_libs/doc/make.bat | 35 ------ vtr_flow/scripts/python_libs/vtr/abc/abc.py | 73 +++++++---- vtr_flow/scripts/python_libs/vtr/ace/ace.py | 36 ++++-- vtr_flow/scripts/python_libs/vtr/flow.py | 66 ++++++---- vtr_flow/scripts/python_libs/vtr/odin/odin.py | 42 ++++--- vtr_flow/scripts/python_libs/vtr/vpr/vpr.py | 114 +++++++++++------- 12 files changed, 257 insertions(+), 242 deletions(-) create mode 100755 doc/src/vtr/python_libs/vtr.rst delete mode 100755 vtr_flow/scripts/python_libs/doc/Makefile delete mode 100755 vtr_flow/scripts/python_libs/doc/conf.py delete mode 100755 vtr_flow/scripts/python_libs/doc/index.rst delete mode 100755 vtr_flow/scripts/python_libs/doc/make.bat diff --git a/doc/src/conf.py b/doc/src/conf.py index b3c3dd67e3e..7213ab31e4f 100644 --- a/doc/src/conf.py +++ b/doc/src/conf.py @@ -22,6 +22,7 @@ import recommonmark sys.path.append(".") +sys.path.insert(0, os.path.abspath('../../vtr_flow/scripts/python_libs')) from markdown_code_symlinks import LinkParser, MarkdownSymlinksDomain # Cool looking ReadTheDocs theme @@ -58,13 +59,16 @@ 'sphinx.ext.todo', 'sphinx.ext.mathjax', 'sphinx.ext.imgmath', + 'sphinx.ext.napoleon', + 'sphinx.ext.coverage', 'breathe', 'notfound.extension', 'sphinx_markdown_tables', 'sdcdomain', 'archdomain', 'rrgraphdomain', - 'recommonmark' + 'recommonmark', + 'sphinx.ext.autodoc' ] if have_sphinxcontrib_bibtex: diff --git a/doc/src/vtr/index.rst b/doc/src/vtr/index.rst index fda8c3a95cb..be581eec596 100644 --- a/doc/src/vtr/index.rst +++ b/doc/src/vtr/index.rst @@ -49,6 +49,7 @@ VTR also includes a set of benchmark designs known to work with the design flow. parse_vtr_task parse_config pass_requirements + python_libs/vtr diff --git a/doc/src/vtr/python_libs/vtr.rst b/doc/src/vtr/python_libs/vtr.rst new file mode 100755 index 00000000000..b35875591df --- /dev/null +++ b/doc/src/vtr/python_libs/vtr.rst @@ -0,0 +1,38 @@ +.. _python_libs/vtr: + +VTR Flow Python library +----------------------- +The VTR flow can be imported and implemented as a python library. Below are the descriptions of the useful functions. + +VTR flow +======== +.. automodule:: vtr.flow + :members: run + +ODIN II +======= + +.. automodule:: vtr.odin.odin + :members: + +ABC +=== + +.. automodule:: vtr.abc.abc + :members: + +ACE +=== + +.. automodule:: vtr.ace.ace + :members: + +VPR +=== + +.. automodule:: vtr.vpr.vpr + :members: + +.. toctree:: + :maxdepth: 2 + :caption: Contents: \ No newline at end of file diff --git a/vtr_flow/scripts/python_libs/doc/Makefile b/vtr_flow/scripts/python_libs/doc/Makefile deleted file mode 100755 index d4bb2cbb9ed..00000000000 --- a/vtr_flow/scripts/python_libs/doc/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line, and also -# from the environment for the first two. -SPHINXOPTS ?= -SPHINXBUILD ?= sphinx-build -SOURCEDIR = . -BUILDDIR = _build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/vtr_flow/scripts/python_libs/doc/conf.py b/vtr_flow/scripts/python_libs/doc/conf.py deleted file mode 100755 index 10b5f0ace62..00000000000 --- a/vtr_flow/scripts/python_libs/doc/conf.py +++ /dev/null @@ -1,44 +0,0 @@ -# -- Path setup -------------------------------------------------------------- -import os -import sys -import sphinx_rtd_theme -sys.path.insert(0, os.path.abspath('..')) - - -# -- Project information ----------------------------------------------------- - -project = u'Verilog-to-Routing' -copyright = u'2016, VTR Developers' -author = u'VTR Developers' - -# -- General configuration --------------------------------------------------- - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - "sphinx_rtd_theme", - 'sphinx.ext.autodoc', - 'sphinx.ext.napoleon', -] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] - - -# -- Options for HTML output ------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = 'sphinx_rtd_theme' - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] diff --git a/vtr_flow/scripts/python_libs/doc/index.rst b/vtr_flow/scripts/python_libs/doc/index.rst deleted file mode 100755 index 4d02cba7824..00000000000 --- a/vtr_flow/scripts/python_libs/doc/index.rst +++ /dev/null @@ -1,24 +0,0 @@ -Welcome to VTR's documentation! -=============================== -.. automodule:: vtr.flow - :members: run -.. automodule:: vtr.odin.odin - :members: -.. automodule:: vtr.abc.abc - :members: -.. automodule:: vtr.ace.ace_flow - :members: -.. automodule:: vtr.vpr.vpr - :members: -.. toctree:: - :maxdepth: 2 - :caption: Contents: - - - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/vtr_flow/scripts/python_libs/doc/make.bat b/vtr_flow/scripts/python_libs/doc/make.bat deleted file mode 100755 index 2119f51099b..00000000000 --- a/vtr_flow/scripts/python_libs/doc/make.bat +++ /dev/null @@ -1,35 +0,0 @@ -@ECHO OFF - -pushd %~dp0 - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set SOURCEDIR=. -set BUILDDIR=_build - -if "%1" == "" goto help - -%SPHINXBUILD% >NUL 2>NUL -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - exit /b 1 -) - -%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% -goto end - -:help -%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% - -:end -popd diff --git a/vtr_flow/scripts/python_libs/vtr/abc/abc.py b/vtr_flow/scripts/python_libs/vtr/abc/abc.py index 37fa14b74d1..69c2d06b86f 100644 --- a/vtr_flow/scripts/python_libs/vtr/abc/abc.py +++ b/vtr_flow/scripts/python_libs/vtr/abc/abc.py @@ -11,34 +11,47 @@ def run(architecture_file, circuit_file, """ Runs ABC to optimize specified file. - Usage: - vtr.abc.run(,,,[OPTIONS]) + .. note :: Usage: vtr.abc.run(,,,[OPTIONS]) - Required arguments: - architecture_file : Architecture file to target + Arguments + ========= + architecture_file : + Architecture file to target - circuit_file : Circuit file to optimize + circuit_file : + Circuit file to optimize - output_netlist : File name to output the resulting circuit to + output_netlist : + File name to output the resulting circuit to - Options: - command_runner : A CommandRunner object used to run system commands + Other Parameters + ---------------- + command_runner : + A CommandRunner object used to run system commands - temp_dir : Directory to run in (created if non-existent) + temp_dir : + Directory to run in (created if non-existent) - log_filename : File to log result to + log_filename : + File to log result to - abc_exec : ABC executable to be run + abc_exec : + ABC executable to be run - abc_script : The script to be run on abc + abc_script : + The script to be run on abc - abc_rc : The ABC rc file + abc_rc : + The ABC rc file - use_old_abc_script : Enables the use of the old ABC script + use_old_abc_script : + Enables the use of the old ABC script - abc_args : A dictionary of keyword arguments to pass on to ABC + abc_args : + A dictionary of keyword arguments to pass on to ABC - keep_intermediate_files : Determines if intermediate files are kept or deleted + keep_intermediate_files : + Determines if intermediate files are kept or deleted """ mkdir_p(temp_dir) @@ -220,22 +233,30 @@ def run_lec(reference_netlist, implementation_netlist, command_runner=CommandRun """ Run Logical Equivalence Checking (LEC) between two netlists using ABC - To run: - vtr.abc.run_lec(args) + .. note :: Usage: vtr.abc.run_lec(,,[OPTIONS]) - Required arguments: - reference_netlist : The reference netlist to be commpared to + Arguments + ========= + reference_netlist : + The reference netlist to be commpared to - implementation_netlist : The implemeted netlist to compare to the reference netlist + implementation_netlist : + The implemeted netlist to compare to the reference netlist - Options: - command_runner : A CommandRunner object used to run system commands + + Other Parameters + ---------------- + command_runner : + A CommandRunner object used to run system commands - temp_dir : Directory to run in (created if non-existent) + temp_dir : + Directory to run in (created if non-existent) - log_filename : File to log result to + log_filename : + File to log result to - abc_exec : ABC executable to be run + abc_exec : + ABC executable to be run """ mkdir_p(temp_dir) diff --git a/vtr_flow/scripts/python_libs/vtr/ace/ace.py b/vtr_flow/scripts/python_libs/vtr/ace/ace.py index c8ddfac8ad8..3896da93224 100644 --- a/vtr_flow/scripts/python_libs/vtr/ace/ace.py +++ b/vtr_flow/scripts/python_libs/vtr/ace/ace.py @@ -6,28 +6,38 @@ def run(circuit_file, old_netlist, output_netlist, output_activity_file, """ Runs ACE for activity estimation - Usage: - vtr.ace.run(,,,[OPTIONS]) + .. note :: Usage: vtr.ace.run(,,,[OPTIONS]) - Required arguments: - circuit_file : Circuit file to optimize + Arguments + ========= + circuit_file : + Circuit file to optimize - old_netlist : netlist to be anylized + old_netlist : + netlist to be anylized - output_netlist : File name to output the resulting circuit to + output_netlist : + File name to output the resulting circuit to - output_activity_file : The output activity file + output_activity_file : + The output activity file - Options: - command_runner : A CommandRunner object used to run system commands + Other Parameters + ---------------- + command_runner : + A CommandRunner object used to run system commands - temp_dir : Directory to run in (created if non-existent) + temp_dir : + Directory to run in (created if non-existent) - log_filename : File to log result to + log_filename : + File to log result to - ace_exec : ACE executable to be run + ace_exec : + ACE executable to be run - ace_seed : The ACE seed + ace_seed : + The ACE seed """ #Verify that files are Paths or convert them to Paths and check that they exist diff --git a/vtr_flow/scripts/python_libs/vtr/flow.py b/vtr_flow/scripts/python_libs/vtr/flow.py index 279351c35ae..94a5fdfc1e8 100644 --- a/vtr_flow/scripts/python_libs/vtr/flow.py +++ b/vtr_flow/scripts/python_libs/vtr/flow.py @@ -31,48 +31,68 @@ def run(architecture_file, circuit_file, """ Runs the VTR CAD flow to map the specified circuit_file onto the target architecture_file - Usage: - vtr.run(,,[OPTIONS]) + .. note :: Usage: vtr.run(,,[OPTIONS]) - Required arguments: - architecture_file : Architecture file to target + Arguments + ========= + architecture_file : + Architecture file to target - circuit_file : Circuit to implement + circuit_file : + Circuit to implement - Options: - power_tech_file : Technology power file. Enables power analysis and runs ace + Other Parameters + ---------------- + power_tech_file : + Technology power file. Enables power analysis and runs ace - start_stage : Stage of the flow to start at + start_stage : + Stage of the flow to start at - end_stage : Stage of the flow to finish at + end_stage : + Stage of the flow to finish at - temp_dir : Directory to run in (created if non-existent) + temp_dir : + Directory to run in (created if non-existent) - command_runner : A CommandRunner object used to run system commands + command_runner : + A CommandRunner object used to run system commands - verbosity : How much output to produce + verbosity : + How much output to produce - odin_args : A dictionary of keyword arguments to pass on to ODIN II + odin_args : + A dictionary of keyword arguments to pass on to ODIN II - abc_args : A dictionary of keyword arguments to pass on to ABC + abc_args : + A dictionary of keyword arguments to pass on to ABC - vpr_args : A dictionary of keyword arguments to pass on to VPR + vpr_args : + A dictionary of keyword arguments to pass on to VPR - keep_intermediate_files : Determines if intermediate files are kept or deleted + keep_intermediate_files : + Determines if intermediate files are kept or deleted - keep_result_files : Determines if the result files are kept or deleted + keep_result_files : + Determines if the result files are kept or deleted - min_hard_mult_size : Tells ODIN II the minimum multiplier size that should be implemented using hard multiplier (if available) + min_hard_mult_size : + Tells ODIN II the minimum multiplier size that should be implemented using hard multiplier (if available) - min_hard_adder_size : Tells ODIN II the minimum adder size that should be implemented using hard adder (if available). + min_hard_adder_size : + Tells ODIN II the minimum adder size that should be implemented using hard adder (if available). - check_equivalent : Enables Logical Equivalence Checks + check_equivalent : + Enables Logical Equivalence Checks - use_old_abc_script : Enables the use of the old ABC script + use_old_abc_script : + Enables the use of the old ABC script - relax_W_factor : Factor by which to relax minimum channel width for critical path delay routing + relax_W_factor : + Factor by which to relax minimum channel width for critical path delay routing - check_incremental_sta_consistency : Do a second-run of the incremental analysis to compare the result files + check_incremental_sta_consistency : + Do a second-run of the incremental analysis to compare the result files """ if odin_args == None: diff --git a/vtr_flow/scripts/python_libs/vtr/odin/odin.py b/vtr_flow/scripts/python_libs/vtr/odin/odin.py index 56ac9b74b5b..d0b1472be77 100644 --- a/vtr_flow/scripts/python_libs/vtr/odin/odin.py +++ b/vtr_flow/scripts/python_libs/vtr/odin/odin.py @@ -15,32 +15,44 @@ def run(architecture_file, circuit_file, """ Runs ODIN II on the specified architecture file and circuit file - Usage: - vtr.odin.run(,,,[OPTIONS]) + .. note :: Usage: vtr.odin.run(,,,[OPTIONS]) - Required arguments: - architecture_file : Architecture file to target + Arguments + ========= + architecture_file : + Architecture file to target - circuit_file : Circuit file to optimize + circuit_file : + Circuit file to optimize - output_netlist : File name to output the resulting circuit to + output_netlist : + File name to output the resulting circuit to - Options: - command_runner : A CommandRunner object used to run system commands + Other Parameters + ---------------- + command_runner : + A CommandRunner object used to run system commands - temp_dir : Directory to run in (created if non-existent) + temp_dir : + Directory to run in (created if non-existent) - odin_args: A dictionary of keyword arguments to pass on to ODIN II + odin_args: + A dictionary of keyword arguments to pass on to ODIN II - log_filename : File to log result to + log_filename : + File to log result to - odin_exec: ODIN II executable to be run + odin_exec: + ODIN II executable to be run - odin_config: The ODIN II configuration file + odin_config: + The ODIN II configuration file - min_hard_mult_size : Tells ODIN II the minimum multiplier size that should be implemented using hard multiplier (if available) + min_hard_mult_size : + Tells ODIN II the minimum multiplier size that should be implemented using hard multiplier (if available) - min_hard_adder_size : Tells ODIN II the minimum adder size that should be implemented using hard adder (if available). + min_hard_adder_size : + Tells ODIN II the minimum adder size that should be implemented using hard adder (if available). """ mkdir_p(temp_dir) diff --git a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py index 1c1a4ded319..c45fb607482 100644 --- a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py +++ b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py @@ -13,31 +13,44 @@ def run_relax_W(architecture, circuit_name, circuit, command_runner=CommandRunne 2nd: At relaxed channel width (e.g. for critical path delay) - Usage: - vtr.vpr.run_relax_W(,,,[OPTIONS]) + .. note :: Usage: vtr.vpr.run_relax_W(,,,[OPTIONS]) - Required arguments: - architecture: Architecture file + Arguments + ========= + architecture: + Architecture file - circuit_name: Curcuit netlist + circuit_name: + Curcuit netlist - circuit: Input circuit netlist - Options: - command_runner: CommandRunner object + circuit: + Input circuit netlist + + Other Parameters + ---------------- + command_runner: + CommandRunner object - temp_dir: Directory to run in + temp_dir: + Directory to run in - relax_W_factor: Factor by which to relax minimum channel width for critical path delay routing + relax_W_factor: + Factor by which to relax minimum channel width for critical path delay routing - vpr_exec: Path to the VPR executable + vpr_exec: + Path to the VPR executable - verbosity: How much progress output to produce + verbosity: + How much progress output to produce - logfile_base: Base name for log files (e.g. "vpr" produces vpr.min_W.out, vpr.relaxed_W.out) + logfile_base: + Base name for log files (e.g. "vpr" produces vpr.min_W.out, vpr.relaxed_W.out) - vpr_args: Extra arguments for VPR + vpr_args: + Extra arguments for VPR - output_netlist: Output implementation netlist to generate + output_netlist: + Output implementation netlist to generate """ if vpr_args is None: vpr_args = OrderedDict() @@ -94,30 +107,41 @@ def run(architecture, circuit_name, circuit, command_runner=CommandRunner(), tem """ Runs VPR with the specified configuration - To run: - vtr.vpr.run(args) + .. note :: Usage: vtr.vpr.run(,,,[OPTIONS]) - Required arguments: - architecture: Architecture file + Arguments + ========= + architecture: + Architecture file - circuit_name: Curcuit netlist + circuit_name: + Curcuit netlist - circuit: Input circuit file + circuit: + Input circuit file - Options: - command_runner: CommandRunner object + Other Parameters + ---------------- + command_runner: + CommandRunner object - temp_dir: Directory to run in + temp_dir: + Directory to run in - output_netlist: Output implementation netlist to generate + output_netlist: + Output implementation netlist to generate - log_filename : File to log result to + log_filename : + File to log result to - vpr_exec: Path to the VPR executable + vpr_exec: + Path to the VPR executable - vpr_args: Extra arguments for VPR + vpr_args: + Extra arguments for VPR - check_for_second_run: enables checking for arguments in vpr_args that require a second run of VPR ie analysis + check_for_second_run: + enables checking for arguments in vpr_args that require a second run of VPR ie analysis """ @@ -201,27 +225,35 @@ def run(architecture, circuit_name, circuit, command_runner=CommandRunner(), tem raise InspectError("failed: vpr (RR Graph XML output not consistent when reloaded)") def cmp_full_vs_incr_STA(architecture,circuit_name,circuit,command_runner=CommandRunner(),vpr_args=None,rr_graph_ext=".xml",temp_dir=".",vpr_exec=None): - """" + """ Sanity check that full STA and the incremental STA produce the same *.net, *.place, *.route files as well as identical timing report files - To run: - vtr.vpr.cmp_full_vs_incr_STA(args) + .. note :: Usage: vtr.vpr.cmp_full_vs_incr_STA(,,,[OPTIONS]) - Required arguments: - architecture: Architecture file + Arguments + ========= + architecture: + Architecture file - circuit_name: Curcuit netlist + circuit_name: + Curcuit netlist - circuit: Input circuit file + circuit: + Input circuit file - Options: - command_runner: CommandRunner object + Other Parameters + ---------------- + command_runner: + CommandRunner object - temp_dir: Directory to run in + temp_dir: + Directory to run in - vpr_exec: Path to the VPR executable + vpr_exec: + Path to the VPR executable - vpr_args: Extra arguments for VPR + vpr_args: + Extra arguments for VPR """ From ba2312bdd556f1b04cbe956ff37897e4d404640d Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Mon, 20 Jul 2020 10:00:03 -0600 Subject: [PATCH 64/86] fixed file default name --- vtr_flow/scripts/python_libs/vtr/abc/abc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vtr_flow/scripts/python_libs/vtr/abc/abc.py b/vtr_flow/scripts/python_libs/vtr/abc/abc.py index 69c2d06b86f..b9da8d03290 100644 --- a/vtr_flow/scripts/python_libs/vtr/abc/abc.py +++ b/vtr_flow/scripts/python_libs/vtr/abc/abc.py @@ -229,7 +229,7 @@ def populate_clock_list(circuit_file,blackbox_latches_script,clk_list,command_ru for line in f.readlines(): clk_list.append(line.strip('\n')) -def run_lec(reference_netlist, implementation_netlist, command_runner=CommandRunner(), temp_dir=".", log_filename="abc.dec.out", abc_exec=None): +def run_lec(reference_netlist, implementation_netlist, command_runner=CommandRunner(), temp_dir=".", log_filename="abc.lec.out", abc_exec=None): """ Run Logical Equivalence Checking (LEC) between two netlists using ABC From 8a9a4a781f5ed163b3942e9c707c9df0d2fb568d Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Mon, 20 Jul 2020 12:28:35 -0600 Subject: [PATCH 65/86] made circuit_name optional argument to vpr --- vtr_flow/scripts/python_libs/vtr/flow.py | 6 +-- vtr_flow/scripts/python_libs/vtr/vpr/vpr.py | 48 +++++++++++---------- 2 files changed, 28 insertions(+), 26 deletions(-) diff --git a/vtr_flow/scripts/python_libs/vtr/flow.py b/vtr_flow/scripts/python_libs/vtr/flow.py index 94a5fdfc1e8..30dc679e8e1 100644 --- a/vtr_flow/scripts/python_libs/vtr/flow.py +++ b/vtr_flow/scripts/python_libs/vtr/flow.py @@ -226,7 +226,7 @@ def run(architecture_file, circuit_file, if route_fixed_W: #The User specified a fixed channel width - vtr.vpr.run(architecture_copy, circuit_copy, pre_vpr_netlist, + vtr.vpr.run(architecture_copy, pre_vpr_netlist, circuit_copy.stem, output_netlist=post_vpr_netlist, command_runner=command_runner, temp_dir=temp_dir, @@ -234,7 +234,7 @@ def run(architecture_file, circuit_file, rr_graph_ext=rr_graph_ext) else: #First find minW and then re-route at a relaxed W - vtr.vpr.run_relax_W(architecture_copy, circuit_copy, pre_vpr_netlist, + vtr.vpr.run_relax_W(architecture_copy, pre_vpr_netlist, circuit_copy.stem, output_netlist=post_vpr_netlist, command_runner=command_runner, relax_W_factor=relax_W_factor, @@ -257,7 +257,7 @@ def run(architecture_file, circuit_file, # Do a second-run of the incremental analysis to compare the result files if check_incremental_sta_consistency: - vtr.vpr.cmp_full_vs_incr_STA(architecture_copy, circuit_copy, pre_vpr_netlist, + vtr.vpr.cmp_full_vs_incr_STA(architecture_copy, pre_vpr_netlist, circuit_copy.stem, command_runner=command_runner, vpr_args=vpr_args, rr_graph_ext=rr_graph_ext, diff --git a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py index c45fb607482..be77b0c96ef 100644 --- a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py +++ b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py @@ -3,7 +3,7 @@ from vtr import mkdir_p, find_vtr_file, CommandRunner, print_verbose, relax_W, determine_lut_size, determine_min_W, verify_file from vtr.error import InspectError -def run_relax_W(architecture, circuit_name, circuit, command_runner=CommandRunner(), temp_dir=".", +def run_relax_W(architecture, circuit, circuit_name=None, command_runner=CommandRunner(), temp_dir=".", relax_W_factor=1.3, vpr_exec=None, verbosity=1, logfile_base="vpr", vpr_args=None, output_netlist=None): """ @@ -20,14 +20,14 @@ def run_relax_W(architecture, circuit_name, circuit, command_runner=CommandRunne architecture: Architecture file - circuit_name: - Curcuit netlist - circuit: Input circuit netlist Other Parameters ---------------- + circuit_name: + Name of the circuit file + command_runner: CommandRunner object @@ -59,7 +59,6 @@ def run_relax_W(architecture, circuit_name, circuit, command_runner=CommandRunne #Verify that files are Paths or convert them to Paths and check that they exist architecture = verify_file(architecture, "Architecture") - circuit_name = verify_file(circuit_name, "Circuit") circuit = verify_file(circuit, "Circuit") vpr_min_W_log = '.'.join([logfile_base, "out"]) @@ -82,7 +81,7 @@ def run_relax_W(architecture, circuit_name, circuit, command_runner=CommandRunne if vpr_exec == None: vpr_exec = find_vtr_file('vpr', is_executable=True) - run(architecture, circuit_name, circuit, command_runner, temp_dir, log_filename=vpr_min_W_log, vpr_exec=vpr_exec, vpr_args=vpr_args) + run(architecture, circuit, circuit_name, command_runner, temp_dir, log_filename=vpr_min_W_log, vpr_exec=vpr_exec, vpr_args=vpr_args) if ('pack' in vpr_args or 'place' in vpr_args) and 'route' not in vpr_args: #Don't look for min W if routing was not run @@ -99,10 +98,10 @@ def run_relax_W(architecture, circuit_name, circuit, command_runner=CommandRunne #VPR does not support performing routing when fixed pins # are specified, and placement is not run; so remove the option - run(architecture, circuit_name, circuit, command_runner, temp_dir, log_filename=vpr_relaxed_W_log, vpr_exec=vpr_exec, vpr_args=vpr_args, check_for_second_run=False) + run(architecture, circuit, circuit_name, command_runner, temp_dir, log_filename=vpr_relaxed_W_log, vpr_exec=vpr_exec, vpr_args=vpr_args, check_for_second_run=False) -def run(architecture, circuit_name, circuit, command_runner=CommandRunner(), temp_dir=".", output_netlist=None, +def run(architecture, circuit, circuit_name=None, command_runner=CommandRunner(), temp_dir=".", output_netlist=None, log_filename="vpr.out", vpr_exec=None, vpr_args=None,check_for_second_run=True,rr_graph_ext=".xml",): """ Runs VPR with the specified configuration @@ -114,14 +113,14 @@ def run(architecture, circuit_name, circuit, command_runner=CommandRunner(), tem architecture: Architecture file - circuit_name: - Curcuit netlist - circuit: Input circuit file Other Parameters ---------------- + circuit_name: + Name of the circuit file + command_runner: CommandRunner object @@ -155,11 +154,12 @@ def run(architecture, circuit_name, circuit, command_runner=CommandRunner(), tem #Verify that files are Paths or convert them to Paths and check that they exist architecture = verify_file(architecture, "Architecture") - circuit_name = verify_file(circuit_name, "Circuit") circuit = verify_file(circuit, "Circuit") - - cmd = [vpr_exec, architecture.name, circuit_name.stem, "--circuit_file", circuit.name] - + cmd = [] + if circuit_name: + cmd = [vpr_exec, architecture.name, circuit_name, "--circuit_file", circuit.name] + else: + cmd = [vpr_exec, architecture.name, circuit.name] #Enable netlist generation #if output_netlist: #vpr_args['gen_postsynthesis_netlist'] = output_netlist @@ -206,8 +206,11 @@ def run(architecture, circuit_name, circuit, command_runner=CommandRunner(), tem second_run_args["write_rr_graph"] = rr_graph_out_file2 second_run_log_file = "vpr_second_run.out" - cmd = [vpr_exec, architecture.name, circuit_name.stem, "--circuit_file", circuit.name] - + cmd = [] + if circuit_name: + cmd = [vpr_exec, architecture.name, circuit_name, "--circuit_file", circuit.name] + else: + cmd = [vpr_exec, architecture.name, circuit.name] for arg, value in vpr_args.items(): if value == True: cmd += ["--" + arg] @@ -224,7 +227,7 @@ def run(architecture, circuit_name, circuit, command_runner=CommandRunner(), tem if diff_result: raise InspectError("failed: vpr (RR Graph XML output not consistent when reloaded)") -def cmp_full_vs_incr_STA(architecture,circuit_name,circuit,command_runner=CommandRunner(),vpr_args=None,rr_graph_ext=".xml",temp_dir=".",vpr_exec=None): +def cmp_full_vs_incr_STA(architecture,circuit,circuit_name=None,command_runner=CommandRunner(),vpr_args=None,rr_graph_ext=".xml",temp_dir=".",vpr_exec=None): """ Sanity check that full STA and the incremental STA produce the same *.net, *.place, *.route files as well as identical timing report files @@ -235,14 +238,14 @@ def cmp_full_vs_incr_STA(architecture,circuit_name,circuit,command_runner=Comman architecture: Architecture file - circuit_name: - Curcuit netlist - circuit: Input circuit file Other Parameters ---------------- + circuit_name: + Name of the circuit file + command_runner: CommandRunner object @@ -259,7 +262,6 @@ def cmp_full_vs_incr_STA(architecture,circuit_name,circuit,command_runner=Comman #Verify that files are Paths or convert them to Paths and check that they exist architecture = verify_file(architecture, "Architecture") - circuit_name = verify_file(circuit_name, "Circuit") circuit = verify_file(circuit, "Circuit") default_output_filenames = [ @@ -284,7 +286,7 @@ def cmp_full_vs_incr_STA(architecture,circuit_name,circuit,command_runner=Comman incremental_vpr_args["timing_update_type"]="incremental" fixed_W_log_file = "vpr.incr_sta.out" - run(architecture, circuit_name, circuit, command_runner, temp_dir, log_filename=fixed_W_log_file, vpr_exec=vpr_exec, vpr_args=incremental_vpr_args, check_for_second_run=False) + run(architecture, circuit, circuit_name, command_runner, temp_dir, log_filename=fixed_W_log_file, vpr_exec=vpr_exec, vpr_args=incremental_vpr_args, check_for_second_run=False) # Rename the incremental STA output files for filename in default_output_filenames: From f7c965fd2769977b937978b33e431ab6d7b54e74 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Mon, 20 Jul 2020 13:31:54 -0600 Subject: [PATCH 66/86] fixing circuit_name issue --- vtr_flow/scripts/python_libs/vtr/vpr/vpr.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py index be77b0c96ef..71d535b9829 100644 --- a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py +++ b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py @@ -263,11 +263,12 @@ def cmp_full_vs_incr_STA(architecture,circuit,circuit_name=None,command_runner=C #Verify that files are Paths or convert them to Paths and check that they exist architecture = verify_file(architecture, "Architecture") circuit = verify_file(circuit, "Circuit") - + if not circuit_name: + circuit_name = circuit.stem default_output_filenames = [ - "{}.net".format(circuit_name.stem), - "{}.place".format(circuit_name.stem), - "{}.route".format(circuit_name.stem), + "{}.net".format(circuit_name), + "{}.place".format(circuit_name), + "{}.route".format(circuit_name), "report_timing.setup.rpt", "report_timing.hold.rpt", "report_unconstrained_timing.setup.rpt", From fc27438907d6ec30a0f313ce3085c08c499d95e8 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Mon, 20 Jul 2020 16:14:34 -0600 Subject: [PATCH 67/86] fixing two small bugs --- vtr_flow/scripts/python_libs/vtr/util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vtr_flow/scripts/python_libs/vtr/util.py b/vtr_flow/scripts/python_libs/vtr/util.py index 1d60dab55d1..b5ad7aaf250 100644 --- a/vtr_flow/scripts/python_libs/vtr/util.py +++ b/vtr_flow/scripts/python_libs/vtr/util.py @@ -405,13 +405,13 @@ def load_config_lines(filepath, allow_includes=True): if blank_regex.match(line): continue - if line.startswith("@include"): + if line.startswith("%include"): if allow_includes: components = line.split() assert len(components) == 2 include_file = components[1].strip('"') #Strip quotes - include_file_abs = str(Path(filepath).paren / include_file) + include_file_abs = str(Path(filepath).parent / include_file) #Recursively load the config config_lines += load_config_lines(include_file_abs, allow_includes=allow_includes) From dab68f8590a6019a777d6ea73dac8428c7ae146d Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Mon, 20 Jul 2020 16:25:08 -0600 Subject: [PATCH 68/86] fixed verbosity confusion and fixed abc_flow_type variable --- vtr_flow/scripts/python_libs/vtr/abc/abc.py | 19 +++++++++---------- vtr_flow/scripts/python_libs/vtr/flow.py | 4 +--- vtr_flow/scripts/python_libs/vtr/util.py | 11 ++++++++--- vtr_flow/scripts/python_libs/vtr/vpr/vpr.py | 5 +---- vtr_flow/scripts/run_vtr_flow.py | 5 +---- 5 files changed, 20 insertions(+), 24 deletions(-) diff --git a/vtr_flow/scripts/python_libs/vtr/abc/abc.py b/vtr_flow/scripts/python_libs/vtr/abc/abc.py index b9da8d03290..baf09dc0b3d 100644 --- a/vtr_flow/scripts/python_libs/vtr/abc/abc.py +++ b/vtr_flow/scripts/python_libs/vtr/abc/abc.py @@ -70,19 +70,19 @@ def run(architecture_file, circuit_file, # # Parse arguments # - abc_flow_type = 2 + abc_flow_type = "iterative_bb" abc_run_args = "" use_old_latches_restoration_script = 0 lut_size = None if("iterative_bb" in abc_args): - abc_flow_type=2 + abc_flow_type="iterative_bb" del abc_args["iterative_bb"] if("blanket_bb" in abc_args): - abc_flow_type=3 + abc_flow_type="blanket_bb" del abc_args["blanket_bb"] if("once_bb" in abc_args): - abc_flow_type=1 + abc_flow_type="once_bb" del abc_args["once_bb"] if("use_old_latches_restoration_script" in abc_args): use_old_latches_restoration_script = 1 @@ -102,8 +102,7 @@ def run(architecture_file, circuit_file, if(lut_size == None): lut_size = determine_lut_size(str(architecture_file)) - if(abc_flow_type): - populate_clock_list(circuit_file,blackbox_latches_script,clk_list,command_runner,temp_dir,"report_clocks.abc.out") + populate_clock_list(circuit_file,blackbox_latches_script,clk_list,command_runner,temp_dir,"report_clocks.abc.out") if abc_exec == None: abc_exec = find_vtr_file('abc', is_executable=True) @@ -117,7 +116,7 @@ def run(architecture_file, circuit_file, iterations=len(clk_list) - if(iterations==0 or abc_flow_type != 2): + if(iterations==0 or abc_flow_type != "iterative_bb"): iterations=1 original_script = abc_script @@ -126,7 +125,7 @@ def run(architecture_file, circuit_file, pre_abc_blif= Path(temp_dir) / (str(i)+"_" + circuit_file.name) post_abc_blif = Path(temp_dir) / (str(i)+"_"+output_netlist.name) post_abc_raw_blif = Path(temp_dir) / (str(i)+"_"+output_netlist.with_suffix('').stem+".raw.abc.blif") - if(abc_flow_type==3): + if(abc_flow_type=="blanket_bb"): cmd = [blackbox_latches_script, "--input", input_file,"--output",pre_abc_blif.name] command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=str(i)+"_blackboxing_latch.out", indent_depth=1) @@ -192,7 +191,7 @@ def run(architecture_file, circuit_file, log_file = Path(log_filename) command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_file.stem+str(i)+log_file.suffix, indent_depth=1) - if(abc_flow_type != 3 and len(clk_list)>i): + if(abc_flow_type != "blanket_bb" and len(clk_list)>i): cmd = [blackbox_latches_script,"--restore", clk_list[i], "--input", post_abc_raw_blif.name,"--output",post_abc_blif.name] command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename="restore_latch" + str(i) + ".out", indent_depth=1) else: @@ -203,7 +202,7 @@ def run(architecture_file, circuit_file, cmd = [restore_multiclock_info_script, pre_abc_blif.name, post_abc_raw_blif.name ,post_abc_blif.name] command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename="restore_latch" + str(i) + ".out", indent_depth=1) - if(abc_flow_type != 2): + if(abc_flow_type != "iterative_bb"): break abc_script = original_script diff --git a/vtr_flow/scripts/python_libs/vtr/flow.py b/vtr_flow/scripts/python_libs/vtr/flow.py index 30dc679e8e1..83c83a1af31 100644 --- a/vtr_flow/scripts/python_libs/vtr/flow.py +++ b/vtr_flow/scripts/python_libs/vtr/flow.py @@ -16,7 +16,6 @@ def run(architecture_file, circuit_file, start_stage=VTR_STAGE.odin, end_stage=VTR_STAGE.vpr, command_runner=vtr.CommandRunner(), temp_dir="./temp", - verbosity=0, odin_args=None, abc_args=None, vpr_args=None, @@ -59,7 +58,7 @@ def run(architecture_file, circuit_file, A CommandRunner object used to run system commands verbosity : - How much output to produce + Weather to output error description or not odin_args : A dictionary of keyword arguments to pass on to ODIN II @@ -239,7 +238,6 @@ def run(architecture_file, circuit_file, command_runner=command_runner, relax_W_factor=relax_W_factor, temp_dir=temp_dir, - verbosity=verbosity, vpr_args=vpr_args) if not lec_base_netlist: diff --git a/vtr_flow/scripts/python_libs/vtr/util.py b/vtr_flow/scripts/python_libs/vtr/util.py index b5ad7aaf250..669bdc825e0 100644 --- a/vtr_flow/scripts/python_libs/vtr/util.py +++ b/vtr_flow/scripts/python_libs/vtr/util.py @@ -23,7 +23,7 @@ class RawDefaultHelpFormatter(argparse.ArgumentDefaultsHelpFormatter, argparse.R class CommandRunner(object): - def __init__(self, timeout_sec=None, max_memory_mb=None, track_memory=True, verbose_error=False, verbose=False, echo_cmd=False, indent="\t", show_failures=False,valgrind=False): + def __init__(self, timeout_sec=None, max_memory_mb=None, track_memory=True, verbose_error=None, verbose=False, echo_cmd=None, indent="\t", show_failures=False,valgrind=False): """ An object for running system commands with timeouts, memory limits and varying verbose-ness @@ -32,12 +32,17 @@ def __init__(self, timeout_sec=None, max_memory_mb=None, track_memory=True, verb timeout_sec: maximum walk-clock-time of the command in seconds. Default: None max_memory_mb: maximum memory usage of the command in megabytes (if supported). Default: None track_memory: Whether to track usage of the command (disabled if not supported). Default: True - verbose_error: Produce more verbose output if the commadn fails. Default: False + verbose_error: Produce more verbose output if the commadn fails. Default: Equal to verbose verbose: Produce more verbose output. Default: False - echo_cmd: Echo the command before running. Default: False + echo_cmd: Echo the command before running. Default: Equal to verbose indent: The string specifying a single indent (used in verbose mode) valgrind: Indicates if commands should be run with valgrind """ + if verbose_error==None: + verbose_error = verbose + if echo_cmd == None: + echo_cmd = verbose + self._timeout_sec = timeout_sec self._max_memory_mb = max_memory_mb self._track_memory = track_memory diff --git a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py index 71d535b9829..9ad780f9deb 100644 --- a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py +++ b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py @@ -4,7 +4,7 @@ from vtr.error import InspectError def run_relax_W(architecture, circuit, circuit_name=None, command_runner=CommandRunner(), temp_dir=".", - relax_W_factor=1.3, vpr_exec=None, verbosity=1, logfile_base="vpr", + relax_W_factor=1.3, vpr_exec=None, logfile_base="vpr", vpr_args=None, output_netlist=None): """ Runs VPR twice: @@ -40,9 +40,6 @@ def run_relax_W(architecture, circuit, circuit_name=None, command_runner=Command vpr_exec: Path to the VPR executable - verbosity: - How much progress output to produce - logfile_base: Base name for log files (e.g. "vpr" produces vpr.min_W.out, vpr.relaxed_W.out) diff --git a/vtr_flow/scripts/run_vtr_flow.py b/vtr_flow/scripts/run_vtr_flow.py index 2e2ad6efe1a..fa8f2b3fe4c 100755 --- a/vtr_flow/scripts/run_vtr_flow.py +++ b/vtr_flow/scripts/run_vtr_flow.py @@ -319,9 +319,7 @@ def vtr_command_main(arg_list, prog=None): command_runner = vtr.CommandRunner(track_memory=True, max_memory_mb=args.limit_memory_usage, timeout_sec=args.timeout, - verbose_error=True if args.verbose == 2 else False, - verbose=True if args.verbose > 2 else False, - echo_cmd=True if args.verbose >= 4 else False, + verbose=args.verbose, show_failures = args.show_failures, valgrind = args.valgrind) exit_status = 0 @@ -396,7 +394,6 @@ def vtr_command_main(arg_list, prog=None): start_stage=args.start, end_stage=args.end, command_runner=command_runner, - verbosity=args.verbose, vpr_args=vpr_args, abc_args=abc_args, odin_args=odin_args, From b04f23520b687603775797d3c96ebb41400a6e0f Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Tue, 21 Jul 2020 08:13:03 -0600 Subject: [PATCH 69/86] fixing help message --- vtr_flow/scripts/run_vtr_flow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vtr_flow/scripts/run_vtr_flow.py b/vtr_flow/scripts/run_vtr_flow.py index fa8f2b3fe4c..f6bdae65727 100755 --- a/vtr_flow/scripts/run_vtr_flow.py +++ b/vtr_flow/scripts/run_vtr_flow.py @@ -211,7 +211,7 @@ def vtr_command_argparser(prog=None): default=False, action="store_true", dest="blanket_bb", - help="Use iterative black-boxing flow with out clocks") #not sure if this is a correct statement. + help="Use iterative black-boxing on all the clocks") abc.add_argument("-use_old_latches_restoration_script", default=False, action="store_true", From a0c6fcdcfcf559185c4215ccd0e00e2c752f7bbb Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Wed, 22 Jul 2020 16:22:22 -0600 Subject: [PATCH 70/86] replacing enum with python enum. replacing mkdir_p with Pathlib.mkdir. Cleaned up run_vtr_flow and flow for pylint --- vtr_flow/scripts/python_libs/vtr/__init__.py | 4 +- vtr_flow/scripts/python_libs/vtr/abc/abc.py | 36 +- vtr_flow/scripts/python_libs/vtr/ace/ace.py | 8 +- vtr_flow/scripts/python_libs/vtr/flow.py | 424 ++++---- vtr_flow/scripts/python_libs/vtr/inspect.py | 6 +- vtr_flow/scripts/python_libs/vtr/odin/odin.py | 17 +- vtr_flow/scripts/python_libs/vtr/util.py | 45 +- vtr_flow/scripts/python_libs/vtr/vpr/vpr.py | 28 +- vtr_flow/scripts/run_vtr_flow.py | 902 ++++++++++-------- 9 files changed, 807 insertions(+), 663 deletions(-) diff --git a/vtr_flow/scripts/python_libs/vtr/__init__.py b/vtr_flow/scripts/python_libs/vtr/__init__.py index f55219b239f..fe4a12b7b10 100644 --- a/vtr_flow/scripts/python_libs/vtr/__init__.py +++ b/vtr_flow/scripts/python_libs/vtr/__init__.py @@ -1,8 +1,8 @@ -from .util import load_config_lines, mkdir_p, find_vtr_file, CommandRunner, print_verbose, relax_W, file_replace, make_enum, RawDefaultHelpFormatter, VERBOSITY_CHOICES,format_elapsed_time, write_tab_delimitted_csv, load_list_file, find_vtr_root, argparse_str2bool, get_next_run_dir, get_latest_run_dir, verify_file +from .util import load_config_lines, find_vtr_file, CommandRunner, print_verbose, relax_W, file_replace, RawDefaultHelpFormatter, VERBOSITY_CHOICES,format_elapsed_time, write_tab_delimitted_csv, load_list_file, find_vtr_root, argparse_str2bool, get_next_run_dir, get_latest_run_dir, verify_file from .inspect import determine_lut_size, determine_min_W, determine_memory_addr_width from .abc import run, run_lec from .vpr import run,run_relax_W,cmp_full_vs_incr_STA from .odin import run from .ace import run from .error import * -from .flow import run, VTR_STAGE +from .flow import run, VtrStage diff --git a/vtr_flow/scripts/python_libs/vtr/abc/abc.py b/vtr_flow/scripts/python_libs/vtr/abc/abc.py index baf09dc0b3d..d0b8aa4fe9e 100644 --- a/vtr_flow/scripts/python_libs/vtr/abc/abc.py +++ b/vtr_flow/scripts/python_libs/vtr/abc/abc.py @@ -1,11 +1,11 @@ import shutil from pathlib import Path -from vtr import mkdir_p, find_vtr_file, determine_lut_size, verify_file, CommandRunner +from vtr import find_vtr_file, determine_lut_size, verify_file, CommandRunner from vtr.error import VtrError, InspectError, CommandError def run(architecture_file, circuit_file, output_netlist, command_runner=CommandRunner(), - temp_dir=".", log_filename="abc.out", + temp_dir=Path("."), log_filename="abc.out", abc_exec=None, abc_script=None, abc_rc=None, use_old_abc_script = False, abc_args = None, keep_intermediate_files=True): """ @@ -54,9 +54,10 @@ def run(architecture_file, circuit_file, Determines if intermediate files are kept or deleted """ - mkdir_p(temp_dir) + temp_dir = Path(temp_dir) if not isinstance(temp_dir, Path) else temp_dir + temp_dir.mkdir(parents=True, exist_ok=True) - if abc_args == None: + if abc_args is None: abc_args = OrderedDict() #Verify that files are Paths or convert them to Paths and check that they exist @@ -99,18 +100,18 @@ def run(architecture_file, circuit_file, else: abc_run_args += ["--" + arg, str(value)] - if(lut_size == None): + if(lut_size is None): lut_size = determine_lut_size(str(architecture_file)) populate_clock_list(circuit_file,blackbox_latches_script,clk_list,command_runner,temp_dir,"report_clocks.abc.out") - if abc_exec == None: + if abc_exec is None: abc_exec = find_vtr_file('abc', is_executable=True) - if abc_rc == None: + if abc_rc is None: abc_rc = Path(abc_exec).parent / 'abc.rc' - shutil.copyfile(str(abc_rc), str(Path(temp_dir) / 'abc.rc')) + shutil.copyfile(str(abc_rc), str(temp_dir / 'abc.rc')) @@ -122,9 +123,9 @@ def run(architecture_file, circuit_file, original_script = abc_script input_file = circuit_file.name for i in range(0, iterations): - pre_abc_blif= Path(temp_dir) / (str(i)+"_" + circuit_file.name) - post_abc_blif = Path(temp_dir) / (str(i)+"_"+output_netlist.name) - post_abc_raw_blif = Path(temp_dir) / (str(i)+"_"+output_netlist.with_suffix('').stem+".raw.abc.blif") + pre_abc_blif= temp_dir / (str(i)+"_" + circuit_file.name) + post_abc_blif = temp_dir / (str(i)+"_"+output_netlist.name) + post_abc_raw_blif = temp_dir / (str(i)+"_"+output_netlist.with_suffix('').stem+".raw.abc.blif") if(abc_flow_type=="blanket_bb"): cmd = [blackbox_latches_script, "--input", input_file,"--output",pre_abc_blif.name] command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=str(i)+"_blackboxing_latch.out", indent_depth=1) @@ -135,7 +136,7 @@ def run(architecture_file, circuit_file, else: pre_abc_blif = input_file - if abc_script == None: + if abc_script is None: abc_script = ['echo ""', 'echo "Load Netlist"', @@ -213,7 +214,7 @@ def run(architecture_file, circuit_file, command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename="restore_latch" + str(i) + ".out", indent_depth=1) if(not keep_intermediate_files): files = [] - for file in Path(temp_dir).iterdir(): + for file in temp_dir.iterdir(): if file.suffix in ('.dot','.v','.rc'): files.append(file) for p in files: @@ -221,14 +222,14 @@ def run(architecture_file, circuit_file, def populate_clock_list(circuit_file,blackbox_latches_script,clk_list,command_runner,temp_dir,log_filename): - clk_list_path = Path(temp_dir) / "report_clk.out" + clk_list_path = temp_dir / "report_clk.out" cmd = [blackbox_latches_script, "--input", circuit_file.name,"--output_list", clk_list_path.name] command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) with clk_list_path.open('r') as f: for line in f.readlines(): clk_list.append(line.strip('\n')) -def run_lec(reference_netlist, implementation_netlist, command_runner=CommandRunner(), temp_dir=".", log_filename="abc.lec.out", abc_exec=None): +def run_lec(reference_netlist, implementation_netlist, command_runner=CommandRunner(), temp_dir=Path("."), log_filename="abc.lec.out", abc_exec=None): """ Run Logical Equivalence Checking (LEC) between two netlists using ABC @@ -258,9 +259,10 @@ def run_lec(reference_netlist, implementation_netlist, command_runner=CommandRun ABC executable to be run """ - mkdir_p(temp_dir) + temp_dir = Path(temp_dir) if not isinstance(temp_dir, Path) else temp_dir + temp_dir.mkdir(parents=True, exist_ok=True) - if abc_exec == None: + if abc_exec is None: abc_exec = find_vtr_file('abc', is_executable=True) abc_script = 'dsec {ref} {imp}'.format(ref=reference_netlist, imp=implementation_netlist), diff --git a/vtr_flow/scripts/python_libs/vtr/ace/ace.py b/vtr_flow/scripts/python_libs/vtr/ace/ace.py index 3896da93224..8af4d157f2e 100644 --- a/vtr_flow/scripts/python_libs/vtr/ace/ace.py +++ b/vtr_flow/scripts/python_libs/vtr/ace/ace.py @@ -1,7 +1,7 @@ from vtr import find_vtr_file, verify_file,CommandRunner from pathlib import Path def run(circuit_file, old_netlist, output_netlist, output_activity_file, - command_runner=CommandRunner(), temp_dir=".", log_filename="ace.out", + command_runner=CommandRunner(), temp_dir=Path("."), log_filename="ace.out", ace_exec=None, ace_seed = 1): """ Runs ACE for activity estimation @@ -39,15 +39,15 @@ def run(circuit_file, old_netlist, output_netlist, output_activity_file, ace_seed : The ACE seed """ - + temp_dir = Path(temp_dir) if not isinstance(temp_dir, Path) else temp_dir #Verify that files are Paths or convert them to Paths and check that they exist circuit_file = verify_file(circuit_file, "Circuit") old_netlist = verify_file(old_netlist, "Previous netlist") output_netlist = verify_file(output_netlist, "Output netlist", should_exist=False) output_activity_file = verify_file(output_activity_file, "Output activity", should_exist=False) - ace_clk_file = Path(temp_dir) / "ace_clk.txt" - ace_raw = Path(temp_dir) / (circuit_file.with_suffix('').stem + ".raw.ace.blif") + ace_clk_file = temp_dir / "ace_clk.txt" + ace_raw = temp_dir / (circuit_file.with_suffix('').stem + ".raw.ace.blif") if ace_exec is None: ace_exec = find_vtr_file('ace') diff --git a/vtr_flow/scripts/python_libs/vtr/flow.py b/vtr_flow/scripts/python_libs/vtr/flow.py index 83c83a1af31..3144266e286 100644 --- a/vtr_flow/scripts/python_libs/vtr/flow.py +++ b/vtr_flow/scripts/python_libs/vtr/flow.py @@ -1,32 +1,49 @@ -import sys +""" + Module to run the VTR flow. This module calls other modules that then access the tools like VPR. +""" import shutil -import subprocess -import time -import glob -import vtr from pathlib import Path from collections import OrderedDict -from vtr import CommandError - -VTR_STAGE = vtr.make_enum("odin", "abc", 'ace', "vpr") -vtr_stages = VTR_STAGE.reverse_mapping.values() - -def run(architecture_file, circuit_file, - power_tech_file=None, - start_stage=VTR_STAGE.odin, end_stage=VTR_STAGE.vpr, - command_runner=vtr.CommandRunner(), - temp_dir="./temp", - odin_args=None, - abc_args=None, - vpr_args=None, - keep_intermediate_files=True, - keep_result_files=True, - min_hard_mult_size=3, - min_hard_adder_size=1, - check_equivalent=False, - check_incremental_sta_consistency = False, - use_old_abc_script=False, - relax_W_factor=1.3): +from enum import Enum +import vtr + +class VtrStage(Enum): + """ + Enum class for the VTR stages\ + """ + odin = 1 + abc = 2 + ace = 3 + vpr = 4 + def __le__(self, other): + if self.__class__ is other.__class__: + return int(self.value) <= other.value + return NotImplemented + def __ge__(self, other): + if self.__class__ is other.__class__: + return int(self.value) >= other.value + return NotImplemented +#pylint: disable=too-many-arguments +def run( + architecture_file, + circuit_file, + power_tech_file=None, + start_stage=VtrStage.odin, + end_stage=VtrStage.vpr, + command_runner=vtr.CommandRunner(), + temp_dir=Path("./temp"), + odin_args=None, + abc_args=None, + vpr_args=None, + keep_intermediate_files=True, + keep_result_files=True, + min_hard_mult_size=3, + min_hard_adder_size=1, + check_equivalent=False, + check_incremental_sta_consistency=False, + use_old_abc_script=False, + relax_w_factor=1.3, +): """ Runs the VTR CAD flow to map the specified circuit_file onto the target architecture_file @@ -34,246 +51,257 @@ def run(architecture_file, circuit_file, Arguments ========= - architecture_file : + architecture_file : Architecture file to target - - circuit_file : + + circuit_file : Circuit to implement - + Other Parameters ---------------- - power_tech_file : + power_tech_file : Technology power file. Enables power analysis and runs ace - - start_stage : + + start_stage : Stage of the flow to start at - - end_stage : + + end_stage : Stage of the flow to finish at - - temp_dir : + + temp_dir : Directory to run in (created if non-existent) - - command_runner : + + command_runner : A CommandRunner object used to run system commands - - verbosity : + + verbosity : Weather to output error description or not - - odin_args : - A dictionary of keyword arguments to pass on to ODIN II - - abc_args : + + odin_args : + A dictionary of keyword arguments to pass on to ODIN II + + abc_args : A dictionary of keyword arguments to pass on to ABC - - vpr_args : + + vpr_args : A dictionary of keyword arguments to pass on to VPR - - keep_intermediate_files : + + keep_intermediate_files : Determines if intermediate files are kept or deleted - - keep_result_files : + + keep_result_files : Determines if the result files are kept or deleted - - min_hard_mult_size : - Tells ODIN II the minimum multiplier size that should be implemented using hard multiplier (if available) - - min_hard_adder_size : - Tells ODIN II the minimum adder size that should be implemented using hard adder (if available). - - check_equivalent : + + min_hard_mult_size : + Tells ODIN II the minimum multiplier size that should + be implemented using hard multiplier (if available) + + min_hard_adder_size : + Tells ODIN II the minimum adder size that should be implemented + using hard adder (if available). + + check_equivalent : Enables Logical Equivalence Checks - - use_old_abc_script : + + use_old_abc_script : Enables the use of the old ABC script - - relax_W_factor : + + relax_w_factor : Factor by which to relax minimum channel width for critical path delay routing - - check_incremental_sta_consistency : + + check_incremental_sta_consistency : Do a second-run of the incremental analysis to compare the result files - + """ - if odin_args == None: - odin_args = OrderedDict() - if abc_args == None: - abc_args = OrderedDict() - if vpr_args == None: - vpr_args = OrderedDict() - + # - #Initial setup + # Initial setup # + vpr_args = OrderedDict() if not vpr_args else vpr_args + odin_args = OrderedDict() if not odin_args else odin_args + abc_args = OrderedDict() if not abc_args else abc_args - #Verify that files are Paths or convert them to Paths and check that they exist + # Verify that files are Paths or convert them to Paths and check that they exist architecture_file = vtr.util.verify_file(architecture_file, "Architecture") circuit_file = vtr.util.verify_file(circuit_file, "Circuit") - if power_tech_file != None: - power_tech_file = vtr.util.verify_file(power_tech_file, "Power tech") - - architecture_file_basename =architecture_file.name - circuit_file_basename = circuit_file.name - - circuit_name = circuit_file.stem - circuit_ext = circuit_file.suffixes - architecture_name = architecture_file.stem - - vtr.mkdir_p(temp_dir) - netlist_ext = ".blif" - if ".eblif" in circuit_ext: - netlist_ext = ".eblif" - #Define useful filenames - post_odin_netlist = Path(temp_dir) / (circuit_name + '.odin' + netlist_ext) - post_abc_netlist =Path(temp_dir) / (circuit_name + '.abc' + netlist_ext) - post_ace_netlist =Path(temp_dir) / (circuit_name + ".ace" + netlist_ext) - post_ace_activity_file = Path(temp_dir) / (circuit_name + ".act") - pre_vpr_netlist = Path(temp_dir) / (circuit_name + ".pre-vpr" + netlist_ext) - post_vpr_netlist = Path(temp_dir) / "vpr.out" #circuit_name + ".vpr.blif" - lec_base_netlist = None #Reference netlist for LEC - gen_postsynthesis_netlist = Path(temp_dir) / (circuit_name + "_post_synthesis." + netlist_ext) - rr_graph_ext=".xml" - - if "blif" in circuit_ext: - #If the user provided a .blif or .eblif netlist, we use that as the baseline for LEC - #(ABC can't LEC behavioural verilog) - lec_base_netlist = circuit_file_basename - - #Copy the circuit and architecture - circuit_copy = Path(temp_dir) / circuit_file.name - architecture_copy = Path(temp_dir) / architecture_file.name + power_tech_file = vtr.util.verify_file(power_tech_file, "Power") if power_tech_file else None + temp_dir = Path(temp_dir) + temp_dir.mkdir(parents=True, exist_ok=True) + netlist_ext = ".blif" if ".eblif" not in circuit_file.suffixes else ".eblif" + + # Define useful filenames + post_odin_netlist = temp_dir / (circuit_file.stem + ".odin" + netlist_ext) + post_abc_netlist = temp_dir / (circuit_file.stem + ".abc" + netlist_ext) + post_ace_netlist = temp_dir / (circuit_file.stem + ".ace" + netlist_ext) + post_ace_activity_file = temp_dir / (circuit_file.stem + ".act") + pre_vpr_netlist = temp_dir / (circuit_file.stem + ".pre-vpr" + netlist_ext) + post_vpr_netlist = temp_dir / "vpr.out" + + # If the user provided a .blif or .eblif netlist, we use that as the baseline for LEC + # (ABC can't LEC behavioural verilog) + lec_base_netlist = circuit_file.name if "blif" in circuit_file.suffixes else None + # Reference netlist for LEC + + gen_postsynthesis_netlist = temp_dir / ( + circuit_file.stem + "_post_synthesis." + netlist_ext + ) + + # Copy the circuit and architecture + circuit_copy = temp_dir / circuit_file.name + architecture_copy = temp_dir / architecture_file.name shutil.copy(str(circuit_file), str(circuit_copy)) shutil.copy(str(architecture_file), str(architecture_copy)) - - #There are multiple potential paths for the netlist to reach a tool - #We initialize it here to the user specified circuit and let downstream - #stages update it + # There are multiple potential paths for the netlist to reach a tool + # We initialize it here to the user specified circuit and let downstream + # stages update it next_stage_netlist = circuit_copy # # RTL Elaboration & Synthesis # - if should_run_stage(VTR_STAGE.odin, start_stage, end_stage): - if circuit_ext != ".blif": - vtr.odin.run(architecture_copy, next_stage_netlist, - output_netlist=post_odin_netlist, - command_runner=command_runner, - temp_dir=temp_dir, - odin_args=odin_args, - min_hard_mult_size=min_hard_mult_size, - min_hard_adder_size=min_hard_adder_size) - - next_stage_netlist = post_odin_netlist - - if not lec_base_netlist: - lec_base_netlist = post_odin_netlist + if should_run_stage(VtrStage.odin, start_stage, end_stage) and circuit_file.suffixes != ".blif": + vtr.odin.run( + architecture_copy, + next_stage_netlist, + output_netlist=post_odin_netlist, + command_runner=command_runner, + temp_dir=temp_dir, + odin_args=odin_args, + min_hard_mult_size=min_hard_mult_size, + min_hard_adder_size=min_hard_adder_size, + ) + + next_stage_netlist = post_odin_netlist + + lec_base_netlist = post_odin_netlist if not lec_base_netlist else lec_base_netlist # # Logic Optimization & Technology Mapping # - if should_run_stage(VTR_STAGE.abc, start_stage, end_stage): - vtr.abc.run(architecture_copy, next_stage_netlist, - output_netlist=post_abc_netlist, - command_runner=command_runner, - temp_dir=temp_dir, - abc_args=abc_args, - keep_intermediate_files=keep_intermediate_files, - use_old_abc_script=use_old_abc_script) + if should_run_stage(VtrStage.abc, start_stage, end_stage): + vtr.abc.run( + architecture_copy, + next_stage_netlist, + output_netlist=post_abc_netlist, + command_runner=command_runner, + temp_dir=temp_dir, + abc_args=abc_args, + keep_intermediate_files=keep_intermediate_files, + use_old_abc_script=use_old_abc_script, + ) next_stage_netlist = post_abc_netlist - - if not lec_base_netlist: - lec_base_netlist = post_abc_netlist - + lec_base_netlist = post_abc_netlist if not lec_base_netlist else lec_base_netlist # # Power Activity Estimation # if power_tech_file: - #The user provided a tech file, so do power analysis - - if should_run_stage(VTR_STAGE.ace, start_stage, end_stage): - vtr.ace.run(next_stage_netlist, old_netlist = post_odin_netlist, output_netlist=post_ace_netlist, - output_activity_file=post_ace_activity_file, - command_runner=command_runner, - temp_dir=temp_dir) + # The user provided a tech file, so do power analysis + + if should_run_stage(VtrStage.ace, start_stage, end_stage): + vtr.ace.run( + next_stage_netlist, + old_netlist=post_odin_netlist, + output_netlist=post_ace_netlist, + output_activity_file=post_ace_activity_file, + command_runner=command_runner, + temp_dir=temp_dir, + ) if not keep_intermediate_files: next_stage_netlist.unlink() post_odin_netlist.unlink() - #Use ACE's output netlist + # Use ACE's output netlist next_stage_netlist = post_ace_netlist + lec_base_netlist = post_ace_netlist if not lec_base_netlist else lec_base_netlist - if not lec_base_netlist: - lec_base_netlist = post_ace_netlist - - #Enable power analysis in VPR + # Enable power analysis in VPR vpr_args["power"] = True vpr_args["tech_properties"] = str(power_tech_file.resolve()) # # Pack/Place/Route # - if should_run_stage(VTR_STAGE.vpr, start_stage, end_stage): - #Copy the input netlist for input to vpr + if should_run_stage(VtrStage.vpr, start_stage, end_stage): + # Copy the input netlist for input to vpr shutil.copyfile(str(next_stage_netlist), str(pre_vpr_netlist)) - route_fixed_W = "route_chan_width" in vpr_args - if ("route" in vpr_args or "place" in vpr_args) and not route_fixed_W: + route_fixed_w = "route_chan_width" in vpr_args + if ("route" in vpr_args or "place" in vpr_args) and not route_fixed_w: vpr_args["route_chan_width"] = 300 - route_fixed_W = True - - if route_fixed_W: - #The User specified a fixed channel width - vtr.vpr.run(architecture_copy, pre_vpr_netlist, circuit_copy.stem, - output_netlist=post_vpr_netlist, - command_runner=command_runner, - temp_dir=temp_dir, - vpr_args=vpr_args, - rr_graph_ext=rr_graph_ext) + route_fixed_w = True + + if route_fixed_w: + # The User specified a fixed channel width + vtr.vpr.run( + architecture_copy, + pre_vpr_netlist, + circuit_copy.stem, + output_netlist=post_vpr_netlist, + command_runner=command_runner, + temp_dir=temp_dir, + vpr_args=vpr_args, + rr_graph_ext=".xml", + ) else: - #First find minW and then re-route at a relaxed W - vtr.vpr.run_relax_W(architecture_copy, pre_vpr_netlist, circuit_copy.stem, - output_netlist=post_vpr_netlist, - command_runner=command_runner, - relax_W_factor=relax_W_factor, - temp_dir=temp_dir, - vpr_args=vpr_args) - - if not lec_base_netlist: - lec_base_netlist = pre_vpr_netlist + # First find minW and then re-route at a relaxed W + vtr.vpr.run_relax_W( + architecture_copy, + pre_vpr_netlist, + circuit_copy.stem, + output_netlist=post_vpr_netlist, + command_runner=command_runner, + relax_w_factor=relax_w_factor, + temp_dir=temp_dir, + vpr_args=vpr_args, + ) + lec_base_netlist = pre_vpr_netlist if not lec_base_netlist else lec_base_netlist # # Logical Equivalence Checks (LEC) # if check_equivalent: - for file in Path(temp_dir).iterdir(): - if "post_synthesis.blif" in str(file) : - gen_postsynthesis_netlist = file.name - break - vtr.abc.run_lec(lec_base_netlist, gen_postsynthesis_netlist, command_runner=command_runner, temp_dir=temp_dir) + vtr.abc.run_lec( + lec_base_netlist, + gen_postsynthesis_netlist, + command_runner=command_runner, + temp_dir=temp_dir, + ) # Do a second-run of the incremental analysis to compare the result files if check_incremental_sta_consistency: - vtr.vpr.cmp_full_vs_incr_STA(architecture_copy, pre_vpr_netlist, circuit_copy.stem, - command_runner=command_runner, - vpr_args=vpr_args, - rr_graph_ext=rr_graph_ext, - temp_dir=temp_dir - ) - + vtr.vpr.cmp_full_vs_incr_STA( + architecture_copy, + pre_vpr_netlist, + circuit_copy.stem, + command_runner=command_runner, + vpr_args=vpr_args, + rr_graph_ext=".xml", + temp_dir=temp_dir, + ) + if not keep_intermediate_files: - next_stage_netlist.unlink() - exts = ('.xml','.sdf','.v') - if not keep_result_files: - exts += ('.net', '.place', '.route') + delete_intermediate_files(next_stage_netlist, post_ace_activity_file, + keep_result_files, temp_dir, power_tech_file) +#pylint: enable=too-many-arguments - for file in Path(temp_dir).iterdir(): - if file.suffix in exts: - file.unlink() +def delete_intermediate_files(next_stage_netlist, post_ace_activity_file, + keep_result_files, temp_dir, power_tech_file): + """ + delete intermediate files + """ + next_stage_netlist.unlink() + exts = (".xml", ".sdf", ".v") + exts += (".net", ".place", ".route") if not keep_result_files else None - if power_tech_file: - post_ace_activity_file.unlink() + for file in temp_dir.iterdir(): + if file.suffix in exts: + file.unlink() + + if power_tech_file: + post_ace_activity_file.unlink() def should_run_stage(stage, flow_start_stage, flow_end_stage): """ @@ -282,7 +310,3 @@ def should_run_stage(stage, flow_start_stage, flow_end_stage): if flow_start_stage <= stage <= flow_end_stage: return True return False - - - - diff --git a/vtr_flow/scripts/python_libs/vtr/inspect.py b/vtr_flow/scripts/python_libs/vtr/inspect.py index 8561f7cf378..15c8eef64b8 100644 --- a/vtr_flow/scripts/python_libs/vtr/inspect.py +++ b/vtr_flow/scripts/python_libs/vtr/inspect.py @@ -78,11 +78,11 @@ def max_value(self): return self._max_value def check_passed(self, golden_value, check_value): - if golden_value == None and check_value == None: + if golden_value is None and check_value is None: return True, "both golden and check are None" - elif golden_value == None and check_value != None: + elif golden_value is None and check_value != None: return False, "golden value is None, but check value is {}".format(check_value) - elif golden_value != None and check_value == None: + elif golden_value != None and check_value is None: return False, "golden value is {}, but check value is None".format(golden_value) assert golden_value != None diff --git a/vtr_flow/scripts/python_libs/vtr/odin/odin.py b/vtr_flow/scripts/python_libs/vtr/odin/odin.py index d0b1472be77..d9a793aedc9 100644 --- a/vtr_flow/scripts/python_libs/vtr/odin/odin.py +++ b/vtr_flow/scripts/python_libs/vtr/odin/odin.py @@ -1,11 +1,11 @@ import shutil from pathlib import Path -from vtr import mkdir_p, find_vtr_file, file_replace, determine_memory_addr_width, verify_file, CommandRunner +from vtr import find_vtr_file, file_replace, determine_memory_addr_width, verify_file, CommandRunner def run(architecture_file, circuit_file, output_netlist, command_runner=CommandRunner(), - temp_dir=".", + temp_dir=Path("."), odin_args="--adder_type default", log_filename="odin.out", odin_exec=None, @@ -55,9 +55,10 @@ def run(architecture_file, circuit_file, Tells ODIN II the minimum adder size that should be implemented using hard adder (if available). """ - mkdir_p(temp_dir) + temp_dir = Path(temp_dir) if not isinstance(temp_dir, Path) else temp_dir + temp_dir.mkdir(parents=True, exist_ok=True) - if odin_args == None: + if odin_args is None: odin_args = OrderedDict() #Verify that files are Paths or convert them to Paths and check that they exist @@ -65,15 +66,15 @@ def run(architecture_file, circuit_file, circuit_file = verify_file(circuit_file, "Circuit") output_netlist = verify_file(output_netlist, "Output netlist", False) - if odin_exec == None: + if odin_exec is None: odin_exec = find_vtr_file('odin_II', is_executable=True) - if odin_config == None: + if odin_config is None: odin_base_config = find_vtr_file('basic_odin_config_split.xml') #Copy the config file odin_config = "odin_config.xml" - odin_config_full_path = str(Path(temp_dir) / odin_config) + odin_config_full_path = str(temp_dir / odin_config) shutil.copyfile(odin_base_config, odin_config_full_path) #Update the config file @@ -109,7 +110,7 @@ def run(architecture_file, circuit_file, command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) if(use_odin_simulation): - sim_dir = Path(temp_dir) / "simulation_init" + sim_dir = temp_dir / "simulation_init" sim_dir.mkdir() cmd = [odin_exec, "-b", output_netlist.name, -a, architecture_file.name, "-sim_dir", str(sim_dir),"-g", "100", "--best_coverage", "-U0"] command_runner.run_system_command(cmd,temp_dir=temp_dir, log_filename="sim_produce_vector.out",indent_depth=1) \ No newline at end of file diff --git a/vtr_flow/scripts/python_libs/vtr/util.py b/vtr_flow/scripts/python_libs/vtr/util.py index 669bdc825e0..aa657d7ff9e 100644 --- a/vtr_flow/scripts/python_libs/vtr/util.py +++ b/vtr_flow/scripts/python_libs/vtr/util.py @@ -38,9 +38,9 @@ def __init__(self, timeout_sec=None, max_memory_mb=None, track_memory=True, verb indent: The string specifying a single indent (used in verbose mode) valgrind: Indicates if commands should be run with valgrind """ - if verbose_error==None: + if verbose_error is None: verbose_error = verbose - if echo_cmd == None: + if echo_cmd is None: echo_cmd = verbose self._timeout_sec = timeout_sec @@ -71,9 +71,10 @@ def run_system_command(self, cmd, temp_dir, log_filename=None, expected_return_c """ #Save the original command orig_cmd = cmd + temp_dir = Path(temp_dir) if not isinstance(temp_dir, Path) else temp_dir #If no log file is specified the name is based on the executed command - if log_filename == None: + if log_filename is None: log_filename = PurePath(orig_cmd[0]).name + '.out' @@ -114,7 +115,7 @@ def run_system_command(self, cmd, temp_dir, log_filename=None, expected_return_c stdout=subprocess.PIPE, #We grab stdout stderr=stderr, #stderr redirected to stderr universal_newlines=True, #Lines always end in \n - cwd=temp_dir, #Where to run the command + cwd=str(temp_dir), #Where to run the command ) # Read the output line-by-line and log it @@ -122,7 +123,7 @@ def run_system_command(self, cmd, temp_dir, log_filename=None, expected_return_c # # We do this rather than use proc.communicate() # to get interactive output - with (Path(temp_dir) / log_filename).open('w') as log_f: + with (temp_dir / log_filename).open('w') as log_f: #Print the command at the top of the log log_f.write(" ".join(cmd)) log_f.write("\n") @@ -148,7 +149,7 @@ def run_system_command(self, cmd, temp_dir, log_filename=None, expected_return_c finally: #Clean-up if we did not exit cleanly if proc: - if proc.returncode == None: + if proc.returncode is None: #Still running, stop it proc.terminate() @@ -165,7 +166,7 @@ def run_system_command(self, cmd, temp_dir, log_filename=None, expected_return_c if (self._show_failures and cmd_errored): raise CommandError(msg="Executable {exec_name} failed".format(exec_name=PurePath(orig_cmd[0]).name), cmd=cmd, - log=str(PurePath(temp_dir).joinpath(log_filename)), + log=str(temp_dir / log_filename), returncode=cmd_returncode) return cmd_output, cmd_returncode @@ -223,7 +224,7 @@ def load_tab_delimited_csv(filepath): header = None for csv_row in reader: - if header == None: + if header is None: header = [val.strip() for val in csv_row] else: data_row = OrderedDict() @@ -235,30 +236,6 @@ def load_tab_delimited_csv(filepath): return data -def make_enum(*sequential, **named): - """ - Makes an enumeration - - >> MY_ENUM = enum(ONE, TWO, THREE) - """ - enums = dict(zip(sequential, range(len(sequential))), **named) - reverse = dict((value, key) for key, value in enums.items()) - enums['reverse_mapping'] = reverse - return type('Enum', (), enums) - -def mkdir_p(path): - """ - Makes a directory (including parents) at the specified path - """ - make_path = Path(path) - try: - make_path.mkdir(parents=True) - except OSError as exc: # Python >2.5 - if exc.errno == errno.EEXIST and make_path.is_dir(): - pass - else: - raise - def print_verbose(min_verbosity, curr_verbosity, string, endl=True): """ Print string if curr_verbosity is gteq min_verbosity @@ -470,7 +447,7 @@ def get_latest_run_dir(base_dir): """ latest_run_number = get_latest_run_number(base_dir) - if latest_run_number == None: + if latest_run_number is None: return None return str(PurePath(base_dir) / run_dir_name(latest_run_number)) @@ -481,7 +458,7 @@ def get_next_run_number(base_dir): """ latest_run_number = get_latest_run_number(base_dir) - if latest_run_number == None: + if latest_run_number is None: next_run_number = 0 else: next_run_number = latest_run_number + 1 diff --git a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py index 9ad780f9deb..a7ec8377205 100644 --- a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py +++ b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py @@ -1,10 +1,10 @@ import shutil from pathlib import Path -from vtr import mkdir_p, find_vtr_file, CommandRunner, print_verbose, relax_W, determine_lut_size, determine_min_W, verify_file +from vtr import find_vtr_file, CommandRunner, print_verbose, relax_W, determine_lut_size, determine_min_W, verify_file from vtr.error import InspectError -def run_relax_W(architecture, circuit, circuit_name=None, command_runner=CommandRunner(), temp_dir=".", - relax_W_factor=1.3, vpr_exec=None, logfile_base="vpr", +def run_relax_W(architecture, circuit, circuit_name=None, command_runner=CommandRunner(), temp_dir=Path("."), + relax_w_factor=1.3, vpr_exec=None, logfile_base="vpr", vpr_args=None, output_netlist=None): """ Runs VPR twice: @@ -34,7 +34,7 @@ def run_relax_W(architecture, circuit, circuit_name=None, command_runner=Command temp_dir: Directory to run in - relax_W_factor: + relax_w_factor: Factor by which to relax minimum channel width for critical path delay routing vpr_exec: @@ -51,8 +51,8 @@ def run_relax_W(architecture, circuit, circuit_name=None, command_runner=Command """ if vpr_args is None: vpr_args = OrderedDict() - - mkdir_p(temp_dir) + temp_dir = Path(temp_dir) if not isinstance(temp_dir, Path) else temp_dir + temp_dir.mkdir(parents=True, exist_ok=True) #Verify that files are Paths or convert them to Paths and check that they exist architecture = verify_file(architecture, "Architecture") @@ -75,7 +75,7 @@ def run_relax_W(architecture, circuit, circuit_name=None, command_runner=Command if "route" in vpr_args: del vpr_args["route"] - if vpr_exec == None: + if vpr_exec is None: vpr_exec = find_vtr_file('vpr', is_executable=True) run(architecture, circuit, circuit_name, command_runner, temp_dir, log_filename=vpr_min_W_log, vpr_exec=vpr_exec, vpr_args=vpr_args) @@ -85,9 +85,9 @@ def run_relax_W(architecture, circuit, circuit_name=None, command_runner=Command return if max_router_iterations: vpr_args["max_router_iterations"]=max_router_iterations - min_W = determine_min_W(str(Path(temp_dir) / vpr_min_W_log)) + min_W = determine_min_W(str(temp_dir / vpr_min_W_log)) - relaxed_W = relax_W(min_W, relax_W_factor) + relaxed_W = relax_W(min_W, relax_w_factor) vpr_args['route'] = True #Re-route only vpr_args['route_chan_width'] = relaxed_W #At a fixed channel width @@ -98,7 +98,7 @@ def run_relax_W(architecture, circuit, circuit_name=None, command_runner=Command run(architecture, circuit, circuit_name, command_runner, temp_dir, log_filename=vpr_relaxed_W_log, vpr_exec=vpr_exec, vpr_args=vpr_args, check_for_second_run=False) -def run(architecture, circuit, circuit_name=None, command_runner=CommandRunner(), temp_dir=".", output_netlist=None, +def run(architecture, circuit, circuit_name=None, command_runner=CommandRunner(), temp_dir=Path("."), output_netlist=None, log_filename="vpr.out", vpr_exec=None, vpr_args=None,check_for_second_run=True,rr_graph_ext=".xml",): """ Runs VPR with the specified configuration @@ -143,10 +143,10 @@ def run(architecture, circuit, circuit_name=None, command_runner=CommandRunner() if vpr_args is None: vpr_args = OrderedDict() + temp_dir = Path(temp_dir) if not isinstance(temp_dir, Path) else temp_dir + temp_dir.mkdir(parents=True, exist_ok=True) - mkdir_p(temp_dir) - - if vpr_exec == None: + if vpr_exec is None: vpr_exec = find_vtr_file('vpr', is_executable=True) #Verify that files are Paths or convert them to Paths and check that they exist @@ -224,7 +224,7 @@ def run(architecture, circuit, circuit_name=None, command_runner=CommandRunner() if diff_result: raise InspectError("failed: vpr (RR Graph XML output not consistent when reloaded)") -def cmp_full_vs_incr_STA(architecture,circuit,circuit_name=None,command_runner=CommandRunner(),vpr_args=None,rr_graph_ext=".xml",temp_dir=".",vpr_exec=None): +def cmp_full_vs_incr_STA(architecture,circuit,circuit_name=None,command_runner=CommandRunner(),vpr_args=None,rr_graph_ext=".xml",temp_dir=Path("."),vpr_exec=None): """ Sanity check that full STA and the incremental STA produce the same *.net, *.place, *.route files as well as identical timing report files diff --git a/vtr_flow/scripts/run_vtr_flow.py b/vtr_flow/scripts/run_vtr_flow.py index f6bdae65727..211a9828b4b 100755 --- a/vtr_flow/scripts/run_vtr_flow.py +++ b/vtr_flow/scripts/run_vtr_flow.py @@ -1,63 +1,57 @@ #!/usr/bin/env python3 +""" + Module to run the VTR Flow +""" import sys from pathlib import Path -import errno import argparse -import subprocess -import time import shutil -import re import textwrap import socket from datetime import datetime - from collections import OrderedDict - -sys.path.insert(0, str(Path(__file__).resolve().parent / 'python_libs')) +#pylint: disable=wrong-import-position, import-error +sys.path.insert(0, str(Path(__file__).resolve().parent / "python_libs")) import vtr -from vtr import mkdir_p - +#pylint: enable=wrong-import-position, import-error BASIC_VERBOSITY = 1 - +#pylint: disable=too-few-public-methods class VtrStageArgparseAction(argparse.Action): + """ + Class to parse the VTR stages to begin and end at. + """ + def __call__(self, parser, namespace, value, option_string=None): if value == "odin": - setattr(namespace, self.dest, vtr.VTR_STAGE.odin) + setattr(namespace, self.dest, vtr.VtrStage.odin) elif value == "abc": - setattr(namespace, self.dest, vtr.VTR_STAGE.abc) + setattr(namespace, self.dest, vtr.VtrStage.abc) elif value == "vpr": - setattr(namespace, self.dest, vtr.VTR_STAGE.vpr) + setattr(namespace, self.dest, vtr.VtrStage.vpr) elif value == "lec": - setattr(namespace, self.dest, vtr.VTR_STAGE.lec) + setattr(namespace, self.dest, vtr.VtrStage.lec) else: raise argparse.ArgumentError(self, "Invalid VTR stage '" + value + "'") - -class OnOffArgparseAction(argparse.Action): - def __call__(self, parser, namespace, value, option_string=None): - if value.lower() == "on": - setattr(namespace, self.dest, True) - elif value.lower() == "off": - setattr(namespace, self.dest, False) - else: - raise argparse.ArgumentError(self, "Invalid yes/no value '" + value + "'") - -on_off_choices = ['on', 'off'] +#pylint: enable=too-few-public-methods def vtr_command_argparser(prog=None): + """ + The VTR command arg parser + """ usage = "%(prog)s circuit_file architecture_file [options]" description = textwrap.dedent( - """ + """ Runs a single architecture and circuit through the VTR CAD flow - mapping the circuit onto the target archietcture. Any unrecognzied arguments are passed to VPR. """ - ) + ) epilog = textwrap.dedent( - """ + """ Examples -------- @@ -70,8 +64,9 @@ def vtr_command_argparser(prog=None): %(prog)s arch.xml circuit.v --route_chan_width 100 - Passing '--route_chan_width 100' will force VPR to route at a fixed channel width, - rather then attempting to find the minimum channel width (VPR's default behaviour). + Passing '--route_chan_width 100' will force VPR to route at a fixed channel + width, rather then attempting to find the minimum channel width + (VPR's default behaviour). Run in the directory 'test' (rather than in a temporary directory): @@ -84,437 +79,582 @@ def vtr_command_argparser(prog=None): %(prog)s arch.xml circuit.v -power_tech power_tech.xml - Only run a specific stage (assumes required results have already been generated): + Only run specific stage (assumes required results have already been generated): %(prog)s arch.xml circuit.blif -start vpr -end vpr """ - ) + ) parser = argparse.ArgumentParser( - prog=prog, - usage=usage, - description=description, - epilog=epilog, - formatter_class=vtr.RawDefaultHelpFormatter, - ) + prog=prog, + usage=usage, + description=description, + epilog=epilog, + formatter_class=vtr.RawDefaultHelpFormatter, + ) # # Major arguments # - parser.add_argument('circuit_file', - help="The circuit to map to the target architecture.") - parser.add_argument('architecture_file', - help="The FPGA architecture to target.") - parser.add_argument("-start", "-starting_stage", - choices=vtr.VTR_STAGE.reverse_mapping.values(), - default=vtr.VTR_STAGE.odin, - action=VtrStageArgparseAction, - help="Starting stage of the VTR flow.") - parser.add_argument("-delete_intermediate_files", - default=True, - action="store_false", - dest="keep_intermediate_files", - help="Determines if intermediate files are deleted.") - parser.add_argument("-delete_result_files", - default=True, - action="store_false", - dest="keep_result_files", - help="Determines if result files are deleted.") - parser.add_argument("-end", "-ending_stage", - choices=vtr.VTR_STAGE.reverse_mapping.values(), - default=vtr.VTR_STAGE.vpr, - action=VtrStageArgparseAction, - help="Ending stage of the VTR flow.") - - parser.add_argument("-verbose", - default=False, - action="store_true", - dest="verbose", - help="Verbosity of the script.") - + parser.add_argument( + "circuit_file", help="The circuit to map to the target architecture." + ) + parser.add_argument("architecture_file", help="The FPGA architecture to target.") + parser.add_argument( + "-start", + "-starting_stage", + choices=str(list(vtr.VtrStage)), + default=vtr.VtrStage.odin, + action=VtrStageArgparseAction, + help="Starting stage of the VTR flow.", + ) + parser.add_argument( + "-delete_intermediate_files", + default=True, + action="store_false", + dest="keep_intermediate_files", + help="Determines if intermediate files are deleted.", + ) + parser.add_argument( + "-delete_result_files", + default=True, + action="store_false", + dest="keep_result_files", + help="Determines if result files are deleted.", + ) + parser.add_argument( + "-end", + "-ending_stage", + choices=str(list(vtr.VtrStage)), + default=vtr.VtrStage.vpr, + action=VtrStageArgparseAction, + help="Ending stage of the VTR flow.", + ) + + parser.add_argument( + "-verbose", + default=False, + action="store_true", + dest="verbose", + help="Verbosity of the script.", + ) # # Power arguments # - power = parser.add_argument_group("Power", description="Power Analysis Related Options") - power.add_argument("-power", - default=False, - action="store_true", - dest="do_power", - help="Track the memory usage for each stage. Requires /usr/bin/time -v, disabled if not available.") - power.add_argument("-cmos_tech", - default=None, - dest="power_tech", - metavar="POWER_TECH_FILE", - help="Enables power analysis using the specified technology file. Power analysis is disabled if not specified.") + power = parser.add_argument_group( + "Power", description="Power Analysis Related Options" + ) + power.add_argument( + "-power", + default=False, + action="store_true", + dest="do_power", + help="Track the memory usage for each stage. Requires /usr/bin/time -v," + + " disabled if not available.", + ) + power.add_argument( + "-cmos_tech", + default=None, + dest="power_tech", + metavar="POWER_TECH_FILE", + help="Enables power analysis using the specified technology file." + + " Power analysis is disabled if not specified.", + ) # # House keeping arguments # - house_keeping = parser.add_argument_group("House Keeping", description="Configuration related to how files/time/memory are used by the script.") - - house_keeping.add_argument("-temp_dir", - default=None, - help="Directory to run the flow in (will be created if non-existant).") - - house_keeping.add_argument("-name", - default=None, - help="Name for this run to be output.") - - house_keeping.add_argument("-track_memory_usage", - default=False, - action="store_true", - dest="track_memory_usage", - help="Track the memory usage for each stage. Requires /usr/bin/time -v, disabled if not available.") - - house_keeping.add_argument("-show_failures", - default=False, - action="store_true", - dest="show_failures", - help="Tells the flow to display failures in the console.") - - house_keeping.add_argument("-limit_memory_usage", - default=None, - metavar="MAX_MEMORY_MB", - help="Specifies the maximum memory usable by any stage. " - "Not supported on some platforms (requires ulimit).") - - house_keeping.add_argument("-timeout", - default=14*24*60*60, #14 days - type=float, - metavar="TIMEOUT_SECONDS", - help="Maximum time in seconds to spend on a single stage.") - house_keeping.add_argument("-expect_fail", - default=None, - type=str, - help="Informs VTR that this run is expected to fail with this message.") - house_keeping.add_argument("-valgrind", - default=False, - action="store_true", - help="Runs the flow with valgrind with the following options (--leak-check=full, --errors-for-leak-kinds=none, --error-exitcode=1, --track-origins=yes)") + house_keeping = parser.add_argument_group( + "House Keeping", + description="Configuration related to how files/time/memory are used by the script.", + ) + + house_keeping.add_argument( + "-temp_dir", + default=None, + help="Directory to run the flow in (will be created if non-existant).", + ) + + house_keeping.add_argument( + "-name", default=None, help="Name for this run to be output." + ) + + house_keeping.add_argument( + "-track_memory_usage", + default=False, + action="store_true", + dest="track_memory_usage", + help="Track the memory usage for each stage." + + " Requires /usr/bin/time -v, disabled if not available.", + ) + + house_keeping.add_argument( + "-show_failures", + default=False, + action="store_true", + dest="show_failures", + help="Tells the flow to display failures in the console.", + ) + + house_keeping.add_argument( + "-limit_memory_usage", + default=None, + metavar="MAX_MEMORY_MB", + help="Specifies the maximum memory usable by any stage. " + "Not supported on some platforms (requires ulimit).", + ) + + house_keeping.add_argument( + "-timeout", + default=14 * 24 * 60 * 60, # 14 days + type=float, + metavar="TIMEOUT_SECONDS", + help="Maximum time in seconds to spend on a single stage.", + ) + house_keeping.add_argument( + "-expect_fail", + default=None, + type=str, + help="Informs VTR that this run is expected to fail with this message.", + ) + house_keeping.add_argument( + "-valgrind", + default=False, + action="store_true", + help="Runs the flow with valgrind with the following options (--leak-check=full," + + " --errors-for-leak-kinds=none, --error-exitcode=1, --track-origins=yes)", + ) # # ABC arguments # abc = parser.add_argument_group("abc", description="Arguments used by ABC") - abc.add_argument("-iterative_bb", - default=False, - action="store_true", - dest="iterative_bb", - help="Use iterative black-boxing flow for multi-clock circuits") - abc.add_argument("-once_bb", - default=False, - action="store_true", - dest="once_bb", - help="Use the black-boxing flow a single time") - abc.add_argument("-blanket_bb", - default=False, - action="store_true", - dest="blanket_bb", - help="Use iterative black-boxing on all the clocks") - abc.add_argument("-use_old_latches_restoration_script", - default=False, - action="store_true", - dest="use_old_latches_restoration_script", - help="Use the old latches restoration script") - abc.add_argument("-check_equivalent", - default=False, - action="store_true", - help="Enables Logical Equivalence Checks") - abc.add_argument("-use_old_abc_script", - default=False, - action="store_true", - help="Enables use of legacy ABC script adapted for new ABC") - abc.add_argument("-lut_size", - type=int, - help="Tells ABC the LUT size of the FPGA architecture") + abc.add_argument( + "-iterative_bb", + default=False, + action="store_true", + dest="iterative_bb", + help="Use iterative black-boxing flow for multi-clock circuits", + ) + abc.add_argument( + "-once_bb", + default=False, + action="store_true", + dest="once_bb", + help="Use the black-boxing flow a single time", + ) + abc.add_argument( + "-blanket_bb", + default=False, + action="store_true", + dest="blanket_bb", + help="Use iterative black-boxing on all the clocks", + ) + abc.add_argument( + "-use_old_latches_restoration_script", + default=False, + action="store_true", + dest="use_old_latches_restoration_script", + help="Use the old latches restoration script", + ) + abc.add_argument( + "-check_equivalent", + default=False, + action="store_true", + help="Enables Logical Equivalence Checks", + ) + abc.add_argument( + "-use_old_abc_script", + default=False, + action="store_true", + help="Enables use of legacy ABC script adapted for new ABC", + ) + abc.add_argument( + "-lut_size", type=int, help="Tells ABC the LUT size of the FPGA architecture" + ) # # ODIN II arguments # - odin = parser.add_argument_group("Odin", description="Arguments to be passed to ODIN II") - odin.add_argument("-adder_type", - default="default", - help="Tells ODIN II the adder type used in this configuration") - odin.add_argument("-adder_cin_global", - default=False, - action="store_true", - dest="adder_cin_global", - help="Tells ODIN II to connect the first cin in an adder/subtractor chain to a global gnd/vdd net.") - odin.add_argument("-disable_odin_xml", - default=False, - action="store_true", - dest="disable_odin_xml", - help="Disables the ODIN xml") - odin.add_argument("-use_odin_simulation", - default=False, - action="store_true", - dest="use_odin_simulation", - help="Tells odin to run simulation.") - odin.add_argument("-min_hard_mult_size", - default=3, - type=int, - metavar="min_hard_mult_size", - help="Tells ODIN II the minimum multiplier size that should be implemented using hard multiplier (if available).") - odin.add_argument("-min_hard_adder_size", - default=1, - type=int, - help="Tells ODIN II the minimum adder size that should be implemented using hard adder (if available).") + odin = parser.add_argument_group( + "Odin", description="Arguments to be passed to ODIN II" + ) + odin.add_argument( + "-adder_type", + default="default", + help="Tells ODIN II the adder type used in this configuration", + ) + odin.add_argument( + "-adder_cin_global", + default=False, + action="store_true", + dest="adder_cin_global", + help="Tells ODIN II to connect the first cin in an adder/subtractor" + + "chain to a global gnd/vdd net.", + ) + odin.add_argument( + "-disable_odin_xml", + default=False, + action="store_true", + dest="disable_odin_xml", + help="Disables the ODIN xml", + ) + odin.add_argument( + "-use_odin_simulation", + default=False, + action="store_true", + dest="use_odin_simulation", + help="Tells odin to run simulation.", + ) + odin.add_argument( + "-min_hard_mult_size", + default=3, + type=int, + metavar="min_hard_mult_size", + help="Tells ODIN II the minimum multiplier size to be implemented using hard multiplier.", + ) + odin.add_argument( + "-min_hard_adder_size", + default=1, + type=int, + help="Tells ODIN II the minimum adder size that should be implemented using hard adder.", + ) # # VPR arguments # - vpr = parser.add_argument_group("Vpr", description="Arguments to be parsed and then passed to VPR") - vpr.add_argument("-crit_path_router_iterations", - type=int, - default=150, - help="Tells VPR the amount of iterations allowed to obtain the critical path.") - vpr.add_argument("-fix_pins", - type=str, - help="Controls how the placer handles I/O pads during placement.") - vpr.add_argument("-relax_W_factor", - type=float, - default=1.3, - help="Factor by which to relax minimum channel width for critical path delay routing") - vpr.add_argument("-verify_rr_graph", - default=False, - action="store_true", - help="Tells VPR to verify the routing resource graph.") - vpr.add_argument("-rr_graph_ext", - default=".xml", - type=str, - help="Determines the output rr_graph files' extention.") - vpr.add_argument("-check_route", - default=False, - action="store_true", - help="Tells VPR to run final analysis stage.") - vpr.add_argument("-check_place", - default=False, - action="store_true", - help="Tells VPR to run routing stage") - vpr.add_argument("-sdc_file", - default=None, - type=str, - help="Path to SDC timing constraints file.") - vpr.add_argument("-check_incremental_sta_consistency", - default=False, - action="store_true", - help="Do a second-run of the incremental analysis to compare the result files") - + vpr = parser.add_argument_group( + "Vpr", description="Arguments to be parsed and then passed to VPR" + ) + vpr.add_argument( + "-crit_path_router_iterations", + type=int, + default=150, + help="Tells VPR the amount of iterations allowed to obtain the critical path.", + ) + vpr.add_argument( + "-fix_pins", + type=str, + help="Controls how the placer handles I/O pads during placement.", + ) + vpr.add_argument( + "-relax_w_factor", + type=float, + default=1.3, + help="Factor by which to relax minimum channel width for critical path delay routing", + ) + vpr.add_argument( + "-verify_rr_graph", + default=False, + action="store_true", + help="Tells VPR to verify the routing resource graph.", + ) + vpr.add_argument( + "-rr_graph_ext", + default=".xml", + type=str, + help="Determines the output rr_graph files' extention.", + ) + vpr.add_argument( + "-check_route", + default=False, + action="store_true", + help="Tells VPR to run final analysis stage.", + ) + vpr.add_argument( + "-check_place", + default=False, + action="store_true", + help="Tells VPR to run routing stage", + ) + vpr.add_argument( + "-sdc_file", default=None, type=str, help="Path to SDC timing constraints file." + ) + vpr.add_argument( + "-check_incremental_sta_consistency", + default=False, + action="store_true", + help="Do a second-run of the incremental analysis to compare the result files", + ) + return parser + def main(): - vtr_command_main(sys.argv[1:],prog = sys.argv[0]) + """ + Main function to call vtr_command_main to run VTR + """ + return vtr_command_main(sys.argv[1:], prog=sys.argv[0]) + def vtr_command_main(arg_list, prog=None): + """ + Running VTR with the specified arguemnts. + """ start = datetime.now() - #Load the arguments + # Load the arguments args, unknown_args = vtr_command_argparser(prog).parse_known_args(arg_list) - path_arch_file = Path(args.architecture_file) - path_circuit_file = Path(args.circuit_file) error_status = "Error" - if (args.temp_dir == None): - temp_dir="./temp" + if args.temp_dir is None: + temp_dir = Path("./temp") else: - temp_dir=args.temp_dir - #Specify how command should be run - command_runner = vtr.CommandRunner(track_memory=True, - max_memory_mb=args.limit_memory_usage, - timeout_sec=args.timeout, - verbose=args.verbose, - show_failures = args.show_failures, - valgrind = args.valgrind) + temp_dir = Path(args.temp_dir) + # Specify how command should be run + command_runner = vtr.CommandRunner( + track_memory=True, + max_memory_mb=args.limit_memory_usage, + timeout_sec=args.timeout, + verbose=args.verbose, + show_failures=args.show_failures, + valgrind=args.valgrind, + ) exit_status = 0 return_status = 0 + try: + vpr_args = process_unknown_args(unknown_args) + vpr_args.update(process_vpr_args(args, prog, temp_dir)) + if args.sdc_file: + vpr_args["sdc_file"] = get_sdc_file(args.sdc_file, prog) + + print(args.name if args.name else Path(args.architecture_file).name + + "/" + + Path(args.circuit_file).name, end="\t\t") + # Run the flow + vtr.run( + Path(args.architecture_file), + Path(args.circuit_file), + power_tech_file=args.power_tech, + temp_dir=temp_dir, + start_stage=args.start, + end_stage=args.end, + command_runner=command_runner, + vpr_args=vpr_args, + abc_args=process_abc_args(args), + odin_args=process_odin_args(args), + keep_intermediate_files=args.keep_intermediate_files, + keep_result_files=args.keep_result_files, + min_hard_mult_size=args.min_hard_mult_size, + min_hard_adder_size=args.min_hard_adder_size, + check_equivalent=args.check_equivalent, + check_incremental_sta_consistency=args.check_incremental_sta_consistency, + use_old_abc_script=args.use_old_abc_script, + relax_w_factor=args.relax_w_factor, + ) + error_status = "OK" + except vtr.VtrError as error: + error_status, return_status, exit_status = except_vtr_error( + error, + args.expect_fail, + args.verbose + ) + + except KeyboardInterrupt as error: + print("{} recieved keyboard interrupt".format(prog)) + exit_status = 4 + return_status = exit_status + + finally: + seconds = datetime.now() - start + print( + "{status} (took {time})".format( + status=error_status, time=vtr.format_elapsed_time(seconds) + ) + ) + temp_dir.mkdir(parents=True, exist_ok=True) + out = temp_dir / "output.txt" + out.touch() + with out.open("w") as file: + file.write("vpr_status=") + if exit_status == 0: + file.write("success\n") + else: + file.write("exited with return code {}\n".format(exit_status)) + file.write( + "vpr_seconds=%d\nrundir=%s\nhostname=%s\nerror=" + % (seconds.total_seconds(), str(Path.cwd()), socket.gethostname()) + ) + file.write("\n") + + sys.exit(return_status) + + +def process_unknown_args(unknown_args): + """ + We convert the unknown_args into a dictionary, which is eventually + used to generate arguments for VPR + """ + vpr_args = OrderedDict() + while len(unknown_args) > 0: + # Get the first argument + arg = unknown_args.pop(0) + + if arg == "": + continue + + if not arg.startswith("-"): + raise vtr.VtrError( + "Extra argument '{}' intended for VPR does not start with '-'".format( + arg + ) + ) + + # To make it a valid kwargs dictionary we trim the initial '-' or '--' from the + # argument name + assert len(arg) >= 2 + if arg[1] == "-": + # Double-dash prefix + arg = arg[2:] + else: + # Single-dash prefix + arg = arg[1:] + + # Determine if there is a value associated with this argument + if len(unknown_args) == 0 or ( + unknown_args[0].startswith("-") and arg != "target_ext_pin_util" + ): + # Single value argument, we place these with value 'True' + # in vpr_args + vpr_args[arg] = True + else: + # Multivalue argument + val = unknown_args.pop(0) + if len(unknown_args) != 0 and not unknown_args[0].startswith("-"): + temp = val + val = [] + val.append(temp) + while len(unknown_args) != 0 and not unknown_args[0].startswith("-"): + val.append(unknown_args.pop(0)) + vpr_args[arg] = val + + return vpr_args + + +def process_abc_args(args): + """ + Finds arguments needed in the ABC stage of the flow + """ abc_args = OrderedDict() - if(args.iterative_bb): + if args.iterative_bb: abc_args["iterative_bb"] = True - if(args.once_bb): + if args.once_bb: abc_args["once_bb"] = True - if(args.blanket_bb): + if args.blanket_bb: abc_args["blanket_bb"] = True - if(args.use_old_latches_restoration_script): + if args.use_old_latches_restoration_script: abc_args["use_old_latches_restoration_script"] = True - if(args.lut_size): + if args.lut_size: abc_args["lut_size"] = args.lut_size + return abc_args + +def process_odin_args(args): + """ + Finds arguments needed in the ODIN stage of the flow + """ odin_args = OrderedDict() - odin_args["adder_type"] = args.adder_type + odin_args["adder_type"] = args.adder_type - if(args.adder_cin_global): + if args.adder_cin_global: odin_args["adder_cin_global"] = True - if(args.disable_odin_xml): + if args.disable_odin_xml: odin_args["disable_odin_xml"] = True - if(args.use_odin_simulation): + if args.use_odin_simulation: odin_args["use_odin_simulation"] = True - try: - vpr_args = process_unknown_args(unknown_args) - if(args.crit_path_router_iterations): - vpr_args["max_router_iterations"] = args.crit_path_router_iterations - if(args.fix_pins): - new_file = str(Path(temp_dir) / Path(args.fix_pins).name) - shutil.copyfile(str((Path(prog).parent.parent / args.fix_pins)), new_file) - vpr_args["fix_pins"] = new_file - if args.verify_rr_graph: - rr_graph_out_file = "rr_graph" + args.rr_graph_ext - vpr_args["write_rr_graph"] = rr_graph_out_file - if args.check_place: - vpr_args["route"] = True - if args.check_route: - vpr_args["analysis"] = True - if args.sdc_file: - if not Path(args.sdc_file).exists(): - sdc_file = None - if args.sdc_file.startswith("/"): - sdc_file = Path(str(Path(prog).parent.parent) + args.sdc_file) - else: - sdc_file = Path(prog).parent.parent / args.sdc_file - if sdc_file.exists(): - args.sdc_file = str(sdc_file) - else: - raise vtr.InspectError("Sdc file {arg},{sdc} was not found.".format(arg=args.sdc_file,sdc=str(sdc_file))) - vpr_args["sdc_file"] = args.sdc_file + return odin_args + - name = "" - if(args.name): - name=args.name +def process_vpr_args(args, prog, temp_dir): + """ + Finds arguments needed in the VPR stage of the flow + """ + vpr_args = OrderedDict() + if args.crit_path_router_iterations: + vpr_args["max_router_iterations"] = args.crit_path_router_iterations + if args.fix_pins: + new_file = str(temp_dir / Path(args.fix_pins).name) + shutil.copyfile(str((Path(prog).parent.parent / args.fix_pins)), new_file) + vpr_args["fix_pins"] = new_file + if args.verify_rr_graph: + rr_graph_out_file = "rr_graph" + args.rr_graph_ext + vpr_args["write_rr_graph"] = rr_graph_out_file + if args.check_place: + vpr_args["route"] = True + if args.check_route: + vpr_args["analysis"] = True + + return vpr_args + +def get_sdc_file(sdc_file, prog): + """ + takes in the sdc_file and returns a path to that file if it exists. + """ + if not Path(sdc_file).exists(): + if sdc_file.startswith("/"): + sdc_file = Path(str(Path(prog).parent.parent) + sdc_file) else: - name=path_arch_file.name+"/"+path_circuit_file.name - print(name, end ="\t\t") - #Run the flow - vtr.run(path_arch_file, - path_circuit_file, - power_tech_file=args.power_tech, - temp_dir=temp_dir, - start_stage=args.start, - end_stage=args.end, - command_runner=command_runner, - vpr_args=vpr_args, - abc_args=abc_args, - odin_args=odin_args, - keep_intermediate_files=args.keep_intermediate_files, - keep_result_files=args.keep_result_files, - min_hard_mult_size=args.min_hard_mult_size, - min_hard_adder_size = args.min_hard_adder_size, - check_equivalent = args.check_equivalent, - check_incremental_sta_consistency = args.check_incremental_sta_consistency, - use_old_abc_script=args.use_old_abc_script, - relax_W_factor=args.relax_W_factor - ) - error_status = "OK" - except vtr.CommandError as e: - #An external command failed + sdc_file = Path(prog).parent.parent / sdc_file + + return str(vtr.verify_file(sdc_file, "sdc file")) + +def except_vtr_error(error, expect_fail, verbose): + """ + Handle vtr exceptions + """ + error_status = None + actual_error = None + exit_status = None + if isinstance(error, vtr.CommandError): + # An external command failed return_status = 1 - if args.expect_fail: - expect_string = args.expect_fail + if expect_fail: + expect_string = expect_fail actual_error = None if "exited with return code" in expect_string: - actual_error = "exited with return code {}".format(e.returncode) + actual_error = "exited with return code {}".format(error.returncode) else: - actual_error = e.msg + actual_error = error.msg if expect_string != actual_error: - error_status = "failed: expected to fail with '{expected}' but was '{actual}'".format(expected=expect_string,actual = actual_error) + error_status = "failed: expected '{expected}' but was '{actual}'".format( + expected=expect_string, actual=actual_error + ) exit_status = 1 else: error_status = "OK" return_status = 0 - if args.verbose: + if verbose: error_status += " (as expected {})".format(expect_string) else: error_status += "*" else: - error_status = "failed: {}".format(e.msg) - if not args.expect_fail or exit_status: - print ("Error: {msg}".format(msg=e.msg)) - print ("\tfull command: ", ' '.join(e.cmd)) - print ("\treturncode : ", e.returncode) - print ("\tlog file : ", e.log) + error_status = "failed: {}".format(error.msg) + if not expect_fail or exit_status: + print("Error: {msg}".format(msg=error.msg)) + print("\tfull command: ", " ".join(error.cmd)) + print("\treturncode : ", error.returncode) + print("\tlog file : ", error.log) exit_status = 1 - except vtr.InspectError as e: - #Something went wrong gathering information - print ("Error: {msg}".format(msg=e.msg)) - print ("\tfile : ", e.filename) + if isinstance(error, vtr.InspectError): + # Something went wrong gathering information + print("Error: {msg}".format(msg=error.msg)) + print("\tfile : ", error.filename) exit_status = 2 return_status = exit_status - error_status = "failed: {}".format(e.msg) + error_status = "failed: {}".format(error.msg) - except vtr.VtrError as e: - #Generic VTR errors - print ("Error: ", e.msg) + if isinstance(error, vtr.VtrError): + # Generic VTR errors + print("Error: ", error.msg) exit_status = 3 return_status = exit_status - error_status = "failed: {}".format(e.msg) - - except KeyboardInterrupt as e: - print ("{} recieved keyboard interrupt".format(prog)) - exit_status = 4 - return_status = exit_status + error_status = "failed: {}".format(error.msg) + return error_status, return_status, exit_status - finally: - seconds = datetime.now() - start - print("{status} (took {time})".format(status = error_status, time=vtr.format_elapsed_time(seconds))) - mkdir_p(temp_dir) - out = Path(temp_dir) / "output.txt" - out.touch() - with out.open('w') as f: - f.write("vpr_status=") - if(exit_status==0): - f.write("success\n") - else: - f.write("exited with return code {}\n".format(exit_status)) - f.write("vpr_seconds=%d\nrundir=%s\nhostname=%s\nerror=" % (seconds.total_seconds(), str(Path.cwd()), socket.gethostname())) - f.write("\n") - - sys.exit(return_status) - -def process_unknown_args(unknown_args): - #We convert the unknown_args into a dictionary, which is eventually - #used to generate arguments for VPR - vpr_args = OrderedDict() - while len(unknown_args) > 0: - #Get the first argument - arg = unknown_args.pop(0) - - if arg == '': - continue - - if not arg.startswith('-'): - raise vtr.VtrError("Extra argument '{}' intended for VPR does not start with '-'".format(arg)) - - #To make it a valid kwargs dictionary we trim the initial '-' or '--' from the - #argument name - assert len(arg) >= 2 - if arg[1] == '-': - #Double-dash prefix - arg = arg[2:] - else: - #Single-dash prefix - arg = arg[1:] - - - #Determine if there is a value associated with this argument - if len(unknown_args) == 0 or (unknown_args[0].startswith('-') and arg != "target_ext_pin_util"): - #Single value argument, we place these with value 'True' - #in vpr_args - vpr_args[arg] = True - else: - #Multivalue argument - val = unknown_args.pop(0) - if len(unknown_args) != 0 and not unknown_args[0].startswith('-'): - temp = val - val =[] - val.append(temp) - while len(unknown_args) != 0 and not unknown_args[0].startswith('-'): - val.append(unknown_args.pop(0)) - vpr_args[arg] = val - - return vpr_args - if __name__ == "__main__": retval = main() sys.exit(retval) From bbc43f64ef3ead3c16229147f53fc0b97155e2e0 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Thu, 23 Jul 2020 11:07:47 -0600 Subject: [PATCH 71/86] cleaned up util.py for pylint --- vtr_flow/scripts/python_libs/vtr/__init__.py | 6 +- vtr_flow/scripts/python_libs/vtr/flow.py | 2 +- vtr_flow/scripts/python_libs/vtr/inspect.py | 8 +- vtr_flow/scripts/python_libs/vtr/util.py | 386 +++++++++++------- .../scripts/python_libs/vtr/vpr/__init__.py | 2 +- vtr_flow/scripts/python_libs/vtr/vpr/vpr.py | 16 +- 6 files changed, 245 insertions(+), 175 deletions(-) diff --git a/vtr_flow/scripts/python_libs/vtr/__init__.py b/vtr_flow/scripts/python_libs/vtr/__init__.py index fe4a12b7b10..77732936da4 100644 --- a/vtr_flow/scripts/python_libs/vtr/__init__.py +++ b/vtr_flow/scripts/python_libs/vtr/__init__.py @@ -1,7 +1,7 @@ -from .util import load_config_lines, find_vtr_file, CommandRunner, print_verbose, relax_W, file_replace, RawDefaultHelpFormatter, VERBOSITY_CHOICES,format_elapsed_time, write_tab_delimitted_csv, load_list_file, find_vtr_root, argparse_str2bool, get_next_run_dir, get_latest_run_dir, verify_file -from .inspect import determine_lut_size, determine_min_W, determine_memory_addr_width +from .util import load_config_lines, find_vtr_file, CommandRunner, print_verbose, relax_w, file_replace, RawDefaultHelpFormatter, VERBOSITY_CHOICES,format_elapsed_time, write_tab_delimitted_csv, load_list_file, find_vtr_root, argparse_str2bool, get_next_run_dir, get_latest_run_dir, verify_file +from .inspect import determine_lut_size, determine_min_w, determine_memory_addr_width from .abc import run, run_lec -from .vpr import run,run_relax_W,cmp_full_vs_incr_STA +from .vpr import run,run_relax_w,cmp_full_vs_incr_STA from .odin import run from .ace import run from .error import * diff --git a/vtr_flow/scripts/python_libs/vtr/flow.py b/vtr_flow/scripts/python_libs/vtr/flow.py index 3144266e286..af336176670 100644 --- a/vtr_flow/scripts/python_libs/vtr/flow.py +++ b/vtr_flow/scripts/python_libs/vtr/flow.py @@ -247,7 +247,7 @@ def run( ) else: # First find minW and then re-route at a relaxed W - vtr.vpr.run_relax_W( + vtr.vpr.run_relax_w( architecture_copy, pre_vpr_netlist, circuit_copy.stem, diff --git a/vtr_flow/scripts/python_libs/vtr/inspect.py b/vtr_flow/scripts/python_libs/vtr/inspect.py index 15c8eef64b8..1a79ada2d12 100644 --- a/vtr_flow/scripts/python_libs/vtr/inspect.py +++ b/vtr_flow/scripts/python_libs/vtr/inspect.py @@ -186,13 +186,13 @@ def determine_memory_addr_width(architecture_file): return mem_addr_width -def determine_min_W(log_filename): - min_W_regex = re.compile(r"\s*Best routing used a channel width factor of (?P\d+).") +def determine_min_w(log_filename): + min_w_regex = re.compile(r"\s*Best routing used a channel width factor of (?P\d+).") with open(log_filename) as f: for line in f: - match = min_W_regex.match(line) + match = min_w_regex.match(line) if match: - return int(match.group("min_W")) + return int(match.group("min_w")) raise InspectError(msg="Failed to find minimum channel width.", filename=log_filename) diff --git a/vtr_flow/scripts/python_libs/vtr/util.py b/vtr_flow/scripts/python_libs/vtr/util.py index aa657d7ff9e..b32c9354f23 100644 --- a/vtr_flow/scripts/python_libs/vtr/util.py +++ b/vtr_flow/scripts/python_libs/vtr/util.py @@ -1,10 +1,12 @@ +""" + Module to utilize many of the tools needed for VTR. +""" import os from pathlib import PurePath from pathlib import Path import sys import re import time -import errno import subprocess import distutils.spawn as distutils_spawn import argparse @@ -15,29 +17,47 @@ VERBOSITY_CHOICES = range(5) -class RawDefaultHelpFormatter(argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescriptionHelpFormatter): + +class RawDefaultHelpFormatter( + argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescriptionHelpFormatter +): """ - An argparse formatter which supports both default arguments and raw formatting of description/epilog + An argparse formatter which supports both default arguments and raw + formatting of description/epilog """ - pass - -class CommandRunner(object): - def __init__(self, timeout_sec=None, max_memory_mb=None, track_memory=True, verbose_error=None, verbose=False, echo_cmd=None, indent="\t", show_failures=False,valgrind=False): - """ - An object for running system commands with timeouts, memory limits and varying verbose-ness +#pylint: disable=too-many-arguments, too-many-instance-attributes, too-few-public-methods, too-many-locals +class CommandRunner(): + """ + An object for running system commands with timeouts, memory limits and varying verbose-ness Arguments ========= timeout_sec: maximum walk-clock-time of the command in seconds. Default: None - max_memory_mb: maximum memory usage of the command in megabytes (if supported). Default: None - track_memory: Whether to track usage of the command (disabled if not supported). Default: True - verbose_error: Produce more verbose output if the commadn fails. Default: Equal to verbose + max_memory_mb: maximum memory usage of the command in megabytes (if supported). + Default: None + track_memory: Whether to track usage of the command (disabled if not supported). + Default: True + verbose_error: Produce more verbose output if the command fails. + Default: Equal to verbose verbose: Produce more verbose output. Default: False echo_cmd: Echo the command before running. Default: Equal to verbose indent: The string specifying a single indent (used in verbose mode) valgrind: Indicates if commands should be run with valgrind """ + def __init__( + self, + timeout_sec=None, + max_memory_mb=None, + track_memory=True, + verbose_error=None, + verbose=False, + echo_cmd=None, + indent="\t", + show_failures=False, + valgrind=False, + ): + if verbose_error is None: verbose_error = verbose if echo_cmd is None: @@ -53,7 +73,9 @@ def __init__(self, timeout_sec=None, max_memory_mb=None, track_memory=True, verb self._show_failures = show_failures self._valgrind = valgrind - def run_system_command(self, cmd, temp_dir, log_filename=None, expected_return_code=0, indent_depth=0): + def run_system_command( + self, cmd, temp_dir, log_filename=None, expected_return_code=0, indent_depth=0 + ): """ Runs the specified command in the system shell. @@ -64,120 +86,126 @@ def run_system_command(self, cmd, temp_dir, log_filename=None, expected_return_c Arguments ========= cmd: list of tokens that form the command to be run - log_filename: name of the log file for the command's output. Default: derived from command + log_filename: the log fiel name for the command's output. Default: derived from command temp_dir: The directory to run the command in. Default: None (uses object default). - expected_return_code: The expected return code from the command. If the actula return code does not match, will generate an exception. Default: 0 + expected_return_code: The expected return code from the command. + If the actula return code does not match, will generate an exception. Default: 0 indent_depth: How deep to indent the tool output in verbose mode. Default 0 """ - #Save the original command + # Save the original command orig_cmd = cmd temp_dir = Path(temp_dir) if not isinstance(temp_dir, Path) else temp_dir - #If no log file is specified the name is based on the executed command - if log_filename is None: - log_filename = PurePath(orig_cmd[0]).name + '.out' - + # If no log file is specified the name is based on the executed command + log_filename = PurePath(orig_cmd[0]).name + ".out" if log_filename is None else log_filename - #Limit memory usage? + # Limit memory usage? memory_limit = ["ulimit", "-Sv", "{val};".format(val=self._max_memory_mb)] - if self._max_memory_mb != None and self.check_command(memory_limit[0]): - cmd = memory_limit + cmd + cmd = memory_limit + cmd if self._max_memory_mb and check_cmd(memory_limit[0]) else cmd - #Enable memory tracking? + # Enable memory tracking? memory_tracking = ["/usr/bin/env", "time", "-v"] - if self._track_memory and self.check_command(memory_tracking[0]): - if self._valgrind: - valgrind_args = ["valgrind", "--leak-check=full", "--suppressions={}".format(find_vtr_file("valgrind.supp")), "--error-exitcode=1", "--errors-for-leak-kinds=none", "--track-origins=yes", "--log-file=valgrind.log","--error-limit=no"] - cmd = memory_tracking + valgrind_args + cmd - else: - cmd = memory_tracking + cmd - - #Flush before calling subprocess to ensure output is ordered - #correctly if stdout is buffered + if self._track_memory and check_cmd(memory_tracking[0]): + cmd = memory_tracking + [ + "valgrind", + "--leak-check=full", + "--suppressions={}".format(find_vtr_file("valgrind.supp")), + "--error-exitcode=1", + "--errors-for-leak-kinds=none", + "--track-origins=yes", + "--log-file=valgrind.log", + "--error-limit=no", + ] + cmd if self._valgrind else memory_tracking + cmd + + # Flush before calling subprocess to ensure output is ordered + # correctly if stdout is buffered sys.stdout.flush() - #Echo the command? + # Echo the command? if self._echo_cmd: print(cmd) - #Begin timing + # Begin timing start_time = time.time() - cmd_output=[] - cmd_returncode=None + cmd_output = [] + cmd_returncode = None proc = None try: - #Call the command - stderr = subprocess.STDOUT - if self._valgrind: - stderr = None - proc = subprocess.Popen(cmd, - stdout=subprocess.PIPE, #We grab stdout - stderr=stderr, #stderr redirected to stderr - universal_newlines=True, #Lines always end in \n - cwd=str(temp_dir), #Where to run the command - ) + # Call the command + stderr = None if self._valgrind else subprocess.STDOUT + proc = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, # We grab stdout + stderr=stderr, # stderr redirected to stderr + universal_newlines=True, # Lines always end in \n + cwd=str(temp_dir), # Where to run the command + ) # Read the output line-by-line and log it # to a file. # - # We do this rather than use proc.communicate() + # We do this rather than use proc.communicate() # to get interactive output - with (temp_dir / log_filename).open('w') as log_f: - #Print the command at the top of the log + with (temp_dir / log_filename).open("w") as log_f: + # Print the command at the top of the log log_f.write(" ".join(cmd)) log_f.write("\n") - #Read from subprocess output + # Read from subprocess output for line in proc.stdout: - #Send to log file + # Send to log file log_f.write(line) - #Save the output + # Save the output cmd_output.append(line) - #Abort if over time limit + # Abort if over time limit elapsed_time = time.time() - start_time if self._timeout_sec and elapsed_time > self._timeout_sec: proc.terminate() - #Should now be finished (since we stopped reading from proc.stdout), - #sets the return code + # Should now be finished (since we stopped reading from proc.stdout), + # sets the return code proc.wait() finally: - #Clean-up if we did not exit cleanly + # Clean-up if we did not exit cleanly if proc: if proc.returncode is None: - #Still running, stop it + # Still running, stop it proc.terminate() cmd_returncode = proc.returncode - cmd_errored = (cmd_returncode != expected_return_code) + cmd_errored = cmd_returncode != expected_return_code - #Send to stdout - if self._show_failures and (self._verbose or (cmd_errored and self._verbose_error)): + # Send to stdout + if self._show_failures and ( + self._verbose or (cmd_errored and self._verbose_error) + ): for line in cmd_output: - print (indent_depth*self._indent + line,) - - - if (self._show_failures and cmd_errored): - raise CommandError(msg="Executable {exec_name} failed".format(exec_name=PurePath(orig_cmd[0]).name), - cmd=cmd, - log=str(temp_dir / log_filename), - returncode=cmd_returncode) + print(indent_depth * self._indent + line,) + + if self._show_failures and cmd_errored: + raise CommandError( + msg="Executable {exec_name} failed".format( + exec_name=PurePath(orig_cmd[0]).name + ), + cmd=cmd, + log=str(temp_dir / log_filename), + returncode=cmd_returncode, + ) return cmd_output, cmd_returncode + #pylint: enable=too-many-arguments, too-many-instance-attributes, too-few-public-methods, too-many-locals +def check_cmd(command): + """ + Return True if command can be run, False otherwise. + """ - def check_command(self, command): - """ - Return True if command can be run, False otherwise. - """ - - #TODO: actually check for this - return True + return Path(command).exists() def write_tab_delimitted_csv(filepath, rows): @@ -185,11 +213,11 @@ def write_tab_delimitted_csv(filepath, rows): Write out the data provied in a tab-delimited CSV format filepath: The filepath to write the data to - rows: An iterable of dictionary-like elements; each element - provides a set key-value pairs corresponding to a row + rows: An iterable of dictionary-like elements; each element + provides a set key-value pairs corresponding to a row in the output file """ - #Calculate the max width of each column + # Calculate the max width of each column columns = OrderedDict() for row in rows: for key, value in row.items(): @@ -199,32 +227,35 @@ def write_tab_delimitted_csv(filepath, rows): else: columns[key] = max(columns[key], len(str(value))) - #Write the elements - with open(filepath, 'w+') as f: - writer = csv.writer(f, delimiter='\t') + # Write the elements + with open(filepath, "w+") as file: + writer = csv.writer(file, delimiter="\t") - #Write out the header + # Write out the header header = [] for col_name, col_width in columns.items(): header.append("{:{width}}".format(col_name, width=col_width)) writer.writerow(header) - #Write rows + # Write rows for row in rows: values = [] for col_name, col_width in columns.items(): values.append("{:{width}}".format(row[col_name], width=col_width)) writer.writerow(values) -def load_tab_delimited_csv(filepath): +def load_tab_delimited_csv(filepath): + """ + loads a tab delimted csv as a list of ordered dictionaries + """ data = [] - with open(filepath) as f: - reader = csv.reader(f, delimiter='\t') + with open(filepath) as file: + reader = csv.reader(file, delimiter="\t") - header = None + header = [] for csv_row in reader: - if header is None: + if len(header) == 0: header = [val.strip() for val in csv_row] else: data_row = OrderedDict() @@ -236,6 +267,7 @@ def load_tab_delimited_csv(filepath): return data + def print_verbose(min_verbosity, curr_verbosity, string, endl=True): """ Print string if curr_verbosity is gteq min_verbosity @@ -244,100 +276,119 @@ def print_verbose(min_verbosity, curr_verbosity, string, endl=True): if endl: print(string) else: - print (string,) + print(string,) + def find_vtr_file(filename, is_executable=False): """ Attempts to find a VTR related file by searching the environment. - + Checking the following places: 1) System path (if is_executable=True) 2) The inferred vtr root from environment variables or the script file location """ - #We assume exectuables are specified in the unix style (no .exe), + # We assume exectuables are specified in the unix style (no .exe), # if it was specified with .exe, strip it off file_path = PurePath(filename) if file_path.suffix == ".exe": filename = file_path.name # - #Check if it is on the path (provided it is executable) + # Check if it is on the path (provided it is executable) # if is_executable: - #Search first for the non-exe version + # Search first for the non-exe version result = distutils_spawn.find_executable(filename) if result: return result - #If not found try the .exe version - result = distutils_spawn.find_executable(filename + '.exe') + # If not found try the .exe version + result = distutils_spawn.find_executable(filename + ".exe") if result: return result vtr_root = find_vtr_root() - #Check the inferred VTR root + # Check the inferred VTR root result = find_file_from_vtr_root(filename, vtr_root, is_executable=is_executable) if result: return result - #Since we stripped off the .exe, try looking for the .exe version as a last resort (i.e. on Windows/cygwin) + # Since we stripped off the .exe, try looking for the .exe version + # as a last resort (i.e. on Windows/cygwin) if is_executable: - result = find_file_from_vtr_root(filename + '.exe', vtr_root, is_executable=is_executable) + result = find_file_from_vtr_root( + filename + ".exe", vtr_root, is_executable=is_executable + ) if result: return result - raise ValueError("Could not find {type} {file}".format(type="executable" if is_executable else "file",file=filename)) + raise ValueError( + "Could not find {type} {file}".format( + type="executable" if is_executable else "file", file=filename + ) + ) + def find_file_from_vtr_root(filename, vtr_root, is_executable=False): """ - Given a vtr_root and a filename searches for the file recursively under some common VTR directories + Given a vtr_root and a filename searches for the file recursively + under some common VTR directories """ - for subdir in ['vpr', 'abc', 'abc_with_bb_support', 'ODIN_II', 'vtr_flow', 'ace2']: - directory_path = (Path(vtr_root) / subdir) - for file_path in directory_path.glob('**/*'): + for subdir in ["vpr", "abc", "abc_with_bb_support", "ODIN_II", "vtr_flow", "ace2"]: + directory_path = Path(vtr_root) / subdir + for file_path in directory_path.glob("**/*"): if file_path.name == filename: if file_path.is_file(): - if is_executable: - #Found an executable file as required - if os.access(str(file_path), os.X_OK): - return str(file_path) - else: - #Found a file as required + if is_executable and os.access(str(file_path), os.X_OK): + # Found an executable file as required return str(file_path) + # Found a file as required + return str(file_path) return None + def find_vtr_root(): - for env_var in ['VTR_ROOT']: + """ + finds the root directory of VTR + """ + for env_var in ["VTR_ROOT"]: if env_var in os.environ: return os.environ[env_var] - #We assume that this file is in /vtr_flow/python_libs/verilogtorouting - inferred_vtr_root = Path(__file__).parent / '../../../..' + # We assume that this file is in /vtr_flow/python_libs/verilogtorouting + inferred_vtr_root = Path(__file__).parent / "../../../.." - if inferred_vtr_root.is_dir: + if inferred_vtr_root.is_dir(): return str(inferred_vtr_root) - else: - raise VtrError("Could not find VTR root directory. Try setting VTR_ROOT environment variable.") + raise VtrError( + "Could not find VTR root directory. Try setting VTR_ROOT environment variable." + ) + def file_replace(filename, search_replace_dict): + """ + searches file for specified values and replaces them with specified values. + """ lines = [] - with open(filename, 'r') as f: - lines = f.readlines() + with open(filename, "r") as file: + lines = file.readlines() - with open(filename, 'w') as f: + with open(filename, "w") as file: for line in lines: for search, replace in search_replace_dict.items(): line = line.replace(search, str(replace)) - print (line,file=f) + print(line, file=file) -def relax_W(min_W, relax_factor, base=2): + +def relax_w(min_w, relax_factor, base=2): """ - Scale min_W by relax_factor and round to the nearest multiple of base. + Scale min_w by relax_factor and round to the nearest multiple of base. """ - relaxed_W = int(base * round(min_W*relax_factor/base)) - return relaxed_W + relaxed_w = int(base * round(min_w * relax_factor / base)) + return relaxed_w + def load_list_file(list_file): """ @@ -345,17 +396,18 @@ def load_list_file(list_file): potentially with '#' comments """ values = [] - with open(list_file) as f: - for line in f: + with open(list_file) as file: + for line in file: line = line.strip() - #Handle comments - if '#' in line: - line = line.split('#')[0] + # Handle comments + if "#" in line: + line = line.split("#")[0] if line == "": continue values.append(line) return values + def load_config_lines(filepath, allow_includes=True): """ Loads the lines of a file, stripping blank lines and '#' comments. @@ -374,16 +426,16 @@ def load_config_lines(filepath, allow_includes=True): blank_regex = re.compile(r"^\s*$") try: - with open(filepath) as f: - for line in f: - #Trim '\n' + with open(filepath) as file: + for line in file: + # Trim '\n' line = line.strip() - #Trim comments - if '#' in line: - line = line.split('#')[0] + # Trim comments + if "#" in line: + line = line.split("#")[0] - #Skip blanks + # Skip blanks if blank_regex.match(line): continue @@ -392,45 +444,60 @@ def load_config_lines(filepath, allow_includes=True): components = line.split() assert len(components) == 2 - include_file = components[1].strip('"') #Strip quotes + include_file = components[1].strip('"') # Strip quotes include_file_abs = str(Path(filepath).parent / include_file) - #Recursively load the config - config_lines += load_config_lines(include_file_abs, allow_includes=allow_includes) + # Recursively load the config + config_lines += load_config_lines( + include_file_abs, allow_includes=allow_includes + ) else: - raise InspectError("@include not allowed in this file", filepath) + raise InspectError( + "@include not allowed in this file", filepath + ) else: config_lines.append(line) - except IOError as e: - raise InspectError("Error opening config file ({})".format(e)) + except IOError as error: + raise InspectError("Error opening config file ({})".format(error)) - return config_lines + return config_lines -def verify_file(file, file_type,should_exist=True): + +def verify_file(file, file_type, should_exist=True): """ - Verifies that the file is a Pathlib object and if not makes it one. + Verifies that the file is a Pathlib object and if not makes it one. Ensures that the file exists by default. - This makes it possible to pass files into the various files as strings or as pathlib objects. + This makes it possible to pass files into the various files as strings or as pathlib objects """ - if(not isinstance(file,Path)): + if not isinstance(file, Path): file = Path(file) - if(should_exist and not file.is_file()): - raise Exception("{file_type} file does not exist: {file} ".format(file_type = file_type, file=file)) + if should_exist and not file.is_file(): + raise Exception( + "{file_type} file does not exist: {file} ".format( + file_type=file_type, file=file + ) + ) return file - + + def format_elapsed_time(time_delta): + """ + formats a time into desired string format + """ return "%.2f seconds" % time_delta.total_seconds() - + def argparse_str2bool(str_val): + """ + parses a string boolean to a boolean + """ str_val = str_val.lower() - if str_val in ['yes', 'on', 'true', '1']: + if str_val in ["yes", "on", "true", "1"]: return True - elif str_val in ['no', 'off', 'false', '0']: + if str_val in ["no", "off", "false", "0"]: return False - else: - raise argparse.ArgumentTypeError('Boolean value expected.') + raise argparse.ArgumentTypeError("Boolean value expected.") def get_next_run_dir(base_dir): @@ -441,6 +508,7 @@ def get_next_run_dir(base_dir): """ return str(PurePath(base_dir) / run_dir_name(get_next_run_number(base_dir))) + def get_latest_run_dir(base_dir): """ Returns the run directory with the highest run number in base_dir @@ -452,6 +520,7 @@ def get_latest_run_dir(base_dir): return str(PurePath(base_dir) / run_dir_name(latest_run_number)) + def get_next_run_number(base_dir): """ Returns the next available (i.e. non-existing) run number in base_dir @@ -465,6 +534,7 @@ def get_next_run_number(base_dir): return next_run_number + def get_latest_run_number(base_dir): """ Returns the highest run number of all run directories with in base_dir @@ -473,15 +543,15 @@ def get_latest_run_number(base_dir): run_dir = Path(base_dir) / run_dir_name(run_number) if not run_dir.exists: - #No existing run directories + # No existing run directories return None while run_dir.exists: run_number += 1 run_dir = Path(base_dir) / run_dir_name(run_number) - #Currently one-past the last existing run dir, - #to get latest existing, subtract one + # Currently one-past the last existing run dir, + # to get latest existing, subtract one return run_number - 1 diff --git a/vtr_flow/scripts/python_libs/vtr/vpr/__init__.py b/vtr_flow/scripts/python_libs/vtr/vpr/__init__.py index 6da4386cc41..3c0a3854793 100644 --- a/vtr_flow/scripts/python_libs/vtr/vpr/__init__.py +++ b/vtr_flow/scripts/python_libs/vtr/vpr/__init__.py @@ -1 +1 @@ -from .vpr import run,run_relax_W,cmp_full_vs_incr_STA +from .vpr import run,run_relax_w,cmp_full_vs_incr_STA diff --git a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py index a7ec8377205..ea826963989 100644 --- a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py +++ b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py @@ -1,9 +1,9 @@ import shutil from pathlib import Path -from vtr import find_vtr_file, CommandRunner, print_verbose, relax_W, determine_lut_size, determine_min_W, verify_file +from vtr import find_vtr_file, CommandRunner, print_verbose, relax_w, determine_lut_size, determine_min_w, verify_file from vtr.error import InspectError -def run_relax_W(architecture, circuit, circuit_name=None, command_runner=CommandRunner(), temp_dir=Path("."), +def run_relax_w(architecture, circuit, circuit_name=None, command_runner=CommandRunner(), temp_dir=Path("."), relax_w_factor=1.3, vpr_exec=None, logfile_base="vpr", vpr_args=None, output_netlist=None): """ @@ -13,7 +13,7 @@ def run_relax_W(architecture, circuit, circuit_name=None, command_runner=Command 2nd: At relaxed channel width (e.g. for critical path delay) - .. note :: Usage: vtr.vpr.run_relax_W(,,,[OPTIONS]) + .. note :: Usage: vtr.vpr.run_relax_w(,,,[OPTIONS]) Arguments ========= @@ -41,7 +41,7 @@ def run_relax_W(architecture, circuit, circuit_name=None, command_runner=Command Path to the VPR executable logfile_base: - Base name for log files (e.g. "vpr" produces vpr.min_W.out, vpr.relaxed_W.out) + Base name for log files (e.g. "vpr" produces vpr.min_w.out, vpr.relaxed_W.out) vpr_args: Extra arguments for VPR @@ -58,7 +58,7 @@ def run_relax_W(architecture, circuit, circuit_name=None, command_runner=Command architecture = verify_file(architecture, "Architecture") circuit = verify_file(circuit, "Circuit") - vpr_min_W_log = '.'.join([logfile_base, "out"]) + vpr_min_w_log = '.'.join([logfile_base, "out"]) vpr_relaxed_W_log = '.'.join([logfile_base, "crit_path", "out"]) max_router_iterations = None @@ -78,16 +78,16 @@ def run_relax_W(architecture, circuit, circuit_name=None, command_runner=Command if vpr_exec is None: vpr_exec = find_vtr_file('vpr', is_executable=True) - run(architecture, circuit, circuit_name, command_runner, temp_dir, log_filename=vpr_min_W_log, vpr_exec=vpr_exec, vpr_args=vpr_args) + run(architecture, circuit, circuit_name, command_runner, temp_dir, log_filename=vpr_min_w_log, vpr_exec=vpr_exec, vpr_args=vpr_args) if ('pack' in vpr_args or 'place' in vpr_args) and 'route' not in vpr_args: #Don't look for min W if routing was not run return if max_router_iterations: vpr_args["max_router_iterations"]=max_router_iterations - min_W = determine_min_W(str(temp_dir / vpr_min_W_log)) + min_w = determine_min_w(str(temp_dir / vpr_min_w_log)) - relaxed_W = relax_W(min_W, relax_w_factor) + relaxed_W = relax_w(min_w, relax_w_factor) vpr_args['route'] = True #Re-route only vpr_args['route_chan_width'] = relaxed_W #At a fixed channel width From 0e39c90085745f9c4774fbfeba4a40b05b9e7525 Mon Sep 17 00:00:00 2001 From: Jeff Goeders Date: Thu, 23 Jul 2020 12:23:28 -0600 Subject: [PATCH 72/86] Switch travis python to 3.6 --- .travis.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4001172230d..7d9004d49db 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,7 +43,6 @@ addons: - libxft-dev - libxml++2.6-dev - perl - - python - python-lxml - texinfo - time @@ -228,9 +227,13 @@ jobs: - ./.github/travis/cron_build.sh before_script: + # Switch to python 3.6.3 + - pyenv install -f 3.6.3 + - pyenv global 3.6.3 + - source .github/travis/common.sh - ./.github/travis/setup.sh - + after_script: - ./.github/travis/setup.sh From b1fc0a0582f1dcf07481cf04c07a87cd3e590c88 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Thu, 23 Jul 2020 17:19:33 -0600 Subject: [PATCH 73/86] cleaning up error.py and inspect.py for pylint, also removing task fuctionality that will be reimplemented later. --- vtr_flow/scripts/python_libs/vtr/error.py | 4 +- vtr_flow/scripts/python_libs/vtr/inspect.py | 138 ++------------------ 2 files changed, 13 insertions(+), 129 deletions(-) diff --git a/vtr_flow/scripts/python_libs/vtr/error.py b/vtr_flow/scripts/python_libs/vtr/error.py index 5c37cc3366b..a93241a0c49 100644 --- a/vtr_flow/scripts/python_libs/vtr/error.py +++ b/vtr_flow/scripts/python_libs/vtr/error.py @@ -17,7 +17,7 @@ class CommandError(VtrError): cmd == The command run """ def __init__(self, msg, cmd, returncode, log=None): - super(CommandError, self).__init__(msg=msg) + super().__init__(msg=msg) self.returncode = returncode self.cmd = cmd self.log = log @@ -27,6 +27,6 @@ class InspectError(VtrError): Raised when some query (inspection) result is not found. """ def __init__(self, msg, filename=None): - super(InspectError, self).__init__(msg=msg) + super().__init__(msg=msg) self.filename=filename diff --git a/vtr_flow/scripts/python_libs/vtr/inspect.py b/vtr_flow/scripts/python_libs/vtr/inspect.py index 1a79ada2d12..58b1bed9eba 100644 --- a/vtr_flow/scripts/python_libs/vtr/inspect.py +++ b/vtr_flow/scripts/python_libs/vtr/inspect.py @@ -1,5 +1,7 @@ +""" + module that contains functions to inspect various files to determine important values +""" import re -from collections import OrderedDict try: #Try for the fast c-based version first @@ -8,130 +10,8 @@ #Fall back on python implementation import xml.etree.ElementTree as ET -from vtr import load_config_lines from vtr.error import InspectError -class ParsePattern: - def __init__(self, name, filename, regex_str, default_value=None): - self._name = name - self._filename = filename - self._regex = re.compile(regex_str) - self._default_value = default_value - - def name(self): - return self._name - - def filename(self): - return self._filename - - def regex(self): - return self._regex - - def default_value(self): - return self._default_value - -class PassRequirement(object): - def __init__(self, metric): - self._metric = metric - self._type = type - - def metric(self): - return self._metric - - def type(self): - raise NotImplementedError() - - def check_passed(golden_value, check_value): - raise NotImplementedError() - -class EqualPassRequirement(PassRequirement): - def __init__(self, metric): - super(EqualPassRequirement, self).__init__(metric) - - def type(self): - return "Equal" - - def check_passed(self, golden_value, check_value): - if golden_value == check_value: - return True, "" - else: - return False, "Task value '{}' does not match golden value '{}'".format(golden_value, check_value) - -class RangePassRequirement(PassRequirement): - - def __init__(self, metric, min_value=None, max_value=None): - super(RangePassRequirement, self).__init__(metric) - - if max_value < min_value: - raise InspectError("Invalid range specification (max value larger than min value)") - - self._min_value = min_value - self._max_value = max_value - - def type(self): - return "Range" - - def min_value(self): - return self._min_value - - def max_value(self): - return self._max_value - - def check_passed(self, golden_value, check_value): - if golden_value is None and check_value is None: - return True, "both golden and check are None" - elif golden_value is None and check_value != None: - return False, "golden value is None, but check value is {}".format(check_value) - elif golden_value != None and check_value is None: - return False, "golden value is {}, but check value is None".format(golden_value) - - assert golden_value != None - assert check_value != None - - try: - golden_value = float(golden_value) - except ValueError as e: - raise InspectError("Failed to convert golden value '{}' to float".format(golden_value)) - - try: - check_value = float(check_value) - except ValueError as e: - raise InspectError("Failed to convert check value '{}' to float".format(check_value)) - - if golden_value == 0.: #Avoid division by zero - - if golden_value == check_value: - return True, "golden and check both equal 0" - else: - return False, "unable to normalize relative value (golden value is zero)".format(norm_check_value, min_value(), max_value()) - - else: - norm_check_value = check_value / golden_value - - if self.min_value() <= norm_check_value <= self.max_value(): - return True, "relative value within range" - else: - return False, "relative value {} outside of range [{},{}]".format(norm_check_value, min_value(), max_value()) - -class ParseResults: - def __init__(self): - self._metrics = OrderedDict() - - def primary_keys(self): - return ("architecture", "circuit") - - def add_result(self, arch, circuit, parse_result): - self._metrics[(arch, circuit)] = parse_result - - def metrics(self, arch, circuit): - if (arch, circuit) in self._metrics: - return self._metrics[(arch, circuit)] - else: - return None - - def all_metrics(self): - return self._metrics - def determine_lut_size(architecture_file): """ Determines the maximum LUT size (K) in an architecture file. @@ -164,7 +44,8 @@ def determine_memory_addr_width(architecture_file): """ Determines the maximum RAM block address width in an architecture file - Assumes RAMS are represented using the standard VTR primitives (.subckt single_port_ram, .subckt dual_port_ram etc.) + Assumes RAMS are represented using the standard VTR primitives + (.subckt single_port_ram, .subckt dual_port_ram etc.) """ arch_xml = ET.parse(architecture_file).getroot() @@ -187,12 +68,15 @@ def determine_memory_addr_width(architecture_file): return mem_addr_width def determine_min_w(log_filename): + """ + determines the miniumum width. + """ min_w_regex = re.compile(r"\s*Best routing used a channel width factor of (?P\d+).") - with open(log_filename) as f: - for line in f: + with open(log_filename) as file: + for line in file: match = min_w_regex.match(line) if match: return int(match.group("min_w")) raise InspectError(msg="Failed to find minimum channel width.", - filename=log_filename) + filename=log_filename) From 4b05c8ec7af0529d26724e76aec97712010dc855 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Thu, 23 Jul 2020 17:26:39 -0600 Subject: [PATCH 74/86] actually fixing error.py to pass pylint --- vtr_flow/scripts/python_libs/vtr/error.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/vtr_flow/scripts/python_libs/vtr/error.py b/vtr_flow/scripts/python_libs/vtr/error.py index a93241a0c49..f44520f5c13 100644 --- a/vtr_flow/scripts/python_libs/vtr/error.py +++ b/vtr_flow/scripts/python_libs/vtr/error.py @@ -1,3 +1,6 @@ +""" + Module to handle VTR flow erroring. +""" class VtrError(Exception): """ Base class for VTR related exceptions @@ -5,8 +8,11 @@ class VtrError(Exception): Attributes: msg -- An explanation of the error """ + def __init__(self, msg=""): self.msg = msg + super().__init__(msg=msg) + class CommandError(VtrError): """ @@ -16,17 +22,19 @@ class CommandError(VtrError): returncode -- The return code from the command cmd == The command run """ + def __init__(self, msg, cmd, returncode, log=None): super().__init__(msg=msg) self.returncode = returncode self.cmd = cmd self.log = log + class InspectError(VtrError): """ Raised when some query (inspection) result is not found. """ + def __init__(self, msg, filename=None): super().__init__(msg=msg) - self.filename=filename - + self.filename = filename From 4c78c00b67457423ab91cbc37dbcd1f0aa64f968 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Fri, 24 Jul 2020 09:15:46 -0600 Subject: [PATCH 75/86] fixed lec issue --- vtr_flow/scripts/python_libs/vtr/flow.py | 63 ++++++++++++++++++------ 1 file changed, 49 insertions(+), 14 deletions(-) diff --git a/vtr_flow/scripts/python_libs/vtr/flow.py b/vtr_flow/scripts/python_libs/vtr/flow.py index af336176670..1bdcab61102 100644 --- a/vtr_flow/scripts/python_libs/vtr/flow.py +++ b/vtr_flow/scripts/python_libs/vtr/flow.py @@ -7,23 +7,29 @@ from enum import Enum import vtr + class VtrStage(Enum): """ Enum class for the VTR stages\ """ + odin = 1 abc = 2 ace = 3 vpr = 4 + def __le__(self, other): if self.__class__ is other.__class__: return int(self.value) <= other.value return NotImplemented + def __ge__(self, other): if self.__class__ is other.__class__: return int(self.value) >= other.value return NotImplemented -#pylint: disable=too-many-arguments + + +# pylint: disable=too-many-arguments, too-many-locals, too-many-branches, too-many-statements def run( architecture_file, circuit_file, @@ -124,10 +130,12 @@ def run( # Verify that files are Paths or convert them to Paths and check that they exist architecture_file = vtr.util.verify_file(architecture_file, "Architecture") circuit_file = vtr.util.verify_file(circuit_file, "Circuit") - power_tech_file = vtr.util.verify_file(power_tech_file, "Power") if power_tech_file else None + power_tech_file = ( + vtr.util.verify_file(power_tech_file, "Power") if power_tech_file else None + ) temp_dir = Path(temp_dir) temp_dir.mkdir(parents=True, exist_ok=True) - netlist_ext = ".blif" if ".eblif" not in circuit_file.suffixes else ".eblif" + netlist_ext = ".blif" if ".eblif" not in circuit_file.suffixes else ".eblif" # Define useful filenames post_odin_netlist = temp_dir / (circuit_file.stem + ".odin" + netlist_ext) @@ -143,7 +151,7 @@ def run( # Reference netlist for LEC gen_postsynthesis_netlist = temp_dir / ( - circuit_file.stem + "_post_synthesis." + netlist_ext + circuit_file.stem + "_post_synthesis" + netlist_ext ) # Copy the circuit and architecture @@ -160,7 +168,10 @@ def run( # # RTL Elaboration & Synthesis # - if should_run_stage(VtrStage.odin, start_stage, end_stage) and circuit_file.suffixes != ".blif": + if ( + should_run_stage(VtrStage.odin, start_stage, end_stage) + and circuit_file.suffixes != ".blif" + ): vtr.odin.run( architecture_copy, next_stage_netlist, @@ -174,7 +185,9 @@ def run( next_stage_netlist = post_odin_netlist - lec_base_netlist = post_odin_netlist if not lec_base_netlist else lec_base_netlist + lec_base_netlist = ( + post_odin_netlist if not lec_base_netlist else lec_base_netlist + ) # # Logic Optimization & Technology Mapping @@ -192,7 +205,9 @@ def run( ) next_stage_netlist = post_abc_netlist - lec_base_netlist = post_abc_netlist if not lec_base_netlist else lec_base_netlist + lec_base_netlist = ( + post_abc_netlist if not lec_base_netlist else lec_base_netlist + ) # # Power Activity Estimation @@ -216,7 +231,9 @@ def run( # Use ACE's output netlist next_stage_netlist = post_ace_netlist - lec_base_netlist = post_ace_netlist if not lec_base_netlist else lec_base_netlist + lec_base_netlist = ( + post_ace_netlist if not lec_base_netlist else lec_base_netlist + ) # Enable power analysis in VPR vpr_args["power"] = True @@ -263,6 +280,10 @@ def run( # Logical Equivalence Checks (LEC) # if check_equivalent: + for file in Path(temp_dir).iterdir(): + if "post_synthesis.blif" in str(file): + gen_postsynthesis_netlist = file.name + break vtr.abc.run_lec( lec_base_netlist, gen_postsynthesis_netlist, @@ -272,7 +293,7 @@ def run( # Do a second-run of the incremental analysis to compare the result files if check_incremental_sta_consistency: - vtr.vpr.cmp_full_vs_incr_STA( + vtr.vpr.cmp_full_vs_incr_sta( architecture_copy, pre_vpr_netlist, circuit_copy.stem, @@ -283,12 +304,25 @@ def run( ) if not keep_intermediate_files: - delete_intermediate_files(next_stage_netlist, post_ace_activity_file, - keep_result_files, temp_dir, power_tech_file) -#pylint: enable=too-many-arguments + delete_intermediate_files( + next_stage_netlist, + post_ace_activity_file, + keep_result_files, + temp_dir, + power_tech_file, + ) + -def delete_intermediate_files(next_stage_netlist, post_ace_activity_file, - keep_result_files, temp_dir, power_tech_file): +# pylint: enable=too-many-arguments, too-many-locals, too-many-branches, too-many-statements + + +def delete_intermediate_files( + next_stage_netlist, + post_ace_activity_file, + keep_result_files, + temp_dir, + power_tech_file, +): """ delete intermediate files """ @@ -303,6 +337,7 @@ def delete_intermediate_files(next_stage_netlist, post_ace_activity_file, if power_tech_file: post_ace_activity_file.unlink() + def should_run_stage(stage, flow_start_stage, flow_end_stage): """ Returns True if stage falls between flow_start_stage and flow_end_stage From e1d352298584c0066527751b4ef4ef21bf4f4a8b Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Fri, 24 Jul 2020 13:00:19 -0600 Subject: [PATCH 76/86] fixed expected fail issue --- vtr_flow/scripts/run_vtr_flow.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/vtr_flow/scripts/run_vtr_flow.py b/vtr_flow/scripts/run_vtr_flow.py index 211a9828b4b..26b4eac4cfa 100755 --- a/vtr_flow/scripts/run_vtr_flow.py +++ b/vtr_flow/scripts/run_vtr_flow.py @@ -639,7 +639,7 @@ def except_vtr_error(error, expect_fail, verbose): print("\treturncode : ", error.returncode) print("\tlog file : ", error.log) exit_status = 1 - if isinstance(error, vtr.InspectError): + elif isinstance(error, vtr.InspectError): # Something went wrong gathering information print("Error: {msg}".format(msg=error.msg)) print("\tfile : ", error.filename) @@ -647,7 +647,7 @@ def except_vtr_error(error, expect_fail, verbose): return_status = exit_status error_status = "failed: {}".format(error.msg) - if isinstance(error, vtr.VtrError): + elif isinstance(error, vtr.VtrError): # Generic VTR errors print("Error: ", error.msg) exit_status = 3 @@ -655,6 +655,7 @@ def except_vtr_error(error, expect_fail, verbose): error_status = "failed: {}".format(error.msg) return error_status, return_status, exit_status + if __name__ == "__main__": retval = main() sys.exit(retval) From fd11257b7447f33e7f00ed040b0ed7d4aea68f7e Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Fri, 24 Jul 2020 15:59:54 -0600 Subject: [PATCH 77/86] enabling all of the python library to pass pylint --- vtr_flow/scripts/python_libs/vtr/__init__.py | 26 +- .../scripts/python_libs/vtr/abc/__init__.py | 3 + vtr_flow/scripts/python_libs/vtr/abc/abc.py | 534 +++++++++++------- .../scripts/python_libs/vtr/ace/__init__.py | 3 + vtr_flow/scripts/python_libs/vtr/ace/ace.py | 105 ++-- vtr_flow/scripts/python_libs/vtr/error.py | 4 +- vtr_flow/scripts/python_libs/vtr/flow.py | 38 +- vtr_flow/scripts/python_libs/vtr/inspect.py | 46 +- .../scripts/python_libs/vtr/odin/__init__.py | 3 + vtr_flow/scripts/python_libs/vtr/odin/odin.py | 174 +++--- vtr_flow/scripts/python_libs/vtr/util.py | 51 +- .../scripts/python_libs/vtr/vpr/__init__.py | 5 +- vtr_flow/scripts/python_libs/vtr/vpr/vpr.py | 444 +++++++++------ 13 files changed, 918 insertions(+), 518 deletions(-) diff --git a/vtr_flow/scripts/python_libs/vtr/__init__.py b/vtr_flow/scripts/python_libs/vtr/__init__.py index 77732936da4..2a701260451 100644 --- a/vtr_flow/scripts/python_libs/vtr/__init__.py +++ b/vtr_flow/scripts/python_libs/vtr/__init__.py @@ -1,8 +1,30 @@ -from .util import load_config_lines, find_vtr_file, CommandRunner, print_verbose, relax_w, file_replace, RawDefaultHelpFormatter, VERBOSITY_CHOICES,format_elapsed_time, write_tab_delimitted_csv, load_list_file, find_vtr_root, argparse_str2bool, get_next_run_dir, get_latest_run_dir, verify_file +""" + __init__ for the VTR python module +""" +from .util import ( + load_config_lines, + find_vtr_file, + CommandRunner, + print_verbose, + relax_w, + file_replace, + RawDefaultHelpFormatter, + VERBOSITY_CHOICES, + format_elapsed_time, + write_tab_delimitted_csv, + load_list_file, + find_vtr_root, + argparse_str2bool, + get_next_run_dir, + get_latest_run_dir, + verify_file, +) from .inspect import determine_lut_size, determine_min_w, determine_memory_addr_width +#pylint: disable=reimported from .abc import run, run_lec -from .vpr import run,run_relax_w,cmp_full_vs_incr_STA +from .vpr import run, run_relax_w, cmp_full_vs_incr_sta, run_second_time from .odin import run from .ace import run from .error import * from .flow import run, VtrStage +#pylint: enable=reimported \ No newline at end of file diff --git a/vtr_flow/scripts/python_libs/vtr/abc/__init__.py b/vtr_flow/scripts/python_libs/vtr/abc/__init__.py index 15cdd920af3..26d376b3420 100644 --- a/vtr_flow/scripts/python_libs/vtr/abc/__init__.py +++ b/vtr_flow/scripts/python_libs/vtr/abc/__init__.py @@ -1 +1,4 @@ +""" + __init__ for abc module +""" from .abc import run, run_lec diff --git a/vtr_flow/scripts/python_libs/vtr/abc/abc.py b/vtr_flow/scripts/python_libs/vtr/abc/abc.py index d0b8aa4fe9e..f08338e70d2 100644 --- a/vtr_flow/scripts/python_libs/vtr/abc/abc.py +++ b/vtr_flow/scripts/python_libs/vtr/abc/abc.py @@ -1,13 +1,27 @@ +""" + Module to run ABC with its various options +""" import shutil +from collections import OrderedDict from pathlib import Path from vtr import find_vtr_file, determine_lut_size, verify_file, CommandRunner -from vtr.error import VtrError, InspectError, CommandError - -def run(architecture_file, circuit_file, - output_netlist, command_runner=CommandRunner(), - temp_dir=Path("."), log_filename="abc.out", - abc_exec=None, abc_script=None, abc_rc=None, - use_old_abc_script = False, abc_args = None, keep_intermediate_files=True): +from vtr.error import InspectError + +#pylint: disable=too-many-arguments, too-many-locals +def run( + architecture_file, + circuit_file, + output_netlist, + command_runner=CommandRunner(), + temp_dir=Path("."), + log_filename="abc.out", + abc_exec=None, + abc_script=None, + abc_rc=None, + use_old_abc_script=False, + abc_args=None, + keep_intermediate_files=True, +): """ Runs ABC to optimize specified file. @@ -15,221 +29,323 @@ def run(architecture_file, circuit_file, Arguments ========= - architecture_file : + architecture_file : Architecture file to target - - circuit_file : + + circuit_file : Circuit file to optimize - - output_netlist : + + output_netlist : File name to output the resulting circuit to - + Other Parameters ---------------- - command_runner : + command_runner : A CommandRunner object used to run system commands - - temp_dir : + + temp_dir : Directory to run in (created if non-existent) - - log_filename : + + log_filename : File to log result to - - abc_exec : + + abc_exec : ABC executable to be run - - abc_script : + + abc_script : The script to be run on abc - - abc_rc : + + abc_rc : The ABC rc file - - use_old_abc_script : + + use_old_abc_script : Enables the use of the old ABC script - - abc_args : + + abc_args : A dictionary of keyword arguments to pass on to ABC - - keep_intermediate_files : + + keep_intermediate_files : Determines if intermediate files are kept or deleted """ temp_dir = Path(temp_dir) if not isinstance(temp_dir, Path) else temp_dir temp_dir.mkdir(parents=True, exist_ok=True) - if abc_args is None: - abc_args = OrderedDict() + abc_args = OrderedDict() if abc_args is None else abc_args - #Verify that files are Paths or convert them to Paths and check that they exist + # Verify that files are Paths or convert them to Paths and check that they exist architecture_file = verify_file(architecture_file, "Architecture") circuit_file = verify_file(circuit_file, "Circuit") output_netlist = verify_file(output_netlist, "Output netlist", should_exist=False) - + blackbox_latches_script = find_vtr_file("blackbox_latches.pl") clk_list = [] - clk_log_file = "report_clk_out.out" # # Parse arguments # - abc_flow_type = "iterative_bb" - abc_run_args = "" - use_old_latches_restoration_script = 0 - lut_size = None + (abc_args, + abc_flow_type, + lut_size, + abc_run_args, + use_old_latches_restoration_script) = parse_abc_args(abc_args) - if("iterative_bb" in abc_args): - abc_flow_type="iterative_bb" - del abc_args["iterative_bb"] - if("blanket_bb" in abc_args): - abc_flow_type="blanket_bb" - del abc_args["blanket_bb"] - if("once_bb" in abc_args): - abc_flow_type="once_bb" - del abc_args["once_bb"] - if("use_old_latches_restoration_script" in abc_args): - use_old_latches_restoration_script = 1 - del abc_args["use_old_latches_restoration_script"] - if("lut_size" in abc_args): - lut_size=abc_args["lut_size"] - del abc_args["lut_size"] - for key, value in abc_args.items(): - if value == True: - abc_run_args += ["--" + arg] - elif value == False: - pass - else: - abc_run_args += ["--" + arg, str(value)] - - if(lut_size is None): - lut_size = determine_lut_size(str(architecture_file)) + lut_size = determine_lut_size(str(architecture_file)) if lut_size is None else lut_size - populate_clock_list(circuit_file,blackbox_latches_script,clk_list,command_runner,temp_dir,"report_clocks.abc.out") - - if abc_exec is None: - abc_exec = find_vtr_file('abc', is_executable=True) - - if abc_rc is None: - abc_rc = Path(abc_exec).parent / 'abc.rc' + populate_clock_list( + circuit_file, + blackbox_latches_script, + clk_list, + command_runner, + temp_dir + ) - shutil.copyfile(str(abc_rc), str(temp_dir / 'abc.rc')) + abc_exec = find_vtr_file("abc", is_executable=True) if abc_exec is None else abc_exec + abc_rc = Path(abc_exec).parent / "abc.rc" if abc_rc is None else abc_rc - + shutil.copyfile(str(abc_rc), str(temp_dir / "abc.rc")) - iterations=len(clk_list) - - if(iterations==0 or abc_flow_type != "iterative_bb"): - iterations=1 + iterations = len(clk_list) + iterations = 1 if iterations == 0 or abc_flow_type != "iterative_bb" else iterations original_script = abc_script input_file = circuit_file.name for i in range(0, iterations): - pre_abc_blif= temp_dir / (str(i)+"_" + circuit_file.name) - post_abc_blif = temp_dir / (str(i)+"_"+output_netlist.name) - post_abc_raw_blif = temp_dir / (str(i)+"_"+output_netlist.with_suffix('').stem+".raw.abc.blif") - if(abc_flow_type=="blanket_bb"): - cmd = [blackbox_latches_script, "--input", input_file,"--output",pre_abc_blif.name] - command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=str(i)+"_blackboxing_latch.out", indent_depth=1) - - elif(len(clk_list)>i): - cmd = [blackbox_latches_script,"--clk_list", clk_list[i], "--input", input_file,"--output",pre_abc_blif.name] - command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=str(i)+"_blackboxing_latch.out", indent_depth=1) + pre_abc_blif = temp_dir / (str(i) + "_" + circuit_file.name) + post_abc_blif = temp_dir / (str(i) + "_" + output_netlist.name) + post_abc_raw_blif = temp_dir / ( + str(i) + "_" + output_netlist.with_suffix("").stem + ".raw.abc.blif" + ) + if abc_flow_type == "blanket_bb": + command_runner.run_system_command( + [ + blackbox_latches_script, + "--input", + input_file, + "--output", + pre_abc_blif.name, + ], + temp_dir=temp_dir, + log_filename=str(i) + "_blackboxing_latch.out", + indent_depth=1, + ) + + elif len(clk_list) > i: + command_runner.run_system_command( + [ + blackbox_latches_script, + "--clk_list", + clk_list[i], + "--input", + input_file, + "--output", + pre_abc_blif.name, + ], + temp_dir=temp_dir, + log_filename=str(i) + "_blackboxing_latch.out", + indent_depth=1, + ) else: pre_abc_blif = input_file - - if abc_script is None: - - abc_script = ['echo ""', - 'echo "Load Netlist"', - 'echo "============"', - 'read {pre_abc_blif}'.format(pre_abc_blif=pre_abc_blif.name), - 'time', - 'echo ""', - 'echo "Circuit Info"', - 'echo "=========="', - 'print_stats', - 'print_latch', - 'time', - 'echo ""', - 'echo "LUT Costs"', - 'echo "========="', - 'print_lut', - 'time', - 'echo ""', - 'echo "Logic Opt + Techmap"', - 'echo "==================="', - 'strash', - 'ifraig -v', - 'scorr -v', - 'dc2 -v', - 'dch -f', - 'if -K {lut_size} -v'.format(lut_size=lut_size), - 'mfs2 -v', - 'print_stats', - 'time', - 'echo ""', - 'echo "Output Netlist"', - 'echo "=============="', - 'write_hie {pre_abc_blif} {post_abc_raw_blif}'.format(pre_abc_blif=pre_abc_blif.name, post_abc_raw_blif=post_abc_raw_blif.name), - 'time;'] - - if(use_old_abc_script): - abc_script = ['read {pre_abc_blif}'.format(pre_abc_blif=pre_abc_blif.name), - 'time', - 'resyn', - 'resyn2', - 'if -K {lut_size}'.format(lut_size=lut_size), - 'time', - 'scleanup', - 'write_hie {pre_abc_blif} {post_abc_raw_blif}'.format(pre_abc_blif=pre_abc_blif.name, post_abc_raw_blif=post_abc_raw_blif.name), - 'print_stats'] - - abc_script = "; ".join(abc_script) + + abc_script = ( + "; ".join([ + 'echo ""', + 'echo "Load Netlist"', + 'echo "============"', + "read {pre_abc_blif}".format(pre_abc_blif=pre_abc_blif.name), + "time", + 'echo ""', + 'echo "Circuit Info"', + 'echo "=========="', + "print_stats", + "print_latch", + "time", + 'echo ""', + 'echo "LUT Costs"', + 'echo "========="', + "print_lut", + "time", + 'echo ""', + 'echo "Logic Opt + Techmap"', + 'echo "==================="', + "strash", + "ifraig -v", + "scorr -v", + "dc2 -v", + "dch -f", + "if -K {lut_size} -v".format(lut_size=lut_size), + "mfs2 -v", + "print_stats", + "time", + 'echo ""', + 'echo "Output Netlist"', + 'echo "=============="', + "write_hie {pre_abc_blif} {post_abc_raw_blif}".format( + pre_abc_blif=pre_abc_blif.name, + post_abc_raw_blif=post_abc_raw_blif.name, + ), + "time;" + ]) + if abc_script is None + else + "; ".join([ + "read {pre_abc_blif}".format(pre_abc_blif=pre_abc_blif.name), + "time", + "resyn", + "resyn2", + "if -K {lut_size}".format(lut_size=lut_size), + "time", + "scleanup", + "write_hie {pre_abc_blif} {post_abc_raw_blif}".format( + pre_abc_blif=pre_abc_blif.name, + post_abc_raw_blif=post_abc_raw_blif.name, + ), + "print_stats" + ]) + if use_old_abc_script + else abc_script + ) cmd = [abc_exec, '-c', abc_script] - - if(abc_run_args): + if abc_run_args: cmd.append(abc_run_args) - log_file = Path(log_filename) - command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_file.stem+str(i)+log_file.suffix, indent_depth=1) - - if(abc_flow_type != "blanket_bb" and len(clk_list)>i): - cmd = [blackbox_latches_script,"--restore", clk_list[i], "--input", post_abc_raw_blif.name,"--output",post_abc_blif.name] - command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename="restore_latch" + str(i) + ".out", indent_depth=1) + + command_runner.run_system_command( + cmd, + temp_dir=temp_dir, + log_filename=Path(log_filename).stem + str(i) + Path(log_filename).suffix, + indent_depth=1, + ) + + if abc_flow_type != "blanket_bb" and len(clk_list) > i: + command_runner.run_system_command( + [ + blackbox_latches_script, + "--restore", + clk_list[i], + "--input", + post_abc_raw_blif.name, + "--output", + post_abc_blif.name, + ], + temp_dir=temp_dir, + log_filename="restore_latch" + str(i) + ".out", + indent_depth=1, + ) else: - if(use_old_latches_restoration_script): - restore_multiclock_info_script = find_vtr_file("restore_multiclock_latch_information.pl") - else: - restore_multiclock_info_script = find_vtr_file("restore_multiclock_latch.pl") - - cmd = [restore_multiclock_info_script, pre_abc_blif.name, post_abc_raw_blif.name ,post_abc_blif.name] - command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename="restore_latch" + str(i) + ".out", indent_depth=1) - if(abc_flow_type != "iterative_bb"): + restore_multiclock_info_script = ( + find_vtr_file( + "restore_multiclock_latch_information.pl" + ) + if use_old_latches_restoration_script + else + find_vtr_file( + "restore_multiclock_latch.pl" + ) + ) + command_runner.run_system_command( + [ + restore_multiclock_info_script, + pre_abc_blif.name, + post_abc_raw_blif.name, + post_abc_blif.name, + ], + temp_dir=temp_dir, + log_filename="restore_latch" + str(i) + ".out", + indent_depth=1, + ) + if abc_flow_type != "iterative_bb": break - + abc_script = original_script input_file = post_abc_blif.name - - cmd = [blackbox_latches_script, "--input", post_abc_blif.name,"--output",output_netlist.name,"--vanilla"] - command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename="restore_latch" + str(i) + ".out", indent_depth=1) - if(not keep_intermediate_files): - files = [] + command_runner.run_system_command( + [ + blackbox_latches_script, + "--input", + post_abc_blif.name, + "--output", + output_netlist.name, + "--vanilla", + ], + temp_dir=temp_dir, + log_filename="restore_latch" + str(i) + ".out", + indent_depth=1, + ) + if not keep_intermediate_files: for file in temp_dir.iterdir(): - if file.suffix in ('.dot','.v','.rc'): - files.append(file) - for p in files: - p.unlink() - + if file.suffix in (".dot", ".v", ".rc"): + file.unlink() +#pylint: enable=too-many-arguments, too-many-locals +def parse_abc_args(abc_args): + """ + function to parse abc_args + """ + abc_flow_type = "iterative_bb" + abc_run_args = "" + lut_size = None + use_old_latches_restoration_script = False + if "iterative_bb" in abc_args: + abc_flow_type = "iterative_bb" + del abc_args["iterative_bb"] + if "blanket_bb" in abc_args: + abc_flow_type = "blanket_bb" + del abc_args["blanket_bb"] + if "once_bb" in abc_args: + abc_flow_type = "once_bb" + del abc_args["once_bb"] + if "use_old_latches_restoration_script" in abc_args: + use_old_latches_restoration_script = True + del abc_args["use_old_latches_restoration_script"] + if "lut_size" in abc_args: + lut_size = abc_args["lut_size"] + del abc_args["lut_size"] -def populate_clock_list(circuit_file,blackbox_latches_script,clk_list,command_runner,temp_dir,log_filename): + for arg, value in abc_args.items(): + if isinstance(value, bool) and value: + abc_run_args += ["--" + arg] + elif isinstance(value, (str, int, float)): + abc_run_args += ["--" + arg, str(value)] + else: + pass + return abc_args, abc_flow_type, lut_size, abc_run_args, use_old_latches_restoration_script + +def populate_clock_list( + circuit_file, + blackbox_latches_script, + clk_list, + command_runner, + temp_dir +): + """ + function to populate the clock list + """ clk_list_path = temp_dir / "report_clk.out" - cmd = [blackbox_latches_script, "--input", circuit_file.name,"--output_list", clk_list_path.name] - command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) - with clk_list_path.open('r') as f: - for line in f.readlines(): - clk_list.append(line.strip('\n')) - -def run_lec(reference_netlist, implementation_netlist, command_runner=CommandRunner(), temp_dir=Path("."), log_filename="abc.lec.out", abc_exec=None): + cmd = [ + blackbox_latches_script, + "--input", + circuit_file.name, + "--output_list", + clk_list_path.name, + ] + command_runner.run_system_command( + cmd, temp_dir=temp_dir, log_filename="report_clocks.abc.out", indent_depth=1 + ) + with clk_list_path.open("r") as file: + for line in file.readlines(): + clk_list.append(line.strip("\n")) + +#pylint: disable=too-many-arguments +def run_lec( + reference_netlist, + implementation_netlist, + command_runner=CommandRunner(), + temp_dir=Path("."), + log_filename="abc.lec.out", + abc_exec=None, +): """ Run Logical Equivalence Checking (LEC) between two netlists using ABC @@ -237,57 +353,82 @@ def run_lec(reference_netlist, implementation_netlist, command_runner=CommandRun Arguments ========= - reference_netlist : + reference_netlist : The reference netlist to be commpared to - - implementation_netlist : + + implementation_netlist : The implemeted netlist to compare to the reference netlist - + Other Parameters ---------------- - command_runner : + command_runner : A CommandRunner object used to run system commands - - temp_dir : + + temp_dir : Directory to run in (created if non-existent) - - log_filename : + + log_filename : File to log result to - - abc_exec : + + abc_exec : ABC executable to be run - + """ temp_dir = Path(temp_dir) if not isinstance(temp_dir, Path) else temp_dir temp_dir.mkdir(parents=True, exist_ok=True) if abc_exec is None: - abc_exec = find_vtr_file('abc', is_executable=True) + abc_exec = find_vtr_file("abc", is_executable=True) - abc_script = 'dsec {ref} {imp}'.format(ref=reference_netlist, imp=implementation_netlist), + abc_script = ( + "dsec {ref} {imp}".format( + ref=reference_netlist, imp=implementation_netlist + ), + ) abc_script = "; ".join(abc_script) - cmd = [abc_exec, '-c', abc_script] + cmd = [abc_exec, "-c", abc_script] - output, returncode = command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) + output, _ = command_runner.run_system_command( + cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1 + ) - #Check if ABC's LEC engine passed + # Check if ABC's LEC engine passed lec_passed, errored = check_abc_lec_status(output) if errored: - abc_script = 'cec {ref} {imp}'.format(ref=reference_netlist, imp=implementation_netlist), + abc_script = ( + "cec {ref} {imp}".format(ref=reference_netlist, imp=implementation_netlist), + ) abc_script = "; ".join(abc_script) - cmd = [abc_exec, '-c', abc_script] - output, returncode = command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename="abc.cec.out", indent_depth=1) + cmd = [abc_exec, "-c", abc_script] + output, _ = command_runner.run_system_command( + cmd, temp_dir=temp_dir, log_filename="abc.cec.out", indent_depth=1 + ) lec_passed, errored = check_abc_lec_status(output) if lec_passed is None: - raise InspectError("Could not determine Logical Equivalence Check status between {input} <-> {output}".format(input=reference_netlist, output=implementation_netlist), filename=log_filename) - elif lec_passed is False: - raise InspectError("Logical Equivalence Check failed between {input} <-> {output}".format(input=reference_netlist, output=implementation_netlist), filename=log_filename) - + raise InspectError( + "Couldn't determine Logical Equivalence status between {input} <-> {output}".format( + input=reference_netlist, output=implementation_netlist + ), + filename=log_filename, + ) + if lec_passed is False: + raise InspectError( + "Logical Equivalence Check failed between {input} <-> {output}".format( + input=reference_netlist, output=implementation_netlist + ), + filename=log_filename, + ) + assert lec_passed +#pylint: enable=too-many-arguments def check_abc_lec_status(output): + """ + Reads abc_lec output and determines if the files were equivelent and + if there were errors when preforming lec. + """ equivalent = None errored = False for line in output: @@ -298,6 +439,5 @@ def check_abc_lec_status(output): elif line.startswith("Networks are equivalent"): equivalent = True - #Returns None if could not determine LEC status + # Returns None if could not determine LEC status return equivalent, errored - diff --git a/vtr_flow/scripts/python_libs/vtr/ace/__init__.py b/vtr_flow/scripts/python_libs/vtr/ace/__init__.py index 9fb6597793e..9666f7710ac 100644 --- a/vtr_flow/scripts/python_libs/vtr/ace/__init__.py +++ b/vtr_flow/scripts/python_libs/vtr/ace/__init__.py @@ -1 +1,4 @@ +""" + init for the ACE module +""" from .ace import run diff --git a/vtr_flow/scripts/python_libs/vtr/ace/ace.py b/vtr_flow/scripts/python_libs/vtr/ace/ace.py index 8af4d157f2e..441dd892e7b 100644 --- a/vtr_flow/scripts/python_libs/vtr/ace/ace.py +++ b/vtr_flow/scripts/python_libs/vtr/ace/ace.py @@ -1,74 +1,99 @@ -from vtr import find_vtr_file, verify_file,CommandRunner +""" + Module to run ACE with its various options +""" from pathlib import Path -def run(circuit_file, old_netlist, output_netlist, output_activity_file, - command_runner=CommandRunner(), temp_dir=Path("."), log_filename="ace.out", - ace_exec=None, ace_seed = 1): +from vtr import find_vtr_file, verify_file, CommandRunner +#pylint: disable=too-many-arguments +def run( + circuit_file, + old_netlist, + output_netlist, + output_activity_file, + command_runner=CommandRunner(), + temp_dir=Path("."), + log_filename="ace.out", + ace_exec=None, + ace_seed=1, +): """ - Runs ACE for activity estimation + Runs ACE for activity estimation .. note :: Usage: vtr.ace.run(,,,[OPTIONS]) Arguments ========= - circuit_file : + circuit_file : Circuit file to optimize - old_netlist : + old_netlist : netlist to be anylized - - output_netlist : + + output_netlist : File name to output the resulting circuit to - output_activity_file : + output_activity_file : The output activity file - + Other Parameters ---------------- - command_runner : + command_runner : A CommandRunner object used to run system commands - - temp_dir : + + temp_dir : Directory to run in (created if non-existent) - - log_filename : + + log_filename : File to log result to - - ace_exec : + + ace_exec : ACE executable to be run - - ace_seed : + + ace_seed : The ACE seed """ temp_dir = Path(temp_dir) if not isinstance(temp_dir, Path) else temp_dir - #Verify that files are Paths or convert them to Paths and check that they exist + # Verify that files are Paths or convert them to Paths and check that they exist circuit_file = verify_file(circuit_file, "Circuit") old_netlist = verify_file(old_netlist, "Previous netlist") output_netlist = verify_file(output_netlist, "Output netlist", should_exist=False) - output_activity_file = verify_file(output_activity_file, "Output activity", should_exist=False) + output_activity_file = verify_file( + output_activity_file, "Output activity", should_exist=False + ) ace_clk_file = temp_dir / "ace_clk.txt" - ace_raw = temp_dir / (circuit_file.with_suffix('').stem + ".raw.ace.blif") + ace_raw = temp_dir / (circuit_file.with_suffix("").stem + ".raw.ace.blif") if ace_exec is None: - ace_exec = find_vtr_file('ace') - - ace_extraction_file = find_vtr_file("extract_clk_from_blif.py") + ace_exec = find_vtr_file("ace") - cmd =[ace_extraction_file, ace_clk_file.name, circuit_file.name] - command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename="ace_clk_extraction.out", indent_depth=1) + cmd = [find_vtr_file("extract_clk_from_blif.py"), ace_clk_file.name, circuit_file.name] + command_runner.run_system_command( + cmd, temp_dir=temp_dir, log_filename="ace_clk_extraction.out", indent_depth=1 + ) ace_clk = "" - with ace_clk_file.open('r') as f: - ace_clk = f.readline().strip('\n') - cmd = [ace_exec, - "-b", circuit_file.name, - "-c", ace_clk, - "-n", ace_raw.name, - "-o", output_activity_file.name, - "-s", str(ace_seed)] + with ace_clk_file.open("r") as file: + ace_clk = file.readline().strip("\n") + cmd = [ + ace_exec, + "-b", + circuit_file.name, + "-c", + ace_clk, + "-n", + ace_raw.name, + "-o", + output_activity_file.name, + "-s", + str(ace_seed), + ] - command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) + command_runner.run_system_command( + cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1 + ) clock_script = find_vtr_file("restore_multiclock_latch.pl") - - cmd = [ clock_script, old_netlist.name, ace_raw.name, output_netlist.name] - command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename="ace_clk_restore.out", indent_depth=1) + cmd = [clock_script, old_netlist.name, ace_raw.name, output_netlist.name] + command_runner.run_system_command( + cmd, temp_dir=temp_dir, log_filename="ace_clk_restore.out", indent_depth=1 + ) + #pylint: enable=too-many-arguments diff --git a/vtr_flow/scripts/python_libs/vtr/error.py b/vtr_flow/scripts/python_libs/vtr/error.py index f44520f5c13..4171fa887aa 100644 --- a/vtr_flow/scripts/python_libs/vtr/error.py +++ b/vtr_flow/scripts/python_libs/vtr/error.py @@ -1,6 +1,8 @@ """ Module to handle VTR flow erroring. """ + + class VtrError(Exception): """ Base class for VTR related exceptions @@ -11,7 +13,7 @@ class VtrError(Exception): def __init__(self, msg=""): self.msg = msg - super().__init__(msg=msg) + super().__init__() class CommandError(VtrError): diff --git a/vtr_flow/scripts/python_libs/vtr/flow.py b/vtr_flow/scripts/python_libs/vtr/flow.py index 1bdcab61102..c7e29591efe 100644 --- a/vtr_flow/scripts/python_libs/vtr/flow.py +++ b/vtr_flow/scripts/python_libs/vtr/flow.py @@ -81,7 +81,7 @@ def run( A CommandRunner object used to run system commands verbosity : - Weather to output error description or not + whether to output error description or not odin_args : A dictionary of keyword arguments to pass on to ODIN II @@ -143,7 +143,6 @@ def run( post_ace_netlist = temp_dir / (circuit_file.stem + ".ace" + netlist_ext) post_ace_activity_file = temp_dir / (circuit_file.stem + ".act") pre_vpr_netlist = temp_dir / (circuit_file.stem + ".pre-vpr" + netlist_ext) - post_vpr_netlist = temp_dir / "vpr.out" # If the user provided a .blif or .eblif netlist, we use that as the baseline for LEC # (ABC can't LEC behavioural verilog) @@ -252,23 +251,51 @@ def run( if route_fixed_w: # The User specified a fixed channel width + do_second_run = False + second_run_args = vpr_args + + if "write_rr_graph" in vpr_args: + do_second_run = True + + if "analysis" in vpr_args: + do_second_run = True + del vpr_args["analysis"] + + if "route" in vpr_args: + do_second_run = True + del vpr_args["route"] + vtr.vpr.run( architecture_copy, pre_vpr_netlist, circuit_copy.stem, - output_netlist=post_vpr_netlist, command_runner=command_runner, temp_dir=temp_dir, vpr_args=vpr_args, - rr_graph_ext=".xml", ) + if do_second_run: + #Run vpr again with additional parameters. + #This is used to ensure that files generated by VPR can be re-loaded by it + rr_graph_ext = ( + Path(second_run_args["write_rr_graph"]).suffix + if "write_rr_graph" in second_run_args + else ".xml" + ) + vtr.vpr.run_second_time( + architecture_copy, + pre_vpr_netlist, + circuit_copy.stem, + command_runner=command_runner, + temp_dir=temp_dir, + second_run_args=second_run_args, + rr_graph_ext=rr_graph_ext + ) else: # First find minW and then re-route at a relaxed W vtr.vpr.run_relax_w( architecture_copy, pre_vpr_netlist, circuit_copy.stem, - output_netlist=post_vpr_netlist, command_runner=command_runner, relax_w_factor=relax_w_factor, temp_dir=temp_dir, @@ -299,7 +326,6 @@ def run( circuit_copy.stem, command_runner=command_runner, vpr_args=vpr_args, - rr_graph_ext=".xml", temp_dir=temp_dir, ) diff --git a/vtr_flow/scripts/python_libs/vtr/inspect.py b/vtr_flow/scripts/python_libs/vtr/inspect.py index 58b1bed9eba..8abbfd57aa9 100644 --- a/vtr_flow/scripts/python_libs/vtr/inspect.py +++ b/vtr_flow/scripts/python_libs/vtr/inspect.py @@ -4,14 +4,15 @@ import re try: - #Try for the fast c-based version first + # Try for the fast c-based version first import xml.etree.cElementTree as ET except ImportError: - #Fall back on python implementation + # Fall back on python implementation import xml.etree.ElementTree as ET from vtr.error import InspectError + def determine_lut_size(architecture_file): """ Determines the maximum LUT size (K) in an architecture file. @@ -22,24 +23,26 @@ def determine_lut_size(architecture_file): lut_size = 0 saw_blif_names = False - for elem in arch_xml.findall('.//pb_type'): #Xpath recrusive search for 'pb_type' - blif_model = elem.get('blif_model') + for elem in arch_xml.findall(".//pb_type"): # Xpath recrusive search for 'pb_type' + blif_model = elem.get("blif_model") if blif_model and blif_model == ".names": saw_blif_names = True - input_port = elem.find('input') + input_port = elem.find("input") - input_width = int(input_port.get('num_pins')) + input_width = int(input_port.get("num_pins")) assert input_width > 0 - #Keep the maximum lut size found (i.e. fracturable architectures) + # Keep the maximum lut size found (i.e. fracturable architectures) lut_size = max(lut_size, input_width) if saw_blif_names and lut_size == 0: - raise InspectError(msg="Could not identify valid LUT size (K)", - filename=architecture_file) + raise InspectError( + msg="Could not identify valid LUT size (K)", filename=architecture_file + ) return lut_size + def determine_memory_addr_width(architecture_file): """ Determines the maximum RAM block address width in an architecture file @@ -51,32 +54,37 @@ def determine_memory_addr_width(architecture_file): mem_addr_width = 0 saw_ram = False - for elem in arch_xml.findall('.//pb_type'): #XPATH for recursive search - blif_model = elem.get('blif_model') + for elem in arch_xml.findall(".//pb_type"): # XPATH for recursive search + blif_model = elem.get("blif_model") if blif_model and "port_ram" in blif_model: saw_ram = True - for input_port in elem.findall('input'): - port_name = input_port.get('name') + for input_port in elem.findall("input"): + port_name = input_port.get("name") if "addr" in port_name: - input_width = int(input_port.get('num_pins')) + input_width = int(input_port.get("num_pins")) mem_addr_width = max(mem_addr_width, input_width) if saw_ram and mem_addr_width == 0: - raise InspectError(msg="Could not identify RAM block address width", - filename=architecture_file) + raise InspectError( + msg="Could not identify RAM block address width", filename=architecture_file + ) return mem_addr_width + def determine_min_w(log_filename): """ determines the miniumum width. """ - min_w_regex = re.compile(r"\s*Best routing used a channel width factor of (?P\d+).") + min_w_regex = re.compile( + r"\s*Best routing used a channel width factor of (?P\d+)." + ) with open(log_filename) as file: for line in file: match = min_w_regex.match(line) if match: return int(match.group("min_w")) - raise InspectError(msg="Failed to find minimum channel width.", - filename=log_filename) + raise InspectError( + msg="Failed to find minimum channel width.", filename=log_filename + ) diff --git a/vtr_flow/scripts/python_libs/vtr/odin/__init__.py b/vtr_flow/scripts/python_libs/vtr/odin/__init__.py index 63d040329cd..b55d5c160cd 100644 --- a/vtr_flow/scripts/python_libs/vtr/odin/__init__.py +++ b/vtr_flow/scripts/python_libs/vtr/odin/__init__.py @@ -1 +1,4 @@ +""" + init for the ODIN module +""" from .odin import run diff --git a/vtr_flow/scripts/python_libs/vtr/odin/odin.py b/vtr_flow/scripts/python_libs/vtr/odin/odin.py index d9a793aedc9..b224c37e4e6 100644 --- a/vtr_flow/scripts/python_libs/vtr/odin/odin.py +++ b/vtr_flow/scripts/python_libs/vtr/odin/odin.py @@ -1,17 +1,31 @@ +""" + Module to run ODIN II with its various arguments +""" import shutil +from collections import OrderedDict from pathlib import Path -from vtr import find_vtr_file, file_replace, determine_memory_addr_width, verify_file, CommandRunner - -def run(architecture_file, circuit_file, - output_netlist, - command_runner=CommandRunner(), - temp_dir=Path("."), - odin_args="--adder_type default", - log_filename="odin.out", - odin_exec=None, - odin_config=None, - min_hard_mult_size=3, - min_hard_adder_size=1): +from vtr import ( + find_vtr_file, + file_replace, + determine_memory_addr_width, + verify_file, + CommandRunner, +) + +#pylint: disable=too-many-arguments, too-many-locals +def run( + architecture_file, + circuit_file, + output_netlist, + command_runner=CommandRunner(), + temp_dir=Path("."), + odin_args="--adder_type default", + log_filename="odin.out", + odin_exec=None, + odin_config=None, + min_hard_mult_size=3, + min_hard_adder_size=1 +): """ Runs ODIN II on the specified architecture file and circuit file @@ -19,41 +33,43 @@ def run(architecture_file, circuit_file, Arguments ========= - architecture_file : + architecture_file : Architecture file to target - - circuit_file : + + circuit_file : Circuit file to optimize - - output_netlist : + + output_netlist : File name to output the resulting circuit to Other Parameters ---------------- - command_runner : + command_runner : A CommandRunner object used to run system commands - - temp_dir : + + temp_dir : Directory to run in (created if non-existent) - - odin_args: - A dictionary of keyword arguments to pass on to ODIN II - - log_filename : + + odin_args: + A dictionary of keyword arguments to pass on to ODIN II + + log_filename : File to log result to - - odin_exec: + + odin_exec: ODIN II executable to be run - - odin_config: + + odin_config: The ODIN II configuration file - - min_hard_mult_size : - Tells ODIN II the minimum multiplier size that should be implemented using hard multiplier (if available) - - min_hard_adder_size : - Tells ODIN II the minimum adder size that should be implemented using hard adder (if available). - + + min_hard_mult_size : + Tells ODIN II the minimum multiplier size that should be implemented using + hard multiplier (if available) + + min_hard_adder_size : + Tells ODIN II the minimum adder size that should be implemented + using hard adder (if available). + """ temp_dir = Path(temp_dir) if not isinstance(temp_dir, Path) else temp_dir temp_dir.mkdir(parents=True, exist_ok=True) @@ -61,56 +77,86 @@ def run(architecture_file, circuit_file, if odin_args is None: odin_args = OrderedDict() - #Verify that files are Paths or convert them to Paths and check that they exist + # Verify that files are Paths or convert them to Paths and check that they exist architecture_file = verify_file(architecture_file, "Architecture") circuit_file = verify_file(circuit_file, "Circuit") output_netlist = verify_file(output_netlist, "Output netlist", False) if odin_exec is None: - odin_exec = find_vtr_file('odin_II', is_executable=True) + odin_exec = find_vtr_file("odin_II", is_executable=True) if odin_config is None: - odin_base_config = find_vtr_file('basic_odin_config_split.xml') + odin_base_config = find_vtr_file("basic_odin_config_split.xml") - #Copy the config file + # Copy the config file odin_config = "odin_config.xml" odin_config_full_path = str(temp_dir / odin_config) shutil.copyfile(odin_base_config, odin_config_full_path) - #Update the config file - file_replace(odin_config_full_path, { - "XXX": circuit_file.name, - "YYY": architecture_file.name, - "ZZZ": output_netlist.name, - "PPP": determine_memory_addr_width(str(architecture_file)), - "MMM": min_hard_mult_size, - "AAA": min_hard_adder_size, - }) + # Update the config file + file_replace( + odin_config_full_path, + { + "XXX": circuit_file.name, + "YYY": architecture_file.name, + "ZZZ": output_netlist.name, + "PPP": determine_memory_addr_width(str(architecture_file)), + "MMM": min_hard_mult_size, + "AAA": min_hard_adder_size, + }, + ) disable_odin_xml = False - if("disable_odin_xml" in odin_args): - disable_odin_xml=True + if "disable_odin_xml" in odin_args: + disable_odin_xml = True del odin_args["disable_odin_xml"] use_odin_simulation = False - if("use_odin_simulation" in odin_args): - use_odin_simulation=True + if "use_odin_simulation" in odin_args: + use_odin_simulation = True del odin_args["use_odin_simulation"] - + cmd = [odin_exec, "-c", odin_config] for arg, value in odin_args.items(): - if value == True: + if isinstance(value, bool) and value: cmd += ["--" + arg] - elif value == False: - pass - else: + elif isinstance(value, (str, int, float)): cmd += ["--" + arg, str(value)] + else: + pass cmd += ["-U0"] - if(disable_odin_xml): - cmd += ["-a",architecture_file.name, "-V", circuit_file.name, "-o",output_netlist.name] + if disable_odin_xml: + cmd += [ + "-a", + architecture_file.name, + "-V", + circuit_file.name, + "-o", + output_netlist.name, + ] - command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) + command_runner.run_system_command( + cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1 + ) - if(use_odin_simulation): + if use_odin_simulation: sim_dir = temp_dir / "simulation_init" sim_dir.mkdir() - cmd = [odin_exec, "-b", output_netlist.name, -a, architecture_file.name, "-sim_dir", str(sim_dir),"-g", "100", "--best_coverage", "-U0"] - command_runner.run_system_command(cmd,temp_dir=temp_dir, log_filename="sim_produce_vector.out",indent_depth=1) \ No newline at end of file + cmd = [ + odin_exec, + "-b", + output_netlist.name, + "-a", + architecture_file.name, + "-sim_dir", + str(sim_dir), + "-g", + "100", + "--best_coverage", + "-U0", + ] + command_runner.run_system_command( + cmd, + temp_dir=temp_dir, + log_filename="sim_produce_vector.out", + indent_depth=1, + ) +#pylint: enable=too-many-arguments, too-many-locals \ No newline at end of file diff --git a/vtr_flow/scripts/python_libs/vtr/util.py b/vtr_flow/scripts/python_libs/vtr/util.py index b32c9354f23..40002daa2f2 100644 --- a/vtr_flow/scripts/python_libs/vtr/util.py +++ b/vtr_flow/scripts/python_libs/vtr/util.py @@ -26,8 +26,9 @@ class RawDefaultHelpFormatter( formatting of description/epilog """ -#pylint: disable=too-many-arguments, too-many-instance-attributes, too-few-public-methods, too-many-locals -class CommandRunner(): + +# pylint: disable=too-many-arguments, too-many-instance-attributes, too-few-public-methods, too-many-locals +class CommandRunner: """ An object for running system commands with timeouts, memory limits and varying verbose-ness @@ -45,6 +46,7 @@ class CommandRunner(): indent: The string specifying a single indent (used in verbose mode) valgrind: Indicates if commands should be run with valgrind """ + def __init__( self, timeout_sec=None, @@ -97,25 +99,39 @@ def run_system_command( temp_dir = Path(temp_dir) if not isinstance(temp_dir, Path) else temp_dir # If no log file is specified the name is based on the executed command - log_filename = PurePath(orig_cmd[0]).name + ".out" if log_filename is None else log_filename + log_filename = ( + PurePath(orig_cmd[0]).name + ".out" + if log_filename is None + else log_filename + ) # Limit memory usage? memory_limit = ["ulimit", "-Sv", "{val};".format(val=self._max_memory_mb)] - cmd = memory_limit + cmd if self._max_memory_mb and check_cmd(memory_limit[0]) else cmd + cmd = ( + memory_limit + cmd + if self._max_memory_mb and check_cmd(memory_limit[0]) + else cmd + ) # Enable memory tracking? memory_tracking = ["/usr/bin/env", "time", "-v"] if self._track_memory and check_cmd(memory_tracking[0]): - cmd = memory_tracking + [ - "valgrind", - "--leak-check=full", - "--suppressions={}".format(find_vtr_file("valgrind.supp")), - "--error-exitcode=1", - "--errors-for-leak-kinds=none", - "--track-origins=yes", - "--log-file=valgrind.log", - "--error-limit=no", - ] + cmd if self._valgrind else memory_tracking + cmd + cmd = ( + memory_tracking + + [ + "valgrind", + "--leak-check=full", + "--suppressions={}".format(find_vtr_file("valgrind.supp")), + "--error-exitcode=1", + "--errors-for-leak-kinds=none", + "--track-origins=yes", + "--log-file=valgrind.log", + "--error-limit=no", + ] + + cmd + if self._valgrind + else memory_tracking + cmd + ) # Flush before calling subprocess to ensure output is ordered # correctly if stdout is buffered @@ -199,13 +215,16 @@ def run_system_command( ) return cmd_output, cmd_returncode - #pylint: enable=too-many-arguments, too-many-instance-attributes, too-few-public-methods, too-many-locals + + # pylint: enable=too-many-arguments, too-many-instance-attributes, too-few-public-methods, too-many-locals + + def check_cmd(command): """ Return True if command can be run, False otherwise. """ - return Path(command).exists() + return Path(command).exists() def write_tab_delimitted_csv(filepath, rows): diff --git a/vtr_flow/scripts/python_libs/vtr/vpr/__init__.py b/vtr_flow/scripts/python_libs/vtr/vpr/__init__.py index 3c0a3854793..895950332ef 100644 --- a/vtr_flow/scripts/python_libs/vtr/vpr/__init__.py +++ b/vtr_flow/scripts/python_libs/vtr/vpr/__init__.py @@ -1 +1,4 @@ -from .vpr import run,run_relax_w,cmp_full_vs_incr_STA +""" + __init__ for the VPR Module +""" +from .vpr import run, run_relax_w, cmp_full_vs_incr_sta, run_second_time diff --git a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py index ea826963989..97f1dc4e7df 100644 --- a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py +++ b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py @@ -1,65 +1,80 @@ -import shutil +""" + Module to interact with VPR and its various options +""" +from collections import OrderedDict from pathlib import Path -from vtr import find_vtr_file, CommandRunner, print_verbose, relax_w, determine_lut_size, determine_min_w, verify_file +from vtr import ( + find_vtr_file, + CommandRunner, + relax_w, + determine_min_w, + verify_file +) from vtr.error import InspectError -def run_relax_w(architecture, circuit, circuit_name=None, command_runner=CommandRunner(), temp_dir=Path("."), - relax_w_factor=1.3, vpr_exec=None, logfile_base="vpr", - vpr_args=None, output_netlist=None): +#pylint: disable=too-many-arguments +def run_relax_w( + architecture, + circuit, + circuit_name=None, + command_runner=CommandRunner(), + temp_dir=Path("."), + relax_w_factor=1.3, + vpr_exec=None, + logfile_base="vpr", + vpr_args=None, +): """ Runs VPR twice: 1st: To find the minimum channel width 2nd: At relaxed channel width (e.g. for critical path delay) - - .. note :: Usage: vtr.vpr.run_relax_w(,,,[OPTIONS]) + + .. note :: Use: vtr.vpr.run_relax_w(,,[OPTIONS]) Arguments ========= - architecture: + architecture: Architecture file - - circuit: + + circuit: Input circuit netlist Other Parameters ---------------- - circuit_name: + circuit_name: Name of the circuit file - command_runner: + command_runner: CommandRunner object - - temp_dir: + + temp_dir: Directory to run in - - relax_w_factor: + + relax_w_factor: Factor by which to relax minimum channel width for critical path delay routing - - vpr_exec: + + vpr_exec: Path to the VPR executable - - logfile_base: - Base name for log files (e.g. "vpr" produces vpr.min_w.out, vpr.relaxed_W.out) - - vpr_args: + + logfile_base: + Base name for log files (e.g. "vpr" produces vpr.min_w.out, vpr.relaxed_w.out) + + vpr_args: Extra arguments for VPR - - output_netlist: - Output implementation netlist to generate """ if vpr_args is None: vpr_args = OrderedDict() temp_dir = Path(temp_dir) if not isinstance(temp_dir, Path) else temp_dir temp_dir.mkdir(parents=True, exist_ok=True) - #Verify that files are Paths or convert them to Paths and check that they exist + # Verify that files are Paths or convert them to Paths and check that they exist architecture = verify_file(architecture, "Architecture") circuit = verify_file(circuit, "Circuit") - vpr_min_w_log = '.'.join([logfile_base, "out"]) - vpr_relaxed_W_log = '.'.join([logfile_base, "crit_path", "out"]) + vpr_min_w_log = ".".join([logfile_base, "out"]) + vpr_relaxed_w_log = ".".join([logfile_base, "crit_path", "out"]) max_router_iterations = None if "max_router_iterations" in vpr_args: @@ -76,230 +91,315 @@ def run_relax_w(architecture, circuit, circuit_name=None, command_runner=Command del vpr_args["route"] if vpr_exec is None: - vpr_exec = find_vtr_file('vpr', is_executable=True) - - run(architecture, circuit, circuit_name, command_runner, temp_dir, log_filename=vpr_min_w_log, vpr_exec=vpr_exec, vpr_args=vpr_args) - - if ('pack' in vpr_args or 'place' in vpr_args) and 'route' not in vpr_args: - #Don't look for min W if routing was not run + vpr_exec = find_vtr_file("vpr", is_executable=True) + + run( + architecture, + circuit, + circuit_name, + command_runner, + temp_dir, + log_filename=vpr_min_w_log, + vpr_exec=vpr_exec, + vpr_args=vpr_args, + ) + + if ("pack" in vpr_args or "place" in vpr_args) and "route" not in vpr_args: + # Don't look for min W if routing was not run return if max_router_iterations: - vpr_args["max_router_iterations"]=max_router_iterations - min_w = determine_min_w(str(temp_dir / vpr_min_w_log)) + vpr_args["max_router_iterations"] = max_router_iterations + min_w = determine_min_w(str(temp_dir / vpr_min_w_log)) - relaxed_W = relax_w(min_w, relax_w_factor) + relaxed_w = relax_w(min_w, relax_w_factor) - vpr_args['route'] = True #Re-route only - vpr_args['route_chan_width'] = relaxed_W #At a fixed channel width + vpr_args["route"] = True # Re-route only + vpr_args["route_chan_width"] = relaxed_w # At a fixed channel width - #VPR does not support performing routing when fixed pins + # VPR does not support performing routing when fixed pins # are specified, and placement is not run; so remove the option - run(architecture, circuit, circuit_name, command_runner, temp_dir, log_filename=vpr_relaxed_W_log, vpr_exec=vpr_exec, vpr_args=vpr_args, check_for_second_run=False) - - -def run(architecture, circuit, circuit_name=None, command_runner=CommandRunner(), temp_dir=Path("."), output_netlist=None, - log_filename="vpr.out", vpr_exec=None, vpr_args=None,check_for_second_run=True,rr_graph_ext=".xml",): + run( + architecture, + circuit, + circuit_name, + command_runner, + temp_dir, + log_filename=vpr_relaxed_w_log, + vpr_exec=vpr_exec, + vpr_args=vpr_args, + ) + + +def run( + architecture, + circuit, + circuit_name=None, + command_runner=CommandRunner(), + temp_dir=Path("."), + log_filename="vpr.out", + vpr_exec=None, + vpr_args=None, +): """ Runs VPR with the specified configuration - .. note :: Usage: vtr.vpr.run(,,,[OPTIONS]) + .. note :: Usage: vtr.vpr.run(,,[OPTIONS]) Arguments ========= - architecture: + architecture: Architecture file - + circuit: Input circuit file - + Other Parameters ---------------- - circuit_name: + circuit_name: Name of the circuit file - command_runner: + command_runner: CommandRunner object - + temp_dir: Directory to run in - output_netlist: - Output implementation netlist to generate - - log_filename : + log_filename : File to log result to - - vpr_exec: + + vpr_exec: Path to the VPR executable - - vpr_args: + + vpr_args: Extra arguments for VPR - - check_for_second_run: - enables checking for arguments in vpr_args that require a second run of VPR ie analysis + """ - + if vpr_args is None: vpr_args = OrderedDict() temp_dir = Path(temp_dir) if not isinstance(temp_dir, Path) else temp_dir temp_dir.mkdir(parents=True, exist_ok=True) if vpr_exec is None: - vpr_exec = find_vtr_file('vpr', is_executable=True) + vpr_exec = find_vtr_file("vpr", is_executable=True) - #Verify that files are Paths or convert them to Paths and check that they exist + # Verify that files are Paths or convert them to Paths and check that they exist architecture = verify_file(architecture, "Architecture") circuit = verify_file(circuit, "Circuit") cmd = [] if circuit_name: - cmd = [vpr_exec, architecture.name, circuit_name, "--circuit_file", circuit.name] + cmd = [ + vpr_exec, + architecture.name, + circuit_name, + "--circuit_file", + circuit.name, + ] else: - cmd = [vpr_exec, architecture.name, circuit.name] - #Enable netlist generation - #if output_netlist: - #vpr_args['gen_postsynthesis_netlist'] = output_netlist - - #Translate arbitrary keyword arguments into options for VPR - do_second_run = False - second_run_args = vpr_args + cmd = [vpr_exec, architecture.name, circuit.name] - if check_for_second_run and "write_rr_graph" in vpr_args: - do_second_run = True - - if check_for_second_run and "analysis" in vpr_args: - do_second_run = True - del vpr_args["analysis"] - - if check_for_second_run and "route" in vpr_args: - do_second_run = True - del vpr_args["route"] + # Translate arbitrary keyword arguments into options for VPR for arg, value in vpr_args.items(): - if value == True: + if isinstance(value, bool) and value: cmd += ["--" + arg] - elif value == False: - pass - else: - if isinstance(value,list): - cmd += ["--" + arg] - for i in range(len(value)): - cmd += [str(value[i])] - else: - cmd += ["--" + arg, str(value)] - - command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1) - - if(do_second_run): - rr_graph_out_file = "" - if "write_rr_graph" in second_run_args: - rr_graph_out_file = second_run_args["write_rr_graph"] - rr_graph_ext = Path(rr_graph_out_file).suffix - - rr_graph_out_file2 = "rr_graph2" + rr_graph_ext - if "write_rr_graph" in second_run_args: - second_run_args["read_rr_graph"] = rr_graph_out_file - second_run_args["write_rr_graph"] = rr_graph_out_file2 - - second_run_log_file = "vpr_second_run.out" - cmd = [] - if circuit_name: - cmd = [vpr_exec, architecture.name, circuit_name, "--circuit_file", circuit.name] + elif isinstance(value, list): + cmd += ["--" + arg] + for i in enumerate(value): + cmd += [str(value[i])] + elif isinstance(value, (str, list, int,)): + cmd += ["--" + arg, str(value)] else: - cmd = [vpr_exec, architecture.name, circuit.name] - for arg, value in vpr_args.items(): - if value == True: - cmd += ["--" + arg] - elif value == False: - pass - else: - cmd += ["--" + arg, str(value)] - - command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename=second_run_log_file, indent_depth=1) - - if "write_rr_graph" in second_run_args: - cmd = ["diff", rr_graph_out_file, rr_graph_out_file2] - output, diff_result = command_runner.run_system_command(cmd,temp_dir,log_filename="diff.rr_graph.out", indent_depth=1) - if diff_result: - raise InspectError("failed: vpr (RR Graph XML output not consistent when reloaded)") - -def cmp_full_vs_incr_STA(architecture,circuit,circuit_name=None,command_runner=CommandRunner(),vpr_args=None,rr_graph_ext=".xml",temp_dir=Path("."),vpr_exec=None): + pass + + command_runner.run_system_command( + cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1 + ) + +def run_second_time( + architecture, + circuit, + circuit_name=None, + command_runner=CommandRunner(), + temp_dir=Path("."), + vpr_exec=None, + second_run_args=None, + rr_graph_ext=".xml", +): + """ + Run vpr again with additional parameters. + This is used to ensure that files generated by VPR can be re-loaded by it + + .. note :: Usage: vtr.vpr.run_second_time(,,[OPTIONS]) + + Arguments + ========= + architecture: + Architecture file + + circuit: + Input circuit file + + Other Parameters + ---------------- + circuit_name: + Name of the circuit file + + command_runner: + CommandRunner object + + temp_dir: + Directory to run in + + log_filename : + File to log result to + + vpr_exec: + Path to the VPR executable + + second_run_args: + Extra arguments for VPR + + """ + temp_dir = Path(temp_dir) if not isinstance(temp_dir, Path) else temp_dir + temp_dir.mkdir(parents=True, exist_ok=True) + + rr_graph_out_file = "" + if "write_rr_graph" in second_run_args: + rr_graph_out_file = second_run_args["write_rr_graph"] + rr_graph_ext = Path(rr_graph_out_file).suffix + + rr_graph_out_file2 = "rr_graph2" + rr_graph_ext + if "write_rr_graph" in second_run_args: + second_run_args["read_rr_graph"] = rr_graph_out_file + second_run_args["write_rr_graph"] = rr_graph_out_file2 + + #run VPR + run( + architecture, + circuit, + circuit_name, + command_runner, + temp_dir, + log_filename="vpr_second_run.out", + vpr_exec=vpr_exec, + vpr_args=second_run_args, + ) + + if "write_rr_graph" in second_run_args: + cmd = ["diff", rr_graph_out_file, rr_graph_out_file2] + _, diff_result = command_runner.run_system_command( + cmd, temp_dir, log_filename="diff.rr_graph.out", indent_depth=1 + ) + if diff_result: + raise InspectError( + "failed: vpr (RR Graph XML output not consistent when reloaded)" + ) + +def cmp_full_vs_incr_sta( + architecture, + circuit, + circuit_name=None, + command_runner=CommandRunner(), + vpr_args=None, + temp_dir=Path("."), + vpr_exec=None, +): """ - Sanity check that full STA and the incremental STA produce the same *.net, *.place, *.route files as well as identical timing report files + Sanity check that full STA and the incremental STA produce the same *.net, *.place, *.route + files as well as identical timing report files - .. note :: Usage: vtr.vpr.cmp_full_vs_incr_STA(,,,[OPTIONS]) + .. note :: Use: vtr.vpr.cmp_full_vs_incr_sta(,,,[OPTIONS]) Arguments ========= - architecture: + architecture: Architecture file - - circuit: + + circuit: Input circuit file - + Other Parameters ---------------- - circuit_name: + circuit_name: Name of the circuit file - command_runner: + command_runner: CommandRunner object - - temp_dir: + + temp_dir: Directory to run in - - vpr_exec: + + vpr_exec: Path to the VPR executable - - vpr_args: + + vpr_args: Extra arguments for VPR - """ - #Verify that files are Paths or convert them to Paths and check that they exist + # Verify that files are Paths or convert them to Paths and check that they exist architecture = verify_file(architecture, "Architecture") circuit = verify_file(circuit, "Circuit") if not circuit_name: circuit_name = circuit.stem default_output_filenames = [ - "{}.net".format(circuit_name), - "{}.place".format(circuit_name), - "{}.route".format(circuit_name), - "report_timing.setup.rpt", - "report_timing.hold.rpt", - "report_unconstrained_timing.setup.rpt", - "report_unconstrained_timing.hold.rpt"] - - dif_exec = "diff" - move_exec = "mv" + "{}.net".format(circuit_name), + "{}.place".format(circuit_name), + "{}.route".format(circuit_name), + "report_timing.setup.rpt", + "report_timing.hold.rpt", + "report_unconstrained_timing.setup.rpt", + "report_unconstrained_timing.hold.rpt", + ] + # The full STA flow should have already been run # directly rename the output files for filename in default_output_filenames: - cmd = [move_exec,filename,"full_sta_{}".format(filename)] - command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename="move.out", indent_depth=1) - + cmd = ["mv", filename, "full_sta_{}".format(filename)] + command_runner.run_system_command( + cmd, temp_dir=temp_dir, log_filename="move.out", indent_depth=1 + ) + # run incremental STA flow incremental_vpr_args = vpr_args - incremental_vpr_args["timing_update_type"]="incremental" - fixed_W_log_file = "vpr.incr_sta.out" - - run(architecture, circuit, circuit_name, command_runner, temp_dir, log_filename=fixed_W_log_file, vpr_exec=vpr_exec, vpr_args=incremental_vpr_args, check_for_second_run=False) + incremental_vpr_args["timing_update_type"] = "incremental" + + run( + architecture, + circuit, + circuit_name, + command_runner, + temp_dir, + log_filename="vpr.incr_sta.out", + vpr_exec=vpr_exec, + vpr_args=incremental_vpr_args, + ) # Rename the incremental STA output files for filename in default_output_filenames: - cmd = [move_exec,filename,"incremental_sta_{}".format(filename)] - command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename="move.out", indent_depth=1) - + cmd = ["mv", filename, "incremental_sta_{}".format(filename)] + command_runner.run_system_command( + cmd, temp_dir=temp_dir, log_filename="move.out", indent_depth=1 + ) + failed_msg = "Failed with these files (not identical):" identical = True for filename in default_output_filenames: - cmd=[dif_exec,"full_sta_{}".format(filename),"incremental_sta_{}".format(filename)] - cmd_output,cmd_return_code = command_runner.run_system_command(cmd, temp_dir=temp_dir, log_filename="diff.out", indent_depth=1) - if(cmd_return_code): + cmd = [ + "diff", + "full_sta_{}".format(filename), + "incremental_sta_{}".format(filename), + ] + _, cmd_return_code = command_runner.run_system_command( + cmd, temp_dir=temp_dir, log_filename="diff.out", indent_depth=1 + ) + if cmd_return_code: identical = False failed_msg += " {}".format(filename) - + if not identical: raise InspectError(failed_msg) +#pylint: disable=too-many-arguments From b3eecd8ffd232dd616ed0b1fabf4b5bcb33de7a5 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Mon, 27 Jul 2020 15:25:15 -0700 Subject: [PATCH 78/86] fixing enumerate issue --- vtr_flow/scripts/python_libs/vtr/vpr/vpr.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py index 97f1dc4e7df..2a17f7bfda7 100644 --- a/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py +++ b/vtr_flow/scripts/python_libs/vtr/vpr/vpr.py @@ -203,16 +203,17 @@ def run( # Translate arbitrary keyword arguments into options for VPR for arg, value in vpr_args.items(): - if isinstance(value, bool) and value: + if isinstance(value, bool): + if not value: + pass cmd += ["--" + arg] - elif isinstance(value, list): - cmd += ["--" + arg] - for i in enumerate(value): - cmd += [str(value[i])] - elif isinstance(value, (str, list, int,)): - cmd += ["--" + arg, str(value)] else: - pass + if isinstance(value, list): + cmd += ["--" + arg] + for item in value: + cmd += [str(item)] + else: + cmd += ["--" + arg, str(value)] command_runner.run_system_command( cmd, temp_dir=temp_dir, log_filename=log_filename, indent_depth=1 From dbd8d93117d4c2a904f4d4b76ae1d83ce18d9cc0 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Mon, 27 Jul 2020 15:30:24 -0700 Subject: [PATCH 79/86] fixing accedental change with merge conflict --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 85ca9cb68a2..e9f1fb1a8f5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,8 +44,6 @@ addons: - libxft-dev - libxml++2.6-dev - perl - - python - - python3.6 - python-lxml - texinfo - time From e825e2171e933e7cc24642f67edb291c0a27988a Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Mon, 27 Jul 2020 15:53:39 -0700 Subject: [PATCH 80/86] made errors accept variable arguments --- vtr_flow/scripts/python_libs/vtr/error.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vtr_flow/scripts/python_libs/vtr/error.py b/vtr_flow/scripts/python_libs/vtr/error.py index 4171fa887aa..10f2fb6259f 100644 --- a/vtr_flow/scripts/python_libs/vtr/error.py +++ b/vtr_flow/scripts/python_libs/vtr/error.py @@ -11,8 +11,8 @@ class VtrError(Exception): msg -- An explanation of the error """ - def __init__(self, msg=""): - self.msg = msg + def __init__(self, *msg): + self.msg = " ".join(str(item) for item in msg) super().__init__() @@ -25,7 +25,7 @@ class CommandError(VtrError): cmd == The command run """ - def __init__(self, msg, cmd, returncode, log=None): + def __init__(self, *msg, cmd, returncode, log=None): super().__init__(msg=msg) self.returncode = returncode self.cmd = cmd @@ -37,6 +37,6 @@ class InspectError(VtrError): Raised when some query (inspection) result is not found. """ - def __init__(self, msg, filename=None): + def __init__(self, *msg, filename=None): super().__init__(msg=msg) self.filename = filename From 07c94f9f5323ffa6cc791cc5a4f9e5fcef621a40 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Mon, 27 Jul 2020 15:54:47 -0700 Subject: [PATCH 81/86] cleaned up pathlib path --- vtr_flow/scripts/python_libs/vtr/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vtr_flow/scripts/python_libs/vtr/util.py b/vtr_flow/scripts/python_libs/vtr/util.py index 40002daa2f2..9d59fb4dcd0 100644 --- a/vtr_flow/scripts/python_libs/vtr/util.py +++ b/vtr_flow/scripts/python_libs/vtr/util.py @@ -377,7 +377,7 @@ def find_vtr_root(): return os.environ[env_var] # We assume that this file is in /vtr_flow/python_libs/verilogtorouting - inferred_vtr_root = Path(__file__).parent / "../../../.." + inferred_vtr_root = Path(__file__).parent.parent.parent.parent.parent if inferred_vtr_root.is_dir(): return str(inferred_vtr_root) From ce43aa50871837baf4d37cb369df0711637e6629 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Tue, 28 Jul 2020 08:44:43 -0700 Subject: [PATCH 82/86] fixing args issue --- vtr_flow/scripts/python_libs/vtr/error.py | 4 ++-- vtr_flow/scripts/python_libs/vtr/inspect.py | 6 +++--- vtr_flow/scripts/python_libs/vtr/util.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/vtr_flow/scripts/python_libs/vtr/error.py b/vtr_flow/scripts/python_libs/vtr/error.py index 10f2fb6259f..d340cd6d4cf 100644 --- a/vtr_flow/scripts/python_libs/vtr/error.py +++ b/vtr_flow/scripts/python_libs/vtr/error.py @@ -26,7 +26,7 @@ class CommandError(VtrError): """ def __init__(self, *msg, cmd, returncode, log=None): - super().__init__(msg=msg) + super().__init__(msg) self.returncode = returncode self.cmd = cmd self.log = log @@ -38,5 +38,5 @@ class InspectError(VtrError): """ def __init__(self, *msg, filename=None): - super().__init__(msg=msg) + super().__init__(msg) self.filename = filename diff --git a/vtr_flow/scripts/python_libs/vtr/inspect.py b/vtr_flow/scripts/python_libs/vtr/inspect.py index 8abbfd57aa9..265c70998c5 100644 --- a/vtr_flow/scripts/python_libs/vtr/inspect.py +++ b/vtr_flow/scripts/python_libs/vtr/inspect.py @@ -37,7 +37,7 @@ def determine_lut_size(architecture_file): if saw_blif_names and lut_size == 0: raise InspectError( - msg="Could not identify valid LUT size (K)", filename=architecture_file + "Could not identify valid LUT size (K)", filename=architecture_file ) return lut_size @@ -66,7 +66,7 @@ def determine_memory_addr_width(architecture_file): if saw_ram and mem_addr_width == 0: raise InspectError( - msg="Could not identify RAM block address width", filename=architecture_file + "Could not identify RAM block address width", filename=architecture_file ) return mem_addr_width @@ -86,5 +86,5 @@ def determine_min_w(log_filename): return int(match.group("min_w")) raise InspectError( - msg="Failed to find minimum channel width.", filename=log_filename + "Failed to find minimum channel width.", filename=log_filename ) diff --git a/vtr_flow/scripts/python_libs/vtr/util.py b/vtr_flow/scripts/python_libs/vtr/util.py index 9d59fb4dcd0..1840d75894f 100644 --- a/vtr_flow/scripts/python_libs/vtr/util.py +++ b/vtr_flow/scripts/python_libs/vtr/util.py @@ -206,7 +206,7 @@ def run_system_command( if self._show_failures and cmd_errored: raise CommandError( - msg="Executable {exec_name} failed".format( + "Executable {exec_name} failed".format( exec_name=PurePath(orig_cmd[0]).name ), cmd=cmd, From b02b1376c71b13b047d65ffdbc442cb810f9e858 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Tue, 28 Jul 2020 08:51:26 -0700 Subject: [PATCH 83/86] fixed doc to only show important function descriptions ' --- doc/src/vtr/python_libs/vtr.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/vtr/python_libs/vtr.rst b/doc/src/vtr/python_libs/vtr.rst index b35875591df..9dfff1dcc13 100755 --- a/doc/src/vtr/python_libs/vtr.rst +++ b/doc/src/vtr/python_libs/vtr.rst @@ -19,7 +19,7 @@ ABC === .. automodule:: vtr.abc.abc - :members: + :members: run, run_lec ACE === From 6ae36da3a2969996b202a59074b048c87910eed9 Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Tue, 28 Jul 2020 10:45:53 -0700 Subject: [PATCH 84/86] cleaning up error reporting --- vtr_flow/scripts/python_libs/vtr/error.py | 4 ++-- vtr_flow/scripts/python_libs/vtr/util.py | 3 ++- vtr_flow/scripts/run_vtr_flow.py | 6 ++---- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/vtr_flow/scripts/python_libs/vtr/error.py b/vtr_flow/scripts/python_libs/vtr/error.py index d340cd6d4cf..ca915e3f933 100644 --- a/vtr_flow/scripts/python_libs/vtr/error.py +++ b/vtr_flow/scripts/python_libs/vtr/error.py @@ -26,7 +26,7 @@ class CommandError(VtrError): """ def __init__(self, *msg, cmd, returncode, log=None): - super().__init__(msg) + super().__init__(*msg) self.returncode = returncode self.cmd = cmd self.log = log @@ -38,5 +38,5 @@ class InspectError(VtrError): """ def __init__(self, *msg, filename=None): - super().__init__(msg) + super().__init__(*msg) self.filename = filename diff --git a/vtr_flow/scripts/python_libs/vtr/util.py b/vtr_flow/scripts/python_libs/vtr/util.py index 1840d75894f..f07b0fc6223 100644 --- a/vtr_flow/scripts/python_libs/vtr/util.py +++ b/vtr_flow/scripts/python_libs/vtr/util.py @@ -213,7 +213,8 @@ def run_system_command( log=str(temp_dir / log_filename), returncode=cmd_returncode, ) - + elif cmd_errored: + raise VtrError("{}".format(PurePath(orig_cmd[0]).name)) return cmd_output, cmd_returncode # pylint: enable=too-many-arguments, too-many-instance-attributes, too-few-public-methods, too-many-locals diff --git a/vtr_flow/scripts/run_vtr_flow.py b/vtr_flow/scripts/run_vtr_flow.py index 26b4eac4cfa..6ed8300f08c 100755 --- a/vtr_flow/scripts/run_vtr_flow.py +++ b/vtr_flow/scripts/run_vtr_flow.py @@ -415,9 +415,9 @@ def vtr_command_main(arg_list, prog=None): if args.sdc_file: vpr_args["sdc_file"] = get_sdc_file(args.sdc_file, prog) - print(args.name if args.name else Path(args.architecture_file).name + print(args.name if args.name else Path(args.architecture_file).stem + "/" - + Path(args.circuit_file).name, end="\t\t") + + Path(args.circuit_file).stem, end="\t\t") # Run the flow vtr.run( Path(args.architecture_file), @@ -641,7 +641,6 @@ def except_vtr_error(error, expect_fail, verbose): exit_status = 1 elif isinstance(error, vtr.InspectError): # Something went wrong gathering information - print("Error: {msg}".format(msg=error.msg)) print("\tfile : ", error.filename) exit_status = 2 return_status = exit_status @@ -649,7 +648,6 @@ def except_vtr_error(error, expect_fail, verbose): elif isinstance(error, vtr.VtrError): # Generic VTR errors - print("Error: ", error.msg) exit_status = 3 return_status = exit_status error_status = "failed: {}".format(error.msg) From dc4bfec77dc104a41a8aedc9117c08c67e38c5db Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Tue, 28 Jul 2020 10:47:54 -0700 Subject: [PATCH 85/86] fixing pylint issue --- vtr_flow/scripts/python_libs/vtr/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vtr_flow/scripts/python_libs/vtr/util.py b/vtr_flow/scripts/python_libs/vtr/util.py index f07b0fc6223..c7cd39681a1 100644 --- a/vtr_flow/scripts/python_libs/vtr/util.py +++ b/vtr_flow/scripts/python_libs/vtr/util.py @@ -213,7 +213,7 @@ def run_system_command( log=str(temp_dir / log_filename), returncode=cmd_returncode, ) - elif cmd_errored: + if cmd_errored: raise VtrError("{}".format(PurePath(orig_cmd[0]).name)) return cmd_output, cmd_returncode From a80427539e99b899265d21924a56f9907db1a5de Mon Sep 17 00:00:00 2001 From: shadtorrie Date: Tue, 28 Jul 2020 11:09:44 -0700 Subject: [PATCH 86/86] Added error handling if architecture file is not xml --- vtr_flow/scripts/python_libs/vtr/flow.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/vtr_flow/scripts/python_libs/vtr/flow.py b/vtr_flow/scripts/python_libs/vtr/flow.py index c7e29591efe..32cdf27f10e 100644 --- a/vtr_flow/scripts/python_libs/vtr/flow.py +++ b/vtr_flow/scripts/python_libs/vtr/flow.py @@ -130,6 +130,10 @@ def run( # Verify that files are Paths or convert them to Paths and check that they exist architecture_file = vtr.util.verify_file(architecture_file, "Architecture") circuit_file = vtr.util.verify_file(circuit_file, "Circuit") + if architecture_file.suffix != ".xml": + raise vtr.VtrError( + "Expected Architecture file as second argument (was {})".format(architecture_file.name) + ) power_tech_file = ( vtr.util.verify_file(power_tech_file, "Power") if power_tech_file else None )