Compiler directives benchmarks (#1331)

* Regression test: Created benchmarks and vector files for compiler directives which can be found in the preprocessor file. Additionally, created an "expected" option for json files that allows us to identify benchmarks that should be passed but fail due to bugs or unsupported functions. This allows developers creating benchmarks
to go more in depth on why a test may be failing and possible solutions. All json files have been updated accordingly to include the expectation.

* Fixed directory path

* Change log so something won't be synthesized

* regenerated results for micro benchmarks, missing expectation

* updating large and full regression test

* Updated results

Co-authored-by: emacdo12 <[email protected]>

Author: emacdo12

ODIN_II/regression_test/benchmark/suite/light_suite/task_list.conf

@@ -1,9 +1,9 @@
 rtl_reg
 regression_test/benchmark/task/arch_sweep
 regression_test/benchmark/task/cmd_line_args/*
+regression_test/benchmark/task/preprocessor
 regression_test/benchmark/task/operators
 regression_test/benchmark/task/binary
 regression_test/benchmark/task/syntax
 regression_test/benchmark/task/FIR
 regression_test/benchmark/task/micro
-

ODIN_II/regression_test/benchmark/task/FIR/simulation_result.json

@@ -6,6 +6,7 @@
         "exit": 0,
         "errors": [],
         "warnings": [],
+        "expectation": [],
         "max_rss(MiB)": 443.4,
         "exec_time(ms)": 3897.9,
         "simulation_time(ms)": 3754.8,
@@ -32,6 +33,7 @@
         "exit": 0,
         "errors": [],
         "warnings": [],
+        "expectation": [],
         "max_rss(MiB)": 445.3,
         "exec_time(ms)": 3820,
         "simulation_time(ms)": 3641,
@@ -58,6 +60,7 @@
         "exit": 0,
         "errors": [],
         "warnings": [],
+        "expectation": [],
         "max_rss(MiB)": 441.1,
         "exec_time(ms)": 3836.8,
         "simulation_time(ms)": 3704.9,
@@ -84,6 +87,7 @@
         "exit": 0,
         "errors": [],
         "warnings": [],
+        "expectation": [],
         "max_rss(MiB)": 118.8,
         "exec_time(ms)": 1241.4,
         "simulation_time(ms)": 1191.4,
@@ -110,6 +114,7 @@
         "exit": 0,
         "errors": [],
         "warnings": [],
+        "expectation": [],
         "max_rss(MiB)": 128,
         "exec_time(ms)": 1299.8,
         "simulation_time(ms)": 1211.5,
@@ -136,6 +141,7 @@
         "exit": 0,
         "errors": [],
         "warnings": [],
+        "expectation": [],
         "max_rss(MiB)": 115.7,
         "exec_time(ms)": 1236.7,
         "simulation_time(ms)": 1188.6,
@@ -162,6 +168,7 @@
         "exit": 0,
         "errors": [],
         "warnings": [],
+        "expectation": [],
         "max_rss(MiB)": 1050.6,
         "exec_time(ms)": 10555.3,
         "simulation_time(ms)": 10127.6,
@@ -188,6 +195,7 @@
         "exit": 0,
         "errors": [],
         "warnings": [],
+        "expectation": [],
         "max_rss(MiB)": 1045.3,
         "exec_time(ms)": 10751.9,
         "simulation_time(ms)": 10268.5,
@@ -214,6 +222,7 @@
         "exit": 0,
         "errors": [],
         "warnings": [],
+        "expectation": [],
         "max_rss(MiB)": 1046.5,
         "exec_time(ms)": 10986.2,
         "simulation_time(ms)": 10565.3,
@@ -240,6 +249,7 @@
         "exit": 0,
         "errors": [],
         "warnings": [],
+        "expectation": [],
         "max_rss(MiB)": 732.1,
         "exec_time(ms)": 5965.6,
         "simulation_time(ms)": 5731,
@@ -266,6 +276,7 @@
         "exit": 0,
         "errors": [],
         "warnings": [],
+        "expectation": [],
         "max_rss(MiB)": 724.2,
         "exec_time(ms)": 6030.5,
         "simulation_time(ms)": 5756.1,
@@ -292,6 +303,7 @@
         "exit": 0,
         "errors": [],
         "warnings": [],
+        "expectation": [],
         "max_rss(MiB)": 729.9,
         "exec_time(ms)": 6181.6,
         "simulation_time(ms)": 5948.3,
@@ -318,6 +330,7 @@
         "exit": 0,
         "errors": [],
         "warnings": [],
+        "expectation": [],
         "max_rss(MiB)": 1267.8,
         "exec_time(ms)": 12757.1,
         "simulation_time(ms)": 12288.7,
@@ -344,6 +357,7 @@
         "exit": 0,
         "errors": [],
         "warnings": [],
+        "expectation": [],
         "max_rss(MiB)": 1217.8,
         "exec_time(ms)": 12707.1,
         "simulation_time(ms)": 12198.2,
@@ -370,6 +384,7 @@
         "exit": 0,
         "errors": [],
         "warnings": [],
+        "expectation": [],
         "max_rss(MiB)": 1265.1,
         "exec_time(ms)": 12320.7,
         "simulation_time(ms)": 11846.3,
@@ -396,6 +411,7 @@
         "exit": 0,
         "errors": [],
         "warnings": [],
+        "expectation": [],
         "max_rss(MiB)": 169.2,
         "exec_time(ms)": 1858.6,
         "simulation_time(ms)": 1786.7,
@@ -422,6 +438,7 @@
         "exit": 0,
         "errors": [],
         "warnings": [],
+        "expectation": [],
         "max_rss(MiB)": 177.5,
         "exec_time(ms)": 1968.8,
         "simulation_time(ms)": 1856.9,
@@ -448,6 +465,7 @@
         "exit": 0,
         "errors": [],
         "warnings": [],
+        "expectation": [],
         "max_rss(MiB)": 165.3,
         "exec_time(ms)": 1925.1,
         "simulation_time(ms)": 1858.1,
@@ -474,6 +492,7 @@
         "exit": 0,
         "errors": [],
         "warnings": [],
+        "expectation": [],
         "max_rss(MiB)": 2004.5,
         "exec_time(ms)": 32841.3,
         "simulation_time(ms)": 31158.4,
@@ -500,6 +519,7 @@
         "exit": 0,
         "errors": [],
         "warnings": [],
+        "expectation": [],
         "max_rss(MiB)": 2003.3,
         "exec_time(ms)": 27031.3,
         "simulation_time(ms)": 25158,
@@ -526,6 +546,7 @@
         "exit": 0,
         "errors": [],
         "warnings": [],
+        "expectation": [],
         "max_rss(MiB)": 2001.2,
         "exec_time(ms)": 30766.1,
         "simulation_time(ms)": 28981.3,
@@ -552,6 +573,7 @@
         "exit": 0,
         "errors": [],
         "warnings": [],
+        "expectation": [],
         "max_rss(MiB)": 1597.4,
         "exec_time(ms)": 17962.9,
         "simulation_time(ms)": 17211,
@@ -578,6 +600,7 @@
         "exit": 0,
         "errors": [],
         "warnings": [],
+        "expectation": [],
         "max_rss(MiB)": 1590.3,
         "exec_time(ms)": 17981,
         "simulation_time(ms)": 17115.1,
@@ -604,6 +627,7 @@
         "exit": 0,
         "errors": [],
         "warnings": [],
+        "expectation": [],
         "max_rss(MiB)": 1599.4,
         "exec_time(ms)": 17273.4,
         "simulation_time(ms)": 16557.4,
@@ -630,6 +654,7 @@
         "exit": 0,
         "errors": [],
         "warnings": [],
+        "expectation": [],
         "max_rss(MiB)": 226,
         "exec_time(ms)": 2356,
         "simulation_time(ms)": 2268.7,
@@ -656,6 +681,7 @@
         "exit": 0,
         "errors": [],
         "warnings": [],
+        "expectation": [],
         "max_rss(MiB)": 235.8,
         "exec_time(ms)": 2416.2,
         "simulation_time(ms)": 2290.5,
@@ -682,6 +708,7 @@
         "exit": 0,
         "errors": [],
         "warnings": [],
+        "expectation": [],
         "max_rss(MiB)": 223.2,
         "exec_time(ms)": 2495.2,
         "simulation_time(ms)": 2411.4,
@@ -708,6 +735,7 @@
         "exit": 0,
         "errors": [],
         "warnings": [],
+        "expectation": [],
         "max_rss(MiB)": 2402.5,
         "exec_time(ms)": 46569.6,
         "simulation_time(ms)": 38923,
@@ -734,6 +762,7 @@
         "exit": 0,
         "errors": [],
         "warnings": [],
+        "expectation": [],
         "max_rss(MiB)": 2356.4,
         "exec_time(ms)": 48542.8,
         "simulation_time(ms)": 41813.5,
@@ -760,6 +789,7 @@
         "exit": 0,
         "errors": [],
         "warnings": [],
+        "expectation": [],
         "max_rss(MiB)": 2354.1,
         "exec_time(ms)": 49500.5,
         "simulation_time(ms)": 41744.8,
@@ -786,6 +816,7 @@
         "exit": 0,
         "errors": [],
         "warnings": [],
+        "expectation": [],
         "max_rss(MiB)": 2032.1,
         "exec_time(ms)": 30013.7,
         "simulation_time(ms)": 27159.3,
@@ -812,6 +843,7 @@
         "exit": 0,
         "errors": [],
         "warnings": [],
+        "expectation": [],
         "max_rss(MiB)": 2038.5,
         "exec_time(ms)": 35038.6,
         "simulation_time(ms)": 32110.1,
@@ -838,6 +870,7 @@
         "exit": 0,
         "errors": [],
         "warnings": [],
+        "expectation": [],
         "max_rss(MiB)": 2025.6,
         "exec_time(ms)": 32326.5,
         "simulation_time(ms)": 29646.2,