add velgrind test suite the odin regression

Author: jeanlego

ODIN_II/regression_test/benchmark/FIR/config.txt

@@ -1 +1 @@
---simulate
+--arch_list small_sweep

ODIN_II/regression_test/benchmark/_bug/config.txt

@@ -1 +1 @@
---no_threading --simulate
+--no_threading

ODIN_II/regression_test/benchmark/arch/config.txt

@@ -1 +1 @@
---arch_list small_sweep --simulate
+--arch_list small_sweep

ODIN_II/regression_test/benchmark/full/config.txt

@@ -1 +1 @@
---arch_list memories --no_threading --simulate
+--arch_list memories --no_threading

ODIN_II/regression_test/benchmark/micro/config.txt

@@ -1 +1 @@
---arch_list small_sweep --simulate
+--arch_list small_sweep

ODIN_II/regression_test/benchmark/operators/config.txt

@@ -1 +1 @@
---arch_list small_sweep --simulate
+--arch_list small_sweep

ODIN_II/regression_test/benchmark/syntax/config.txt

@@ -1 +1 @@
---simulate
+--arch_list small_sweep

ODIN_II/regression_test/benchmark/valgrind_operators/config.txt

@@ -0,0 +1 @@
+--arch_list small_sweep --disable_simulation --source ../operators --valgrind

ODIN_II/verify_odin.sh

@@ -42,7 +42,7 @@ _TEST=""
 _NUMBER_OF_PROCESS="1"
 _SIM_THREADS="1"
 _VECTORS="100"
-_TIMEOUT="1200"
+_TIMEOUT="1200s"
 _ADDER_DEF="default"
 _SIM_COUNT="1"
 _RUN_DIR_OVERRIDE=""
@@ -117,7 +117,7 @@ printf "Called program with $INPUT
 		-j|--nb_of_process < N >        $(_prt_cur_arg ${_NUMBER_OF_PROCESS}) Number of process requested to be used
 		-s|--sim_threads < N >          $(_prt_cur_arg ${_SIM_THREADS}) Use multithreaded simulation using N threads
 		-V|--vectors < N >              $(_prt_cur_arg ${_VECTORS}) Use N vectors to generate per simulation
-		-T|--timeout < N sec >          $(_prt_cur_arg ${_TIMEOUT}) Timeout a simulation/synthesis after N seconds
+		-T|--timeout < N[s|m|h] >          $(_prt_cur_arg ${_TIMEOUT}) Timeout a simulation/synthesis after N seconds
 		-a|--adder_def < /abs/path >    $(_prt_cur_arg ${_ADDER_DEF}) Use template to build adders
 		-n|--simulation_count < N >     $(_prt_cur_arg ${_SIM_COUNT}) Allow to run the simulation N times to benchmark the simulator
 		-d|--output_dir < /abs/path >   $(_prt_cur_arg ${_RUN_DIR_OVERRIDE}) Change the run directory output
@@ -145,8 +145,9 @@ printf "
 								*small_sweep     use a small set of timing architecture
 								*full_sweep  	 sweep the whole vtr directory *** WILL FAIL ***
 								*default         use the sample_arch.xml
-				--simulate                       request simulation to be ran
+				--disable_simulation             request simulation NOT to be ran
 				--no_threading                   do not use multithreading for this test ** useful if you have large test **
+				--source [relative_path]         change where the benchmark files are, (relative to this config file) <default: ./>
 "
 }
 
@@ -287,40 +288,10 @@ function flag_is_number() {
 	esac
 }
 
-
-# boolean type flags
-_low_ressource_flag=""
-_valgrind_flag=""
-_batch_sim_flag=""
-_use_best_coverage_flag=""
-_perf_flag=""
-
-# number type flags
-_vector_flag=""
-_timeout_flag=""
-_simulation_threads_flag=""
-
-_adder_definition_flag=""
-
 function _set_if() {
 	[ "$1" == "on" ] && echo "$2" || echo ""
 }
 
-function _set_flag() {
-	_low_ressource_flag=$(_set_if ${_LIMIT_RESSOURCE} "--limit_ressource")
-	_valgrind_flag=$(_set_if ${_VALGRIND} "--tool valgrind")
-	_batch_sim_flag=$(_set_if ${_BATCH_SIM} "--batch")
-	_use_best_coverage_flag=$(_set_if ${_BEST_COVERAGE_OFF} "--best_coverage")
-	_perf_flag=$(_set_if ${_USE_PERF} "--tool perf")
-	
-	_vector_flag="-g ${_VECTORS}"
-	_timeout_flag="--time_limit ${_TIMEOUT}s"
-	_simulation_threads_flag=$([ "${_SIM_THREADS}" != "1" ] && echo "-j ${_SIM_THREADS}")
-
-	_adder_definition_flag="--adder_type ${_ADDER_DEF}"
-
-}
-
 function parse_args() {
 	while [[ "$#" > 0 ]]
 	do 
@@ -463,10 +434,28 @@ function sim() {
 	arch_list="no_arch"
 	with_sim="0"
 	threads=${_NUMBER_OF_PROCESS}
-	DEFAULT_CMD_PARAM="${_adder_definition_flag} ${_simulation_threads_flag} ${_batch_sim_flag}"
+	test_src="./"
+
+	# default flags
+	_low_ressource_flag="--limit_ressource"
+	_valgrind_flag="--tool valgrind"
+	_batch_sim_flag="--batch"
+	_best_coverage_flag="--best_coverage"
+	_perf_flag="--tool perf"
+
+	use_timeout="--time_limit ${_TIMEOUT}"
+	use_valgrind=$(_set_if ${_VALGRIND} ${_valgrind_flag})
+	use_low_ressource=$(_set_if ${_LIMIT_RESSOURCE} ${_low_ressource_flag})
+	use_batch_sim=$(_set_if ${_BATCH_SIM} ${_batch_sim_flag})
+	use_best_coverage=$(_set_if ${_BEST_COVERAGE_OFF} ${_best_coverage_flag})
+	use_perf=$(_set_if ${_USE_PERF} ${_perf_flag})
+
+	_vector_flag="-g ${_VECTORS}"
+	_simulation_threads_flag=$([ "${_SIM_THREADS}" != "1" ] && echo "-j ${_SIM_THREADS}")
+	_adder_definition_flag="--adder_type ${_ADDER_DEF}"
 
 	_SYNTHESIS="on"
-	_SIMULATE=${_FORCE_SIM}
+	_SIMULATE="on"
 
 	if [ ! -d "$1" ]
 	then
@@ -479,6 +468,11 @@ function sim() {
 	while [[ "$#" > 0 ]]
 	do 
 		case $1 in
+			--source)
+				test_src=$2
+				shift
+				;;
+
 			--custom_args_file) 
 				with_custom_args=1
 				;;
@@ -506,15 +500,20 @@ function sim() {
 				shift
 				;;
 
-			--simulate)
-				_SIMULATE="on"
+			--disable_simulation)
+				_SIMULATE=${_FORCE_SIM}
 				;;
 
 			--no_threading)
 				echo "This test will not be multithreaded"
 				threads="1"
 				;;
 
+			--valgrind)
+				echo "This test will be ran with valgrind"
+				use_valgrind=${_valgrind_flag}
+				;;
+
 			*)
 				echo "Unknown internal parameter passed: $1"
 				config_help 
@@ -526,9 +525,23 @@ function sim() {
 
 	###########################################
 	# run custom benchmark
-	bench_type=${benchmark_dir##*/}
+	real_bench="${benchmark_dir}"
+	benchmark_dir="${benchmark_dir}/${test_src}"
+	if [ "_${benchmark_dir}" == "_" ] || [ ! -d ${benchmark_dir} ]; then
+		echo "invalid benchmark directory parameter passed: ${benchmark_dir} from ${test_src}"
+		ctrl_c
+	fi
+
+	benchmark_dir=$(readlink -f "${benchmark_dir}")
+	bench_type=${real_bench##*/}
 	echo " BENCHMARK IS: ${bench_type}"
 
+
+	##########################################
+	# setup defaults
+	DEFAULT_CMD_PARAM="${_adder_definition_flag} ${_simulation_threads_flag} ${_batch_sim_flag}"
+	DEFAULT_WRAPPER_CMD="${_timeout_flag} ${_low_ressource_flag} ${use_valgrind}"
+
 	if [ "_${with_custom_args}" == "_1" ]
 	then
 
@@ -552,7 +565,7 @@ function sim() {
 											--failure_log ${global_odin_failure}.log
 											${_timeout_flag}
 											${_low_ressource_flag}
-											${_valgrind_flag}"
+											${use_valgrind}"
 
 				if [ "${_USE_PERF}" == "on" ]
 				then
@@ -636,9 +649,9 @@ function sim() {
 												--failure_log ${global_synthesis_failure}.log
 												${_timeout_flag}
 												${_low_ressource_flag}
-												${_valgrind_flag}"
+												${use_valgrind}"
 
-					if [ "${_USE_PERF}" == "on" ]
+					if [ "${use_perf}" == "on" ]
 					then
 						wrapper_synthesis_command="${wrapper_synthesis_command} ${_perf_flag} ${DIR}/perf.data"
 					fi
@@ -658,9 +671,9 @@ function sim() {
 											--log_file ${DIR}/simulation.log
 											--test_name ${TEST_FULL_REF}
 											--failure_log ${global_simulation_failure}.log
-											${_timeout_flag}
-											${_low_ressource_flag}
-											${_valgrind_flag}"
+											${use_timeout}
+											${use_low_ressource}
+											${use_valgrind}"
 
 					if [ "${_USE_PERF}" == "on" ]
 					then
@@ -675,7 +688,7 @@ function sim() {
 
 					if [ "${_GENERATE_BENCH}" == "on" ] || [ ! -f ${input_vector_file} ]
 					then
-						simulation_command="${simulation_command} ${_use_best_coverage_flag} ${_vector_flag}"
+						simulation_command="${simulation_command} ${use_best_coverage} ${_vector_flag}"
 					else
 						simulation_command="${simulation_command} -t ${input_vector_file}"
 						if [ "${_GENERATE_OUTPUT}" == "off" ] && [ -f ${output_vector_file} ]
@@ -753,6 +766,7 @@ function sim() {
 		fi
 
 	fi
+
 }
 
 function run_failed() {
@@ -907,7 +921,6 @@ START=`get_current_time`
 init_temp
 
 parse_args $INPUT
-_set_flag
 
 if [ "_${_TEST}" == "_" ]
 then

ODIN_II/wrapper_odin.sh

@@ -20,7 +20,7 @@ if [ ! -f ${EXEC} ]; then
 fi
 
 TIME_EXEC=$($SHELL -c "which time") 
-VALGRIND_EXEC="valgrind --leak-check=full --max-stackframe=128000000"
+VALGRIND_EXEC="valgrind --leak-check=full --max-stackframe=128000000 --error-exitcode=1"
 PERF_EXEC="perf stat record -a -d -d -d -o"
 GDB_EXEC="gdb --args"
 LOG=""
@@ -221,17 +221,18 @@ do
 
 			if [ "${USE_TIMEOUT}" == "on" ]; then
 				if [ "${CANCEL_LOGS}" == "off" ] && [ "_${LOG_FILE}" != "_" ]; then 
-					timeout ${TIME_LIMIT} /bin/bash -c "${TIME_EXEC} --output=${LOG_FILE} --append ${EXEC} ${ODIN_ARGS} &>> ${LOG_FILE}" &> /dev/null && EXIT_STATUS=0 || EXIT_STATUS=1 
+					timeout ${TIME_LIMIT} /bin/bash -c "${TIME_EXEC} --output=${LOG_FILE} --append ${EXEC} ${ODIN_ARGS} &>> ${LOG_FILE}" &> /dev/null
 				else
-					timeout ${TIME_LIMIT} /bin/bash -c "${TIME_EXEC} ${EXEC} ${ODIN_ARGS}" && EXIT_STATUS=0 || EXIT_STATUS=1
+					timeout ${TIME_LIMIT} /bin/bash -c "${TIME_EXEC} ${EXEC} ${ODIN_ARGS}"
 				fi
 			else
 				if [ "${CANCEL_LOGS}" == "off" ] && [ "_${LOG_FILE}" != "_" ]; then 
-					/bin/bash -c "${EXEC} ${ODIN_ARGS} &>> ${LOG_FILE}" &> /dev/null && EXIT_STATUS=0 || EXIT_STATUS=1 
+					/bin/bash -c "${EXEC} ${ODIN_ARGS} &>> ${LOG_FILE}" &> /dev/null
 				else
 					/bin/bash -c "${EXEC} ${ODIN_ARGS}"
 				fi
 			fi
+			EXIT_STATUS=$?
 
 			break
 			;;