Change regression testing suite to support building with cvc4 as well as z3

Author: NlightNFotis

regression/cbmc-incr-smt2/CMakeLists.txt

@@ -5,6 +5,16 @@ else()
     set(exclude_win_broken_tests "")
 endif()
 
-add_test_pl_tests(
-    "$<TARGET_FILE:cbmc> --validate-goto-model --validate-ssa-equation --slice-formula" ${exclude_win_broken_tests}
+add_test_pl_profile(
+    "cbmc-new-smt-backend-z3"
+    "$<TARGET_FILE:cbmc> --incremental-smt2-solver 'z3 --smt2 -in' --validate-goto-model --validate-ssa-equation --slice-formula ${exclude_win_broken_tests}"
+    "-C;-s;new-smt-z3"
+    "CORE"
+)
+
+add_test_pl_profile(
+    "cbmc-new-smt-backend-cv4"
+    "$<TARGET_FILE:cbmc> --incremental-smt2-solver 'cvc4 --lang smt' --validate-goto-model --validate-ssa-equation --slice-formula ${exclude_win_broken_tests}"
+    "-C;-s;new-smt-cvc4"
+    "CORE"
 )

regression/cbmc-incr-smt2/bitvector-arithmetic-operators/overflow_behaviour.desc

@@ -1,6 +1,6 @@
 CORE
 overflow_behaviour.c
---incremental-smt2-solver 'z3 --smt2 -in' --trace
+--trace
 \[main\.assertion\.1\] line \d+ Wrap-around to INT_MIN when adding to INT_MAX: SUCCESS
 \[main\.assertion\.2\] line \d+ Wrap-around to INT_MAX when subtracting from INT_MIN: SUCCESS
 \[main\.assertion\.3\] line \d+ INT_MAX minus INT_MIN equals -1: SUCCESS

regression/cbmc-incr-smt2/bitvector-arithmetic-operators/polynomial.desc

@@ -1,6 +1,6 @@
 CORE
 polynomial.c
---incremental-smt2-solver 'z3 --smt2 -in' --trace
+--trace
 \[main\.assertion\.1\] line \d+ No negative solution: FAILURE
 \[main\.assertion\.2\] line \d+ No positive solution: FAILURE
 x=-8\ \(11111111 11111111 11111111 11111000\)

regression/cbmc-incr-smt2/bitvector-arithmetic-operators/simple_equation.desc

@@ -1,6 +1,6 @@
 CORE
 simple_equation.c
---incremental-smt2-solver 'z3 --smt2 -in' --trace --verbosity 10
+--trace --verbosity 10
 \[main\.assertion\.1\] line \d+ a plus a always equals two times a: SUCCESS
 \[main\.assertion\.2\] line \d+ a minus a always equals 0: SUCCESS
 \[main\.assertion\.3\] line \d+ a plus its additive inverse equals 0: SUCCESS

regression/cbmc-incr-smt2/bitvector-arithmetic-operators/unsigned_behaviour.desc

@@ -1,6 +1,6 @@
 CORE
 unsigned_behaviour.c
---incremental-smt2-solver 'z3 --smt2 -in' --trace
+--trace
 \[main\.assertion\.1\] line \d+ a plus b should be more than 27: FAILURE
 \[main\.assertion\.2\] line \d+ a plus b should be more than 27: FAILURE
 \[main\.assertion\.3\] line \d+ c plus d should be more than 27: FAILURE

regression/cbmc-incr-smt2/bitvector-bitwise-operators/bitwise.desc

@@ -1,6 +1,6 @@
 CORE
 bitwise_ops.c
---incremental-smt2-solver 'z3 --smt2 -in' --slice-formula
+--slice-formula
 \[main\.assertion\.1\] line \d+ This is going to fail for bit-opposites: FAILURE
 \[main\.assertion\.2\] line \d+ This is going to hold for all values != 0: SUCCESS
 \[main\.assertion\.3\] line \d+ This is going to fail for the same value in A and B: FAILURE

regression/cbmc-incr-smt2/bitvector-bitwise-operators/shift_left.desc

@@ -1,6 +1,6 @@
 CORE
 shift_left.c
---incremental-smt2-solver 'z3 --smt2 -in' --slice-formula
+--slice-formula
 \[main\.assertion\.1\] line \d Shifted result should be greater than one: FAILURE
 ^EXIT=10$
 ^SIGNAL=0$

regression/cbmc-incr-smt2/bitvector-bitwise-operators/shift_right.desc

@@ -1,6 +1,6 @@
 CORE
 shift_right.c
---incremental-smt2-solver 'z3 --smt2 -in' --slice-formula --trace
+--slice-formula --trace
 \[main\.assertion\.1\] line \d+ Right shifting a uint with leftmost bit set is a logical shift: FAILURE
 \[main\.assertion\.2\] line \d+ Right shifting a positive number has a lower bound of 0: SUCCESS
 \[main\.assertion\.3\] line \d+ Right shifting a negative number has a lower bound value of -1: SUCCESS

regression/cbmc-incr-smt2/bitvector-flag-tests/div_by_zero.desc

@@ -1,6 +1,6 @@
 CORE
 div_by_zero.c
---incremental-smt2-solver 'z3 --smt2 -in' --div-by-zero-check --trace
+--div-by-zero-check --trace
 \[main\.division-by-zero\.1\] line \d division by zero in x / y: FAILURE
 \[main\.division-by-zero\.2\] line \d+ division by zero in x / z: SUCCESS
 y=0

regression/cbmc-incr-smt2/bitvector-flag-tests/signed_overflow.desc

@@ -1,6 +1,6 @@
 KNOWNBUG
 signed_overflow.c
---incremental-smt2-solver 'z3 --smt2 -in' --signed-overflow-check --trace
+--signed-overflow-check --trace
 ^VERIFICATION FAILED$
 ^EXIT=10$
 ^SIGNAL=0$

regression/cbmc-incr-smt2/nondeterministic-int-assert/control_flow.desc

@@ -1,6 +1,6 @@
 CORE
 control_flow.c
---incremental-smt2-solver 'z3 --smt2 -in' --verbosity 10
+--verbosity 10
 Passing problem to incremental SMT2 solving via "z3 --smt2 -in"
 Sending command to SMT2 solver - \(set-option :produce-models true\)
 Sending command to SMT2 solver - \(set-logic QF_UFBV\)

regression/cbmc-incr-smt2/nondeterministic-int-assert/incremental_solver_called.desc

@@ -1,6 +1,6 @@
 CORE
 test.c
---incremental-smt2-solver 'z3 --smt2 -in' --verbosity 10
+--verbosity 10
 Passing problem to incremental SMT2 solving via "z3 --smt2 -in"
 Sending command to SMT2 solver - \(set-option :produce-models true\)
 Sending command to SMT2 solver - \(set-logic QF_UFBV\)

regression/cbmc-incr-smt2/nondeterministic-int-assert/stdbool.desc

@@ -1,6 +1,6 @@
 CORE
 stdbool_example.c
---incremental-smt2-solver 'z3 --smt2 -in' --trace
+--trace
 Passing problem to incremental SMT2 solving via "z3 --smt2 -in"
 VERIFICATION FAILED
 equal=FALSE\s*\([0 ]+\)

regression/cbmc-incr-smt2/nondeterministic-int-assert/trace.desc

@@ -1,6 +1,6 @@
 CORE
 trace.c
---incremental-smt2-solver 'z3 --smt2 -in' --trace
+--trace
 Passing problem to incremental SMT2 solving via "z3 --smt2 -in"
 Assert of inequality to 4\.: FAILURE
 Assert of inequality to 2\.: FAILURE

regression/cbmc-incr-smt2/nondeterministic-int-assert/valid_unsat.desc

@@ -1,6 +1,6 @@
 CORE
 valid_unsat.c
---incremental-smt2-solver 'z3 --smt2 -in' --verbosity 10
+--verbosity 10
 Passing problem to incremental SMT2 solving via "z3 --smt2 -in"
 Sending command to SMT2 solver - \(check-sat\)
 Solver response - unsat