Change regression testing suite to support running tests with cvc5.

This allows us to test against more than one solvers in the new
backend (currently tested - cvc5, z3).

This ensures that we don't invariably issue output that is solver
specific, tying the new backend to a single solver by accident.

Author: NlightNFotis

.github/workflows/pull-request-checks.yaml

@@ -634,6 +634,12 @@ jobs:
           sudo apt-get install -y --allow-downgrades --reinstall gcc g++ libgcc-s1- libstdc++6=$target liblsan0=$target libtsan0=$target libcc1-0=$target libgcc1=1:$target gdb=8.1.1-0ubuntu1
       - name: Confirm z3 solver is available and log the version installed
         run: z3 --version
+      - name: Download cvc-5 from the releases page and make sure it can be deployed
+        run: |
+          wget -O cvc5 https://github.com/cvc5/cvc5/releases/download/cvc5-${{env.cvc5-version}}/cvc5-Linux
+          chmod u+x cvc5
+          mv cvc5 /usr/local/bin
+          cvc5 --version
       - name: Prepare ccache
         uses: actions/cache@v2
         with:

regression/cbmc-incr-smt2/CMakeLists.txt

@@ -1,10 +1,20 @@
 
 if("${CMAKE_SYSTEM_NAME}" STREQUAL "Windows")
-    set(exclude_win_broken_tests -X winbug)
+    set(exclude_win_broken_tests "-X;winbug")
 else()
     set(exclude_win_broken_tests "")
 endif()
 
-add_test_pl_tests(
-    "$<TARGET_FILE:cbmc> --validate-goto-model --validate-ssa-equation --slice-formula" ${exclude_win_broken_tests}
+add_test_pl_profile(
+    "cbmc-new-smt-backend-z3"
+    "$<TARGET_FILE:cbmc> --incremental-smt2-solver 'z3 --smt2 -in' --validate-goto-model --validate-ssa-equation --slice-formula"
+    "-C;${exclude_win_broken_tests};-s;new-smt-z3"
+    "CORE"
+)
+
+add_test_pl_profile(
+    "cbmc-new-smt-backend-cvc5"
+    "$<TARGET_FILE:cbmc> --incremental-smt2-solver 'cvc5 --lang=smtlib2.6 --incremental' --validate-goto-model --validate-ssa-equation --slice-formula"
+    "-C;${exclude_win_broken_tests};-s;new-smt-cvc5"
+    "CORE"
 )

regression/cbmc-incr-smt2/bitvector-arithmetic-operators/overflow_behaviour.desc

@@ -1,6 +1,6 @@
 CORE
 overflow_behaviour.c
---incremental-smt2-solver 'z3 --smt2 -in' --trace
+--trace
 \[main\.assertion\.1\] line \d+ Wrap-around to INT_MIN when adding to INT_MAX: SUCCESS
 \[main\.assertion\.2\] line \d+ Wrap-around to INT_MAX when subtracting from INT_MIN: SUCCESS
 \[main\.assertion\.3\] line \d+ INT_MAX minus INT_MIN equals -1: SUCCESS

regression/cbmc-incr-smt2/bitvector-arithmetic-operators/polynomial.desc

@@ -1,6 +1,6 @@
 CORE
 polynomial.c
---incremental-smt2-solver 'z3 --smt2 -in' --trace
+--trace
 \[main\.assertion\.1\] line \d+ No negative solution: FAILURE
 \[main\.assertion\.2\] line \d+ No positive solution: FAILURE
 x=-8\ \(11111111 11111111 11111111 11111000\)

regression/cbmc-incr-smt2/bitvector-arithmetic-operators/simple_equation.desc

@@ -1,6 +1,6 @@
 CORE
 simple_equation.c
---incremental-smt2-solver 'z3 --smt2 -in' --trace --verbosity 10
+--trace --verbosity 10
 \[main\.assertion\.1\] line \d+ a plus a always equals two times a: SUCCESS
 \[main\.assertion\.2\] line \d+ a minus a always equals 0: SUCCESS
 \[main\.assertion\.3\] line \d+ a plus its additive inverse equals 0: SUCCESS

regression/cbmc-incr-smt2/bitvector-arithmetic-operators/unsigned_behaviour.desc

@@ -1,6 +1,6 @@
 CORE
 unsigned_behaviour.c
---incremental-smt2-solver 'z3 --smt2 -in' --trace
+--trace
 \[main\.assertion\.1\] line \d+ a plus b should be more than 27: FAILURE
 \[main\.assertion\.2\] line \d+ a plus b should be more than 27: FAILURE
 \[main\.assertion\.3\] line \d+ c plus d should be more than 27: FAILURE

regression/cbmc-incr-smt2/bitvector-bitwise-operators/bitwise.desc

@@ -1,6 +1,6 @@
 CORE
 bitwise_ops.c
---incremental-smt2-solver 'z3 --smt2 -in' --slice-formula
+--slice-formula
 \[main\.assertion\.1\] line \d+ This is going to fail for bit-opposites: FAILURE
 \[main\.assertion\.2\] line \d+ This is going to hold for all values != 0: SUCCESS
 \[main\.assertion\.3\] line \d+ This is going to fail for the same value in A and B: FAILURE

regression/cbmc-incr-smt2/bitvector-bitwise-operators/shift_left.desc

@@ -1,6 +1,6 @@
 CORE
 shift_left.c
---incremental-smt2-solver 'z3 --smt2 -in' --slice-formula
+--slice-formula
 \[main\.assertion\.1\] line \d Shifted result should be greater than one: FAILURE
 ^EXIT=10$
 ^SIGNAL=0$

regression/cbmc-incr-smt2/bitvector-bitwise-operators/shift_right.desc

@@ -1,10 +1,10 @@
 CORE
 shift_right.c
---incremental-smt2-solver 'z3 --smt2 -in' --slice-formula --trace
+--slice-formula --trace
 \[main\.assertion\.1\] line \d+ Right shifting a uint with leftmost bit set is a logical shift: FAILURE
 \[main\.assertion\.2\] line \d+ Right shifting a positive number has a lower bound of 0: SUCCESS
 \[main\.assertion\.3\] line \d+ Right shifting a negative number has a lower bound value of -1: SUCCESS
-second=128
+second=[128|\d+]
 result_unsigned=64
 ^EXIT=10$
 ^SIGNAL=0$

regression/cbmc-incr-smt2/bitvector-flag-tests/div_by_zero.desc

@@ -1,6 +1,6 @@
 CORE
 div_by_zero.c
---incremental-smt2-solver 'z3 --smt2 -in' --div-by-zero-check --trace
+--div-by-zero-check --trace
 \[main\.division-by-zero\.1\] line \d division by zero in x / y: FAILURE
 \[main\.division-by-zero\.2\] line \d+ division by zero in x / z: SUCCESS
 y=0

regression/cbmc-incr-smt2/bitvector-flag-tests/signed_overflow.desc

@@ -1,6 +1,6 @@
 CORE
 signed_overflow.c
---incremental-smt2-solver 'z3 --smt2 -in' --signed-overflow-check --trace
+--signed-overflow-check --trace
 ^VERIFICATION FAILED$
 ^EXIT=10$
 ^SIGNAL=0$

regression/cbmc-incr-smt2/nondeterministic-int-assert/control_flow.desc

@@ -1,7 +1,7 @@
 CORE
 control_flow.c
---incremental-smt2-solver 'z3 --smt2 -in' --verbosity 10
-Passing problem to incremental SMT2 solving via "z3 --smt2 -in"
+--verbosity 10
+Passing problem to incremental SMT2 solving via
 Sending command to SMT2 solver - \(set-option :produce-models true\)
 Sending command to SMT2 solver - \(set-logic QF_UFBV\)
 Sending command to SMT2 solver - \(declare-fun |goto_symex::&92;guard#1| \(\) Bool\)

regression/cbmc-incr-smt2/nondeterministic-int-assert/incremental_solver_called.desc

@@ -1,7 +1,7 @@
 CORE
 test.c
---incremental-smt2-solver 'z3 --smt2 -in' --verbosity 10
-Passing problem to incremental SMT2 solving via "z3 --smt2 -in"
+--verbosity 10
+Passing problem to incremental SMT2 solving via
 Sending command to SMT2 solver - \(set-option :produce-models true\)
 Sending command to SMT2 solver - \(set-logic QF_UFBV\)
 Sending command to SMT2 solver - \(define-fun |B0| \(\) Bool true\)

regression/cbmc-incr-smt2/nondeterministic-int-assert/stdbool.desc

@@ -1,7 +1,7 @@
 CORE
 stdbool_example.c
---incremental-smt2-solver 'z3 --smt2 -in' --trace
-Passing problem to incremental SMT2 solving via "z3 --smt2 -in"
+--trace
+Passing problem to incremental SMT2 solving via
 VERIFICATION FAILED
 equal=FALSE\s*\([0 ]+\)
 equal=TRUE\s*\([0 ]+1\)

regression/cbmc-incr-smt2/nondeterministic-int-assert/trace.desc

@@ -1,7 +1,7 @@
 CORE
 trace.c
---incremental-smt2-solver 'z3 --smt2 -in' --trace
-Passing problem to incremental SMT2 solving via "z3 --smt2 -in"
+--trace
+Passing problem to incremental SMT2 solving via
 Assert of inequality to 4\.: FAILURE
 Assert of inequality to 2\.: FAILURE
 y=4

regression/cbmc-incr-smt2/nondeterministic-int-assert/valid_unsat.desc

@@ -1,7 +1,7 @@
 CORE
 valid_unsat.c
---incremental-smt2-solver 'z3 --smt2 -in' --verbosity 10
-Passing problem to incremental SMT2 solving via "z3 --smt2 -in"
+--verbosity 10
+Passing problem to incremental SMT2 solving via
 Sending command to SMT2 solver - \(check-sat\)
 Solver response - unsat
 VERIFICATION SUCCESSFUL