Made the Perl script not using grep

Includes tests for excluded lines and multi lines.

Includes tests that the test runner should fail, these are set as KNOWNBUG
so we can run the regressions folder.

We also don't print out when grep options activated as  no chance of doing
by accident as very clearly different to any other syntax. The printing
messes up the formatting of the output so better to not have it.

Added test script to the Makefile

Also ordered the tests into alphabetical orde.r

Author: thk123

regression/Makefile

@@ -1,5 +1,13 @@
 
-DIRS = ansi-c cbmc cpp goto-instrument goto-analyzer cbmc-java
+DIRS = ansi-c \
+       cbmc \
+       cpp \
+       cbmc-java \
+       goto-analyzer \
+       goto-instrument \
+       test-script \
+       # Empty last line
+
 
 test:
 	$(foreach var,$(DIRS), $(MAKE) -C $(var) test || exit 1;)

regression/test-script/Makefile

@@ -0,0 +1,32 @@
+
+default: tests.log
+
+test:
+	@if ! ../test.pl -c ../program_runner.sh ; then \
+		../failed-tests-printer.pl ; \
+		exit 1; \
+	fi
+
+tests.log:
+	pwd
+	@if ! ../test.pl -c ../program_runner.sh ; then \
+		../failed-tests-printer.pl ; \
+		exit 1; \
+	fi
+
+show:
+	@for dir in *; do \
+		if [ -d "$$dir" ]; then \
+			vim -o "$$dir/*.c" "$$dir/*.out"; \
+		fi; \
+	done;
+
+clean:
+	@for dir in *; do \
+		rm -f tests.log; \
+		if [ -d "$$dir" ]; then \
+			cd "$$dir"; \
+			rm -f *.out *.gb; \
+			cd ..; \
+		fi \
+	done

regression/test-script/excluded-line/program.c

@@ -0,0 +1,7 @@
+#include <stdio.h>
+
+int main(void) {
+  printf("Hello\n");
+  printf("World\n");
+  return 0;
+}

regression/test-script/excluded-line/test.desc

@@ -0,0 +1,6 @@
+CORE
+program.c
+
+^Hello$
+--
+^Goodbye$

regression/test-script/failing-excluded-line/program.c

@@ -0,0 +1,7 @@
+#include <stdio.h>
+
+int main(void) {
+  printf("Hello\n");
+  printf("World\n");
+  return 0;
+}

regression/test-script/failing-excluded-line/test.desc

@@ -0,0 +1,9 @@
+KNOWNBUG
+program.c
+
+^Hello$
+--
+^World$
+--
+This isn't really a known bug perse, instead this test should fail (since these
+are tests of the test runner).

regression/test-script/failing-multi-line/program.c

@@ -0,0 +1,7 @@
+#include <stdio.h>
+
+int main(void) {
+  printf("Hello\n");
+  printf("World\n");
+  return 0;
+}

regression/test-script/failing-multi-line/test.desc

@@ -0,0 +1,9 @@
+KNOWNBUG
+program.c
+
+activate-multi-line-match
+Hello\nAnother\nWorld
+--
+--
+This isn't really a known bug perse, instead this test should fail (since these
+are tests of the test runner).

regression/test-script/failing-single-line/program.c

@@ -0,0 +1,7 @@
+#include <stdio.h>
+
+int main(void) {
+  printf("Hello\n");
+  printf("World\n");
+  return 0;
+}

regression/test-script/failing-single-line/test.desc

@@ -0,0 +1,8 @@
+KNOWNBUG
+program.c
+
+^Goodbye$
+--
+--
+This isn't really a known bug perse, instead this test should fail (since these
+are tests of the test runner).

regression/test-script/multi-line/program.c

@@ -0,0 +1,7 @@
+#include <stdio.h>
+
+int main(void) {
+  printf("Hello\n");
+  printf("World\n");
+  return 0;
+}

regression/test-script/multi-line/test.desc

@@ -0,0 +1,6 @@
+CORE
+program.c
+
+activate-multi-line-match
+Hello\nWorld
+--

regression/test-script/program_runner.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -e
+
+gcc $1 -o a.out
+./a.out

regression/test-script/single-line/program.c

@@ -0,0 +1,7 @@
+#include <stdio.h>
+
+int main(void) {
+  printf("Hello\n");
+  printf("World\n");
+  return 0;
+}

regression/test-script/single-line/test.desc

@@ -0,0 +1,5 @@
+CORE
+program.c
+
+^Hello$
+--

regression/test.pl

@@ -4,6 +4,8 @@
 use strict;
 use warnings;
 
+use Cwd;
+
 my $has_thread_pool = eval
 {
   # "http://www.cpan.org/authors/id/J/JW/JWU/Thread-Pool-Simple/Thread-Pool-Simple-0.25.tar.gz"
@@ -61,13 +63,9 @@ ($$$$$)
   my ($name, $test, $t_level, $cmd, $ign) = @_;
   my ($level, $input, $options, $grep_options, @results) = load("$test");
 
-  # If the 4th line starts with a '-' we use that line as options to pass to
-  # grep when matching all lines in this test
-  if($grep_options =~ /^-/) {
-    print "\nActivating perl flags: $grep_options\n";
-  }
-  else {
-    # No grep options so stick this back into the results array
+  # If the 4th line is activate-multi-line-match we enable multi-line checks
+  if($grep_options ne "activate-multi-line-match") {
+    # No such flag, so we add it back in
     unshift @results, $grep_options;
     $grep_options = "";
   }
@@ -117,10 +115,43 @@ ($$$$$)
           $included--;
         } else {
           my $r;
-          $result =~ s/\\/\\\\/g;
-          $result =~ s/([^\\])\$/$1\\r\\\\?\$/;
-          system("bash", "-c", "grep $grep_options \$'$result' '$name/$output' >/dev/null");
-          $r = ($included ? $? != 0 : $? == 0);
+
+          my $dir = getcwd();
+          my $abs_path = "$dir/$name/$output";
+          open(my $fh => $abs_path) || die "Cannot open '$name/$output': $!";
+
+          # Multi line therefore we run each check across the whole output
+          if($grep_options eq "activate-multi-line-match") {
+            local $/ = undef;
+            binmode $fh;
+            my $whole_file = <$fh>;
+            my $is_match = $whole_file =~ /$result/;
+            $r = ($included ? !$is_match : $is_match);
+          }
+          else
+          {
+            my $found_line = 0;
+            while(my $line = <$fh>) {
+              if($line =~ /$result/) {
+                # We've found the line, therefore if it is included we set
+                # the result to 0 (OK) If it is excluded, we set the result
+                # to 1 (FAILED)
+                $r = !$included;
+                $found_line = 1;
+                last;
+              }
+            }
+
+            if($found_line == 0) {
+              # None of the lines matched, therefore the result is set to
+              # 0 (OK) if and only if the line was not meant to be included
+              $r = $included;
+            }
+
+          }
+          close($fh);
+
+
           if($r) {
             print LOG "$result [FAILED]\n";
             $failed = 1;
@@ -185,24 +216,23 @@ ($$$$)
 <level>
 <main source>
 <options>
-<grep_options>
+<activate-multi-line-match>
 <required patterns>
 --
 <disallowed patterns>
 --
 <comment text>
 
 where
-  <level>                is one of CORE, THOROUGH, FUTURE or KNOWNBUG
-  <main source>          is a file with extension .c/.i/.gb/.cpp/.ii/.xml/.class/.jar
-  <options>              additional options to be passed to CMD
-  <grep_options>         additional flags to be passed to grep when checking required
-                         patterns (this is optional, if the line stats with a `-'
-                         it will be used as grep options. Otherwise, it will be
-                         considered part of the required patterns)
-  <required patterns>    one or more lines of regualar expressions that must occur in the output
-  <disallowed patterns>  one or more lines of expressions that must not occur in output
-  <comment text>         free form text
+  <level>                         is one of CORE, THOROUGH, FUTURE or KNOWNBUG
+  <main source>                   is a file with extension .c/.i/.gb/.cpp/.ii/.xml/.class/.jar
+  <options>                       additional options to be passed to CMD
+  <activate-multi-line-match>     The fourth line can optionally be activate-multi-line-match, if this is the
+                                  case then each of the rules will be matched over multiple lines (so can contain)
+                                  the new line symbol (\\n) inside them.
+  <required patterns>             one or more lines of regualar expressions that must occur in the output
+  <disallowed patterns>           one or more lines of expressions that must not occur in output
+  <comment text>                  free form text
 
 EOF
   exit 1;