Merge pull request diffblue#511 from diffblue/owen-jones-diffblue/benchmarks-cron-job

Set up cron job for genuine benchmarks [SEC-471]

Author: owen-jones-diffblue

.travis.yml

@@ -7,6 +7,7 @@ jobs:
   include:
 
     - &formatting-stage
+      if: type != cron
       stage: Linter + Doxygen + non-debug Ubuntu/gcc-5 test
       env: NAME="clang-format"
       addons:
@@ -37,14 +38,16 @@ jobs:
 # LOCAL CHANGES: added cbmc/ prefix to both these scripts
 
     - &linter-stage
+      if: type != cron
       stage: Linter + Doxygen + non-debug Ubuntu/gcc-5 test
       env: NAME="CPP-LINT"
       install:
       script: cbmc/scripts/travis_lint.sh
       before_cache:
 
     - &doxygen-stage
-    - stage: Linter + Doxygen + non-debug Ubuntu/gcc-5 test
+      if: type != cron
+      stage: Linter + Doxygen + non-debug Ubuntu/gcc-5 test
       env: NAME="DOXYGEN-CHECK"
       addons:
         apt:
@@ -55,7 +58,8 @@ jobs:
       before_cache:
 
     # Ubuntu Linux with glibc using g++-5
-    - stage: Linter + Doxygen + non-debug Ubuntu/gcc-5 test
+    - if: type != cron
+      stage: Linter + Doxygen + non-debug Ubuntu/gcc-5 test
       os: linux
       sudo: false
       compiler: gcc
@@ -85,7 +89,8 @@ jobs:
 # LOCAL CHANGES: Added 'ant' to the OSX package list (needed by security-scanner regression tests)
 
     # OS X using g++
-    - stage: Test different OS/CXX/Flags
+    - if: type != cron
+      stage: Test different OS/CXX/Flags
       os: osx
       jdk: openjdk8
       osx_image: xcode8.3
@@ -105,7 +110,8 @@ jobs:
         - RUN_SECURITY_TESTS="true"
 
     # OS X using clang++
-    - stage: Test different OS/CXX/Flags
+    - if: type != cron
+      stage: Test different OS/CXX/Flags
       os: osx
       jdk: openjdk8
       osx_image: xcode8.3
@@ -124,7 +130,8 @@ jobs:
         - CCACHE_CPP2=yes
 
     # Ubuntu Linux with glibc using g++-5, debug mode
-    - stage: Test different OS/CXX/Flags
+    - if: type != cron
+      stage: Test different OS/CXX/Flags
       os: linux
       dist: trusty
       sudo: false
@@ -157,7 +164,8 @@ jobs:
         - TEST_MODELS_LIB="false"
 
     # Ubuntu Linux with glibc using clang++-3.7
-    - stage: Test different OS/CXX/Flags
+    - if: type != cron
+      stage: Test different OS/CXX/Flags
       os: linux
       dist: trusty
       sudo: false
@@ -193,7 +201,8 @@ jobs:
         - OS="ubuntu"
 
     # Ubuntu Linux with glibc using clang++-3.7, debug mode
-    - stage: Test different OS/CXX/Flags
+    - if: type != cron
+      stage: Test different OS/CXX/Flags
       os: linux
       sudo: false
       compiler: clang
@@ -224,6 +233,39 @@ jobs:
         - RUN_SECURITY_TESTS="true"
         - CCACHE_CPP2=yes
 
+# LOCAL CHANGES: added cron-job stage
+
+    - &cron-job-stage
+      if: type == cron
+      stage: cron-job
+      os: linux
+      sudo: false
+      compiler: gcc
+      cache: ccache
+      addons:
+        apt:
+          sources:
+            - ubuntu-toolchain-r-test
+            - deadsnakes
+          packages:
+            - libwww-perl
+            - g++-5
+            - libubsan0
+            - python3-pip
+            - libc6-dev-i386
+      before_install:
+        - mkdir bin ; ln -s /usr/bin/gcc-5 bin/gcc
+          # Install pytest locally and adjust PATH so pip3 can find it
+        - pip3 install -q pytest fasteners --user
+        - export PATH=$HOME/.local/bin:$PATH
+      env:
+        - COMPILER="/usr/bin/g++-5"
+        - BUILD_TYPE="Release"
+        - RUN_CBMC_TESTS="false"
+        - RUN_SECURITY_TESTS="true"
+        - SECURITY_SCANNER_HOME=${TRAVIS_BUILD_DIR}/build
+      script: (cd regression; python3 -m pytest -v -s --run-slow-only)
+
 
   allow_failures:
     - <<: *linter-stage

benchmarks/GENUINE/Alfresco.sh

@@ -11,10 +11,14 @@ set -e
 SCRIPT_DIR="$( cd "$(dirname "$0")" ; pwd -P )"
 REPO_DIR=$SCRIPT_DIR/Alfresco
 DEPLOY_DIR=$REPO_DIR/__dist__
+FILES_DIR=$SCRIPT_DIR/Alfresco_files
+LIB_DIR=$SCRIPT_DIR/../LIBRARIES
 if [ -z "$OUTPUT_DIR" ]; then
     OUTPUT_DIR=$SECURITY_SCANNER_HOME/GENUINE
 fi
 
+(cd $LIB_DIR && ./install_apache_tomcat.sh)
+
 if [[ ! -d $REPO_DIR ]]; then
     # First switch to Java 7
     java_version=$(java -version 2>&1 | awk -F '"' '/version/ {print $2}')
@@ -40,10 +44,8 @@ if [[ ! -d $REPO_DIR ]]; then
             echo "  where <name> is one of the options returned from the first command."
             exit 1
         fi
-        echo "TEMPORARILY SWITCHING JAVA TO VERSION 7 LOCATED HERE:"
-        echo "    /usr/lib/jvm/java-7-openjdk-amd64/bin"
         PATH=/usr/lib/jvm/java-7-openjdk-amd64/bin:$PATH
-        JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64/bin
+        JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64
     fi
 
     # Clone the repository and check out a commit which has the issue.
@@ -55,8 +57,11 @@ if [[ ! -d $REPO_DIR ]]; then
     cd $REPO_DIR
     svn checkout -r 74720 https://svn.alfresco.com/repos/alfresco-open-mirror/alfresco/HEAD/root .
 
+    # Fix dependency version
+    patch -p1 -f < $FILES_DIR/update_dependency.patch
+
     # Build and install
-    mvn install -DskipTests
+    mvn install -DskipTests -Dhttps.protocols=TLSv1.2
 
     # Deploy web apps
     mkdir -p $DEPLOY_DIR/webapps

benchmarks/GENUINE/Alfresco_files/update_dependency.patch

@@ -0,0 +1,11 @@
+--- Alfresco/projects/repository/pom.xml	2018-07-19 17:39:07.513596311 +0100
++++ Alfresco-fixed/projects/repository/pom.xml	2018-07-20 10:32:46.083159162 +0100
+@@ -579,7 +579,7 @@
+         <dependency>
+             <groupId>org.springframework.social</groupId>
+             <artifactId>spring-social-facebook</artifactId>
+-            <version>1.0.0.RC1</version>
++            <version>1.0.0.RELEASE</version>
+         </dependency>
+         <dependency>
+             <groupId>org.springframework.social</groupId>

benchmarks/GENUINE/Sakai.sh

@@ -40,7 +40,7 @@ python3 $SCRIPT_DIR/../../driver/run.py \
 -R $OUTPUT_DIR/Sakai/results \
 -T $OUTPUT_DIR/Sakai/temp \
 --name sakai_assignment_tool \
---use-models-library --use-apache-tomcat --use-spring-framework \
+--use-models-library \
 --timeout 10000000 --verbosity 9 --rebuild \
 --do-not-use-precise-access-paths \
 --entry-point org.sakaiproject.assignment.tool.AssignmentAction.doUpload_all

benchmarks/GENUINE/WebGoat.sh

@@ -52,19 +52,25 @@ fi
 # Run security-analyser on each lesson which works separately
 cd $SECURITY_SCANNER_HOME
 
+rm -rf ${OUTPUT_DIR}/WebGoat
+mkdir -p ${OUTPUT_DIR}/WebGoat/results
+
 for LESSON in $LESSONS_WHICH_WORK_SQL
 do
 	python3 $SCRIPT_DIR/../../driver/run.py \
 	-C $SCRIPT_DIR/WebGoatRulesSQL.json \
 	-I $DEPLOY_DIR \
 	-L $DEPLOY_DIR \
-	-R $OUTPUT_DIR/WebGoat/results/$LESSON \
-	-T $OUTPUT_DIR/WebGoat/temp \
+	-R $OUTPUT_DIR/WebGoat/${LESSON}/results \
+	-T $OUTPUT_DIR/WebGoat/${LESSON}/temp \
 	--name WebGoat \
 	--use-models-library \
 	--timeout 10000000  --verbosity 9 --rebuild \
 	--do-not-use-precise-access-paths \
 	--entry-point Main.$LESSON
+
+	mv ${OUTPUT_DIR}/WebGoat/${LESSON}/results/* ${OUTPUT_DIR}/WebGoat/results
+	rm -rf ${OUTPUT_DIR}/WebGoat/${LESSON}
 done
 
 for LESSON in $LESSONS_WHICH_WORK_XSS
@@ -73,11 +79,14 @@ do
 	-C $SCRIPT_DIR/WebGoatRulesXSS.json \
 	-I $DEPLOY_DIR \
 	-L $DEPLOY_DIR \
-	-R $OUTPUT_DIR/WebGoat/results/$LESSON \
-	-T $OUTPUT_DIR/WebGoat/temp \
+	-R $OUTPUT_DIR/WebGoat/${LESSON}/results \
+	-T $OUTPUT_DIR/WebGoat/${LESSON}/temp \
 	--name WebGoat \
 	--use-models-library \
 	--timeout 10000000  --verbosity 9 --rebuild \
 	--do-not-use-precise-access-paths \
 	--entry-point Main.$LESSON
+
+	mv ${OUTPUT_DIR}/WebGoat/${LESSON}/results/* ${OUTPUT_DIR}/WebGoat/results
+	rm -rf ${OUTPUT_DIR}/WebGoat/${LESSON}
 done

driver/run.py

@@ -462,8 +462,7 @@ def __main():
     if cmdline.name is None:
         cmdline.name = os.path.basename(cmdline.install_dir)
 
-    # If you're preparing a scan you implicitly want to rebuild everything.
-    if cmdline.rebuild or cmdline.prepare_scan:
+    if cmdline.rebuild:
         print("First performing cleanup.")
         print("  Deleting " + cmdline.temp_dir)
         if os.path.exists(cmdline.temp_dir):

regression/conftest.py

@@ -11,3 +11,24 @@ class LoadStrategy(Enum):
 def load_strategy(request):
     """Use fixture to parametrize tests"""
     return request.param
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--run-slow-only", action="store_true", default=False, help="run slow tests only"
+    )
+
+
+def pytest_collection_modifyitems(config, items):
+    if config.getoption("--run-slow-only"):
+        # --run-slow-only given in cli: skip tests that are not marked slow
+        skip_non_slow = pytest.mark.skip(reason="need to not have --run-slow-only option to run this test")
+        for item in items:
+            if "slow" not in item.keywords:
+                item.add_marker(skip_non_slow)
+    else:
+        # --run-slow-only not given in cli: skip tests that are marked slow
+        skip_slow = pytest.mark.skip(reason="need --run-slow-only option to run this test")
+        for item in items:
+            if "slow" in item.keywords:
+                item.add_marker(skip_slow)

regression/executable_runner.py

@@ -1,35 +1,88 @@
+import os
+import signal
 import subprocess
+import time
 
 
 class ExecutableRunner:
     """Run an executable"""
 
-    def __init__(self, cmd, timeout=600, encoding='utf-8'):
+    def __init__(self, cmd, timeout=10, encoding='utf-8'):
         """Set the command to run, the timeout (optional) and the encoding (optional)
 
         cmd should be in the format excepted by subprocess.Popen, i.e. [executable_path, arg1, arg2, ...]
+        timeout is in minutes
         """
         self.cmd = cmd
         self.timeout = timeout
         self.encoding = encoding
 
-    def run(self, pipe_stderr_to_stdout=False):
+    def run(self, pipe_stderr_to_stdout=False, env_dict=None, regular_log_messages=False):
         """Run the executable and capture the output
 
-        If the process exceeds the timeout then we assert false so that the test fails.
+        If the process exceeds the timeout then we raise a TimeoutError.
 
         Return the tuple (output to stdout (string), output to stderr (string), exit code (int)).
         """
         # We limit ourselves to the API available in python 3.3 to make life easier on Travis. There are API
         # improvements in python 3.5 and 3.6 which would make this code simpler.
+
+        # Allow the caller to pipe stderr to stdout
         stderr = subprocess.STDOUT if pipe_stderr_to_stdout else subprocess.PIPE
-        proc = subprocess.Popen(self.cmd, stdout=subprocess.PIPE, stderr=stderr)
-        try:
-            (stdout_data, stderr_data) = proc.communicate(timeout=self.timeout)
-        except subprocess.TimeoutExpired:
-            proc.kill()
-            print('The executable ran for longer than {} seconds'.format(self.timeout))
-            assert False
+
+        # Allow the caller to set environment variables in the subprocess
+        env = None
+        if env_dict:
+            env = os.environ.copy()
+            for key, value in env_dict.items():
+                env[key] = value
+
+        # We must pass start_new_session=True so that we can kill the subprocess and all of its children using the
+        # pid of the subprocess, without killing the process that pytest is running in
+        proc = subprocess.Popen(self.cmd, stdout=subprocess.PIPE, stderr=stderr, env=env, start_new_session=True)
+        if regular_log_messages:
+            stdout_data, stderr_data = get_proc_output_with_regular_log_messages(proc, self.timeout)
+        else:
+            stdout_data, stderr_data = get_proc_output(proc, self.timeout)
+
         return (stdout_data.decode(self.encoding) if stdout_data else None,
                 stderr_data.decode(self.encoding) if stderr_data else None,
                 proc.returncode)
+
+
+def get_proc_output(proc, timeout):
+    try:
+        stdout_data, stderr_data = proc.communicate(timeout=60*timeout)
+    except subprocess.TimeoutExpired:
+        kill_process_and_children(proc.pid)
+        print('The executable ran for longer than {} minutes'.format(timeout))
+        raise TimeoutError
+    return stdout_data, stderr_data
+
+
+def get_proc_output_with_regular_log_messages(proc, timeout):
+    stdout_data = None
+    stderr_data = None
+    num_minutes_run = 0
+    print("")
+    process_completed = False
+    while not process_completed:
+        try:
+            stdout_data, stderr_data = proc.communicate(timeout=60)
+            process_completed = True
+        except subprocess.TimeoutExpired:
+            print("pytest still running... ")
+            num_minutes_run += 1
+            if num_minutes_run >= timeout:
+                kill_process_and_children(proc.pid)
+                print('The executable ran for longer than {} minutes'.format(timeout))
+                raise TimeoutError
+    return stdout_data, stderr_data
+
+
+def kill_process_and_children(pid):
+    # We can't just run `proc.kill()` because we have run a shell script which has run a python script,
+    # which has run jbmc, and `proc.kill()` would only kill the shell script and not its children
+    os.killpg(os.getpgid(pid), signal.SIGTERM)
+    time.sleep(5)
+    os.killpg(os.getpgid(pid), signal.SIGKILL)