Merge pull request diffblue#1907 from chrisr-diffblue/travis-test-speedups

Travis test speedups

Author: tautschnig

.travis.yml

@@ -62,6 +62,7 @@ jobs:
             - libwww-perl
             - g++-5
             - libubsan0
+            - parallel
       before_install:
         - mkdir bin ; ln -s /usr/bin/gcc-5 bin/gcc
       # env: COMPILER=g++-5 SAN_FLAGS="-fsanitize=undefined -fno-sanitize-recover -fno-omit-frame-pointer"
@@ -76,7 +77,7 @@ jobs:
       compiler: gcc
       cache: ccache
       before_install:
-        - HOMEBREW_NO_AUTO_UPDATE=1 brew install ccache
+        - HOMEBREW_NO_AUTO_UPDATE=1 brew install ccache parallel
         - export PATH=$PATH:/usr/local/opt/ccache/libexec
       env: COMPILER="ccache g++"
 
@@ -87,7 +88,7 @@ jobs:
       compiler: clang
       cache: ccache
       before_install:
-        - HOMEBREW_NO_AUTO_UPDATE=1 brew install ccache
+        - HOMEBREW_NO_AUTO_UPDATE=1 brew install ccache parallel
         - export PATH=$PATH:/usr/local/opt/ccache/libexec
       env:
         - COMPILER="ccache clang++"
@@ -132,6 +133,7 @@ jobs:
             - clang-3.7
             - libstdc++-5-dev
             - libubsan0
+            - parallel
       before_install:
         - mkdir bin ; ln -s /usr/bin/clang-3.7 bin/gcc
         - export CCACHE_CPP2=yes
@@ -258,7 +260,7 @@ install:
 
 script:
   - if [ -e bin/gcc ] ; then export PATH=$PWD/bin:$PATH ; fi ;
-  - env UBSAN_OPTIONS=print_stacktrace=1 make -C regression test "CXX=${COMPILER} ${EXTRA_CXXFLAGS}" -j2
+  - env UBSAN_OPTIONS=print_stacktrace=1 make -C regression test-parallel "CXX=${COMPILER} ${EXTRA_CXXFLAGS}" -j2 JOBS=2
   - make -C unit "CXX=${COMPILER} ${EXTRA_CXXFLAGS}" -j2
   - make -C unit test
 

regression/CMakeLists.txt

@@ -20,25 +20,29 @@ macro(add_test_pl_tests cmdline)
     add_test_pl_profile("${TEST_DIR_NAME}" "${cmdline}" -K KNOWNBUG ${ARGN})
 endmacro(add_test_pl_tests)
 
-add_subdirectory(ansi-c)
+# For the best possible utilisation of multiple cores when
+# running tests in parallel, it is important that these directories are
+# listed with decreasing runtimes (i.e. longest running at the top)
 add_subdirectory(cbmc)
-add_subdirectory(cbmc-cover)
-add_subdirectory(cbmc-cpp)
 add_subdirectory(cbmc-java)
-add_subdirectory(cbmc-java-inheritance)
-add_subdirectory(cpp)
 add_subdirectory(goto-analyzer)
-add_subdirectory(goto-analyzer-taint)
-add_subdirectory(goto-cc-cbmc)
-add_subdirectory(goto-cc-goto-analyzer)
-add_subdirectory(goto-diff)
+add_subdirectory(ansi-c)
+add_subdirectory(jbmc-strings)
 add_subdirectory(goto-instrument)
+add_subdirectory(cpp)
+add_subdirectory(strings-smoke-tests)
+add_subdirectory(cbmc-cover)
 add_subdirectory(goto-instrument-typedef)
+add_subdirectory(strings)
+add_subdirectory(invariants)
+add_subdirectory(goto-diff)
+add_subdirectory(test-script)
+add_subdirectory(goto-analyzer-taint)
+add_subdirectory(cbmc-java-inheritance)
 if(NOT WIN32)
   add_subdirectory(goto-gcc)
 endif()
-add_subdirectory(invariants)
-add_subdirectory(jbmc-strings)
-add_subdirectory(strings)
-add_subdirectory(strings-smoke-tests)
-add_subdirectory(test-script)
+add_subdirectory(goto-cc-cbmc)
+add_subdirectory(cbmc-cpp)
+add_subdirectory(goto-cc-goto-analyzer)
+

regression/Makefile

@@ -1,34 +1,62 @@
-DIRS = ansi-c \
-       cbmc \
-       cbmc-cover \
-       cbmc-cpp \
+# For the best possible utilisation of multiple cores when
+# running tests in parallel, it is important that these directories are
+# listed with decreasing runtimes (i.e. longest running at the top)
+DIRS = cbmc \
        cbmc-java \
-       cbmc-java-inheritance \
-       cpp \
        goto-analyzer \
-       goto-analyzer-taint \
-       goto-cc-cbmc \
-       goto-cc-goto-analyzer \
-       goto-diff \
-       goto-gcc \
+       ansi-c \
+       jbmc-strings \
        goto-instrument \
+       cpp \
+       strings-smoke-tests \
+       cbmc-cover \
        goto-instrument-typedef \
-       invariants \
        strings \
-       jbmc-strings \
-       strings-smoke-tests \
+       invariants \
+       goto-diff \
        test-script \
+       goto-analyzer-taint \
+       cbmc-java-inheritance \
+       goto-gcc \
+       goto-cc-cbmc \
+       cbmc-cpp \
+       goto-cc-goto-analyzer \
        # Empty last line
 
-# Check for the existence of $dir. Tests under goto-gcc cannot be run on
-# Windows, so appveyor.yml unlinks the entire directory under Windows.
+# Tests under goto-gcc cannot be run on Windows, so appveyor.yml unlinks
+# the entire directory under Windows. This variable will contain the list
+# of directories that actually exist on the current platform.
+PLATFORM_DIRS = $(wildcard $(DIRS))
+
+# Run all test directories in sequence
+.PHONY: test
 test:
-	@for dir in $(DIRS); do \
-	  if [ -d "$$dir" ]; then \
-	    $(MAKE) -C "$$dir" test || exit 1; \
-	  fi; \
+	@for dir in $(PLATFORM_DIRS); do \
+		$(MAKE) "$$dir" || exit 1; \
 	done;
 
+# Pattern to execute a single test suite directory
+.PHONY: $(PLATFORM_DIRS)
+$(PLATFORM_DIRS):
+	@echo "Running $@..." ;
+	$(MAKE) -C "$@" test || exit 1;
+
+# Run all test directories using GNU Parallel
+.PHONY: test-parallel
+.NOTPARALLEL: test-parallel
+test-parallel:
+	@echo "Building with $(JOBS) jobs"
+	parallel \
+		--halt soon,fail=1 \
+		--tag \
+		--tagstring '{#}:' \
+		--linebuffer \
+		--jobs $(JOBS) \
+		$(MAKE) "{}" \
+		::: $(PLATFORM_DIRS)
+
+
+.PHONY: clean
 clean:
 	@for dir in *; do \
 		if [ -d "$$dir" ]; then \