add Travis CI test isolation controls

Author: mattsb42-aws

.travis.yml

@@ -4,33 +4,45 @@ matrix:
   include:
     # CPython 2.7
     - python: 2.7
-      env: TOXENV=py27-local-fast
+      env: TOXENV=py27-travis-local-slow
     - python: 2.7
-      env: TOXENV=py27-integ-fast
+      env: TOXENV=py27-travis-integ-slow
+    - python: 2.7
+      env: TOXENV=py27-travis-isolation
     # CPython 3.4
     - python: 3.4
-      env: TOXENV=py34-local-fast
+      env: TOXENV=py34-travis-local-slow
+    - python: 3.4
+      env: TOXENV=py34-travis-integ-slow
     - python: 3.4
-      env: TOXENV=py34-integ-fast
+      env: TOXENV=py34-travis-isolation
     # CPython 3.5
     - python: 3.5
-      env: TOXENV=py35-local-fast
+      env: TOXENV=py35-travis-local-slow
     - python: 3.5
-      env: TOXENV=py35-integ-fast
+      env: TOXENV=py35-travis-integ-slow
+    - python: 3.5
+      env: TOXENV=py35-travis-isolation
     # CPython 3.6
     - python: 3.6
-      env: TOXENV=py36-local-fast
+      env: TOXENV=py36-travis-local-slow
+    - python: 3.6
+      env: TOXENV=py36-travis-integ-slow
     - python: 3.6
-      env: TOXENV=py36-integ-fast
+      env: TOXENV=py36-travis-isolation
     # CPython 3.7
     # xenial + sudo are currently needed to get 3.7
     # https://github.com/travis-ci/travis-ci/issues/9815
     - python: 3.7
-      env: TOXENV=py37-local-fast
+      env: TOXENV=py37-travis-local-slow
+      dist: xenial
+      sudo: true
+    - python: 3.7
+      env: TOXENV=py37-travis-integ-slow
       dist: xenial
       sudo: true
     - python: 3.7
-      env: TOXENV=py37-integ-fast
+      env: TOXENV=py37-travis-isolation
       dist: xenial
       sudo: true
     # Upstream tests

setup.cfg

@@ -27,6 +27,7 @@ markers =
     slow: mark a test as being known to take a long time to complete (order 5s < t < 60s)
     veryslow: mark a test as being known to take a very long time to complete (order t > 60s)
     nope: mark a test as being so slow that it should only be very infrequently (order t > 30m)
+    travis_isolation: mark a test that crashes Travis CI when run with other tests
 log_level=NOTSET
 
 # Flake8 Configuration

test/functional/delegated_keys/test_jce.py

@@ -74,6 +74,7 @@ def build_short_key_cases():
     yield ("RSA", MinimumKeySizes.RSA.value // 2, True, message)
 
 
+@pytest.mark.travis_isolation
 @pytest.mark.parametrize("algorithm, key_bits, too_short, error_message", build_short_key_cases())
 def test_warn_on_short_keys(caplog, algorithm, key_bits, too_short, error_message):
     with caplog.at_level(logging.DEBUG):

tox.ini

@@ -63,6 +63,10 @@ commands =
     integ-slow: {[testenv:base-command]commands} test/ -m "integ and not ddb_integ and not veryslow and not nope"
     ddb-slow: {[testenv:base-command]commands} test/ -m "ddb_integ and not veryslow and not nope"
     all-slow: {[testenv:base-command]commands} test/ -m "not veryslow and not nope"
+    # Only run those tests that need to be isolated in Travis CI
+    travis-isolation: {[testenv:base-command]commands} test/ -m travis_isolation
+    travis-local-slow: {[testenv:base-command]commands} test/ -m "local and not veryslow and not nope and not travis_isolation"
+    travis-integ-slow: {[testenv:base-command]commands} test/ -m "integ and not ddb_integ and not veryslow and not nope and not travis_isolation"
     # Also run very large test scenario sets
     local-full: {[testenv:base-command]commands} test/ -m "local and not nope"
     integ-full: {[testenv:base-command]commands} test/ -m "integ and not ddb_integ and not nope"