change: build spec improvements. (aws#888)

Author: mvsusp

buildspec.yml

@@ -8,39 +8,46 @@ phases:
   build:
     commands:
       # run linters
-      - tox -e flake8,pylint
-      # run package and docbuild checks
-      - tox -e twine
+      - TOX_PARALLEL_NO_SPINNER=1
+      - PY_COLORS=0
+      - tox -e flake8,pylint,twine,black-check --parallel all
       - tox -e sphinx
 
-      # run format verification
-      - tox -e black-check
-
       # run unit tests
       - AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= AWS_SESSION_TOKEN=
         AWS_CONTAINER_CREDENTIALS_RELATIVE_URI= AWS_DEFAULT_REGION=
-        tox -e py36,py27 -- tests/unit
+        tox -e py36,py27 --parallel all -- tests/unit
 
-      # run notebook test
+      # local mode tests
       - |
-        if has-matching-changes "src/*.py" "setup.py" "setup.cfg" "buildspec.yml"; then
-          echo "running notebook test"
-          ./tests/scripts/run-notebook-test.sh
-         else
-           echo "skipping notebook test"
-         fi
-
+        if has-matching-changes "tests/" "src/*.py" "setup.py" "setup.cfg" "buildspec.yml"; then
+          IGNORE_COVERAGE=- tox -e py36 -- tests/integ -m local_mode --durations 50
+          IGNORE_COVERAGE=- tox -e py27 -- tests/integ -m local_mode --durations 50
+        else
+          echo "skipping integration tests"
+        fi
+        
       # run integration tests
       - |
         if has-matching-changes "tests/" "src/*.py" "setup.py" "setup.cfg" "buildspec.yml"; then
           python3 -u ci-scripts/queue_build.py
-          IGNORE_COVERAGE=- tox -e py36,py27 -- tests/integ -n 24 --reruns 3
+          IGNORE_COVERAGE=- tox -e py36 -- tests/integ -m "not local_mode" -n 48  --reruns 3 --reruns-delay 5 --durations 50
+          IGNORE_COVERAGE=- tox -e py27 -- tests/integ -m "not local_mode" -n 48  --reruns 3 --reruns-delay 5 --durations 50
         else
           echo "skipping integration tests"
         fi
+        
+      # run notebook test
+      - |
+        if has-matching-changes "src/*.py" "setup.py" "setup.cfg" "buildspec.yml"; then
+          echo "running notebook test"
+          ./tests/scripts/run-notebook-test.sh
+         else
+           echo "skipping notebook test"
+         fi
   post_build:
     finally:
       - FILENAME=$(ls ci-lock/)
       - ACCOUNT=$(aws sts get-caller-identity --output text | awk '{print $1}')
       - S3_BUCKET_DIR=s3://sagemaker-us-west-2-${ACCOUNT}/ci-lock/
-      - aws s3 rm ${S3_BUCKET_DIR}${FILENAME}
+      - aws s3 rm ${S3_BUCKET_DIR}${FILENAME}

setup.py

@@ -67,7 +67,7 @@ def read_version():
     install_requires=required_packages,
     extras_require={
         "test": [
-            "tox",
+            "tox==3.13.1",
             "flake8",
             "pytest==4.4.1",
             "pytest-cov",
@@ -79,6 +79,7 @@ def read_version():
             "awslogs",
             "pandas",
             "black==19.3b0 ; python_version >= '3.6'",
+            "stopit==1.1.2",
         ]
     },
     entry_points={"console_scripts": ["sagemaker=sagemaker.cli.main:main"]},

tests/integ/test_local_mode.py

@@ -20,9 +20,10 @@
 import pytest
 import tempfile
 
+import stopit
+
 import tests.integ.lock as lock
 from tests.integ import DATA_DIR, PYTHON_VERSION
-from tests.integ.timeout import timeout
 
 from sagemaker.local import LocalSession, LocalSagemakerRuntimeClient, LocalSagemakerClient
 from sagemaker.mxnet import MXNet
@@ -86,7 +87,7 @@ def _create_model(output_path):
 @pytest.mark.local_mode
 @pytest.mark.skipif(PYTHON_VERSION != "py2", reason="TensorFlow image supports only python 2.")
 def test_tf_local_mode(sagemaker_local_session):
-    with timeout(minutes=5):
+    with stopit.ThreadingTimeout(5 * 60, swallow_exc=False):
         script_path = os.path.join(DATA_DIR, "iris", "iris-dnn-classifier.py")
 
         estimator = TensorFlow(
@@ -129,7 +130,7 @@ def test_tf_local_mode(sagemaker_local_session):
 @pytest.mark.local_mode
 @pytest.mark.skipif(PYTHON_VERSION != "py2", reason="TensorFlow image supports only python 2.")
 def test_tf_distributed_local_mode(sagemaker_local_session):
-    with timeout(minutes=5):
+    with stopit.ThreadingTimeout(5 * 60, swallow_exc=False):
         script_path = os.path.join(DATA_DIR, "iris", "iris-dnn-classifier.py")
 
         estimator = TensorFlow(
@@ -171,7 +172,7 @@ def test_tf_distributed_local_mode(sagemaker_local_session):
 @pytest.mark.local_mode
 @pytest.mark.skipif(PYTHON_VERSION != "py2", reason="TensorFlow image supports only python 2.")
 def test_tf_local_data(sagemaker_local_session):
-    with timeout(minutes=5):
+    with stopit.ThreadingTimeout(5 * 60, swallow_exc=False):
         script_path = os.path.join(DATA_DIR, "iris", "iris-dnn-classifier.py")
 
         estimator = TensorFlow(
@@ -212,7 +213,7 @@ def test_tf_local_data(sagemaker_local_session):
 @pytest.mark.local_mode
 @pytest.mark.skipif(PYTHON_VERSION != "py2", reason="TensorFlow image supports only python 2.")
 def test_tf_local_data_local_script():
-    with timeout(minutes=5):
+    with stopit.ThreadingTimeout(5 * 60, swallow_exc=False):
         script_path = os.path.join(DATA_DIR, "iris", "iris-dnn-classifier.py")
 
         estimator = TensorFlow(
@@ -391,7 +392,7 @@ def test_local_transform_mxnet(sagemaker_local_session, tmpdir, mxnet_full_versi
         path=os.path.join(data_path, "test"), key_prefix="integ-test-data/mxnet_mnist/test"
     )
 
-    with timeout(minutes=15):
+    with stopit.ThreadingTimeout(5 * 60, swallow_exc=False):
         mx.fit({"train": train_input, "test": test_input})
 
     transform_input_path = os.path.join(data_path, "transform")

tests/integ/timeout.py

@@ -12,19 +12,16 @@
 # language governing permissions and limitations under the License.
 from __future__ import absolute_import
 
-import signal
 from contextlib import contextmanager
 import logging
 from time import sleep
 
 from awslogs.core import AWSLogs
 from botocore.exceptions import ClientError
-
-LOGGER = logging.getLogger("timeout")
+import stopit
 
 
-class TimeoutError(Exception):
-    pass
+LOGGER = logging.getLogger("timeout")
 
 
 @contextmanager
@@ -43,23 +40,17 @@ def timeout(seconds=0, minutes=0, hours=0):
 
     limit = seconds + 60 * minutes + 3600 * hours
 
-    def handler(signum, frame):
-        raise TimeoutError("timed out after {} seconds".format(limit))
-
-    try:
-        signal.signal(signal.SIGALRM, handler)
-        signal.alarm(limit)
-
-        yield
-    finally:
-        signal.alarm(0)
+    with stopit.ThreadingTimeout(limit, swallow_exc=False) as t:
+        yield [t]
 
 
 @contextmanager
 def timeout_and_delete_endpoint_by_name(
     endpoint_name, sagemaker_session, seconds=0, minutes=45, hours=0
 ):
-    with timeout(seconds=seconds, minutes=minutes, hours=hours) as t:
+    limit = seconds + 60 * minutes + 3600 * hours
+
+    with stopit.ThreadingTimeout(limit, swallow_exc=False) as t:
         no_errors = False
         try:
             yield [t]
@@ -89,7 +80,9 @@ def timeout_and_delete_endpoint_by_name(
 def timeout_and_delete_model_with_transformer(
     transformer, sagemaker_session, seconds=0, minutes=0, hours=0
 ):
-    with timeout(seconds=seconds, minutes=minutes, hours=hours) as t:
+    limit = seconds + 60 * minutes + 3600 * hours
+
+    with stopit.ThreadingTimeout(limit, swallow_exc=False) as t:
         no_errors = False
         try:
             yield [t]