Skip to content

change: build spec improvements. #888

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 14 commits into from
Jun 27, 2019
41 changes: 24 additions & 17 deletions buildspec.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,39 +8,46 @@ phases:
build:
commands:
# run linters
- tox -e flake8,pylint
# run package and docbuild checks
- tox -e twine
- TOX_PARALLEL_NO_SPINNER=1
- PY_COLORS=0
- tox -e flake8,pylint,twine,black-check --parallel all
- tox -e sphinx

# run format verification
- tox -e black-check

# run unit tests
- AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= AWS_SESSION_TOKEN=
AWS_CONTAINER_CREDENTIALS_RELATIVE_URI= AWS_DEFAULT_REGION=
tox -e py36,py27 -- tests/unit
tox -e py36,py27 --parallel all -- tests/unit

# run notebook test
# local mode tests
- |
if has-matching-changes "src/*.py" "setup.py" "setup.cfg" "buildspec.yml"; then
echo "running notebook test"
./tests/scripts/run-notebook-test.sh
else
echo "skipping notebook test"
fi

if has-matching-changes "tests/" "src/*.py" "setup.py" "setup.cfg" "buildspec.yml"; then
IGNORE_COVERAGE=- tox -e py36 -- tests/integ -m local_mode --durations 50
IGNORE_COVERAGE=- tox -e py27 -- tests/integ -m local_mode --durations 50
else
echo "skipping integration tests"
fi
# run integration tests
- |
if has-matching-changes "tests/" "src/*.py" "setup.py" "setup.cfg" "buildspec.yml"; then
python3 -u ci-scripts/queue_build.py
IGNORE_COVERAGE=- tox -e py36,py27 -- tests/integ -n 24 --reruns 3
IGNORE_COVERAGE=- tox -e py36 -- tests/integ -m "not local_mode" -n 48 --reruns 3 --reruns-delay 5 --durations 50
IGNORE_COVERAGE=- tox -e py27 -- tests/integ -m "not local_mode" -n 48 --reruns 3 --reruns-delay 5 --durations 50
else
echo "skipping integration tests"
fi

# run notebook test
- |
if has-matching-changes "src/*.py" "setup.py" "setup.cfg" "buildspec.yml"; then
echo "running notebook test"
./tests/scripts/run-notebook-test.sh
else
echo "skipping notebook test"
fi
post_build:
finally:
- FILENAME=$(ls ci-lock/)
- ACCOUNT=$(aws sts get-caller-identity --output text | awk '{print $1}')
- S3_BUCKET_DIR=s3://sagemaker-us-west-2-${ACCOUNT}/ci-lock/
- aws s3 rm ${S3_BUCKET_DIR}${FILENAME}
- aws s3 rm ${S3_BUCKET_DIR}${FILENAME}
3 changes: 2 additions & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ def read_version():
install_requires=required_packages,
extras_require={
"test": [
"tox",
"tox==3.13.1",
"flake8",
"pytest==4.4.1",
"pytest-cov",
Expand All @@ -79,6 +79,7 @@ def read_version():
"awslogs",
"pandas",
"black==19.3b0 ; python_version >= '3.6'",
"stopit==1.1.2",
]
},
entry_points={"console_scripts": ["sagemaker=sagemaker.cli.main:main"]},
Expand Down
13 changes: 7 additions & 6 deletions tests/integ/test_local_mode.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,10 @@
import pytest
import tempfile

import stopit

import tests.integ.lock as lock
from tests.integ import DATA_DIR, PYTHON_VERSION
from tests.integ.timeout import timeout

from sagemaker.local import LocalSession, LocalSagemakerRuntimeClient, LocalSagemakerClient
from sagemaker.mxnet import MXNet
Expand Down Expand Up @@ -86,7 +87,7 @@ def _create_model(output_path):
@pytest.mark.local_mode
@pytest.mark.skipif(PYTHON_VERSION != "py2", reason="TensorFlow image supports only python 2.")
def test_tf_local_mode(sagemaker_local_session):
with timeout(minutes=5):
with stopit.ThreadingTimeout(5 * 60, swallow_exc=False):
script_path = os.path.join(DATA_DIR, "iris", "iris-dnn-classifier.py")

estimator = TensorFlow(
Expand Down Expand Up @@ -129,7 +130,7 @@ def test_tf_local_mode(sagemaker_local_session):
@pytest.mark.local_mode
@pytest.mark.skipif(PYTHON_VERSION != "py2", reason="TensorFlow image supports only python 2.")
def test_tf_distributed_local_mode(sagemaker_local_session):
with timeout(minutes=5):
with stopit.ThreadingTimeout(5 * 60, swallow_exc=False):
script_path = os.path.join(DATA_DIR, "iris", "iris-dnn-classifier.py")

estimator = TensorFlow(
Expand Down Expand Up @@ -171,7 +172,7 @@ def test_tf_distributed_local_mode(sagemaker_local_session):
@pytest.mark.local_mode
@pytest.mark.skipif(PYTHON_VERSION != "py2", reason="TensorFlow image supports only python 2.")
def test_tf_local_data(sagemaker_local_session):
with timeout(minutes=5):
with stopit.ThreadingTimeout(5 * 60, swallow_exc=False):
script_path = os.path.join(DATA_DIR, "iris", "iris-dnn-classifier.py")

estimator = TensorFlow(
Expand Down Expand Up @@ -212,7 +213,7 @@ def test_tf_local_data(sagemaker_local_session):
@pytest.mark.local_mode
@pytest.mark.skipif(PYTHON_VERSION != "py2", reason="TensorFlow image supports only python 2.")
def test_tf_local_data_local_script():
with timeout(minutes=5):
with stopit.ThreadingTimeout(5 * 60, swallow_exc=False):
script_path = os.path.join(DATA_DIR, "iris", "iris-dnn-classifier.py")

estimator = TensorFlow(
Expand Down Expand Up @@ -391,7 +392,7 @@ def test_local_transform_mxnet(sagemaker_local_session, tmpdir, mxnet_full_versi
path=os.path.join(data_path, "test"), key_prefix="integ-test-data/mxnet_mnist/test"
)

with timeout(minutes=15):
with stopit.ThreadingTimeout(5 * 60, swallow_exc=False):
mx.fit({"train": train_input, "test": test_input})

transform_input_path = os.path.join(data_path, "transform")
Expand Down
27 changes: 10 additions & 17 deletions tests/integ/timeout.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,19 +12,16 @@
# language governing permissions and limitations under the License.
from __future__ import absolute_import

import signal
from contextlib import contextmanager
import logging
from time import sleep

from awslogs.core import AWSLogs
from botocore.exceptions import ClientError

LOGGER = logging.getLogger("timeout")
import stopit


class TimeoutError(Exception):
pass
LOGGER = logging.getLogger("timeout")


@contextmanager
Expand All @@ -43,23 +40,17 @@ def timeout(seconds=0, minutes=0, hours=0):

limit = seconds + 60 * minutes + 3600 * hours

def handler(signum, frame):
raise TimeoutError("timed out after {} seconds".format(limit))

try:
signal.signal(signal.SIGALRM, handler)
signal.alarm(limit)

yield
finally:
signal.alarm(0)
with stopit.ThreadingTimeout(limit, swallow_exc=False) as t:
yield [t]


@contextmanager
def timeout_and_delete_endpoint_by_name(
endpoint_name, sagemaker_session, seconds=0, minutes=45, hours=0
):
with timeout(seconds=seconds, minutes=minutes, hours=hours) as t:
limit = seconds + 60 * minutes + 3600 * hours

with stopit.ThreadingTimeout(limit, swallow_exc=False) as t:
no_errors = False
try:
yield [t]
Expand Down Expand Up @@ -89,7 +80,9 @@ def timeout_and_delete_endpoint_by_name(
def timeout_and_delete_model_with_transformer(
transformer, sagemaker_session, seconds=0, minutes=0, hours=0
):
with timeout(seconds=seconds, minutes=minutes, hours=hours) as t:
limit = seconds + 60 * minutes + 3600 * hours

with stopit.ThreadingTimeout(limit, swallow_exc=False) as t:
no_errors = False
try:
yield [t]
Expand Down