From 229052aa914c6228df3d406e17027524ce8019db Mon Sep 17 00:00:00 2001 From: Nadia Yakimakha <32335935+nadiaya@users.noreply.github.com> Date: Fri, 31 Aug 2018 14:41:26 -0700 Subject: [PATCH 1/3] Do not run tensorflow integ tests when running python 3 since we do not have python 3 image for tensorflow. --- tests/integ/__init__.py | 2 ++ tests/integ/test_tf.py | 5 ++++- tests/integ/test_tf_cifar.py | 3 ++- tests/integ/test_tf_keras.py | 3 ++- 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/integ/__init__.py b/tests/integ/__init__.py index 344efad12c..3c1a4a549a 100644 --- a/tests/integ/__init__.py +++ b/tests/integ/__init__.py @@ -14,10 +14,12 @@ import logging import os +import sys DATA_DIR = os.path.join(os.path.dirname(__file__), '..', 'data') TRAINING_DEFAULT_TIMEOUT_MINUTES = 20 TUNING_DEFAULT_TIMEOUT_MINUTES = 20 +PYTHON_VERSION = 'py' + str(sys.version_info.major) logging.getLogger('boto3').setLevel(logging.INFO) logging.getLogger('botocore').setLevel(logging.INFO) diff --git a/tests/integ/test_tf.py b/tests/integ/test_tf.py index 329436ccd8..cba3d63973 100644 --- a/tests/integ/test_tf.py +++ b/tests/integ/test_tf.py @@ -18,7 +18,7 @@ import pytest from sagemaker.tensorflow import TensorFlow -from tests.integ import DATA_DIR, TRAINING_DEFAULT_TIMEOUT_MINUTES +from tests.integ import DATA_DIR, TRAINING_DEFAULT_TIMEOUT_MINUTES, PYTHON_VERSION from tests.integ.timeout import timeout_and_delete_endpoint_by_name, timeout from tests.integ.vpc_utils import get_or_create_subnet_and_security_group @@ -27,6 +27,7 @@ @pytest.mark.continuous_testing +@pytest.mark.skipif(PYTHON_VERSION != 'py2', reason="TensorFlow image supports only python 2.") def test_tf(sagemaker_session, tf_full_version): with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES): script_path = os.path.join(DATA_DIR, 'iris', 'iris-dnn-classifier.py') @@ -60,6 +61,7 @@ def test_tf(sagemaker_session, tf_full_version): assert dict_result == list_result +@pytest.mark.skipif(PYTHON_VERSION != 'py2', reason="TensorFlow image supports only python 2.") def test_tf_async(sagemaker_session): with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES): script_path = os.path.join(DATA_DIR, 'iris', 'iris-dnn-classifier.py') @@ -89,6 +91,7 @@ def test_tf_async(sagemaker_session): print('predict result: {}'.format(result)) +@pytest.mark.skipif(PYTHON_VERSION != 'py2', reason="TensorFlow image supports only python 2.") def test_failed_tf_training(sagemaker_session, tf_full_version): with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES): script_path = os.path.join(DATA_DIR, 'iris', 'failure_script.py') diff --git a/tests/integ/test_tf_cifar.py b/tests/integ/test_tf_cifar.py index 05abd3b0d1..f569b782ff 100644 --- a/tests/integ/test_tf_cifar.py +++ b/tests/integ/test_tf_cifar.py @@ -19,7 +19,7 @@ import pytest from sagemaker.tensorflow import TensorFlow -from tests.integ import DATA_DIR +from tests.integ import DATA_DIR, PYTHON_VERSION from tests.integ.timeout import timeout_and_delete_endpoint_by_name, timeout PICKLE_CONTENT_TYPE = 'application/python-pickle' @@ -34,6 +34,7 @@ def __call__(self, data): @pytest.mark.continuous_testing +@pytest.mark.skipif(PYTHON_VERSION != 'py2', reason="TensorFlow image supports only python 2.") def test_cifar(sagemaker_session, tf_full_version): with timeout(minutes=45): script_path = os.path.join(DATA_DIR, 'cifar_10', 'source') diff --git a/tests/integ/test_tf_keras.py b/tests/integ/test_tf_keras.py index 37880dbe72..ae33316e71 100644 --- a/tests/integ/test_tf_keras.py +++ b/tests/integ/test_tf_keras.py @@ -18,11 +18,12 @@ import pytest from sagemaker.tensorflow import TensorFlow -from tests.integ import DATA_DIR +from tests.integ import DATA_DIR, PYTHON_VERSION from tests.integ.timeout import timeout_and_delete_endpoint_by_name, timeout @pytest.mark.continuous_testing +@pytest.mark.skipif(PYTHON_VERSION != 'py2', reason="TensorFlow image supports only python 2.") def test_keras(sagemaker_session, tf_full_version): script_path = os.path.join(DATA_DIR, 'cifar_10', 'source') dataset_path = os.path.join(DATA_DIR, 'cifar_10', 'data') From 152e89852746953ce1a7a167b79052578665039a Mon Sep 17 00:00:00 2001 From: Nadia Yakimakha <32335935+nadiaya@users.noreply.github.com> Date: Fri, 31 Aug 2018 14:41:43 -0700 Subject: [PATCH 2/3] Correct docs to reflect that upload_data would create only default bucket. (#373) --- src/sagemaker/session.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sagemaker/session.py b/src/sagemaker/session.py index e904d0531c..68a89a6ac4 100644 --- a/src/sagemaker/session.py +++ b/src/sagemaker/session.py @@ -119,8 +119,8 @@ def upload_data(self, path, bucket=None, key_prefix='data'): Args: path (str): Path (absolute or relative) of local file or directory to upload. bucket (str): Name of the S3 Bucket to upload to (default: None). If not specified, the - default bucket of the ``Session`` is used. If the bucket does not exist, the ``Session`` - creates the bucket. + default bucket of the ``Session`` is used (if default bucket does not exist, the ``Session`` + creates it). key_prefix (str): Optional S3 object key name prefix (default: 'data'). S3 uses the prefix to create a directory structure for the bucket content that it display in the S3 console. From 6a945b4818e73dd29a794177125923bcdad1389e Mon Sep 17 00:00:00 2001 From: Nadia Yakimakha <32335935+nadiaya@users.noreply.github.com> Date: Fri, 31 Aug 2018 15:21:13 -0700 Subject: [PATCH 3/3] Skip TF when running local mode and tuner integ tests in python3. --- tests/integ/test_local_mode.py | 6 +++++- tests/integ/test_tuner.py | 3 ++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/integ/test_local_mode.py b/tests/integ/test_local_mode.py index e25ca86575..7b831471df 100644 --- a/tests/integ/test_local_mode.py +++ b/tests/integ/test_local_mode.py @@ -24,7 +24,7 @@ from sagemaker.mxnet import MXNet, MXNetModel from sagemaker.tensorflow import TensorFlow from sagemaker.fw_utils import tar_and_upload_dir -from tests.integ import DATA_DIR +from tests.integ import DATA_DIR, PYTHON_VERSION from tests.integ.timeout import timeout DATA_PATH = os.path.join(DATA_DIR, 'iris', 'data') @@ -75,6 +75,7 @@ def mxnet_model(sagemaker_local_session): return model +@pytest.mark.skipif(PYTHON_VERSION != 'py2', reason="TensorFlow image supports only python 2.") def test_tf_local_mode(tf_full_version, sagemaker_local_session): local_mode_lock_fd = open(LOCK_PATH, 'w') local_mode_lock = local_mode_lock_fd.fileno() @@ -120,6 +121,7 @@ def test_tf_local_mode(tf_full_version, sagemaker_local_session): fcntl.lockf(local_mode_lock, fcntl.LOCK_UN) +@pytest.mark.skipif(PYTHON_VERSION != 'py2', reason="TensorFlow image supports only python 2.") def test_tf_distributed_local_mode(sagemaker_local_session): local_mode_lock_fd = open(LOCK_PATH, 'w') local_mode_lock = local_mode_lock_fd.fileno() @@ -164,6 +166,7 @@ def test_tf_distributed_local_mode(sagemaker_local_session): fcntl.lockf(local_mode_lock, fcntl.LOCK_UN) +@pytest.mark.skipif(PYTHON_VERSION != 'py2', reason="TensorFlow image supports only python 2.") def test_tf_local_data(sagemaker_local_session): local_mode_lock_fd = open(LOCK_PATH, 'w') local_mode_lock = local_mode_lock_fd.fileno() @@ -206,6 +209,7 @@ def test_tf_local_data(sagemaker_local_session): fcntl.lockf(local_mode_lock, fcntl.LOCK_UN) +@pytest.mark.skipif(PYTHON_VERSION != 'py2', reason="TensorFlow image supports only python 2.") def test_tf_local_data_local_script(): local_mode_lock_fd = open(LOCK_PATH, 'w') local_mode_lock = local_mode_lock_fd.fileno() diff --git a/tests/integ/test_tuner.py b/tests/integ/test_tuner.py index 3599f04412..ee1b80b677 100644 --- a/tests/integ/test_tuner.py +++ b/tests/integ/test_tuner.py @@ -32,7 +32,7 @@ from sagemaker.pytorch import PyTorch from sagemaker.tensorflow import TensorFlow from sagemaker.tuner import IntegerParameter, ContinuousParameter, CategoricalParameter, HyperparameterTuner -from tests.integ import DATA_DIR, TUNING_DEFAULT_TIMEOUT_MINUTES +from tests.integ import DATA_DIR, PYTHON_VERSION, TUNING_DEFAULT_TIMEOUT_MINUTES from tests.integ.record_set import prepare_record_set_from_local_files from tests.integ.timeout import timeout, timeout_and_delete_endpoint_by_name @@ -218,6 +218,7 @@ def test_tuning_mxnet(sagemaker_session): @pytest.mark.continuous_testing +@pytest.mark.skipif(PYTHON_VERSION != 'py2', reason="TensorFlow image supports only python 2.") def test_tuning_tf(sagemaker_session): with timeout(minutes=TUNING_DEFAULT_TIMEOUT_MINUTES): script_path = os.path.join(DATA_DIR, 'iris', 'iris-dnn-classifier.py')