Skip to content

Commit 3b7cbd4

Browse files
jesterhazyjesterhazy
authored
fix tests with incorrect region-skipping code (#537)
1 parent 5b3ab50 commit 3b7cbd4

7 files changed

+81
-45
lines changed

‎tests/conftest.py

Lines changed: 22 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -13,21 +13,21 @@
1313
from __future__ import absolute_import
1414

1515
import json
16+
import os
1617

1718
import boto3
1819
import pytest
1920
from botocore.config import Config
2021

2122
from sagemaker import Session
22-
from sagemaker.local import LocalSession
2323
from sagemaker.chainer import Chainer
24+
from sagemaker.local import LocalSession
2425
from sagemaker.mxnet import MXNet
25-
from sagemaker.rl import RLEstimator
2626
from sagemaker.pytorch.defaults import PYTORCH_VERSION
27+
from sagemaker.rl import RLEstimator
2728
from sagemaker.sklearn.defaults import SKLEARN_VERSION
2829
from sagemaker.tensorflow.defaults import TF_VERSION
2930

30-
3131
DEFAULT_REGION = 'us-west-2'
3232

3333

@@ -38,12 +38,23 @@ def pytest_addoption(parser):
3838
parser.addoption('--chainer-full-version', action='store', default=Chainer.LATEST_VERSION)
3939
parser.addoption('--mxnet-full-version', action='store', default=MXNet.LATEST_VERSION)
4040
parser.addoption('--pytorch-full-version', action='store', default=PYTORCH_VERSION)
41-
parser.addoption('--rl-coach-full-version', action='store', default=RLEstimator.COACH_LATEST_VERSION)
42-
parser.addoption('--rl-ray-full-version', action='store', default=RLEstimator.RAY_LATEST_VERSION)
41+
parser.addoption('--rl-coach-full-version', action='store',
42+
default=RLEstimator.COACH_LATEST_VERSION)
43+
parser.addoption('--rl-ray-full-version', action='store',
44+
default=RLEstimator.RAY_LATEST_VERSION)
4345
parser.addoption('--sklearn-full-version', action='store', default=SKLEARN_VERSION)
4446
parser.addoption('--tf-full-version', action='store', default=TF_VERSION)
4547

4648

49+
def pytest_configure(config):
50+
bc = config.getoption('--boto-config')
51+
parsed = json.loads(bc) if bc else {}
52+
region = parsed.get('region_name', boto3.session.Session().region_name)
53+
54+
if region:
55+
os.environ['TEST_AWS_REGION_NAME'] = region
56+
57+
4758
@pytest.fixture(scope='session')
4859
def sagemaker_client_config(request):
4960
config = request.config.getoption('--sagemaker-client-config')
@@ -64,10 +75,13 @@ def boto_config(request):
6475

6576
@pytest.fixture(scope='session')
6677
def sagemaker_session(sagemaker_client_config, sagemaker_runtime_config, boto_config):
67-
boto_session = boto3.Session(**boto_config) if boto_config else boto3.Session(region_name=DEFAULT_REGION)
78+
boto_session = boto3.Session(**boto_config) if boto_config else boto3.Session(
79+
region_name=DEFAULT_REGION)
6880
sagemaker_client_config.setdefault('config', Config(retries=dict(max_attempts=10)))
69-
sagemaker_client = boto_session.client('sagemaker', **sagemaker_client_config) if sagemaker_client_config else None
70-
runtime_client = (boto_session.client('sagemaker-runtime', **sagemaker_runtime_config) if sagemaker_runtime_config
81+
sagemaker_client = boto_session.client('sagemaker',
82+
**sagemaker_client_config) if sagemaker_client_config else None
83+
runtime_client = (boto_session.client('sagemaker-runtime',
84+
**sagemaker_runtime_config) if sagemaker_runtime_config
7185
else None)
7286

7387
return Session(boto_session=boto_session,

‎tests/integ/__init__.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,13 @@
2323
TUNING_DEFAULT_TIMEOUT_MINUTES = 20
2424
TRANSFORM_DEFAULT_TIMEOUT_MINUTES = 20
2525
PYTHON_VERSION = 'py' + str(sys.version_info.major)
26-
REGION = boto3.session.Session().region_name
27-
2826
HOSTING_P2_UNAVAILABLE_REGIONS = ['ca-central-1', 'us-west-1', 'eu-west-2']
2927
HOSTING_P3_UNAVAILABLE_REGIONS = ['ap-southeast-1', 'ap-southeast-2', 'ap-south-1', 'ca-central-1',
3028
'us-west-1']
3129

3230
logging.getLogger('boto3').setLevel(logging.INFO)
3331
logging.getLogger('botocore').setLevel(logging.INFO)
32+
33+
34+
def test_region():
35+
return os.environ.get('TEST_AWS_REGION_NAME', boto3.session.Session().region_name)

‎tests/integ/test_chainer_train.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@
2222
from sagemaker.chainer.estimator import Chainer
2323
from sagemaker.chainer.model import ChainerModel
2424
from sagemaker.utils import sagemaker_timestamp
25-
from tests.integ import DATA_DIR, PYTHON_VERSION, TRAINING_DEFAULT_TIMEOUT_MINUTES, REGION
25+
import tests.integ
26+
from tests.integ import DATA_DIR, PYTHON_VERSION, TRAINING_DEFAULT_TIMEOUT_MINUTES
2627
from tests.integ.timeout import timeout, timeout_and_delete_endpoint_by_name
2728

2829

@@ -35,7 +36,7 @@ def test_distributed_cpu_training(sagemaker_session, chainer_full_version):
3536
_run_mnist_training_job(sagemaker_session, "ml.c4.xlarge", 2, chainer_full_version)
3637

3738

38-
@pytest.mark.skipif(REGION in ['us-west-1', 'eu-west-2', 'ca-central-1'],
39+
@pytest.mark.skipif(tests.integ.test_region() in ['us-west-1', 'eu-west-2', 'ca-central-1'],
3940
reason='No ml.p2.xlarge supported in these regions')
4041
def test_distributed_gpu_training(sagemaker_session, chainer_full_version):
4142
_run_mnist_training_job(sagemaker_session, "ml.p2.xlarge", 2, chainer_full_version)

‎tests/integ/test_pytorch_train.py

Lines changed: 20 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -11,15 +11,19 @@
1111
# ANY KIND, either express or implied. See the License for the specific
1212
# language governing permissions and limitations under the License.
1313
from __future__ import absolute_import
14-
import numpy
14+
1515
import os
1616
import time
17+
18+
import numpy
1719
import pytest
20+
import tests.integ
21+
from tests.integ import DATA_DIR, PYTHON_VERSION, TRAINING_DEFAULT_TIMEOUT_MINUTES
22+
from tests.integ.timeout import timeout, timeout_and_delete_endpoint_by_name
23+
1824
from sagemaker.pytorch.estimator import PyTorch
1925
from sagemaker.pytorch.model import PyTorchModel
2026
from sagemaker.utils import sagemaker_timestamp
21-
from tests.integ import DATA_DIR, PYTHON_VERSION, TRAINING_DEFAULT_TIMEOUT_MINUTES, REGION
22-
from tests.integ.timeout import timeout, timeout_and_delete_endpoint_by_name
2327

2428
MNIST_DIR = os.path.join(DATA_DIR, 'pytorch_mnist')
2529
MNIST_SCRIPT = os.path.join(MNIST_DIR, 'mnist.py')
@@ -57,9 +61,11 @@ def test_deploy_model(pytorch_training_job, sagemaker_session):
5761
endpoint_name = 'test-pytorch-deploy-model-{}'.format(sagemaker_timestamp())
5862

5963
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
60-
desc = sagemaker_session.sagemaker_client.describe_training_job(TrainingJobName=pytorch_training_job)
64+
desc = sagemaker_session.sagemaker_client.describe_training_job(
65+
TrainingJobName=pytorch_training_job)
6166
model_data = desc['ModelArtifacts']['S3ModelArtifacts']
62-
model = PyTorchModel(model_data, 'SageMakerRole', entry_point=MNIST_SCRIPT, sagemaker_session=sagemaker_session)
67+
model = PyTorchModel(model_data, 'SageMakerRole', entry_point=MNIST_SCRIPT,
68+
sagemaker_session=sagemaker_session)
6369
predictor = model.deploy(1, 'ml.m4.xlarge', endpoint_name=endpoint_name)
6470

6571
batch_size = 100
@@ -69,7 +75,7 @@ def test_deploy_model(pytorch_training_job, sagemaker_session):
6975
assert output.shape == (batch_size, 10)
7076

7177

72-
@pytest.mark.skipif(REGION in ['us-west-1', 'eu-west-2', 'ca-central-1'],
78+
@pytest.mark.skipif(tests.integ.test_region() in ['us-west-1', 'eu-west-2', 'ca-central-1'],
7379
reason='No ml.p2.xlarge supported in these regions')
7480
def test_async_fit_deploy(sagemaker_session, pytorch_full_version):
7581
training_job_name = ""
@@ -90,7 +96,8 @@ def test_async_fit_deploy(sagemaker_session, pytorch_full_version):
9096

9197
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
9298
print("Re-attaching now to: %s" % training_job_name)
93-
estimator = PyTorch.attach(training_job_name=training_job_name, sagemaker_session=sagemaker_session)
99+
estimator = PyTorch.attach(training_job_name=training_job_name,
100+
sagemaker_session=sagemaker_session)
94101
predictor = estimator.deploy(1, instance_type, endpoint_name=endpoint_name)
95102

96103
batch_size = 100
@@ -105,7 +112,8 @@ def test_failed_training_job(sagemaker_session, pytorch_full_version):
105112
script_path = os.path.join(MNIST_DIR, 'failure_script.py')
106113

107114
with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
108-
pytorch = _get_pytorch_estimator(sagemaker_session, pytorch_full_version, entry_point=script_path)
115+
pytorch = _get_pytorch_estimator(sagemaker_session, pytorch_full_version,
116+
entry_point=script_path)
109117

110118
with pytest.raises(ValueError) as e:
111119
pytorch.fit()
@@ -119,8 +127,10 @@ def _upload_training_data(pytorch):
119127

120128
def _get_pytorch_estimator(sagemaker_session, pytorch_full_version, instance_type='ml.c4.xlarge',
121129
entry_point=MNIST_SCRIPT):
122-
return PyTorch(entry_point=entry_point, role='SageMakerRole', framework_version=pytorch_full_version,
123-
py_version=PYTHON_VERSION, train_instance_count=1, train_instance_type=instance_type,
130+
return PyTorch(entry_point=entry_point, role='SageMakerRole',
131+
framework_version=pytorch_full_version,
132+
py_version=PYTHON_VERSION, train_instance_count=1,
133+
train_instance_type=instance_type,
124134
sagemaker_session=sagemaker_session)
125135

126136

‎tests/integ/test_tf_cifar.py

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,11 @@
1818
import numpy as np
1919
import pytest
2020

21-
from sagemaker.tensorflow import TensorFlow
22-
from tests.integ import DATA_DIR, PYTHON_VERSION, REGION
21+
import tests.integ
2322
from tests.integ.timeout import timeout_and_delete_endpoint_by_name, timeout
2423

24+
from sagemaker.tensorflow import TensorFlow
25+
2526
PICKLE_CONTENT_TYPE = 'application/python-pickle'
2627

2728

@@ -34,22 +35,26 @@ def __call__(self, data):
3435

3536

3637
@pytest.mark.continuous_testing
37-
@pytest.mark.skipif(PYTHON_VERSION != 'py2', reason="TensorFlow image supports only python 2.")
38-
@pytest.mark.skipif(REGION in ['us-west-1', 'eu-west-2', 'ca-central-1'],
38+
@pytest.mark.skipif(tests.integ.PYTHON_VERSION != 'py2',
39+
reason="TensorFlow image supports only python 2.")
40+
@pytest.mark.skipif(tests.integ.test_region() in ['us-west-1', 'eu-west-2', 'ca-central-1'],
3941
reason='No ml.p2.xlarge supported in these regions')
4042
def test_cifar(sagemaker_session, tf_full_version):
4143
with timeout(minutes=45):
42-
script_path = os.path.join(DATA_DIR, 'cifar_10', 'source')
44+
script_path = os.path.join(tests.integ.DATA_DIR, 'cifar_10', 'source')
4345

44-
dataset_path = os.path.join(DATA_DIR, 'cifar_10', 'data')
46+
dataset_path = os.path.join(tests.integ.DATA_DIR, 'cifar_10', 'data')
4547

46-
estimator = TensorFlow(entry_point='resnet_cifar_10.py', source_dir=script_path, role='SageMakerRole',
47-
framework_version=tf_full_version, training_steps=500, evaluation_steps=5,
48+
estimator = TensorFlow(entry_point='resnet_cifar_10.py', source_dir=script_path,
49+
role='SageMakerRole',
50+
framework_version=tf_full_version, training_steps=500,
51+
evaluation_steps=5,
4852
train_instance_count=2, train_instance_type='ml.p2.xlarge',
4953
sagemaker_session=sagemaker_session, train_max_run=45 * 60,
5054
base_job_name='test-cifar')
5155

52-
inputs = estimator.sagemaker_session.upload_data(path=dataset_path, key_prefix='data/cifar10')
56+
inputs = estimator.sagemaker_session.upload_data(path=dataset_path,
57+
key_prefix='data/cifar10')
5358
estimator.fit(inputs, logs=False)
5459
print('job succeeded: {}'.format(estimator.latest_training_job.name))
5560

‎tests/integ/test_tf_keras.py

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -17,18 +17,20 @@
1717
import numpy as np
1818
import pytest
1919

20-
from sagemaker.tensorflow import TensorFlow
21-
from tests.integ import DATA_DIR, PYTHON_VERSION, REGION
20+
import tests.integ
2221
from tests.integ.timeout import timeout_and_delete_endpoint_by_name, timeout
2322

23+
from sagemaker.tensorflow import TensorFlow
24+
2425

2526
@pytest.mark.continuous_testing
26-
@pytest.mark.skipif(PYTHON_VERSION != 'py2', reason="TensorFlow image supports only python 2.")
27-
@pytest.mark.skipif(REGION in ['us-west-1', 'eu-west-2', 'ca-central-1'],
27+
@pytest.mark.skipif(tests.integ.PYTHON_VERSION != 'py2',
28+
reason="TensorFlow image supports only python 2.")
29+
@pytest.mark.skipif(tests.integ.test_region() in ['us-west-1', 'eu-west-2', 'ca-central-1'],
2830
reason='No ml.p2.xlarge supported in these regions')
2931
def test_keras(sagemaker_session, tf_full_version):
30-
script_path = os.path.join(DATA_DIR, 'cifar_10', 'source')
31-
dataset_path = os.path.join(DATA_DIR, 'cifar_10', 'data')
32+
script_path = os.path.join(tests.integ.DATA_DIR, 'cifar_10', 'source')
33+
dataset_path = os.path.join(tests.integ.DATA_DIR, 'cifar_10', 'data')
3234

3335
with timeout(minutes=45):
3436
estimator = TensorFlow(entry_point='keras_cnn_cifar_10.py',
@@ -39,7 +41,8 @@ def test_keras(sagemaker_session, tf_full_version):
3941
train_instance_count=1, train_instance_type='ml.c4.xlarge',
4042
train_max_run=45 * 60)
4143

42-
inputs = estimator.sagemaker_session.upload_data(path=dataset_path, key_prefix='data/cifar10')
44+
inputs = estimator.sagemaker_session.upload_data(path=dataset_path,
45+
key_prefix='data/cifar10')
4346

4447
estimator.fit(inputs)
4548

‎tests/integ/test_tfs.py

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,19 @@
2222
from sagemaker.tensorflow.serving import Model, Predictor
2323

2424

25-
@pytest.fixture(scope='session', params=['ml.c5.xlarge', 'ml.p3.2xlarge'])
25+
@pytest.fixture(scope='session', params=[
26+
'ml.c5.xlarge',
27+
pytest.param('ml.p3.2xlarge',
28+
marks=pytest.mark.skipif(
29+
tests.integ.test_region() in tests.integ.HOSTING_P3_UNAVAILABLE_REGIONS,
30+
reason='no ml.p3 instances in this region'))])
2631
def instance_type(request):
2732
return request.param
2833

2934

3035
@pytest.fixture(scope='module')
3136
def tfs_predictor(instance_type, sagemaker_session, tf_full_version):
32-
endpoint_name = sagemaker.utils.name_from_base('sagemaker-tensorflow-serving')
37+
endpoint_name = sagemaker.utils.unique_name_from_base('sagemaker-tensorflow-serving')
3338
model_data = sagemaker_session.upload_data(
3439
path='tests/data/tensorflow-serving-test-model.tar.gz',
3540
key_prefix='tensorflow-serving/models')
@@ -42,11 +47,7 @@ def tfs_predictor(instance_type, sagemaker_session, tf_full_version):
4247

4348

4449
@pytest.mark.continuous_testing
45-
def test_predict(tfs_predictor, instance_type):
46-
if ('p3' in instance_type) and (
47-
tests.integ.REGION in tests.integ.HOSTING_P3_UNAVAILABLE_REGIONS):
48-
pytest.skip('no ml.p3 instances in this region')
49-
50+
def test_predict(tfs_predictor, instance_type): # pylint: disable=W0613
5051
input_data = {'instances': [1.0, 2.0, 5.0]}
5152
expected_result = {'predictions': [3.5, 4.0, 5.5]}
5253

0 commit comments

Comments
 (0)