Skip to content

fix additional p2 and job/endpoint naming issues #544

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Dec 11, 2018
Merged

fix additional p2 and job/endpoint naming issues #544

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
996da4e
fix additional p2 and job/endpoint naming issues
Dec 11, 2018
2dac94b
Merge branch 'master' into je-flaky
jesterhazy Dec 11, 2018
4c7eca4
combine scarce/no p2 region list
Dec 11, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions tests/integ/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,10 @@
TUNING_DEFAULT_TIMEOUT_MINUTES = 20
TRANSFORM_DEFAULT_TIMEOUT_MINUTES = 20
PYTHON_VERSION = 'py' + str(sys.version_info.major)
HOSTING_P2_UNAVAILABLE_REGIONS = ['ca-central-1', 'us-west-1', 'eu-west-2']
HOSTING_P3_UNAVAILABLE_REGIONS = ['ap-southeast-1', 'ap-southeast-2', 'ap-south-1', 'ca-central-1',
'eu-west-2', 'us-west-1']
HOSTING_NO_P2_REGIONS = ['ca-central-1', 'eu-west-2', 'us-west-1']
HOSTING_SCARCE_P2_REGIONS = HOSTING_NO_P2_REGIONS + ['eu-central-1']
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what's the benefit of separating these?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

hmm. not much the way i've implemented it. i'll combine them.

HOSTING_NO_P3_REGIONS = ['ap-southeast-1', 'ap-southeast-2', 'ap-south-1', 'ca-central-1',
'eu-west-2', 'us-west-1']

logging.getLogger('boto3').setLevel(logging.INFO)
logging.getLogger('botocore').setLevel(logging.INFO)
Expand Down
6 changes: 3 additions & 3 deletions tests/integ/test_byo_estimator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
import sagemaker
from sagemaker.amazon.amazon_estimator import registry
from sagemaker.estimator import Estimator
from sagemaker.utils import name_from_base
from sagemaker.utils import unique_name_from_base
from tests.integ import DATA_DIR, TRAINING_DEFAULT_TIMEOUT_MINUTES
from tests.integ.timeout import timeout, timeout_and_delete_endpoint_by_name

Expand Down Expand Up @@ -81,7 +81,7 @@ def test_byo_estimator(sagemaker_session, region):
# training labels must be 'float32'
estimator.fit({'train': s3_train_data})

endpoint_name = name_from_base('byo')
endpoint_name = unique_name_from_base('byo')

with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
model = estimator.create_model()
Expand All @@ -99,7 +99,7 @@ def test_byo_estimator(sagemaker_session, region):

def test_async_byo_estimator(sagemaker_session, region):
image_name = registry(region) + "/factorization-machines:1"
endpoint_name = name_from_base('byo')
endpoint_name = unique_name_from_base('byo')
training_data_path = os.path.join(DATA_DIR, 'dummy_tensor')
training_job_name = ""

Expand Down
4 changes: 2 additions & 2 deletions tests/integ/test_chainer_train.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,8 @@ def test_distributed_cpu_training(sagemaker_session, chainer_full_version):
_run_mnist_training_job(sagemaker_session, "ml.c4.xlarge", 2, chainer_full_version)


@pytest.mark.skipif(tests.integ.test_region() in ['us-west-1', 'eu-west-2', 'ca-central-1'],
reason='No ml.p2.xlarge supported in these regions')
@pytest.mark.skipif(tests.integ.test_region() in tests.integ.HOSTING_SCARCE_P2_REGIONS,
reason='no ml.p2 instances in these regions')
def test_distributed_gpu_training(sagemaker_session, chainer_full_version):
_run_mnist_training_job(sagemaker_session, "ml.p2.xlarge", 2, chainer_full_version)

Expand Down
6 changes: 3 additions & 3 deletions tests/integ/test_factorization_machines.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
import pytest

from sagemaker import FactorizationMachines, FactorizationMachinesModel
from sagemaker.utils import name_from_base
from sagemaker.utils import unique_name_from_base
from tests.integ import DATA_DIR, TRAINING_DEFAULT_TIMEOUT_MINUTES
from tests.integ.timeout import timeout, timeout_and_delete_endpoint_by_name

Expand All @@ -45,7 +45,7 @@ def test_factorization_machines(sagemaker_session):
# training labels must be 'float32'
fm.fit(fm.record_set(train_set[0][:200], train_set[1][:200].astype('float32')))

endpoint_name = name_from_base('fm')
endpoint_name = unique_name_from_base('fm')
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
model = FactorizationMachinesModel(fm.model_data, role='SageMakerRole', sagemaker_session=sagemaker_session)
predictor = model.deploy(1, 'ml.c4.xlarge', endpoint_name=endpoint_name)
Expand All @@ -58,7 +58,7 @@ def test_factorization_machines(sagemaker_session):

def test_async_factorization_machines(sagemaker_session):
training_job_name = ""
endpoint_name = name_from_base('factorizationMachines')
endpoint_name = unique_name_from_base('factorizationMachines')

with timeout(minutes=5):
data_path = os.path.join(DATA_DIR, 'one_p_mnist', 'mnist.pkl.gz')
Expand Down
4 changes: 2 additions & 2 deletions tests/integ/test_ipinsights.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

from sagemaker import IPInsights, IPInsightsModel
from sagemaker.predictor import RealTimePredictor
from sagemaker.utils import name_from_base
from sagemaker.utils import unique_name_from_base
from tests.integ import DATA_DIR, TRAINING_DEFAULT_TIMEOUT_MINUTES
from tests.integ.record_set import prepare_record_set_from_local_files
from tests.integ.timeout import timeout, timeout_and_delete_endpoint_by_name
Expand Down Expand Up @@ -47,7 +47,7 @@ def test_ipinsights(sagemaker_session):
num_records, FEATURE_DIM, sagemaker_session)
ipinsights.fit(record_set, None)

endpoint_name = name_from_base('ipinsights')
endpoint_name = unique_name_from_base('ipinsights')
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
model = IPInsightsModel(ipinsights.model_data, role='SageMakerRole', sagemaker_session=sagemaker_session)
predictor = model.deploy(1, 'ml.c4.xlarge', endpoint_name=endpoint_name)
Expand Down
6 changes: 3 additions & 3 deletions tests/integ/test_kmeans.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
import pytest

from sagemaker import KMeans, KMeansModel
from sagemaker.utils import name_from_base
from sagemaker.utils import unique_name_from_base
from tests.integ import DATA_DIR, TRAINING_DEFAULT_TIMEOUT_MINUTES
from tests.integ.timeout import timeout, timeout_and_delete_endpoint_by_name

Expand Down Expand Up @@ -64,7 +64,7 @@ def test_kmeans(sagemaker_session):

kmeans.fit(kmeans.record_set(train_set[0][:100]))

endpoint_name = name_from_base('kmeans')
endpoint_name = unique_name_from_base('kmeans')
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
model = KMeansModel(kmeans.model_data, role='SageMakerRole', sagemaker_session=sagemaker_session)
predictor = model.deploy(1, 'ml.c4.xlarge', endpoint_name=endpoint_name)
Expand All @@ -78,7 +78,7 @@ def test_kmeans(sagemaker_session):

def test_async_kmeans(sagemaker_session):
training_job_name = ""
endpoint_name = name_from_base('kmeans')
endpoint_name = unique_name_from_base('kmeans')

with timeout(minutes=5):
data_path = os.path.join(DATA_DIR, 'one_p_mnist', 'mnist.pkl.gz')
Expand Down
6 changes: 3 additions & 3 deletions tests/integ/test_knn.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
import pytest

from sagemaker import KNN, KNNModel
from sagemaker.utils import name_from_base
from sagemaker.utils import unique_name_from_base
from tests.integ import DATA_DIR, TRAINING_DEFAULT_TIMEOUT_MINUTES
from tests.integ.timeout import timeout, timeout_and_delete_endpoint_by_name

Expand All @@ -44,7 +44,7 @@ def test_knn_regressor(sagemaker_session):
# training labels must be 'float32'
knn.fit(knn.record_set(train_set[0][:200], train_set[1][:200].astype('float32')))

endpoint_name = name_from_base('knn')
endpoint_name = unique_name_from_base('knn')
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
model = KNNModel(knn.model_data, role='SageMakerRole', sagemaker_session=sagemaker_session)
predictor = model.deploy(1, 'ml.c4.xlarge', endpoint_name=endpoint_name)
Expand All @@ -57,7 +57,7 @@ def test_knn_regressor(sagemaker_session):

def test_async_knn_classifier(sagemaker_session):
training_job_name = ""
endpoint_name = name_from_base('knn')
endpoint_name = unique_name_from_base('knn')

with timeout(minutes=5):
data_path = os.path.join(DATA_DIR, 'one_p_mnist', 'mnist.pkl.gz')
Expand Down
4 changes: 2 additions & 2 deletions tests/integ/test_lda.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

from sagemaker import LDA, LDAModel
from sagemaker.amazon.common import read_records
from sagemaker.utils import name_from_base
from sagemaker.utils import unique_name_from_base
from tests.integ import DATA_DIR, TRAINING_DEFAULT_TIMEOUT_MINUTES
from tests.integ.timeout import timeout, timeout_and_delete_endpoint_by_name
from tests.integ.record_set import prepare_record_set_from_local_files
Expand All @@ -44,7 +44,7 @@ def test_lda(sagemaker_session):
len(all_records), feature_num, sagemaker_session)
lda.fit(record_set, 100)

endpoint_name = name_from_base('lda')
endpoint_name = unique_name_from_base('lda')
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
model = LDAModel(lda.model_data, role='SageMakerRole', sagemaker_session=sagemaker_session)
predictor = model.deploy(1, 'ml.c4.xlarge', endpoint_name=endpoint_name)
Expand Down
6 changes: 3 additions & 3 deletions tests/integ/test_linear_learner.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import pytest

from sagemaker.amazon.linear_learner import LinearLearner, LinearLearnerModel
from sagemaker.utils import name_from_base, sagemaker_timestamp
from sagemaker.utils import unique_name_from_base, sagemaker_timestamp
from tests.integ import DATA_DIR, TRAINING_DEFAULT_TIMEOUT_MINUTES
from tests.integ.timeout import timeout, timeout_and_delete_endpoint_by_name

Expand Down Expand Up @@ -80,7 +80,7 @@ def test_linear_learner(sagemaker_session):
ll.early_stopping_patience = 3
ll.fit(ll.record_set(train_set[0][:200], train_set[1][:200]))

endpoint_name = name_from_base('linear-learner')
endpoint_name = unique_name_from_base('linear-learner')
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):

predictor = ll.deploy(1, 'ml.c4.xlarge', endpoint_name=endpoint_name)
Expand Down Expand Up @@ -109,7 +109,7 @@ def test_linear_learner_multiclass(sagemaker_session):
ll.epochs = 1
ll.fit(ll.record_set(train_set[0][:200], train_set[1][:200]))

endpoint_name = name_from_base('linear-learner')
endpoint_name = unique_name_from_base('linear-learner')
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):

predictor = ll.deploy(1, 'ml.c4.xlarge', endpoint_name=endpoint_name)
Expand Down
4 changes: 2 additions & 2 deletions tests/integ/test_ntm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

from sagemaker import NTM, NTMModel
from sagemaker.amazon.common import read_records
from sagemaker.utils import name_from_base
from sagemaker.utils import unique_name_from_base
from tests.integ import DATA_DIR, TRAINING_DEFAULT_TIMEOUT_MINUTES
from tests.integ.timeout import timeout, timeout_and_delete_endpoint_by_name
from tests.integ.record_set import prepare_record_set_from_local_files
Expand All @@ -44,7 +44,7 @@ def test_ntm(sagemaker_session):
len(all_records), feature_num, sagemaker_session)
ntm.fit(record_set, None)

endpoint_name = name_from_base('ntm')
endpoint_name = unique_name_from_base('ntm')
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
model = NTMModel(ntm.model_data, role='SageMakerRole', sagemaker_session=sagemaker_session)
predictor = model.deploy(1, 'ml.c4.xlarge', endpoint_name=endpoint_name)
Expand Down
4 changes: 2 additions & 2 deletions tests/integ/test_object2vec.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

from sagemaker.predictor import RealTimePredictor
from sagemaker import Object2Vec, Object2VecModel
from sagemaker.utils import name_from_base
from sagemaker.utils import unique_name_from_base
from tests.integ import DATA_DIR, TRAINING_DEFAULT_TIMEOUT_MINUTES
from tests.integ.timeout import timeout, timeout_and_delete_endpoint_by_name
from tests.integ.record_set import prepare_record_set_from_local_files
Expand Down Expand Up @@ -52,7 +52,7 @@ def test_object2vec(sagemaker_session):

object2vec.fit(record_set, None)

endpoint_name = name_from_base('object2vec')
endpoint_name = unique_name_from_base('object2vec')

with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
model = Object2VecModel(object2vec.model_data, role='SageMakerRole', sagemaker_session=sagemaker_session)
Expand Down
6 changes: 3 additions & 3 deletions tests/integ/test_pca.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
import pytest

import sagemaker.amazon.pca
from sagemaker.utils import name_from_base
from sagemaker.utils import unique_name_from_base
from tests.integ import DATA_DIR, TRAINING_DEFAULT_TIMEOUT_MINUTES
from tests.integ.timeout import timeout, timeout_and_delete_endpoint_by_name

Expand All @@ -45,7 +45,7 @@ def test_pca(sagemaker_session):
pca.extra_components = 5
pca.fit(pca.record_set(train_set[0][:100]))

endpoint_name = name_from_base('pca')
endpoint_name = unique_name_from_base('pca')
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
pca_model = sagemaker.amazon.pca.PCAModel(model_data=pca.model_data, role='SageMakerRole',
sagemaker_session=sagemaker_session)
Expand All @@ -61,7 +61,7 @@ def test_pca(sagemaker_session):

def test_async_pca(sagemaker_session):
training_job_name = ""
endpoint_name = name_from_base('pca')
endpoint_name = unique_name_from_base('pca')

with timeout(minutes=5):
data_path = os.path.join(DATA_DIR, 'one_p_mnist', 'mnist.pkl.gz')
Expand Down
4 changes: 2 additions & 2 deletions tests/integ/test_pytorch_train.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,8 +75,8 @@ def test_deploy_model(pytorch_training_job, sagemaker_session):
assert output.shape == (batch_size, 10)


@pytest.mark.skipif(tests.integ.test_region() in ['us-west-1', 'eu-west-2', 'ca-central-1'],
reason='No ml.p2.xlarge supported in these regions')
@pytest.mark.skipif(tests.integ.test_region() in tests.integ.HOSTING_SCARCE_P2_REGIONS,
reason='no ml.p2 instances in these regions')
def test_async_fit_deploy(sagemaker_session, pytorch_full_version):
training_job_name = ""
# TODO: add tests against local mode when it's ready to be used
Expand Down
4 changes: 2 additions & 2 deletions tests/integ/test_randomcutforest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
import pytest

from sagemaker import RandomCutForest, RandomCutForestModel
from sagemaker.utils import name_from_base
from sagemaker.utils import unique_name_from_base
from tests.integ import TRAINING_DEFAULT_TIMEOUT_MINUTES
from tests.integ.timeout import timeout, timeout_and_delete_endpoint_by_name

Expand All @@ -34,7 +34,7 @@ def test_randomcutforest(sagemaker_session):

rcf.fit(rcf.record_set(train_input))

endpoint_name = name_from_base('randomcutforest')
endpoint_name = unique_name_from_base('randomcutforest')
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
model = RandomCutForestModel(rcf.model_data, role='SageMakerRole', sagemaker_session=sagemaker_session)
predictor = model.deploy(1, 'ml.c4.xlarge', endpoint_name=endpoint_name)
Expand Down
4 changes: 2 additions & 2 deletions tests/integ/test_tf_cifar.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,8 @@ def __call__(self, data):
@pytest.mark.continuous_testing
@pytest.mark.skipif(tests.integ.PYTHON_VERSION != 'py2',
reason="TensorFlow image supports only python 2.")
@pytest.mark.skipif(tests.integ.test_region() in ['us-west-1', 'eu-west-2', 'ca-central-1'],
reason='No ml.p2.xlarge supported in these regions')
@pytest.mark.skipif(tests.integ.test_region() in tests.integ.HOSTING_SCARCE_P2_REGIONS,
reason='no ml.p2 instances in these regions')
def test_cifar(sagemaker_session, tf_full_version):
with timeout(minutes=45):
script_path = os.path.join(tests.integ.DATA_DIR, 'cifar_10', 'source')
Expand Down
4 changes: 2 additions & 2 deletions tests/integ/test_tf_keras.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@
@pytest.mark.continuous_testing
@pytest.mark.skipif(tests.integ.PYTHON_VERSION != 'py2',
reason="TensorFlow image supports only python 2.")
@pytest.mark.skipif(tests.integ.test_region() in ['us-west-1', 'eu-west-2', 'ca-central-1'],
reason='No ml.p2.xlarge supported in these regions')
@pytest.mark.skipif(tests.integ.test_region() in tests.integ.HOSTING_SCARCE_P2_REGIONS,
reason='no ml.p2 instances in these regions')
def test_keras(sagemaker_session, tf_full_version):
script_path = os.path.join(tests.integ.DATA_DIR, 'cifar_10', 'source')
dataset_path = os.path.join(tests.integ.DATA_DIR, 'cifar_10', 'data')
Expand Down
2 changes: 1 addition & 1 deletion tests/integ/test_tfs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
'ml.c5.xlarge',
pytest.param('ml.p3.2xlarge',
marks=pytest.mark.skipif(
tests.integ.test_region() in tests.integ.HOSTING_P3_UNAVAILABLE_REGIONS,
tests.integ.test_region() in tests.integ.HOSTING_NO_P3_REGIONS,
reason='no ml.p3 instances in this region'))])
def instance_type(request):
return request.param
Expand Down