diff --git a/tests/integ/test_chainer_train.py b/tests/integ/test_chainer_train.py index a4afe785c2..f932bea06a 100644 --- a/tests/integ/test_chainer_train.py +++ b/tests/integ/test_chainer_train.py @@ -66,7 +66,7 @@ def test_training_with_additional_hyperparameters(sagemaker_session, chainer_ful def test_attach_deploy(chainer_training_job, sagemaker_session): endpoint_name = 'test-chainer-attach-deploy-{}'.format(sagemaker_timestamp()) - with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session, minutes=20): + with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session): estimator = Chainer.attach(chainer_training_job, sagemaker_session=sagemaker_session) predictor = estimator.deploy(1, 'ml.m4.xlarge', endpoint_name=endpoint_name) _predict_and_assert(predictor) @@ -74,7 +74,7 @@ def test_attach_deploy(chainer_training_job, sagemaker_session): def test_deploy_model(chainer_training_job, sagemaker_session): endpoint_name = 'test-chainer-deploy-model-{}'.format(sagemaker_timestamp()) - with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session, minutes=20): + with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session): desc = sagemaker_session.sagemaker_client.describe_training_job(TrainingJobName=chainer_training_job) model_data = desc['ModelArtifacts']['S3ModelArtifacts'] script_path = os.path.join(DATA_DIR, 'chainer_mnist', 'mnist.py') @@ -93,7 +93,7 @@ def test_async_fit(sagemaker_session): print("Waiting to re-attach to the training job: %s" % training_job_name) time.sleep(20) - with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session, minutes=35): + with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session): print("Re-attaching now to: %s" % training_job_name) estimator = Chainer.attach(training_job_name=training_job_name, sagemaker_session=sagemaker_session) predictor = estimator.deploy(1, "ml.c4.xlarge", endpoint_name=endpoint_name) diff --git a/tests/integ/test_pytorch_train.py b/tests/integ/test_pytorch_train.py index cb46fc8fd0..3ef5f26bca 100644 --- a/tests/integ/test_pytorch_train.py +++ b/tests/integ/test_pytorch_train.py @@ -41,7 +41,7 @@ def fixture_training_job(sagemaker_session, pytorch_full_version): def test_sync_fit_deploy(pytorch_training_job, sagemaker_session): # TODO: add tests against local mode when it's ready to be used endpoint_name = 'test-pytorch-sync-fit-attach-deploy{}'.format(sagemaker_timestamp()) - with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session, minutes=20): + with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session): estimator = PyTorch.attach(pytorch_training_job, sagemaker_session=sagemaker_session) predictor = estimator.deploy(1, 'ml.c4.xlarge', endpoint_name=endpoint_name) data = numpy.zeros(shape=(1, 1, 28, 28), dtype=numpy.float32) diff --git a/tests/integ/test_randomcutforest.py b/tests/integ/test_randomcutforest.py index a749af985c..30624695fd 100644 --- a/tests/integ/test_randomcutforest.py +++ b/tests/integ/test_randomcutforest.py @@ -34,7 +34,7 @@ def test_randomcutforest(sagemaker_session): rcf.fit(rcf.record_set(train_input)) endpoint_name = name_from_base('randomcutforest') - with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session, minutes=20): + with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session): model = RandomCutForestModel(rcf.model_data, role='SageMakerRole', sagemaker_session=sagemaker_session) predictor = model.deploy(1, 'ml.c4.xlarge', endpoint_name=endpoint_name) diff --git a/tests/integ/test_tuner.py b/tests/integ/test_tuner.py index e20d238f8d..f8101ff6f9 100644 --- a/tests/integ/test_tuner.py +++ b/tests/integ/test_tuner.py @@ -344,7 +344,7 @@ def test_attach_tuning_pytorch(sagemaker_session): attached_tuner = HyperparameterTuner.attach(tuning_job_name, sagemaker_session=sagemaker_session) best_training_job = tuner.best_training_job() - with timeout_and_delete_endpoint_by_name(best_training_job, sagemaker_session, minutes=20): + with timeout_and_delete_endpoint_by_name(best_training_job, sagemaker_session): predictor = attached_tuner.deploy(1, 'ml.c4.xlarge') data = np.zeros(shape=(1, 1, 28, 28), dtype=np.float32) predictor.predict(data) diff --git a/tests/integ/timeout.py b/tests/integ/timeout.py index 7c5d43e357..43bd6f7691 100644 --- a/tests/integ/timeout.py +++ b/tests/integ/timeout.py @@ -60,7 +60,7 @@ def handler(signum, frame): @contextmanager -def timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session, seconds=0, minutes=35, hours=0): +def timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session, seconds=0, minutes=45, hours=0): with timeout(seconds=seconds, minutes=minutes, hours=hours) as t: no_errors = False try: