From 4f9550da079c1c1ecd68744639cf602ac5e8b1a4 Mon Sep 17 00:00:00 2001 From: Rui Wang Date: Tue, 7 Aug 2018 14:24:54 -0700 Subject: [PATCH 1/3] Change timeout to 45 minutes for createEndpoint in integ tests --- tests/integ/test_chainer_train.py | 6 +++--- tests/integ/test_pytorch_train.py | 2 +- tests/integ/test_randomcutforest.py | 2 +- tests/integ/test_tuner.py | 2 +- tests/integ/timeout.py | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/integ/test_chainer_train.py b/tests/integ/test_chainer_train.py index a4afe785c2..f932bea06a 100644 --- a/tests/integ/test_chainer_train.py +++ b/tests/integ/test_chainer_train.py @@ -66,7 +66,7 @@ def test_training_with_additional_hyperparameters(sagemaker_session, chainer_ful def test_attach_deploy(chainer_training_job, sagemaker_session): endpoint_name = 'test-chainer-attach-deploy-{}'.format(sagemaker_timestamp()) - with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session, minutes=20): + with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session): estimator = Chainer.attach(chainer_training_job, sagemaker_session=sagemaker_session) predictor = estimator.deploy(1, 'ml.m4.xlarge', endpoint_name=endpoint_name) _predict_and_assert(predictor) @@ -74,7 +74,7 @@ def test_attach_deploy(chainer_training_job, sagemaker_session): def test_deploy_model(chainer_training_job, sagemaker_session): endpoint_name = 'test-chainer-deploy-model-{}'.format(sagemaker_timestamp()) - with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session, minutes=20): + with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session): desc = sagemaker_session.sagemaker_client.describe_training_job(TrainingJobName=chainer_training_job) model_data = desc['ModelArtifacts']['S3ModelArtifacts'] script_path = os.path.join(DATA_DIR, 'chainer_mnist', 'mnist.py') @@ -93,7 +93,7 @@ def test_async_fit(sagemaker_session): print("Waiting to re-attach to the training job: %s" % training_job_name) time.sleep(20) - with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session, minutes=35): + with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session): print("Re-attaching now to: %s" % training_job_name) estimator = Chainer.attach(training_job_name=training_job_name, sagemaker_session=sagemaker_session) predictor = estimator.deploy(1, "ml.c4.xlarge", endpoint_name=endpoint_name) diff --git a/tests/integ/test_pytorch_train.py b/tests/integ/test_pytorch_train.py index cb46fc8fd0..3ef5f26bca 100644 --- a/tests/integ/test_pytorch_train.py +++ b/tests/integ/test_pytorch_train.py @@ -41,7 +41,7 @@ def fixture_training_job(sagemaker_session, pytorch_full_version): def test_sync_fit_deploy(pytorch_training_job, sagemaker_session): # TODO: add tests against local mode when it's ready to be used endpoint_name = 'test-pytorch-sync-fit-attach-deploy{}'.format(sagemaker_timestamp()) - with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session, minutes=20): + with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session): estimator = PyTorch.attach(pytorch_training_job, sagemaker_session=sagemaker_session) predictor = estimator.deploy(1, 'ml.c4.xlarge', endpoint_name=endpoint_name) data = numpy.zeros(shape=(1, 1, 28, 28), dtype=numpy.float32) diff --git a/tests/integ/test_randomcutforest.py b/tests/integ/test_randomcutforest.py index a749af985c..30624695fd 100644 --- a/tests/integ/test_randomcutforest.py +++ b/tests/integ/test_randomcutforest.py @@ -34,7 +34,7 @@ def test_randomcutforest(sagemaker_session): rcf.fit(rcf.record_set(train_input)) endpoint_name = name_from_base('randomcutforest') - with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session, minutes=20): + with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session): model = RandomCutForestModel(rcf.model_data, role='SageMakerRole', sagemaker_session=sagemaker_session) predictor = model.deploy(1, 'ml.c4.xlarge', endpoint_name=endpoint_name) diff --git a/tests/integ/test_tuner.py b/tests/integ/test_tuner.py index e20d238f8d..f8101ff6f9 100644 --- a/tests/integ/test_tuner.py +++ b/tests/integ/test_tuner.py @@ -344,7 +344,7 @@ def test_attach_tuning_pytorch(sagemaker_session): attached_tuner = HyperparameterTuner.attach(tuning_job_name, sagemaker_session=sagemaker_session) best_training_job = tuner.best_training_job() - with timeout_and_delete_endpoint_by_name(best_training_job, sagemaker_session, minutes=20): + with timeout_and_delete_endpoint_by_name(best_training_job, sagemaker_session): predictor = attached_tuner.deploy(1, 'ml.c4.xlarge') data = np.zeros(shape=(1, 1, 28, 28), dtype=np.float32) predictor.predict(data) diff --git a/tests/integ/timeout.py b/tests/integ/timeout.py index 7c5d43e357..43bd6f7691 100644 --- a/tests/integ/timeout.py +++ b/tests/integ/timeout.py @@ -60,7 +60,7 @@ def handler(signum, frame): @contextmanager -def timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session, seconds=0, minutes=35, hours=0): +def timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session, seconds=0, minutes=45, hours=0): with timeout(seconds=seconds, minutes=minutes, hours=hours) as t: no_errors = False try: From 050e83f5d47d35dbf44b4c59af577f3805c42870 Mon Sep 17 00:00:00 2001 From: Rui Wang Date: Tue, 7 Aug 2018 14:29:59 -0700 Subject: [PATCH 2/3] Update changelog and bump version --- CHANGELOG.rst | 5 +++++ setup.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 66e1923573..325d45b351 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,11 @@ CHANGELOG ========= +1.9.1 +===== + +* Increase timeout for create_endpoint calls in integ tests + 1.9.0 ===== diff --git a/setup.py b/setup.py index 21f7db8098..092b02038e 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ def read(fname): setup(name="sagemaker", - version="1.9.0", + version="1.9.1", description="Open source library for training and deploying models on Amazon SageMaker.", packages=find_packages('src'), package_dir={'': 'src'}, From 99dd0f2d0274b26f28a1f18c893b193973b9e6d4 Mon Sep 17 00:00:00 2001 From: Rui Wang Date: Tue, 7 Aug 2018 14:54:19 -0700 Subject: [PATCH 3/3] Remove version bump --- CHANGELOG.rst | 5 ----- setup.py | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 325d45b351..66e1923573 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,11 +2,6 @@ CHANGELOG ========= -1.9.1 -===== - -* Increase timeout for create_endpoint calls in integ tests - 1.9.0 ===== diff --git a/setup.py b/setup.py index 092b02038e..21f7db8098 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ def read(fname): setup(name="sagemaker", - version="1.9.1", + version="1.9.0", description="Open source library for training and deploying models on Amazon SageMaker.", packages=find_packages('src'), package_dir={'': 'src'},