Classify canary_quick in integ tests (aws#677)

Author: yangaws

tests/integ/test_byo_estimator.py

@@ -40,7 +40,7 @@ def fm_serializer(data):
     return json.dumps(js)
 
 
-@pytest.mark.continuous_testing
+@pytest.mark.canary_quick
 def test_byo_estimator(sagemaker_session, region):
     """Use Factorization Machines algorithm as an example here.
 

tests/integ/test_chainer_train.py

@@ -66,7 +66,7 @@ def test_training_with_additional_hyperparameters(sagemaker_session, chainer_ful
         return chainer.latest_training_job.name
 
 
-@pytest.mark.continuous_testing
+@pytest.mark.canary_quick
 @pytest.mark.regional_testing
 def test_attach_deploy(chainer_training_job, sagemaker_session):
     endpoint_name = 'test-chainer-attach-deploy-{}'.format(sagemaker_timestamp())

tests/integ/test_factorization_machines.py

@@ -18,15 +18,12 @@
 import sys
 import time
 
-import pytest
-
 from sagemaker import FactorizationMachines, FactorizationMachinesModel
 from sagemaker.utils import unique_name_from_base
 from tests.integ import DATA_DIR, TRAINING_DEFAULT_TIMEOUT_MINUTES
 from tests.integ.timeout import timeout, timeout_and_delete_endpoint_by_name
 
 
-@pytest.mark.continuous_testing
 def test_factorization_machines(sagemaker_session):
     with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
         data_path = os.path.join(DATA_DIR, 'one_p_mnist', 'mnist.pkl.gz')

tests/integ/test_horovod.py

@@ -27,6 +27,7 @@
 horovod_dir = os.path.join(os.path.dirname(__file__), '..', 'data', 'horovod')
 
 
+@pytest.mark.canary_quick
 @pytest.mark.parametrize('instance_type', ['ml.c5.xlarge', 'ml.p3.2xlarge'])
 def test_horovod(sagemaker_session, instance_type, tmpdir):
 

tests/integ/test_inference_pipeline.py

@@ -28,7 +28,7 @@
 from sagemaker.utils import sagemaker_timestamp
 
 
-@pytest.mark.continuous_testing
+@pytest.mark.canary_quick
 @pytest.mark.regional_testing
 def test_inference_pipeline_model_deploy(sagemaker_session):
     sparkml_data_path = os.path.join(DATA_DIR, 'sparkml_model')

tests/integ/test_ipinsights.py

@@ -13,7 +13,6 @@
 from __future__ import absolute_import
 
 import os
-import pytest
 
 from sagemaker import IPInsights, IPInsightsModel
 from sagemaker.predictor import RealTimePredictor
@@ -25,7 +24,6 @@
 FEATURE_DIM = None
 
 
-@pytest.mark.continuous_testing
 def test_ipinsights(sagemaker_session):
     with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
         data_path = os.path.join(DATA_DIR, 'ipinsights')

tests/integ/test_kmeans.py

@@ -26,7 +26,6 @@
 from tests.integ.timeout import timeout, timeout_and_delete_endpoint_by_name
 
 
-@pytest.mark.continuous_testing
 def test_kmeans(sagemaker_session):
     with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
         data_path = os.path.join(DATA_DIR, 'one_p_mnist', 'mnist.pkl.gz')

tests/integ/test_knn.py

@@ -18,15 +18,12 @@
 import sys
 import time
 
-import pytest
-
 from sagemaker import KNN, KNNModel
 from sagemaker.utils import unique_name_from_base
 from tests.integ import DATA_DIR, TRAINING_DEFAULT_TIMEOUT_MINUTES
 from tests.integ.timeout import timeout, timeout_and_delete_endpoint_by_name
 
 
-@pytest.mark.continuous_testing
 def test_knn_regressor(sagemaker_session):
     with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
         data_path = os.path.join(DATA_DIR, 'one_p_mnist', 'mnist.pkl.gz')

tests/integ/test_lda.py

@@ -15,7 +15,6 @@
 import os
 
 import numpy as np
-import pytest
 
 from sagemaker import LDA, LDAModel
 from sagemaker.amazon.common import read_records
@@ -25,7 +24,6 @@
 from tests.integ.record_set import prepare_record_set_from_local_files
 
 
-@pytest.mark.continuous_testing
 def test_lda(sagemaker_session):
     with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
         data_path = os.path.join(DATA_DIR, 'lda')

tests/integ/test_linear_learner.py

@@ -27,7 +27,7 @@
 from tests.integ.timeout import timeout, timeout_and_delete_endpoint_by_name
 
 
-@pytest.mark.continuous_testing
+@pytest.mark.canary_quick
 def test_linear_learner(sagemaker_session):
     with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
         data_path = os.path.join(DATA_DIR, 'one_p_mnist', 'mnist.pkl.gz')

tests/integ/test_marketplace.py

@@ -17,6 +17,7 @@
 import time
 
 import pandas
+import pytest
 
 import sagemaker
 from sagemaker import AlgorithmEstimator, ModelPackage
@@ -42,6 +43,7 @@
                     '154230595-8f00905c1f927a512b73ea29dd09ae30'
 
 
+@pytest.mark.canary_quick
 def test_marketplace_estimator(sagemaker_session):
     with timeout(minutes=15):
         data_path = os.path.join(DATA_DIR, 'marketplace', 'training')
@@ -112,6 +114,7 @@ def test_marketplace_attach(sagemaker_session):
         print(predictor.predict(test_x.values).decode('utf-8'))
 
 
+@pytest.mark.canary_quick
 def test_marketplace_model(sagemaker_session):
 
     def predict_wrapper(endpoint, session):

tests/integ/test_mxnet_train.py

@@ -45,7 +45,7 @@ def mxnet_training_job(sagemaker_session, mxnet_full_version):
         return mx.latest_training_job.name
 
 
-@pytest.mark.continuous_testing
+@pytest.mark.canary_quick
 @pytest.mark.regional_testing
 def test_attach_deploy(mxnet_training_job, sagemaker_session):
     endpoint_name = 'test-mxnet-attach-deploy-{}'.format(sagemaker_timestamp())
@@ -122,7 +122,7 @@ def test_deploy_model_with_update_non_existing_endpoint(mxnet_training_job, sage
             model.deploy(1, 'ml.m4.xlarge', update_endpoint=True, endpoint_name='non-existing-endpoint')
 
 
-@pytest.mark.continuous_testing
+@pytest.mark.canary_quick
 @pytest.mark.regional_testing
 @pytest.mark.skipif(tests.integ.test_region() not in tests.integ.EI_SUPPORTED_REGIONS,
                     reason="EI isn't supported in that specific region.")

tests/integ/test_neo_mxnet.py

@@ -44,7 +44,7 @@ def mxnet_training_job(sagemaker_session, mxnet_full_version):
         return mx.latest_training_job.name
 
 
-@pytest.mark.continuous_testing
+@pytest.mark.canary_quick
 @pytest.mark.regional_testing
 @pytest.mark.skip(reason="This should be enabled along with the Boto SDK release for Neo API changes")
 def test_attach_deploy(mxnet_training_job, sagemaker_session):

tests/integ/test_ntm.py

@@ -25,7 +25,7 @@
 from tests.integ.record_set import prepare_record_set_from_local_files
 
 
-@pytest.mark.continuous_testing
+@pytest.mark.canary_quick
 def test_ntm(sagemaker_session):
     with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
         data_path = os.path.join(DATA_DIR, 'ntm')

tests/integ/test_object2vec.py

@@ -13,7 +13,6 @@
 from __future__ import absolute_import
 
 import os
-import pytest
 
 from sagemaker.predictor import RealTimePredictor
 from sagemaker import Object2Vec, Object2VecModel
@@ -26,7 +25,6 @@
 FEATURE_NUM = None
 
 
-@pytest.mark.continuous_testing
 def test_object2vec(sagemaker_session):
     with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
         data_path = os.path.join(DATA_DIR, 'object2vec')

tests/integ/test_pca.py

@@ -18,15 +18,12 @@
 import sys
 import time
 
-import pytest
-
 import sagemaker.amazon.pca
 from sagemaker.utils import unique_name_from_base
 from tests.integ import DATA_DIR, TRAINING_DEFAULT_TIMEOUT_MINUTES
 from tests.integ.timeout import timeout, timeout_and_delete_endpoint_by_name
 
 
-@pytest.mark.continuous_testing
 def test_pca(sagemaker_session):
     with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
         data_path = os.path.join(DATA_DIR, 'one_p_mnist', 'mnist.pkl.gz')

tests/integ/test_pytorch_train.py

@@ -39,7 +39,7 @@ def fixture_training_job(sagemaker_session, pytorch_full_version):
         return pytorch.latest_training_job.name
 
 
-@pytest.mark.continuous_testing
+@pytest.mark.canary_quick
 @pytest.mark.regional_testing
 def test_sync_fit_deploy(pytorch_training_job, sagemaker_session):
     # TODO: add tests against local mode when it's ready to be used

tests/integ/test_randomcutforest.py

@@ -13,15 +13,13 @@
 from __future__ import absolute_import
 
 import numpy as np
-import pytest
 
 from sagemaker import RandomCutForest, RandomCutForestModel
 from sagemaker.utils import unique_name_from_base
 from tests.integ import TRAINING_DEFAULT_TIMEOUT_MINUTES
 from tests.integ.timeout import timeout, timeout_and_delete_endpoint_by_name
 
 
-@pytest.mark.continuous_testing
 def test_randomcutforest(sagemaker_session):
     with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
         # Generate a thousand 14-dimensional datapoints.

tests/integ/test_rl.py

@@ -25,7 +25,7 @@
 CPU_INSTANCE = 'ml.m4.xlarge'
 
 
-@pytest.mark.continuous_testing
+@pytest.mark.canary_quick
 @pytest.mark.skipif(PYTHON_VERSION != 'py3', reason="RL images supports only Python 3.")
 def test_coach_mxnet(sagemaker_session, rl_coach_full_version):
     estimator = _test_coach(sagemaker_session, RLFramework.MXNET, rl_coach_full_version)
@@ -89,7 +89,7 @@ def _test_coach(sagemaker_session, rl_framework, rl_coach_version):
                        })
 
 
-@pytest.mark.continuous_testing
+@pytest.mark.canary_quick
 @pytest.mark.skipif(PYTHON_VERSION != 'py3', reason="RL images supports only Python 3.")
 def test_ray_tf(sagemaker_session, rl_ray_full_version):
     source_dir = os.path.join(DATA_DIR, 'ray_cartpole')

tests/integ/test_sklearn_train.py

@@ -54,7 +54,7 @@ def test_training_with_additional_hyperparameters(sagemaker_session, sklearn_ful
         return sklearn.latest_training_job.name
 
 
-@pytest.mark.continuous_testing
+@pytest.mark.canary_quick
 @pytest.mark.regional_testing
 @pytest.mark.skipif(PYTHON_VERSION != 'py3', reason="Scikit-learn image supports only python 3.")
 def test_attach_deploy(sklearn_training_job, sagemaker_session):

tests/integ/test_sparkml_serving.py

@@ -23,7 +23,7 @@
 from tests.integ.timeout import timeout_and_delete_endpoint_by_name
 
 
-@pytest.mark.continuous_testing
+@pytest.mark.canary_quick
 @pytest.mark.regional_testing
 def test_sparkml_model_deploy(sagemaker_session):
     # Uploads an MLeap serialized MLeap model to S3 and use that to deploy a SparkML model to perform inference

tests/integ/test_tf.py

@@ -52,7 +52,7 @@ def tf_training_job(sagemaker_session, tf_full_version):
         return estimator.latest_training_job.name
 
 
-@pytest.mark.continuous_testing
+@pytest.mark.canary_quick
 @pytest.mark.regional_testing
 @pytest.mark.skipif(PYTHON_VERSION != 'py2', reason="TensorFlow image supports only python 2.")
 def test_deploy_model(sagemaker_session, tf_training_job):
@@ -78,7 +78,7 @@ def test_deploy_model(sagemaker_session, tf_training_job):
         assert dict_result == list_result
 
 
-@pytest.mark.continuous_testing
+@pytest.mark.canary_quick
 @pytest.mark.regional_testing
 @pytest.mark.skipif(tests.integ.test_region() not in tests.integ.EI_SUPPORTED_REGIONS,
                     reason="EI isn't supported in that specific region.")

tests/integ/test_tf_cifar.py

@@ -34,7 +34,7 @@ def __call__(self, data):
         return pickle.dumps(data, protocol=2)
 
 
-@pytest.mark.continuous_testing
+@pytest.mark.canary_quick
 @pytest.mark.skipif(tests.integ.PYTHON_VERSION != 'py2',
                     reason="TensorFlow image supports only python 2.")
 @pytest.mark.skipif(tests.integ.test_region() in tests.integ.HOSTING_NO_P2_REGIONS,

tests/integ/test_tf_keras.py

@@ -23,7 +23,7 @@
 from sagemaker.tensorflow import TensorFlow
 
 
-@pytest.mark.continuous_testing
+@pytest.mark.canary_quick
 @pytest.mark.skipif(tests.integ.PYTHON_VERSION != 'py2',
                     reason="TensorFlow image supports only python 2.")
 @pytest.mark.skipif(tests.integ.test_region() in tests.integ.HOSTING_NO_P2_REGIONS,

tests/integ/test_tf_script_mode.py

@@ -52,6 +52,7 @@ def test_mnist(sagemaker_session, instance_type):
                            ['graph.pbtxt', 'model.ckpt-0.index', 'model.ckpt-0.meta', 'saved_model.pb'])
 
 
+@pytest.mark.canary_quick
 @pytest.mark.skipif(integ.PYTHON_VERSION != 'py3', reason="Script Mode tests are only configured to run with Python 3")
 def test_mnist_distributed(sagemaker_session, instance_type):
     estimator = TensorFlow(entry_point=SCRIPT,

tests/integ/test_tfs.py

@@ -46,7 +46,7 @@ def tfs_predictor(instance_type, sagemaker_session, tf_full_version):
         yield predictor
 
 
-@pytest.mark.continuous_testing
+@pytest.mark.canary_quick
 def test_predict(tfs_predictor, instance_type):  # pylint: disable=W0613
     input_data = {'instances': [1.0, 2.0, 5.0]}
     expected_result = {'predictions': [3.5, 4.0, 5.5]}

tests/integ/test_transformer.py

@@ -28,7 +28,7 @@
 from tests.integ.vpc_test_utils import get_or_create_vpc_resources
 
 
-@pytest.mark.continuous_testing
+@pytest.mark.canary_quick
 def test_transform_mxnet(sagemaker_session, mxnet_full_version):
     data_path = os.path.join(DATA_DIR, 'mxnet_mnist')
     script_path = os.path.join(data_path, 'mnist.py')
@@ -65,7 +65,7 @@ def test_transform_mxnet(sagemaker_session, mxnet_full_version):
     assert kms_key_arn == job_desc['TransformResources']['VolumeKmsKeyId']
 
 
-@pytest.mark.continuous_testing
+@pytest.mark.canary_quick
 def test_attach_transform_kmeans(sagemaker_session):
     data_path = os.path.join(DATA_DIR, 'one_p_mnist')
     pickle_args = {} if sys.version_info.major == 2 else {'encoding': 'latin1'}