Run chainer and mxnet integ tests for python 2 and 3 images. (aws#376)

Author: nadiaya

tests/integ/test_chainer_train.py

@@ -22,7 +22,7 @@
 from sagemaker.chainer.estimator import Chainer
 from sagemaker.chainer.model import ChainerModel
 from sagemaker.utils import sagemaker_timestamp
-from tests.integ import DATA_DIR, TRAINING_DEFAULT_TIMEOUT_MINUTES
+from tests.integ import DATA_DIR, PYTHON_VERSION, TRAINING_DEFAULT_TIMEOUT_MINUTES
 from tests.integ.timeout import timeout, timeout_and_delete_endpoint_by_name
 
 
@@ -47,6 +47,7 @@ def test_training_with_additional_hyperparameters(sagemaker_session, chainer_ful
         chainer = Chainer(entry_point=script_path, role='SageMakerRole',
                           train_instance_count=1, train_instance_type="ml.c4.xlarge",
                           framework_version=chainer_full_version,
+                          py_version=PYTHON_VERSION,
                           sagemaker_session=sagemaker_session, hyperparameters={'epochs': 1},
                           use_mpi=True,
                           num_processes=2,
@@ -106,7 +107,7 @@ def test_failed_training_job(sagemaker_session, chainer_full_version):
         data_path = os.path.join(DATA_DIR, 'chainer_mnist')
 
         chainer = Chainer(entry_point=script_path, role='SageMakerRole',
-                          framework_version=chainer_full_version,
+                          framework_version=chainer_full_version, py_version=PYTHON_VERSION,
                           train_instance_count=1, train_instance_type='ml.c4.xlarge',
                           sagemaker_session=sagemaker_session)
 
@@ -127,7 +128,7 @@ def _run_mnist_training_job(sagemaker_session, instance_type, instance_count,
         data_path = os.path.join(DATA_DIR, 'chainer_mnist')
 
         chainer = Chainer(entry_point=script_path, role='SageMakerRole',
-                          framework_version=chainer_full_version,
+                          framework_version=chainer_full_version, py_version=PYTHON_VERSION,
                           train_instance_count=instance_count, train_instance_type=instance_type,
                           sagemaker_session=sagemaker_session, hyperparameters={'epochs': 1})
 

tests/integ/test_local_mode.py

@@ -313,7 +313,7 @@ def test_mxnet_local_mode(sagemaker_local_session):
     script_path = os.path.join(DATA_DIR, 'mxnet_mnist', 'mnist.py')
     data_path = os.path.join(DATA_DIR, 'mxnet_mnist')
 
-    mx = MXNet(entry_point=script_path, role='SageMakerRole',
+    mx = MXNet(entry_point=script_path, role='SageMakerRole', py_version=PYTHON_VERSION,
                train_instance_count=1, train_instance_type='local',
                sagemaker_session=sagemaker_local_session)
 

tests/integ/test_mxnet_train.py

@@ -21,7 +21,7 @@
 from sagemaker.mxnet.estimator import MXNet
 from sagemaker.mxnet.model import MXNetModel
 from sagemaker.utils import sagemaker_timestamp
-from tests.integ import DATA_DIR, TRAINING_DEFAULT_TIMEOUT_MINUTES
+from tests.integ import DATA_DIR, PYTHON_VERSION, TRAINING_DEFAULT_TIMEOUT_MINUTES
 from tests.integ.timeout import timeout, timeout_and_delete_endpoint_by_name
 
 
@@ -32,7 +32,7 @@ def mxnet_training_job(sagemaker_session, mxnet_full_version):
         data_path = os.path.join(DATA_DIR, 'mxnet_mnist')
 
         mx = MXNet(entry_point=script_path, role='SageMakerRole', framework_version=mxnet_full_version,
-                   train_instance_count=1, train_instance_type='ml.c4.xlarge',
+                   py_version=PYTHON_VERSION, train_instance_count=1, train_instance_type='ml.c4.xlarge',
                    sagemaker_session=sagemaker_session)
 
         train_input = mx.sagemaker_session.upload_data(path=os.path.join(data_path, 'train'),
@@ -62,7 +62,8 @@ def test_deploy_model(mxnet_training_job, sagemaker_session):
         desc = sagemaker_session.sagemaker_client.describe_training_job(TrainingJobName=mxnet_training_job)
         model_data = desc['ModelArtifacts']['S3ModelArtifacts']
         script_path = os.path.join(DATA_DIR, 'mxnet_mnist', 'mnist.py')
-        model = MXNetModel(model_data, 'SageMakerRole', entry_point=script_path, sagemaker_session=sagemaker_session)
+        model = MXNetModel(model_data, 'SageMakerRole', entry_point=script_path,
+                           py_version=PYTHON_VERSION, sagemaker_session=sagemaker_session)
         predictor = model.deploy(1, 'ml.m4.xlarge', endpoint_name=endpoint_name)
 
         data = numpy.zeros(shape=(1, 1, 28, 28))
@@ -76,7 +77,7 @@ def test_async_fit(sagemaker_session):
         script_path = os.path.join(DATA_DIR, 'mxnet_mnist', 'mnist.py')
         data_path = os.path.join(DATA_DIR, 'mxnet_mnist')
 
-        mx = MXNet(entry_point=script_path, role='SageMakerRole',
+        mx = MXNet(entry_point=script_path, role='SageMakerRole', py_version=PYTHON_VERSION,
                    train_instance_count=1, train_instance_type='ml.c4.xlarge',
                    sagemaker_session=sagemaker_session)
 
@@ -105,7 +106,7 @@ def test_failed_training_job(sagemaker_session, mxnet_full_version):
         data_path = os.path.join(DATA_DIR, 'mxnet_mnist')
 
         mx = MXNet(entry_point=script_path, role='SageMakerRole', framework_version=mxnet_full_version,
-                   train_instance_count=1, train_instance_type='ml.c4.xlarge',
+                   py_version=PYTHON_VERSION, train_instance_count=1, train_instance_type='ml.c4.xlarge',
                    sagemaker_session=sagemaker_session)
 
         train_input = mx.sagemaker_session.upload_data(path=os.path.join(data_path, 'train'),

tests/integ/test_pytorch_train.py

@@ -13,18 +13,16 @@
 from __future__ import absolute_import
 import numpy
 import os
-import sys
 import time
 import pytest
 from sagemaker.pytorch.estimator import PyTorch
 from sagemaker.pytorch.model import PyTorchModel
 from sagemaker.utils import sagemaker_timestamp
-from tests.integ import DATA_DIR, TRAINING_DEFAULT_TIMEOUT_MINUTES
+from tests.integ import DATA_DIR, PYTHON_VERSION, TRAINING_DEFAULT_TIMEOUT_MINUTES
 from tests.integ.timeout import timeout, timeout_and_delete_endpoint_by_name
 
 MNIST_DIR = os.path.join(DATA_DIR, 'pytorch_mnist')
 MNIST_SCRIPT = os.path.join(MNIST_DIR, 'mnist.py')
-PYTHON_VERSION = 'py' + str(sys.version_info.major)
 
 
 @pytest.fixture(scope='module', name='pytorch_training_job')

tests/integ/test_tuner.py

@@ -188,6 +188,7 @@ def test_tuning_mxnet(sagemaker_session):
 
         estimator = MXNet(entry_point=script_path,
                           role='SageMakerRole',
+                          py_version=PYTHON_VERSION,
                           train_instance_count=1,
                           train_instance_type='ml.m4.xlarge',
                           sagemaker_session=sagemaker_session,
@@ -270,6 +271,7 @@ def test_tuning_chainer(sagemaker_session):
 
         estimator = Chainer(entry_point=script_path,
                             role='SageMakerRole',
+                            py_version=PYTHON_VERSION,
                             train_instance_count=1,
                             train_instance_type='ml.c4.xlarge',
                             sagemaker_session=sagemaker_session,
@@ -319,7 +321,8 @@ def test_attach_tuning_pytorch(sagemaker_session):
     mnist_dir = os.path.join(DATA_DIR, 'pytorch_mnist')
     mnist_script = os.path.join(mnist_dir, 'mnist.py')
 
-    estimator = PyTorch(entry_point=mnist_script, role='SageMakerRole', train_instance_count=1,
+    estimator = PyTorch(entry_point=mnist_script, role='SageMakerRole',
+                        train_instance_count=1, py_version=PYTHON_VERSION,
                         train_instance_type='ml.c4.xlarge', sagemaker_session=sagemaker_session)
 
     with timeout(minutes=TUNING_DEFAULT_TIMEOUT_MINUTES):