Skip to content

Add new unit and integration tests #155

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Oct 16, 2023
Merged

Add new unit and integration tests #155

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d0b94e3
Add new unit and integration tests
sachanub Oct 13, 2023
824fd61
Merge branch 'master' into add_new_tests
sachanub Oct 14, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions buildspec.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,10 +55,10 @@ phases:
# build DLC GPU image because the base DLC image is too big and takes too long to build as part of the test
- python3 setup.py sdist
- $(aws ecr get-login --registry-ids $DLC_ACCOUNT --no-include-email --region $AWS_DEFAULT_REGION)
- create-key-pair
- |
for FRAMEWORK_VERSION in $FRAMEWORK_VERSIONS;
do
create-key-pair;
launch-ec2-instance --instance-type $instance_type --ami-name ami-03e3ef8c92fdb39ad;
DLC_GPU_TAG="$FRAMEWORK_VERSION-dlc-gpu-$BUILD_ID";
build_dir="test/container/$FRAMEWORK_VERSION";
Expand All @@ -71,8 +71,10 @@ phases:
execute-command-if-has-matching-changes "$test_cmd" "test/" "src/*.py" "setup.py" "setup.cfg";
docker system prune --all --force;
cleanup-gpu-instances;
cleanup-key-pairs;
rm ~/.instance_id;
rm ~/.ip_address;
done
- cleanup-key-pairs;

# run CPU sagemaker integration tests
- |
Expand Down
92 changes: 35 additions & 57 deletions test/integration/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,77 +13,55 @@
from __future__ import absolute_import

import os
import json

from utils import file_utils

resources_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'resources'))
model_gpu_context_dir = os.path.join(resources_path, 'model_gpu_context')
mnist_path = os.path.join(resources_path, 'mnist')
resnet18_path = os.path.join(resources_path, 'resnet18')
data_dir = os.path.join(mnist_path, 'data')
training_dir = os.path.join(data_dir, 'training')
cpu_sub_dir = 'model_cpu'
gpu_sub_dir = 'model_gpu'
eia_sub_dir = 'model_eia'
code_sub_dir = 'code'
default_sub_dir = 'default_model'
default_sub_eia_dir = 'default_model_eia'
default_sub_traced_resnet_dir = 'default_traced_resnet'

model_cpu_dir = os.path.join(mnist_path, cpu_sub_dir)
mnist_cpu_script = os.path.join(model_cpu_dir, code_sub_dir, 'mnist.py')
model_cpu_tar = file_utils.make_tarfile(mnist_cpu_script,
os.path.join(model_cpu_dir, "torch_model.pth"),
model_cpu_dir,
script_path="code")
all_models_info_json = os.path.abspath(os.path.join(os.path.dirname(__file__), 'all_models_info.json'))

model_cpu_1d_dir = os.path.join(model_cpu_dir, '1d')
mnist_1d_script = os.path.join(model_cpu_1d_dir, code_sub_dir, 'mnist_1d.py')
model_cpu_1d_tar = file_utils.make_tarfile(mnist_1d_script,
os.path.join(model_cpu_1d_dir, "torch_model.pth"),
model_cpu_1d_dir,
script_path="code")
with open(all_models_info_json, 'r') as json_file:
all_models_info = json.load(json_file)

model_gpu_dir = os.path.join(mnist_path, gpu_sub_dir)
mnist_gpu_script = os.path.join(model_gpu_dir, code_sub_dir, 'mnist.py')
model_gpu_tar = file_utils.make_tarfile(mnist_gpu_script,
os.path.join(model_gpu_dir, "torch_model.pth"),
model_gpu_dir,
script_path="code")
for model_name in all_models_info.keys():
model_info = all_models_info[model_name]

model_eia_dir = os.path.join(mnist_path, eia_sub_dir)
mnist_eia_script = os.path.join(model_eia_dir, 'mnist.py')
model_eia_tar = file_utils.make_tarfile(mnist_eia_script,
os.path.join(model_eia_dir, "torch_model.pth"),
model_eia_dir)
dir_path = model_info['dir_path']
model_dir = os.path.join(resources_path, *dir_path)
setattr(__import__('integration'), model_name + '_dir', model_dir)

call_model_fn_once_script = os.path.join(model_cpu_dir, code_sub_dir, 'call_model_fn_once.py')
call_model_fn_once_tar = file_utils.make_tarfile(call_model_fn_once_script,
os.path.join(model_cpu_dir, "torch_model.pth"),
model_cpu_dir,
"model_call_model_fn_once.tar.gz",
script_path="code")
script_name = model_info['script_name']
model = model_info['model']
if 'filename' in model_info:
filename = model_info['filename']
else:
filename = 'model.tar.gz'
if 'code_path' in model_info:
code_path = model_info['code_path']
script_path = model_info['code_path']
else:
code_path = ''
script_path = None
if 'requirements' in model_info:
requirements = os.path.join(model_dir, code_path, 'requirements.txt')
else:
requirements = None

default_model_dir = os.path.join(resnet18_path, default_sub_dir)
default_model_script = os.path.join(default_model_dir, code_sub_dir, "resnet18.py")
default_model_tar = file_utils.make_tarfile(
default_model_script, os.path.join(default_model_dir, "model.pt"), default_model_dir, script_path="code"
)
model_script = os.path.join(model_dir, code_path, script_name)
model_tar = file_utils.make_tarfile(model_script,
os.path.join(model_dir, model),
model_dir,
filename=filename,
script_path=script_path,
requirements=requirements)

default_traced_resnet_dir = os.path.join(resnet18_path, default_sub_traced_resnet_dir)
default_traced_resnet_script = os.path.join(default_traced_resnet_dir, code_sub_dir, "resnet18.py")
default_model_traced_resnet18_tar = file_utils.make_tarfile(
default_traced_resnet_script,
os.path.join(default_traced_resnet_dir, "traced_resnet18.pt"),
default_traced_resnet_dir,
filename="traced_resnet18.tar.gz",
script_path="code",
)

default_model_eia_dir = os.path.join(mnist_path, default_sub_eia_dir)
default_model_eia_script = os.path.join(default_model_eia_dir, code_sub_dir, "mnist.py")
default_model_eia_tar = file_utils.make_tarfile(
default_model_eia_script, os.path.join(default_model_eia_dir, "model.pt"), default_model_eia_dir
)
setattr(__import__('integration'), model_name + '_script', model_script)
setattr(__import__('integration'), model_name + '_tar', model_tar)

ROLE = 'dummy/unused-role'
DEFAULT_TIMEOUT = 20
Expand Down
83 changes: 83 additions & 0 deletions test/integration/all_models_info.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
{
"mnist_cpu":
{
"script_name": "mnist.py",
"model": "torch_model.pth",
"code_path": "code",
"requirements": "requirements.txt",
"dir_path": ["mnist", "model_cpu"]
},
"mnist_cpu_1d":
{
"script_name": "mnist_1d.py",
"model": "torch_model.pth",
"code_path": "code",
"dir_path": ["mnist", "model_cpu", "1d"]
},
"mnist_gpu":
{
"script_name": "mnist.py",
"model": "torch_model.pth",
"code_path": "code",
"dir_path": ["mnist", "model_gpu"]
},
"mnist_eia":
{
"script_name": "mnist.py",
"model": "torch_model.pth",
"dir_path": ["mnist", "model_eia"]
},
"mnist_inductor":
{
"script_name": "mnist.py",
"model": "torch_model.pth",
"code_path": "code",
"dir_path": ["mnist", "model_inductor"]
},
"call_model_fn_once":
{
"script_name": "call_model_fn_once.py",
"model": "torch_model.pth",
"filename": "model_call_model_fn_once.tar.gz",
"code_path": "code",
"dir_path": ["mnist", "model_cpu"]
},
"default_model":
{
"script_name": "resnet18.py",
"model": "model.pt",
"code_path": "code",
"dir_path": ["resnet18", "default_model"]
},
"default_model_traced_resnet":
{
"script_name": "resnet18.py",
"model": "traced_resnet18.pt",
"filename": "traced_resnet18.tar.gz",
"code_path": "code",
"dir_path": ["resnet18", "default_traced_resnet"]
},
"default_model_eia":
{
"script_name": "mnist.py",
"model": "model.pt",
"code_path": "code",
"dir_path": ["mnist", "default_model_eia"]
},
"resnet18":
{
"script_name": "inference.py",
"model": "model.pt",
"filename": "resnet18.tar.gz",
"code_path": "code",
"dir_path": ["mme", "resnet18"]
},
"traced_resnet18":
{
"script_name": "inference.py",
"model": "traced_resnet18.pt",
"filename": "traced_resnet18.tar.gz",
"code_path": "code",
"dir_path": ["mme", "traced_resnet18"]
}
}
12 changes: 6 additions & 6 deletions test/integration/local/test_serving.py → test/integration/local/test_mnist_serving.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@
from sagemaker_inference import content_types
from torchvision import datasets, transforms

from integration import training_dir, mnist_1d_script, model_cpu_tar, mnist_cpu_script, \
model_gpu_tar, mnist_gpu_script, model_cpu_1d_tar, call_model_fn_once_script, ROLE, \
from integration import training_dir, mnist_cpu_1d_script, mnist_cpu_tar, mnist_cpu_script, \
mnist_gpu_tar, mnist_gpu_script, mnist_cpu_1d_tar, call_model_fn_once_script, ROLE, \
call_model_fn_once_tar
from utils import local_mode_utils

Expand All @@ -50,7 +50,7 @@ def fixture_test_loader():


def test_serve_json(test_loader, use_gpu, image_uri, sagemaker_local_session, instance_type):
model_tar = model_gpu_tar if use_gpu else model_cpu_tar
model_tar = mnist_gpu_tar if use_gpu else mnist_cpu_tar
mnist_script = mnist_gpu_script if use_gpu else mnist_cpu_script
with _predictor(model_tar, mnist_script, image_uri, sagemaker_local_session,
instance_type) as predictor:
Expand All @@ -60,7 +60,7 @@ def test_serve_json(test_loader, use_gpu, image_uri, sagemaker_local_session, in


def test_serve_npy(test_loader, use_gpu, image_uri, sagemaker_local_session, instance_type):
model_tar = model_gpu_tar if use_gpu else model_cpu_tar
model_tar = mnist_gpu_tar if use_gpu else mnist_cpu_tar
mnist_script = mnist_gpu_script if use_gpu else mnist_cpu_script
with _predictor(model_tar, mnist_script, image_uri, sagemaker_local_session,
instance_type) as predictor:
Expand All @@ -70,7 +70,7 @@ def test_serve_npy(test_loader, use_gpu, image_uri, sagemaker_local_session, ins


def test_serve_csv(test_loader, use_gpu, image_uri, sagemaker_local_session, instance_type):
with _predictor(model_cpu_1d_tar, mnist_1d_script, image_uri, sagemaker_local_session,
with _predictor(mnist_cpu_1d_tar, mnist_cpu_1d_script, image_uri, sagemaker_local_session,
instance_type) as predictor:
for accept in (content_types.JSON, content_types.CSV, content_types.NPY):
_assert_prediction_csv(predictor, test_loader, accept)
Expand All @@ -79,7 +79,7 @@ def test_serve_csv(test_loader, use_gpu, image_uri, sagemaker_local_session, ins
def test_serve_cpu_model_on_gpu(test_loader, image_uri, sagemaker_local_session, instance_type):
if 'cpu' in image_uri:
pytest.skip("Skipping because running on CPU instance")
with _predictor(model_cpu_1d_tar, mnist_1d_script, image_uri, sagemaker_local_session,
with _predictor(mnist_cpu_1d_tar, mnist_cpu_1d_script, image_uri, sagemaker_local_session,
instance_type) as predictor:
_assert_prediction_npy_json(predictor, test_loader, content_types.NPY, content_types.JSON)

Expand Down
Loading