diff --git a/artifacts/mms-entrypoint.py b/artifacts/ts-entrypoint.py similarity index 100% rename from artifacts/mms-entrypoint.py rename to artifacts/ts-entrypoint.py diff --git a/setup.py b/setup.py index f665a83a..ce6d778f 100644 --- a/setup.py +++ b/setup.py @@ -31,6 +31,7 @@ def read(fname): packages=find_packages(where='src', exclude=('test',)), package_dir={'': 'src'}, + package_data={'': ["etc/*"]}, py_modules=[splitext(basename(path))[0] for path in glob('src/*.py')], long_description=read('README.rst'), @@ -56,7 +57,7 @@ def read(fname): 'test': ['boto3==1.10.32', 'coverage==4.5.3', 'docker-compose==1.23.2', 'flake8==3.7.7', 'Flask==1.1.1', 'mock==2.0.0', 'pytest==4.4.0', 'pytest-cov==2.7.1', 'pytest-xdist==1.28.0', 'PyYAML==3.10', 'sagemaker==1.56.3', 'sagemaker-containers>=2.5.4', 'six==1.12.0', 'requests==2.20.0', - 'requests_mock==1.6.0', 'torch==1.5.0', 'torchvision==0.6.0', 'tox==3.7.0'] + 'requests_mock==1.6.0', 'torch==1.6.0', 'torchvision==0.7.0', 'tox==3.7.0'] }, entry_points={ diff --git a/src/sagemaker_pytorch_serving_container/default_inference_handler.py b/src/sagemaker_pytorch_serving_container/default_pytorch_inference_handler.py similarity index 100% rename from src/sagemaker_pytorch_serving_container/default_inference_handler.py rename to src/sagemaker_pytorch_serving_container/default_pytorch_inference_handler.py diff --git a/src/sagemaker_pytorch_serving_container/etc/default-ts.properties b/src/sagemaker_pytorch_serving_container/etc/default-ts.properties new file mode 100644 index 00000000..bc0f996a --- /dev/null +++ b/src/sagemaker_pytorch_serving_container/etc/default-ts.properties @@ -0,0 +1,4 @@ +# Based on https://github.com/pytorch/serve/blob/master/docs/configuration.md +enable_envvars_config=true +decode_input_request=false +load_models=ALL diff --git a/src/sagemaker_pytorch_serving_container/etc/log4j.properties b/src/sagemaker_pytorch_serving_container/etc/log4j.properties new file mode 100644 index 00000000..eabaf6fa --- /dev/null +++ b/src/sagemaker_pytorch_serving_container/etc/log4j.properties @@ -0,0 +1,50 @@ +log4j.rootLogger = INFO, console + +log4j.appender.console = org.apache.log4j.ConsoleAppender +log4j.appender.console.Target = System.out +log4j.appender.console.layout = org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern = %d{ISO8601} [%-5p] %t %c - %m%n + +log4j.appender.access_log = org.apache.log4j.RollingFileAppender +log4j.appender.access_log.File = ${LOG_LOCATION}/access_log.log +log4j.appender.access_log.MaxFileSize = 10MB +log4j.appender.access_log.MaxBackupIndex = 5 +log4j.appender.access_log.layout = org.apache.log4j.PatternLayout +log4j.appender.access_log.layout.ConversionPattern = %d{ISO8601} - %m%n + +log4j.appender.ts_log = org.apache.log4j.RollingFileAppender +log4j.appender.ts_log.File = ${LOG_LOCATION}/ts_log.log +log4j.appender.ts_log.MaxFileSize = 10MB +log4j.appender.ts_log.MaxBackupIndex = 5 +log4j.appender.ts_log.layout = org.apache.log4j.PatternLayout +log4j.appender.ts_log.layout.ConversionPattern = %d{ISO8601} [%-5p] %t %c - %m%n + +log4j.appender.ts_metrics = org.apache.log4j.RollingFileAppender +log4j.appender.ts_metrics.File = ${METRICS_LOCATION}/ts_metrics.log +log4j.appender.ts_metrics.MaxFileSize = 10MB +log4j.appender.ts_metrics.MaxBackupIndex = 5 +log4j.appender.ts_metrics.layout = org.apache.log4j.PatternLayout +log4j.appender.ts_metrics.layout.ConversionPattern = %d{ISO8601} - %m%n + +log4j.appender.model_log = org.apache.log4j.RollingFileAppender +log4j.appender.model_log.File = ${LOG_LOCATION}/model_log.log +log4j.appender.model_log.MaxFileSize = 10MB +log4j.appender.model_log.MaxBackupIndex = 5 +log4j.appender.model_log.layout = org.apache.log4j.PatternLayout +log4j.appender.model_log.layout.ConversionPattern = %d{ISO8601} [%-5p] %c - %m%n + +log4j.appender.model_metrics = org.apache.log4j.RollingFileAppender +log4j.appender.model_metrics.File = ${METRICS_LOCATION}/model_metrics.log +log4j.appender.model_metrics.MaxFileSize = 10MB +log4j.appender.model_metrics.MaxBackupIndex = 5 +log4j.appender.model_metrics.layout = org.apache.log4j.PatternLayout +log4j.appender.model_metrics.layout.ConversionPattern = %d{ISO8601} - %m%n + +log4j.logger.com.amazonaws.ml.ts = INFO, ts_log +log4j.logger.ACCESS_LOG = INFO, access_log +log4j.logger.TS_METRICS = INFO, ts_metrics +log4j.logger.MODEL_METRICS = INFO, model_metrics +log4j.logger.MODEL_LOG = INFO, model_log + +log4j.logger.org.apache = OFF +log4j.logger.io.netty = ERROR diff --git a/src/sagemaker_pytorch_serving_container/handler_service.py b/src/sagemaker_pytorch_serving_container/handler_service.py index 408758d9..07e81c7b 100644 --- a/src/sagemaker_pytorch_serving_container/handler_service.py +++ b/src/sagemaker_pytorch_serving_container/handler_service.py @@ -14,8 +14,7 @@ from sagemaker_inference.default_handler_service import DefaultHandlerService from sagemaker_inference.transformer import Transformer -from sagemaker_pytorch_serving_container.default_inference_handler import \ - DefaultPytorchInferenceHandler +from sagemaker_pytorch_serving_container.default_pytorch_inference_handler import DefaultPytorchInferenceHandler import os import sys diff --git a/src/sagemaker_pytorch_serving_container/serving.py b/src/sagemaker_pytorch_serving_container/serving.py index 47011207..5e70c961 100644 --- a/src/sagemaker_pytorch_serving_container/serving.py +++ b/src/sagemaker_pytorch_serving_container/serving.py @@ -15,11 +15,10 @@ from subprocess import CalledProcessError from retrying import retry -from sagemaker_inference import model_server - +from sagemaker_pytorch_serving_container import torchserve from sagemaker_pytorch_serving_container import handler_service -HANDLER_SERVICE = handler_service.__name__ +HANDLER_SERVICE = handler_service.__file__ def _retry_if_error(exception): @@ -28,12 +27,12 @@ def _retry_if_error(exception): @retry(stop_max_delay=1000 * 30, retry_on_exception=_retry_if_error) -def _start_model_server(): +def _start_torchserve(): # there's a race condition that causes the model server command to # sometimes fail with 'bad address'. more investigation needed # retry starting mms until it's ready - model_server.start_model_server(handler_service=HANDLER_SERVICE) + torchserve.start_torchserve(handler_service=HANDLER_SERVICE) def main(): - _start_model_server() + _start_torchserve() diff --git a/src/sagemaker_pytorch_serving_container/torchserve.py b/src/sagemaker_pytorch_serving_container/torchserve.py new file mode 100644 index 00000000..58c770d0 --- /dev/null +++ b/src/sagemaker_pytorch_serving_container/torchserve.py @@ -0,0 +1,213 @@ +# Copyright 2019-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""This module contains functionality to configure and start Torchserve.""" +from __future__ import absolute_import + +import os +import signal +import subprocess +import sys + +import pkg_resources +import psutil +import logging +from retrying import retry + +import sagemaker_pytorch_serving_container +from sagemaker_inference import default_handler_service, environment, utils +from sagemaker_inference.environment import code_dir + +logger = logging.getLogger() + +TS_CONFIG_FILE = os.path.join("/etc", "sagemaker-ts.properties") +DEFAULT_HANDLER_SERVICE = default_handler_service.__name__ +DEFAULT_TS_CONFIG_FILE = pkg_resources.resource_filename( + sagemaker_pytorch_serving_container.__name__, "/etc/default-ts.properties" +) +MME_TS_CONFIG_FILE = pkg_resources.resource_filename( + sagemaker_pytorch_serving_container.__name__, "/etc/mme-ts.properties" +) +DEFAULT_TS_LOG_FILE = pkg_resources.resource_filename( + sagemaker_pytorch_serving_container.__name__, "/etc/log4j.properties" +) +DEFAULT_TS_MODEL_DIRECTORY = os.path.join(os.getcwd(), ".sagemaker", "ts", "models") +DEFAULT_TS_MODEL_NAME = "model" +DEFAULT_TS_MODEL_SERIALIZED_FILE = "model.pth" +DEFAULT_HANDLER_SERVICE = "sagemaker_pytorch_serving_container.handler_service" + +ENABLE_MULTI_MODEL = os.getenv("SAGEMAKER_MULTI_MODEL", "false") == "true" +MODEL_STORE = "/" if ENABLE_MULTI_MODEL else DEFAULT_TS_MODEL_DIRECTORY + +PYTHON_PATH_ENV = "PYTHONPATH" +REQUIREMENTS_PATH = os.path.join(code_dir, "requirements.txt") +TS_NAMESPACE = "org.pytorch.serve.ModelServer" + + +def start_torchserve(handler_service=DEFAULT_HANDLER_SERVICE): + """Configure and start the model server. + + Args: + handler_service (str): Python path pointing to a module that defines + a class with the following: + + - A ``handle`` method, which is invoked for all incoming inference + requests to the model server. + - A ``initialize`` method, which is invoked at model server start up + for loading the model. + + Defaults to ``sagemaker_pytorch_serving_container.default_handler_service``. + + """ + + if ENABLE_MULTI_MODEL: + if "SAGEMAKER_HANDLER" not in os.environ: + os.environ["SAGEMAKER_HANDLER"] = handler_service + _set_python_path() + else: + _adapt_to_ts_format(handler_service) + + _create_torchserve_config_file() + + if os.path.exists(REQUIREMENTS_PATH): + _install_requirements() + + ts_torchserve_cmd = [ + "torchserve", + "--start", + "--model-store", + MODEL_STORE, + "--ts-config", + TS_CONFIG_FILE, + "--log-config", + DEFAULT_TS_LOG_FILE, + "--models", + "model.mar" + ] + + print(ts_torchserve_cmd) + + logger.info(ts_torchserve_cmd) + subprocess.Popen(ts_torchserve_cmd) + + ts_process = _retrieve_ts_server_process() + + _add_sigterm_handler(ts_process) + + ts_process.wait() + + +def _adapt_to_ts_format(handler_service): + if not os.path.exists(DEFAULT_TS_MODEL_DIRECTORY): + os.makedirs(DEFAULT_TS_MODEL_DIRECTORY) + + model_archiver_cmd = [ + "torch-model-archiver", + "--model-name", + DEFAULT_TS_MODEL_NAME, + "--handler", + handler_service, + "--serialized-file", + os.path.join(environment.model_dir, DEFAULT_TS_MODEL_SERIALIZED_FILE), + "--export-path", + DEFAULT_TS_MODEL_DIRECTORY, + "--extra-files", + os.path.join(environment.model_dir, environment.Environment().module_name + ".py"), + "--version", + "1", + ] + + logger.info(model_archiver_cmd) + subprocess.check_call(model_archiver_cmd) + + _set_python_path() + + +def _set_python_path(): + # Torchserve handles code execution by appending the export path, provided + # to the model archiver, to the PYTHONPATH env var. + # The code_dir has to be added to the PYTHONPATH otherwise the + # user provided module can not be imported properly. + if PYTHON_PATH_ENV in os.environ: + os.environ[PYTHON_PATH_ENV] = "{}:{}".format(environment.code_dir, os.environ[PYTHON_PATH_ENV]) + else: + os.environ[PYTHON_PATH_ENV] = environment.code_dir + + +def _create_torchserve_config_file(): + configuration_properties = _generate_ts_config_properties() + + utils.write_file(TS_CONFIG_FILE, configuration_properties) + + +def _generate_ts_config_properties(): + env = environment.Environment() + + user_defined_configuration = { + "default_response_timeout": env.model_server_timeout, + "default_workers_per_model": env.model_server_workers, + "inference_address": "http://0.0.0.0:{}".format(env.inference_http_port), + "management_address": "http://0.0.0.0:{}".format(env.management_http_port), + } + + custom_configuration = str() + + for key in user_defined_configuration: + value = user_defined_configuration.get(key) + if value: + custom_configuration += "{}={}\n".format(key, value) + + if ENABLE_MULTI_MODEL: + default_configuration = utils.read_file(MME_TS_CONFIG_FILE) + else: + default_configuration = utils.read_file(DEFAULT_TS_CONFIG_FILE) + + return default_configuration + custom_configuration + + +def _add_sigterm_handler(ts_process): + def _terminate(signo, frame): # pylint: disable=unused-argument + try: + os.kill(ts_process.pid, signal.SIGTERM) + except OSError: + pass + + signal.signal(signal.SIGTERM, _terminate) + + +def _install_requirements(): + logger.info("installing packages from requirements.txt...") + pip_install_cmd = [sys.executable, "-m", "pip", "install", "-r", REQUIREMENTS_PATH] + + try: + subprocess.check_call(pip_install_cmd) + except subprocess.CalledProcessError: + logger.exception("failed to install required packages, exiting") + raise ValueError("failed to install required packages") + + +# retry for 10 seconds +@retry(stop_max_delay=10 * 1000) +def _retrieve_ts_server_process(): + ts_server_processes = list() + + for process in psutil.process_iter(): + if TS_NAMESPACE in process.cmdline(): + ts_server_processes.append(process) + + if not ts_server_processes: + raise Exception("Torchserve model server was unsuccessfully started") + + if len(ts_server_processes) > 1: + raise Exception("multiple ts model servers are not supported") + + return ts_server_processes[0] diff --git a/test/conftest.py b/test/conftest.py index 6a201251..312c8acb 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -53,7 +53,7 @@ def pytest_addoption(parser): parser.addoption('--accelerator-type') parser.addoption('--docker-base-name', default='sagemaker-pytorch-inference') parser.addoption('--region', default='us-west-2') - parser.addoption('--framework-version', default="1.5.0") + parser.addoption('--framework-version', default="1.6.0") parser.addoption('--py-version', choices=['2', '3'], default='3') # Processor is still "cpu" for EIA tests parser.addoption('--processor', choices=['gpu', 'cpu'], default='cpu') diff --git a/test/container/1.5.0/Dockerfile.dlc.cpu b/test/container/1.5.0/Dockerfile.dlc.cpu index 5fa5f7b4..b968a592 100644 --- a/test/container/1.5.0/Dockerfile.dlc.cpu +++ b/test/container/1.5.0/Dockerfile.dlc.cpu @@ -1,6 +1,18 @@ ARG region FROM 763104351884.dkr.ecr.$region.amazonaws.com/pytorch-inference:1.5.0-cpu-py3 +ARG TS_VERSION=0.1.1 +RUN apt-get update \ + && apt-get install -y --no-install-recommends software-properties-common \ + && add-apt-repository ppa:openjdk-r/ppa \ + && apt-get update \ + && apt-get install -y --no-install-recommends openjdk-11-jdk + +RUN pip install torchserve==$TS_VERSION \ + && pip install torch-model-archiver==$TS_VERSION + COPY dist/sagemaker_pytorch_inference-*.tar.gz /sagemaker_pytorch_inference.tar.gz RUN pip install --upgrade --no-cache-dir /sagemaker_pytorch_inference.tar.gz && \ rm /sagemaker_pytorch_inference.tar.gz + +CMD ["torchserve", "--start", "--ts-config", "/home/model-server/config.properties", "--model-store", "/home/model-server/"] diff --git a/test/container/1.5.0/Dockerfile.dlc.gpu b/test/container/1.5.0/Dockerfile.dlc.gpu index 078147cc..b968a592 100644 --- a/test/container/1.5.0/Dockerfile.dlc.gpu +++ b/test/container/1.5.0/Dockerfile.dlc.gpu @@ -1,6 +1,18 @@ ARG region -FROM 763104351884.dkr.ecr.$region.amazonaws.com/pytorch-inference:1.5.0-gpu-py3 +FROM 763104351884.dkr.ecr.$region.amazonaws.com/pytorch-inference:1.5.0-cpu-py3 + +ARG TS_VERSION=0.1.1 +RUN apt-get update \ + && apt-get install -y --no-install-recommends software-properties-common \ + && add-apt-repository ppa:openjdk-r/ppa \ + && apt-get update \ + && apt-get install -y --no-install-recommends openjdk-11-jdk + +RUN pip install torchserve==$TS_VERSION \ + && pip install torch-model-archiver==$TS_VERSION COPY dist/sagemaker_pytorch_inference-*.tar.gz /sagemaker_pytorch_inference.tar.gz RUN pip install --upgrade --no-cache-dir /sagemaker_pytorch_inference.tar.gz && \ rm /sagemaker_pytorch_inference.tar.gz + +CMD ["torchserve", "--start", "--ts-config", "/home/model-server/config.properties", "--model-store", "/home/model-server/"] diff --git a/test/container/1.5.0/Dockerfile.pytorch b/test/container/1.5.0/Dockerfile.pytorch index 9282ba36..95e94717 100644 --- a/test/container/1.5.0/Dockerfile.pytorch +++ b/test/container/1.5.0/Dockerfile.pytorch @@ -3,25 +3,29 @@ FROM pytorch/pytorch:1.5-cuda10.1-cudnn7-runtime LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true LABEL com.amazonaws.sagemaker.capabilities.multi-models=true -ARG MMS_VERSION=1.0.8 +ARG TS_VERSION=0.1.1 ENV SAGEMAKER_SERVING_MODULE sagemaker_pytorch_serving_container.serving:main ENV TEMP=/home/model-server/tmp RUN apt-get update \ + && apt-get install -y --no-install-recommends software-properties-common \ + && add-apt-repository ppa:openjdk-r/ppa \ + && apt-get update \ && apt-get install -y --no-install-recommends \ libgl1-mesa-glx \ libglib2.0-0 \ libsm6 \ libxext6 \ libxrender-dev \ - openjdk-8-jdk-headless \ + openjdk-11-jdk-headless \ && rm -rf /var/lib/apt/lists/* RUN conda install -c conda-forge opencv==4.0.1 \ && ln -s /opt/conda/bin/pip /usr/local/bin/pip3 -RUN pip install mxnet-model-server==$MMS_VERSION +RUN pip install torchserve==$TS_VERSION \ + && pip install torch-model-archiver==$TS_VERSION COPY dist/sagemaker_pytorch_inference-*.tar.gz /sagemaker_pytorch_inference.tar.gz RUN pip install --no-cache-dir /sagemaker_pytorch_inference.tar.gz && \ @@ -31,11 +35,11 @@ RUN useradd -m model-server \ && mkdir -p /home/model-server/tmp \ && chown -R model-server /home/model-server -COPY artifacts/mms-entrypoint.py /usr/local/bin/dockerd-entrypoint.py +COPY artifacts/ts-entrypoint.py /usr/local/bin/dockerd-entrypoint.py COPY artifacts/config.properties /home/model-server RUN chmod +x /usr/local/bin/dockerd-entrypoint.py EXPOSE 8080 8081 ENTRYPOINT ["python", "/usr/local/bin/dockerd-entrypoint.py"] -CMD ["mxnet-model-server", "--start", "--mms-config", "/home/model-server/config.properties"] +CMD ["torchserve", "--start", "--ts-config", "/home/model-server/config.properties", "--model-store", "/home/model-server/"] diff --git a/test/container/1.6.0/Dockerfile.dlc.cpu b/test/container/1.6.0/Dockerfile.dlc.cpu new file mode 100644 index 00000000..ea35cb94 --- /dev/null +++ b/test/container/1.6.0/Dockerfile.dlc.cpu @@ -0,0 +1,23 @@ +ARG region +FROM 763104351884.dkr.ecr.$region.amazonaws.com/pytorch-inference:1.5.0-cpu-py3 + +ARG TS_VERSION=0.1.1 +RUN apt-get update \ + && apt-get install -y --no-install-recommends software-properties-common \ + && add-apt-repository ppa:openjdk-r/ppa \ + && apt-get update \ + && apt-get install -y --no-install-recommends openjdk-11-jdk + +RUN pip install torchserve==$TS_VERSION \ + && pip install torch-model-archiver==$TS_VERSION + +RUN pip uninstall torch \ + && pip uninstall torchvision \ + && pip install torch=1.6.0 \ + && pip install torchvision=0.7.0 + +COPY dist/sagemaker_pytorch_inference-*.tar.gz /sagemaker_pytorch_inference.tar.gz +RUN pip install --upgrade --no-cache-dir /sagemaker_pytorch_inference.tar.gz && \ + rm /sagemaker_pytorch_inference.tar.gz + +CMD ["torchserve", "--start", "--ts-config", "/home/model-server/config.properties", "--model-store", "/home/model-server/"] diff --git a/test/container/1.6.0/Dockerfile.dlc.gpu b/test/container/1.6.0/Dockerfile.dlc.gpu new file mode 100644 index 00000000..ea35cb94 --- /dev/null +++ b/test/container/1.6.0/Dockerfile.dlc.gpu @@ -0,0 +1,23 @@ +ARG region +FROM 763104351884.dkr.ecr.$region.amazonaws.com/pytorch-inference:1.5.0-cpu-py3 + +ARG TS_VERSION=0.1.1 +RUN apt-get update \ + && apt-get install -y --no-install-recommends software-properties-common \ + && add-apt-repository ppa:openjdk-r/ppa \ + && apt-get update \ + && apt-get install -y --no-install-recommends openjdk-11-jdk + +RUN pip install torchserve==$TS_VERSION \ + && pip install torch-model-archiver==$TS_VERSION + +RUN pip uninstall torch \ + && pip uninstall torchvision \ + && pip install torch=1.6.0 \ + && pip install torchvision=0.7.0 + +COPY dist/sagemaker_pytorch_inference-*.tar.gz /sagemaker_pytorch_inference.tar.gz +RUN pip install --upgrade --no-cache-dir /sagemaker_pytorch_inference.tar.gz && \ + rm /sagemaker_pytorch_inference.tar.gz + +CMD ["torchserve", "--start", "--ts-config", "/home/model-server/config.properties", "--model-store", "/home/model-server/"] diff --git a/test/container/1.6.0/Dockerfile.pytorch b/test/container/1.6.0/Dockerfile.pytorch new file mode 100644 index 00000000..debf50ff --- /dev/null +++ b/test/container/1.6.0/Dockerfile.pytorch @@ -0,0 +1,45 @@ +FROM pytorch/pytorch:1.6.0-cuda10.1-cudnn7-runtime + +LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true +LABEL com.amazonaws.sagemaker.capabilities.multi-models=true + +ARG TS_VERSION=0.1.1 + +ENV SAGEMAKER_SERVING_MODULE sagemaker_pytorch_serving_container.serving:main +ENV TEMP=/home/model-server/tmp + +RUN apt-get update \ + && apt-get install -y --no-install-recommends software-properties-common \ + && add-apt-repository ppa:openjdk-r/ppa \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + libgl1-mesa-glx \ + libglib2.0-0 \ + libsm6 \ + libxext6 \ + libxrender-dev \ + openjdk-11-jdk-headless \ + && rm -rf /var/lib/apt/lists/* + +RUN conda install -c conda-forge opencv==4.0.1 \ + && ln -s /opt/conda/bin/pip /usr/local/bin/pip3 + +RUN pip install torchserve==$TS_VERSION \ + && pip install torch-model-archiver==$TS_VERSION + +COPY dist/sagemaker_pytorch_inference-*.tar.gz /sagemaker_pytorch_inference.tar.gz +RUN pip install --no-cache-dir /sagemaker_pytorch_inference.tar.gz && \ + rm /sagemaker_pytorch_inference.tar.gz + +RUN useradd -m model-server \ + && mkdir -p /home/model-server/tmp \ + && chown -R model-server /home/model-server + +COPY artifacts/ts-entrypoint.py /usr/local/bin/dockerd-entrypoint.py +COPY artifacts/config.properties /home/model-server + +RUN chmod +x /usr/local/bin/dockerd-entrypoint.py + +EXPOSE 8080 8081 +ENTRYPOINT ["python", "/usr/local/bin/dockerd-entrypoint.py"] +CMD ["torchserve", "--start", "--ts-config", "/home/model-server/config.properties", "--model-store", "/home/model-server/"] diff --git a/test/integration/sagemaker/test_mnist.py b/test/integration/sagemaker/test_mnist.py index 1dbfa5f8..912eba5b 100644 --- a/test/integration/sagemaker/test_mnist.py +++ b/test/integration/sagemaker/test_mnist.py @@ -34,6 +34,7 @@ def test_mnist_gpu(sagemaker_session, image_uri, instance_type): _test_mnist_distributed(sagemaker_session, image_uri, instance_type, model_gpu_tar, mnist_gpu_script) +@pytest.mark.skip(reason="Latest EIA version is too old - 1.3.1. Remove this after a new DLC release") @pytest.mark.eia_test def test_mnist_eia(sagemaker_session, image_uri, instance_type, accelerator_type): instance_type = instance_type or 'ml.c4.xlarge' diff --git a/test/unit/test_default_inference_handler.py b/test/unit/test_default_inference_handler.py index e43e026e..b8a37449 100644 --- a/test/unit/test_default_inference_handler.py +++ b/test/unit/test_default_inference_handler.py @@ -24,7 +24,7 @@ from six import StringIO, BytesIO from torch.autograd import Variable -from sagemaker_pytorch_serving_container import default_inference_handler +from sagemaker_pytorch_serving_container import default_pytorch_inference_handler device = torch.device("cuda" if torch.cuda.is_available() else "cpu") @@ -49,12 +49,12 @@ def fixture_tensor(): @pytest.fixture() def inference_handler(): - return default_inference_handler.DefaultPytorchInferenceHandler() + return default_pytorch_inference_handler.DefaultPytorchInferenceHandler() @pytest.fixture() def eia_inference_handler(): - return default_inference_handler.DefaultPytorchInferenceHandler() + return default_pytorch_inference_handler.DefaultPytorchInferenceHandler() def test_default_model_fn(inference_handler): @@ -181,7 +181,7 @@ def test_default_output_fn_gpu(inference_handler): def test_eia_default_model_fn(eia_inference_handler): - with mock.patch("sagemaker_pytorch_serving_container.default_inference_handler.os") as mock_os: + with mock.patch("sagemaker_pytorch_serving_container.default_pytorch_inference_handler.os") as mock_os: mock_os.getenv.return_value = "true" mock_os.path.join.return_value = "model_dir" mock_os.path.exists.return_value = True @@ -192,7 +192,7 @@ def test_eia_default_model_fn(eia_inference_handler): def test_eia_default_model_fn_error(eia_inference_handler): - with mock.patch("sagemaker_pytorch_serving_container.default_inference_handler.os") as mock_os: + with mock.patch("sagemaker_pytorch_serving_container.default_pytorch_inference_handler.os") as mock_os: mock_os.getenv.return_value = "true" mock_os.path.join.return_value = "model_dir" mock_os.path.exists.return_value = False @@ -202,7 +202,7 @@ def test_eia_default_model_fn_error(eia_inference_handler): def test_eia_default_predict_fn(eia_inference_handler, tensor): model = DummyModel() - with mock.patch("sagemaker_pytorch_serving_container.default_inference_handler.os") as mock_os: + with mock.patch("sagemaker_pytorch_serving_container.default_pytorch_inference_handler.os") as mock_os: mock_os.getenv.return_value = "true" with mock.patch("torch.jit.optimized_execution") as mock_torch: mock_torch.__enter__.return_value = "dummy" diff --git a/test/unit/test_handler_service.py b/test/unit/test_handler_service.py index f3fb468e..fd3dfc60 100644 --- a/test/unit/test_handler_service.py +++ b/test/unit/test_handler_service.py @@ -12,10 +12,10 @@ # language governing permissions and limitations under the License. from __future__ import absolute_import -from mock import patch +from mock import patch, Mock -@patch('sagemaker_pytorch_serving_container.default_inference_handler.DefaultPytorchInferenceHandler') +@patch('sagemaker_pytorch_serving_container.default_pytorch_inference_handler.DefaultPytorchInferenceHandler') @patch('sagemaker_inference.transformer.Transformer') def test_hosting_start(Transformer, DefaultPytorchInferenceHandler): from sagemaker_pytorch_serving_container import handler_service @@ -23,3 +23,16 @@ def test_hosting_start(Transformer, DefaultPytorchInferenceHandler): handler_service.HandlerService() Transformer.assert_called_with(default_inference_handler=DefaultPytorchInferenceHandler()) + + +@patch('sagemaker_pytorch_serving_container.default_pytorch_inference_handler.DefaultPytorchInferenceHandler') +@patch('sagemaker_inference.transformer.Transformer') +def test_hosting_start_enable_multi_model(Transformer, DefaultPytorchInferenceHandler): + from sagemaker_pytorch_serving_container import handler_service + + context = Mock() + context.system_properties.get.return_value = "/" + handler_service.ENABLE_MULTI_MODEL = True + handler = handler_service.HandlerService() + handler.initialize(context) + handler_service.ENABLE_MULTI_MODEL = False diff --git a/test/unit/test_model_server.py b/test/unit/test_model_server.py new file mode 100644 index 00000000..552a691d --- /dev/null +++ b/test/unit/test_model_server.py @@ -0,0 +1,331 @@ +# Copyright 2019-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the 'license' file accompanying this file. This file is +# distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +from __future__ import absolute_import +import os +import signal +import subprocess +import types + +from mock import Mock, patch +import pytest + +from sagemaker_inference import environment +from sagemaker_pytorch_serving_container import torchserve +from sagemaker_pytorch_serving_container.torchserve import TS_NAMESPACE, REQUIREMENTS_PATH + +PYTHON_PATH = "python_path" +DEFAULT_CONFIGURATION = "default_configuration" + + +@patch("subprocess.call") +@patch("subprocess.Popen") +@patch("sagemaker_pytorch_serving_container.torchserve._retrieve_ts_server_process") +@patch("sagemaker_pytorch_serving_container.torchserve._add_sigterm_handler") +@patch("sagemaker_pytorch_serving_container.torchserve._install_requirements") +@patch("os.path.exists", return_value=True) +@patch("sagemaker_pytorch_serving_container.torchserve._create_torchserve_config_file") +@patch("sagemaker_pytorch_serving_container.torchserve._adapt_to_ts_format") +def test_start_torchserve_default_service_handler( + adapt, + create_config, + exists, + install_requirements, + sigterm, + retrieve, + subprocess_popen, + subprocess_call, +): + torchserve.start_torchserve() + + adapt.assert_called_once_with(torchserve.DEFAULT_HANDLER_SERVICE) + create_config.assert_called_once_with() + exists.assert_called_once_with(REQUIREMENTS_PATH) + install_requirements.assert_called_once_with() + + ts_model_server_cmd = [ + "torchserve", + "--start", + "--model-store", + torchserve.MODEL_STORE, + "--ts-config", + torchserve.TS_CONFIG_FILE, + "--log-config", + torchserve.DEFAULT_TS_LOG_FILE, + "--models", + "model.mar" + ] + + subprocess_popen.assert_called_once_with(ts_model_server_cmd) + sigterm.assert_called_once_with(retrieve.return_value) + + +@patch("subprocess.call") +@patch("subprocess.Popen") +@patch("sagemaker_pytorch_serving_container.torchserve._retrieve_ts_server_process") +@patch("sagemaker_pytorch_serving_container.torchserve._add_sigterm_handler") +@patch("sagemaker_pytorch_serving_container.torchserve._install_requirements") +@patch("os.path.exists", return_value=True) +@patch("sagemaker_pytorch_serving_container.torchserve._create_torchserve_config_file") +@patch("sagemaker_pytorch_serving_container.torchserve._adapt_to_ts_format") +def test_start_torchserve_default_service_handler_multi_model( + adapt, + create_config, + exists, + install_requirements, + sigterm, + retrieve, + subprocess_popen, + subprocess_call, +): + torchserve.ENABLE_MULTI_MODEL = True + torchserve.start_torchserve() + torchserve.ENABLE_MULTI_MODEL = False + create_config.assert_called_once_with() + exists.assert_called_once_with(REQUIREMENTS_PATH) + install_requirements.assert_called_once_with() + + ts_model_server_cmd = [ + "torchserve", + "--start", + "--model-store", + torchserve.MODEL_STORE, + "--ts-config", + torchserve.TS_CONFIG_FILE, + "--log-config", + torchserve.DEFAULT_TS_LOG_FILE, + "--models", + "model.mar" + ] + + subprocess_popen.assert_called_once_with(ts_model_server_cmd) + sigterm.assert_called_once_with(retrieve.return_value) + + +@patch("subprocess.call") +@patch("subprocess.Popen") +@patch("sagemaker_pytorch_serving_container.torchserve._retrieve_ts_server_process") +@patch("sagemaker_pytorch_serving_container.torchserve._add_sigterm_handler") +@patch("sagemaker_pytorch_serving_container.torchserve._create_torchserve_config_file") +@patch("sagemaker_pytorch_serving_container.torchserve._adapt_to_ts_format") +def test_start_torchserve_custom_handler_service( + adapt, create_config, sigterm, retrieve, subprocess_popen, subprocess_call +): + handler_service = Mock() + + torchserve.start_torchserve(handler_service) + + adapt.assert_called_once_with(handler_service) + + +@patch("sagemaker_pytorch_serving_container.torchserve._set_python_path") +@patch("subprocess.check_call") +@patch("os.makedirs") +@patch("os.path.exists", return_value=False) +def test_adapt_to_ts_format(path_exists, make_dir, subprocess_check_call, set_python_path): + handler_service = Mock() + + torchserve._adapt_to_ts_format(handler_service) + + path_exists.assert_called_once_with(torchserve.DEFAULT_TS_MODEL_DIRECTORY) + make_dir.assert_called_once_with(torchserve.DEFAULT_TS_MODEL_DIRECTORY) + + model_archiver_cmd = [ + "torch-model-archiver", + "--model-name", + torchserve.DEFAULT_TS_MODEL_NAME, + "--handler", + handler_service, + "--serialized-file", + os.path.join(environment.model_dir, torchserve.DEFAULT_TS_MODEL_SERIALIZED_FILE), + "--export-path", + torchserve.DEFAULT_TS_MODEL_DIRECTORY, + "--extra-files", + os.path.join(environment.model_dir, environment.Environment().module_name + ".py"), + "--version", + "1", + ] + + subprocess_check_call.assert_called_once_with(model_archiver_cmd) + set_python_path.assert_called_once_with() + + +@patch("sagemaker_pytorch_serving_container.torchserve._set_python_path") +@patch("subprocess.check_call") +@patch("os.makedirs") +@patch("os.path.exists", return_value=True) +def test_adapt_to_ts_format_existing_path( + path_exists, make_dir, subprocess_check_call, set_python_path +): + handler_service = Mock() + + torchserve._adapt_to_ts_format(handler_service) + + path_exists.assert_called_once_with(torchserve.DEFAULT_TS_MODEL_DIRECTORY) + make_dir.assert_not_called() + + +@patch.dict(os.environ, {torchserve.PYTHON_PATH_ENV: PYTHON_PATH}, clear=True) +def test_set_existing_python_path(): + torchserve._set_python_path() + + code_dir_path = "{}:{}".format(environment.code_dir, PYTHON_PATH) + + assert os.environ[torchserve.PYTHON_PATH_ENV] == code_dir_path + + +@patch.dict(os.environ, {}, clear=True) +def test_new_python_path(): + torchserve._set_python_path() + + code_dir_path = environment.code_dir + + assert os.environ[torchserve.PYTHON_PATH_ENV] == code_dir_path + + +@patch("sagemaker_pytorch_serving_container.torchserve._generate_ts_config_properties") +@patch("sagemaker_inference.utils.write_file") +def test_create_torchserve_config_file(write_file, generate_ts_config_props): + torchserve._create_torchserve_config_file() + + write_file.assert_called_once_with( + torchserve.TS_CONFIG_FILE, generate_ts_config_props.return_value + ) + + +@patch("sagemaker_inference.utils.read_file", return_value=DEFAULT_CONFIGURATION) +@patch("sagemaker_inference.environment.Environment") +def test_generate_ts_config_properties(env, read_file): + model_server_timeout = "torchserve_timeout" + model_server_workers = "torchserve_workers" + http_port = "http_port" + + env.return_value.model_server_timeout = model_server_timeout + env.return_value.model_sever_workerse = model_server_workers + env.return_value.inference_http_port = http_port + + ts_config_properties = torchserve._generate_ts_config_properties() + + inference_address = "inference_address=http://0.0.0.0:{}\n".format(http_port) + server_timeout = "default_response_timeout={}\n".format(model_server_timeout) + + read_file.assert_called_once_with(torchserve.DEFAULT_TS_CONFIG_FILE) + + assert ts_config_properties.startswith(DEFAULT_CONFIGURATION) + assert inference_address in ts_config_properties + assert server_timeout in ts_config_properties + + +@patch("sagemaker_inference.utils.read_file", return_value=DEFAULT_CONFIGURATION) +@patch("sagemaker_inference.environment.Environment") +def test_generate_ts_config_properties_default_workers(env, read_file): + env.return_value.model_server_workers = None + + ts_config_properties = torchserve._generate_ts_config_properties() + + workers = "default_workers_per_model={}".format(None) + + read_file.assert_called_once_with(torchserve.DEFAULT_TS_CONFIG_FILE) + + assert ts_config_properties.startswith(DEFAULT_CONFIGURATION) + assert workers not in ts_config_properties + + +@patch("sagemaker_inference.utils.read_file", return_value=DEFAULT_CONFIGURATION) +@patch("sagemaker_inference.environment.Environment") +def test_generate_ts_config_properties_multi_model(env, read_file): + env.return_value.model_server_workers = None + + torchserve.ENABLE_MULTI_MODEL = True + ts_config_properties = torchserve._generate_ts_config_properties() + torchserve.ENABLE_MULTI_MODEL = False + + workers = "default_workers_per_model={}".format(None) + + read_file.assert_called_once_with(torchserve.MME_TS_CONFIG_FILE) + + assert ts_config_properties.startswith(DEFAULT_CONFIGURATION) + assert workers not in ts_config_properties + + +@patch("signal.signal") +def test_add_sigterm_handler(signal_call): + ts = Mock() + + torchserve._add_sigterm_handler(ts) + + mock_calls = signal_call.mock_calls + first_argument = mock_calls[0][1][0] + second_argument = mock_calls[0][1][1] + + assert len(mock_calls) == 1 + assert first_argument == signal.SIGTERM + assert isinstance(second_argument, types.FunctionType) + + +@patch("subprocess.check_call") +def test_install_requirements(check_call): + torchserve._install_requirements() + for i in ['pip', 'install', '-r', '/opt/ml/model/code/requirements.txt']: + assert i in check_call.call_args.args[0] + + +@patch("subprocess.check_call", side_effect=subprocess.CalledProcessError(0, "cmd")) +def test_install_requirements_installation_failed(check_call): + with pytest.raises(ValueError) as e: + torchserve._install_requirements() + assert "failed to install required packages" in str(e.value) + + +@patch("retrying.Retrying.should_reject", return_value=False) +@patch("psutil.process_iter") +def test_retrieve_ts_server_process(process_iter, retry): + server = Mock() + server.cmdline.return_value = TS_NAMESPACE + + processes = list() + processes.append(server) + + process_iter.return_value = processes + + process = torchserve._retrieve_ts_server_process() + + assert process == server + + +@patch("retrying.Retrying.should_reject", return_value=False) +@patch("psutil.process_iter", return_value=list()) +def test_retrieve_ts_server_process_no_server(process_iter, retry): + with pytest.raises(Exception) as e: + torchserve._retrieve_ts_server_process() + + assert "Torchserve model server was unsuccessfully started" in str(e.value) + + +@patch("retrying.Retrying.should_reject", return_value=False) +@patch("psutil.process_iter") +def test_retrieve_ts_server_process_too_many_servers(process_iter, retry): + server = Mock() + second_server = Mock() + server.cmdline.return_value = TS_NAMESPACE + second_server.cmdline.return_value = TS_NAMESPACE + + processes = list() + processes.append(server) + processes.append(second_server) + + process_iter.return_value = processes + + with pytest.raises(Exception) as e: + torchserve._retrieve_ts_server_process() + + assert "multiple ts model servers are not supported" in str(e.value) diff --git a/test/unit/test_serving.py b/test/unit/test_serving.py index dfeca8f6..662674aa 100644 --- a/test/unit/test_serving.py +++ b/test/unit/test_serving.py @@ -15,11 +15,14 @@ from mock import patch -@patch('sagemaker_inference.model_server.start_model_server') -def test_hosting_start(start_model_server): +@patch('sagemaker_pytorch_serving_container.torchserve.start_torchserve') +def test_hosting_start(start_torchserve): from sagemaker_pytorch_serving_container import serving serving.main() + start_torchserve.assert_called() - start_model_server.assert_called_with( - handler_service='sagemaker_pytorch_serving_container.handler_service') + +def test_retry_if_error(): + from sagemaker_pytorch_serving_container import serving + serving._retry_if_error(Exception)