Merge branch 'master' into mme-fix

Author: satishpasumarthi

.gitignore

@@ -2,3 +2,6 @@ __pycache__
 .tox/
 log.txt
 .idea/
+node_modules/
+package.json
+package-lock.json

README.md

@@ -42,7 +42,9 @@ For notebook examples, see: [Amazon SageMaker Examples](https://github.com/awsla
 3. [Running the tests](#running-the-tests)
 4. [Pre/Post-Processing](#pre/post-processing)
 5. [Deploying a TensorFlow Serving Model](#deploying-a-tensorflow-serving-model)
-6. [Deploying to Multi-Model Endpoint](#deploying-to-multi-model-endpoint)
+6. [Enable Batching](#enabling-batching)
+7. [Configurable SageMaker Environment Variables](#configurable-sagemaker-environment-variables)
+8. [Deploying to Multi-Model Endpoint](#deploying-to-multi-model-endpoint)
 
 ## Getting Started
 
@@ -612,6 +614,22 @@ SAGEMAKER_TFS_NUM_BATCH_THREADS="16"
 SAGEMAKER_TFS_MAX_ENQUEUED_BATCHES="10000"
 ```
 
+## Configurable SageMaker Environment Variables
+The following environment variables can be set on a SageMaker Model or Transform Job if further configuration is required:
+
+[Configures](https://docs.gunicorn.org/en/stable/settings.html#loglevel)
+the logging level for Gunicorn.
+```bash
+# Defaults to "info"
+SAGEMAKER_GUNICORN_LOGLEVEL="debug"
+```
+[Configures](https://docs.gunicorn.org/en/stable/settings.html#timeout)
+how long a Gunicorn worker may be silent before it is killed and restarted.
+```bash
+# Defaults to 30.
+SAGEMAKER_GUNICORN_TIMEOUT_SECONDS="60"
+```
+
 ## Deploying to Multi-Model Endpoint
 
 SageMaker TensorFlow Serving container (version 1.5.0 and 2.1.0, CPU) now supports Multi-Model Endpoint. With this feature, you can deploy different models (not just different versions of a model) to a single endpoint.

docker/1.15/Dockerfile.cpu

@@ -1,4 +1,4 @@
-FROM ubuntu:18.04
+FROM public.ecr.aws/ubuntu/ubuntu:18.04
 
 LABEL maintainer="Amazon AI"
 # Specify LABEL for inference pipelines to use SAGEMAKER_BIND_TO_PORT

docker/2.1/Dockerfile.cpu

@@ -1,4 +1,4 @@
-FROM ubuntu:18.04
+FROM public.ecr.aws/ubuntu/ubuntu:18.04
 
 LABEL maintainer="Amazon AI"
 LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true

docker/build_artifacts/deep_learning_container.py

@@ -49,19 +49,33 @@ def _retrieve_instance_region():
     Retrieve instance region from instance metadata service
     """
     region = None
-    valid_regions = ['ap-northeast-1', 'ap-northeast-2', 'ap-southeast-1', 'ap-southeast-2',
-                     'ap-south-1', 'ca-central-1', 'eu-central-1', 'eu-north-1',
-                     'eu-west-1', 'eu-west-2', 'eu-west-3', 'sa-east-1',
-                     'us-east-1', 'us-east-2', 'us-west-1', 'us-west-2']
+    valid_regions = [
+        "ap-northeast-1",
+        "ap-northeast-2",
+        "ap-southeast-1",
+        "ap-southeast-2",
+        "ap-south-1",
+        "ca-central-1",
+        "eu-central-1",
+        "eu-north-1",
+        "eu-west-1",
+        "eu-west-2",
+        "eu-west-3",
+        "sa-east-1",
+        "us-east-1",
+        "us-east-2",
+        "us-west-1",
+        "us-west-2",
+    ]
 
     url = "http://169.254.169.254/latest/dynamic/instance-identity/document"
     response = requests_helper(url, timeout=0.1)
 
     if response is not None:
         response_json = json.loads(response.text)
 
-        if response_json['region'] in valid_regions:
-            region = response_json['region']
+        if response_json["region"] in valid_regions:
+            region = response_json["region"]
 
     return region
 
@@ -75,8 +89,10 @@ def query_bucket():
     region = _retrieve_instance_region()
 
     if instance_id is not None and region is not None:
-        url = ("https://aws-deep-learning-containers-{0}.s3.{0}.amazonaws.com"
-               "/dlc-containers.txt?x-instance-id={1}".format(region, instance_id))
+        url = (
+            "https://aws-deep-learning-containers-{0}.s3.{0}.amazonaws.com"
+            "/dlc-containers.txt?x-instance-id={1}".format(region, instance_id)
+        )
         response = requests_helper(url, timeout=0.2)
 
     logging.debug("Query bucket finished: {}".format(response))
@@ -105,5 +121,5 @@ def main():
     query_bucket()
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

docker/build_artifacts/dockerd-entrypoint.py

@@ -19,4 +19,4 @@
 if not os.path.exists("/opt/ml/input/config"):
     subprocess.call(["python", "/usr/local/bin/deep_learning_container.py", "&>/dev/null", "&"])
 
-subprocess.check_call(shlex.split(' '.join(sys.argv[1:])))
+subprocess.check_call(shlex.split(" ".join(sys.argv[1:])))

docker/build_artifacts/sagemaker/python_service.py

@@ -58,7 +58,6 @@ def default_handler(data, context):
 
 
 class PythonServiceResource:
-
     def __init__(self):
         if SAGEMAKER_MULTI_MODEL_ENABLED:
             self._model_tfs_rest_port = {}
@@ -83,9 +82,9 @@ def __init__(self):
             log.info("Inference script exists, importing handlers.")
             # Single-Model Mode & Multi-Model Mode both use one inference.py
             self._handler, self._input_handler, self._output_handler = self._import_handlers()
-            self._handlers = self._make_handler(self._handler,
-                                                self._input_handler,
-                                                self._output_handler)
+            self._handlers = self._make_handler(
+                self._handler, self._input_handler, self._output_handler
+            )
         else:
             log.info("Inference script does not exist, using default handlers.")
             self._handlers = default_handler
@@ -108,7 +107,7 @@ def _pick_port(self, ports):
         return random.choice(ports)
 
     def _parse_sagemaker_port_range_mme(self, port_range):
-        lower, upper = port_range.split('-')
+        lower, upper = port_range.split("-")
         lower = int(lower)
         upper = lower + int((int(upper) - lower) * 0.9)  # only utilizing 90% of the ports
         rest_port = lower
@@ -132,16 +131,14 @@ def _handle_load_model_post(self, res, data):  # noqa: C901
         # model is already loaded
         if model_name in self._model_tfs_pid:
             res.status = falcon.HTTP_409
-            res.body = json.dumps({
-                "error": "Model {} is already loaded.".format(model_name)
-            })
+            res.body = json.dumps({"error": "Model {} is already loaded.".format(model_name)})
 
         # check if there are available ports
         if not self._ports_available():
             res.status = falcon.HTTP_507
-            res.body = json.dumps({
-                "error": "Memory exhausted: no available ports to load the model."
-            })
+            res.body = json.dumps(
+                {"error": "Memory exhausted: no available ports to load the model."}
+            )
         with lock():
             self._model_tfs_rest_port[model_name] = self._tfs_ports["rest_port"].pop()
             self._model_tfs_grpc_port[model_name] = self._tfs_ports["grpc_port"].pop()
@@ -157,7 +154,8 @@ def _handle_load_model_post(self, res, data):  # noqa: C901
                     f.write(tfs_config)
 
                 batching_config_file = "/sagemaker/batching/{}/batching-config.cfg".format(
-                    model_name)
+                    model_name
+                )
                 if self._tfs_enable_batching:
                     tfs_utils.create_batching_config(batching_config_file)
 
@@ -170,22 +168,26 @@ def _handle_load_model_post(self, res, data):  # noqa: C901
                 )
                 p = subprocess.Popen(cmd.split())
 
-                tfs_utils.wait_for_model(self._model_tfs_rest_port[model_name], model_name,
-                                         self._tfs_wait_time_seconds)
+                tfs_utils.wait_for_model(
+                    self._model_tfs_rest_port[model_name], model_name, self._tfs_wait_time_seconds
+                )
 
                 log.info("started tensorflow serving (pid: %d)", p.pid)
                 # update model name <-> tfs pid map
                 self._model_tfs_pid[model_name] = p
 
                 res.status = falcon.HTTP_200
-                res.body = json.dumps({
-                    "success":
-                        "Successfully loaded model {}, "
+                res.body = json.dumps(
+                    {
+                        "success": "Successfully loaded model {}, "
                         "listening on rest port {} "
-                        "and grpc port {}.".format(model_name,
-                                                   self._model_tfs_rest_port,
-                                                   self._model_tfs_grpc_port,)
-                })
+                        "and grpc port {}.".format(
+                            model_name,
+                            self._model_tfs_rest_port,
+                            self._model_tfs_grpc_port,
+                        )
+                    }
+                )
             except MultiModelException as multi_model_exception:
                 self._cleanup_config_file(tfs_config_file)
                 self._cleanup_config_file(batching_config_file)
@@ -199,25 +201,28 @@ def _handle_load_model_post(self, res, data):  # noqa: C901
                     raise MultiModelException(falcon.HTTP_500, multi_model_exception.msg)
             except FileExistsError as e:
                 res.status = falcon.HTTP_409
-                res.body = json.dumps({
-                    "error": "Model {} is already loaded. {}".format(model_name, str(e))
-                })
+                res.body = json.dumps(
+                    {"error": "Model {} is already loaded. {}".format(model_name, str(e))}
+                )
             except OSError as os_error:
                 self._cleanup_config_file(tfs_config_file)
                 self._cleanup_config_file(batching_config_file)
                 if os_error.errno == 12:
-                    raise MultiModelException(falcon.HTTP_507,
-                                              "Memory exhausted: "
-                                              "not enough memory to start TFS instance")
+                    raise MultiModelException(
+                        falcon.HTTP_507,
+                        "Memory exhausted: " "not enough memory to start TFS instance",
+                    )
                 else:
                     raise MultiModelException(falcon.HTTP_500, os_error.strerror)
         else:
             res.status = falcon.HTTP_404
-            res.body = json.dumps({
-                "error":
-                    "Could not find valid base path {} for servable {}".format(base_path,
-                                                                               model_name)
-            })
+            res.body = json.dumps(
+                {
+                    "error": "Could not find valid base path {} for servable {}".format(
+                        base_path, model_name
+                    )
+                }
+            )
 
     def _cleanup_config_file(self, config_file):
         if os.path.exists(config_file):
@@ -228,31 +233,37 @@ def _handle_invocation_post(self, req, res, model_name=None):
             if model_name:
                 if model_name not in self._model_tfs_rest_port:
                     res.status = falcon.HTTP_404
-                    res.body = json.dumps({
-                        "error": "Model {} is not loaded yet.".format(model_name)
-                    })
+                    res.body = json.dumps(
+                        {"error": "Model {} is not loaded yet.".format(model_name)}
+                    )
                     return
                 else:
                     log.info("model name: {}".format(model_name))
                     rest_port = self._model_tfs_rest_port[model_name]
                     log.info("rest port: {}".format(str(self._model_tfs_rest_port[model_name])))
                     grpc_port = self._model_tfs_grpc_port[model_name]
                     log.info("grpc port: {}".format(str(self._model_tfs_grpc_port[model_name])))
-                    data, context = tfs_utils.parse_request(req, rest_port, grpc_port,
-                                                            self._tfs_default_model_name,
-                                                            model_name=model_name)
+                    data, context = tfs_utils.parse_request(
+                        req,
+                        rest_port,
+                        grpc_port,
+                        self._tfs_default_model_name,
+                        model_name=model_name,
+                    )
             else:
                 res.status = falcon.HTTP_400
-                res.body = json.dumps({
-                    "error": "Invocation request does not contain model name."
-                })
+                res.body = json.dumps({"error": "Invocation request does not contain model name."})
         else:
             # Randomly pick port used for routing incoming request.
             grpc_port = self._pick_port(self._tfs_grpc_ports)
             rest_port = self._pick_port(self._tfs_rest_ports)
-            data, context = tfs_utils.parse_request(req, rest_port, grpc_port,
-                                                    self._tfs_default_model_name,
-                                                    channel=self._channels[grpc_port])
+            data, context = tfs_utils.parse_request(
+                req,
+                rest_port,
+                grpc_port,
+                self._tfs_default_model_name,
+                channel=self._channels[grpc_port],
+            )
 
         try:
             res.status = falcon.HTTP_200
@@ -261,9 +272,7 @@ def _handle_invocation_post(self, req, res, model_name=None):
         except Exception as e:  # pylint: disable=broad-except
             log.exception("exception handling request: {}".format(e))
             res.status = falcon.HTTP_500
-            res.body = json.dumps({
-                "error": str(e)
-            }).encode("utf-8")  # pylint: disable=E1101
+            res.body = json.dumps({"error": str(e)}).encode("utf-8")  # pylint: disable=E1101
 
     def _setup_channel(self, grpc_port):
         if grpc_port not in self._channels:
@@ -309,39 +318,31 @@ def on_get(self, req, res, model_name=None):  # pylint: disable=W0613
                 except ValueError as e:
                     log.exception("exception handling request: {}".format(e))
                     res.status = falcon.HTTP_500
-                    res.body = json.dumps({
-                        "error": str(e)
-                    }).encode("utf-8")
+                    res.body = json.dumps({"error": str(e)}).encode("utf-8")
             res.status = falcon.HTTP_200
             res.body = json.dumps(models_info)
         else:
             if model_name not in self._model_tfs_rest_port:
                 res.status = falcon.HTTP_404
-                res.body = json.dumps({
-                    "error": "Model {} is loaded yet.".format(model_name)
-                }).encode("utf-8")
+                res.body = json.dumps(
+                    {"error": "Model {} is loaded yet.".format(model_name)}
+                ).encode("utf-8")
             else:
                 port = self._model_tfs_rest_port[model_name]
                 uri = "http://localhost:{}/v1/models/{}".format(port, model_name)
                 try:
                     info = requests.get(uri)
                     res.status = falcon.HTTP_200
-                    res.body = json.dumps({
-                        "model": info
-                    }).encode("utf-8")
+                    res.body = json.dumps({"model": info}).encode("utf-8")
                 except ValueError as e:
                     log.exception("exception handling GET models request.")
                     res.status = falcon.HTTP_500
-                    res.body = json.dumps({
-                        "error": str(e)
-                    }).encode("utf-8")
+                    res.body = json.dumps({"error": str(e)}).encode("utf-8")
 
     def on_delete(self, req, res, model_name):  # pylint: disable=W0613
         if model_name not in self._model_tfs_pid:
             res.status = falcon.HTTP_404
-            res.body = json.dumps({
-                "error": "Model {} is not loaded yet".format(model_name)
-            })
+            res.body = json.dumps({"error": "Model {} is not loaded yet".format(model_name)})
         else:
             try:
                 self._model_tfs_pid[model_name].kill()
@@ -356,14 +357,12 @@ def on_delete(self, req, res, model_name):  # pylint: disable=W0613
                 del self._model_tfs_grpc_port[model_name]
                 del self._model_tfs_pid[model_name]
                 res.status = falcon.HTTP_200
-                res.body = json.dumps({
-                    "success": "Successfully unloaded model {}.".format(model_name)
-                })
+                res.body = json.dumps(
+                    {"success": "Successfully unloaded model {}.".format(model_name)}
+                )
             except OSError as error:
                 res.status = falcon.HTTP_500
-                res.body = json.dumps({
-                    "error": str(error)
-                }).encode("utf-8")
+                res.body = json.dumps({"error": str(error)}).encode("utf-8")
 
     def validate_model_dir(self, model_path):
         # model base path doesn't exits