From 596a03b0bfd2c74c553489a72837c40a041300db Mon Sep 17 00:00:00 2001 From: "Isaac T. Mathers" Date: Thu, 3 Mar 2022 22:12:02 +0000 Subject: [PATCH 01/12] Add env var that will manage GUnicorn setup timeout. --- docker/build_artifacts/sagemaker/serve.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docker/build_artifacts/sagemaker/serve.py b/docker/build_artifacts/sagemaker/serve.py index 68f22f4c..f00d6f6b 100644 --- a/docker/build_artifacts/sagemaker/serve.py +++ b/docker/build_artifacts/sagemaker/serve.py @@ -63,6 +63,9 @@ def __init__(self): self._tfs_inter_op_parallelism = os.environ.get("SAGEMAKER_TFS_INTER_OP_PARALLELISM", 0) self._tfs_intra_op_parallelism = os.environ.get("SAGEMAKER_TFS_INTRA_OP_PARALLELISM", 0) self._gunicorn_worker_class = os.environ.get("SAGEMAKER_GUNICORN_WORKER_CLASS", "gevent") + self._gunicorn_setup_timeout_seconds = os.environ.get( + "SAGEMAKER_GUNICORN_SETUP_TIMEOUT_SECONDS", 30 + ) if os.environ.get("OMP_NUM_THREADS") is None: os.environ["OMP_NUM_THREADS"] = "1" @@ -449,7 +452,7 @@ def start(self): self._setup_gunicorn() self._start_gunicorn() # make sure gunicorn is up - with self._timeout(seconds=30): + with self._timeout(seconds=self._gunicorn_setup_timeout_seconds): self._wait_for_gunicorn() self._start_nginx() From 54b8e82e81a7f1d14d112882af5972aaea59c3f5 Mon Sep 17 00:00:00 2001 From: "Isaac T. Mathers" Date: Fri, 4 Mar 2022 17:55:15 +0000 Subject: [PATCH 02/12] Ensure env var is treated as innt. --- docker/build_artifacts/sagemaker/serve.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/build_artifacts/sagemaker/serve.py b/docker/build_artifacts/sagemaker/serve.py index f00d6f6b..d54c5f71 100644 --- a/docker/build_artifacts/sagemaker/serve.py +++ b/docker/build_artifacts/sagemaker/serve.py @@ -63,9 +63,9 @@ def __init__(self): self._tfs_inter_op_parallelism = os.environ.get("SAGEMAKER_TFS_INTER_OP_PARALLELISM", 0) self._tfs_intra_op_parallelism = os.environ.get("SAGEMAKER_TFS_INTRA_OP_PARALLELISM", 0) self._gunicorn_worker_class = os.environ.get("SAGEMAKER_GUNICORN_WORKER_CLASS", "gevent") - self._gunicorn_setup_timeout_seconds = os.environ.get( + self._gunicorn_setup_timeout_seconds = int(os.environ.get( "SAGEMAKER_GUNICORN_SETUP_TIMEOUT_SECONDS", 30 - ) + )) if os.environ.get("OMP_NUM_THREADS") is None: os.environ["OMP_NUM_THREADS"] = "1" From 739c82ac19fbcbbba3ad2b40ee87a39c12afc614 Mon Sep 17 00:00:00 2001 From: "Isaac T. Mathers" Date: Fri, 4 Mar 2022 18:38:49 +0000 Subject: [PATCH 03/12] Update readme to include new env var section and update TOC. --- README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d4729851..6654d9b0 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,9 @@ For notebook examples, see: [Amazon SageMaker Examples](https://github.com/awsla 3. [Running the tests](#running-the-tests) 4. [Pre/Post-Processing](#pre/post-processing) 5. [Deploying a TensorFlow Serving Model](#deploying-a-tensorflow-serving-model) -6. [Deploying to Multi-Model Endpoint](#deploying-to-multi-model-endpoint) +6. [Enable Batching](#enabling-batching) +7. [Other Configurable Environment Variables](#other-configurable-environment-variables) +8. [Deploying to Multi-Model Endpoint](#deploying-to-multi-model-endpoint) ## Getting Started @@ -612,6 +614,16 @@ SAGEMAKER_TFS_NUM_BATCH_THREADS="16" SAGEMAKER_TFS_MAX_ENQUEUED_BATCHES="10000" ``` +## Other Configurable Environment Variables +The following environment variables can be set on a SageMaker Model or Transform Job if further configuration is required: + +```bash +# Configures how long to wait in seconds for GUnicorn +# to finish starting up before timing out. +# Defaults to 30. +SAGEMAKER_GUNICORN_SETUP_TIMEOUT_SECONDS="60" +``` + ## Deploying to Multi-Model Endpoint SageMaker TensorFlow Serving container (version 1.5.0 and 2.1.0, CPU) now supports Multi-Model Endpoint. With this feature, you can deploy different models (not just different versions of a model) to a single endpoint. From 6e99c4cbce598df0477c4c8984ce2dae4a1ac095 Mon Sep 17 00:00:00 2001 From: "Isaac T. Mathers" Date: Mon, 7 Mar 2022 16:16:33 +0000 Subject: [PATCH 04/12] Add SAGEMAKER_GUNICORN_LOGLEVEL description to readme. --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 6654d9b0..92ed9eba 100644 --- a/README.md +++ b/README.md @@ -618,6 +618,11 @@ SAGEMAKER_TFS_MAX_ENQUEUED_BATCHES="10000" The following environment variables can be set on a SageMaker Model or Transform Job if further configuration is required: ```bash +# Configures the logging level for GUnicorn. +# Valid values can be found here: https://docs.gunicorn.org/en/stable/settings.html#loglevel +# Defaults to "info" +SAGEMAKER_GUNICORN_LOGLEVEL="debug" + # Configures how long to wait in seconds for GUnicorn # to finish starting up before timing out. # Defaults to 30. From 984455ceeb4a0ccb15e14b0eb9eed88d1d65bc9a Mon Sep 17 00:00:00 2001 From: Manu Seth <22492939+mseth10@users.noreply.github.com> Date: Fri, 4 Mar 2022 17:32:44 -0800 Subject: [PATCH 05/12] expose gunicorn logging (#219) --- docker/build_artifacts/sagemaker/serve.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/build_artifacts/sagemaker/serve.py b/docker/build_artifacts/sagemaker/serve.py index d54c5f71..17d0831a 100644 --- a/docker/build_artifacts/sagemaker/serve.py +++ b/docker/build_artifacts/sagemaker/serve.py @@ -51,6 +51,7 @@ def __init__(self): self._sagemaker_port_range = os.environ.get("SAGEMAKER_SAFE_PORT_RANGE", None) self._gunicorn_workers = os.environ.get("SAGEMAKER_GUNICORN_WORKERS", 1) self._gunicorn_threads = os.environ.get("SAGEMAKER_GUNICORN_THREADS", 1) + self._gunicorn_loglevel = os.environ.get("SAGEMAKER_GUNICORN_LOGLEVEL", "info") self._tfs_config_path = "/sagemaker/model-config.cfg" self._tfs_batching_config_path = "/sagemaker/batching-config.cfg" @@ -204,7 +205,7 @@ def _setup_gunicorn(self): gunicorn_command = ( "gunicorn -b unix:/tmp/gunicorn.sock -k {} --chdir /sagemaker " - "--workers {} --threads {} " + "--workers {} --threads {} --log-level {} " "{}{} -e TFS_GRPC_PORTS={} -e TFS_REST_PORTS={} " "-e SAGEMAKER_MULTI_MODEL={} -e SAGEMAKER_SAFE_PORT_RANGE={} " "-e SAGEMAKER_TFS_WAIT_TIME_SECONDS={} " @@ -213,6 +214,7 @@ def _setup_gunicorn(self): self._gunicorn_worker_class, self._gunicorn_workers, self._gunicorn_threads, + self._gunicorn_loglevel, python_path_option, ",".join(python_path_content), self._tfs_grpc_concat_ports, From 45b0373d89439b80a15e19067fb7b90e9918a592 Mon Sep 17 00:00:00 2001 From: "Isaac T. Mathers" Date: Mon, 7 Mar 2022 22:43:49 +0000 Subject: [PATCH 06/12] Add in SAGEMAKER_GUNICORN_TIMEOUT_SECONDS. --- docker/build_artifacts/sagemaker/serve.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/docker/build_artifacts/sagemaker/serve.py b/docker/build_artifacts/sagemaker/serve.py index 17d0831a..2d30bf73 100644 --- a/docker/build_artifacts/sagemaker/serve.py +++ b/docker/build_artifacts/sagemaker/serve.py @@ -64,9 +64,12 @@ def __init__(self): self._tfs_inter_op_parallelism = os.environ.get("SAGEMAKER_TFS_INTER_OP_PARALLELISM", 0) self._tfs_intra_op_parallelism = os.environ.get("SAGEMAKER_TFS_INTRA_OP_PARALLELISM", 0) self._gunicorn_worker_class = os.environ.get("SAGEMAKER_GUNICORN_WORKER_CLASS", "gevent") - self._gunicorn_setup_timeout_seconds = int(os.environ.get( - "SAGEMAKER_GUNICORN_SETUP_TIMEOUT_SECONDS", 30 - )) + self._gunicorn_timeout_seconds = int( + os.environ.get("SAGEMAKER_GUNICORN_TIMEOUT_SECONDS", 30) + ) + self._gunicorn_setup_timeout_seconds = int( + os.environ.get("SAGEMAKER_GUNICORN_SETUP_TIMEOUT_SECONDS", 30) + ) if os.environ.get("OMP_NUM_THREADS") is None: os.environ["OMP_NUM_THREADS"] = "1" @@ -205,7 +208,7 @@ def _setup_gunicorn(self): gunicorn_command = ( "gunicorn -b unix:/tmp/gunicorn.sock -k {} --chdir /sagemaker " - "--workers {} --threads {} --log-level {} " + "--workers {} --threads {} --log-level {} --timeout {} " "{}{} -e TFS_GRPC_PORTS={} -e TFS_REST_PORTS={} " "-e SAGEMAKER_MULTI_MODEL={} -e SAGEMAKER_SAFE_PORT_RANGE={} " "-e SAGEMAKER_TFS_WAIT_TIME_SECONDS={} " @@ -215,6 +218,8 @@ def _setup_gunicorn(self): self._gunicorn_workers, self._gunicorn_threads, self._gunicorn_loglevel, + self._gunicorn_timeout_seconds, + self._gunicorn_setup_timeout_seconds, python_path_option, ",".join(python_path_content), self._tfs_grpc_concat_ports, From 5cabd4cc2d164a07c4d3a97adc392f55766e7b8c Mon Sep 17 00:00:00 2001 From: "Isaac T. Mathers" Date: Mon, 7 Mar 2022 22:44:00 +0000 Subject: [PATCH 07/12] Update readme with PR feedback. --- README.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 92ed9eba..4790e1e7 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ For notebook examples, see: [Amazon SageMaker Examples](https://github.com/awsla 4. [Pre/Post-Processing](#pre/post-processing) 5. [Deploying a TensorFlow Serving Model](#deploying-a-tensorflow-serving-model) 6. [Enable Batching](#enabling-batching) -7. [Other Configurable Environment Variables](#other-configurable-environment-variables) +7. [Configurable SageMaker Environment Variables](#configurable-sagemaker-environment-variables) 8. [Deploying to Multi-Model Endpoint](#deploying-to-multi-model-endpoint) ## Getting Started @@ -614,15 +614,22 @@ SAGEMAKER_TFS_NUM_BATCH_THREADS="16" SAGEMAKER_TFS_MAX_ENQUEUED_BATCHES="10000" ``` -## Other Configurable Environment Variables +## Configurable SageMaker Environment Variables The following environment variables can be set on a SageMaker Model or Transform Job if further configuration is required: ```bash # Configures the logging level for GUnicorn. -# Valid values can be found here: https://docs.gunicorn.org/en/stable/settings.html#loglevel +# When looking to set this environment variable, please refer to: +# https://docs.gunicorn.org/en/stable/settings.html#loglevel # Defaults to "info" SAGEMAKER_GUNICORN_LOGLEVEL="debug" +# Configures how long a GUnicorn worker may be silent before it is killed and restarted. +# When looking to set this environment variable, please refer to: +# https://docs.gunicorn.org/en/stable/settings.html#timeout +# Defaults to 30. +SAGEMAKER_GUNICORN_TIMEOUT_SECONDS="60" + # Configures how long to wait in seconds for GUnicorn # to finish starting up before timing out. # Defaults to 30. From d71c0400c52332e5e84469297553f74bdcd61bf1 Mon Sep 17 00:00:00 2001 From: "Isaac T. Mathers" Date: Mon, 7 Mar 2022 23:01:33 +0000 Subject: [PATCH 08/12] Remove extra argument in format. --- docker/build_artifacts/sagemaker/serve.py | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/build_artifacts/sagemaker/serve.py b/docker/build_artifacts/sagemaker/serve.py index 2d30bf73..b82f5eed 100644 --- a/docker/build_artifacts/sagemaker/serve.py +++ b/docker/build_artifacts/sagemaker/serve.py @@ -219,7 +219,6 @@ def _setup_gunicorn(self): self._gunicorn_threads, self._gunicorn_loglevel, self._gunicorn_timeout_seconds, - self._gunicorn_setup_timeout_seconds, python_path_option, ",".join(python_path_content), self._tfs_grpc_concat_ports, From 368319913f9c5d1cbdd81a147e6d4da882bf91de Mon Sep 17 00:00:00 2001 From: "Isaac T. Mathers" Date: Tue, 8 Mar 2022 00:23:42 +0000 Subject: [PATCH 09/12] Remove setup timeout. --- README.md | 5 ----- docker/build_artifacts/sagemaker/serve.py | 5 +---- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/README.md b/README.md index 4790e1e7..812b3270 100644 --- a/README.md +++ b/README.md @@ -629,11 +629,6 @@ SAGEMAKER_GUNICORN_LOGLEVEL="debug" # https://docs.gunicorn.org/en/stable/settings.html#timeout # Defaults to 30. SAGEMAKER_GUNICORN_TIMEOUT_SECONDS="60" - -# Configures how long to wait in seconds for GUnicorn -# to finish starting up before timing out. -# Defaults to 30. -SAGEMAKER_GUNICORN_SETUP_TIMEOUT_SECONDS="60" ``` ## Deploying to Multi-Model Endpoint diff --git a/docker/build_artifacts/sagemaker/serve.py b/docker/build_artifacts/sagemaker/serve.py index b82f5eed..a31e3e62 100644 --- a/docker/build_artifacts/sagemaker/serve.py +++ b/docker/build_artifacts/sagemaker/serve.py @@ -67,9 +67,6 @@ def __init__(self): self._gunicorn_timeout_seconds = int( os.environ.get("SAGEMAKER_GUNICORN_TIMEOUT_SECONDS", 30) ) - self._gunicorn_setup_timeout_seconds = int( - os.environ.get("SAGEMAKER_GUNICORN_SETUP_TIMEOUT_SECONDS", 30) - ) if os.environ.get("OMP_NUM_THREADS") is None: os.environ["OMP_NUM_THREADS"] = "1" @@ -458,7 +455,7 @@ def start(self): self._setup_gunicorn() self._start_gunicorn() # make sure gunicorn is up - with self._timeout(seconds=self._gunicorn_setup_timeout_seconds): + with self._timeout(seconds=self._gunicorn_timeout_seconds): self._wait_for_gunicorn() self._start_nginx() From 188095ac4e4be6b74b3374731a9ab16ebbacd1ec Mon Sep 17 00:00:00 2001 From: "Isaac T. Mathers" Date: Tue, 8 Mar 2022 00:44:45 +0000 Subject: [PATCH 10/12] Make readme links clickable. --- README.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 812b3270..581a4211 100644 --- a/README.md +++ b/README.md @@ -617,16 +617,15 @@ SAGEMAKER_TFS_MAX_ENQUEUED_BATCHES="10000" ## Configurable SageMaker Environment Variables The following environment variables can be set on a SageMaker Model or Transform Job if further configuration is required: +When setting this environment variable, please refer to [this.](https://docs.gunicorn.org/en/stable/settings.html#loglevel) ```bash # Configures the logging level for GUnicorn. -# When looking to set this environment variable, please refer to: -# https://docs.gunicorn.org/en/stable/settings.html#loglevel # Defaults to "info" SAGEMAKER_GUNICORN_LOGLEVEL="debug" - +``` +When setting this environment variable, please refer to [this.](https://docs.gunicorn.org/en/stable/settings.html#timeout) +```bash # Configures how long a GUnicorn worker may be silent before it is killed and restarted. -# When looking to set this environment variable, please refer to: -# https://docs.gunicorn.org/en/stable/settings.html#timeout # Defaults to 30. SAGEMAKER_GUNICORN_TIMEOUT_SECONDS="60" ``` From 260a8c560a9edf1b663174634afe983de7854837 Mon Sep 17 00:00:00 2001 From: "Isaac T. Mathers" Date: Tue, 8 Mar 2022 00:47:53 +0000 Subject: [PATCH 11/12] Remove boiler plate statements and instead link in descriptive comment of var. --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 581a4211..ca488909 100644 --- a/README.md +++ b/README.md @@ -617,15 +617,15 @@ SAGEMAKER_TFS_MAX_ENQUEUED_BATCHES="10000" ## Configurable SageMaker Environment Variables The following environment variables can be set on a SageMaker Model or Transform Job if further configuration is required: -When setting this environment variable, please refer to [this.](https://docs.gunicorn.org/en/stable/settings.html#loglevel) +[Configures](https://docs.gunicorn.org/en/stable/settings.html#loglevel) +the logging level for GUnicorn. ```bash -# Configures the logging level for GUnicorn. # Defaults to "info" SAGEMAKER_GUNICORN_LOGLEVEL="debug" ``` -When setting this environment variable, please refer to [this.](https://docs.gunicorn.org/en/stable/settings.html#timeout) +[Configures](https://docs.gunicorn.org/en/stable/settings.html#timeout) +how long a GUnicorn worker may be silent before it is killed and restarted. ```bash -# Configures how long a GUnicorn worker may be silent before it is killed and restarted. # Defaults to 30. SAGEMAKER_GUNICORN_TIMEOUT_SECONDS="60" ``` From f50b1b43dbd1e89a31ead83856e376fb6d5cc0e6 Mon Sep 17 00:00:00 2001 From: "Isaac T. Mathers" Date: Tue, 8 Mar 2022 03:15:18 +0000 Subject: [PATCH 12/12] Update capitalization of Gunicorn. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ca488909..bedb1bea 100644 --- a/README.md +++ b/README.md @@ -618,13 +618,13 @@ SAGEMAKER_TFS_MAX_ENQUEUED_BATCHES="10000" The following environment variables can be set on a SageMaker Model or Transform Job if further configuration is required: [Configures](https://docs.gunicorn.org/en/stable/settings.html#loglevel) -the logging level for GUnicorn. +the logging level for Gunicorn. ```bash # Defaults to "info" SAGEMAKER_GUNICORN_LOGLEVEL="debug" ``` [Configures](https://docs.gunicorn.org/en/stable/settings.html#timeout) -how long a GUnicorn worker may be silent before it is killed and restarted. +how long a Gunicorn worker may be silent before it is killed and restarted. ```bash # Defaults to 30. SAGEMAKER_GUNICORN_TIMEOUT_SECONDS="60"