From 1d28892bb006880eba5df2816f81e40acada65ea Mon Sep 17 00:00:00 2001 From: Jinpeng Qi Date: Wed, 9 Jun 2021 23:26:19 +0800 Subject: [PATCH 1/8] refactor naming of some variables --- .../sagemaker/python_service.py | 12 +++++----- docker/build_artifacts/sagemaker/serve.py | 23 +++++++++---------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/docker/build_artifacts/sagemaker/python_service.py b/docker/build_artifacts/sagemaker/python_service.py index 35d40eb8..b99b341a 100644 --- a/docker/build_artifacts/sagemaker/python_service.py +++ b/docker/build_artifacts/sagemaker/python_service.py @@ -31,8 +31,8 @@ SAGEMAKER_BATCHING_ENABLED = os.environ.get("SAGEMAKER_TFS_ENABLE_BATCHING", "false").lower() MODEL_CONFIG_FILE_PATH = "/sagemaker/model-config.cfg" -TFS_GRPC_PORT_RANGE = os.environ.get("TFS_GRPC_PORT_RANGE") -TFS_REST_PORT_RANGE = os.environ.get("TFS_REST_PORT_RANGE") +TFS_GRPC_SELECTED_PORTS = os.environ.get("TFS_GRPC_SELECTED_PORTS") +TFS_REST_SELECTED_PORTS = os.environ.get("TFS_REST_SELECTED_PORTS") SAGEMAKER_TFS_PORT_RANGE = os.environ.get("SAGEMAKER_SAFE_PORT_RANGE") TFS_INSTANCE_COUNT = int(os.environ.get("SAGEMAKER_TFS_INSTANCE_COUNT", "1")) @@ -69,8 +69,8 @@ def __init__(self): # during the _handle_load_model_post() self.model_handlers = {} else: - self._tfs_grpc_ports = self._parse_sagemaker_port_range(TFS_GRPC_PORT_RANGE) - self._tfs_rest_ports = self._parse_sagemaker_port_range(TFS_REST_PORT_RANGE) + self._tfs_grpc_ports = self._reconstruct_sagemaker_selected_ports(TFS_GRPC_SELECTED_PORTS) + self._tfs_rest_ports = self._reconstruct_sagemaker_selected_ports(TFS_REST_SELECTED_PORTS) self._channels = {} for grpc_port in self._tfs_grpc_ports: @@ -98,8 +98,8 @@ def on_post(self, req, res, model_name=None): data = json.loads(req.stream.read().decode("utf-8")) self._handle_load_model_post(res, data) - def _parse_sagemaker_port_range(self, port_range): - lower, upper = port_range.split('-') + def _reconstruct_sagemaker_selected_ports(self, selected_ports): + lower, upper = selected_ports.split('-') lower = int(lower) upper = int(upper) if lower == upper: diff --git a/docker/build_artifacts/sagemaker/serve.py b/docker/build_artifacts/sagemaker/serve.py index d834142a..5a157373 100644 --- a/docker/build_artifacts/sagemaker/serve.py +++ b/docker/build_artifacts/sagemaker/serve.py @@ -94,25 +94,24 @@ def __init__(self): if low + 2 * self._tfs_instance_count > hi: raise ValueError("not enough ports available in SAGEMAKER_SAFE_PORT_RANGE ({})" .format(self._sagemaker_port_range)) - self._tfs_grpc_port_range = "{}-{}".format(low, + # select non-overlapping grpc and rest ports based on tfs instance count + # record the init and end value in order to reconstruct them in python service + self._tfs_grpc_selected_ports = "{}-{}".format(low, low + 2 * self._tfs_instance_count) - self._tfs_rest_port_range = "{}-{}".format(low + 1, + self._tfs_rest_selected_ports = "{}-{}".format(low + 1, low + 2 * self._tfs_instance_count + 1) for i in range(self._tfs_instance_count): self._tfs_grpc_port.append(str(low + 2 * i)) self._tfs_rest_port.append(str(low + 2 * i + 1)) - # set environment variable for python service - os.environ["TFS_GRPC_PORT_RANGE"] = self._tfs_grpc_port_range - os.environ["TFS_REST_PORT_RANGE"] = self._tfs_rest_port_range else: # just use the standard default ports self._tfs_grpc_port = ["9000"] self._tfs_rest_port = ["8501"] - self._tfs_grpc_port_range = "9000-9000" - self._tfs_rest_port_range = "8501-8501" - # set environment variable for python service - os.environ["TFS_GRPC_PORT_RANGE"] = self._tfs_grpc_port_range - os.environ["TFS_REST_PORT_RANGE"] = self._tfs_rest_port_range + self._tfs_grpc_selected_ports = "9000-9000" + self._tfs_rest_selected_ports = "8501-8501" + # set environment variable for python service + os.environ["TFS_GRPC_SELECTED_PORTS"] = self._tfs_grpc_selected_ports + os.environ["TFS_REST_SELECTED_PORTS"] = self._tfs_rest_selected_ports def _need_python_service(self): if os.path.exists(INFERENCE_PATH): @@ -194,13 +193,13 @@ def _setup_gunicorn(self): gunicorn_command = ( "gunicorn -b unix:/tmp/gunicorn.sock -k {} --chdir /sagemaker " "--workers {} --threads {} " - "{}{} -e TFS_GRPC_PORT_RANGE={} -e TFS_REST_PORT_RANGE={} " + "{}{} -e TFS_GRPC_SELECTED_PORTS={} -e TFS_REST_SELECTED_PORTS={} " "-e SAGEMAKER_MULTI_MODEL={} -e SAGEMAKER_SAFE_PORT_RANGE={} " "-e SAGEMAKER_TFS_WAIT_TIME_SECONDS={} " "python_service:app").format(self._gunicorn_worker_class, self._gunicorn_workers, self._gunicorn_threads, python_path_option, ",".join(python_path_content), - self._tfs_grpc_port_range, self._tfs_rest_port_range, + self._tfs_grpc_selected_ports, self._tfs_rest_selected_ports, self._tfs_enable_multi_model_endpoint, self._sagemaker_port_range, self._tfs_wait_time_seconds) From bd41141a64367e982f4b181a4ec861dd794fd767 Mon Sep 17 00:00:00 2001 From: Jinpeng Qi Date: Wed, 9 Jun 2021 23:54:03 +0800 Subject: [PATCH 2/8] fix sanity test failure --- docker/build_artifacts/sagemaker/python_service.py | 6 ++++-- docker/build_artifacts/sagemaker/serve.py | 7 ++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/docker/build_artifacts/sagemaker/python_service.py b/docker/build_artifacts/sagemaker/python_service.py index b99b341a..216ec545 100644 --- a/docker/build_artifacts/sagemaker/python_service.py +++ b/docker/build_artifacts/sagemaker/python_service.py @@ -69,8 +69,10 @@ def __init__(self): # during the _handle_load_model_post() self.model_handlers = {} else: - self._tfs_grpc_ports = self._reconstruct_sagemaker_selected_ports(TFS_GRPC_SELECTED_PORTS) - self._tfs_rest_ports = self._reconstruct_sagemaker_selected_ports(TFS_REST_SELECTED_PORTS) + self._tfs_grpc_ports = self._reconstruct_sagemaker_selected_ports( + TFS_GRPC_SELECTED_PORTS) + self._tfs_rest_ports = self._reconstruct_sagemaker_selected_ports( + TFS_REST_SELECTED_PORTS) self._channels = {} for grpc_port in self._tfs_grpc_ports: diff --git a/docker/build_artifacts/sagemaker/serve.py b/docker/build_artifacts/sagemaker/serve.py index 5a157373..affe6854 100644 --- a/docker/build_artifacts/sagemaker/serve.py +++ b/docker/build_artifacts/sagemaker/serve.py @@ -97,9 +97,9 @@ def __init__(self): # select non-overlapping grpc and rest ports based on tfs instance count # record the init and end value in order to reconstruct them in python service self._tfs_grpc_selected_ports = "{}-{}".format(low, - low + 2 * self._tfs_instance_count) + low + 2 * self._tfs_instance_count) self._tfs_rest_selected_ports = "{}-{}".format(low + 1, - low + 2 * self._tfs_instance_count + 1) + low + 2 * self._tfs_instance_count + 1) for i in range(self._tfs_instance_count): self._tfs_grpc_port.append(str(low + 2 * i)) self._tfs_rest_port.append(str(low + 2 * i + 1)) @@ -199,7 +199,8 @@ def _setup_gunicorn(self): "python_service:app").format(self._gunicorn_worker_class, self._gunicorn_workers, self._gunicorn_threads, python_path_option, ",".join(python_path_content), - self._tfs_grpc_selected_ports, self._tfs_rest_selected_ports, + self._tfs_grpc_selected_ports, + self._tfs_rest_selected_ports, self._tfs_enable_multi_model_endpoint, self._sagemaker_port_range, self._tfs_wait_time_seconds) From e2f0d1819a39799ef2d098fb96028c146814488e Mon Sep 17 00:00:00 2001 From: Jinpeng Qi Date: Thu, 10 Jun 2021 22:14:44 +0800 Subject: [PATCH 3/8] modify to pick ports in linear fashion --- .../sagemaker/python_service.py | 16 +++++------- docker/build_artifacts/sagemaker/serve.py | 26 +++++++++---------- 2 files changed, 20 insertions(+), 22 deletions(-) diff --git a/docker/build_artifacts/sagemaker/python_service.py b/docker/build_artifacts/sagemaker/python_service.py index 216ec545..3c7ac81f 100644 --- a/docker/build_artifacts/sagemaker/python_service.py +++ b/docker/build_artifacts/sagemaker/python_service.py @@ -31,8 +31,8 @@ SAGEMAKER_BATCHING_ENABLED = os.environ.get("SAGEMAKER_TFS_ENABLE_BATCHING", "false").lower() MODEL_CONFIG_FILE_PATH = "/sagemaker/model-config.cfg" -TFS_GRPC_SELECTED_PORTS = os.environ.get("TFS_GRPC_SELECTED_PORTS") -TFS_REST_SELECTED_PORTS = os.environ.get("TFS_REST_SELECTED_PORTS") +TFS_GRPC_PORT_RANGE = os.environ.get("TFS_GRPC_PORT_RANGE") +TFS_REST_PORT_RANGE = os.environ.get("TFS_REST_PORT_RANGE") SAGEMAKER_TFS_PORT_RANGE = os.environ.get("SAGEMAKER_SAFE_PORT_RANGE") TFS_INSTANCE_COUNT = int(os.environ.get("SAGEMAKER_TFS_INSTANCE_COUNT", "1")) @@ -69,10 +69,8 @@ def __init__(self): # during the _handle_load_model_post() self.model_handlers = {} else: - self._tfs_grpc_ports = self._reconstruct_sagemaker_selected_ports( - TFS_GRPC_SELECTED_PORTS) - self._tfs_rest_ports = self._reconstruct_sagemaker_selected_ports( - TFS_REST_SELECTED_PORTS) + self._tfs_grpc_ports = self._parse_sagemaker_port_range(TFS_GRPC_PORT_RANGE) + self._tfs_rest_ports = self._parse_sagemaker_port_range(TFS_REST_PORT_RANGE) self._channels = {} for grpc_port in self._tfs_grpc_ports: @@ -100,13 +98,13 @@ def on_post(self, req, res, model_name=None): data = json.loads(req.stream.read().decode("utf-8")) self._handle_load_model_post(res, data) - def _reconstruct_sagemaker_selected_ports(self, selected_ports): - lower, upper = selected_ports.split('-') + def _parse_sagemaker_port_range(self, port_range): + lower, upper = port_range.split('-') lower = int(lower) upper = int(upper) if lower == upper: return [lower] - return [lower + 2 * i for i in range(TFS_INSTANCE_COUNT)] + return [lower + i for i in range(TFS_INSTANCE_COUNT)] def _pick_port(self, ports): return str(random.choice(ports)) diff --git a/docker/build_artifacts/sagemaker/serve.py b/docker/build_artifacts/sagemaker/serve.py index affe6854..fd8e9417 100644 --- a/docker/build_artifacts/sagemaker/serve.py +++ b/docker/build_artifacts/sagemaker/serve.py @@ -96,22 +96,23 @@ def __init__(self): .format(self._sagemaker_port_range)) # select non-overlapping grpc and rest ports based on tfs instance count # record the init and end value in order to reconstruct them in python service - self._tfs_grpc_selected_ports = "{}-{}".format(low, - low + 2 * self._tfs_instance_count) - self._tfs_rest_selected_ports = "{}-{}".format(low + 1, - low + 2 * self._tfs_instance_count + 1) + self._tfs_grpc_port_range = "{}-{}".format(low, + low + self._tfs_instance_count) + self._tfs_rest_port_range = "{}-{}".format(low + self._tfs_instance_count, + low + 2 * self._tfs_instance_count) for i in range(self._tfs_instance_count): - self._tfs_grpc_port.append(str(low + 2 * i)) - self._tfs_rest_port.append(str(low + 2 * i + 1)) + self._tfs_grpc_port.append(str(low + i)) + self._tfs_rest_port.append(str(low + self._tfs_instance_count + i)) else: # just use the standard default ports self._tfs_grpc_port = ["9000"] self._tfs_rest_port = ["8501"] - self._tfs_grpc_selected_ports = "9000-9000" - self._tfs_rest_selected_ports = "8501-8501" + self._tfs_grpc_port_range = "9000-9000" + self._tfs_rest_port_range = "8501-8501" + # set environment variable for python service - os.environ["TFS_GRPC_SELECTED_PORTS"] = self._tfs_grpc_selected_ports - os.environ["TFS_REST_SELECTED_PORTS"] = self._tfs_rest_selected_ports + os.environ["TFS_GRPC_PORT_RANGE"] = self._tfs_grpc_port_range + os.environ["TFS_REST_PORT_RANGE"] = self._tfs_rest_port_range def _need_python_service(self): if os.path.exists(INFERENCE_PATH): @@ -193,14 +194,13 @@ def _setup_gunicorn(self): gunicorn_command = ( "gunicorn -b unix:/tmp/gunicorn.sock -k {} --chdir /sagemaker " "--workers {} --threads {} " - "{}{} -e TFS_GRPC_SELECTED_PORTS={} -e TFS_REST_SELECTED_PORTS={} " + "{}{} -e TFS_GRPC_PORT_RANGE={} -e TFS_REST_PORT_RANGE={} " "-e SAGEMAKER_MULTI_MODEL={} -e SAGEMAKER_SAFE_PORT_RANGE={} " "-e SAGEMAKER_TFS_WAIT_TIME_SECONDS={} " "python_service:app").format(self._gunicorn_worker_class, self._gunicorn_workers, self._gunicorn_threads, python_path_option, ",".join(python_path_content), - self._tfs_grpc_selected_ports, - self._tfs_rest_selected_ports, + self._tfs_grpc_port_range, self._tfs_rest_port_range, self._tfs_enable_multi_model_endpoint, self._sagemaker_port_range, self._tfs_wait_time_seconds) From f783aefa4d45835fcb3c8cf9814855737b76c3c3 Mon Sep 17 00:00:00 2001 From: Jinpeng Qi Date: Fri, 11 Jun 2021 19:39:00 +0800 Subject: [PATCH 4/8] modify the way port number passing --- .../sagemaker/python_service.py | 23 ++++++-------- docker/build_artifacts/sagemaker/serve.py | 30 +++++++++++-------- 2 files changed, 26 insertions(+), 27 deletions(-) diff --git a/docker/build_artifacts/sagemaker/python_service.py b/docker/build_artifacts/sagemaker/python_service.py index 3c7ac81f..858d43a8 100644 --- a/docker/build_artifacts/sagemaker/python_service.py +++ b/docker/build_artifacts/sagemaker/python_service.py @@ -31,8 +31,8 @@ SAGEMAKER_BATCHING_ENABLED = os.environ.get("SAGEMAKER_TFS_ENABLE_BATCHING", "false").lower() MODEL_CONFIG_FILE_PATH = "/sagemaker/model-config.cfg" -TFS_GRPC_PORT_RANGE = os.environ.get("TFS_GRPC_PORT_RANGE") -TFS_REST_PORT_RANGE = os.environ.get("TFS_REST_PORT_RANGE") +TFS_GRPC_PORT = os.environ.get("TFS_GRPC_PORT") +TFS_REST_PORT = os.environ.get("TFS_REST_PORT") SAGEMAKER_TFS_PORT_RANGE = os.environ.get("SAGEMAKER_SAFE_PORT_RANGE") TFS_INSTANCE_COUNT = int(os.environ.get("SAGEMAKER_TFS_INSTANCE_COUNT", "1")) @@ -69,11 +69,11 @@ def __init__(self): # during the _handle_load_model_post() self.model_handlers = {} else: - self._tfs_grpc_ports = self._parse_sagemaker_port_range(TFS_GRPC_PORT_RANGE) - self._tfs_rest_ports = self._parse_sagemaker_port_range(TFS_REST_PORT_RANGE) + self._tfs_grpc_port = self._parse_concat_port(TFS_GRPC_PORT) + self._tfs_rest_port = self._parse_concat_port(TFS_REST_PORT) self._channels = {} - for grpc_port in self._tfs_grpc_ports: + for grpc_port in self._tfs_grpc_port: # Initialize grpc channel here so gunicorn worker could have mapping # between each grpc port and channel self._setup_channel(grpc_port) @@ -98,13 +98,8 @@ def on_post(self, req, res, model_name=None): data = json.loads(req.stream.read().decode("utf-8")) self._handle_load_model_post(res, data) - def _parse_sagemaker_port_range(self, port_range): - lower, upper = port_range.split('-') - lower = int(lower) - upper = int(upper) - if lower == upper: - return [lower] - return [lower + i for i in range(TFS_INSTANCE_COUNT)] + def _parse_concat_port(self, concat_port): + return concat_port.split(",") def _pick_port(self, ports): return str(random.choice(ports)) @@ -250,8 +245,8 @@ def _handle_invocation_post(self, req, res, model_name=None): }) else: # Randomly pick port used for routing incoming request. - grpc_port = self._pick_port(self._tfs_grpc_ports) - rest_port = self._pick_port(self._tfs_rest_ports) + grpc_port = self._pick_port(self._tfs_grpc_port) + rest_port = self._pick_port(self._tfs_rest_port) data, context = tfs_utils.parse_request(req, rest_port, grpc_port, self._tfs_default_model_name, channel=self._channels[int(grpc_port)]) diff --git a/docker/build_artifacts/sagemaker/serve.py b/docker/build_artifacts/sagemaker/serve.py index fd8e9417..ac9c0097 100644 --- a/docker/build_artifacts/sagemaker/serve.py +++ b/docker/build_artifacts/sagemaker/serve.py @@ -95,24 +95,23 @@ def __init__(self): raise ValueError("not enough ports available in SAGEMAKER_SAFE_PORT_RANGE ({})" .format(self._sagemaker_port_range)) # select non-overlapping grpc and rest ports based on tfs instance count - # record the init and end value in order to reconstruct them in python service - self._tfs_grpc_port_range = "{}-{}".format(low, - low + self._tfs_instance_count) - self._tfs_rest_port_range = "{}-{}".format(low + self._tfs_instance_count, - low + 2 * self._tfs_instance_count) for i in range(self._tfs_instance_count): - self._tfs_grpc_port.append(str(low + i)) - self._tfs_rest_port.append(str(low + self._tfs_instance_count + i)) + self._tfs_grpc_port.append(str(low + 2 * i)) + self._tfs_rest_port.append(str(low + 2 * i + 1)) + # concat selected ports respectively in order to pass them to python service + self._tfs_grpc_concat_port = self._concat_port(self._tfs_grpc_port) + self._tfs_rest_concat_port = self._concat_port(self._tfs_rest_port) else: # just use the standard default ports self._tfs_grpc_port = ["9000"] self._tfs_rest_port = ["8501"] - self._tfs_grpc_port_range = "9000-9000" - self._tfs_rest_port_range = "8501-8501" + # provide single concat port here for default case + self._tfs_grpc_concat_port = "9000" + self._tfs_rest_concat_port = "8501" # set environment variable for python service - os.environ["TFS_GRPC_PORT_RANGE"] = self._tfs_grpc_port_range - os.environ["TFS_REST_PORT_RANGE"] = self._tfs_rest_port_range + os.environ["TFS_GRPC_PORT"] = self._tfs_grpc_concat_port + os.environ["TFS_REST_PORT"] = self._tfs_rest_concat_port def _need_python_service(self): if os.path.exists(INFERENCE_PATH): @@ -121,6 +120,11 @@ def _need_python_service(self): and os.environ.get("SAGEMAKER_MULTI_MODEL_UNIVERSAL_PREFIX"): self._enable_python_service = True + def _concat_port(self, ports): + str_ports = [str(port) for port in ports] + concat_str_ports = ",".join(str_ports) + return concat_str_ports + def _create_tfs_config(self): models = tfs_utils.find_models() @@ -194,13 +198,13 @@ def _setup_gunicorn(self): gunicorn_command = ( "gunicorn -b unix:/tmp/gunicorn.sock -k {} --chdir /sagemaker " "--workers {} --threads {} " - "{}{} -e TFS_GRPC_PORT_RANGE={} -e TFS_REST_PORT_RANGE={} " + "{}{} -e TFS_GRPC_PORT={} -e TFS_REST_PORT={} " "-e SAGEMAKER_MULTI_MODEL={} -e SAGEMAKER_SAFE_PORT_RANGE={} " "-e SAGEMAKER_TFS_WAIT_TIME_SECONDS={} " "python_service:app").format(self._gunicorn_worker_class, self._gunicorn_workers, self._gunicorn_threads, python_path_option, ",".join(python_path_content), - self._tfs_grpc_port_range, self._tfs_rest_port_range, + self._tfs_grpc_concat_port, self._tfs_rest_concat_port, self._tfs_enable_multi_model_endpoint, self._sagemaker_port_range, self._tfs_wait_time_seconds) From bb54ee300917a3bf98e864d01490e274cb858c78 Mon Sep 17 00:00:00 2001 From: Jinpeng Qi Date: Sat, 12 Jun 2021 00:23:28 +0800 Subject: [PATCH 5/8] using plural ports instead of switching to singular --- .../sagemaker/python_service.py | 18 ++++----- docker/build_artifacts/sagemaker/serve.py | 38 +++++++++---------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/docker/build_artifacts/sagemaker/python_service.py b/docker/build_artifacts/sagemaker/python_service.py index 858d43a8..ab9cb1c0 100644 --- a/docker/build_artifacts/sagemaker/python_service.py +++ b/docker/build_artifacts/sagemaker/python_service.py @@ -31,8 +31,8 @@ SAGEMAKER_BATCHING_ENABLED = os.environ.get("SAGEMAKER_TFS_ENABLE_BATCHING", "false").lower() MODEL_CONFIG_FILE_PATH = "/sagemaker/model-config.cfg" -TFS_GRPC_PORT = os.environ.get("TFS_GRPC_PORT") -TFS_REST_PORT = os.environ.get("TFS_REST_PORT") +TFS_GRPC_PORTS = os.environ.get("TFS_GRPC_PORTS") +TFS_REST_PORTS = os.environ.get("TFS_REST_PORTS") SAGEMAKER_TFS_PORT_RANGE = os.environ.get("SAGEMAKER_SAFE_PORT_RANGE") TFS_INSTANCE_COUNT = int(os.environ.get("SAGEMAKER_TFS_INSTANCE_COUNT", "1")) @@ -69,11 +69,11 @@ def __init__(self): # during the _handle_load_model_post() self.model_handlers = {} else: - self._tfs_grpc_port = self._parse_concat_port(TFS_GRPC_PORT) - self._tfs_rest_port = self._parse_concat_port(TFS_REST_PORT) + self._tfs_grpc_ports = self._parse_concat_ports(TFS_GRPC_PORTS) + self._tfs_rest_ports = self._parse_concat_ports(TFS_REST_PORTS) self._channels = {} - for grpc_port in self._tfs_grpc_port: + for grpc_port in self._tfs_grpc_ports: # Initialize grpc channel here so gunicorn worker could have mapping # between each grpc port and channel self._setup_channel(grpc_port) @@ -98,8 +98,8 @@ def on_post(self, req, res, model_name=None): data = json.loads(req.stream.read().decode("utf-8")) self._handle_load_model_post(res, data) - def _parse_concat_port(self, concat_port): - return concat_port.split(",") + def _parse_concat_ports(self, concat_ports): + return concat_ports.split(",") def _pick_port(self, ports): return str(random.choice(ports)) @@ -245,8 +245,8 @@ def _handle_invocation_post(self, req, res, model_name=None): }) else: # Randomly pick port used for routing incoming request. - grpc_port = self._pick_port(self._tfs_grpc_port) - rest_port = self._pick_port(self._tfs_rest_port) + grpc_port = self._pick_port(self._tfs_grpc_ports) + rest_port = self._pick_port(self._tfs_rest_ports) data, context = tfs_utils.parse_request(req, rest_port, grpc_port, self._tfs_default_model_name, channel=self._channels[int(grpc_port)]) diff --git a/docker/build_artifacts/sagemaker/serve.py b/docker/build_artifacts/sagemaker/serve.py index ac9c0097..f8b87614 100644 --- a/docker/build_artifacts/sagemaker/serve.py +++ b/docker/build_artifacts/sagemaker/serve.py @@ -89,29 +89,29 @@ def __init__(self): parts = self._sagemaker_port_range.split("-") low = int(parts[0]) hi = int(parts[1]) - self._tfs_grpc_port = [] - self._tfs_rest_port = [] + self._tfs_grpc_ports = [] + self._tfs_rest_ports = [] if low + 2 * self._tfs_instance_count > hi: raise ValueError("not enough ports available in SAGEMAKER_SAFE_PORT_RANGE ({})" .format(self._sagemaker_port_range)) # select non-overlapping grpc and rest ports based on tfs instance count for i in range(self._tfs_instance_count): - self._tfs_grpc_port.append(str(low + 2 * i)) - self._tfs_rest_port.append(str(low + 2 * i + 1)) + self._tfs_grpc_ports.append(str(low + 2 * i)) + self._tfs_rest_ports.append(str(low + 2 * i + 1)) # concat selected ports respectively in order to pass them to python service - self._tfs_grpc_concat_port = self._concat_port(self._tfs_grpc_port) - self._tfs_rest_concat_port = self._concat_port(self._tfs_rest_port) + self._tfs_grpc_concat_ports = self._concat_ports(self._tfs_grpc_ports) + self._tfs_rest_concat_ports = self._concat_ports(self._tfs_rest_ports) else: # just use the standard default ports - self._tfs_grpc_port = ["9000"] - self._tfs_rest_port = ["8501"] + self._tfs_grpc_ports = ["9000"] + self._tfs_rest_ports = ["8501"] # provide single concat port here for default case - self._tfs_grpc_concat_port = "9000" - self._tfs_rest_concat_port = "8501" + self._tfs_grpc_concat_ports = "9000" + self._tfs_rest_concat_ports = "8501" # set environment variable for python service - os.environ["TFS_GRPC_PORT"] = self._tfs_grpc_concat_port - os.environ["TFS_REST_PORT"] = self._tfs_rest_concat_port + os.environ["TFS_GRPC_PORTS"] = self._tfs_grpc_concat_ports + os.environ["TFS_REST_PORTS"] = self._tfs_rest_concat_ports def _need_python_service(self): if os.path.exists(INFERENCE_PATH): @@ -120,7 +120,7 @@ def _need_python_service(self): and os.environ.get("SAGEMAKER_MULTI_MODEL_UNIVERSAL_PREFIX"): self._enable_python_service = True - def _concat_port(self, ports): + def _concat_ports(self, ports): str_ports = [str(port) for port in ports] concat_str_ports = ",".join(str_ports) return concat_str_ports @@ -198,13 +198,13 @@ def _setup_gunicorn(self): gunicorn_command = ( "gunicorn -b unix:/tmp/gunicorn.sock -k {} --chdir /sagemaker " "--workers {} --threads {} " - "{}{} -e TFS_GRPC_PORT={} -e TFS_REST_PORT={} " + "{}{} -e TFS_GRPC_PORTS={} -e TFS_REST_PORTS={} " "-e SAGEMAKER_MULTI_MODEL={} -e SAGEMAKER_SAFE_PORT_RANGE={} " "-e SAGEMAKER_TFS_WAIT_TIME_SECONDS={} " "python_service:app").format(self._gunicorn_worker_class, self._gunicorn_workers, self._gunicorn_threads, python_path_option, ",".join(python_path_content), - self._tfs_grpc_concat_port, self._tfs_rest_concat_port, + self._tfs_grpc_concat_ports, self._tfs_rest_concat_ports, self._tfs_enable_multi_model_endpoint, self._sagemaker_port_range, self._tfs_wait_time_seconds) @@ -234,7 +234,7 @@ def _download_scripts(self, bucket, prefix): def _create_nginx_tfs_upstream(self): indentation = " " tfs_upstream = "" - for port in self._tfs_rest_port: + for port in self._tfs_rest_ports: tfs_upstream += "{}server localhost:{};\n".format(indentation, port) tfs_upstream = tfs_upstream[len(indentation):-2] @@ -338,7 +338,7 @@ def _wait_for_gunicorn(self): def _wait_for_tfs(self): for i in range(self._tfs_instance_count): - tfs_utils.wait_for_model(self._tfs_rest_port[i], + tfs_utils.wait_for_model(self._tfs_rest_ports[i], self._tfs_default_model_name, self._tfs_wait_time_seconds) @contextmanager @@ -374,8 +374,8 @@ def _restart_single_tfs(self, pid): def _start_single_tfs(self, instance_id): cmd = tfs_utils.tfs_command( - self._tfs_grpc_port[instance_id], - self._tfs_rest_port[instance_id], + self._tfs_grpc_ports[instance_id], + self._tfs_rest_ports[instance_id], self._tfs_config_path, self._tfs_enable_batching, self._tfs_batching_config_path, From 8199a01c7afd872da2367f36a2df2c6c62a09953 Mon Sep 17 00:00:00 2001 From: Jinpeng Qi Date: Tue, 15 Jun 2021 10:10:07 +0800 Subject: [PATCH 6/8] trigger rebuild test From ba83486a0d08e36dcf58d4167c4a2290353075d6 Mon Sep 17 00:00:00 2001 From: Jinpeng Qi Date: Tue, 15 Jun 2021 11:43:28 +0800 Subject: [PATCH 7/8] fix KeyError --- docker/build_artifacts/sagemaker/python_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/build_artifacts/sagemaker/python_service.py b/docker/build_artifacts/sagemaker/python_service.py index ab9cb1c0..e294e5fc 100644 --- a/docker/build_artifacts/sagemaker/python_service.py +++ b/docker/build_artifacts/sagemaker/python_service.py @@ -102,7 +102,7 @@ def _parse_concat_ports(self, concat_ports): return concat_ports.split(",") def _pick_port(self, ports): - return str(random.choice(ports)) + return random.choice(ports) def _parse_sagemaker_port_range_mme(self, port_range): lower, upper = port_range.split('-') @@ -249,7 +249,7 @@ def _handle_invocation_post(self, req, res, model_name=None): rest_port = self._pick_port(self._tfs_rest_ports) data, context = tfs_utils.parse_request(req, rest_port, grpc_port, self._tfs_default_model_name, - channel=self._channels[int(grpc_port)]) + channel=self._channels[grpc_port]) try: res.status = falcon.HTTP_200 From 9455619c4440b27ace8b357e73687dc4513e385b Mon Sep 17 00:00:00 2001 From: Jinpeng Qi Date: Tue, 15 Jun 2021 21:52:30 +0800 Subject: [PATCH 8/8] retrigger rebuild test