fix: Increase timeout for starting model server to 10 minutes (#92)

* Increase timeout for starting model server to 10 minutes

* fix: Increase timeout for starting model server to 10 minutes

* Fix tabs

* Trigger build

* Modify test to wait for 10 minutes

* Sleep initially for 3 sec

* Add later sleep within the except block

* Print number of attempts

* Print number of attempts

* Disable output capturing

* Print more debuggin info

* Add wait_fixed for retry

* Trigger build

* Revert test changes

Co-authored-by: David Thomas <[email protected]>

Author: nikhil-sk

buildspec-release.yml

@@ -1,5 +1,10 @@
 version: 0.2
 
+env:
+  variables:
+    LC_ALL: "C.UTF-8"
+    LANG: "C.UTF-8"
+
 phases:
   pre_build:
     commands:

buildspec.yml

@@ -1,5 +1,10 @@
 version: 0.2
 
+env:
+  variables:
+    LC_ALL: "C.UTF-8"
+    LANG: "C.UTF-8"
+
 phases:
   pre_build:
     commands:
@@ -30,4 +35,4 @@ phases:
       - cd ../..
 
       # run local integration tests
-      - IGNORE_COVERAGE=- tox -e py27,py36,py37 -- test/integration/local
+      - IGNORE_COVERAGE=- tox -e py27,py36,py37 -- -s test/integration/local 

src/sagemaker_inference/model_server.py

@@ -189,8 +189,8 @@ def _install_requirements():
         raise ValueError("failed to install required packages")
 
 
-# retry for 10 seconds
-@retry(stop_max_delay=10 * 1000)
+# retry for 10 minutes
+@retry(wait_fixed=1000, stop_max_delay=10 * 60 * 1000)
 def _retrieve_mms_server_process():
     mms_server_processes = list()
 

test/integration/local/test_dummy_multi_model.py

@@ -38,7 +38,7 @@ def container():
         proc = subprocess.Popen(command.split(), stdout=sys.stdout, stderr=subprocess.STDOUT)
 
         attempts = 0
-        while attempts < 5:
+        while attempts < 10:
             time.sleep(3)
             try:
                 requests.get(PING_URL)

test/integration/local/test_mxnet_multi_model.py

@@ -38,7 +38,7 @@ def container():
         proc = subprocess.Popen(command.split(), stdout=sys.stdout, stderr=subprocess.STDOUT)
 
         attempts = 0
-        while attempts < 5:
+        while attempts < 10:
             time.sleep(3)
             try:
                 requests.get(PING_URL)